diff --git a/CHANGELOG.md b/CHANGELOG.md index c5615e3b..cb009544 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,10 @@ All notable changes to this project will be documented in this file. ### Changed - BREAKING: Use current S3 connection/bucket structs ([#86]) +- Add node selector to top-level job and specify node selection in PVC-relevant tests ([#90]) [#86]: https://github.com/stackabletech/spark-k8s-operator/pull/86 +[#90]: https://github.com/stackabletech/spark-k8s-operator/pull/90 ## [0.2.0] - 2022-06-21 diff --git a/rust/operator-binary/src/spark_k8s_controller.rs b/rust/operator-binary/src/spark_k8s_controller.rs index 71411a8e..d34bdc75 100644 --- a/rust/operator-binary/src/spark_k8s_controller.rs +++ b/rust/operator-binary/src/spark_k8s_controller.rs @@ -388,6 +388,7 @@ fn spark_job( .fs_group(1000) .build() .into(), // Needed for secret-operator + node_selector: spark_application.driver_node_selector(), ..PodSpec::default() }), }; diff --git a/tests/templates/kuttl/spark-ny-public-s3/02-deps-volume.yaml b/tests/templates/kuttl/spark-ny-public-s3/02-deps-volume.yaml index bdcd9577..c8e81a73 100644 --- a/tests/templates/kuttl/spark-ny-public-s3/02-deps-volume.yaml +++ b/tests/templates/kuttl/spark-ny-public-s3/02-deps-volume.yaml @@ -17,6 +17,8 @@ metadata: spec: template: spec: + nodeSelector: + node: "1" restartPolicy: Never volumes: - name: job-deps diff --git a/tests/templates/kuttl/spark-ny-public-s3/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/spark-ny-public-s3/10-deploy-spark-app.yaml.j2 index 07e3cf17..7e3a98ce 100644 --- a/tests/templates/kuttl/spark-ny-public-s3/10-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/spark-ny-public-s3/10-deploy-spark-app.yaml.j2 @@ -48,6 +48,8 @@ spec: mountPath: /dependencies - name: cm-job-arguments mountPath: /arguments + nodeSelector: + node: "1" executor: cores: 1 instances: 3 @@ -57,3 +59,5 @@ spec: mountPath: /dependencies - name: cm-job-arguments mountPath: /arguments + nodeSelector: + node: "1" diff --git a/tests/templates/kuttl/spark-pi-private-s3/02-deps-volume.yaml b/tests/templates/kuttl/spark-pi-private-s3/02-deps-volume.yaml index 8e7f5783..7e9edca7 100644 --- a/tests/templates/kuttl/spark-pi-private-s3/02-deps-volume.yaml +++ b/tests/templates/kuttl/spark-pi-private-s3/02-deps-volume.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: spark-pi-pvc + name: spark-pi-private-pvc spec: accessModes: - ReadWriteOnce @@ -17,11 +17,13 @@ metadata: spec: template: spec: + nodeSelector: + node: "1" restartPolicy: Never volumes: - name: job-deps persistentVolumeClaim: - claimName: spark-pi-pvc + claimName: spark-pi-private-pvc containers: - name: aws-deps image: docker.stackable.tech/stackable/tools:0.2.0-stackable0 diff --git a/tests/templates/kuttl/spark-pi-private-s3/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/spark-pi-private-s3/10-deploy-spark-app.yaml.j2 index 2ced72ce..d50ab139 100644 --- a/tests/templates/kuttl/spark-pi-private-s3/10-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/spark-pi-private-s3/10-deploy-spark-app.yaml.j2 @@ -22,7 +22,7 @@ spec: volumes: - name: spark-pi-deps persistentVolumeClaim: - claimName: spark-pi-pvc + claimName: spark-pi-private-pvc sparkConf: spark.hadoop.fs.s3a.aws.credentials.provider: "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider" spark.driver.extraClassPath: "/dependencies/jars/hadoop-aws-3.2.0.jar:/dependencies/jars/aws-java-sdk-bundle-1.11.375.jar" @@ -34,6 +34,8 @@ spec: volumeMounts: - name: spark-pi-deps mountPath: /dependencies + nodeSelector: + node: "1" executor: cores: 1 instances: 1 @@ -41,3 +43,5 @@ spec: volumeMounts: - name: spark-pi-deps mountPath: /dependencies + nodeSelector: + node: "1" diff --git a/tests/templates/kuttl/spark-pi-public-s3/02-deps-volume.yaml b/tests/templates/kuttl/spark-pi-public-s3/02-deps-volume.yaml index 8e7f5783..9880525b 100644 --- a/tests/templates/kuttl/spark-pi-public-s3/02-deps-volume.yaml +++ b/tests/templates/kuttl/spark-pi-public-s3/02-deps-volume.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: spark-pi-pvc + name: spark-pi-public-pvc spec: accessModes: - ReadWriteOnce @@ -17,11 +17,13 @@ metadata: spec: template: spec: + nodeSelector: + node: "1" restartPolicy: Never volumes: - name: job-deps persistentVolumeClaim: - claimName: spark-pi-pvc + claimName: spark-pi-public-pvc containers: - name: aws-deps image: docker.stackable.tech/stackable/tools:0.2.0-stackable0 diff --git a/tests/templates/kuttl/spark-pi-public-s3/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/spark-pi-public-s3/10-deploy-spark-app.yaml.j2 index f7143249..c2f7f3c2 100644 --- a/tests/templates/kuttl/spark-pi-public-s3/10-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/spark-pi-public-s3/10-deploy-spark-app.yaml.j2 @@ -13,7 +13,7 @@ spec: volumes: - name: spark-pi-deps persistentVolumeClaim: - claimName: spark-pi-pvc + claimName: spark-pi-public-pvc s3bucket: inline: bucketName: my-bucket @@ -33,6 +33,8 @@ spec: volumeMounts: - name: spark-pi-deps mountPath: /dependencies + nodeSelector: + node: "1" executor: cores: 1 instances: 1 @@ -40,3 +42,5 @@ spec: volumeMounts: - name: spark-pi-deps mountPath: /dependencies + nodeSelector: + node: "1"