Merge branch '3-configure-node-selection' into handle-spark-app-termination

Author: razvan

.github/workflows/build.yml

@@ -112,7 +112,7 @@ jobs:
       - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2
         with:
           submodules: recursive
-      - uses: EmbarkStudios/cargo-deny-action@3481b77dfd7b1d3c62bebdbfb57695131cd59c8f # tag=v1.2.17
+      - uses: EmbarkStudios/cargo-deny-action@30ecad1d5873c1cc2cad10a33637371ca094768b # tag=v1.3.1
         with:
           command: check ${{ matrix.checks }}
 

.pre-commit-config.yaml

@@ -33,8 +33,3 @@ repos:
     rev: 4.0.1
     hooks:
       - id: flake8
-
-  - repo: https://github.com/PyCQA/pylint
-    rev: v2.13.9
-    hooks:
-      - id: pylint

Cargo.lock

@@ -5,7 +5,7 @@ edition = "2021"
 license = "OSL-3.0"
 name = "stackable-spark-k8s-crd"
 repository = "https://github.com/stackabletech/spark-k8s-operator"
-version = "0.1.0"
+version = "0.2.0-nightly"
 
 [dependencies]
 stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag="0.19.0" }

rust/crd/Cargo.toml

@@ -5,7 +5,7 @@ edition = "2021"
 license = "OSL-3.0"
 name = "stackable-spark-k8s-operator"
 repository = "https://github.com/stackabletech/spark-k8s-operator"
-version = "0.1.0"
+version = "0.2.0-nightly"
 
 [dependencies]
 stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag="0.19.0" }

rust/operator-binary/Cargo.toml

@@ -8,3 +8,4 @@ testDirs:
 
 startKIND: false
 suppress: ["events"]
+parallel: 2

tests/kuttl-test.yaml.jinja2

@@ -3,7 +3,7 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
 metadata:
   name: node-selector-assert
-timeout: 900
+timeout: 300
 ---
 # The Job starting the whole process
 apiVersion: spark.stackable.tech/v1alpha1

tests/templates/kuttl/node-selector/00-assert.yaml

@@ -6,41 +6,19 @@ metadata:
 spec:
   version: "1.0"
   sparkImage: docker.stackable.tech/stackable/spark-k8s:{{ test_scenario['values']['spark'] }}-hadoop{{ test_scenario['values']['hadoop'][:-2] }}-stackable{{ test_scenario['values']['stackable'] }}
+  sparkImagePullPolicy: IfNotPresent
   mode: cluster
-  mainClass: org.apache.spark.examples.SparkPi
-  mainApplicationFile: s3a://my-bucket/spark-examples_2.12-{{ test_scenario['values']['spark'] }}.jar
-  s3bucket:
-    inline:
-      bucketName: my-bucket
-      connection:
-        inline:
-          host: test-minio
-          port: 9000
-          secretClass: minio-credentials
-  volumes:
-    - name: spark-pi-deps
-      persistentVolumeClaim:
-        claimName: spark-pi-pvc
-  sparkConf:
-    spark.hadoop.fs.s3a.aws.credentials.provider: "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider"
-    spark.hadoop.fs.s3a.path.style.access: "true"
-    spark.driver.extraClassPath: "/dependencies/jars/hadoop-aws-3.2.0.jar:/dependencies/jars/aws-java-sdk-bundle-1.11.375.jar"
-    spark.executor.extraClassPath: "/dependencies/jars/hadoop-aws-3.2.0.jar:/dependencies/jars/aws-java-sdk-bundle-1.11.375.jar"
+  mainClass: org.apache.spark.examples.SparkALS
+  mainApplicationFile: local:///stackable/spark/examples/jars/spark-examples_2.12-{{ test_scenario['values']['spark'] }}.jar
   driver:
     cores: 1
     coreLimit: "1200m"
     memory: "512m"
-    volumeMounts:
-      - name: spark-pi-deps
-        mountPath: /dependencies
     nodeSelector:
       node: "1"
   executor:
     cores: 1
     instances: 1
     memory: "512m"
-    volumeMounts:
-      - name: spark-pi-deps
-        mountPath: /dependencies
     nodeSelector:
       kubernetes.io/os: linux