stackabletech · sbernauer · May 22, 2023 · May 22, 2023 · May 22, 2023 · May 22, 2023
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,7 @@ All notable changes to this project will be documented in this file.
 ### Added
 
 - Generate OLM bundle for Release 23.4.0 ([#238]).
+- Add support for Spark 3.4.0 ([#243])
 
 ### Changed
 
@@ -15,10 +16,15 @@ All notable changes to this project will be documented in this file.
 - Use testing-tools 0.2.0 ([#236])
 - Run as root group ([#241]).
 
+### Fixed
+
+- Fix quoting issues when spark config values contain spaces ([#243])
+
 [#235]: https://github.com/stackabletech/spark-k8s-operator/pull/235
 [#236]: https://github.com/stackabletech/spark-k8s-operator/pull/236
 [#238]: https://github.com/stackabletech/spark-k8s-operator/pull/238
 [#241]: https://github.com/stackabletech/spark-k8s-operator/pull/241
+[#243]: https://github.com/stackabletech/spark-k8s-operator/pull/243
 
 ## [23.4.0] - 2023-04-17
 

diff --git a/deploy/helm/spark-k8s-operator/templates/roles.yaml b/deploy/helm/spark-k8s-operator/templates/roles.yaml
@@ -29,6 +29,7 @@ rules:
     verbs:
       - create
       - delete
+      - deletecollection
       - get
       - list
       - patch

diff --git a/docs/modules/spark-k8s/examples/example-encapsulated.yaml b/docs/modules/spark-k8s/examples/example-encapsulated.yaml
@@ -8,6 +8,6 @@ spec:
   sparkImage: docker.stackable.tech/stackable/spark-k8s:3.3.0-stackable0.3.0 # <1>
   mode: cluster
   mainClass: org.apache.spark.examples.SparkPi
-  mainApplicationFile: /stackable/spark/examples/jars/spark-examples_2.12-3.3.0.jar # <2>
+  mainApplicationFile: /stackable/spark/examples/jars/spark-examples.jar # <2>
   executor:
     instances: 3
diff --git a/docs/modules/spark-k8s/examples/example-history-app.yaml b/docs/modules/spark-k8s/examples/example-history-app.yaml
@@ -9,7 +9,7 @@ spec:
   sparkImagePullPolicy: IfNotPresent
   mode: cluster
   mainClass: org.apache.spark.examples.SparkPi
-  mainApplicationFile: s3a://my-bucket/spark-examples_2.12-3.3.0.jar
+  mainApplicationFile: s3a://my-bucket/spark-examples.jar
   s3connection: # <1>
     inline:
       host: test-minio

diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml
@@ -7,7 +7,7 @@ spec:
   version: "1.0"
   sparkImage: docker.stackable.tech/stackable/spark-k8s:3.3.0-stackable0.3.0
   mode: cluster
-  mainApplicationFile: s3a://my-bucket/spark-examples_2.12-3.3.0.jar # <1>
+  mainApplicationFile: s3a://my-bucket/spark-examples.jar # <1>
   mainClass: org.apache.spark.examples.SparkPi # <2>
   s3connection: # <3>
     inline:

diff --git a/docs/modules/spark-k8s/partials/supported-versions.adoc b/docs/modules/spark-k8s/partials/supported-versions.adoc
@@ -5,3 +5,4 @@
 - 3.2.1-hadoop3.2
 - 3.2.1-hadoop3.2-python39
 - 3.3.0-hadoop3
+- 3.4.0-hadoop3
diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs
@@ -618,7 +618,7 @@ impl SparkApplication {
         }
         // ...before being added to the command collection
         for (key, value) in submit_conf {
-            submit_cmd.push(format!("--conf {key}={value}"));
+            submit_cmd.push(format!("--conf \"{key}={value}\""));
         }
 
         submit_cmd.extend(
@@ -939,7 +939,7 @@ spec:
   sparkImage: docker.stackable.tech/stackable/spark-k8s:3.2.1-hadoop3.2-python39-aws1.11.375-stackable0.3.0
   mode: cluster
   mainClass: org.apache.spark.examples.SparkPi
-  mainApplicationFile: s3a://stackable-spark-k8s-jars/jobs/spark-examples_2.12-3.2.1.jar
+  mainApplicationFile: s3a://stackable-spark-k8s-jars/jobs/spark-examples.jar
   sparkConf:
     "spark.hadoop.fs.s3a.aws.credentials.provider": "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider"
   driver:
@@ -960,7 +960,7 @@ spec:
             spark_application.spec.main_class
         );
         assert_eq!(
-            Some("s3a://stackable-spark-k8s-jars/jobs/spark-examples_2.12-3.2.1.jar".to_string()),
+            Some("s3a://stackable-spark-k8s-jars/jobs/spark-examples.jar".to_string()),
             spark_application.spec.main_application_file
         );
         assert_eq!(
@@ -1113,7 +1113,7 @@ spec:
     - name: myregistrykey
   mode: cluster
   mainClass: org.apache.spark.examples.SparkPi
-  mainApplicationFile: local:///stackable/spark/examples/jars/spark-examples_2.12-3.2.1.jar
+  mainApplicationFile: local:///stackable/spark/examples/jars/spark-examples.jar
   sparkConf:
     spark.kubernetes.node.selector.node: "2"
   driver:

diff --git a/rust/operator-binary/src/history_controller.rs b/rust/operator-binary/src/history_controller.rs
@@ -536,9 +536,9 @@ fn command_args(s3logdir: &S3LogDir) -> Vec<String> {
 
     if let Some(secret_dir) = s3logdir.credentials_mount_path() {
         command.extend(vec![
-            format!("export AWS_ACCESS_KEY_ID=$(cat {secret_dir}/{ACCESS_KEY_ID})"),
+            format!("export AWS_ACCESS_KEY_ID=\"$(cat {secret_dir}/{ACCESS_KEY_ID})\""),
             "&&".to_string(),
-            format!("export AWS_SECRET_ACCESS_KEY=$(cat {secret_dir}/{SECRET_ACCESS_KEY})"),
+            format!("export AWS_SECRET_ACCESS_KEY=\"$(cat {secret_dir}/{SECRET_ACCESS_KEY})\""),
             "&&".to_string(),
         ]);
     }

diff --git a/tests/templates/kuttl/logging/05-deploy-automatic-log-config-spark-app.yaml.j2 b/tests/templates/kuttl/logging/05-deploy-automatic-log-config-spark-app.yaml.j2
@@ -5,13 +5,15 @@ metadata:
   name: spark-automatic-log-config
 spec:
   version: "1.0"
-  sparkImage: docker.stackable.tech/stackable/spark-k8s:{{ test_scenario['values']['spark'].split('-stackable')[0] }}-stackable{{ test_scenario['values']['spark'].split('-stackable')[1] }}
-  sparkImagePullPolicy: IfNotPresent
+  sparkImage: docker.stackable.tech/sandbox/sbernauer/spark-k8s:{{ test_scenario['values']['spark'].split('-stackable')[0] }}-stackable{{ test_scenario['values']['spark'].split('-stackable')[1] }}
+  sparkImagePullPolicy: Always
   image: docker.stackable.tech/stackable/ny-tlc-report:{{ test_scenario['values']['ny-tlc-report'] }}
   vectorAggregatorConfigMapName: spark-vector-aggregator-discovery
   mode: cluster
   mainClass: org.apache.spark.examples.SparkALS
-  mainApplicationFile: local:///stackable/spark/examples/jars/spark-examples_2.12-{{ test_scenario['values']['examples'] }}.jar
+  mainApplicationFile: local:///stackable/spark/examples/jars/spark-examples.jar
+  sparkConf:
+    spark.kubernetes.container.image.pullPolicy: Always
   job:
     logging:
       enableVectorAgent: true

diff --git a/tests/templates/kuttl/logging/06-deploy-custom-log-config-spark-app.yaml.j2 b/tests/templates/kuttl/logging/06-deploy-custom-log-config-spark-app.yaml.j2
@@ -39,13 +39,15 @@ metadata:
   name: spark-custom-log-config
 spec:
   version: "1.0"
-  sparkImage: docker.stackable.tech/stackable/spark-k8s:{{ test_scenario['values']['spark'].split('-stackable')[0] }}-stackable{{ test_scenario['values']['spark'].split('-stackable')[1] }}
-  sparkImagePullPolicy: IfNotPresent
+  sparkImage: docker.stackable.tech/sandbox/sbernauer/spark-k8s:{{ test_scenario['values']['spark'].split('-stackable')[0] }}-stackable{{ test_scenario['values']['spark'].split('-stackable')[1] }}
+  sparkImagePullPolicy: Always
   image: docker.stackable.tech/stackable/ny-tlc-report:{{ test_scenario['values']['ny-tlc-report'] }}
   vectorAggregatorConfigMapName: spark-vector-aggregator-discovery
   mode: cluster
   mainClass: org.apache.spark.examples.SparkALS
-  mainApplicationFile: local:///stackable/spark/examples/jars/spark-examples_2.12-{{ test_scenario['values']['examples'] }}.jar
+  mainApplicationFile: local:///stackable/spark/examples/jars/spark-examples.jar
+  sparkConf:
+    spark.kubernetes.container.image.pullPolicy: Always
   job:
     logging:
       enableVectorAgent: true

diff --git a/tests/templates/kuttl/logging/07-deploy-automatic-log-config-pyspark-app.yaml.j2 b/tests/templates/kuttl/logging/07-deploy-automatic-log-config-pyspark-app.yaml.j2
@@ -5,11 +5,13 @@ metadata:
   name: pyspark-automatic-log-config
 spec:
   version: "1.0"
-  sparkImage: docker.stackable.tech/stackable/pyspark-k8s:{{ test_scenario['values']['spark'].split('-stackable')[0] }}-stackable{{ test_scenario['values']['spark'].split('-stackable')[1] }}
-  sparkImagePullPolicy: IfNotPresent
+  sparkImage: docker.stackable.tech/sandbox/sbernauer/pyspark-k8s:{{ test_scenario['values']['spark'].split('-stackable')[0] }}-stackable{{ test_scenario['values']['spark'].split('-stackable')[1] }}
+  sparkImagePullPolicy: Always
   vectorAggregatorConfigMapName: spark-vector-aggregator-discovery
   mode: cluster
   mainApplicationFile: local:///stackable/spark/examples/src/main/python/als.py
+  sparkConf:
+    spark.kubernetes.container.image.pullPolicy: Always
   deps:
     requirements:
       - numpy==1.24.2

diff --git a/tests/templates/kuttl/logging/08-deploy-custom-log-config-pyspark-app.yaml.j2 b/tests/templates/kuttl/logging/08-deploy-custom-log-config-pyspark-app.yaml.j2
@@ -39,11 +39,13 @@ metadata:
   name: pyspark-custom-log-config
 spec:
   version: "1.0"
-  sparkImage: docker.stackable.tech/stackable/pyspark-k8s:{{ test_scenario['values']['spark'].split('-stackable')[0] }}-stackable{{ test_scenario['values']['spark'].split('-stackable')[1] }}
-  sparkImagePullPolicy: IfNotPresent
+  sparkImage: docker.stackable.tech/sandbox/sbernauer/pyspark-k8s:{{ test_scenario['values']['spark'].split('-stackable')[0] }}-stackable{{ test_scenario['values']['spark'].split('-stackable')[1] }}
+  sparkImagePullPolicy: Always
   vectorAggregatorConfigMapName: spark-vector-aggregator-discovery
   mode: cluster
   mainApplicationFile: local:///stackable/spark/examples/src/main/python/als.py
+  sparkConf:
+    spark.kubernetes.container.image.pullPolicy: Always
   deps:
     requirements:
       - numpy==1.24.2

diff --git a/tests/templates/kuttl/pyspark-ny-public-s3-image/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/pyspark-ny-public-s3-image/10-deploy-spark-app.yaml.j2
@@ -10,7 +10,7 @@ spec:
 {% if lookup('env', 'VECTOR_AGGREGATOR') %}
   vectorAggregatorConfigMapName: vector-aggregator-discovery
 {% endif %}
-  sparkImage: "docker.stackable.tech/stackable/pyspark-k8s:{{ test_scenario['values']['spark'].split('-stackable')[0] }}-stackable{{ test_scenario['values']['spark'].split('-stackable')[1] }}"
+  sparkImage: "docker.stackable.tech/sandbox/sbernauer/pyspark-k8s:{{ test_scenario['values']['spark'].split('-stackable')[0] }}-stackable{{ test_scenario['values']['spark'].split('-stackable')[1] }}"
   sparkImagePullPolicy: IfNotPresent
   mode: cluster
   mainApplicationFile: local:///stackable/spark/jobs/ny_tlc_report.py

diff --git a/tests/templates/kuttl/pyspark-ny-public-s3/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/pyspark-ny-public-s3/10-deploy-spark-app.yaml.j2
@@ -9,7 +9,7 @@ spec:
   vectorAggregatorConfigMapName: vector-aggregator-discovery
 {% endif %}
   # everything under /jobs will be copied to /stackable/spark/jobs
-  sparkImage: "docker.stackable.tech/stackable/pyspark-k8s:{{ test_scenario['values']['spark'].split('-stackable')[0] }}-stackable{{ test_scenario['values']['spark'].split('-stackable')[1] }}"
+  sparkImage: "docker.stackable.tech/sandbox/sbernauer/pyspark-k8s:{{ test_scenario['values']['spark'].split('-stackable')[0] }}-stackable{{ test_scenario['values']['spark'].split('-stackable')[1] }}"
   sparkImagePullPolicy: IfNotPresent
   mode: cluster
   mainApplicationFile: s3a://my-bucket/ny_tlc_report.py

diff --git a/tests/templates/kuttl/resources/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/resources/10-deploy-spark-app.yaml.j2
@@ -8,7 +8,7 @@ spec:
 {% if lookup('env', 'VECTOR_AGGREGATOR') %}
   vectorAggregatorConfigMapName: vector-aggregator-discovery
 {% endif %}
-  sparkImage: "docker.stackable.tech/stackable/pyspark-k8s:{{ test_scenario['values']['spark'].split('-stackable')[0] }}-stackable{{ test_scenario['values']['spark'].split('-stackable')[1] }}"
+  sparkImage: "docker.stackable.tech/sandbox/sbernauer/pyspark-k8s:{{ test_scenario['values']['spark'].split('-stackable')[0] }}-stackable{{ test_scenario['values']['spark'].split('-stackable')[1] }}"
   mode: cluster
   mainApplicationFile: local:///stackable/spark/examples/src/main/python/streaming/hdfs_wordcount.py
   args:

diff --git a/tests/templates/kuttl/resources/12-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/resources/12-deploy-spark-app.yaml.j2
@@ -8,7 +8,7 @@ spec:
 {% if lookup('env', 'VECTOR_AGGREGATOR') %}
   vectorAggregatorConfigMapName: vector-aggregator-discovery
 {% endif %}
-  sparkImage: "docker.stackable.tech/stackable/pyspark-k8s:{{ test_scenario['values']['spark'].split('-stackable')[0] }}-stackable{{ test_scenario['values']['spark'].split('-stackable')[1] }}"
+  sparkImage: "docker.stackable.tech/sandbox/sbernauer/pyspark-k8s:{{ test_scenario['values']['spark'].split('-stackable')[0] }}-stackable{{ test_scenario['values']['spark'].split('-stackable')[1] }}"
   mode: cluster
   mainApplicationFile: local:///stackable/spark/examples/src/main/python/streaming/hdfs_wordcount.py
   args:

diff --git a/tests/templates/kuttl/resources/14-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/resources/14-deploy-spark-app.yaml.j2
@@ -8,7 +8,7 @@ spec:
 {% if lookup('env', 'VECTOR_AGGREGATOR') %}
   vectorAggregatorConfigMapName: vector-aggregator-discovery
 {% endif %}
-  sparkImage: "docker.stackable.tech/stackable/pyspark-k8s:{{ test_scenario['values']['spark'].split('-stackable')[0] }}-stackable{{ test_scenario['values']['spark'].split('-stackable')[1] }}"
+  sparkImage: "docker.stackable.tech/sandbox/sbernauer/pyspark-k8s:{{ test_scenario['values']['spark'].split('-stackable')[0] }}-stackable{{ test_scenario['values']['spark'].split('-stackable')[1] }}"
   mode: cluster
   mainApplicationFile: local:///stackable/spark/examples/src/main/python/streaming/hdfs_wordcount.py
   args:

diff --git a/tests/templates/kuttl/spark-examples/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/spark-examples/10-deploy-spark-app.yaml.j2
@@ -8,11 +8,11 @@ spec:
 {% if lookup('env', 'VECTOR_AGGREGATOR') %}
   vectorAggregatorConfigMapName: vector-aggregator-discovery
 {% endif %}
-  sparkImage: "docker.stackable.tech/stackable/spark-k8s:{{ test_scenario['values']['spark'].split('-stackable')[0] }}-stackable{{ test_scenario['values']['spark'].split('-stackable')[1] }}"
+  sparkImage: "docker.stackable.tech/sandbox/sbernauer/spark-k8s:{{ test_scenario['values']['spark'].split('-stackable')[0] }}-stackable{{ test_scenario['values']['spark'].split('-stackable')[1] }}"
   sparkImagePullPolicy: IfNotPresent
   mode: cluster
   mainClass: org.apache.spark.examples.SparkALS
-  mainApplicationFile: "local:///stackable/spark/examples/jars/spark-examples_2.12-{{ test_scenario['values']['examples'] }}.jar"
+  mainApplicationFile: "local:///stackable/spark/examples/jars/spark-examples.jar"
   job:
     logging:
       enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}

diff --git a/tests/templates/kuttl/spark-history-server/04-prepare-bucket.yaml.j2 b/tests/templates/kuttl/spark-history-server/04-prepare-bucket.yaml.j2
@@ -4,10 +4,9 @@ kind: TestStep
 commands:
   # give minio enough time to start
   - command: sleep 10
-  - command: kubectl cp -n $NAMESPACE spark-examples_2.12-{{ test_scenario['values']['examples'] }}.jar minio-client:/tmp
+  - command: kubectl cp -n $NAMESPACE spark-examples_{{ test_scenario['values']['spark'].split('-stackable')[0] }}.jar minio-client:/tmp/spark-examples.jar
   - command: kubectl exec -n $NAMESPACE minio-client -- sh -c 'mc alias set test-minio http://test-minio:9000 $$MINIO_SERVER_ACCESS_KEY $$MINIO_SERVER_SECRET_KEY'
   - command: kubectl exec -n $NAMESPACE minio-client -- mc mb test-minio/my-bucket
   - command: kubectl exec -n $NAMESPACE eventlog-minio-client -- sh -c 'mc alias set eventlog-minio http://eventlog-minio:9000 $$MINIO_SERVER_ACCESS_KEY $$MINIO_SERVER_SECRET_KEY'
   - command: kubectl exec -n $NAMESPACE eventlog-minio-client -- mc mb eventlog-minio/spark-logs/eventlogs
-  - script: >-
-      kubectl exec -n $NAMESPACE minio-client -- mc cp /tmp/spark-examples_2.12-{{ test_scenario['values']['examples'] }}.jar test-minio/my-bucket
+  - command: kubectl exec -n $NAMESPACE minio-client -- mc cp /tmp/spark-examples.jar test-minio/my-bucket
diff --git a/tests/templates/kuttl/spark-history-server/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/spark-history-server/10-deploy-spark-app.yaml.j2
@@ -8,11 +8,11 @@ spec:
 {% if lookup('env', 'VECTOR_AGGREGATOR') %}
   vectorAggregatorConfigMapName: vector-aggregator-discovery
 {% endif %}
-  sparkImage: "docker.stackable.tech/stackable/spark-k8s:{{ test_scenario['values']['spark'].split('-stackable')[0] }}-stackable{{ test_scenario['values']['spark'].split('-stackable')[1] }}"
+  sparkImage: "docker.stackable.tech/sandbox/sbernauer/spark-k8s:{{ test_scenario['values']['spark'].split('-stackable')[0] }}-stackable{{ test_scenario['values']['spark'].split('-stackable')[1] }}"
   sparkImagePullPolicy: IfNotPresent
   mode: cluster
   mainClass: org.apache.spark.examples.SparkPi
-  mainApplicationFile: "s3a://my-bucket/spark-examples_2.12-{{ test_scenario['values']['examples'] }}.jar"
+  mainApplicationFile: "s3a://my-bucket/spark-examples.jar"
   s3connection:
     reference: spark-data-s3-connection
   logFileDirectory:

diff --git a/tests/templates/kuttl/spark-history-server/12-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/spark-history-server/12-deploy-spark-app.yaml.j2
@@ -8,11 +8,11 @@ spec:
 {% if lookup('env', 'VECTOR_AGGREGATOR') %}
   vectorAggregatorConfigMapName: vector-aggregator-discovery
 {% endif %}
-  sparkImage: "docker.stackable.tech/stackable/spark-k8s:{{ test_scenario['values']['spark'].split('-stackable')[0] }}-stackable{{ test_scenario['values']['spark'].split('-stackable')[1] }}"
+  sparkImage: "docker.stackable.tech/sandbox/sbernauer/spark-k8s:{{ test_scenario['values']['spark'].split('-stackable')[0] }}-stackable{{ test_scenario['values']['spark'].split('-stackable')[1] }}"
   sparkImagePullPolicy: IfNotPresent
   mode: cluster
   mainClass: org.apache.spark.examples.SparkPi
-  mainApplicationFile: "s3a://my-bucket/spark-examples_2.12-{{ test_scenario['values']['examples'] }}.jar"
+  mainApplicationFile: "s3a://my-bucket/spark-examples.jar"
   s3connection:
     reference: spark-data-s3-connection
   logFileDirectory:

diff --git a/...tory-server/spark-examples_2.12-3.3.0.jar → ...k-history-server/spark-examples_3.3.0.jar b/...tory-server/spark-examples_2.12-3.3.0.jar → ...k-history-server/spark-examples_3.3.0.jar
diff --git a/tests/templates/kuttl/spark-history-server/spark-examples_3.4.0.jar b/tests/templates/kuttl/spark-history-server/spark-examples_3.4.0.jar
diff --git a/tests/templates/kuttl/spark-ny-public-s3/03-prepare-bucket.yaml.j2 b/tests/templates/kuttl/spark-ny-public-s3/03-prepare-bucket.yaml.j2
@@ -4,14 +4,10 @@ kind: TestStep
 commands:
   # give minio enough time to start
   - command: sleep 10
-  - command: kubectl cp -n $NAMESPACE ny-tlc-report-1.1.0-{{ test_scenario['values']['examples'] }}.jar  minio-client:/tmp
+  - command: kubectl cp -n $NAMESPACE ny-tlc-report-1.1.0-{{ test_scenario['values']['spark'].split('-stackable')[0] }}.jar  minio-client:/tmp/ny-tlc-report.jar
   - command: kubectl cp -n $NAMESPACE yellow_tripdata_2021-07.csv  minio-client:/tmp
   - command: kubectl exec -n $NAMESPACE minio-client -- sh -c 'mc alias set test-minio http://test-minio:9000 $$MINIO_SERVER_ACCESS_KEY $$MINIO_SERVER_SECRET_KEY'
   - command: kubectl exec -n $NAMESPACE minio-client -- mc mb test-minio/my-bucket
   - command: kubectl exec -n $NAMESPACE minio-client -- mc policy set public test-minio/my-bucket
-  - script: >-
-      kubectl exec -n $NAMESPACE minio-client --
-      mc cp /tmp/ny-tlc-report-1.1.0-{{ test_scenario['values']['examples'] }}.jar test-minio/my-bucket
-  - script: >-
-      kubectl exec -n $NAMESPACE minio-client --
-      mc cp /tmp/yellow_tripdata_2021-07.csv test-minio/my-bucket
+  - command: kubectl exec -n $NAMESPACE minio-client -- mc cp /tmp/ny-tlc-report.jar test-minio/my-bucket
+  - command: kubectl exec -n $NAMESPACE minio-client -- mc cp /tmp/yellow_tripdata_2021-07.csv test-minio/my-bucket
diff --git a/tests/templates/kuttl/spark-ny-public-s3/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/spark-ny-public-s3/10-deploy-spark-app.yaml.j2
@@ -16,11 +16,11 @@ spec:
 {% if lookup('env', 'VECTOR_AGGREGATOR') %}
   vectorAggregatorConfigMapName: vector-aggregator-discovery
 {% endif %}
-  sparkImage: "docker.stackable.tech/stackable/spark-k8s:{{ test_scenario['values']['spark'].split('-stackable')[0] }}-stackable{{ test_scenario['values']['spark'].split('-stackable')[1] }}"
+  sparkImage: "docker.stackable.tech/sandbox/sbernauer/spark-k8s:{{ test_scenario['values']['spark'].split('-stackable')[0] }}-stackable{{ test_scenario['values']['spark'].split('-stackable')[1] }}"
   sparkImagePullPolicy: IfNotPresent
   mode: cluster
   mainClass: tech.stackable.demo.spark.NYTLCReport
-  mainApplicationFile: "s3a://my-bucket/ny-tlc-report-1.1.0-{{ test_scenario['values']['examples'] }}.jar"
+  mainApplicationFile: "s3a://my-bucket/ny-tlc-report.jar"
   volumes:
     - name: cm-job-arguments
       configMap:

diff --git a/tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.4.0.jar b/tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.4.0.jar
diff --git a/tests/templates/kuttl/spark-pi-private-s3/04-prepare-bucket.yaml.j2 b/tests/templates/kuttl/spark-pi-private-s3/04-prepare-bucket.yaml.j2
@@ -4,8 +4,7 @@ kind: TestStep
 commands:
   # give minio enough time to start
   - command: sleep 10
-  - command: kubectl cp -n $NAMESPACE spark-examples_2.12-{{ test_scenario['values']['examples'] }}.jar minio-client:/tmp
+  - command: kubectl cp -n $NAMESPACE spark-examples_{{ test_scenario['values']['spark'].split('-stackable')[0] }}.jar minio-client:/tmp/spark-examples.jar
   - command: kubectl exec -n $NAMESPACE minio-client -- sh -c 'mc alias set test-minio http://test-minio:9000 $$MINIO_SERVER_ACCESS_KEY $$MINIO_SERVER_SECRET_KEY'
   - command: kubectl exec -n $NAMESPACE minio-client -- mc mb test-minio/my-bucket
-  - script: >-
-      kubectl exec -n $NAMESPACE minio-client -- mc cp /tmp/spark-examples_2.12-{{ test_scenario['values']['examples'] }}.jar test-minio/my-bucket
+  - command: kubectl exec -n $NAMESPACE minio-client -- mc cp /tmp/spark-examples.jar test-minio/my-bucket
diff --git a/tests/templates/kuttl/spark-pi-private-s3/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/spark-pi-private-s3/10-deploy-spark-app.yaml.j2
@@ -8,11 +8,11 @@ spec:
 {% if lookup('env', 'VECTOR_AGGREGATOR') %}
   vectorAggregatorConfigMapName: vector-aggregator-discovery
 {% endif %}
-  sparkImage: docker.stackable.tech/stackable/spark-k8s:{{ test_scenario['values']['spark'].split('-stackable')[0] }}-stackable{{ test_scenario['values']['spark'].split('-stackable')[1] }}
+  sparkImage: docker.stackable.tech/sandbox/sbernauer/spark-k8s:{{ test_scenario['values']['spark'].split('-stackable')[0] }}-stackable{{ test_scenario['values']['spark'].split('-stackable')[1] }}
   sparkImagePullPolicy: IfNotPresent
   mode: cluster
   mainClass: org.apache.spark.examples.SparkPi
-  mainApplicationFile: s3a://my-bucket/spark-examples_2.12-{{ test_scenario['values']['examples'] }}.jar
+  mainApplicationFile: s3a://my-bucket/spark-examples.jar
   s3connection:
     inline:
       host: test-minio

diff --git a/...-private-s3/spark-examples_2.12-3.3.0.jar → ...rk-pi-private-s3/spark-examples_3.3.0.jar b/...-private-s3/spark-examples_2.12-3.3.0.jar → ...rk-pi-private-s3/spark-examples_3.3.0.jar
diff --git a/tests/templates/kuttl/spark-pi-private-s3/spark-examples_3.4.0.jar b/tests/templates/kuttl/spark-pi-private-s3/spark-examples_3.4.0.jar
-Original file line number
+Diff line change
@@ Expand Up / @@ -29,6 +29,7 @@ rules: @@
         verbs:
           - create
           - delete
+          - deletecollection
           - get
           - list
           - patch
@@ Expand Down @@