Add support for Spark 3.4.0

Author: sbernauer

CHANGELOG.md

@@ -7,6 +7,7 @@ All notable changes to this project will be documented in this file.
 ### Added
 
 - Generate OLM bundle for Release 23.4.0 ([#238]).
+- Add support for Spark 3.4.0 ([#243])
 
 ### Changed
 
@@ -15,10 +16,15 @@ All notable changes to this project will be documented in this file.
 - Use testing-tools 0.2.0 ([#236])
 - Run as root group ([#241]).
 
+### Fixed
+
+- Fix quoting issues when spark config values contain spaces ([#243])
+
 [#235]: https://github.com/stackabletech/spark-k8s-operator/pull/235
 [#236]: https://github.com/stackabletech/spark-k8s-operator/pull/236
 [#238]: https://github.com/stackabletech/spark-k8s-operator/pull/238
 [#241]: https://github.com/stackabletech/spark-k8s-operator/pull/241
+[#243]: https://github.com/stackabletech/spark-k8s-operator/pull/243
 
 ## [23.4.0] - 2023-04-17
 

deploy/helm/spark-k8s-operator/templates/roles.yaml

@@ -29,6 +29,7 @@ rules:
     verbs:
       - create
       - delete
+      - deletecollection
       - get
       - list
       - patch

docs/modules/spark-k8s/partials/supported-versions.adoc

@@ -5,3 +5,4 @@
 - 3.2.1-hadoop3.2
 - 3.2.1-hadoop3.2-python39
 - 3.3.0-hadoop3
+- 3.4.0-hadoop3

rust/crd/src/lib.rs

@@ -618,7 +618,7 @@ impl SparkApplication {
         }
         // ...before being added to the command collection
         for (key, value) in submit_conf {
-            submit_cmd.push(format!("--conf {key}={value}"));
+            submit_cmd.push(format!("--conf \"{key}={value}\""));
         }
 
         submit_cmd.extend(

rust/operator-binary/src/history_controller.rs

@@ -536,9 +536,9 @@ fn command_args(s3logdir: &S3LogDir) -> Vec<String> {
 
     if let Some(secret_dir) = s3logdir.credentials_mount_path() {
         command.extend(vec![
-            format!("export AWS_ACCESS_KEY_ID=$(cat {secret_dir}/{ACCESS_KEY_ID})"),
+            format!("export AWS_ACCESS_KEY_ID=\"$(cat {secret_dir}/{ACCESS_KEY_ID})\""),
             "&&".to_string(),
-            format!("export AWS_SECRET_ACCESS_KEY=$(cat {secret_dir}/{SECRET_ACCESS_KEY})"),
+            format!("export AWS_SECRET_ACCESS_KEY=\"$(cat {secret_dir}/{SECRET_ACCESS_KEY})\""),
             "&&".to_string(),
         ]);
     }

tests/test-definition.yaml

@@ -10,6 +10,7 @@ dimensions:
   - name: spark
     values:
       - 3.3.0-stackable0.0.0-dev
+      - 3.4.0-stackable0.0.0-dev
   - name: ny-tlc-report
     values:
       - 0.1.0