feat: Support Spark 3.5.1 (#373)

sbernauer · web-flow · commit e7870b7660c0 · 2024-03-12T08:31:38.000Z
* feat: Support Spark 3.5.1

* changelog

* changelog

* changelog
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,7 @@ All notable changes to this project will be documented in this file.
 ### Added
 
 - Helm: support labels in values.yaml ([#344]).
+- Support version `3.5.1` ([#373]).
 - Support version `3.4.2` ([#357]).
 - `spec.job.config.volumeMounts` property to easily mount volumes on the job pod ([#359])
 
@@ -31,6 +32,7 @@ All notable changes to this project will be documented in this file.
 [#355]: https://github.com/stackabletech/spark-k8s-operator/pull/355
 [#357]: https://github.com/stackabletech/spark-k8s-operator/pull/357
 [#359]: https://github.com/stackabletech/spark-k8s-operator/pull/359
+[#373]: https://github.com/stackabletech/spark-k8s-operator/pull/373
 
 ## [23.11.0] - 2023-11-24
 
diff --git a/docs/modules/spark-k8s/examples/example-encapsulated.yaml b/docs/modules/spark-k8s/examples/example-encapsulated.yaml
@@ -5,7 +5,7 @@ metadata:
   name: spark-pi
 spec:
   sparkImage:
-    productVersion: 3.5.0 # <1>
+    productVersion: 3.5.1 # <1>
   mode: cluster
   mainClass: org.apache.spark.examples.SparkPi
   mainApplicationFile: /stackable/spark/examples/jars/spark-examples.jar # <2>
diff --git a/docs/modules/spark-k8s/examples/example-history-app.yaml b/docs/modules/spark-k8s/examples/example-history-app.yaml
@@ -5,7 +5,7 @@ metadata:
   name: spark-pi-s3-1
 spec:
   sparkImage:
-    productVersion: 3.5.0
+    productVersion: 3.5.1
     pullPolicy: IfNotPresent
   mode: cluster
   mainClass: org.apache.spark.examples.SparkPi
diff --git a/docs/modules/spark-k8s/examples/example-history-server.yaml b/docs/modules/spark-k8s/examples/example-history-server.yaml
@@ -5,7 +5,7 @@ metadata:
   name: spark-history
 spec:
   image:
-    productVersion: 3.5.0
+    productVersion: 3.5.1
   logFileDirectory:  # <1>
     s3:
       prefix: eventlogs/  # <2>
diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml
@@ -6,7 +6,7 @@ metadata:
   namespace: default
 spec:
   sparkImage:
-    productVersion: 3.5.0
+    productVersion: 3.5.1
   mode: cluster
   mainApplicationFile: s3a://stackable-spark-k8s-jars/jobs/ny-tlc-report-1.1.0.jar # <3>
   mainClass: tech.stackable.demo.spark.NYTLCReport
diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-external-dependencies.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-external-dependencies.yaml
@@ -6,7 +6,7 @@ metadata:
   namespace: default
 spec:
   sparkImage:
-    productVersion: 3.5.0
+    productVersion: 3.5.1
   mode: cluster
   mainApplicationFile: s3a://stackable-spark-k8s-jars/jobs/ny_tlc_report.py # <1>
   args:
diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml
@@ -7,7 +7,7 @@ metadata:
 spec:
   image: docker.stackable.tech/stackable/ny-tlc-report:0.1.0 # <1>
   sparkImage:
-    productVersion: 3.5.0
+    productVersion: 3.5.1
   mode: cluster
   mainApplicationFile: local:///stackable/spark/jobs/ny_tlc_report.py # <2>
   args:
diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml
@@ -6,7 +6,7 @@ metadata:
   namespace: default
 spec:
   sparkImage:
-    productVersion: 3.5.0
+    productVersion: 3.5.1
   mode: cluster
   mainApplicationFile: s3a://stackable-spark-k8s-jars/jobs/ny-tlc-report-1.0-SNAPSHOT.jar # <1>
   mainClass: org.example.App # <2>
diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml
@@ -5,7 +5,7 @@ metadata:
   name: example-sparkapp-s3-private
 spec:
   sparkImage:
-    productVersion: 3.5.0
+    productVersion: 3.5.1
   mode: cluster
   mainApplicationFile: s3a://my-bucket/spark-examples.jar # <1>
   mainClass: org.apache.spark.examples.SparkPi # <2>
diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml
@@ -6,7 +6,7 @@ metadata:
   namespace: default
 spec:
   sparkImage:
-    productVersion: 3.5.0
+    productVersion: 3.5.1
   mode: cluster
   mainApplicationFile: local:///stackable/spark/examples/src/main/python/streaming/hdfs_wordcount.py
   args:
diff --git a/docs/modules/spark-k8s/examples/getting_started/getting_started.sh b/docs/modules/spark-k8s/examples/getting_started/getting_started.sh
@@ -60,7 +60,7 @@ metadata:
   namespace: default
 spec:
   sparkImage:
-    productVersion: 3.5.0
+    productVersion: 3.5.1
   mode: cluster
   mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py
   driver:
diff --git a/docs/modules/spark-k8s/examples/getting_started/getting_started.sh.j2 b/docs/modules/spark-k8s/examples/getting_started/getting_started.sh.j2
@@ -60,7 +60,7 @@ metadata:
   namespace: default
 spec:
   sparkImage:
-    productVersion: 3.5.0
+    productVersion: 3.5.1
   mode: cluster
   mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py
   driver:
diff --git a/docs/modules/spark-k8s/partials/supported-versions.adoc b/docs/modules/spark-k8s/partials/supported-versions.adoc
@@ -3,6 +3,7 @@
 // Stackable Platform documentation.
 // Please sort the versions in descending order (newest first)
 
-- 3.5.0 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 11)
+- 3.5.1 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 11)
+- 3.5.0 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 11) (deprecated)
 - 3.4.2 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 11) (LTS)
 - 3.4.1 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 11) (deprecated)
diff --git a/examples/README-examples.md b/examples/README-examples.md
@@ -50,10 +50,10 @@ Several resources are needed in this store. These can be loaded like this:
 
 ````text
 kubectl exec minio-mc-0 -- sh -c 'mc alias set test-minio http://test-minio:9000/'
-kubectl cp examples/ny-tlc-report-1.1.0-3.5.0.jar  minio-mc-0:/tmp
+kubectl cp examples/ny-tlc-report-1.1.0-3.5.1.jar  minio-mc-0:/tmp
 kubectl cp apps/ny_tlc_report.py  minio-mc-0:/tmp
 kubectl cp examples/yellow_tripdata_2021-07.csv  minio-mc-0:/tmp
-kubectl exec minio-mc-0 -- mc cp /tmp/ny-tlc-report-1.1.0-3.5.0.jar test-minio/my-bucket
+kubectl exec minio-mc-0 -- mc cp /tmp/ny-tlc-report-1.1.0-3.5.1.jar test-minio/my-bucket
 kubectl exec minio-mc-0 -- mc cp /tmp/ny_tlc_report.py test-minio/my-bucket
 kubectl exec minio-mc-0 -- mc cp /tmp/yellow_tripdata_2021-07.csv test-minio/my-bucket
 ````
diff --git a/examples/ny-tlc-report-1.1.0-3.5.1.jar b/examples/ny-tlc-report-1.1.0-3.5.1.jar
diff --git a/examples/ny-tlc-report-external-dependencies.yaml b/examples/ny-tlc-report-external-dependencies.yaml
@@ -6,7 +6,7 @@ metadata:
   namespace: default
 spec:
   sparkImage:
-    productVersion: 3.5.0
+    productVersion: 3.5.1
     pullPolicy: IfNotPresent
   mode: cluster
   mainApplicationFile: s3a://my-bucket/ny_tlc_report.py
diff --git a/examples/ny-tlc-report-image.yaml b/examples/ny-tlc-report-image.yaml
@@ -7,7 +7,7 @@ metadata:
 spec:
   # everything under /jobs will be copied to /stackable/spark/jobs
   image: docker.stackable.tech/stackable/ny-tlc-report:0.1.0
-  sparkImage: docker.stackable.tech/stackable/spark-k8s:3.5.0-stackable0.0.0-dev
+  sparkImage: docker.stackable.tech/stackable/spark-k8s:3.5.1-stackable0.0.0-dev
   sparkImagePullPolicy: IfNotPresent
   mode: cluster
   mainApplicationFile: local:///stackable/spark/jobs/ny_tlc_report.py
diff --git a/examples/ny-tlc-report.yaml b/examples/ny-tlc-report.yaml
@@ -13,9 +13,9 @@ metadata:
   name: spark-ny-cm
 spec:
   sparkImage:
-    productVersion: 3.5.0
+    productVersion: 3.5.1
   mode: cluster
-  mainApplicationFile: s3a://my-bucket/ny-tlc-report-1.1.0-3.5.0.jar
+  mainApplicationFile: s3a://my-bucket/ny-tlc-report-1.1.0-3.5.1.jar
   mainClass: tech.stackable.demo.spark.NYTLCReport
   volumes:
     - name: cm-job-arguments
diff --git a/rust/crd/src/affinity.rs b/rust/crd/src/affinity.rs
@@ -47,7 +47,7 @@ mod test {
           name: spark-history
         spec:
           image:
-            productVersion: 3.5.0
+            productVersion: 3.5.1
           logFileDirectory:
             s3:
               prefix: eventlogs/
diff --git a/tests/templates/kuttl/pod_overrides/spark-examples_3.5.1.jar b/tests/templates/kuttl/pod_overrides/spark-examples_3.5.1.jar
diff --git a/tests/templates/kuttl/smoke/spark-examples_3.5.1.jar b/tests/templates/kuttl/smoke/spark-examples_3.5.1.jar
diff --git a/tests/templates/kuttl/spark-history-server/spark-examples_3.5.1.jar b/tests/templates/kuttl/spark-history-server/spark-examples_3.5.1.jar
diff --git a/tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.5.1.jar b/tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.5.1.jar
diff --git a/tests/templates/kuttl/spark-pi-private-s3/spark-examples_3.5.1.jar b/tests/templates/kuttl/spark-pi-private-s3/spark-examples_3.5.1.jar
diff --git a/tests/templates/kuttl/spark-pi-public-s3/spark-examples_3.5.1.jar b/tests/templates/kuttl/spark-pi-public-s3/spark-examples_3.5.1.jar
diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml
@@ -8,13 +8,14 @@ dimensions:
       - 3.4.1
       - 3.4.2
       - 3.5.0
+      - 3.5.1
       # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version
       # as in the example below.
-      # - 3.5.0,docker.stackable.tech/sandbox/spark-k8s:3.5.0-stackable0.0.0-dev
+      # - 3.5.1,docker.stackable.tech/sandbox/spark-k8s:3.5.1-stackable0.0.0-dev
   - name: spark-delta-lake
     values:
-      - 3.5.0
-      # - 3.5.0,docker.stackable.tech/sandbox/spark-k8s:3.5.0-stackable0.0.0-dev
+      - 3.5.1
+      # - 3.5.1,docker.stackable.tech/sandbox/spark-k8s:3.5.1-stackable0.0.0-dev
   - name: ny-tlc-report
     values:
       - 0.1.0
