diff --git a/CHANGELOG.md b/CHANGELOG.md index d5dc6eba..489f99fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ All notable changes to this project will be documented in this file. ### Added - Helm: support labels in values.yaml ([#344]). +- Support version `3.5.1` ([#373]). - Support version `3.4.2` ([#357]). - `spec.job.config.volumeMounts` property to easily mount volumes on the job pod ([#359]) @@ -31,6 +32,7 @@ All notable changes to this project will be documented in this file. [#355]: https://github.com/stackabletech/spark-k8s-operator/pull/355 [#357]: https://github.com/stackabletech/spark-k8s-operator/pull/357 [#359]: https://github.com/stackabletech/spark-k8s-operator/pull/359 +[#373]: https://github.com/stackabletech/spark-k8s-operator/pull/373 ## [23.11.0] - 2023-11-24 diff --git a/docs/modules/spark-k8s/examples/example-encapsulated.yaml b/docs/modules/spark-k8s/examples/example-encapsulated.yaml index e8edac3c..a8d5b33c 100644 --- a/docs/modules/spark-k8s/examples/example-encapsulated.yaml +++ b/docs/modules/spark-k8s/examples/example-encapsulated.yaml @@ -5,7 +5,7 @@ metadata: name: spark-pi spec: sparkImage: - productVersion: 3.5.0 # <1> + productVersion: 3.5.1 # <1> mode: cluster mainClass: org.apache.spark.examples.SparkPi mainApplicationFile: /stackable/spark/examples/jars/spark-examples.jar # <2> diff --git a/docs/modules/spark-k8s/examples/example-history-app.yaml b/docs/modules/spark-k8s/examples/example-history-app.yaml index cfd37074..93945ce2 100644 --- a/docs/modules/spark-k8s/examples/example-history-app.yaml +++ b/docs/modules/spark-k8s/examples/example-history-app.yaml @@ -5,7 +5,7 @@ metadata: name: spark-pi-s3-1 spec: sparkImage: - productVersion: 3.5.0 + productVersion: 3.5.1 pullPolicy: IfNotPresent mode: cluster mainClass: org.apache.spark.examples.SparkPi diff --git a/docs/modules/spark-k8s/examples/example-history-server.yaml b/docs/modules/spark-k8s/examples/example-history-server.yaml index 64afd14e..11132fd8 100644 --- a/docs/modules/spark-k8s/examples/example-history-server.yaml +++ b/docs/modules/spark-k8s/examples/example-history-server.yaml @@ -5,7 +5,7 @@ metadata: name: spark-history spec: image: - productVersion: 3.5.0 + productVersion: 3.5.1 logFileDirectory: # <1> s3: prefix: eventlogs/ # <2> diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml index 27173724..b94200b6 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml @@ -6,7 +6,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.0 + productVersion: 3.5.1 mode: cluster mainApplicationFile: s3a://stackable-spark-k8s-jars/jobs/ny-tlc-report-1.1.0.jar # <3> mainClass: tech.stackable.demo.spark.NYTLCReport diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-external-dependencies.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-external-dependencies.yaml index c9a700ba..35faf6c6 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-external-dependencies.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-external-dependencies.yaml @@ -6,7 +6,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.0 + productVersion: 3.5.1 mode: cluster mainApplicationFile: s3a://stackable-spark-k8s-jars/jobs/ny_tlc_report.py # <1> args: diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml index 15428794..98cfc3a3 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml @@ -7,7 +7,7 @@ metadata: spec: image: docker.stackable.tech/stackable/ny-tlc-report:0.1.0 # <1> sparkImage: - productVersion: 3.5.0 + productVersion: 3.5.1 mode: cluster mainApplicationFile: local:///stackable/spark/jobs/ny_tlc_report.py # <2> args: diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml index 322fd315..8a8fc1cf 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml @@ -6,7 +6,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.0 + productVersion: 3.5.1 mode: cluster mainApplicationFile: s3a://stackable-spark-k8s-jars/jobs/ny-tlc-report-1.0-SNAPSHOT.jar # <1> mainClass: org.example.App # <2> diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml index 0c0b6167..12869a16 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml @@ -5,7 +5,7 @@ metadata: name: example-sparkapp-s3-private spec: sparkImage: - productVersion: 3.5.0 + productVersion: 3.5.1 mode: cluster mainApplicationFile: s3a://my-bucket/spark-examples.jar # <1> mainClass: org.apache.spark.examples.SparkPi # <2> diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml index 572bb7e5..60e0c244 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml @@ -6,7 +6,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.0 + productVersion: 3.5.1 mode: cluster mainApplicationFile: local:///stackable/spark/examples/src/main/python/streaming/hdfs_wordcount.py args: diff --git a/docs/modules/spark-k8s/examples/getting_started/getting_started.sh b/docs/modules/spark-k8s/examples/getting_started/getting_started.sh index e6b1ef29..8dcc2188 100755 --- a/docs/modules/spark-k8s/examples/getting_started/getting_started.sh +++ b/docs/modules/spark-k8s/examples/getting_started/getting_started.sh @@ -60,7 +60,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.0 + productVersion: 3.5.1 mode: cluster mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py driver: diff --git a/docs/modules/spark-k8s/examples/getting_started/getting_started.sh.j2 b/docs/modules/spark-k8s/examples/getting_started/getting_started.sh.j2 index a7e2a954..1af6ea83 100755 --- a/docs/modules/spark-k8s/examples/getting_started/getting_started.sh.j2 +++ b/docs/modules/spark-k8s/examples/getting_started/getting_started.sh.j2 @@ -60,7 +60,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.0 + productVersion: 3.5.1 mode: cluster mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py driver: diff --git a/docs/modules/spark-k8s/partials/supported-versions.adoc b/docs/modules/spark-k8s/partials/supported-versions.adoc index 5564f48b..cd1a86c2 100644 --- a/docs/modules/spark-k8s/partials/supported-versions.adoc +++ b/docs/modules/spark-k8s/partials/supported-versions.adoc @@ -3,6 +3,7 @@ // Stackable Platform documentation. // Please sort the versions in descending order (newest first) -- 3.5.0 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 11) +- 3.5.1 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 11) +- 3.5.0 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 11) (deprecated) - 3.4.2 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 11) (LTS) - 3.4.1 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 11) (deprecated) diff --git a/examples/README-examples.md b/examples/README-examples.md index f89364d9..b14e0a93 100644 --- a/examples/README-examples.md +++ b/examples/README-examples.md @@ -50,10 +50,10 @@ Several resources are needed in this store. These can be loaded like this: ````text kubectl exec minio-mc-0 -- sh -c 'mc alias set test-minio http://test-minio:9000/' -kubectl cp examples/ny-tlc-report-1.1.0-3.5.0.jar minio-mc-0:/tmp +kubectl cp examples/ny-tlc-report-1.1.0-3.5.1.jar minio-mc-0:/tmp kubectl cp apps/ny_tlc_report.py minio-mc-0:/tmp kubectl cp examples/yellow_tripdata_2021-07.csv minio-mc-0:/tmp -kubectl exec minio-mc-0 -- mc cp /tmp/ny-tlc-report-1.1.0-3.5.0.jar test-minio/my-bucket +kubectl exec minio-mc-0 -- mc cp /tmp/ny-tlc-report-1.1.0-3.5.1.jar test-minio/my-bucket kubectl exec minio-mc-0 -- mc cp /tmp/ny_tlc_report.py test-minio/my-bucket kubectl exec minio-mc-0 -- mc cp /tmp/yellow_tripdata_2021-07.csv test-minio/my-bucket ```` diff --git a/examples/ny-tlc-report-1.1.0-3.5.0.jar b/examples/ny-tlc-report-1.1.0-3.5.1.jar similarity index 89% rename from examples/ny-tlc-report-1.1.0-3.5.0.jar rename to examples/ny-tlc-report-1.1.0-3.5.1.jar index c059ccbc..7ea1fbf8 100644 Binary files a/examples/ny-tlc-report-1.1.0-3.5.0.jar and b/examples/ny-tlc-report-1.1.0-3.5.1.jar differ diff --git a/examples/ny-tlc-report-external-dependencies.yaml b/examples/ny-tlc-report-external-dependencies.yaml index 965e3122..3fbb936f 100644 --- a/examples/ny-tlc-report-external-dependencies.yaml +++ b/examples/ny-tlc-report-external-dependencies.yaml @@ -6,7 +6,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.0 + productVersion: 3.5.1 pullPolicy: IfNotPresent mode: cluster mainApplicationFile: s3a://my-bucket/ny_tlc_report.py diff --git a/examples/ny-tlc-report-image.yaml b/examples/ny-tlc-report-image.yaml index 5adb3328..7f6f3c36 100644 --- a/examples/ny-tlc-report-image.yaml +++ b/examples/ny-tlc-report-image.yaml @@ -7,7 +7,7 @@ metadata: spec: # everything under /jobs will be copied to /stackable/spark/jobs image: docker.stackable.tech/stackable/ny-tlc-report:0.1.0 - sparkImage: docker.stackable.tech/stackable/spark-k8s:3.5.0-stackable0.0.0-dev + sparkImage: docker.stackable.tech/stackable/spark-k8s:3.5.1-stackable0.0.0-dev sparkImagePullPolicy: IfNotPresent mode: cluster mainApplicationFile: local:///stackable/spark/jobs/ny_tlc_report.py diff --git a/examples/ny-tlc-report.yaml b/examples/ny-tlc-report.yaml index eda7a0cd..f659086e 100644 --- a/examples/ny-tlc-report.yaml +++ b/examples/ny-tlc-report.yaml @@ -13,9 +13,9 @@ metadata: name: spark-ny-cm spec: sparkImage: - productVersion: 3.5.0 + productVersion: 3.5.1 mode: cluster - mainApplicationFile: s3a://my-bucket/ny-tlc-report-1.1.0-3.5.0.jar + mainApplicationFile: s3a://my-bucket/ny-tlc-report-1.1.0-3.5.1.jar mainClass: tech.stackable.demo.spark.NYTLCReport volumes: - name: cm-job-arguments diff --git a/rust/crd/src/affinity.rs b/rust/crd/src/affinity.rs index 6de3ebad..bbe30dd1 100644 --- a/rust/crd/src/affinity.rs +++ b/rust/crd/src/affinity.rs @@ -47,7 +47,7 @@ mod test { name: spark-history spec: image: - productVersion: 3.5.0 + productVersion: 3.5.1 logFileDirectory: s3: prefix: eventlogs/ diff --git a/tests/templates/kuttl/pod_overrides/spark-examples_3.5.1.jar b/tests/templates/kuttl/pod_overrides/spark-examples_3.5.1.jar new file mode 100644 index 00000000..f5808840 Binary files /dev/null and b/tests/templates/kuttl/pod_overrides/spark-examples_3.5.1.jar differ diff --git a/tests/templates/kuttl/smoke/spark-examples_3.5.1.jar b/tests/templates/kuttl/smoke/spark-examples_3.5.1.jar new file mode 100644 index 00000000..f5808840 Binary files /dev/null and b/tests/templates/kuttl/smoke/spark-examples_3.5.1.jar differ diff --git a/tests/templates/kuttl/spark-history-server/spark-examples_3.5.1.jar b/tests/templates/kuttl/spark-history-server/spark-examples_3.5.1.jar new file mode 100644 index 00000000..f5808840 Binary files /dev/null and b/tests/templates/kuttl/spark-history-server/spark-examples_3.5.1.jar differ diff --git a/tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.5.1.jar b/tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.5.1.jar new file mode 100644 index 00000000..7ea1fbf8 Binary files /dev/null and b/tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.5.1.jar differ diff --git a/tests/templates/kuttl/spark-pi-private-s3/spark-examples_3.5.1.jar b/tests/templates/kuttl/spark-pi-private-s3/spark-examples_3.5.1.jar new file mode 100644 index 00000000..f5808840 Binary files /dev/null and b/tests/templates/kuttl/spark-pi-private-s3/spark-examples_3.5.1.jar differ diff --git a/tests/templates/kuttl/spark-pi-public-s3/spark-examples_3.5.1.jar b/tests/templates/kuttl/spark-pi-public-s3/spark-examples_3.5.1.jar new file mode 100644 index 00000000..f5808840 Binary files /dev/null and b/tests/templates/kuttl/spark-pi-public-s3/spark-examples_3.5.1.jar differ diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index e6c58523..d752701e 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -8,13 +8,14 @@ dimensions: - 3.4.1 - 3.4.2 - 3.5.0 + - 3.5.1 # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version # as in the example below. - # - 3.5.0,docker.stackable.tech/sandbox/spark-k8s:3.5.0-stackable0.0.0-dev + # - 3.5.1,docker.stackable.tech/sandbox/spark-k8s:3.5.1-stackable0.0.0-dev - name: spark-delta-lake values: - - 3.5.0 - # - 3.5.0,docker.stackable.tech/sandbox/spark-k8s:3.5.0-stackable0.0.0-dev + - 3.5.1 + # - 3.5.1,docker.stackable.tech/sandbox/spark-k8s:3.5.1-stackable0.0.0-dev - name: ny-tlc-report values: - 0.1.0