diff --git a/CHANGELOG.md b/CHANGELOG.md index 17960ea0..0b9f308c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,13 +11,19 @@ All notable changes to this project will be documented in this file. - `affinity` - `volumes` - `volumeMounts` +- Update tests and docs to Spark version 3.5.2 ([#459]) ### Fixed - Fix `envOverrides` for SparkApplication and SparkHistoryServer ([#451]). +### Removed + +- Support for Spark versions 3.4.2 and 3.4.3 has been dropped ([#459]). + [#450]: https://github.com/stackabletech/spark-k8s-operator/pull/450 [#451]: https://github.com/stackabletech/spark-k8s-operator/pull/451 +[#459]: https://github.com/stackabletech/spark-k8s-operator/pull/459 ## [24.7.0] - 2024-07-24 diff --git a/docs/modules/spark-k8s/examples/example-history-app.yaml b/docs/modules/spark-k8s/examples/example-history-app.yaml index 93945ce2..e3062f96 100644 --- a/docs/modules/spark-k8s/examples/example-history-app.yaml +++ b/docs/modules/spark-k8s/examples/example-history-app.yaml @@ -5,7 +5,7 @@ metadata: name: spark-pi-s3-1 spec: sparkImage: - productVersion: 3.5.1 + productVersion: 3.5.2 pullPolicy: IfNotPresent mode: cluster mainClass: org.apache.spark.examples.SparkPi diff --git a/docs/modules/spark-k8s/examples/example-history-server.yaml b/docs/modules/spark-k8s/examples/example-history-server.yaml index 11132fd8..4b7e1efb 100644 --- a/docs/modules/spark-k8s/examples/example-history-server.yaml +++ b/docs/modules/spark-k8s/examples/example-history-server.yaml @@ -5,7 +5,7 @@ metadata: name: spark-history spec: image: - productVersion: 3.5.1 + productVersion: 3.5.2 logFileDirectory: # <1> s3: prefix: eventlogs/ # <2> diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml index b94200b6..c95898dc 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml @@ -6,7 +6,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.1 + productVersion: 3.5.2 mode: cluster mainApplicationFile: s3a://stackable-spark-k8s-jars/jobs/ny-tlc-report-1.1.0.jar # <3> mainClass: tech.stackable.demo.spark.NYTLCReport diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml index b2540c41..06dfe88a 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml @@ -7,7 +7,7 @@ metadata: spec: image: docker.stackable.tech/stackable/ny-tlc-report:0.2.0 # <1> sparkImage: - productVersion: 3.5.1 + productVersion: 3.5.2 mode: cluster mainApplicationFile: local:///stackable/spark/jobs/ny_tlc_report.py # <2> args: diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml index 8a8fc1cf..cfd8addd 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml @@ -6,7 +6,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.1 + productVersion: 3.5.2 mode: cluster mainApplicationFile: s3a://stackable-spark-k8s-jars/jobs/ny-tlc-report-1.0-SNAPSHOT.jar # <1> mainClass: org.example.App # <2> diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml index 12869a16..04535100 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml @@ -5,7 +5,7 @@ metadata: name: example-sparkapp-s3-private spec: sparkImage: - productVersion: 3.5.1 + productVersion: 3.5.2 mode: cluster mainApplicationFile: s3a://my-bucket/spark-examples.jar # <1> mainClass: org.apache.spark.examples.SparkPi # <2> diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml index 60e0c244..37c09637 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml @@ -6,7 +6,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.1 + productVersion: 3.5.2 mode: cluster mainApplicationFile: local:///stackable/spark/examples/src/main/python/streaming/hdfs_wordcount.py args: diff --git a/docs/modules/spark-k8s/examples/getting_started/getting_started.sh b/docs/modules/spark-k8s/examples/getting_started/getting_started.sh index 7dc9fac9..2c04e4ac 100755 --- a/docs/modules/spark-k8s/examples/getting_started/getting_started.sh +++ b/docs/modules/spark-k8s/examples/getting_started/getting_started.sh @@ -73,7 +73,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.1 + productVersion: 3.5.2 mode: cluster mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py driver: diff --git a/docs/modules/spark-k8s/examples/getting_started/getting_started.sh.j2 b/docs/modules/spark-k8s/examples/getting_started/getting_started.sh.j2 index 9cbda5a2..0a34b9b3 100755 --- a/docs/modules/spark-k8s/examples/getting_started/getting_started.sh.j2 +++ b/docs/modules/spark-k8s/examples/getting_started/getting_started.sh.j2 @@ -73,7 +73,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.1 + productVersion: 3.5.2 mode: cluster mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py driver: diff --git a/docs/modules/spark-k8s/pages/usage-guide/job-dependencies.adoc b/docs/modules/spark-k8s/pages/usage-guide/job-dependencies.adoc index f22b8840..ee366676 100644 --- a/docs/modules/spark-k8s/pages/usage-guide/job-dependencies.adoc +++ b/docs/modules/spark-k8s/pages/usage-guide/job-dependencies.adoc @@ -57,7 +57,7 @@ Below is an example of a custom image that includes a JDBC driver: [source, Dockerfile] ---- -FROM docker.stackable.tech/stackable/spark-k8s:3.5.1-stackable24.3.0 # <1> +FROM docker.stackable.tech/stackable/spark-k8s:3.5.2-stackable24.11.0 # <1> RUN curl --fail -o /stackable/spark/jars/postgresql-42.6.0.jar "https://jdbc.postgresql.org/download/postgresql-42.6.0.jar" ---- @@ -74,8 +74,8 @@ metadata: name: spark-jdbc spec: sparkImage: - custom: "docker.stackable.tech/sandbox/spark-k8s:3.5.1-stackable0.0.0-dev" # <1> - productVersion: "3.5.1" # <2> + custom: "docker.stackable.tech/sandbox/spark-k8s:3.5.2-stackable0.0.0-dev" # <1> + productVersion: "3.5.2" # <2> pullPolicy: IfNotPresent # <3> ... ---- @@ -160,9 +160,8 @@ spec: deps: requirements: - tabulate==0.8.9 # <2> -... +... ---- <1> The main application file. In this example it is assumed that the file is part of a custom image. <2> A Python package that is used by the application and installed when the application is submitted. - diff --git a/docs/modules/spark-k8s/partials/supported-versions.adoc b/docs/modules/spark-k8s/partials/supported-versions.adoc index 4ec5fbdb..3928356f 100644 --- a/docs/modules/spark-k8s/partials/supported-versions.adoc +++ b/docs/modules/spark-k8s/partials/supported-versions.adoc @@ -3,6 +3,5 @@ // Stackable Platform documentation. // Please sort the versions in descending order (newest first) -- 3.5.1 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 17) (LTS) -- 3.4.3 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 11) (deprecated) -- 3.4.2 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 11) (deprecated) +- 3.5.2 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 17) (LTS) +- 3.5.1 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 17) diff --git a/examples/README-examples.md b/examples/README-examples.md index 6dbffb7b..4466c7ea 100644 --- a/examples/README-examples.md +++ b/examples/README-examples.md @@ -50,10 +50,10 @@ Several resources are needed in this store. These can be loaded like this: ````text kubectl exec minio-mc-0 -- sh -c 'mc alias set test-minio http://test-minio:9000/' -kubectl cp examples/ny-tlc-report-1.1.0-3.5.1.jar minio-mc-0:/tmp +kubectl cp examples/ny-tlc-report-1.1.0-3.5.2.jar minio-mc-0:/tmp kubectl cp apps/ny_tlc_report.py minio-mc-0:/tmp kubectl cp examples/yellow_tripdata_2021-07.csv minio-mc-0:/tmp -kubectl exec minio-mc-0 -- mc cp /tmp/ny-tlc-report-1.1.0-3.5.1.jar test-minio/my-bucket +kubectl exec minio-mc-0 -- mc cp /tmp/ny-tlc-report-1.1.0-3.5.2.jar test-minio/my-bucket kubectl exec minio-mc-0 -- mc cp /tmp/ny_tlc_report.py test-minio/my-bucket kubectl exec minio-mc-0 -- mc cp /tmp/yellow_tripdata_2021-07.csv test-minio/my-bucket ```` diff --git a/examples/ny-tlc-report-1.1.0-3.5.1.jar b/examples/ny-tlc-report-1.1.0-3.5.2.jar similarity index 90% rename from examples/ny-tlc-report-1.1.0-3.5.1.jar rename to examples/ny-tlc-report-1.1.0-3.5.2.jar index 7ea1fbf8..c88b521e 100644 Binary files a/examples/ny-tlc-report-1.1.0-3.5.1.jar and b/examples/ny-tlc-report-1.1.0-3.5.2.jar differ diff --git a/examples/ny-tlc-report-external-dependencies.yaml b/examples/ny-tlc-report-external-dependencies.yaml index 3fbb936f..39cc12b8 100644 --- a/examples/ny-tlc-report-external-dependencies.yaml +++ b/examples/ny-tlc-report-external-dependencies.yaml @@ -6,7 +6,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.1 + productVersion: 3.5.2 pullPolicy: IfNotPresent mode: cluster mainApplicationFile: s3a://my-bucket/ny_tlc_report.py diff --git a/examples/ny-tlc-report-image.yaml b/examples/ny-tlc-report-image.yaml index 507080e9..64d13bae 100644 --- a/examples/ny-tlc-report-image.yaml +++ b/examples/ny-tlc-report-image.yaml @@ -7,7 +7,7 @@ metadata: spec: # everything under /jobs will be copied to /stackable/spark/jobs image: docker.stackable.tech/stackable/ny-tlc-report:0.2.0 - sparkImage: docker.stackable.tech/stackable/spark-k8s:3.5.1-stackable0.0.0-dev + sparkImage: docker.stackable.tech/stackable/spark-k8s:3.5.2-stackable0.0.0-dev sparkImagePullPolicy: IfNotPresent mode: cluster mainApplicationFile: local:///stackable/spark/jobs/ny_tlc_report.py diff --git a/examples/ny-tlc-report.yaml b/examples/ny-tlc-report.yaml index f659086e..3481281c 100644 --- a/examples/ny-tlc-report.yaml +++ b/examples/ny-tlc-report.yaml @@ -13,9 +13,9 @@ metadata: name: spark-ny-cm spec: sparkImage: - productVersion: 3.5.1 + productVersion: 3.5.2 mode: cluster - mainApplicationFile: s3a://my-bucket/ny-tlc-report-1.1.0-3.5.1.jar + mainApplicationFile: s3a://my-bucket/ny-tlc-report-1.1.0-3.5.2.jar mainClass: tech.stackable.demo.spark.NYTLCReport volumes: - name: cm-job-arguments diff --git a/rust/crd/src/affinity.rs b/rust/crd/src/affinity.rs index e4511481..e3ae617e 100644 --- a/rust/crd/src/affinity.rs +++ b/rust/crd/src/affinity.rs @@ -47,7 +47,7 @@ mod test { name: spark-history spec: image: - productVersion: 3.5.1 + productVersion: 3.5.2 logFileDirectory: s3: prefix: eventlogs/ diff --git a/rust/crd/src/history.rs b/rust/crd/src/history.rs index 882dec2c..f2983864 100644 --- a/rust/crd/src/history.rs +++ b/rust/crd/src/history.rs @@ -473,7 +473,7 @@ mod test { name: spark-history spec: image: - productVersion: 3.5.1 + productVersion: 3.5.2 logFileDirectory: s3: prefix: eventlogs/ diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 3302a4ec..ade3b6ee 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -5,15 +5,14 @@ dimensions: - "false" - name: spark values: - - 3.4.2 - - 3.4.3 - 3.5.1 + - 3.5.2 # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version # as in the example below. # - 3.5.1,docker.stackable.tech/sandbox/spark-k8s:3.5.1-stackable0.0.0-dev - name: spark-delta-lake values: - - 3.5.1 + - 3.5.2 # - 3.5.1,docker.stackable.tech/sandbox/spark-k8s:3.5.1-stackable0.0.0-dev - name: ny-tlc-report values: