New versions 3.4.1 and 3.5.0 (#291)

* new examples for tests

* bump versions

* add jmx exporter for history server

* adapt test matrix to new versions

* fix tests

* adapt test versions

* fix test

* fix examples

* adapted changelog

* bump operator-rs to 0.55.0

* Update tests/templates/kuttl/iceberg/10-deploy-spark-app.yaml.j2

Co-authored-by: Razvan-Daniel Mihai <[email protected]>

* do not use pyspark image anymore

* vector and operator-rs bumps

---------

Co-authored-by: Razvan-Daniel Mihai <[email protected]>

Author: maltesander

CHANGELOG.md

@@ -21,7 +21,7 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 serde_yaml = "0.9"
 snafu = "0.7"
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.52.1" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.55.0" }
 strum = { version = "0.25", features = ["derive"] }
 tokio = { version = "1.29", features = ["full"] }
 tracing = "0.1"

Cargo.lock

@@ -6,7 +6,7 @@ metadata:
 spec:
   version: "1.0"
   sparkImage:
-    productVersion: 3.3.0 # <1>
+    productVersion: 3.5.0 # <1>
   mode: cluster
   mainClass: org.apache.spark.examples.SparkPi
   mainApplicationFile: /stackable/spark/examples/jars/spark-examples.jar # <2>

Cargo.toml

@@ -6,7 +6,7 @@ metadata:
 spec:
   version: "1.0"
   sparkImage:
-    productVersion: 3.3.0
+    productVersion: 3.5.0
     pullPolicy: IfNotPresent
   mode: cluster
   mainClass: org.apache.spark.examples.SparkPi

deploy/helm/spark-k8s-operator/crds/crds.yaml

@@ -5,7 +5,7 @@ metadata:
   name: spark-history
 spec:
   image:
-    productVersion: 3.3.0
+    productVersion: 3.5.0
   logFileDirectory:  # <1>
     s3:
       prefix: eventlogs/  # <2>

docs/modules/spark-k8s/examples/example-encapsulated.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   version: "1.0"
   sparkImage:
-    productVersion: 3.3.0
+    productVersion: 3.5.0
   mode: cluster
   mainApplicationFile: s3a://stackable-spark-k8s-jars/jobs/ny-tlc-report-1.1.0.jar # <3>
   mainClass: tech.stackable.demo.spark.NYTLCReport

docs/modules/spark-k8s/examples/example-history-app.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   version: "1.0"
   sparkImage:
-    productVersion: 3.3.0
+    productVersion: 3.5.0
   mode: cluster
   mainApplicationFile: s3a://stackable-spark-k8s-jars/jobs/ny_tlc_report.py # <1>
   args:

docs/modules/spark-k8s/examples/example-history-server.yaml

@@ -8,7 +8,7 @@ spec:
   version: "1.0"
   image: docker.stackable.tech/stackable/ny-tlc-report:0.1.0 # <1>
   sparkImage:
-    productVersion: 3.3.0
+    productVersion: 3.5.0
   mode: cluster
   mainApplicationFile: local:///stackable/spark/jobs/ny_tlc_report.py # <2>
   args:

docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   version: "1.0"
   sparkImage:
-    productVersion: 3.3.0
+    productVersion: 3.5.0
   mode: cluster
   mainApplicationFile: s3a://stackable-spark-k8s-jars/jobs/ny-tlc-report-1.0-SNAPSHOT.jar # <1>
   mainClass: org.example.App # <2>

docs/modules/spark-k8s/examples/example-sparkapp-external-dependencies.yaml

@@ -6,7 +6,7 @@ metadata:
 spec:
   version: "1.0"
   sparkImage:
-    productVersion: 3.3.0
+    productVersion: 3.5.0
   mode: cluster
   mainApplicationFile: s3a://my-bucket/spark-examples.jar # <1>
   mainClass: org.apache.spark.examples.SparkPi # <2>

docs/modules/spark-k8s/examples/example-sparkapp-image.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   version: "1.0"
   sparkImage:
-    productVersion: 3.3.0
+    productVersion: 3.5.0
   mode: cluster
   mainApplicationFile: local:///stackable/spark/examples/src/main/python/streaming/hdfs_wordcount.py
   args:

docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml

@@ -59,7 +59,7 @@ metadata:
 spec:
   version: "1.0"
   sparkImage:
-    productVersion: 3.3.0
+    productVersion: 3.5.0
   mode: cluster
   mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py
   driver:

docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml

@@ -59,7 +59,7 @@ metadata:
 spec:
   version: "1.0"
   sparkImage:
-    productVersion: 3.3.0
+    productVersion: 3.5.0
   mode: cluster
   mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py
   driver:

docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   version: "1.0"
   sparkImage:
-    productVersion: 3.3.0
+    productVersion: 3.5.0
     pullPolicy: IfNotPresent
   mode: cluster
   mainApplicationFile: s3a://my-bucket/ny_tlc_report.py

docs/modules/spark-k8s/examples/getting_started/getting_started.sh

@@ -8,7 +8,7 @@ spec:
   version: "1.0"
   # everything under /jobs will be copied to /stackable/spark/jobs
   image: docker.stackable.tech/stackable/ny-tlc-report:0.1.0
-  sparkImage: docker.stackable.tech/stackable/pyspark-k8s:3.3.0-stackable0.0.0-dev
+  sparkImage: docker.stackable.tech/stackable/spark-k8s:3.5.0-stackable0.0.0-dev
   sparkImagePullPolicy: IfNotPresent
   mode: cluster
   mainApplicationFile: local:///stackable/spark/jobs/ny_tlc_report.py

docs/modules/spark-k8s/examples/getting_started/getting_started.sh.j2

@@ -14,9 +14,9 @@ metadata:
 spec:
   version: "1.0"
   sparkImage:
-    productVersion: 3.3.0
+    productVersion: 3.5.0
   mode: cluster
-  mainApplicationFile: s3a://my-bucket/ny-tlc-report-1.1.0-3.3.0.jar
+  mainApplicationFile: s3a://my-bucket/ny-tlc-report-1.1.0-3.5.0.jar
   mainClass: tech.stackable.demo.spark.NYTLCReport
   volumes:
     - name: cm-job-arguments

docs/modules/spark-k8s/pages/crd-reference.adoc

@@ -47,7 +47,7 @@ mod test {
           name: spark-history
         spec:
           image:
-            productVersion: 3.3.0
+            productVersion: 3.5.0
           logFileDirectory:
             s3:
               prefix: eventlogs/

docs/modules/spark-k8s/pages/usage-guide/history-server.adoc

@@ -5,7 +5,6 @@ use stackable_operator::{
     builder::{ConfigMapBuilder, ContainerBuilder, ObjectMetaBuilder, PodBuilder, VolumeBuilder},
     cluster_resources::{ClusterResourceApplyStrategy, ClusterResources},
     commons::product_image_selection::ResolvedProductImage,
-    duration::Duration,
     k8s_openapi::{
         api::{
             apps::v1::{StatefulSet, StatefulSetSpec},
@@ -31,6 +30,7 @@ use stackable_operator::{
         },
     },
     role_utils::RoleGroupRef,
+    time::Duration,
 };
 use stackable_spark_k8s_crd::{
     constants::{
@@ -55,6 +55,8 @@ use stackable_operator::k8s_openapi::DeepMerge;
 use stackable_operator::logging::controller::ReconcilerError;
 use strum::{EnumDiscriminants, IntoStaticStr};
 
+const METRICS_PORT: u16 = 18081;
+
 #[derive(Snafu, Debug, EnumDiscriminants)]
 #[strum_discriminants(derive(IntoStaticStr))]
 #[allow(clippy::enum_variant_names)]
@@ -415,6 +417,7 @@ fn build_stateful_set(
         .command(vec!["/bin/bash".to_string()])
         .args(command_args(s3_log_dir))
         .add_container_port("http", 18080)
+        .add_container_port("metrics", METRICS_PORT.into())
         .add_env_vars(env_vars(s3_log_dir))
         .add_volume_mounts(s3_log_dir.volume_mounts())
         .add_volume_mount(VOLUME_MOUNT_NAME_CONFIG, VOLUME_MOUNT_PATH_CONFIG)
@@ -515,15 +518,23 @@ fn build_service(
             .ownerreference_from_resource(shs, None, Some(true))
             .context(ObjectMissingMetadataForOwnerRefSnafu)?
             .with_recommended_labels(labels(shs, app_version_label, &group_name))
+            .with_label("prometheus.io/scrape", "true")
             .build(),
         spec: Some(ServiceSpec {
             type_: Some(service_type),
             cluster_ip: service_cluster_ip,
-            ports: Some(vec![ServicePort {
-                name: Some(String::from("http")),
-                port: 18080,
-                ..ServicePort::default()
-            }]),
+            ports: Some(vec![
+                ServicePort {
+                    name: Some(String::from("http")),
+                    port: 18080,
+                    ..ServicePort::default()
+                },
+                ServicePort {
+                    name: Some(String::from("metrics")),
+                    port: METRICS_PORT.into(),
+                    ..ServicePort::default()
+                },
+            ]),
             selector: Some(selector),
             ..ServiceSpec::default()
         }),
@@ -634,6 +645,7 @@ fn env_vars(s3logdir: &S3LogDir) -> Vec<EnvVar> {
         format!(
             "-Djava.security.properties={VOLUME_MOUNT_PATH_CONFIG}/{JVM_SECURITY_PROPERTIES_FILE}"
         ),
+        format!("-javaagent:/stackable/jmx/jmx_prometheus_javaagent.jar={METRICS_PORT}:/stackable/jmx/config.yaml")
     ];
     if tlscerts::tls_secret_name(&s3logdir.bucket.connection).is_some() {
         history_opts.extend(

docs/modules/spark-k8s/partials/supported-versions.adoc

@@ -1,6 +1,6 @@
 use stackable_operator::{
-    client::Client, duration::Duration, k8s_openapi::api::core::v1::Pod,
-    kube::runtime::controller::Action,
+    client::Client, k8s_openapi::api::core::v1::Pod, kube::runtime::controller::Action,
+    time::Duration,
 };
 use stackable_spark_k8s_crd::{
     constants::POD_DRIVER_CONTROLLER_NAME, SparkApplication, SparkApplicationStatus,

examples/README-examples.md

@@ -6,7 +6,7 @@ use std::{
     vec,
 };
 
-use stackable_operator::{duration::Duration, product_config::writer::to_java_properties_string};
+use stackable_operator::{product_config::writer::to_java_properties_string, time::Duration};
 use stackable_spark_k8s_crd::{
     constants::*, s3logdir::S3LogDir, tlscerts, RoleConfig, SparkApplication, SparkApplicationRole,
     SparkContainer, SubmitConfig,

tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.4.0.jar renamed to examples/ny-tlc-report-1.1.0-3.5.0.jar

@@ -9,9 +9,4 @@ kind: SparkApplication
 metadata:
   name: pyspark-iceberg
 status:
-{% if test_scenario['values']['spark'].startswith("3.3") %}
-  # Spark 3.3 is expected to fail because of this https://issues.apache.org/jira/browse/SPARK-35084
-  phase: Failed
-{% else %}
   phase: Succeeded
-{% endif %}

examples/ny-tlc-report-external-dependencies.yaml

@@ -45,7 +45,8 @@ spec:
           mountPath: /stackable/spark/jobs
   deps:
     packages:
-      - org.apache.iceberg:iceberg-spark-runtime-{{ test_scenario['values']['spark'].rstrip('.0') }}_2.12:1.3.1
+      # need to extract only the major and minor versions
+      - org.apache.iceberg:iceberg-spark-runtime-{{ test_scenario['values']['spark'].rsplit('.', maxsplit=1)[0] }}_2.12:1.4.0
   volumes:
     - name: script
       configMap: