Revert extra-jars removal but don't set the userClassPathFirst properties.

Author: razvan

rust/crd/src/lib.rs

@@ -483,7 +483,12 @@ impl SparkApplication {
             format!("--conf spark.kubernetes.executor.container.image={}", spark_image.to_string()),
             format!("--conf spark.kubernetes.authenticate.driver.serviceAccountName={}", serviceaccount_name),
             format!("--conf spark.driver.defaultJavaOptions=-Dlog4j.configurationFile={VOLUME_MOUNT_PATH_LOG_CONFIG}/{LOG4J2_CONFIG_FILE}"),
-            format!("--conf spark.executor.defaultJavaOptions=-Dlog4j.configurationFile={VOLUME_MOUNT_PATH_LOG_CONFIG}/{LOG4J2_CONFIG_FILE}")
+            format!("--conf spark.driver.extraClassPath=/stackable/spark/extra-jars/*"),
+            //"--conf spark.driver.userClassPathFirst=true".to_string(),
+            "--conf spark.driver.userClassPathFirst=true".to_string(),
+            format!("--conf spark.executor.defaultJavaOptions=-Dlog4j.configurationFile={VOLUME_MOUNT_PATH_LOG_CONFIG}/{LOG4J2_CONFIG_FILE}"),
+            format!("--conf spark.executor.extraClassPath=/stackable/spark/extra-jars/*"),
+            //"--conf spark.executor.userClassPathFirst=true".to_string(),
         ]);
 
         // See https://spark.apache.org/docs/latest/running-on-kubernetes.html#dependency-management

rust/operator-binary/src/history/history_controller.rs

@@ -671,6 +671,11 @@ fn env_vars(s3logdir: &S3LogDir) -> Vec<EnvVar> {
         value: Some("true".into()),
         value_from: None,
     });
+    vars.push(EnvVar {
+        name: "SPARK_DAEMON_CLASSPATH".to_string(),
+        value: Some("/stackable/spark/extra-jars/*".into()),
+        value_from: None,
+    });
 
     let mut history_opts = vec![
         format!("-Dlog4j.configurationFile={VOLUME_MOUNT_PATH_LOG_CONFIG}/{LOG4J2_CONFIG_FILE}"),

tests/test-definition.yaml

@@ -6,11 +6,11 @@ dimensions:
   - name: spark
     values:
       # - 3.4.1
-      # - 3.5.0
+      - 3.5.0
       # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version
       # as in the example below.
-      - 3.4.1,docker.stackable.tech/sandbox/spark-k8s:3.4.1-stackable0.0.0-dev
-      - 3.5.0,docker.stackable.tech/sandbox/spark-k8s:3.5.0-stackable0.0.0-dev
+      #- 3.4.1,docker.stackable.tech/sandbox/spark-k8s:3.4.1-stackable0.0.0-dev
+      #- 3.5.0,docker.stackable.tech/sandbox/spark-k8s:3.5.0-stackable0.0.0-dev
   - name: spark-delta
     values:
       # - 3.5.0