feat: remove all refs to the extra-jars folder

Author: razvan

CHANGELOG.md

@@ -13,6 +13,8 @@ All notable changes to this project will be documented in this file.
 - Various documentation of the CRD ([#319]).
 - [BREAKING] Removed version field. Several attributes have been changed to mandatory. While this change is
   technically breaking, existing Spark jobs would not have worked before as these attributes were necessary ([#319]).
+- [BREAKING] Remove `userClassPathFirst` properties from `spark-submit` and all references to the
+  `/stackable/spark/extra-jars` folder. This PR is paired with the `docker-images` PR #556 which removes that folder. ([#350])
 
 ### Fixed
 
@@ -22,6 +24,7 @@ All notable changes to this project will be documented in this file.
 [#313]: https://github.com/stackabletech/spark-k8s-operator/pull/313
 [#319]: https://github.com/stackabletech/spark-k8s-operator/pull/319
 [#344]: https://github.com/stackabletech/spark-k8s-operator/pull/344
+[#350]: https://github.com/stackabletech/spark-k8s-operator/pull/350
 
 ## [23.11.0] - 2023-11-24
 

rust/operator-binary/src/history/history_controller.rs

@@ -671,11 +671,6 @@ fn env_vars(s3logdir: &S3LogDir) -> Vec<EnvVar> {
         value: Some("true".into()),
         value_from: None,
     });
-    vars.push(EnvVar {
-        name: "SPARK_DAEMON_CLASSPATH".to_string(),
-        value: Some("/stackable/spark/extra-jars/*".into()),
-        value_from: None,
-    });
 
     let mut history_opts = vec![
         format!("-Dlog4j.configurationFile={VOLUME_MOUNT_PATH_LOG_CONFIG}/{LOG4J2_CONFIG_FILE}"),

rust/operator-binary/src/spark_k8s_controller.rs

@@ -762,8 +762,7 @@ fn spark_job(
         .add_env_var(
             "SPARK_SUBMIT_OPTS",
             format!(
-                "-cp /stackable/spark/extra-jars/*:/stackable/spark/jars/* \
-                -Dlog4j.configurationFile={VOLUME_MOUNT_PATH_LOG_CONFIG}/{LOG4J2_CONFIG_FILE}"
+            "-Dlog4j.configurationFile={VOLUME_MOUNT_PATH_LOG_CONFIG}/{LOG4J2_CONFIG_FILE}"
             ),
         )
         // TODO: move this to the image

tests/templates/kuttl/delta/40-spark-app.yaml.j2

@@ -8,11 +8,11 @@ spec:
   vectorAggregatorConfigMapName: vector-aggregator-discovery
 {% endif %}
   sparkImage:
-{% if test_scenario['values']['spark'].find(",") > 0 %}
-    custom: "{{ test_scenario['values']['spark'].split(',')[1] }}"
-    productVersion: "{{ test_scenario['values']['spark'].split(',')[0] }}"
+{% if test_scenario['values']['spark-delta'].find(",") > 0 %}
+    custom: "{{ test_scenario['values']['spark-delta'].split(',')[1] }}"
+    productVersion: "{{ test_scenario['values']['spark-delta'].split(',')[0] }}"
 {% else %}
-    productVersion: "{{ test_scenario['values']['spark'] }}"
+    productVersion: "{{ test_scenario['values']['spark-delta'] }}"
 {% endif %}
     pullPolicy: IfNotPresent
   mode: cluster

tests/test-definition.yaml

@@ -5,11 +5,18 @@ dimensions:
       - "false"
   - name: spark
     values:
-      - 3.4.1
-      - 3.5.0
+      #- 3.4.1
+      #- 3.5.0
       # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version
       # as in the example below.
-      # - 3.4.0,docker.stackable.tech/sandbox/spark-k8s:3.4.0-stackable0.0.0-dev
+      - 3.4.1,docker.stackable.tech/sandbox/spark-k8s:3.4.1-stackable0.0.0-dev
+      - 3.5.0,docker.stackable.tech/sandbox/spark-k8s:3.5.0-stackable0.0.0-dev
+  - name: spark-delta
+    values:
+      #- 3.5.0
+      - 3.4.1,docker.stackable.tech/sandbox/spark-k8s:3.4.1-stackable0.0.0-dev
+      - 3.5.0,docker.stackable.tech/sandbox/spark-k8s:3.5.0-stackable0.0.0-dev
+
   - name: ny-tlc-report
     values:
       - 0.1.0
@@ -75,7 +82,7 @@ tests:
       - spark
   - name: delta
     dimensions:
-      - spark
+      - spark-delta
       - delta
 suites:
   - name: nightly