fix(tests): update Python script for logging tests

Author: razvan

rust/crd/src/lib.rs

@@ -484,11 +484,8 @@ impl SparkApplication {
             format!("--conf spark.kubernetes.authenticate.driver.serviceAccountName={}", serviceaccount_name),
             format!("--conf spark.driver.defaultJavaOptions=-Dlog4j.configurationFile={VOLUME_MOUNT_PATH_LOG_CONFIG}/{LOG4J2_CONFIG_FILE}"),
             format!("--conf spark.driver.extraClassPath=/stackable/spark/extra-jars/*"),
-            //"--conf spark.driver.userClassPathFirst=true".to_string(),
-            "--conf spark.driver.userClassPathFirst=true".to_string(),
             format!("--conf spark.executor.defaultJavaOptions=-Dlog4j.configurationFile={VOLUME_MOUNT_PATH_LOG_CONFIG}/{LOG4J2_CONFIG_FILE}"),
             format!("--conf spark.executor.extraClassPath=/stackable/spark/extra-jars/*"),
-            //"--conf spark.executor.userClassPathFirst=true".to_string(),
         ]);
 
         // See https://spark.apache.org/docs/latest/running-on-kubernetes.html#dependency-management

tests/templates/kuttl/logging/test_log_aggregation.py

@@ -37,6 +37,12 @@ def check_sent_events():
             assert sentEvents is None or \
                 sentEvents['sentEventsTotal'] == 0, \
                 'Invalid log events were sent.'
+        elif componentId.find('LogConfigSubmitSpark') > 0:
+            # [*LogConfigSubmitSpark] components do not log in XML format
+            # because spark-submit doesn't know about the /stackable/spark/extra-jars folder.
+            assert sentEvents is None or \
+                sentEvents['sentEventsTotal'] == 0, \
+                f'Found events for {componentId} but did not expect any.'
         else:
             assert sentEvents is not None and \
                 sentEvents['sentEventsTotal'] > 0, \

tests/test-definition.yaml

@@ -5,17 +5,15 @@ dimensions:
       - "false"
   - name: spark
     values:
-      # - 3.4.1
+      - 3.4.1
       - 3.5.0
       # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version
       # as in the example below.
-      #- 3.4.1,docker.stackable.tech/sandbox/spark-k8s:3.4.1-stackable0.0.0-dev
-      #- 3.5.0,docker.stackable.tech/sandbox/spark-k8s:3.5.0-stackable0.0.0-dev
+      # - 3.5.0,docker.stackable.tech/sandbox/spark-k8s:3.5.0-stackable0.0.0-dev
   - name: spark-delta
     values:
-      # - 3.5.0
-      - 3.5.0,docker.stackable.tech/sandbox/spark-k8s:3.5.0-stackable0.0.0-dev
-
+      - 3.5.0
+      # - 3.5.0,docker.stackable.tech/sandbox/spark-k8s:3.5.0-stackable0.0.0-dev
   - name: ny-tlc-report
     values:
       - 0.1.0