re-include executor/driver cores in spark-config (#302)

* re-include executor/driver cores in spark-config

* updated changelog

* add executor/cores check by inspecting env-vars

* adapted docs

* check env vars with jsonpath

* replace jq usage with yq

* revert jq usage for testing

* wip: test jq with single test

* re-include tests

* re-include tests II

Author: adwk67

CHANGELOG.md

@@ -25,6 +25,7 @@ All notable changes to this project will be documented in this file.
 ### Fixed
 
 - Dynamic loading of Maven packages ([#281]).
+- Re-instated driver/executor cores setting ([#302]).
 
 ### Removed
 
@@ -41,6 +42,7 @@ All notable changes to this project will be documented in this file.
 [#288]: https://github.com/stackabletech/spark-k8s-operator/pull/288
 [#291]: https://github.com/stackabletech/spark-k8s-operator/pull/291
 [#297]: https://github.com/stackabletech/spark-k8s-operator/pull/297
+[#302]: https://github.com/stackabletech/spark-k8s-operator/pull/302
 
 ## [23.7.0] - 2023-07-14
 

docs/modules/spark-k8s/pages/usage-guide/resources.adoc

@@ -77,7 +77,7 @@ CPU request and limit will be rounded up to the next integer value, resulting in
 |2
 |===
 
-Spark allows CPU limits to be set for the driver and executor using standard Spark settings (`spark.{driver|executor}.cores}`) as well as Kubernetes-specific ones (`spark.kubernetes.{driver,executor}.{request|limit}.cores`). Since `spark.kubernetes.{driver,executor}.request.cores` takes precedence over `spark.{driver|executor}.cores}`, `spark.{driver|executor}.cores}` is not specified by the operator when building the spark-submit configuration.
+Spark allows CPU limits to be set for the driver and executor using Spark settings (`spark.{driver|executor}.cores}`) as well as Kubernetes-specific ones (`spark.kubernetes.{driver,executor}.{request|limit}.cores`). `spark.kubernetes.executor.request.cores` takes precedence over `spark.executor.cores` in determining the pod CPU request, but does not affect task parallelism (the number of tasks an executor can run concurrently), so for this reason `spark.executor.cores` is set to the value of `spark.kubernetes.executor.limit.cores`.
 
 === Memory
 

rust/crd/src/lib.rs

@@ -822,6 +822,7 @@ fn resources_to_driver_props(
         let min_cores = cores_from_quantity(min.0.clone())?;
         let max_cores = cores_from_quantity(max.0.clone())?;
         // will have default value from resources to apply if nothing set specifically
+        props.insert("spark.driver.cores".to_string(), max_cores.clone());
         props.insert(
             "spark.kubernetes.driver.request.cores".to_string(),
             min_cores,
@@ -861,6 +862,7 @@ fn resources_to_executor_props(
         let min_cores = cores_from_quantity(min.0.clone())?;
         let max_cores = cores_from_quantity(max.0.clone())?;
         // will have default value from resources to apply if nothing set specifically
+        props.insert("spark.executor.cores".to_string(), max_cores.clone());
         props.insert(
             "spark.kubernetes.executor.request.cores".to_string(),
             min_cores,
@@ -1041,6 +1043,7 @@ mod tests {
         resources_to_driver_props(true, &driver_config, &mut props).expect("blubb");
 
         let expected: BTreeMap<String, String> = vec![
+            ("spark.driver.cores".to_string(), "1".to_string()),
             ("spark.driver.memory".to_string(), "128m".to_string()),
             (
                 "spark.kubernetes.driver.limit.cores".to_string(),
@@ -1084,6 +1087,7 @@ mod tests {
         resources_to_executor_props(true, &executor_config, &mut props).expect("blubb");
 
         let expected: BTreeMap<String, String> = vec![
+            ("spark.executor.cores".to_string(), "2".to_string()),
             ("spark.executor.memory".to_string(), "128m".to_string()), // 128 and not 512 because memory overhead is subtracted
             (
                 "spark.kubernetes.executor.request.cores".to_string(),

tests/templates/kuttl/resources/10-assert.yaml.j2

@@ -55,8 +55,8 @@ spec:
       resources:
       # these resources are set via Spark submit properties like "spark.executor.cores"
         limits:
-          cpu: "1"
+          cpu: "2"
           memory: 1Gi
         requests:
-          cpu: "1"
+          cpu: "2"
           memory: 1Gi

tests/templates/kuttl/resources/10-deploy-spark-app.yaml.j2

@@ -50,7 +50,7 @@ spec:
         enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
       resources:
         cpu:
-          min: 250m
-          max: 1000m
+          min: 1250m
+          max: 2000m
         memory:
           limit: 1024Mi

tests/templates/kuttl/resources/12-deploy-spark-app.yaml.j2

@@ -25,10 +25,12 @@ spec:
     spark.kubernetes.executor.podNamePrefix: "resources-sparkconf"
     spark.kubernetes.driver.request.cores: "1"
     spark.kubernetes.driver.limit.cores: "1"
+    spark.driver.cores: "1"
     spark.driver.memory: "1g"
     spark.driver.memoryOverheadFactor: "0.4"
     spark.kubernetes.executor.request.cores: "1"
     spark.kubernetes.executor.limit.cores: "2"
+    spark.executor.cores: "2"
     spark.executor.memory: "2g"
     spark.executor.memoryOverheadFactor: "0.4"
     spark.executor.instances: "1"

tests/templates/kuttl/resources/20-assert.yaml.j2

@@ -0,0 +1,7 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 120
+commands:
+  - script: kubectl get pods -n $NAMESPACE resources-crd-exec-1 -o json | jq '.spec.containers[].env[] | select(.name == "SPARK_EXECUTOR_CORES").value' | grep -w "2"
+  - script: kubectl get pods -n $NAMESPACE resources-sparkconf-exec-1 -o json | jq '.spec.containers[].env[] | select(.name == "SPARK_EXECUTOR_CORES").value' | grep -w "2"