Use the CommonConfiguration structure for roles. (#277)

* wip

* Re-add pod overrides for the submit pod.

* Unify driver and executor configurations.

* Successful smoke tests.

* Rename config structs.

* smoke tests pass on kind.

* Refactor, cleanup and split configuration between driver and executor again.

* Update docs, examples and changelog.

* fix typo

* Move replicas under spec.executor.config in tests and examples.

* Clean up  tests.

* Apply suggestions

* Remove old node selector struct

* Added module doc for roles.rs

* Use RoleGroup for executors to make replicas on the same level as executor configuration.

* Update tests with "replicas" directly under "executor".

* Update docs/examples with "replicas" directly under "executor".

* Update rust/crd/src/roles.rs

Co-authored-by: Andrew Kenworthy <[email protected]>

* Implement review feedback.

---------

Co-authored-by: Andrew Kenworthy <[email protected]>

Author: razvan

CHANGELOG.md

@@ -15,12 +15,14 @@ All notable changes to this project will be documented in this file.
 - `operator-rs` `0.44.0` -> `0.48.0` ([#267], [#275]).
 - Removed usages of SPARK_DAEMON_JAVA_OPTS since it's not a reliable way to pass extra JVM options ([#272]).
 - [BREAKING] use product image selection instead of version ([#275]).
+- BREAKING refactored application roles to use `CommonConfiguration` structures from the operator framework ([#277]).
 
 [#267]: https://github.com/stackabletech/spark-k8s-operator/pull/267
 [#268]: https://github.com/stackabletech/spark-k8s-operator/pull/268
 [#269]: https://github.com/stackabletech/spark-k8s-operator/pull/269
 [#272]: https://github.com/stackabletech/spark-k8s-operator/pull/272
 [#275]: https://github.com/stackabletech/spark-k8s-operator/pull/275
+[#277]: https://github.com/stackabletech/spark-k8s-operator/pull/277
 
 ## [23.7.0] - 2023-07-14
 

Cargo.lock

@@ -26,6 +26,7 @@ strum = { version = "0.25", features = ["derive"] }
 tokio = { version = "1.29", features = ["full"] }
 tracing = "0.1"
 tracing-futures = { version = "0.2", features = ["futures-03"] }
+indoc = "2"
 
 # [patch."https://github.com/stackabletech/operator-rs.git"]
 # stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" }

Cargo.toml

@@ -17,7 +17,11 @@ properties:
         - fromVersion: "0.0.0"
           value: "30"
       roles:
-        - name: "node"
+        - name: "submit"
+          required: true
+        - name: "driver"
+          required: true
+        - name: "executor"
           required: true
       asOfVersion: "0.0.0"
       comment: "History server - TTL for successfully resolved domain names."
@@ -36,7 +40,11 @@ properties:
         - fromVersion: "0.0.0"
           value: "0"
       roles:
-        - name: "node"
+        - name: "submit"
+          required: true
+        - name: "driver"
+          required: true
+        - name: "executor"
           required: true
       asOfVersion: "0.0.0"
       comment: "History server - TTL for domain names that cannot be resolved."

deploy/config-spec/properties.yaml

@@ -17,7 +17,11 @@ properties:
         - fromVersion: "0.0.0"
           value: "30"
       roles:
-        - name: "node"
+        - name: "submit"
+          required: true
+        - name: "driver"
+          required: true
+        - name: "executor"
           required: true
       asOfVersion: "0.0.0"
       comment: "History server - TTL for successfully resolved domain names."
@@ -36,7 +40,11 @@ properties:
         - fromVersion: "0.0.0"
           value: "0"
       roles:
-        - name: "node"
+        - name: "submit"
+          required: true
+        - name: "driver"
+          required: true
+        - name: "executor"
           required: true
       asOfVersion: "0.0.0"
       comment: "History server - TTL for domain names that cannot be resolved."

deploy/helm/spark-k8s-operator/configs/properties.yaml

@@ -32,4 +32,4 @@ spec:
               credentials:
                 secretClass: history-credentials-class # <6>
   executor:
-    instances: 1
+    replicas: 1

deploy/helm/spark-k8s-operator/crds/crds.yaml

@@ -20,11 +20,13 @@ spec:
   sparkConf:
     "spark.hadoop.fs.s3a.aws.credentials.provider": "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider"
   driver:
-    volumeMounts:
-      - name: cm-job-arguments # <6>
-        mountPath: /arguments  # <7>
+    config:
+      volumeMounts:
+        - name: cm-job-arguments # <6>
+          mountPath: /arguments  # <7>
   executor:
-    instances: 3
-    volumeMounts:
-      - name: cm-job-arguments # <6>
-        mountPath: /arguments # <7>
+    replicas: 3
+    config:
+      volumeMounts:
+        - name: cm-job-arguments # <6>
+          mountPath: /arguments # <7>

docs/modules/spark-k8s/examples/example-history-app.yaml

@@ -24,11 +24,13 @@ spec:
       persistentVolumeClaim:
         claimName: pvc-ksv
   driver:
-    volumeMounts:
-      - name: job-deps
-        mountPath: /dependencies  # <6>
+    config:
+      volumeMounts:
+        - name: job-deps
+          mountPath: /dependencies  # <6>
   executor:
-    instances: 3
-    volumeMounts:
-      - name: job-deps
-        mountPath: /dependencies  # <6>
+    replicas: 3
+    config:
+      volumeMounts:
+        - name: job-deps
+          mountPath: /dependencies  # <6>

docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml

@@ -19,24 +19,27 @@ spec:
   sparkConf: # <5>
     "spark.hadoop.fs.s3a.aws.credentials.provider": "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider"
   job:
-    resources:
-      cpu:
-        min: "1"
-        max: "1"
-      memory:
-        limit: "1Gi"
+    config:
+      resources:
+        cpu:
+          min: "1"
+          max: "1"
+        memory:
+          limit: "1Gi"
   driver:
-    resources:
-      cpu:
-        min: "1"
-        max: "1500m"
-      memory:
-        limit: "1Gi"
+    config:
+      resources:
+        cpu:
+          min: "1"
+          max: "1500m"
+        memory:
+          limit: "1Gi"
   executor:
-    instances: 3
-    resources:
-      cpu:
-        min: "1"
-        max: "4"
-      memory:
-        limit: "2Gi"
+    replicas: 3
+    config:
+      resources:
+        cpu:
+          min: "1"
+          max: "4"
+        memory:
+          limit: "2Gi"

docs/modules/spark-k8s/examples/example-sparkapp-external-dependencies.yaml

@@ -22,11 +22,13 @@ spec:
       persistentVolumeClaim:
         claimName: pvc-ksv
   driver:
-    volumeMounts:
-      - name: job-deps
-        mountPath: /dependencies # <5>
+    config:
+      volumeMounts:
+        - name: job-deps
+          mountPath: /dependencies # <5>
   executor:
-    instances: 3
-    volumeMounts:
-      - name: job-deps
-        mountPath: /dependencies # <5>
+    replicas: 3
+    config:
+      volumeMounts:
+        - name: job-deps
+          mountPath: /dependencies # <5>

docs/modules/spark-k8s/examples/example-sparkapp-image.yaml

@@ -22,4 +22,4 @@ spec:
     spark.driver.extraClassPath: "/dependencies/jars/hadoop-aws-3.2.0.jar:/dependencies/jars/aws-java-sdk-bundle-1.11.375.jar"
     spark.executor.extraClassPath: "/dependencies/jars/hadoop-aws-3.2.0.jar:/dependencies/jars/aws-java-sdk-bundle-1.11.375.jar"
   executor:
-    instances: 3
+    replicas: 3

docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml

@@ -17,17 +17,19 @@ spec:
     spark.kubernetes.driver.pod.name: "pyspark-streaming-driver"
     spark.kubernetes.executor.podNamePrefix: "pyspark-streaming"
   driver:
-    resources:
-      cpu:
-        min: "1"
-        max: "2"
-      memory:
-        limit: "1Gi"
+    config:
+      resources:
+        cpu:
+          min: "1"
+          max: "2"
+        memory:
+          limit: "1Gi"
   executor:
-    instances: 1
-    resources:
-      cpu:
-        min: "1700m"
-        max: "3"
-      memory:
-        limit: "2Gi"
+    replicas: 1
+    config:
+      resources:
+        cpu:
+          min: "1700m"
+          max: "3"
+        memory:
+          limit: "2Gi"

docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml

@@ -7,26 +7,29 @@ If no resources are configured explicitly, the operator uses the following defau
 [source,yaml]
 ----
 job:
-  resources:
-    cpu:
-      min: '100m'
-      max: "400m"
-    memory:
-      limit: '512Mi'
+  config:
+    resources:
+      cpu:
+        min: '100m'
+        max: "400m"
+      memory:
+        limit: '512Mi'
 driver:
-  resources:
-    cpu:
-      min: '250m'
-      max: "1"
-    memory:
-      limit: '1Gi'
+  config:
+    resources:
+      cpu:
+        min: '250m'
+        max: "1"
+      memory:
+        limit: '1Gi'
 executor:
-  resources:
-    cpu:
-      min: '250m'
-      max: "1"
-    memory:
-      limit: '4Gi'
+  config:
+    resources:
+      cpu:
+        min: '250m'
+        max: "1"
+      memory:
+        limit: '4Gi'
 ----
 
 For `SparkHistoryServer`s the following defaults are used:
@@ -48,4 +51,3 @@ For more details regarding Kubernetes CPU limits see: https://kubernetes.io/docs
 Spark allocates a default amount of non-heap memory based on the type of job (JVM or non-JVM). This is taken into account when defining memory settings based exclusively on the resource limits, so that the "declared" value is the actual total value (i.e. including memory overhead). This may result in minor deviations from the stated resource value due to rounding differences.
 
 NOTE: It is possible to define Spark resources either directly by setting configuration properties listed under `sparkConf`, or by using resource limits. If both are used, then `sparkConf` properties take precedence. It is recommended for the sake of clarity to use *_either_* one *_or_* the other.
-

docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml

@@ -30,11 +30,13 @@ spec:
       persistentVolumeClaim:
         claimName: pvc-ksv
   driver:
-    volumeMounts:
-      - name: job-deps
-        mountPath: /dependencies
+    config:
+      volumeMounts:
+        - name: job-deps
+          mountPath: /dependencies
   executor:
-    instances: 3
-    volumeMounts:
-      - name: job-deps
-        mountPath: /dependencies
+    config:
+      replicas: 3
+      volumeMounts:
+        - name: job-deps
+          mountPath: /dependencies

docs/modules/spark-k8s/pages/usage-guide/resources.adoc

@@ -25,4 +25,4 @@ spec:
   sparkConf:
     spark.hadoop.fs.s3a.aws.credentials.provider: "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider"
   executor:
-    instances: 3
+    replicas: 3

examples/ny-tlc-report-external-dependencies.yaml

@@ -32,11 +32,13 @@ spec:
   sparkConf:
     spark.hadoop.fs.s3a.aws.credentials.provider: "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider"
   driver:
-    volumeMounts:
-      - name: cm-job-arguments
-        mountPath: /arguments
+    config:
+      volumeMounts:
+        - name: cm-job-arguments
+          mountPath: /arguments
   executor:
-    instances: 3
-    volumeMounts:
-      - name: cm-job-arguments
-        mountPath: /arguments
+    replicas: 3
+    config:
+      volumeMounts:
+        - name: cm-job-arguments
+          mountPath: /arguments

examples/ny-tlc-report-image.yaml

@@ -20,3 +20,4 @@ tracing.workspace = true
 [dev-dependencies]
 rstest.workspace = true
 serde_yaml.workspace = true
+indoc.workspace = true