stackabletech · razvan · Sep 25, 2023 · Sep 1, 2023 · Sep 2, 2023 · Sep 4, 2023
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -15,12 +15,14 @@ All notable changes to this project will be documented in this file.
 - `operator-rs` `0.44.0` -> `0.48.0` ([#267], [#275]).
 - Removed usages of SPARK_DAEMON_JAVA_OPTS since it's not a reliable way to pass extra JVM options ([#272]).
 - [BREAKING] use product image selection instead of version ([#275]).
+- BREAKING refactored application roles to use `CommonConfiguration` structures from the operator framework ([#277]).
 
 [#267]: https://github.com/stackabletech/spark-k8s-operator/pull/267
 [#268]: https://github.com/stackabletech/spark-k8s-operator/pull/268
 [#269]: https://github.com/stackabletech/spark-k8s-operator/pull/269
 [#272]: https://github.com/stackabletech/spark-k8s-operator/pull/272
 [#275]: https://github.com/stackabletech/spark-k8s-operator/pull/275
+[#277]: https://github.com/stackabletech/spark-k8s-operator/pull/277
 
 ## [23.7.0] - 2023-07-14
 

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -26,6 +26,7 @@ strum = { version = "0.25", features = ["derive"] }
 tokio = { version = "1.29", features = ["full"] }
 tracing = "0.1"
 tracing-futures = { version = "0.2", features = ["futures-03"] }
+indoc = "2"
 
 # [patch."https://github.com/stackabletech/operator-rs.git"]
 # stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" }
diff --git a/deploy/config-spec/properties.yaml b/deploy/config-spec/properties.yaml
@@ -17,7 +17,11 @@ properties:
         - fromVersion: "0.0.0"
           value: "30"
       roles:
-        - name: "node"
+        - name: "submit"
+          required: true
+        - name: "driver"
+          required: true
+        - name: "executor"
           required: true
       asOfVersion: "0.0.0"
       comment: "History server - TTL for successfully resolved domain names."
@@ -36,7 +40,11 @@ properties:
         - fromVersion: "0.0.0"
           value: "0"
       roles:
-        - name: "node"
+        - name: "submit"
+          required: true
+        - name: "driver"
+          required: true
+        - name: "executor"
           required: true
       asOfVersion: "0.0.0"
       comment: "History server - TTL for domain names that cannot be resolved."

diff --git a/deploy/helm/spark-k8s-operator/configs/properties.yaml b/deploy/helm/spark-k8s-operator/configs/properties.yaml
@@ -17,7 +17,11 @@ properties:
         - fromVersion: "0.0.0"
           value: "30"
       roles:
-        - name: "node"
+        - name: "submit"
+          required: true
+        - name: "driver"
+          required: true
+        - name: "executor"
           required: true
       asOfVersion: "0.0.0"
       comment: "History server - TTL for successfully resolved domain names."
@@ -36,7 +40,11 @@ properties:
         - fromVersion: "0.0.0"
           value: "0"
       roles:
-        - name: "node"
+        - name: "submit"
+          required: true
+        - name: "driver"
+          required: true
+        - name: "executor"
           required: true
       asOfVersion: "0.0.0"
       comment: "History server - TTL for domain names that cannot be resolved."

diff --git a/deploy/helm/spark-k8s-operator/crds/crds.yaml b/deploy/helm/spark-k8s-operator/crds/crds.yaml
diff --git a/docs/modules/spark-k8s/examples/example-history-app.yaml b/docs/modules/spark-k8s/examples/example-history-app.yaml
@@ -32,4 +32,5 @@ spec:
               credentials:
                 secretClass: history-credentials-class # <6>
   executor:
-    instances: 1
+    config:
+      replicas: 1
diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml
@@ -20,11 +20,13 @@ spec:
   sparkConf:
     "spark.hadoop.fs.s3a.aws.credentials.provider": "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider"
   driver:
-    volumeMounts:
-      - name: cm-job-arguments # <6>
-        mountPath: /arguments  # <7>
+    config:
+      volumeMounts:
+        - name: cm-job-arguments # <6>
+          mountPath: /arguments  # <7>
   executor:
-    instances: 3
-    volumeMounts:
-      - name: cm-job-arguments # <6>
-        mountPath: /arguments # <7>
+    config:
+      replicas: 3
+      volumeMounts:
+        - name: cm-job-arguments # <6>
+          mountPath: /arguments # <7>
diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-external-dependencies.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-external-dependencies.yaml
@@ -24,11 +24,13 @@ spec:
       persistentVolumeClaim:
         claimName: pvc-ksv
   driver:
-    volumeMounts:
-      - name: job-deps
-        mountPath: /dependencies  # <6>
+    config:
+      volumeMounts:
+        - name: job-deps
+          mountPath: /dependencies  # <6>
   executor:
-    instances: 3
-    volumeMounts:
-      - name: job-deps
-        mountPath: /dependencies  # <6>
+    config:
+      replicas: 3
+      volumeMounts:
+        - name: job-deps
+          mountPath: /dependencies  # <6>
diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml
@@ -19,24 +19,27 @@ spec:
   sparkConf: # <5>
     "spark.hadoop.fs.s3a.aws.credentials.provider": "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider"
   job:
-    resources:
-      cpu:
-        min: "1"
-        max: "1"
-      memory:
-        limit: "1Gi"
+    config:
+      resources:
+        cpu:
+          min: "1"
+          max: "1"
+        memory:
+          limit: "1Gi"
   driver:
-    resources:
-      cpu:
-        min: "1"
-        max: "1500m"
-      memory:
-        limit: "1Gi"
+    config:
+      resources:
+        cpu:
+          min: "1"
+          max: "1500m"
+        memory:
+          limit: "1Gi"
   executor:
-    instances: 3
-    resources:
-      cpu:
-        min: "1"
-        max: "4"
-      memory:
-        limit: "2Gi"
+    config:
+      replicas: 3
+      resources:
+        cpu:
+          min: "1"
+          max: "4"
+        memory:
+          limit: "2Gi"
diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml
@@ -22,11 +22,13 @@ spec:
       persistentVolumeClaim:
         claimName: pvc-ksv
   driver:
-    volumeMounts:
-      - name: job-deps
-        mountPath: /dependencies # <5>
+    config:
+      volumeMounts:
+        - name: job-deps
+          mountPath: /dependencies # <5>
   executor:
-    instances: 3
-    volumeMounts:
-      - name: job-deps
-        mountPath: /dependencies # <5>
+    config:
+      replicas: 3
+      volumeMounts:
+        - name: job-deps
+          mountPath: /dependencies # <5>
diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml
@@ -22,4 +22,5 @@ spec:
     spark.driver.extraClassPath: "/dependencies/jars/hadoop-aws-3.2.0.jar:/dependencies/jars/aws-java-sdk-bundle-1.11.375.jar"
     spark.executor.extraClassPath: "/dependencies/jars/hadoop-aws-3.2.0.jar:/dependencies/jars/aws-java-sdk-bundle-1.11.375.jar"
   executor:
-    instances: 3
+    config:
+      replicas: 3
diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml
@@ -17,17 +17,19 @@ spec:
     spark.kubernetes.driver.pod.name: "pyspark-streaming-driver"
     spark.kubernetes.executor.podNamePrefix: "pyspark-streaming"
   driver:
-    resources:
-      cpu:
-        min: "1"
-        max: "2"
-      memory:
-        limit: "1Gi"
+    config:
+      resources:
+        cpu:
+          min: "1"
+          max: "2"
+        memory:
+          limit: "1Gi"
   executor:
-    instances: 1
-    resources:
-      cpu:
-        min: "1700m"
-        max: "3"
-      memory:
-        limit: "2Gi"
+    config:
+      replicas: 1
+      resources:
+        cpu:
+          min: "1700m"
+          max: "3"
+        memory:
+          limit: "2Gi"
diff --git a/docs/modules/spark-k8s/pages/usage-guide/resources.adoc b/docs/modules/spark-k8s/pages/usage-guide/resources.adoc
@@ -7,26 +7,29 @@ If no resources are configured explicitly, the operator uses the following defau
 [source,yaml]
 ----
 job:
-  resources:
-    cpu:
-      min: '100m'
-      max: "400m"
-    memory:
-      limit: '512Mi'
+  config:
+    resources:
+      cpu:
+        min: '100m'
+        max: "400m"
+      memory:
+        limit: '512Mi'
 driver:
-  resources:
-    cpu:
-      min: '250m'
-      max: "1"
-    memory:
-      limit: '1Gi'
+  config:
+    resources:
+      cpu:
+        min: '250m'
+        max: "1"
+      memory:
+        limit: '1Gi'
 executor:
-  resources:
-    cpu:
-      min: '250m'
-      max: "1"
-    memory:
-      limit: '4Gi'
+  config:
+    resources:
+      cpu:
+        min: '250m'
+        max: "1"
+      memory:
+        limit: '4Gi'
 ----
 
 For `SparkHistoryServer`s the following defaults are used:
@@ -48,4 +51,3 @@ For more details regarding Kubernetes CPU limits see: https://kubernetes.io/docs
 Spark allocates a default amount of non-heap memory based on the type of job (JVM or non-JVM). This is taken into account when defining memory settings based exclusively on the resource limits, so that the "declared" value is the actual total value (i.e. including memory overhead). This may result in minor deviations from the stated resource value due to rounding differences.
 
 NOTE: It is possible to define Spark resources either directly by setting configuration properties listed under `sparkConf`, or by using resource limits. If both are used, then `sparkConf` properties take precedence. It is recommended for the sake of clarity to use *_either_* one *_or_* the other.
-
diff --git a/examples/ny-tlc-report-external-dependencies.yaml b/examples/ny-tlc-report-external-dependencies.yaml
@@ -30,11 +30,13 @@ spec:
       persistentVolumeClaim:
         claimName: pvc-ksv
   driver:
-    volumeMounts:
-      - name: job-deps
-        mountPath: /dependencies
+    config:
+      volumeMounts:
+        - name: job-deps
+          mountPath: /dependencies
   executor:
-    instances: 3
-    volumeMounts:
-      - name: job-deps
-        mountPath: /dependencies
+    config:
+      replicas: 3
+      volumeMounts:
+        - name: job-deps
+          mountPath: /dependencies
diff --git a/examples/ny-tlc-report-image.yaml b/examples/ny-tlc-report-image.yaml
@@ -25,4 +25,5 @@ spec:
   sparkConf:
     spark.hadoop.fs.s3a.aws.credentials.provider: "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider"
   executor:
-    instances: 3
+    config:
+      replicas: 3
diff --git a/examples/ny-tlc-report.yaml b/examples/ny-tlc-report.yaml
@@ -32,11 +32,13 @@ spec:
   sparkConf:
     spark.hadoop.fs.s3a.aws.credentials.provider: "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider"
   driver:
-    volumeMounts:
-      - name: cm-job-arguments
-        mountPath: /arguments
+    config:
+      volumeMounts:
+        - name: cm-job-arguments
+          mountPath: /arguments
   executor:
-    instances: 3
-    volumeMounts:
-      - name: cm-job-arguments
-        mountPath: /arguments
+    config:
+      replicas: 3
+      volumeMounts:
+        - name: cm-job-arguments
+          mountPath: /arguments
diff --git a/rust/crd/Cargo.toml b/rust/crd/Cargo.toml
@@ -20,3 +20,4 @@ tracing.workspace = true
 [dev-dependencies]
 rstest.workspace = true
 serde_yaml.workspace = true
+indoc.workspace = true