use standard configmap-backed volume

adwk67 · adwk67 · commit b48436f911a2 · 2022-04-19T13:24:59.000+02:00
diff --git a/deploy/crd/sparkapplication.crd.yaml b/deploy/crd/sparkapplication.crd.yaml
@@ -85,34 +85,6 @@ spec:
                 driver:
                   nullable: true
                   properties:
-                    configMapMounts:
-                      items:
-                        description: VolumeMount describes a mounting of a Volume within a container.
-                        properties:
-                          mountPath:
-                            description: "Path within the container at which the volume should be mounted.  Must not contain ':'."
-                            type: string
-                          mountPropagation:
-                            description: "mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10."
-                            type: string
-                          name:
-                            description: This must match the Name of a Volume.
-                            type: string
-                          readOnly:
-                            description: "Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false."
-                            type: boolean
-                          subPath:
-                            description: "Path within the volume from which the container's volume should be mounted. Defaults to \"\" (volume's root)."
-                            type: string
-                          subPathExpr:
-                            description: "Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to \"\" (volume's root). SubPathExpr and SubPath are mutually exclusive."
-                            type: string
-                        required:
-                          - mountPath
-                          - name
-                        type: object
-                      nullable: true
-                      type: array
                     coreLimit:
                       nullable: true
                       type: string
@@ -237,34 +209,6 @@ spec:
                 executor:
                   nullable: true
                   properties:
-                    configMapMounts:
-                      items:
-                        description: VolumeMount describes a mounting of a Volume within a container.
-                        properties:
-                          mountPath:
-                            description: "Path within the container at which the volume should be mounted.  Must not contain ':'."
-                            type: string
-                          mountPropagation:
-                            description: "mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10."
-                            type: string
-                          name:
-                            description: This must match the Name of a Volume.
-                            type: string
-                          readOnly:
-                            description: "Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false."
-                            type: boolean
-                          subPath:
-                            description: "Path within the volume from which the container's volume should be mounted. Defaults to \"\" (volume's root)."
-                            type: string
-                          subPathExpr:
-                            description: "Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to \"\" (volume's root). SubPathExpr and SubPath are mutually exclusive."
-                            type: string
-                        required:
-                          - mountPath
-                          - name
-                        type: object
-                      nullable: true
-                      type: array
                     cores:
                       format: uint
                       minimum: 0.0
diff --git a/deploy/helm/spark-k8s-operator/crds/crds.yaml b/deploy/helm/spark-k8s-operator/crds/crds.yaml
@@ -87,34 +87,6 @@ spec:
                 driver:
                   nullable: true
                   properties:
-                    configMapMounts:
-                      items:
-                        description: VolumeMount describes a mounting of a Volume within a container.
-                        properties:
-                          mountPath:
-                            description: "Path within the container at which the volume should be mounted.  Must not contain ':'."
-                            type: string
-                          mountPropagation:
-                            description: "mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10."
-                            type: string
-                          name:
-                            description: This must match the Name of a Volume.
-                            type: string
-                          readOnly:
-                            description: "Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false."
-                            type: boolean
-                          subPath:
-                            description: "Path within the volume from which the container's volume should be mounted. Defaults to \"\" (volume's root)."
-                            type: string
-                          subPathExpr:
-                            description: "Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to \"\" (volume's root). SubPathExpr and SubPath are mutually exclusive."
-                            type: string
-                        required:
-                          - mountPath
-                          - name
-                        type: object
-                      nullable: true
-                      type: array
                     coreLimit:
                       nullable: true
                       type: string
@@ -239,34 +211,6 @@ spec:
                 executor:
                   nullable: true
                   properties:
-                    configMapMounts:
-                      items:
-                        description: VolumeMount describes a mounting of a Volume within a container.
-                        properties:
-                          mountPath:
-                            description: "Path within the container at which the volume should be mounted.  Must not contain ':'."
-                            type: string
-                          mountPropagation:
-                            description: "mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10."
-                            type: string
-                          name:
-                            description: This must match the Name of a Volume.
-                            type: string
-                          readOnly:
-                            description: "Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false."
-                            type: boolean
-                          subPath:
-                            description: "Path within the volume from which the container's volume should be mounted. Defaults to \"\" (volume's root)."
-                            type: string
-                          subPathExpr:
-                            description: "Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to \"\" (volume's root). SubPathExpr and SubPath are mutually exclusive."
-                            type: string
-                        required:
-                          - mountPath
-                          - name
-                        type: object
-                      nullable: true
-                      type: array
                     cores:
                       format: uint
                       minimum: 0.0
diff --git a/deploy/manifests/crds.yaml b/deploy/manifests/crds.yaml
@@ -88,34 +88,6 @@ spec:
                 driver:
                   nullable: true
                   properties:
-                    configMapMounts:
-                      items:
-                        description: VolumeMount describes a mounting of a Volume within a container.
-                        properties:
-                          mountPath:
-                            description: "Path within the container at which the volume should be mounted.  Must not contain ':'."
-                            type: string
-                          mountPropagation:
-                            description: "mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10."
-                            type: string
-                          name:
-                            description: This must match the Name of a Volume.
-                            type: string
-                          readOnly:
-                            description: "Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false."
-                            type: boolean
-                          subPath:
-                            description: "Path within the volume from which the container's volume should be mounted. Defaults to \"\" (volume's root)."
-                            type: string
-                          subPathExpr:
-                            description: "Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to \"\" (volume's root). SubPathExpr and SubPath are mutually exclusive."
-                            type: string
-                        required:
-                          - mountPath
-                          - name
-                        type: object
-                      nullable: true
-                      type: array
                     coreLimit:
                       nullable: true
                       type: string
@@ -240,34 +212,6 @@ spec:
                 executor:
                   nullable: true
                   properties:
-                    configMapMounts:
-                      items:
-                        description: VolumeMount describes a mounting of a Volume within a container.
-                        properties:
-                          mountPath:
-                            description: "Path within the container at which the volume should be mounted.  Must not contain ':'."
-                            type: string
-                          mountPropagation:
-                            description: "mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10."
-                            type: string
-                          name:
-                            description: This must match the Name of a Volume.
-                            type: string
-                          readOnly:
-                            description: "Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false."
-                            type: boolean
-                          subPath:
-                            description: "Path within the volume from which the container's volume should be mounted. Defaults to \"\" (volume's root)."
-                            type: string
-                          subPathExpr:
-                            description: "Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to \"\" (volume's root). SubPathExpr and SubPath are mutually exclusive."
-                            type: string
-                        required:
-                          - mountPath
-                          - name
-                        type: object
-                      nullable: true
-                      type: array
                     cores:
                       format: uint
                       minimum: 0.0
diff --git a/docs/modules/ROOT/examples/example-sparkapp-configmap.yaml b/docs/modules/ROOT/examples/example-sparkapp-configmap.yaml
@@ -14,8 +14,11 @@ spec:
     - name: job-deps
       persistentVolumeClaim:
         claimName: pvc-ksv
+    - name: cm-job-arguments
+      configMap:
+        name: cm-job-arguments # <4>
   args:
-    - "--input /arguments/job-args.txt" # <4>
+    - "--input /arguments/job-args.txt" # <5>
   sparkConf:
     "spark.hadoop.fs.s3a.aws.credentials.provider": "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider"
     "spark.driver.extraClassPath": "/dependencies/jars/hadoop-aws-3.2.0.jar:/dependencies/jars/aws-java-sdk-bundle-1.11.375.jar"
@@ -27,16 +30,14 @@ spec:
     volumeMounts:
       - name: job-deps
         mountPath: /dependencies
-    configMapMounts:
-      - name: cm-job-arguments # <5>
-        mountPath: /arguments  # <6>
+      - name: cm-job-arguments # <6>
+        mountPath: /arguments  # <7>
   executor:
     cores: 1
     instances: 3
     memory: "512m"
     volumeMounts:
       - name: job-deps
         mountPath: /dependencies
-    configMapMounts:
-      - name: cm-job-arguments # <5>
-        mountPath: /arguments # <6>
+      - name: cm-job-arguments # <6>
+        mountPath: /arguments # <7>
diff --git a/docs/modules/ROOT/pages/usage.adoc b/docs/modules/ROOT/pages/usage.adoc
@@ -114,9 +114,10 @@ include::example$example-sparkapp-configmap.yaml[]
 <1> Name of the configuration map
 <2> Argument required by the job
 <3> Job scala artifact that requires an input argument
-<4> The expected job argument, accessed via the mounted configuration map file
-<5> The name of the configuration map that will be mounted to the driver/executor
-<6> The mount location of the configuration map (this will contain a file `/arguments/job-args.txt`)
+<4> The volume backed by the configuration map
+<5> The expected job argument, accessed via the mounted configuration map file
+<6> The name of the volume backed by the configuration map that will be mounted to the driver/executor
+<7> The mount location of the volume (this will contain a file `/arguments/job-args.txt`)
 
 == CRD argument coverage
 
@@ -203,12 +204,6 @@ Below are listed the CRD fields that can be defined by the user:
 |`spec.driver.volumeMounts.mountPath`
 |Volume mount path
 
-|`spec.driver.configMapMounts.configMapName`
-|Name of configuration map to be mounted in the driver
-
-|`spec.driver.configMapMounts.path`
-|Mount path of the configuration map in the driver
-
 |`spec.executor.cores`
 |Number of cores for each executor
 
@@ -226,11 +221,5 @@ Below are listed the CRD fields that can be defined by the user:
 
 |`spec.executor.volumeMounts.mountPath`
 |Volume mount path
-
-|`spec.executor.configMapMounts.configMapName`
-|Name of configuration map to be mounted in the executor
-
-|`spec.executor.configMapMounts.path`
-|Mount path of the configuration map in the executor
 |===
 
diff --git a/examples/ny-tlc-report-configmap.yaml b/examples/ny-tlc-report-configmap.yaml
@@ -22,6 +22,9 @@ spec:
     - name: job-deps
       persistentVolumeClaim:
         claimName: pvc-ksv
+    - name: cm-job-arguments
+      configMap:
+        name: cm-job-arguments
   args:
     - "--input /arguments/job-args.txt"
   sparkConf:
@@ -35,7 +38,6 @@ spec:
     volumeMounts:
       - name: job-deps
         mountPath: /dependencies
-    configMapMounts:
       - name: cm-job-arguments
         mountPath: /arguments
   executor:
@@ -45,6 +47,5 @@ spec:
     volumeMounts:
       - name: job-deps
         mountPath: /dependencies
-    configMapMounts:
       - name: cm-job-arguments
         mountPath: /arguments
diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs
@@ -169,26 +169,6 @@ impl SparkApplication {
         tmp.iter().flat_map(|v| v.iter()).cloned().collect()
     }
 
-    pub fn executor_config_map_mounts(&self) -> Vec<VolumeMount> {
-        let tmp = self
-            .spec
-            .executor
-            .as_ref()
-            .and_then(|executor_conf| executor_conf.config_map_mounts.clone());
-
-        tmp.iter().flat_map(|v| v.iter()).cloned().collect()
-    }
-
-    pub fn driver_config_map_mounts(&self) -> Vec<VolumeMount> {
-        let tmp = self
-            .spec
-            .driver
-            .as_ref()
-            .and_then(|driver_conf| driver_conf.config_map_mounts.clone());
-
-        tmp.iter().flat_map(|v| v.iter()).cloned().collect()
-    }
-
     pub fn executor_volume_mounts(&self) -> Vec<VolumeMount> {
         let tmp = self
             .spec
@@ -315,8 +295,6 @@ pub struct DriverConfig {
     pub memory: Option<String>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub volume_mounts: Option<Vec<VolumeMount>>,
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub config_map_mounts: Option<Vec<VolumeMount>>,
 }
 
 impl DriverConfig {
@@ -345,8 +323,6 @@ pub struct ExecutorConfig {
     pub memory: Option<String>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub volume_mounts: Option<Vec<VolumeMount>>,
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub config_map_mounts: Option<Vec<VolumeMount>>,
 }
 
 impl ExecutorConfig {
diff --git a/rust/operator-binary/src/spark_k8s_controller.rs b/rust/operator-binary/src/spark_k8s_controller.rs
diff --git a/tests/templates/kuttl/spark-ny-public-s3/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/spark-ny-public-s3/10-deploy-spark-app.yaml.j2
