[Merged by Bors] - OpenShift compatibility

CHANGELOG.md

@@ -14,9 +14,11 @@ All notable changes to this project will be documented in this file.
 
 - Add missing role to read S3Connection and S3Bucket objects ([#112]).
 - Update annotation due to update to rust version ([#114]).
+- Update RBAC properties for OpenShift compatibility ([#126]).
 
 [#112]: https://github.com/stackabletech/spark-k8s-operator/pull/112
 [#114]: https://github.com/stackabletech/spark-k8s-operator/pull/114
+[#126]: https://github.com/stackabletech/spark-k8s-operator/pull/126
 
 ## [0.4.0] - 2022-08-03
 

deploy/helm/spark-k8s-operator/templates/roles.yaml

@@ -104,3 +104,11 @@ rules:
       - get
       - list
       - watch
+  - apiGroups:
+      - rbac.authorization.k8s.io
+    resources:
+      - clusterroles
+    verbs:
+      - bind
+    resourceNames:
+      - {{ include "operator.name" . }}-clusterrole

deploy/helm/spark-k8s-operator/templates/spark-clusterrole.yaml

@@ -1,15 +1,82 @@
+{{ if .Capabilities.APIVersions.Has "security.openshift.io/v1" }}
+---
+apiVersion: security.openshift.io/v1
+kind: SecurityContextConstraints
+metadata:
+  name: spark-k8s-scc
+  annotations:
+    kubernetes.io/description: |-
+      This resource is derived from hostmount-anyuid. It provides all the features of the
+      restricted SCC but allows host mounts and any UID by a pod.  This is primarily
+      used by the persistent volume recycler. WARNING: this SCC allows host file
+      system access as any UID, including UID 0.  Grant with caution.
+    release.openshift.io/create-only: "true"
+allowHostDirVolumePlugin: true
+allowHostIPC: false
+allowHostNetwork: false
+allowHostPID: false
+allowHostPorts: false
+allowPrivilegeEscalation: true
+allowPrivilegedContainer: false
+allowedCapabilities: null
+defaultAddCapabilities: null
+fsGroup:
+  type: RunAsAny
+groups: []
+priority: null
+readOnlyRootFilesystem: false
+runAsUser:
+  type: RunAsAny
+seLinuxContext:
+  type: MustRunAs
+supplementalGroups:
+  type: RunAsAny
+volumes:
+- configMap
+- downwardAPI
+- emptyDir
+- hostPath
+- nfs
+- persistentVolumeClaim
+- projected
+- secret
+- ephemeral
+{{ end }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  name: spark-driver-edit-role
+  name: {{ include "operator.name" . }}-clusterrole
 rules:
-  - apiGroups: [""]
-    resources: ["pods", "services", "configmaps"]
-    verbs: ["get", "list", "watch", "create", "delete"]
   - apiGroups:
       - ""
     resources:
-      - persistentvolumeclaims
+      - configmaps
+      - pods
+      - secrets
+      - serviceaccounts
+      - services
     verbs:
+      - create
+      - delete
+      - get
       - list
+      - patch
+      - update
+      - watch
+  - apiGroups:
+      - events.k8s.io
+    resources:
+      - events
+    verbs:
+      - create
+{{ if .Capabilities.APIVersions.Has "security.openshift.io/v1" }}
+  - apiGroups:
+      - security.openshift.io
+    resources:
+      - securitycontextconstraints
+    resourceNames:
+      - spark-k8s-scc
+    verbs:
+      - use
+{{ end }}

deploy/helm/spark-k8s-operator/values.yaml

@@ -21,14 +21,22 @@ podAnnotations: {}
 
 podSecurityContext: {}
   # fsGroup: 2000
-
-securityContext: {}
-  # capabilities:
-  #   drop:
-  #   - ALL
-  # readOnlyRootFilesystem: true
-  # runAsNonRoot: true
-  # runAsUser: 1000
+#
+# OpenShift 4.11 replaces the PodSecurityPolicy with a new pod security
+# admission mechanism as described in this blog post [1].
+# This requires Pods to explicitely specify the securityContext.
+#
+# [1]: https://cloud.redhat.com/blog/pod-security-admission-in-openshift-4.11
+securityContext:
+  capabilities:
+    drop:
+      - ALL
+  readOnlyRootFilesystem: false
+  allowPrivilegeEscalation: false
+  seccompProfile:
+    type: RuntimeDefault
+  runAsNonRoot: true
+  runAsUser: 1000
 
 resources: {}
   # We usually recommend not to specify default resources and to leave this as a conscious

deploy/manifests/deployment.yaml

@@ -25,7 +25,16 @@ spec:
       securityContext: {}
       containers:
         - name: spark-k8s-operator
-          securityContext: {}
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+            readOnlyRootFilesystem: false
+            runAsNonRoot: true
+            runAsUser: 1000
+            seccompProfile:
+              type: RuntimeDefault
           image: "docker.stackable.tech/stackable/spark-k8s-operator:0.5.0-nightly"
           imagePullPolicy: IfNotPresent
           resources: {}

deploy/manifests/roles.yaml

@@ -104,3 +104,11 @@ rules:
       - get
       - list
       - watch
+  - apiGroups:
+      - rbac.authorization.k8s.io
+    resources:
+      - clusterroles
+    verbs:
+      - bind
+    resourceNames:
+      - spark-k8s-clusterrole

deploy/manifests/spark-clusterrole.yaml

@@ -2,14 +2,27 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  name: spark-driver-edit-role
+  name: spark-k8s-clusterrole
 rules:
-  - apiGroups: [""]
-    resources: ["pods", "services", "configmaps"]
-    verbs: ["get", "list", "watch", "create", "delete"]
   - apiGroups:
       - ""
     resources:
-      - persistentvolumeclaims
+      - configmaps
+      - pods
+      - secrets
+      - serviceaccounts
+      - services
     verbs:
+      - create
+      - delete
+      - get
       - list
+      - patch
+      - update
+      - watch
+  - apiGroups:
+      - events.k8s.io
+    resources:
+      - events
+    verbs:
+      - create

docs/modules/ROOT/pages/rbac.adoc

@@ -6,6 +6,6 @@ The https://spark.apache.org/docs/latest/running-on-kubernetes.html#rbac[Spark-K
 
 However, to add security, each `spark-submit` job launched by the spark-k8s operator will be assigned its own service account.
 
-When the spark-k8s operator is installed via helm, a cluster role named `spark-driver-edit-role` is created with pre-defined permissions.
+When the spark-k8s operator is installed via Helm, a cluster role named `spark-k8s-clusterrole` is created with pre-defined permissions.
 
-When a new Spark application is submitted, the operator creates a new service account with the same name as the application and binds this account to the cluster role `spark-driver-edit-role` created by helm.
+When a new Spark application is submitted, the operator creates a new service account with the same name as the application and binds this account to the cluster role `spark-k8s-clusterrole` created by Helm.

rust/crd/src/constants.rs

@@ -20,3 +20,5 @@ pub const CONTAINER_NAME_EXECUTOR: &str = "spark-executor";
 pub const ACCESS_KEY_ID: &str = "accessKeyId";
 pub const SECRET_ACCESS_KEY: &str = "secretAccessKey";
 pub const S3_SECRET_DIR_NAME: &str = "/stackable/secrets";
+
+pub const SPARK_UID: i64 = 1000;

rust/operator-binary/src/spark_k8s_controller.rs

@@ -7,8 +7,8 @@ use stackable_operator::commons::s3::InlinedS3BucketSpec;
 use stackable_operator::commons::tls::{CaCert, TlsVerification};
 use stackable_operator::k8s_openapi::api::batch::v1::{Job, JobSpec};
 use stackable_operator::k8s_openapi::api::core::v1::{
-    ConfigMap, ConfigMapVolumeSource, Container, EnvVar, Pod, PodSpec, PodTemplateSpec,
-    ServiceAccount, Volume, VolumeMount,
+    ConfigMap, ConfigMapVolumeSource, Container, EnvVar, Pod, PodSecurityContext, PodSpec,
+    PodTemplateSpec, ServiceAccount, Volume, VolumeMount,
 };
 use stackable_operator::k8s_openapi::api::rbac::v1::{ClusterRole, RoleBinding, RoleRef, Subject};
 use stackable_operator::k8s_openapi::Resource;
@@ -21,7 +21,7 @@ use std::{sync::Arc, time::Duration};
 use strum::{EnumDiscriminants, IntoStaticStr};
 
 const FIELD_MANAGER_SCOPE: &str = "sparkapplication";
-const SPARK_CLUSTER_ROLE: &str = "spark-driver-edit-role";
+const SPARK_CLUSTER_ROLE: &str = "spark-k8s-clusterrole";
 
 pub struct Ctx {
     pub client: stackable_operator::client::Client,
@@ -234,10 +234,7 @@ fn pod_template(
     let mut pod_spec = PodSpec {
         containers: vec![cb.build()],
         volumes: Some(volumes.to_vec()),
-        security_context: PodSecurityContextBuilder::new()
-            .fs_group(1000)
-            .build()
-            .into(), // Needed for secret-operator
+        security_context: Some(security_context()),
         ..PodSpec::default()
     };
 
@@ -370,10 +367,7 @@ fn spark_job(
             service_account_name: serviceaccount.metadata.name.clone(),
             volumes: Some(volumes),
             image_pull_secrets: spark_application.spark_image_pull_secrets(),
-            security_context: PodSecurityContextBuilder::new()
-                .fs_group(1000)
-                .build()
-                .into(), // Needed for secret-operator
+            security_context: Some(security_context()),
             node_selector: spark_application.driver_node_selector(),
             ..PodSpec::default()
         }),
@@ -439,6 +433,20 @@ fn build_spark_role_serviceaccount(
     Ok((sa, binding))
 }
 
+fn security_context() -> PodSecurityContext {
+    PodSecurityContextBuilder::new()
+        .fs_group(1000)
+        // OpenShift generates UIDs for processes inside Pods. Setting the UID is optional,
+        // *but* if specified, OpenShift will check that the value is within the
+        // valid range generated by the SCC (security context constraints) for this Pod.
+        // On the other hand, it is *required* to set the process UID in KinD, K3S as soon
+        // as the runAsGroup property is set.
+        .run_as_user(SPARK_UID)
+        // Required to access files in mounted volumes on OpenShift.
+        .run_as_group(0)
+        .build()
+}
+
 pub fn error_policy(_error: &Error, _ctx: Arc<Ctx>) -> Action {
     Action::requeue(Duration::from_secs(5))
 }

tests/templates/kuttl/pyspark-ny-public-s3-image/00-assert.yaml

@@ -5,17 +5,23 @@ metadata:
   name: minio
 timeout: 900
 ---
-apiVersion: v1
-kind: Service
+apiVersion: apps/v1
+kind: Deployment
 metadata:
   name: test-minio
-  labels:
-    app: minio
+status:
+  readyReplicas: 1
 ---
-apiVersion: apps/v1
-kind: StatefulSet
+apiVersion: v1
+kind: Pod
 metadata:
-  name: minio-mc
+  name: minio-client
+  labels:
+    app: minio-client
 status:
-  readyReplicas: 1
-  replicas: 1
+  phase: Running
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: integration-tests-sa