From 0b459b2f82bd4e567b00a50631e909ed0f82e2c0 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 24 Aug 2022 14:46:09 +0200 Subject: [PATCH 01/22] Added SCC and update cluster role name --- .../spark-k8s-operator/templates/roles.yaml | 8 +++ .../templates/spark-clusterrole.yaml | 72 +++++++++++++++++-- deploy/helm/spark-k8s-operator/values.yaml | 13 ++-- deploy/manifests/deployment.yaml | 9 ++- deploy/manifests/roles.yaml | 8 +++ deploy/manifests/spark-clusterrole.yaml | 18 +++-- docs/modules/ROOT/pages/rbac.adoc | 4 +- .../src/spark_k8s_controller.rs | 2 +- 8 files changed, 113 insertions(+), 21 deletions(-) diff --git a/deploy/helm/spark-k8s-operator/templates/roles.yaml b/deploy/helm/spark-k8s-operator/templates/roles.yaml index 81117249..3c74456f 100644 --- a/deploy/helm/spark-k8s-operator/templates/roles.yaml +++ b/deploy/helm/spark-k8s-operator/templates/roles.yaml @@ -104,3 +104,11 @@ rules: - get - list - watch + - apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterroles + verbs: + - bind + resourceNames: + - {{ include "operator.name" . }}-clusterrole diff --git a/deploy/helm/spark-k8s-operator/templates/spark-clusterrole.yaml b/deploy/helm/spark-k8s-operator/templates/spark-clusterrole.yaml index 25b80c05..75b652c1 100644 --- a/deploy/helm/spark-k8s-operator/templates/spark-clusterrole.yaml +++ b/deploy/helm/spark-k8s-operator/templates/spark-clusterrole.yaml @@ -1,15 +1,75 @@ +{{ if .Capabilities.APIVersions.Has "security.openshift.io/v1" }} +--- +apiVersion: security.openshift.io/v1 +kind: SecurityContextConstraints +metadata: + name: spark-k8s-scc + annotations: + kubernetes.io/description: |- + This resource is derived from hostmount-anyuid. It provides all the features of the + restricted SCC but allows host mounts and any UID by a pod. This is primarily + used by the persistent volume recycler. WARNING: this SCC allows host file + system access as any UID, including UID 0. Grant with caution. + release.openshift.io/create-only: "true" +allowHostDirVolumePlugin: true +allowHostIPC: false +allowHostNetwork: false +allowHostPID: false +allowHostPorts: false +allowPrivilegeEscalation: true +allowPrivilegedContainer: false +allowedCapabilities: null +defaultAddCapabilities: null +fsGroup: + type: RunAsAny +groups: [] +priority: null +readOnlyRootFilesystem: false +runAsUser: + type: RunAsAny +seLinuxContext: + type: MustRunAs +supplementalGroups: + type: RunAsAny +volumes: +- configMap +- downwardAPI +- emptyDir +- hostPath +- nfs +- persistentVolumeClaim +- projected +- secret +- ephemeral +{{ end }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: spark-driver-edit-role + name: {{ include "operator.name" . }}-clusterrole rules: - - apiGroups: [""] - resources: ["pods", "services", "configmaps"] - verbs: ["get", "list", "watch", "create", "delete"] - apiGroups: - "" resources: - - persistentvolumeclaims + - configmaps + - secrets + - serviceaccounts + - services + verbs: + - get + - apiGroups: + - events.k8s.io + resources: + - events + verbs: + - create +{{ if .Capabilities.APIVersions.Has "security.openshift.io/v1" }} + - apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + resourceNames: + - spark-k8s-scc verbs: - - list + - use +{{ end }} diff --git a/deploy/helm/spark-k8s-operator/values.yaml b/deploy/helm/spark-k8s-operator/values.yaml index 2211dd3e..3e9c0319 100644 --- a/deploy/helm/spark-k8s-operator/values.yaml +++ b/deploy/helm/spark-k8s-operator/values.yaml @@ -22,11 +22,14 @@ podAnnotations: {} podSecurityContext: {} # fsGroup: 2000 -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true +securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: false + allowPrivilegeEscalation: false + seccompProfile: + type: RuntimeDefault # runAsNonRoot: true # runAsUser: 1000 diff --git a/deploy/manifests/deployment.yaml b/deploy/manifests/deployment.yaml index d34746db..f22374bf 100644 --- a/deploy/manifests/deployment.yaml +++ b/deploy/manifests/deployment.yaml @@ -25,7 +25,14 @@ spec: securityContext: {} containers: - name: spark-k8s-operator - securityContext: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: false + seccompProfile: null + type: RuntimeDefault image: "docker.stackable.tech/stackable/spark-k8s-operator:0.5.0-nightly" imagePullPolicy: IfNotPresent resources: {} diff --git a/deploy/manifests/roles.yaml b/deploy/manifests/roles.yaml index 44b95148..4ad6e338 100644 --- a/deploy/manifests/roles.yaml +++ b/deploy/manifests/roles.yaml @@ -104,3 +104,11 @@ rules: - get - list - watch + - apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterroles + verbs: + - bind + resourceNames: + - spark-k8s-clusterrole diff --git a/deploy/manifests/spark-clusterrole.yaml b/deploy/manifests/spark-clusterrole.yaml index 25b80c05..e5837fa1 100644 --- a/deploy/manifests/spark-clusterrole.yaml +++ b/deploy/manifests/spark-clusterrole.yaml @@ -2,14 +2,20 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: spark-driver-edit-role + name: spark-k8s-clusterrole rules: - - apiGroups: [""] - resources: ["pods", "services", "configmaps"] - verbs: ["get", "list", "watch", "create", "delete"] - apiGroups: - "" resources: - - persistentvolumeclaims + - configmaps + - secrets + - serviceaccounts + - services verbs: - - list + - get + - apiGroups: + - events.k8s.io + resources: + - events + verbs: + - create diff --git a/docs/modules/ROOT/pages/rbac.adoc b/docs/modules/ROOT/pages/rbac.adoc index 7abb691a..ed9de94d 100644 --- a/docs/modules/ROOT/pages/rbac.adoc +++ b/docs/modules/ROOT/pages/rbac.adoc @@ -6,6 +6,6 @@ The https://spark.apache.org/docs/latest/running-on-kubernetes.html#rbac[Spark-K However, to add security, each `spark-submit` job launched by the spark-k8s operator will be assigned its own service account. -When the spark-k8s operator is installed via helm, a cluster role named `spark-driver-edit-role` is created with pre-defined permissions. +When the spark-k8s operator is installed via Helm, a cluster role named `spark-k8s-clusterrole` is created with pre-defined permissions. -When a new Spark application is submitted, the operator creates a new service account with the same name as the application and binds this account to the cluster role `spark-driver-edit-role` created by helm. +When a new Spark application is submitted, the operator creates a new service account with the same name as the application and binds this account to the cluster role `spark-k8s-clusterrole` created by Helm. diff --git a/rust/operator-binary/src/spark_k8s_controller.rs b/rust/operator-binary/src/spark_k8s_controller.rs index 54ddc317..488cfb7a 100644 --- a/rust/operator-binary/src/spark_k8s_controller.rs +++ b/rust/operator-binary/src/spark_k8s_controller.rs @@ -21,7 +21,7 @@ use std::{sync::Arc, time::Duration}; use strum::{EnumDiscriminants, IntoStaticStr}; const FIELD_MANAGER_SCOPE: &str = "sparkapplication"; -const SPARK_CLUSTER_ROLE: &str = "spark-driver-edit-role"; +const SPARK_CLUSTER_ROLE: &str = "spark-k8s-clusterrole"; pub struct Ctx { pub client: stackable_operator::client::Client, From a5ca51e8571c8ea205e55849f6866efbdbc879a8 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 24 Aug 2022 15:16:38 +0200 Subject: [PATCH 02/22] Fix indentantion. --- deploy/helm/spark-k8s-operator/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/helm/spark-k8s-operator/values.yaml b/deploy/helm/spark-k8s-operator/values.yaml index 3e9c0319..68338999 100644 --- a/deploy/helm/spark-k8s-operator/values.yaml +++ b/deploy/helm/spark-k8s-operator/values.yaml @@ -29,7 +29,7 @@ securityContext: readOnlyRootFilesystem: false allowPrivilegeEscalation: false seccompProfile: - type: RuntimeDefault + type: RuntimeDefault # runAsNonRoot: true # runAsUser: 1000 From 08b47f420d9bfaea4a905ee059a6ce2308138c6c Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 24 Aug 2022 15:25:38 +0200 Subject: [PATCH 03/22] make regenerate-charts --- deploy/manifests/deployment.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/manifests/deployment.yaml b/deploy/manifests/deployment.yaml index f22374bf..40cad702 100644 --- a/deploy/manifests/deployment.yaml +++ b/deploy/manifests/deployment.yaml @@ -31,8 +31,8 @@ spec: drop: - ALL readOnlyRootFilesystem: false - seccompProfile: null - type: RuntimeDefault + seccompProfile: + type: RuntimeDefault image: "docker.stackable.tech/stackable/spark-k8s-operator:0.5.0-nightly" imagePullPolicy: IfNotPresent resources: {} From 35f37f2807de4d6c908ff17a2bd05787008fe2a9 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 24 Aug 2022 16:09:22 +0200 Subject: [PATCH 04/22] Update the spark-k8s-clusterrole to allow spark-submit to create pods. --- .../spark-k8s-operator/templates/spark-clusterrole.yaml | 7 +++++++ deploy/helm/spark-k8s-operator/values.yaml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/deploy/helm/spark-k8s-operator/templates/spark-clusterrole.yaml b/deploy/helm/spark-k8s-operator/templates/spark-clusterrole.yaml index 75b652c1..4828006c 100644 --- a/deploy/helm/spark-k8s-operator/templates/spark-clusterrole.yaml +++ b/deploy/helm/spark-k8s-operator/templates/spark-clusterrole.yaml @@ -52,11 +52,18 @@ rules: - "" resources: - configmaps + - pods - secrets - serviceaccounts - services verbs: + - create + - delete - get + - list + - patch + - update + - watch - apiGroups: - events.k8s.io resources: diff --git a/deploy/helm/spark-k8s-operator/values.yaml b/deploy/helm/spark-k8s-operator/values.yaml index 68338999..1484683a 100644 --- a/deploy/helm/spark-k8s-operator/values.yaml +++ b/deploy/helm/spark-k8s-operator/values.yaml @@ -30,7 +30,7 @@ securityContext: allowPrivilegeEscalation: false seccompProfile: type: RuntimeDefault - # runAsNonRoot: true + runAsNonRoot: true # runAsUser: 1000 resources: {} From f12f35ec9636aa3ac0560c9d5282015a425fe1a8 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 24 Aug 2022 17:54:58 +0200 Subject: [PATCH 05/22] wip: minio setup on openshift. --- .../spark-pi-public-s3/00-setup-minio.yaml | 10 ++--- .../helm-bitnami-minio-values.yaml | 38 +++++++++++++++++++ 2 files changed, 41 insertions(+), 7 deletions(-) create mode 100644 tests/templates/kuttl/spark-pi-public-s3/helm-bitnami-minio-values.yaml diff --git a/tests/templates/kuttl/spark-pi-public-s3/00-setup-minio.yaml b/tests/templates/kuttl/spark-pi-public-s3/00-setup-minio.yaml index b74ca5ff..403df831 100644 --- a/tests/templates/kuttl/spark-pi-public-s3/00-setup-minio.yaml +++ b/tests/templates/kuttl/spark-pi-public-s3/00-setup-minio.yaml @@ -5,11 +5,7 @@ commands: - script: >- helm install test-minio --namespace $NAMESPACE - --version 4.0.2 - --set mode=standalone - --set replicas=1 - --set persistence.enabled=false - --set buckets[0].name=my-bucket,buckets[0].policy=public - --set resources.requests.memory=1Gi - --repo https://charts.min.io/ minio + --version 11.9.2 + -f helm-bitnami-minio-values.yaml + --repo https://charts.bitnami.com/bitnami minio timeout: 240 diff --git a/tests/templates/kuttl/spark-pi-public-s3/helm-bitnami-minio-values.yaml b/tests/templates/kuttl/spark-pi-public-s3/helm-bitnami-minio-values.yaml new file mode 100644 index 00000000..18f039a3 --- /dev/null +++ b/tests/templates/kuttl/spark-pi-public-s3/helm-bitnami-minio-values.yaml @@ -0,0 +1,38 @@ +--- +volumePermissions: + enabled: false + +podSecurityContext: + enabled: false + +containerSecurityContext: + enabled: false + +mode: standalone +defaultBuckets: my-bucket +persistence: + enabled: false +policies: + - name: my-bucket-public-policy + statements: + - resources: + - "arn:aws:s3:::my-bucket" + actions: + - "s3:GetBucketLocation" + - "s3:ListBucket" + - "s3:ListBucketMultipartUploads" + - resources: + - "arn:aws:s3:::my-bucket/*" + # Allowed values: "Allow" | "Deny" + # Defaults to "Deny" if not specified + effect: "Allow" + actions: + - "s3:AbortMultipartUpload" + - "s3:DeleteObject" + - "s3:GetObject" + - "s3:ListMultipartUploadParts" + - "s3:PutObject" + +resources: + requests: + memory: 1Gi From 78663be28a2ec11d78a7b9cec6fd4f472e3237a0 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Thu, 25 Aug 2022 14:57:27 +0200 Subject: [PATCH 06/22] kuttl tests: still wip --- .../kuttl/spark-pi-public-s3/00-assert.yaml | 18 ++++++---- .../00-s3-upload-container.yaml | 36 ------------------- .../00-serviceaccount.yaml.j2 | 29 +++++++++++++++ .../spark-pi-public-s3/00-setup-minio.yaml | 33 +++++++++++++++++ .../01-prepare-bucket.yaml.j2 | 10 ++---- .../spark-pi-public-s3/02-deps-volume.yaml | 5 +-- tests/test-definition.yaml | 4 +++ 7 files changed, 83 insertions(+), 52 deletions(-) delete mode 100644 tests/templates/kuttl/spark-pi-public-s3/00-s3-upload-container.yaml create mode 100644 tests/templates/kuttl/spark-pi-public-s3/00-serviceaccount.yaml.j2 diff --git a/tests/templates/kuttl/spark-pi-public-s3/00-assert.yaml b/tests/templates/kuttl/spark-pi-public-s3/00-assert.yaml index 20f246c3..f7295267 100644 --- a/tests/templates/kuttl/spark-pi-public-s3/00-assert.yaml +++ b/tests/templates/kuttl/spark-pi-public-s3/00-assert.yaml @@ -9,13 +9,17 @@ apiVersion: v1 kind: Service metadata: name: test-minio - labels: - app: minio --- -apiVersion: apps/v1 -kind: StatefulSet +apiVersion: v1 +kind: Pod metadata: - name: minio-mc + name: minio-client + labels: + app: minio-client status: - readyReplicas: 1 - replicas: 1 + phase: Running +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: integration-tests-sa diff --git a/tests/templates/kuttl/spark-pi-public-s3/00-s3-upload-container.yaml b/tests/templates/kuttl/spark-pi-public-s3/00-s3-upload-container.yaml deleted file mode 100644 index dbd4c93d..00000000 --- a/tests/templates/kuttl/spark-pi-public-s3/00-s3-upload-container.yaml +++ /dev/null @@ -1,36 +0,0 @@ ---- -apiVersion: v1 -kind: Service -metadata: - name: minio-mc - labels: - app: minio-mc -timeout: 240 -spec: - clusterIP: None - selector: - app: minio-mc ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: minio-mc - labels: - app: minio-mc -timeout: 240 -spec: - replicas: 1 - serviceName: "minio-mc" - selector: - matchLabels: - app: minio-mc - template: - metadata: - labels: - app: minio-mc - spec: - containers: - - name: minio-mc - image: bitnami/minio:2022-debian-10 - stdin: true - tty: true diff --git a/tests/templates/kuttl/spark-pi-public-s3/00-serviceaccount.yaml.j2 b/tests/templates/kuttl/spark-pi-public-s3/00-serviceaccount.yaml.j2 new file mode 100644 index 00000000..9cbf0351 --- /dev/null +++ b/tests/templates/kuttl/spark-pi-public-s3/00-serviceaccount.yaml.j2 @@ -0,0 +1,29 @@ +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: use-integration-tests-scc +rules: +{% if test_scenario['values']['openshift'] == "true" %} + - apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + resourceNames: ["privileged"] + verbs: ["use"] +{% endif %} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: integration-tests-sa +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: use-integration-tests-scc +subjects: + - kind: ServiceAccount + name: integration-tests-sa +roleRef: + kind: Role + name: use-integration-tests-scc + apiGroup: rbac.authorization.k8s.io diff --git a/tests/templates/kuttl/spark-pi-public-s3/00-setup-minio.yaml b/tests/templates/kuttl/spark-pi-public-s3/00-setup-minio.yaml index 403df831..f3820faa 100644 --- a/tests/templates/kuttl/spark-pi-public-s3/00-setup-minio.yaml +++ b/tests/templates/kuttl/spark-pi-public-s3/00-setup-minio.yaml @@ -9,3 +9,36 @@ commands: -f helm-bitnami-minio-values.yaml --repo https://charts.bitnami.com/bitnami minio timeout: 240 +--- +apiVersion: v1 +kind: Pod +metadata: + name: minio-client + labels: + app: minio-client +spec: + selector: + matchLabels: + app: minio-client + restartPolicy: Never + containers: + - name: minio-client + image: docker.io/bitnami/minio-client:2022.8.11-debian-11-r3 + command: ["bash", "-c", "sleep infinity"] + stdin: true + tty: true + env: + - name: MINIO_SERVER_ROOT_USER + valueFrom: + secretKeyRef: + name: test-minio + key: root-user + optional: false + - name: MINIO_SERVER_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: test-minio + key: root-password + optional: false + - name: MINIO_SERVER_HOST + value: test-minio diff --git a/tests/templates/kuttl/spark-pi-public-s3/01-prepare-bucket.yaml.j2 b/tests/templates/kuttl/spark-pi-public-s3/01-prepare-bucket.yaml.j2 index e155f6e8..98c22d3c 100644 --- a/tests/templates/kuttl/spark-pi-public-s3/01-prepare-bucket.yaml.j2 +++ b/tests/templates/kuttl/spark-pi-public-s3/01-prepare-bucket.yaml.j2 @@ -2,10 +2,6 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - - script: >- - kubectl exec -n $NAMESPACE minio-mc-0 -- - sh -c 'mc alias set test-minio http://test-minio:9000/' - - script: kubectl cp -n $NAMESPACE spark-examples_2.12-{{ test_scenario['values']['spark'] }}.jar minio-mc-0:/tmp - - script: >- - kubectl exec -n $NAMESPACE minio-mc-0 -- - mc cp /tmp/spark-examples_2.12-{{ test_scenario['values']['spark'] }}.jar test-minio/my-bucket + - script: kubectl cp -n $NAMESPACE spark-examples_2.12-{{ test_scenario['values']['spark'] }}.jar minio-client:/tmp + - script: >- + kubectl exec -n $NAMESPACE minio-client -- mc cp /tmp/spark-examples_2.12-{{ test_scenario['values']['spark'] }}.jar test-minio/my-bucket diff --git a/tests/templates/kuttl/spark-pi-public-s3/02-deps-volume.yaml b/tests/templates/kuttl/spark-pi-public-s3/02-deps-volume.yaml index 302a7679..1273fa32 100644 --- a/tests/templates/kuttl/spark-pi-public-s3/02-deps-volume.yaml +++ b/tests/templates/kuttl/spark-pi-public-s3/02-deps-volume.yaml @@ -24,6 +24,9 @@ spec: - name: job-deps persistentVolumeClaim: claimName: spark-pi-public-pvc + serviceAccountName: integration-tests-sa + securityContext: + runAsUser: 0 containers: - name: aws-deps image: docker.stackable.tech/stackable/tools:0.2.0-stackable0 @@ -46,5 +49,3 @@ spec: volumeMounts: - name: job-deps mountPath: /dependencies - securityContext: - runAsUser: 0 diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 8a587733..8d0cc2d5 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -4,6 +4,9 @@ # --- dimensions: + - name: openshift + values: + - "false" - name: spark values: - 3.3.0 @@ -22,6 +25,7 @@ tests: dimensions: - spark - stackable + - openshift - name: spark-ny-public-s3 dimensions: - spark From 1ad692429271bffc5a2d1eb53ae76f6bcfaa62a1 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Fri, 26 Aug 2022 11:08:17 +0200 Subject: [PATCH 07/22] Add run_as_group and use the same node label as the airflow tests. --- .../src/spark_k8s_controller.rs | 2 + .../spark-pi-public-s3/02-deps-volume.yaml | 10 ++-- .../10-deploy-spark-app.yaml.j2 | 12 ++--- tests/test-definition.yaml | 52 +++++++++---------- 4 files changed, 39 insertions(+), 37 deletions(-) diff --git a/rust/operator-binary/src/spark_k8s_controller.rs b/rust/operator-binary/src/spark_k8s_controller.rs index 488cfb7a..61297d35 100644 --- a/rust/operator-binary/src/spark_k8s_controller.rs +++ b/rust/operator-binary/src/spark_k8s_controller.rs @@ -236,6 +236,7 @@ fn pod_template( volumes: Some(volumes.to_vec()), security_context: PodSecurityContextBuilder::new() .fs_group(1000) + .run_as_group(0) // needed for pvc file access .build() .into(), // Needed for secret-operator ..PodSpec::default() @@ -372,6 +373,7 @@ fn spark_job( image_pull_secrets: spark_application.spark_image_pull_secrets(), security_context: PodSecurityContextBuilder::new() .fs_group(1000) + .run_as_group(0) // needed for pvc file access .build() .into(), // Needed for secret-operator node_selector: spark_application.driver_node_selector(), diff --git a/tests/templates/kuttl/spark-pi-public-s3/02-deps-volume.yaml b/tests/templates/kuttl/spark-pi-public-s3/02-deps-volume.yaml index 1273fa32..3704346b 100644 --- a/tests/templates/kuttl/spark-pi-public-s3/02-deps-volume.yaml +++ b/tests/templates/kuttl/spark-pi-public-s3/02-deps-volume.yaml @@ -18,7 +18,7 @@ spec: template: spec: nodeSelector: - node: "1" + node: "2" restartPolicy: Never volumes: - name: job-deps @@ -26,13 +26,13 @@ spec: claimName: spark-pi-public-pvc serviceAccountName: integration-tests-sa securityContext: - runAsUser: 0 + fsGroup: 1000 containers: - name: aws-deps image: docker.stackable.tech/stackable/tools:0.2.0-stackable0 env: - name: DEST_DIR - value: "/dependencies/jars" + value: "/stackable/dependencies/jars" - name: AWS value: "1.11.1026" - name: HADOOP @@ -44,8 +44,8 @@ spec: "-o", "pipefail", "-c", - "mkdir -p ${DEST_DIR} && curl -L https://search.maven.org/remotecontent?filepath=org/apache/hadoop/hadoop-aws/${HADOOP}/hadoop-aws-${HADOOP}.jar -o ${DEST_DIR}/hadoop-aws-${HADOOP}.jar && curl -L https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${AWS}/aws-java-sdk-bundle-${AWS}.jar -o ${DEST_DIR}/aws-java-sdk-bundle-${AWS}.jar && chown -R stackable:stackable ${DEST_DIR} && chmod -R a=,u=rwX ${DEST_DIR}", + "curl -L https://search.maven.org/remotecontent?filepath=org/apache/hadoop/hadoop-aws/${HADOOP}/hadoop-aws-${HADOOP}.jar -o ${DEST_DIR}/hadoop-aws-${HADOOP}.jar && curl -L https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${AWS}/aws-java-sdk-bundle-${AWS}.jar -o ${DEST_DIR}/aws-java-sdk-bundle-${AWS}.jar", ] volumeMounts: - name: job-deps - mountPath: /dependencies + mountPath: /stackable/dependencies/jars diff --git a/tests/templates/kuttl/spark-pi-public-s3/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/spark-pi-public-s3/10-deploy-spark-app.yaml.j2 index 940e7bbb..bb049c03 100644 --- a/tests/templates/kuttl/spark-pi-public-s3/10-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/spark-pi-public-s3/10-deploy-spark-app.yaml.j2 @@ -24,23 +24,23 @@ spec: accessStyle: Path sparkConf: spark.hadoop.fs.s3a.aws.credentials.provider: "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider" - spark.driver.extraClassPath: "/dependencies/jars/*" - spark.executor.extraClassPath: "/dependencies/jars/*" + spark.driver.extraClassPath: "/stackable/dependencies/jars/*" + spark.executor.extraClassPath: "/stackable/dependencies/jars/*" driver: cores: 1 coreLimit: "1200m" memory: "512m" volumeMounts: - name: spark-pi-deps - mountPath: /dependencies + mountPath: /stackable/dependencies/jars nodeSelector: - node: "1" + node: "2" executor: cores: 1 instances: 1 memory: "512m" volumeMounts: - name: spark-pi-deps - mountPath: /dependencies + mountPath: /stackable/dependencies/jars nodeSelector: - node: "1" + node: "2" diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 8d0cc2d5..bc4456fc 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -6,7 +6,7 @@ dimensions: - name: openshift values: - - "false" + - "true" - name: spark values: - 3.3.0 @@ -17,33 +17,33 @@ dimensions: values: - 0.1.0 tests: - - name: spark-pi-private-s3 - dimensions: - - spark - - stackable +# - name: spark-pi-private-s3 +# dimensions: +# - spark +# - stackable - name: spark-pi-public-s3 dimensions: - spark - stackable - openshift - - name: spark-ny-public-s3 - dimensions: - - spark - - stackable - - name: node-selector - dimensions: - - spark - - stackable - - name: spark-examples - dimensions: - - spark - - stackable - - name: pyspark-ny-public-s3 - dimensions: - - spark - - stackable - - name: pyspark-ny-public-s3-image - dimensions: - - spark - - stackable - - ny-tlc-report +# - name: spark-ny-public-s3 +# dimensions: +# - spark +# - stackable +# - name: node-selector +# dimensions: +# - spark +# - stackable +# - name: spark-examples +# dimensions: +# - spark +# - stackable +# - name: pyspark-ny-public-s3 +# dimensions: +# - spark +# - stackable +# - name: pyspark-ny-public-s3-image +# dimensions: +# - spark +# - stackable +# - ny-tlc-report From d32ed11ce841d5ee65e1e66d01136c711b56a36e Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Fri, 26 Aug 2022 11:18:23 +0200 Subject: [PATCH 08/22] make regenerate-charts --- deploy/manifests/deployment.yaml | 1 + deploy/manifests/spark-clusterrole.yaml | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/deploy/manifests/deployment.yaml b/deploy/manifests/deployment.yaml index 40cad702..925a4f83 100644 --- a/deploy/manifests/deployment.yaml +++ b/deploy/manifests/deployment.yaml @@ -31,6 +31,7 @@ spec: drop: - ALL readOnlyRootFilesystem: false + runAsNonRoot: true seccompProfile: type: RuntimeDefault image: "docker.stackable.tech/stackable/spark-k8s-operator:0.5.0-nightly" diff --git a/deploy/manifests/spark-clusterrole.yaml b/deploy/manifests/spark-clusterrole.yaml index e5837fa1..b4f142a4 100644 --- a/deploy/manifests/spark-clusterrole.yaml +++ b/deploy/manifests/spark-clusterrole.yaml @@ -8,11 +8,18 @@ rules: - "" resources: - configmaps + - pods - secrets - serviceaccounts - services verbs: + - create + - delete - get + - list + - patch + - update + - watch - apiGroups: - events.k8s.io resources: From 88a93667241b3ade0758524a97034abac00f0abd Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Fri, 26 Aug 2022 12:31:04 +0200 Subject: [PATCH 09/22] Remove runAsNonRoot --- deploy/helm/spark-k8s-operator/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/helm/spark-k8s-operator/values.yaml b/deploy/helm/spark-k8s-operator/values.yaml index 1484683a..881f02d4 100644 --- a/deploy/helm/spark-k8s-operator/values.yaml +++ b/deploy/helm/spark-k8s-operator/values.yaml @@ -30,8 +30,8 @@ securityContext: allowPrivilegeEscalation: false seccompProfile: type: RuntimeDefault - runAsNonRoot: true - # runAsUser: 1000 + runAsUser: 1000 + # runAsNonRoot: true resources: {} # We usually recommend not to specify default resources and to leave this as a conscious From ff89e542ea2c0ea8f06f03ed115b93a5cf245826 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Fri, 26 Aug 2022 12:41:27 +0200 Subject: [PATCH 10/22] make regenerate-charts --- deploy/manifests/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/manifests/deployment.yaml b/deploy/manifests/deployment.yaml index 925a4f83..7ea0735b 100644 --- a/deploy/manifests/deployment.yaml +++ b/deploy/manifests/deployment.yaml @@ -31,7 +31,7 @@ spec: drop: - ALL readOnlyRootFilesystem: false - runAsNonRoot: true + runAsUser: 1000 seccompProfile: type: RuntimeDefault image: "docker.stackable.tech/stackable/spark-k8s-operator:0.5.0-nightly" From 652fe77396506b4af149b18f87ce70286c82eeab Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Fri, 26 Aug 2022 16:31:43 +0200 Subject: [PATCH 11/22] Fix minio public bucket setup. --- .../kuttl/spark-pi-public-s3/00-setup-minio.yaml | 8 ++++++-- .../kuttl/spark-pi-public-s3/01-prepare-bucket.yaml.j2 | 3 ++- .../kuttl/spark-pi-public-s3/02-deps-volume.yaml | 1 + .../spark-pi-public-s3/helm-bitnami-minio-values.yaml | 3 ++- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/templates/kuttl/spark-pi-public-s3/00-setup-minio.yaml b/tests/templates/kuttl/spark-pi-public-s3/00-setup-minio.yaml index f3820faa..2ba4dcac 100644 --- a/tests/templates/kuttl/spark-pi-public-s3/00-setup-minio.yaml +++ b/tests/templates/kuttl/spark-pi-public-s3/00-setup-minio.yaml @@ -28,13 +28,13 @@ spec: stdin: true tty: true env: - - name: MINIO_SERVER_ROOT_USER + - name: MINIO_SERVER_ACCESS_KEY valueFrom: secretKeyRef: name: test-minio key: root-user optional: false - - name: MINIO_SERVER_ROOT_PASSWORD + - name: MINIO_SERVER_SECRET_KEY valueFrom: secretKeyRef: name: test-minio @@ -42,3 +42,7 @@ spec: optional: false - name: MINIO_SERVER_HOST value: test-minio + - name: MINIO_SERVER_PORT_NUMBER + value: "9000" + - name: MINIO_SERVER_SCHEME + value: http diff --git a/tests/templates/kuttl/spark-pi-public-s3/01-prepare-bucket.yaml.j2 b/tests/templates/kuttl/spark-pi-public-s3/01-prepare-bucket.yaml.j2 index 98c22d3c..dbc00ff4 100644 --- a/tests/templates/kuttl/spark-pi-public-s3/01-prepare-bucket.yaml.j2 +++ b/tests/templates/kuttl/spark-pi-public-s3/01-prepare-bucket.yaml.j2 @@ -2,6 +2,7 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - - script: kubectl cp -n $NAMESPACE spark-examples_2.12-{{ test_scenario['values']['spark'] }}.jar minio-client:/tmp + - command: kubectl cp -n $NAMESPACE spark-examples_2.12-{{ test_scenario['values']['spark'] }}.jar minio-client:/tmp + - command: kubectl exec -n $NAMESPACE minio-client -- sh -c mc alias set test-minio http://test-minio:9000 $$MINIO_SERVER_ACCESS_KEY $$MINIO_SERVER_SECRET_KEY - script: >- kubectl exec -n $NAMESPACE minio-client -- mc cp /tmp/spark-examples_2.12-{{ test_scenario['values']['spark'] }}.jar test-minio/my-bucket diff --git a/tests/templates/kuttl/spark-pi-public-s3/02-deps-volume.yaml b/tests/templates/kuttl/spark-pi-public-s3/02-deps-volume.yaml index 3704346b..d104e7fe 100644 --- a/tests/templates/kuttl/spark-pi-public-s3/02-deps-volume.yaml +++ b/tests/templates/kuttl/spark-pi-public-s3/02-deps-volume.yaml @@ -27,6 +27,7 @@ spec: serviceAccountName: integration-tests-sa securityContext: fsGroup: 1000 + runAsGroup: 0 containers: - name: aws-deps image: docker.stackable.tech/stackable/tools:0.2.0-stackable0 diff --git a/tests/templates/kuttl/spark-pi-public-s3/helm-bitnami-minio-values.yaml b/tests/templates/kuttl/spark-pi-public-s3/helm-bitnami-minio-values.yaml index 18f039a3..2df048ab 100644 --- a/tests/templates/kuttl/spark-pi-public-s3/helm-bitnami-minio-values.yaml +++ b/tests/templates/kuttl/spark-pi-public-s3/helm-bitnami-minio-values.yaml @@ -9,7 +9,8 @@ containerSecurityContext: enabled: false mode: standalone -defaultBuckets: my-bucket +disableWebUI: true +defaultBuckets: "my-bucket" persistence: enabled: false policies: From 6ad750b86f696c83f8fdf418d137d622371116ec Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Fri, 26 Aug 2022 17:27:02 +0200 Subject: [PATCH 12/22] spark-pi-public-s3 test now works --- .../01-prepare-bucket.yaml.j2 | 4 +++- .../helm-bitnami-minio-values.yaml | 23 ++----------------- 2 files changed, 5 insertions(+), 22 deletions(-) diff --git a/tests/templates/kuttl/spark-pi-public-s3/01-prepare-bucket.yaml.j2 b/tests/templates/kuttl/spark-pi-public-s3/01-prepare-bucket.yaml.j2 index dbc00ff4..6ac27d07 100644 --- a/tests/templates/kuttl/spark-pi-public-s3/01-prepare-bucket.yaml.j2 +++ b/tests/templates/kuttl/spark-pi-public-s3/01-prepare-bucket.yaml.j2 @@ -3,6 +3,8 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - command: kubectl cp -n $NAMESPACE spark-examples_2.12-{{ test_scenario['values']['spark'] }}.jar minio-client:/tmp - - command: kubectl exec -n $NAMESPACE minio-client -- sh -c mc alias set test-minio http://test-minio:9000 $$MINIO_SERVER_ACCESS_KEY $$MINIO_SERVER_SECRET_KEY + - command: kubectl exec -n $NAMESPACE minio-client -- sh -c 'mc alias set test-minio http://test-minio:9000 $$MINIO_SERVER_ACCESS_KEY $$MINIO_SERVER_SECRET_KEY' + - command: kubectl exec -n $NAMESPACE minio-client -- mc mb test-minio/my-bucket + - command: kubectl exec -n $NAMESPACE minio-client -- mc policy set public test-minio/my-bucket - script: >- kubectl exec -n $NAMESPACE minio-client -- mc cp /tmp/spark-examples_2.12-{{ test_scenario['values']['spark'] }}.jar test-minio/my-bucket diff --git a/tests/templates/kuttl/spark-pi-public-s3/helm-bitnami-minio-values.yaml b/tests/templates/kuttl/spark-pi-public-s3/helm-bitnami-minio-values.yaml index 2df048ab..27705b9b 100644 --- a/tests/templates/kuttl/spark-pi-public-s3/helm-bitnami-minio-values.yaml +++ b/tests/templates/kuttl/spark-pi-public-s3/helm-bitnami-minio-values.yaml @@ -9,30 +9,11 @@ containerSecurityContext: enabled: false mode: standalone + disableWebUI: true -defaultBuckets: "my-bucket" + persistence: enabled: false -policies: - - name: my-bucket-public-policy - statements: - - resources: - - "arn:aws:s3:::my-bucket" - actions: - - "s3:GetBucketLocation" - - "s3:ListBucket" - - "s3:ListBucketMultipartUploads" - - resources: - - "arn:aws:s3:::my-bucket/*" - # Allowed values: "Allow" | "Deny" - # Defaults to "Deny" if not specified - effect: "Allow" - actions: - - "s3:AbortMultipartUpload" - - "s3:DeleteObject" - - "s3:GetObject" - - "s3:ListMultipartUploadParts" - - "s3:PutObject" resources: requests: From a505a71265a9b2355e9b883351b188dd0466d88b Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Mon, 29 Aug 2022 17:03:39 +0200 Subject: [PATCH 13/22] private s3 test works --- .../kuttl/spark-pi-private-s3/00-assert.yaml | 18 ++++--- .../spark-pi-private-s3/00-s3-secret.yaml | 4 ++ .../00-s3-upload-container.yaml | 36 -------------- .../00-serviceaccount.yaml.j2 | 29 +++++++++++ .../spark-pi-private-s3/00-setup-minio.yaml | 48 +++++++++++++++---- .../01-prepare-bucket.yaml.j2 | 12 ++--- .../spark-pi-private-s3/02-deps-volume.yaml | 7 ++- .../10-deploy-spark-app.yaml.j2 | 4 +- .../helm-bitnami-minio-values.yaml | 23 +++++++++ .../spark-pi-public-s3/02-deps-volume.yaml | 3 +- .../10-deploy-spark-app.yaml.j2 | 4 +- tests/test-definition.yaml | 9 ++-- 12 files changed, 127 insertions(+), 70 deletions(-) delete mode 100644 tests/templates/kuttl/spark-pi-private-s3/00-s3-upload-container.yaml create mode 100644 tests/templates/kuttl/spark-pi-private-s3/00-serviceaccount.yaml.j2 create mode 100644 tests/templates/kuttl/spark-pi-private-s3/helm-bitnami-minio-values.yaml diff --git a/tests/templates/kuttl/spark-pi-private-s3/00-assert.yaml b/tests/templates/kuttl/spark-pi-private-s3/00-assert.yaml index 20f246c3..f7295267 100644 --- a/tests/templates/kuttl/spark-pi-private-s3/00-assert.yaml +++ b/tests/templates/kuttl/spark-pi-private-s3/00-assert.yaml @@ -9,13 +9,17 @@ apiVersion: v1 kind: Service metadata: name: test-minio - labels: - app: minio --- -apiVersion: apps/v1 -kind: StatefulSet +apiVersion: v1 +kind: Pod metadata: - name: minio-mc + name: minio-client + labels: + app: minio-client status: - readyReplicas: 1 - replicas: 1 + phase: Running +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: integration-tests-sa diff --git a/tests/templates/kuttl/spark-pi-private-s3/00-s3-secret.yaml b/tests/templates/kuttl/spark-pi-private-s3/00-s3-secret.yaml index 0b9c799a..235e1049 100644 --- a/tests/templates/kuttl/spark-pi-private-s3/00-s3-secret.yaml +++ b/tests/templates/kuttl/spark-pi-private-s3/00-s3-secret.yaml @@ -9,6 +9,10 @@ timeout: 240 stringData: accessKeyId: minioAccessKey secretAccessKey: minioSecretKey + # The following two entries are used by the Bitnami chart for MinIO to + # set up creedentials for accessing buckets managed by the MinIO tenant. + root-user: minioAccessKey + root-password: minioSecretKey --- apiVersion: secrets.stackable.tech/v1alpha1 kind: SecretClass diff --git a/tests/templates/kuttl/spark-pi-private-s3/00-s3-upload-container.yaml b/tests/templates/kuttl/spark-pi-private-s3/00-s3-upload-container.yaml deleted file mode 100644 index dbd4c93d..00000000 --- a/tests/templates/kuttl/spark-pi-private-s3/00-s3-upload-container.yaml +++ /dev/null @@ -1,36 +0,0 @@ ---- -apiVersion: v1 -kind: Service -metadata: - name: minio-mc - labels: - app: minio-mc -timeout: 240 -spec: - clusterIP: None - selector: - app: minio-mc ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: minio-mc - labels: - app: minio-mc -timeout: 240 -spec: - replicas: 1 - serviceName: "minio-mc" - selector: - matchLabels: - app: minio-mc - template: - metadata: - labels: - app: minio-mc - spec: - containers: - - name: minio-mc - image: bitnami/minio:2022-debian-10 - stdin: true - tty: true diff --git a/tests/templates/kuttl/spark-pi-private-s3/00-serviceaccount.yaml.j2 b/tests/templates/kuttl/spark-pi-private-s3/00-serviceaccount.yaml.j2 new file mode 100644 index 00000000..9cbf0351 --- /dev/null +++ b/tests/templates/kuttl/spark-pi-private-s3/00-serviceaccount.yaml.j2 @@ -0,0 +1,29 @@ +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: use-integration-tests-scc +rules: +{% if test_scenario['values']['openshift'] == "true" %} + - apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + resourceNames: ["privileged"] + verbs: ["use"] +{% endif %} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: integration-tests-sa +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: use-integration-tests-scc +subjects: + - kind: ServiceAccount + name: integration-tests-sa +roleRef: + kind: Role + name: use-integration-tests-scc + apiGroup: rbac.authorization.k8s.io diff --git a/tests/templates/kuttl/spark-pi-private-s3/00-setup-minio.yaml b/tests/templates/kuttl/spark-pi-private-s3/00-setup-minio.yaml index 0aaafb93..ef60c970 100644 --- a/tests/templates/kuttl/spark-pi-private-s3/00-setup-minio.yaml +++ b/tests/templates/kuttl/spark-pi-private-s3/00-setup-minio.yaml @@ -5,12 +5,44 @@ commands: - script: >- helm install test-minio --namespace $NAMESPACE - --version 4.0.2 - --set mode=standalone - --set replicas=1 - --set persistence.enabled=false - --set buckets[0].name=my-bucket,buckets[0].policy=none - --set users[0].accessKey=minioAccessKey,users[0].secretKey=minioSecretKey,users[0].policy=readwrite - --set resources.requests.memory=1Gi - --repo https://charts.min.io/ minio + --version 11.9.2 + -f helm-bitnami-minio-values.yaml + --repo https://charts.bitnami.com/bitnami minio timeout: 240 +--- +apiVersion: v1 +kind: Pod +metadata: + name: minio-client + labels: + app: minio-client +spec: + selector: + matchLabels: + app: minio-client + restartPolicy: Never + containers: + - name: minio-client + image: docker.io/bitnami/minio-client:2022.8.11-debian-11-r3 + command: ["bash", "-c", "sleep infinity"] + stdin: true + tty: true + env: + - name: MINIO_SERVER_ACCESS_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: root-user + optional: false + - name: MINIO_SERVER_SECRET_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: root-password + optional: false + - name: MINIO_SERVER_HOST + value: test-minio + - name: MINIO_SERVER_PORT_NUMBER + value: "9000" + - name: MINIO_SERVER_SCHEME + value: http diff --git a/tests/templates/kuttl/spark-pi-private-s3/01-prepare-bucket.yaml.j2 b/tests/templates/kuttl/spark-pi-private-s3/01-prepare-bucket.yaml.j2 index c1f1e994..d917f328 100644 --- a/tests/templates/kuttl/spark-pi-private-s3/01-prepare-bucket.yaml.j2 +++ b/tests/templates/kuttl/spark-pi-private-s3/01-prepare-bucket.yaml.j2 @@ -2,10 +2,8 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - - script: >- - kubectl exec -n $NAMESPACE minio-mc-0 -- - sh -c 'mc alias set test-minio http://test-minio:9000/ minioAccessKey minioSecretKey' - - script: kubectl cp -n $NAMESPACE spark-examples_2.12-{{ test_scenario['values']['spark'] }}.jar minio-mc-0:/tmp - - script: >- - kubectl exec -n $NAMESPACE minio-mc-0 -- - mc cp /tmp/spark-examples_2.12-{{ test_scenario['values']['spark'] }}.jar test-minio/my-bucket + - command: kubectl cp -n $NAMESPACE spark-examples_2.12-{{ test_scenario['values']['spark'] }}.jar minio-client:/tmp + - command: kubectl exec -n $NAMESPACE minio-client -- sh -c 'mc alias set test-minio http://test-minio:9000 $$MINIO_SERVER_ACCESS_KEY $$MINIO_SERVER_SECRET_KEY' + - command: kubectl exec -n $NAMESPACE minio-client -- mc mb test-minio/my-bucket + - script: >- + kubectl exec -n $NAMESPACE minio-client -- mc cp /tmp/spark-examples_2.12-{{ test_scenario['values']['spark'] }}.jar test-minio/my-bucket diff --git a/tests/templates/kuttl/spark-pi-private-s3/02-deps-volume.yaml b/tests/templates/kuttl/spark-pi-private-s3/02-deps-volume.yaml index c1e8a013..848c7a0e 100644 --- a/tests/templates/kuttl/spark-pi-private-s3/02-deps-volume.yaml +++ b/tests/templates/kuttl/spark-pi-private-s3/02-deps-volume.yaml @@ -24,6 +24,9 @@ spec: - name: job-deps persistentVolumeClaim: claimName: spark-pi-private-pvc + serviceAccountName: integration-tests-sa + securityContext: + runAsGroup: 0 containers: - name: aws-deps image: docker.stackable.tech/stackable/tools:0.2.0-stackable0 @@ -41,10 +44,10 @@ spec: "-o", "pipefail", "-c", - "mkdir -p ${DEST_DIR} && curl -L https://search.maven.org/remotecontent?filepath=org/apache/hadoop/hadoop-aws/${HADOOP}/hadoop-aws-${HADOOP}.jar -o ${DEST_DIR}/hadoop-aws-${HADOOP}.jar && curl -L https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${AWS}/aws-java-sdk-bundle-${AWS}.jar -o ${DEST_DIR}/aws-java-sdk-bundle-${AWS}.jar && chown -R stackable:stackable ${DEST_DIR} && chmod -R a=,u=rwX ${DEST_DIR}", + "curl -L https://search.maven.org/remotecontent?filepath=org/apache/hadoop/hadoop-aws/${HADOOP}/hadoop-aws-${HADOOP}.jar -o ${DEST_DIR}/hadoop-aws-${HADOOP}.jar && curl -L https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${AWS}/aws-java-sdk-bundle-${AWS}.jar -o ${DEST_DIR}/aws-java-sdk-bundle-${AWS}.jar", ] volumeMounts: - name: job-deps - mountPath: /dependencies + mountPath: /dependencies/jars securityContext: runAsUser: 0 diff --git a/tests/templates/kuttl/spark-pi-private-s3/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/spark-pi-private-s3/10-deploy-spark-app.yaml.j2 index 9f6c8088..e6b701a2 100644 --- a/tests/templates/kuttl/spark-pi-private-s3/10-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/spark-pi-private-s3/10-deploy-spark-app.yaml.j2 @@ -34,7 +34,7 @@ spec: memory: "512m" volumeMounts: - name: spark-pi-deps - mountPath: /dependencies + mountPath: /dependencies/jars nodeSelector: node: "1" executor: @@ -43,6 +43,6 @@ spec: memory: "512m" volumeMounts: - name: spark-pi-deps - mountPath: /dependencies + mountPath: /dependencies/jars nodeSelector: node: "1" diff --git a/tests/templates/kuttl/spark-pi-private-s3/helm-bitnami-minio-values.yaml b/tests/templates/kuttl/spark-pi-private-s3/helm-bitnami-minio-values.yaml new file mode 100644 index 00000000..c8891024 --- /dev/null +++ b/tests/templates/kuttl/spark-pi-private-s3/helm-bitnami-minio-values.yaml @@ -0,0 +1,23 @@ +--- +volumePermissions: + enabled: false + +podSecurityContext: + enabled: false + +containerSecurityContext: + enabled: false + +mode: standalone + +disableWebUI: true + +persistence: + enabled: false + +resources: + requests: + memory: 1Gi + +auth: + existingSecret: minio-credentials diff --git a/tests/templates/kuttl/spark-pi-public-s3/02-deps-volume.yaml b/tests/templates/kuttl/spark-pi-public-s3/02-deps-volume.yaml index d104e7fe..cf1493fc 100644 --- a/tests/templates/kuttl/spark-pi-public-s3/02-deps-volume.yaml +++ b/tests/templates/kuttl/spark-pi-public-s3/02-deps-volume.yaml @@ -18,7 +18,7 @@ spec: template: spec: nodeSelector: - node: "2" + node: "1" restartPolicy: Never volumes: - name: job-deps @@ -26,7 +26,6 @@ spec: claimName: spark-pi-public-pvc serviceAccountName: integration-tests-sa securityContext: - fsGroup: 1000 runAsGroup: 0 containers: - name: aws-deps diff --git a/tests/templates/kuttl/spark-pi-public-s3/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/spark-pi-public-s3/10-deploy-spark-app.yaml.j2 index bb049c03..3b27cbce 100644 --- a/tests/templates/kuttl/spark-pi-public-s3/10-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/spark-pi-public-s3/10-deploy-spark-app.yaml.j2 @@ -34,7 +34,7 @@ spec: - name: spark-pi-deps mountPath: /stackable/dependencies/jars nodeSelector: - node: "2" + node: "1" executor: cores: 1 instances: 1 @@ -43,4 +43,4 @@ spec: - name: spark-pi-deps mountPath: /stackable/dependencies/jars nodeSelector: - node: "2" + node: "1" diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index bc4456fc..38d2b05c 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -17,10 +17,11 @@ dimensions: values: - 0.1.0 tests: -# - name: spark-pi-private-s3 -# dimensions: -# - spark -# - stackable + - name: spark-pi-private-s3 + dimensions: + - spark + - stackable + - openshift - name: spark-pi-public-s3 dimensions: - spark From bc66a60ee14aad22c8375d4f8c2823a94b005172 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Tue, 30 Aug 2022 10:33:21 +0200 Subject: [PATCH 14/22] kuttl test spark-ny-public-s3 works. --- .../kuttl/spark-ny-public-s3/00-assert.yaml | 18 ++++--- .../00-s3-upload-container.yaml | 36 -------------- .../00-serviceaccount.yaml.j2 | 29 ++++++++++++ .../spark-ny-public-s3/00-setup-minio.yaml | 47 ++++++++++++++++--- .../01-prepare-bucket.yaml.j2 | 22 ++++----- .../spark-ny-public-s3/02-deps-volume.yaml | 9 ++-- .../10-deploy-spark-app.yaml.j2 | 4 +- .../helm-bitnami-minio-values.yaml | 20 ++++++++ tests/test-definition.yaml | 9 ++-- 9 files changed, 123 insertions(+), 71 deletions(-) delete mode 100644 tests/templates/kuttl/spark-ny-public-s3/00-s3-upload-container.yaml create mode 100644 tests/templates/kuttl/spark-ny-public-s3/00-serviceaccount.yaml.j2 create mode 100644 tests/templates/kuttl/spark-ny-public-s3/helm-bitnami-minio-values.yaml diff --git a/tests/templates/kuttl/spark-ny-public-s3/00-assert.yaml b/tests/templates/kuttl/spark-ny-public-s3/00-assert.yaml index 20f246c3..f7295267 100644 --- a/tests/templates/kuttl/spark-ny-public-s3/00-assert.yaml +++ b/tests/templates/kuttl/spark-ny-public-s3/00-assert.yaml @@ -9,13 +9,17 @@ apiVersion: v1 kind: Service metadata: name: test-minio - labels: - app: minio --- -apiVersion: apps/v1 -kind: StatefulSet +apiVersion: v1 +kind: Pod metadata: - name: minio-mc + name: minio-client + labels: + app: minio-client status: - readyReplicas: 1 - replicas: 1 + phase: Running +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: integration-tests-sa diff --git a/tests/templates/kuttl/spark-ny-public-s3/00-s3-upload-container.yaml b/tests/templates/kuttl/spark-ny-public-s3/00-s3-upload-container.yaml deleted file mode 100644 index dbd4c93d..00000000 --- a/tests/templates/kuttl/spark-ny-public-s3/00-s3-upload-container.yaml +++ /dev/null @@ -1,36 +0,0 @@ ---- -apiVersion: v1 -kind: Service -metadata: - name: minio-mc - labels: - app: minio-mc -timeout: 240 -spec: - clusterIP: None - selector: - app: minio-mc ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: minio-mc - labels: - app: minio-mc -timeout: 240 -spec: - replicas: 1 - serviceName: "minio-mc" - selector: - matchLabels: - app: minio-mc - template: - metadata: - labels: - app: minio-mc - spec: - containers: - - name: minio-mc - image: bitnami/minio:2022-debian-10 - stdin: true - tty: true diff --git a/tests/templates/kuttl/spark-ny-public-s3/00-serviceaccount.yaml.j2 b/tests/templates/kuttl/spark-ny-public-s3/00-serviceaccount.yaml.j2 new file mode 100644 index 00000000..9cbf0351 --- /dev/null +++ b/tests/templates/kuttl/spark-ny-public-s3/00-serviceaccount.yaml.j2 @@ -0,0 +1,29 @@ +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: use-integration-tests-scc +rules: +{% if test_scenario['values']['openshift'] == "true" %} + - apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + resourceNames: ["privileged"] + verbs: ["use"] +{% endif %} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: integration-tests-sa +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: use-integration-tests-scc +subjects: + - kind: ServiceAccount + name: integration-tests-sa +roleRef: + kind: Role + name: use-integration-tests-scc + apiGroup: rbac.authorization.k8s.io diff --git a/tests/templates/kuttl/spark-ny-public-s3/00-setup-minio.yaml b/tests/templates/kuttl/spark-ny-public-s3/00-setup-minio.yaml index b74ca5ff..2ba4dcac 100644 --- a/tests/templates/kuttl/spark-ny-public-s3/00-setup-minio.yaml +++ b/tests/templates/kuttl/spark-ny-public-s3/00-setup-minio.yaml @@ -5,11 +5,44 @@ commands: - script: >- helm install test-minio --namespace $NAMESPACE - --version 4.0.2 - --set mode=standalone - --set replicas=1 - --set persistence.enabled=false - --set buckets[0].name=my-bucket,buckets[0].policy=public - --set resources.requests.memory=1Gi - --repo https://charts.min.io/ minio + --version 11.9.2 + -f helm-bitnami-minio-values.yaml + --repo https://charts.bitnami.com/bitnami minio timeout: 240 +--- +apiVersion: v1 +kind: Pod +metadata: + name: minio-client + labels: + app: minio-client +spec: + selector: + matchLabels: + app: minio-client + restartPolicy: Never + containers: + - name: minio-client + image: docker.io/bitnami/minio-client:2022.8.11-debian-11-r3 + command: ["bash", "-c", "sleep infinity"] + stdin: true + tty: true + env: + - name: MINIO_SERVER_ACCESS_KEY + valueFrom: + secretKeyRef: + name: test-minio + key: root-user + optional: false + - name: MINIO_SERVER_SECRET_KEY + valueFrom: + secretKeyRef: + name: test-minio + key: root-password + optional: false + - name: MINIO_SERVER_HOST + value: test-minio + - name: MINIO_SERVER_PORT_NUMBER + value: "9000" + - name: MINIO_SERVER_SCHEME + value: http diff --git a/tests/templates/kuttl/spark-ny-public-s3/01-prepare-bucket.yaml.j2 b/tests/templates/kuttl/spark-ny-public-s3/01-prepare-bucket.yaml.j2 index 0154e5ae..00d89bf2 100644 --- a/tests/templates/kuttl/spark-ny-public-s3/01-prepare-bucket.yaml.j2 +++ b/tests/templates/kuttl/spark-ny-public-s3/01-prepare-bucket.yaml.j2 @@ -2,14 +2,14 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - - script: >- - kubectl exec -n $NAMESPACE minio-mc-0 -- - sh -c 'mc alias set test-minio http://test-minio:9000/' - - script: kubectl cp -n $NAMESPACE ny-tlc-report-1.1.0-{{ test_scenario['values']['spark'] }}.jar minio-mc-0:/tmp - - script: kubectl cp -n $NAMESPACE yellow_tripdata_2021-07.csv minio-mc-0:/tmp - - script: >- - kubectl exec -n $NAMESPACE minio-mc-0 -- - mc cp /tmp/ny-tlc-report-1.1.0-{{ test_scenario['values']['spark'] }}.jar test-minio/my-bucket - - script: >- - kubectl exec -n $NAMESPACE minio-mc-0 -- - mc cp /tmp/yellow_tripdata_2021-07.csv test-minio/my-bucket + - command: kubectl cp -n $NAMESPACE ny-tlc-report-1.1.0-{{ test_scenario['values']['spark'] }}.jar minio-client:/tmp + - command: kubectl cp -n $NAMESPACE yellow_tripdata_2021-07.csv minio-client:/tmp + - command: kubectl exec -n $NAMESPACE minio-client -- sh -c 'mc alias set test-minio http://test-minio:9000 $$MINIO_SERVER_ACCESS_KEY $$MINIO_SERVER_SECRET_KEY' + - command: kubectl exec -n $NAMESPACE minio-client -- mc mb test-minio/my-bucket + - command: kubectl exec -n $NAMESPACE minio-client -- mc policy set public test-minio/my-bucket + - script: >- + kubectl exec -n $NAMESPACE minio-client -- + mc cp /tmp/ny-tlc-report-1.1.0-{{ test_scenario['values']['spark'] }}.jar test-minio/my-bucket + - script: >- + kubectl exec -n $NAMESPACE minio-client -- + mc cp /tmp/yellow_tripdata_2021-07.csv test-minio/my-bucket diff --git a/tests/templates/kuttl/spark-ny-public-s3/02-deps-volume.yaml b/tests/templates/kuttl/spark-ny-public-s3/02-deps-volume.yaml index af2e490a..618c74d9 100644 --- a/tests/templates/kuttl/spark-ny-public-s3/02-deps-volume.yaml +++ b/tests/templates/kuttl/spark-ny-public-s3/02-deps-volume.yaml @@ -24,6 +24,9 @@ spec: - name: job-deps persistentVolumeClaim: claimName: spark-ny-pvc + securityContext: + runAsGroup: 0 + serviceAccountName: integration-tests-sa containers: - name: aws-deps image: docker.stackable.tech/stackable/tools:0.2.0-stackable0 @@ -41,10 +44,8 @@ spec: "-o", "pipefail", "-c", - "mkdir -p ${DEST_DIR} && curl -L https://search.maven.org/remotecontent?filepath=org/apache/hadoop/hadoop-aws/${HADOOP}/hadoop-aws-${HADOOP}.jar -o ${DEST_DIR}/hadoop-aws-${HADOOP}.jar && curl -L https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${AWS}/aws-java-sdk-bundle-${AWS}.jar -o ${DEST_DIR}/aws-java-sdk-bundle-${AWS}.jar && chown -R stackable:stackable ${DEST_DIR} && chmod -R a=,u=rwX ${DEST_DIR}", + "curl -L https://search.maven.org/remotecontent?filepath=org/apache/hadoop/hadoop-aws/${HADOOP}/hadoop-aws-${HADOOP}.jar -o ${DEST_DIR}/hadoop-aws-${HADOOP}.jar && curl -L https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${AWS}/aws-java-sdk-bundle-${AWS}.jar -o ${DEST_DIR}/aws-java-sdk-bundle-${AWS}.jar", ] volumeMounts: - name: job-deps - mountPath: /dependencies - securityContext: - runAsUser: 0 + mountPath: /dependencies/jars diff --git a/tests/templates/kuttl/spark-ny-public-s3/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/spark-ny-public-s3/10-deploy-spark-app.yaml.j2 index dcc9ce63..8831e58d 100644 --- a/tests/templates/kuttl/spark-ny-public-s3/10-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/spark-ny-public-s3/10-deploy-spark-app.yaml.j2 @@ -45,7 +45,7 @@ spec: memory: "512m" volumeMounts: - name: spark-ny-deps - mountPath: /dependencies + mountPath: /dependencies/jars - name: cm-job-arguments mountPath: /arguments nodeSelector: @@ -56,7 +56,7 @@ spec: memory: "512m" volumeMounts: - name: spark-ny-deps - mountPath: /dependencies + mountPath: /dependencies/jars - name: cm-job-arguments mountPath: /arguments nodeSelector: diff --git a/tests/templates/kuttl/spark-ny-public-s3/helm-bitnami-minio-values.yaml b/tests/templates/kuttl/spark-ny-public-s3/helm-bitnami-minio-values.yaml new file mode 100644 index 00000000..27705b9b --- /dev/null +++ b/tests/templates/kuttl/spark-ny-public-s3/helm-bitnami-minio-values.yaml @@ -0,0 +1,20 @@ +--- +volumePermissions: + enabled: false + +podSecurityContext: + enabled: false + +containerSecurityContext: + enabled: false + +mode: standalone + +disableWebUI: true + +persistence: + enabled: false + +resources: + requests: + memory: 1Gi diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 38d2b05c..6fe3ee6f 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -27,10 +27,11 @@ tests: - spark - stackable - openshift -# - name: spark-ny-public-s3 -# dimensions: -# - spark -# - stackable + - name: spark-ny-public-s3 + dimensions: + - spark + - stackable + - openshift # - name: node-selector # dimensions: # - spark From a7a3c09a08d46a3de23e3a9c10ceabac17b7cd2d Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Tue, 30 Aug 2022 11:52:51 +0200 Subject: [PATCH 15/22] All kuttl tests work now. --- .../pyspark-ny-public-s3-image/00-assert.yaml | 18 ++++--- .../00-s3-upload-container.yaml | 36 -------------- .../00-serviceaccount.yaml.j2 | 29 ++++++++++++ .../00-setup-minio.yaml | 47 ++++++++++++++++--- .../01-prepare-bucket.yaml.j2 | 12 ++--- .../02-deps-volume.yaml | 9 ++-- .../10-deploy-spark-app.yaml.j2 | 4 +- .../helm-bitnami-minio-values.yaml | 20 ++++++++ .../kuttl/pyspark-ny-public-s3/00-assert.yaml | 18 ++++--- .../00-s3-upload-container.yaml | 36 -------------- .../00-serviceaccount.yaml.j2 | 29 ++++++++++++ .../pyspark-ny-public-s3/00-setup-minio.yaml | 47 ++++++++++++++++--- .../01-prepare-bucket.yaml.j2 | 18 +++---- .../pyspark-ny-public-s3/02-deps-volume.yaml | 9 ++-- .../10-deploy-spark-app.yaml.j2 | 4 +- .../helm-bitnami-minio-values.yaml | 20 ++++++++ tests/test-definition.yaml | 43 +++++++++-------- 17 files changed, 249 insertions(+), 150 deletions(-) delete mode 100644 tests/templates/kuttl/pyspark-ny-public-s3-image/00-s3-upload-container.yaml create mode 100644 tests/templates/kuttl/pyspark-ny-public-s3-image/00-serviceaccount.yaml.j2 create mode 100644 tests/templates/kuttl/pyspark-ny-public-s3-image/helm-bitnami-minio-values.yaml delete mode 100644 tests/templates/kuttl/pyspark-ny-public-s3/00-s3-upload-container.yaml create mode 100644 tests/templates/kuttl/pyspark-ny-public-s3/00-serviceaccount.yaml.j2 create mode 100644 tests/templates/kuttl/pyspark-ny-public-s3/helm-bitnami-minio-values.yaml diff --git a/tests/templates/kuttl/pyspark-ny-public-s3-image/00-assert.yaml b/tests/templates/kuttl/pyspark-ny-public-s3-image/00-assert.yaml index 20f246c3..f7295267 100644 --- a/tests/templates/kuttl/pyspark-ny-public-s3-image/00-assert.yaml +++ b/tests/templates/kuttl/pyspark-ny-public-s3-image/00-assert.yaml @@ -9,13 +9,17 @@ apiVersion: v1 kind: Service metadata: name: test-minio - labels: - app: minio --- -apiVersion: apps/v1 -kind: StatefulSet +apiVersion: v1 +kind: Pod metadata: - name: minio-mc + name: minio-client + labels: + app: minio-client status: - readyReplicas: 1 - replicas: 1 + phase: Running +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: integration-tests-sa diff --git a/tests/templates/kuttl/pyspark-ny-public-s3-image/00-s3-upload-container.yaml b/tests/templates/kuttl/pyspark-ny-public-s3-image/00-s3-upload-container.yaml deleted file mode 100644 index dbd4c93d..00000000 --- a/tests/templates/kuttl/pyspark-ny-public-s3-image/00-s3-upload-container.yaml +++ /dev/null @@ -1,36 +0,0 @@ ---- -apiVersion: v1 -kind: Service -metadata: - name: minio-mc - labels: - app: minio-mc -timeout: 240 -spec: - clusterIP: None - selector: - app: minio-mc ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: minio-mc - labels: - app: minio-mc -timeout: 240 -spec: - replicas: 1 - serviceName: "minio-mc" - selector: - matchLabels: - app: minio-mc - template: - metadata: - labels: - app: minio-mc - spec: - containers: - - name: minio-mc - image: bitnami/minio:2022-debian-10 - stdin: true - tty: true diff --git a/tests/templates/kuttl/pyspark-ny-public-s3-image/00-serviceaccount.yaml.j2 b/tests/templates/kuttl/pyspark-ny-public-s3-image/00-serviceaccount.yaml.j2 new file mode 100644 index 00000000..9cbf0351 --- /dev/null +++ b/tests/templates/kuttl/pyspark-ny-public-s3-image/00-serviceaccount.yaml.j2 @@ -0,0 +1,29 @@ +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: use-integration-tests-scc +rules: +{% if test_scenario['values']['openshift'] == "true" %} + - apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + resourceNames: ["privileged"] + verbs: ["use"] +{% endif %} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: integration-tests-sa +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: use-integration-tests-scc +subjects: + - kind: ServiceAccount + name: integration-tests-sa +roleRef: + kind: Role + name: use-integration-tests-scc + apiGroup: rbac.authorization.k8s.io diff --git a/tests/templates/kuttl/pyspark-ny-public-s3-image/00-setup-minio.yaml b/tests/templates/kuttl/pyspark-ny-public-s3-image/00-setup-minio.yaml index b74ca5ff..2ba4dcac 100644 --- a/tests/templates/kuttl/pyspark-ny-public-s3-image/00-setup-minio.yaml +++ b/tests/templates/kuttl/pyspark-ny-public-s3-image/00-setup-minio.yaml @@ -5,11 +5,44 @@ commands: - script: >- helm install test-minio --namespace $NAMESPACE - --version 4.0.2 - --set mode=standalone - --set replicas=1 - --set persistence.enabled=false - --set buckets[0].name=my-bucket,buckets[0].policy=public - --set resources.requests.memory=1Gi - --repo https://charts.min.io/ minio + --version 11.9.2 + -f helm-bitnami-minio-values.yaml + --repo https://charts.bitnami.com/bitnami minio timeout: 240 +--- +apiVersion: v1 +kind: Pod +metadata: + name: minio-client + labels: + app: minio-client +spec: + selector: + matchLabels: + app: minio-client + restartPolicy: Never + containers: + - name: minio-client + image: docker.io/bitnami/minio-client:2022.8.11-debian-11-r3 + command: ["bash", "-c", "sleep infinity"] + stdin: true + tty: true + env: + - name: MINIO_SERVER_ACCESS_KEY + valueFrom: + secretKeyRef: + name: test-minio + key: root-user + optional: false + - name: MINIO_SERVER_SECRET_KEY + valueFrom: + secretKeyRef: + name: test-minio + key: root-password + optional: false + - name: MINIO_SERVER_HOST + value: test-minio + - name: MINIO_SERVER_PORT_NUMBER + value: "9000" + - name: MINIO_SERVER_SCHEME + value: http diff --git a/tests/templates/kuttl/pyspark-ny-public-s3-image/01-prepare-bucket.yaml.j2 b/tests/templates/kuttl/pyspark-ny-public-s3-image/01-prepare-bucket.yaml.j2 index fd517e3d..9b84c9e9 100644 --- a/tests/templates/kuttl/pyspark-ny-public-s3-image/01-prepare-bucket.yaml.j2 +++ b/tests/templates/kuttl/pyspark-ny-public-s3-image/01-prepare-bucket.yaml.j2 @@ -2,10 +2,8 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - - script: >- - kubectl exec -n $NAMESPACE minio-mc-0 -- - sh -c 'mc alias set test-minio http://test-minio:9000/' - - script: kubectl cp -n $NAMESPACE yellow_tripdata_2021-07.csv minio-mc-0:/tmp - - script: >- - kubectl exec -n $NAMESPACE minio-mc-0 -- - mc cp /tmp/yellow_tripdata_2021-07.csv test-minio/my-bucket + - command: kubectl cp -n $NAMESPACE yellow_tripdata_2021-07.csv minio-client:/tmp + - command: kubectl exec -n $NAMESPACE minio-client -- sh -c 'mc alias set test-minio http://test-minio:9000 $$MINIO_SERVER_ACCESS_KEY $$MINIO_SERVER_SECRET_KEY' + - command: kubectl exec -n $NAMESPACE minio-client -- mc mb test-minio/my-bucket + - command: kubectl exec -n $NAMESPACE minio-client -- mc policy set public test-minio/my-bucket + - command: kubectl exec -n $NAMESPACE minio-client -- mc cp /tmp/yellow_tripdata_2021-07.csv test-minio/my-bucket diff --git a/tests/templates/kuttl/pyspark-ny-public-s3-image/02-deps-volume.yaml b/tests/templates/kuttl/pyspark-ny-public-s3-image/02-deps-volume.yaml index fe80e464..2f5ba82b 100644 --- a/tests/templates/kuttl/pyspark-ny-public-s3-image/02-deps-volume.yaml +++ b/tests/templates/kuttl/pyspark-ny-public-s3-image/02-deps-volume.yaml @@ -20,6 +20,9 @@ spec: nodeSelector: node: "1" restartPolicy: Never + securityContext: + runAsGroup: 0 + serviceAccountName: integration-tests-sa volumes: - name: job-deps persistentVolumeClaim: @@ -41,10 +44,8 @@ spec: "-o", "pipefail", "-c", - "mkdir -p ${DEST_DIR} && curl -L https://search.maven.org/remotecontent?filepath=org/apache/hadoop/hadoop-aws/${HADOOP}/hadoop-aws-${HADOOP}.jar -o ${DEST_DIR}/hadoop-aws-${HADOOP}.jar && curl -L https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${AWS}/aws-java-sdk-bundle-${AWS}.jar -o ${DEST_DIR}/aws-java-sdk-bundle-${AWS}.jar && chown -R stackable:stackable ${DEST_DIR} && chmod -R a=,u=rwX ${DEST_DIR}", + "curl -L https://search.maven.org/remotecontent?filepath=org/apache/hadoop/hadoop-aws/${HADOOP}/hadoop-aws-${HADOOP}.jar -o ${DEST_DIR}/hadoop-aws-${HADOOP}.jar && curl -L https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${AWS}/aws-java-sdk-bundle-${AWS}.jar -o ${DEST_DIR}/aws-java-sdk-bundle-${AWS}.jar", ] volumeMounts: - name: job-deps - mountPath: /dependencies - securityContext: - runAsUser: 0 + mountPath: /dependencies/jars diff --git a/tests/templates/kuttl/pyspark-ny-public-s3-image/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/pyspark-ny-public-s3-image/10-deploy-spark-app.yaml.j2 index 91f8ebf4..ed35659d 100644 --- a/tests/templates/kuttl/pyspark-ny-public-s3-image/10-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/pyspark-ny-public-s3-image/10-deploy-spark-app.yaml.j2 @@ -38,11 +38,11 @@ spec: memory: "512m" volumeMounts: - name: job-deps - mountPath: /dependencies + mountPath: /dependencies/jars executor: cores: 1 instances: 3 memory: "512m" volumeMounts: - name: job-deps - mountPath: /dependencies + mountPath: /dependencies/jars diff --git a/tests/templates/kuttl/pyspark-ny-public-s3-image/helm-bitnami-minio-values.yaml b/tests/templates/kuttl/pyspark-ny-public-s3-image/helm-bitnami-minio-values.yaml new file mode 100644 index 00000000..27705b9b --- /dev/null +++ b/tests/templates/kuttl/pyspark-ny-public-s3-image/helm-bitnami-minio-values.yaml @@ -0,0 +1,20 @@ +--- +volumePermissions: + enabled: false + +podSecurityContext: + enabled: false + +containerSecurityContext: + enabled: false + +mode: standalone + +disableWebUI: true + +persistence: + enabled: false + +resources: + requests: + memory: 1Gi diff --git a/tests/templates/kuttl/pyspark-ny-public-s3/00-assert.yaml b/tests/templates/kuttl/pyspark-ny-public-s3/00-assert.yaml index 20f246c3..f7295267 100644 --- a/tests/templates/kuttl/pyspark-ny-public-s3/00-assert.yaml +++ b/tests/templates/kuttl/pyspark-ny-public-s3/00-assert.yaml @@ -9,13 +9,17 @@ apiVersion: v1 kind: Service metadata: name: test-minio - labels: - app: minio --- -apiVersion: apps/v1 -kind: StatefulSet +apiVersion: v1 +kind: Pod metadata: - name: minio-mc + name: minio-client + labels: + app: minio-client status: - readyReplicas: 1 - replicas: 1 + phase: Running +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: integration-tests-sa diff --git a/tests/templates/kuttl/pyspark-ny-public-s3/00-s3-upload-container.yaml b/tests/templates/kuttl/pyspark-ny-public-s3/00-s3-upload-container.yaml deleted file mode 100644 index dbd4c93d..00000000 --- a/tests/templates/kuttl/pyspark-ny-public-s3/00-s3-upload-container.yaml +++ /dev/null @@ -1,36 +0,0 @@ ---- -apiVersion: v1 -kind: Service -metadata: - name: minio-mc - labels: - app: minio-mc -timeout: 240 -spec: - clusterIP: None - selector: - app: minio-mc ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: minio-mc - labels: - app: minio-mc -timeout: 240 -spec: - replicas: 1 - serviceName: "minio-mc" - selector: - matchLabels: - app: minio-mc - template: - metadata: - labels: - app: minio-mc - spec: - containers: - - name: minio-mc - image: bitnami/minio:2022-debian-10 - stdin: true - tty: true diff --git a/tests/templates/kuttl/pyspark-ny-public-s3/00-serviceaccount.yaml.j2 b/tests/templates/kuttl/pyspark-ny-public-s3/00-serviceaccount.yaml.j2 new file mode 100644 index 00000000..9cbf0351 --- /dev/null +++ b/tests/templates/kuttl/pyspark-ny-public-s3/00-serviceaccount.yaml.j2 @@ -0,0 +1,29 @@ +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: use-integration-tests-scc +rules: +{% if test_scenario['values']['openshift'] == "true" %} + - apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + resourceNames: ["privileged"] + verbs: ["use"] +{% endif %} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: integration-tests-sa +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: use-integration-tests-scc +subjects: + - kind: ServiceAccount + name: integration-tests-sa +roleRef: + kind: Role + name: use-integration-tests-scc + apiGroup: rbac.authorization.k8s.io diff --git a/tests/templates/kuttl/pyspark-ny-public-s3/00-setup-minio.yaml b/tests/templates/kuttl/pyspark-ny-public-s3/00-setup-minio.yaml index b74ca5ff..2ba4dcac 100644 --- a/tests/templates/kuttl/pyspark-ny-public-s3/00-setup-minio.yaml +++ b/tests/templates/kuttl/pyspark-ny-public-s3/00-setup-minio.yaml @@ -5,11 +5,44 @@ commands: - script: >- helm install test-minio --namespace $NAMESPACE - --version 4.0.2 - --set mode=standalone - --set replicas=1 - --set persistence.enabled=false - --set buckets[0].name=my-bucket,buckets[0].policy=public - --set resources.requests.memory=1Gi - --repo https://charts.min.io/ minio + --version 11.9.2 + -f helm-bitnami-minio-values.yaml + --repo https://charts.bitnami.com/bitnami minio timeout: 240 +--- +apiVersion: v1 +kind: Pod +metadata: + name: minio-client + labels: + app: minio-client +spec: + selector: + matchLabels: + app: minio-client + restartPolicy: Never + containers: + - name: minio-client + image: docker.io/bitnami/minio-client:2022.8.11-debian-11-r3 + command: ["bash", "-c", "sleep infinity"] + stdin: true + tty: true + env: + - name: MINIO_SERVER_ACCESS_KEY + valueFrom: + secretKeyRef: + name: test-minio + key: root-user + optional: false + - name: MINIO_SERVER_SECRET_KEY + valueFrom: + secretKeyRef: + name: test-minio + key: root-password + optional: false + - name: MINIO_SERVER_HOST + value: test-minio + - name: MINIO_SERVER_PORT_NUMBER + value: "9000" + - name: MINIO_SERVER_SCHEME + value: http diff --git a/tests/templates/kuttl/pyspark-ny-public-s3/01-prepare-bucket.yaml.j2 b/tests/templates/kuttl/pyspark-ny-public-s3/01-prepare-bucket.yaml.j2 index a43d7439..aa38a10b 100644 --- a/tests/templates/kuttl/pyspark-ny-public-s3/01-prepare-bucket.yaml.j2 +++ b/tests/templates/kuttl/pyspark-ny-public-s3/01-prepare-bucket.yaml.j2 @@ -2,14 +2,10 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - - script: >- - kubectl exec -n $NAMESPACE minio-mc-0 -- - sh -c 'mc alias set test-minio http://test-minio:9000/' - - script: kubectl cp -n $NAMESPACE ny_tlc_report.py minio-mc-0:/tmp - - script: kubectl cp -n $NAMESPACE yellow_tripdata_2021-07.csv minio-mc-0:/tmp - - script: >- - kubectl exec -n $NAMESPACE minio-mc-0 -- - mc cp /tmp/ny_tlc_report.py test-minio/my-bucket - - script: >- - kubectl exec -n $NAMESPACE minio-mc-0 -- - mc cp /tmp/yellow_tripdata_2021-07.csv test-minio/my-bucket + - command: kubectl cp -n $NAMESPACE ny_tlc_report.py minio-client:/tmp + - command: kubectl cp -n $NAMESPACE yellow_tripdata_2021-07.csv minio-client:/tmp + - command: kubectl exec -n $NAMESPACE minio-client -- sh -c 'mc alias set test-minio http://test-minio:9000 $$MINIO_SERVER_ACCESS_KEY $$MINIO_SERVER_SECRET_KEY' + - command: kubectl exec -n $NAMESPACE minio-client -- mc mb test-minio/my-bucket + - command: kubectl exec -n $NAMESPACE minio-client -- mc policy set public test-minio/my-bucket + - command: kubectl exec -n $NAMESPACE minio-client -- mc cp /tmp/ny_tlc_report.py test-minio/my-bucket + - command: kubectl exec -n $NAMESPACE minio-client -- mc cp /tmp/yellow_tripdata_2021-07.csv test-minio/my-bucket diff --git a/tests/templates/kuttl/pyspark-ny-public-s3/02-deps-volume.yaml b/tests/templates/kuttl/pyspark-ny-public-s3/02-deps-volume.yaml index fe80e464..2f5ba82b 100644 --- a/tests/templates/kuttl/pyspark-ny-public-s3/02-deps-volume.yaml +++ b/tests/templates/kuttl/pyspark-ny-public-s3/02-deps-volume.yaml @@ -20,6 +20,9 @@ spec: nodeSelector: node: "1" restartPolicy: Never + securityContext: + runAsGroup: 0 + serviceAccountName: integration-tests-sa volumes: - name: job-deps persistentVolumeClaim: @@ -41,10 +44,8 @@ spec: "-o", "pipefail", "-c", - "mkdir -p ${DEST_DIR} && curl -L https://search.maven.org/remotecontent?filepath=org/apache/hadoop/hadoop-aws/${HADOOP}/hadoop-aws-${HADOOP}.jar -o ${DEST_DIR}/hadoop-aws-${HADOOP}.jar && curl -L https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${AWS}/aws-java-sdk-bundle-${AWS}.jar -o ${DEST_DIR}/aws-java-sdk-bundle-${AWS}.jar && chown -R stackable:stackable ${DEST_DIR} && chmod -R a=,u=rwX ${DEST_DIR}", + "curl -L https://search.maven.org/remotecontent?filepath=org/apache/hadoop/hadoop-aws/${HADOOP}/hadoop-aws-${HADOOP}.jar -o ${DEST_DIR}/hadoop-aws-${HADOOP}.jar && curl -L https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${AWS}/aws-java-sdk-bundle-${AWS}.jar -o ${DEST_DIR}/aws-java-sdk-bundle-${AWS}.jar", ] volumeMounts: - name: job-deps - mountPath: /dependencies - securityContext: - runAsUser: 0 + mountPath: /dependencies/jars diff --git a/tests/templates/kuttl/pyspark-ny-public-s3/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/pyspark-ny-public-s3/10-deploy-spark-app.yaml.j2 index b36a61dd..c02a3844 100644 --- a/tests/templates/kuttl/pyspark-ny-public-s3/10-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/pyspark-ny-public-s3/10-deploy-spark-app.yaml.j2 @@ -37,11 +37,11 @@ spec: memory: "512m" volumeMounts: - name: job-deps - mountPath: /dependencies + mountPath: /dependencies/jars executor: cores: 1 instances: 3 memory: "512m" volumeMounts: - name: job-deps - mountPath: /dependencies + mountPath: /dependencies/jars diff --git a/tests/templates/kuttl/pyspark-ny-public-s3/helm-bitnami-minio-values.yaml b/tests/templates/kuttl/pyspark-ny-public-s3/helm-bitnami-minio-values.yaml new file mode 100644 index 00000000..27705b9b --- /dev/null +++ b/tests/templates/kuttl/pyspark-ny-public-s3/helm-bitnami-minio-values.yaml @@ -0,0 +1,20 @@ +--- +volumePermissions: + enabled: false + +podSecurityContext: + enabled: false + +containerSecurityContext: + enabled: false + +mode: standalone + +disableWebUI: true + +persistence: + enabled: false + +resources: + requests: + memory: 1Gi diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 6fe3ee6f..8a0162ad 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -1,12 +1,13 @@ # -# Currently the tests only run against one version of Spark but will be changed -# soon to support multiple Spark versions. +# To run these tests on OpenShift you have ensure that: +# 1. The "openshift" dimension below is set to "true" +# 2. At least one node in the cluster is labeled with "node: 1" # --- dimensions: - name: openshift values: - - "true" + - "false" - name: spark values: - 3.3.0 @@ -32,20 +33,22 @@ tests: - spark - stackable - openshift -# - name: node-selector -# dimensions: -# - spark -# - stackable -# - name: spark-examples -# dimensions: -# - spark -# - stackable -# - name: pyspark-ny-public-s3 -# dimensions: -# - spark -# - stackable -# - name: pyspark-ny-public-s3-image -# dimensions: -# - spark -# - stackable -# - ny-tlc-report + - name: node-selector + dimensions: + - spark + - stackable + - name: spark-examples + dimensions: + - spark + - stackable + - name: pyspark-ny-public-s3 + dimensions: + - spark + - stackable + - openshift + - name: pyspark-ny-public-s3-image + dimensions: + - spark + - stackable + - ny-tlc-report + - openshift From a27c9ed2463448caff1f82ade4ba627456034d73 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Tue, 30 Aug 2022 13:53:28 +0200 Subject: [PATCH 16/22] Update the security context of the operator Pod This allows the operator installation to pass the pod security admission and complete successfuly. --- deploy/helm/spark-k8s-operator/values.yaml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/deploy/helm/spark-k8s-operator/values.yaml b/deploy/helm/spark-k8s-operator/values.yaml index 881f02d4..0812b6b8 100644 --- a/deploy/helm/spark-k8s-operator/values.yaml +++ b/deploy/helm/spark-k8s-operator/values.yaml @@ -21,17 +21,22 @@ podAnnotations: {} podSecurityContext: {} # fsGroup: 2000 - +# +# OpenShift 4.11 replaces the PodSecurityPolicy with a new pod security +# admission mechanism as described in this blog post [1]. +# This requires Pods to explicitely specify the securityContext. +# +# [1]: https://cloud.redhat.com/blog/pod-security-admission-in-openshift-4.11 securityContext: capabilities: drop: - - ALL + - ALL readOnlyRootFilesystem: false allowPrivilegeEscalation: false seccompProfile: type: RuntimeDefault + runAsNonRoot: true runAsUser: 1000 - # runAsNonRoot: true resources: {} # We usually recommend not to specify default resources and to leave this as a conscious From e869395a8494ba72c4ce3e4073c41c369f7df6c0 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Tue, 30 Aug 2022 13:57:01 +0200 Subject: [PATCH 17/22] make regenerate-charts --- deploy/manifests/deployment.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy/manifests/deployment.yaml b/deploy/manifests/deployment.yaml index 7ea0735b..f0f0398b 100644 --- a/deploy/manifests/deployment.yaml +++ b/deploy/manifests/deployment.yaml @@ -31,6 +31,7 @@ spec: drop: - ALL readOnlyRootFilesystem: false + runAsNonRoot: true runAsUser: 1000 seccompProfile: type: RuntimeDefault From 6aeaced70187fdca27ef18215dce3bc776c5ccc1 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Tue, 30 Aug 2022 14:32:11 +0200 Subject: [PATCH 18/22] Update CHANGELOG. --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e8f594e..b61cb423 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,9 +14,11 @@ All notable changes to this project will be documented in this file. - Add missing role to read S3Connection and S3Bucket objects ([#112]). - Update annotation due to update to rust version ([#114]). +- Update RBAC properties for OpenShift compatibility ([#126]). [#112]: https://github.com/stackabletech/spark-k8s-operator/pull/112 [#114]: https://github.com/stackabletech/spark-k8s-operator/pull/114 +[#126]: https://github.com/stackabletech/spark-k8s-operator/pull/126 ## [0.4.0] - 2022-08-03 From 39d6aa98c0ca90ddb2c76de650dc5f4b393396a7 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 31 Aug 2022 12:46:37 +0200 Subject: [PATCH 19/22] Add runAsUser to the Spark application security context. Setting runAsGroup automatically makes runAsUser required in non OpenShift environments. --- rust/crd/src/constants.rs | 2 ++ rust/operator-binary/src/spark_k8s_controller.rs | 9 ++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/rust/crd/src/constants.rs b/rust/crd/src/constants.rs index ffe871e3..a28462b0 100644 --- a/rust/crd/src/constants.rs +++ b/rust/crd/src/constants.rs @@ -20,3 +20,5 @@ pub const CONTAINER_NAME_EXECUTOR: &str = "spark-executor"; pub const ACCESS_KEY_ID: &str = "accessKeyId"; pub const SECRET_ACCESS_KEY: &str = "secretAccessKey"; pub const S3_SECRET_DIR_NAME: &str = "/stackable/secrets"; + +pub const SPARK_UID: i64 = 1000; diff --git a/rust/operator-binary/src/spark_k8s_controller.rs b/rust/operator-binary/src/spark_k8s_controller.rs index 61297d35..0e381012 100644 --- a/rust/operator-binary/src/spark_k8s_controller.rs +++ b/rust/operator-binary/src/spark_k8s_controller.rs @@ -373,7 +373,14 @@ fn spark_job( image_pull_secrets: spark_application.spark_image_pull_secrets(), security_context: PodSecurityContextBuilder::new() .fs_group(1000) - .run_as_group(0) // needed for pvc file access + // OpenShift generates UIDs for processes inside Pods. Setting the UID is optional, + // *but* if specified, OpenShift will check that the value is within the + // valid range generated by the SCC (security context constraints) for this Pod. + // On the other hand, it is *required* to set the process UID in KinD, K3S as soon + // as the runAsGroup property is set. + .run_as_user(SPARK_UID) + // Required to access files in mounted volumes on OpenShift. + .run_as_group(0) .build() .into(), // Needed for secret-operator node_selector: spark_application.driver_node_selector(), From 0bcaa033bd82710e28324713c50feefc992e5841 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 31 Aug 2022 13:45:30 +0200 Subject: [PATCH 20/22] Add runAsUser to Spark pod templates too. --- .../src/spark_k8s_controller.rs | 37 +++++++++---------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/rust/operator-binary/src/spark_k8s_controller.rs b/rust/operator-binary/src/spark_k8s_controller.rs index 0e381012..c6c76566 100644 --- a/rust/operator-binary/src/spark_k8s_controller.rs +++ b/rust/operator-binary/src/spark_k8s_controller.rs @@ -7,8 +7,8 @@ use stackable_operator::commons::s3::InlinedS3BucketSpec; use stackable_operator::commons::tls::{CaCert, TlsVerification}; use stackable_operator::k8s_openapi::api::batch::v1::{Job, JobSpec}; use stackable_operator::k8s_openapi::api::core::v1::{ - ConfigMap, ConfigMapVolumeSource, Container, EnvVar, Pod, PodSpec, PodTemplateSpec, - ServiceAccount, Volume, VolumeMount, + ConfigMap, ConfigMapVolumeSource, Container, EnvVar, Pod, PodSecurityContext, PodSpec, + PodTemplateSpec, ServiceAccount, Volume, VolumeMount, }; use stackable_operator::k8s_openapi::api::rbac::v1::{ClusterRole, RoleBinding, RoleRef, Subject}; use stackable_operator::k8s_openapi::Resource; @@ -234,11 +234,7 @@ fn pod_template( let mut pod_spec = PodSpec { containers: vec![cb.build()], volumes: Some(volumes.to_vec()), - security_context: PodSecurityContextBuilder::new() - .fs_group(1000) - .run_as_group(0) // needed for pvc file access - .build() - .into(), // Needed for secret-operator + security_context: Some(security_context()), ..PodSpec::default() }; @@ -371,18 +367,7 @@ fn spark_job( service_account_name: serviceaccount.metadata.name.clone(), volumes: Some(volumes), image_pull_secrets: spark_application.spark_image_pull_secrets(), - security_context: PodSecurityContextBuilder::new() - .fs_group(1000) - // OpenShift generates UIDs for processes inside Pods. Setting the UID is optional, - // *but* if specified, OpenShift will check that the value is within the - // valid range generated by the SCC (security context constraints) for this Pod. - // On the other hand, it is *required* to set the process UID in KinD, K3S as soon - // as the runAsGroup property is set. - .run_as_user(SPARK_UID) - // Required to access files in mounted volumes on OpenShift. - .run_as_group(0) - .build() - .into(), // Needed for secret-operator + security_context: Some(security_context()), node_selector: spark_application.driver_node_selector(), ..PodSpec::default() }), @@ -448,6 +433,20 @@ fn build_spark_role_serviceaccount( Ok((sa, binding)) } +fn security_context() -> PodSecurityContext { + PodSecurityContextBuilder::new() + .fs_group(1000) + // OpenShift generates UIDs for processes inside Pods. Setting the UID is optional, + // *but* if specified, OpenShift will check that the value is within the + // valid range generated by the SCC (security context constraints) for this Pod. + // On the other hand, it is *required* to set the process UID in KinD, K3S as soon + // as the runAsGroup property is set. + .run_as_user(SPARK_UID) + // Required to access files in mounted volumes on OpenShift. + .run_as_group(0) + .build() +} + pub fn error_policy(_error: &Error, _ctx: Arc) -> Action { Action::requeue(Duration::from_secs(5)) } From d1c1a877c8375ed1d5ace262ab8c61576084c884 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 31 Aug 2022 15:57:34 +0200 Subject: [PATCH 21/22] Wait for the Minio deployment to be ready. And add runAsUser to Jobs preapring dependency volumes. --- .../kuttl/pyspark-ny-public-s3-image/00-assert.yaml | 6 ++++-- .../kuttl/pyspark-ny-public-s3-image/02-deps-volume.yaml | 1 + tests/templates/kuttl/pyspark-ny-public-s3/00-assert.yaml | 6 ++++-- .../kuttl/pyspark-ny-public-s3/02-deps-volume.yaml | 1 + tests/templates/kuttl/spark-ny-public-s3/00-assert.yaml | 6 ++++-- .../templates/kuttl/spark-ny-public-s3/02-deps-volume.yaml | 1 + tests/templates/kuttl/spark-pi-private-s3/00-assert.yaml | 6 ++++-- .../templates/kuttl/spark-pi-private-s3/02-deps-volume.yaml | 1 + tests/templates/kuttl/spark-pi-public-s3/00-assert.yaml | 6 ++++-- .../templates/kuttl/spark-pi-public-s3/02-deps-volume.yaml | 1 + 10 files changed, 25 insertions(+), 10 deletions(-) diff --git a/tests/templates/kuttl/pyspark-ny-public-s3-image/00-assert.yaml b/tests/templates/kuttl/pyspark-ny-public-s3-image/00-assert.yaml index f7295267..4a33d6c0 100644 --- a/tests/templates/kuttl/pyspark-ny-public-s3-image/00-assert.yaml +++ b/tests/templates/kuttl/pyspark-ny-public-s3-image/00-assert.yaml @@ -5,10 +5,12 @@ metadata: name: minio timeout: 900 --- -apiVersion: v1 -kind: Service +apiVersion: apps/v1 +kind: Deployment metadata: name: test-minio +status: + readyReplicas: 1 --- apiVersion: v1 kind: Pod diff --git a/tests/templates/kuttl/pyspark-ny-public-s3-image/02-deps-volume.yaml b/tests/templates/kuttl/pyspark-ny-public-s3-image/02-deps-volume.yaml index 2f5ba82b..97b8d49a 100644 --- a/tests/templates/kuttl/pyspark-ny-public-s3-image/02-deps-volume.yaml +++ b/tests/templates/kuttl/pyspark-ny-public-s3-image/02-deps-volume.yaml @@ -22,6 +22,7 @@ spec: restartPolicy: Never securityContext: runAsGroup: 0 + runAsUser: 1000 serviceAccountName: integration-tests-sa volumes: - name: job-deps diff --git a/tests/templates/kuttl/pyspark-ny-public-s3/00-assert.yaml b/tests/templates/kuttl/pyspark-ny-public-s3/00-assert.yaml index f7295267..4a33d6c0 100644 --- a/tests/templates/kuttl/pyspark-ny-public-s3/00-assert.yaml +++ b/tests/templates/kuttl/pyspark-ny-public-s3/00-assert.yaml @@ -5,10 +5,12 @@ metadata: name: minio timeout: 900 --- -apiVersion: v1 -kind: Service +apiVersion: apps/v1 +kind: Deployment metadata: name: test-minio +status: + readyReplicas: 1 --- apiVersion: v1 kind: Pod diff --git a/tests/templates/kuttl/pyspark-ny-public-s3/02-deps-volume.yaml b/tests/templates/kuttl/pyspark-ny-public-s3/02-deps-volume.yaml index 2f5ba82b..97b8d49a 100644 --- a/tests/templates/kuttl/pyspark-ny-public-s3/02-deps-volume.yaml +++ b/tests/templates/kuttl/pyspark-ny-public-s3/02-deps-volume.yaml @@ -22,6 +22,7 @@ spec: restartPolicy: Never securityContext: runAsGroup: 0 + runAsUser: 1000 serviceAccountName: integration-tests-sa volumes: - name: job-deps diff --git a/tests/templates/kuttl/spark-ny-public-s3/00-assert.yaml b/tests/templates/kuttl/spark-ny-public-s3/00-assert.yaml index f7295267..4a33d6c0 100644 --- a/tests/templates/kuttl/spark-ny-public-s3/00-assert.yaml +++ b/tests/templates/kuttl/spark-ny-public-s3/00-assert.yaml @@ -5,10 +5,12 @@ metadata: name: minio timeout: 900 --- -apiVersion: v1 -kind: Service +apiVersion: apps/v1 +kind: Deployment metadata: name: test-minio +status: + readyReplicas: 1 --- apiVersion: v1 kind: Pod diff --git a/tests/templates/kuttl/spark-ny-public-s3/02-deps-volume.yaml b/tests/templates/kuttl/spark-ny-public-s3/02-deps-volume.yaml index 618c74d9..9cf648a4 100644 --- a/tests/templates/kuttl/spark-ny-public-s3/02-deps-volume.yaml +++ b/tests/templates/kuttl/spark-ny-public-s3/02-deps-volume.yaml @@ -26,6 +26,7 @@ spec: claimName: spark-ny-pvc securityContext: runAsGroup: 0 + runAsUser: 1000 serviceAccountName: integration-tests-sa containers: - name: aws-deps diff --git a/tests/templates/kuttl/spark-pi-private-s3/00-assert.yaml b/tests/templates/kuttl/spark-pi-private-s3/00-assert.yaml index f7295267..4a33d6c0 100644 --- a/tests/templates/kuttl/spark-pi-private-s3/00-assert.yaml +++ b/tests/templates/kuttl/spark-pi-private-s3/00-assert.yaml @@ -5,10 +5,12 @@ metadata: name: minio timeout: 900 --- -apiVersion: v1 -kind: Service +apiVersion: apps/v1 +kind: Deployment metadata: name: test-minio +status: + readyReplicas: 1 --- apiVersion: v1 kind: Pod diff --git a/tests/templates/kuttl/spark-pi-private-s3/02-deps-volume.yaml b/tests/templates/kuttl/spark-pi-private-s3/02-deps-volume.yaml index 848c7a0e..0b97e947 100644 --- a/tests/templates/kuttl/spark-pi-private-s3/02-deps-volume.yaml +++ b/tests/templates/kuttl/spark-pi-private-s3/02-deps-volume.yaml @@ -27,6 +27,7 @@ spec: serviceAccountName: integration-tests-sa securityContext: runAsGroup: 0 + runAsUser: 1000 containers: - name: aws-deps image: docker.stackable.tech/stackable/tools:0.2.0-stackable0 diff --git a/tests/templates/kuttl/spark-pi-public-s3/00-assert.yaml b/tests/templates/kuttl/spark-pi-public-s3/00-assert.yaml index f7295267..4a33d6c0 100644 --- a/tests/templates/kuttl/spark-pi-public-s3/00-assert.yaml +++ b/tests/templates/kuttl/spark-pi-public-s3/00-assert.yaml @@ -5,10 +5,12 @@ metadata: name: minio timeout: 900 --- -apiVersion: v1 -kind: Service +apiVersion: apps/v1 +kind: Deployment metadata: name: test-minio +status: + readyReplicas: 1 --- apiVersion: v1 kind: Pod diff --git a/tests/templates/kuttl/spark-pi-public-s3/02-deps-volume.yaml b/tests/templates/kuttl/spark-pi-public-s3/02-deps-volume.yaml index cf1493fc..15d54839 100644 --- a/tests/templates/kuttl/spark-pi-public-s3/02-deps-volume.yaml +++ b/tests/templates/kuttl/spark-pi-public-s3/02-deps-volume.yaml @@ -27,6 +27,7 @@ spec: serviceAccountName: integration-tests-sa securityContext: runAsGroup: 0 + runAsUser: 1000 containers: - name: aws-deps image: docker.stackable.tech/stackable/tools:0.2.0-stackable0 From 98b5eb125cd75a2a14fa313cb69c8d2c9bd12844 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Fri, 2 Sep 2022 09:40:24 +0200 Subject: [PATCH 22/22] Merge node-selector and spark-examples Moved nodeSelector property to the spark-examples and delete the node-selector test. --- .../kuttl/node-selector/10-assert.yaml | 14 ----------- .../node-selector/10-deploy-spark-app.yaml.j2 | 24 ------------------- .../10-deploy-spark-app.yaml.j2 | 2 ++ 3 files changed, 2 insertions(+), 38 deletions(-) delete mode 100644 tests/templates/kuttl/node-selector/10-assert.yaml delete mode 100644 tests/templates/kuttl/node-selector/10-deploy-spark-app.yaml.j2 diff --git a/tests/templates/kuttl/node-selector/10-assert.yaml b/tests/templates/kuttl/node-selector/10-assert.yaml deleted file mode 100644 index 8ed6eb40..00000000 --- a/tests/templates/kuttl/node-selector/10-assert.yaml +++ /dev/null @@ -1,14 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -metadata: - name: node-selector-assert -timeout: 300 ---- -# The Job starting the whole process -apiVersion: spark.stackable.tech/v1alpha1 -kind: SparkApplication -metadata: - name: node-selector-test -status: - phase: Succeeded diff --git a/tests/templates/kuttl/node-selector/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/node-selector/10-deploy-spark-app.yaml.j2 deleted file mode 100644 index 0ea3db18..00000000 --- a/tests/templates/kuttl/node-selector/10-deploy-spark-app.yaml.j2 +++ /dev/null @@ -1,24 +0,0 @@ ---- -apiVersion: spark.stackable.tech/v1alpha1 -kind: SparkApplication -metadata: - name: node-selector-test -spec: - version: "1.0" - sparkImage: docker.stackable.tech/stackable/spark-k8s:{{ test_scenario['values']['spark'] }}-stackable{{ test_scenario['values']['stackable'] }} - sparkImagePullPolicy: IfNotPresent - mode: cluster - mainClass: org.apache.spark.examples.SparkALS - mainApplicationFile: local:///stackable/spark/examples/jars/spark-examples_2.12-{{ test_scenario['values']['spark'] }}.jar - driver: - cores: 1 - coreLimit: "1200m" - memory: "512m" - nodeSelector: - node: "1" - executor: - cores: 1 - instances: 1 - memory: "512m" - nodeSelector: - kubernetes.io/os: linux diff --git a/tests/templates/kuttl/spark-examples/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/spark-examples/10-deploy-spark-app.yaml.j2 index 71848c1e..b40d9898 100644 --- a/tests/templates/kuttl/spark-examples/10-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/spark-examples/10-deploy-spark-app.yaml.j2 @@ -14,6 +14,8 @@ spec: cores: 1 coreLimit: "1200m" memory: "512m" + nodeSelector: + node: "1" executor: cores: 1 instances: 1