Skip to content

Commit d43f584

Browse files
bors[bot]bors[bot]
authored
Try #126:
2 parents 142ca13 + d1c1a87 commit d43f584

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+772
-369
lines changed

‎CHANGELOG.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,11 @@ All notable changes to this project will be documented in this file.
1414

1515
- Add missing role to read S3Connection and S3Bucket objects ([#112]).
1616
- Update annotation due to update to rust version ([#114]).
17+
- Update RBAC properties for OpenShift compatibility ([#126]).
1718

1819
[#112]: https://github.com/stackabletech/spark-k8s-operator/pull/112
1920
[#114]: https://github.com/stackabletech/spark-k8s-operator/pull/114
21+
[#126]: https://github.com/stackabletech/spark-k8s-operator/pull/126
2022

2123
## [0.4.0] - 2022-08-03
2224

‎deploy/helm/spark-k8s-operator/templates/roles.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,3 +104,11 @@ rules:
104104
- get
105105
- list
106106
- watch
107+
- apiGroups:
108+
- rbac.authorization.k8s.io
109+
resources:
110+
- clusterroles
111+
verbs:
112+
- bind
113+
resourceNames:
114+
- {{ include "operator.name" . }}-clusterrole

‎deploy/helm/spark-k8s-operator/templates/spark-clusterrole.yaml

Lines changed: 72 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,82 @@
1+
{{ if .Capabilities.APIVersions.Has "security.openshift.io/v1" }}
2+
---
3+
apiVersion: security.openshift.io/v1
4+
kind: SecurityContextConstraints
5+
metadata:
6+
name: spark-k8s-scc
7+
annotations:
8+
kubernetes.io/description: |-
9+
This resource is derived from hostmount-anyuid. It provides all the features of the
10+
restricted SCC but allows host mounts and any UID by a pod. This is primarily
11+
used by the persistent volume recycler. WARNING: this SCC allows host file
12+
system access as any UID, including UID 0. Grant with caution.
13+
release.openshift.io/create-only: "true"
14+
allowHostDirVolumePlugin: true
15+
allowHostIPC: false
16+
allowHostNetwork: false
17+
allowHostPID: false
18+
allowHostPorts: false
19+
allowPrivilegeEscalation: true
20+
allowPrivilegedContainer: false
21+
allowedCapabilities: null
22+
defaultAddCapabilities: null
23+
fsGroup:
24+
type: RunAsAny
25+
groups: []
26+
priority: null
27+
readOnlyRootFilesystem: false
28+
runAsUser:
29+
type: RunAsAny
30+
seLinuxContext:
31+
type: MustRunAs
32+
supplementalGroups:
33+
type: RunAsAny
34+
volumes:
35+
- configMap
36+
- downwardAPI
37+
- emptyDir
38+
- hostPath
39+
- nfs
40+
- persistentVolumeClaim
41+
- projected
42+
- secret
43+
- ephemeral
44+
{{ end }}
145
---
246
apiVersion: rbac.authorization.k8s.io/v1
347
kind: ClusterRole
448
metadata:
5-
name: spark-driver-edit-role
49+
name: {{ include "operator.name" . }}-clusterrole
650
rules:
7-
- apiGroups: [""]
8-
resources: ["pods", "services", "configmaps"]
9-
verbs: ["get", "list", "watch", "create", "delete"]
1051
- apiGroups:
1152
- ""
1253
resources:
13-
- persistentvolumeclaims
54+
- configmaps
55+
- pods
56+
- secrets
57+
- serviceaccounts
58+
- services
1459
verbs:
60+
- create
61+
- delete
62+
- get
1563
- list
64+
- patch
65+
- update
66+
- watch
67+
- apiGroups:
68+
- events.k8s.io
69+
resources:
70+
- events
71+
verbs:
72+
- create
73+
{{ if .Capabilities.APIVersions.Has "security.openshift.io/v1" }}
74+
- apiGroups:
75+
- security.openshift.io
76+
resources:
77+
- securitycontextconstraints
78+
resourceNames:
79+
- spark-k8s-scc
80+
verbs:
81+
- use
82+
{{ end }}

‎deploy/helm/spark-k8s-operator/values.yaml

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,22 @@ podAnnotations: {}
2121

2222
podSecurityContext: {}
2323
# fsGroup: 2000
24-
25-
securityContext: {}
26-
# capabilities:
27-
# drop:
28-
# - ALL
29-
# readOnlyRootFilesystem: true
30-
# runAsNonRoot: true
31-
# runAsUser: 1000
24+
#
25+
# OpenShift 4.11 replaces the PodSecurityPolicy with a new pod security
26+
# admission mechanism as described in this blog post [1].
27+
# This requires Pods to explicitely specify the securityContext.
28+
#
29+
# [1]: https://cloud.redhat.com/blog/pod-security-admission-in-openshift-4.11
30+
securityContext:
31+
capabilities:
32+
drop:
33+
- ALL
34+
readOnlyRootFilesystem: false
35+
allowPrivilegeEscalation: false
36+
seccompProfile:
37+
type: RuntimeDefault
38+
runAsNonRoot: true
39+
runAsUser: 1000
3240

3341
resources: {}
3442
# We usually recommend not to specify default resources and to leave this as a conscious

‎deploy/manifests/deployment.yaml

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,16 @@ spec:
2525
securityContext: {}
2626
containers:
2727
- name: spark-k8s-operator
28-
securityContext: {}
28+
securityContext:
29+
allowPrivilegeEscalation: false
30+
capabilities:
31+
drop:
32+
- ALL
33+
readOnlyRootFilesystem: false
34+
runAsNonRoot: true
35+
runAsUser: 1000
36+
seccompProfile:
37+
type: RuntimeDefault
2938
image: "docker.stackable.tech/stackable/spark-k8s-operator:0.5.0-nightly"
3039
imagePullPolicy: IfNotPresent
3140
resources: {}

‎deploy/manifests/roles.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,3 +104,11 @@ rules:
104104
- get
105105
- list
106106
- watch
107+
- apiGroups:
108+
- rbac.authorization.k8s.io
109+
resources:
110+
- clusterroles
111+
verbs:
112+
- bind
113+
resourceNames:
114+
- spark-k8s-clusterrole

‎deploy/manifests/spark-clusterrole.yaml

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,27 @@
22
apiVersion: rbac.authorization.k8s.io/v1
33
kind: ClusterRole
44
metadata:
5-
name: spark-driver-edit-role
5+
name: spark-k8s-clusterrole
66
rules:
7-
- apiGroups: [""]
8-
resources: ["pods", "services", "configmaps"]
9-
verbs: ["get", "list", "watch", "create", "delete"]
107
- apiGroups:
118
- ""
129
resources:
13-
- persistentvolumeclaims
10+
- configmaps
11+
- pods
12+
- secrets
13+
- serviceaccounts
14+
- services
1415
verbs:
16+
- create
17+
- delete
18+
- get
1519
- list
20+
- patch
21+
- update
22+
- watch
23+
- apiGroups:
24+
- events.k8s.io
25+
resources:
26+
- events
27+
verbs:
28+
- create

‎docs/modules/ROOT/pages/rbac.adoc

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,6 @@ The https://spark.apache.org/docs/latest/running-on-kubernetes.html#rbac[Spark-K
66

77
However, to add security, each `spark-submit` job launched by the spark-k8s operator will be assigned its own service account.
88

9-
When the spark-k8s operator is installed via helm, a cluster role named `spark-driver-edit-role` is created with pre-defined permissions.
9+
When the spark-k8s operator is installed via Helm, a cluster role named `spark-k8s-clusterrole` is created with pre-defined permissions.
1010

11-
When a new Spark application is submitted, the operator creates a new service account with the same name as the application and binds this account to the cluster role `spark-driver-edit-role` created by helm.
11+
When a new Spark application is submitted, the operator creates a new service account with the same name as the application and binds this account to the cluster role `spark-k8s-clusterrole` created by Helm.

‎rust/crd/src/constants.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,3 +20,5 @@ pub const CONTAINER_NAME_EXECUTOR: &str = "spark-executor";
2020
pub const ACCESS_KEY_ID: &str = "accessKeyId";
2121
pub const SECRET_ACCESS_KEY: &str = "secretAccessKey";
2222
pub const S3_SECRET_DIR_NAME: &str = "/stackable/secrets";
23+
24+
pub const SPARK_UID: i64 = 1000;

‎rust/operator-binary/src/spark_k8s_controller.rs

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ use stackable_operator::commons::s3::InlinedS3BucketSpec;
77
use stackable_operator::commons::tls::{CaCert, TlsVerification};
88
use stackable_operator::k8s_openapi::api::batch::v1::{Job, JobSpec};
99
use stackable_operator::k8s_openapi::api::core::v1::{
10-
ConfigMap, ConfigMapVolumeSource, Container, EnvVar, Pod, PodSpec, PodTemplateSpec,
11-
ServiceAccount, Volume, VolumeMount,
10+
ConfigMap, ConfigMapVolumeSource, Container, EnvVar, Pod, PodSecurityContext, PodSpec,
11+
PodTemplateSpec, ServiceAccount, Volume, VolumeMount,
1212
};
1313
use stackable_operator::k8s_openapi::api::rbac::v1::{ClusterRole, RoleBinding, RoleRef, Subject};
1414
use stackable_operator::k8s_openapi::Resource;
@@ -21,7 +21,7 @@ use std::{sync::Arc, time::Duration};
2121
use strum::{EnumDiscriminants, IntoStaticStr};
2222

2323
const FIELD_MANAGER_SCOPE: &str = "sparkapplication";
24-
const SPARK_CLUSTER_ROLE: &str = "spark-driver-edit-role";
24+
const SPARK_CLUSTER_ROLE: &str = "spark-k8s-clusterrole";
2525

2626
pub struct Ctx {
2727
pub client: stackable_operator::client::Client,
@@ -234,10 +234,7 @@ fn pod_template(
234234
let mut pod_spec = PodSpec {
235235
containers: vec![cb.build()],
236236
volumes: Some(volumes.to_vec()),
237-
security_context: PodSecurityContextBuilder::new()
238-
.fs_group(1000)
239-
.build()
240-
.into(), // Needed for secret-operator
237+
security_context: Some(security_context()),
241238
..PodSpec::default()
242239
};
243240

@@ -370,10 +367,7 @@ fn spark_job(
370367
service_account_name: serviceaccount.metadata.name.clone(),
371368
volumes: Some(volumes),
372369
image_pull_secrets: spark_application.spark_image_pull_secrets(),
373-
security_context: PodSecurityContextBuilder::new()
374-
.fs_group(1000)
375-
.build()
376-
.into(), // Needed for secret-operator
370+
security_context: Some(security_context()),
377371
node_selector: spark_application.driver_node_selector(),
378372
..PodSpec::default()
379373
}),
@@ -439,6 +433,20 @@ fn build_spark_role_serviceaccount(
439433
Ok((sa, binding))
440434
}
441435

436+
fn security_context() -> PodSecurityContext {
437+
PodSecurityContextBuilder::new()
438+
.fs_group(1000)
439+
// OpenShift generates UIDs for processes inside Pods. Setting the UID is optional,
440+
// *but* if specified, OpenShift will check that the value is within the
441+
// valid range generated by the SCC (security context constraints) for this Pod.
442+
// On the other hand, it is *required* to set the process UID in KinD, K3S as soon
443+
// as the runAsGroup property is set.
444+
.run_as_user(SPARK_UID)
445+
// Required to access files in mounted volumes on OpenShift.
446+
.run_as_group(0)
447+
.build()
448+
}
449+
442450
pub fn error_policy(_error: &Error, _ctx: Arc<Ctx>) -> Action {
443451
Action::requeue(Duration::from_secs(5))
444452
}

‎tests/templates/kuttl/pyspark-ny-public-s3-image/00-assert.yaml

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,17 +5,23 @@ metadata:
55
name: minio
66
timeout: 900
77
---
8-
apiVersion: v1
9-
kind: Service
8+
apiVersion: apps/v1
9+
kind: Deployment
1010
metadata:
1111
name: test-minio
12-
labels:
13-
app: minio
12+
status:
13+
readyReplicas: 1
1414
---
15-
apiVersion: apps/v1
16-
kind: StatefulSet
15+
apiVersion: v1
16+
kind: Pod
1717
metadata:
18-
name: minio-mc
18+
name: minio-client
19+
labels:
20+
app: minio-client
1921
status:
20-
readyReplicas: 1
21-
replicas: 1
22+
phase: Running
23+
---
24+
apiVersion: v1
25+
kind: ServiceAccount
26+
metadata:
27+
name: integration-tests-sa

‎tests/templates/kuttl/pyspark-ny-public-s3-image/00-s3-upload-container.yaml

Lines changed: 0 additions & 36 deletions
This file was deleted.

‎tests/templates/kuttl/pyspark-ny-public-s3-image/00-serviceaccount.yaml.j2

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
---
2+
kind: Role
3+
apiVersion: rbac.authorization.k8s.io/v1
4+
metadata:
5+
name: use-integration-tests-scc
6+
rules:
7+
{% if test_scenario['values']['openshift'] == "true" %}
8+
- apiGroups: ["security.openshift.io"]
9+
resources: ["securitycontextconstraints"]
10+
resourceNames: ["privileged"]
11+
verbs: ["use"]
12+
{% endif %}
13+
---
14+
apiVersion: v1
15+
kind: ServiceAccount
16+
metadata:
17+
name: integration-tests-sa
18+
---
19+
kind: RoleBinding
20+
apiVersion: rbac.authorization.k8s.io/v1
21+
metadata:
22+
name: use-integration-tests-scc
23+
subjects:
24+
- kind: ServiceAccount
25+
name: integration-tests-sa
26+
roleRef:
27+
kind: Role
28+
name: use-integration-tests-scc
29+
apiGroup: rbac.authorization.k8s.io

0 commit comments

Comments
 (0)