Skip to content

Commit e6cae12

Browse files
sjbermansjberman
authored
CP/DP Split: Openshift support (#3278)
Problem: Now that we have additional pods in the new architecture, we need the proper SecurityContextConstraints for running in Openshift. Solution: Create an SCC for the cert-generator and an SCC for nginx data plane pods on startup. A Role and RoleBinding are created when deploying nginx to link to the SCC.
1 parent de0fdc2 commit e6cae12

22 files changed

+720
-33
lines changed

‎charts/nginx-gateway-fabric/README.md

-6
Original file line numberDiff line numberDiff line change
@@ -115,12 +115,6 @@ To use a NodePort Service instead:
115115
helm install ngf oci://ghcr.io/nginx/charts/nginx-gateway-fabric --create-namespace -n nginx-gateway --set nginx.service.type=NodePort
116116
```
117117

118-
To disable the creation of a Service:
119-
120-
```shell
121-
helm install ngf oci://ghcr.io/nginx/charts/nginx-gateway-fabric --create-namespace -n nginx-gateway --set nginx.service.create=false
122-
```
123-
124118
## Upgrading the Chart
125119

126120
> [!NOTE]

‎charts/nginx-gateway-fabric/README.md.gotmpl

-6
Original file line numberDiff line numberDiff line change
@@ -113,12 +113,6 @@ To use a NodePort Service instead:
113113
helm install ngf oci://ghcr.io/nginx/charts/nginx-gateway-fabric --create-namespace -n nginx-gateway --set nginx.service.type=NodePort
114114
```
115115

116-
To disable the creation of a Service:
117-
118-
```shell
119-
helm install ngf oci://ghcr.io/nginx/charts/nginx-gateway-fabric --create-namespace -n nginx-gateway --set nginx.service.create=false
120-
```
121-
122116
## Upgrading the Chart
123117

124118
> [!NOTE]

‎charts/nginx-gateway-fabric/templates/certs-job.yaml

+44
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,50 @@ subjects:
5656
name: {{ include "nginx-gateway.fullname" . }}-cert-generator
5757
namespace: {{ .Release.Namespace }}
5858
---
59+
{{- if .Capabilities.APIVersions.Has "security.openshift.io/v1/SecurityContextConstraints" }}
60+
kind: SecurityContextConstraints
61+
apiVersion: security.openshift.io/v1
62+
metadata:
63+
name: {{ include "nginx-gateway.scc-name" . }}-cert-generator
64+
labels:
65+
{{- include "nginx-gateway.labels" . | nindent 4 }}
66+
annotations:
67+
"helm.sh/hook-weight": "-1"
68+
"helm.sh/hook": pre-install
69+
allowPrivilegeEscalation: false
70+
allowHostDirVolumePlugin: false
71+
allowHostIPC: false
72+
allowHostNetwork: false
73+
allowHostPID: false
74+
allowHostPorts: false
75+
allowPrivilegedContainer: false
76+
readOnlyRootFilesystem: true
77+
runAsUser:
78+
type: MustRunAsRange
79+
uidRangeMin: 101
80+
uidRangeMax: 101
81+
fsGroup:
82+
type: MustRunAs
83+
ranges:
84+
- min: 1001
85+
max: 1001
86+
supplementalGroups:
87+
type: MustRunAs
88+
ranges:
89+
- min: 1001
90+
max: 1001
91+
seLinuxContext:
92+
type: MustRunAs
93+
seccompProfiles:
94+
- runtime/default
95+
users:
96+
- {{ printf "system:serviceaccount:%s:%s-cert-generator" .Release.Namespace (include "nginx-gateway.fullname" .) }}
97+
requiredDropCapabilities:
98+
- ALL
99+
volumes:
100+
- projected
101+
---
102+
{{- end }}
59103
apiVersion: batch/v1
60104
kind: Job
61105
metadata:

‎charts/nginx-gateway-fabric/templates/clusterrole.yaml

+11
Original file line numberDiff line numberDiff line change
@@ -150,8 +150,19 @@ rules:
150150
- securitycontextconstraints
151151
resourceNames:
152152
- {{ include "nginx-gateway.scc-name" . }}
153+
- {{ include "nginx-gateway.scc-name" . }}-nginx
153154
verbs:
154155
- use
156+
- apiGroups:
157+
- rbac.authorization.k8s.io
158+
resources:
159+
- roles
160+
- rolebindings
161+
verbs:
155162
- create
163+
- update
164+
- delete
165+
- list
166+
- get
156167
- watch
157168
{{- end }}

‎charts/nginx-gateway-fabric/templates/deployment.yaml

+3
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,9 @@ spec:
9393
{{- if .Values.nginxGateway.snippetsFilters.enable }}
9494
- --snippets-filters
9595
{{- end }}
96+
{{- if .Capabilities.APIVersions.Has "security.openshift.io/v1/SecurityContextConstraints" }}
97+
- --nginx-scc={{ include "nginx-gateway.scc-name" . }}-nginx
98+
{{- end}}
9699
env:
97100
- name: POD_NAMESPACE
98101
valueFrom:

‎charts/nginx-gateway-fabric/templates/scc.yaml

+43-1
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
1-
# TODO(sberman): will need an SCC for nginx ServiceAccounts as well.
21
{{- if .Capabilities.APIVersions.Has "security.openshift.io/v1/SecurityContextConstraints" }}
32
kind: SecurityContextConstraints
43
apiVersion: security.openshift.io/v1
54
metadata:
65
name: {{ include "nginx-gateway.scc-name" . }}
6+
labels:
7+
{{- include "nginx-gateway.labels" . | nindent 4 }}
78
allowPrivilegeEscalation: false
89
allowHostDirVolumePlugin: false
910
allowHostIPC: false
@@ -36,4 +37,45 @@ requiredDropCapabilities:
3637
- ALL
3738
volumes:
3839
- secret
40+
---
41+
kind: SecurityContextConstraints
42+
apiVersion: security.openshift.io/v1
43+
metadata:
44+
name: {{ include "nginx-gateway.scc-name" . }}-nginx
45+
labels:
46+
{{- include "nginx-gateway.labels" . | nindent 4 }}
47+
allowHostDirVolumePlugin: false
48+
allowHostIPC: false
49+
allowHostNetwork: false
50+
allowHostPID: false
51+
allowHostPorts: false
52+
allowPrivilegedContainer: false
53+
readOnlyRootFilesystem: true
54+
runAsUser:
55+
type: MustRunAsRange
56+
uidRangeMin: 101
57+
uidRangeMax: 101
58+
fsGroup:
59+
type: MustRunAs
60+
ranges:
61+
- min: 1001
62+
max: 1001
63+
supplementalGroups:
64+
type: MustRunAs
65+
ranges:
66+
- min: 1001
67+
max: 1001
68+
seLinuxContext:
69+
type: MustRunAs
70+
seccompProfiles:
71+
- runtime/default
72+
allowedCapabilities:
73+
- NET_BIND_SERVICE
74+
requiredDropCapabilities:
75+
- ALL
76+
volumes:
77+
- emptyDir
78+
- secret
79+
- configMap
80+
- projected
3981
{{- end }}

‎cmd/gateway/commands.go

+12
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ func createControllerCommand() *cobra.Command {
8181
usageReportClientSSLSecretFlag = "usage-report-client-ssl-secret" //nolint:gosec // not credentials
8282
usageReportCASecretFlag = "usage-report-ca-secret" //nolint:gosec // not credentials
8383
snippetsFiltersFlag = "snippets-filters"
84+
nginxSCCFlag = "nginx-scc"
8485
)
8586

8687
// flag values
@@ -105,6 +106,9 @@ func createControllerCommand() *cobra.Command {
105106
validator: validateResourceName,
106107
value: agentTLSSecret,
107108
}
109+
nginxSCCName = stringValidatingValue{
110+
validator: validateResourceName,
111+
}
108112
disableMetrics bool
109113
metricsSecure bool
110114
metricsListenPort = intValidatingValue{
@@ -264,6 +268,7 @@ func createControllerCommand() *cobra.Command {
264268
SnippetsFilters: snippetsFilters,
265269
NginxDockerSecretNames: nginxDockerSecrets.values,
266270
AgentTLSSecretName: agentTLSSecretName.value,
271+
NGINXSCCName: nginxSCCName.value,
267272
}
268273

269274
if err := static.StartManager(conf); err != nil {
@@ -457,6 +462,13 @@ func createControllerCommand() *cobra.Command {
457462
"generated NGINX config for HTTPRoute and GRPCRoute resources.",
458463
)
459464

465+
cmd.Flags().Var(
466+
&nginxSCCName,
467+
nginxSCCFlag,
468+
`The name of the SecurityContextConstraints to be used with the NGINX data plane Pods.`+
469+
` Only applicable in OpenShift.`,
470+
)
471+
460472
return cmd
461473
}
462474

‎cmd/gateway/commands_test.go

+17
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,7 @@ func TestControllerCmdFlagValidation(t *testing.T) {
158158
"--usage-report-ca-secret=ca-secret",
159159
"--usage-report-client-ssl-secret=client-secret",
160160
"--snippets-filters",
161+
"--nginx-scc=nginx-sscc-name",
161162
},
162163
wantErr: false,
163164
},
@@ -445,6 +446,22 @@ func TestControllerCmdFlagValidation(t *testing.T) {
445446
},
446447
wantErr: true,
447448
},
449+
{
450+
name: "nginx-scc is set to empty string",
451+
args: []string{
452+
"--nginx-scc=",
453+
},
454+
wantErr: true,
455+
expectedErrPrefix: `invalid argument "" for "--nginx-scc" flag: must be set`,
456+
},
457+
{
458+
name: "nginx-scc is invalid",
459+
args: []string{
460+
"--nginx-scc=!@#$",
461+
},
462+
wantErr: true,
463+
expectedErrPrefix: `invalid argument "!@#$" for "--nginx-scc" flag: invalid format: `,
464+
},
448465
}
449466

450467
// common flags validation is tested separately

‎deploy/openshift/deploy.yaml

+100
Original file line numberDiff line numberDiff line change
@@ -175,11 +175,22 @@ rules:
175175
- security.openshift.io
176176
resourceNames:
177177
- nginx-gateway-scc
178+
- nginx-gateway-scc-nginx
178179
resources:
179180
- securitycontextconstraints
180181
verbs:
181182
- use
183+
- apiGroups:
184+
- rbac.authorization.k8s.io
185+
resources:
186+
- roles
187+
- rolebindings
188+
verbs:
182189
- create
190+
- update
191+
- delete
192+
- list
193+
- get
183194
- watch
184195
---
185196
apiVersion: rbac.authorization.k8s.io/v1
@@ -272,6 +283,7 @@ spec:
272283
- --metrics-port=9113
273284
- --health-port=8081
274285
- --leader-election-lock-name=nginx-gateway-leader-election
286+
- --nginx-scc=nginx-gateway-scc-nginx
275287
env:
276288
- name: POD_NAMESPACE
277289
valueFrom:
@@ -442,6 +454,10 @@ fsGroup:
442454
type: MustRunAs
443455
kind: SecurityContextConstraints
444456
metadata:
457+
labels:
458+
app.kubernetes.io/instance: nginx-gateway
459+
app.kubernetes.io/name: nginx-gateway
460+
app.kubernetes.io/version: edge
445461
name: nginx-gateway-scc
446462
readOnlyRootFilesystem: true
447463
requiredDropCapabilities:
@@ -463,3 +479,87 @@ users:
463479
- system:serviceaccount:nginx-gateway:nginx-gateway
464480
volumes:
465481
- secret
482+
---
483+
allowHostDirVolumePlugin: false
484+
allowHostIPC: false
485+
allowHostNetwork: false
486+
allowHostPID: false
487+
allowHostPorts: false
488+
allowPrivilegeEscalation: false
489+
allowPrivilegedContainer: false
490+
apiVersion: security.openshift.io/v1
491+
fsGroup:
492+
ranges:
493+
- max: 1001
494+
min: 1001
495+
type: MustRunAs
496+
kind: SecurityContextConstraints
497+
metadata:
498+
labels:
499+
app.kubernetes.io/instance: nginx-gateway
500+
app.kubernetes.io/name: nginx-gateway
501+
app.kubernetes.io/version: edge
502+
name: nginx-gateway-scc-cert-generator
503+
readOnlyRootFilesystem: true
504+
requiredDropCapabilities:
505+
- ALL
506+
runAsUser:
507+
type: MustRunAsRange
508+
uidRangeMax: 101
509+
uidRangeMin: 101
510+
seLinuxContext:
511+
type: MustRunAs
512+
seccompProfiles:
513+
- runtime/default
514+
supplementalGroups:
515+
ranges:
516+
- max: 1001
517+
min: 1001
518+
type: MustRunAs
519+
users:
520+
- system:serviceaccount:nginx-gateway:nginx-gateway-cert-generator
521+
volumes:
522+
- projected
523+
---
524+
allowHostDirVolumePlugin: false
525+
allowHostIPC: false
526+
allowHostNetwork: false
527+
allowHostPID: false
528+
allowHostPorts: false
529+
allowPrivilegedContainer: false
530+
allowedCapabilities:
531+
- NET_BIND_SERVICE
532+
apiVersion: security.openshift.io/v1
533+
fsGroup:
534+
ranges:
535+
- max: 1001
536+
min: 1001
537+
type: MustRunAs
538+
kind: SecurityContextConstraints
539+
metadata:
540+
labels:
541+
app.kubernetes.io/instance: nginx-gateway
542+
app.kubernetes.io/name: nginx-gateway
543+
app.kubernetes.io/version: edge
544+
name: nginx-gateway-scc-nginx
545+
readOnlyRootFilesystem: true
546+
requiredDropCapabilities:
547+
- ALL
548+
runAsUser:
549+
type: MustRunAsRange
550+
uidRangeMax: 101
551+
uidRangeMin: 101
552+
seLinuxContext:
553+
type: MustRunAs
554+
seccompProfiles:
555+
- runtime/default
556+
supplementalGroups:
557+
ranges:
558+
- max: 1001
559+
min: 1001
560+
type: MustRunAs
561+
volumes:
562+
- emptyDir
563+
- secret
564+
- configMap
565+
- projected

‎internal/mode/static/config/config.go

+2
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ type Config struct {
3434
GatewayClassName string
3535
// AgentTLSSecretName is the name of the TLS Secret used by NGINX Agent to communicate with the control plane.
3636
AgentTLSSecretName string
37+
// NGINXSCCName is the name of the SecurityContextConstraints for the NGINX Pods. Only applicable in OpenShift.
38+
NGINXSCCName string
3739
// NginxDockerSecretNames are the names of any Docker registry Secrets for the NGINX container.
3840
NginxDockerSecretNames []string
3941
// LeaderElection contains the configuration for leader election.

‎internal/mode/static/manager.go

+3
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
authv1 "k8s.io/api/authentication/v1"
1515
apiv1 "k8s.io/api/core/v1"
1616
discoveryV1 "k8s.io/api/discovery/v1"
17+
rbacv1 "k8s.io/api/rbac/v1"
1718
apiext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
1819
apierrors "k8s.io/apimachinery/pkg/api/errors"
1920
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -93,6 +94,7 @@ func init() {
9394
utilruntime.Must(apiext.AddToScheme(scheme))
9495
utilruntime.Must(appsv1.AddToScheme(scheme))
9596
utilruntime.Must(authv1.AddToScheme(scheme))
97+
utilruntime.Must(rbacv1.AddToScheme(scheme))
9698
}
9799

98100
func StartManager(cfg config.Config) error {
@@ -216,6 +218,7 @@ func StartManager(cfg config.Config) error {
216218
GatewayPodConfig: &cfg.GatewayPodConfig,
217219
GCName: cfg.GatewayClassName,
218220
AgentTLSSecretName: cfg.AgentTLSSecretName,
221+
NGINXSCCName: cfg.NGINXSCCName,
219222
Plus: cfg.Plus,
220223
NginxDockerSecretNames: cfg.NginxDockerSecretNames,
221224
PlusUsageConfig: &cfg.UsageReportConfig,

0 commit comments

Comments
 (0)