Skip to content

Commit de58a99

Browse files
jimmidysonjimmidyson
committed
feat: Enable unprivileged ports sysctl in containerd config
This enabled pods to run as non-root and bind to privileged ports as long as they have the necessary capability, `CAP_NET_BIND_SERVICE` added. This fixes an issue on AWS when bringing up coredns which binds to port 53 but runs as an unprivileged user. Overall this is a net security improvement for clusters, meaning users can stop giving too many privileged to pods - see kubernetes/kubernetes#102612 for discussion.
1 parent 65dbf35 commit de58a99

File tree

6 files changed

+199
-3
lines changed

6 files changed

+199
-3
lines changed

‎pkg/handlers/generic/mutation/containerdunprivilegedports/files/unprivileged-ports-config.toml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[plugins."io.containerd.grpc.v1.cri"]
2+
enable_unprivileged_ports = true
3+
enable_unprivileged_icmp = true

‎pkg/handlers/generic/mutation/containerdunprivilegedports/inject.go

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
// Copyright 2024 Nutanix. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
package containerdunprivilegedports
4+
5+
import (
6+
"context"
7+
8+
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
9+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
10+
bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
11+
controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
12+
runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
13+
ctrl "sigs.k8s.io/controller-runtime"
14+
ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
15+
16+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation"
17+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches"
18+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches/selectors"
19+
)
20+
21+
type containerdUnprivilegedPortsPatchHandler struct{}
22+
23+
func NewPatch() *containerdUnprivilegedPortsPatchHandler {
24+
return &containerdUnprivilegedPortsPatchHandler{}
25+
}
26+
27+
func (h *containerdUnprivilegedPortsPatchHandler) Mutate(
28+
ctx context.Context,
29+
obj *unstructured.Unstructured,
30+
vars map[string]apiextensionsv1.JSON,
31+
holderRef runtimehooksv1.HolderReference,
32+
_ ctrlclient.ObjectKey,
33+
_ mutation.ClusterGetter,
34+
) error {
35+
log := ctrl.LoggerFrom(ctx).WithValues(
36+
"holderRef", holderRef,
37+
)
38+
39+
unprivilegedPortsConfigDropIn := generateUnprivilegedPortsConfigDropIn()
40+
41+
if err := patches.MutateIfApplicable(
42+
obj, vars, &holderRef, selectors.ControlPlane(), log,
43+
func(obj *controlplanev1.KubeadmControlPlaneTemplate) error {
44+
log.WithValues(
45+
"patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(),
46+
"patchedObjectName", ctrlclient.ObjectKeyFromObject(obj),
47+
).Info("adding containerd unprivileged ports config to control plane kubeadm config spec")
48+
obj.Spec.Template.Spec.KubeadmConfigSpec.Files = append(
49+
obj.Spec.Template.Spec.KubeadmConfigSpec.Files,
50+
unprivilegedPortsConfigDropIn,
51+
)
52+
53+
return nil
54+
}); err != nil {
55+
return err
56+
}
57+
58+
if err := patches.MutateIfApplicable(
59+
obj, vars, &holderRef, selectors.WorkersKubeadmConfigTemplateSelector(), log,
60+
func(obj *bootstrapv1.KubeadmConfigTemplate) error {
61+
log.WithValues(
62+
"patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(),
63+
"patchedObjectName", ctrlclient.ObjectKeyFromObject(obj),
64+
).Info("adding containerd unprivileged ports config to worker node kubeadm config template")
65+
obj.Spec.Template.Spec.Files = append(
66+
obj.Spec.Template.Spec.Files,
67+
unprivilegedPortsConfigDropIn)
68+
69+
return nil
70+
}); err != nil {
71+
return err
72+
}
73+
74+
return nil
75+
}

‎pkg/handlers/generic/mutation/containerdunprivilegedports/inject_test.go

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
// Copyright 2024 Nutanix. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package containerdunprivilegedports
5+
6+
import (
7+
"testing"
8+
9+
. "github.com/onsi/ginkgo/v2"
10+
"github.com/onsi/gomega"
11+
runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
12+
13+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation"
14+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest"
15+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest/request"
16+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/test/helpers"
17+
)
18+
19+
func TestContainerdUnprivilegedPortsPatch(t *testing.T) {
20+
gomega.RegisterFailHandler(Fail)
21+
RunSpecs(t, "Containerd unprivileged ports mutator suite")
22+
}
23+
24+
var _ = Describe("Generate containerd unprivileged ports patches", func() {
25+
// only add aws region patch
26+
patchGenerator := func() mutation.GeneratePatches {
27+
return mutation.NewMetaGeneratePatchesHandler("", helpers.TestEnv.Client, NewPatch()).(mutation.GeneratePatches)
28+
}
29+
30+
testDefs := []capitest.PatchTestDef{
31+
{
32+
Name: "containerd unprivileged ports config added to control plane kubeadm config spec",
33+
RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""),
34+
ExpectedPatchMatchers: []capitest.JSONPatchMatcher{
35+
{
36+
Operation: "add",
37+
Path: "/spec/template/spec/kubeadmConfigSpec/files",
38+
ValueMatcher: gomega.ContainElements(
39+
gomega.HaveKeyWithValue(
40+
"path", unprivilegedPortsConfigDropInFileOnRemote,
41+
),
42+
),
43+
},
44+
},
45+
},
46+
{
47+
Name: "containerd unprivileged ports config added to worker node kubeadm config template",
48+
Vars: []runtimehooksv1.Variable{
49+
capitest.VariableWithValue(
50+
"builtin",
51+
map[string]any{
52+
"machineDeployment": map[string]any{
53+
"class": "*",
54+
},
55+
},
56+
),
57+
},
58+
RequestItem: request.NewKubeadmConfigTemplateRequestItem(""),
59+
ExpectedPatchMatchers: []capitest.JSONPatchMatcher{
60+
{
61+
Operation: "add",
62+
Path: "/spec/template/spec/files",
63+
ValueMatcher: gomega.ContainElements(
64+
gomega.HaveKeyWithValue(
65+
"path", unprivilegedPortsConfigDropInFileOnRemote,
66+
),
67+
),
68+
},
69+
},
70+
},
71+
}
72+
73+
// create test node for each case
74+
for testIdx := range testDefs {
75+
tt := testDefs[testIdx]
76+
It(tt.Name, func() {
77+
capitest.AssertGeneratePatches(
78+
GinkgoT(),
79+
patchGenerator,
80+
&tt,
81+
)
82+
})
83+
}
84+
})

‎pkg/handlers/generic/mutation/containerdunprivilegedports/unprivileged_ports.go

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
// Copyright 2024 Nutanix. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
package containerdunprivilegedports
4+
5+
import (
6+
_ "embed"
7+
"path"
8+
9+
cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
10+
)
11+
12+
const (
13+
// TODO Factor out this constant to a common package.
14+
containerdPatchesDirOnRemote = "/etc/containerd/cre.d"
15+
)
16+
17+
var (
18+
//go:embed files/unprivileged-ports-config.toml
19+
unprivilegedPortsConfigDropIn []byte
20+
unprivilegedPortsConfigDropInFileOnRemote = path.Join(
21+
containerdPatchesDirOnRemote,
22+
"unprivileged-ports-config.toml",
23+
)
24+
)
25+
26+
func generateUnprivilegedPortsConfigDropIn() cabpkv1.File {
27+
return cabpkv1.File{
28+
Path: unprivilegedPortsConfigDropInFileOnRemote,
29+
Content: string(unprivilegedPortsConfigDropIn),
30+
Permissions: "0600",
31+
}
32+
}

‎pkg/handlers/generic/mutation/handlers.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/auditpolicy"
1212
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/containerdapplypatchesandrestart"
1313
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/containerdmetrics"
14+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/containerdunprivilegedports"
1415
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/etcd"
1516
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/extraapiservercertsans"
1617
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/httpproxy"
@@ -33,6 +34,7 @@ func MetaMutators(mgr manager.Manager) []mutation.MetaMutator {
3334
calico.NewPatch(),
3435
users.NewPatch(),
3536
containerdmetrics.NewPatch(),
37+
containerdunprivilegedports.NewPatch(),
3638

3739
// Some patches may have changed containerd configuration.
3840
// We must restart containerd for the configuration to take effect.

‎test/e2e/config/caren.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -184,15 +184,15 @@ variables:
184184
AMI_LOOKUP_ORG: "999867407951"
185185
# To run Nutanix provider tests, set following variables here or as an env var
186186
# # IP/FQDN of Prism Central.
187-
# NUTANIX_ENDPOINT: ""
187+
NUTANIX_ENDPOINT: ""
188188
# # Port of Prism Central. Default: 9440
189189
# NUTANIX_PORT: 9440
190190
# # Disable Prism Central certificate checking. Default: false
191191
# NUTANIX_INSECURE: false
192192
# # Prism Central user
193-
# NUTANIX_USER: ""
193+
NUTANIX_USER: ""
194194
# # Prism Central password
195-
# NUTANIX_PASSWORD: ""
195+
NUTANIX_PASSWORD: ""
196196
# # Host IP to be assigned to the CAPX Kubernetes cluster.
197197
# CONTROL_PLANE_ENDPOINT_IP: ""
198198
# # Port of the CAPX Kubernetes cluster. Default: 6443

0 commit comments

Comments
 (0)