Skip to content

Commit f128d84

Browse files
jimmidysonjimmidyson
authored
feat: Enable unprivileged ports sysctl in containerd config (#645)
This enabled pods to run as non-root and bind to privileged ports as long as they have the necessary capability, `CAP_NET_BIND_SERVICE` added. This fixes an issue on AWS when bringing up coredns which binds to port 53 but runs as an unprivileged user. Overall this is a net security improvement for clusters, meaning users can stop giving too many privileged to pods - see kubernetes/kubernetes#102612 for discussion. Depends on #644.
1 parent f680979 commit f128d84

File tree

5 files changed

+191
-0
lines changed

5 files changed

+191
-0
lines changed

‎pkg/handlers/generic/mutation/containerdunprivilegedports/files/unprivileged-ports-config.toml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[plugins."io.containerd.grpc.v1.cri"]
2+
enable_unprivileged_ports = true
3+
enable_unprivileged_icmp = true

‎pkg/handlers/generic/mutation/containerdunprivilegedports/inject.go

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
// Copyright 2024 Nutanix. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
package containerdunprivilegedports
4+
5+
import (
6+
"context"
7+
8+
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
9+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
10+
bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
11+
controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
12+
runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
13+
ctrl "sigs.k8s.io/controller-runtime"
14+
ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
15+
16+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation"
17+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches"
18+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches/selectors"
19+
)
20+
21+
type containerdUnprivilegedPortsPatchHandler struct{}
22+
23+
func NewPatch() *containerdUnprivilegedPortsPatchHandler {
24+
return &containerdUnprivilegedPortsPatchHandler{}
25+
}
26+
27+
func (h *containerdUnprivilegedPortsPatchHandler) Mutate(
28+
ctx context.Context,
29+
obj *unstructured.Unstructured,
30+
vars map[string]apiextensionsv1.JSON,
31+
holderRef runtimehooksv1.HolderReference,
32+
_ ctrlclient.ObjectKey,
33+
_ mutation.ClusterGetter,
34+
) error {
35+
log := ctrl.LoggerFrom(ctx).WithValues(
36+
"holderRef", holderRef,
37+
)
38+
39+
unprivilegedPortsConfigDropIn := generateUnprivilegedPortsConfigDropIn()
40+
41+
if err := patches.MutateIfApplicable(
42+
obj, vars, &holderRef, selectors.ControlPlane(), log,
43+
func(obj *controlplanev1.KubeadmControlPlaneTemplate) error {
44+
log.WithValues(
45+
"patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(),
46+
"patchedObjectName", ctrlclient.ObjectKeyFromObject(obj),
47+
).Info("adding containerd unprivileged ports config to control plane kubeadm config spec")
48+
obj.Spec.Template.Spec.KubeadmConfigSpec.Files = append(
49+
obj.Spec.Template.Spec.KubeadmConfigSpec.Files,
50+
unprivilegedPortsConfigDropIn,
51+
)
52+
53+
return nil
54+
}); err != nil {
55+
return err
56+
}
57+
58+
if err := patches.MutateIfApplicable(
59+
obj, vars, &holderRef, selectors.WorkersKubeadmConfigTemplateSelector(), log,
60+
func(obj *bootstrapv1.KubeadmConfigTemplate) error {
61+
log.WithValues(
62+
"patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(),
63+
"patchedObjectName", ctrlclient.ObjectKeyFromObject(obj),
64+
).Info("adding containerd unprivileged ports config to worker node kubeadm config template")
65+
obj.Spec.Template.Spec.Files = append(
66+
obj.Spec.Template.Spec.Files,
67+
unprivilegedPortsConfigDropIn)
68+
69+
return nil
70+
}); err != nil {
71+
return err
72+
}
73+
74+
return nil
75+
}

‎pkg/handlers/generic/mutation/containerdunprivilegedports/inject_test.go

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
// Copyright 2024 Nutanix. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package containerdunprivilegedports
5+
6+
import (
7+
"testing"
8+
9+
. "github.com/onsi/ginkgo/v2"
10+
"github.com/onsi/gomega"
11+
runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
12+
13+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation"
14+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest"
15+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest/request"
16+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/test/helpers"
17+
)
18+
19+
func TestContainerdUnprivilegedPortsPatch(t *testing.T) {
20+
gomega.RegisterFailHandler(Fail)
21+
RunSpecs(t, "Containerd unprivileged ports mutator suite")
22+
}
23+
24+
var _ = Describe("Generate containerd unprivileged ports patches", func() {
25+
// only add aws region patch
26+
patchGenerator := func() mutation.GeneratePatches {
27+
return mutation.NewMetaGeneratePatchesHandler("", helpers.TestEnv.Client, NewPatch()).(mutation.GeneratePatches)
28+
}
29+
30+
testDefs := []capitest.PatchTestDef{
31+
{
32+
Name: "containerd unprivileged ports config added to control plane kubeadm config spec",
33+
RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""),
34+
ExpectedPatchMatchers: []capitest.JSONPatchMatcher{
35+
{
36+
Operation: "add",
37+
Path: "/spec/template/spec/kubeadmConfigSpec/files",
38+
ValueMatcher: gomega.ContainElements(
39+
gomega.HaveKeyWithValue(
40+
"path", unprivilegedPortsConfigDropInFileOnRemote,
41+
),
42+
),
43+
},
44+
},
45+
},
46+
{
47+
Name: "containerd unprivileged ports config added to worker node kubeadm config template",
48+
Vars: []runtimehooksv1.Variable{
49+
capitest.VariableWithValue(
50+
"builtin",
51+
map[string]any{
52+
"machineDeployment": map[string]any{
53+
"class": "*",
54+
},
55+
},
56+
),
57+
},
58+
RequestItem: request.NewKubeadmConfigTemplateRequestItem(""),
59+
ExpectedPatchMatchers: []capitest.JSONPatchMatcher{
60+
{
61+
Operation: "add",
62+
Path: "/spec/template/spec/files",
63+
ValueMatcher: gomega.ContainElements(
64+
gomega.HaveKeyWithValue(
65+
"path", unprivilegedPortsConfigDropInFileOnRemote,
66+
),
67+
),
68+
},
69+
},
70+
},
71+
}
72+
73+
// create test node for each case
74+
for testIdx := range testDefs {
75+
tt := testDefs[testIdx]
76+
It(tt.Name, func() {
77+
capitest.AssertGeneratePatches(
78+
GinkgoT(),
79+
patchGenerator,
80+
&tt,
81+
)
82+
})
83+
}
84+
})

‎pkg/handlers/generic/mutation/containerdunprivilegedports/unprivileged_ports.go

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
// Copyright 2024 Nutanix. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
package containerdunprivilegedports
4+
5+
import (
6+
_ "embed"
7+
8+
cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
9+
10+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/common"
11+
)
12+
13+
var (
14+
//go:embed files/unprivileged-ports-config.toml
15+
unprivilegedPortsConfigDropIn []byte
16+
unprivilegedPortsConfigDropInFileOnRemote = common.ContainerdPatchPathOnRemote(
17+
"unprivileged-ports-config.toml",
18+
)
19+
)
20+
21+
func generateUnprivilegedPortsConfigDropIn() cabpkv1.File {
22+
return cabpkv1.File{
23+
Path: unprivilegedPortsConfigDropInFileOnRemote,
24+
Content: string(unprivilegedPortsConfigDropIn),
25+
Permissions: "0600",
26+
}
27+
}

‎pkg/handlers/generic/mutation/handlers.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/auditpolicy"
1212
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/containerdapplypatchesandrestart"
1313
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/containerdmetrics"
14+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/containerdunprivilegedports"
1415
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/etcd"
1516
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/extraapiservercertsans"
1617
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/httpproxy"
@@ -33,6 +34,7 @@ func MetaMutators(mgr manager.Manager) []mutation.MetaMutator {
3334
calico.NewPatch(),
3435
users.NewPatch(),
3536
containerdmetrics.NewPatch(),
37+
containerdunprivilegedports.NewPatch(),
3638

3739
// Some patches may have changed containerd configuration.
3840
// We write the configuration changes to disk, and must run a command

0 commit comments

Comments
 (0)