diff --git a/charts/capi-runtime-extensions/templates/role.yaml b/charts/capi-runtime-extensions/templates/role.yaml index eb212dbee..ba9a58a75 100644 --- a/charts/capi-runtime-extensions/templates/role.yaml +++ b/charts/capi-runtime-extensions/templates/role.yaml @@ -30,3 +30,11 @@ rules: - patch - update - watch +- apiGroups: + - cluster.x-k8s.io + resources: + - clusters + verbs: + - get + - list + - watch diff --git a/cmd/capi-runtime-extensions/main.go b/cmd/capi-runtime-extensions/main.go index 0abb23aa3..b88f516b3 100644 --- a/cmd/capi-runtime-extensions/main.go +++ b/cmd/capi-runtime-extensions/main.go @@ -79,7 +79,7 @@ func main() { servicelbgc.New(client), calico.New(client, calicoCNIConfig), httpproxy.NewVariable(), - httpproxy.NewPatch(), + httpproxy.NewPatch(client), extraapiservercertsans.NewVariable(), extraapiservercertsans.NewPatch(), ) diff --git a/docs/content/http-proxy.md b/docs/content/http-proxy.md index 1e7c9b753..5af21e0da 100644 --- a/docs/content/http-proxy.md +++ b/docs/content/http-proxy.md @@ -32,14 +32,17 @@ spec: topology: variables: - name: proxy - value: + values: http: http://example.com - https: https://example.com - no: + https: http://example.com + additionalNo: - no-proxy-1.example.com - no-proxy-2.example.com ``` +The `additionalNo` list will be added to default pre-calculated values that apply on k8s networking +`localhost,127.0.0.1,,,kubernetes,kubernetes.default,.svc,.svc.cluster.local`. + Applying this configuration will result in new bootstrap files on the `KubeadmControlPlaneTemplate` and `KubeadmConfigTemplate`. diff --git a/pkg/handlers/httpproxy/inject.go b/pkg/handlers/httpproxy/inject.go index 55a90f701..d360ee28f 100644 --- a/pkg/handlers/httpproxy/inject.go +++ b/pkg/handlers/httpproxy/inject.go @@ -1,20 +1,27 @@ // Copyright 2023 D2iQ, Inc. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters,verbs=watch;list;get + package httpproxy import ( "context" + "errors" + "fmt" + "strings" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/types" + capiv1 "sigs.k8s.io/cluster-api/api/v1beta1" bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" "sigs.k8s.io/cluster-api/exp/runtime/topologymutation" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/handlers" "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/handlers/mutation" @@ -30,6 +37,7 @@ const ( type httpProxyPatchHandler struct { decoder runtime.Decoder + client ctrlclient.Reader } var ( @@ -37,7 +45,7 @@ var ( _ mutation.GeneratePatches = &httpProxyPatchHandler{} ) -func NewPatch() *httpProxyPatchHandler { +func NewPatch(cl ctrlclient.Reader) *httpProxyPatchHandler { scheme := runtime.NewScheme() _ = bootstrapv1.AddToScheme(scheme) _ = controlplanev1.AddToScheme(scheme) @@ -46,6 +54,7 @@ func NewPatch() *httpProxyPatchHandler { controlplanev1.GroupVersion, bootstrapv1.GroupVersion, ), + client: cl, } } @@ -58,6 +67,12 @@ func (h *httpProxyPatchHandler) GeneratePatches( req *runtimehooksv1.GeneratePatchesRequest, resp *runtimehooksv1.GeneratePatchesResponse, ) { + log := ctrl.LoggerFrom(ctx) + noProxy, err := h.detectNoProxy(ctx, req) + if err != nil { + log.Error(err, "failed to resolve no proxy value") + } + topologymutation.WalkTemplates( ctx, h.decoder, @@ -69,7 +84,7 @@ func (h *httpProxyPatchHandler) GeneratePatches( vars map[string]apiextensionsv1.JSON, holderRef runtimehooksv1.HolderReference, ) error { - log := ctrl.LoggerFrom(ctx).WithValues( + log = log.WithValues( "holderRef", holderRef, ) @@ -92,11 +107,11 @@ func (h *httpProxyPatchHandler) GeneratePatches( func(obj *controlplanev1.KubeadmControlPlaneTemplate) error { log.WithValues( "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), - "patchedObjectName", client.ObjectKeyFromObject(obj), + "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), ).Info("adding files to control plane kubeadm config spec") obj.Spec.Template.Spec.KubeadmConfigSpec.Files = append( obj.Spec.Template.Spec.KubeadmConfigSpec.Files, - generateSystemdFiles(httpProxyVariable)..., + generateSystemdFiles(httpProxyVariable, noProxy)..., ) return nil }); err != nil { @@ -108,11 +123,11 @@ func (h *httpProxyPatchHandler) GeneratePatches( func(obj *bootstrapv1.KubeadmConfigTemplate) error { log.WithValues( "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), - "patchedObjectName", client.ObjectKeyFromObject(obj), + "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), ).Info("adding files to worker node kubeadm config template") obj.Spec.Template.Spec.Files = append( obj.Spec.Template.Spec.Files, - generateSystemdFiles(httpProxyVariable)..., + generateSystemdFiles(httpProxyVariable, noProxy)..., ) return nil }); err != nil { @@ -123,3 +138,70 @@ func (h *httpProxyPatchHandler) GeneratePatches( }, ) } + +func (h *httpProxyPatchHandler) detectNoProxy( + ctx context.Context, + req *runtimehooksv1.GeneratePatchesRequest, +) ([]string, error) { + clusterKey := types.NamespacedName{} + + for i := range req.Items { + item := req.Items[i] + if item.HolderReference.Kind == "Cluster" && + item.HolderReference.APIVersion == capiv1.GroupVersion.String() { + clusterKey.Name = item.HolderReference.Name + clusterKey.Namespace = item.HolderReference.Namespace + } + } + + if clusterKey.Name == "" { + return nil, errors.New("failed to detect cluster name from GeneratePatch request") + } + + cluster := &capiv1.Cluster{} + if err := h.client.Get(ctx, clusterKey, cluster); err != nil { + return nil, err + } + + return generateNoProxy(cluster), nil +} + +// generateNoProxy creates default NO_PROXY values that should be applied on cluster +// in any environment and are preventing the use of proxy for cluster internal +// networking. +func generateNoProxy(cluster *capiv1.Cluster) []string { + noProxy := []string{ + "localhost", + "127.0.0.1", + } + + if cluster.Spec.ClusterNetwork != nil && + cluster.Spec.ClusterNetwork.Pods != nil { + noProxy = append(noProxy, cluster.Spec.ClusterNetwork.Pods.CIDRBlocks...) + } + + if cluster.Spec.ClusterNetwork != nil && + cluster.Spec.ClusterNetwork.Services != nil { + noProxy = append(noProxy, cluster.Spec.ClusterNetwork.Services.CIDRBlocks...) + } + + serviceDomain := "cluster.local" + if cluster.Spec.ClusterNetwork != nil && + cluster.Spec.ClusterNetwork.ServiceDomain != "" { + serviceDomain = cluster.Spec.ClusterNetwork.ServiceDomain + } + + noProxy = append(noProxy, []string{ + "kubernetes", + "kubernetes.default", + ".svc", + }...) + + // append .svc. + noProxy = append( + noProxy, + fmt.Sprintf(".svc.%s", strings.TrimLeft(serviceDomain, ".")), + ) + + return noProxy +} diff --git a/pkg/handlers/httpproxy/inject_test.go b/pkg/handlers/httpproxy/inject_test.go index 305d6b1a8..8243c2060 100644 --- a/pkg/handlers/httpproxy/inject_test.go +++ b/pkg/handlers/httpproxy/inject_test.go @@ -8,7 +8,9 @@ import ( . "github.com/onsi/gomega" "k8s.io/apiserver/pkg/storage/names" + capiv1 "sigs.k8s.io/cluster-api/api/v1beta1" runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/testutils/capitest" ) @@ -16,7 +18,10 @@ import ( func TestGeneratePatches(t *testing.T) { capitest.ValidateGeneratePatches( t, - NewPatch, + func() *httpProxyPatchHandler { + fakeClient := fake.NewClientBuilder().Build() + return NewPatch(fakeClient) + }, capitest.PatchTestDef{ Name: "unset variable", }, @@ -26,9 +31,9 @@ func TestGeneratePatches(t *testing.T) { capitest.VariableWithValue( VariableName, HTTPProxyVariables{ - HTTP: "http://example.com", - HTTPS: "https://example.com", - No: []string{"no-proxy.example.com"}, + HTTP: "http://example.com", + HTTPS: "https://example.com", + AdditionalNo: []string{"no-proxy.example.com"}, }, ), capitest.VariableWithValue( @@ -53,9 +58,9 @@ func TestGeneratePatches(t *testing.T) { capitest.VariableWithValue( VariableName, HTTPProxyVariables{ - HTTP: "http://example.com", - HTTPS: "https://example.com", - No: []string{"no-proxy.example.com"}, + HTTP: "http://example.com", + HTTPS: "https://example.com", + AdditionalNo: []string{"no-proxy.example.com"}, }, ), capitest.VariableWithValue( @@ -80,9 +85,9 @@ func TestGeneratePatches(t *testing.T) { capitest.VariableWithValue( VariableName, HTTPProxyVariables{ - HTTP: "http://example.com", - HTTPS: "https://example.com", - No: []string{"no-proxy.example.com"}, + HTTP: "http://example.com", + HTTPS: "https://example.com", + AdditionalNo: []string{"no-proxy.example.com"}, }, ), }, @@ -95,3 +100,97 @@ func TestGeneratePatches(t *testing.T) { }, ) } + +func TestGenerateNoProxy(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + testCases := []struct { + name string + cluster *capiv1.Cluster + expectedNoProxy []string + }{ + { + name: "no networking config", + cluster: &capiv1.Cluster{}, + expectedNoProxy: []string{ + "localhost", "127.0.0.1", "kubernetes", "kubernetes.default", + ".svc", ".svc.cluster.local", + }, + }, + { + name: "custom pod network", + cluster: &capiv1.Cluster{ + Spec: capiv1.ClusterSpec{ + ClusterNetwork: &capiv1.ClusterNetwork{ + Pods: &capiv1.NetworkRanges{ + CIDRBlocks: []string{"10.0.0.0/24", "10.0.1.0/24"}, + }, + }, + }, + }, + expectedNoProxy: []string{ + "localhost", "127.0.0.1", "10.0.0.0/24", "10.0.1.0/24", "kubernetes", + "kubernetes.default", ".svc", ".svc.cluster.local", + }, + }, + { + name: "custom service network", + cluster: &capiv1.Cluster{ + Spec: capiv1.ClusterSpec{ + ClusterNetwork: &capiv1.ClusterNetwork{ + Services: &capiv1.NetworkRanges{ + CIDRBlocks: []string{"172.16.0.0/24", "172.16.1.0/24"}, + }, + }, + }, + }, + expectedNoProxy: []string{ + "localhost", "127.0.0.1", "172.16.0.0/24", "172.16.1.0/24", "kubernetes", + "kubernetes.default", ".svc", ".svc.cluster.local", + }, + }, + { + name: "custom servicedomain", + cluster: &capiv1.Cluster{ + Spec: capiv1.ClusterSpec{ + ClusterNetwork: &capiv1.ClusterNetwork{ + ServiceDomain: "foo.bar", + }, + }, + }, + expectedNoProxy: []string{ + "localhost", "127.0.0.1", "kubernetes", "kubernetes.default", + ".svc", ".svc.foo.bar", + }, + }, + { + name: "all options", + cluster: &capiv1.Cluster{ + Spec: capiv1.ClusterSpec{ + ClusterNetwork: &capiv1.ClusterNetwork{ + Pods: &capiv1.NetworkRanges{ + CIDRBlocks: []string{"10.10.0.0/16"}, + }, + Services: &capiv1.NetworkRanges{ + CIDRBlocks: []string{"172.16.0.0/16"}, + }, + ServiceDomain: "foo.bar", + }, + }, + }, + expectedNoProxy: []string{ + "localhost", "127.0.0.1", "10.10.0.0/16", "172.16.0.0/16", "kubernetes", + "kubernetes.default", ".svc", ".svc.foo.bar", + }, + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(tt *testing.T) { + tt.Parallel() + g.Expect(generateNoProxy(tc.cluster)).To(Equal(tc.expectedNoProxy)) + }) + } +} diff --git a/pkg/handlers/httpproxy/systemd_proxy_config.go b/pkg/handlers/httpproxy/systemd_proxy_config.go index dbb612861..ddec0c414 100644 --- a/pkg/handlers/httpproxy/systemd_proxy_config.go +++ b/pkg/handlers/httpproxy/systemd_proxy_config.go @@ -26,11 +26,15 @@ var ( } ) -func generateSystemdFiles(vars HTTPProxyVariables) []bootstrapv1.File { - if vars.HTTP == "" && vars.HTTPS == "" && len(vars.No) == 0 { +func generateSystemdFiles(vars HTTPProxyVariables, noProxy []string) []bootstrapv1.File { + if vars.HTTP == "" && vars.HTTPS == "" && len(vars.AdditionalNo) == 0 { return nil } + allNoProxy := []string{} + allNoProxy = append(allNoProxy, noProxy...) + allNoProxy = append(allNoProxy, vars.AdditionalNo...) + tplVars := struct { HTTP string HTTPS string @@ -38,7 +42,7 @@ func generateSystemdFiles(vars HTTPProxyVariables) []bootstrapv1.File { }{ HTTP: vars.HTTP, HTTPS: vars.HTTPS, - NO: strings.Join(vars.No, ","), + NO: strings.Join(allNoProxy, ","), } var buf bytes.Buffer diff --git a/pkg/handlers/httpproxy/systemd_proxy_config_test.go b/pkg/handlers/httpproxy/systemd_proxy_config_test.go index 158b2dc0f..f89347b02 100644 --- a/pkg/handlers/httpproxy/systemd_proxy_config_test.go +++ b/pkg/handlers/httpproxy/systemd_proxy_config_test.go @@ -16,68 +16,84 @@ func TestGenerateSystemdFiles(t *testing.T) { tests := []struct { name string vars HTTPProxyVariables + noProxy []string expectedContents string - }{{ - name: "no proxy configuration", - }, { - name: "all vars set", - vars: HTTPProxyVariables{ - HTTP: "http://example.com", - HTTPS: "https://example.com", - No: []string{ - "https://no-proxy.example.com", + }{ + { + name: "no proxy configuration", + }, { + name: "all vars set", + vars: HTTPProxyVariables{ + HTTP: "http://example.com", + HTTPS: "https://example.com", + AdditionalNo: []string{ + "no-proxy.example.com", + }, }, - }, - expectedContents: `[Service] + expectedContents: `[Service] Environment="HTTP_PROXY=http://example.com" Environment="http_proxy=http://example.com" Environment="HTTPS_PROXY=https://example.com" Environment="https_proxy=https://example.com" -Environment="NO_PROXY=https://no-proxy.example.com" -Environment="no_proxy=https://no-proxy.example.com" +Environment="NO_PROXY=no-proxy.example.com" +Environment="no_proxy=no-proxy.example.com" `, - }, { - name: "http only", - vars: HTTPProxyVariables{ - HTTP: "http://example.com", - }, - expectedContents: `[Service] + }, { + name: "http only", + vars: HTTPProxyVariables{ + HTTP: "http://example.com", + }, + expectedContents: `[Service] Environment="HTTP_PROXY=http://example.com" Environment="http_proxy=http://example.com" `, - }, { - name: "https only", - vars: HTTPProxyVariables{ - HTTPS: "https://example.com", - }, - expectedContents: `[Service] + }, { + name: "https only", + vars: HTTPProxyVariables{ + HTTPS: "https://example.com", + }, + expectedContents: `[Service] Environment="HTTPS_PROXY=https://example.com" Environment="https_proxy=https://example.com" `, - }, { - name: "no proxy only", - vars: HTTPProxyVariables{ - No: []string{ - "https://no-proxy.example.com", + }, { + name: "no proxy only", + vars: HTTPProxyVariables{ + AdditionalNo: []string{ + "no-proxy.example.com", + }, }, - }, - expectedContents: `[Service] -Environment="NO_PROXY=https://no-proxy.example.com" -Environment="no_proxy=https://no-proxy.example.com" + expectedContents: `[Service] +Environment="NO_PROXY=no-proxy.example.com" +Environment="no_proxy=no-proxy.example.com" `, - }, { - name: "multiple no proxy only", - vars: HTTPProxyVariables{ - No: []string{ - "https://no-proxy.example.com", - "https://no-proxy-1.example.com", + }, { + name: "multiple no proxy only", + vars: HTTPProxyVariables{ + AdditionalNo: []string{ + "no-proxy.example.com", + "no-proxy-1.example.com", + }, }, - }, - expectedContents: `[Service] -Environment="NO_PROXY=https://no-proxy.example.com,https://no-proxy-1.example.com" -Environment="no_proxy=https://no-proxy.example.com,https://no-proxy-1.example.com" + expectedContents: `[Service] +Environment="NO_PROXY=no-proxy.example.com,no-proxy-1.example.com" +Environment="no_proxy=no-proxy.example.com,no-proxy-1.example.com" +`, + }, { + name: "default no proxy values", + vars: HTTPProxyVariables{ + AdditionalNo: []string{ + "no-proxy.example.com", + "no-proxy-1.example.com", + }, + }, + noProxy: []string{"localhost", "127.0.0.1"}, + expectedContents: `[Service] +Environment="NO_PROXY=localhost,127.0.0.1,no-proxy.example.com,no-proxy-1.example.com" +Environment="no_proxy=localhost,127.0.0.1,no-proxy.example.com,no-proxy-1.example.com" `, - }} + }, + } for idx := range tests { tt := tests[idx] @@ -101,7 +117,7 @@ Environment="no_proxy=https://no-proxy.example.com,https://no-proxy-1.example.co }} } - g.Expect(generateSystemdFiles(tt.vars)).Should(Equal(expectedFiles)) + g.Expect(generateSystemdFiles(tt.vars, tt.noProxy)).Should(Equal(expectedFiles)) }) } } diff --git a/pkg/handlers/httpproxy/variables.go b/pkg/handlers/httpproxy/variables.go index 2f0cf28bf..f9acac74f 100644 --- a/pkg/handlers/httpproxy/variables.go +++ b/pkg/handlers/httpproxy/variables.go @@ -58,8 +58,11 @@ type HTTPProxyVariables struct { // HTTPS proxy. HTTPS string `json:"https,omitempty"` - // No Proxy list. - No []string `json:"no,omitempty"` + // AdditionalNo Proxy list that will be added to the automatically calculated + // values that will apply no_proxy configuration for cluster internal network. + // Default values: localhost,127.0.0.1,,,kubernetes + // ,kubernetes.default,.svc,.svc. + AdditionalNo []string `json:"additionalNo"` } // VariableSchema provides Cluster Class variable schema definition. @@ -76,9 +79,12 @@ func (HTTPProxyVariables) VariableSchema() clusterv1.VariableSchema { Description: "HTTPS proxy value.", Type: "string", }, - "no": { - Description: "No Proxy list.", - Type: "array", + "additionalNo": { + Description: "Additional No Proxy list that will be added to the automatically calculated " + + "values that will apply no_proxy configuration for cluster internal network. " + + "Default value: localhost,127.0.0.1,,,kubernetes," + + "kubernetes.default,.svc,.svc.", + Type: "array", Items: &clusterv1.JSONSchemaProps{ Type: "string", }, diff --git a/pkg/handlers/httpproxy/variables_test.go b/pkg/handlers/httpproxy/variables_test.go index b20ed5ac3..8f794366e 100644 --- a/pkg/handlers/httpproxy/variables_test.go +++ b/pkg/handlers/httpproxy/variables_test.go @@ -20,9 +20,9 @@ func TestVariableValidation(t *testing.T) { capitest.VariableTestDef{ Name: "valid values", Vals: HTTPProxyVariables{ - HTTP: "http://a.b.c.example.com", - HTTPS: "https://a.b.c.example.com", - No: []string{"d.e.f.example.com"}, + HTTP: "http://a.b.c.example.com", + HTTPS: "https://a.b.c.example.com", + AdditionalNo: []string{"d.e.f.example.com"}, }, }, )