diff --git a/api/v1alpha1/clusterconfig_types.go b/api/v1alpha1/clusterconfig_types.go index caeb673af..72e684cd7 100644 --- a/api/v1alpha1/clusterconfig_types.go +++ b/api/v1alpha1/clusterconfig_types.go @@ -210,21 +210,6 @@ type Etcd struct { Image *Image `json:"image,omitempty"` } -// HTTPProxy required for providing proxy configuration. -type HTTPProxy struct { - // HTTP proxy value. - HTTP string `json:"http,omitempty"` - - // HTTPS proxy value. - HTTPS string `json:"https,omitempty"` - - // AdditionalNo Proxy list that will be added to the automatically calculated - // values that will apply no_proxy configuration for cluster internal network. - // Default values: localhost,127.0.0.1,,,kubernetes - // ,kubernetes.default,.svc,.svc. - AdditionalNo []string `json:"additionalNo"` -} - type RegistryCredentials struct { // A reference to the Secret containing the registry credentials and optional CA certificate // using the keys `username`, `password` and `ca.crt`. diff --git a/api/v1alpha1/http_proxy.go b/api/v1alpha1/http_proxy.go new file mode 100644 index 000000000..6847d4b7f --- /dev/null +++ b/api/v1alpha1/http_proxy.go @@ -0,0 +1,102 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import ( + "fmt" + "strings" + + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" +) + +const ( + // instanceMetadataIP is the IPv4 address used to retrieve + // instance metadata in AWS, Azure, OpenStack, etc. + instanceMetadataIP = "169.254.169.254" +) + +// HTTPProxy required for providing proxy configuration. +type HTTPProxy struct { + // HTTP proxy value. + HTTP string `json:"http,omitempty"` + + // HTTPS proxy value. + HTTPS string `json:"https,omitempty"` + + // AdditionalNo Proxy list that will be added to the automatically calculated + // values that will apply no_proxy configuration for cluster internal network. + // Default values: localhost,127.0.0.1,,,kubernetes + // ,kubernetes.default,.svc,.svc. + AdditionalNo []string `json:"additionalNo"` +} + +// GenerateNoProxy creates default NO_PROXY values that should be applied on cluster +// in any environment and are preventing the use of proxy for cluster internal +// networking. It appends additional values from HTTPProxy.AdditionalNo. +func (p *HTTPProxy) GenerateNoProxy(cluster *clusterv1.Cluster) []string { + noProxy := []string{ + "localhost", + "127.0.0.1", + } + + if cluster.Spec.ClusterNetwork != nil && + cluster.Spec.ClusterNetwork.Pods != nil { + noProxy = append(noProxy, cluster.Spec.ClusterNetwork.Pods.CIDRBlocks...) + } + + if cluster.Spec.ClusterNetwork != nil && + cluster.Spec.ClusterNetwork.Services != nil { + noProxy = append(noProxy, cluster.Spec.ClusterNetwork.Services.CIDRBlocks...) + } + + serviceDomain := "cluster.local" + if cluster.Spec.ClusterNetwork != nil && + cluster.Spec.ClusterNetwork.ServiceDomain != "" { + serviceDomain = cluster.Spec.ClusterNetwork.ServiceDomain + } + + noProxy = append( + noProxy, + "kubernetes", + "kubernetes.default", + ".svc", + // append .svc. + fmt.Sprintf(".svc.%s", strings.TrimLeft(serviceDomain, ".")), + ) + + if cluster.Spec.InfrastructureRef == nil { + return append(noProxy, p.AdditionalNo...) + } + + // Add infra-specific entries + switch cluster.Spec.InfrastructureRef.Kind { + case "AWSCluster", "AWSManagedCluster": + noProxy = append( + noProxy, + // Exclude the instance metadata service + instanceMetadataIP, + // Exclude the control plane endpoint + ".elb.amazonaws.com", + ) + case "AzureCluster", "AzureManagedControlPlane": + noProxy = append( + noProxy, + // Exclude the instance metadata service + instanceMetadataIP, + ) + case "GCPCluster": + noProxy = append( + noProxy, + // Exclude the instance metadata service + instanceMetadataIP, + // Exclude aliases for instance metadata service. + // See https://cloud.google.com/vpc/docs/special-configurations + "metadata", + "metadata.google.internal", + ) + default: + // Unknown infrastructure. Do nothing. + } + return append(noProxy, p.AdditionalNo...) +} diff --git a/api/v1alpha1/http_proxy_test.go b/api/v1alpha1/http_proxy_test.go new file mode 100644 index 000000000..e8424b1ef --- /dev/null +++ b/api/v1alpha1/http_proxy_test.go @@ -0,0 +1,195 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1_test + +import ( + "testing" + + "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" +) + +func TestGenerateNoProxy(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + cluster *clusterv1.Cluster + expectedNoProxy []string + additonalNo []string + }{{ + name: "no networking config", + cluster: &clusterv1.Cluster{}, + expectedNoProxy: []string{ + "localhost", "127.0.0.1", "kubernetes", "kubernetes.default", + ".svc", ".svc.cluster.local", + }, + }, { + name: "no networking config with additional no proxy", + cluster: &clusterv1.Cluster{}, + additonalNo: []string{"example.com"}, + expectedNoProxy: []string{ + "localhost", "127.0.0.1", "kubernetes", "kubernetes.default", + ".svc", ".svc.cluster.local", "example.com", + }, + }, { + name: "custom pod network", + cluster: &clusterv1.Cluster{ + Spec: clusterv1.ClusterSpec{ + ClusterNetwork: &clusterv1.ClusterNetwork{ + Pods: &clusterv1.NetworkRanges{ + CIDRBlocks: []string{"10.0.0.0/24", "10.0.1.0/24"}, + }, + }, + }, + }, + expectedNoProxy: []string{ + "localhost", "127.0.0.1", "10.0.0.0/24", "10.0.1.0/24", "kubernetes", + "kubernetes.default", ".svc", ".svc.cluster.local", + }, + }, { + name: "Unknown infrastructure cluster", + cluster: &clusterv1.Cluster{ + Spec: clusterv1.ClusterSpec{ + InfrastructureRef: &v1.ObjectReference{ + Kind: "SomeFakeInfrastructureCluster", + }, + }, + }, + expectedNoProxy: []string{ + "localhost", "127.0.0.1", "kubernetes", "kubernetes.default", + ".svc", ".svc.cluster.local", + }, + }, { + name: "AWS cluster", + cluster: &clusterv1.Cluster{ + Spec: clusterv1.ClusterSpec{ + InfrastructureRef: &v1.ObjectReference{ + Kind: "AWSCluster", + }, + }, + }, + expectedNoProxy: []string{ + "localhost", "127.0.0.1", "kubernetes", "kubernetes.default", + ".svc", ".svc.cluster.local", "169.254.169.254", ".elb.amazonaws.com", + }, + }, { + name: "AWS managed (EKS) cluster", + cluster: &clusterv1.Cluster{ + Spec: clusterv1.ClusterSpec{ + InfrastructureRef: &v1.ObjectReference{ + Kind: "AWSManagedCluster", + }, + }, + }, + expectedNoProxy: []string{ + "localhost", "127.0.0.1", "kubernetes", "kubernetes.default", + ".svc", ".svc.cluster.local", "169.254.169.254", ".elb.amazonaws.com", + }, + }, { + name: "Azure cluster", + cluster: &clusterv1.Cluster{ + Spec: clusterv1.ClusterSpec{ + InfrastructureRef: &v1.ObjectReference{ + Kind: "AzureCluster", + }, + }, + }, + expectedNoProxy: []string{ + "localhost", "127.0.0.1", "kubernetes", "kubernetes.default", + ".svc", ".svc.cluster.local", "169.254.169.254", + }, + }, { + name: "Azure managed (AKS) cluster", + cluster: &clusterv1.Cluster{ + Spec: clusterv1.ClusterSpec{ + InfrastructureRef: &v1.ObjectReference{ + Kind: "AzureCluster", + }, + }, + }, + expectedNoProxy: []string{ + "localhost", "127.0.0.1", "kubernetes", "kubernetes.default", + ".svc", ".svc.cluster.local", "169.254.169.254", + }, + }, { + name: "GCP cluster", + cluster: &clusterv1.Cluster{ + Spec: clusterv1.ClusterSpec{ + InfrastructureRef: &v1.ObjectReference{ + Kind: "GCPCluster", + }, + }, + }, + expectedNoProxy: []string{ + "localhost", "127.0.0.1", "kubernetes", "kubernetes.default", + ".svc", ".svc.cluster.local", "169.254.169.254", "metadata", "metadata.google.internal", + }, + }, { + name: "custom service network", + cluster: &clusterv1.Cluster{ + Spec: clusterv1.ClusterSpec{ + ClusterNetwork: &clusterv1.ClusterNetwork{ + Services: &clusterv1.NetworkRanges{ + CIDRBlocks: []string{"172.16.0.0/24", "172.16.1.0/24"}, + }, + }, + }, + }, + expectedNoProxy: []string{ + "localhost", "127.0.0.1", "172.16.0.0/24", "172.16.1.0/24", "kubernetes", + "kubernetes.default", ".svc", ".svc.cluster.local", + }, + }, { + name: "custom servicedomain", + cluster: &clusterv1.Cluster{ + Spec: clusterv1.ClusterSpec{ + ClusterNetwork: &clusterv1.ClusterNetwork{ + ServiceDomain: "foo.bar", + }, + }, + }, + expectedNoProxy: []string{ + "localhost", "127.0.0.1", "kubernetes", "kubernetes.default", + ".svc", ".svc.foo.bar", + }, + }, { + name: "all options", + cluster: &clusterv1.Cluster{ + Spec: clusterv1.ClusterSpec{ + ClusterNetwork: &clusterv1.ClusterNetwork{ + Pods: &clusterv1.NetworkRanges{ + CIDRBlocks: []string{"10.10.0.0/16"}, + }, + Services: &clusterv1.NetworkRanges{ + CIDRBlocks: []string{"172.16.0.0/16"}, + }, + ServiceDomain: "foo.bar", + }, + }, + }, + additonalNo: []string{"example.com"}, + expectedNoProxy: []string{ + "localhost", "127.0.0.1", "10.10.0.0/16", "172.16.0.0/16", "kubernetes", + "kubernetes.default", ".svc", ".svc.foo.bar", "example.com", + }, + }} + + for idx := range testCases { + tt := testCases[idx] + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + g := gomega.NewWithT(t) + + g.Expect((&v1alpha1.HTTPProxy{ + AdditionalNo: tt.additonalNo, + }).GenerateNoProxy(tt.cluster)).To(gomega.Equal(tt.expectedNoProxy)) + }) + } +} diff --git a/pkg/handlers/generic/mutation/httpproxy/inject.go b/pkg/handlers/generic/mutation/httpproxy/inject.go index 61bd1539d..57025d888 100644 --- a/pkg/handlers/generic/mutation/httpproxy/inject.go +++ b/pkg/handlers/generic/mutation/httpproxy/inject.go @@ -5,12 +5,9 @@ package httpproxy import ( "context" - "fmt" - "strings" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" @@ -28,10 +25,6 @@ import ( const ( // VariableName is the external patch variable name. VariableName = "proxy" - - // instanceMetadataIP is the IPv4 address used to retrieve - // instance metadata in AWS, Azure, OpenStack, etc. - instanceMetadataIP = "169.254.169.254" ) type httpProxyPatchHandler struct { @@ -73,7 +66,6 @@ func (h *httpProxyPatchHandler) Mutate( log.Error(err, "failed to fetch cluster") return err } - noProxy := generateNoProxy(cluster) httpProxyVariable, err := variables.Get[v1alpha1.HTTPProxy]( vars, h.variableName, @@ -105,7 +97,7 @@ func (h *httpProxyPatchHandler) Mutate( ).Info("adding files to control plane kubeadm config spec") obj.Spec.Template.Spec.KubeadmConfigSpec.Files = append( obj.Spec.Template.Spec.KubeadmConfigSpec.Files, - generateSystemdFiles(httpProxyVariable, noProxy)..., + generateSystemdFiles(httpProxyVariable, httpProxyVariable.GenerateNoProxy(cluster))..., ) return nil }); err != nil { @@ -121,7 +113,7 @@ func (h *httpProxyPatchHandler) Mutate( ).Info("adding files to worker node kubeadm config template") obj.Spec.Template.Spec.Files = append( obj.Spec.Template.Spec.Files, - generateSystemdFiles(httpProxyVariable, noProxy)..., + generateSystemdFiles(httpProxyVariable, httpProxyVariable.GenerateNoProxy(cluster))..., ) return nil }); err != nil { @@ -130,73 +122,3 @@ func (h *httpProxyPatchHandler) Mutate( return nil } - -// generateNoProxy creates default NO_PROXY values that should be applied on cluster -// in any environment and are preventing the use of proxy for cluster internal -// networking. -func generateNoProxy(cluster *clusterv1.Cluster) []string { - noProxy := []string{ - "localhost", - "127.0.0.1", - } - - if cluster.Spec.ClusterNetwork != nil && - cluster.Spec.ClusterNetwork.Pods != nil { - noProxy = append(noProxy, cluster.Spec.ClusterNetwork.Pods.CIDRBlocks...) - } - - if cluster.Spec.ClusterNetwork != nil && - cluster.Spec.ClusterNetwork.Services != nil { - noProxy = append(noProxy, cluster.Spec.ClusterNetwork.Services.CIDRBlocks...) - } - - serviceDomain := "cluster.local" - if cluster.Spec.ClusterNetwork != nil && - cluster.Spec.ClusterNetwork.ServiceDomain != "" { - serviceDomain = cluster.Spec.ClusterNetwork.ServiceDomain - } - - noProxy = append( - noProxy, - "kubernetes", - "kubernetes.default", - ".svc", - // append .svc. - fmt.Sprintf(".svc.%s", strings.TrimLeft(serviceDomain, ".")), - ) - - if cluster.Spec.InfrastructureRef == nil { - return noProxy - } - - // Add infra-specific entries - switch cluster.Spec.InfrastructureRef.Kind { - case "AWSCluster", "AWSManagedCluster": - noProxy = append( - noProxy, - // Exclude the instance metadata service - instanceMetadataIP, - // Exclude the control plane endpoint - ".elb.amazonaws.com", - ) - case "AzureCluster", "AzureManagedControlPlane": - noProxy = append( - noProxy, - // Exclude the instance metadata service - instanceMetadataIP, - ) - case "GCPCluster": - noProxy = append( - noProxy, - // Exclude the instance metadata service - instanceMetadataIP, - // Exclude aliases for instance metadata service. - // See https://cloud.google.com/vpc/docs/special-configurations - "metadata", - "metadata.google.internal", - ) - default: - // Unknown infrastructure. Do nothing. - } - return noProxy -} diff --git a/pkg/handlers/generic/mutation/httpproxy/inject_test.go b/pkg/handlers/generic/mutation/httpproxy/inject_test.go index 4308d39ad..e190aee50 100644 --- a/pkg/handlers/generic/mutation/httpproxy/inject_test.go +++ b/pkg/handlers/generic/mutation/httpproxy/inject_test.go @@ -9,7 +9,6 @@ import ( . "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -26,175 +25,6 @@ import ( "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/test/helpers" ) -func TestGenerateNoProxy(t *testing.T) { - t.Parallel() - - testCases := []struct { - name string - cluster *clusterv1.Cluster - expectedNoProxy []string - }{{ - name: "no networking config", - cluster: &clusterv1.Cluster{}, - expectedNoProxy: []string{ - "localhost", "127.0.0.1", "kubernetes", "kubernetes.default", - ".svc", ".svc.cluster.local", - }, - }, { - name: "custom pod network", - cluster: &clusterv1.Cluster{ - Spec: clusterv1.ClusterSpec{ - ClusterNetwork: &clusterv1.ClusterNetwork{ - Pods: &clusterv1.NetworkRanges{ - CIDRBlocks: []string{"10.0.0.0/24", "10.0.1.0/24"}, - }, - }, - }, - }, - expectedNoProxy: []string{ - "localhost", "127.0.0.1", "10.0.0.0/24", "10.0.1.0/24", "kubernetes", - "kubernetes.default", ".svc", ".svc.cluster.local", - }, - }, { - name: "Unknown infrastructure cluster", - cluster: &clusterv1.Cluster{ - Spec: clusterv1.ClusterSpec{ - InfrastructureRef: &v1.ObjectReference{ - Kind: "SomeFakeInfrastructureCluster", - }, - }, - }, - expectedNoProxy: []string{ - "localhost", "127.0.0.1", "kubernetes", "kubernetes.default", - ".svc", ".svc.cluster.local", - }, - }, { - name: "AWS cluster", - cluster: &clusterv1.Cluster{ - Spec: clusterv1.ClusterSpec{ - InfrastructureRef: &v1.ObjectReference{ - Kind: "AWSCluster", - }, - }, - }, - expectedNoProxy: []string{ - "localhost", "127.0.0.1", "kubernetes", "kubernetes.default", - ".svc", ".svc.cluster.local", "169.254.169.254", ".elb.amazonaws.com", - }, - }, { - name: "AWS managed (EKS) cluster", - cluster: &clusterv1.Cluster{ - Spec: clusterv1.ClusterSpec{ - InfrastructureRef: &v1.ObjectReference{ - Kind: "AWSManagedCluster", - }, - }, - }, - expectedNoProxy: []string{ - "localhost", "127.0.0.1", "kubernetes", "kubernetes.default", - ".svc", ".svc.cluster.local", "169.254.169.254", ".elb.amazonaws.com", - }, - }, { - name: "Azure cluster", - cluster: &clusterv1.Cluster{ - Spec: clusterv1.ClusterSpec{ - InfrastructureRef: &v1.ObjectReference{ - Kind: "AzureCluster", - }, - }, - }, - expectedNoProxy: []string{ - "localhost", "127.0.0.1", "kubernetes", "kubernetes.default", - ".svc", ".svc.cluster.local", "169.254.169.254", - }, - }, { - name: "Azure managed (AKS) cluster", - cluster: &clusterv1.Cluster{ - Spec: clusterv1.ClusterSpec{ - InfrastructureRef: &v1.ObjectReference{ - Kind: "AzureCluster", - }, - }, - }, - expectedNoProxy: []string{ - "localhost", "127.0.0.1", "kubernetes", "kubernetes.default", - ".svc", ".svc.cluster.local", "169.254.169.254", - }, - }, { - name: "GCP cluster", - cluster: &clusterv1.Cluster{ - Spec: clusterv1.ClusterSpec{ - InfrastructureRef: &v1.ObjectReference{ - Kind: "GCPCluster", - }, - }, - }, - expectedNoProxy: []string{ - "localhost", "127.0.0.1", "kubernetes", "kubernetes.default", - ".svc", ".svc.cluster.local", "169.254.169.254", "metadata", "metadata.google.internal", - }, - }, { - name: "custom service network", - cluster: &clusterv1.Cluster{ - Spec: clusterv1.ClusterSpec{ - ClusterNetwork: &clusterv1.ClusterNetwork{ - Services: &clusterv1.NetworkRanges{ - CIDRBlocks: []string{"172.16.0.0/24", "172.16.1.0/24"}, - }, - }, - }, - }, - expectedNoProxy: []string{ - "localhost", "127.0.0.1", "172.16.0.0/24", "172.16.1.0/24", "kubernetes", - "kubernetes.default", ".svc", ".svc.cluster.local", - }, - }, { - name: "custom servicedomain", - cluster: &clusterv1.Cluster{ - Spec: clusterv1.ClusterSpec{ - ClusterNetwork: &clusterv1.ClusterNetwork{ - ServiceDomain: "foo.bar", - }, - }, - }, - expectedNoProxy: []string{ - "localhost", "127.0.0.1", "kubernetes", "kubernetes.default", - ".svc", ".svc.foo.bar", - }, - }, { - name: "all options", - cluster: &clusterv1.Cluster{ - Spec: clusterv1.ClusterSpec{ - ClusterNetwork: &clusterv1.ClusterNetwork{ - Pods: &clusterv1.NetworkRanges{ - CIDRBlocks: []string{"10.10.0.0/16"}, - }, - Services: &clusterv1.NetworkRanges{ - CIDRBlocks: []string{"172.16.0.0/16"}, - }, - ServiceDomain: "foo.bar", - }, - }, - }, - expectedNoProxy: []string{ - "localhost", "127.0.0.1", "10.10.0.0/16", "172.16.0.0/16", "kubernetes", - "kubernetes.default", ".svc", ".svc.foo.bar", - }, - }} - - for idx := range testCases { - tt := testCases[idx] - - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - g := gomega.NewWithT(t) - - g.Expect(generateNoProxy(tt.cluster)).To(gomega.Equal(tt.expectedNoProxy)) - }) - } -} - func TestHTTPProxyPatch(t *testing.T) { gomega.RegisterFailHandler(Fail) RunSpecs(t, "HTTP Proxy mutator suite") diff --git a/pkg/handlers/generic/mutation/httpproxy/systemd_proxy_config.go b/pkg/handlers/generic/mutation/httpproxy/systemd_proxy_config.go index 5d25a3c65..647a9dc8f 100644 --- a/pkg/handlers/generic/mutation/httpproxy/systemd_proxy_config.go +++ b/pkg/handlers/generic/mutation/httpproxy/systemd_proxy_config.go @@ -33,10 +33,6 @@ func generateSystemdFiles(vars v1alpha1.HTTPProxy, noProxy []string) []bootstrap return nil } - allNoProxy := []string{} - allNoProxy = append(allNoProxy, noProxy...) - allNoProxy = append(allNoProxy, vars.AdditionalNo...) - tplVars := struct { HTTP string HTTPS string @@ -44,7 +40,7 @@ func generateSystemdFiles(vars v1alpha1.HTTPProxy, noProxy []string) []bootstrap }{ HTTP: vars.HTTP, HTTPS: vars.HTTPS, - NO: strings.Join(allNoProxy, ","), + NO: strings.Join(noProxy, ","), } var buf bytes.Buffer diff --git a/pkg/handlers/generic/mutation/httpproxy/systemd_proxy_config_test.go b/pkg/handlers/generic/mutation/httpproxy/systemd_proxy_config_test.go index 7f7f1bfb7..7a1ee4c19 100644 --- a/pkg/handlers/generic/mutation/httpproxy/systemd_proxy_config_test.go +++ b/pkg/handlers/generic/mutation/httpproxy/systemd_proxy_config_test.go @@ -119,7 +119,8 @@ Environment="no_proxy=localhost,127.0.0.1,no-proxy.example.com,no-proxy-1.exampl }} } - g.Expect(generateSystemdFiles(tt.vars, tt.noProxy)).Should(Equal(expectedFiles)) + g.Expect(generateSystemdFiles(tt.vars, append(tt.noProxy, tt.vars.AdditionalNo...))). + Should(Equal(expectedFiles)) }) } }