nutanix-cloud-native · jimmidyson · Sep 5, 2023 · Sep 4, 2023
diff --git a/charts/capi-runtime-extensions/templates/role.yaml b/charts/capi-runtime-extensions/templates/role.yaml
@@ -30,3 +30,11 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - cluster.x-k8s.io
+  resources:
+  - clusters
+  verbs:
+  - get
+  - list
+  - watch
diff --git a/cmd/capi-runtime-extensions/main.go b/cmd/capi-runtime-extensions/main.go
@@ -79,7 +79,7 @@ func main() {
 		servicelbgc.New(client),
 		calico.New(client, calicoCNIConfig),
 		httpproxy.NewVariable(),
-		httpproxy.NewPatch(),
+		httpproxy.NewPatch(client),
 		extraapiservercertsans.NewVariable(),
 		extraapiservercertsans.NewPatch(),
 	)

diff --git a/docs/content/http-proxy.md b/docs/content/http-proxy.md
@@ -32,14 +32,17 @@ spec:
   topology:
     variables:
       - name: proxy
-        value:
+        values:
           http: http://example.com
-          https: https://example.com
-          no:
+          https: http://example.com
+          additionalNo:
             - no-proxy-1.example.com
             - no-proxy-2.example.com
 ```
 
+The `additionalNo` list will be added to default pre-calculated values that apply on k8s networking
+`localhost,127.0.0.1,<POD CIDRS>,<SERVICE CIDRS>,kubernetes,kubernetes.default,.svc,.svc.cluster.local`.
+
 Applying this configuration will result in new bootstrap files on the `KubeadmControlPlaneTemplate`
 and `KubeadmConfigTemplate`.
 

diff --git a/pkg/handlers/httpproxy/inject.go b/pkg/handlers/httpproxy/inject.go
@@ -1,20 +1,27 @@
 // Copyright 2023 D2iQ, Inc. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
+// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters,verbs=watch;list;get
+
 package httpproxy
 
 import (
 	"context"
+	"errors"
+	"fmt"
+	"strings"
 
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/types"
+	capiv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
 	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
 	runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
 	"sigs.k8s.io/cluster-api/exp/runtime/topologymutation"
 	ctrl "sigs.k8s.io/controller-runtime"
-	"sigs.k8s.io/controller-runtime/pkg/client"
+	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/d2iq-labs/capi-runtime-extensions/common/pkg/handlers"
 	"github.com/d2iq-labs/capi-runtime-extensions/common/pkg/handlers/mutation"
@@ -30,14 +37,15 @@ const (
 
 type httpProxyPatchHandler struct {
 	decoder runtime.Decoder
+	client  ctrlclient.Reader
 }
 
 var (
 	_ handlers.Named           = &httpProxyPatchHandler{}
 	_ mutation.GeneratePatches = &httpProxyPatchHandler{}
 )
 
-func NewPatch() *httpProxyPatchHandler {
+func NewPatch(cl ctrlclient.Reader) *httpProxyPatchHandler {
 	scheme := runtime.NewScheme()
 	_ = bootstrapv1.AddToScheme(scheme)
 	_ = controlplanev1.AddToScheme(scheme)
@@ -46,6 +54,7 @@ func NewPatch() *httpProxyPatchHandler {
 			controlplanev1.GroupVersion,
 			bootstrapv1.GroupVersion,
 		),
+		client: cl,
 	}
 }
 
@@ -58,6 +67,12 @@ func (h *httpProxyPatchHandler) GeneratePatches(
 	req *runtimehooksv1.GeneratePatchesRequest,
 	resp *runtimehooksv1.GeneratePatchesResponse,
 ) {
+	log := ctrl.LoggerFrom(ctx)
+	noProxy, err := h.detectNoProxy(ctx, req)
+	if err != nil {
+		log.Error(err, "failed to resolve no proxy value")
+	}
+
 	topologymutation.WalkTemplates(
 		ctx,
 		h.decoder,
@@ -69,7 +84,7 @@ func (h *httpProxyPatchHandler) GeneratePatches(
 			vars map[string]apiextensionsv1.JSON,
 			holderRef runtimehooksv1.HolderReference,
 		) error {
-			log := ctrl.LoggerFrom(ctx).WithValues(
+			log = log.WithValues(
 				"holderRef", holderRef,
 			)
 
@@ -92,11 +107,11 @@ func (h *httpProxyPatchHandler) GeneratePatches(
 				func(obj *controlplanev1.KubeadmControlPlaneTemplate) error {
 					log.WithValues(
 						"patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(),
-						"patchedObjectName", client.ObjectKeyFromObject(obj),
+						"patchedObjectName", ctrlclient.ObjectKeyFromObject(obj),
 					).Info("adding files to control plane kubeadm config spec")
 					obj.Spec.Template.Spec.KubeadmConfigSpec.Files = append(
 						obj.Spec.Template.Spec.KubeadmConfigSpec.Files,
-						generateSystemdFiles(httpProxyVariable)...,
+						generateSystemdFiles(httpProxyVariable, noProxy)...,
 					)
 					return nil
 				}); err != nil {
@@ -108,11 +123,11 @@ func (h *httpProxyPatchHandler) GeneratePatches(
 				func(obj *bootstrapv1.KubeadmConfigTemplate) error {
 					log.WithValues(
 						"patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(),
-						"patchedObjectName", client.ObjectKeyFromObject(obj),
+						"patchedObjectName", ctrlclient.ObjectKeyFromObject(obj),
 					).Info("adding files to worker node kubeadm config template")
 					obj.Spec.Template.Spec.Files = append(
 						obj.Spec.Template.Spec.Files,
-						generateSystemdFiles(httpProxyVariable)...,
+						generateSystemdFiles(httpProxyVariable, noProxy)...,
 					)
 					return nil
 				}); err != nil {
@@ -123,3 +138,70 @@ func (h *httpProxyPatchHandler) GeneratePatches(
 		},
 	)
 }
+
+func (h *httpProxyPatchHandler) detectNoProxy(
+	ctx context.Context,
+	req *runtimehooksv1.GeneratePatchesRequest,
+) ([]string, error) {
+	clusterKey := types.NamespacedName{}
+
+	for i := range req.Items {
+		item := req.Items[i]
+		if item.HolderReference.Kind == "Cluster" &&
+			item.HolderReference.APIVersion == capiv1.GroupVersion.String() {
+			clusterKey.Name = item.HolderReference.Name
+			clusterKey.Namespace = item.HolderReference.Namespace
+		}
+	}
+
+	if clusterKey.Name == "" {
+		return nil, errors.New("failed to detect cluster name from GeneratePatch request")
+	}
+
+	cluster := &capiv1.Cluster{}
+	if err := h.client.Get(ctx, clusterKey, cluster); err != nil {
+		return nil, err
+	}
+
+	return generateNoProxy(cluster), nil
+}
+
+// generateNoProxy creates default NO_PROXY values that should be applied on cluster
+// in any environment and are preventing the use of proxy for cluster internal
+// networking.
+func generateNoProxy(cluster *capiv1.Cluster) []string {
+	noProxy := []string{
+		"localhost",
+		"127.0.0.1",
+	}
+
+	if cluster.Spec.ClusterNetwork != nil &&
+		cluster.Spec.ClusterNetwork.Pods != nil {
+		noProxy = append(noProxy, cluster.Spec.ClusterNetwork.Pods.CIDRBlocks...)
+	}
+
+	if cluster.Spec.ClusterNetwork != nil &&
+		cluster.Spec.ClusterNetwork.Services != nil {
+		noProxy = append(noProxy, cluster.Spec.ClusterNetwork.Services.CIDRBlocks...)
+	}
+
+	serviceDomain := "cluster.local"
+	if cluster.Spec.ClusterNetwork != nil &&
+		cluster.Spec.ClusterNetwork.ServiceDomain != "" {
+		serviceDomain = cluster.Spec.ClusterNetwork.ServiceDomain
+	}
+
+	noProxy = append(noProxy, []string{
+		"kubernetes",
+		"kubernetes.default",
+		".svc",
+	}...)
+
+	// append .svc.<SERVICE_DOMAIN>
+	noProxy = append(
+		noProxy,
+		fmt.Sprintf(".svc.%s", strings.TrimLeft(serviceDomain, ".")),
+	)
+
+	return noProxy
+}
diff --git a/pkg/handlers/httpproxy/inject_test.go b/pkg/handlers/httpproxy/inject_test.go
@@ -8,15 +8,20 @@ import (
 
 	. "github.com/onsi/gomega"
 	"k8s.io/apiserver/pkg/storage/names"
+	capiv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 
 	"github.com/d2iq-labs/capi-runtime-extensions/common/pkg/testutils/capitest"
 )
 
 func TestGeneratePatches(t *testing.T) {
 	capitest.ValidateGeneratePatches(
 		t,
-		NewPatch,
+		func() *httpProxyPatchHandler {
+			fakeClient := fake.NewClientBuilder().Build()
+			return NewPatch(fakeClient)
+		},
 		capitest.PatchTestDef{
 			Name: "unset variable",
 		},
@@ -26,9 +31,9 @@ func TestGeneratePatches(t *testing.T) {
 				capitest.VariableWithValue(
 					VariableName,
 					HTTPProxyVariables{
-						HTTP:  "http://example.com",
-						HTTPS: "https://example.com",
-						No:    []string{"no-proxy.example.com"},
+						HTTP:         "http://example.com",
+						HTTPS:        "https://example.com",
+						AdditionalNo: []string{"no-proxy.example.com"},
 					},
 				),
 				capitest.VariableWithValue(
@@ -53,9 +58,9 @@ func TestGeneratePatches(t *testing.T) {
 				capitest.VariableWithValue(
 					VariableName,
 					HTTPProxyVariables{
-						HTTP:  "http://example.com",
-						HTTPS: "https://example.com",
-						No:    []string{"no-proxy.example.com"},
+						HTTP:         "http://example.com",
+						HTTPS:        "https://example.com",
+						AdditionalNo: []string{"no-proxy.example.com"},
 					},
 				),
 				capitest.VariableWithValue(
@@ -80,9 +85,9 @@ func TestGeneratePatches(t *testing.T) {
 				capitest.VariableWithValue(
 					VariableName,
 					HTTPProxyVariables{
-						HTTP:  "http://example.com",
-						HTTPS: "https://example.com",
-						No:    []string{"no-proxy.example.com"},
+						HTTP:         "http://example.com",
+						HTTPS:        "https://example.com",
+						AdditionalNo: []string{"no-proxy.example.com"},
 					},
 				),
 			},
@@ -95,3 +100,97 @@ func TestGeneratePatches(t *testing.T) {
 		},
 	)
 }
+
+func TestGenerateNoProxy(t *testing.T) {
+	t.Parallel()
+	g := NewWithT(t)
+
+	testCases := []struct {
+		name            string
+		cluster         *capiv1.Cluster
+		expectedNoProxy []string
+	}{
+		{
+			name:    "no networking config",
+			cluster: &capiv1.Cluster{},
+			expectedNoProxy: []string{
+				"localhost", "127.0.0.1", "kubernetes", "kubernetes.default",
+				".svc", ".svc.cluster.local",
+			},
+		},
+		{
+			name: "custom pod network",
+			cluster: &capiv1.Cluster{
+				Spec: capiv1.ClusterSpec{
+					ClusterNetwork: &capiv1.ClusterNetwork{
+						Pods: &capiv1.NetworkRanges{
+							CIDRBlocks: []string{"10.0.0.0/24", "10.0.1.0/24"},
+						},
+					},
+				},
+			},
+			expectedNoProxy: []string{
+				"localhost", "127.0.0.1", "10.0.0.0/24", "10.0.1.0/24", "kubernetes",
+				"kubernetes.default", ".svc", ".svc.cluster.local",
+			},
+		},
+		{
+			name: "custom service network",
+			cluster: &capiv1.Cluster{
+				Spec: capiv1.ClusterSpec{
+					ClusterNetwork: &capiv1.ClusterNetwork{
+						Services: &capiv1.NetworkRanges{
+							CIDRBlocks: []string{"172.16.0.0/24", "172.16.1.0/24"},
+						},
+					},
+				},
+			},
+			expectedNoProxy: []string{
+				"localhost", "127.0.0.1", "172.16.0.0/24", "172.16.1.0/24", "kubernetes",
+				"kubernetes.default", ".svc", ".svc.cluster.local",
+			},
+		},
+		{
+			name: "custom servicedomain",
+			cluster: &capiv1.Cluster{
+				Spec: capiv1.ClusterSpec{
+					ClusterNetwork: &capiv1.ClusterNetwork{
+						ServiceDomain: "foo.bar",
+					},
+				},
+			},
+			expectedNoProxy: []string{
+				"localhost", "127.0.0.1", "kubernetes", "kubernetes.default",
+				".svc", ".svc.foo.bar",
+			},
+		},
+		{
+			name: "all options",
+			cluster: &capiv1.Cluster{
+				Spec: capiv1.ClusterSpec{
+					ClusterNetwork: &capiv1.ClusterNetwork{
+						Pods: &capiv1.NetworkRanges{
+							CIDRBlocks: []string{"10.10.0.0/16"},
+						},
+						Services: &capiv1.NetworkRanges{
+							CIDRBlocks: []string{"172.16.0.0/16"},
+						},
+						ServiceDomain: "foo.bar",
+					},
+				},
+			},
+			expectedNoProxy: []string{
+				"localhost", "127.0.0.1", "10.10.0.0/16", "172.16.0.0/16", "kubernetes",
+				"kubernetes.default", ".svc", ".svc.foo.bar",
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.name, func(tt *testing.T) {
+			tt.Parallel()
+			g.Expect(generateNoProxy(tc.cluster)).To(Equal(tc.expectedNoProxy))
+		})
+	}
+}