refactor: run a script to configure kube-vip

dkoshkin · dkoshkin · commit 4cf43a32fba1 · 2024-06-10T08:11:09.000-07:00
This makes it more consistent with the other preKubeadmCommands
diff --git a/pkg/handlers/generic/mutation/controlplanevirtualip/inject.go b/pkg/handlers/generic/mutation/controlplanevirtualip/inject.go
@@ -134,12 +134,14 @@ func (h *ControlPlaneVirtualIP) Mutate(
 		selectors.ControlPlane(),
 		log,
 		func(obj *controlplanev1.KubeadmControlPlaneTemplate) error {
-			virtualIPProviderFile, getFileErr := virtualIPProvider.GetFile(
-				ctx,
-				controlPlaneEndpointVar,
-			)
-			if getFileErr != nil {
-				return getFileErr
+			virtualIPProviderFiles, preKubeadmCommands, postKubeadmCommands, generateErr :=
+				virtualIPProvider.GenerateFilesAndCommands(
+					ctx,
+					controlPlaneEndpointVar,
+					cluster,
+				)
+			if generateErr != nil {
+				return generateErr
 			}
 
 			log.WithValues(
@@ -151,16 +153,9 @@ func (h *ControlPlaneVirtualIP) Mutate(
 			))
 			obj.Spec.Template.Spec.KubeadmConfigSpec.Files = append(
 				obj.Spec.Template.Spec.KubeadmConfigSpec.Files,
-				*virtualIPProviderFile,
+				virtualIPProviderFiles...,
 			)
 
-			preKubeadmCommands, postKubeadmCommands, getCommandsErr := virtualIPProvider.GetCommands(
-				cluster,
-			)
-			if getCommandsErr != nil {
-				return getCommandsErr
-			}
-
 			if len(preKubeadmCommands) > 0 {
 				log.WithValues(
 					"patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(),
diff --git a/pkg/handlers/generic/mutation/controlplanevirtualip/inject_test.go b/pkg/handlers/generic/mutation/controlplanevirtualip/inject_test.go
@@ -21,7 +21,6 @@ import (
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest/request"
-	virtuialipproviders "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/controlplanevirtualip/providers"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/options"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/test/helpers"
 )
@@ -83,14 +82,14 @@ var _ = Describe("Generate ControlPlane virtual IP patches", func() {
 						Operation: "add",
 						Path:      "/spec/template/spec/kubeadmConfigSpec/preKubeadmCommands",
 						ValueMatcher: gomega.ContainElements(
-							virtuialipproviders.KubeVIPPreKubeadmCommands,
+							"/bin/bash /etc/caren/configure-kube-vip.sh use-super-admin.conf",
 						),
 					},
 					{
 						Operation: "add",
 						Path:      "/spec/template/spec/kubeadmConfigSpec/postKubeadmCommands",
 						ValueMatcher: gomega.ContainElements(
-							virtuialipproviders.KubeVIPPostKubeadmCommands,
+							"/bin/bash /etc/caren/configure-kube-vip.sh use-admin.conf",
 						),
 					},
 				},
@@ -148,20 +147,28 @@ var _ = Describe("Generate ControlPlane virtual IP patches", func() {
 								),
 								gomega.HaveKey("permissions"),
 							),
+							gomega.SatisfyAll(
+								gomega.HaveKey("content"),
+								gomega.HaveKeyWithValue(
+									"path",
+									gomega.ContainSubstring("configure-kube-vip.sh"),
+								),
+								gomega.HaveKey("permissions"),
+							),
 						),
 					},
 					{
 						Operation: "add",
 						Path:      "/spec/template/spec/kubeadmConfigSpec/preKubeadmCommands",
 						ValueMatcher: gomega.ContainElements(
-							virtuialipproviders.KubeVIPPreKubeadmCommands,
+							"/bin/bash /etc/caren/configure-kube-vip.sh use-super-admin.conf",
 						),
 					},
 					{
 						Operation: "add",
 						Path:      "/spec/template/spec/kubeadmConfigSpec/postKubeadmCommands",
 						ValueMatcher: gomega.ContainElements(
-							virtuialipproviders.KubeVIPPostKubeadmCommands,
+							"/bin/bash /etc/caren/configure-kube-vip.sh use-admin.conf",
 						),
 					},
 				},
diff --git a/pkg/handlers/generic/mutation/controlplanevirtualip/providers/kubevip.go b/pkg/handlers/generic/mutation/controlplanevirtualip/providers/kubevip.go
@@ -5,6 +5,7 @@ package providers
 
 import (
 	"context"
+	_ "embed"
 	"fmt"
 
 	"github.com/blang/semver/v4"
@@ -14,17 +15,28 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/common"
+)
+
+const (
+	kubeVIPFileOwner       = "root:root"
+	kubeVIPFilePath        = "/etc/kubernetes/manifests/kube-vip.yaml"
+	kubeVIPFilePermissions = "0600"
+
+	configureKubeVIPScriptPermissions = "0700"
+)
+
+var (
+	configureKubeVIPScriptOnRemote = common.ConfigFilePathOnRemote(
+		"configure-kube-vip.sh")
+
+	configureKubeVIPScriptOnRemotePreKubeadmCommand  = "/bin/bash " + configureKubeVIPScriptOnRemote + " use-super-admin.conf"
+	configureKubeVIPScriptOnRemotePostKubeadmCommand = "/bin/bash " + configureKubeVIPScriptOnRemote + " use-admin.conf"
 )
 
 var (
-	//nolint:lll // for readability prefer to keep the long line
-	KubeVIPPreKubeadmCommands = []string{`if [ -f /run/kubeadm/kubeadm.yaml ]; then
-  sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' /etc/kubernetes/manifests/kube-vip.yaml;
-fi`}
-	//nolint:lll // for readability prefer to keep the long line
-	KubeVIPPostKubeadmCommands = []string{`if [ -f /run/kubeadm/kubeadm.yaml ]; then
-  sed -i 's#path: /etc/kubernetes/super-admin.conf#path: /etc/kubernetes/admin.conf#' /etc/kubernetes/manifests/kube-vip.yaml;
-fi`}
+	//go:embed templates/configure-kube-vip.sh
+	configureKubeVIPScript []byte
 )
 
 type kubeVIPFromConfigMapProvider struct {
@@ -50,35 +62,34 @@ func (p *kubeVIPFromConfigMapProvider) Name() string {
 	return "kube-vip"
 }
 
-// GetFile reads the kube-vip template from the ConfigMap
-// and returns the content a File, templating the required variables.
-func (p *kubeVIPFromConfigMapProvider) GetFile(
+// GenerateFilesAndCommands returns files and pre/post kubeadm commands for kube-vip.
+// It reads kube-vip template from a ConfigMap and returns the content a File, templating the required variables.
+// If required, it also returns a script file and pre/post kubeadm commands to change the kube-vip Pod to use the new
+// super-admin.conf file.
+func (p *kubeVIPFromConfigMapProvider) GenerateFilesAndCommands(
 	ctx context.Context,
 	spec v1alpha1.ControlPlaneEndpointSpec,
-) (*bootstrapv1.File, error) {
+	cluster *clusterv1.Cluster,
+) ([]bootstrapv1.File, []string, []string, error) {
 	data, err := getTemplateFromConfigMap(ctx, p.client, p.configMapKey)
 	if err != nil {
-		return nil, fmt.Errorf("failed getting template data: %w", err)
+		return nil, nil, nil, fmt.Errorf("failed getting template data: %w", err)
 	}
 
 	kubeVIPStaticPod, err := templateValues(spec, data)
 	if err != nil {
-		return nil, fmt.Errorf("failed templating static Pod: %w", err)
+		return nil, nil, nil, fmt.Errorf("failed templating static Pod: %w", err)
 	}
 
-	return &bootstrapv1.File{
-		Content:     kubeVIPStaticPod,
-		Owner:       kubeVIPFileOwner,
-		Path:        kubeVIPFilePath,
-		Permissions: kubeVIPFilePermissions,
-	}, nil
-}
+	files := []bootstrapv1.File{
+		{
+			Content:     kubeVIPStaticPod,
+			Owner:       kubeVIPFileOwner,
+			Path:        kubeVIPFilePath,
+			Permissions: kubeVIPFilePermissions,
+		},
+	}
 
-//
-//nolint:gocritic // No need for named return values
-func (p *kubeVIPFromConfigMapProvider) GetCommands(
-	cluster *clusterv1.Cluster,
-) ([]string, []string, error) {
 	// The kube-vip static Pod uses admin.conf on the host to connect to the API server.
 	// But, starting with Kubernetes 1.29, admin.conf first gets created with no RBAC permissions.
 	// At the same time, 'kubeadm init' command waits for the API server to be reachable on the kube-vip IP.
@@ -91,13 +102,24 @@ func (p *kubeVIPFromConfigMapProvider) GetCommands(
 	// See https://github.com/kube-vip/kube-vip/issues/684
 	needCommands, err := needHackCommands(cluster)
 	if err != nil {
-		return nil, nil, fmt.Errorf("failed to determine if kube-vip commands are needed: %w", err)
+		return nil, nil, nil, fmt.Errorf("failed to determine if kube-vip commands are needed: %w", err)
 	}
 	if !needCommands {
-		return nil, nil, nil
+		return files, nil, nil, nil
 	}
 
-	return KubeVIPPreKubeadmCommands, KubeVIPPostKubeadmCommands, nil
+	files = append(
+		files,
+		bootstrapv1.File{
+			Content:     string(configureKubeVIPScript),
+			Path:        configureKubeVIPScriptOnRemote,
+			Permissions: configureKubeVIPScriptPermissions,
+		},
+	)
+	preKubeadmCommands := []string{configureKubeVIPScriptOnRemotePreKubeadmCommand}
+	postKubeadmCommands := []string{configureKubeVIPScriptOnRemotePostKubeadmCommand}
+
+	return files, preKubeadmCommands, postKubeadmCommands, nil
 }
 
 type multipleKeysError struct {
diff --git a/pkg/handlers/generic/mutation/controlplanevirtualip/providers/kubevip_test.go b/pkg/handlers/generic/mutation/controlplanevirtualip/providers/kubevip_test.go
@@ -11,22 +11,69 @@ import (
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
 )
 
-func Test_GetFile(t *testing.T) {
+func Test_GenerateFilesAndCommands(t *testing.T) {
 	t.Parallel()
 
 	tests := []struct {
-		name                     string
-		controlPlaneEndpointSpec v1alpha1.ControlPlaneEndpointSpec
-		configMap                *corev1.ConfigMap
-		expectedContent          string
-		expectedErr              error
+		name                        string
+		controlPlaneEndpointSpec    v1alpha1.ControlPlaneEndpointSpec
+		cluster                     *clusterv1.Cluster
+		configMap                   *corev1.ConfigMap
+		expectedFiles               []bootstrapv1.File
+		expectedPreKubeadmCommands  []string
+		expectedPostKubeadmCommands []string
+		expectedErr                 error
 	}{
+		{
+			name: "should return templated data with both host and port and pre/post kubeadm hack commands",
+			controlPlaneEndpointSpec: v1alpha1.ControlPlaneEndpointSpec{
+				Host: "10.20.100.10",
+				Port: 6443,
+			},
+			configMap: &corev1.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "default-kube-vip-template",
+					Namespace: "default",
+				},
+				Data: map[string]string{
+					"data": validKubeVIPTemplate,
+				},
+			},
+			cluster: &clusterv1.Cluster{
+				Spec: clusterv1.ClusterSpec{
+					Topology: &clusterv1.Topology{
+						Version: "v1.29.0",
+					},
+				},
+			},
+			expectedFiles: []bootstrapv1.File{
+				{
+					Content:     expectedKubeVIPPod,
+					Owner:       kubeVIPFileOwner,
+					Path:        kubeVIPFilePath,
+					Permissions: kubeVIPFilePermissions,
+				},
+				{
+					Content:     string(configureKubeVIPScript),
+					Path:        configureKubeVIPScriptOnRemote,
+					Permissions: configureKubeVIPScriptPermissions,
+				},
+			},
+			expectedPreKubeadmCommands: []string{
+				configureKubeVIPScriptOnRemotePreKubeadmCommand,
+			},
+			expectedPostKubeadmCommands: []string{
+				configureKubeVIPScriptOnRemotePostKubeadmCommand,
+			},
+		},
 		{
 			name: "should return templated data with both host and port",
 			controlPlaneEndpointSpec: v1alpha1.ControlPlaneEndpointSpec{
@@ -42,7 +89,21 @@ func Test_GetFile(t *testing.T) {
 					"data": validKubeVIPTemplate,
 				},
 			},
-			expectedContent: expectedKubeVIPPod,
+			cluster: &clusterv1.Cluster{
+				Spec: clusterv1.ClusterSpec{
+					Topology: &clusterv1.Topology{
+						Version: "v1.28.0",
+					},
+				},
+			},
+			expectedFiles: []bootstrapv1.File{
+				{
+					Content:     expectedKubeVIPPod,
+					Owner:       kubeVIPFileOwner,
+					Path:        kubeVIPFilePath,
+					Permissions: kubeVIPFilePermissions,
+				},
+			},
 		},
 	}
 
@@ -57,12 +118,16 @@ func Test_GetFile(t *testing.T) {
 				configMapKey: client.ObjectKeyFromObject(tt.configMap),
 			}
 
-			file, err := provider.GetFile(context.TODO(), tt.controlPlaneEndpointSpec)
+			files, preKubeadmCommands, postKubeadmCommands, err :=
+				provider.GenerateFilesAndCommands(
+					context.TODO(),
+					tt.controlPlaneEndpointSpec,
+					tt.cluster,
+				)
 			require.Equal(t, tt.expectedErr, err)
-			assert.Equal(t, tt.expectedContent, file.Content)
-			assert.NotEmpty(t, file.Path)
-			assert.NotEmpty(t, file.Owner)
-			assert.NotEmpty(t, file.Permissions)
+			assert.Equal(t, tt.expectedFiles, files)
+			assert.Equal(t, tt.expectedPreKubeadmCommands, preKubeadmCommands)
+			assert.Equal(t, tt.expectedPostKubeadmCommands, postKubeadmCommands)
 		})
 	}
 }
diff --git a/pkg/handlers/generic/mutation/controlplanevirtualip/providers/providers.go b/pkg/handlers/generic/mutation/controlplanevirtualip/providers/providers.go
@@ -15,24 +15,21 @@ import (
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
 )
 
-const (
-	kubeVIPFileOwner       = "root:root"
-	kubeVIPFilePath        = "/etc/kubernetes/manifests/kube-vip.yaml"
-	kubeVIPFilePermissions = "0600"
-)
-
-// Provider is an interface for getting the kube-vip static Pod as a file.
+// Provider is an interface for getting the virtual IP provider static Pod as a file.
 type Provider interface {
 	Name() string
-	GetFile(ctx context.Context, spec v1alpha1.ControlPlaneEndpointSpec) (*bootstrapv1.File, error)
-	GetCommands(cluster *clusterv1.Cluster) ([]string, []string, error)
+	GenerateFilesAndCommands(
+		ctx context.Context,
+		spec v1alpha1.ControlPlaneEndpointSpec,
+		cluster *clusterv1.Cluster,
+	) ([]bootstrapv1.File, []string, []string, error)
 }
 
 func templateValues(
 	controlPlaneEndpoint v1alpha1.ControlPlaneEndpointSpec,
 	text string,
 ) (string, error) {
-	kubeVIPTemplate, err := template.New("").Parse(text)
+	virtualIPTemplate, err := template.New("").Parse(text)
 	if err != nil {
 		return "", fmt.Errorf("failed to parse template: %w", err)
 	}
@@ -46,7 +43,7 @@ func templateValues(
 	}
 
 	var b bytes.Buffer
-	err = kubeVIPTemplate.Execute(&b, templateInput)
+	err = virtualIPTemplate.Execute(&b, templateInput)
 	if err != nil {
 		return "", fmt.Errorf("failed setting API endpoint configuration in template: %w", err)
 	}
diff --git a/pkg/handlers/generic/mutation/controlplanevirtualip/providers/templates/configure-kube-vip.sh b/pkg/handlers/generic/mutation/controlplanevirtualip/providers/templates/configure-kube-vip.sh
diff --git a/pkg/handlers/generic/mutation/encryptionatrest/inject.go b/pkg/handlers/generic/mutation/encryptionatrest/inject.go
