use rootfs to store manifests of different controlplane into different directory

Author: charleszheng44

Dockerfile

@@ -50,8 +50,7 @@ ENTRYPOINT [ "/start.sh", "/workspace/manager" ]
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
-# FROM gcr.io/distroless/static:nonroot
-FROM ubuntu
+FROM gcr.io/distroless/static-debian10
 # Copy the controller-manager into a thin image
 WORKDIR /
 COPY --from=builder /workspace/manager .

controlplane/nested/kubeadm/kubeadm.go

@@ -19,6 +19,7 @@ limitations under the License.
 package kubeadm
 
 import (
+	"bytes"
 	"io/ioutil"
 	"os"
 	"os/exec"
@@ -87,40 +88,62 @@ etcd:
       name:                        $(HOSTNAME)
       data-dir:                    /var/lib/etcd/data`
 
-func execCommand(command string, subcommand ...string) error {
+func execCommand(log logr.Logger, command string, subcommand ...string) error {
 	cmd := exec.Command(command, subcommand...)
-	return cmd.Run()
+	var (
+		out    bytes.Buffer
+		stderr bytes.Buffer
+	)
+	cmd.Stdout = &out
+	cmd.Stderr = &stderr
+	if err := cmd.Run(); err != nil {
+		log.Error(err, "fail to run kubeadm", "stderr", stderr.String())
+		return err
+	}
+	log.Info("successfully execute the kubeadm command", "stdout", out.String())
+	return nil
 }
 
 // GenerateTemplates generates the manifests for the nested apiserver,
 // controller-manager and etcd by calling the `kubeadm init phase control-plane/etcd`.
 func GenerateTemplates(log logr.Logger, clusterName string) (map[string]string, error) {
+	// create the cluster manifests directory if not exist
+	if err := os.MkdirAll("/"+clusterName, 0755); err != nil {
+		errors.Wrap(err, "fail to create the cluster manifests directory")
+	}
+	// defer os.RemoveAll("/" + clusterName)
+	log.Info("cluster manifests directory is created")
 	if err := generateKubeadmConfig(clusterName); err != nil {
 		return nil, err
 	}
 	log.Info("kubeadmconfig is generated")
+
+	// store manifests of different nested cluster in different directory
+	KASSubcommand = append(KASSubcommand, "--rootfs", "/"+clusterName)
+	KCMSubcommand = append(KCMSubcommand, "--rootfs", "/"+clusterName)
+	EtcdSubcommand = append(EtcdSubcommand, "--rootfs", "/"+clusterName)
 	// generate the manifests
-	if err := execCommand(KubeadmExecPath, KASSubcommand...); err != nil {
+	if err := execCommand(log, KubeadmExecPath, KASSubcommand...); err != nil {
 		return nil, errors.Wrap(err, "fail to generate the apiserver manifests")
 	}
-	if err := execCommand(KubeadmExecPath, KCMSubcommand...); err != nil {
+	if err := execCommand(log, KubeadmExecPath, KCMSubcommand...); err != nil {
 		return nil, errors.Wrap(err, "fail to generate the controller-manager manifests")
 	}
-	if err := execCommand(KubeadmExecPath, EtcdSubcommand...); err != nil {
+	if err := execCommand(log, KubeadmExecPath, EtcdSubcommand...); err != nil {
 		return nil, errors.Wrap(err, "fail to generate the etcd manifests")
 	}
 	log.Info("static pod manifests generated")
 
 	var loadErr error
-	KASManifests, loadErr := ioutil.ReadFile(KASManifestsPath)
+	KASManifests, loadErr := ioutil.ReadFile("/" + clusterName + KASManifestsPath)
 	if loadErr != nil {
 		return nil, errors.Wrap(loadErr, "fail to load the apiserver manifests")
 	}
-	KCMManifests, loadErr := ioutil.ReadFile(KCMManifestsPath)
+	KCMManifests, loadErr := ioutil.ReadFile("/" + clusterName + KCMManifestsPath)
 	if loadErr != nil {
 		return nil, errors.Wrap(loadErr, "fail to load the controller-manager manifests")
 	}
-	EtcdManifests, loadErr := ioutil.ReadFile(EtcdManifestsPath)
+	EtcdManifests, loadErr := ioutil.ReadFile("/" + clusterName + EtcdManifestsPath)
 	if loadErr != nil {
 		return nil, errors.Wrap(loadErr, "fail to load the etcd manifests")
 	}
@@ -139,7 +162,7 @@ func generateKubeadmConfig(clusterName string) error {
 	if err != nil {
 		return err
 	}
-	return os.WriteFile(DefaultKubeadmConfigPath, []byte(completedKubeadmConfig), 0600)
+	return os.WriteFile("/"+clusterName+DefaultKubeadmConfigPath, []byte(completedKubeadmConfig), 0600)
 }
 
 func completeDefaultKubeadmConfig(clusterName string) (string, error) {