Add additional support for external etcd

Supporting external etcd should be as easy as possible. It was very
difficult before this patch to support external etcd.

Signed-off-by: Chuck Ha <[email protected]>

Author: chuckha

cloudinit/cloudinit_test.go

@@ -46,7 +46,7 @@ func TestNewInitControlPlaneAdditionalFileEncodings(t *testing.T) {
 			Users:      nil,
 			NTP:        nil,
 		},
-		Certificates:         cluster.NewCertificates(),
+		Certificates:         cluster.Certificates{},
 		ClusterConfiguration: "my-cluster-config",
 		InitConfiguration:    "my-init-config",
 	}

cloudinit/controlplane_init.go

@@ -52,10 +52,6 @@ type ControlPlaneInput struct {
 // NewInitControlPlane returns the user data string to be used on a controlplane instance.
 func NewInitControlPlane(input *ControlPlaneInput) ([]byte, error) {
 	input.Header = cloudConfigHeader
-	if err := input.Certificates.EnsureAllExist(); err != nil {
-		return nil, err
-	}
-
 	input.WriteFiles = input.Certificates.AsFiles()
 	input.WriteFiles = append(input.WriteFiles, input.AdditionalFiles...)
 	userData, err := generate("InitControlplane", controlPlaneCloudInit, input)

cloudinit/controlplane_join.go

@@ -50,10 +50,7 @@ type ControlPlaneJoinInput struct {
 // NewJoinControlPlane returns the user data string to be used on a new control plane instance.
 func NewJoinControlPlane(input *ControlPlaneJoinInput) ([]byte, error) {
 	input.Header = cloudConfigHeader
-	if err := input.Certificates.EnsureAllExist(); err != nil {
-		return nil, err
-	}
-
+	// TODO: Consider validating that the correct certificates exist. It is different for external/stacked etcd
 	input.WriteFiles = input.Certificates.AsFiles()
 	input.WriteFiles = append(input.WriteFiles, input.AdditionalFiles...)
 	userData, err := generate("JoinControlplane", controlPlaneJoinCloudInit, input)

controllers/kubeadmconfig_controller.go

@@ -227,7 +227,7 @@ func (r *KubeadmConfigReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, re
 			return ctrl.Result{}, err
 		}
 
-		certificates := internalcluster.NewCertificates()
+		certificates := internalcluster.NewCertificatesForControlPlane(config.Spec.ClusterConfiguration)
 		if err := certificates.LookupOrGenerate(ctx, r.Client, cluster, config); err != nil {
 			log.Error(err, "unable to lookup or create cluster certificates")
 			return ctrl.Result{}, err
@@ -271,7 +271,7 @@ func (r *KubeadmConfigReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, re
 		}
 	}
 
-	certificates := internalcluster.NewCertificates()
+	certificates := internalcluster.NewCertificatesForWorker(config.Spec.JoinConfiguration.CACertPath)
 	if err := certificates.Lookup(ctx, r.Client, cluster); err != nil {
 		log.Error(err, "unable to lookup cluster certificates")
 		return ctrl.Result{}, err

controllers/kubeadmconfig_controller_test.go

@@ -1267,7 +1267,10 @@ func newControlPlaneInitKubeadmConfig(machine *clusterv1.Machine, name string) *
 
 func createSecrets(t *testing.T, cluster *clusterv1.Cluster, owner *bootstrapv1.KubeadmConfig) []runtime.Object {
 	out := []runtime.Object{}
-	certificates := internalcluster.NewCertificates()
+	if owner.Spec.ClusterConfiguration == nil {
+		owner.Spec.ClusterConfiguration = &kubeadmv1beta1.ClusterConfiguration{}
+	}
+	certificates := internalcluster.NewCertificatesForControlPlane(owner.Spec.ClusterConfiguration)
 	if err := certificates.Generate(); err != nil {
 		t.Fatal(err)
 	}

docs/external-etcd.md

@@ -0,0 +1,77 @@
+# Support for external etcd
+
+Cluster API Bootstrap Provider Kubeadm  supports using an external etcd cluster for your workload Kubernetes clusters.
+
+### ⚠️ Warnings ⚠️
+
+Before getting started you should be aware of the expectations that come with using an external etcd cluster.
+
+* Cluster API is unable to manage any aspect of the external etcd cluster.
+* Depending on how you configure your etcd nodes you may incur additional cloud costs in data transfer.
+    * As an example, cross availability zone traffic can cost money on cloud providers. You don't have to deploy etcd
+    across availability zones, but if you do please be aware of the costs.
+
+### Getting started
+
+To use this, you will need to create an etcd cluster and generate an apiserver-etcd-client key/pair.
+[`etcdadm`](https://github.com/kubernetes-sigs/etcdadm) is a good way to get started if you'd like to test this
+behavior.
+
+Once you create an etcd cluster, you will want to base64 encode the `/etc/etcd/pki/apiserver-etcd-client.crt`,
+`/etc/etcd/pki/apiserver-etcd-client.key`, and `/etc/etcd/pki/server.crt` files and put them in two secrets. The secrets
+must be formatted as follows and the cert material must be base64 encoded:
+
+```yaml
+# Kubernetes APIServer etcd client certificate
+kind: Secret
+apiVersion: v1
+metadata:
+  name: $CLUSTER_NAME-apiserver-etcd-client
+  namespace: $CLUSTER_NAMESPACE
+data:
+  tls.crt: |
+    LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURCRENDQWV5Z0F3SUJBZ0lJZFlkclZUMzV0
+    NW93RFFZSktvWklodmNOQVFFTEJRQXdEekVOTUFzR0ExVUUKQXhNRVpYUmpaREFlRncweE9UQTVN
+    ...
+  tls.key: |
+    LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBdlFlTzVKOE5j
+    VCtDeGRubFR3alpuQ3YwRzByY0tETklhZzlSdFdrZ1p4MEcxVm1yClA4Zy9BRkhXVHdxSTUrNi81
+    ...
+```
+
+```yaml
+# Etcd's CA crt file to validate the generated client certificates
+kind: Secret
+apiVersion: v1
+metadata:
+  name: $CLUSTER_NAME-etcd
+  namespace: $CLUSTER_NAMESPACE
+data:
+  tls.crt: |
+    LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURBRENDQWVpZ0F3SUJBZ0lJRDNrVVczaDIy
+    K013RFFZSktvWklodmNOQVFFTEJRQXdEekVOTUFzR0ExVUUKQXhNRVpYUmpaREFlRncweE9UQTVN
+    ...
+```
+
+After that the rest is standard Kubeadm. Config your ClusterConfiguration as follows:
+
+```yaml
+apiVersion: bootstrap.cluster.x-k8s.io/v1alpha2
+kind: KubeadmConfig
+metadata:
+  name: CLUSTER_NAME-controlplane-0
+  namespace: CLUSTER_NAMESPACE
+spec:
+  ... # initConfiguration goes here
+  clusterConfiguration:
+    etcd:
+      external:
+        endpoints:
+          - https://10.0.0.230:2379
+        caFile: /etc/kubernetes/pki/etcd/ca.crt
+        certFile: /etc/kubernetes/pki/apiserver-etcd-client.crt
+        keyFile: /etc/kubernetes/pki/apiserver-etcd-client.key
+    ... # other clusterConfiguration goes here
+```
+
+Create your cluster as normal!

internal/cluster/certificates.go

@@ -25,6 +25,7 @@ import (
 	"crypto/x509/pkix"
 	"encoding/hex"
 	"math/big"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -34,6 +35,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/util/cert"
 	bootstrapv1 "sigs.k8s.io/cluster-api-bootstrap-provider-kubeadm/api/v1alpha2"
+	"sigs.k8s.io/cluster-api-bootstrap-provider-kubeadm/kubeadm/v1beta1"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha2"
 	"sigs.k8s.io/cluster-api/util/certs"
 	"sigs.k8s.io/cluster-api/util/secret"
@@ -51,6 +53,11 @@ const (
 
 	// FrontProxyCA is the secret name suffix for Front Proxy CA
 	FrontProxyCA secret.Purpose = "proxy"
+
+	// APIServerEtcdClient is the secret name of user-supplied secret containing the apiserver-etcd-client key/cert
+	APIServerEtcdClient secret.Purpose = "apiserver-etcd-client"
+
+	defaultCertificatesDir = "/etc/kubernetes/pki"
 )
 
 var (
@@ -67,13 +74,65 @@ var (
 // Certificates are the certificates necessary to bootstrap a cluster.
 type Certificates []*Certificate
 
-// NewCertificates return an initialized but empty set of CA certificates needed to bootstrap a cluster.
-func NewCertificates() Certificates {
+// NewCertificatesForControlPlane returns a list of certificates configured for a control plane node
+func NewCertificatesForControlPlane(config *v1beta1.ClusterConfiguration) Certificates {
+	if config.CertificatesDir == "" {
+		config.CertificatesDir = defaultCertificatesDir
+	}
+
+	certificates := Certificates{
+		&Certificate{
+			Purpose:  secret.ClusterCA,
+			CertFile: filepath.Join(config.CertificatesDir, "ca.crt"),
+			KeyFile:  filepath.Join(config.CertificatesDir, "ca.key"),
+		},
+		&Certificate{
+			Purpose:  ServiceAccount,
+			CertFile: filepath.Join(config.CertificatesDir, "sa.pub"),
+			KeyFile:  filepath.Join(config.CertificatesDir, "sa.key"),
+		},
+		&Certificate{
+			Purpose:  FrontProxyCA,
+			CertFile: filepath.Join(config.CertificatesDir, "front-proxy-ca.crt"),
+			KeyFile:  filepath.Join(config.CertificatesDir, "front-proxy-ca.key"),
+		},
+	}
+
+	etcdCert := &Certificate{
+		Purpose:  EtcdCA,
+		CertFile: filepath.Join(config.CertificatesDir, "etcd", "ca.crt"),
+		KeyFile:  filepath.Join(config.CertificatesDir, "etcd", "ca.key"),
+	}
+
+	// TODO make sure all the fields are actually defined and return an error if not
+	if config.Etcd.External != nil {
+		etcdCert = &Certificate{
+			Purpose:  EtcdCA,
+			CertFile: config.Etcd.External.CAFile,
+		}
+		apiserverEtcdClientCert := &Certificate{
+			Purpose:  APIServerEtcdClient,
+			CertFile: config.Etcd.External.CertFile,
+			KeyFile:  config.Etcd.External.KeyFile,
+		}
+		certificates = append(certificates, apiserverEtcdClientCert)
+	}
+
+	certificates = append(certificates, etcdCert)
+	return certificates
+}
+
+// NewCertificatesForWorker return an initialized but empty set of CA certificates needed to bootstrap a cluster.
+func NewCertificatesForWorker(caCertPath string) Certificates {
+	if caCertPath == "" {
+		caCertPath = filepath.Join(defaultCertificatesDir, "ca.crt")
+	}
+
 	return Certificates{
-		&Certificate{Purpose: secret.ClusterCA},
-		&Certificate{Purpose: EtcdCA},
-		&Certificate{Purpose: ServiceAccount},
-		&Certificate{Purpose: FrontProxyCA},
+		&Certificate{
+			Purpose:  secret.ClusterCA,
+			CertFile: caCertPath,
+		},
 	}
 }
 
@@ -138,6 +197,8 @@ func (c Certificates) Generate() error {
 		if certificate.KeyPair == nil {
 			var generator certGenerator
 			switch certificate.Purpose {
+			case APIServerEtcdClient: // Do not generate the APIServerEtcdClient key pair. It is user supplied
+				continue
 			case ServiceAccount:
 				generator = generateServiceAccountKeys
 			default:
@@ -191,9 +252,10 @@ func (c Certificates) LookupOrGenerate(ctx context.Context, ctrlclient client.Cl
 
 // Certificate represents a single certificate CA.
 type Certificate struct {
-	Generated bool
-	Purpose   secret.Purpose
-	KeyPair   *certs.KeyPair
+	Generated         bool
+	Purpose           secret.Purpose
+	KeyPair           *certs.KeyPair
+	CertFile, KeyFile string
 }
 
 // Hashes hashes all the certificates stored in a CA certificate.
@@ -244,63 +306,56 @@ func (c *Certificate) AsSecret(cluster *clusterv1.Cluster, config *bootstrapv1.K
 	return s
 }
 
+// AsFiles converts the certificate to a slice of Files that may have 0, 1 or 2 Files.
+func (c *Certificate) AsFiles() []bootstrapv1.File {
+	out := make([]bootstrapv1.File, 0)
+	if len(c.KeyPair.Cert) > 0 {
+		out = append(out, bootstrapv1.File{
+			Path:        c.CertFile,
+			Owner:       rootOwnerValue,
+			Permissions: "0640",
+			Content:     string(c.KeyPair.Cert),
+		})
+	}
+	if len(c.KeyPair.Key) > 0 {
+		out = append(out, bootstrapv1.File{
+			Path:        c.KeyFile,
+			Owner:       rootOwnerValue,
+			Permissions: "0600",
+			Content:     string(c.KeyPair.Key),
+		})
+	}
+	return out
+}
+
 // AsFiles converts a slice of certificates into bootstrap files.
 func (c Certificates) AsFiles() []bootstrapv1.File {
 	clusterCA := c.GetByPurpose(secret.ClusterCA)
 	etcdCA := c.GetByPurpose(EtcdCA)
 	frontProxyCA := c.GetByPurpose(FrontProxyCA)
 	serviceAccountKey := c.GetByPurpose(ServiceAccount)
 
-	return []bootstrapv1.File{
-		{
-			Path:        "/etc/kubernetes/pki/ca.crt",
-			Owner:       rootOwnerValue,
-			Permissions: "0640",
-			Content:     string(clusterCA.KeyPair.Cert),
-		},
-		{
-			Path:        "/etc/kubernetes/pki/ca.key",
-			Owner:       rootOwnerValue,
-			Permissions: "0600",
-			Content:     string(clusterCA.KeyPair.Key),
-		},
-		{
-			Path:        "/etc/kubernetes/pki/etcd/ca.crt",
-			Owner:       rootOwnerValue,
-			Permissions: "0640",
-			Content:     string(etcdCA.KeyPair.Cert),
-		},
-		{
-			Path:        "/etc/kubernetes/pki/etcd/ca.key",
-			Owner:       rootOwnerValue,
-			Permissions: "0600",
-			Content:     string(etcdCA.KeyPair.Key),
-		},
-		{
-			Path:        "/etc/kubernetes/pki/front-proxy-ca.crt",
-			Owner:       rootOwnerValue,
-			Permissions: "0640",
-			Content:     string(frontProxyCA.KeyPair.Cert),
-		},
-		{
-			Path:        "/etc/kubernetes/pki/front-proxy-ca.key",
-			Owner:       rootOwnerValue,
-			Permissions: "0600",
-			Content:     string(frontProxyCA.KeyPair.Key),
-		},
-		{
-			Path:        "/etc/kubernetes/pki/sa.pub",
-			Owner:       rootOwnerValue,
-			Permissions: "0640",
-			Content:     string(serviceAccountKey.KeyPair.Cert),
-		},
-		{
-			Path:        "/etc/kubernetes/pki/sa.key",
-			Owner:       rootOwnerValue,
-			Permissions: "0600",
-			Content:     string(serviceAccountKey.KeyPair.Key),
-		},
+	certFiles := make([]bootstrapv1.File, 0)
+	if clusterCA != nil {
+		certFiles = append(certFiles, clusterCA.AsFiles()...)
+	}
+	if etcdCA != nil {
+		certFiles = append(certFiles, etcdCA.AsFiles()...)
+	}
+	if frontProxyCA != nil {
+		certFiles = append(certFiles, frontProxyCA.AsFiles()...)
+	}
+	if serviceAccountKey != nil {
+		certFiles = append(certFiles, serviceAccountKey.AsFiles()...)
 	}
+
+	// these will only exist if external etcd was defined and supplied by the user
+	apiserverEtcdClientCert := c.GetByPurpose(APIServerEtcdClient)
+	if apiserverEtcdClientCert != nil {
+		certFiles = append(certFiles, apiserverEtcdClientCert.AsFiles()...)
+	}
+
+	return certFiles
 }
 
 func secretToKeyPair(s *corev1.Secret) (*certs.KeyPair, error) {
@@ -309,9 +364,11 @@ func secretToKeyPair(s *corev1.Secret) (*certs.KeyPair, error) {
 		return nil, errors.Errorf("missing data for key %s", secret.TLSCrtDataName)
 	}
 
+	// In some cases (external etcd) it's ok if the etcd.key does not exist.
+	// TODO: some other function should ensure that the certificates we need exist.
 	key, exists := s.Data[secret.TLSKeyDataName]
 	if !exists {
-		return nil, errors.Errorf("missing data for key %s", secret.TLSKeyDataName)
+		key = []byte("")
 	}
 
 	return &certs.KeyPair{