Implement the NestedEtcd controller

Makefile

@@ -143,7 +143,7 @@ apidiff: $(GO_APIDIFF) ## Check for API differences
 generate: ## Generate code
 	$(MAKE) generate-manifests
 	$(MAKE) generate-go
-	$(MAKE) generate-bindata
+	# $(MAKE) generate-bindata
 
 .PHONY: generate-go
 generate-go: ## Runs Go related generate targets
@@ -163,8 +163,8 @@ generate-manifests: ## Generate manifests e.g. CRD, RBAC etc.
 		output:webhook:dir=./config/webhook \
 		webhook
 	## Copy files in CI folders.
-	cp -f ./config/rbac/*.yaml ./config/ci/rbac/
-	cp -f ./config/manager/manager*.yaml ./config/ci/manager/
+	# cp -f ./config/rbac/*.yaml ./config/ci/rbac/
+	# cp -f ./config/manager/manager*.yaml ./config/ci/manager/
 
 .PHONY: modules
 modules: ## Runs go mod to ensure modules are up to date.
@@ -184,8 +184,8 @@ docker-pull-prerequisites:
 .PHONY: docker-build
 docker-build: docker-pull-prerequisites ## Build the docker images for controller managers
 	DOCKER_BUILDKIT=1 docker build --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg ldflags="$(LDFLAGS)" . -t $(CONTROLLER_IMG)-$(ARCH):$(TAG)
-	$(MAKE) set-manifest-image MANIFEST_IMG=$(CONTROLLER_IMG)-$(ARCH) MANIFEST_TAG=$(TAG) TARGET_RESOURCE="./config/manager/manager_image_patch.yaml"
-	$(MAKE) set-manifest-pull-policy TARGET_RESOURCE="./config/manager/manager_pull_policy.yaml"
+	# $(MAKE) set-manifest-image MANIFEST_IMG=$(CONTROLLER_IMG)-$(ARCH) MANIFEST_TAG=$(TAG) TARGET_RESOURCE="./config/manager/manager_image_patch.yaml"
+	# $(MAKE) set-manifest-pull-policy TARGET_RESOURCE="./config/manager/manager_pull_policy.yaml"
 
 .PHONY: docker-push
 docker-push: ## Push the docker images

apis/controlplane/v1alpha4/nestedcomponentspec_types.go

@@ -22,19 +22,21 @@ import (
 )
 
 type NestedComponentSpec struct {
-	// NestedComponentSpec defines the common information for creating the component
+	// NestedComponentSpec defines the common information for creating the
+	// component
 	// +optional
 	addonv1alpha1.CommonSpec `json:",inline"`
 
 	// PatchSpecs includes the user specifed settings
 	// +optional
 	addonv1alpha1.PatchSpec `json:",inline"`
 
-	// Resources defines the amount of computing resources that will be used by this component
+	// Resources defines the amount of computing resources that will be used
+	// by this component
 	// +optional
-	Resources corev1.ResourceRequirements `json:"resources",omitempty`
+	Resources corev1.ResourceRequirements `json:"resources,omitempty"`
 
 	// Replicas defines the number of replicas in the component's workload
 	// +optional
-	Replicas int32 `json:"replicas",omitempty`
+	Replicas int32 `json:"replicas,omitempty"`
 }

apis/controlplane/v1alpha4/nestedetcd_types.go

@@ -21,19 +21,23 @@ import (
 	addonv1alpha1 "sigs.k8s.io/kubebuilder-declarative-pattern/pkg/patterns/addon/pkg/apis/v1alpha1"
 )
 
+type NestedEtcdPhase string
+
+const (
+	NestedEtcdReady   NestedEtcdPhase = "ready"
+	NestedEtcdUnready NestedEtcdPhase = "unready"
+)
+
 // NestedEtcdSpec defines the desired state of NestedEtcd
 type NestedEtcdSpec struct {
-	// NestedComponentSpec contains the common and user-specified information that are
-	// required for creating the component
+	// NestedComponentSpec contains the common and user-specified information
+	// that are required for creating the component
 	// +optional
 	NestedComponentSpec `json:",inline"`
 }
 
 // NestedEtcdStatus defines the observed state of NestedEtcd
 type NestedEtcdStatus struct {
-	// Ready is set if all resources have been created
-	Ready bool `json:"ready,omitempty"`
-
 	// EtcdDomain defines how to address the etcd instance
 	Addresses []NestedEtcdAddress `json:"addresses,omitempty"`
 
@@ -56,6 +60,10 @@ type NestedEtcdAddress struct {
 }
 
 //+kubebuilder:object:root=true
+//+kubebuilder:resource:scope=Namespaced,path=nestedetcds,shortName=netcd
+//+kubebuilder:categories=capi,capn
+//+kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase"
+//+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
 //+kubebuilder:subresource:status
 
 // NestedEtcd is the Schema for the nestedetcds API

config/component-templates/nested-etcd/nested-etcd-service-template.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{.nestedetcdName}}
+  namespace: {{.nestedetcdNamespace}}
+  labels:
+    component-name: {{.nestedetcdName}} 
+spec:
+  publishNotReadyAddresses: true
+  clusterIP: None
+  selector:
+    component-name: {{.nestedetcdName}}

config/component-templates/nested-etcd/nested-etcd-statefulset-template.yaml

@@ -0,0 +1,87 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{.nestedetcdName}}
+  namespace: {{.nestedetcdNamespace}}
+spec:
+  replicas: {{.nestedetcdStsReplicas}}
+  revisionHistoryLimit: 10
+  serviceName: {{.nestedetcdName}}
+  selector:
+    matchLabels:
+      component-name: {{.nestedetcdName}}
+  # etcd will not be updated, unless it is deleted
+  updateStrategy:
+    type: OnDelete
+  template:
+    metadata:
+      labels:
+        component-name: {{.nestedetcdName}}
+    spec:
+      subdomain: etcd
+      containers:
+      - name: {{.nestedetcdName}}
+        image: virtualcluster/etcd-v3.4.0
+        imagePullPolicy: Always
+        command:
+        - etcd
+        # pass the pod name(hostname) to container for composing the advertise-urls args
+        env:
+        - name: HOSTNAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        args:
+        - --name=$(HOSTNAME)
+        - --trusted-ca-file=/etc/kubernetes/pki/root/tls.crt
+        - --client-cert-auth
+        - --cert-file=/etc/kubernetes/pki/etcd/tls.crt
+        - --key-file=/etc/kubernetes/pki/etcd/tls.key
+        - --peer-client-cert-auth
+        - --peer-trusted-ca-file=/etc/kubernetes/pki/root/tls.crt
+        - --peer-cert-file=/etc/kubernetes/pki/etcd/tls.crt
+        - --peer-key-file=/etc/kubernetes/pki/etcd/tls.key
+        - --listen-peer-urls=https://0.0.0.0:2380
+        - --listen-client-urls=https://0.0.0.0:2379
+        - --initial-advertise-peer-urls=https://$(HOSTNAME).{{.nestedetcdName}}:2380
+        # we use a headless service to encapsulate each pod
+        - --advertise-client-urls=https://$(HOSTNAME).{{.nestedetcdName}}:2379
+        - --initial-cluster-state=new
+        - --initial-cluster-token=vc-etcd
+        - --data-dir=/var/lib/etcd/data
+        # --initial-cluster option will be set during runtime based on the number of replicas
+        livenessProbe:
+          exec:
+            command:
+            - sh
+            - -c
+            - ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/root/tls.crt --cert=/etc/kubernetes/pki/etcd/tls.crt --key=/etc/kubernetes/pki/etcd/tls.key endpoint health
+          failureThreshold: 8
+          initialDelaySeconds: 60
+          timeoutSeconds: 15
+        readinessProbe:
+          exec:
+            command:
+            - sh
+            - -c
+            - ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/root/tls.crt --cert=/etc/kubernetes/pki/etcd/tls.crt --key=/etc/kubernetes/pki/etcd/tls.key endpoint health
+          failureThreshold: 8
+          initialDelaySeconds: 15
+          periodSeconds: 2
+          timeoutSeconds: 15
+        volumeMounts:
+        - mountPath: /etc/kubernetes/pki/etcd
+          name: {{.nestedControlPlaneName}}-etcd-client-cert
+          readOnly: true
+        - mountPath: /etc/kubernetes/pki/root
+          name: {{.nestedControlPlaneName}}-etcd
+          readOnly: true
+      volumes:
+      - name: {{.nestedControlPlaneName}}-etcd-client-cert
+        secret:
+          defaultMode: 420
+          secretName: {{.nestedControlPlaneName}}-etcd-client-cert
+      - name: {{.nestedControlPlaneName}}-etcd
+        secret:
+          defaultMode: 420
+          secretName: {{.nestedControlPlaneName}}-etcd

config/crd/bases/controlplane.cluster.x-k8s.io_nestedetcds.yaml

@@ -13,10 +13,19 @@ spec:
     kind: NestedEtcd
     listKind: NestedEtcdList
     plural: nestedetcds
+    shortNames:
+    - netcd
     singular: nestedetcd
   scope: Namespaced
   versions:
-  - name: v1alpha4
+  - additionalPrinterColumns:
+    - jsonPath: .status.phase
+      name: Phase
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1alpha4
     schema:
       openAPIV3Schema:
         description: NestedEtcd is the Schema for the nestedetcds API
@@ -97,9 +106,6 @@ spec:
                 type: boolean
               phase:
                 type: string
-              ready:
-                description: Ready is set if all resources have been created
-                type: boolean
             required:
             - healthy
             type: object