diff --git a/Dockerfile b/Dockerfile index 2cd3bbdb..f5de2bc1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,14 +45,15 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ --mount=type=cache,target=/root/.local/share/golang \ CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} go build -ldflags "${LDFLAGS} -extldflags '-static'" -o manager ${package} +RUN curl -L https://dl.k8s.io/release/v1.22.0/bin/linux/amd64/kubeadm -o kubeadm && chmod +x kubeadm ENTRYPOINT [ "/start.sh", "/workspace/manager" ] # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details -FROM gcr.io/distroless/static:nonroot +FROM gcr.io/distroless/static-debian10 # Copy the controller-manager into a thin image WORKDIR / COPY --from=builder /workspace/manager . -COPY controlplane/nested/component-templates/ ./component-templates/ -USER 65532:65532 +COPY --from=builder /workspace/kubeadm kubeadm +# USER 65532:65532 ENTRYPOINT ["/manager"] diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 4295b194..ed14fec7 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -24,4 +24,5 @@ commonLabels: # capi system can cross reference the proper controlplane & infra refs # https://cluster-api.sigs.k8s.io/developer/providers/v1alpha2-to-v1alpha3.html#apply-the-contract-version-label-clusterx-k8sioversion-version1_version2_version3-to-your-crds cluster.x-k8s.io/v1alpha3: v1alpha4 - cluster.x-k8s.io/v1alpha4: v1alpha4 \ No newline at end of file + cluster.x-k8s.io/v1alpha4: v1alpha4 + cluster.x-k8s.io/v1beta1: v1alpha4 diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 84d5775a..f46b06d7 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -48,3 +48,14 @@ rules: - get - patch - update +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - patch + - update + - list + - create + - delete diff --git a/config/samples/infrastructure_v1alpha4_nestedcluster.yaml b/config/samples/infrastructure_v1alpha4_nestedcluster.yaml index d27a72a0..20d47cfd 100644 --- a/config/samples/infrastructure_v1alpha4_nestedcluster.yaml +++ b/config/samples/infrastructure_v1alpha4_nestedcluster.yaml @@ -2,6 +2,8 @@ apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 kind: NestedCluster metadata: name: nestedcluster-sample + labels: + cluster.x-k8s.io/v1beta1: v1alpha4_v1beta1 spec: controlPlaneEndpoint: host: "localhost" diff --git a/config/samples/v1alpha4_cluster.yaml b/config/samples/v1alpha4_cluster.yaml index 1a67d446..0f55e7d8 100644 --- a/config/samples/v1alpha4_cluster.yaml +++ b/config/samples/v1alpha4_cluster.yaml @@ -19,4 +19,4 @@ spec: apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 kind: NestedCluster name: nestedcluster-sample - namespace: default \ No newline at end of file + namespace: default diff --git a/controlplane/nested/component-templates/nested-apiserver/nested-apiserver-service-template.yaml b/controlplane/nested/component-templates/nested-apiserver/nested-apiserver-service-template.yaml deleted file mode 100644 index 0c7fcf20..00000000 --- a/controlplane/nested/component-templates/nested-apiserver/nested-apiserver-service-template.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{.clusterName}}-apiserver - namespace: {{.componentNamespace}} - labels: - component-name: {{.componentName}} -spec: - selector: - component-name: {{.componentName}} - type: ClusterIP - ports: - - name: api - port: 6443 - protocol: TCP - targetPort: api diff --git a/controlplane/nested/component-templates/nested-apiserver/nested-apiserver-statefulset-template.yaml b/controlplane/nested/component-templates/nested-apiserver/nested-apiserver-statefulset-template.yaml deleted file mode 100644 index 7aebe3ea..00000000 --- a/controlplane/nested/component-templates/nested-apiserver/nested-apiserver-statefulset-template.yaml +++ /dev/null @@ -1,149 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: {{.clusterName}}-apiserver - namespace: {{.componentNamespace}} -spec: - revisionHistoryLimit: 10 - serviceName: {{.componentName}} - selector: - matchLabels: - component-name: {{.componentName}} - # apiserver will not be updated, unless it is deleted - updateStrategy: - type: OnDelete - template: - metadata: - labels: - component-name: {{.componentName}} - spec: - hostname: apiserver - subdomain: {{.clusterName}}-apiserver - containers: - - name: {{.componentName}} - image: virtualcluster/apiserver-v1.16.2 - imagePullPolicy: Always - command: - - kube-apiserver - env: - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - args: - - --bind-address=0.0.0.0 - - --allow-privileged=true - - --anonymous-auth=true - - --client-ca-file=/etc/kubernetes/pki/apiserver/ca/tls.crt - - --tls-cert-file=/etc/kubernetes/pki/apiserver/tls.crt - - --tls-private-key-file=/etc/kubernetes/pki/apiserver/tls.key - - --kubelet-https=true - - --kubelet-certificate-authority=/etc/kubernetes/pki/apiserver/ca/tls.crt - - --kubelet-client-certificate=/etc/kubernetes/pki/kubelet/tls.crt - - --kubelet-client-key=/etc/kubernetes/pki/kubelet/tls.key - - --kubelet-preferred-address-types=InternalIP,ExternalIP - - --enable-bootstrap-token-auth=true - - --etcd-servers=https://{{.clusterName}}-etcd-0.{{.clusterName}}-etcd.$(NAMESPACE):2379 - - --etcd-cafile=/etc/kubernetes/pki/etcd/ca/tls.crt - - --etcd-certfile=/etc/kubernetes/pki/etcd/tls.crt - - --etcd-keyfile=/etc/kubernetes/pki/etcd/tls.key - - --service-account-key-file=/etc/kubernetes/pki/service-account/tls.key - - --service-cluster-ip-range=10.32.0.0/16 - - --service-node-port-range=30000-32767 - - --authorization-mode=Node,RBAC - - --runtime-config=api/all - - --enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota - - --apiserver-count=1 - - --endpoint-reconciler-type=master-count - - --enable-aggregator-routing=true - - --requestheader-client-ca-file=/etc/kubernetes/pki/proxy/ca/tls.crt - - --requestheader-allowed-names=front-proxy-client - - --requestheader-username-headers=X-Remote-User - - --requestheader-group-headers=X-Remote-Group - - --requestheader-extra-headers-prefix=X-Remote-Extra- - - --proxy-client-key-file=/etc/kubernetes/pki/proxy/tls.key - - --proxy-client-cert-file=/etc/kubernetes/pki/proxy/tls.crt - - --v=2 - ports: - - containerPort: 6443 - protocol: TCP - name: api - livenessProbe: - # since we set anonymous-auth to false, we use tcp instead of https - tcpSocket: - port: 6443 - failureThreshold: 8 - initialDelaySeconds: 15 - periodSeconds: 10 - timeoutSeconds: 15 - readinessProbe: - httpGet: - port: 6443 - path: /healthz - scheme: HTTPS - failureThreshold: 8 - initialDelaySeconds: 5 - periodSeconds: 2 - timeoutSeconds: 30 - volumeMounts: - - mountPath: /etc/kubernetes/pki/proxy/ca - name: {{.clusterName}}-proxy-client-ca - readOnly: true - - mountPath: /etc/kubernetes/pki/proxy - name: {{.clusterName}}-proxy-client - readOnly: true - - mountPath: /etc/kubernetes/pki/etcd/ca - name: {{.clusterName}}-etcd-ca - readOnly: true - - mountPath: /etc/kubernetes/pki/etcd - name: {{.clusterName}}-etcd-client - readOnly: true - - mountPath: /etc/kubernetes/pki/apiserver/ca - name: {{.clusterName}}-ca - readOnly: true - - mountPath: /etc/kubernetes/pki/apiserver - name: {{.clusterName}}-apiserver-client - readOnly: true - - mountPath: /etc/kubernetes/pki/kubelet - name: {{.clusterName}}-kubelet-client - readOnly: true - - mountPath: /etc/kubernetes/pki/service-account - name: {{.clusterName}}-sa - readOnly: true - terminationGracePeriodSeconds: 30 - dnsConfig: - searches: - - cluster.local - volumes: - - name: {{.clusterName}}-apiserver-client - secret: - defaultMode: 420 - secretName: {{.clusterName}}-apiserver-client - - name: {{.clusterName}}-etcd-ca - secret: - defaultMode: 420 - secretName: {{.clusterName}}-etcd - - name: {{.clusterName}}-etcd-client - secret: - defaultMode: 420 - secretName: {{.clusterName}}-etcd-client - - name: {{.clusterName}}-proxy-client-ca - secret: - defaultMode: 420 - secretName: {{.clusterName}}-proxy - - name: {{.clusterName}}-proxy-client - secret: - defaultMode: 420 - secretName: {{.clusterName}}-proxy-client - - name: {{.clusterName}}-kubelet-client - secret: - defaultMode: 420 - secretName: {{.clusterName}}-kubelet-client - - name: {{.clusterName}}-ca - secret: - defaultMode: 420 - secretName: {{.clusterName}}-ca - - name: {{.clusterName}}-sa - secret: - defaultMode: 420 - secretName: {{.clusterName}}-sa diff --git a/controlplane/nested/component-templates/nested-controllermanager/nested-controllermanager-statefulset-template.yaml b/controlplane/nested/component-templates/nested-controllermanager/nested-controllermanager-statefulset-template.yaml deleted file mode 100644 index c48176ef..00000000 --- a/controlplane/nested/component-templates/nested-controllermanager/nested-controllermanager-statefulset-template.yaml +++ /dev/null @@ -1,91 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: {{.clusterName}}-controller-manager - namespace: {{.componentNamespace}} -spec: - selector: - matchLabels: - component-name: {{.componentName}} - updateStrategy: - type: OnDelete - template: - metadata: - labels: - component-name: {{.componentName}} - spec: - containers: - - name: {{.componentName}} - image: virtualcluster/controller-manager-v1.16.2 - imagePullPolicy: Always - command: - - kube-controller-manager - args: - - --bind-address=0.0.0.0 - - --cluster-cidr=10.200.0.0/16 - - --cluster-signing-cert-file=/etc/kubernetes/pki/root/tls.crt - - --cluster-signing-key-file=/etc/kubernetes/pki/root/tls.key - - --kubeconfig=/etc/kubernetes/kubeconfig/controller-manager-kubeconfig - - --authorization-kubeconfig=/etc/kubernetes/kubeconfig/controller-manager-kubeconfig - - --authentication-kubeconfig=/etc/kubernetes/kubeconfig/controller-manager-kubeconfig - # control plane contains only one instance for now - - --leader-elect=false - - --root-ca-file=/etc/kubernetes/pki/root/ca/tls.crt - - --service-account-private-key-file=/etc/kubernetes/pki/service-account/tls.key - - --service-cluster-ip-range=10.32.0.0/24 - - --use-service-account-credentials=true - - --experimental-cluster-signing-duration=87600h - - --node-monitor-grace-period=200s - - --controllers=*,-nodelifecycle - - --v=2 - livenessProbe: - httpGet: - path: /healthz - port: 10252 - scheme: HTTP - failureThreshold: 8 - initialDelaySeconds: 15 - periodSeconds: 10 - timeoutSeconds: 15 - readinessProbe: - httpGet: - port: 10252 - path: /healthz - scheme: HTTP - failureThreshold: 8 - initialDelaySeconds: 15 - periodSeconds: 2 - timeoutSeconds: 15 - volumeMounts: - - mountPath: /etc/kubernetes/pki/root/ca - name: {{.clusterName}}-ca - readOnly: true - - mountPath: /etc/kubernetes/pki/root - name: {{.clusterName}}-apiserver-client - readOnly: true - - mountPath: /etc/kubernetes/pki/service-account - name: {{.clusterName}}-sa - readOnly: true - - mountPath: /etc/kubernetes/kubeconfig - name: {{.clusterName}}-kubeconfig - readOnly: true - volumes: - - name: {{.clusterName}}-ca - secret: - defaultMode: 420 - secretName: {{.clusterName}}-ca - - name: {{.clusterName}}-apiserver-client - secret: - defaultMode: 420 - secretName: {{.clusterName}}-apiserver-client - - name: {{.clusterName}}-sa - secret: - defaultMode: 420 - secretName: {{.clusterName}}-sa - - name: {{.clusterName}}-kubeconfig - secret: - defaultMode: 420 - secretName: {{.clusterName}}-kubeconfig - items: - - key: value - path: controller-manager-kubeconfig diff --git a/controlplane/nested/component-templates/nested-etcd/nested-etcd-service-template.yaml b/controlplane/nested/component-templates/nested-etcd/nested-etcd-service-template.yaml deleted file mode 100644 index 58cd080b..00000000 --- a/controlplane/nested/component-templates/nested-etcd/nested-etcd-service-template.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{.clusterName}}-etcd - namespace: {{.componentNamespace}} - labels: - component-name: {{.componentName}} -spec: - publishNotReadyAddresses: true - clusterIP: None - selector: - component-name: {{.componentName}} diff --git a/controlplane/nested/component-templates/nested-etcd/nested-etcd-statefulset-template.yaml b/controlplane/nested/component-templates/nested-etcd/nested-etcd-statefulset-template.yaml deleted file mode 100644 index 95c53009..00000000 --- a/controlplane/nested/component-templates/nested-etcd/nested-etcd-statefulset-template.yaml +++ /dev/null @@ -1,96 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: {{.clusterName}}-etcd - namespace: {{.componentNamespace}} -spec: - revisionHistoryLimit: 10 - serviceName: {{.clusterName}}-etcd - selector: - matchLabels: - component-name: {{.componentName}} - # etcd will not be updated, unless it is deleted - updateStrategy: - type: OnDelete - template: - metadata: - labels: - component-name: {{.componentName}} - spec: - containers: - - name: {{.componentName}} - image: virtualcluster/etcd-v3.4.0 - imagePullPolicy: Always - command: - - etcd - # pass the pod name(hostname) to container for composing the advertise-urls args - env: - - name: HOSTNAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - args: - - --name=$(HOSTNAME) - - --trusted-ca-file=/etc/kubernetes/pki/ca/tls.crt - - --client-cert-auth - - --cert-file=/etc/kubernetes/pki/etcd/tls.crt - - --key-file=/etc/kubernetes/pki/etcd/tls.key - - --peer-client-cert-auth - - --peer-trusted-ca-file=/etc/kubernetes/pki/ca/tls.crt - - --peer-cert-file=/etc/kubernetes/pki/etcd/tls.crt - - --peer-key-file=/etc/kubernetes/pki/etcd/tls.key - - --listen-peer-urls=https://0.0.0.0:2380 - - --listen-client-urls=https://0.0.0.0:2379 - - --initial-advertise-peer-urls=https://$(HOSTNAME).{{.clusterName}}-etcd.$(NAMESPACE):2380 - # we use a headless service to encapsulate each pod - - --advertise-client-urls=https://$(HOSTNAME).{{.clusterName}}-etcd.$(NAMESPACE):2379 - - --initial-cluster-state=new - - --initial-cluster-token=vc-etcd - - --data-dir=/var/lib/etcd/data - # --initial-cluster option will be set during runtime based on the number of replicas - livenessProbe: - exec: - command: - - sh - - -c - - ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/ca/tls.crt --cert=/etc/kubernetes/pki/health/tls.crt --key=/etc/kubernetes/pki/health/tls.key endpoint health - failureThreshold: 8 - initialDelaySeconds: 60 - timeoutSeconds: 15 - readinessProbe: - exec: - command: - - sh - - -c - - ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/ca/tls.crt --cert=/etc/kubernetes/pki/health/tls.crt --key=/etc/kubernetes/pki/health/tls.key endpoint health - failureThreshold: 8 - initialDelaySeconds: 15 - periodSeconds: 2 - timeoutSeconds: 15 - volumeMounts: - - mountPath: /etc/kubernetes/pki/ca - name: {{.clusterName}}-etcd-ca - readOnly: true - - mountPath: /etc/kubernetes/pki/etcd - name: {{.clusterName}}-etcd-client - readOnly: true - - mountPath: /etc/kubernetes/pki/health - name: {{.clusterName}}-etcd-health-client - readOnly: true - volumes: - - name: {{.clusterName}}-etcd-ca - secret: - defaultMode: 420 - secretName: {{.clusterName}}-etcd - - name: {{.clusterName}}-etcd-client - secret: - defaultMode: 420 - secretName: {{.clusterName}}-etcd-client - - name: {{.clusterName}}-etcd-health-client - secret: - defaultMode: 420 - secretName: {{.clusterName}}-etcd-health-client diff --git a/controlplane/nested/config/crd/kustomization.yaml b/controlplane/nested/config/crd/kustomization.yaml index 0e41c51a..23fa547b 100644 --- a/controlplane/nested/config/crd/kustomization.yaml +++ b/controlplane/nested/config/crd/kustomization.yaml @@ -27,4 +27,5 @@ commonLabels: # capi system can cross reference the proper controlplane & infra refs # https://cluster-api.sigs.k8s.io/developer/providers/v1alpha2-to-v1alpha3.html#apply-the-contract-version-label-clusterx-k8sioversion-version1_version2_version3-to-your-crds cluster.x-k8s.io/v1alpha3: v1alpha4 - cluster.x-k8s.io/v1alpha4: v1alpha4 \ No newline at end of file + cluster.x-k8s.io/v1alpha4: v1alpha4 + cluster.x-k8s.io/v1beta1: v1alpha4 diff --git a/controlplane/nested/config/rbac/role.yaml b/controlplane/nested/config/rbac/role.yaml index c21cafa5..fa3b11ef 100644 --- a/controlplane/nested/config/rbac/role.yaml +++ b/controlplane/nested/config/rbac/role.yaml @@ -9,6 +9,7 @@ rules: - apiGroups: - "" resources: + - configmaps - secrets verbs: - create diff --git a/controlplane/nested/controllers/consts.go b/controlplane/nested/controllers/consts.go index d5cb2bce..7f30c560 100644 --- a/controlplane/nested/controllers/consts.go +++ b/controlplane/nested/controllers/consts.go @@ -19,12 +19,14 @@ limitations under the License. package controllers const ( - statefulsetOwnerKeyNEtcd = ".metadata.netcd.controller" - statefulsetOwnerKeyNKas = ".metadata.nkas.controller" - statefulsetOwnerKeyNKcm = ".metadata.nkcm.controller" - defaultEtcdStatefulSetURL = "/nested-etcd/nested-etcd-statefulset-template.yaml" - defaultEtcdServiceURL = "/nested-etcd/nested-etcd-service-template.yaml" - defaultKASStatefulSetURL = "/nested-apiserver/nested-apiserver-statefulset-template.yaml" - defaultKASServiceURL = "/nested-apiserver/nested-apiserver-service-template.yaml" - defaultKCMStatefulSetURL = "/nested-controllermanager/nested-controllermanager-statefulset-template.yaml" + statefulsetOwnerKeyNEtcd = ".metadata.netcd.controller" + statefulsetOwnerKeyNKas = ".metadata.nkas.controller" + statefulsetOwnerKeyNKcm = ".metadata.nkcm.controller" + // KASManifestConfigmapName is the key name of the apiserver manifest in the configmap. + KASManifestConfigmapName = "nkas-manifest" + // KCMManifestConfigmapName is the key name of the controller-manager manifest in the configmap. + KCMManifestConfigmapName = "nkcm-manifest" + // EtcdManifestConfigmapName is the key name of the etcd manifest in the configmap. + EtcdManifestConfigmapName = "netcd-manifest" + loopbackAddress = "127.0.0.1" ) diff --git a/controlplane/nested/controllers/controller_util.go b/controlplane/nested/controllers/controller_util.go index 499133aa..12b6ee6c 100644 --- a/controlplane/nested/controllers/controller_util.go +++ b/controlplane/nested/controllers/controller_util.go @@ -20,23 +20,26 @@ import ( "bytes" "context" "fmt" - "io/ioutil" + "strings" "text/template" - openuri "github.com/utahta/go-openuri" - "github.com/go-logr/logr" + "github.com/pkg/errors" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/runtime/serializer/json" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/kubernetes/scheme" ctrlcli "sigs.k8s.io/controller-runtime/pkg/client" controlplanev1 "sigs.k8s.io/cluster-api-provider-nested/controlplane/nested/api/v1alpha4" + "sigs.k8s.io/cluster-api-provider-nested/controlplane/nested/kubeadm" addonv1alpha1 "sigs.k8s.io/kubebuilder-declarative-pattern/pkg/patterns/addon/pkg/apis/v1alpha1" ) @@ -48,31 +51,21 @@ import ( func createNestedComponentSts(ctx context.Context, cli ctrlcli.Client, ncMeta metav1.ObjectMeta, ncSpec controlplanev1.NestedComponentSpec, - ncKind controlplanev1.ComponentKind, - controlPlaneName, clusterName, templatePath string, log logr.Logger) error { - ncSts := &appsv1.StatefulSet{} - ncSvc := &corev1.Service{} - // Setup the ownerReferences for all objects + ncKind, clusterName string, log logr.Logger) error { + // setup the ownerReferences for all objects or := metav1.NewControllerRef(&ncMeta, - controlplanev1.GroupVersion.WithKind(string(ncKind))) - - // 1. Using the template defined by version/channel to create the - // StatefulSet and the Service - // TODO check the template version/channel, if not set, use the default. - if ncSpec.Version != "" && ncSpec.Channel != "" { - panic("NOT IMPLEMENT YET") - } + controlplanev1.GroupVersion.WithKind(ncKind)) - log.V(4).Info("The Version and Channel are not set, " + - "will use the default template.") - if err := genStatefulSetObject(templatePath, ncMeta, ncSpec, ncKind, controlPlaneName, clusterName, log, ncSts); err != nil { - return fmt.Errorf("fail to generate the Statefulset object: %v", err) + ncSts, err := genStatefulSetObject(cli, ncMeta, ncSpec, ncKind, clusterName, log) + if err != nil { + return errors.Errorf("fail to generate the Statefulset object: %v", err) } - if ncKind != controlplanev1.ControllerManager { + if ncKind != kubeadm.ControllerManager { // no need to create the service for the NestedControllerManager - if err := genServiceObject(templatePath, ncMeta, ncSpec, ncKind, controlPlaneName, clusterName, log, ncSvc); err != nil { - return fmt.Errorf("fail to generate the Service object: %v", err) + ncSvc, err := genServiceObject(ncKind, clusterName, ncMeta.GetName(), ncMeta.GetNamespace()) + if err != nil { + return errors.Errorf("fail to generate the Service object: %v", err) } ncSvc.SetOwnerReferences([]metav1.OwnerReference{*or}) @@ -83,101 +76,140 @@ func createNestedComponentSts(ctx context.Context, "component", ncKind) } - // 2. set the NestedComponent object as the owner of the StatefulSet + // set the NestedComponent object as the owner of the StatefulSet ncSts.SetOwnerReferences([]metav1.OwnerReference{*or}) - // 4. create the NestedComponent StatefulSet + // create the NestedComponent StatefulSet return cli.Create(ctx, ncSts) } -// genServiceObject generates the Service object corresponding to the -// NestedComponent. -func genServiceObject( - templatePath string, - ncMeta metav1.ObjectMeta, - ncSpec controlplanev1.NestedComponentSpec, ncKind controlplanev1.ComponentKind, - controlPlaneName, clusterName string, log logr.Logger, svc *corev1.Service) error { - var templateURL string - if ncSpec.Version == "" && ncSpec.Channel == "" { - switch ncKind { - case controlplanev1.APIServer: - templateURL = templatePath + defaultKASServiceURL - case controlplanev1.Etcd: - templateURL = templatePath + defaultEtcdServiceURL - default: - panic("Unreachable") - } - } else { - panic("NOT IMPLEMENT YET") - } - svcTmpl, err := fetchTemplate(templateURL) - if err != nil { - return fmt.Errorf("fail to fetch the default template "+ - "for the %s service: %v", ncKind, err) +// genServiceObject generates the Service object corresponding to the NestedComponent. +func genServiceObject(ncKind, clusterName, componentName, componentNamespace string) (*corev1.Service, error) { + switch ncKind { + case kubeadm.APIServer: + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName + "-apiserver", + Namespace: componentNamespace, + Labels: map[string]string{ + "component-name": componentName, + }, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{ + "component-name": componentName, + }, + Type: corev1.ServiceTypeClusterIP, + Ports: []corev1.ServicePort{ + { + Name: "api", + Port: 6443, + Protocol: corev1.ProtocolTCP, + TargetPort: intstr.FromString("api"), + }, + }, + }, + }, nil + case kubeadm.Etcd: + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName + "-etcd", + Namespace: componentNamespace, + Labels: map[string]string{ + "component-name": componentName, + }, + }, + Spec: corev1.ServiceSpec{ + PublishNotReadyAddresses: true, + ClusterIP: "None", + Selector: map[string]string{ + "component-name": componentName, + }, + }, + }, nil + default: + return nil, errors.Errorf("unknown component type: %s", ncKind) } +} - templateCtx := getTemplateArgs(ncMeta, controlPlaneName, clusterName) +// objectToYaml serialize the runtime object to the yaml. +func objectToYaml(obj runtime.Object) ([]byte, error) { + serializer := json.NewYAMLSerializer(json.DefaultMetaFactory, nil, nil) + buf := bytes.NewBuffer([]byte{}) + if err := serializer.Encode(obj, buf); err != nil { + return nil, err + } + return buf.Bytes(), nil +} - svcStr, err := substituteTemplate(templateCtx, svcTmpl) - if err != nil { - return fmt.Errorf("fail to substitute the default template "+ - "for the nestedetcd Service: %v", err) +// genStatefulSetManifest complete the pod spec and use it to generate the +// statefulset manifest. +func genStatefulSetManifest(podManifest, ncKind, clusterName, componentName, componentNamespace string) (*appsv1.StatefulSet, error) { + pod := corev1.Pod{} + if err := yamlToObject([]byte(podManifest), &pod); err != nil { + return nil, errors.Errorf("fail to convert yaml file to pod: %v", err) } - if err := yamlToObject([]byte(svcStr), svc); err != nil { - return fmt.Errorf("fail to convert yaml file to Serivce: %v", err) + switch ncKind { + case kubeadm.APIServer: + case kubeadm.Etcd: + case kubeadm.ControllerManager: + default: + return nil, errors.Errorf("invalid component type: %s", ncKind) } - log.Info("deserialize yaml to runtime object(Service)") - - return nil + return &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName + "-" + ncKind, + Namespace: componentNamespace, + }, + Spec: appsv1.StatefulSetSpec{ + ServiceName: clusterName + "-" + ncKind, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "component-name": componentName, + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "component-name": componentName, + }, + }, + Spec: pod.Spec, + }, + }, + }, nil } -// genStatefulSetObject generates the StatefulSet object corresponding to the -// NestedComponent. +// genStatefulSetObject generates the StatefulSet object corresponding to the NestedComponent. func genStatefulSetObject( - templatePath string, + cli ctrlcli.Client, ncMeta metav1.ObjectMeta, ncSpec controlplanev1.NestedComponentSpec, - ncKind controlplanev1.ComponentKind, controlPlaneName, clusterName string, - log logr.Logger, ncSts *appsv1.StatefulSet) error { - var templateURL string - if ncSpec.Version == "" && ncSpec.Channel == "" { - log.V(4).Info("The Version and Channel are not set, " + - "will use the default template.") - switch ncKind { - case controlplanev1.APIServer: - templateURL = templatePath + defaultKASStatefulSetURL - case controlplanev1.Etcd: - templateURL = templatePath + defaultEtcdStatefulSetURL - case controlplanev1.ControllerManager: - templateURL = templatePath + defaultKCMStatefulSetURL - default: - panic("Unreachable") + ncKind, clusterName string, + log logr.Logger) (*appsv1.StatefulSet, error) { + cm := corev1.ConfigMap{} + if err := cli.Get(context.TODO(), types.NamespacedName{ + Namespace: ncMeta.Namespace, + Name: clusterName + "-" + kubeadm.ManifestsConfigmapSuffix, + }, &cm); err != nil { + if apierrors.IsNotFound(err) { + log.Error(err, "manifests configmap not found") } - } else { - panic("NOT IMPLEMENT YET") + return nil, err } - // 1 fetch the statefulset template - stsTmpl, err := fetchTemplate(templateURL) - if err != nil { - return fmt.Errorf("fail to fetch the default template "+ - "for the %s StatefulSet: %v", ncKind, err) + // 1. get the pod spec + podManifest := cm.Data[ncKind] + if podManifest == "" { + return nil, errors.Errorf("data %s is not found", ncKind) } - // 2 substitute the statefulset template - templateCtx := getTemplateArgs(ncMeta, controlPlaneName, clusterName) - stsStr, err := substituteTemplate(templateCtx, stsTmpl) + // 2. generate the statefulset manifest + ncSts, err := genStatefulSetManifest(podManifest, ncKind, clusterName, ncMeta.GetName(), ncMeta.GetNamespace()) if err != nil { - return fmt.Errorf("fail to substitute the default template "+ - "for the %s StatefulSet: %v", ncKind, err) - } - // 3 deserialize the yaml string to the StatefulSet object - - if err := yamlToObject([]byte(stsStr), ncSts); err != nil { - return fmt.Errorf("fail to convert yaml file to StatefulSet: %v", err) + return nil, errors.Wrap(err, "failed to generate the statefulset manifest") } - log.V(5).Info("deserialize yaml to runtime object(StatefulSet)") - // 5 apply NestedComponent.Spec.Resources and NestedComponent.Spec.Replicas + // 3. apply NestedComponent.Spec.Resources and NestedComponent.Spec.Replicas // to the NestedComponent StatefulSet for i := range ncSts.Spec.Template.Spec.Containers { ncSts.Spec.Template.Spec.Containers[i].Resources = @@ -190,26 +222,15 @@ func genStatefulSetObject( "Replicas fields are set", "StatefulSet", ncSts.GetName()) - // 6 set the "--initial-cluster" command line flag for the Etcd container - if ncKind == controlplanev1.Etcd { + // 4. set the "--initial-cluster" command line flag for the Etcd container + if ncKind == kubeadm.Etcd { icaVal := genInitialClusterArgs(1, clusterName, clusterName, ncMeta.GetNamespace()) - stsArgs := append(ncSts.Spec.Template.Spec.Containers[0].Args, - "--initial-cluster", icaVal) - ncSts.Spec.Template.Spec.Containers[0].Args = stsArgs + ncSts.Spec.Template.Spec.Containers[0].Command = append( + ncSts.Spec.Template.Spec.Containers[0].Command, + fmt.Sprintf("--initial-cluster=%s", icaVal)) log.V(5).Info("The '--initial-cluster' command line option is set") } - - // 7 TODO validate the patch and apply it to the template. - return nil -} - -func getTemplateArgs(ncMeta metav1.ObjectMeta, controlPlaneName, clusterName string) map[string]string { - return map[string]string{ - "componentName": ncMeta.GetName(), - "componentNamespace": ncMeta.GetNamespace(), - "clusterName": clusterName, - "controlPlaneName": controlPlaneName, - } + return ncSts, nil } // yamlToObject deserialize the yaml to the runtime object. @@ -238,22 +259,6 @@ func substituteTemplate(context interface{}, tmpl string) (string, error) { return writer.String(), nil } -// fetchTemplate fetches the component template through the tmplateURL. -func fetchTemplate(templateURL string) (string, error) { - rep, err := openuri.Open(templateURL) - if err != nil { - return "", err - } - defer rep.Close() - - bodyBytes, err := ioutil.ReadAll(rep) - if err != nil { - return "", err - } - - return string(bodyBytes), nil -} - // getOwner gets the ownerreference of the NestedComponent. func getOwner(ncMeta metav1.ObjectMeta) metav1.OwnerReference { owners := ncMeta.GetOwnerReferences() @@ -302,3 +307,371 @@ func genObjRefFromObj(obj ctrlcli.Object) corev1.ObjectReference { func IsComponentReady(status addonv1alpha1.CommonStatus) bool { return status.Phase == string(controlplanev1.Ready) } + +// createManifestsConfigMap create the configmap that holds the manifests of +// the NestedComponent. NOTE this function will be deprecated once the +// nestedmachine_controller is implemented. +func createManifestsConfigMap(cli ctrlcli.Client, manifests map[string]corev1.Pod, clusterName, namespace string) error { + data := map[string]string{} + for name, pod := range manifests { + tmpPod := pod + podYaml, err := objectToYaml(&tmpPod) + if err != nil { + return err + } + data[name] = string(podYaml) + } + cm := corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: clusterName + "-" + kubeadm.ManifestsConfigmapSuffix, + }, + Data: data, + } + return cli.Create(context.TODO(), &cm) +} + +// completeTemplates completes the pod templates of nested control plane +// components. +func completeTemplates(templates map[string]string, clusterName string) (map[string]corev1.Pod, error) { + var ret map[string]corev1.Pod = make(map[string]corev1.Pod) + for name, podTemplate := range templates { + pod := corev1.Pod{} + if err := yamlToObject([]byte(podTemplate), &pod); err != nil { + return nil, err + } + switch name { + case kubeadm.APIServer: + ret[kubeadm.APIServer] = completeKASPodSpec(pod, clusterName) + case kubeadm.ControllerManager: + ret[kubeadm.ControllerManager] = completeKCMPodSpec(pod, clusterName) + case kubeadm.Etcd: + ret[kubeadm.Etcd] = completeEtcdPodSpec(pod, clusterName) + default: + return nil, errors.New("unknown component: " + name) + } + } + return ret, nil +} + +// completeKASPodSpec sets volumes, envs and other fields for the kube-apiserver pod spec. +func completeKASPodSpec(pod corev1.Pod, clusterName string) corev1.Pod { + ps := pod.Spec + pod.Spec.DNSConfig = &corev1.PodDNSConfig{ + Searches: []string{"cluster.local"}, + } + ps.Hostname = kubeadm.APIServer + ps.Subdomain = clusterName + "-apiserver" + ps.Containers[0].Env = []corev1.EnvVar{ + { + Name: "NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.namespace", + }, + }, + }, + } + ps.Containers[0].VolumeMounts = []corev1.VolumeMount{ + { + MountPath: "/etc/kubernetes/pki/etcd/ca", + Name: clusterName + "-etcd-ca", + ReadOnly: true, + }, + { + MountPath: "/etc/kubernetes/pki/etcd", + Name: clusterName + "-etcd-client", + ReadOnly: true, + }, + { + MountPath: "/etc/kubernetes/pki/apiserver/ca", + Name: clusterName + "-ca", + ReadOnly: true, + }, + { + MountPath: "/etc/kubernetes/pki/apiserver", + Name: clusterName + "-apiserver-client", + ReadOnly: true, + }, + { + MountPath: "/etc/kubernetes/pki/kubelet", + Name: clusterName + "-kubelet-client", + ReadOnly: true, + }, + { + MountPath: "/etc/kubernetes/pki/service-account", + Name: clusterName + "-sa", + ReadOnly: true, + }, + { + MountPath: "/etc/kubernetes/pki/proxy/ca", + Name: clusterName + "-proxy", + ReadOnly: true, + }, + { + MountPath: "/etc/kubernetes/pki/proxy", + Name: clusterName + "-proxy-client", + ReadOnly: true, + }, + } + ps.Containers[0].Ports = append(ps.Containers[0].Ports, corev1.ContainerPort{ + Name: "api", + ContainerPort: 6443, + }) + ps.Containers[0].LivenessProbe.Handler.HTTPGet.Host = loopbackAddress + ps.Containers[0].ReadinessProbe.Handler.HTTPGet.Host = loopbackAddress + ps.Containers[0].StartupProbe.Handler.HTTPGet.Host = loopbackAddress + + // disable the hostnetwork + ps.HostNetwork = false + var volSrtMode int32 = 420 + ps.Volumes = []corev1.Volume{ + { + Name: clusterName + "-apiserver-client", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &volSrtMode, + SecretName: clusterName + "-apiserver-client", + }, + }, + }, + { + Name: clusterName + "-etcd-ca", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &volSrtMode, + SecretName: clusterName + "-etcd", + }, + }, + }, + { + Name: clusterName + "-etcd-client", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &volSrtMode, + SecretName: clusterName + "-etcd-client", + }, + }, + }, + { + Name: clusterName + "-kubelet-client", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &volSrtMode, + SecretName: clusterName + "-kubelet-client", + }, + }, + }, + { + Name: clusterName + "-ca", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &volSrtMode, + SecretName: clusterName + "-ca", + }, + }, + }, + { + Name: clusterName + "-sa", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &volSrtMode, + SecretName: clusterName + "-sa", + }, + }, + }, + { + Name: clusterName + "-proxy", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &volSrtMode, + SecretName: clusterName + "-proxy", + }, + }, + }, + { + Name: clusterName + "-proxy-client", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &volSrtMode, + SecretName: clusterName + "-proxy-client", + }, + }, + }, + } + pod.Spec = ps + return pod +} + +// completeKCMPodSpec sets volumes and other fields for the kube-controller-manager pod spec. +func completeKCMPodSpec(pod corev1.Pod, clusterName string) corev1.Pod { + ps := pod.Spec + ps.Containers[0].VolumeMounts = []corev1.VolumeMount{ + { + MountPath: "/etc/kubernetes/pki/root/ca", + Name: clusterName + "-ca", + ReadOnly: true, + }, + { + MountPath: "/etc/kubernetes/pki/root", + Name: clusterName + "-apiserver-client", + ReadOnly: true, + }, + { + MountPath: "/etc/kubernetes/pki/service-account", + Name: clusterName + "-sa", + ReadOnly: true, + }, + { + MountPath: "/etc/kubernetes/pki/proxy/ca", + Name: clusterName + "-proxy", + ReadOnly: true, + }, + { + MountPath: "/etc/kubernetes/kubeconfig", + Name: clusterName + "-kubeconfig", + ReadOnly: true, + }, + } + + // disable the hostnetwork + ps.HostNetwork = false + + var volSrtMode int32 = 420 + ps.Volumes = []corev1.Volume{ + { + Name: clusterName + "-ca", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &volSrtMode, + SecretName: clusterName + "-ca", + }, + }, + }, + { + Name: clusterName + "-apiserver-client", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &volSrtMode, + SecretName: clusterName + "-apiserver-client", + }, + }, + }, + { + Name: clusterName + "-sa", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &volSrtMode, + SecretName: clusterName + "-sa", + }, + }, + }, + { + Name: clusterName + "-proxy", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &volSrtMode, + SecretName: clusterName + "-proxy", + }, + }, + }, + { + Name: clusterName + "-kubeconfig", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &volSrtMode, + SecretName: clusterName + "-kubeconfig", + Items: []corev1.KeyToPath{ + { + Key: "value", + Path: "controller-manager-kubeconfig", + }, + }, + }, + }, + }, + } + pod.Spec = ps + return pod +} + +// completeEtcdPodSpec sets volumes, envs and other fields for the etcd pod spec. +func completeEtcdPodSpec(pod corev1.Pod, clusterName string) corev1.Pod { + ps := pod.Spec + ps.Containers[0].Env = []corev1.EnvVar{ + { + Name: "HOSTNAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, + { + Name: "NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.namespace", + }, + }, + }, + } + ps.Containers[0].VolumeMounts = []corev1.VolumeMount{ + { + MountPath: "/etc/kubernetes/pki/ca", + Name: clusterName + "-etcd-ca", + ReadOnly: true, + }, + { + MountPath: "/etc/kubernetes/pki/etcd", + Name: clusterName + "-etcd-client", + ReadOnly: true, + }, + { + MountPath: "/etc/kubernetes/pki/health", + Name: clusterName + "-etcd-health-client", + ReadOnly: true, + }, + } + // remove the --initial-cluster option + for i, cmd := range ps.Containers[0].Command { + if strings.Contains(cmd, "initial-cluster") { + ps.Containers[0].Command = append(ps.Containers[0].Command[:i], ps.Containers[0].Command[i+1:]...) + } + } + // disable the hostnetwork + ps.HostNetwork = false + + var volSrtMode int32 = 420 + ps.Volumes = []corev1.Volume{ + { + Name: clusterName + "-etcd-ca", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &volSrtMode, + SecretName: clusterName + "-etcd", + }, + }, + }, + { + Name: clusterName + "-etcd-client", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &volSrtMode, + SecretName: clusterName + "-etcd-client", + }, + }, + }, + { + Name: clusterName + "-etcd-health-client", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &volSrtMode, + SecretName: clusterName + "-etcd-health-client", + }, + }, + }, + } + pod.Spec = ps + return pod +} diff --git a/controlplane/nested/controllers/nestedapiserver_controller.go b/controlplane/nested/controllers/nestedapiserver_controller.go index 1f4308c9..53ad9524 100644 --- a/controlplane/nested/controllers/nestedapiserver_controller.go +++ b/controlplane/nested/controllers/nestedapiserver_controller.go @@ -34,15 +34,15 @@ import ( controlplanev1 "sigs.k8s.io/cluster-api-provider-nested/controlplane/nested/api/v1alpha4" "sigs.k8s.io/cluster-api-provider-nested/controlplane/nested/certificate" + "sigs.k8s.io/cluster-api-provider-nested/controlplane/nested/kubeadm" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4" ) // NestedAPIServerReconciler reconciles a NestedAPIServer object. type NestedAPIServerReconciler struct { client.Client - Log logr.Logger - Scheme *runtime.Scheme - TemplatePath string + Log logr.Logger + Scheme *runtime.Scheme } // +kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedapiservers,verbs=get;list;watch;create;update;patch;delete @@ -113,7 +113,7 @@ func (r *NestedAPIServerReconciler) Reconcile(ctx context.Context, req ctrl.Requ // the statefulset is not found, create one. if err := createNestedComponentSts(ctx, r.Client, nkas.ObjectMeta, nkas.Spec.NestedComponentSpec, - controlplanev1.APIServer, owner.Name, cluster.GetName(), r.TemplatePath, log); err != nil { + kubeadm.APIServer, cluster.GetName(), log); err != nil { log.Error(err, "fail to create NestedAPIServer StatefulSet") return ctrl.Result{}, err } diff --git a/controlplane/nested/controllers/nestedcontrollermanager_controller.go b/controlplane/nested/controllers/nestedcontrollermanager_controller.go index 22feb217..be906011 100644 --- a/controlplane/nested/controllers/nestedcontrollermanager_controller.go +++ b/controlplane/nested/controllers/nestedcontrollermanager_controller.go @@ -30,14 +30,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" controlplanev1 "sigs.k8s.io/cluster-api-provider-nested/controlplane/nested/api/v1alpha4" + "sigs.k8s.io/cluster-api-provider-nested/controlplane/nested/kubeadm" ) // NestedControllerManagerReconciler reconciles a NestedControllerManager object. type NestedControllerManagerReconciler struct { client.Client - Log logr.Logger - Scheme *runtime.Scheme - TemplatePath string + Log logr.Logger + Scheme *runtime.Scheme } // +kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedcontrollermanagers,verbs=get;list;watch;create;update;patch;delete @@ -104,7 +104,7 @@ func (r *NestedControllerManagerReconciler) Reconcile(ctx context.Context, req c // the statefulset is not found, create one if err := createNestedComponentSts(ctx, r.Client, nkcm.ObjectMeta, nkcm.Spec.NestedComponentSpec, - controlplanev1.ControllerManager, owner.Name, cluster.GetName(), r.TemplatePath, log); err != nil { + kubeadm.ControllerManager, cluster.GetName(), log); err != nil { log.Error(err, "fail to create NestedControllerManager StatefulSet") return ctrl.Result{}, err } diff --git a/controlplane/nested/controllers/nestedcontrolplane_controller.go b/controlplane/nested/controllers/nestedcontrolplane_controller.go index 38564681..9ae4753c 100644 --- a/controlplane/nested/controllers/nestedcontrolplane_controller.go +++ b/controlplane/nested/controllers/nestedcontrolplane_controller.go @@ -43,6 +43,7 @@ import ( addonv1alpha1 "sigs.k8s.io/kubebuilder-declarative-pattern/pkg/patterns/addon/pkg/apis/v1alpha1" controlplanev1 "sigs.k8s.io/cluster-api-provider-nested/controlplane/nested/api/v1alpha4" + "sigs.k8s.io/cluster-api-provider-nested/controlplane/nested/kubeadm" ) // +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters,verbs=get;list;watch @@ -50,6 +51,7 @@ import ( // +kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedcontrolplanes/status,verbs=get;update;patch // +kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedcontrollermanagers/finalizers,verbs=update // +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete. +// +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete. // NestedControlPlaneReconciler reconciles a NestedControlPlane object. type NestedControlPlaneReconciler struct { @@ -216,6 +218,25 @@ func (r *NestedControlPlaneReconciler) reconcile(ctx context.Context, log logr.L &controlplanev1.NestedControllerManager{}: ncp.Spec.ControllerManagerRef, } + // generate manifests by calling the kubeadm + templates, err := kubeadm.GenerateTemplates(log, cluster.GetName()) + if err != nil { + return ctrl.Result{}, err + } + + // complete the manifests with CAPN specific configurations + manifests, err := completeTemplates(templates, cluster.GetName()) + if err != nil { + return ctrl.Result{}, err + } + + // create the configmap that holds the manifest of each component + if err := createManifestsConfigMap(r.Client, + manifests, cluster.GetName(), + ncp.GetNamespace()); err != nil && !apierrors.IsAlreadyExists(err) { + return ctrl.Result{}, err + } + // Adopt NestedComponents in the same Namespace for component, nestedComponent := range nestedComponents { if nestedComponent != nil { diff --git a/controlplane/nested/controllers/nestedetcd_controller.go b/controlplane/nested/controllers/nestedetcd_controller.go index 19bb22b4..4cc5cd43 100644 --- a/controlplane/nested/controllers/nestedetcd_controller.go +++ b/controlplane/nested/controllers/nestedetcd_controller.go @@ -36,15 +36,15 @@ import ( controlplanev1 "sigs.k8s.io/cluster-api-provider-nested/controlplane/nested/api/v1alpha4" "sigs.k8s.io/cluster-api-provider-nested/controlplane/nested/certificate" + "sigs.k8s.io/cluster-api-provider-nested/controlplane/nested/kubeadm" "sigs.k8s.io/cluster-api/util" ) // NestedEtcdReconciler reconciles a NestedEtcd object. type NestedEtcdReconciler struct { client.Client - Log logr.Logger - Scheme *runtime.Scheme - TemplatePath string + Log logr.Logger + Scheme *runtime.Scheme } // +kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedetcds,verbs=get;list;watch;create;update;patch;delete @@ -115,7 +115,7 @@ func (r *NestedEtcdReconciler) Reconcile(ctx context.Context, req ctrl.Request) if err := createNestedComponentSts(ctx, r.Client, netcd.ObjectMeta, netcd.Spec.NestedComponentSpec, - controlplanev1.Etcd, owner.Name, cluster.GetName(), r.TemplatePath, log); err != nil { + kubeadm.Etcd, cluster.GetName(), log); err != nil { log.Error(err, "fail to create NestedEtcd StatefulSet") return ctrl.Result{}, err } diff --git a/controlplane/nested/kubeadm/consts.go b/controlplane/nested/kubeadm/consts.go new file mode 100644 index 00000000..ec77e1d5 --- /dev/null +++ b/controlplane/nested/kubeadm/consts.go @@ -0,0 +1,48 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubeadm + +const ( + // KubeadmExecPath denotes the path to the kubeadm executable. + KubeadmExecPath = "/kubeadm" + // KASManifestsPath denotes the Path to the apiserver static pod manifest. + KASManifestsPath = "/etc/kubernetes/manifests/kube-apiserver.yaml" + // KCMManifestsPath denotes the Path to the controller-manager static pod manifest. + KCMManifestsPath = "/etc/kubernetes/manifests/kube-controller-manager.yaml" + // EtcdManifestsPath denotes the Path to the etcd static pod manifest. + EtcdManifestsPath = "/etc/kubernetes/manifests/etcd.yaml" + // DefaultKubeadmConfigPath denotes the Path to the default kubeadm config. + DefaultKubeadmConfigPath = "/kubeadm.config" + // ManifestsConfigmapSuffix is the name of the configmap that will store the + // manifests of the nested components' manifests. + ManifestsConfigmapSuffix = "ncp-manifests" + // APIServer denotes the name of the apiserver. + APIServer = "apiserver" + // ControllerManager denotes the name of the controller-manager. + ControllerManager = "controller-manager" + // Etcd denotes the name of the etcd. + Etcd = "etcd" +) + +var ( + // KASSubcommand is the command that generates the apiserver manifest. + KASSubcommand = []string{"init", "phase", "control-plane", "apiserver", "--config", DefaultKubeadmConfigPath} + // KCMSubcommand is the command that generates the controller-manager manifest. + KCMSubcommand = []string{"init", "phase", "control-plane", "controller-manager", "--config", DefaultKubeadmConfigPath} + // EtcdSubcommand is the command that generates the etcd manifest. + EtcdSubcommand = []string{"init", "phase", "etcd", "local", "--config", DefaultKubeadmConfigPath} +) diff --git a/controlplane/nested/kubeadm/kubeadm.go b/controlplane/nested/kubeadm/kubeadm.go new file mode 100644 index 00000000..bcb34475 --- /dev/null +++ b/controlplane/nested/kubeadm/kubeadm.go @@ -0,0 +1,200 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package kubeadm contains functions that used to generate pod manifests +// of the nested control-plane using the kubeadm. +package kubeadm + +import ( + "bytes" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + + "github.com/go-logr/logr" + "github.com/pkg/errors" + "gopkg.in/yaml.v2" +) + +// DefaultKubeadmConfig denotes the content of the default kubeadm config. +const DefaultKubeadmConfig = ` +apiVersion: kubeadm.k8s.io/v1beta3 +certificatesDir: /etc/kubernetes/pki +clusterName: kubernetes +imageRepository: k8s.gcr.io +kind: ClusterConfiguration +kubernetesVersion: 1.21.1 +apiServer: + timeoutForControlPlane: 4m0s + extraArgs: + advertise-address: 0.0.0.0 + client-ca-file: /etc/kubernetes/pki/apiserver/ca/tls.crt + tls-cert-file: /etc/kubernetes/pki/apiserver/tls.crt + tls-private-key-file: /etc/kubernetes/pki/apiserver/tls.key + kubelet-certificate-authority: /etc/kubernetes/pki/apiserver/ca/tls.crt + kubelet-client-certificate: /etc/kubernetes/pki/kubelet/tls.crt + kubelet-client-key: /etc/kubernetes/pki/kubelet/tls.key + etcd-cafile: /etc/kubernetes/pki/etcd/ca/tls.crt + etcd-certfile: /etc/kubernetes/pki/etcd/tls.crt + etcd-keyfile: /etc/kubernetes/pki/etcd/tls.key + service-account-key-file: /etc/kubernetes/pki/service-account/tls.key + service-account-signing-key-file: /etc/kubernetes/pki/service-account/tls.key + proxy-client-cert-file: /etc/kubernetes/pki/proxy/tls.crt + proxy-client-key-file: /etc/kubernetes/pki/proxy/tls.key + requestheader-client-ca-file: /etc/kubernetes/pki/proxy/ca/tls.crt + +controllerManager: + extraArgs: + bind-address: 0.0.0.0 + cluster-signing-cert-file: /etc/kubernetes/pki/root/tls.crt + cluster-signing-key-file: /etc/kubernetes/pki/root/tls.key + kubeconfig: /etc/kubernetes/kubeconfig/controller-manager-kubeconfig + authorization-kubeconfig: /etc/kubernetes/kubeconfig/controller-manager-kubeconfig + authentication-kubeconfig: /etc/kubernetes/kubeconfig/controller-manager-kubeconfig + leader-elect: "false" + requestheader-client-ca-file: /etc/kubernetes/pki/proxy/ca/tls.crt + client-ca-file: "" + root-ca-file: /etc/kubernetes/pki/root/ca/tls.crt + service-account-private-key-file: /etc/kubernetes/pki/service-account/tls.key + controllers: "*,-nodelifecycle,bootstrapsigner,tokencleaner" + +etcd: + local: + dataDir: /var/lib/etcd + extraArgs: + trusted-ca-file: /etc/kubernetes/pki/ca/tls.crt + client-cert-auth: "true" + cert-file: /etc/kubernetes/pki/etcd/tls.crt + key-file: /etc/kubernetes/pki/etcd/tls.key + peer-client-cert-auth: "true" + peer-trusted-ca-file: /etc/kubernetes/pki/ca/tls.crt + peer-cert-file: /etc/kubernetes/pki/etcd/tls.crt + peer-key-file: /etc/kubernetes/pki/etcd/tls.key + listen-peer-urls: https://0.0.0.0:2380 + listen-client-urls: https://0.0.0.0:2379 + name: $(HOSTNAME) + data-dir: /var/lib/etcd/data` + +func execCommand(log logr.Logger, command string, subcommand ...string) error { + cmd := exec.Command(command, subcommand...) + var ( + out bytes.Buffer + stderr bytes.Buffer + ) + cmd.Stdout = &out + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + log.Error(err, "fail to run kubeadm", "stderr", stderr.String()) + return err + } + log.Info("successfully execute the kubeadm command", "stdout", out.String()) + return nil +} + +// GenerateTemplates generates the manifests for the nested apiserver, +// controller-manager and etcd by calling the `kubeadm init phase control-plane/etcd`. +func GenerateTemplates(log logr.Logger, clusterName string) (map[string]string, error) { + // create the cluster manifests directory if not exist + if err := os.MkdirAll("/"+clusterName, 0755); err != nil { + return nil, errors.Wrap(err, "fail to create the cluster manifests directory") + } + // defer os.RemoveAll("/" + clusterName) + log.Info("cluster manifests directory is created") + if err := generateKubeadmConfig(clusterName); err != nil { + return nil, err + } + log.Info("kubeadmconfig is generated") + + // store manifests of different nested cluster in different directory + KASSubcommand = append(KASSubcommand, "--rootfs", "/"+clusterName) + KCMSubcommand = append(KCMSubcommand, "--rootfs", "/"+clusterName) + EtcdSubcommand = append(EtcdSubcommand, "--rootfs", "/"+clusterName) + // generate the manifests + if err := execCommand(log, KubeadmExecPath, KASSubcommand...); err != nil { + return nil, errors.Wrap(err, "fail to generate the apiserver manifests") + } + if err := execCommand(log, KubeadmExecPath, KCMSubcommand...); err != nil { + return nil, errors.Wrap(err, "fail to generate the controller-manager manifests") + } + if err := execCommand(log, KubeadmExecPath, EtcdSubcommand...); err != nil { + return nil, errors.Wrap(err, "fail to generate the etcd manifests") + } + log.Info("static pod manifests generated") + + var loadErr error + kasPath := filepath.Join("/", clusterName, KASManifestsPath) + KASManifests, loadErr := ioutil.ReadFile(filepath.Clean(kasPath)) + if loadErr != nil { + return nil, errors.Wrap(loadErr, "fail to load the apiserver manifests") + } + kcmPath := filepath.Join("/", clusterName, KCMManifestsPath) + KCMManifests, loadErr := ioutil.ReadFile(filepath.Clean(kcmPath)) + if loadErr != nil { + return nil, errors.Wrap(loadErr, "fail to load the controller-manager manifests") + } + etcdPath := filepath.Join("/", clusterName, EtcdManifestsPath) + EtcdManifests, loadErr := ioutil.ReadFile(filepath.Clean(etcdPath)) + if loadErr != nil { + return nil, errors.Wrap(loadErr, "fail to load the etcd manifests") + } + + return map[string]string{ + APIServer: string(KASManifests), + ControllerManager: string(KCMManifests), + Etcd: string(EtcdManifests), + }, nil +} + +// generateKubeadmConfig writes the DefaultKubeadmConfig to the DefaultKubeadmConfigPath, +// which will be read by the `kubeadm init` command. +func generateKubeadmConfig(clusterName string) error { + completedKubeadmConfig, err := completeDefaultKubeadmConfig(clusterName) + if err != nil { + return err + } + return os.WriteFile("/"+clusterName+DefaultKubeadmConfigPath, []byte(completedKubeadmConfig), 0600) +} + +func completeDefaultKubeadmConfig(clusterName string) (string, error) { + config := make(map[string]interface{}) + if err := yaml.Unmarshal([]byte(DefaultKubeadmConfig), &config); err != nil { + return "", err + } + kasConfig := config["apiServer"].(map[interface{}]interface{}) + kasExtraConfig, ok := kasConfig["extraArgs"].(map[interface{}]interface{}) + if !ok { + return "", errors.Errorf("fail to assert apiServer.extraArgs to map") + } + kasExtraConfig["etcd-servers"] = "https://" + clusterName + "-etcd-0." + clusterName + "-etcd.$(NAMESPACE):2379" + + etcdConfig := config["etcd"].(map[interface{}]interface{}) + etcdLocalConfig, ok := etcdConfig["local"].(map[interface{}]interface{}) + if !ok { + return "", errors.Errorf("fail to assert etcd.local to map") + } + etcdExtraConfig, ok := etcdLocalConfig["extraArgs"].(map[interface{}]interface{}) + if !ok { + return "", errors.Errorf("fail to assert etcd.local.extraArgs to map") + } + etcdExtraConfig["initial-advertise-peer-urls"] = "https://$(HOSTNAME)." + clusterName + "-etcd.$(NAMESPACE).svc:2380" + etcdExtraConfig["advertise-client-urls"] = "https://$(HOSTNAME)." + clusterName + "-etcd.$(NAMESPACE).svc:2379" + completedConfigYaml, err := yaml.Marshal(config) + if err != nil { + return "", err + } + return string(completedConfigYaml), nil +} diff --git a/controlplane/nested/main.go b/controlplane/nested/main.go index 892f3093..2a65d6dd 100644 --- a/controlplane/nested/main.go +++ b/controlplane/nested/main.go @@ -55,7 +55,6 @@ var ( syncPeriod time.Duration webhookPort int healthAddr string - templatePath string ) func init() { @@ -97,9 +96,6 @@ func InitFlags(fs *pflag.FlagSet) { fs.StringVar(&healthAddr, "health-addr", ":9440", "The address the health endpoint binds to.") - fs.StringVar(&templatePath, "template-path", "/component-templates", - "The address the health endpoint binds to.") - feature.MutableGates.AddFlag(fs) } @@ -159,30 +155,27 @@ func main() { } if err = (&controllers.NestedEtcdReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("controlplane").WithName("NestedEtcd"), - Scheme: mgr.GetScheme(), - TemplatePath: templatePath, + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("controlplane").WithName("NestedEtcd"), + Scheme: mgr.GetScheme(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "NestedEtcd") os.Exit(1) } if err = (&controllers.NestedAPIServerReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("controlplane").WithName("NestedAPIServer"), - Scheme: mgr.GetScheme(), - TemplatePath: templatePath, + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("controlplane").WithName("NestedAPIServer"), + Scheme: mgr.GetScheme(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "NestedAPIServer") os.Exit(1) } if err = (&controllers.NestedControllerManagerReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("controlplane").WithName("NestedControllerManager"), - Scheme: mgr.GetScheme(), - TemplatePath: templatePath, + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("controlplane").WithName("NestedControllerManager"), + Scheme: mgr.GetScheme(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "NestedControllerManager") os.Exit(1) diff --git a/go.mod b/go.mod index 87c1c1e3..d189e337 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/spf13/pflag v1.0.5 github.com/utahta/go-openuri v0.1.0 + gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/api v0.21.3 k8s.io/apimachinery v0.21.3 k8s.io/client-go v0.21.3