diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..321b636e --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,36 @@ +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) + +SOURCES := $(shell find ${ROOT_DIR} -name \*.plantuml) +DIAGRAMS := $(SOURCES:%.plantuml=%.png) + +# Hosts running SELinux need :z added to volume mounts +SELINUX_ENABLED := $(shell cat /sys/fs/selinux/enforce 2> /dev/null || echo 0) + +ifeq ($(SELINUX_ENABLED),1) + DOCKER_VOL_OPTS?=:z +endif + +.PHONY: diagrams +diagrams: $(DIAGRAMS) + +%.png: %.plantuml + docker run \ + --rm \ + --volume ${ROOT_DIR}:/workdir$(DOCKER_VOL_OPTS) \ + --user $(shell id -u):$(shell id -g) \ + k8s.gcr.io/cluster-api/plantuml:1.2019.6 \ + -v /workdir/$(shell echo '$^' | sed -e 's,.*docs/,,g' ) \ No newline at end of file diff --git a/proposals/00_capn-glossary.md b/docs/proposals/00_capn-glossary.md similarity index 100% rename from proposals/00_capn-glossary.md rename to docs/proposals/00_capn-glossary.md diff --git a/proposals/20201026-creating-control-plane-components.md b/docs/proposals/20201026-creating-control-plane-components.md similarity index 98% rename from proposals/20201026-creating-control-plane-components.md rename to docs/proposals/20201026-creating-control-plane-components.md index a74436bd..d64429fb 100644 --- a/proposals/20201026-creating-control-plane-components.md +++ b/docs/proposals/20201026-creating-control-plane-components.md @@ -146,7 +146,7 @@ The component controller will create the component under the in-tree mode, which Each component's controller will generate necessary certificates for the component and store them to the [secret resources](https://cluster-api.sigs.k8s.io/tasks/certs/using-custom-certificates.html) defined by CAPI. Also, The KAS controller will store the content of the kubeconfig file in a secret named `[clustername]-kubeconfig`. -![Control Plane Creating Process](in-tree.png) +![Control Plane Creating Process](images/componentcontrollers/in-tree.png) The creating process will include six steps: @@ -164,7 +164,7 @@ The creating process will include six steps: If users intend to use an external controller to create the NCP component, they may need to implement a new component controller that can interact with the component CR and the external controller to create the component. For example, if the user wanted to use the [etcd-cluster-operator](https://github.com/improbable-eng/etcd-cluster-operator) that requires the [EtcdCluster](https://github.com/improbable-eng/etcd-cluster-operator/blob/master/api/v1alpha1/etcdcluster_types.go) CR. They need to implement a custom controller that watches the `NestedEtcd` resource, creates the necessary CRs for that implementation, and updates the required status fields on `NestedEtcd` to allow dependent services to be provisioned. This can be done using the [kubebuilder-declarative-pattern](https://github.com/kubernetes-sigs/kubebuilder-declarative-pattern) like is done for in-tree component controllers. -![Creating a Control Plane using out-of-tree provisioners](out-of-tree.png) +![Creating a Control Plane using out-of-tree provisioners](images/componentcontrollers/out-of-tree.png) In the following example, we assume that the user intends to use Etcd-cluster-operator(ECO) as the Etcd controller. The creating process will include seven steps: @@ -203,11 +203,11 @@ type NestedEtcdStatus struct { // Ready is set if all resources have been created Ready bool `json:"ready,omitempty"` - // EtcdDomain defines how to address the etcd instance + // Addresses defines how to address the etcd instance Addresses []NestedEtcdAddress `json:"addresses,omitempty"` // CommonStatus allows addons status monitoring - addonv1alpha1. CommonStatus `json:",inline"` + addonv1alpha1.CommonStatus `json:",inline"` } // EtcdAddress defines the observed addresses for etcd diff --git a/docs/proposals/20210126-nc-and-ncp.md b/docs/proposals/20210126-nc-and-ncp.md new file mode 100644 index 00000000..3774950d --- /dev/null +++ b/docs/proposals/20210126-nc-and-ncp.md @@ -0,0 +1,313 @@ +--- +title: CAPN NestedCluster & NestedControlPlane +authors: + - "@christopherhein" +reviewers: + - "@Fei-Guo" + - "@charleszheng44" + - "@salaxander" + - "@vincepri" +creation-date: 2020-10-21 +last-updated: 2021-01-26 +status: provisional +see-also: +- https://sigs.k8s.io/cluster-api-provider-nested/proposals/20201026-creating-control-plane-components.md +replaces: [] +superseded-by: [] +--- + +# CAPN NestedCluster & NestedControlPlane + +## Table of Contents + + + + * [CAPN NestedCluster & NestedControlPlane](#capn-nestedcluster--nestedcontrolplane) + * [Table of Contents](#table-of-contents) + * [Glossary](#glossary) + * [Summary](#summary) + * [Motivation](#motivation) + * [Goals](#goals) + * [Non-Goals/Future Work](#non-goalsfuture-work) + * [Proposal](#proposal) + * [User Stories](#user-stories) + * [Features from user stories](#features-from-user-stories) + * [NestedCluster](#nestedcluster) + * [NestedCluster Lifecycle](#nestedcluster-lifecycle) + * [NestedControlPlane](#nestedcontrolplane) + * [NestedControlPlane Lifecycle](#nestedcontrolplane-lifecycle) + * [Component CustomResources (NEtcd, NKAS, NKCM)](#component-customresources-netcd-nkas-nkcm) + * [PKI/Certificate Management](#pkicertificate-management) + * [In-Tree Certificate Management](#in-tree-certificate-management) + * [ControlPlaneEndpoint Setting](#controlplaneendpoint-setting) + * [KUBECONFIG Management](#kubeconfig-management) + * [Risks and Mitigations](#risks-and-mitigations) + * [Upgrade Strategy](#upgrade-strategy) + * [Version Skew Strategy](#version-skew-strategy) + * [Implementation History](#implementation-history) + + + + +## Glossary + +Refer to the [Cluster API Provider Nested Glossary](/proposals/00_capn-glossary.md). + +If this proposal adds new terms, or defines some, make the changes to the book's glossary when in the PR stage. + +## Summary + +This proposal outlines the base architecture for Cluster API Provider Nested (CAPN) `NestedCluster` & `NestedControlPlane`. CAPN enables you to use Cluster API to manage control planes that are run as Pods "nested" within a [super cluster](/proposals/00_capn-glossary.md#super-cluster). These nested control planes, paired with a "sync controller" (PROPOSAL LINK TBD), that allows the super cluster to run workloads that have been scheduled against the CAPN provided control planes. + +Doing this allows you to take a multi-tenant super cluster and break it up into multiple single-tenant control planes, thus allowing the users of those control planes the ability to use cluster level resources like CRDs, AdmissionWebhooks, Cluster RBAC and more, while still getting the benefits of utilization of multi-tenant clusters. + +## Motivation + +This project is based after the work from the [multi-tenancy working group](http://sigs.k8s.io/multi-tenancy) on `virtualcluster`. Virtual cluster was implemented with a component internally called `vc-manager` which allowed you to provision a pod based control plane with the exception of the `kube-scheduler`. The motivation for CAPN is to reimplement this design with CAPI in-mind. + +### Goals + +- To design a new custom resource definition to orchestrate the creation of the nested control planes. +- To design a new custom resource definition to orchestrate the creation of the "cluster" +- To support declarative orchestrated control plane upgrades for apiserver and controller manager pods in built-in controllers. +- To support managing custom cluster add ons in the CAPN control plane, such as DNS, auth, etc using CAPI's `ClusterResourceSet`. + +### Non-Goals/Future Work + +Non-goals are limited to the scope of this document, these features will evolve +over time. + +- To support managing real nodes in CAPN. _Technically, CAPN can use real nodes. But we have to incorporate an entire new machine provision mechanism/workflow which we leave as the future work._ +- To support managing components running in the super cluster worker nodes. _All the node plugins such as Kubeproxy, CNI/CSI will be managed in the super cluster by the super cluster administrator. They cannot be configured through the CAPN APIs._ +- To support a separate scheduler in each CAPN control plane. _There are few use cases which require two level schedulers but it is out of the scope for CAPN as of now._ +- Define how each [component controller](/proposals/00_capn-glossary.md#component-controller) operates, this is defined in [Creating Control Plane Components](/proposals/20201026-creating-control-plane-components.md) + + +## Proposal + +### User Stories + +1. As a control plane operator, I want to be able to set the namespace where my cluster is provisioned +1. As a control plane operator, I want to be able to set the name of my control plane +1. As a control plane operator, I want to be able to specify the [component controllers](/proposals/00_capn-glossary.md#component-controller) for my cluster. +1. As a control plane operator, I want CAPN to manage my certificates. +1. As a control plane operator, I want to supply my certificates and PKI. +1. As a control plane operator, I want CAPN to manage my KUBECONFIGs. +1. As a control plane operator, I want CAPN to setup object relationships. +1. As a machine, I want to know how to address my control plane. + +### Components + +#### NestedCluster + +Kubernetes API Group: `infrastructure.cluster.x-k8s.io/v1alpha4` + +This resource is responsible for getting the Cluster API `Cluster` type and setting the `spec.controlPlaneRef` pointing back to to this `NestedCluster`, this allows the CAPI controllers to understand the state of the cluster. As well It listens to the `NestedControlPlane` resource which is associated and keeps it's status in sync as the cluster is provided. + + +```go +type NestedClusterSpec struct { + // ControlPlaneRef is an optional reference to a provider-specific resource that holds + // the details for provisioning the Control Plane for a Cluster. + // +optional + ControlPlaneRef *corev1.ObjectReference `json:"controlPlaneRef,omitempty"` + + // ControlPlaneEndpoint represents the endpoint used to communicate with the control plane. + // +optional + ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint"` +} + +type NestedClusterStatus struct { + // Ready is when the NestedControlPlane has a API server URL. + // +optional + Ready bool `json:"ready,omitempty"` +} + +type NestedCluster struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec NestedClusterSpec `json:"spec,omitempty"` + Status NestedClusterStatus `json:"status,omitempty"` +} +``` + +### NestedCluster Lifecycle + +![Creating a NestedCluster](images/nestedcontrolplane/nc-activity.png) + + +#### NestedControlPlane + +Kubernetes API Group: `controlplane.cluster.x-k8s.io/v1alpha4` + +The NestedControlPlane resource is responsible for orchestrating the overarching cluster, this controller doesn't create any of the downstream objects instead it creates a place where the downstream objects can look up shared values. + +```go +type NestedControlPlaneSpec struct { + // ControlPlaneEndpoint represents the endpoint used to communicate with the control plane. + // +optional + ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint"` + + // EtcdRef is the reference to the NestedEtcd + EtcdRef *corev1.ObjectReference `json:"etcd,omitempty"` + + // APIServerRef is the reference to the NestedAPIServer + // +optional + APIServerRef *corev1.ObjectReference `json:"apiserver,omitempty"` + + // ContollerManagerRef is the reference to the NestedControllerManager + // +optional + ControllerManagerRef *corev1.ObjectReference `json:"controllerManager,omitempty"` +} + +type NestedControlPlaneStatus struct { + // Etcd stores the connection information from the downstream etcd + // implementation if the NestedEtcd type isn't used this + // allows other component controllers to fetch the endpoints. + // +optional + Etcd *NestedControlPlaneStatusEtcd `json:"etcd,omitempty"` + + // APIServer stores the connection information from the control plane + // this should contain anything shared between control plane components + // +optional + APIServer *NestedControlPlaneStatusAPIServer `json:"apiserver,omitempty"` + + // Initialized denotes whether or not the control plane has the + // uploaded kubernetes config-map. + // +optional + Initialized bool `json:"initialized"` + + // Ready denotes that the AWSManagedControlPlane API Server is ready to + // receive requests and that the VPC infra is ready. + // +kubebuilder:default=false + Ready bool `json:"ready"` + + // ErrorMessage indicates that there is a terminal problem reconciling the + // state, and will be set to a descriptive error message. + // +optional + FailureMessage *string `json:"failureMessage,omitempty"` + + // Conditions specifies the cpnditions for the managed control plane + Conditions clusterv1.Conditions `json:"conditions,omitempty"` +} + +type NestedControlPlaneStatusEtcd struct { + // Addresses defines how to address the etcd instance + Addresses []NestedEtcdAddress `json:"addresses,omitempty"` +} + +type NestedControlPlaneStatusAPIServer struct { + // ServiceCIDRs which is provided to kube-apiserver and kube-controller-manager + // +optional + ServiceCIDR string `json:"serviceCidr,omitempty"` +} +``` + +### NestedControlPlane Lifecycle + +![Creating a NestedControl Plane](images/nestedcontrolplane/ncp-activity.png) + +#### Component CustomResources (NEtcd, NKAS, NKCM) + +After the controller has fetched the `NestedControlPlane` resource and that it's not in a terminating state it will look for all the [Nested* Component Controllers](20201026-creating-control-plane-components.md). It will do this by following the `*Refs` defined on the `NestedControlPlaneSpec`. Once all of the Nested* component Custom Resources have been found the controller will patch these objects setting the `metadata.ownerReferences` back to the `NestedControlPlane`. + +```go +if err := controllerutil.SetControllerReference(ncp, component, r.scheme); err != nil { + // Return errors +} +``` + + +### PKI/Certificate Management + +Within a CAPN Cluster we need to manage the certificates and CA's which are used for the components in the control plane, this MUST be done outside of each of the component controllers so that we can share this information. As well these should be overridable by supplying your own certificates using `cert-manager` or other solutions by the use of shared common Kubernetes secret names. + +#### In-Tree Certificate Management + +When the `NestedControlPlane` controller receives a new `NCP` resource it first goes through checks to validate that the necessary resources have been created first. In these checks it will look to see if the cluster CA and initial certificates have been issued. These are looked up in the management cluster via Kubernetes secrets as per [Using Custom Certificates](https://cluster-api.sigs.k8s.io/tasks/certs/using-custom-certificates.html?highlight=certificate#using-custom-certificates) with the addition + + +| Name | Type | Example | +| ---------------------- | -------- | ------------------------------------------------------------ | +| *[cluster name]***-ca** | CA | openssl req -x509 -subj "/CN=Kubernetes API" -new -newkey rsa:2048 -nodes -keyout tls.key -sha256 -days 3650 -out tls.crt | +| *[cluster name]***-etcd** | CA | openssl req -x509 -subj "/CN=ETCD CA" -new -newkey rsa:2048 -nodes -keyout tls.key -sha256 -days 3650 -out tls.crt | +| *[cluster name]***-proxy** | CA | openssl req -x509 -subj "/CN=Front-End Proxy" -new -newkey rsa:2048 -nodes -keyout tls.key -sha256 -days 3650 -out tls.crt | +| *[cluster name]***-sa** | Key Pair | openssl genrsa -out tls.key 2048 && openssl rsa -in tls.key -pubout -out tls.crt | +| *[cluster name]***-[component]-client-cert** | Key Pair | openssl genrsa -out tls.key 2048 && openssl rsa -in tls.key -pubout -out tls.crt | + +If these secrets aren't found, the controller will attempt to create these using the funcs from `kubeadm` and CAPI. These funcs will check if the secrets exist, if they do not they generate the certificates and save them. + +```go +certificates := secret.NewCertificatesForInitialControlPlane(&kubeadmv1.ClusterConfiguration{}) +controllerRef := metav1.NewControllerRef(kcp, controlplanev1.GroupVersion.WithKind("NestedControlPlane")) +if err := certificates.LookupOrGenerate(ctx, r.Client, util.ObjectKey(cluster), *controllerRef); err != nil { + // Log errors, Set Certificate Condition, Return +} +// Set Certificate Condition +``` + +The only addition to the default CA and SA certificates that are generated are the key pairs for each component, if there is not a secret found using the format `*[cluster name]***-[component]-client-cert**` then one will be created for those components using the defined CA. The possible `component` names are as follows but not limited to: + +* etcd +* proxy +* controller-manager +* kubelet +* TBD + +#### ControlPlaneEndpoint Setting + +After the certificates for the cluster have been found or generated we then need to check if the `ControlPlaneEndpoint` is set, if not we need to return and wait for the endpoint to be available. _Setting the `ControlPlaneEndpoint` field on the `NestedControlPlane` CR should be done by the `NestedAPIServer` resource._ + +#### KUBECONFIG Management + +After the controller has checked that the `ControlPlaneEndpoint` is set, it moves on to verifying the `KUBECONFIG` has been created. To do this the controller will lookup the Secret again, `[cluster name]-kubeconfig`, If the secret is NOT found the controller will begin to generate the `KUBECONFIG` using the Cluster API `kubeconfig` helper functions. + +```go +createErr := kubeconfig.CreateSecretWithOwner( + ctx, + r.Client, + clusterName, + endpoint.String(), + controllerOwnerRef, +) +``` + +If the `KUBECONFIG` secret did exist the controller will check it's ownership, if it's missing owner references the kubeconfig will be `adopted` and owned by this control plane. Following ownership checks it will verify that the `KUBECONFIG` certificates are still valid, following CAPI's design this will default to 1 year expiry and a 6 month rotation timing, if those need rotation that operation will be done and the secret is updated. + +```go +needsRotation, err := kubeconfig.NeedsClientCertRotation(configSecret, certs.ClientCertificateRenewalDuration) +if err != nil { + // Return errors +} + +if needsRotation { + if err := kubeconfig.RegenerateSecret(ctx, r.Client, configSecret); err != nil { + // Return errors + } +} +``` + +## Upgrade Strategy + +Both `NestedCluster` & `NestedControlPlane` resources aren't actually responsible for the upgrading of the physical components, this is left to the downstream component controllers. When they receive an updated CR for `NestedEtcd` or `NestedAPIserver` those controllers are responsible for keeping the control planes updated. The `NestedControlPlane` and `NestedCluster` both need to be kept in-sync if update do happen to make sure `spec.controlPlaneEndpoint` is still pointed at the right access. + +The exception to this rule is the certificate management, because the control planes get their certificates through the `NestedControlPlane` controller creating the artifacts and storing in Kubernetes secrets (if not supplied) then we need to make sure the secrets receive that same updated secret when the certificates expire and are renewed, this is an action the component controllers MUST watch for but COULD be triggered via NCP. + + +### Version Skew Strategy + +Version skew is a problem that can easily happen, since the NCP doesn't prescribe the overall version of the stack, these must be managed manually by the individual component controllers and any updates that happen should be synced back through the Nested* CRs where necessary (eg. sync control plane endpoint, etcd addresses, etc.) + +## Implementation History + +- [x] 10/21/2020: Proposed idea in an issue or [community meeting] +- [x] 01/11/2021: Compile a Google Doc following the CAEP template (link here) +- [ ] MM/DD/YYYY: First round of feedback from community +- [ ] MM/DD/YYYY: Present proposal at a [community meeting] +- [ ] MM/DD/YYYY: Open proposal PR + + +[community meeting]: https://docs.google.com/document/d/10aTeq2lhXW_3aFQAd_MdGjY8PtZPslKhZCCcXxFp3_Q/edit#heading=h.ejz1103gmaij + + diff --git a/proposals/YYYYMMDD-template.md b/docs/proposals/YYYYMMDD-template.md similarity index 100% rename from proposals/YYYYMMDD-template.md rename to docs/proposals/YYYYMMDD-template.md diff --git a/proposals/in-tree.png b/docs/proposals/images/componentcontrollers/in-tree.png similarity index 100% rename from proposals/in-tree.png rename to docs/proposals/images/componentcontrollers/in-tree.png diff --git a/proposals/out-of-tree.png b/docs/proposals/images/componentcontrollers/out-of-tree.png similarity index 100% rename from proposals/out-of-tree.png rename to docs/proposals/images/componentcontrollers/out-of-tree.png diff --git a/docs/proposals/images/nestedcontrolplane/nc-activity.plantuml b/docs/proposals/images/nestedcontrolplane/nc-activity.plantuml new file mode 100644 index 00000000..d6a45d60 --- /dev/null +++ b/docs/proposals/images/nestedcontrolplane/nc-activity.plantuml @@ -0,0 +1,81 @@ +@startuml +skinparam roundcorner 20 +skinparam ParticipantPadding 20 +skinparam BoxPadding 50 +skinparam Shadowing false +skinparam NoteBorderColor #444 +skinparam NoteBackgroundColor #fff +skinparam NoteFontColor #444 +skinparam EntityBackgroundColor #fff +skinparam EntityBorderColor #444 +skinparam ArrowFontColor #444 +skinparam ArrowColor #444 +skinparam ArrowLollipopColor #444 +skinparam ArrowThickness 1 +skinparam ControlBorderColor #444 +skinparam ControlBackgroundColor #fff +skinparam ParticipantBorderColor #444 +skinparam ParticipantBackgroundColor #fff +skinparam ParticipantFontSize 17 +skinparam ParticipantFontColor #444 +skinparam ActorBorderColor #444 +skinparam ActorFontColor #444 +skinparam ActorFontSize 17 +skinparam ActorBackgroundColor #fff +skinparam GroupBorderColor #444 +skinparam GroupBorderThickness 1 +skinparam GroupHeaderFontColor #444 +skinparam GroupFontColor #444 +skinparam SequenceLifeLineBorderColor #444 +skinparam ActivityBorderColor #444 +skinparam ActivityBackgroundColor #fff +skinparam ActivityDiamondBorderColor #444 +skinparam ActivityDiamondBackgroundColor #fff + +title NestedCluster Creation + +start + +if (NC exists?) then (no) + :return nil; + end +else (yes) +endif + +if (deletionTimestamp?) then (yes) + :handle deletion; + end +else (no) +endif + +if (owningCluster exists?) then (no) + :return nil; + end +else (yes) +endif + +if (cluster paused?) then (yes) + :return nil; + end +else (no) +endif + +if (controlPlane exists?) then (no) + :return failed to get control plane; + end +endif + +:set cluster.Status.Ready == ncp.Status.Ready; +:set cluster.Spec.ControlPlaneEndpoint == ncp.Spec.ControlPlaneEndpoint; + +:patch cluster; +if (errors?) then (yes) + :return error; + end +endif + +:return nil; + +end + +@enduml \ No newline at end of file diff --git a/docs/proposals/images/nestedcontrolplane/nc-activity.png b/docs/proposals/images/nestedcontrolplane/nc-activity.png new file mode 100644 index 00000000..b4503896 Binary files /dev/null and b/docs/proposals/images/nestedcontrolplane/nc-activity.png differ diff --git a/docs/proposals/images/nestedcontrolplane/ncp-activity.plantuml b/docs/proposals/images/nestedcontrolplane/ncp-activity.plantuml new file mode 100644 index 00000000..12ff271a --- /dev/null +++ b/docs/proposals/images/nestedcontrolplane/ncp-activity.plantuml @@ -0,0 +1,117 @@ +@startuml +skinparam roundcorner 20 +skinparam ParticipantPadding 20 +skinparam BoxPadding 50 +skinparam Shadowing false +skinparam NoteBorderColor #444 +skinparam NoteBackgroundColor #fff +skinparam NoteFontColor #444 +skinparam EntityBackgroundColor #fff +skinparam EntityBorderColor #444 +skinparam ArrowFontColor #444 +skinparam ArrowColor #444 +skinparam ArrowLollipopColor #444 +skinparam ArrowThickness 1 +skinparam ControlBorderColor #444 +skinparam ControlBackgroundColor #fff +skinparam ParticipantBorderColor #444 +skinparam ParticipantBackgroundColor #fff +skinparam ParticipantFontSize 17 +skinparam ParticipantFontColor #444 +skinparam ActorBorderColor #444 +skinparam ActorFontColor #444 +skinparam ActorFontSize 17 +skinparam ActorBackgroundColor #fff +skinparam GroupBorderColor #444 +skinparam GroupBorderThickness 1 +skinparam GroupHeaderFontColor #444 +skinparam GroupFontColor #444 +skinparam SequenceLifeLineBorderColor #444 +skinparam ActivityBorderColor #444 +skinparam ActivityBackgroundColor #fff +skinparam ActivityDiamondBorderColor #444 +skinparam ActivityDiamondBackgroundColor #fff + +title NestedControlPlane Creation + +start + +if (NCP exists?) then (no) + :return nil; + end +else (yes) +endif + +if (deletionTimestamp?) then (yes) + :handle deletion; + end +endif + +if (component CRs exist?) then (no) + :emit event; + :return retryAfter; + end +else (yes) + if (has ownerReferences?) then (no) + :setup OwnerReferences back; + endif +endif + +if (cluster CA/Certs exists?) then (no) + :call secret.NewCertificatesForInitialControlPlane; + :store certs as secrets; + if (errors?) then (yes) + :return err; + end + elseif (error with Conflicts?) then (yes) + :return retryAfter; + end + else (no) + endif +endif + +if (ControlPlaneEndpoint set?) then (no) + :return nil; + end +endif + + +if (admin kubeconfig exists?) then (no) + :get ControlPlaneEndpoint; + :create kubeconfig; + if (errors?) then (yes) + :return err; + end + endif +elseif (kubeconfig owned?) then (no) + :adopt kubeconfig; +else (yes) + if (certs need rotation?) then (yes) + :regenerate and store; + if (errors?) then (yes) + :return err; + end + endif + endif +endif + +if (conditions updated?) then (no) + if (etcd managed?) then (yes) + :try to fetch common status fields; + if (errors?) then (yes) + :set etcd as unknown state; + endif + endif + :get nested component statuses; + :update NestedControlPlane status; + if (errors?) then (yes) + :return err; + end + endif +endif + +:return nil; + +end + +@enduml \ No newline at end of file diff --git a/docs/proposals/images/nestedcontrolplane/ncp-activity.png b/docs/proposals/images/nestedcontrolplane/ncp-activity.png new file mode 100644 index 00000000..2f582c54 Binary files /dev/null and b/docs/proposals/images/nestedcontrolplane/ncp-activity.png differ diff --git a/docs/proposals/out-of-tree.png b/docs/proposals/out-of-tree.png new file mode 100644 index 00000000..876d56a1 Binary files /dev/null and b/docs/proposals/out-of-tree.png differ