diff --git a/.dockerignore b/.dockerignore index 8650cf8d..e38aaadd 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,7 +2,7 @@ .github .vscode bin/ -**/*.yaml +# **/*.yaml hack/ docs/ logos/ diff --git a/Dockerfile b/Dockerfile index 27faa3c3..6b830096 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,28 +1,62 @@ -# Build the manager binary -FROM golang:1.15 as builder +# syntax=docker/dockerfile:1.1-experimental + +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Build the manager binary +FROM golang:1.16.2 as builder WORKDIR /workspace + +# Run this with docker build --build_arg $(go env GOPROXY) to override the goproxy +ARG goproxy=https://proxy.golang.org +ENV GOPROXY=$goproxy + # Copy the Go Modules manifests COPY go.mod go.mod COPY go.sum go.sum -# cache deps before building and copying source so that we don't need to re-download as much +# Cache deps before building and copying source so that we don't need to re-download as much # and so that source changes don't invalidate our downloaded layer -RUN go mod download +RUN --mount=type=cache,target=/root/.local/share/golang \ + --mount=type=cache,target=/go/pkg/mod \ + go mod download -# Copy the go source +# Copy the sources COPY main.go main.go -COPY apis/ apis/ +COPY api/ api/ COPY controllers/ controllers/ -COPY certificate/ certificate/ +COPY controlplane/ controlplane/ + +RUN wget --output-document /restart.sh --quiet https://raw.githubusercontent.com/windmilleng/rerun-process-wrapper/master/restart.sh && \ + wget --output-document /start.sh --quiet https://raw.githubusercontent.com/windmilleng/rerun-process-wrapper/master/start.sh && \ + chmod +x /start.sh && chmod +x /restart.sh # Build -RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o manager main.go +ARG package=. +ARG ARCH +ARG LDFLAGS +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/root/.local/share/golang \ + CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} go build -ldflags "${LDFLAGS} -extldflags '-static'" -o manager ${package} +ENTRYPOINT [ "/start.sh", "/workspace/manager" ] # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details FROM gcr.io/distroless/static:nonroot +# Copy the controller-manager into a thin image WORKDIR / COPY --from=builder /workspace/manager . +COPY controlplane/nested/component-templates/ ./component-templates/ USER 65532:65532 - -ENTRYPOINT ["/manager"] +ENTRYPOINT ["/manager"] \ No newline at end of file diff --git a/Makefile b/Makefile index 881d92d5..96a5161d 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -# Copyright 2018 The Kubernetes Authors. +# Copyright 2021 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,9 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# If you update this file, please follow -# https://suva.sh/posts/well-documented-makefiles - # Ensure Make is run with bash shell as some syntax below is bash-specific SHELL:=/usr/bin/env bash @@ -61,9 +58,15 @@ REGISTRY ?= gcr.io/$(shell gcloud config get-value project) endif STAGING_REGISTRY ?= gcr.io/k8s-staging-cluster-api-provider-nested PROD_REGISTRY ?= us.gcr.io/k8s-artifacts-prod/cluster-api-provider-nested + +# Infrastructure IMAGE_NAME ?= cluster-api-nested-controller CONTROLLER_IMG ?= $(REGISTRY)/$(IMAGE_NAME) +# Control Plane +CONTROLPLANE_IMAGE_NAME ?= nested-controlplane-controller +CONTROLPLANE_CONTROLLER_IMG ?= $(REGISTRY)/$(CONTROLPLANE_IMAGE_NAME) + TAG ?= dev ARCH ?= amd64 ALL_ARCH = amd64 arm arm64 ppc64le s390x @@ -93,13 +96,18 @@ test: ## Run tests. .PHONY: binaries binaries: managers -.PHONY: manager -manager-core: ## Build manager binary - go build -ldflags "$(LDFLAGS)" -o $(BIN_DIR)/manager sigs.k8s.io/cluster-api-provider-nested - .PHONY: managers managers: ## Build all managers - $(MAKE) manager-core + $(MAKE) manager-nested-infrastructure + $(MAKE) manager-nested-controlplane + +.PHONY: manager-nested-infrastructure +manager-nested-infrastructure: + go build -ldflags "$(LDFLAGS)" -o $(BIN_DIR)/manager sigs.k8s.io/cluster-api-provider-nested + +.PHONY: manager-nested-controlplane +manager-nested-controlplane: ## Build manager binary + go build -ldflags "$(LDFLAGS)" -o $(BIN_DIR)/nested-controlplane-manager sigs.k8s.io/cluster-api-provider-nested/controlplane/nested $(CONTROLLER_GEN): $(TOOLS_DIR)/go.mod # Build controller-gen from tools folder. cd $(TOOLS_DIR); go build -tags=tools -o $(BIN_DIR)/controller-gen sigs.k8s.io/controller-tools/cmd/controller-gen @@ -150,24 +158,52 @@ generate: .PHONY: generate-go generate-go: $(CONTROLLER_GEN) ## Runs Go related generate targets go generate ./... + $(MAKE) generate-go-infrastructure + $(MAKE) generate-go-controlplane + +.PHONY: generate-go-infrastructure +generate-go-infrastructure: $(CONTROLLER_GEN) + $(CONTROLLER_GEN) \ + object:headerFile=./hack/boilerplate/boilerplate.generatego.txt \ + paths=./api/... + +generate-go-controlplane: $(CONTROLLER_GEN) $(CONTROLLER_GEN) \ object:headerFile=./hack/boilerplate/boilerplate.generatego.txt \ - paths=./apis/... + paths=./controlplane/nested/api/... .PHONY: generate-manifests generate-manifests: $(CONTROLLER_GEN) ## Generate manifests e.g. CRD, RBAC etc. + $(MAKE) generate-manifests-infrastructure + $(MAKE) generate-manifests-controlplane + ## Copy files in CI folders. + mkdir -p ./config/ci/{rbac,manager} + cp -f ./config/rbac/*.yaml ./config/ci/rbac/ + cp -f ./config/manager/manager*.yaml ./config/ci/manager/ + +.PHONY: generate-manifests-infrastructure +generate-manifests-infrastructure: $(CONTROLLER_GEN) \ - paths=./apis/... \ + paths=./api/... \ paths=./controllers/... \ crd:crdVersions=v1 \ rbac:roleName=manager-role \ output:crd:dir=./config/crd/bases \ output:webhook:dir=./config/webhook \ + output:rbac:dir=./config/rbac \ + webhook + +.PHONY: generate-manifests-controlplane +generate-manifests-controlplane: + $(CONTROLLER_GEN) \ + paths=./controlplane/nested/api/... \ + paths=./controlplane/nested/controllers/... \ + crd:crdVersions=v1 \ + rbac:roleName=manager-role \ + output:crd:dir=./controlplane/nested/config/crd/bases \ + output:webhook:dir=./controlplane/nested/config/webhook \ + output:rbac:dir=./controlplane/nested/config/rbac \ webhook - ## Copy files in CI folders. - mkdir -p ./config/ci/{rbac,manager} - cp -f ./config/rbac/*.yaml ./config/ci/rbac/ - cp -f ./config/manager/manager*.yaml ./config/ci/manager/ .PHONY: modules modules: ## Runs go mod to ensure modules are up to date. @@ -184,32 +220,63 @@ docker-pull-prerequisites: docker pull docker.io/library/golang:1.15.3 docker pull gcr.io/distroless/static:latest -.PHONY: docker-build -docker-build: docker-pull-prerequisites ## Build the docker images for controller managers +.PHONY: docker-infrastructure-build +docker-infrastructure-build: docker-pull-prerequisites ## Build the docker images for controller managers DOCKER_BUILDKIT=1 docker build --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg ldflags="$(LDFLAGS)" . -t $(CONTROLLER_IMG)-$(ARCH):$(TAG) - # $(MAKE) set-manifest-image MANIFEST_IMG=$(CONTROLLER_IMG)-$(ARCH) MANIFEST_TAG=$(TAG) TARGET_RESOURCE="./config/manager/manager_image_patch.yaml" - # $(MAKE) set-manifest-pull-policy TARGET_RESOURCE="./config/manager/manager_pull_policy.yaml" + $(MAKE) set-manifest-image MANIFEST_IMG=$(CONTROLLER_IMG)-$(ARCH) MANIFEST_TAG=$(TAG) TARGET_RESOURCE="./config/default/manager_image_patch.yaml" + $(MAKE) set-manifest-pull-policy TARGET_RESOURCE="./config/default/manager_pull_policy.yaml" -.PHONY: docker-push -docker-push: ## Push the docker images +.PHONY: docker-controlplane-build +docker-controlplane-build: docker-pull-prerequisites ## Build the docker images for controller managers + DOCKER_BUILDKIT=1 docker build --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg ldflags="$(LDFLAGS)" --build-arg package=./controlplane/nested . -t $(CONTROLPLANE_CONTROLLER_IMG)-$(ARCH):$(TAG) + $(MAKE) set-manifest-image MANIFEST_IMG=$(CONTROLPLANE_CONTROLLER_IMG)-$(ARCH) MANIFEST_TAG=$(TAG) TARGET_RESOURCE="./controlplane/nested/config/default/manager_image_patch.yaml" + $(MAKE) set-manifest-pull-policy TARGET_RESOURCE="./controlplane/nested/config/default/manager_pull_policy.yaml" + + +.PHONY: docker-infrastructure-push +docker-infrastructure-push: ## Push the docker images docker push $(CONTROLLER_IMG)-$(ARCH):$(TAG) +.PHONY: docker-controlplane-push +docker-controlplane-push: ## Push the docker images + docker push $(CONTROLPLANE_CONTROLLER_IMG)-$(ARCH):$(TAG) + ## -------------------------------------- ## Docker — All ARCH ## -------------------------------------- .PHONY: docker-build-all ## Build all the architecture docker images -docker-build-all: $(addprefix docker-build-,$(ALL_ARCH)) +docker-build-all: $(addprefix docker-infrastructure-build-,$(ALL_ARCH)) $(addprefix docker-controlplane-build-,$(ALL_ARCH)) + +.PHONY: docker-build +docker-build: + $(MAKE) docker-infrastructure-build + $(MAKE) docker-controlplane-build -docker-build-%: - $(MAKE) ARCH=$* docker-build +.PHONY: docker-infrastructure-build +docker-infrastructure-build-%: + $(MAKE) ARCH=$* docker-infrastructure-build + +.PHONY: docker-controlplane-build +docker-controlplane-build-%: + $(MAKE) ARCH=$* docker-controlplane-build .PHONY: docker-push-all ## Push all the architecture docker images -docker-push-all: $(addprefix docker-push-,$(ALL_ARCH)) +docker-push-all: $(addprefix docker-infrastructure-push-,$(ALL_ARCH)) $(addprefix docker-controlplane-push-,$(ALL_ARCH)) $(MAKE) docker-push-core-manifest -docker-push-%: - $(MAKE) ARCH=$* docker-push +.PHONY: docker-push +docker-push: + $(MAKE) docker-infrastructure-push + $(MAKE) docker-controlplane-push + +.PHONY: docker-infrastructure-push +docker-infrastructure-push-%: + $(MAKE) ARCH=$* docker-infrastructure-push + +.PHONY: docker-controlplane-push +docker-controlplane-push-%: + $(MAKE) ARCH=$* docker-controlplane-push .PHONY: docker-push-core-manifest docker-push-core-manifest: ## Push the fat manifest docker image for the core image. @@ -260,17 +327,28 @@ release: clean-release ## Builds and push container images using the latest git .PHONY: release-manifests release-manifests: $(RELEASE_DIR) $(KUSTOMIZE) ## Builds the manifests to publish with a release # Build infrastructure-components. - $(KUSTOMIZE) build config > $(RELEASE_DIR)/infrastructure-components.yaml + $(KUSTOMIZE) build config/default > $(RELEASE_DIR)/infrastructure-components.yaml + # Build control-plane-components. + $(KUSTOMIZE) build controlplane/nested/config/default > $(RELEASE_DIR)/control-plane-components.yaml + + ## Build cluster-api-provider-nested-components (aggregate of all of the above). + cat $(RELEASE_DIR)/infrastructure-components.yaml > $(RELEASE_DIR)/cluster-api-provider-nested-components.yaml + echo "---" >> $(RELEASE_DIR)/cluster-api-provider-nested-components.yaml + cat $(RELEASE_DIR)/control-plane-components.yaml >> $(RELEASE_DIR)/cluster-api-provider-nested-components.yaml + # Add metadata to the release artifacts + cp metadata.yaml $(RELEASE_DIR)/metadata.yaml + .PHONY: release-staging release-staging: ## Builds and push container images to the staging bucket. - REGISTRY=$(STAGING_REGISTRY) $(MAKE) docker-build docker-push release-alias-tag + REGISTRY=$(STAGING_REGISTRY) $(MAKE) docker-build-all docker-push-all release-alias-tag RELEASE_ALIAS_TAG=$(PULL_BASE_REF) .PHONY: release-alias-tag release-alias-tag: ## Adds the tag to the last build tag. gcloud container images add-tag $(CONTROLLER_IMG):$(TAG) $(CONTROLLER_IMG):$(RELEASE_ALIAS_TAG) + gcloud container images add-tag $(CONTROLPLANE_CONTROLLER_IMG):$(TAG) $(CONTROLPLANE_CONTROLLER_IMG):$(RELEASE_ALIAS_TAG) .PHONY: release-notes release-notes: $(RELEASE_NOTES) ## Generates a release notes template to be used with a release. diff --git a/PROJECT b/PROJECT index 6c309cf7..e378aca8 100644 --- a/PROJECT +++ b/PROJECT @@ -1,28 +1,9 @@ domain: cluster.x-k8s.io layout: - go.kubebuilder.io/v3 -multigroup: true projectName: cluster-api-provider-nested repo: sigs.k8s.io/cluster-api-provider-nested resources: -- group: controlplane - kind: NestedControlPlane - version: v1alpha4 -- api: - crdVersion: v1 - group: controlplane - kind: NestedEtcd - version: v1alpha4 -- api: - crdVersion: v1 - group: controlplane - kind: NestedAPIServer - version: v1alpha4 -- api: - crdVersion: v1 - group: controlplane - kind: NestedControllerManager - version: v1alpha4 - api: crdVersion: v1 namespaced: true @@ -30,6 +11,6 @@ resources: domain: cluster.x-k8s.io group: infrastructure kind: NestedCluster - path: sigs.k8s.io/cluster-api-provider-nested/apis/infrastructure/v1alpha4 + path: sigs.k8s.io/cluster-api-provider-nested/api/v1alpha4 version: v1alpha4 version: "3" diff --git a/apis/infrastructure/v1alpha4/groupversion_info.go b/api/v1alpha4/groupversion_info.go similarity index 100% rename from apis/infrastructure/v1alpha4/groupversion_info.go rename to api/v1alpha4/groupversion_info.go diff --git a/apis/infrastructure/v1alpha4/nestedcluster_types.go b/api/v1alpha4/nestedcluster_types.go similarity index 100% rename from apis/infrastructure/v1alpha4/nestedcluster_types.go rename to api/v1alpha4/nestedcluster_types.go diff --git a/apis/infrastructure/v1alpha4/zz_generated.deepcopy.go b/api/v1alpha4/zz_generated.deepcopy.go similarity index 100% rename from apis/infrastructure/v1alpha4/zz_generated.deepcopy.go rename to api/v1alpha4/zz_generated.deepcopy.go diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 899557ad..4295b194 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -3,10 +3,6 @@ # It should be run by config/default resources: - bases/infrastructure.cluster.x-k8s.io_nestedclusters.yaml -- bases/controlplane.cluster.x-k8s.io_nestedcontrolplanes.yaml -- bases/controlplane.cluster.x-k8s.io_nestedetcds.yaml -- bases/controlplane.cluster.x-k8s.io_nestedapiservers.yaml -- bases/controlplane.cluster.x-k8s.io_nestedcontrollermanagers.yaml #+kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index bf803201..4e32ab36 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -6,7 +6,7 @@ namespace: capn-system commonLabels: # Label to denote name of the infra provider # https://cluster-api.sigs.k8s.io/clusterctl/provider-contract.html#labels - cluster.x-k8s.io/provider: "infrastructure-aws" + cluster.x-k8s.io/provider: "infrastructure-nested" bases: - ../crd @@ -25,6 +25,8 @@ patchesStrategicMerge: # If you want your controller-manager to expose the /metrics # endpoint w/o any authn/z, please comment the following line. - manager_auth_proxy_patch.yaml +- manager_image_patch.yaml +- manager_pull_policy.yaml # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml index 77e743d1..56e123cf 100644 --- a/config/default/manager_auth_proxy_patch.yaml +++ b/config/default/manager_auth_proxy_patch.yaml @@ -21,5 +21,5 @@ spec: name: https - name: manager args: - - "--metrics-addr=127.0.0.1:8080" - - "--enable-leader-election" + - "--metrics-bind-address=127.0.0.1:8080" + - "--leader-elect" diff --git a/config/default/manager_image_patch.yaml b/config/default/manager_image_patch.yaml new file mode 100644 index 00000000..35c6fe6c --- /dev/null +++ b/config/default/manager_image_patch.yaml @@ -0,0 +1,11 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - image: gcr.io/cluster-api-nested-controller-amd64:dev + name: manager \ No newline at end of file diff --git a/config/default/manager_pull_policy.yaml b/config/default/manager_pull_policy.yaml new file mode 100644 index 00000000..91eb7d60 --- /dev/null +++ b/config/default/manager_pull_policy.yaml @@ -0,0 +1,11 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + imagePullPolicy: Never diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index b6c85a52..23b85a29 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -26,7 +26,7 @@ spec: - command: - /manager args: - - --enable-leader-election + - --leader-elect image: controller:latest name: manager resources: diff --git a/config/rbac/leader_election_role.yaml b/config/rbac/leader_election_role.yaml index 7dc16c42..d9a56542 100644 --- a/config/rbac/leader_election_role.yaml +++ b/config/rbac/leader_election_role.yaml @@ -6,8 +6,10 @@ metadata: rules: - apiGroups: - "" + - coordination.k8s.io resources: - configmaps + - leases verbs: - get - list @@ -18,8 +20,10 @@ rules: - delete - apiGroups: - "" + - coordination.k8s.io resources: - configmaps/status + - leases/status verbs: - get - update @@ -30,4 +34,4 @@ rules: - events verbs: - create - - patch + - patch \ No newline at end of file diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 265223b9..84d5775a 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -6,134 +6,22 @@ metadata: creationTimestamp: null name: manager-role rules: -- resources: - - service - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- resources: - - service/status - verbs: - - get - - patch - - update - apiGroups: - - apps + - cluster.x-k8s.io resources: - - statefulset + - clusters verbs: - - create - - delete - get - list - - patch - - update - watch -- apiGroups: - - apps - resources: - - statefulset/status - verbs: - - get - - patch - - update -- apiGroups: - - controlplane.cluster.x-k8s.io - resources: - - nestedapiservers - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - controlplane.cluster.x-k8s.io - resources: - - nestedapiservers/finalizers - verbs: - - update -- apiGroups: - - controlplane.cluster.x-k8s.io - resources: - - nestedapiservers/status - verbs: - - get - - patch - - update -- apiGroups: - - controlplane.cluster.x-k8s.io - resources: - - nestedcontrollermanagers - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - controlplane.cluster.x-k8s.io - resources: - - nestedcontrollermanagers/finalizers - verbs: - - update -- apiGroups: - - controlplane.cluster.x-k8s.io - resources: - - nestedcontrollermanagers/status - verbs: - - get - - patch - - update - apiGroups: - controlplane.cluster.x-k8s.io resources: - nestedcontrolplanes verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - controlplane.cluster.x-k8s.io - resources: - - nestedcontrolplanes/status - verbs: - - get - - patch - - update -- apiGroups: - - controlplane.cluster.x-k8s.io - resources: - - nestedetcds - verbs: - - create - - delete - get - list - - patch - - update - watch -- apiGroups: - - controlplane.cluster.x-k8s.io - resources: - - nestedetcds/status - verbs: - - get - - patch - - update - apiGroups: - infrastructure.cluster.x-k8s.io resources: diff --git a/controllers/infrastructure/nestedcluster_controller.go b/controllers/nestedcluster_controller.go similarity index 87% rename from controllers/infrastructure/nestedcluster_controller.go rename to controllers/nestedcluster_controller.go index e668ca03..3a3269f1 100644 --- a/controllers/infrastructure/nestedcluster_controller.go +++ b/controllers/nestedcluster_controller.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package infrastructure +package controllers import ( "context" @@ -27,13 +27,15 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - clusterv1 "sigs.k8s.io/cluster-api-provider-nested/apis/controlplane/v1alpha4" - infrav1 "sigs.k8s.io/cluster-api-provider-nested/apis/infrastructure/v1alpha4" + infrav1 "sigs.k8s.io/cluster-api-provider-nested/api/v1alpha4" + controlplanev1 "sigs.k8s.io/cluster-api-provider-nested/controlplane/nested/api/v1alpha4" ) +// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters,verbs=get;list;watch //+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=nestedclusters,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=nestedclusters/status,verbs=get;update;patch //+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=nestedclusters/finalizers,verbs=update +//+kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedcontrolplanes,verbs=get;list;watch // NestedClusterReconciler reconciles a NestedCluster object type NestedClusterReconciler struct { @@ -46,7 +48,7 @@ type NestedClusterReconciler struct { func (r *NestedClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&infrav1.NestedCluster{}). - Owns(&clusterv1.NestedControlPlane{}). + Owns(&controlplanev1.NestedControlPlane{}). Complete(r) } @@ -80,7 +82,7 @@ func (r *NestedClusterReconciler) Reconcile(ctx context.Context, req ctrl.Reques Namespace: cluster.Spec.ControlPlaneRef.Namespace, Name: cluster.Spec.ControlPlaneRef.Name, } - ncp := &clusterv1.NestedControlPlane{} + ncp := &controlplanev1.NestedControlPlane{} if err := r.Get(ctx, objectKey, ncp); err != nil { if apierrors.IsNotFound(err) { return ctrl.Result{Requeue: true}, nil diff --git a/controllers/infrastructure/suite_test.go b/controllers/suite_test.go similarity index 93% rename from controllers/infrastructure/suite_test.go rename to controllers/suite_test.go index 5343e507..ccc765b8 100644 --- a/controllers/infrastructure/suite_test.go +++ b/controllers/suite_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package infrastructure +package controllers import ( "path/filepath" @@ -30,7 +30,7 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" - infrastructurev1alpha4 "sigs.k8s.io/cluster-api-provider-nested/apis/infrastructure/v1alpha4" + infrastructurev1alpha4 "sigs.k8s.io/cluster-api-provider-nested/api/v1alpha4" //+kubebuilder:scaffold:imports ) @@ -54,7 +54,7 @@ var _ = BeforeSuite(func() { By("bootstrapping test environment") testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, ErrorIfCRDPathMissing: true, } diff --git a/controlplane/nested/PROJECT b/controlplane/nested/PROJECT new file mode 100644 index 00000000..1103720e --- /dev/null +++ b/controlplane/nested/PROJECT @@ -0,0 +1,43 @@ +domain: cluster.x-k8s.io +layout: +- go.kubebuilder.io/v3 +projectName: cluster-api-provider-nested +repo: sigs.k8s.io/cluster-api-provider-nested +resources: +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: cluster.x-k8s.io + group: controlplane + kind: NestedControlPlane + path: sigs.k8s.io/cluster-api-provider-nested/controlplane/nested/api/v1alpha4 + version: v1alpha4 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: cluster.x-k8s.io + group: controlplane + kind: NestedEtcd + path: sigs.k8s.io/cluster-api-provider-nested/controlplane/nested/api/v1alpha4 + version: v1alpha4 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: cluster.x-k8s.io + group: controlplane + kind: NestedAPIServer + path: sigs.k8s.io/cluster-api-provider-nested/controlplane/api/v1alpha4 + version: v1alpha4 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: cluster.x-k8s.io + group: controlplane + kind: NestedControllerManager + path: sigs.k8s.io/cluster-api-provider-nested/controlplane/api/v1alpha4 + version: v1alpha4 +version: "3" diff --git a/apis/controlplane/v1alpha4/groupversion_info.go b/controlplane/nested/api/v1alpha4/groupversion_info.go similarity index 100% rename from apis/controlplane/v1alpha4/groupversion_info.go rename to controlplane/nested/api/v1alpha4/groupversion_info.go diff --git a/apis/controlplane/v1alpha4/nestedapiserver_types.go b/controlplane/nested/api/v1alpha4/nestedapiserver_types.go similarity index 100% rename from apis/controlplane/v1alpha4/nestedapiserver_types.go rename to controlplane/nested/api/v1alpha4/nestedapiserver_types.go diff --git a/apis/controlplane/v1alpha4/nestedcomponent_types.go b/controlplane/nested/api/v1alpha4/nestedcomponent_types.go similarity index 100% rename from apis/controlplane/v1alpha4/nestedcomponent_types.go rename to controlplane/nested/api/v1alpha4/nestedcomponent_types.go diff --git a/apis/controlplane/v1alpha4/nestedcontrollermanager_types.go b/controlplane/nested/api/v1alpha4/nestedcontrollermanager_types.go similarity index 100% rename from apis/controlplane/v1alpha4/nestedcontrollermanager_types.go rename to controlplane/nested/api/v1alpha4/nestedcontrollermanager_types.go diff --git a/apis/controlplane/v1alpha4/nestedcontrolplane_types.go b/controlplane/nested/api/v1alpha4/nestedcontrolplane_types.go similarity index 100% rename from apis/controlplane/v1alpha4/nestedcontrolplane_types.go rename to controlplane/nested/api/v1alpha4/nestedcontrolplane_types.go diff --git a/apis/controlplane/v1alpha4/nestedetcd_types.go b/controlplane/nested/api/v1alpha4/nestedetcd_types.go similarity index 100% rename from apis/controlplane/v1alpha4/nestedetcd_types.go rename to controlplane/nested/api/v1alpha4/nestedetcd_types.go diff --git a/apis/controlplane/v1alpha4/zz_generated.deepcopy.go b/controlplane/nested/api/v1alpha4/zz_generated.deepcopy.go similarity index 100% rename from apis/controlplane/v1alpha4/zz_generated.deepcopy.go rename to controlplane/nested/api/v1alpha4/zz_generated.deepcopy.go diff --git a/certificate/consts.go b/controlplane/nested/certificate/consts.go similarity index 100% rename from certificate/consts.go rename to controlplane/nested/certificate/consts.go diff --git a/certificate/helpers.go b/controlplane/nested/certificate/helpers.go similarity index 98% rename from certificate/helpers.go rename to controlplane/nested/certificate/helpers.go index ee405426..37280ba3 100644 --- a/certificate/helpers.go +++ b/controlplane/nested/certificate/helpers.go @@ -22,7 +22,7 @@ import ( "github.com/pkg/errors" "k8s.io/client-go/util/cert" - "sigs.k8s.io/cluster-api-provider-nested/certificate/util" + "sigs.k8s.io/cluster-api-provider-nested/controlplane/nested/certificate/util" ) // NewAPIServerCertAndKey creates crt and key for apiserver using ca. diff --git a/certificate/keypair.go b/controlplane/nested/certificate/keypair.go similarity index 94% rename from certificate/keypair.go rename to controlplane/nested/certificate/keypair.go index 47bbfcfa..57c9e242 100644 --- a/certificate/keypair.go +++ b/controlplane/nested/certificate/keypair.go @@ -20,7 +20,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4" - "sigs.k8s.io/cluster-api-provider-nested/certificate/util" + "sigs.k8s.io/cluster-api-provider-nested/controlplane/nested/certificate/util" "sigs.k8s.io/cluster-api/util/secret" "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/certificate/keypair_test.go b/controlplane/nested/certificate/keypair_test.go similarity index 97% rename from certificate/keypair_test.go rename to controlplane/nested/certificate/keypair_test.go index 3a8bcf7b..dde076b1 100644 --- a/certificate/keypair_test.go +++ b/controlplane/nested/certificate/keypair_test.go @@ -23,7 +23,7 @@ import ( "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - controlplanev1 "sigs.k8s.io/cluster-api-provider-nested/apis/controlplane/v1alpha4" + controlplanev1 "sigs.k8s.io/cluster-api-provider-nested/controlplane/nested/api/v1alpha4" "sigs.k8s.io/cluster-api/util/secret" "sigs.k8s.io/controller-runtime/pkg/client" ) diff --git a/certificate/keypairs.go b/controlplane/nested/certificate/keypairs.go similarity index 100% rename from certificate/keypairs.go rename to controlplane/nested/certificate/keypairs.go diff --git a/certificate/keypairs_test.go b/controlplane/nested/certificate/keypairs_test.go similarity index 97% rename from certificate/keypairs_test.go rename to controlplane/nested/certificate/keypairs_test.go index 1b58634f..ba541354 100644 --- a/certificate/keypairs_test.go +++ b/controlplane/nested/certificate/keypairs_test.go @@ -19,7 +19,7 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - controlplanev1 "sigs.k8s.io/cluster-api-provider-nested/apis/controlplane/v1alpha4" + controlplanev1 "sigs.k8s.io/cluster-api-provider-nested/controlplane/nested/api/v1alpha4" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) diff --git a/certificate/types.go b/controlplane/nested/certificate/types.go similarity index 100% rename from certificate/types.go rename to controlplane/nested/certificate/types.go diff --git a/certificate/util/util.go b/controlplane/nested/certificate/util/util.go similarity index 100% rename from certificate/util/util.go rename to controlplane/nested/certificate/util/util.go diff --git a/config/component-templates/nested-apiserver/nested-apiserver-service-template.yaml b/controlplane/nested/component-templates/nested-apiserver/nested-apiserver-service-template.yaml similarity index 100% rename from config/component-templates/nested-apiserver/nested-apiserver-service-template.yaml rename to controlplane/nested/component-templates/nested-apiserver/nested-apiserver-service-template.yaml diff --git a/config/component-templates/nested-apiserver/nested-apiserver-statefulset-template.yaml b/controlplane/nested/component-templates/nested-apiserver/nested-apiserver-statefulset-template.yaml similarity index 100% rename from config/component-templates/nested-apiserver/nested-apiserver-statefulset-template.yaml rename to controlplane/nested/component-templates/nested-apiserver/nested-apiserver-statefulset-template.yaml diff --git a/config/component-templates/nested-controllermanager/nested-controllermanager-statefulset-template.yaml b/controlplane/nested/component-templates/nested-controllermanager/nested-controllermanager-statefulset-template.yaml similarity index 100% rename from config/component-templates/nested-controllermanager/nested-controllermanager-statefulset-template.yaml rename to controlplane/nested/component-templates/nested-controllermanager/nested-controllermanager-statefulset-template.yaml diff --git a/config/component-templates/nested-etcd/nested-etcd-service-template.yaml b/controlplane/nested/component-templates/nested-etcd/nested-etcd-service-template.yaml similarity index 100% rename from config/component-templates/nested-etcd/nested-etcd-service-template.yaml rename to controlplane/nested/component-templates/nested-etcd/nested-etcd-service-template.yaml diff --git a/config/component-templates/nested-etcd/nested-etcd-statefulset-template.yaml b/controlplane/nested/component-templates/nested-etcd/nested-etcd-statefulset-template.yaml similarity index 100% rename from config/component-templates/nested-etcd/nested-etcd-statefulset-template.yaml rename to controlplane/nested/component-templates/nested-etcd/nested-etcd-statefulset-template.yaml diff --git a/controlplane/nested/config/certmanager/certificate.yaml b/controlplane/nested/config/certmanager/certificate.yaml new file mode 100644 index 00000000..58db114f --- /dev/null +++ b/controlplane/nested/config/certmanager/certificate.yaml @@ -0,0 +1,26 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager 0.11 check https://docs.cert-manager.io/en/latest/tasks/upgrading/index.html for +# breaking changes +apiVersion: cert-manager.io/v1alpha2 +kind: Issuer +metadata: + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1alpha2 +kind: Certificate +metadata: + name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize + dnsNames: + - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc + - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/controlplane/nested/config/certmanager/kustomization.yaml b/controlplane/nested/config/certmanager/kustomization.yaml new file mode 100644 index 00000000..bebea5a5 --- /dev/null +++ b/controlplane/nested/config/certmanager/kustomization.yaml @@ -0,0 +1,5 @@ +resources: +- certificate.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/controlplane/nested/config/certmanager/kustomizeconfig.yaml b/controlplane/nested/config/certmanager/kustomizeconfig.yaml new file mode 100644 index 00000000..90d7c313 --- /dev/null +++ b/controlplane/nested/config/certmanager/kustomizeconfig.yaml @@ -0,0 +1,16 @@ +# This configuration is for teaching kustomize how to update name ref and var substitution +nameReference: +- kind: Issuer + group: cert-manager.io + fieldSpecs: + - kind: Certificate + group: cert-manager.io + path: spec/issuerRef/name + +varReference: +- kind: Certificate + group: cert-manager.io + path: spec/commonName +- kind: Certificate + group: cert-manager.io + path: spec/dnsNames diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_nestedapiservers.yaml b/controlplane/nested/config/crd/bases/controlplane.cluster.x-k8s.io_nestedapiservers.yaml similarity index 100% rename from config/crd/bases/controlplane.cluster.x-k8s.io_nestedapiservers.yaml rename to controlplane/nested/config/crd/bases/controlplane.cluster.x-k8s.io_nestedapiservers.yaml diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_nestedcontrollermanagers.yaml b/controlplane/nested/config/crd/bases/controlplane.cluster.x-k8s.io_nestedcontrollermanagers.yaml similarity index 100% rename from config/crd/bases/controlplane.cluster.x-k8s.io_nestedcontrollermanagers.yaml rename to controlplane/nested/config/crd/bases/controlplane.cluster.x-k8s.io_nestedcontrollermanagers.yaml diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_nestedcontrolplanes.yaml b/controlplane/nested/config/crd/bases/controlplane.cluster.x-k8s.io_nestedcontrolplanes.yaml similarity index 100% rename from config/crd/bases/controlplane.cluster.x-k8s.io_nestedcontrolplanes.yaml rename to controlplane/nested/config/crd/bases/controlplane.cluster.x-k8s.io_nestedcontrolplanes.yaml diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_nestedetcds.yaml b/controlplane/nested/config/crd/bases/controlplane.cluster.x-k8s.io_nestedetcds.yaml similarity index 100% rename from config/crd/bases/controlplane.cluster.x-k8s.io_nestedetcds.yaml rename to controlplane/nested/config/crd/bases/controlplane.cluster.x-k8s.io_nestedetcds.yaml diff --git a/controlplane/nested/config/crd/kustomization.yaml b/controlplane/nested/config/crd/kustomization.yaml new file mode 100644 index 00000000..0e41c51a --- /dev/null +++ b/controlplane/nested/config/crd/kustomization.yaml @@ -0,0 +1,30 @@ +# This kustomization.yaml is not intended to be run by itself, +# since it depends on service name and namespace that are out of this kustomize package. +# It should be run by config/default +resources: +- bases/controlplane.cluster.x-k8s.io_nestedcontrolplanes.yaml +- bases/controlplane.cluster.x-k8s.io_nestedetcds.yaml +- bases/controlplane.cluster.x-k8s.io_nestedapiservers.yaml +- bases/controlplane.cluster.x-k8s.io_nestedcontrollermanagers.yaml +#+kubebuilder:scaffold:crdkustomizeresource + +patchesStrategicMerge: +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. +# patches here are for enabling the conversion webhook for each CRD +#+kubebuilder:scaffold:crdkustomizewebhookpatch + +# [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix. +# patches here are for enabling the CA injection for each CRD +#+kubebuilder:scaffold:crdkustomizecainjectionpatch + +# the following config is for teaching kustomize how to do kustomization for CRDs. +configurations: +# - kustomizeconfig.yaml + + +commonLabels: + # When using CAPI we need to define the contract version labels so that the + # capi system can cross reference the proper controlplane & infra refs + # https://cluster-api.sigs.k8s.io/developer/providers/v1alpha2-to-v1alpha3.html#apply-the-contract-version-label-clusterx-k8sioversion-version1_version2_version3-to-your-crds + cluster.x-k8s.io/v1alpha3: v1alpha4 + cluster.x-k8s.io/v1alpha4: v1alpha4 \ No newline at end of file diff --git a/controlplane/nested/config/default/kustomization.yaml b/controlplane/nested/config/default/kustomization.yaml new file mode 100644 index 00000000..6148f964 --- /dev/null +++ b/controlplane/nested/config/default/kustomization.yaml @@ -0,0 +1,68 @@ +namePrefix: capn-nested-control-plane- +namespace: capn-nested-control-plane-system + + +# Labels to add to all resources and selectors. +commonLabels: + # Label to denote name of the infra provider + # https://cluster-api.sigs.k8s.io/clusterctl/provider-contract.html#labels + cluster.x-k8s.io/provider: "control-plane-nested" + +bases: +- ../crd +- ../rbac +- ../manager +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# crd/kustomization.yaml +#- ../webhook +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. +#- ../certmanager +# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. +#- ../prometheus + +patchesStrategicMerge: + # Protect the /metrics endpoint by putting it behind auth. + # If you want your controller-manager to expose the /metrics + # endpoint w/o any authn/z, please comment the following line. +- manager_auth_proxy_patch.yaml +- manager_image_patch.yaml +- manager_pull_policy.yaml + +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# crd/kustomization.yaml +#- manager_webhook_patch.yaml + +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. +# Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. +# 'CERTMANAGER' needs to be enabled to use ca injection +#- webhookcainjection_patch.yaml + +# the following config is for teaching kustomize how to do var substitution +vars: +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. +#- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR +# objref: +# kind: Certificate +# group: cert-manager.io +# version: v1alpha2 +# name: serving-cert # this name should match the one in certificate.yaml +# fieldref: +# fieldpath: metadata.namespace +#- name: CERTIFICATE_NAME +# objref: +# kind: Certificate +# group: cert-manager.io +# version: v1alpha2 +# name: serving-cert # this name should match the one in certificate.yaml +#- name: SERVICE_NAMESPACE # namespace of the service +# objref: +# kind: Service +# version: v1 +# name: webhook-service +# fieldref: +# fieldpath: metadata.namespace +#- name: SERVICE_NAME +# objref: +# kind: Service +# version: v1 +# name: webhook-service diff --git a/controlplane/nested/config/default/manager_auth_proxy_patch.yaml b/controlplane/nested/config/default/manager_auth_proxy_patch.yaml new file mode 100644 index 00000000..56e123cf --- /dev/null +++ b/controlplane/nested/config/default/manager_auth_proxy_patch.yaml @@ -0,0 +1,25 @@ +# This patch inject a sidecar container which is a HTTP proxy for the +# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: kube-rbac-proxy + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.5.0 + args: + - "--secure-listen-address=0.0.0.0:8443" + - "--upstream=http://127.0.0.1:8080/" + - "--logtostderr=true" + - "--v=10" + ports: + - containerPort: 8443 + name: https + - name: manager + args: + - "--metrics-bind-address=127.0.0.1:8080" + - "--leader-elect" diff --git a/controlplane/nested/config/default/manager_image_patch.yaml b/controlplane/nested/config/default/manager_image_patch.yaml new file mode 100644 index 00000000..a4de91cd --- /dev/null +++ b/controlplane/nested/config/default/manager_image_patch.yaml @@ -0,0 +1,11 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - image: gcr.io/nested-controlplane-controller-amd64:dev + name: manager \ No newline at end of file diff --git a/controlplane/nested/config/default/manager_pull_policy.yaml b/controlplane/nested/config/default/manager_pull_policy.yaml new file mode 100644 index 00000000..91eb7d60 --- /dev/null +++ b/controlplane/nested/config/default/manager_pull_policy.yaml @@ -0,0 +1,11 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + imagePullPolicy: Never diff --git a/controlplane/nested/config/default/manager_webhook_patch.yaml b/controlplane/nested/config/default/manager_webhook_patch.yaml new file mode 100644 index 00000000..738de350 --- /dev/null +++ b/controlplane/nested/config/default/manager_webhook_patch.yaml @@ -0,0 +1,23 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: webhook-server-cert diff --git a/controlplane/nested/config/default/webhookcainjection_patch.yaml b/controlplane/nested/config/default/webhookcainjection_patch.yaml new file mode 100644 index 00000000..7e79bf99 --- /dev/null +++ b/controlplane/nested/config/default/webhookcainjection_patch.yaml @@ -0,0 +1,15 @@ +# This patch add annotation to admission webhook config and +# the variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) will be substituted by kustomize. +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: MutatingWebhookConfiguration +metadata: + name: mutating-webhook-configuration + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) +--- +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: ValidatingWebhookConfiguration +metadata: + name: validating-webhook-configuration + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) diff --git a/controlplane/nested/config/manager/kustomization.yaml b/controlplane/nested/config/manager/kustomization.yaml new file mode 100644 index 00000000..5c5f0b84 --- /dev/null +++ b/controlplane/nested/config/manager/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- manager.yaml diff --git a/controlplane/nested/config/manager/manager.yaml b/controlplane/nested/config/manager/manager.yaml new file mode 100644 index 00000000..23b85a29 --- /dev/null +++ b/controlplane/nested/config/manager/manager.yaml @@ -0,0 +1,39 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + control-plane: controller-manager + name: system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system + labels: + control-plane: controller-manager +spec: + selector: + matchLabels: + control-plane: controller-manager + replicas: 1 + template: + metadata: + labels: + control-plane: controller-manager + spec: + containers: + - command: + - /manager + args: + - --leader-elect + image: controller:latest + name: manager + resources: + limits: + cpu: 100m + memory: 30Mi + requests: + cpu: 100m + memory: 20Mi + terminationGracePeriodSeconds: 10 diff --git a/controlplane/nested/config/rbac/auth_proxy_client_clusterrole.yaml b/controlplane/nested/config/rbac/auth_proxy_client_clusterrole.yaml new file mode 100644 index 00000000..bd4af137 --- /dev/null +++ b/controlplane/nested/config/rbac/auth_proxy_client_clusterrole.yaml @@ -0,0 +1,7 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-reader +rules: +- nonResourceURLs: ["/metrics"] + verbs: ["get"] diff --git a/controlplane/nested/config/rbac/auth_proxy_role.yaml b/controlplane/nested/config/rbac/auth_proxy_role.yaml new file mode 100644 index 00000000..618f5e41 --- /dev/null +++ b/controlplane/nested/config/rbac/auth_proxy_role.yaml @@ -0,0 +1,13 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: proxy-role +rules: +- apiGroups: ["authentication.k8s.io"] + resources: + - tokenreviews + verbs: ["create"] +- apiGroups: ["authorization.k8s.io"] + resources: + - subjectaccessreviews + verbs: ["create"] diff --git a/controlplane/nested/config/rbac/auth_proxy_role_binding.yaml b/controlplane/nested/config/rbac/auth_proxy_role_binding.yaml new file mode 100644 index 00000000..48ed1e4b --- /dev/null +++ b/controlplane/nested/config/rbac/auth_proxy_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: proxy-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: proxy-role +subjects: +- kind: ServiceAccount + name: default + namespace: system diff --git a/controlplane/nested/config/rbac/auth_proxy_service.yaml b/controlplane/nested/config/rbac/auth_proxy_service.yaml new file mode 100644 index 00000000..6cf656be --- /dev/null +++ b/controlplane/nested/config/rbac/auth_proxy_service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + control-plane: controller-manager + name: controller-manager-metrics-service + namespace: system +spec: + ports: + - name: https + port: 8443 + targetPort: https + selector: + control-plane: controller-manager diff --git a/controlplane/nested/config/rbac/kustomization.yaml b/controlplane/nested/config/rbac/kustomization.yaml new file mode 100644 index 00000000..66c28338 --- /dev/null +++ b/controlplane/nested/config/rbac/kustomization.yaml @@ -0,0 +1,12 @@ +resources: +- role.yaml +- role_binding.yaml +- leader_election_role.yaml +- leader_election_role_binding.yaml +# Comment the following 4 lines if you want to disable +# the auth proxy (https://github.com/brancz/kube-rbac-proxy) +# which protects your /metrics endpoint. +- auth_proxy_service.yaml +- auth_proxy_role.yaml +- auth_proxy_role_binding.yaml +- auth_proxy_client_clusterrole.yaml diff --git a/controlplane/nested/config/rbac/leader_election_role.yaml b/controlplane/nested/config/rbac/leader_election_role.yaml new file mode 100644 index 00000000..627cb138 --- /dev/null +++ b/controlplane/nested/config/rbac/leader_election_role.yaml @@ -0,0 +1,37 @@ +# permissions to do leader election. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: leader-election-role +rules: +- apiGroups: + - "" + - coordination.k8s.io + resources: + - configmaps + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + - coordination.k8s.io + resources: + - configmaps/status + - leases/status + verbs: + - get + - update + - patch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch diff --git a/controlplane/nested/config/rbac/leader_election_role_binding.yaml b/controlplane/nested/config/rbac/leader_election_role_binding.yaml new file mode 100644 index 00000000..eed16906 --- /dev/null +++ b/controlplane/nested/config/rbac/leader_election_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: leader-election-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: leader-election-role +subjects: +- kind: ServiceAccount + name: default + namespace: system diff --git a/config/rbac/nestedapiserver_editor_role.yaml b/controlplane/nested/config/rbac/nestedapiserver_editor_role.yaml similarity index 100% rename from config/rbac/nestedapiserver_editor_role.yaml rename to controlplane/nested/config/rbac/nestedapiserver_editor_role.yaml diff --git a/config/rbac/nestedapiserver_viewer_role.yaml b/controlplane/nested/config/rbac/nestedapiserver_viewer_role.yaml similarity index 100% rename from config/rbac/nestedapiserver_viewer_role.yaml rename to controlplane/nested/config/rbac/nestedapiserver_viewer_role.yaml diff --git a/config/rbac/nestedcontrollermanager_editor_role.yaml b/controlplane/nested/config/rbac/nestedcontrollermanager_editor_role.yaml similarity index 100% rename from config/rbac/nestedcontrollermanager_editor_role.yaml rename to controlplane/nested/config/rbac/nestedcontrollermanager_editor_role.yaml diff --git a/config/rbac/nestedcontrollermanager_viewer_role.yaml b/controlplane/nested/config/rbac/nestedcontrollermanager_viewer_role.yaml similarity index 100% rename from config/rbac/nestedcontrollermanager_viewer_role.yaml rename to controlplane/nested/config/rbac/nestedcontrollermanager_viewer_role.yaml diff --git a/config/rbac/nestedcontrolplane_editor_role.yaml b/controlplane/nested/config/rbac/nestedcontrolplane_editor_role.yaml similarity index 100% rename from config/rbac/nestedcontrolplane_editor_role.yaml rename to controlplane/nested/config/rbac/nestedcontrolplane_editor_role.yaml diff --git a/config/rbac/nestedcontrolplane_viewer_role.yaml b/controlplane/nested/config/rbac/nestedcontrolplane_viewer_role.yaml similarity index 100% rename from config/rbac/nestedcontrolplane_viewer_role.yaml rename to controlplane/nested/config/rbac/nestedcontrolplane_viewer_role.yaml diff --git a/config/rbac/nestedetcd_editor_role.yaml b/controlplane/nested/config/rbac/nestedetcd_editor_role.yaml similarity index 100% rename from config/rbac/nestedetcd_editor_role.yaml rename to controlplane/nested/config/rbac/nestedetcd_editor_role.yaml diff --git a/config/rbac/nestedetcd_viewer_role.yaml b/controlplane/nested/config/rbac/nestedetcd_viewer_role.yaml similarity index 100% rename from config/rbac/nestedetcd_viewer_role.yaml rename to controlplane/nested/config/rbac/nestedetcd_viewer_role.yaml diff --git a/controlplane/nested/config/rbac/role.yaml b/controlplane/nested/config/rbac/role.yaml new file mode 100644 index 00000000..d1c8bc4d --- /dev/null +++ b/controlplane/nested/config/rbac/role.yaml @@ -0,0 +1,150 @@ + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: manager-role +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + - apps + resources: + - services + - statefulsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + - apps + resources: + - services/status + - statefulsets/status + verbs: + - get + - patch + - update +- apiGroups: + - cluster.x-k8s.io + resources: + - clusters + verbs: + - get + - list + - watch +- apiGroups: + - controlplane.cluster.x-k8s.io + resources: + - nestedapiservers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - controlplane.cluster.x-k8s.io + resources: + - nestedapiservers/finalizers + verbs: + - update +- apiGroups: + - controlplane.cluster.x-k8s.io + resources: + - nestedapiservers/status + verbs: + - get + - patch + - update +- apiGroups: + - controlplane.cluster.x-k8s.io + resources: + - nestedcontrollermanagers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - controlplane.cluster.x-k8s.io + resources: + - nestedcontrollermanagers/finalizers + verbs: + - update +- apiGroups: + - controlplane.cluster.x-k8s.io + resources: + - nestedcontrollermanagers/status + verbs: + - get + - patch + - update +- apiGroups: + - controlplane.cluster.x-k8s.io + resources: + - nestedcontrolplanes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - controlplane.cluster.x-k8s.io + resources: + - nestedcontrolplanes/status + verbs: + - get + - patch + - update +- apiGroups: + - controlplane.cluster.x-k8s.io + resources: + - nestedetcds + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - controlplane.cluster.x-k8s.io + resources: + - nestedetcds/finalizers + verbs: + - update +- apiGroups: + - controlplane.cluster.x-k8s.io + resources: + - nestedetcds/status + verbs: + - get + - patch + - update diff --git a/controlplane/nested/config/rbac/role_binding.yaml b/controlplane/nested/config/rbac/role_binding.yaml new file mode 100644 index 00000000..8f265870 --- /dev/null +++ b/controlplane/nested/config/rbac/role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: manager-role +subjects: +- kind: ServiceAccount + name: default + namespace: system diff --git a/controlplane/nested/config/webhook/kustomization.yaml b/controlplane/nested/config/webhook/kustomization.yaml new file mode 100644 index 00000000..9cf26134 --- /dev/null +++ b/controlplane/nested/config/webhook/kustomization.yaml @@ -0,0 +1,6 @@ +resources: +- manifests.yaml +- service.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/controlplane/nested/config/webhook/kustomizeconfig.yaml b/controlplane/nested/config/webhook/kustomizeconfig.yaml new file mode 100644 index 00000000..25e21e3c --- /dev/null +++ b/controlplane/nested/config/webhook/kustomizeconfig.yaml @@ -0,0 +1,25 @@ +# the following config is for teaching kustomize where to look at when substituting vars. +# It requires kustomize v2.1.0 or newer to work properly. +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + - kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + +namespace: +- kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true +- kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true + +varReference: +- path: metadata/annotations diff --git a/controlplane/nested/config/webhook/service.yaml b/controlplane/nested/config/webhook/service.yaml new file mode 100644 index 00000000..31e0f829 --- /dev/null +++ b/controlplane/nested/config/webhook/service.yaml @@ -0,0 +1,12 @@ + +apiVersion: v1 +kind: Service +metadata: + name: webhook-service + namespace: system +spec: + ports: + - port: 443 + targetPort: 9443 + selector: + control-plane: controller-manager diff --git a/controllers/controlplane/consts.go b/controlplane/nested/controllers/consts.go similarity index 54% rename from controllers/controlplane/consts.go rename to controlplane/nested/controllers/consts.go index d1e57ae4..95c8b107 100644 --- a/controllers/controlplane/consts.go +++ b/controlplane/nested/controllers/consts.go @@ -14,20 +14,15 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controlplane +package controllers const ( statefulsetOwnerKeyNEtcd = ".metadata.netcd.controller" statefulsetOwnerKeyNKas = ".metadata.nkas.controller" statefulsetOwnerKeyNKcm = ".metadata.nkcm.controller" - defaultEtcdStatefulSetURL = "./config/component-templates/" + - "nested-etcd/nested-etcd-statefulset-template.yaml" - defaultEtcdServiceURL = "./config/component-templates/" + - "nested-etcd/nested-etcd-service-template.yaml" - defaultKASStatefulSetURL = "./config/component-templates/" + - "nested-apiserver/nested-apiserver-statefulset-template.yaml" - defaultKASServiceURL = "./config/component-templates/" + - "nested-apiserver/nested-apiserver-service-template.yaml" - defaultKCMStatefulSetURL = "./config/component-templates/" + - "nested-controllermanager/nested-controllermanager-statefulset-template.yaml" + defaultEtcdStatefulSetURL = "/nested-etcd/nested-etcd-statefulset-template.yaml" + defaultEtcdServiceURL = "/nested-etcd/nested-etcd-service-template.yaml" + defaultKASStatefulSetURL = "/nested-apiserver/nested-apiserver-statefulset-template.yaml" + defaultKASServiceURL = "/nested-apiserver/nested-apiserver-service-template.yaml" + defaultKCMStatefulSetURL = "/nested-controllermanager/nested-controllermanager-statefulset-template.yaml" ) diff --git a/controllers/controlplane/controller_util.go b/controlplane/nested/controllers/controller_util.go similarity index 81% rename from controllers/controlplane/controller_util.go rename to controlplane/nested/controllers/controller_util.go index 1177c353..0e3d216a 100644 --- a/controllers/controlplane/controller_util.go +++ b/controlplane/nested/controllers/controller_util.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controlplane +package controllers import ( "bytes" @@ -36,24 +36,25 @@ import ( "k8s.io/client-go/kubernetes/scheme" ctrlcli "sigs.k8s.io/controller-runtime/pkg/client" - clusterv1 "sigs.k8s.io/cluster-api-provider-nested/apis/controlplane/v1alpha4" + controlplanev1 "sigs.k8s.io/cluster-api-provider-nested/controlplane/nested/api/v1alpha4" addonv1alpha1 "sigs.k8s.io/kubebuilder-declarative-pattern/pkg/patterns/addon/pkg/apis/v1alpha1" ) +// +kubebuilder:rbac:groups="";apps,resources=services;statefulsets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="";apps,resources=services/status;statefulsets/status,verbs=get;update;patch + // createNestedComponentSts will create the StatefulSet that runs the // NestedComponent func createNestedComponentSts(ctx context.Context, cli ctrlcli.Client, ncMeta metav1.ObjectMeta, - ncSpec clusterv1.NestedComponentSpec, - ncKind clusterv1.ComponentKind, - controlPlaneName, clusterName string, log logr.Logger) error { - var ( - ncSts *appsv1.StatefulSet - ncSvc *corev1.Service - ) + ncSpec controlplanev1.NestedComponentSpec, + ncKind controlplanev1.ComponentKind, + controlPlaneName, clusterName, templatePath string, log logr.Logger) error { + ncSts := &appsv1.StatefulSet{} + ncSvc := &corev1.Service{} // Setup the ownerReferences for all objects or := metav1.NewControllerRef(&ncMeta, - clusterv1.GroupVersion.WithKind(string(ncKind))) + controlplanev1.GroupVersion.WithKind(string(ncKind))) // 1. Using the template defined by version/channel to create the // StatefulSet and the Service @@ -64,13 +65,13 @@ func createNestedComponentSts(ctx context.Context, log.V(4).Info("The Version and Channel are not set, " + "will use the default template.") - if err := genStatefulSetObject(ncMeta, ncSpec, ncKind, controlPlaneName, clusterName, log, ncSts); err != nil { + if err := genStatefulSetObject(templatePath, ncMeta, ncSpec, ncKind, controlPlaneName, clusterName, log, ncSts); err != nil { return fmt.Errorf("fail to generate the Statefulset object: %v", err) } - if ncKind != clusterv1.ControllerManager { + if ncKind != controlplanev1.ControllerManager { // no need to create the service for the NestedControllerManager - if err := genServiceObject(ncMeta, ncSpec, ncKind, controlPlaneName, clusterName, log, ncSvc); err != nil { + if err := genServiceObject(templatePath, ncMeta, ncSpec, ncKind, controlPlaneName, clusterName, log, ncSvc); err != nil { return fmt.Errorf("fail to generate the Service object: %v", err) } @@ -91,16 +92,18 @@ func createNestedComponentSts(ctx context.Context, // genServiceObject generates the Service object corresponding to the // NestedComponent -func genServiceObject(ncMeta metav1.ObjectMeta, - ncSpec clusterv1.NestedComponentSpec, ncKind clusterv1.ComponentKind, +func genServiceObject( + templatePath string, + ncMeta metav1.ObjectMeta, + ncSpec controlplanev1.NestedComponentSpec, ncKind controlplanev1.ComponentKind, controlPlaneName, clusterName string, log logr.Logger, svc *corev1.Service) error { var templateURL string if ncSpec.Version == "" && ncSpec.Channel == "" { switch ncKind { - case clusterv1.APIServer: - templateURL = defaultKASServiceURL - case clusterv1.Etcd: - templateURL = defaultEtcdServiceURL + case controlplanev1.APIServer: + templateURL = templatePath + defaultKASServiceURL + case controlplanev1.Etcd: + templateURL = templatePath + defaultEtcdServiceURL default: panic("Unreachable") } @@ -131,21 +134,22 @@ func genServiceObject(ncMeta metav1.ObjectMeta, // genStatefulSetObject generates the StatefulSet object corresponding to the // NestedComponent func genStatefulSetObject( + templatePath string, ncMeta metav1.ObjectMeta, - ncSpec clusterv1.NestedComponentSpec, - ncKind clusterv1.ComponentKind, controlPlaneName, clusterName string, + ncSpec controlplanev1.NestedComponentSpec, + ncKind controlplanev1.ComponentKind, controlPlaneName, clusterName string, log logr.Logger, ncSts *appsv1.StatefulSet) error { var templateURL string if ncSpec.Version == "" && ncSpec.Channel == "" { log.V(4).Info("The Version and Channel are not set, " + "will use the default template.") switch ncKind { - case clusterv1.APIServer: - templateURL = defaultKASStatefulSetURL - case clusterv1.Etcd: - templateURL = defaultEtcdStatefulSetURL - case clusterv1.ControllerManager: - templateURL = defaultKCMStatefulSetURL + case controlplanev1.APIServer: + templateURL = templatePath + defaultKASStatefulSetURL + case controlplanev1.Etcd: + templateURL = templatePath + defaultEtcdStatefulSetURL + case controlplanev1.ControllerManager: + templateURL = templatePath + defaultKCMStatefulSetURL default: panic("Unreachable") } @@ -187,7 +191,7 @@ func genStatefulSetObject( "StatefulSet", ncSts.GetName()) // 6 set the "--initial-cluster" command line flag for the Etcd container - if ncKind == clusterv1.Etcd { + if ncKind == controlplanev1.Etcd { icaVal := genInitialClusterArgs(1, clusterName, clusterName, ncMeta.GetNamespace()) stsArgs := append(ncSts.Spec.Template.Spec.Containers[0].Args, "--initial-cluster", icaVal) @@ -257,7 +261,7 @@ func getOwner(ncMeta metav1.ObjectMeta) metav1.OwnerReference { return metav1.OwnerReference{} } for _, owner := range owners { - if owner.APIVersion == clusterv1.GroupVersion.String() && + if owner.APIVersion == controlplanev1.GroupVersion.String() && owner.Kind == "NestedControlPlane" { return owner } @@ -268,7 +272,7 @@ func getOwner(ncMeta metav1.ObjectMeta) metav1.OwnerReference { // genAPIServerSvcRef generates the ObjectReference that points to the // APISrver service func genAPIServerSvcRef(cli ctrlcli.Client, - nkas clusterv1.NestedAPIServer, clusterName string) (corev1.ObjectReference, error) { + nkas controlplanev1.NestedAPIServer, clusterName string) (corev1.ObjectReference, error) { var ( svc corev1.Service objRef corev1.ObjectReference @@ -295,5 +299,5 @@ func genObjRefFromObj(obj ctrlcli.Object) corev1.ObjectReference { } func IsComponentReady(status addonv1alpha1.CommonStatus) bool { - return status.Phase == string(clusterv1.Ready) + return status.Phase == string(controlplanev1.Ready) } diff --git a/controllers/controlplane/controller_util_test.go b/controlplane/nested/controllers/controller_util_test.go similarity index 90% rename from controllers/controlplane/controller_util_test.go rename to controlplane/nested/controllers/controller_util_test.go index 4c7c1716..14d9c42e 100644 --- a/controllers/controlplane/controller_util_test.go +++ b/controlplane/nested/controllers/controller_util_test.go @@ -14,14 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controlplane +package controllers import ( "reflect" "testing" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - controlplanev1alpha4 "sigs.k8s.io/cluster-api-provider-nested/apis/controlplane/v1alpha4" + controlplanev1 "sigs.k8s.io/cluster-api-provider-nested/controlplane/nested/api/v1alpha4" ) const ( @@ -76,12 +76,12 @@ func TestSubstituteTemplate(t *testing.T) { func TestGetOwner(t *testing.T) { tests := []struct { name string - netcd controlplanev1alpha4.NestedEtcd + netcd controlplanev1.NestedEtcd expect metav1.OwnerReference }{ { "no owner", - controlplanev1alpha4.NestedEtcd{ + controlplanev1.NestedEtcd{ ObjectMeta: metav1.ObjectMeta{ Name: "test-netcd", OwnerReferences: []metav1.OwnerReference{}, @@ -91,7 +91,7 @@ func TestGetOwner(t *testing.T) { }, { "owner APIVersion not matched", - controlplanev1alpha4.NestedEtcd{ + controlplanev1.NestedEtcd{ ObjectMeta: metav1.ObjectMeta{ Name: "test-netcd", OwnerReferences: []metav1.OwnerReference{ @@ -108,12 +108,12 @@ func TestGetOwner(t *testing.T) { }, { "owner kind not matched", - controlplanev1alpha4.NestedEtcd{ + controlplanev1.NestedEtcd{ ObjectMeta: metav1.ObjectMeta{ Name: "test-netcd", OwnerReferences: []metav1.OwnerReference{ { - APIVersion: controlplanev1alpha4.GroupVersion.String(), + APIVersion: controlplanev1.GroupVersion.String(), Kind: "test-kind", Name: "test-name", UID: "xxxxx-xxxxx-xxxxx-xxxxx", @@ -125,12 +125,12 @@ func TestGetOwner(t *testing.T) { }, { "owner found", - controlplanev1alpha4.NestedEtcd{ + controlplanev1.NestedEtcd{ ObjectMeta: metav1.ObjectMeta{ Name: "test-netcd", OwnerReferences: []metav1.OwnerReference{ { - APIVersion: controlplanev1alpha4.GroupVersion.String(), + APIVersion: controlplanev1.GroupVersion.String(), Kind: "NestedControlPlane", Name: "test-name", UID: "xxxxx-xxxxx-xxxxx-xxxxx", @@ -139,7 +139,7 @@ func TestGetOwner(t *testing.T) { }, }, metav1.OwnerReference{ - APIVersion: controlplanev1alpha4.GroupVersion.String(), + APIVersion: controlplanev1.GroupVersion.String(), Kind: "NestedControlPlane", Name: "test-name", UID: "xxxxx-xxxxx-xxxxx-xxxxx", diff --git a/controllers/controlplane/nestedapiserver_controller.go b/controlplane/nested/controllers/nestedapiserver_controller.go similarity index 82% rename from controllers/controlplane/nestedapiserver_controller.go rename to controlplane/nested/controllers/nestedapiserver_controller.go index 1afe4a3c..1a4c60c3 100644 --- a/controllers/controlplane/nestedapiserver_controller.go +++ b/controlplane/nested/controllers/nestedapiserver_controller.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controlplane +package controllers import ( "context" @@ -33,30 +33,27 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ctrlcli "sigs.k8s.io/controller-runtime/pkg/client" - clusterv1 "sigs.k8s.io/cluster-api-provider-nested/apis/controlplane/v1alpha4" - "sigs.k8s.io/cluster-api-provider-nested/certificate" - clusterv1alpha4 "sigs.k8s.io/cluster-api/api/v1alpha4" + controlplanev1 "sigs.k8s.io/cluster-api-provider-nested/controlplane/nested/api/v1alpha4" + "sigs.k8s.io/cluster-api-provider-nested/controlplane/nested/certificate" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4" ) // NestedAPIServerReconciler reconciles a NestedAPIServer object type NestedAPIServerReconciler struct { client.Client - Log logr.Logger - Scheme *runtime.Scheme + Log logr.Logger + Scheme *runtime.Scheme + TemplatePath string } -//+kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedapiservers,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedapiservers/status,verbs=get;update;patch -//+kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedapiservers/finalizers,verbs=update -//+kubebuilder:rbac:groups=apps,resources=statefulset,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=apps,resources=statefulset/status,verbs=get;update;patch -//+kubebuilder:rbac:groups=,resources=service,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=,resources=service/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedapiservers,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedapiservers/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedapiservers/finalizers,verbs=update func (r *NestedAPIServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.Log.WithValues("nestedapiserver", req.NamespacedName) log.Info("Reconciling NestedAPIServer...") - var nkas clusterv1.NestedAPIServer + var nkas controlplanev1.NestedAPIServer if err := r.Get(ctx, req.NamespacedName, &nkas); err != nil { return ctrl.Result{}, ctrlcli.IgnoreNotFound(err) } @@ -76,7 +73,7 @@ func (r *NestedAPIServerReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{Requeue: true}, nil } - var ncp clusterv1.NestedControlPlane + var ncp controlplanev1.NestedControlPlane if err := r.Get(ctx, types.NamespacedName{Namespace: nkas.GetNamespace(), Name: owner.Name}, &ncp); err != nil { log.Info("the owner could not be found, will retry later", "namespace", nkas.GetNamespace(), @@ -101,7 +98,7 @@ func (r *NestedAPIServerReconciler) Reconcile(ctx context.Context, req ctrl.Requ // as the statefulset is not found, mark the NestedAPIServer as unready if IsComponentReady(nkas.Status.CommonStatus) { nkas.Status.Phase = - string(clusterv1.Unready) + string(controlplanev1.Unready) log.V(5).Info("The corresponding statefulset is not found, " + "will mark the NestedAPIServer as unready") if err := r.Status().Update(ctx, &nkas); err != nil { @@ -117,7 +114,7 @@ func (r *NestedAPIServerReconciler) Reconcile(ctx context.Context, req ctrl.Requ // the statefulset is not found, create one if err := createNestedComponentSts(ctx, r.Client, nkas.ObjectMeta, nkas.Spec.NestedComponentSpec, - clusterv1.APIServer, owner.Name, cluster.GetName(), log); err != nil { + controlplanev1.APIServer, owner.Name, cluster.GetName(), r.TemplatePath, log); err != nil { log.Error(err, "fail to create NestedAPIServer StatefulSet") return ctrl.Result{}, err } @@ -135,7 +132,7 @@ func (r *NestedAPIServerReconciler) Reconcile(ctx context.Context, req ctrl.Requ if !IsComponentReady(nkas.Status.CommonStatus) { // As the NestedAPIServer StatefulSet is ready, update // NestedAPIServer status - nkas.Status.Phase = string(clusterv1.Ready) + nkas.Status.Phase = string(controlplanev1.Ready) objRef, err := genAPIServerSvcRef(r.Client, nkas, cluster.GetName()) if err != nil { log.Error(err, "fail to generate NestedAPIServer Service Reference") @@ -157,7 +154,7 @@ func (r *NestedAPIServerReconciler) Reconcile(ctx context.Context, req ctrl.Requ // mark the NestedAPIServer as unready, if the NestedAPIServer // StatefulSet is unready, if IsComponentReady(nkas.Status.CommonStatus) { - nkas.Status.Phase = string(clusterv1.Unready) + nkas.Status.Phase = string(controlplanev1.Unready) if err := r.Status().Update(ctx, &nkas); err != nil { log.Error(err, "fail to update NestedAPIServer Object") return ctrl.Result{}, err @@ -181,8 +178,8 @@ func (r *NestedAPIServerReconciler) SetupWithManager(mgr ctrl.Manager) error { return nil } // make sure it's a NestedAPIServer - if owner.APIVersion != clusterv1.GroupVersion.String() || - owner.Kind != string(clusterv1.APIServer) { + if owner.APIVersion != controlplanev1.GroupVersion.String() || + owner.Kind != string(controlplanev1.APIServer) { return nil } @@ -192,13 +189,13 @@ func (r *NestedAPIServerReconciler) SetupWithManager(mgr ctrl.Manager) error { return err } return ctrl.NewControllerManagedBy(mgr). - For(&clusterv1.NestedAPIServer{}). + For(&controlplanev1.NestedAPIServer{}). Owns(&appsv1.StatefulSet{}). Complete(r) } // createAPIServerClientCrts will find of create client certs for the etcd cluster -func (r *NestedAPIServerReconciler) createAPIServerClientCrts(ctx context.Context, cluster *clusterv1alpha4.Cluster, ncp *clusterv1.NestedControlPlane, nkas *clusterv1.NestedAPIServer) error { +func (r *NestedAPIServerReconciler) createAPIServerClientCrts(ctx context.Context, cluster *clusterv1.Cluster, ncp *controlplanev1.NestedControlPlane, nkas *controlplanev1.NestedAPIServer) error { certificates := secret.NewCertificatesForInitialControlPlane(nil) if err := certificates.Lookup(ctx, r.Client, util.ObjectKey(cluster)); err != nil { return err @@ -255,7 +252,7 @@ func (r *NestedAPIServerReconciler) createAPIServerClientCrts(ctx context.Contex frontProxyKeyPair, } - controllerRef := metav1.NewControllerRef(ncp, clusterv1.GroupVersion.WithKind("NestedControlPlane")) + controllerRef := metav1.NewControllerRef(ncp, controlplanev1.GroupVersion.WithKind("NestedControlPlane")) if err := certs.LookupOrSave(ctx, r.Client, util.ObjectKey(cluster), *controllerRef); err != nil { return err } diff --git a/controllers/controlplane/nestedcontrollermanager_controller.go b/controlplane/nested/controllers/nestedcontrollermanager_controller.go similarity index 81% rename from controllers/controlplane/nestedcontrollermanager_controller.go rename to controlplane/nested/controllers/nestedcontrollermanager_controller.go index ebec066e..eb631534 100644 --- a/controllers/controlplane/nestedcontrollermanager_controller.go +++ b/controlplane/nested/controllers/nestedcontrollermanager_controller.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controlplane +package controllers import ( "context" @@ -30,27 +30,25 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ctrlcli "sigs.k8s.io/controller-runtime/pkg/client" - clusterv1 "sigs.k8s.io/cluster-api-provider-nested/apis/controlplane/v1alpha4" - controlplanev1alpha4 "sigs.k8s.io/cluster-api-provider-nested/apis/controlplane/v1alpha4" + controlplanev1 "sigs.k8s.io/cluster-api-provider-nested/controlplane/nested/api/v1alpha4" ) // NestedControllerManagerReconciler reconciles a NestedControllerManager object type NestedControllerManagerReconciler struct { client.Client - Log logr.Logger - Scheme *runtime.Scheme + Log logr.Logger + Scheme *runtime.Scheme + TemplatePath string } -//+kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedcontrollermanagers,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedcontrollermanagers/status,verbs=get;update;patch -//+kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedcontrollermanagers/finalizers,verbs=update -//+kubebuilder:rbac:groups=apps,resources=statefulset,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=apps,resources=statefulset/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedcontrollermanagers,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedcontrollermanagers/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedcontrollermanagers/finalizers,verbs=update func (r *NestedControllerManagerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.Log.WithValues("nestedcontrollermanager", req.NamespacedName) log.Info("Reconciling NestedControllerManager...") - var nkcm clusterv1.NestedControllerManager + var nkcm controlplanev1.NestedControllerManager if err := r.Get(ctx, req.NamespacedName, &nkcm); err != nil { return ctrl.Result{}, ctrlcli.IgnoreNotFound(err) } @@ -70,7 +68,7 @@ func (r *NestedControllerManagerReconciler) Reconcile(ctx context.Context, req c return ctrl.Result{Requeue: true}, nil } - var ncp clusterv1.NestedControlPlane + var ncp controlplanev1.NestedControlPlane if err := r.Get(ctx, types.NamespacedName{Namespace: nkcm.GetNamespace(), Name: owner.Name}, &ncp); err != nil { log.Info("the owner could not be found, will retry later", "namespace", nkcm.GetNamespace(), @@ -96,7 +94,7 @@ func (r *NestedControllerManagerReconciler) Reconcile(ctx context.Context, req c // as unready if IsComponentReady(nkcm.Status.CommonStatus) { nkcm.Status.Phase = - string(clusterv1.Unready) + string(controlplanev1.Unready) log.V(5).Info("The corresponding statefulset is not found, " + "will mark the NestedControllerManager as unready") if err := r.Status().Update(ctx, &nkcm); err != nil { @@ -107,7 +105,7 @@ func (r *NestedControllerManagerReconciler) Reconcile(ctx context.Context, req c // the statefulset is not found, create one if err := createNestedComponentSts(ctx, r.Client, nkcm.ObjectMeta, nkcm.Spec.NestedComponentSpec, - clusterv1.ControllerManager, owner.Name, cluster.GetName(), log); err != nil { + controlplanev1.ControllerManager, owner.Name, cluster.GetName(), r.TemplatePath, log); err != nil { log.Error(err, "fail to create NestedControllerManager StatefulSet") return ctrl.Result{}, err } @@ -125,7 +123,7 @@ func (r *NestedControllerManagerReconciler) Reconcile(ctx context.Context, req c if !IsComponentReady(nkcm.Status.CommonStatus) { // As the NestedControllerManager StatefulSet is ready, update // NestedControllerManager status - nkcm.Status.Phase = string(clusterv1.Ready) + nkcm.Status.Phase = string(controlplanev1.Ready) log.V(5).Info("The corresponding statefulset is ready, " + "will mark the NestedControllerManager as ready") if err := r.Status().Update(ctx, &nkcm); err != nil { @@ -140,7 +138,7 @@ func (r *NestedControllerManagerReconciler) Reconcile(ctx context.Context, req c // mark the NestedControllerManager as unready, if the NestedControllerManager // StatefulSet is unready, if IsComponentReady(nkcm.Status.CommonStatus) { - nkcm.Status.Phase = string(clusterv1.Unready) + nkcm.Status.Phase = string(controlplanev1.Unready) if err := r.Status().Update(ctx, &nkcm); err != nil { log.Error(err, "fail to update NestedControllerManager Object") return ctrl.Result{}, err @@ -164,8 +162,8 @@ func (r *NestedControllerManagerReconciler) SetupWithManager(mgr ctrl.Manager) e return nil } // make sure it's a NestedControllerManager - if owner.APIVersion != clusterv1.GroupVersion.String() || - owner.Kind != string(clusterv1.ControllerManager) { + if owner.APIVersion != controlplanev1.GroupVersion.String() || + owner.Kind != string(controlplanev1.ControllerManager) { return nil } @@ -175,7 +173,7 @@ func (r *NestedControllerManagerReconciler) SetupWithManager(mgr ctrl.Manager) e return err } return ctrl.NewControllerManagedBy(mgr). - For(&controlplanev1alpha4.NestedControllerManager{}). + For(&controlplanev1.NestedControllerManager{}). Owns(&appsv1.StatefulSet{}). Complete(r) } diff --git a/controllers/controlplane/nestedcontrolplane_controller.go b/controlplane/nested/controllers/nestedcontrolplane_controller.go similarity index 94% rename from controllers/controlplane/nestedcontrolplane_controller.go rename to controlplane/nested/controllers/nestedcontrolplane_controller.go index eeaf2379..14919244 100644 --- a/controllers/controlplane/nestedcontrolplane_controller.go +++ b/controlplane/nested/controllers/nestedcontrolplane_controller.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controlplane +package controllers import ( "context" @@ -27,8 +27,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/cluster-api-provider-nested/apis/controlplane/v1alpha4" - controlplanev1 "sigs.k8s.io/cluster-api-provider-nested/apis/controlplane/v1alpha4" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4" kcpv1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1alpha4" "sigs.k8s.io/cluster-api/util" @@ -42,11 +41,15 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" addonv1alpha1 "sigs.k8s.io/kubebuilder-declarative-pattern/pkg/patterns/addon/pkg/apis/v1alpha1" + + controlplanev1 "sigs.k8s.io/cluster-api-provider-nested/controlplane/nested/api/v1alpha4" ) +// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters,verbs=get;list;watch // +kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedcontrolplanes,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedcontrolplanes/status,verbs=get;update;patch -//+kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedcontrollermanagers/finalizers,verbs=update +// +kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedcontrollermanagers/finalizers,verbs=update +// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete // NestedControlPlaneReconciler reconciles a NestedControlPlane object type NestedControlPlaneReconciler struct { @@ -70,7 +73,7 @@ func (r *NestedControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R log := r.Log.WithValues("nestedcontrolplane", req.NamespacedName) log.Info("Reconciling NestedControlPlane...") // Fetch the NestedControlPlane - ncp := &v1alpha4.NestedControlPlane{} + ncp := &controlplanev1.NestedControlPlane{} if err := r.Get(ctx, req.NamespacedName, ncp); err != nil { // check for not found and don't requeue if apierrors.IsNotFound(err) { @@ -84,7 +87,7 @@ func (r *NestedControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R cluster, err := ncp.GetOwnerCluster(ctx, r.Client) if err != nil || cluster == nil { log.Error(err, "Failed to retrieve owner Cluster from the API Server") - return ctrl.Result{}, err + return ctrl.Result{Requeue: true}, err } log = log.WithValues("cluster", cluster.Name) @@ -112,7 +115,7 @@ func (r *NestedControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R return ctrl.Result{}, err } - return ctrl.Result{}, nil + return ctrl.Result{Requeue: true}, nil } // TODO(christopherhein) handle deletion @@ -259,6 +262,8 @@ func (r *NestedControlPlaneReconciler) reconcile(ctx context.Context, log logr.L if err := r.Status().Update(ctx, ncp); err != nil { return ctrl.Result{}, err } + } else if !ncp.Status.Ready && len(isReady) < 3 { + return ctrl.Result{Requeue: true}, nil } return ctrl.Result{}, nil diff --git a/controllers/controlplane/nestedetcd_controller.go b/controlplane/nested/controllers/nestedetcd_controller.go similarity index 84% rename from controllers/controlplane/nestedetcd_controller.go rename to controlplane/nested/controllers/nestedetcd_controller.go index 2d69ec76..9de7fafd 100644 --- a/controllers/controlplane/nestedetcd_controller.go +++ b/controlplane/nested/controllers/nestedetcd_controller.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controlplane +package controllers import ( "context" @@ -28,36 +28,34 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - clusterv1alpha4 "sigs.k8s.io/cluster-api/api/v1alpha4" + controlplanev1alpha4 "sigs.k8s.io/cluster-api/api/v1alpha4" "sigs.k8s.io/cluster-api/util/certs" "sigs.k8s.io/cluster-api/util/secret" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" ctrlcli "sigs.k8s.io/controller-runtime/pkg/client" - clusterv1 "sigs.k8s.io/cluster-api-provider-nested/apis/controlplane/v1alpha4" - "sigs.k8s.io/cluster-api-provider-nested/certificate" + controlplanev1 "sigs.k8s.io/cluster-api-provider-nested/controlplane/nested/api/v1alpha4" + "sigs.k8s.io/cluster-api-provider-nested/controlplane/nested/certificate" "sigs.k8s.io/cluster-api/util" ) // NestedEtcdReconciler reconciles a NestedEtcd object type NestedEtcdReconciler struct { client.Client - Log logr.Logger - Scheme *runtime.Scheme + Log logr.Logger + Scheme *runtime.Scheme + TemplatePath string } -//+kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedetcds,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedetcds/status,verbs=get;update;patch -//+kubebuilder:rbac:groups=apps,resources=statefulset,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=apps,resources=statefulset/status,verbs=get;update;patch -//+kubebuilder:rbac:groups=,resources=service,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=,resources=service/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedetcds,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedetcds/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=nestedetcds/finalizers,verbs=update func (r *NestedEtcdReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.Log.WithValues("nestedetcd", req.NamespacedName) log.Info("Reconciling NestedEtcd...") - var netcd clusterv1.NestedEtcd + var netcd controlplanev1.NestedEtcd if err := r.Get(ctx, req.NamespacedName, &netcd); err != nil { return ctrl.Result{}, ctrlcli.IgnoreNotFound(err) } @@ -76,7 +74,7 @@ func (r *NestedEtcdReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{Requeue: true}, nil } - var ncp clusterv1.NestedControlPlane + var ncp controlplanev1.NestedControlPlane if err := r.Get(ctx, types.NamespacedName{Namespace: netcd.GetNamespace(), Name: owner.Name}, &ncp); err != nil { log.Info("the owner could not be found, will retry later", "namespace", netcd.GetNamespace(), @@ -100,7 +98,7 @@ func (r *NestedEtcdReconciler) Reconcile(ctx context.Context, req ctrl.Request) // as the statefulset is not found, mark the NestedEtcd as unready if IsComponentReady(netcd.Status.CommonStatus) { netcd.Status.Phase = - string(clusterv1.Unready) + string(controlplanev1.Unready) log.V(5).Info("The corresponding statefulset is not found, " + "will mark the NestedEtcd as unready") if err := r.Status().Update(ctx, &netcd); err != nil { @@ -118,7 +116,7 @@ func (r *NestedEtcdReconciler) Reconcile(ctx context.Context, req ctrl.Request) if err := createNestedComponentSts(ctx, r.Client, netcd.ObjectMeta, netcd.Spec.NestedComponentSpec, - clusterv1.Etcd, owner.Name, cluster.GetName(), log); err != nil { + controlplanev1.Etcd, owner.Name, cluster.GetName(), r.TemplatePath, log); err != nil { log.Error(err, "fail to create NestedEtcd StatefulSet") return ctrl.Result{}, err } @@ -138,8 +136,8 @@ func (r *NestedEtcdReconciler) Reconcile(ctx context.Context, req ctrl.Request) log.Error(err, "fail to get NestedEtcd Service ClusterIP") return ctrl.Result{}, err } - netcd.Status.Phase = string(clusterv1.Ready) - netcd.Status.Addresses = []clusterv1.NestedEtcdAddress{ + netcd.Status.Phase = string(controlplanev1.Ready) + netcd.Status.Addresses = []controlplanev1.NestedEtcdAddress{ { IP: ip, Port: 2379, @@ -160,7 +158,7 @@ func (r *NestedEtcdReconciler) Reconcile(ctx context.Context, req ctrl.Request) // As the NestedEtcd StatefulSet is unready, mark the NestedEtcd as unready // if its current status is ready if IsComponentReady(netcd.Status.CommonStatus) { - netcd.Status.Phase = string(clusterv1.Unready) + netcd.Status.Phase = string(controlplanev1.Unready) if err := r.Status().Update(ctx, &netcd); err != nil { log.Error(err, "fail to update NestedEtcd Object") return ctrl.Result{}, err @@ -184,7 +182,7 @@ func (r *NestedEtcdReconciler) SetupWithManager(mgr ctrl.Manager) error { return nil } // make sure it's a NestedEtcd - if owner.APIVersion != clusterv1.GroupVersion.String() || + if owner.APIVersion != controlplanev1.GroupVersion.String() || owner.Kind != "NestedEtcd" { return nil } @@ -196,13 +194,13 @@ func (r *NestedEtcdReconciler) SetupWithManager(mgr ctrl.Manager) error { } return ctrl.NewControllerManagedBy(mgr). - For(&clusterv1.NestedEtcd{}). + For(&controlplanev1.NestedEtcd{}). Owns(&appsv1.StatefulSet{}). Complete(r) } func getNestedEtcdSvcClusterIP(ctx context.Context, cli ctrlcli.Client, - clusterName string, netcd *clusterv1.NestedEtcd) (string, error) { + clusterName string, netcd *controlplanev1.NestedEtcd) (string, error) { var svc corev1.Service if err := cli.Get(ctx, types.NamespacedName{ Namespace: netcd.GetNamespace(), @@ -241,7 +239,7 @@ func getEtcdServers(name, namespace string, replicas int32) (etcdServers []strin } // createEtcdClientCrts will find of create client certs for the etcd cluster -func (r *NestedEtcdReconciler) createEtcdClientCrts(ctx context.Context, cluster *clusterv1alpha4.Cluster, ncp *clusterv1.NestedControlPlane, netcd *clusterv1.NestedEtcd) error { +func (r *NestedEtcdReconciler) createEtcdClientCrts(ctx context.Context, cluster *controlplanev1alpha4.Cluster, ncp *controlplanev1.NestedControlPlane, netcd *controlplanev1.NestedEtcd) error { certificates := secret.NewCertificatesForInitialControlPlane(nil) if err := certificates.Lookup(ctx, r.Client, util.ObjectKey(cluster)); err != nil { return err @@ -276,7 +274,7 @@ func (r *NestedEtcdReconciler) createEtcdClientCrts(ctx context.Context, cluster etcdHealthKeyPair, } - controllerRef := metav1.NewControllerRef(ncp, clusterv1.GroupVersion.WithKind("NestedControlPlane")) + controllerRef := metav1.NewControllerRef(ncp, controlplanev1.GroupVersion.WithKind("NestedControlPlane")) if err := certs.LookupOrSave(ctx, r.Client, util.ObjectKey(cluster), *controllerRef); err != nil { return err } diff --git a/controllers/controlplane/suite_test.go b/controlplane/nested/controllers/suite_test.go similarity index 83% rename from controllers/controlplane/suite_test.go rename to controlplane/nested/controllers/suite_test.go index 50a0118d..15b4c00a 100644 --- a/controllers/controlplane/suite_test.go +++ b/controlplane/nested/controllers/suite_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controlplane +package controllers import ( "path/filepath" @@ -30,7 +30,7 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" - controlplanev1alpha4 "sigs.k8s.io/cluster-api-provider-nested/apis/controlplane/v1alpha4" + controlplanev1 "sigs.k8s.io/cluster-api-provider-nested/controlplane/nested/api/v1alpha4" // +kubebuilder:scaffold:imports ) @@ -54,7 +54,7 @@ var _ = BeforeSuite(func(done Done) { By("bootstrapping test environment") testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, } var err error @@ -62,16 +62,16 @@ var _ = BeforeSuite(func(done Done) { Expect(err).ToNot(HaveOccurred()) Expect(cfg).ToNot(BeNil()) - err = controlplanev1alpha4.AddToScheme(scheme.Scheme) + err = controlplanev1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) - err = controlplanev1alpha4.AddToScheme(scheme.Scheme) + err = controlplanev1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) - err = controlplanev1alpha4.AddToScheme(scheme.Scheme) + err = controlplanev1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) - err = controlplanev1alpha4.AddToScheme(scheme.Scheme) + err = controlplanev1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) // +kubebuilder:scaffold:scheme diff --git a/controlplane/nested/main.go b/controlplane/nested/main.go new file mode 100644 index 00000000..39cf3ffa --- /dev/null +++ b/controlplane/nested/main.go @@ -0,0 +1,199 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package main + +import ( + "flag" + "math/rand" + "net/http" + _ "net/http/pprof" + "os" + "time" + + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/klog/v2" + "k8s.io/klog/v2/klogr" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4" + "sigs.k8s.io/cluster-api/cmd/version" + "sigs.k8s.io/cluster-api/feature" + "sigs.k8s.io/cluster-api/util" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" + + infrastructurev1alpha4 "sigs.k8s.io/cluster-api-provider-nested/api/v1alpha4" + controlplanev1alpha4 "sigs.k8s.io/cluster-api-provider-nested/controlplane/nested/api/v1alpha4" + "sigs.k8s.io/cluster-api-provider-nested/controlplane/nested/controllers" + // +kubebuilder:scaffold:imports +) + +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + + // flags + metricsAddr string + enableLeaderElection bool + leaderElectionLeaseDuration time.Duration + leaderElectionRenewDeadline time.Duration + leaderElectionRetryPeriod time.Duration + profilerAddress string + syncPeriod time.Duration + webhookPort int + healthAddr string + templatePath string +) + +func init() { + klog.InitFlags(nil) + + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(clusterv1.AddToScheme(scheme)) + utilruntime.Must(controlplanev1alpha4.AddToScheme(scheme)) + utilruntime.Must(infrastructurev1alpha4.AddToScheme(scheme)) + // +kubebuilder:scaffold:scheme +} + +// InitFlags initializes the flags. +func InitFlags(fs *pflag.FlagSet) { + fs.StringVar(&metricsAddr, "metrics-bind-address", ":8080", + "The address the metric endpoint binds to.") + + fs.BoolVar(&enableLeaderElection, "leader-elect", false, + "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") + + fs.DurationVar(&leaderElectionLeaseDuration, "leader-elect-lease-duration", 15*time.Second, + "Interval at which non-leader candidates will wait to force acquire leadership (duration string)") + + fs.DurationVar(&leaderElectionRenewDeadline, "leader-elect-renew-deadline", 10*time.Second, + "Duration that the leading controller manager will retry refreshing leadership before giving up (duration string)") + + fs.DurationVar(&leaderElectionRetryPeriod, "leader-elect-retry-period", 2*time.Second, + "Duration the LeaderElector clients should wait between tries of actions (duration string)") + + fs.StringVar(&profilerAddress, "profiler-address", "", + "Bind address to expose the pprof profiler (e.g. localhost:6060)") + + fs.DurationVar(&syncPeriod, "sync-period", 10*time.Minute, + "The minimum interval at which watched resources are reconciled (e.g. 15m)") + + fs.IntVar(&webhookPort, "webhook-port", 0, + "Webhook Server port, disabled by default. When enabled, the manager will only work as webhook server, no reconcilers are installed.") + + fs.StringVar(&healthAddr, "health-addr", ":9440", + "The address the health endpoint binds to.") + + fs.StringVar(&templatePath, "template-path", "/component-templates", + "The address the health endpoint binds to.") + + feature.MutableGates.AddFlag(fs) +} + +func main() { + rand.Seed(time.Now().UnixNano()) + + InitFlags(pflag.CommandLine) + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) + pflag.Parse() + + ctrl.SetLogger(klogr.New()) + + if profilerAddress != "" { + klog.Infof("Profiler listening for requests at %s", profilerAddress) + go func() { + klog.Info(http.ListenAndServe(profilerAddress, nil)) + }() + } + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + MetricsBindAddress: metricsAddr, + LeaderElection: enableLeaderElection, + LeaderElectionID: "controller-leader-election-capn", + LeaseDuration: &leaderElectionLeaseDuration, + RenewDeadline: &leaderElectionRenewDeadline, + RetryPeriod: &leaderElectionRetryPeriod, + SyncPeriod: &syncPeriod, + NewClient: util.ManagerDelegatingClientFunc, + Port: webhookPort, + HealthProbeBindAddress: healthAddr, + }) + if err != nil { + setupLog.Error(err, "unable to start manager") + os.Exit(1) + } + + // Setup the context that's going to be used in controllers and for the manager. + ctx := ctrl.SetupSignalHandler() + + // Setup health checks. + if err := mgr.AddReadyzCheck("ping", healthz.Ping); err != nil { + setupLog.Error(err, "unable to create ready check") + os.Exit(1) + } + if err := mgr.AddHealthzCheck("ping", healthz.Ping); err != nil { + setupLog.Error(err, "unable to create health check") + os.Exit(1) + } + + if err = (&controllers.NestedControlPlaneReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("controlplane").WithName("NestedControlPlane"), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "NestedControlPlane") + os.Exit(1) + } + + if err = (&controllers.NestedEtcdReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("controlplane").WithName("NestedEtcd"), + Scheme: mgr.GetScheme(), + TemplatePath: templatePath, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "NestedEtcd") + os.Exit(1) + } + + if err = (&controllers.NestedAPIServerReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("controlplane").WithName("NestedAPIServer"), + Scheme: mgr.GetScheme(), + TemplatePath: templatePath, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "NestedAPIServer") + os.Exit(1) + } + + if err = (&controllers.NestedControllerManagerReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("controlplane").WithName("NestedControllerManager"), + Scheme: mgr.GetScheme(), + TemplatePath: templatePath, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "NestedControllerManager") + os.Exit(1) + } + // +kubebuilder:scaffold:builder + + setupLog.Info("Starting manager", "version", version.Get().String()) + if err := mgr.Start(ctx); err != nil { + setupLog.Error(err, "problem running manager") + os.Exit(1) + } +} diff --git a/hack/boilerplate/boilerplate.Dockerfile.txt b/hack/boilerplate/boilerplate.Dockerfile.txt index 8168b3a2..f6a6f689 100644 --- a/hack/boilerplate/boilerplate.Dockerfile.txt +++ b/hack/boilerplate/boilerplate.Dockerfile.txt @@ -1,4 +1,4 @@ -# syntax=docker/dockerfile:experimental +# syntax=docker/dockerfile:1.1-experimental # Copyright YEAR The Kubernetes Authors. # diff --git a/main.go b/main.go index e11ab47e..9e62dc58 100644 --- a/main.go +++ b/main.go @@ -36,10 +36,9 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" - controlplanev1alpha4 "sigs.k8s.io/cluster-api-provider-nested/apis/controlplane/v1alpha4" - infrastructurev1alpha4 "sigs.k8s.io/cluster-api-provider-nested/apis/infrastructure/v1alpha4" - controlplanecontrollers "sigs.k8s.io/cluster-api-provider-nested/controllers/controlplane" - infrastructurecontrollers "sigs.k8s.io/cluster-api-provider-nested/controllers/infrastructure" + infrastructurev1 "sigs.k8s.io/cluster-api-provider-nested/api/v1alpha4" + "sigs.k8s.io/cluster-api-provider-nested/controllers" + controlplanev1 "sigs.k8s.io/cluster-api-provider-nested/controlplane/nested/api/v1alpha4" // +kubebuilder:scaffold:imports ) @@ -64,8 +63,8 @@ func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(clusterv1.AddToScheme(scheme)) - utilruntime.Must(controlplanev1alpha4.AddToScheme(scheme)) - utilruntime.Must(infrastructurev1alpha4.AddToScheme(scheme)) + utilruntime.Must(controlplanev1.AddToScheme(scheme)) + utilruntime.Must(infrastructurev1.AddToScheme(scheme)) // +kubebuilder:scaffold:scheme } @@ -148,15 +147,7 @@ func main() { os.Exit(1) } - if err = (&controlplanecontrollers.NestedControlPlaneReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("controlplane").WithName("NestedControlPlane"), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "NestedControlPlane") - os.Exit(1) - } - if err = (&infrastructurecontrollers.NestedClusterReconciler{ + if err = (&controllers.NestedClusterReconciler{ Client: mgr.GetClient(), Log: ctrl.Log.WithName("controllers").WithName("infrastructure").WithName("NestedCluster"), Scheme: mgr.GetScheme(), @@ -164,31 +155,6 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "NestedCluster") os.Exit(1) } - - if err = (&controlplanecontrollers.NestedEtcdReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("controlplane").WithName("NestedEtcd"), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "NestedEtcd") - os.Exit(1) - } - if err = (&controlplanecontrollers.NestedAPIServerReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("controlplane").WithName("NestedAPIServer"), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "NestedAPIServer") - os.Exit(1) - } - if err = (&controlplanecontrollers.NestedControllerManagerReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("controlplane").WithName("NestedControllerManager"), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "NestedControllerManager") - os.Exit(1) - } // +kubebuilder:scaffold:builder setupLog.Info("Starting manager", "version", version.Get().String()) diff --git a/metadata.yaml b/metadata.yaml new file mode 100644 index 00000000..77ccf892 --- /dev/null +++ b/metadata.yaml @@ -0,0 +1,10 @@ +# maps release series of major.minor to cluster-api contract version +# the contract version may change between minor or major versions, but *not* +# between patch versions. +# +# update this file only when a new major or minor version is released +apiVersion: clusterctl.cluster.x-k8s.io/v1alpha4 +releaseSeries: +- major: 0 + minor: 1 + contract: v1alpha4 \ No newline at end of file diff --git a/virtualcluster/Dockerfile b/virtualcluster/Dockerfile index 1e1b2d6f..917a35df 100644 --- a/virtualcluster/Dockerfile +++ b/virtualcluster/Dockerfile @@ -1,3 +1,19 @@ +# syntax=docker/dockerfile:1.1-experimental + +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Build the manager binary FROM golang:1.12 as builder diff --git a/virtualcluster/Makefile b/virtualcluster/Makefile index c237a641..43f6f1f9 100644 --- a/virtualcluster/Makefile +++ b/virtualcluster/Makefile @@ -1,3 +1,18 @@ +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + # Explicitly opt into go modules, even though we're inside a GOPATH directory export GO111MODULE=on