Support Kubernetes 1.18

Adds and modifies e2e tests to test scenario
Makefile updated for local testing

Signed-off-by: Naadir Jeewa <[email protected]>

Author: Naadir Jeewa

Diff for: .dockerignore

@@ -15,3 +15,5 @@ Tiltfile
 test/infrastructure/docker/e2e/logs/**
 **/config/**/*.yaml
 _artifacts
+Makefile
+**/Makefile

Diff for: Makefile

@@ -86,6 +86,8 @@ TAG ?= dev
 ARCH ?= amd64
 ALL_ARCH = amd64 arm arm64 ppc64le s390x
 
+GINKGO_NODES ?= 1
+
 # Allow overriding the imagePullPolicy
 PULL_POLICY ?= Always
 
@@ -132,6 +134,22 @@ test-capd-e2e: ## Rebuild the docker provider and run the capd-e2e tests
 	$(MAKE) -C test/infrastructure/docker docker-build REGISTRY=gcr.io/k8s-staging-capi-docker
 	$(MAKE) -C test/infrastructure/docker run-e2e
 
+E2E_FLAGS := REGISTRY=gcr.io/k8s-staging-cluster-api TAG=dev ARCH=amd64 PULL_POLICY=IfNotPresent GINKGO_FOCUS="$(GINKGO_FOCUS)" GINKGO_NODES=$(GINKGO_NODES) GINKGO_NOCOLOR=false E2E_CONF_FILE=config/docker-dev.yaml ARTIFACTS=$(abspath _artifacts) SKIP_RESOURCE_CLEANUP=false USE_EXISTING_CLUSTER=false
+
+.PHONY: e2e
+e2e: ## Run clusterctl e2e
+	mkdir -p _artifacts
+	$(E2E_FLAGS) $(MAKE) docker-build
+	$(E2E_FLAGS) $(MAKE) -C test/infrastructure/docker docker-build
+	docker pull quay.io/jetstack/cert-manager-cainjector:v0.11.0
+	docker pull quay.io/jetstack/cert-manager-webhook:v0.11.0
+	docker pull quay.io/jetstack/cert-manager-controller:v0.11.0
+	$(E2E_FLAGS) $(MAKE) -C test/e2e run
+
+.PHONY: clean-e2e
+clean-e2e: clean-manifests ## Clean-up after clusterctl e2e
+	rm -rf _artifacts
+
 ## --------------------------------------
 ## Binaries
 ## --------------------------------------
@@ -201,6 +219,9 @@ lint-full: $(GOLANGCI_LINT) ## Run slower linters to detect possible issues
 	cd $(E2E_FRAMEWORK_DIR); $(GOLANGCI_LINT) run -v --fast=false
 	cd $(CAPD_DIR); $(GOLANGCI_LINT) run -v --fast=false
 
+apidiff: $(GO_APIDIFF) ## Check for API differences
+	$(GO_APIDIFF) $(shell git rev-parse origin/master) --print-compatible
+
 ## --------------------------------------
 ## Generate / Manifests
 ## --------------------------------------
@@ -529,6 +550,11 @@ clean-book: ## Remove all generated GitBook files
 clean-bindata: ## Remove bindata generated folder
 	rm -rf $(GOBINDATA_CLUSTERCTL_DIR)/manifest
 
+.PHONY: clean-manifests ## Reset manifests in config directories back to master
+clean-manifests:
+	@read -p "WARNING: This will reset all config directories to local master. Press [ENTER] to continue."
+	git checkout master config bootstrap/kubeadm/config controlplane/kubeadm/config test/infrastructure/docker/config
+
 .PHONY: format-tiltfile
 format-tiltfile: ## Format Tiltfile
 	./hack/verify-starlark.sh fix

Diff for: controlplane/kubeadm/controllers/controller.go

@@ -254,6 +254,11 @@ func (r *KubeadmControlPlaneReconciler) reconcile(ctx context.Context, cluster *
 		return ctrl.Result{Requeue: true}, nil
 	}
 
+	// Ensure kubeadm role bindings for v1.18+
+	if err := workloadCluster.AllowBootstrapTokensToGetNodes(ctx); err != nil {
+		return ctrl.Result{}, errors.Wrap(err, "failed to set role and role binding for kubeadm")
+	}
+
 	// Update kube-proxy daemonset.
 	if err := workloadCluster.UpdateKubeProxyImageInfo(ctx, kcp); err != nil {
 		logger.Error(err, "failed to update kube-proxy daemonset")

Diff for: controlplane/kubeadm/controllers/fakes_test.go

@@ -78,6 +78,10 @@ func (f fakeWorkloadCluster) ClusterStatus(_ context.Context) (internal.ClusterS
 	return f.Status, nil
 }
 
+func (f fakeWorkloadCluster) AllowBootstrapTokensToGetNodes(ctx context.Context) error {
+	return nil
+}
+
 func (f fakeWorkloadCluster) ReconcileKubeletRBACRole(ctx context.Context, version semver.Version) error {
 	return nil
 }

Diff for: controlplane/kubeadm/controllers/upgrade.go

@@ -57,6 +57,12 @@ func (r *KubeadmControlPlaneReconciler) upgradeControlPlane(
 		return ctrl.Result{}, errors.Wrap(err, "failed to reconcile the remote kubelet RBAC binding")
 	}
 
+	// Ensure kubeadm cluster role  & bindings for v1.18+
+	// as per https://github.com/kubernetes/kubernetes/commit/b117a928a6c3f650931bdac02a41fca6680548c4
+	if err := workloadCluster.AllowBootstrapTokensToGetNodes(ctx); err != nil {
+		return ctrl.Result{}, errors.Wrap(err, "failed to set role and role binding for kubeadm")
+	}
+
 	if err := workloadCluster.UpdateKubernetesVersionInKubeadmConfigMap(ctx, parsedVersion); err != nil {
 		return ctrl.Result{}, errors.Wrap(err, "failed to update the kubernetes version in the kubeadm config map")
 	}

Diff for: controlplane/kubeadm/internal/workload_cluster.go

@@ -31,7 +31,6 @@ import (
 	"github.com/pkg/errors"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
-	rbacv1 "k8s.io/api/rbac/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
@@ -73,6 +72,7 @@ type WorkloadCluster interface {
 	RemoveMachineFromKubeadmConfigMap(ctx context.Context, machine *clusterv1.Machine) error
 	RemoveNodeFromKubeadmConfigMap(ctx context.Context, nodeName string) error
 	ForwardEtcdLeadership(ctx context.Context, machine *clusterv1.Machine, leaderCandidate *clusterv1.Machine) error
+	AllowBootstrapTokensToGetNodes(ctx context.Context) error
 
 	// State recovery tasks.
 	ReconcileEtcdMembers(ctx context.Context) error
@@ -251,83 +251,6 @@ func (w *Workload) RemoveNodeFromKubeadmConfigMap(ctx context.Context, name stri
 	return nil
 }
 
-// ReconcileKubeletRBACBinding will create a RoleBinding for the new kubelet version during upgrades.
-// If the role binding already exists this function is a no-op.
-func (w *Workload) ReconcileKubeletRBACBinding(ctx context.Context, version semver.Version) error {
-	roleName := fmt.Sprintf("kubeadm:kubelet-config-%d.%d", version.Major, version.Minor)
-	roleBinding := &rbacv1.RoleBinding{}
-	err := w.Client.Get(ctx, ctrlclient.ObjectKey{Name: roleName, Namespace: metav1.NamespaceSystem}, roleBinding)
-	if err != nil && !apierrors.IsNotFound(err) {
-		return errors.Wrapf(err, "failed to determine if kubelet config rbac role binding %q already exists", roleName)
-	} else if err == nil {
-		// The required role binding already exists, nothing left to do
-		return nil
-	}
-
-	newRoleBinding := &rbacv1.RoleBinding{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      roleName,
-			Namespace: metav1.NamespaceSystem,
-		},
-		Subjects: []rbacv1.Subject{
-			{
-				APIGroup: "rbac.authorization.k8s.io",
-				Kind:     "Group",
-				Name:     "system:nodes",
-			},
-			{
-				APIGroup: "rbac.authorization.k8s.io",
-				Kind:     "Group",
-				Name:     "system:bootstrappers:kubeadm:default-node-token",
-			},
-		},
-		RoleRef: rbacv1.RoleRef{
-			APIGroup: "rbac.authorization.k8s.io",
-			Kind:     "Role",
-			Name:     roleName,
-		},
-	}
-	if err := w.Client.Create(ctx, newRoleBinding); err != nil && !apierrors.IsAlreadyExists(err) {
-		return errors.Wrapf(err, "failed to create kubelet rbac role binding %q", roleName)
-	}
-
-	return nil
-}
-
-// ReconcileKubeletRBACRole will create a Role for the new kubelet version during upgrades.
-// If the role already exists this function is a no-op.
-func (w *Workload) ReconcileKubeletRBACRole(ctx context.Context, version semver.Version) error {
-	majorMinor := fmt.Sprintf("%d.%d", version.Major, version.Minor)
-	roleName := fmt.Sprintf("kubeadm:kubelet-config-%s", majorMinor)
-	role := &rbacv1.Role{}
-	if err := w.Client.Get(ctx, ctrlclient.ObjectKey{Name: roleName, Namespace: metav1.NamespaceSystem}, role); err != nil && !apierrors.IsNotFound(err) {
-		return errors.Wrapf(err, "failed to determine if kubelet config rbac role %q already exists", roleName)
-	} else if err == nil {
-		// The required role already exists, nothing left to do
-		return nil
-	}
-
-	newRole := &rbacv1.Role{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      roleName,
-			Namespace: metav1.NamespaceSystem,
-		},
-		Rules: []rbacv1.PolicyRule{
-			{
-				Verbs:         []string{"get"},
-				APIGroups:     []string{""},
-				Resources:     []string{"configmaps"},
-				ResourceNames: []string{fmt.Sprintf("kubelet-config-%s", majorMinor)},
-			},
-		},
-	}
-	if err := w.Client.Create(ctx, newRole); err != nil && !apierrors.IsAlreadyExists(err) {
-		return errors.Wrapf(err, "failed to create kubelet rbac role %q", roleName)
-	}
-
-	return nil
-}
-
 // ClusterStatus holds stats information about the cluster.
 type ClusterStatus struct {
 	// Nodes are a total count of nodes

Diff for: controlplane/kubeadm/internal/workload_cluster_etcd.go

@@ -249,6 +249,9 @@ func (w *Workload) ForwardEtcdLeadership(ctx context.Context, machine *clusterv1
 	if leaderCandidate == nil {
 		return errors.New("leader candidate cannot be nil")
 	}
+	if leaderCandidate.Status.NodeRef == nil {
+		return errors.New("leader has no node reference")
+	}
 
 	nodes, err := w.getControlPlaneNodes(ctx)
 	if err != nil {

Diff for: controlplane/kubeadm/internal/workload_cluster_etcd_test.go

@@ -339,6 +339,14 @@ func TestForwardEtcdLeadership(t *testing.T) {
 				leaderCandidate: nil,
 				expectErr:       true,
 			},
+			{
+				name:    "returns an error if the leader candidate's noderef is nil",
+				machine: defaultMachine(),
+				leaderCandidate: defaultMachine(func(m *clusterv1.Machine) {
+					m.Status.NodeRef = nil
+				}),
+				expectErr: true,
+			},
 			{
 				name:            "returns an error if it can't retrieve the list of control plane nodes",
 				machine:         defaultMachine(),

Diff for: controlplane/kubeadm/internal/workload_cluster_rbac.go

@@ -0,0 +1,163 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package internal
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/blang/semver"
+	"github.com/pkg/errors"
+	rbac "k8s.io/api/rbac/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	// NodeBootstrapTokenAuthGroup specifies which group a Node Bootstrap Token should be authenticated in
+	NodeBootstrapTokenAuthGroup = "system:bootstrappers:kubeadm:default-node-token"
+
+	// GetNodesClusterRoleName defines the name of the ClusterRole and ClusterRoleBinding to get nodes
+	GetNodesClusterRoleName = "kubeadm:get-nodes"
+
+	// NodesGroup defines the well-known group for all nodes.
+	NodesGroup = "system:nodes"
+
+	// KubeletConfigMapRolePrefix defines base kubelet configuration ConfigMap role prefix.
+	KubeletConfigMapRolePrefix = "kubeadm:"
+
+	// KubeletConfigMapName defines base kubelet configuration ConfigMap name.
+	KubeletConfigMapName = "kubelet-config-%d.%d"
+)
+
+// EnsureResource creates a resoutce if the target resource doesn't exist. If the resource exists already, this function will ignore the resource instead.
+func (w *Workload) EnsureResource(ctx context.Context, obj runtime.Object) error {
+	testObj := obj.DeepCopyObject()
+	key, err := ctrlclient.ObjectKeyFromObject(obj)
+	if err != nil {
+		return errors.Wrap(err, "unable to derive key for resource")
+	}
+	if err := w.Client.Get(ctx, key, testObj); err != nil && !apierrors.IsNotFound(err) {
+		return errors.Wrapf(err, "failed to determine if resource %s/%s already exists", key.Namespace, key.Name)
+	} else if err == nil {
+		// If object already exists, nothing left to do
+		return nil
+	}
+	if err := w.Client.Create(ctx, obj); err != nil {
+		if !apierrors.IsAlreadyExists(err) {
+			return errors.Wrapf(err, "unable to create resource %s/%s on workload cluster", key.Namespace, key.Name)
+		}
+	}
+	return nil
+}
+
+// AllowBootstrapTokensToGetNodes creates RBAC rules to allow Node Bootstrap Tokens to list nodes
+func (w *Workload) AllowBootstrapTokensToGetNodes(ctx context.Context) error {
+	if err := w.EnsureResource(ctx, &rbac.ClusterRole{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      GetNodesClusterRoleName,
+			Namespace: metav1.NamespaceSystem,
+		},
+		Rules: []rbac.PolicyRule{
+			{
+				Verbs:     []string{"get"},
+				APIGroups: []string{""},
+				Resources: []string{"nodes"},
+			},
+		},
+	}); err != nil {
+		return err
+	}
+
+	return w.EnsureResource(ctx, &rbac.ClusterRoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      GetNodesClusterRoleName,
+			Namespace: metav1.NamespaceSystem,
+		},
+		RoleRef: rbac.RoleRef{
+			APIGroup: rbac.GroupName,
+			Kind:     "ClusterRole",
+			Name:     GetNodesClusterRoleName,
+		},
+		Subjects: []rbac.Subject{
+			{
+				Kind: rbac.GroupKind,
+				Name: NodeBootstrapTokenAuthGroup,
+			},
+		},
+	})
+}
+
+func generateKubeletConfigName(version semver.Version) string {
+	return fmt.Sprintf(KubeletConfigMapName, version.Major, version.Minor)
+}
+
+func generateKubeletConfigRoleName(version semver.Version) string {
+	return KubeletConfigMapRolePrefix + generateKubeletConfigName(version)
+}
+
+// ReconcileKubeletRBACBinding will create a RoleBinding for the new kubelet version during upgrades.
+// If the role binding already exists this function is a no-op.
+func (w *Workload) ReconcileKubeletRBACBinding(ctx context.Context, version semver.Version) error {
+	roleName := generateKubeletConfigRoleName(version)
+	return w.EnsureResource(ctx, &rbac.RoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: metav1.NamespaceSystem,
+			Name:      roleName,
+		},
+		Subjects: []rbacv1.Subject{
+			{
+				APIGroup: rbac.GroupName,
+				Kind:     rbac.GroupKind,
+				Name:     NodesGroup,
+			},
+			{
+				APIGroup: rbac.GroupName,
+				Kind:     rbac.GroupKind,
+				Name:     NodeBootstrapTokenAuthGroup,
+			},
+		},
+		RoleRef: rbacv1.RoleRef{
+			APIGroup: rbac.GroupName,
+			Kind:     "Role",
+			Name:     roleName,
+		},
+	})
+
+}
+
+// ReconcileKubeletRBACRole will create a Role for the new kubelet version during upgrades.
+// If the role already exists this function is a no-op.
+func (w *Workload) ReconcileKubeletRBACRole(ctx context.Context, version semver.Version) error {
+	return w.EnsureResource(ctx, &rbacv1.Role{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      generateKubeletConfigRoleName(version),
+			Namespace: metav1.NamespaceSystem,
+		},
+		Rules: []rbacv1.PolicyRule{
+			{
+				Verbs:         []string{"get"},
+				APIGroups:     []string{""},
+				Resources:     []string{"configmaps"},
+				ResourceNames: []string{generateKubeletConfigName(version)},
+			},
+		},
+	})
+}