Merge pull request #1522 from tormath1/tormath1/flatcar

k8s-ci-robot · web-flow · commit 41674c89e994 · 2023-04-17T00:04:43.000-07:00
⚠️ flatcar: make external-cloud-provider-flatcar the default one
diff --git a/Makefile b/Makefile
@@ -162,8 +162,7 @@ e2e-templates: $(addprefix $(E2E_TEMPLATES_DIR)/, \
 		 cluster-template-without-lb.yaml \
 		 cluster-template.yaml) \
 		 $(addprefix $(E2E_NO_ARTIFACT_TEMPLATES_DIR)/, \
-		 cluster-template-flatcar.yaml \
-		 cluster-template-external-cloud-provider-flatcar.yaml) \
+		 cluster-template-flatcar.yaml)
 
 $(E2E_TEMPLATES_DIR)/cluster-template.yaml: $(E2E_KUSTOMIZE_DIR)/with-tags $(KUSTOMIZE) FORCE
 	$(KUSTOMIZE) build "$<" > "$@"
@@ -424,8 +423,7 @@ release-notes: $(RELEASE_NOTES) ## Generate release notes
 templates: ## Generate cluster templates
 templates: templates/cluster-template.yaml \
 	templates/cluster-template-without-lb.yaml \
-	templates/cluster-template-flatcar.yaml \
-	templates/cluster-template-external-cloud-provider-flatcar.yaml \
+	templates/cluster-template-flatcar.yaml
 
 templates/cluster-template.yaml: kustomize/v1alpha7/default $(KUSTOMIZE) FORCE
 	$(KUSTOMIZE) build "$<" > "$@"
diff --git a/RELEASE.md b/RELEASE.md
@@ -72,6 +72,7 @@ It is recommended to create at least one release candidate when bumping `X` or `
     - `./out/infrastructure-components.yaml`
     - `./out/cluster-template.yaml`
     - `./out/cluster-template-without-lb.yaml`
+    - `./out/cluster-template-flatcar.yaml`
     - `./out/metadata.yaml`
 1. Ensure that the release image has been promoted.
 1. Publish release.
diff --git a/docs/book/src/clusteropenstack/configuration.md b/docs/book/src/clusteropenstack/configuration.md
@@ -77,7 +77,7 @@ Some OS like [Fedora CoreOS](https://getfedora.org/en/coreos) or [Flatcar](https
 To use Flatcar image:
 * Build the image with the [image-builder](https://image-builder.sigs.k8s.io/capi/providers/openstack.html): `make OEM_ID=openstack build-qemu-flatcar`
 * Export the name of the uploaded image: `export OPENSTACK_FLATCAR_IMAGE_NAME=flatcar-stable-3374.2.5-kube-v1.25.6`
-* When generating the cluster configuration, use one of the following Cluster API [flavor](https://cluster-api.sigs.k8s.io/clusterctl/commands/generate-cluster.html?#flavors): `--flavor flatcar` or `--flavor external-cloud-provider-flatcar` (_NOTE_: If you select `external-cloud-provider-flatcar`, don't forget to refer to the [external-cloud-provider](https://cluster-api-openstack.sigs.k8s.io/topics/external-cloud-provider.html) section)
+* When generating the cluster configuration, use the following Cluster API [flavor](https://cluster-api.sigs.k8s.io/clusterctl/commands/generate-cluster.html?#flavors): `--flavor flatcar` (_NOTE_: Don't forget to refer to the [external-cloud-provider](https://cluster-api-openstack.sigs.k8s.io/topics/external-cloud-provider.html) section)
 
 ## SSH key pair
 
diff --git a/kustomize/v1alpha7/flatcar/kustomization.yaml b/kustomize/v1alpha7/flatcar/kustomization.yaml
@@ -0,0 +1,5 @@
+resources:
+- ../default
+
+patchesStrategicMerge:
+- patch-flatcar.yaml
diff --git a/kustomize/v1alpha7/flatcar/patch-flatcar.yaml b/kustomize/v1alpha7/flatcar/patch-flatcar.yaml
@@ -0,0 +1,97 @@
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha7
+kind: OpenStackCluster
+metadata:
+  name: ${CLUSTER_NAME}
+spec:
+  apiServerLoadBalancer:
+    $patch: delete
+---
+apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+kind: KubeadmControlPlane
+metadata:
+  name: "${CLUSTER_NAME}-control-plane"
+spec:
+  replicas: ${CONTROL_PLANE_MACHINE_COUNT}
+  kubeadmConfigSpec:
+    joinConfiguration:
+      nodeRegistration:
+        name: $${COREOS_OPENSTACK_HOSTNAME}
+    initConfiguration:
+      nodeRegistration:
+        name: $${COREOS_OPENSTACK_HOSTNAME}
+    format: ignition
+    ignition:
+      containerLinuxConfig:
+        additionalConfig: |
+          systemd:
+            units:
+            - name: coreos-metadata-sshkeys@.service
+              enabled: true
+            - name: kubeadm.service
+              enabled: true
+              dropins:
+              - name: 10-flatcar.conf
+                contents: |
+                  [Unit]
+                  Requires=containerd.service coreos-metadata.service
+                  After=containerd.service coreos-metadata.service
+
+                  [Service]
+                  EnvironmentFile=/run/metadata/flatcar
+    preKubeadmCommands:
+      - export COREOS_OPENSTACK_HOSTNAME=$${COREOS_OPENSTACK_HOSTNAME%.*}
+      - envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp
+      - mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml
+---
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+kind: KubeadmConfigTemplate
+metadata:
+  name: ${CLUSTER_NAME}-md-0
+spec:
+  template:
+    spec:
+      joinConfiguration:
+        nodeRegistration:
+          name: $${COREOS_OPENSTACK_HOSTNAME}
+      preKubeadmCommands:
+        - export COREOS_OPENSTACK_HOSTNAME=$${COREOS_OPENSTACK_HOSTNAME%.*}
+        - envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp
+        - mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml
+      format: ignition
+      ignition:
+        containerLinuxConfig:
+          additionalConfig: |
+            systemd:
+              units:
+              - name: coreos-metadata-sshkeys@.service
+                enabled: true
+              - name: kubeadm.service
+                enabled: true
+                dropins:
+                - name: 10-flatcar.conf
+                  contents: |
+                    [Unit]
+                    Requires=containerd.service coreos-metadata.service
+                    After=containerd.service coreos-metadata.service
+
+                    [Service]
+                    EnvironmentFile=/run/metadata/flatcar
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha7
+kind: OpenStackMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-md-0
+spec:
+  template:
+    spec:
+      image: ${OPENSTACK_FLATCAR_IMAGE_NAME}
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha7
+kind: OpenStackMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-control-plane
+spec:
+  template:
+    spec:
+      image: ${OPENSTACK_FLATCAR_IMAGE_NAME}
diff --git a/templates/cluster-template-flatcar.yaml b/templates/cluster-template-flatcar.yaml
@@ -15,17 +15,7 @@ metadata:
 spec:
   template:
     spec:
-      files:
-      - content: ${OPENSTACK_CLOUD_PROVIDER_CONF_B64}
-        encoding: base64
-        owner: root
-        path: /etc/kubernetes/cloud.conf
-        permissions: "0600"
-      - content: ${OPENSTACK_CLOUD_CACERT_B64}
-        encoding: base64
-        owner: root
-        path: /etc/certs/cacert
-        permissions: "0600"
+      files: []
       format: ignition
       ignition:
         containerLinuxConfig:
@@ -48,8 +38,7 @@ spec:
       joinConfiguration:
         nodeRegistration:
           kubeletExtraArgs:
-            cloud-config: /etc/kubernetes/cloud.conf
-            cloud-provider: openstack
+            cloud-provider: external
           name: $${COREOS_OPENSTACK_HOSTNAME}
       preKubeadmCommands:
       - export COREOS_OPENSTACK_HOSTNAME=$${COREOS_OPENSTACK_HOSTNAME%.*}
@@ -71,7 +60,7 @@ spec:
     kind: KubeadmControlPlane
     name: ${CLUSTER_NAME}-control-plane
   infrastructureRef:
-    apiVersion: infrastructure.cluster.x-k8s.io/v1alpha6
+    apiVersion: infrastructure.cluster.x-k8s.io/v1alpha7
     kind: OpenStackCluster
     name: ${CLUSTER_NAME}
 ---
@@ -94,7 +83,7 @@ spec:
       clusterName: ${CLUSTER_NAME}
       failureDomain: ${OPENSTACK_FAILURE_DOMAIN}
       infrastructureRef:
-        apiVersion: infrastructure.cluster.x-k8s.io/v1alpha6
+        apiVersion: infrastructure.cluster.x-k8s.io/v1alpha7
         kind: OpenStackMachineTemplate
         name: ${CLUSTER_NAME}-md-0
       version: ${KUBERNETES_VERSION}
@@ -108,37 +97,11 @@ spec:
     clusterConfiguration:
       apiServer:
         extraArgs:
-          cloud-config: /etc/kubernetes/cloud.conf
-          cloud-provider: openstack
-        extraVolumes:
-        - hostPath: /etc/kubernetes/cloud.conf
-          mountPath: /etc/kubernetes/cloud.conf
-          name: cloud
-          readOnly: true
+          cloud-provider: external
       controllerManager:
         extraArgs:
-          cloud-config: /etc/kubernetes/cloud.conf
-          cloud-provider: openstack
-        extraVolumes:
-        - hostPath: /etc/kubernetes/cloud.conf
-          mountPath: /etc/kubernetes/cloud.conf
-          name: cloud
-          readOnly: true
-        - hostPath: /etc/certs/cacert
-          mountPath: /etc/certs/cacert
-          name: cacerts
-          readOnly: true
-    files:
-    - content: ${OPENSTACK_CLOUD_PROVIDER_CONF_B64}
-      encoding: base64
-      owner: root
-      path: /etc/kubernetes/cloud.conf
-      permissions: "0600"
-    - content: ${OPENSTACK_CLOUD_CACERT_B64}
-      encoding: base64
-      owner: root
-      path: /etc/certs/cacert
-      permissions: "0600"
+          cloud-provider: external
+    files: []
     format: ignition
     ignition:
       containerLinuxConfig:
@@ -161,28 +124,26 @@ spec:
     initConfiguration:
       nodeRegistration:
         kubeletExtraArgs:
-          cloud-config: /etc/kubernetes/cloud.conf
-          cloud-provider: openstack
+          cloud-provider: external
         name: $${COREOS_OPENSTACK_HOSTNAME}
     joinConfiguration:
       nodeRegistration:
         kubeletExtraArgs:
-          cloud-config: /etc/kubernetes/cloud.conf
-          cloud-provider: openstack
+          cloud-provider: external
         name: $${COREOS_OPENSTACK_HOSTNAME}
     preKubeadmCommands:
     - export COREOS_OPENSTACK_HOSTNAME=$${COREOS_OPENSTACK_HOSTNAME%.*}
     - envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp
     - mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml
   machineTemplate:
     infrastructureRef:
-      apiVersion: infrastructure.cluster.x-k8s.io/v1alpha6
+      apiVersion: infrastructure.cluster.x-k8s.io/v1alpha7
       kind: OpenStackMachineTemplate
       name: ${CLUSTER_NAME}-control-plane
   replicas: ${CONTROL_PLANE_MACHINE_COUNT}
   version: ${KUBERNETES_VERSION}
 ---
-apiVersion: infrastructure.cluster.x-k8s.io/v1alpha6
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha7
 kind: OpenStackCluster
 metadata:
   name: ${CLUSTER_NAME}
@@ -197,7 +158,7 @@ spec:
   managedSecurityGroups: true
   nodeCidr: 10.6.0.0/24
 ---
-apiVersion: infrastructure.cluster.x-k8s.io/v1alpha6
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha7
 kind: OpenStackMachineTemplate
 metadata:
   name: ${CLUSTER_NAME}-control-plane
@@ -212,7 +173,7 @@ spec:
       image: ${OPENSTACK_FLATCAR_IMAGE_NAME}
       sshKeyName: ${OPENSTACK_SSH_KEY_NAME}
 ---
-apiVersion: infrastructure.cluster.x-k8s.io/v1alpha6
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha7
 kind: OpenStackMachineTemplate
 metadata:
   name: ${CLUSTER_NAME}-md-0
diff --git a/test/e2e/data/kustomize/external-cloud-provider-flatcar/kustomization.yaml b/test/e2e/data/kustomize/external-cloud-provider-flatcar/kustomization.yaml
diff --git a/test/e2e/data/kustomize/flatcar/ccm.yaml b/test/e2e/data/kustomize/flatcar/ccm.yaml
diff --git a/test/e2e/data/kustomize/flatcar/kustomization.yaml b/test/e2e/data/kustomize/flatcar/kustomization.yaml
@@ -1,5 +1,8 @@
 ---
 resources:
-- ../../../../../kustomize/v1alpha6/flatcar
+- ../../../../../kustomize/v1alpha7/flatcar
+- ccm.yaml
+
 components:
 - ../common-patches/cni
+- ../common-patches/ccm
diff --git a/test/e2e/shared/defaults.go b/test/e2e/shared/defaults.go
@@ -30,30 +30,29 @@ import (
 )
 
 const (
-	DefaultSSHKeyPairName              = "cluster-api-provider-openstack-sigs-k8s-io"
-	KubeContext                        = "KUBE_CONTEXT"
-	KubernetesVersion                  = "KUBERNETES_VERSION"
-	CCMPath                            = "CCM"
-	CCMResources                       = "CCM_RESOURCES"
-	OpenStackCloudYAMLFile             = "OPENSTACK_CLOUD_YAML_FILE"
-	OpenStackCloud                     = "OPENSTACK_CLOUD"
-	OpenStackCloudAdmin                = "OPENSTACK_CLOUD_ADMIN"
-	OpenStackFailureDomain             = "OPENSTACK_FAILURE_DOMAIN"
-	OpenStackFailureDomainAlt          = "OPENSTACK_FAILURE_DOMAIN_ALT"
-	OpenStackVolumeTypeAlt             = "OPENSTACK_VOLUME_TYPE_ALT"
-	OpenStackImageName                 = "OPENSTACK_IMAGE_NAME"
-	OpenStackNodeMachineFlavor         = "OPENSTACK_NODE_MACHINE_FLAVOR"
-	SSHUserMachine                     = "SSH_USER_MACHINE"
-	FlavorDefault                      = "ci-artifacts"
-	FlavorWithoutLB                    = "without-lb-ci-artifacts"
-	FlavorMultiNetwork                 = "multi-network-ci-artifacts"
-	FlavorMultiAZ                      = "multi-az-ci-artifacts"
-	FlavorV1alpha5                     = "v1alpha5-ci-artifacts"
-	FlavorV1alpha6                     = "v1alpha6-ci-artifacts"
-	FlavorMDRemediation                = "md-remediation-ci-artifacts"
-	FlavorKCPRemediation               = "kcp-remediation-ci-artifacts"
-	FlavorFlatcar                      = "flatcar"
-	FlavorExternalCloudProviderFlatcar = "external-cloud-provider-flatcar"
+	DefaultSSHKeyPairName      = "cluster-api-provider-openstack-sigs-k8s-io"
+	KubeContext                = "KUBE_CONTEXT"
+	KubernetesVersion          = "KUBERNETES_VERSION"
+	CCMPath                    = "CCM"
+	CCMResources               = "CCM_RESOURCES"
+	OpenStackCloudYAMLFile     = "OPENSTACK_CLOUD_YAML_FILE"
+	OpenStackCloud             = "OPENSTACK_CLOUD"
+	OpenStackCloudAdmin        = "OPENSTACK_CLOUD_ADMIN"
+	OpenStackFailureDomain     = "OPENSTACK_FAILURE_DOMAIN"
+	OpenStackFailureDomainAlt  = "OPENSTACK_FAILURE_DOMAIN_ALT"
+	OpenStackVolumeTypeAlt     = "OPENSTACK_VOLUME_TYPE_ALT"
+	OpenStackImageName         = "OPENSTACK_IMAGE_NAME"
+	OpenStackNodeMachineFlavor = "OPENSTACK_NODE_MACHINE_FLAVOR"
+	SSHUserMachine             = "SSH_USER_MACHINE"
+	FlavorDefault              = "ci-artifacts"
+	FlavorWithoutLB            = "without-lb-ci-artifacts"
+	FlavorMultiNetwork         = "multi-network-ci-artifacts"
+	FlavorMultiAZ              = "multi-az-ci-artifacts"
+	FlavorV1alpha5             = "v1alpha5-ci-artifacts"
+	FlavorV1alpha6             = "v1alpha6-ci-artifacts"
+	FlavorMDRemediation        = "md-remediation-ci-artifacts"
+	FlavorKCPRemediation       = "kcp-remediation-ci-artifacts"
+	FlavorFlatcar              = "flatcar"
 )
 
 // DefaultScheme returns the default scheme to use for testing.
diff --git a/test/e2e/suites/e2e/e2e_test.go b/test/e2e/suites/e2e/e2e_test.go
@@ -184,35 +184,6 @@ var _ = Describe("e2e tests [PR-Blocking]", func() {
 			})
 			Expect(workerMachines).To(HaveLen(1))
 			Expect(controlPlaneMachines).To(HaveLen(3))
-		})
-	})
-
-	Describe("Workload cluster (external cloud provider with Flatcar)", func() {
-		It("should be creatable and deletable", func() {
-			// Flatcar default user is "core"
-			shared.SetEnvVar(shared.SSHUserMachine, "core", false)
-
-			shared.Logf("Creating a cluster")
-			clusterName := fmt.Sprintf("cluster-%s", namespace.Name)
-			configCluster := defaultConfigCluster(clusterName, namespace.Name)
-			configCluster.ControlPlaneMachineCount = pointer.Int64(1)
-			configCluster.WorkerMachineCount = pointer.Int64(1)
-			configCluster.Flavor = shared.FlavorExternalCloudProviderFlatcar
-			md := createCluster(ctx, configCluster)
-
-			workerMachines := framework.GetMachinesByMachineDeployments(ctx, framework.GetMachinesByMachineDeploymentsInput{
-				Lister:            e2eCtx.Environment.BootstrapClusterProxy.GetClient(),
-				ClusterName:       clusterName,
-				Namespace:         namespace.Name,
-				MachineDeployment: *md[0],
-			})
-			controlPlaneMachines := framework.GetControlPlaneMachinesByCluster(ctx, framework.GetControlPlaneMachinesByClusterInput{
-				Lister:      e2eCtx.Environment.BootstrapClusterProxy.GetClient(),
-				ClusterName: clusterName,
-				Namespace:   namespace.Name,
-			})
-			Expect(workerMachines).To(HaveLen(1))
-			Expect(controlPlaneMachines).To(HaveLen(1))
 
 			shared.Logf("Waiting for worker nodes to be in Running phase")
 			statusChecks := []framework.MachineStatusCheck{framework.MachinePhaseCheck(string(clusterv1.MachinePhaseRunning))}
@@ -227,7 +198,7 @@ var _ = Describe("e2e tests [PR-Blocking]", func() {
 
 			waitForDaemonSetRunning(ctx, workloadCluster.GetClient(), "kube-system", "openstack-cloud-controller-manager")
 
-			waitForNodesReadyWithoutCCMTaint(ctx, workloadCluster.GetClient(), 2)
+			waitForNodesReadyWithoutCCMTaint(ctx, workloadCluster.GetClient(), 4)
 		})
 	})
 
