Merge pull request kubernetes-sigs#1699 from Nordix/lentzi90/e2e-pre-built-node-images

🌱 E2e: Use pre-build node images

Author: k8s-ci-robot

Makefile

@@ -156,22 +156,20 @@ E2E_NO_ARTIFACT_TEMPLATES_DIR=test/e2e/data/infrastructure-openstack-no-artifact
 
 .PHONY: e2e-templates
 e2e-templates: ## Generate cluster templates for e2e tests
-e2e-templates: $(addprefix $(E2E_TEMPLATES_DIR)/, \
+e2e-templates: $(addprefix $(E2E_NO_ARTIFACT_TEMPLATES_DIR)/, \
 		 cluster-template-v1alpha5.yaml \
 		 cluster-template-v1alpha6.yaml \
 		 cluster-template-md-remediation.yaml \
 		 cluster-template-kcp-remediation.yaml \
 		 cluster-template-multi-az.yaml \
 		 cluster-template-multi-network.yaml \
 		 cluster-template-without-lb.yaml \
-		 cluster-template.yaml) \
-		 $(addprefix $(E2E_NO_ARTIFACT_TEMPLATES_DIR)/, \
+		 cluster-template.yaml \
 		 cluster-template-flatcar.yaml)
+# Currently no templates that require CI artifacts
+# $(addprefix $(E2E_TEMPLATES_DIR)/, add-templates-here.yaml) \
 
-$(E2E_TEMPLATES_DIR)/cluster-template.yaml: $(E2E_KUSTOMIZE_DIR)/with-tags $(KUSTOMIZE) FORCE
-	$(KUSTOMIZE) build "$<" > "$@"
-
-$(E2E_TEMPLATES_DIR)/cluster-template-%.yaml: $(E2E_KUSTOMIZE_DIR)/% $(KUSTOMIZE) FORCE
+$(E2E_NO_ARTIFACT_TEMPLATES_DIR)/cluster-template.yaml: $(E2E_KUSTOMIZE_DIR)/with-tags $(KUSTOMIZE) FORCE
 	$(KUSTOMIZE) build "$<" > "$@"
 
 $(E2E_NO_ARTIFACT_TEMPLATES_DIR)/cluster-template-%.yaml: $(E2E_KUSTOMIZE_DIR)/% $(KUSTOMIZE) FORCE

hack/ci/cloud-init/controller.yaml.tpl

@@ -53,8 +53,17 @@
 
     # Don't download default images, just our test images
     DOWNLOAD_DEFAULT_IMAGES=False
+    # Increase the total image size limit
+    GLANCE_LIMIT_IMAGE_SIZE_TOTAL=20000
     # We upload the Amphora image so it doesn't have to be build
-    IMAGE_URLS="https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/amphora/2022-12-05/amphora-x64-haproxy.qcow2"
+    # Upload the images so we don't have to upload them from Prow
+    # NOTE: If you get issues when changing/adding images, check if the limits
+    # are sufficient and change the variable above if needed.
+    # https://docs.openstack.org/glance/latest/admin/quotas.html
+    IMAGE_URLS="https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/amphora/2022-12-05/amphora-x64-haproxy.qcow2,"
+    IMAGE_URLS+="https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/cirros/2022-12-05/cirros-0.6.1-x86_64-disk.img,"
+    IMAGE_URLS+="https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/ubuntu/2023-09-29/ubuntu-2204-kube-v1.27.2.img,"
+    IMAGE_URLS+="https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/flatcar/flatcar-stable-3602.2.0-kube-v1.27.2.img"
 
     [[post-config|$NOVA_CONF]]
     [DEFAULT]
@@ -133,13 +142,6 @@
 
     source /opt/stack/devstack/openrc admin admin
 
-    # Upload the images so we don't have to upload them from Prow
-    # Upload cirros image first in order to avoid reach limit of project
-    # https://docs.openstack.org/glance/latest/admin/quotas.html
-    /opt/stack/devstack/tools/upload_image.sh https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/cirros/2022-12-05/cirros-0.6.1-x86_64-disk.img
-    /opt/stack/devstack/tools/upload_image.sh https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/ubuntu/2023-01-14/focal-server-cloudimg-amd64.img
-    /opt/stack/devstack/tools/upload_image.sh https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/flatcar/flatcar-stable-3602.2.0-kube-v1.27.2.img
-
     # Add the controller to its own host aggregate and availability zone
     aggregateid=$(openstack aggregate create --zone "${PRIMARY_AZ}" "${PRIMARY_AZ}" -f value -c id)
     for host in $(openstack compute service list --service nova-compute -f value -c Host)

test/e2e/data/e2e_conf.yaml

@@ -117,7 +117,7 @@ providers:
     contract: v1beta1
     files:
     - sourcePath: "../data/shared/v1beta1_provider/metadata.yaml"
-    - sourcePath: "./infrastructure-openstack/cluster-template.yaml"
+    - sourcePath: "./infrastructure-openstack-no-artifact/cluster-template.yaml"
     replacements:
     - old: "imagePullPolicy: Always"
       new: "imagePullPolicy: IfNotPresent"
@@ -131,7 +131,7 @@ providers:
     contract: v1beta1
     files:
     - sourcePath: "../data/shared/v1beta1_provider/metadata.yaml"
-    - sourcePath: "./infrastructure-openstack/cluster-template.yaml"
+    - sourcePath: "./infrastructure-openstack-no-artifact/cluster-template.yaml"
     replacements:
     - old: "imagePullPolicy: Always"
       new: "imagePullPolicy: IfNotPresent"
@@ -146,8 +146,8 @@ providers:
     # contract: v1beta1
     files:
     - sourcePath: "../data/shared/v1beta1_provider/metadata.yaml"
-    - sourcePath: "./infrastructure-openstack/cluster-template.yaml"
-    - sourcePath: "./infrastructure-openstack/cluster-template-without-lb.yaml"
+    - sourcePath: "./infrastructure-openstack-no-artifact/cluster-template.yaml"
+    - sourcePath: "./infrastructure-openstack-no-artifact/cluster-template-without-lb.yaml"
     replacements:
     - old: gcr.io/k8s-staging-capi-openstack/capi-openstack-controller:dev
       new: gcr.io/k8s-staging-capi-openstack/capi-openstack-controller:e2e
@@ -165,10 +165,6 @@ variables:
   # used to ensure we deploy to the correct management cluster
   KUBE_CONTEXT: "kind-capo-e2e"
   KUBERNETES_VERSION: "v1.27.2"
-  KUBERNETES_VERSION_UPGRADE_FROM: "v1.26.5"
-  KUBERNETES_VERSION_UPGRADE_TO: "v1.27.2"
-  ETCD_VERSION_UPGRADE_TO: "3.5.6-0"
-  COREDNS_VERSION_UPGRADE_TO: "v1.9.3"
   CNI: "../../data/cni/calico.yaml"
   CCM: "../../data/ccm/cloud-controller-manager.yaml"
   EXP_CLUSTER_RESOURCE_SET: "true"
@@ -182,7 +178,7 @@ variables:
   OPENSTACK_DNS_NAMESERVERS: "8.8.8.8"
   OPENSTACK_FAILURE_DOMAIN: "testaz1"
   OPENSTACK_FAILURE_DOMAIN_ALT: "testaz2"
-  OPENSTACK_IMAGE_NAME: "focal-server-cloudimg-amd64"
+  OPENSTACK_IMAGE_NAME: "ubuntu-2204-kube-v1.27.2"
   OPENSTACK_NODE_MACHINE_FLAVOR: "m1.small"
   OPENSTACK_SSH_KEY_NAME: "cluster-api-provider-openstack-sigs-k8s-io"
   OPENSTACK_EXTERNAL_NETWORK_ID: ""

test/e2e/data/kustomize/common-patches/containerd/containerd-kcp.yaml

@@ -4,6 +4,5 @@ resources:
 
 components:
 - ../common-patches/cni
-- ../common-patches/containerd
 - ../upgrade-patches
 - ../common-patches/ccm

test/e2e/data/kustomize/common-patches/containerd/containerd-kct.yaml

@@ -2,7 +2,7 @@
 - op: add
   path: /spec/template/spec/rootVolume
   value:
-    diskSize: 15
+    diskSize: 25
 - op: add
   path: /spec/template/spec/additionalBlockDevices
   value:

test/e2e/data/kustomize/common-patches/containerd/kustomization.yaml

@@ -2,7 +2,7 @@
 - op: add
   path: /spec/template/spec/rootVolume
   value:
-    diskSize: 15
+    diskSize: 25
     volumeType: ${OPENSTACK_VOLUME_TYPE_ALT}
     availabilityZone: ${OPENSTACK_FAILURE_DOMAIN}
 - op: add

test/e2e/data/kustomize/default/kustomization.yaml

@@ -4,5 +4,4 @@ resources:
 
 components:
 - ../common-patches/cni
-- ../common-patches/containerd
 - ../common-patches/ccm

test/e2e/data/kustomize/multi-az/patch-machine-template-control-plane.yaml

@@ -4,7 +4,6 @@ resources:
 
 components:
 - ../common-patches/cni
-- ../common-patches/containerd
 - ../common-patches/ccm
 
 patches:

test/e2e/data/kustomize/multi-az/patch-machine-template-worker.yaml

@@ -4,5 +4,4 @@ resources:
 
 components:
 - ../common-patches/cni
-- ../common-patches/containerd
 - ../common-patches/ccm

test/e2e/data/kustomize/v1alpha5/kustomization.yaml

@@ -44,14 +44,14 @@ const (
 	OpenStackImageName         = "OPENSTACK_IMAGE_NAME"
 	OpenStackNodeMachineFlavor = "OPENSTACK_NODE_MACHINE_FLAVOR"
 	SSHUserMachine             = "SSH_USER_MACHINE"
-	FlavorDefault              = "ci-artifacts"
-	FlavorWithoutLB            = "without-lb-ci-artifacts"
-	FlavorMultiNetwork         = "multi-network-ci-artifacts"
-	FlavorMultiAZ              = "multi-az-ci-artifacts"
-	FlavorV1alpha5             = "v1alpha5-ci-artifacts"
-	FlavorV1alpha6             = "v1alpha6-ci-artifacts"
-	FlavorMDRemediation        = "md-remediation-ci-artifacts"
-	FlavorKCPRemediation       = "kcp-remediation-ci-artifacts"
+	FlavorDefault              = ""
+	FlavorWithoutLB            = "without-lb"
+	FlavorMultiNetwork         = "multi-network"
+	FlavorMultiAZ              = "multi-az"
+	FlavorV1alpha5             = "v1alpha5"
+	FlavorV1alpha6             = "v1alpha6"
+	FlavorMDRemediation        = "md-remediation"
+	FlavorKCPRemediation       = "kcp-remediation"
 	FlavorFlatcar              = "flatcar"
 )
 

test/e2e/data/kustomize/v1alpha6/kustomization.yaml

@@ -58,6 +58,8 @@ func Node1BeforeSuite(e2eCtx *E2EContext) []byte {
 	flag.Parse()
 	Expect(e2eCtx.Settings.ConfigPath).To(BeAnExistingFile(), "Invalid test suite argument. configPath should be an existing file.")
 	Expect(os.MkdirAll(e2eCtx.Settings.ArtifactFolder, 0o750)).To(Succeed(), "Invalid test suite argument. Can't create artifacts-folder %q", e2eCtx.Settings.ArtifactFolder)
+	templatesDir := path.Join(e2eCtx.Settings.ArtifactFolder, "templates")
+	Expect(os.MkdirAll(templatesDir, 0o750)).To(Succeed(), "Can't create templates folder %q", templatesDir)
 	Logf("Loading the e2e test configuration from %q", e2eCtx.Settings.ConfigPath)
 	e2eCtx.E2EConfig = LoadE2EConfig(e2eCtx.Settings.ConfigPath)
 
@@ -66,7 +68,8 @@ func Node1BeforeSuite(e2eCtx *E2EContext) []byte {
 
 	templates := []clusterctl.Files{}
 
-	// TODO(sbuerin): we always need ci artifacts, because we don't have images for every Kubernetes version
+	// Cluster templates in this folder will get ci artifacts injected. It makes it possible to use generic cloud images
+	// without kubernetes pre-installed.
 	err := filepath.WalkDir(path.Join(e2eCtx.Settings.DataFolder, "infrastructure-openstack"), func(f string, d fs.DirEntry, _ error) error {
 		filename := filepath.Base(f)
 		fileExtension := filepath.Ext(filename)
@@ -90,7 +93,7 @@ func Node1BeforeSuite(e2eCtx *E2EContext) []byte {
 		Expect(err).NotTo(HaveOccurred())
 
 		targetName := fmt.Sprintf("%s-ci-artifacts.yaml", strings.TrimSuffix(filename, fileExtension))
-		targetTemplate := path.Join(e2eCtx.Settings.ArtifactFolder, "templates", targetName)
+		targetTemplate := path.Join(templatesDir, targetName)
 
 		// We have to copy the file from ciTemplate to targetTemplate. Otherwise it would be overwritten because
 		// ciTemplate is the same for all templates
@@ -120,7 +123,7 @@ func Node1BeforeSuite(e2eCtx *E2EContext) []byte {
 		t, err := os.ReadFile(f)
 		Expect(err).NotTo(HaveOccurred())
 
-		targetTemplate := path.Join(e2eCtx.Settings.ArtifactFolder, "templates", filename)
+		targetTemplate := path.Join(templatesDir, filename)
 
 		err = os.WriteFile(targetTemplate, t, 0o600)
 		Expect(err).NotTo(HaveOccurred())

test/e2e/data/kustomize/without-lb/kustomization.yaml

@@ -81,7 +81,7 @@ var _ = Describe("e2e tests [PR-Blocking]", func() {
 		clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult)
 		Expect(e2eCtx.E2EConfig).ToNot(BeNil(), "Invalid argument. e2eConfig can't be nil when calling %s spec", specName)
 		Expect(e2eCtx.E2EConfig.Variables).To(HaveKey(shared.KubernetesVersion))
-		shared.SetEnvVar("USE_CI_ARTIFACTS", "true", false)
+		shared.SetEnvVar("USE_CI_ARTIFACTS", "false", false)
 		postClusterCleanup = nil
 	})
 
@@ -600,7 +600,7 @@ var _ = Describe("e2e tests [PR-Blocking]", func() {
 				rootVolumes[machine.Name] = rootVolume
 				Expect(*rootVolume).To(MatchFields(IgnoreExtras, Fields{
 					"Name":     Equal(fmt.Sprintf("%s-root", server.Name)),
-					"Size":     Equal(15),
+					"Size":     Equal(25),
 					"Bootable": Equal("true"), // This is genuinely a string, not a bool
 				}), "Boot volume %s for machine %s not as expected", rootVolume.ID, machine.Name)