From 569c83bb22081e1a64c984fa13d00100a9ecd788 Mon Sep 17 00:00:00 2001 From: Sid Shukla Date: Wed, 26 Jun 2024 16:01:58 +0200 Subject: [PATCH 01/11] ci: Run Nutanix E2E on Self Hosted runners This is necessary as we want some e2e jobs to run on hosted runners and some to run on self-hosted runners. The actions runner controller does not support using multiple labels to target the runners See: https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/using-actions-runner-controller-runners-in-a-workflow#using-runner-scale-set-names --- .github/actionlint.yaml | 7 + .github/workflows/checks.yml | 6 +- .github/workflows/e2e.yml | 71 +++++-- api/v1alpha1/zz_generated.deepcopy.go | 4 +- devbox.json | 1 + devbox.lock | 48 +++++ .../nutanix-cluster-calico-crs.yaml | 2 +- .../nutanix-cluster-cilium-crs.yaml | 2 +- .../calico/crs/kustomization.yaml.tmpl | 3 + .../calico/helm-addon/kustomization.yaml.tmpl | 3 + .../cilium/crs/kustomization.yaml.tmpl | 3 + .../cilium/helm-addon/kustomization.yaml.tmpl | 3 + .../patches/nutanix/ccm-crs-strategy.yaml | 6 + .../nutanix/ccm-helm-addon-strategy.yaml | 6 + hack/kind/dns-override.yaml | 28 +++ hack/tools/redact-artifacts/main.go | 100 +++++++++ make/all.mk | 1 + make/redact.mk | 11 + test/e2e/config/caren.yaml | 14 +- test/e2e/e2e_suite_test.go | 39 ++++ test/e2e/self_hosted_test.go | 192 +++++++++--------- 21 files changed, 431 insertions(+), 119 deletions(-) create mode 100644 .github/actionlint.yaml create mode 100644 hack/examples/patches/nutanix/ccm-crs-strategy.yaml create mode 100644 hack/examples/patches/nutanix/ccm-helm-addon-strategy.yaml create mode 100644 hack/kind/dns-override.yaml create mode 100644 hack/tools/redact-artifacts/main.go create mode 100644 make/redact.mk diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 000000000..8c04ab7e5 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,7 @@ +# Copyright 2024 Nutanix. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +self-hosted-runner: + # Labels of self-hosted runner in array of string + labels: + - self-hosted-ncn-dind diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 40844fd4b..4ed3bcce9 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -59,14 +59,15 @@ jobs: matrix: provider: - Docker + - Nutanix # Uncomment below once we have the ability to run e2e tests on other providers from GHA. # - AWS - # - Nutanix fail-fast: false uses: ./.github/workflows/e2e.yml with: provider: ${{ matrix.provider }} focus: Quick start + runs-on: ${{ matrix.provider == 'Nutanix' && 'self-hosted-ncn-dind' || 'ubuntu-22.04' }} secrets: inherit permissions: contents: read @@ -77,14 +78,15 @@ jobs: matrix: provider: - Docker + - Nutanix # Uncomment below once we have the ability to run e2e tests on other providers from GHA. # - AWS - # - Nutanix fail-fast: false uses: ./.github/workflows/e2e.yml with: provider: ${{ matrix.provider }} focus: Self-hosted + runs-on: ${{ matrix.provider == 'Nutanix' && 'self-hosted-ncn-dind' || 'ubuntu-22.04' }} secrets: inherit permissions: contents: read diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 487e9af92..8dba3cb71 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -14,10 +14,14 @@ on: focus: description: e2e tests to focus type: string + runs-on: + description: The runner to run the e2e tests on + type: string + required: true jobs: e2e-test: - runs-on: ubuntu-22.04 + runs-on: ${{ inputs.runs-on }} permissions: contents: read checks: write @@ -27,41 +31,68 @@ jobs: with: fetch-depth: 0 - - name: Install devbox - uses: jetify-com/devbox-install-action@v0.11.0 + - uses: cachix/install-nix-action@V27 + if: inputs.provider == 'Nutanix' with: - enable-cache: true + github_access_token: ${{ secrets.GITHUB_TOKEN }} - - name: Go cache - uses: actions/cache@v4 + - name: Install devbox + uses: jetify-com/devbox-install-action@v0.11.0 with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- + enable-cache: "true" + skip-nix-installation: ${{ inputs.provider == 'Nutanix' }} # The default disk size of these runners is ~14GB, this is not enough to run the e2e tests. # Cleanup the disk, see upstream discussion https://github.com/actions/runner-images/issues/2840. - name: Cleanup Disk Space + if: inputs.runs-on != 'self-hosted-ncn-dind' run: | echo "Before removing files:" df -h sudo rm -rf /usr/share/dotnet sudo rm -rf /opt/ghc sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf "${AGENT_TOOLSDIRECTORY}" echo "After removing files:" df -h + - name: Get Control Plane endpoint IP + id: get-control-plane-endpoint-ip + if: inputs.provider == 'Nutanix' + run: | + CONTROL_PLANE_ENDPOINT_RANGE_START="${{ vars.NUTANIX_CONTROL_PLANE_ENDPOINT_RANGE_START }}" + CONTROL_PLANE_ENDPOINT_RANGE_END="${{ vars.NUTANIX_CONTROL_PLANE_ENDPOINT_RANGE_END }}" + control_plane_endpoint_ip="$(devbox run -- fping -g -u "${CONTROL_PLANE_ENDPOINT_RANGE_START}" "${CONTROL_PLANE_ENDPOINT_RANGE_END}" 2>/dev/null | devbox run -- shuf --head-count=1)" + echo "control_plane_endpoint_ip=${control_plane_endpoint_ip}" >> "${GITHUB_OUTPUT}" + - name: Go cache + uses: actions/cache@v4 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Run e2e tests - run: devbox run -- make e2e-test E2E_LABEL='provider:${{ inputs.provider }}' E2E_SKIP='${{ inputs.skip }}' E2E_FOCUS='${{ inputs.focus }}' + run: devbox run -- make e2e-test E2E_LABEL='provider:${{ inputs.provider }} && cni:Cilium && addonStrategy:HelmAddon' E2E_SKIP='${{ inputs.skip }}' E2E_FOCUS='${{ inputs.focus }}' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} NUTANIX_ENDPOINT: ${{ secrets.NUTANIX_ENDPOINT }} - NUTANIX_PASSWORD: ${{ secrets.NUTANIX_PASSWORD }} NUTANIX_USER: ${{ secrets.NUTANIX_USER }} + NUTANIX_PASSWORD: ${{ secrets.NUTANIX_PASSWORD }} + NUTANIX_PORT: ${{ vars.NUTANIX_PORT }} + NUTANIX_INSECURE: false + NUTANIX_PRISM_ELEMENT_CLUSTER_NAME: ${{ vars.NUTANIX_PRISM_ELEMENT_CLUSTER_NAME }} + NUTANIX_SUBNET_NAME: ${{ vars.NUTANIX_SUBNET_NAME }} + NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME: ${{ vars.NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME }} + NUTANIX_STORAGE_CONTAINER_NAME: ${{ vars.NUTANIX_STORAGE_CONTAINER_NAME }} + CONTROL_PLANE_ENDPOINT_IP: ${{ steps.get-control-plane-endpoint-ip.outputs.control_plane_endpoint_ip }} + + - name: Redact secrets from artifacts + run: devbox run -- make redact-artifacts - if: success() || failure() # always run even if the previous step fails name: Publish e2e test report @@ -71,3 +102,13 @@ jobs: check_name: 'e2e test report' detailed_summary: true require_passed_tests: true + + - if: success() || failure() # always run even if the previous step fails + name: Upload e2e test artifacts + uses: actions/upload-artifact@v4 + with: + if-no-files-found: warn + overwrite: false + name: ${{ inputs.provider }} ${{ inputs.focus }} e2e artifacts + path: | + _artifacts diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index c546643ad..34592e92b 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1294,7 +1294,9 @@ func (in *NutanixNodeSpec) DeepCopy() *NutanixNodeSpec { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NutanixPrismCentralEndpointCredentials) DeepCopyInto(out *NutanixPrismCentralEndpointCredentials) { +func (in *NutanixPrismCentralEndpointCredentials) DeepCopyInto( + out *NutanixPrismCentralEndpointCredentials, +) { *out = *in out.SecretRef = in.SecretRef } diff --git a/devbox.json b/devbox.json index 369f18c24..0a80a8fc3 100644 --- a/devbox.json +++ b/devbox.json @@ -6,6 +6,7 @@ "crane@latest", "envsubst@latest", "findutils@latest", + "fping@latest", "gh@latest", "ginkgo@latest", "git@latest", diff --git a/devbox.lock b/devbox.lock index f5e4ccb95..8e54bb4c7 100644 --- a/devbox.lock +++ b/devbox.lock @@ -381,6 +381,54 @@ } } }, + "fping@latest": { + "last_modified": "2024-06-12T20:55:33Z", + "resolved": "github:NixOS/nixpkgs/a9858885e197f984d92d7fe64e9fff6b2e488d40#fping", + "source": "devbox-search", + "version": "5.2", + "systems": { + "aarch64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/clxfp6jl0d2fs1bp2d1278534n2gixbj-fping-5.2", + "default": true + } + ], + "store_path": "/nix/store/clxfp6jl0d2fs1bp2d1278534n2gixbj-fping-5.2" + }, + "aarch64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/ilzq042wih0h5vdzxcpf6sd826h37g6w-fping-5.2", + "default": true + } + ], + "store_path": "/nix/store/ilzq042wih0h5vdzxcpf6sd826h37g6w-fping-5.2" + }, + "x86_64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/hrh3202f2njx3skj3xn33fish5az5691-fping-5.2", + "default": true + } + ], + "store_path": "/nix/store/hrh3202f2njx3skj3xn33fish5az5691-fping-5.2" + }, + "x86_64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/2nr99jpa9g7b5z8pwj85awzh4qbhas28-fping-5.2", + "default": true + } + ], + "store_path": "/nix/store/2nr99jpa9g7b5z8pwj85awzh4qbhas28-fping-5.2" + } + } + }, "gh@latest": { "last_modified": "2024-05-30T12:09:21Z", "resolved": "github:NixOS/nixpkgs/aa61b27554a5fc282758bf0324781e3464ef2cde#gh", diff --git a/examples/capi-quick-start/nutanix-cluster-calico-crs.yaml b/examples/capi-quick-start/nutanix-cluster-calico-crs.yaml index 1b31ea277..79b7e74f6 100644 --- a/examples/capi-quick-start/nutanix-cluster-calico-crs.yaml +++ b/examples/capi-quick-start/nutanix-cluster-calico-crs.yaml @@ -67,7 +67,7 @@ spec: credentials: secretRef: name: ${CLUSTER_NAME}-pc-creds - strategy: HelmAddon + strategy: ClusterResourceSet clusterAutoscaler: strategy: ClusterResourceSet cni: diff --git a/examples/capi-quick-start/nutanix-cluster-cilium-crs.yaml b/examples/capi-quick-start/nutanix-cluster-cilium-crs.yaml index 5073709e0..44cca319f 100644 --- a/examples/capi-quick-start/nutanix-cluster-cilium-crs.yaml +++ b/examples/capi-quick-start/nutanix-cluster-cilium-crs.yaml @@ -67,7 +67,7 @@ spec: credentials: secretRef: name: ${CLUSTER_NAME}-pc-creds - strategy: HelmAddon + strategy: ClusterResourceSet clusterAutoscaler: strategy: ClusterResourceSet cni: diff --git a/hack/examples/overlays/clusters/nutanix/calico/crs/kustomization.yaml.tmpl b/hack/examples/overlays/clusters/nutanix/calico/crs/kustomization.yaml.tmpl index f7dd00af3..0d7c8921f 100644 --- a/hack/examples/overlays/clusters/nutanix/calico/crs/kustomization.yaml.tmpl +++ b/hack/examples/overlays/clusters/nutanix/calico/crs/kustomization.yaml.tmpl @@ -17,3 +17,6 @@ patches: - target: kind: Cluster path: ../../../../../patches/crs-strategy.yaml + - target: + kind: Cluster + path: ../../../../../patches/nutanix/ccm-crs-strategy.yaml diff --git a/hack/examples/overlays/clusters/nutanix/calico/helm-addon/kustomization.yaml.tmpl b/hack/examples/overlays/clusters/nutanix/calico/helm-addon/kustomization.yaml.tmpl index d2ffaa757..03498c0de 100644 --- a/hack/examples/overlays/clusters/nutanix/calico/helm-addon/kustomization.yaml.tmpl +++ b/hack/examples/overlays/clusters/nutanix/calico/helm-addon/kustomization.yaml.tmpl @@ -17,3 +17,6 @@ patches: - target: kind: Cluster path: ../../../../../patches/helm-addon-strategy.yaml + - target: + kind: Cluster + path: ../../../../../patches/nutanix/ccm-helm-addon-strategy.yaml diff --git a/hack/examples/overlays/clusters/nutanix/cilium/crs/kustomization.yaml.tmpl b/hack/examples/overlays/clusters/nutanix/cilium/crs/kustomization.yaml.tmpl index 4d4053efb..9b1b24b61 100644 --- a/hack/examples/overlays/clusters/nutanix/cilium/crs/kustomization.yaml.tmpl +++ b/hack/examples/overlays/clusters/nutanix/cilium/crs/kustomization.yaml.tmpl @@ -17,3 +17,6 @@ patches: - target: kind: Cluster path: ../../../../../patches/crs-strategy.yaml + - target: + kind: Cluster + path: ../../../../../patches/nutanix/ccm-crs-strategy.yaml diff --git a/hack/examples/overlays/clusters/nutanix/cilium/helm-addon/kustomization.yaml.tmpl b/hack/examples/overlays/clusters/nutanix/cilium/helm-addon/kustomization.yaml.tmpl index 82bb89906..9c2d5a64b 100644 --- a/hack/examples/overlays/clusters/nutanix/cilium/helm-addon/kustomization.yaml.tmpl +++ b/hack/examples/overlays/clusters/nutanix/cilium/helm-addon/kustomization.yaml.tmpl @@ -17,3 +17,6 @@ patches: - target: kind: Cluster path: ../../../../../patches/helm-addon-strategy.yaml + - target: + kind: Cluster + path: ../../../../../patches/nutanix/ccm-helm-addon-strategy.yaml diff --git a/hack/examples/patches/nutanix/ccm-crs-strategy.yaml b/hack/examples/patches/nutanix/ccm-crs-strategy.yaml new file mode 100644 index 000000000..30df2772e --- /dev/null +++ b/hack/examples/patches/nutanix/ccm-crs-strategy.yaml @@ -0,0 +1,6 @@ +# Copyright 2024 Nutanix. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +- op: "add" + path: "/spec/topology/variables/0/value/addons/ccm/strategy" + value: ClusterResourceSet diff --git a/hack/examples/patches/nutanix/ccm-helm-addon-strategy.yaml b/hack/examples/patches/nutanix/ccm-helm-addon-strategy.yaml new file mode 100644 index 000000000..32fdd0560 --- /dev/null +++ b/hack/examples/patches/nutanix/ccm-helm-addon-strategy.yaml @@ -0,0 +1,6 @@ +# Copyright 2024 Nutanix. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +- op: "add" + path: "/spec/topology/variables/0/value/addons/ccm/strategy" + value: HelmAddon diff --git a/hack/kind/dns-override.yaml b/hack/kind/dns-override.yaml new file mode 100644 index 000000000..ed3fc3b63 --- /dev/null +++ b/hack/kind/dns-override.yaml @@ -0,0 +1,28 @@ +# Copyright 2024 Nutanix. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: v1 +data: + Corefile: | + .:53 { + errors + health { + lameduck 5s + } + ready + kubernetes cluster.local in-addr.arpa ip6.arpa { + pods insecure + fallthrough in-addr.arpa ip6.arpa + ttl 30 + } + prometheus :9153 + forward . 8.8.8.8 + cache 30 + loop + reload + loadbalance + } +kind: ConfigMap +metadata: + name: coredns + namespace: kube-system diff --git a/hack/tools/redact-artifacts/main.go b/hack/tools/redact-artifacts/main.go new file mode 100644 index 000000000..83bd427d1 --- /dev/null +++ b/hack/tools/redact-artifacts/main.go @@ -0,0 +1,100 @@ +// Copyright 2024 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "sigs.k8s.io/yaml" +) + +const ( + artifactDir = "_artifacts" + extensionYAML = ".yaml" + extensionYML = ".yml" + kindKey = "kind" + secretKind = "Secret" + dataKey = "data" + stringDataKey = "stringData" + redactedValue = "***REDACTED***" + documentSeparator = "---" +) + +func main() { + err := filepath.Walk(artifactDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if !info.IsDir() && isYAMLFile(path) { + processYAMLFile(path) + } + return nil + }) + if err != nil { + fmt.Printf("Error walking the path: %v\n", err) + } +} + +func isYAMLFile(path string) bool { + ext := strings.ToLower(filepath.Ext(path)) + return ext == extensionYAML || ext == extensionYML +} + +func processYAMLFile(path string) { + data, err := os.ReadFile(path) + if err != nil { + fmt.Printf("Failed to read file %s: %v\n", path, err) + return + } + + output := make([]string, 0) + // Split the file into multiple documents + docs := strings.Split(string(data), documentSeparator) + for _, doc := range docs { + doc = strings.TrimSpace(doc) + if doc == "" { + continue + } + + var unstructured map[string]interface{} + if err := yaml.Unmarshal([]byte(doc), &unstructured); err != nil { + fmt.Printf("Failed to unmarshal document in file %s: %v\n", path, err) + continue + } + + if kind, ok := unstructured[kindKey].(string); ok && kind == secretKind { + redactSecret(unstructured) + } + + redacted, err := yaml.Marshal(unstructured) + if err != nil { + fmt.Printf("Failed to marshal document in file %s: %v\n", path, err) + continue + } + + output = append(output, string(redacted)) + } + + if err := os.WriteFile(path, []byte(strings.Join(output, "\n"+documentSeparator+"\n")), 0o600); err != nil { + fmt.Printf("Failed to write file %s: %v\n", path, err) + } +} + +func redactSecret(unstructured map[string]interface{}) { + if data, ok := unstructured[dataKey].(map[string]interface{}); ok { + for key := range data { + data[key] = redactedValue + } + } + + if stringData, ok := unstructured[stringDataKey].(map[string]interface{}); ok { + for key := range stringData { + stringData[key] = redactedValue + } + } +} diff --git a/make/all.mk b/make/all.mk index d1fdc8509..0181b3269 100644 --- a/make/all.mk +++ b/make/all.mk @@ -19,3 +19,4 @@ include $(INCLUDE_DIR)helm.mk include $(INCLUDE_DIR)examples.mk include $(INCLUDE_DIR)apis.mk include $(INCLUDE_DIR)docs.mk +include $(INCLUDE_DIR)redact.mk diff --git a/make/redact.mk b/make/redact.mk new file mode 100644 index 000000000..82d9fc452 --- /dev/null +++ b/make/redact.mk @@ -0,0 +1,11 @@ +# Copyright 2024 Nutanix. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +ARTIFACT_DIR := $(REPO_ROOT)/_artifacts + +# Define the placeholder value +REDACTED_PLACEHOLDER="***REDACTED***" + +.PHONY: redact-artifacts +redact-artifacts: ## Redacts all secrets in $(ARTIFACT_DIR) + go run hack/tools/redact-artifacts/main.go diff --git a/test/e2e/config/caren.yaml b/test/e2e/config/caren.yaml index 37aa6b7f7..6d696bc28 100644 --- a/test/e2e/config/caren.yaml +++ b/test/e2e/config/caren.yaml @@ -8,6 +8,14 @@ images: loadBehavior: mustLoad - name: ghcr.io/nutanix-cloud-native/caren-helm-reg:${E2E_IMAGE_TAG}-${GOARCH} loadBehavior: mustLoad + - name: docker.io/mesosphere/cluster-api-controller:${CAPI_VERSION}-d2iq.0 + loadBehavior: mustLoad + - name: docker.io/mesosphere/kubeadm-bootstrap-controller:${CAPI_VERSION}-d2iq.0 + loadBehavior: mustLoad + - name: docker.io/mesosphere/kubeadm-control-plane-controller:${CAPI_VERSION}-d2iq.0 + loadBehavior: mustLoad + - name: docker.io/mesosphere/capd-manager:${CAPD_VERSION}-d2iq.0 + loadBehavior: mustLoad providers: - name: cluster-api @@ -188,15 +196,15 @@ variables: # IP/FQDN of Prism Central. # NOTE: This has to be overridden by specifying the env var NUTANIX_ENDPOINT when running the e2e tests. It is # set as empty here to enable running the e2e tests for non-nutanix providers locally without setting the env var. - NUTANIX_ENDPOINT: "" + # NUTANIX_ENDPOINT: "" # # Port of Prism Central. Default: 9440 # NUTANIX_PORT: 9440 # # Disable Prism Central certificate checking. Default: false # NUTANIX_INSECURE: false # # Prism Central user - NUTANIX_USER: "" + # NUTANIX_USER: "" # # Prism Central password - NUTANIX_PASSWORD: "" + # NUTANIX_PASSWORD: "" # # Host IP to be assigned to the CAPX Kubernetes cluster. # CONTROL_PLANE_ENDPOINT_IP: "" # # Port of the CAPX Kubernetes cluster. Default: 6443 diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 9c45ba0e5..dd3305f10 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -11,16 +11,22 @@ import ( "encoding/base64" "encoding/gob" "flag" + "fmt" "os" "path/filepath" "slices" "strings" "testing" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/klog/v2" capie2e "sigs.k8s.io/cluster-api/test/e2e" capie2eframework "sigs.k8s.io/cluster-api/test/framework" @@ -110,6 +116,39 @@ var _ = SynchronizedBeforeSuite(func() []byte { useExistingCluster, ) + By("Overriding coreDNS resolver") + // override coredns resolver to 8.8.8.8 and restart coredns deployment + // read the dns-override.yaml file + filePath, _ := filepath.Abs("../../hack/kind/dns-override.yaml") + yamlFile, err := os.ReadFile(filePath) + Expect(err).To(BeNil(), "Failed to read the dns-override.yaml file") + + // decode the yaml file into a Kubernetes object + decode := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(yamlFile), 4096) + configMap := &corev1.ConfigMap{} + err = decode.Decode(&configMap) + Expect(err).To(BeNil(), "Failed to decode the yaml file into a Kubernetes object") + + _, err = bootstrapClusterProxy.GetClientSet(). + CoreV1(). + ConfigMaps(configMap.GetNamespace()). + Update(context.Background(), configMap, metav1.UpdateOptions{}) + Expect( + err, + ).To(BeNil(), "Failed to update the coredns deployment with the dns-override.yaml file") + + timeNow := time.Now().Format(time.RFC3339) + patch := fmt.Sprintf( + `{"spec":{"template":{"metadata":{"annotations":{"kubectl.kubernetes.io/restartedAt":%q}}}}}`, + timeNow, + ) + + _, err = bootstrapClusterProxy.GetClientSet(). + AppsV1(). + Deployments("kube-system"). + Patch(context.Background(), "coredns", types.StrategicMergePatchType, []byte(patch), metav1.PatchOptions{}) + Expect(err).To(BeNil(), "Failed to restart the coredns deployment") + By("Initializing the bootstrap cluster") initBootstrapCluster(bootstrapClusterProxy, e2eConfig, clusterctlConfigPath, artifactFolder) diff --git a/test/e2e/self_hosted_test.go b/test/e2e/self_hosted_test.go index 8e15d69a9..9b465a89d 100644 --- a/test/e2e/self_hosted_test.go +++ b/test/e2e/self_hosted_test.go @@ -22,103 +22,103 @@ import ( ) var _ = Describe("Self-hosted", Serial, func() { - provider := "Docker" - lowercaseProvider := strings.ToLower(provider) - for _, cniProvider := range []string{"Cilium", "Calico"} { - for _, addonStrategy := range []string{"HelmAddon", "ClusterResourceSet"} { - strategy := "" - switch addonStrategy { - case "HelmAddon": - strategy = "helm-addon" - case "ClusterResourceSet": - strategy = "crs" - default: - Fail("unknown addon strategy: " + addonStrategy) - } - flavour := fmt.Sprintf( - "topology-%s-%s", - strings.ToLower(cniProvider), - strategy, - ) - Context( - flavour, - Label("provider:"+provider), - Label("cni:"+cniProvider), - Label("addonStrategy:"+addonStrategy), - func() { - framework.SelfHostedSpec(ctx, func() framework.SelfHostedSpecInput { - return framework.SelfHostedSpecInput{ - E2EConfig: e2eConfig, - ClusterctlConfigPath: clusterctlConfigPath, - BootstrapClusterProxy: bootstrapClusterProxy, - ArtifactFolder: artifactFolder, - SkipCleanup: skipCleanup, - Flavor: flavour, - InfrastructureProvider: ptr.To(lowercaseProvider), - PostClusterMoved: func(proxy capiframework.ClusterProxy, cluster *clusterv1.Cluster) { - By( - "Waiting for all requested addons to be ready in workload cluster", - ) - workloadCluster := capiframework.GetClusterByName( - ctx, - capiframework.GetClusterByNameInput{ - Namespace: cluster.GetNamespace(), - Name: cluster.GetName(), - Getter: proxy.GetClient(), - }, - ) - Expect(workloadCluster.Spec.Topology).ToNot(BeNil()) - clusterVars := variables.ClusterVariablesToVariablesMap( - workloadCluster.Spec.Topology.Variables, - ) - addonsConfig, err := variables.Get[apivariables.Addons]( - clusterVars, - v1alpha1.ClusterConfigVariableName, - "addons", - ) - Expect(err).ToNot(HaveOccurred()) - WaitForAddonsToBeReadyInWorkloadCluster( - ctx, - WaitForAddonsToBeReadyInWorkloadClusterInput{ - AddonsConfig: addonsConfig, - ClusterProxy: proxy, - WorkloadCluster: workloadCluster, - InfrastructureProvider: lowercaseProvider, - DeploymentIntervals: e2eConfig.GetIntervals( - flavour, - "wait-deployment", - ), - DaemonSetIntervals: e2eConfig.GetIntervals( - flavour, - "wait-daemonset", - ), - HelmReleaseIntervals: e2eConfig.GetIntervals( - flavour, - "wait-helmrelease", - ), - ClusterResourceSetIntervals: e2eConfig.GetIntervals( - flavour, - "wait-clusterresourceset", - ), - }, - ) + for _, provider := range []string{"Docker", "Nutanix"} { + lowercaseProvider := strings.ToLower(provider) + for _, cniProvider := range []string{"Cilium", "Calico"} { + for _, addonStrategy := range []string{"HelmAddon", "ClusterResourceSet"} { + strategy := "" + switch addonStrategy { + case "HelmAddon": + strategy = "helm-addon" + case "ClusterResourceSet": + strategy = "crs" + default: + Fail("unknown addon strategy: " + addonStrategy) + } + flavour := fmt.Sprintf( + "topology-%s-%s", + strings.ToLower(cniProvider), + strategy, + ) + Context( + flavour, + Label("provider:"+provider), + Label("cni:"+cniProvider), + Label("addonStrategy:"+addonStrategy), + func() { + framework.SelfHostedSpec(ctx, func() framework.SelfHostedSpecInput { + return framework.SelfHostedSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + Flavor: flavour, + InfrastructureProvider: ptr.To(lowercaseProvider), + PostClusterMoved: func(proxy capiframework.ClusterProxy, cluster *clusterv1.Cluster) { + By( + "Waiting for all requested addons to be ready in workload cluster", + ) + workloadCluster := capiframework.GetClusterByName( + ctx, + capiframework.GetClusterByNameInput{ + Namespace: cluster.GetNamespace(), + Name: cluster.GetName(), + Getter: proxy.GetClient(), + }, + ) + Expect(workloadCluster.Spec.Topology).ToNot(BeNil()) + clusterVars := variables.ClusterVariablesToVariablesMap( + workloadCluster.Spec.Topology.Variables, + ) + addonsConfig, err := variables.Get[apivariables.Addons]( + clusterVars, + v1alpha1.ClusterConfigVariableName, + "addons", + ) + Expect(err).ToNot(HaveOccurred()) + WaitForAddonsToBeReadyInWorkloadCluster( + ctx, + WaitForAddonsToBeReadyInWorkloadClusterInput{ + AddonsConfig: addonsConfig, + ClusterProxy: proxy, + WorkloadCluster: workloadCluster, + DeploymentIntervals: e2eConfig.GetIntervals( + flavour, + "wait-deployment", + ), + DaemonSetIntervals: e2eConfig.GetIntervals( + flavour, + "wait-daemonset", + ), + HelmReleaseIntervals: e2eConfig.GetIntervals( + flavour, + "wait-helmrelease", + ), + ClusterResourceSetIntervals: e2eConfig.GetIntervals( + flavour, + "wait-clusterresourceset", + ), + }, + ) - WaitForCoreDNSToBeReadyInWorkloadCluster( - ctx, - WaitForCoreDNSToBeReadyInWorkloadClusterInput{ - WorkloadCluster: workloadCluster, - ClusterProxy: proxy, - DeploymentIntervals: e2eConfig.GetIntervals( - flavour, - "wait-deployment", - ), - }, - ) - }, - } - }) - }, - ) + WaitForCoreDNSToBeReadyInWorkloadCluster( + ctx, + WaitForCoreDNSToBeReadyInWorkloadClusterInput{ + WorkloadCluster: workloadCluster, + ClusterProxy: proxy, + DeploymentIntervals: e2eConfig.GetIntervals( + flavour, + "wait-deployment", + ), + }, + ) + }, + } + }) + }, + ) + } } } }) From 61852a7be005781877ce20255fa0a7ac314bcfc4 Mon Sep 17 00:00:00 2001 From: Sid Shukla Date: Mon, 1 Jul 2024 14:21:39 +0200 Subject: [PATCH 02/11] ci: update redact-artifacts workflow to always run artifacts should be redacted regardless of success/failure of test. --- .github/workflows/e2e.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 8dba3cb71..eec947952 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -92,6 +92,7 @@ jobs: CONTROL_PLANE_ENDPOINT_IP: ${{ steps.get-control-plane-endpoint-ip.outputs.control_plane_endpoint_ip }} - name: Redact secrets from artifacts + if: success() || failure() # always run even if the previous step fails run: devbox run -- make redact-artifacts - if: success() || failure() # always run even if the previous step fails From 83d4cbec7b5ef8f74e93844d1d5ab420a53fd0b9 Mon Sep 17 00:00:00 2001 From: Sid Shukla Date: Mon, 1 Jul 2024 15:42:53 +0200 Subject: [PATCH 03/11] ci: Add step to validate control plane endpoint is not empty We have seen instances where the controlplane endpoint is empty and then tests fail down the line. It's better to catch that early on and fail fast. --- .github/workflows/e2e.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index eec947952..6ecce487f 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -64,6 +64,14 @@ jobs: CONTROL_PLANE_ENDPOINT_RANGE_END="${{ vars.NUTANIX_CONTROL_PLANE_ENDPOINT_RANGE_END }}" control_plane_endpoint_ip="$(devbox run -- fping -g -u "${CONTROL_PLANE_ENDPOINT_RANGE_START}" "${CONTROL_PLANE_ENDPOINT_RANGE_END}" 2>/dev/null | devbox run -- shuf --head-count=1)" echo "control_plane_endpoint_ip=${control_plane_endpoint_ip}" >> "${GITHUB_OUTPUT}" + + - name: Check Control Plane endpoint IP + run: | + if [[ -z "${{ steps.get-control-plane-endpoint-ip.outputs.control_plane_endpoint_ip }}" ]]; then + echo "control_plane_endpoint_ip is empty; cannot proceed with e2e tests" + exit 1 + fi + - name: Go cache uses: actions/cache@v4 with: From 1421ecd9b04c83d77be8fc1e4cbad79f97fbb8fe Mon Sep 17 00:00:00 2001 From: Sid Shukla Date: Mon, 1 Jul 2024 16:24:57 +0200 Subject: [PATCH 04/11] ci: Remove upload artifacts until we have more robust mechanism --- .github/workflows/e2e.yml | 17 +---- hack/tools/redact-artifacts/main.go | 100 ---------------------------- make/all.mk | 1 - make/redact.mk | 11 --- 4 files changed, 2 insertions(+), 127 deletions(-) delete mode 100644 hack/tools/redact-artifacts/main.go delete mode 100644 make/redact.mk diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 6ecce487f..ee4ae5bc7 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -62,10 +62,11 @@ jobs: run: | CONTROL_PLANE_ENDPOINT_RANGE_START="${{ vars.NUTANIX_CONTROL_PLANE_ENDPOINT_RANGE_START }}" CONTROL_PLANE_ENDPOINT_RANGE_END="${{ vars.NUTANIX_CONTROL_PLANE_ENDPOINT_RANGE_END }}" - control_plane_endpoint_ip="$(devbox run -- fping -g -u "${CONTROL_PLANE_ENDPOINT_RANGE_START}" "${CONTROL_PLANE_ENDPOINT_RANGE_END}" 2>/dev/null | devbox run -- shuf --head-count=1)" + control_plane_endpoint_ip="$(devbox run -- fping -g -u "${CONTROL_PLANE_ENDPOINT_RANGE_START}" "${CONTROL_PLANE_ENDPOINT_RANGE_END}" | devbox run -- shuf --head-count=1)" echo "control_plane_endpoint_ip=${control_plane_endpoint_ip}" >> "${GITHUB_OUTPUT}" - name: Check Control Plane endpoint IP + if: inputs.provider == 'Nutanix' run: | if [[ -z "${{ steps.get-control-plane-endpoint-ip.outputs.control_plane_endpoint_ip }}" ]]; then echo "control_plane_endpoint_ip is empty; cannot proceed with e2e tests" @@ -99,10 +100,6 @@ jobs: NUTANIX_STORAGE_CONTAINER_NAME: ${{ vars.NUTANIX_STORAGE_CONTAINER_NAME }} CONTROL_PLANE_ENDPOINT_IP: ${{ steps.get-control-plane-endpoint-ip.outputs.control_plane_endpoint_ip }} - - name: Redact secrets from artifacts - if: success() || failure() # always run even if the previous step fails - run: devbox run -- make redact-artifacts - - if: success() || failure() # always run even if the previous step fails name: Publish e2e test report uses: mikepenz/action-junit-report@v4 @@ -111,13 +108,3 @@ jobs: check_name: 'e2e test report' detailed_summary: true require_passed_tests: true - - - if: success() || failure() # always run even if the previous step fails - name: Upload e2e test artifacts - uses: actions/upload-artifact@v4 - with: - if-no-files-found: warn - overwrite: false - name: ${{ inputs.provider }} ${{ inputs.focus }} e2e artifacts - path: | - _artifacts diff --git a/hack/tools/redact-artifacts/main.go b/hack/tools/redact-artifacts/main.go deleted file mode 100644 index 83bd427d1..000000000 --- a/hack/tools/redact-artifacts/main.go +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2024 Nutanix. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "fmt" - "os" - "path/filepath" - "strings" - - "sigs.k8s.io/yaml" -) - -const ( - artifactDir = "_artifacts" - extensionYAML = ".yaml" - extensionYML = ".yml" - kindKey = "kind" - secretKind = "Secret" - dataKey = "data" - stringDataKey = "stringData" - redactedValue = "***REDACTED***" - documentSeparator = "---" -) - -func main() { - err := filepath.Walk(artifactDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - if !info.IsDir() && isYAMLFile(path) { - processYAMLFile(path) - } - return nil - }) - if err != nil { - fmt.Printf("Error walking the path: %v\n", err) - } -} - -func isYAMLFile(path string) bool { - ext := strings.ToLower(filepath.Ext(path)) - return ext == extensionYAML || ext == extensionYML -} - -func processYAMLFile(path string) { - data, err := os.ReadFile(path) - if err != nil { - fmt.Printf("Failed to read file %s: %v\n", path, err) - return - } - - output := make([]string, 0) - // Split the file into multiple documents - docs := strings.Split(string(data), documentSeparator) - for _, doc := range docs { - doc = strings.TrimSpace(doc) - if doc == "" { - continue - } - - var unstructured map[string]interface{} - if err := yaml.Unmarshal([]byte(doc), &unstructured); err != nil { - fmt.Printf("Failed to unmarshal document in file %s: %v\n", path, err) - continue - } - - if kind, ok := unstructured[kindKey].(string); ok && kind == secretKind { - redactSecret(unstructured) - } - - redacted, err := yaml.Marshal(unstructured) - if err != nil { - fmt.Printf("Failed to marshal document in file %s: %v\n", path, err) - continue - } - - output = append(output, string(redacted)) - } - - if err := os.WriteFile(path, []byte(strings.Join(output, "\n"+documentSeparator+"\n")), 0o600); err != nil { - fmt.Printf("Failed to write file %s: %v\n", path, err) - } -} - -func redactSecret(unstructured map[string]interface{}) { - if data, ok := unstructured[dataKey].(map[string]interface{}); ok { - for key := range data { - data[key] = redactedValue - } - } - - if stringData, ok := unstructured[stringDataKey].(map[string]interface{}); ok { - for key := range stringData { - stringData[key] = redactedValue - } - } -} diff --git a/make/all.mk b/make/all.mk index 0181b3269..d1fdc8509 100644 --- a/make/all.mk +++ b/make/all.mk @@ -19,4 +19,3 @@ include $(INCLUDE_DIR)helm.mk include $(INCLUDE_DIR)examples.mk include $(INCLUDE_DIR)apis.mk include $(INCLUDE_DIR)docs.mk -include $(INCLUDE_DIR)redact.mk diff --git a/make/redact.mk b/make/redact.mk deleted file mode 100644 index 82d9fc452..000000000 --- a/make/redact.mk +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright 2024 Nutanix. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -ARTIFACT_DIR := $(REPO_ROOT)/_artifacts - -# Define the placeholder value -REDACTED_PLACEHOLDER="***REDACTED***" - -.PHONY: redact-artifacts -redact-artifacts: ## Redacts all secrets in $(ARTIFACT_DIR) - go run hack/tools/redact-artifacts/main.go From d0355145a40f183204d8a177821a9c4fc7dbb3e5 Mon Sep 17 00:00:00 2001 From: Sid Shukla Date: Mon, 1 Jul 2024 16:48:32 +0200 Subject: [PATCH 05/11] ci: Skip nutanix self-hosted tests until we fix image upload CAREN image is not pushed to a registry during build process. This causes failures on the clusterctl move step where image is not available in the workload cluster. We will in future push an image out to a registry and unskip this. --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 4ed3bcce9..39072938f 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -78,9 +78,9 @@ jobs: matrix: provider: - Docker - - Nutanix # Uncomment below once we have the ability to run e2e tests on other providers from GHA. # - AWS + # - Nutanix fail-fast: false uses: ./.github/workflows/e2e.yml with: From 7b6dbdd962de8ab86ad9fda4014db82659d589d6 Mon Sep 17 00:00:00 2001 From: Sid Shukla Date: Mon, 1 Jul 2024 18:20:27 +0200 Subject: [PATCH 06/11] ci: Address review comments --- .github/workflows/e2e.yml | 2 +- api/v1alpha1/zz_generated.deepcopy.go | 4 +--- examples/capi-quick-start/nutanix-cluster-calico-crs.yaml | 2 +- examples/capi-quick-start/nutanix-cluster-cilium-crs.yaml | 2 +- .../clusters/nutanix/calico/crs/kustomization.yaml.tmpl | 3 --- .../nutanix/calico/helm-addon/kustomization.yaml.tmpl | 3 --- .../clusters/nutanix/cilium/crs/kustomization.yaml.tmpl | 3 --- .../nutanix/cilium/helm-addon/kustomization.yaml.tmpl | 3 --- test/e2e/config/caren.yaml | 2 +- 9 files changed, 5 insertions(+), 19 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index ee4ae5bc7..db4183992 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -84,7 +84,7 @@ jobs: ${{ runner.os }}-go- - name: Run e2e tests - run: devbox run -- make e2e-test E2E_LABEL='provider:${{ inputs.provider }} && cni:Cilium && addonStrategy:HelmAddon' E2E_SKIP='${{ inputs.skip }}' E2E_FOCUS='${{ inputs.focus }}' + run: devbox run -- make e2e-test E2E_LABEL='provider:${{ inputs.provider }}' E2E_SKIP='${{ inputs.skip }}' E2E_FOCUS='${{ inputs.focus }}' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 34592e92b..c546643ad 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1294,9 +1294,7 @@ func (in *NutanixNodeSpec) DeepCopy() *NutanixNodeSpec { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NutanixPrismCentralEndpointCredentials) DeepCopyInto( - out *NutanixPrismCentralEndpointCredentials, -) { +func (in *NutanixPrismCentralEndpointCredentials) DeepCopyInto(out *NutanixPrismCentralEndpointCredentials) { *out = *in out.SecretRef = in.SecretRef } diff --git a/examples/capi-quick-start/nutanix-cluster-calico-crs.yaml b/examples/capi-quick-start/nutanix-cluster-calico-crs.yaml index 79b7e74f6..1b31ea277 100644 --- a/examples/capi-quick-start/nutanix-cluster-calico-crs.yaml +++ b/examples/capi-quick-start/nutanix-cluster-calico-crs.yaml @@ -67,7 +67,7 @@ spec: credentials: secretRef: name: ${CLUSTER_NAME}-pc-creds - strategy: ClusterResourceSet + strategy: HelmAddon clusterAutoscaler: strategy: ClusterResourceSet cni: diff --git a/examples/capi-quick-start/nutanix-cluster-cilium-crs.yaml b/examples/capi-quick-start/nutanix-cluster-cilium-crs.yaml index 44cca319f..5073709e0 100644 --- a/examples/capi-quick-start/nutanix-cluster-cilium-crs.yaml +++ b/examples/capi-quick-start/nutanix-cluster-cilium-crs.yaml @@ -67,7 +67,7 @@ spec: credentials: secretRef: name: ${CLUSTER_NAME}-pc-creds - strategy: ClusterResourceSet + strategy: HelmAddon clusterAutoscaler: strategy: ClusterResourceSet cni: diff --git a/hack/examples/overlays/clusters/nutanix/calico/crs/kustomization.yaml.tmpl b/hack/examples/overlays/clusters/nutanix/calico/crs/kustomization.yaml.tmpl index 0d7c8921f..f7dd00af3 100644 --- a/hack/examples/overlays/clusters/nutanix/calico/crs/kustomization.yaml.tmpl +++ b/hack/examples/overlays/clusters/nutanix/calico/crs/kustomization.yaml.tmpl @@ -17,6 +17,3 @@ patches: - target: kind: Cluster path: ../../../../../patches/crs-strategy.yaml - - target: - kind: Cluster - path: ../../../../../patches/nutanix/ccm-crs-strategy.yaml diff --git a/hack/examples/overlays/clusters/nutanix/calico/helm-addon/kustomization.yaml.tmpl b/hack/examples/overlays/clusters/nutanix/calico/helm-addon/kustomization.yaml.tmpl index 03498c0de..d2ffaa757 100644 --- a/hack/examples/overlays/clusters/nutanix/calico/helm-addon/kustomization.yaml.tmpl +++ b/hack/examples/overlays/clusters/nutanix/calico/helm-addon/kustomization.yaml.tmpl @@ -17,6 +17,3 @@ patches: - target: kind: Cluster path: ../../../../../patches/helm-addon-strategy.yaml - - target: - kind: Cluster - path: ../../../../../patches/nutanix/ccm-helm-addon-strategy.yaml diff --git a/hack/examples/overlays/clusters/nutanix/cilium/crs/kustomization.yaml.tmpl b/hack/examples/overlays/clusters/nutanix/cilium/crs/kustomization.yaml.tmpl index 9b1b24b61..4d4053efb 100644 --- a/hack/examples/overlays/clusters/nutanix/cilium/crs/kustomization.yaml.tmpl +++ b/hack/examples/overlays/clusters/nutanix/cilium/crs/kustomization.yaml.tmpl @@ -17,6 +17,3 @@ patches: - target: kind: Cluster path: ../../../../../patches/crs-strategy.yaml - - target: - kind: Cluster - path: ../../../../../patches/nutanix/ccm-crs-strategy.yaml diff --git a/hack/examples/overlays/clusters/nutanix/cilium/helm-addon/kustomization.yaml.tmpl b/hack/examples/overlays/clusters/nutanix/cilium/helm-addon/kustomization.yaml.tmpl index 9c2d5a64b..82bb89906 100644 --- a/hack/examples/overlays/clusters/nutanix/cilium/helm-addon/kustomization.yaml.tmpl +++ b/hack/examples/overlays/clusters/nutanix/cilium/helm-addon/kustomization.yaml.tmpl @@ -17,6 +17,3 @@ patches: - target: kind: Cluster path: ../../../../../patches/helm-addon-strategy.yaml - - target: - kind: Cluster - path: ../../../../../patches/nutanix/ccm-helm-addon-strategy.yaml diff --git a/test/e2e/config/caren.yaml b/test/e2e/config/caren.yaml index 6d696bc28..a3d8cbc31 100644 --- a/test/e2e/config/caren.yaml +++ b/test/e2e/config/caren.yaml @@ -196,7 +196,7 @@ variables: # IP/FQDN of Prism Central. # NOTE: This has to be overridden by specifying the env var NUTANIX_ENDPOINT when running the e2e tests. It is # set as empty here to enable running the e2e tests for non-nutanix providers locally without setting the env var. - # NUTANIX_ENDPOINT: "" + NUTANIX_ENDPOINT: "" # # Port of Prism Central. Default: 9440 # NUTANIX_PORT: 9440 # # Disable Prism Central certificate checking. Default: false From dc5b5f39a7686d610e84606282466665179ba33f Mon Sep 17 00:00:00 2001 From: Sid Shukla Date: Mon, 1 Jul 2024 18:26:23 +0200 Subject: [PATCH 07/11] ci: Remove redundant strategy patches from nutanix --- hack/examples/patches/nutanix/ccm-crs-strategy.yaml | 6 ------ hack/examples/patches/nutanix/ccm-helm-addon-strategy.yaml | 6 ------ 2 files changed, 12 deletions(-) delete mode 100644 hack/examples/patches/nutanix/ccm-crs-strategy.yaml delete mode 100644 hack/examples/patches/nutanix/ccm-helm-addon-strategy.yaml diff --git a/hack/examples/patches/nutanix/ccm-crs-strategy.yaml b/hack/examples/patches/nutanix/ccm-crs-strategy.yaml deleted file mode 100644 index 30df2772e..000000000 --- a/hack/examples/patches/nutanix/ccm-crs-strategy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright 2024 Nutanix. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -- op: "add" - path: "/spec/topology/variables/0/value/addons/ccm/strategy" - value: ClusterResourceSet diff --git a/hack/examples/patches/nutanix/ccm-helm-addon-strategy.yaml b/hack/examples/patches/nutanix/ccm-helm-addon-strategy.yaml deleted file mode 100644 index 32fdd0560..000000000 --- a/hack/examples/patches/nutanix/ccm-helm-addon-strategy.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright 2024 Nutanix. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -- op: "add" - path: "/spec/topology/variables/0/value/addons/ccm/strategy" - value: HelmAddon From 79a4af3867a0ff79f100c4935497aa02cca5d00a Mon Sep 17 00:00:00 2001 From: Sid Shukla Date: Mon, 1 Jul 2024 19:27:18 +0200 Subject: [PATCH 08/11] ci: use matrix to spawn e2e tests in parallel --- .github/workflows/checks.yml | 16 ++++++++++++++++ .github/workflows/e2e.yml | 32 ++++++++++++++++++++------------ 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 39072938f..2af3aaeef 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -62,10 +62,18 @@ jobs: - Nutanix # Uncomment below once we have the ability to run e2e tests on other providers from GHA. # - AWS + cni: + - Calico + - Cilium + addon-strategy: + - ClusterResourceSet + - HelmAddon fail-fast: false uses: ./.github/workflows/e2e.yml with: provider: ${{ matrix.provider }} + cni: ${{ matrix.cni }} + addon-strategy: ${{ matrix.addon-strategy }} focus: Quick start runs-on: ${{ matrix.provider == 'Nutanix' && 'self-hosted-ncn-dind' || 'ubuntu-22.04' }} secrets: inherit @@ -81,10 +89,18 @@ jobs: # Uncomment below once we have the ability to run e2e tests on other providers from GHA. # - AWS # - Nutanix + cni: + - Calico + - Cilium + addon-strategy: + - ClusterResourceSet + - HelmAddon fail-fast: false uses: ./.github/workflows/e2e.yml with: provider: ${{ matrix.provider }} + cni: ${{ matrix.cni }} + addon-strategy: ${{ matrix.addon-strategy }} focus: Self-hosted runs-on: ${{ matrix.provider == 'Nutanix' && 'self-hosted-ncn-dind' || 'ubuntu-22.04' }} secrets: inherit diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index db4183992..cbde3dbca 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -8,6 +8,14 @@ on: description: Infrastructure provider to run e2e tests with type: string required: true + cni: + description: CNI to run e2e tests with + type: string + required: true + addon-strategy: + description: Addon strategy to run e2e tests with + type: string + required: true skip: description: e2e tests to skip type: string @@ -42,6 +50,16 @@ jobs: enable-cache: "true" skip-nix-installation: ${{ inputs.provider == 'Nutanix' }} + - name: Go cache + uses: actions/cache@v4 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + # The default disk size of these runners is ~14GB, this is not enough to run the e2e tests. # Cleanup the disk, see upstream discussion https://github.com/actions/runner-images/issues/2840. - name: Cleanup Disk Space @@ -56,7 +74,7 @@ jobs: echo "After removing files:" df -h - - name: Get Control Plane endpoint IP + - name: Get Control Plane endpoint IP id: get-control-plane-endpoint-ip if: inputs.provider == 'Nutanix' run: | @@ -73,18 +91,8 @@ jobs: exit 1 fi - - name: Go cache - uses: actions/cache@v4 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- - - name: Run e2e tests - run: devbox run -- make e2e-test E2E_LABEL='provider:${{ inputs.provider }}' E2E_SKIP='${{ inputs.skip }}' E2E_FOCUS='${{ inputs.focus }}' + run: devbox run -- make e2e-test E2E_LABEL='provider:${{ inputs.provider }} && cni:${{ inputs.cni }} && addonStrategy:${{ inputs.addon-strategy }}' E2E_SKIP='${{ inputs.skip }}' E2E_FOCUS='${{ inputs.focus }}' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} From 0f1842d85b2f865d1da0ff27f193c5a1c929bc72 Mon Sep 17 00:00:00 2001 From: Sid Shukla Date: Tue, 2 Jul 2024 12:23:10 +0200 Subject: [PATCH 09/11] ci: Change conditional for cachix action change conditional from provider:nutanix to runs-on:self-hosted-ncn-dind --- .github/workflows/e2e.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index cbde3dbca..90721b0e3 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -39,8 +39,10 @@ jobs: with: fetch-depth: 0 + # Install nix using cachix/install-nix-action if running on ARC runners + # See: https://github.com/DeterminateSystems/nix-installer-action/issues/68 - uses: cachix/install-nix-action@V27 - if: inputs.provider == 'Nutanix' + if: inputs.runs-on == 'self-hosted-ncn-dind' with: github_access_token: ${{ secrets.GITHUB_TOKEN }} @@ -48,7 +50,7 @@ jobs: uses: jetify-com/devbox-install-action@v0.11.0 with: enable-cache: "true" - skip-nix-installation: ${{ inputs.provider == 'Nutanix' }} + skip-nix-installation: ${{ inputs.runs-on == 'self-hosted-ncn-dind' }} - name: Go cache uses: actions/cache@v4 @@ -60,7 +62,7 @@ jobs: restore-keys: | ${{ runner.os }}-go- - # The default disk size of these runners is ~14GB, this is not enough to run the e2e tests. + # The default disk size of Github hosted runners is ~14GB, this is not enough to run the e2e tests. # Cleanup the disk, see upstream discussion https://github.com/actions/runner-images/issues/2840. - name: Cleanup Disk Space if: inputs.runs-on != 'self-hosted-ncn-dind' From 806ef05cf2f8b06d1ec1023bfbfd322b341f7b20 Mon Sep 17 00:00:00 2001 From: Sid Shukla Date: Tue, 2 Jul 2024 12:25:35 +0200 Subject: [PATCH 10/11] ci: Add name for cachix/install-nix-action --- .github/workflows/e2e.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 90721b0e3..1865d6044 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -41,7 +41,8 @@ jobs: # Install nix using cachix/install-nix-action if running on ARC runners # See: https://github.com/DeterminateSystems/nix-installer-action/issues/68 - - uses: cachix/install-nix-action@V27 + - name: Install Nix on self-hosted ARC runners + uses: cachix/install-nix-action@V27 if: inputs.runs-on == 'self-hosted-ncn-dind' with: github_access_token: ${{ secrets.GITHUB_TOKEN }} From ac257388ff0ee44a3aee0b76b51daf2aae7fe7bc Mon Sep 17 00:00:00 2001 From: Sid Shukla Date: Tue, 2 Jul 2024 15:40:57 +0200 Subject: [PATCH 11/11] ci: uncomment NUTANIX_USER and NUTANIX_PASSWORD in e2e config --- test/e2e/config/caren.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/config/caren.yaml b/test/e2e/config/caren.yaml index a3d8cbc31..fe0fbc251 100644 --- a/test/e2e/config/caren.yaml +++ b/test/e2e/config/caren.yaml @@ -202,9 +202,9 @@ variables: # # Disable Prism Central certificate checking. Default: false # NUTANIX_INSECURE: false # # Prism Central user - # NUTANIX_USER: "" + NUTANIX_USER: "" # # Prism Central password - # NUTANIX_PASSWORD: "" + NUTANIX_PASSWORD: "" # # Host IP to be assigned to the CAPX Kubernetes cluster. # CONTROL_PLANE_ENDPOINT_IP: "" # # Port of the CAPX Kubernetes cluster. Default: 6443