ci: Run Nutanix provider e2e tests on self-hosted runner (#755)

Modify e2e workflow to Run Nutanix jobs on self-hosted-github runner

This is necessary as we want some e2e jobs to run on hosted runners
and some to run on self-hosted runners. The actions runner controller
does not support using multiple labels to target the runners
See:
https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/using-actions-runner-controller-runners-in-a-workflow#using-runner-scale-set-names

Depends on #754.

---------

Co-authored-by: Sid Shukla <[email protected]>

Author: jimmidyson

.github/actionlint.yaml

@@ -0,0 +1,7 @@
+# Copyright 2024 Nutanix. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+self-hosted-runner:
+  # Labels of self-hosted runner in array of string
+  labels:
+    - self-hosted-ncn-dind

.github/workflows/checks.yml

@@ -59,14 +59,23 @@ jobs:
       matrix:
         provider:
           - Docker
+          - Nutanix
           # Uncomment below once we have the ability to run e2e tests on other providers from GHA.
           # - AWS
-          # - Nutanix
+        cni:
+          - Calico
+          - Cilium
+        addon-strategy:
+          - ClusterResourceSet
+          - HelmAddon
       fail-fast: false
     uses: ./.github/workflows/e2e.yml
     with:
       provider: ${{ matrix.provider }}
+      cni: ${{ matrix.cni }}
+      addon-strategy: ${{ matrix.addon-strategy }}
       focus: Quick start
+      runs-on: ${{ matrix.provider == 'Nutanix' && 'self-hosted-ncn-dind' || 'ubuntu-22.04' }}
     secrets: inherit
     permissions:
       contents: read
@@ -80,11 +89,20 @@ jobs:
           # Uncomment below once we have the ability to run e2e tests on other providers from GHA.
           # - AWS
           # - Nutanix
+        cni:
+          - Calico
+          - Cilium
+        addon-strategy:
+          - ClusterResourceSet
+          - HelmAddon
       fail-fast: false
     uses: ./.github/workflows/e2e.yml
     with:
       provider: ${{ matrix.provider }}
+      cni: ${{ matrix.cni }}
+      addon-strategy: ${{ matrix.addon-strategy }}
       focus: Self-hosted
+      runs-on: ${{ matrix.provider == 'Nutanix' && 'self-hosted-ncn-dind' || 'ubuntu-22.04' }}
     secrets: inherit
     permissions:
       contents: read

.github/workflows/e2e.yml

@@ -8,16 +8,28 @@ on:
         description: Infrastructure provider to run e2e tests with
         type: string
         required: true
+      cni:
+        description: CNI to run e2e tests with
+        type: string
+        required: true
+      addon-strategy:
+        description: Addon strategy to run e2e tests with
+        type: string
+        required: true
       skip:
         description: e2e tests to skip
         type: string
       focus:
         description: e2e tests to focus
         type: string
+      runs-on:
+        description: The runner to run the e2e tests on
+        type: string
+        required: true
 
 jobs:
   e2e-test:
-    runs-on: ubuntu-22.04
+    runs-on: ${{ inputs.runs-on }}
     permissions:
       contents: read
       checks: write
@@ -27,10 +39,19 @@ jobs:
         with:
           fetch-depth: 0
 
+      # Install nix using cachix/install-nix-action if running on ARC runners
+      # See: https://github.com/DeterminateSystems/nix-installer-action/issues/68
+      - name: Install Nix on self-hosted ARC runners
+        uses: cachix/install-nix-action@V27
+        if: inputs.runs-on == 'self-hosted-ncn-dind'
+        with:
+          github_access_token: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Install devbox
         uses: jetify-com/[email protected]
         with:
-          enable-cache: true
+          enable-cache: "true"
+          skip-nix-installation: ${{ inputs.runs-on == 'self-hosted-ncn-dind' }}
 
       - name: Go cache
         uses: actions/cache@v4
@@ -42,26 +63,53 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-go-
 
-      # The default disk size of these runners is ~14GB, this is not enough to run the e2e tests.
+      # The default disk size of Github hosted runners is ~14GB, this is not enough to run the e2e tests.
       # Cleanup the disk, see upstream discussion https://github.com/actions/runner-images/issues/2840.
       - name: Cleanup Disk Space
+        if: inputs.runs-on != 'self-hosted-ncn-dind'
         run: |
           echo "Before removing files:"
           df -h
           sudo rm -rf /usr/share/dotnet
           sudo rm -rf /opt/ghc
           sudo rm -rf "/usr/local/share/boost"
-          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          sudo rm -rf "${AGENT_TOOLSDIRECTORY}"
           echo "After removing files:"
           df -h
 
+      - name: Get Control Plane endpoint IP
+        id: get-control-plane-endpoint-ip
+        if: inputs.provider == 'Nutanix'
+        run: |
+          CONTROL_PLANE_ENDPOINT_RANGE_START="${{ vars.NUTANIX_CONTROL_PLANE_ENDPOINT_RANGE_START }}"
+          CONTROL_PLANE_ENDPOINT_RANGE_END="${{ vars.NUTANIX_CONTROL_PLANE_ENDPOINT_RANGE_END }}"
+          control_plane_endpoint_ip="$(devbox run -- fping -g -u "${CONTROL_PLANE_ENDPOINT_RANGE_START}" "${CONTROL_PLANE_ENDPOINT_RANGE_END}" | devbox run -- shuf --head-count=1)"
+          echo "control_plane_endpoint_ip=${control_plane_endpoint_ip}" >> "${GITHUB_OUTPUT}"
+
+      - name: Check Control Plane endpoint IP
+        if: inputs.provider == 'Nutanix'
+        run: |
+          if [[ -z "${{ steps.get-control-plane-endpoint-ip.outputs.control_plane_endpoint_ip }}" ]]; then
+            echo "control_plane_endpoint_ip is empty; cannot proceed with e2e tests"
+            exit 1
+          fi
+
       - name: Run e2e tests
-        run: devbox run -- make e2e-test E2E_LABEL='provider:${{ inputs.provider }}' E2E_SKIP='${{ inputs.skip }}' E2E_FOCUS='${{ inputs.focus }}'
+        run: devbox run -- make e2e-test E2E_LABEL='provider:${{ inputs.provider }} && cni:${{ inputs.cni }} && addonStrategy:${{ inputs.addon-strategy }}' E2E_SKIP='${{ inputs.skip }}' E2E_FOCUS='${{ inputs.focus }}'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
+          DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
           NUTANIX_ENDPOINT: ${{ secrets.NUTANIX_ENDPOINT }}
-          NUTANIX_PASSWORD: ${{ secrets.NUTANIX_PASSWORD }}
           NUTANIX_USER: ${{ secrets.NUTANIX_USER }}
+          NUTANIX_PASSWORD: ${{ secrets.NUTANIX_PASSWORD }}
+          NUTANIX_PORT: ${{ vars.NUTANIX_PORT }}
+          NUTANIX_INSECURE: false
+          NUTANIX_PRISM_ELEMENT_CLUSTER_NAME: ${{ vars.NUTANIX_PRISM_ELEMENT_CLUSTER_NAME }}
+          NUTANIX_SUBNET_NAME: ${{ vars.NUTANIX_SUBNET_NAME }}
+          NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME: ${{ vars.NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME }}
+          NUTANIX_STORAGE_CONTAINER_NAME: ${{ vars.NUTANIX_STORAGE_CONTAINER_NAME }}
+          CONTROL_PLANE_ENDPOINT_IP: ${{ steps.get-control-plane-endpoint-ip.outputs.control_plane_endpoint_ip }}
 
       - if: success() || failure() # always run even if the previous step fails
         name: Publish e2e test report

devbox.json

@@ -6,6 +6,7 @@
     "crane@latest",
     "envsubst@latest",
     "findutils@latest",
+    "fping@latest",
     "gh@latest",
     "ginkgo@latest",
     "git@latest",

devbox.lock

@@ -381,6 +381,54 @@
         }
       }
     },
+    "fping@latest": {
+      "last_modified": "2024-06-12T20:55:33Z",
+      "resolved": "github:NixOS/nixpkgs/a9858885e197f984d92d7fe64e9fff6b2e488d40#fping",
+      "source": "devbox-search",
+      "version": "5.2",
+      "systems": {
+        "aarch64-darwin": {
+          "outputs": [
+            {
+              "name": "out",
+              "path": "/nix/store/clxfp6jl0d2fs1bp2d1278534n2gixbj-fping-5.2",
+              "default": true
+            }
+          ],
+          "store_path": "/nix/store/clxfp6jl0d2fs1bp2d1278534n2gixbj-fping-5.2"
+        },
+        "aarch64-linux": {
+          "outputs": [
+            {
+              "name": "out",
+              "path": "/nix/store/ilzq042wih0h5vdzxcpf6sd826h37g6w-fping-5.2",
+              "default": true
+            }
+          ],
+          "store_path": "/nix/store/ilzq042wih0h5vdzxcpf6sd826h37g6w-fping-5.2"
+        },
+        "x86_64-darwin": {
+          "outputs": [
+            {
+              "name": "out",
+              "path": "/nix/store/hrh3202f2njx3skj3xn33fish5az5691-fping-5.2",
+              "default": true
+            }
+          ],
+          "store_path": "/nix/store/hrh3202f2njx3skj3xn33fish5az5691-fping-5.2"
+        },
+        "x86_64-linux": {
+          "outputs": [
+            {
+              "name": "out",
+              "path": "/nix/store/2nr99jpa9g7b5z8pwj85awzh4qbhas28-fping-5.2",
+              "default": true
+            }
+          ],
+          "store_path": "/nix/store/2nr99jpa9g7b5z8pwj85awzh4qbhas28-fping-5.2"
+        }
+      }
+    },
     "gh@latest": {
       "last_modified": "2024-05-30T12:09:21Z",
       "resolved": "github:NixOS/nixpkgs/aa61b27554a5fc282758bf0324781e3464ef2cde#gh",

hack/kind/dns-override.yaml

@@ -0,0 +1,28 @@
+# Copyright 2024 Nutanix. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: v1
+data:
+  Corefile: |
+    .:53 {
+        errors
+        health {
+           lameduck 5s
+        }
+        ready
+        kubernetes cluster.local in-addr.arpa ip6.arpa {
+           pods insecure
+           fallthrough in-addr.arpa ip6.arpa
+           ttl 30
+        }
+        prometheus :9153
+        forward . 8.8.8.8
+        cache 30
+        loop
+        reload
+        loadbalance
+    }
+kind: ConfigMap
+metadata:
+  name: coredns
+  namespace: kube-system

test/e2e/config/caren.yaml

@@ -8,6 +8,14 @@ images:
     loadBehavior: mustLoad
   - name: ghcr.io/nutanix-cloud-native/caren-helm-reg:${E2E_IMAGE_TAG}-${GOARCH}
     loadBehavior: mustLoad
+  - name: docker.io/mesosphere/cluster-api-controller:${CAPI_VERSION}-d2iq.0
+    loadBehavior: mustLoad
+  - name: docker.io/mesosphere/kubeadm-bootstrap-controller:${CAPI_VERSION}-d2iq.0
+    loadBehavior: mustLoad
+  - name: docker.io/mesosphere/kubeadm-control-plane-controller:${CAPI_VERSION}-d2iq.0
+    loadBehavior: mustLoad
+  - name: docker.io/mesosphere/capd-manager:${CAPD_VERSION}-d2iq.0
+    loadBehavior: mustLoad
 
 providers:
 - name: cluster-api

test/e2e/e2e_suite_test.go

@@ -11,16 +11,22 @@ import (
 	"encoding/base64"
 	"encoding/gob"
 	"flag"
+	"fmt"
 	"os"
 	"path/filepath"
 	"slices"
 	"strings"
 	"testing"
+	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
 	storagev1 "k8s.io/api/storage/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/yaml"
 	"k8s.io/klog/v2"
 	capie2e "sigs.k8s.io/cluster-api/test/e2e"
 	capie2eframework "sigs.k8s.io/cluster-api/test/framework"
@@ -110,6 +116,39 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 		useExistingCluster,
 	)
 
+	By("Overriding coreDNS resolver")
+	// override coredns resolver to 8.8.8.8 and restart coredns deployment
+	// read the dns-override.yaml file
+	filePath, _ := filepath.Abs("../../hack/kind/dns-override.yaml")
+	yamlFile, err := os.ReadFile(filePath)
+	Expect(err).To(BeNil(), "Failed to read the dns-override.yaml file")
+
+	// decode the yaml file into a Kubernetes object
+	decode := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(yamlFile), 4096)
+	configMap := &corev1.ConfigMap{}
+	err = decode.Decode(&configMap)
+	Expect(err).To(BeNil(), "Failed to decode the yaml file into a Kubernetes object")
+
+	_, err = bootstrapClusterProxy.GetClientSet().
+		CoreV1().
+		ConfigMaps(configMap.GetNamespace()).
+		Update(context.Background(), configMap, metav1.UpdateOptions{})
+	Expect(
+		err,
+	).To(BeNil(), "Failed to update the coredns deployment with the dns-override.yaml file")
+
+	timeNow := time.Now().Format(time.RFC3339)
+	patch := fmt.Sprintf(
+		`{"spec":{"template":{"metadata":{"annotations":{"kubectl.kubernetes.io/restartedAt":%q}}}}}`,
+		timeNow,
+	)
+
+	_, err = bootstrapClusterProxy.GetClientSet().
+		AppsV1().
+		Deployments("kube-system").
+		Patch(context.Background(), "coredns", types.StrategicMergePatchType, []byte(patch), metav1.PatchOptions{})
+	Expect(err).To(BeNil(), "Failed to restart the coredns deployment")
+
 	By("Initializing the bootstrap cluster")
 	initBootstrapCluster(bootstrapClusterProxy, e2eConfig, clusterctlConfigPath, artifactFolder)