ci: Run Nutanix provider e2e tests on self-hosted runner

Author: jimmidyson

.github/actionlint.yaml

@@ -1,3 +1,6 @@
+# Copyright 2024 Nutanix. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
 self-hosted-runner:
   # Labels of self-hosted runner in array of string
   labels:

.github/workflows/checks.yml

@@ -59,32 +59,15 @@ jobs:
       matrix:
         provider:
           - Docker
+          - Nutanix
           # Uncomment below once we have the ability to run e2e tests on other providers from GHA.
           # - AWS
       fail-fast: false
     uses: ./.github/workflows/e2e.yml
     with:
       provider: ${{ matrix.provider }}
       focus: Quick start
-    secrets: inherit
-    permissions:
-      contents: read
-      checks: write
-
-  # The actions runner controller does not support using multiple labels to target the runners so we have to create separate jobs for self-hosted runners.
-  # https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/using-actions-runner-controller-runners-in-a-workflow#using-runner-scale-set-names
-  e2e-quick-start-custom-runner:
-    strategy:
-      matrix:
-        provider:
-          - Nutanix
-          # Uncomment below once we have the ability to run e2e tests on other providers via self-hosted runners.
-          # - Vsphere (example)
-      fail-fast: false
-    uses: ./.github/workflows/e2e-custom-runner.yml
-    with:
-      provider: ${{ matrix.provider }}
-      focus: Quick start
+      runs-on: ${{ matrix.provider == 'Nutanix' && 'self-hosted-ncn-dind' || 'ubuntu-22.04' }}
     secrets: inherit
     permissions:
       contents: read
@@ -95,32 +78,15 @@ jobs:
       matrix:
         provider:
           - Docker
+          - Nutanix
           # Uncomment below once we have the ability to run e2e tests on other providers from GHA.
           # - AWS
       fail-fast: false
     uses: ./.github/workflows/e2e.yml
     with:
       provider: ${{ matrix.provider }}
       focus: Self-hosted
-    secrets: inherit
-    permissions:
-      contents: read
-      checks: write
-
-  # The actions runner controller does not support using multiple labels to target the runners so we have to create separate jobs for self-hosted runners.
-  # https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/using-actions-runner-controller-runners-in-a-workflow#using-runner-scale-set-names
-  e2e-self-hosted-custom-runner:
-    strategy:
-      matrix:
-        provider:
-          - Nutanix
-          # Uncomment below once we have the ability to run e2e tests on other providers via self-hosted runners.
-          # - Vsphere (example)
-      fail-fast: false
-    uses: ./.github/workflows/e2e-custom-runner.yml
-    with:
-      provider: ${{ matrix.provider }}
-      focus: Self-hosted
+      runs-on: ${{ matrix.provider == 'Nutanix' && 'self-hosted-ncn-dind' || 'ubuntu-22.04' }}
     secrets: inherit
     permissions:
       contents: read

.github/workflows/e2e-custom-runner.yml

@@ -14,10 +14,14 @@ on:
       focus:
         description: e2e tests to focus
         type: string
+      runs-on:
+        description: The runner to run the e2e tests on
+        type: string
+        required: true
 
 jobs:
   e2e-test:
-    runs-on: ubuntu-22.04
+    runs-on: ${{ inputs.runs-on }}
     permissions:
       contents: read
       checks: write
@@ -27,20 +31,16 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Install devbox
-        uses: jetify-com/[email protected]
+      - uses: cachix/install-nix-action@V27
+        if: inputs.provider == 'Nutanix'
         with:
-          enable-cache: true
+          github_access_token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Go cache
-        uses: actions/cache@v4
+      - name: Install devbox
+        uses: jetify-com/[email protected]
         with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
+          enable-cache: "true"
+          skip-nix-installation: ${{ inputs.provider == 'Nutanix' }}
 
       # The default disk size of these runners is ~14GB, this is not enough to run the e2e tests.
       # Cleanup the disk, see upstream discussion https://github.com/actions/runner-images/issues/2840.
@@ -51,17 +51,42 @@ jobs:
           sudo rm -rf /usr/share/dotnet
           sudo rm -rf /opt/ghc
           sudo rm -rf "/usr/local/share/boost"
-          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          sudo rm -rf "${AGENT_TOOLSDIRECTORY}"
           echo "After removing files:"
           df -h
 
+      - name:  Get Control Plane endpoint IP
+        id: get-control-plane-endpoint-ip
+        if: inputs.provider == 'Nutanix'
+        run: |
+          control_plane_endpoint_ip="$(devbox run -- fping -g -u "${CONTROL_PLANE_ENDPOINT_RANGE_START}" "${CONTROL_PLANE_ENDPOINT_RANGE_END}" 2>/dev/null | devbox run -- shuf --head-count=1)"
+          echo "control_plane_endpoint_ip=${control_plane_endpoint_ip}" >> "${GITHUB_OUTPUT}"
+      - name: Go cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+
       - name: Run e2e tests
         run: devbox run -- make e2e-test E2E_LABEL='provider:${{ inputs.provider }}' E2E_SKIP='${{ inputs.skip }}' E2E_FOCUS='${{ inputs.focus }}'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
+          DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
           NUTANIX_ENDPOINT: ${{ secrets.NUTANIX_ENDPOINT }}
-          NUTANIX_PASSWORD: ${{ secrets.NUTANIX_PASSWORD }}
           NUTANIX_USER: ${{ secrets.NUTANIX_USER }}
+          NUTANIX_PASSWORD: ${{ secrets.NUTANIX_PASSWORD }}
+          NUTANIX_PORT: ${{ vars.NUTANIX_PORT }}
+          NUTANIX_INSECURE: ${{ vars.NUTANIX_INSECURE }}
+          NUTANIX_PRISM_ELEMENT_CLUSTER_NAME: ${{ vars.NUTANIX_PRISM_ELEMENT_CLUSTER_NAME }}
+          NUTANIX_SUBNET_NAME: ${{ vars.NUTANIX_SUBNET_NAME }}
+          NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME: ${{ vars.NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME }}
+          NUTANIX_STORAGE_CONTAINER_NAME: ${{ vars.NUTANIX_STORAGE_CONTAINER_NAME }}
+          CONTROL_PLANE_ENDPOINT_IP: ${{ steps.get-control-plane-endpoint-ip.outputs.control_plane_endpoint_ip }}
 
       - if: success() || failure() # always run even if the previous step fails
         name: Publish e2e test report

.github/workflows/e2e.yml

@@ -22,7 +22,8 @@ import (
 )
 
 var _ = Describe("Self-hosted", Serial, func() {
-	provider := "Docker"
+	for _, provider := range []string{"Docker", "Nutanix"} {
+		lowercaseProvider := strings.ToLower(provider)
 	lowercaseProvider := strings.ToLower(provider)
 	for _, cniProvider := range []string{"Cilium", "Calico"} {
 		for _, addonStrategy := range []string{"HelmAddon", "ClusterResourceSet"} {