ci: Run Nutanix provider e2e tests on self-hosted runner

Author: jimmidyson

.github/workflows/checks.yml

@@ -59,32 +59,15 @@ jobs:
       matrix:
         provider:
           - Docker
+          - Nutanix
           # Uncomment below once we have the ability to run e2e tests on other providers from GHA.
           # - AWS
       fail-fast: false
     uses: ./.github/workflows/e2e.yml
     with:
       provider: ${{ matrix.provider }}
       focus: Quick start
-    secrets: inherit
-    permissions:
-      contents: read
-      checks: write
-
-  # The actions runner controller does not support using multiple labels to target the runners so we have to create separate jobs for self-hosted runners.
-  # https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/using-actions-runner-controller-runners-in-a-workflow#using-runner-scale-set-names
-  e2e-quick-start-custom-runner:
-    strategy:
-      matrix:
-        provider:
-          - Nutanix
-          # Uncomment below once we have the ability to run e2e tests on other providers via self-hosted runners.
-          # - Vsphere (example)
-      fail-fast: false
-    uses: ./.github/workflows/e2e-custom-runner.yml
-    with:
-      provider: ${{ matrix.provider }}
-      focus: Quick start
+      runs-on: ${{ matrix.provider == 'Nutanix' && 'self-hosted-ncn-dind' || 'ubuntu-22.04' }}
     secrets: inherit
     permissions:
       contents: read
@@ -95,32 +78,15 @@ jobs:
       matrix:
         provider:
           - Docker
+          - Nutanix
           # Uncomment below once we have the ability to run e2e tests on other providers from GHA.
           # - AWS
       fail-fast: false
     uses: ./.github/workflows/e2e.yml
     with:
       provider: ${{ matrix.provider }}
       focus: Self-hosted
-    secrets: inherit
-    permissions:
-      contents: read
-      checks: write
-
-  # The actions runner controller does not support using multiple labels to target the runners so we have to create separate jobs for self-hosted runners.
-  # https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/using-actions-runner-controller-runners-in-a-workflow#using-runner-scale-set-names
-  e2e-self-hosted-custom-runner:
-    strategy:
-      matrix:
-        provider:
-          - Nutanix
-          # Uncomment below once we have the ability to run e2e tests on other providers via self-hosted runners.
-          # - Vsphere (example)
-      fail-fast: false
-    uses: ./.github/workflows/e2e-custom-runner.yml
-    with:
-      provider: ${{ matrix.provider }}
-      focus: Self-hosted
+      runs-on: ${{ matrix.provider == 'Nutanix' && 'self-hosted-ncn-dind' || 'ubuntu-22.04' }}
     secrets: inherit
     permissions:
       contents: read

.github/workflows/e2e-custom-runner.yml

@@ -14,10 +14,14 @@ on:
       focus:
         description: e2e tests to focus
         type: string
+      runs-on:
+        description: The runner to run the e2e tests on
+        type: string
+        required: true
 
 jobs:
   e2e-test:
-    runs-on: ubuntu-22.04
+    runs-on: ${{ inputs.runs-on }}
     permissions:
       contents: read
       checks: write
@@ -30,17 +34,7 @@ jobs:
       - name: Install devbox
         uses: jetify-com/[email protected]
         with:
-          enable-cache: true
-
-      - name: Go cache
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
+          enable-cache: "true"
 
       # The default disk size of these runners is ~14GB, this is not enough to run the e2e tests.
       # Cleanup the disk, see upstream discussion https://github.com/actions/runner-images/issues/2840.
@@ -51,17 +45,60 @@ jobs:
           sudo rm -rf /usr/share/dotnet
           sudo rm -rf /opt/ghc
           sudo rm -rf "/usr/local/share/boost"
-          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          sudo rm -rf "${AGENT_TOOLSDIRECTORY}"
           echo "After removing files:"
           df -h
 
+      - name:  Get Control Plane endpoint IP
+        id: get-control-plane-endpoint-ip
+        if: inputs.provider == 'Nutanix'
+        run: |
+          # Get all available IPs in the range
+          mapfile -t available_ips <<< "$(fping -g -u '${{ secrets.NUTANIX_CONTROL_PLANE_ENDPOINT_RANGE_START }}' '${{ secrets.NUTANIX_CONTROL_PLANE_ENDPOINT_RANGE_END }}')"
+
+          # Get the number of available IPs
+          num_ips="${#available_ips[@]}"
+
+          if [ "${num_ips}" -eq 0 ]; then
+            echo "No available IPs found"
+            exit 1
+          fi
+
+          # Get a random index
+          random_index="$((RANDOM % num_ips +1))"
+
+          # Get the random IP
+          control_plane_endpoint_ip="${available_ips[${random_index}]}"
+
+          # Set the random IP as an output variable
+          echo "control_plane_endpoint_ip=${control_plane_endpoint_ip}" >> "${GITHUB_OUTPUT}"
+
+      - name: Go cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+
       - name: Run e2e tests
         run: devbox run -- make e2e-test E2E_LABEL='provider:${{ inputs.provider }}' E2E_SKIP='${{ inputs.skip }}' E2E_FOCUS='${{ inputs.focus }}'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
+          DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
           NUTANIX_ENDPOINT: ${{ secrets.NUTANIX_ENDPOINT }}
-          NUTANIX_PASSWORD: ${{ secrets.NUTANIX_PASSWORD }}
           NUTANIX_USER: ${{ secrets.NUTANIX_USER }}
+          NUTANIX_PASSWORD: ${{ secrets.NUTANIX_PASSWORD }}
+          NUTANIX_PORT: ${{ vars.NUTANIX_PORT }}
+          NUTANIX_INSECURE: ${{ vars.NUTANIX_INSECURE }}
+          NUTANIX_PRISM_ELEMENT_CLUSTER_NAME: ${{ vars.NUTANIX_PRISM_ELEMENT_CLUSTER_NAME }}
+          NUTANIX_SUBNET_NAME: ${{ vars.NUTANIX_SUBNET_NAME }}
+          NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME: ${{ vars.NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME }}
+          NUTANIX_STORAGE_CONTAINER_NAME: ${{ vars.NUTANIX_STORAGE_CONTAINER_NAME }}
+          CONTROL_PLANE_ENDPOINT_IP: ${{ steps.get-control-plane-endpoint-ip.outputs.control_plane_endpoint_ip }}
 
       - if: success() || failure() # always run even if the previous step fails
         name: Publish e2e test report