Update e2e workflow to conditionally run on self-hosted-github runner

thunderboltsid · thunderboltsid · commit e6d23e34c446 · 2024-06-27T17:07:23.000+02:00
This is necessary as we want some e2e jobs to run on hosted runners and some to run on self-hosted runners. The actions runner controller does not support using multiple labels to target the runners See: https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/using-actions-runner-controller-runners-in-a-workflow#using-runner-scale-set-names ci: Run Nutanix provider e2e tests on self-hosted runner
diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml
@@ -0,0 +1,7 @@
+# Copyright 2024 Nutanix. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+self-hosted-runner:
+  # Labels of self-hosted runner in array of string
+  labels:
+    - self-hosted-ncn-dind
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
@@ -59,14 +59,15 @@ jobs:
       matrix:
         provider:
           - Docker
+          - Nutanix
           # Uncomment below once we have the ability to run e2e tests on other providers from GHA.
           # - AWS
-          # - Nutanix
       fail-fast: false
     uses: ./.github/workflows/e2e.yml
     with:
       provider: ${{ matrix.provider }}
       focus: Quick start
+      runs-on: ${{ matrix.provider == 'Nutanix' && 'self-hosted-ncn-dind' || 'ubuntu-22.04' }}
     secrets: inherit
     permissions:
       contents: read
@@ -77,14 +78,15 @@ jobs:
       matrix:
         provider:
           - Docker
+          - Nutanix
           # Uncomment below once we have the ability to run e2e tests on other providers from GHA.
           # - AWS
-          # - Nutanix
       fail-fast: false
     uses: ./.github/workflows/e2e.yml
     with:
       provider: ${{ matrix.provider }}
       focus: Self-hosted
+      runs-on: ${{ matrix.provider == 'Nutanix' && 'self-hosted-ncn-dind' || 'ubuntu-22.04' }}
     secrets: inherit
     permissions:
       contents: read
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -14,10 +14,14 @@ on:
       focus:
         description: e2e tests to focus
         type: string
+      runs-on:
+        description: The runner to run the e2e tests on
+        type: string
+        required: true
 
 jobs:
   e2e-test:
-    runs-on: ubuntu-22.04
+    runs-on: ${{ inputs.runs-on }}
     permissions:
       contents: read
       checks: write
@@ -27,41 +31,65 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Install devbox
-        uses: jetify-com/devbox-install-action@v0.11.0
+      - uses: cachix/install-nix-action@V27
+        if: inputs.provider == 'Nutanix'
         with:
-          enable-cache: true
+          github_access_token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Go cache
-        uses: actions/cache@v4
+      - name: Install devbox
+        uses: jetify-com/devbox-install-action@v0.11.0
         with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
+          enable-cache: "true"
+          skip-nix-installation: ${{ inputs.provider == 'Nutanix' }}
 
       # The default disk size of these runners is ~14GB, this is not enough to run the e2e tests.
       # Cleanup the disk, see upstream discussion https://github.com/actions/runner-images/issues/2840.
       - name: Cleanup Disk Space
+        if: inputs.runs-on != 'self-hosted-ncn-dind'
         run: |
           echo "Before removing files:"
           df -h
           sudo rm -rf /usr/share/dotnet
           sudo rm -rf /opt/ghc
           sudo rm -rf "/usr/local/share/boost"
-          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          sudo rm -rf "${AGENT_TOOLSDIRECTORY}"
           echo "After removing files:"
           df -h
 
+      - name:  Get Control Plane endpoint IP
+        id: get-control-plane-endpoint-ip
+        if: inputs.provider == 'Nutanix'
+        run: |
+          CONTROL_PLANE_ENDPOINT_RANGE_START="${{ secrets.NUTANIX_CONTROL_PLANE_ENDPOINT_RANGE_START }}"
+          CONTROL_PLANE_ENDPOINT_RANGE_END="${{ secrets.NUTANIX_CONTROL_PLANE_ENDPOINT_RANGE_END }}"
+          control_plane_endpoint_ip="$(devbox run -- fping -g -u "${CONTROL_PLANE_ENDPOINT_RANGE_START}" "${CONTROL_PLANE_ENDPOINT_RANGE_END}" 2>/dev/null | devbox run -- shuf --head-count=1)"
+          echo "control_plane_endpoint_ip=${control_plane_endpoint_ip}" >> "${GITHUB_OUTPUT}"
+      - name: Go cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+
       - name: Run e2e tests
         run: devbox run -- make e2e-test E2E_LABEL='provider:${{ inputs.provider }}' E2E_SKIP='${{ inputs.skip }}' E2E_FOCUS='${{ inputs.focus }}'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
+          DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
           NUTANIX_ENDPOINT: ${{ secrets.NUTANIX_ENDPOINT }}
-          NUTANIX_PASSWORD: ${{ secrets.NUTANIX_PASSWORD }}
           NUTANIX_USER: ${{ secrets.NUTANIX_USER }}
+          NUTANIX_PASSWORD: ${{ secrets.NUTANIX_PASSWORD }}
+          NUTANIX_PORT: ${{ vars.NUTANIX_PORT }}
+          NUTANIX_INSECURE: ${{ vars.NUTANIX_INSECURE }}
+          NUTANIX_PRISM_ELEMENT_CLUSTER_NAME: ${{ vars.NUTANIX_PRISM_ELEMENT_CLUSTER_NAME }}
+          NUTANIX_SUBNET_NAME: ${{ vars.NUTANIX_SUBNET_NAME }}
+          NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME: ${{ vars.NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME }}
+          NUTANIX_STORAGE_CONTAINER_NAME: ${{ vars.NUTANIX_STORAGE_CONTAINER_NAME }}
+          CONTROL_PLANE_ENDPOINT_IP: ${{ steps.get-control-plane-endpoint-ip.outputs.control_plane_endpoint_ip }}
 
       - if: success() || failure() # always run even if the previous step fails
         name: Publish e2e test report
@@ -71,3 +99,13 @@ jobs:
           check_name: 'e2e test report'
           detailed_summary: true
           require_passed_tests: true
+
+      - if: success() || failure() # always run even if the previous step fails
+        name: Upload e2e test artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          if-no-files-found: warn
+          overwrite: false
+          name: ${{ inputs.provider }} ${{ inputs.focus }} e2e artifacts
+          path: |
+            _artifacts
diff --git a/devbox.json b/devbox.json
@@ -6,6 +6,7 @@
     "crane@latest",
     "envsubst@latest",
     "findutils@latest",
+    "fping@latest",
     "gh@latest",
     "ginkgo@latest",
     "git@latest",
diff --git a/devbox.lock b/devbox.lock
@@ -381,6 +381,54 @@
         }
       }
     },
+    "fping@latest": {
+      "last_modified": "2024-06-12T20:55:33Z",
+      "resolved": "github:NixOS/nixpkgs/a9858885e197f984d92d7fe64e9fff6b2e488d40#fping",
+      "source": "devbox-search",
+      "version": "5.2",
+      "systems": {
+        "aarch64-darwin": {
+          "outputs": [
+            {
+              "name": "out",
+              "path": "/nix/store/clxfp6jl0d2fs1bp2d1278534n2gixbj-fping-5.2",
+              "default": true
+            }
+          ],
+          "store_path": "/nix/store/clxfp6jl0d2fs1bp2d1278534n2gixbj-fping-5.2"
+        },
+        "aarch64-linux": {
+          "outputs": [
+            {
+              "name": "out",
+              "path": "/nix/store/ilzq042wih0h5vdzxcpf6sd826h37g6w-fping-5.2",
+              "default": true
+            }
+          ],
+          "store_path": "/nix/store/ilzq042wih0h5vdzxcpf6sd826h37g6w-fping-5.2"
+        },
+        "x86_64-darwin": {
+          "outputs": [
+            {
+              "name": "out",
+              "path": "/nix/store/hrh3202f2njx3skj3xn33fish5az5691-fping-5.2",
+              "default": true
+            }
+          ],
+          "store_path": "/nix/store/hrh3202f2njx3skj3xn33fish5az5691-fping-5.2"
+        },
+        "x86_64-linux": {
+          "outputs": [
+            {
+              "name": "out",
+              "path": "/nix/store/2nr99jpa9g7b5z8pwj85awzh4qbhas28-fping-5.2",
+              "default": true
+            }
+          ],
+          "store_path": "/nix/store/2nr99jpa9g7b5z8pwj85awzh4qbhas28-fping-5.2"
+        }
+      }
+    },
     "gh@latest": {
       "last_modified": "2024-05-30T12:09:21Z",
       "resolved": "github:NixOS/nixpkgs/aa61b27554a5fc282758bf0324781e3464ef2cde#gh",
diff --git a/test/e2e/self_hosted_test.go b/test/e2e/self_hosted_test.go
