diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 87d00eaeb..60c68e79e 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -68,18 +68,10 @@ jobs: - Nutanix # Uncomment below once we have the ability to run e2e tests on other providers from GHA. # - AWS - cni: - - Calico - - Cilium - addon-strategy: - - ClusterResourceSet - - HelmAddon fail-fast: false uses: ./.github/workflows/e2e.yml with: provider: ${{ matrix.provider }} - cni: ${{ matrix.cni }} - addon-strategy: ${{ matrix.addon-strategy }} focus: Quick start runs-on: ${{ matrix.provider == 'Nutanix' && 'self-hosted-ncn-dind' || 'ubuntu-22.04' }} secrets: inherit @@ -101,18 +93,10 @@ jobs: # Uncomment below once we have the ability to run e2e tests on other providers from GHA. # - AWS # - Nutanix - cni: - - Calico - - Cilium - addon-strategy: - - ClusterResourceSet - - HelmAddon fail-fast: false uses: ./.github/workflows/e2e.yml with: provider: ${{ matrix.provider }} - cni: ${{ matrix.cni }} - addon-strategy: ${{ matrix.addon-strategy }} focus: Self-hosted runs-on: ${{ matrix.provider == 'Nutanix' && 'self-hosted-ncn-dind' || 'ubuntu-22.04' }} secrets: inherit diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index c0bcbfb39..78ebc2694 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -8,14 +8,6 @@ on: description: Infrastructure provider to run e2e tests with type: string required: true - cni: - description: CNI to run e2e tests with - type: string - required: true - addon-strategy: - description: Addon strategy to run e2e tests with - type: string - required: true skip: description: e2e tests to skip type: string @@ -77,25 +69,8 @@ jobs: echo "After removing files:" df -h - - name: Get Control Plane endpoint IP - id: get-control-plane-endpoint-ip - if: inputs.provider == 'Nutanix' - run: | - export CONTROL_PLANE_ENDPOINT_RANGE_START="${{ vars.NUTANIX_CONTROL_PLANE_ENDPOINT_RANGE_START }}" - export CONTROL_PLANE_ENDPOINT_RANGE_END="${{ vars.NUTANIX_CONTROL_PLANE_ENDPOINT_RANGE_END }}" - control_plane_endpoint_ip="$(devbox run -- make nutanix-cp-endpoint-ip)" - echo "control_plane_endpoint_ip=${control_plane_endpoint_ip}" >> "${GITHUB_OUTPUT}" - - - name: Check Control Plane endpoint IP - if: inputs.provider == 'Nutanix' - run: | - if [[ -z "${{ steps.get-control-plane-endpoint-ip.outputs.control_plane_endpoint_ip }}" ]]; then - echo "control_plane_endpoint_ip is empty; cannot proceed with e2e tests" - exit 1 - fi - - name: Run e2e tests - run: devbox run -- make e2e-test E2E_LABEL='provider:${{ inputs.provider }} && cni:${{ inputs.cni }} && addonStrategy:${{ inputs.addon-strategy }}' E2E_SKIP='${{ inputs.skip }}' E2E_FOCUS='${{ inputs.focus }}' E2E_VERBOSE=true + run: devbox run -- make e2e-test E2E_LABEL='provider:${{ inputs.provider }}' E2E_SKIP='${{ inputs.skip }}' E2E_FOCUS='${{ inputs.focus }}' E2E_VERBOSE=true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} @@ -109,7 +84,6 @@ jobs: NUTANIX_SUBNET_NAME: ${{ vars.NUTANIX_SUBNET_NAME }} NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME: ${{ vars.NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME }} NUTANIX_STORAGE_CONTAINER_NAME: ${{ vars.NUTANIX_STORAGE_CONTAINER_NAME }} - CONTROL_PLANE_ENDPOINT_IP: ${{ steps.get-control-plane-endpoint-ip.outputs.control_plane_endpoint_ip }} - if: success() || failure() # always run even if the previous step fails name: Publish e2e test report diff --git a/api/go.mod b/api/go.mod index 82db046e4..c517197eb 100644 --- a/api/go.mod +++ b/api/go.mod @@ -12,7 +12,7 @@ replace github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/c require ( github.com/aws/aws-sdk-go v1.54.19 github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common v0.7.0 - github.com/nutanix-cloud-native/prism-go-client v0.4.0 + github.com/nutanix-cloud-native/prism-go-client v0.4.1-0.20240704131014-072b7a88a7f5 github.com/onsi/gomega v1.33.1 k8s.io/api v0.29.6 k8s.io/apiextensions-apiserver v0.29.6 diff --git a/api/go.sum b/api/go.sum index 7537933ba..0cb5c4e0a 100644 --- a/api/go.sum +++ b/api/go.sum @@ -85,8 +85,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/nutanix-cloud-native/prism-go-client v0.4.0 h1:P9mLW6eyKMUXVQBzuVL5k7WjV1YwVu8XNpu2XAsRgGo= -github.com/nutanix-cloud-native/prism-go-client v0.4.0/go.mod h1:bHxgYigeclzjuaMEdjpsIEO4k7sjzP4Gr7ooF6nWXcI= +github.com/nutanix-cloud-native/prism-go-client v0.4.1-0.20240704131014-072b7a88a7f5 h1:R9Aaat25nA9mAIRGciy8QpkVzLOcuk+TV7U4s1kH8fQ= +github.com/nutanix-cloud-native/prism-go-client v0.4.1-0.20240704131014-072b7a88a7f5/go.mod h1:C2eEpgqsMqwhliCKeoJOvkkWobeZr4d4DiaCR09pdHA= github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= diff --git a/common/go.mod b/common/go.mod index 7b6772b1f..a337fc4d8 100644 --- a/common/go.mod +++ b/common/go.mod @@ -59,7 +59,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/nutanix-cloud-native/prism-go-client v0.4.0 // indirect + github.com/nutanix-cloud-native/prism-go-client v0.4.1-0.20240704131014-072b7a88a7f5 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.18.0 // indirect diff --git a/common/go.sum b/common/go.sum index 1028947ea..60890db06 100644 --- a/common/go.sum +++ b/common/go.sum @@ -106,8 +106,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/nutanix-cloud-native/prism-go-client v0.4.0 h1:P9mLW6eyKMUXVQBzuVL5k7WjV1YwVu8XNpu2XAsRgGo= -github.com/nutanix-cloud-native/prism-go-client v0.4.0/go.mod h1:bHxgYigeclzjuaMEdjpsIEO4k7sjzP4Gr7ooF6nWXcI= +github.com/nutanix-cloud-native/prism-go-client v0.4.1-0.20240704131014-072b7a88a7f5 h1:R9Aaat25nA9mAIRGciy8QpkVzLOcuk+TV7U4s1kH8fQ= +github.com/nutanix-cloud-native/prism-go-client v0.4.1-0.20240704131014-072b7a88a7f5/go.mod h1:C2eEpgqsMqwhliCKeoJOvkkWobeZr4d4DiaCR09pdHA= github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= diff --git a/devbox.json b/devbox.json index 0a80a8fc3..369f18c24 100644 --- a/devbox.json +++ b/devbox.json @@ -6,7 +6,6 @@ "crane@latest", "envsubst@latest", "findutils@latest", - "fping@latest", "gh@latest", "ginkgo@latest", "git@latest", diff --git a/devbox.lock b/devbox.lock index 8e54bb4c7..f5e4ccb95 100644 --- a/devbox.lock +++ b/devbox.lock @@ -381,54 +381,6 @@ } } }, - "fping@latest": { - "last_modified": "2024-06-12T20:55:33Z", - "resolved": "github:NixOS/nixpkgs/a9858885e197f984d92d7fe64e9fff6b2e488d40#fping", - "source": "devbox-search", - "version": "5.2", - "systems": { - "aarch64-darwin": { - "outputs": [ - { - "name": "out", - "path": "/nix/store/clxfp6jl0d2fs1bp2d1278534n2gixbj-fping-5.2", - "default": true - } - ], - "store_path": "/nix/store/clxfp6jl0d2fs1bp2d1278534n2gixbj-fping-5.2" - }, - "aarch64-linux": { - "outputs": [ - { - "name": "out", - "path": "/nix/store/ilzq042wih0h5vdzxcpf6sd826h37g6w-fping-5.2", - "default": true - } - ], - "store_path": "/nix/store/ilzq042wih0h5vdzxcpf6sd826h37g6w-fping-5.2" - }, - "x86_64-darwin": { - "outputs": [ - { - "name": "out", - "path": "/nix/store/hrh3202f2njx3skj3xn33fish5az5691-fping-5.2", - "default": true - } - ], - "store_path": "/nix/store/hrh3202f2njx3skj3xn33fish5az5691-fping-5.2" - }, - "x86_64-linux": { - "outputs": [ - { - "name": "out", - "path": "/nix/store/2nr99jpa9g7b5z8pwj85awzh4qbhas28-fping-5.2", - "default": true - } - ], - "store_path": "/nix/store/2nr99jpa9g7b5z8pwj85awzh4qbhas28-fping-5.2" - } - } - }, "gh@latest": { "last_modified": "2024-05-30T12:09:21Z", "resolved": "github:NixOS/nixpkgs/aa61b27554a5fc282758bf0324781e3464ef2cde#gh", diff --git a/go.mod b/go.mod index 8ea535bb3..bdc264ddf 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix -go 1.21 +go 1.22 toolchain go1.22.5 @@ -17,9 +17,13 @@ require ( github.com/go-logr/logr v1.4.2 github.com/gobuffalo/flect v1.0.2 github.com/google/go-cmp v0.6.0 + github.com/google/uuid v1.4.0 github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api v0.0.0-00010101000000-000000000000 github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common v0.7.0 - github.com/nutanix-cloud-native/prism-go-client v0.4.0 + github.com/nutanix-cloud-native/prism-go-client v0.4.1-0.20240704131014-072b7a88a7f5 + github.com/nutanix/ntnx-api-golang-clients/clustermgmt-go-client/v4 v4.0.1-beta.2 + github.com/nutanix/ntnx-api-golang-clients/networking-go-client/v4 v4.0.2-beta.1 + github.com/nutanix/ntnx-api-golang-clients/prism-go-client/v4 v4.0.1-beta.1 github.com/onsi/ginkgo/v2 v2.19.0 github.com/onsi/gomega v1.33.1 github.com/pkg/errors v0.9.1 @@ -84,7 +88,8 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 // indirect - github.com/google/uuid v1.4.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-retryablehttp v0.7.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/huandu/xstrings v1.4.0 // indirect github.com/imdario/mergo v0.3.13 // indirect @@ -103,6 +108,8 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/nutanix/ntnx-api-golang-clients/storage-go-client/v4 v4.0.2-alpha.3 // indirect + github.com/nutanix/ntnx-api-golang-clients/vmm-go-client/v4 v4.0.1-beta.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0-rc5 // indirect github.com/pelletier/go-toml v1.9.5 // indirect @@ -115,6 +122,7 @@ require ( github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/shopspring/decimal v1.3.1 // indirect + github.com/sirupsen/logrus v1.9.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect diff --git a/go.sum b/go.sum index a1860c89e..6fdbcbe58 100644 --- a/go.sum +++ b/go.sum @@ -70,6 +70,8 @@ github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= @@ -130,6 +132,14 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= +github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-retryablehttp v0.7.1 h1:sUiuQAnLlbvmExtFQs72iFW/HXeUn8Z1aJLQ4LJJbTQ= +github.com/hashicorp/go-retryablehttp v0.7.1/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= @@ -161,6 +171,8 @@ github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0V github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= @@ -184,8 +196,18 @@ github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/nutanix-cloud-native/prism-go-client v0.4.0 h1:P9mLW6eyKMUXVQBzuVL5k7WjV1YwVu8XNpu2XAsRgGo= -github.com/nutanix-cloud-native/prism-go-client v0.4.0/go.mod h1:bHxgYigeclzjuaMEdjpsIEO4k7sjzP4Gr7ooF6nWXcI= +github.com/nutanix-cloud-native/prism-go-client v0.4.1-0.20240704131014-072b7a88a7f5 h1:R9Aaat25nA9mAIRGciy8QpkVzLOcuk+TV7U4s1kH8fQ= +github.com/nutanix-cloud-native/prism-go-client v0.4.1-0.20240704131014-072b7a88a7f5/go.mod h1:C2eEpgqsMqwhliCKeoJOvkkWobeZr4d4DiaCR09pdHA= +github.com/nutanix/ntnx-api-golang-clients/clustermgmt-go-client/v4 v4.0.1-beta.2 h1:s1u5/GEw3mTZakepJoTD1OvPVU1YuioRxmKZin+W99s= +github.com/nutanix/ntnx-api-golang-clients/clustermgmt-go-client/v4 v4.0.1-beta.2/go.mod h1:sd4Fnk6MVfEDVY+8WyRoQTmLhi2SgZ3riySWErVHf8E= +github.com/nutanix/ntnx-api-golang-clients/networking-go-client/v4 v4.0.2-beta.1 h1:PvZQwYhhJtxmzLpnzEhHTpp2fV6woc6W65PHGsHzVfs= +github.com/nutanix/ntnx-api-golang-clients/networking-go-client/v4 v4.0.2-beta.1/go.mod h1:+eZgV1+xL/r84qmuFSVt5R8OFRO70rEz92jOnVgJNco= +github.com/nutanix/ntnx-api-golang-clients/prism-go-client/v4 v4.0.1-beta.1 h1:hvy3QCc2SgVidYxTq0rRPOazJOt1PP8A86kW7j6sywU= +github.com/nutanix/ntnx-api-golang-clients/prism-go-client/v4 v4.0.1-beta.1/go.mod h1:Yhk+xD4mN90OKEHnk5ARf97CX5p4+MEC/B/YIVoZeZ0= +github.com/nutanix/ntnx-api-golang-clients/storage-go-client/v4 v4.0.2-alpha.3 h1:K3I9YtqKcKKxSL4+tcxnFeLOoaptiVlpsOJ9Xzq3shM= +github.com/nutanix/ntnx-api-golang-clients/storage-go-client/v4 v4.0.2-alpha.3/go.mod h1:kz3gO87xtWnPOCP2kN7yw5LvCDVRnvg8BOWL7CarqXA= +github.com/nutanix/ntnx-api-golang-clients/vmm-go-client/v4 v4.0.1-beta.1 h1:XuTRvYu1kiNjdXOYVwyjhKlFWyo9nMit6GsOYV8+5Cg= +github.com/nutanix/ntnx-api-golang-clients/vmm-go-client/v4 v4.0.1-beta.1/go.mod h1:CaWm4GFpAjQQDc6YXl/dUDrHpuW54h8j6Cj7EslE4Qk= github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= @@ -331,6 +353,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/make/dev.mk b/make/dev.mk index f5b4a7975..3b823d0a7 100644 --- a/make/dev.mk +++ b/make/dev.mk @@ -55,7 +55,3 @@ endif gojq --yaml-input --raw-output '.variables | to_entries | map("export \(.key)=\(.value|tostring)")|.[]' < test/e2e/config/caren.yaml | envsubst > .envrc.e2e setup-envtest use -p env $(ENVTEST_VERSION) >> .envrc.e2e direnv reload - -.PHONY: nutanix-cp-endpoint-ip -nutanix-cp-endpoint-ip: ## Gets a random IP from the control plane endpoint range. - shuf --head-count=1 < <(fping -g -u "$(CONTROL_PLANE_ENDPOINT_RANGE_START)" "$(CONTROL_PLANE_ENDPOINT_RANGE_END)") diff --git a/test/e2e/config/caren.yaml b/test/e2e/config/caren.yaml index 38ccfce3b..2b0dca0b4 100644 --- a/test/e2e/config/caren.yaml +++ b/test/e2e/config/caren.yaml @@ -190,15 +190,11 @@ variables: # set as empty here to enable running the e2e tests for non-nutanix providers locally without setting the env var. NUTANIX_ENDPOINT: "" # # Port of Prism Central. Default: 9440 - # NUTANIX_PORT: 9440 - # # Disable Prism Central certificate checking. Default: false - # NUTANIX_INSECURE: false + NUTANIX_PORT: 9440 # # Prism Central user NUTANIX_USER: "" # # Prism Central password NUTANIX_PASSWORD: "" - # # Host IP to be assigned to the CAPX Kubernetes cluster. - # CONTROL_PLANE_ENDPOINT_IP: "" # # Port of the CAPX Kubernetes cluster. Default: 6443 # CONTROL_PLANE_ENDPOINT_PORT: 6443 # # Name of the Prism Element cluster. diff --git a/test/e2e/framework/nutanix/client.go b/test/e2e/framework/nutanix/client.go new file mode 100644 index 000000000..f6b446948 --- /dev/null +++ b/test/e2e/framework/nutanix/client.go @@ -0,0 +1,84 @@ +//go:build e2e + +// Copyright 2024 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package nutanix + +import ( + "context" + "fmt" + "time" + + prismcommonapi "github.com/nutanix/ntnx-api-golang-clients/prism-go-client/v4/models/common/v1/config" + prismapi "github.com/nutanix/ntnx-api-golang-clients/prism-go-client/v4/models/prism/v4/config" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/utils/ptr" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" + + prismgoclient "github.com/nutanix-cloud-native/prism-go-client" + prismclientv4 "github.com/nutanix-cloud-native/prism-go-client/v4" +) + +const ( + prismEndpointVariableName = "NUTANIX_ENDPOINT" + prismPortVariableName = "NUTANIX_PORT" + prismUsernameVariableName = "NUTANIX_USER" + prismPasswordVariableName = "NUTANIX_PASSWORD" +) + +func CredentialsFromCAPIE2EConfig(e2eConfig *clusterctl.E2EConfig) *prismgoclient.Credentials { + return &prismgoclient.Credentials{ + Endpoint: e2eConfig.GetVariable(prismEndpointVariableName), + Port: e2eConfig.GetVariable(prismPortVariableName), + Username: e2eConfig.GetVariable(prismUsernameVariableName), + Password: e2eConfig.GetVariable(prismPasswordVariableName), + Insecure: false, + } +} + +func NewV4Client(credentials *prismgoclient.Credentials) (*prismclientv4.Client, error) { + v4Client, err := prismclientv4.NewV4Client(*credentials) + if err != nil { + return nil, fmt.Errorf("failed to create Nutanix V4 API client: %w", err) + } + + return v4Client, nil +} + +func WaitForTaskCompletion( + ctx context.Context, + taskID string, + v4Client *prismclientv4.Client, +) ([]prismcommonapi.KVPair, error) { + var data []prismcommonapi.KVPair + + if err := wait.PollUntilContextCancel( + ctx, + 100*time.Millisecond, + true, + func(ctx context.Context) (done bool, err error) { + task, err := v4Client.TasksApiInstance.GetTaskById(ptr.To(taskID)) + if err != nil { + return false, fmt.Errorf("failed to get task %s: %w", taskID, err) + } + + taskData, ok := task.GetData().(prismapi.Task) + if !ok { + return false, fmt.Errorf("unexpected task data type %[1]T: %+[1]v", task.GetData()) + } + + if ptr.Deref(taskData.Status, prismapi.TASKSTATUS_UNKNOWN) != prismapi.TASKSTATUS_SUCCEEDED { + return false, nil + } + + data = taskData.CompletionDetails + + return true, nil + }, + ); err != nil { + return nil, fmt.Errorf("failed to wait for task %s to complete: %w", taskID, err) + } + + return data, nil +} diff --git a/test/e2e/framework/nutanix/cluster.go b/test/e2e/framework/nutanix/cluster.go new file mode 100644 index 000000000..c72febfc8 --- /dev/null +++ b/test/e2e/framework/nutanix/cluster.go @@ -0,0 +1,59 @@ +//go:build e2e + +// Copyright 2024 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package nutanix + +import ( + "fmt" + + "github.com/google/uuid" + clustersapi "github.com/nutanix/ntnx-api-golang-clients/clustermgmt-go-client/v4/models/clustermgmt/v4/config" + "k8s.io/utils/ptr" + + prismclientv4 "github.com/nutanix-cloud-native/prism-go-client/v4" +) + +func GetClusterUUIDFromName(cluster string, v4Client *prismclientv4.Client) (uuid.UUID, error) { + clusterUUID, err := uuid.Parse(cluster) + if err == nil { + return clusterUUID, nil + } + + response, err := v4Client.ClustersApiInstance.ListClusters( + nil, + nil, + ptr.To(`name eq '`+cluster+`'`), + nil, + nil, + nil, + ) + if err != nil { + return uuid.UUID{}, fmt.Errorf( + "failed to find cluster uuid for cluster %s: %w", + cluster, + err, + ) + } + clusters := response.GetData() + if clusters == nil { + return uuid.UUID{}, fmt.Errorf("no cluster found with name %s", cluster) + } + + switch apiClusters := clusters.(type) { + case []clustersapi.Cluster: + if len(apiClusters) == 0 { + return uuid.UUID{}, fmt.Errorf("no subnet found with name %s", cluster) + } + + clusterUUID, err := uuid.Parse(*apiClusters[0].ExtId) + if err != nil { + return uuid.UUID{}, fmt.Errorf("failed to parse cluster uuid for cluster %s: %w", cluster, err) + } + + return clusterUUID, nil + default: + return uuid.UUID{}, fmt.Errorf("unknown response: %+v", clusters) + } +} diff --git a/test/e2e/framework/nutanix/networking.go b/test/e2e/framework/nutanix/networking.go new file mode 100644 index 000000000..31f7050ce --- /dev/null +++ b/test/e2e/framework/nutanix/networking.go @@ -0,0 +1,230 @@ +//go:build e2e + +// Copyright 2024 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package nutanix + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "time" + + "github.com/google/uuid" + networkingcommonapi "github.com/nutanix/ntnx-api-golang-clients/networking-go-client/v4/models/common/v1/config" + networkingapi "github.com/nutanix/ntnx-api-golang-clients/networking-go-client/v4/models/networking/v4/config" + networkingprismapi "github.com/nutanix/ntnx-api-golang-clients/networking-go-client/v4/models/prism/v4/config" + "k8s.io/utils/ptr" + + prismclientv4 "github.com/nutanix-cloud-native/prism-go-client/v4" +) + +func GetSubnetUUIDFromNameAndCluster( + subnet, cluster string, v4Client *prismclientv4.Client, +) (uuid.UUID, error) { + clusterUUID, err := uuid.Parse(cluster) + if err != nil { + clusterUUID, err = GetClusterUUIDFromName(cluster, v4Client) + if err != nil { + return uuid.UUID{}, fmt.Errorf( + "failed to get cluster uuid for cluster %s: %w", + cluster, + err, + ) + } + } + + listSubnetsResponse, err := v4Client.SubnetsApiInstance.ListSubnets( + nil, + nil, + ptr.To(`name eq '`+subnet+`' and clusterReference eq '`+clusterUUID.String()+`'`), + nil, + nil, + nil, + ) + if err != nil { + return uuid.UUID{}, fmt.Errorf("failed to find subnet uuid for subnet %s: %w", subnet, err) + } + subnets := listSubnetsResponse.GetData() + if subnets == nil { + return uuid.UUID{}, fmt.Errorf("no subnet found with name %s", subnet) + } + + switch apiSubnets := subnets.(type) { + case []networkingapi.Subnet: + if len(apiSubnets) == 0 { + return uuid.UUID{}, fmt.Errorf("no subnet found with name %s", subnet) + } + + subnetUUID, err := uuid.Parse(*apiSubnets[0].ExtId) + if err != nil { + return uuid.UUID{}, fmt.Errorf("failed to parse subnet uuid for subnet %s: %w", subnet, err) + } + + return subnetUUID, nil + case []networkingapi.SubnetProjection: + if len(apiSubnets) == 0 { + return uuid.UUID{}, fmt.Errorf("no subnet found with name %s", subnet) + } + + subnetUUID, err := uuid.Parse(*apiSubnets[0].ExtId) + if err != nil { + return uuid.UUID{}, fmt.Errorf("failed to parse subnet uuid for subnet %s: %w", subnet, err) + } + + return subnetUUID, nil + default: + return uuid.UUID{}, fmt.Errorf("unknown response: %+v", subnets) + } +} + +type reservedIPs struct { + ReservedIPs []string `json:"reserved_ips"` +} + +func ReserveIP( + subnet, cluster string, + v4Client *prismclientv4.Client, +) (ip string, unreserve func() error, err error) { + clusterUUID, err := uuid.Parse(cluster) + if err != nil { + clusterUUID, err = GetClusterUUIDFromName(cluster, v4Client) + if err != nil { + return "", nil, fmt.Errorf( + "failed to get cluster uuid for cluster %s: %w", + cluster, + err, + ) + } + } + + subnetUUID, err := uuid.Parse(subnet) + if err != nil { + subnetUUID, err = GetSubnetUUIDFromNameAndCluster(subnet, clusterUUID.String(), v4Client) + if err != nil { + return "", nil, fmt.Errorf("failed to get subnet uuid for subnet %s: %w", subnet, err) + } + } + + reserveIPResponse, err := v4Client.SubnetIPReservationApi.ReserveIpsBySubnetId( + ptr.To(subnetUUID.String()), + &networkingapi.IpReserveSpec{ + Count: ptr.To[int64](1), + ReserveType: ptr.To(networkingapi.RESERVETYPE_IP_ADDRESS_COUNT), + }, + ) + if err != nil { + return "", nil, fmt.Errorf("failed to reserve IP in subnet %s: %w", subnet, err) + } + + responseData, ok := reserveIPResponse.GetData().(networkingprismapi.TaskReference) + if !ok { + return "", nil, fmt.Errorf( + "unexpected response data type %[1]T: %+[1]v", + reserveIPResponse.GetData(), + ) + } + if responseData.ExtId == nil { + return "", nil, fmt.Errorf( + "no task id found in response: %+[1]v", + reserveIPResponse.GetData(), + ) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + result, err := WaitForTaskCompletion(ctx, *responseData.ExtId, v4Client) + if err != nil { + return "", nil, fmt.Errorf("failed to wait for task completion: %w", err) + } + + if len(result) == 0 { + return "", nil, fmt.Errorf("no IP address reserved") + } + + marshaledResponseBytes, _ := json.Marshal(result[0].Value) + marshaledResponse, err := strconv.Unquote(string(marshaledResponseBytes)) + if err != nil { + return "", nil, fmt.Errorf( + "failed to unquote reserved IP response %s: %w", + marshaledResponseBytes, + err, + ) + } + + var response reservedIPs + if err := json.Unmarshal([]byte(marshaledResponse), &response); err != nil { + return "", nil, fmt.Errorf( + "failed to unmarshal reserved IP response %s: %w", + marshaledResponse, + err, + ) + } + + return response.ReservedIPs[0], + func() error { + return UnreserveIP( + response.ReservedIPs[0], + subnetUUID.String(), + clusterUUID.String(), + v4Client, + ) + }, + nil +} + +func UnreserveIP(ip, subnet, cluster string, v4Client *prismclientv4.Client) error { + clusterUUID, err := uuid.Parse(cluster) + if err != nil { + clusterUUID, err = GetClusterUUIDFromName(cluster, v4Client) + if err != nil { + return fmt.Errorf("failed to get cluster uuid for cluster %s: %w", cluster, err) + } + } + + subnetUUID, err := uuid.Parse(subnet) + if err != nil { + subnetUUID, err = GetSubnetUUIDFromNameAndCluster(subnet, clusterUUID.String(), v4Client) + if err != nil { + return fmt.Errorf("failed to get subnet uuid for subnet %s: %w", subnet, err) + } + } + + ipAddress := networkingcommonapi.NewIPAddress() + ipAddress.Ipv4 = networkingcommonapi.NewIPv4Address() + ipAddress.Ipv4.Value = ptr.To(ip) + unreserveIPResponse, err := v4Client.SubnetIPReservationApi.UnreserveIpsBySubnetId( + ptr.To(subnetUUID.String()), + &networkingapi.IpUnreserveSpec{ + UnreserveType: ptr.To(networkingapi.UNRESERVETYPE_IP_ADDRESS_LIST), + IpAddresses: []networkingcommonapi.IPAddress{*ipAddress}, + }, + ) + if err != nil { + return fmt.Errorf("failed to reserve IP in subnet %s: %w", subnet, err) + } + + responseData, ok := unreserveIPResponse.GetData().(networkingprismapi.TaskReference) + if !ok { + return fmt.Errorf( + "unexpected response data type %[1]T: %+[1]v", + unreserveIPResponse.GetData(), + ) + } + if responseData.ExtId == nil { + return fmt.Errorf("no task id found in response: %+v", unreserveIPResponse.GetData()) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + _, err = WaitForTaskCompletion(ctx, *responseData.ExtId, v4Client) + if err != nil { + return fmt.Errorf("failed to wait for task completion: %w", err) + } + + return nil +} diff --git a/test/e2e/quick_start_test.go b/test/e2e/quick_start_test.go index 7218ba78e..96e820703 100644 --- a/test/e2e/quick_start_test.go +++ b/test/e2e/quick_start_test.go @@ -7,6 +7,7 @@ package e2e import ( "fmt" + "os" "slices" "strconv" "strings" @@ -18,207 +19,264 @@ import ( clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" clusterctlcluster "sigs.k8s.io/cluster-api/cmd/clusterctl/client/cluster" capie2e "sigs.k8s.io/cluster-api/test/e2e" - "sigs.k8s.io/cluster-api/test/framework" + capiframework "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" apivariables "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/variables" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/test/e2e/framework/nutanix" ) -var _ = Describe("Quick start", Serial, func() { +var _ = Describe("Quick start", func() { for _, provider := range []string{"Docker", "AWS", "Nutanix"} { - lowercaseProvider := strings.ToLower(provider) - for _, cniProvider := range []string{"Cilium", "Calico"} { - for _, addonStrategy := range []string{"HelmAddon", "ClusterResourceSet"} { - strategy := "" - switch addonStrategy { - case "HelmAddon": - strategy = "helm-addon" - case "ClusterResourceSet": - strategy = "crs" - default: - Fail("unknown addon strategy: " + addonStrategy) - } - flavour := fmt.Sprintf( - "topology-%s-%s", - strings.ToLower(cniProvider), - strategy, - ) - Context( - flavour, - Label("provider:"+provider), - Label("cni:"+cniProvider), - Label("addonStrategy:"+addonStrategy), - func() { - capie2e.QuickStartSpec(ctx, func() capie2e.QuickStartSpecInput { - if !slices.Contains( - e2eConfig.InfrastructureProviders(), - lowercaseProvider, - ) { - Fail(fmt.Sprintf( - "provider %s is not enabled - check environment setup for provider specific requirements", - lowercaseProvider, - )) - } + // Add any provider specific decorators here. + // Currently, only Docker requires Serial decorator to ensure the machine running the Docker e2e tests + // doesn't have resources exhausted and lead to flaky tests. + // Other provider tests will run in parallel. + var providerSpecificDecorators []interface{} + if provider == "Docker" { + providerSpecificDecorators = append(providerSpecificDecorators, Serial) + } - // Check if a provider-specific Kubernetes version is set in the environment and use that. This allows - // for testing against different Kubernetes versions, as some providers (e.g. Docker) have machine images - // available that are not available in other providers. - // This version can be specified in `test/e2e/config/caren.yaml` with a variable named - // `KUBERNETES_VERSION_`, where `` is the uppercase provider name, e.g. - // `KUBERNETES_VERSION_DOCKER: v1.29.5`. - testE2EConfig := e2eConfig.DeepCopy() - varName := capie2e.KubernetesVersion + "_" + strings.ToUpper( - lowercaseProvider, - ) - if testE2EConfig.HasVariable(varName) { - testE2EConfig.Variables[capie2e.KubernetesVersion] = testE2EConfig.GetVariable( - varName, - ) + Context(provider, Label("provider:"+provider), providerSpecificDecorators, func() { + lowercaseProvider := strings.ToLower(provider) + for _, cniProvider := range []string{"Cilium", "Calico"} { + Context(cniProvider, Label("cni:"+cniProvider), func() { + for _, addonStrategy := range []string{"HelmAddon", "ClusterResourceSet"} { + Context(addonStrategy, Label("addonStrategy:"+addonStrategy), func() { + strategy := "" + switch addonStrategy { + case "HelmAddon": + strategy = "helm-addon" + case "ClusterResourceSet": + strategy = "crs" + default: + Fail("unknown addon strategy: " + addonStrategy) } - - return capie2e.QuickStartSpecInput{ - E2EConfig: testE2EConfig, - ClusterctlConfigPath: clusterctlConfigPath, - BootstrapClusterProxy: bootstrapClusterProxy, - ArtifactFolder: artifactFolder, - SkipCleanup: skipCleanup, - Flavor: ptr.To(flavour), - InfrastructureProvider: ptr.To(lowercaseProvider), - PostMachinesProvisioned: func(proxy framework.ClusterProxy, namespace, clusterName string) { - framework.AssertOwnerReferences( - namespace, - proxy.GetKubeconfigPath(), - clusterctlcluster.FilterClusterObjectsWithNameFilter( - clusterName, - ), - framework.CoreOwnerReferenceAssertion, - framework.DockerInfraOwnerReferenceAssertions, - framework.KubeadmBootstrapOwnerReferenceAssertions, - framework.KubeadmControlPlaneOwnerReferenceAssertions, - AWSInfraOwnerReferenceAssertions, - NutanixInfraOwnerReferenceAssertions, - AddonReferenceAssertions, - KubernetesReferenceAssertions, + flavour := fmt.Sprintf( + "topology-%s-%s", + strings.ToLower(cniProvider), + strategy, + ) + Context( + flavour, + func() { + var ( + testE2EConfig *clusterctl.E2EConfig + clusterLocalClusterctlConfigPath string ) - workloadCluster := framework.GetClusterByName( - ctx, - framework.GetClusterByNameInput{ - Namespace: namespace, - Name: clusterName, - Getter: proxy.GetClient(), - }, - ) - Expect(workloadCluster.Spec.Topology).ToNot(BeNil()) + BeforeEach(func() { + testE2EConfig = e2eConfig.DeepCopy() - By("Waiting until nodes are ready") - workloadProxy := proxy.GetWorkloadCluster( - ctx, - namespace, - clusterName, - ) - workloadClient := workloadProxy.GetClient() - - nodeCount := int( - ptr.Deref( - workloadCluster.Spec.Topology.ControlPlane.Replicas, - 0, - ), - ) + - lo.Reduce( - workloadCluster.Spec.Topology.Workers.MachineDeployments, - func(agg int, md clusterv1.MachineDeploymentTopology, _ int) int { - switch { - case md.Replicas != nil: - return agg + int(ptr.Deref(md.Replicas, 0)) - case md.Metadata.Annotations["cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size"] != "": - minSize, err := strconv.Atoi( - md.Metadata.Annotations["cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size"], - ) - Expect(err).ToNot(HaveOccurred()) - return agg + minSize - default: - return agg - } - }, - 0, + // Check if a provider-specific Kubernetes version is set in the environment and use that. This allows + // for testing against different Kubernetes versions, as some providers (e.g. Docker) have machine images + // available that are not available in other providers. + // This version can be specified in `test/e2e/config/caren.yaml` with a variable named + // `KUBERNETES_VERSION_`, where `` is the uppercase provider name, e.g. + // `KUBERNETES_VERSION_DOCKER: v1.29.5`. + varName := capie2e.KubernetesVersion + "_" + strings.ToUpper( + lowercaseProvider, ) + if testE2EConfig.HasVariable(varName) { + testE2EConfig.Variables[capie2e.KubernetesVersion] = testE2EConfig.GetVariable( + varName, + ) + } - framework.WaitForNodesReady( - ctx, - framework.WaitForNodesReadyInput{ - Lister: workloadClient, - KubernetesVersion: testE2EConfig.GetVariable( - capie2e.KubernetesVersion, - ), - Count: nodeCount, - WaitForNodesReady: testE2EConfig.GetIntervals( - flavour, - "wait-nodes-ready", - ), - }, - ) + // For Nutanix provider, reserve an IP address for the workload cluster control plane endpoint - + // remember to unreserve it! + if provider == "Nutanix" { + By( + "Reserving an IP address for the workload cluster control plane endpoint", + ) + nutanixClient, err := nutanix.NewV4Client( + nutanix.CredentialsFromCAPIE2EConfig(testE2EConfig), + ) + Expect(err).ToNot(HaveOccurred()) - By( - "Waiting for all requested addons to be ready in workload cluster", - ) - clusterVars := variables.ClusterVariablesToVariablesMap( - workloadCluster.Spec.Topology.Variables, - ) - addonsConfig, err := variables.Get[apivariables.Addons]( - clusterVars, - v1alpha1.ClusterConfigVariableName, - "addons", - ) - Expect(err).ToNot(HaveOccurred()) - WaitForAddonsToBeReadyInWorkloadCluster( - ctx, - WaitForAddonsToBeReadyInWorkloadClusterInput{ - AddonsConfig: addonsConfig, - ClusterProxy: proxy, - WorkloadCluster: workloadCluster, - InfrastructureProvider: lowercaseProvider, - DeploymentIntervals: testE2EConfig.GetIntervals( - flavour, - "wait-deployment", - ), - DaemonSetIntervals: testE2EConfig.GetIntervals( - flavour, - "wait-daemonset", - ), - HelmReleaseIntervals: testE2EConfig.GetIntervals( - flavour, - "wait-helmrelease", - ), - ClusterResourceSetIntervals: testE2EConfig.GetIntervals( - flavour, - "wait-clusterresourceset", - ), - ResourceIntervals: testE2EConfig.GetIntervals( - flavour, - "wait-resource", - ), - }, - ) + controlPlaneEndpointIP, unreserveControlPlaneEndpointIP, err := nutanix.ReserveIP( + testE2EConfig.GetVariable("NUTANIX_SUBNET_NAME"), + testE2EConfig.GetVariable( + "NUTANIX_PRISM_ELEMENT_CLUSTER_NAME", + ), + nutanixClient, + ) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(unreserveControlPlaneEndpointIP) + testE2EConfig.Variables["CONTROL_PLANE_ENDPOINT_IP"] = controlPlaneEndpointIP + } - WaitForCoreDNSToBeReadyInWorkloadCluster( - ctx, - WaitForCoreDNSToBeReadyInWorkloadClusterInput{ - WorkloadCluster: workloadCluster, - ClusterProxy: proxy, - DeploymentIntervals: testE2EConfig.GetIntervals( - flavour, - "wait-deployment", - ), - }, - ) + clusterLocalTempDir, err := os.MkdirTemp("", "clusterctl-") + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { + Expect(os.RemoveAll(clusterLocalTempDir)).To(Succeed()) + }) + clusterLocalClusterctlConfigPath = createClusterctlLocalRepository( + testE2EConfig, + clusterLocalTempDir, + ) + }) + + capie2e.QuickStartSpec(ctx, func() capie2e.QuickStartSpecInput { + if !slices.Contains( + e2eConfig.InfrastructureProviders(), + lowercaseProvider, + ) { + Fail(fmt.Sprintf( + "provider %s is not enabled - check environment setup for provider specific requirements", + lowercaseProvider, + )) + } + + return capie2e.QuickStartSpecInput{ + E2EConfig: testE2EConfig, + ClusterctlConfigPath: clusterLocalClusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + Flavor: ptr.To(flavour), + InfrastructureProvider: ptr.To(lowercaseProvider), + PostMachinesProvisioned: func(proxy capiframework.ClusterProxy, namespace, clusterName string) { + capiframework.AssertOwnerReferences( + namespace, + proxy.GetKubeconfigPath(), + clusterctlcluster.FilterClusterObjectsWithNameFilter( + clusterName, + ), + capiframework.CoreOwnerReferenceAssertion, + capiframework.DockerInfraOwnerReferenceAssertions, + capiframework.KubeadmBootstrapOwnerReferenceAssertions, + capiframework.KubeadmControlPlaneOwnerReferenceAssertions, + AWSInfraOwnerReferenceAssertions, + NutanixInfraOwnerReferenceAssertions, + AddonReferenceAssertions, + KubernetesReferenceAssertions, + ) + + workloadCluster := capiframework.GetClusterByName( + ctx, + capiframework.GetClusterByNameInput{ + Namespace: namespace, + Name: clusterName, + Getter: proxy.GetClient(), + }, + ) + Expect(workloadCluster.Spec.Topology).ToNot(BeNil()) + + By("Waiting until nodes are ready") + workloadProxy := proxy.GetWorkloadCluster( + ctx, + namespace, + clusterName, + ) + workloadClient := workloadProxy.GetClient() + + nodeCount := int( + ptr.Deref( + workloadCluster.Spec.Topology.ControlPlane.Replicas, + 0, + ), + ) + + lo.Reduce( + workloadCluster.Spec.Topology.Workers.MachineDeployments, + func(agg int, md clusterv1.MachineDeploymentTopology, _ int) int { + switch { + case md.Replicas != nil: + return agg + int( + ptr.Deref(md.Replicas, 0), + ) + case md.Metadata.Annotations["cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size"] != "": + minSize, err := strconv.Atoi( + md.Metadata.Annotations["cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size"], + ) + Expect(err).ToNot(HaveOccurred()) + return agg + minSize + default: + return agg + } + }, + 0, + ) + + capiframework.WaitForNodesReady( + ctx, + capiframework.WaitForNodesReadyInput{ + Lister: workloadClient, + KubernetesVersion: testE2EConfig.GetVariable( + capie2e.KubernetesVersion, + ), + Count: nodeCount, + WaitForNodesReady: testE2EConfig.GetIntervals( + flavour, + "wait-nodes-ready", + ), + }, + ) + + By( + "Waiting for all requested addons to be ready in workload cluster", + ) + clusterVars := variables.ClusterVariablesToVariablesMap( + workloadCluster.Spec.Topology.Variables, + ) + addonsConfig, err := variables.Get[apivariables.Addons]( + clusterVars, + v1alpha1.ClusterConfigVariableName, + "addons", + ) + Expect(err).ToNot(HaveOccurred()) + WaitForAddonsToBeReadyInWorkloadCluster( + ctx, + WaitForAddonsToBeReadyInWorkloadClusterInput{ + AddonsConfig: addonsConfig, + ClusterProxy: proxy, + WorkloadCluster: workloadCluster, + InfrastructureProvider: lowercaseProvider, + DeploymentIntervals: testE2EConfig.GetIntervals( + flavour, + "wait-deployment", + ), + DaemonSetIntervals: testE2EConfig.GetIntervals( + flavour, + "wait-daemonset", + ), + HelmReleaseIntervals: testE2EConfig.GetIntervals( + flavour, + "wait-helmrelease", + ), + ClusterResourceSetIntervals: testE2EConfig.GetIntervals( + flavour, + "wait-clusterresourceset", + ), + ResourceIntervals: testE2EConfig.GetIntervals( + flavour, + "wait-resource", + ), + }, + ) + + WaitForCoreDNSToBeReadyInWorkloadCluster( + ctx, + WaitForCoreDNSToBeReadyInWorkloadClusterInput{ + WorkloadCluster: workloadCluster, + ClusterProxy: proxy, + DeploymentIntervals: testE2EConfig.GetIntervals( + flavour, + "wait-deployment", + ), + }, + ) + }, + } + }) }, - } + ) }) - }, - ) + } + }) } - } + }) } }) diff --git a/test/e2e/self_hosted_test.go b/test/e2e/self_hosted_test.go index e8d2319e8..ab39d63de 100644 --- a/test/e2e/self_hosted_test.go +++ b/test/e2e/self_hosted_test.go @@ -23,106 +23,122 @@ import ( var _ = Describe("Self-hosted", Serial, func() { for _, provider := range []string{"Docker", "Nutanix"} { - lowercaseProvider := strings.ToLower(provider) - for _, cniProvider := range []string{"Cilium", "Calico"} { - for _, addonStrategy := range []string{"HelmAddon", "ClusterResourceSet"} { - strategy := "" - switch addonStrategy { - case "HelmAddon": - strategy = "helm-addon" - case "ClusterResourceSet": - strategy = "crs" - default: - Fail("unknown addon strategy: " + addonStrategy) - } - flavour := fmt.Sprintf( - "topology-%s-%s", - strings.ToLower(cniProvider), - strategy, - ) - Context( - flavour, - Label("provider:"+provider), - Label("cni:"+cniProvider), - Label("addonStrategy:"+addonStrategy), - func() { - framework.SelfHostedSpec(ctx, func() framework.SelfHostedSpecInput { - return framework.SelfHostedSpecInput{ - E2EConfig: e2eConfig, - ClusterctlConfigPath: clusterctlConfigPath, - BootstrapClusterProxy: bootstrapClusterProxy, - ArtifactFolder: artifactFolder, - SkipCleanup: skipCleanup, - Flavor: flavour, - InfrastructureProvider: ptr.To(lowercaseProvider), - PostClusterMoved: func(proxy capiframework.ClusterProxy, cluster *clusterv1.Cluster) { - By( - "Waiting for all requested addons to be ready in workload cluster", - ) - workloadCluster := capiframework.GetClusterByName( - ctx, - capiframework.GetClusterByNameInput{ - Namespace: cluster.GetNamespace(), - Name: cluster.GetName(), - Getter: proxy.GetClient(), - }, - ) - Expect(workloadCluster.Spec.Topology).ToNot(BeNil()) - clusterVars := variables.ClusterVariablesToVariablesMap( - workloadCluster.Spec.Topology.Variables, - ) - addonsConfig, err := variables.Get[apivariables.Addons]( - clusterVars, - v1alpha1.ClusterConfigVariableName, - "addons", - ) - Expect(err).ToNot(HaveOccurred()) - WaitForAddonsToBeReadyInWorkloadCluster( + // Add any provider specific decorators here. + // Currently, only Docker requires Serial decorator to ensure the machine running the Docker e2e tests + // doesn't have resources exhausted and lead to flaky tests. + // Other provider tests will run in parallel. + var providerSpecificDecorators []interface{} + if provider == "Docker" { + providerSpecificDecorators = append(providerSpecificDecorators, Serial) + } + Context(provider, Label("provider:"+provider), providerSpecificDecorators, func() { + lowercaseProvider := strings.ToLower(provider) + for _, cniProvider := range []string{"Cilium", "Calico"} { + Context(cniProvider, Label("cni:"+cniProvider), func() { + for _, addonStrategy := range []string{"HelmAddon", "ClusterResourceSet"} { + Context(addonStrategy, Label("addonStrategy:"+addonStrategy), func() { + strategy := "" + switch addonStrategy { + case "HelmAddon": + strategy = "helm-addon" + case "ClusterResourceSet": + strategy = "crs" + default: + Fail("unknown addon strategy: " + addonStrategy) + } + flavour := fmt.Sprintf( + "topology-%s-%s", + strings.ToLower(cniProvider), + strategy, + ) + Context( + flavour, + func() { + framework.SelfHostedSpec( ctx, - WaitForAddonsToBeReadyInWorkloadClusterInput{ - AddonsConfig: addonsConfig, - ClusterProxy: proxy, - WorkloadCluster: workloadCluster, - DeploymentIntervals: e2eConfig.GetIntervals( - flavour, - "wait-deployment", - ), - DaemonSetIntervals: e2eConfig.GetIntervals( - flavour, - "wait-daemonset", - ), - HelmReleaseIntervals: e2eConfig.GetIntervals( - flavour, - "wait-helmrelease", - ), - ClusterResourceSetIntervals: e2eConfig.GetIntervals( - flavour, - "wait-clusterresourceset", - ), - ResourceIntervals: e2eConfig.GetIntervals( - flavour, - "wait-resource", - ), - }, - ) + func() framework.SelfHostedSpecInput { + return framework.SelfHostedSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + Flavor: flavour, + InfrastructureProvider: ptr.To(lowercaseProvider), + PostClusterMoved: func(proxy capiframework.ClusterProxy, cluster *clusterv1.Cluster) { + By( + "Waiting for all requested addons to be ready in workload cluster", + ) + workloadCluster := capiframework.GetClusterByName( + ctx, + capiframework.GetClusterByNameInput{ + Namespace: cluster.GetNamespace(), + Name: cluster.GetName(), + Getter: proxy.GetClient(), + }, + ) + Expect( + workloadCluster.Spec.Topology, + ).ToNot(BeNil()) + clusterVars := variables.ClusterVariablesToVariablesMap( + workloadCluster.Spec.Topology.Variables, + ) + addonsConfig, err := variables.Get[apivariables.Addons]( + clusterVars, + v1alpha1.ClusterConfigVariableName, + "addons", + ) + Expect(err).ToNot(HaveOccurred()) + WaitForAddonsToBeReadyInWorkloadCluster( + ctx, + WaitForAddonsToBeReadyInWorkloadClusterInput{ + AddonsConfig: addonsConfig, + ClusterProxy: proxy, + WorkloadCluster: workloadCluster, + DeploymentIntervals: e2eConfig.GetIntervals( + flavour, + "wait-deployment", + ), + DaemonSetIntervals: e2eConfig.GetIntervals( + flavour, + "wait-daemonset", + ), + HelmReleaseIntervals: e2eConfig.GetIntervals( + flavour, + "wait-helmrelease", + ), + ClusterResourceSetIntervals: e2eConfig.GetIntervals( + flavour, + "wait-clusterresourceset", + ), + ResourceIntervals: e2eConfig.GetIntervals( + flavour, + "wait-resource", + ), + }, + ) - WaitForCoreDNSToBeReadyInWorkloadCluster( - ctx, - WaitForCoreDNSToBeReadyInWorkloadClusterInput{ - WorkloadCluster: workloadCluster, - ClusterProxy: proxy, - DeploymentIntervals: e2eConfig.GetIntervals( - flavour, - "wait-deployment", - ), + WaitForCoreDNSToBeReadyInWorkloadCluster( + ctx, + WaitForCoreDNSToBeReadyInWorkloadClusterInput{ + WorkloadCluster: workloadCluster, + ClusterProxy: proxy, + DeploymentIntervals: e2eConfig.GetIntervals( + flavour, + "wait-deployment", + ), + }, + ) + }, + } }, ) }, - } + ) }) - }, - ) + } + }) } - } + }) } })