test: Add initial e2e tests (#360)

Blocked by #361.

Added Docker e2e tests running using the CAPI e2e test utilities,
clusters being created, waiting for all nodes to become ready, etc.
tests include both Calico and Cilium deployed via both
ClusterResourceSet and HelmAddon strategies. we can add more tests but
this is a good start.

~There is a data race in the test libraries which I don't know how to
work around but it does not affect the usefulness of these tests and we
can look at that later, unless someone knows what the problem is?~
Disabled race in ginkgo call, I think this is OK, it doesn't show races
in deployed code only in tests.

Author: jimmidyson

.github/workflows/checks.yml

@@ -56,24 +56,24 @@ jobs:
 
   e2e-test:
     runs-on: ubuntu-22.04
+    permissions:
+      checks: write
+    strategy:
+      matrix:
+        PROVIDER:
+          - Docker
     steps:
       - name: Check out code
         uses: actions/checkout@v4
-
-      - name: "Check e2e tests exist"
-        id: check_e2e_tests_exist
-        uses: andstor/file-existence-action@v3
         with:
-          files: "test/e2e/**"
+          fetch-depth: 0
 
-      - if: steps.check_e2e_tests_exist.outputs.files_exists == 'true'
-        name: Install devbox
+      - name: Install devbox
         uses: jetpack-io/[email protected]
         with:
           enable-cache: true
 
-      - if: steps.check_e2e_tests_exist.outputs.files_exists == 'true'
-        name: Go cache
+      - name: Go cache
         uses: actions/cache@v4
         with:
           path: |
@@ -83,21 +83,18 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-go-
 
-      - if: steps.check_e2e_tests_exist.outputs.files_exists == 'true'
-        name: Build and install on KinD
-        run: devbox run -- make dev.run-on-kind
+      - name: Run e2e tests
+        run: devbox run -- make e2e-test E2E_FOCUS='\[${{ matrix.PROVIDER }}\]'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-      - if: steps.check_e2e_tests_exist.outputs.files_exists == 'true'
-        name: Run e2e tests
-        run: devbox run -- make e2e-test
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - if: steps.check_e2e_tests_exist.outputs.files_exists == 'true' && always()
-        name: Cleanup KinD cluster
-        run: devbox run -- make kind.delete
+      - if: success() || failure() # always run even if the previous step fails
+        name: Publish e2e test report
+        uses: mikepenz/action-junit-report@v4
+        with:
+          report_paths: 'junit-e2e.xml'
+          check_name: 'e2e test report'
+          detailed_summary: true
 
   lint-go:
     runs-on: ubuntu-22.04

.gitignore

@@ -39,3 +39,5 @@ node_modules/
 
 /cluster.yaml
 /runtime-extension-components.yaml
+/_artifacts/
+test/e2e/config/cre-envsubst.yaml

.golangci.yml

@@ -7,6 +7,8 @@ run:
     # Although this is actually in api/external, api is a go module
     # and so need to specify the dir relative to the module root.
     - external
+  build-tags:
+    - e2e
 
 linters:
   disable-all: true
@@ -73,6 +75,11 @@ linters-settings:
     extra-rules: true
   lll:
     line-length: 120
+  stylecheck:
+    # https://staticcheck.io/docs/configuration/options/#dot_import_whitelist
+    dot-import-whitelist:
+      - github.com/onsi/ginkgo/v2
+      - github.com/onsi/gomega
 
 issues:
   exclude-rules:

.goreleaser.yml

@@ -30,18 +30,18 @@ release:
 before:
   hooks:
     - |
-      sh -ec 'if [ {{ .IsSnapshot }} != true ]; then
-        cat <<EOF > runtime-extension-components.yaml
+      sh -ec 'cat <<EOF > runtime-extension-components.yaml
       apiVersion: v1
       kind: Namespace
       metadata:
-        name: capi-runtime-extensions-system
-      $(helm template capi-runtime-extensions ./charts/capi-runtime-extensions \
-        --namespace capi-runtime-extensions-system \
+        name: {{.ProjectName}}-system
+      $(helm template {{.ProjectName}} ./charts/{{.ProjectName}} \
+        --namespace {{.ProjectName}}-system \
         --set-string image.tag={{ .Version }} \
+        {{ if .IsSnapshot }}--set-string image.repository=ko.local/{{ .ProjectName }}{{ end }} \
       )
-      EOF
-      fi'
+      EOF'
+    - sed -i 's/\${/$${/g' runtime-extension-components.yaml
 
 builds:
   - id: capi-runtime-extensions
@@ -72,7 +72,7 @@ builds:
           sh -ec 'if [ {{ .IsSnapshot }} == true ] && [ {{ .Runtime.Goarch }} == {{ .Arch }} ]; then
             env GOOS=linux GOARCH={{ .Arch }} \
                 SOURCE_DATE_EPOCH=$(date +%s) \
-                KO_DOCKER_REPO=ko.local/capi-runtime-extensions \
+                KO_DOCKER_REPO=ko.local/{{ .ProjectName }} \
                 ko build \
                   --bare \
                   -t {{ .Version }} \
@@ -100,7 +100,7 @@ kos:
       - -X 'k8s.io/component-base/version/verflag.programName={{ .ProjectName }}'
     labels:
       org.opencontainers.image.created: "{{ .CommitDate }}"
-      org.opencontainers.image.title: capi-runtime-extensions
+      org.opencontainers.image.title: "{{ .ProjectName }}"
       org.opencontainers.image.revision: "{{ .FullCommit }}"
       org.opencontainers.image.version: v{{trimprefix .Version "v"}}
       org.opencontainers.image.source: "{{ .GitURL }}"

capi-runtime-extensions.code-workspace

@@ -14,6 +14,6 @@
         }
     ],
     "settings": {
-        "go.buildTags": "e2e"
+        "go.buildTags": "e2e",
     }
 }

go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/d2iq-labs/capi-runtime-extensions/api v0.0.0-00010101000000-000000000000
 	github.com/d2iq-labs/capi-runtime-extensions/common v0.0.0-00010101000000-000000000000
 	github.com/go-logr/logr v1.4.1
+	github.com/onsi/ginkgo/v2 v2.15.0
 	github.com/onsi/gomega v1.31.1
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.8.4
@@ -26,71 +27,119 @@ require (
 	k8s.io/component-base v0.29.2
 	k8s.io/klog/v2 v2.120.1
 	k8s.io/kubelet v0.29.2
-	k8s.io/utils v0.0.0-20230726121419-3b25d923346b
+	k8s.io/utils v0.0.0-20240102154912-e7106e64919e
 	sigs.k8s.io/cluster-api v1.6.1
+	sigs.k8s.io/cluster-api-addon-provider-helm v0.1.1-alpha.1
+	sigs.k8s.io/cluster-api/test v1.6.1
 	sigs.k8s.io/controller-runtime v0.17.1
 )
 
 require (
+	github.com/BurntSushi/toml v1.3.2 // indirect
 	github.com/MakeNowJust/heredoc v1.0.0 // indirect
+	github.com/Masterminds/goutils v1.1.1 // indirect
+	github.com/Masterminds/semver/v3 v3.2.1 // indirect
+	github.com/Masterminds/sprig/v3 v3.2.3 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect
+	github.com/adrg/xdg v0.4.0 // indirect
+	github.com/alessio/shellescape v1.4.1 // indirect
 	github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect
 	github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect
 	github.com/avast/retry-go/v4 v4.5.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cloudflare/circl v1.3.7 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/distribution/reference v0.5.0 // indirect
+	github.com/docker/distribution v2.8.3+incompatible // indirect
+	github.com/docker/docker v24.0.7+incompatible // indirect
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/gobuffalo/flect v1.0.2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/cel-go v0.17.7 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/go-github/v53 v53.2.0 // indirect
+	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
+	github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 // indirect
 	github.com/google/uuid v1.3.1 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/huandu/xstrings v1.4.0 // indirect
 	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
+	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.0-rc5 // indirect
+	github.com/pelletier/go-toml v1.9.5 // indirect
+	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_golang v1.18.0 // indirect
 	github.com/prometheus/client_model v0.5.0 // indirect
 	github.com/prometheus/common v0.45.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
+	github.com/sagikazarmark/locafero v0.3.0 // indirect
+	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/shopspring/decimal v1.3.1 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spf13/afero v1.10.0 // indirect
+	github.com/spf13/cast v1.5.1 // indirect
 	github.com/spf13/cobra v1.8.0 // indirect
+	github.com/spf13/viper v1.17.0 // indirect
 	github.com/stoewer/go-strcase v1.2.0 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/valyala/fastjson v1.6.4 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/crypto v0.17.0 // indirect
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
+	golang.org/x/mod v0.14.0 // indirect
 	golang.org/x/net v0.19.0 // indirect
 	golang.org/x/oauth2 v0.14.0 // indirect
+	golang.org/x/sync v0.5.0 // indirect
 	golang.org/x/sys v0.16.0 // indirect
 	golang.org/x/term v0.15.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
+	golang.org/x/tools v0.16.1 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/cluster-bootstrap v0.29.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/kind v0.20.0 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect
 )