diff --git a/.cloudbuild.sh b/.cloudbuild.sh new file mode 120000 index 00000000..94226bc6 --- /dev/null +++ b/.cloudbuild.sh @@ -0,0 +1 @@ +release-tools/cloudbuild.sh \ No newline at end of file diff --git a/cloudbuild.yaml b/cloudbuild.yaml new file mode 120000 index 00000000..bee22c46 --- /dev/null +++ b/cloudbuild.yaml @@ -0,0 +1 @@ +release-tools/cloudbuild.yaml \ No newline at end of file diff --git a/release-tools/README.md b/release-tools/README.md index 6315a736..b394724c 100644 --- a/release-tools/README.md +++ b/release-tools/README.md @@ -21,7 +21,11 @@ The expected repository layout is: Dockerfile in the root when only building a single command - `Makefile` - includes `release-tools/build.make` and sets configuration variables - - `.travis.yml` - a symlink to `release-tools/.travis.yml` + - `.prow.sh` script which imports `release-tools/prow.sh` + and may contain further customization + - `.cloudbuild.sh` and `cloudbuild.yaml` as symlinks to + the corresponding files in `release-tools` or (if necessary) + as custom files To create a release, tag a certain revision with a name that starts with `v`, for example `v1.0.0`, then `make push` diff --git a/release-tools/SIDECAR_RELEASE_PROCESS.md b/release-tools/SIDECAR_RELEASE_PROCESS.md index 3aee7a81..f5ec71b9 100644 --- a/release-tools/SIDECAR_RELEASE_PROCESS.md +++ b/release-tools/SIDECAR_RELEASE_PROCESS.md @@ -104,3 +104,47 @@ naming convention `-on-`. CSI hostpath driver with the new sidecars in the [CSI repo](https://github.com/kubernetes-csi/csi-driver-host-path/tree/master/deploy) and [k/k in-tree](https://github.com/kubernetes/kubernetes/tree/master/test/e2e/testing-manifests/storage-csi/hostpath/hostpath) + +## Adding support for a new Kubernetes release + +1. Add the new release to `k8s_versions` in + https://github.com/kubernetes/test-infra/blob/090dec5dd535d5f61b7ba52e671a810f5fc13dfd/config/jobs/kubernetes-csi/gen-jobs.sh#L25 + to enable generating a job for it. Set `experimental_k8s_version` + in + https://github.com/kubernetes/test-infra/blob/090dec5dd535d5f61b7ba52e671a810f5fc13dfd/config/jobs/kubernetes-csi/gen-jobs.sh#L40 + to ensure that the new jobs aren't run for PRs unless explicitly + requested. Generate and submit the new jobs. +1. Create a test PR to try out the new job in some repo with `/test + pull-kubernetes-csi---on-kubernetes-` where x.y + matches the Kubernetes release. Alternatively, run .prow.sh in that + repo locally with `CSI_PROW_KUBERNETES_VERSION=x.y.z`. +1. Optional: update to a [new + release](https://github.com/kubernetes-sigs/kind/tags) of kind with + pre-built images for the new Kubernetes release. This is optional + if the current version of kind is able to build images for the new + Kubernetes release. However, jobs require less resources when they + don't need to build those images from the Kubernetes source code. + This change needs to be tried out in a PR against a component + first, then get submitted against csi-release-tools. +1. Optional: propagate the updated csi-release-tools to all components + with the script from + https://github.com/kubernetes-csi/csi-release-tools/issues/7#issuecomment-707025402 +1. Once it is likely to work in all components, unset + `experimental_k8s_version` and submit the updated jobs. +1. Once all sidecars for the new Kubernetes release are released, + either bump the version number of the images in the existing + [csi-driver-host-path + deployments](https://github.com/kubernetes-csi/csi-driver-host-path/tree/master/deploy) + and/or create a new deployment, depending on what Kubernetes + release an updated sidecar is compatible with. If no new deployment + is needed, then add a symlink to document that there intentionally + isn't a separate deployment. This symlink is not needed for Prow + testing because that will use "kubernetes-latest" as fallback. + Update that link when creating a new deployment. +1. Create a new csi-driver-host-path release. +1. Bump `CSI_PROW_DRIVER_VERSION` in prow.sh to that new release and + (eventually) roll that change out to all repos by updating + `release-tools` in them. This is used when testing manually. The + Prow jobs override that value, so also update + `hostpath_driver_version` in + https://github.com/kubernetes/test-infra/blob/91b04e6af3a40a9bcff25aa030850a4721e2dd2b/config/jobs/kubernetes-csi/gen-jobs.sh#L46-L47 diff --git a/release-tools/build.make b/release-tools/build.make index 9b8306c0..1faaf3b9 100644 --- a/release-tools/build.make +++ b/release-tools/build.make @@ -149,6 +149,7 @@ $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-% --platform=$$os/$$arch \ --file $$(eval echo \$${dockerfile_$$os}) \ --build-arg binary=./bin/$*$$suffix \ + --build-arg ARCH=$$arch \ --label revision=$(REV) \ .; \ done; \ diff --git a/release-tools/cloudbuild.yaml b/release-tools/cloudbuild.yaml index 7f0a4341..1e02ba6c 100644 --- a/release-tools/cloudbuild.yaml +++ b/release-tools/cloudbuild.yaml @@ -25,7 +25,7 @@ options: substitution_option: ALLOW_LOOSE steps: # The image must contain bash and curl. Ideally it should also contain - # the desired version of Go (currently defined in release-tools/travis.yml), + # the desired version of Go (currently defined in release-tools/prow.sh), # but that just speeds up the build and is not required. - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20200421-a2bf5f8' entrypoint: ./.cloudbuild.sh diff --git a/release-tools/go-get-kubernetes.sh b/release-tools/go-get-kubernetes.sh index 8c4e3024..cbbbb7c3 100755 --- a/release-tools/go-get-kubernetes.sh +++ b/release-tools/go-get-kubernetes.sh @@ -55,6 +55,12 @@ mods=$( (set -x; curl --silent --show-error --fail "https://raw.githubuserconten sed -n 's|.*k8s.io/\(.*\) => ./staging/src/k8s.io/.*|k8s.io/\1|p' ) || die "failed to determine Kubernetes staging modules" for mod in $mods; do + if ! (env GO111MODULE=on go mod graph) | grep "$mod@" > /dev/null; then + echo "Kubernetes module $mod is not used, skipping" + # Remove the module from go.mod "replace" that was added by an older version of this script. + (set -x; env GO111MODULE=on go mod edit "-dropreplace=$mod") || die "'go mod edit' failed" + continue + fi # The presence of a potentially incomplete go.mod file affects this command, # so move elsewhere. modinfo=$(set -x; cd /; env GO111MODULE=on go mod download -json "$mod@kubernetes-${k8s}") || diff --git a/release-tools/prow.sh b/release-tools/prow.sh index ec695ec6..aa471c0b 100755 --- a/release-tools/prow.sh +++ b/release-tools/prow.sh @@ -85,37 +85,12 @@ configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; windows amd64 .exe; linux ppc64 # which is disabled with GOFLAGS=-mod=vendor). configvar GOFLAGS_VENDOR "$( [ -d vendor ] && echo '-mod=vendor' )" "Go flags for using the vendor directory" -# Go versions can be specified separately for different tasks -# If the pre-installed Go is missing or a different -# version, the required version here will get installed -# from https://golang.org/dl/. -go_from_travis_yml () { - grep "^ *- go:" "${RELEASE_TOOLS_ROOT}/travis.yml" | sed -e 's/.*go: *//' -} -configvar CSI_PROW_GO_VERSION_BUILD "$(go_from_travis_yml)" "Go version for building the component" # depends on component's source code +configvar CSI_PROW_GO_VERSION_BUILD "1.16" "Go version for building the component" # depends on component's source code configvar CSI_PROW_GO_VERSION_E2E "" "override Go version for building the Kubernetes E2E test suite" # normally doesn't need to be set, see install_e2e configvar CSI_PROW_GO_VERSION_SANITY "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building the csi-sanity test suite" # depends on CSI_PROW_SANITY settings below configvar CSI_PROW_GO_VERSION_KIND "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building 'kind'" # depends on CSI_PROW_KIND_VERSION below configvar CSI_PROW_GO_VERSION_GINKGO "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building ginkgo" # depends on CSI_PROW_GINKGO_VERSION below -# kind version to use. If the pre-installed version is different, -# the desired version is downloaded from https://github.com/kubernetes-sigs/kind/releases -# (if available), otherwise it is built from source. -configvar CSI_PROW_KIND_VERSION "v0.9.0" "kind" - -# kind images to use. Must match the kind version. -# The release notes of each kind release list the supported images. -configvar CSI_PROW_KIND_IMAGES "kindest/node:v1.19.1@sha256:98cf5288864662e37115e362b23e4369c8c4a408f99cbc06e58ac30ddc721600 -kindest/node:v1.18.8@sha256:f4bcc97a0ad6e7abaf3f643d890add7efe6ee4ab90baeb374b4f41a4c95567eb -kindest/node:v1.17.11@sha256:5240a7a2c34bf241afb54ac05669f8a46661912eab05705d660971eeb12f6555 -kindest/node:v1.16.15@sha256:a89c771f7de234e6547d43695c7ab047809ffc71a0c3b65aa54eda051c45ed20 -kindest/node:v1.15.12@sha256:d9b939055c1e852fe3d86955ee24976cab46cba518abcb8b13ba70917e6547a6 -kindest/node:v1.14.10@sha256:ce4355398a704fca68006f8a29f37aafb49f8fc2f64ede3ccd0d9198da910146 -kindest/node:v1.13.12@sha256:1c1a48c2bfcbae4d5f4fa4310b5ed10756facad0b7a2ca93c7a4b5bae5db29f5" "kind images" - -# Use kind node-image --type=bazel by default, but allow to disable that. -configvar CSI_PROW_USE_BAZEL true "use Bazel during 'kind node-image' invocation" - # ginkgo test runner version to use. If the pre-installed version is # different, the desired version is built from source. configvar CSI_PROW_GINKGO_VERSION v1.7.0 "Ginkgo" @@ -152,10 +127,37 @@ configvar CSI_PROW_KUBERNETES_VERSION 1.17.0 "Kubernetes" # when a Prow job just defines the Kubernetes version. csi_prow_kubernetes_version_suffix="$(echo "${CSI_PROW_KUBERNETES_VERSION}" | tr . _ | tr '[:lower:]' '[:upper:]' | sed -e 's/^RELEASE-//' -e 's/\([0-9]*\)_\([0-9]*\).*/\1_\2/')" -# Work directory. It has to allow running executables, therefore /tmp -# is avoided. Cleaning up after the script is intentionally left to -# the caller. -configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csiprow.XXXXXXXXXX")" "work directory" +# Only the latest KinD is (eventually) guaranteed to work with the +# latest Kubernetes. For example, KinD 0.10.0 failed with Kubernetes +# 1.21.0-beta1. Therefore the default version of KinD is "main" +# for that, otherwise the latest stable release for which we then +# list the officially supported images below. +kind_version_default () { + case "${CSI_PROW_KUBERNETES_VERSION}" in + latest|master) + echo main;; + *) + echo v0.10.0;; + esac +} + +# kind version to use. If the pre-installed version is different, +# the desired version is downloaded from https://github.com/kubernetes-sigs/kind/releases +# (if available), otherwise it is built from source. +configvar CSI_PROW_KIND_VERSION "$(kind_version_default)" "kind" + +# kind images to use. Must match the kind version. +# The release notes of each kind release list the supported images. +configvar CSI_PROW_KIND_IMAGES "kindest/node:v1.20.2@sha256:8f7ea6e7642c0da54f04a7ee10431549c0257315b3a634f6ef2fecaaedb19bab +kindest/node:v1.19.7@sha256:a70639454e97a4b733f9d9b67e12c01f6b0297449d5b9cbbef87473458e26dca +kindest/node:v1.18.15@sha256:5c1b980c4d0e0e8e7eb9f36f7df525d079a96169c8a8f20d8bd108c0d0889cc4 +kindest/node:v1.17.17@sha256:7b6369d27eee99c7a85c48ffd60e11412dc3f373658bc59b7f4d530b7056823e +kindest/node:v1.16.15@sha256:c10a63a5bda231c0a379bf91aebf8ad3c79146daca59db816fb963f731852a99 +kindest/node:v1.15.12@sha256:67181f94f0b3072fb56509107b380e38c55e23bf60e6f052fbd8052d26052fb5 +kindest/node:v1.14.10@sha256:3fbed72bcac108055e46e7b4091eb6858ad628ec51bf693c21f5ec34578f6180" "kind images" + +# Use kind node-image --type=bazel by default, but allow to disable that. +configvar CSI_PROW_USE_BAZEL true "use Bazel during 'kind node-image' invocation" # By default, this script tests sidecars with the CSI hostpath driver, # using the install_csi_driver function. That function depends on @@ -354,10 +356,23 @@ configvar CSI_SNAPSHOTTER_VERSION "$(default_csi_snapshotter_version)" "external # to all the K8s versions we test against configvar CSI_PROW_E2E_SKIP 'Disruptive|different\s+node' "tests that need to be skipped" -# This is the directory for additional result files. Usually set by Prow, but -# if not (for example, when invoking manually) it defaults to the work directory. -configvar ARTIFACTS "${CSI_PROW_WORK}/artifacts" "artifacts" -mkdir -p "${ARTIFACTS}" +# This creates directories that are required for testing. +ensure_paths () { + # Work directory. It has to allow running executables, therefore /tmp + # is avoided. Cleaning up after the script is intentionally left to + # the caller. + configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csiprow.XXXXXXXXXX")" "work directory" + + # This is the directory for additional result files. Usually set by Prow, but + # if not (for example, when invoking manually) it defaults to the work directory. + configvar ARTIFACTS "${CSI_PROW_WORK}/artifacts" "artifacts" + mkdir -p "${ARTIFACTS}" + + # For additional tools. + CSI_PROW_BIN="${CSI_PROW_WORK}/bin" + mkdir -p "${CSI_PROW_BIN}" + PATH="${CSI_PROW_BIN}:$PATH" +} run () { echo "$(date) $(go version | sed -e 's/.*version \(go[^ ]*\).*/\1/') $(if [ "$(pwd)" != "${REPO_DIR}" ]; then pwd; fi)\$" "$@" >&2 @@ -377,11 +392,6 @@ die () { exit 1 } -# For additional tools. -CSI_PROW_BIN="${CSI_PROW_WORK}/bin" -mkdir -p "${CSI_PROW_BIN}" -PATH="${CSI_PROW_BIN}:$PATH" - # Ensure that PATH has the desired version of the Go tools, then run command given as argument. # Empty parameter uses the already installed Go. In Prow, that version is kept up-to-date by # bumping the container image regularly. @@ -410,7 +420,7 @@ install_kind () { chmod u+x "${CSI_PROW_WORK}/bin/kind" else git_checkout https://github.com/kubernetes-sigs/kind "${GOPATH}/src/sigs.k8s.io/kind" "${CSI_PROW_KIND_VERSION}" --depth=1 && - (cd "${GOPATH}/src/sigs.k8s.io/kind" && make install INSTALL_DIR="${CSI_PROW_WORK}/bin") + (cd "${GOPATH}/src/sigs.k8s.io/kind" && run_with_go "$CSI_PROW_GO_VERSION_KIND" make install INSTALL_DIR="${CSI_PROW_WORK}/bin") fi } @@ -576,7 +586,8 @@ start_cluster () { go_version="$(go_version_for_kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$version")" || die "cannot proceed without knowing Go version for Kubernetes" # Changing into the Kubernetes source code directory is a workaround for https://github.com/kubernetes-sigs/kind/issues/1910 - (cd "${CSI_PROW_WORK}/src/kubernetes" && run_with_go "$go_version" kind build node-image --image csiprow/node:latest --type="$type" --kube-root "${CSI_PROW_WORK}/src/kubernetes") || die "'kind build node-image' failed" + # shellcheck disable=SC2046 + (cd "${CSI_PROW_WORK}/src/kubernetes" && run_with_go "$go_version" kind build node-image --image csiprow/node:latest $(if [ "$CSI_PROW_KIND_VERSION" != "main" ]; then echo --type="$type"; fi) --kube-root "${CSI_PROW_WORK}/src/kubernetes") || die "'kind build node-image' failed" csi_prow_kind_have_kubernetes=true fi image="csiprow/node:latest" @@ -1090,6 +1101,9 @@ main () { local images ret ret=0 + # Set up work directory. + ensure_paths + images= if ${CSI_PROW_BUILD_JOB}; then # A successful build is required for testing. @@ -1251,6 +1265,9 @@ gcr_cloud_build () { # Required for "docker buildx build --push". gcloud auth configure-docker + # Might not be needed here, but call it just in case. + ensure_paths + if find . -name Dockerfile | grep -v ^./vendor | xargs --no-run-if-empty cat | grep -q ^RUN; then # Needed for "RUN" steps on non-linux/amd64 platforms. # See https://github.com/multiarch/qemu-user-static#getting-started diff --git a/release-tools/travis.yml b/release-tools/travis.yml deleted file mode 100644 index 1ab13aef..00000000 --- a/release-tools/travis.yml +++ /dev/null @@ -1,21 +0,0 @@ -language: go -sudo: required -services: - - docker -git: - depth: false -matrix: - include: - - go: 1.15 -before_script: -- mkdir -p bin -- wget https://github.com/golang/dep/releases/download/v0.5.1/dep-linux-amd64 -O bin/dep -- chmod u+x bin/dep -- export PATH=$PWD/bin:$PATH -script: -- make -k all test GOFLAGS_VENDOR=$( [ -d vendor ] && echo '-mod=vendor' ) -after_success: - - if [ "${TRAVIS_PULL_REQUEST}" == "false" ]; then - docker login -u "${DOCKER_USERNAME}" -p "${DOCKER_PASSWORD}" quay.io; - make push GOFLAGS_VENDOR=$( [ -d vendor ] && echo '-mod=vendor' ); - fi diff --git a/release-tools/verify-go-version.sh b/release-tools/verify-go-version.sh index f242e769..c235e74f 100755 --- a/release-tools/verify-go-version.sh +++ b/release-tools/verify-go-version.sh @@ -29,8 +29,9 @@ die () { version=$("$GO" version) || die "determining version of $GO failed" # shellcheck disable=SC2001 majorminor=$(echo "$version" | sed -e 's/.*go\([0-9]*\)\.\([0-9]*\).*/\1.\2/') -# shellcheck disable=SC2001 -expected=$(grep "^ *- go:" "release-tools/travis.yml" | sed -e 's/.*go: *\([0-9]*\)\.\([0-9]*\).*/\1.\2/') +# SC1091: Not following: release-tools/prow.sh was not specified as input (see shellcheck -x). +# shellcheck disable=SC1091 +expected=$(. release-tools/prow.sh >/dev/null && echo "$CSI_PROW_GO_VERSION_BUILD") if [ "$majorminor" != "$expected" ]; then cat >&2 <