kubernetes-csi · k8s-ci-robot · Apr 29, 2021 · Apr 26, 2021 · Apr 27, 2021 · Apr 27, 2021
diff --git a/.cloudbuild.sh b/.cloudbuild.sh
@@ -0,0 +1 @@
+release-tools/cloudbuild.sh
diff --git a/cloudbuild.yaml b/cloudbuild.yaml
@@ -0,0 +1 @@
+release-tools/cloudbuild.yaml
diff --git a/release-tools/README.md b/release-tools/README.md
@@ -21,7 +21,11 @@ The expected repository layout is:
    Dockerfile in the root when only building a single command
  - `Makefile` - includes `release-tools/build.make` and sets
    configuration variables
- - `.travis.yml` - a symlink to `release-tools/.travis.yml`
+ - `.prow.sh` script which imports `release-tools/prow.sh`
+   and may contain further customization
+ - `.cloudbuild.sh` and `cloudbuild.yaml` as symlinks to
+   the corresponding files in `release-tools` or (if necessary)
+   as custom files
 
 To create a release, tag a certain revision with a name that
 starts with `v`, for example `v1.0.0`, then `make push`

diff --git a/release-tools/SIDECAR_RELEASE_PROCESS.md b/release-tools/SIDECAR_RELEASE_PROCESS.md
@@ -104,3 +104,47 @@ naming convention `<hostpath-deployment-version>-on-<kubernetes-version>`.
    CSI hostpath driver with the new sidecars in the [CSI repo](https://github.com/kubernetes-csi/csi-driver-host-path/tree/master/deploy)
    and [k/k
    in-tree](https://github.com/kubernetes/kubernetes/tree/master/test/e2e/testing-manifests/storage-csi/hostpath/hostpath)
+
+## Adding support for a new Kubernetes release
+
+1. Add the new release to `k8s_versions` in
+   https://github.com/kubernetes/test-infra/blob/090dec5dd535d5f61b7ba52e671a810f5fc13dfd/config/jobs/kubernetes-csi/gen-jobs.sh#L25
+   to enable generating a job for it. Set `experimental_k8s_version`
+   in
+   https://github.com/kubernetes/test-infra/blob/090dec5dd535d5f61b7ba52e671a810f5fc13dfd/config/jobs/kubernetes-csi/gen-jobs.sh#L40
+   to ensure that the new jobs aren't run for PRs unless explicitly
+   requested. Generate and submit the new jobs.
+1. Create a test PR to try out the new job in some repo with `/test
+   pull-kubernetes-csi-<repo>-<x.y>-on-kubernetes-<x.y>` where x.y
+   matches the Kubernetes release. Alternatively, run .prow.sh in that
+   repo locally with `CSI_PROW_KUBERNETES_VERSION=x.y.z`.
+1. Optional: update to a [new
+   release](https://github.com/kubernetes-sigs/kind/tags) of kind with
+   pre-built images for the new Kubernetes release. This is optional
+   if the current version of kind is able to build images for the new
+   Kubernetes release. However, jobs require less resources when they
+   don't need to build those images from the Kubernetes source code.
+   This change needs to be tried out in a PR against a component
+   first, then get submitted against csi-release-tools.
+1. Optional: propagate the updated csi-release-tools to all components
+   with the script from
+   https://github.com/kubernetes-csi/csi-release-tools/issues/7#issuecomment-707025402
+1. Once it is likely to work in all components, unset
+   `experimental_k8s_version` and submit the updated jobs.
+1. Once all sidecars for the new Kubernetes release are released,
+   either bump the version number of the images in the existing
+   [csi-driver-host-path
+   deployments](https://github.com/kubernetes-csi/csi-driver-host-path/tree/master/deploy)
+   and/or create a new deployment, depending on what Kubernetes
+   release an updated sidecar is compatible with. If no new deployment
+   is needed, then add a symlink to document that there intentionally
+   isn't a separate deployment. This symlink is not needed for Prow
+   testing because that will use "kubernetes-latest" as fallback.
+   Update that link when creating a new deployment.
+1. Create a new csi-driver-host-path release.
+1. Bump `CSI_PROW_DRIVER_VERSION` in prow.sh to that new release and
+   (eventually) roll that change out to all repos by updating
+   `release-tools` in them. This is used when testing manually. The
+   Prow jobs override that value, so also update
+   `hostpath_driver_version` in
+   https://github.com/kubernetes/test-infra/blob/91b04e6af3a40a9bcff25aa030850a4721e2dd2b/config/jobs/kubernetes-csi/gen-jobs.sh#L46-L47
diff --git a/release-tools/build.make b/release-tools/build.make
@@ -149,6 +149,7 @@ $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-%
 				--platform=$$os/$$arch \
 				--file $$(eval echo \$${dockerfile_$$os}) \
 				--build-arg binary=./bin/$*$$suffix \
+				--build-arg ARCH=$$arch \
 				--label revision=$(REV) \
 				.; \
 		done; \

diff --git a/release-tools/cloudbuild.yaml b/release-tools/cloudbuild.yaml
@@ -25,7 +25,7 @@ options:
   substitution_option: ALLOW_LOOSE
 steps:
   # The image must contain bash and curl. Ideally it should also contain
-  # the desired version of Go (currently defined in release-tools/travis.yml),
+  # the desired version of Go (currently defined in release-tools/prow.sh),
   # but that just speeds up the build and is not required.
   - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20200421-a2bf5f8'
     entrypoint: ./.cloudbuild.sh

diff --git a/release-tools/go-get-kubernetes.sh b/release-tools/go-get-kubernetes.sh
@@ -55,6 +55,12 @@ mods=$( (set -x; curl --silent --show-error --fail "https://raw.githubuserconten
           sed -n 's|.*k8s.io/\(.*\) => ./staging/src/k8s.io/.*|k8s.io/\1|p'
    ) || die "failed to determine Kubernetes staging modules"
 for mod in $mods; do
+    if ! (env GO111MODULE=on go mod graph) | grep "$mod@" > /dev/null; then
+        echo "Kubernetes module $mod is not used, skipping"
+        # Remove the module from go.mod "replace" that was added by an older version of this script.
+        (set -x;  env GO111MODULE=on go mod edit "-dropreplace=$mod") || die "'go mod edit' failed"
+        continue
+    fi
     # The presence of a potentially incomplete go.mod file affects this command,
     # so move elsewhere.
     modinfo=$(set -x; cd /; env GO111MODULE=on go mod download -json "$mod@kubernetes-${k8s}") ||

diff --git a/release-tools/prow.sh b/release-tools/prow.sh
@@ -85,37 +85,12 @@ configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; windows amd64 .exe; linux ppc64
 # which is disabled with GOFLAGS=-mod=vendor).
 configvar GOFLAGS_VENDOR "$( [ -d vendor ] && echo '-mod=vendor' )" "Go flags for using the vendor directory"
 
-# Go versions can be specified separately for different tasks
-# If the pre-installed Go is missing or a different
-# version, the required version here will get installed
-# from https://golang.org/dl/.
-go_from_travis_yml () {
-    grep "^ *- go:" "${RELEASE_TOOLS_ROOT}/travis.yml" | sed -e 's/.*go: *//'
-}
-configvar CSI_PROW_GO_VERSION_BUILD "$(go_from_travis_yml)" "Go version for building the component" # depends on component's source code
+configvar CSI_PROW_GO_VERSION_BUILD "1.16" "Go version for building the component" # depends on component's source code
 configvar CSI_PROW_GO_VERSION_E2E "" "override Go version for building the Kubernetes E2E test suite" # normally doesn't need to be set, see install_e2e
 configvar CSI_PROW_GO_VERSION_SANITY "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building the csi-sanity test suite" # depends on CSI_PROW_SANITY settings below
 configvar CSI_PROW_GO_VERSION_KIND "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building 'kind'" # depends on CSI_PROW_KIND_VERSION below
 configvar CSI_PROW_GO_VERSION_GINKGO "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building ginkgo" # depends on CSI_PROW_GINKGO_VERSION below
 
-# kind version to use. If the pre-installed version is different,
-# the desired version is downloaded from https://github.com/kubernetes-sigs/kind/releases
-# (if available), otherwise it is built from source.
-configvar CSI_PROW_KIND_VERSION "v0.9.0" "kind"
-
-# kind images to use. Must match the kind version.
-# The release notes of each kind release list the supported images.
-configvar CSI_PROW_KIND_IMAGES "kindest/node:v1.19.1@sha256:98cf5288864662e37115e362b23e4369c8c4a408f99cbc06e58ac30ddc721600
-kindest/node:v1.18.8@sha256:f4bcc97a0ad6e7abaf3f643d890add7efe6ee4ab90baeb374b4f41a4c95567eb
-kindest/node:v1.17.11@sha256:5240a7a2c34bf241afb54ac05669f8a46661912eab05705d660971eeb12f6555
-kindest/node:v1.16.15@sha256:a89c771f7de234e6547d43695c7ab047809ffc71a0c3b65aa54eda051c45ed20
-kindest/node:v1.15.12@sha256:d9b939055c1e852fe3d86955ee24976cab46cba518abcb8b13ba70917e6547a6
-kindest/node:v1.14.10@sha256:ce4355398a704fca68006f8a29f37aafb49f8fc2f64ede3ccd0d9198da910146
-kindest/node:v1.13.12@sha256:1c1a48c2bfcbae4d5f4fa4310b5ed10756facad0b7a2ca93c7a4b5bae5db29f5" "kind images"
-
-# Use kind node-image --type=bazel by default, but allow to disable that.
-configvar CSI_PROW_USE_BAZEL true "use Bazel during 'kind node-image' invocation"
-
 # ginkgo test runner version to use. If the pre-installed version is
 # different, the desired version is built from source.
 configvar CSI_PROW_GINKGO_VERSION v1.7.0 "Ginkgo"
@@ -152,10 +127,37 @@ configvar CSI_PROW_KUBERNETES_VERSION 1.17.0 "Kubernetes"
 # when a Prow job just defines the Kubernetes version.
 csi_prow_kubernetes_version_suffix="$(echo "${CSI_PROW_KUBERNETES_VERSION}" | tr . _ | tr '[:lower:]' '[:upper:]' | sed -e 's/^RELEASE-//' -e 's/\([0-9]*\)_\([0-9]*\).*/\1_\2/')"
 
-# Work directory. It has to allow running executables, therefore /tmp
-# is avoided. Cleaning up after the script is intentionally left to
-# the caller.
-configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csiprow.XXXXXXXXXX")" "work directory"
+# Only the latest KinD is (eventually) guaranteed to work with the
+# latest Kubernetes. For example, KinD 0.10.0 failed with Kubernetes
+# 1.21.0-beta1.  Therefore the default version of KinD is "main"
+# for that, otherwise the latest stable release for which we then
+# list the officially supported images below.
+kind_version_default () {
+    case "${CSI_PROW_KUBERNETES_VERSION}" in
+        latest|master)
+            echo main;;
+        *)
+            echo v0.10.0;;
+    esac
+}
+
+# kind version to use. If the pre-installed version is different,
+# the desired version is downloaded from https://github.com/kubernetes-sigs/kind/releases
+# (if available), otherwise it is built from source.
+configvar CSI_PROW_KIND_VERSION "$(kind_version_default)" "kind"
+
+# kind images to use. Must match the kind version.
+# The release notes of each kind release list the supported images.
+configvar CSI_PROW_KIND_IMAGES "kindest/node:v1.20.2@sha256:8f7ea6e7642c0da54f04a7ee10431549c0257315b3a634f6ef2fecaaedb19bab
+kindest/node:v1.19.7@sha256:a70639454e97a4b733f9d9b67e12c01f6b0297449d5b9cbbef87473458e26dca
+kindest/node:v1.18.15@sha256:5c1b980c4d0e0e8e7eb9f36f7df525d079a96169c8a8f20d8bd108c0d0889cc4
+kindest/node:v1.17.17@sha256:7b6369d27eee99c7a85c48ffd60e11412dc3f373658bc59b7f4d530b7056823e
+kindest/node:v1.16.15@sha256:c10a63a5bda231c0a379bf91aebf8ad3c79146daca59db816fb963f731852a99
+kindest/node:v1.15.12@sha256:67181f94f0b3072fb56509107b380e38c55e23bf60e6f052fbd8052d26052fb5
+kindest/node:v1.14.10@sha256:3fbed72bcac108055e46e7b4091eb6858ad628ec51bf693c21f5ec34578f6180" "kind images"
+
+# Use kind node-image --type=bazel by default, but allow to disable that.
+configvar CSI_PROW_USE_BAZEL true "use Bazel during 'kind node-image' invocation"
 
 # By default, this script tests sidecars with the CSI hostpath driver,
 # using the install_csi_driver function. That function depends on
@@ -354,10 +356,23 @@ configvar CSI_SNAPSHOTTER_VERSION "$(default_csi_snapshotter_version)" "external
 # to all the K8s versions we test against
 configvar CSI_PROW_E2E_SKIP 'Disruptive|different\s+node' "tests that need to be skipped"
 
-# This is the directory for additional result files. Usually set by Prow, but
-# if not (for example, when invoking manually) it defaults to the work directory.
-configvar ARTIFACTS "${CSI_PROW_WORK}/artifacts" "artifacts"
-mkdir -p "${ARTIFACTS}"
+# This creates directories that are required for testing.
+ensure_paths () {
+    # Work directory. It has to allow running executables, therefore /tmp
+    # is avoided. Cleaning up after the script is intentionally left to
+    # the caller.
+    configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csiprow.XXXXXXXXXX")" "work directory"
+
+    # This is the directory for additional result files. Usually set by Prow, but
+    # if not (for example, when invoking manually) it defaults to the work directory.
+    configvar ARTIFACTS "${CSI_PROW_WORK}/artifacts" "artifacts"
+    mkdir -p "${ARTIFACTS}"
+
+    # For additional tools.
+    CSI_PROW_BIN="${CSI_PROW_WORK}/bin"
+    mkdir -p "${CSI_PROW_BIN}"
+    PATH="${CSI_PROW_BIN}:$PATH"
+}
 
 run () {
     echo "$(date) $(go version | sed -e 's/.*version \(go[^ ]*\).*/\1/') $(if [ "$(pwd)" != "${REPO_DIR}" ]; then pwd; fi)\$" "$@" >&2
@@ -377,11 +392,6 @@ die () {
     exit 1
 }
 
-# For additional tools.
-CSI_PROW_BIN="${CSI_PROW_WORK}/bin"
-mkdir -p "${CSI_PROW_BIN}"
-PATH="${CSI_PROW_BIN}:$PATH"
-
 # Ensure that PATH has the desired version of the Go tools, then run command given as argument.
 # Empty parameter uses the already installed Go. In Prow, that version is kept up-to-date by
 # bumping the container image regularly.
@@ -410,7 +420,7 @@ install_kind () {
         chmod u+x "${CSI_PROW_WORK}/bin/kind"
     else
         git_checkout https://github.com/kubernetes-sigs/kind "${GOPATH}/src/sigs.k8s.io/kind" "${CSI_PROW_KIND_VERSION}" --depth=1 &&
-        (cd "${GOPATH}/src/sigs.k8s.io/kind" && make install INSTALL_DIR="${CSI_PROW_WORK}/bin")
+        (cd "${GOPATH}/src/sigs.k8s.io/kind" && run_with_go "$CSI_PROW_GO_VERSION_KIND" make install INSTALL_DIR="${CSI_PROW_WORK}/bin")
     fi
 }
 
@@ -576,7 +586,8 @@ start_cluster () {
 
             go_version="$(go_version_for_kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$version")" || die "cannot proceed without knowing Go version for Kubernetes"
             # Changing into the Kubernetes source code directory is a workaround for https://github.com/kubernetes-sigs/kind/issues/1910
-            (cd "${CSI_PROW_WORK}/src/kubernetes" && run_with_go "$go_version" kind build node-image --image csiprow/node:latest --type="$type" --kube-root "${CSI_PROW_WORK}/src/kubernetes") || die "'kind build node-image' failed"
+            # shellcheck disable=SC2046
+            (cd "${CSI_PROW_WORK}/src/kubernetes" && run_with_go "$go_version" kind build node-image --image csiprow/node:latest $(if [ "$CSI_PROW_KIND_VERSION" != "main" ]; then echo --type="$type"; fi) --kube-root "${CSI_PROW_WORK}/src/kubernetes") || die "'kind build node-image' failed"
             csi_prow_kind_have_kubernetes=true
         fi
         image="csiprow/node:latest"
@@ -1090,6 +1101,9 @@ main () {
     local images ret
     ret=0
 
+    # Set up work directory.
+    ensure_paths
+
     images=
     if ${CSI_PROW_BUILD_JOB}; then
         # A successful build is required for testing.
@@ -1251,6 +1265,9 @@ gcr_cloud_build () {
     # Required for "docker buildx build --push".
     gcloud auth configure-docker
 
+    # Might not be needed here, but call it just in case.
+    ensure_paths
+
     if find . -name Dockerfile | grep -v ^./vendor | xargs --no-run-if-empty cat | grep -q ^RUN; then
         # Needed for "RUN" steps on non-linux/amd64 platforms.
         # See https://github.com/multiarch/qemu-user-static#getting-started

diff --git a/release-tools/travis.yml b/release-tools/travis.yml
diff --git a/release-tools/verify-go-version.sh b/release-tools/verify-go-version.sh
@@ -29,8 +29,9 @@ die () {
 version=$("$GO" version) || die "determining version of $GO failed"
 # shellcheck disable=SC2001
 majorminor=$(echo "$version" | sed -e 's/.*go\([0-9]*\)\.\([0-9]*\).*/\1.\2/')
-# shellcheck disable=SC2001
-expected=$(grep "^ *- go:" "release-tools/travis.yml" | sed -e 's/.*go: *\([0-9]*\)\.\([0-9]*\).*/\1.\2/')
+# SC1091: Not following: release-tools/prow.sh was not specified as input (see shellcheck -x).
+# shellcheck disable=SC1091
+expected=$(. release-tools/prow.sh >/dev/null && echo "$CSI_PROW_GO_VERSION_BUILD")
 
 if [ "$majorminor" != "$expected" ]; then
     cat >&2 <<EOF