From 26f1f1fa33537ad894df4b7e0d6411a0f9e386ef Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 2 Aug 2021 12:09:06 +0200 Subject: [PATCH 1/2] Squashed 'release-tools/' changes from 60e1cd3..c0a4fb1 https://github.com/kubernetes-csi/csi-release-tools/commit/c0a4fb1 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/164 from anubha-v-ardhan/patch-1 https://github.com/kubernetes-csi/csi-release-tools/commit/9c6a6c0 Master to main cleanup https://github.com/kubernetes-csi/csi-release-tools/commit/682c686 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/162 from pohly/pod-name-via-shell-command https://github.com/kubernetes-csi/csi-release-tools/commit/36a29f5 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/163 from pohly/remove-bazel https://github.com/kubernetes-csi/csi-release-tools/commit/68e43ca prow.sh: remove Bazel build support https://github.com/kubernetes-csi/csi-release-tools/commit/c5f59c5 prow.sh: allow shell commands in CSI_PROW_SANITY_POD https://github.com/kubernetes-csi/csi-release-tools/commit/71c810a Merge https://github.com/kubernetes-csi/csi-release-tools/pull/161 from pohly/mock-test-fixes https://github.com/kubernetes-csi/csi-release-tools/commit/9e438f8 prow.sh: fix mock testing https://github.com/kubernetes-csi/csi-release-tools/commit/d7146c7 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/160 from pohly/kind-update https://github.com/kubernetes-csi/csi-release-tools/commit/4b6aa60 prow.sh: update to KinD v0.11.0 https://github.com/kubernetes-csi/csi-release-tools/commit/7cdc76f Merge https://github.com/kubernetes-csi/csi-release-tools/pull/159 from pohly/fix-deployment-selection https://github.com/kubernetes-csi/csi-release-tools/commit/ef8bd33 prow.sh: more flexible CSI_PROW_DEPLOYMENT, part II https://github.com/kubernetes-csi/csi-release-tools/commit/204bc89 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/158 from pohly/fix-deployment-selection https://github.com/kubernetes-csi/csi-release-tools/commit/61538bb prow.sh: more flexible CSI_PROW_DEPLOYMENT https://github.com/kubernetes-csi/csi-release-tools/commit/2b0e6db Merge https://github.com/kubernetes-csi/csi-release-tools/pull/157 from humblec/csi-release https://github.com/kubernetes-csi/csi-release-tools/commit/a2fcd6d Adding myself to csi reviewers group https://github.com/kubernetes-csi/csi-release-tools/commit/f325590 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/149 from pohly/cluster-logs https://github.com/kubernetes-csi/csi-release-tools/commit/4b03b30 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/155 from pohly/owners https://github.com/kubernetes-csi/csi-release-tools/commit/a6453c8 owners: introduce aliases https://github.com/kubernetes-csi/csi-release-tools/commit/ad83def Merge https://github.com/kubernetes-csi/csi-release-tools/pull/153 from pohly/fix-image-builds https://github.com/kubernetes-csi/csi-release-tools/commit/5561780 build.make: fix image publishng https://github.com/kubernetes-csi/csi-release-tools/commit/29bd39b Merge https://github.com/kubernetes-csi/csi-release-tools/pull/152 from pohly/bump-csi-test https://github.com/kubernetes-csi/csi-release-tools/commit/bc42793 prow.sh: use csi-test v4.2.0 https://github.com/kubernetes-csi/csi-release-tools/commit/b546baa Merge https://github.com/kubernetes-csi/csi-release-tools/pull/150 from mauriciopoppe/windows-multiarch-args https://github.com/kubernetes-csi/csi-release-tools/commit/bfbb6f3 add parameter base_image and addon_image to BUILD_PARAMETERS https://github.com/kubernetes-csi/csi-release-tools/commit/2d61d3b Merge https://github.com/kubernetes-csi/csi-release-tools/pull/151 from humblec/cm https://github.com/kubernetes-csi/csi-release-tools/commit/48e71f0 Replace `which` command ( non standard) with `command -v` builtin https://github.com/kubernetes-csi/csi-release-tools/commit/feb20e2 prow.sh: collect cluster logs https://github.com/kubernetes-csi/csi-release-tools/commit/7b96bea Merge https://github.com/kubernetes-csi/csi-release-tools/pull/148 from dobsonj/add-checkpathcmd-to-prow https://github.com/kubernetes-csi/csi-release-tools/commit/2d2e03b prow.sh: enable -csi.checkpathcmd option in csi-sanity https://github.com/kubernetes-csi/csi-release-tools/commit/09d4151 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/147 from pohly/mock-testing https://github.com/kubernetes-csi/csi-release-tools/commit/74cfbc9 prow.sh: support mock tests https://github.com/kubernetes-csi/csi-release-tools/commit/4a3f110 prow.sh: remove obsolete test suppression https://github.com/kubernetes-csi/csi-release-tools/commit/6616a6b Merge https://github.com/kubernetes-csi/csi-release-tools/pull/146 from pohly/kubernetes-1.21 https://github.com/kubernetes-csi/csi-release-tools/commit/510fb0f prow.sh: support Kubernetes 1.21 https://github.com/kubernetes-csi/csi-release-tools/commit/c63c61b prow.sh: add CSI_PROW_DEPLOYMENT_SUFFIX https://github.com/kubernetes-csi/csi-release-tools/commit/51ac11c Merge https://github.com/kubernetes-csi/csi-release-tools/pull/144 from pohly/pull-jobs https://github.com/kubernetes-csi/csi-release-tools/commit/dd54c92 pull-test.sh: test importing csi-release-tools into other repo https://github.com/kubernetes-csi/csi-release-tools/commit/7d2643a Merge https://github.com/kubernetes-csi/csi-release-tools/pull/143 from pohly/path-setup https://github.com/kubernetes-csi/csi-release-tools/commit/6880b0c prow.sh: avoid creating paths unless really running tests https://github.com/kubernetes-csi/csi-release-tools/commit/bc0504a Merge https://github.com/kubernetes-csi/csi-release-tools/pull/140 from jsafrane/remove-unused-k8s-libs https://github.com/kubernetes-csi/csi-release-tools/commit/5b1de1a go-get-kubernetes.sh: remove unused k8s libs https://github.com/kubernetes-csi/csi-release-tools/commit/49b4269 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/120 from pohly/add-kubernetes-release https://github.com/kubernetes-csi/csi-release-tools/commit/a1e1127 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/139 from pohly/kind-for-kubernetes-latest https://github.com/kubernetes-csi/csi-release-tools/commit/1c0fb09 prow.sh: use KinD main for latest Kubernetes https://github.com/kubernetes-csi/csi-release-tools/commit/1d77cfc Merge https://github.com/kubernetes-csi/csi-release-tools/pull/138 from pohly/kind-update-0.10 https://github.com/kubernetes-csi/csi-release-tools/commit/bff2fb7 prow.sh: KinD 0.10.0 https://github.com/kubernetes-csi/csi-release-tools/commit/95eac33 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/137 from pohly/fix-go-version-check https://github.com/kubernetes-csi/csi-release-tools/commit/437e431 verify-go-version.sh: fix check after removal of travis.yml https://github.com/kubernetes-csi/csi-release-tools/commit/1748b16 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/136 from pohly/go-1.16 https://github.com/kubernetes-csi/csi-release-tools/commit/ec844ea remove travis.yml, Go 1.16 https://github.com/kubernetes-csi/csi-release-tools/commit/df76aba Merge https://github.com/kubernetes-csi/csi-release-tools/pull/134 from andyzhangx/add-build-arg https://github.com/kubernetes-csi/csi-release-tools/commit/e314a56 add build-arg ARCH for building multi-arch images, e.g. ARG ARCH FROM k8s.gcr.io/build-image/debian-base-${ARCH}:v2.1.3 https://github.com/kubernetes-csi/csi-release-tools/commit/7bc70e5 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/129 from pohly/squash-documentation https://github.com/kubernetes-csi/csi-release-tools/commit/e0b02e7 README.md: document usage of --squash https://github.com/kubernetes-csi/csi-release-tools/commit/316cb95 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/132 from yiyang5055/bugfix/boilerplate https://github.com/kubernetes-csi/csi-release-tools/commit/26e2ab1 fix: default boilerplate path https://github.com/kubernetes-csi/csi-release-tools/commit/1add8c1 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/133 from pohly/kubernetes-1.20-tag https://github.com/kubernetes-csi/csi-release-tools/commit/3e811d6 prow.sh: fix "on-master" prow jobs https://github.com/kubernetes-csi/csi-release-tools/commit/1d60e77 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/131 from pohly/kubernetes-1.20-tag https://github.com/kubernetes-csi/csi-release-tools/commit/9f10459 prow.sh: support building Kubernetes for a specific version https://github.com/kubernetes-csi/csi-release-tools/commit/f7e7ee4 docs: steps for adding testing against new Kubernetes release https://github.com/kubernetes-csi/csi-release-tools/commit/fe1f284 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/121 from kvaps/namespace-check https://github.com/kubernetes-csi/csi-release-tools/commit/8fdf0f7 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/128 from fengzixu/master https://github.com/kubernetes-csi/csi-release-tools/commit/1c94220 fix: fix a bug of csi-sanity https://github.com/kubernetes-csi/csi-release-tools/commit/a4c41e6 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/127 from pohly/fix-boilerplate https://github.com/kubernetes-csi/csi-release-tools/commit/ece0f50 check namespace for snapshot-controller https://github.com/kubernetes-csi/csi-release-tools/commit/dbd8967 verify-boilerplate.sh: fix path to script https://github.com/kubernetes-csi/csi-release-tools/commit/9289fd1 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/125 from sachinkumarsingh092/optional-spelling-boilerplate-checks https://github.com/kubernetes-csi/csi-release-tools/commit/ad29307 Make the spelling and boilerplate checks optional https://github.com/kubernetes-csi/csi-release-tools/commit/5f06d02 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/124 from sachinkumarsingh092/fix-spellcheck-boilerplate-tests https://github.com/kubernetes-csi/csi-release-tools/commit/48186eb Fix spelling and boilerplate errors https://github.com/kubernetes-csi/csi-release-tools/commit/71690af Merge https://github.com/kubernetes-csi/csi-release-tools/pull/122 from sachinkumarsingh092/include-spellcheck-boilerplate-tests https://github.com/kubernetes-csi/csi-release-tools/commit/981be3f Adding spelling and boilerplate checks. https://github.com/kubernetes-csi/csi-release-tools/commit/2bb7525 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/117 from fengzixu/master https://github.com/kubernetes-csi/csi-release-tools/commit/3b6d17b Merge https://github.com/kubernetes-csi/csi-release-tools/pull/118 from pohly/cloud-build-timeout https://github.com/kubernetes-csi/csi-release-tools/commit/9318c6c cloud build: double the timeout, now 1 hour https://github.com/kubernetes-csi/csi-release-tools/commit/4ab8b15 use the tag to replace commit of csi-test https://github.com/kubernetes-csi/csi-release-tools/commit/5d74e45 change the csi-test import path to v4 https://github.com/kubernetes-csi/csi-release-tools/commit/7dcd0a9 upgrade csi-test to v4.0.2 https://github.com/kubernetes-csi/csi-release-tools/commit/86ff580 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/116 from andyzhangx/export-image-name https://github.com/kubernetes-csi/csi-release-tools/commit/c3a9662 allow export image name and registry name https://github.com/kubernetes-csi/csi-release-tools/commit/c6a88c6 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/113 from xing-yang/install_snapshot_controller https://github.com/kubernetes-csi/csi-release-tools/commit/45ec4c6 Fix the install of snapshot CRDs and controller https://github.com/kubernetes-csi/csi-release-tools/commit/5d874cc Merge https://github.com/kubernetes-csi/csi-release-tools/pull/112 from xing-yang/cleanup https://github.com/kubernetes-csi/csi-release-tools/commit/79bbca7 Cleanup https://github.com/kubernetes-csi/csi-release-tools/commit/d437673 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/111 from xing-yang/update_snapshot_v1_rc https://github.com/kubernetes-csi/csi-release-tools/commit/57718f8 Update snapshot CRD version https://github.com/kubernetes-csi/csi-release-tools/commit/4aff857 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/109 from pohly/alpha-test-defaults https://github.com/kubernetes-csi/csi-release-tools/commit/0427289 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/110 from pohly/kind-0.9-bazel-build-workaround https://github.com/kubernetes-csi/csi-release-tools/commit/9a370ab prow.sh: work around "kind build node-image" failure https://github.com/kubernetes-csi/csi-release-tools/commit/522361e prow.sh: only run alpha tests for latest Kubernetes release https://github.com/kubernetes-csi/csi-release-tools/commit/22c0395 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/108 from bnrjee/master https://github.com/kubernetes-csi/csi-release-tools/commit/b5b447b Add go ldflags using LDFLAGS at the time of compilation https://github.com/kubernetes-csi/csi-release-tools/commit/16f4afb Merge https://github.com/kubernetes-csi/csi-release-tools/pull/107 from pohly/kind-update https://github.com/kubernetes-csi/csi-release-tools/commit/7bcee13 prow.sh: update to kind 0.9, support Kubernetes 1.19 https://github.com/kubernetes-csi/csi-release-tools/commit/df518fb prow.sh: usage of Bazel optional https://github.com/kubernetes-csi/csi-release-tools/commit/c3afd42 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/104 from xing-yang/snapshot https://github.com/kubernetes-csi/csi-release-tools/commit/dde93b2 Update to snapshot-controller v3.0.0 https://github.com/kubernetes-csi/csi-release-tools/commit/a0f195c Merge https://github.com/kubernetes-csi/csi-release-tools/pull/106 from msau42/fix-canary https://github.com/kubernetes-csi/csi-release-tools/commit/7100c12 Only set staging registry when running canary job https://github.com/kubernetes-csi/csi-release-tools/commit/b3c65f9 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/99 from msau42/add-release-process https://github.com/kubernetes-csi/csi-release-tools/commit/e53f3e8 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/103 from msau42/fix-canary https://github.com/kubernetes-csi/csi-release-tools/commit/d129462 Document new method for adding CI jobs are new K8s versions https://github.com/kubernetes-csi/csi-release-tools/commit/e73c2ce Use staging registry for canary tests https://github.com/kubernetes-csi/csi-release-tools/commit/2c09846 Add cleanup instructions to release-notes generation git-subtree-dir: release-tools git-subtree-split: c0a4fb1dd00b02879dbcea987e2e9524e4f5a905 --- .prow.sh | 16 + KUBERNETES_CSI_OWNERS_ALIASES | 45 ++ OWNERS | 7 +- OWNERS_ALIASES | 1 + README.md | 17 +- SIDECAR_RELEASE_PROCESS.md | 64 ++- boilerplate/boilerplate.Dockerfile.txt | 13 + boilerplate/boilerplate.Makefile.txt | 13 + boilerplate/boilerplate.bzl.txt | 13 + boilerplate/boilerplate.go.txt | 15 + boilerplate/boilerplate.py | 200 +++++++++ boilerplate/boilerplate.py.txt | 13 + boilerplate/boilerplate.sh.txt | 13 + build.make | 66 ++- cloudbuild.sh | 14 + cloudbuild.yaml | 10 +- go-get-kubernetes.sh | 6 + prow.sh | 568 ++++++++++++++++--------- pull-test.sh | 32 ++ travis.yml | 21 - verify-boilerplate.sh | 54 +++ verify-go-version.sh | 5 +- verify-shellcheck.sh | 2 +- verify-spelling.sh | 59 +++ verify-subtree.sh | 2 +- 25 files changed, 1004 insertions(+), 265 deletions(-) create mode 100644 KUBERNETES_CSI_OWNERS_ALIASES create mode 120000 OWNERS_ALIASES create mode 100644 boilerplate/boilerplate.Dockerfile.txt create mode 100644 boilerplate/boilerplate.Makefile.txt create mode 100644 boilerplate/boilerplate.bzl.txt create mode 100644 boilerplate/boilerplate.go.txt create mode 100755 boilerplate/boilerplate.py create mode 100644 boilerplate/boilerplate.py.txt create mode 100644 boilerplate/boilerplate.sh.txt create mode 100755 pull-test.sh delete mode 100644 travis.yml create mode 100755 verify-boilerplate.sh create mode 100755 verify-spelling.sh diff --git a/.prow.sh b/.prow.sh index b18c53581..b92678291 100755 --- a/.prow.sh +++ b/.prow.sh @@ -1,7 +1,23 @@ #! /bin/bash -e + +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # This is for testing csi-release-tools itself in Prow. All other # repos use prow.sh for that, but as csi-release-tools isn't a normal # repo with some Go code in it, it has a custom Prow test script. ./verify-shellcheck.sh "$(pwd)" +./verify-spelling.sh "$(pwd)" +./verify-boilerplate.sh "$(pwd)" diff --git a/KUBERNETES_CSI_OWNERS_ALIASES b/KUBERNETES_CSI_OWNERS_ALIASES new file mode 100644 index 000000000..8ea922edb --- /dev/null +++ b/KUBERNETES_CSI_OWNERS_ALIASES @@ -0,0 +1,45 @@ +# See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md + +aliases: + + # SIG-Storage chairs and leads should always have approval rights in all repos. + # Others may be added as needed here or in each repo. + kubernetes-csi-approvers: + - jsafrane + - msau42 + - saad-ali + - xing-yang + + # Reviewers are automatically assigned to new PRs. The following + # reviewers will be active in all repos. Other reviewers can be + # added in each repo. + # + # Reviewers are encouraged to set the "Busy" flag in their GitHub status + # when they are temporarily unable to review PRs. + kubernetes-csi-reviewers: + - andyzhangx + - chrishenzie + - ggriffiths + - gnufied + - humblec + - j-griffith + - Jiawei0227 + - jingxu97 + - jsafrane + - pohly + - xing-yang + +# This documents who previously contributed to Kubernetes-CSI +# as approver. +emeritus_approver: +- lpabon +- sbezverk +- vladimirvivien + +# This documents who previously contributed to Kubernetes-CSI +# as reviewer. +emeritus_reviewer: +- lpabon +- saad-ali +- sbezverk +- vladimirvivien diff --git a/OWNERS b/OWNERS index 6d2f474e1..1fb74587b 100644 --- a/OWNERS +++ b/OWNERS @@ -1,11 +1,8 @@ # See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md approvers: -- saad-ali -- msau42 +- kubernetes-csi-approvers - pohly reviewers: -- saad-ali -- msau42 -- pohly +- kubernetes-csi-reviewers diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES new file mode 120000 index 000000000..7ec6c034d --- /dev/null +++ b/OWNERS_ALIASES @@ -0,0 +1 @@ +KUBERNETES_CSI_OWNERS_ALIASES \ No newline at end of file diff --git a/README.md b/README.md index 60eab2a98..b394724ce 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,11 @@ The expected repository layout is: Dockerfile in the root when only building a single command - `Makefile` - includes `release-tools/build.make` and sets configuration variables - - `.travis.yml` - a symlink to `release-tools/.travis.yml` + - `.prow.sh` script which imports `release-tools/prow.sh` + and may contain further customization + - `.cloudbuild.sh` and `cloudbuild.yaml` as symlinks to + the corresponding files in `release-tools` or (if necessary) + as custom files To create a release, tag a certain revision with a name that starts with `v`, for example `v1.0.0`, then `make push` @@ -44,10 +48,17 @@ is the recommended way of maintaining a copy of the rules inside the changes also locally, test them and then push them back to the shared repository at a later time. +We no longer care about importing the full commit history, so `--squash` should be used +when submitting a `release-tools` update. Also make sure that the PR for that +contains the automatically generated commit message in the PR description. +It contains the list of individual commits that were squashed. The script from +https://github.com/kubernetes-csi/csi-release-tools/issues/7 can create such +PRs automatically. + Cheat sheet: -- `git subtree add --prefix=release-tools https://github.com/kubernetes-csi/csi-release-tools.git master` - add release tools to a repo which does not have them yet (only once) -- `git subtree pull --prefix=release-tools https://github.com/kubernetes-csi/csi-release-tools.git master` - update local copy to latest upstream (whenever upstream changes) +- `git subtree add --squash --prefix=release-tools https://github.com/kubernetes-csi/csi-release-tools.git master` - add release tools to a repo which does not have them yet (only once) +- `git subtree pull --squash --prefix=release-tools https://github.com/kubernetes-csi/csi-release-tools.git master` - update local copy to latest upstream (whenever upstream changes) - edit, `git commit`, `git subtree push --prefix=release-tools git@github.com:/csi-release-tools.git ` - push to a new branch before submitting a PR verify-shellcheck.sh diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md index f6434c825..0cab8233c 100644 --- a/SIDECAR_RELEASE_PROCESS.md +++ b/SIDECAR_RELEASE_PROCESS.md @@ -11,7 +11,7 @@ The release manager must: kubernetes/org to request membership * Be a top level approver for the repository. To become a top level approver, the candidate must demonstrate ownership and deep knowledge of the repository - through active maintainence, responding to and fixing issues, reviewing PRs, + through active maintenance, responding to and fixing issues, reviewing PRs, test triage. * Be part of the maintainers or admin group for the repository. admin is a superset of maintainers, only maintainers level is required for cutting a @@ -39,10 +39,11 @@ naming convention `-on-`. 1. Changes can then be updated in all the sidecar repos and hostpath driver repo by following the [update instructions](https://github.com/kubernetes-csi/csi-release-tools/blob/master/README.md#sharing-and-updating). -1. New pull and CI jobs are configured by +1. New pull and CI jobs are configured by adding new K8s versions to the top of [gen-jobs.sh](https://github.com/kubernetes/test-infra/blob/master/config/jobs/kubernetes-csi/gen-jobs.sh). - New pull jobs that have been unverified should be initially made optional. - [Example](https://github.com/kubernetes/test-infra/pull/15055) + New pull jobs that have been unverified should be initially made optional by + setting the new K8s version as + [experimental](https://github.com/kubernetes/test-infra/blob/a1858f46d6014480b130789df58b230a49203a64/config/jobs/kubernetes-csi/gen-jobs.sh#L40). 1. Once new pull and CI jobs have been verified, and the new Kubernetes version is released, we can make the optional jobs required, and also remove the Kubernetes versions that are no longer supported. @@ -54,14 +55,19 @@ naming convention `-on-`. generator](https://github.com/kubernetes/release/tree/master/cmd/release-notes) 1. Generate release notes for the release. Replace arguments with the relevant information. + * Clean up old cached information (also needed if you are generating release + notes for multiple repos) + ```bash + rm -rf /tmp/k8s-repo + ``` * For new minor releases on master: - ``` + ```bash GITHUB_TOKEN= release-notes --discover=mergebase-to-latest --github-org=kubernetes-csi --github-repo=external-provisioner --required-author="" --output out.md ``` * For new patch releases on a release branch: - ``` + ```bash GITHUB_TOKEN= release-notes --discover=patch-to-latest --branch=release-1.1 --github-org=kubernetes-csi --github-repo=external-provisioner --required-author="" --output out.md @@ -89,7 +95,7 @@ naming convention `-on-`. 1. Check [image build status](https://k8s-testgrid.appspot.com/sig-storage-image-build). 1. Promote images from k8s-staging-sig-storage to k8s.gcr.io/sig-storage. From the [k8s image - repo](https://github.com/kubernetes/k8s.io/tree/master/k8s.gcr.io/images/k8s-staging-sig-storage), + repo](https://github.com/kubernetes/k8s.io/tree/main/k8s.gcr.io/images/k8s-staging-sig-storage), run `./generate.sh > images.yaml`, and send a PR with the updated images. Once merged, the image promoter will copy the images from staging to prod. 1. Update [kubernetes-csi/docs](https://github.com/kubernetes-csi/docs) sidecar @@ -98,3 +104,47 @@ naming convention `-on-`. CSI hostpath driver with the new sidecars in the [CSI repo](https://github.com/kubernetes-csi/csi-driver-host-path/tree/master/deploy) and [k/k in-tree](https://github.com/kubernetes/kubernetes/tree/master/test/e2e/testing-manifests/storage-csi/hostpath/hostpath) + +## Adding support for a new Kubernetes release + +1. Add the new release to `k8s_versions` in + https://github.com/kubernetes/test-infra/blob/090dec5dd535d5f61b7ba52e671a810f5fc13dfd/config/jobs/kubernetes-csi/gen-jobs.sh#L25 + to enable generating a job for it. Set `experimental_k8s_version` + in + https://github.com/kubernetes/test-infra/blob/090dec5dd535d5f61b7ba52e671a810f5fc13dfd/config/jobs/kubernetes-csi/gen-jobs.sh#L40 + to ensure that the new jobs aren't run for PRs unless explicitly + requested. Generate and submit the new jobs. +1. Create a test PR to try out the new job in some repo with `/test + pull-kubernetes-csi---on-kubernetes-` where x.y + matches the Kubernetes release. Alternatively, run .prow.sh in that + repo locally with `CSI_PROW_KUBERNETES_VERSION=x.y.z`. +1. Optional: update to a [new + release](https://github.com/kubernetes-sigs/kind/tags) of kind with + pre-built images for the new Kubernetes release. This is optional + if the current version of kind is able to build images for the new + Kubernetes release. However, jobs require less resources when they + don't need to build those images from the Kubernetes source code. + This change needs to be tried out in a PR against a component + first, then get submitted against csi-release-tools. +1. Optional: propagate the updated csi-release-tools to all components + with the script from + https://github.com/kubernetes-csi/csi-release-tools/issues/7#issuecomment-707025402 +1. Once it is likely to work in all components, unset + `experimental_k8s_version` and submit the updated jobs. +1. Once all sidecars for the new Kubernetes release are released, + either bump the version number of the images in the existing + [csi-driver-host-path + deployments](https://github.com/kubernetes-csi/csi-driver-host-path/tree/master/deploy) + and/or create a new deployment, depending on what Kubernetes + release an updated sidecar is compatible with. If no new deployment + is needed, then add a symlink to document that there intentionally + isn't a separate deployment. This symlink is not needed for Prow + testing because that will use "kubernetes-latest" as fallback. + Update that link when creating a new deployment. +1. Create a new csi-driver-host-path release. +1. Bump `CSI_PROW_DRIVER_VERSION` in prow.sh to that new release and + (eventually) roll that change out to all repos by updating + `release-tools` in them. This is used when testing manually. The + Prow jobs override that value, so also update + `hostpath_driver_version` in + https://github.com/kubernetes/test-infra/blob/91b04e6af3a40a9bcff25aa030850a4721e2dd2b/config/jobs/kubernetes-csi/gen-jobs.sh#L46-L47 diff --git a/boilerplate/boilerplate.Dockerfile.txt b/boilerplate/boilerplate.Dockerfile.txt new file mode 100644 index 000000000..34cb349c4 --- /dev/null +++ b/boilerplate/boilerplate.Dockerfile.txt @@ -0,0 +1,13 @@ +# Copyright YEAR The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/boilerplate/boilerplate.Makefile.txt b/boilerplate/boilerplate.Makefile.txt new file mode 100644 index 000000000..d0d526523 --- /dev/null +++ b/boilerplate/boilerplate.Makefile.txt @@ -0,0 +1,13 @@ +# Copyright YEAR The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file diff --git a/boilerplate/boilerplate.bzl.txt b/boilerplate/boilerplate.bzl.txt new file mode 100644 index 000000000..d0d526523 --- /dev/null +++ b/boilerplate/boilerplate.bzl.txt @@ -0,0 +1,13 @@ +# Copyright YEAR The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file diff --git a/boilerplate/boilerplate.go.txt b/boilerplate/boilerplate.go.txt new file mode 100644 index 000000000..3249913bd --- /dev/null +++ b/boilerplate/boilerplate.go.txt @@ -0,0 +1,15 @@ +/* +Copyright YEAR The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ \ No newline at end of file diff --git a/boilerplate/boilerplate.py b/boilerplate/boilerplate.py new file mode 100755 index 000000000..5618b9ab8 --- /dev/null +++ b/boilerplate/boilerplate.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python + +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import argparse +import difflib +import glob +import os +import re +import sys +from datetime import date + +parser = argparse.ArgumentParser() +parser.add_argument( + "filenames", + help="list of files to check, all files if unspecified", + nargs='*') + +# Rootdir defaults to the directory **above** the repo-infra dir. +rootdir = os.path.dirname(__file__) + "./../../" +rootdir = os.path.abspath(rootdir) +parser.add_argument( + "--rootdir", default=rootdir, help="root directory to examine") + +default_boilerplate_dir = os.path.abspath(os.path.dirname(__file__)) + +parser.add_argument( + "--boilerplate-dir", default=default_boilerplate_dir) + +parser.add_argument( + "-v", "--verbose", + help="give verbose output regarding why a file does not pass", + action="store_true") + +args = parser.parse_args() + +verbose_out = sys.stderr if args.verbose else open("/dev/null", "w") + +def get_refs(): + refs = {} + + for path in glob.glob(os.path.join(args.boilerplate_dir, "boilerplate.*.txt")): + extension = os.path.basename(path).split(".")[1] + + ref_file = open(path, 'r') + ref = ref_file.read().splitlines() + ref_file.close() + refs[extension] = ref + + return refs + +def file_passes(filename, refs, regexs): + try: + f = open(filename, 'r') + except Exception as exc: + print("Unable to open %s: %s" % (filename, exc), file=verbose_out) + return False + + data = f.read() + f.close() + + basename = os.path.basename(filename) + extension = file_extension(filename) + if extension != "": + ref = refs[extension] + else: + ref = refs[basename] + + # remove build tags from the top of Go files + if extension == "go": + p = regexs["go_build_constraints"] + (data, found) = p.subn("", data, 1) + + # remove shebang from the top of shell files + if extension == "sh" or extension == "py": + p = regexs["shebang"] + (data, found) = p.subn("", data, 1) + + data = data.splitlines() + + # if our test file is smaller than the reference it surely fails! + if len(ref) > len(data): + print('File %s smaller than reference (%d < %d)' % + (filename, len(data), len(ref)), + file=verbose_out) + return False + + # trim our file to the same number of lines as the reference file + data = data[:len(ref)] + + p = regexs["year"] + for d in data: + if p.search(d): + print('File %s is missing the year' % filename, file=verbose_out) + return False + + # Replace all occurrences of the regex "CURRENT_YEAR|...|2016|2015|2014" with "YEAR" + p = regexs["date"] + for i, d in enumerate(data): + (data[i], found) = p.subn('YEAR', d) + if found != 0: + break + + # if we don't match the reference at this point, fail + if ref != data: + print("Header in %s does not match reference, diff:" % filename, file=verbose_out) + if args.verbose: + print(file=verbose_out) + for line in difflib.unified_diff(ref, data, 'reference', filename, lineterm=''): + print(line, file=verbose_out) + print(file=verbose_out) + return False + + return True + +def file_extension(filename): + return os.path.splitext(filename)[1].split(".")[-1].lower() + +skipped_dirs = ['Godeps', 'third_party', '_gopath', '_output', '.git', + 'cluster/env.sh', 'vendor', 'test/e2e/generated/bindata.go', + 'repo-infra/verify/boilerplate/test', '.glide'] + +def normalize_files(files): + newfiles = [] + for pathname in files: + if any(x in pathname for x in skipped_dirs): + continue + newfiles.append(pathname) + return newfiles + +def get_files(extensions): + files = [] + if len(args.filenames) > 0: + files = args.filenames + else: + for root, dirs, walkfiles in os.walk(args.rootdir): + # don't visit certain dirs. This is just a performance improvement + # as we would prune these later in normalize_files(). But doing it + # cuts down the amount of filesystem walking we do and cuts down + # the size of the file list + for d in skipped_dirs: + if d in dirs: + dirs.remove(d) + + for name in walkfiles: + pathname = os.path.join(root, name) + files.append(pathname) + + files = normalize_files(files) + + outfiles = [] + for pathname in files: + basename = os.path.basename(pathname) + extension = file_extension(pathname) + if extension in extensions or basename in extensions: + outfiles.append(pathname) + return outfiles + +def get_regexs(): + regexs = {} + # Search for "YEAR" which exists in the boilerplate, but shouldn't in the real thing + regexs["year"] = re.compile( 'YEAR' ) + # dates can be 2014, 2015, 2016, ..., CURRENT_YEAR, company holder names can be anything + years = range(2014, date.today().year + 1) + regexs["date"] = re.compile( '(%s)' % "|".join(map(lambda l: str(l), years)) ) + # strip // +build \n\n build constraints + regexs["go_build_constraints"] = re.compile(r"^(// \+build.*\n)+\n", re.MULTILINE) + # strip #!.* from shell scripts + regexs["shebang"] = re.compile(r"^(#!.*\n)\n*", re.MULTILINE) + return regexs + + + +def main(): + regexs = get_regexs() + refs = get_refs() + filenames = get_files(refs.keys()) + + for filename in filenames: + if not file_passes(filename, refs, regexs): + print(filename, file=sys.stdout) + + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/boilerplate/boilerplate.py.txt b/boilerplate/boilerplate.py.txt new file mode 100644 index 000000000..34cb349c4 --- /dev/null +++ b/boilerplate/boilerplate.py.txt @@ -0,0 +1,13 @@ +# Copyright YEAR The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/boilerplate/boilerplate.sh.txt b/boilerplate/boilerplate.sh.txt new file mode 100644 index 000000000..d0d526523 --- /dev/null +++ b/boilerplate/boilerplate.sh.txt @@ -0,0 +1,13 @@ +# Copyright YEAR The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file diff --git a/build.make b/build.make index c17a1c194..0ed51e80a 100644 --- a/build.make +++ b/build.make @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# force the usage of /bin/bash instead of /bin/sh +SHELL := /bin/bash + .PHONY: build-% build container-% container push-% push clean test # A space-separated list of all commands in the repository, must be @@ -20,7 +23,7 @@ # This is the default. It can be overridden in the main Makefile after # including build.make. -REGISTRY_NAME=quay.io/k8scsi +REGISTRY_NAME?=quay.io/k8scsi # Can be set to -mod=vendor to ensure that the "vendor" directory is used. GOFLAGS_VENDOR= @@ -63,21 +66,35 @@ endif # Specific packages can be excluded from each of the tests below by setting the *_FILTER_CMD variables # to something like "| grep -v 'github.com/kubernetes-csi/project/pkg/foobar'". See usage below. -# BUILD_PLATFORMS contains a set of triplets, +# BUILD_PLATFORMS contains a set of tuples [os arch suffix base_image addon_image] # separated by semicolon. An empty variable or empty entry (= just a # semicolon) builds for the default platform of the current Go # toolchain. BUILD_PLATFORMS = +# Add go ldflags using LDFLAGS at the time of compilation. +IMPORTPATH_LDFLAGS = -X main.version=$(REV) +EXT_LDFLAGS = -extldflags "-static" +LDFLAGS = +FULL_LDFLAGS = $(LDFLAGS) $(IMPORTPATH_LDFLAGS) $(EXT_LDFLAGS) # This builds each command (= the sub-directories of ./cmd) for the target platform(s) # defined by BUILD_PLATFORMS. $(CMDS:%=build-%): build-%: check-go-version-go mkdir -p bin - echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix; do \ - if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o "./bin/$*$$suffix" ./cmd/$*); then \ + # os_arch_seen captures all of the $$os-$$arch seen for the current binary + # that we want to build, if we've seen an $$os-$$arch before it means that + # we don't need to build it again, this is done to avoid building + # the windows binary multiple times (see the default value of $$BUILD_PLATFORMS) + export os_arch_seen="" && echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \ + os_arch_seen_pre=$${os_arch_seen%%$$os-$$arch*}; \ + if ! [ $${#os_arch_seen_pre} = $${#os_arch_seen} ]; then \ + continue; \ + fi; \ + if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '$(FULL_LDFLAGS)' -o "./bin/$*$$suffix" ./cmd/$*); then \ echo "Building $* for GOOS=$$os GOARCH=$$arch failed, see error(s) above."; \ exit 1; \ fi; \ + os_arch_seen+=";$$os-$$arch"; \ done $(CMDS:%=container-%): container-%: build-% @@ -126,29 +143,46 @@ DOCKER_BUILDX_CREATE_ARGS ?= # the tag for the resulting multiarch image. $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-% set -ex; \ - DOCKER_CLI_EXPERIMENTAL=enabled; \ - export DOCKER_CLI_EXPERIMENTAL; \ + export DOCKER_CLI_EXPERIMENTAL=enabled; \ docker buildx create $(DOCKER_BUILDX_CREATE_ARGS) --use --name multiarchimage-buildertest; \ trap "docker buildx rm multiarchimage-buildertest" EXIT; \ dockerfile_linux=$$(if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi); \ dockerfile_windows=$$(if [ -e ./cmd/$*/Dockerfile.Windows ]; then echo ./cmd/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi); \ if [ '$(BUILD_PLATFORMS)' ]; then build_platforms='$(BUILD_PLATFORMS)'; else build_platforms="linux amd64"; fi; \ if ! [ -f "$$dockerfile_windows" ]; then \ - build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe//g' -e 's/; *;/;/g')"; \ + build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe *[^ ]* *[^ ]*//g' -e 's/; *;/;/g' -e 's/;[ ]*$$//')"; \ fi; \ pushMultiArch () { \ tag=$$1; \ - echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do \ + echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \ + escaped_base_image=$${base_image/:/-}; \ + if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \ docker buildx build --push \ - --tag $(IMAGE_NAME):$$arch-$$os-$$tag \ + --tag $(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag \ --platform=$$os/$$arch \ --file $$(eval echo \$${dockerfile_$$os}) \ --build-arg binary=./bin/$*$$suffix \ + --build-arg ARCH=$$arch \ + --build-arg BASE_IMAGE=$$base_image \ + --build-arg ADDON_IMAGE=$$addon_image \ --label revision=$(REV) \ .; \ done; \ - images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do echo $(IMAGE_NAME):$$arch-$$os-$$tag; done); \ + images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \ + escaped_base_image=$${base_image/:/-}; \ + if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \ + echo $(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag; \ + done); \ docker manifest create --amend $(IMAGE_NAME):$$tag $$images; \ + echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \ + if [ $$os = "windows" ]; then \ + escaped_base_image=$${base_image/:/-}; \ + if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \ + image=$(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag; \ + os_version=$$(docker manifest inspect mcr.microsoft.com/windows/$${base_image} | grep "os.version" | head -n 1 | awk '{print $$2}' | sed -e 's/"//g') || true; \ + docker manifest annotate --os-version $$os_version $(IMAGE_NAME):$$tag $$image; \ + fi; \ + done; \ docker manifest push -p $(IMAGE_NAME):$$tag; \ }; \ if [ $(PULL_BASE_REF) = "master" ]; then \ @@ -270,3 +304,15 @@ test-shellcheck: .PHONY: check-go-version-% check-go-version-%: ./release-tools/verify-go-version.sh "$*" + +# Test for spelling errors. +.PHONY: test-spelling +test-spelling: + @ echo; echo "### $@:" + @ ./release-tools/verify-spelling.sh "$(pwd)" + +# Test the boilerplates of the files. +.PHONY: test-boilerplate +test-boilerplate: + @ echo; echo "### $@:" + @ ./release-tools/verify-boilerplate.sh "$(pwd)" diff --git a/cloudbuild.sh b/cloudbuild.sh index 3ba11ecad..1edda4d3f 100755 --- a/cloudbuild.sh +++ b/cloudbuild.sh @@ -1,5 +1,19 @@ #! /bin/bash +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # shellcheck disable=SC1091 . release-tools/prow.sh diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 8f678924e..823fd1c5a 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -13,19 +13,21 @@ # See https://github.com/kubernetes/test-infra/blob/master/config/jobs/image-pushing/README.md # for more details on image pushing process in Kubernetes. # -# To promote release images, see https://github.com/kubernetes/k8s.io/tree/master/k8s.gcr.io/images/k8s-staging-sig-storage. +# To promote release images, see https://github.com/kubernetes/k8s.io/tree/main/k8s.gcr.io/images/k8s-staging-sig-storage. # This must be specified in seconds. If omitted, defaults to 600s (10 mins). -timeout: 1800s +# Building three images in external-snapshotter takes roughly half an hour, +# sometimes more. +timeout: 3600s # This prevents errors if you don't use both _GIT_TAG and _PULL_BASE_REF, # or any new substitutions added in the future. options: substitution_option: ALLOW_LOOSE steps: # The image must contain bash and curl. Ideally it should also contain - # the desired version of Go (currently defined in release-tools/travis.yml), + # the desired version of Go (currently defined in release-tools/prow.sh), # but that just speeds up the build and is not required. - - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20200421-a2bf5f8' + - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20210331-c732583' entrypoint: ./.cloudbuild.sh env: - GIT_TAG=${_GIT_TAG} diff --git a/go-get-kubernetes.sh b/go-get-kubernetes.sh index 8c4e30244..cbbbb7c3c 100755 --- a/go-get-kubernetes.sh +++ b/go-get-kubernetes.sh @@ -55,6 +55,12 @@ mods=$( (set -x; curl --silent --show-error --fail "https://raw.githubuserconten sed -n 's|.*k8s.io/\(.*\) => ./staging/src/k8s.io/.*|k8s.io/\1|p' ) || die "failed to determine Kubernetes staging modules" for mod in $mods; do + if ! (env GO111MODULE=on go mod graph) | grep "$mod@" > /dev/null; then + echo "Kubernetes module $mod is not used, skipping" + # Remove the module from go.mod "replace" that was added by an older version of this script. + (set -x; env GO111MODULE=on go mod edit "-dropreplace=$mod") || die "'go mod edit' failed" + continue + fi # The presence of a potentially incomplete go.mod file affects this command, # so move elsewhere. modinfo=$(set -x; cd /; env GO111MODULE=on go mod download -json "$mod@kubernetes-${k8s}") || diff --git a/prow.sh b/prow.sh index 05d397a97..d1c7b27fa 100755 --- a/prow.sh +++ b/prow.sh @@ -1,5 +1,5 @@ #! /bin/bash -# + # Copyright 2019 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -34,8 +34,6 @@ # - $GOPATH/src/ for the repository that is to be tested, # with PR branch merged (when testing a PR) # - running on linux-amd64 -# - bazel installed (when testing against Kubernetes master), must be recent -# enough for Kubernetes master # - kind (https://github.com/kubernetes-sigs/kind) installed # - optional: Go already installed @@ -52,26 +50,6 @@ configvar () { eval echo "\$3:" "$1=\${$1}" } -# Takes the minor version of $CSI_PROW_KUBERNETES_VERSION and overrides it to -# $1 if they are equal minor versions. Ignores versions that begin with -# "release-". -override_k8s_version () { - local current_minor_version - local override_minor_version - - # Ignore: See if you can use ${variable//search/replace} instead. - # shellcheck disable=SC2001 - current_minor_version="$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1\.\2/')" - - # Ignore: See if you can use ${variable//search/replace} instead. - # shellcheck disable=SC2001 - override_minor_version="$(echo "${1}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1\.\2/')" - if [ "${current_minor_version}" == "${override_minor_version}" ]; then - CSI_PROW_KUBERNETES_VERSION="$1" - echo "Overriding CSI_PROW_KUBERNETES_VERSION with $1: $CSI_PROW_KUBERNETES_VERSION" - fi -} - # Prints the value of a variable + version suffix, falling back to variable + "LATEST". get_versioned_variable () { local var="$1" @@ -85,7 +63,22 @@ get_versioned_variable () { echo "$value" } -configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x; linux arm64 -arm64" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries" +# This takes a version string like CSI_PROW_KUBERNETES_VERSION and +# maps it to the corresponding git tag, branch or commit. +version_to_git () { + version="$1" + shift + case "$version" in + latest|master) echo "master";; + release-*) echo "$version";; + *) echo "v$version";; + esac +} + +# the list of windows versions was matched from: +# - https://hub.docker.com/_/microsoft-windows-nanoserver +# - https://hub.docker.com/_/microsoft-windows-servercore +configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; linux ppc64le -ppc64le; linux s390x -s390x; linux arm64 -arm64; windows amd64 .exe nanoserver:1809 servercore:ltsc2019; windows amd64 .exe nanoserver:1909 servercore:1909; windows amd64 .exe nanoserver:2004 servercore:2004; windows amd64 .exe nanoserver:20H2 servercore:20H2" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries" # If we have a vendor directory, then use it. We must be careful to only # use this for "make" invocations inside the project's repo itself because @@ -93,24 +86,12 @@ configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; windows amd64 .exe; linux ppc64 # which is disabled with GOFLAGS=-mod=vendor). configvar GOFLAGS_VENDOR "$( [ -d vendor ] && echo '-mod=vendor' )" "Go flags for using the vendor directory" -# Go versions can be specified seperately for different tasks -# If the pre-installed Go is missing or a different -# version, the required version here will get installed -# from https://golang.org/dl/. -go_from_travis_yml () { - grep "^ *- go:" "${RELEASE_TOOLS_ROOT}/travis.yml" | sed -e 's/.*go: *//' -} -configvar CSI_PROW_GO_VERSION_BUILD "$(go_from_travis_yml)" "Go version for building the component" # depends on component's source code +configvar CSI_PROW_GO_VERSION_BUILD "1.16" "Go version for building the component" # depends on component's source code configvar CSI_PROW_GO_VERSION_E2E "" "override Go version for building the Kubernetes E2E test suite" # normally doesn't need to be set, see install_e2e configvar CSI_PROW_GO_VERSION_SANITY "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building the csi-sanity test suite" # depends on CSI_PROW_SANITY settings below configvar CSI_PROW_GO_VERSION_KIND "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building 'kind'" # depends on CSI_PROW_KIND_VERSION below configvar CSI_PROW_GO_VERSION_GINKGO "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building ginkgo" # depends on CSI_PROW_GINKGO_VERSION below -# kind version to use. If the pre-installed version is different, -# the desired version is downloaded from https://github.com/kubernetes-sigs/kind/releases/download/ -# (if available), otherwise it is built from source. -configvar CSI_PROW_KIND_VERSION "v0.6.0" "kind" - # ginkgo test runner version to use. If the pre-installed version is # different, the desired version is built from source. configvar CSI_PROW_GINKGO_VERSION v1.7.0 "Ginkgo" @@ -124,10 +105,13 @@ configvar CSI_PROW_GINKO_PARALLEL "-p" "Ginko parallelism parameter(s)" configvar CSI_PROW_BUILD_JOB true "building code in repo enabled" # Kubernetes version to test against. This must be a version number -# (like 1.13.3) for which there is a pre-built kind image (see -# https://hub.docker.com/r/kindest/node/tags), "latest" (builds -# Kubernetes from the master branch) or "release-x.yy" (builds -# Kubernetes from a release branch). +# (like 1.13.3), "latest" (builds Kubernetes from the master branch) +# or "release-x.yy" (builds Kubernetes from a release branch). +# +# The patch version is only relevant for picking the E2E test suite +# that is used for testing. The script automatically picks +# the kind images for the major/minor version of Kubernetes +# that the kind release supports. # # This can also be a version that was not released yet at the time # that the settings below were chose. The script will then @@ -136,16 +120,6 @@ configvar CSI_PROW_BUILD_JOB true "building code in repo enabled" # deprecating or changing the implementation of an alpha feature. configvar CSI_PROW_KUBERNETES_VERSION 1.17.0 "Kubernetes" -# This is a hack to workaround the issue that each version -# of kind currently only supports specific patch versions of -# Kubernetes. We need to override CSI_PROW_KUBERNETES_VERSION -# passed in by our CI/pull jobs to the versions that -# kind v0.5.0 supports. -# -# If the version is prefixed with "release-", then nothing -# is overridden. -override_k8s_version "1.15.3" - # CSI_PROW_KUBERNETES_VERSION reduced to first two version numbers and # with underscore (1_13 instead of 1.13.3) and in uppercase (LATEST # instead of latest). @@ -154,10 +128,35 @@ override_k8s_version "1.15.3" # when a Prow job just defines the Kubernetes version. csi_prow_kubernetes_version_suffix="$(echo "${CSI_PROW_KUBERNETES_VERSION}" | tr . _ | tr '[:lower:]' '[:upper:]' | sed -e 's/^RELEASE-//' -e 's/\([0-9]*\)_\([0-9]*\).*/\1_\2/')" -# Work directory. It has to allow running executables, therefore /tmp -# is avoided. Cleaning up after the script is intentionally left to -# the caller. -configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csiprow.XXXXXXXXXX")" "work directory" +# Only the latest KinD is (eventually) guaranteed to work with the +# latest Kubernetes. For example, KinD 0.10.0 failed with Kubernetes +# 1.21.0-beta1. Therefore the default version of KinD is "main" +# for that, otherwise the latest stable release for which we then +# list the officially supported images below. +kind_version_default () { + case "${CSI_PROW_KUBERNETES_VERSION}" in + latest|master) + echo main;; + *) + echo v0.11.0;; + esac +} + +# kind version to use. If the pre-installed version is different, +# the desired version is downloaded from https://github.com/kubernetes-sigs/kind/releases +# (if available), otherwise it is built from source. +configvar CSI_PROW_KIND_VERSION "$(kind_version_default)" "kind" + +# kind images to use. Must match the kind version. +# The release notes of each kind release list the supported images. +configvar CSI_PROW_KIND_IMAGES "kindest/node:v1.21.1@sha256:fae9a58f17f18f06aeac9772ca8b5ac680ebbed985e266f711d936e91d113bad +kindest/node:v1.20.7@sha256:e645428988191fc824529fd0bb5c94244c12401cf5f5ea3bd875eb0a787f0fe9 +kindest/node:v1.19.11@sha256:7664f21f9cb6ba2264437de0eb3fe99f201db7a3ac72329547ec4373ba5f5911 +kindest/node:v1.18.19@sha256:530378628c7c518503ade70b1df698b5de5585dcdba4f349328d986b8849b1ee +kindest/node:v1.17.17@sha256:c581fbf67f720f70aaabc74b44c2332cc753df262b6c0bca5d26338492470c17 +kindest/node:v1.16.15@sha256:430c03034cd856c1f1415d3e37faf35a3ea9c5aaa2812117b79e6903d1fc9651 +kindest/node:v1.15.12@sha256:8d575f056493c7778935dd855ded0e95c48cb2fab90825792e8fc9af61536bf9 +kindest/node:v1.14.10@sha256:6033e04bcfca7c5f2a9c4ce77551e1abf385bcd2709932ec2f6a9c8c0aff6d4f" "kind images" # By default, this script tests sidecars with the CSI hostpath driver, # using the install_csi_driver function. That function depends on @@ -185,8 +184,8 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip # CSI_PROW_DEPLOYMENT variable can be set in the # .prow.sh of each component when there are breaking changes # that require using a non-default deployment. The default -# is a deployment named "kubernetes-x.yy" (if available), -# otherwise "kubernetes-latest". +# is a deployment named "kubernetes-x.yy${CSI_PROW_DEPLOYMENT_SUFFIX}" (if available), +# otherwise "kubernetes-latest${CSI_PROW_DEPLOYMENT_SUFFIX}". # "none" disables the deployment of the hostpath driver. # # When no deploy script is found (nothing in `deploy` directory, @@ -198,6 +197,7 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip configvar CSI_PROW_DRIVER_VERSION "v1.3.0" "CSI driver version" configvar CSI_PROW_DRIVER_REPO https://github.com/kubernetes-csi/csi-driver-host-path "CSI driver repo" configvar CSI_PROW_DEPLOYMENT "" "deployment" +configvar CSI_PROW_DEPLOYMENT_SUFFIX "" "additional suffix in kubernetes-x.yy[suffix].yaml files" # The install_csi_driver function may work also for other CSI drivers, # as long as they follow the conventions of the CSI hostpath driver. @@ -211,6 +211,10 @@ configvar CSI_PROW_DRIVER_INSTALL "install_csi_driver" "name of the shell functi # still use that name. configvar CSI_PROW_DRIVER_CANARY "${CSI_PROW_HOSTPATH_CANARY}" "driver image override for canary images" +# Image registry to use for canary images. +# Only valid if CSI_PROW_DRIVER_CANARY == "canary". +configvar CSI_PROW_DRIVER_CANARY_REGISTRY "gcr.io/k8s-staging-sig-storage" "registry for canary images" + # The E2E testing can come from an arbitrary repo. The expectation is that # the repo supports "go test ./test/e2e -args --storage.testdriver" (https://github.com/kubernetes/kubernetes/pull/72836) # after setting KUBECONFIG. As a special case, if the repository is Kubernetes, @@ -218,16 +222,7 @@ configvar CSI_PROW_DRIVER_CANARY "${CSI_PROW_HOSTPATH_CANARY}" "driver image ove # all generated files are present. # # CSI_PROW_E2E_REPO=none disables E2E testing. -tag_from_version () { - version="$1" - shift - case "$version" in - latest) echo "master";; - release-*) echo "$version";; - *) echo "v$version";; - esac -} -configvar CSI_PROW_E2E_VERSION "$(tag_from_version "${CSI_PROW_KUBERNETES_VERSION}")" "E2E version" +configvar CSI_PROW_E2E_VERSION "$(version_to_git "${CSI_PROW_KUBERNETES_VERSION}")" "E2E version" configvar CSI_PROW_E2E_REPO "https://github.com/kubernetes/kubernetes" "E2E repo" configvar CSI_PROW_E2E_IMPORT_PATH "k8s.io/kubernetes" "E2E package" @@ -237,8 +232,8 @@ configvar CSI_PROW_E2E_IMPORT_PATH "k8s.io/kubernetes" "E2E package" # of the cluster. The alternative would have been to (cross-)compile csi-sanity # and install it inside the cluster, which is not necessarily easier. configvar CSI_PROW_SANITY_REPO https://github.com/kubernetes-csi/csi-test "csi-test repo" -configvar CSI_PROW_SANITY_VERSION 5421d9f3c37be3b95b241b44a094a3db11bee789 "csi-test version" # latest master -configvar CSI_PROW_SANITY_IMPORT_PATH github.com/kubernetes-csi/csi-test "csi-test package" +configvar CSI_PROW_SANITY_VERSION v4.2.0 "csi-test version" +configvar CSI_PROW_SANITY_PACKAGE_PATH github.com/kubernetes-csi/csi-test "csi-test package" configvar CSI_PROW_SANITY_SERVICE "hostpath-service" "Kubernetes TCP service name that exposes csi.sock" configvar CSI_PROW_SANITY_POD "csi-hostpathplugin-0" "Kubernetes pod with CSI driver" configvar CSI_PROW_SANITY_CONTAINER "hostpath" "Kubernetes container with CSI driver" @@ -258,11 +253,16 @@ configvar CSI_PROW_DEP_VERSION v0.5.1 "golang dep version to be used for vendor # # Unknown or unsupported entries are ignored. # +# Testing of alpha features is only supported for CSI_PROW_KUBERNETES_VERSION=latest +# because CSI_PROW_E2E_ALPHA and CSI_PROW_E2E_ALPHA_GATES are not set for +# older Kubernetes releases. The script supports that, it just isn't done because +# it is not needed and would cause additional maintenance effort. +# # Sanity testing with csi-sanity only covers the CSI driver itself and # thus only makes sense in repos which provide their own CSI # driver. Repos can enable sanity testing by setting # CSI_PROW_TESTS_SANITY=sanity. -configvar CSI_PROW_TESTS "unit parallel serial parallel-alpha serial-alpha sanity" "tests to run" +configvar CSI_PROW_TESTS "unit parallel serial $(if [ "${CSI_PROW_KUBERNETES_VERSION}" = "latest" ]; then echo parallel-alpha serial-alpha; fi) sanity" "tests to run" tests_enabled () { local t1 t2 # We want word-splitting here, so ignore: Quote to prevent word splitting, or split robustly with mapfile or read -a. @@ -292,17 +292,26 @@ tests_need_alpha_cluster () { tests_enabled "parallel-alpha" "serial-alpha" } +# Enabling mock tests adds the "CSI mock volume" tests from https://github.com/kubernetes/kubernetes/blob/master/test/e2e/storage/csi_mock_volume.go +# to the e2e.test invocations (serial, parallel, and the corresponding alpha variants). +# When testing canary images, those get used instead of the images specified +# in the e2e.test's normal YAML files. +# +# The default is to enable this for all jobs which use canary images +# and the latest Kubernetes because those images will be used for mock +# testing once they are released. Using them for mock testing with +# older Kubernetes releases is too risky because the deployment files +# can be very old (for example, still using a removed -provisioner +# parameter in external-provisioner). +configvar CSI_PROW_E2E_MOCK "$(if [ "${CSI_PROW_DRIVER_CANARY}" = "canary" ] && [ "${CSI_PROW_KUBERNETES_VERSION}" = "latest" ]; then echo true; else echo false; fi)" "enable CSI mock volume tests" + # Regex for non-alpha, feature-tagged tests that should be run. # -# Starting with 1.17, snapshots is beta, but the E2E tests still have the -# [Feature:] tag. They need to be explicitly enabled. -configvar CSI_PROW_E2E_FOCUS_1_15 '^' "non-alpha, feature-tagged tests for Kubernetes = 1.15" # no tests to run, match nothing -configvar CSI_PROW_E2E_FOCUS_1_16 '^' "non-alpha, feature-tagged tests for Kubernetes = 1.16" # no tests to run, match nothing -configvar CSI_PROW_E2E_FOCUS_LATEST '\[Feature:VolumeSnapshotDataSource\]' "non-alpha, feature-tagged tests for Kubernetes >= 1.17" +configvar CSI_PROW_E2E_FOCUS_LATEST '\[Feature:VolumeSnapshotDataSource\]' "non-alpha, feature-tagged tests for latest Kubernetes version" configvar CSI_PROW_E2E_FOCUS "$(get_versioned_variable CSI_PROW_E2E_FOCUS "${csi_prow_kubernetes_version_suffix}")" "non-alpha, feature-tagged tests" # Serial vs. parallel is always determined by these regular expressions. -# Individual regular expressions are seperated by spaces for readability +# Individual regular expressions are separated by spaces for readability # and expected to not contain spaces. Use dots instead. The complete # regex for Ginkgo will be created by joining the individual terms. configvar CSI_PROW_E2E_SERIAL '\[Serial\] \[Disruptive\]' "tags for serial E2E tests" @@ -320,7 +329,7 @@ regex_join () { # alpha in previous Kubernetes releases. This was considered too # error prone. Therefore we use E2E tests that match the Kubernetes # version that is getting tested. -configvar CSI_PROW_E2E_ALPHA_LATEST '\[Feature:' "alpha tests for Kubernetes >= 1.14" # there's no need to update this, adding a new case for CSI_PROW_E2E for a new Kubernetes is enough +configvar CSI_PROW_E2E_ALPHA_LATEST '\[Feature:' "alpha tests for latest Kubernetes version" # there's no need to update this, adding a new case for CSI_PROW_E2E for a new Kubernetes is enough configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi_prow_kubernetes_version_suffix}")" "alpha tests" # After the parallel E2E test without alpha features, a test cluster @@ -335,15 +344,18 @@ configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi # kubernetes-csi components must be updated, either by disabling # the failing test for "latest" or by updating the test and not running # it anymore for older releases. -configvar CSI_PROW_E2E_ALPHA_GATES_1_15 'VolumeSnapshotDataSource=true,ExpandCSIVolumes=true' "alpha feature gates for Kubernetes 1.15" -configvar CSI_PROW_E2E_ALPHA_GATES_1_16 'VolumeSnapshotDataSource=true' "alpha feature gates for Kubernetes 1.16" -# TODO: add new CSI_PROW_ALPHA_GATES_xxx entry for future Kubernetes releases and -# add new gates to CSI_PROW_E2E_ALPHA_GATES_LATEST. -configvar CSI_PROW_E2E_ALPHA_GATES_LATEST '' "alpha feature gates for latest Kubernetes" +configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'GenericEphemeralVolume=true,CSIStorageCapacity=true' "alpha feature gates for latest Kubernetes" configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates" # Which external-snapshotter tag to use for the snapshotter CRD and snapshot-controller deployment -configvar CSI_SNAPSHOTTER_VERSION 'v2.0.1' "external-snapshotter version tag" +default_csi_snapshotter_version () { + if [ "${CSI_PROW_KUBERNETES_VERSION}" = "latest" ] || [ "${CSI_PROW_DRIVER_CANARY}" = "canary" ]; then + echo "master" + else + echo "v3.0.2" + fi +} +configvar CSI_SNAPSHOTTER_VERSION "$(default_csi_snapshotter_version)" "external-snapshotter version tag" # Some tests are known to be unusable in a KinD cluster. For example, # stopping kubelet with "ssh systemctl stop kubelet" simply @@ -351,16 +363,25 @@ configvar CSI_SNAPSHOTTER_VERSION 'v2.0.1' "external-snapshotter version tag" # whether they can run with the current cluster provider, but until # they are, we filter them out by name. Like the other test selection # variables, this is again a space separated list of regular expressions. -# -# "different node" test skips can be removed once -# https://github.com/kubernetes/kubernetes/pull/82678 has been backported -# to all the K8s versions we test against -configvar CSI_PROW_E2E_SKIP 'Disruptive|different\s+node' "tests that need to be skipped" - -# This is the directory for additional result files. Usually set by Prow, but -# if not (for example, when invoking manually) it defaults to the work directory. -configvar ARTIFACTS "${CSI_PROW_WORK}/artifacts" "artifacts" -mkdir -p "${ARTIFACTS}" +configvar CSI_PROW_E2E_SKIP 'Disruptive' "tests that need to be skipped" + +# This creates directories that are required for testing. +ensure_paths () { + # Work directory. It has to allow running executables, therefore /tmp + # is avoided. Cleaning up after the script is intentionally left to + # the caller. + configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csiprow.XXXXXXXXXX")" "work directory" + + # This is the directory for additional result files. Usually set by Prow, but + # if not (for example, when invoking manually) it defaults to the work directory. + configvar ARTIFACTS "${CSI_PROW_WORK}/artifacts" "artifacts" + mkdir -p "${ARTIFACTS}" + + # For additional tools. + CSI_PROW_BIN="${CSI_PROW_WORK}/bin" + mkdir -p "${CSI_PROW_BIN}" + PATH="${CSI_PROW_BIN}:$PATH" +} run () { echo "$(date) $(go version | sed -e 's/.*version \(go[^ ]*\).*/\1/') $(if [ "$(pwd)" != "${REPO_DIR}" ]; then pwd; fi)\$" "$@" >&2 @@ -380,11 +401,6 @@ die () { exit 1 } -# For additional tools. -CSI_PROW_BIN="${CSI_PROW_WORK}/bin" -mkdir -p "${CSI_PROW_BIN}" -PATH="${CSI_PROW_BIN}:$PATH" - # Ensure that PATH has the desired version of the Go tools, then run command given as argument. # Empty parameter uses the already installed Go. In Prow, that version is kept up-to-date by # bumping the container image regularly. @@ -413,7 +429,7 @@ install_kind () { chmod u+x "${CSI_PROW_WORK}/bin/kind" else git_checkout https://github.com/kubernetes-sigs/kind "${GOPATH}/src/sigs.k8s.io/kind" "${CSI_PROW_KIND_VERSION}" --depth=1 && - (cd "${GOPATH}/src/sigs.k8s.io/kind" && make install INSTALL_DIR="${CSI_PROW_WORK}/bin") + (cd "${GOPATH}/src/sigs.k8s.io/kind" && run_with_go "$CSI_PROW_GO_VERSION_KIND" make install INSTALL_DIR="${CSI_PROW_WORK}/bin") fi } @@ -471,20 +487,22 @@ git_checkout () { # This clones a repo ("https://github.com/kubernetes/kubernetes") # in a certain location ("$GOPATH/src/k8s.io/kubernetes") at -# a the head of a specific branch (i.e., release-1.13, master). -# The directory cannot exist. -git_clone_branch () { - local repo path branch parent +# a the head of a specific branch (i.e., release-1.13, master), +# tag (v1.20.0) or commit. +# +# The directory must not exist. +git_clone () { + local repo path name parent repo="$1" shift path="$1" shift - branch="$1" + name="$1" shift parent="$(dirname "$path")" mkdir -p "$parent" - (cd "$parent" && run git clone --single-branch --branch "$branch" "$repo" "$path") || die "cloning $repo" failed + (cd "$parent" && run git clone --single-branch --branch "$name" "$repo" "$path") || die "cloning $repo" failed # This is useful for local testing or when switching between different revisions in the same # repo. (cd "$path" && run git clean -fdx) || die "failed to clean $path" @@ -503,6 +521,22 @@ list_gates () ( done ) +# Turn feature gates in the format foo=true,bar=false into +# a YAML map with the corresponding API groups for use +# with https://kind.sigs.k8s.io/docs/user/configuration/#runtime-config +list_api_groups () ( + set -f; IFS=',' + # Ignore: Double quote to prevent globbing and word splitting. + # shellcheck disable=SC2086 + set -- $1 + while [ "$1" ]; do + if [ "$1" = 'CSIStorageCapacity=true' ]; then + echo ' "storage.k8s.io/v1alpha1": "true"' + fi + shift + done +) + go_version_for_kubernetes () ( local path="$1" local version="$2" @@ -532,77 +566,48 @@ start_cluster () { run kind delete cluster --name=csi-prow || die "kind delete failed" fi - # Build from source? - if [[ "${CSI_PROW_KUBERNETES_VERSION}" =~ ^release-|^latest$ ]]; then + # Try to find a pre-built kind image if asked to use a specific version. + if ! [[ "${CSI_PROW_KUBERNETES_VERSION}" =~ ^release-|^latest$ ]]; then + # Ignore: See if you can use ${variable//search/replace} instead. + # shellcheck disable=SC2001 + major_minor=$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/^\([0-9]*\)\.\([0-9]*\).*/\1.\2/') + for i in ${CSI_PROW_KIND_IMAGES}; do + if echo "$i" | grep -q "kindest/node:v${major_minor}"; then + image="$i" + break + fi + done + fi + + # Need to build from source? + if ! [ "$image" ]; then if ! ${csi_prow_kind_have_kubernetes}; then local version="${CSI_PROW_KUBERNETES_VERSION}" if [ "$version" = "latest" ]; then version=master fi - git_clone_branch https://github.com/kubernetes/kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$version" || die "checking out Kubernetes $version failed" + git_clone https://github.com/kubernetes/kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$(version_to_git "$version")" || die "checking out Kubernetes $version failed" go_version="$(go_version_for_kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$version")" || die "cannot proceed without knowing Go version for Kubernetes" - run_with_go "$go_version" kind build node-image --type bazel --image csiprow/node:latest --kube-root "${CSI_PROW_WORK}/src/kubernetes" || die "'kind build node-image' failed" + # Changing into the Kubernetes source code directory is a workaround for https://github.com/kubernetes-sigs/kind/issues/1910 + # shellcheck disable=SC2046 + (cd "${CSI_PROW_WORK}/src/kubernetes" && run_with_go "$go_version" kind build node-image --image csiprow/node:latest --kube-root "${CSI_PROW_WORK}/src/kubernetes") || die "'kind build node-image' failed" csi_prow_kind_have_kubernetes=true fi image="csiprow/node:latest" - else - image="kindest/node:v${CSI_PROW_KUBERNETES_VERSION}" fi cat >"${CSI_PROW_WORK}/kind-config.yaml" <>"${CSI_PROW_WORK}/kind-config.yaml" < kubernetes-latest-test. + # Ignore: See if you can use ${variable//search/replace} instead. + # shellcheck disable=SC2001 + file="$dir/$(echo "$deployment" | sed -e 's/[0-9][0-9]*\.[0-9][0-9]*/latest/')/deploy.sh" if ! [ -e "$file" ]; then return 1 fi @@ -693,7 +717,11 @@ install_csi_driver () { fi if [ "${CSI_PROW_DRIVER_CANARY}" != "stable" ]; then + if [ "${CSI_PROW_DRIVER_CANARY}" == "canary" ]; then + images="$images IMAGE_TAG=${CSI_PROW_DRIVER_CANARY} IMAGE_REGISTRY=${CSI_PROW_DRIVER_CANARY_REGISTRY}" + else images="$images IMAGE_TAG=${CSI_PROW_DRIVER_CANARY}" + fi fi # Ignore: Double quote to prevent globbing and word splitting. # It's intentional here for $images. @@ -707,10 +735,14 @@ install_csi_driver () { fi } -# Installs all nessesary snapshotter CRDs +# Installs all nessesary snapshotter CRDs install_snapshot_crds() { # Wait until volumesnapshot CRDs are in place. - CRD_BASE_DIR="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/config/crd" + CRD_BASE_DIR="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/client/config/crd" + if [[ ${REPO_DIR} == *"external-snapshotter"* ]]; then + CRD_BASE_DIR="${REPO_DIR}/client/config/crd" + fi + echo "Installing snapshot CRDs from ${CRD_BASE_DIR}" kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshotclasses.yaml" --validate=false kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshots.yaml" --validate=false kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshotcontents.yaml" --validate=false @@ -730,7 +762,16 @@ install_snapshot_crds() { # Install snapshot controller and associated RBAC, retrying until the pod is running. install_snapshot_controller() { - kubectl apply -f "https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml" + CONTROLLER_DIR="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}" + if [[ ${REPO_DIR} == *"external-snapshotter"* ]]; then + CONTROLLER_DIR="${REPO_DIR}" + fi + SNAPSHOT_RBAC_YAML="${CONTROLLER_DIR}/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml" + echo "kubectl apply -f ${SNAPSHOT_RBAC_YAML}" + # Ignore: Double quote to prevent globbing and word splitting. + # shellcheck disable=SC2086 + kubectl apply -f ${SNAPSHOT_RBAC_YAML} + cnt=0 until kubectl get clusterrolebinding snapshot-controller-role; do if [ $cnt -gt 30 ]; then @@ -741,23 +782,76 @@ install_snapshot_controller() { fi echo "$(date +%H:%M:%S)" "waiting for snapshot RBAC setup complete, attempt #$cnt" cnt=$((cnt + 1)) - sleep 10 + sleep 10 done + SNAPSHOT_CONTROLLER_YAML="${CONTROLLER_DIR}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml" + if [[ ${REPO_DIR} == *"external-snapshotter"* ]]; then + # snapshot-controller image built from the PR will get a "csiprow" tag. + # Load it into the "kind" cluster so that we can deploy it. + NEW_TAG="csiprow" + NEW_IMG="snapshot-controller:${NEW_TAG}" + echo "kind load docker-image --name csi-prow ${NEW_IMG}" + kind load docker-image --name csi-prow ${NEW_IMG} || die "could not load the snapshot-controller:csiprow image into the kind cluster" + + # deploy snapshot-controller + echo "Deploying snapshot-controller" + # Replace image in SNAPSHOT_CONTROLLER_YAML with snapshot-controller:csiprow and deploy + # NOTE: This logic is similar to the logic here: + # https://github.com/kubernetes-csi/csi-driver-host-path/blob/v1.4.0/deploy/util/deploy-hostpath.sh#L155 + # Ignore: Double quote to prevent globbing and word splitting. + # shellcheck disable=SC2086 + # Ignore: Use find instead of ls to better handle non-alphanumeric filenames. + # shellcheck disable=SC2012 + for i in $(ls ${SNAPSHOT_CONTROLLER_YAML} | sort); do + echo " $i" + # Ignore: Useless cat. Consider 'cmd < file | ..' or 'cmd file | ..' instead. + # shellcheck disable=SC2002 + # Ignore: See if you can use ${variable//search/replace} instead. + # shellcheck disable=SC2001 + modified="$(cat "$i" | while IFS= read -r line; do + nocomments="$(echo "$line" | sed -e 's/ *#.*$//')" + if echo "$nocomments" | grep -q '^[[:space:]]*image:[[:space:]]*'; then + # Split 'image: k8s.gcr.io/sig-storage/snapshot-controller:v3.0.0' + # into image (snapshot-controller:v3.0.0), + # name (snapshot-controller), + # tag (v3.0.0). + image=$(echo "$nocomments" | sed -e 's;.*image:[[:space:]]*;;') + name=$(echo "$image" | sed -e 's;.*/\([^:]*\).*;\1;') + tag=$(echo "$image" | sed -e 's;.*:;;') + + # Now replace registry and/or tag + NEW_TAG="csiprow" + line="$(echo "$nocomments" | sed -e "s;$image;${name}:${NEW_TAG};")" + echo " using $line" >&2 + fi + echo "$line" + done)" + if ! echo "$modified" | kubectl apply -f -; then + echo "modified version of $i:" + echo "$modified" + exit 1 + fi + echo "kubectl apply -f ${SNAPSHOT_CONTROLLER_YAML}(modified)" + done + else + echo "kubectl apply -f $SNAPSHOT_CONTROLLER_YAML" + kubectl apply -f "$SNAPSHOT_CONTROLLER_YAML" + fi - kubectl apply -f "https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml" cnt=0 - expected_running_pods=$(curl https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${CSI_SNAPSHOTTER_VERSION}"/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml | grep replicas | cut -d ':' -f 2-) - while [ "$(kubectl get pods -l app=snapshot-controller | grep 'Running' -c)" -lt "$expected_running_pods" ]; do + expected_running_pods=$(kubectl apply --dry-run=client -o "jsonpath={.spec.replicas}" -f "$SNAPSHOT_CONTROLLER_YAML") + expected_namespace=$(kubectl apply --dry-run=client -o "jsonpath={.metadata.namespace}" -f "$SNAPSHOT_CONTROLLER_YAML") + while [ "$(kubectl get pods -n "$expected_namespace" -l app=snapshot-controller | grep 'Running' -c)" -lt "$expected_running_pods" ]; do if [ $cnt -gt 30 ]; then echo "snapshot-controller pod status:" - kubectl describe pods -l app=snapshot-controller + kubectl describe pods -n "$expected_namespace" -l app=snapshot-controller echo >&2 "ERROR: snapshot controller not ready after over 5 min" exit 1 fi echo "$(date +%H:%M:%S)" "waiting for snapshot controller deployment to complete, attempt #$cnt" cnt=$((cnt + 1)) - sleep 10 + sleep 10 done } @@ -802,6 +896,29 @@ start_loggers () { done } +# Patches the image versions of test/e2e/testing-manifests/storage-csi/mock in the k/k +# source code, if needed. +patch_kubernetes () { + local source="$1" target="$2" + + if [ "${CSI_PROW_DRIVER_CANARY}" = "canary" ]; then + # We cannot replace k8s.gcr.io/sig-storage with gcr.io/k8s-staging-sig-storage because + # e2e.test does not support it (see test/utils/image/manifest.go). Instead we + # invoke the e2e.test binary with KUBE_TEST_REPO_LIST set to a file that + # overrides that registry. + find "$source/test/e2e/testing-manifests/storage-csi/mock" -name '*.yaml' -print0 | xargs -0 sed -i -e 's;k8s.gcr.io/sig-storage/\(.*\):v.*;k8s.gcr.io/sig-storage/\1:canary;' + cat >"$target/e2e-repo-list" <&2 <&1) + +EOF + fi +} + # Makes the E2E test suite binary available as "${CSI_PROW_WORK}/e2e.test". install_e2e () { if [ -e "${CSI_PROW_WORK}/e2e.test" ]; then @@ -810,6 +927,7 @@ install_e2e () { git_checkout "${CSI_PROW_E2E_REPO}" "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_E2E_VERSION}" --depth=1 && if [ "${CSI_PROW_E2E_IMPORT_PATH}" = "k8s.io/kubernetes" ]; then + patch_kubernetes "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_WORK}" && go_version="${CSI_PROW_GO_VERSION_E2E:-$(go_version_for_kubernetes "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_E2E_VERSION}")}" && run_with_go "$go_version" make WHAT=test/e2e/e2e.test "-C${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" && ln -s "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}/_output/bin/e2e.test" "${CSI_PROW_WORK}" @@ -825,8 +943,8 @@ install_sanity () ( return fi - git_checkout "${CSI_PROW_SANITY_REPO}" "${GOPATH}/src/${CSI_PROW_SANITY_IMPORT_PATH}" "${CSI_PROW_SANITY_VERSION}" --depth=1 || die "checking out csi-sanity failed" - run_with_go "${CSI_PROW_GO_VERSION_SANITY}" go test -c -o "${CSI_PROW_WORK}/csi-sanity" "${CSI_PROW_SANITY_IMPORT_PATH}/cmd/csi-sanity" || die "building csi-sanity failed" + git_checkout "${CSI_PROW_SANITY_REPO}" "${GOPATH}/src/${CSI_PROW_SANITY_PACKAGE_PATH}" "${CSI_PROW_SANITY_VERSION}" --depth=1 || die "checking out csi-sanity failed" + ( cd "${GOPATH}/src/${CSI_PROW_SANITY_PACKAGE_PATH}/cmd/csi-sanity" && run_with_go "${CSI_PROW_GO_VERSION_SANITY}" go build -o "${CSI_PROW_WORK}/csi-sanity" ) || die "building csi-sanity failed" ) # Captures pod output while running some other command. @@ -855,25 +973,33 @@ run_e2e () ( # the full Kubernetes E2E testsuite while only running a few tests. move_junit () { if ls "${ARTIFACTS}"/junit_[0-9]*.xml 2>/dev/null >/dev/null; then - run_filter_junit -t="External Storage" -o "${ARTIFACTS}/junit_${name}.xml" "${ARTIFACTS}"/junit_[0-9]*.xml && rm -f "${ARTIFACTS}"/junit_[0-9]*.xml + run_filter_junit -t="External.Storage|CSI.mock.volume" -o "${ARTIFACTS}/junit_${name}.xml" "${ARTIFACTS}"/junit_[0-9]*.xml && rm -f "${ARTIFACTS}"/junit_[0-9]*.xml fi } trap move_junit EXIT cd "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" && - run_with_loggers ginkgo -v "$@" "${CSI_PROW_WORK}/e2e.test" -- -report-dir "${ARTIFACTS}" -storage.testdriver="${CSI_PROW_WORK}/test-driver.yaml" + run_with_loggers env KUBECONFIG="$KUBECONFIG" KUBE_TEST_REPO_LIST="$(if [ -e "${CSI_PROW_WORK}/e2e-repo-list" ]; then echo "${CSI_PROW_WORK}/e2e-repo-list"; fi)" ginkgo -v "$@" "${CSI_PROW_WORK}/e2e.test" -- -report-dir "${ARTIFACTS}" -storage.testdriver="${CSI_PROW_WORK}/test-driver.yaml" ) # Run csi-sanity against installed CSI driver. run_sanity () ( install_sanity || die "installing csi-sanity failed" + if [[ "${CSI_PROW_SANITY_POD}" =~ " " ]]; then + # Contains spaces, more complex than a simple pod name. + # Evaluate as a shell command. + pod=$(eval "${CSI_PROW_SANITY_POD}") || die "evaluation failed: CSI_PROW_SANITY_POD=${CSI_PROW_SANITY_POD}" + else + pod="${CSI_PROW_SANITY_POD}" + fi + cat >"${CSI_PROW_WORK}/mkdir_in_pod.sh" <"${CSI_PROW_WORK}/rmdir_in_pod.sh" <"${CSI_PROW_WORK}/checkdir_in_pod.sh" <" >>"$out" echo " " >>"$out" fi @@ -1018,7 +1163,7 @@ make_test_to_junit () { # version_gt 1.3.1 v1.2.0 (returns true) # version_gt 1.1.1 release-1.2.0 (returns false) # version_gt 1.2.0 1.2.2 (returns false) -function version_gt() { +function version_gt() { versions=$(for ver in "$@"; do ver=${ver#release-}; ver=${ver#kubernetes-}; echo "${ver#v}"; done) greaterVersion=${1#"release-"}; greaterVersion=${greaterVersion#"kubernetes-"}; @@ -1030,6 +1175,9 @@ main () { local images ret ret=0 + # Set up work directory. + ensure_paths + images= if ${CSI_PROW_BUILD_JOB}; then # A successful build is required for testing. @@ -1078,7 +1226,7 @@ main () { if [ "$rbac_file_path" == "" ]; then rbac_file_path="$(pwd)/deploy/kubernetes/rbac.yaml" fi - + if [ -e "$rbac_file_path" ]; then # This is one of those components which has its own RBAC rules (like external-provisioner). # We are testing a locally built image and also want to test with the the current, @@ -1089,18 +1237,19 @@ main () { done fi + # Run the external driver tests and optionally also mock tests. + local focus="External.Storage" + if "$CSI_PROW_E2E_MOCK"; then + focus="($focus|CSI.mock.volume)" + fi + if tests_need_non_alpha_cluster; then start_cluster || die "starting the non-alpha cluster failed" # Install necessary snapshot CRDs and snapshot controller - # For Kubernetes 1.17+, we will install the CRDs and snapshot controller. - if version_gt "${CSI_PROW_KUBERNETES_VERSION}" "1.16.255" || "${CSI_PROW_KUBERNETES_VERSION}" == "latest"; then - info "Version ${CSI_PROW_KUBERNETES_VERSION}, installing CRDs and snapshot controller" - install_snapshot_crds - install_snapshot_controller - else - info "Version ${CSI_PROW_KUBERNETES_VERSION}, skipping CRDs and snapshot controller" - fi + install_snapshot_crds + install_snapshot_controller + # Installing the driver might be disabled. if ${CSI_PROW_DRIVER_INSTALL} "$images"; then @@ -1116,7 +1265,7 @@ main () { # Ignore: Double quote to prevent globbing and word splitting. # shellcheck disable=SC2086 if ! run_e2e parallel ${CSI_PROW_GINKO_PARALLEL} \ - -focus="External.Storage" \ + -focus="$focus" \ -skip="$(regex_join "${CSI_PROW_E2E_SERIAL}" "${CSI_PROW_E2E_ALPHA}" "${CSI_PROW_E2E_SKIP}")"; then warn "E2E parallel failed" ret=1 @@ -1126,7 +1275,7 @@ main () { # Ignore: Double quote to prevent globbing and word splitting. # shellcheck disable=SC2086 if ! run_e2e parallel-features ${CSI_PROW_GINKO_PARALLEL} \ - -focus="External.Storage.*($(regex_join "${CSI_PROW_E2E_FOCUS}"))" \ + -focus="$focus.*($(regex_join "${CSI_PROW_E2E_FOCUS}"))" \ -skip="$(regex_join "${CSI_PROW_E2E_SERIAL}")"; then warn "E2E parallel features failed" ret=1 @@ -1135,14 +1284,14 @@ main () { if tests_enabled "serial"; then if ! run_e2e serial \ - -focus="External.Storage.*($(regex_join "${CSI_PROW_E2E_SERIAL}"))" \ + -focus="$focus.*($(regex_join "${CSI_PROW_E2E_SERIAL}"))" \ -skip="$(regex_join "${CSI_PROW_E2E_ALPHA}" "${CSI_PROW_E2E_SKIP}")"; then warn "E2E serial failed" ret=1 fi fi fi - delete_cluster_inside_prow_job + delete_cluster_inside_prow_job non-alpha fi if tests_need_alpha_cluster && [ "${CSI_PROW_E2E_ALPHA_GATES}" ]; then @@ -1150,14 +1299,8 @@ main () { start_cluster "${CSI_PROW_E2E_ALPHA_GATES}" || die "starting alpha cluster failed" # Install necessary snapshot CRDs and snapshot controller - # For Kubernetes 1.17+, we will install the CRDs and snapshot controller. - if version_gt "${CSI_PROW_KUBERNETES_VERSION}" "1.16.255" || "${CSI_PROW_KUBERNETES_VERSION}" == "latest"; then - info "Version ${CSI_PROW_KUBERNETES_VERSION}, installing CRDs and snapshot controller" - install_snapshot_crds - install_snapshot_controller - else - info "Version ${CSI_PROW_KUBERNETES_VERSION}, skipping CRDs and snapshot controller" - fi + install_snapshot_crds + install_snapshot_controller # Installing the driver might be disabled. if ${CSI_PROW_DRIVER_INSTALL} "$images"; then @@ -1167,7 +1310,7 @@ main () { # Ignore: Double quote to prevent globbing and word splitting. # shellcheck disable=SC2086 if ! run_e2e parallel-alpha ${CSI_PROW_GINKO_PARALLEL} \ - -focus="External.Storage.*($(regex_join "${CSI_PROW_E2E_ALPHA}"))" \ + -focus="$focus.*($(regex_join "${CSI_PROW_E2E_ALPHA}"))" \ -skip="$(regex_join "${CSI_PROW_E2E_SERIAL}" "${CSI_PROW_E2E_SKIP}")"; then warn "E2E parallel alpha failed" ret=1 @@ -1176,14 +1319,14 @@ main () { if tests_enabled "serial-alpha"; then if ! run_e2e serial-alpha \ - -focus="External.Storage.*(($(regex_join "${CSI_PROW_E2E_SERIAL}")).*($(regex_join "${CSI_PROW_E2E_ALPHA}"))|($(regex_join "${CSI_PROW_E2E_ALPHA}")).*($(regex_join "${CSI_PROW_E2E_SERIAL}")))" \ + -focus="$focus.*(($(regex_join "${CSI_PROW_E2E_SERIAL}")).*($(regex_join "${CSI_PROW_E2E_ALPHA}"))|($(regex_join "${CSI_PROW_E2E_ALPHA}")).*($(regex_join "${CSI_PROW_E2E_SERIAL}")))" \ -skip="$(regex_join "${CSI_PROW_E2E_SKIP}")"; then warn "E2E serial alpha failed" ret=1 fi fi fi - delete_cluster_inside_prow_job + delete_cluster_inside_prow_job alpha fi fi @@ -1203,6 +1346,9 @@ gcr_cloud_build () { # Required for "docker buildx build --push". gcloud auth configure-docker + # Might not be needed here, but call it just in case. + ensure_paths + if find . -name Dockerfile | grep -v ^./vendor | xargs --no-run-if-empty cat | grep -q ^RUN; then # Needed for "RUN" steps on non-linux/amd64 platforms. # See https://github.com/multiarch/qemu-user-static#getting-started diff --git a/pull-test.sh b/pull-test.sh new file mode 100755 index 000000000..b019c1776 --- /dev/null +++ b/pull-test.sh @@ -0,0 +1,32 @@ +#! /bin/sh + +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is called by pull Prow jobs for the csi-release-tools +# repo to ensure that the changes in the PR work when imported into +# some other repo. + +set -ex + +# It must be called inside the updated csi-release-tools repo. +CSI_RELEASE_TOOLS_DIR="$(pwd)" + +# Update the other repo. +cd "$PULL_TEST_REPO_DIR" +git subtree pull --squash --prefix=release-tools "$CSI_RELEASE_TOOLS_DIR" master +git log -n2 + +# Now fall through to testing. +exec ./.prow.sh diff --git a/travis.yml b/travis.yml deleted file mode 100644 index 1ab13aef2..000000000 --- a/travis.yml +++ /dev/null @@ -1,21 +0,0 @@ -language: go -sudo: required -services: - - docker -git: - depth: false -matrix: - include: - - go: 1.15 -before_script: -- mkdir -p bin -- wget https://github.com/golang/dep/releases/download/v0.5.1/dep-linux-amd64 -O bin/dep -- chmod u+x bin/dep -- export PATH=$PWD/bin:$PATH -script: -- make -k all test GOFLAGS_VENDOR=$( [ -d vendor ] && echo '-mod=vendor' ) -after_success: - - if [ "${TRAVIS_PULL_REQUEST}" == "false" ]; then - docker login -u "${DOCKER_USERNAME}" -p "${DOCKER_PASSWORD}" quay.io; - make push GOFLAGS_VENDOR=$( [ -d vendor ] && echo '-mod=vendor' ); - fi diff --git a/verify-boilerplate.sh b/verify-boilerplate.sh new file mode 100755 index 000000000..815939574 --- /dev/null +++ b/verify-boilerplate.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +echo "Verifying boilerplate" + +if [[ -z "$(command -v python)" ]]; then + echo "Cannot find python. Make link to python3..." + update-alternatives --install /usr/bin/python python /usr/bin/python3 1 +fi + +# The csi-release-tools directory (absolute path). +TOOLS="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" + +# Directory to check. Default is the parent of the tools themselves. +ROOT="${1:-${TOOLS}/..}" + +boiler="${TOOLS}/boilerplate/boilerplate.py" + +mapfile -t files_need_boilerplate < <("${boiler}" --rootdir="${ROOT}" --verbose) + +# Run boilerplate.py unit tests +unitTestOut="$(mktemp)" +trap cleanup EXIT +cleanup() { + rm "${unitTestOut}" +} + +# Run boilerplate check +if [[ ${#files_need_boilerplate[@]} -gt 0 ]]; then + for file in "${files_need_boilerplate[@]}"; do + echo "Boilerplate header is wrong for: ${file}" + done + + exit 1 +fi + +echo "Done" diff --git a/verify-go-version.sh b/verify-go-version.sh index f242e769d..c235e74fe 100755 --- a/verify-go-version.sh +++ b/verify-go-version.sh @@ -29,8 +29,9 @@ die () { version=$("$GO" version) || die "determining version of $GO failed" # shellcheck disable=SC2001 majorminor=$(echo "$version" | sed -e 's/.*go\([0-9]*\)\.\([0-9]*\).*/\1.\2/') -# shellcheck disable=SC2001 -expected=$(grep "^ *- go:" "release-tools/travis.yml" | sed -e 's/.*go: *\([0-9]*\)\.\([0-9]*\).*/\1.\2/') +# SC1091: Not following: release-tools/prow.sh was not specified as input (see shellcheck -x). +# shellcheck disable=SC1091 +expected=$(. release-tools/prow.sh >/dev/null && echo "$CSI_PROW_GO_VERSION_BUILD") if [ "$majorminor" != "$expected" ]; then cat >&2 </dev/null; then +if command -v shellcheck &>/dev/null; then detected_version="$(shellcheck --version | grep 'version: .*')" if [[ "${detected_version}" = "version: ${SHELLCHECK_VERSION}" ]]; then HAVE_SHELLCHECK=true diff --git a/verify-spelling.sh b/verify-spelling.sh new file mode 100755 index 000000000..4aeb34d68 --- /dev/null +++ b/verify-spelling.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +TOOL_VERSION="v0.3.4" + +# The csi-release-tools directory (absolute path). +TOOLS="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" + +# Directory to check. Default is the parent of the tools themselves. +ROOT="${1:-${TOOLS}/..}" + +# create a temporary directory +TMP_DIR=$(mktemp -d) + +# cleanup +exitHandler() ( + echo "Cleaning up..." + rm -rf "${TMP_DIR}" +) +trap exitHandler EXIT + +if [[ -z "$(command -v misspell)" ]]; then + echo "Cannot find misspell. Installing misspell..." + # perform go get in a temp dir as we are not tracking this version in a go module + # if we do the go get in the repo, it will create / update a go.mod and go.sum + cd "${TMP_DIR}" + GO111MODULE=on GOBIN="${TMP_DIR}" go get "github.com/client9/misspell/cmd/misspell@${TOOL_VERSION}" + export PATH="${TMP_DIR}:${PATH}" +fi + +# check spelling +RES=0 +echo "Checking spelling..." +ERROR_LOG="${TMP_DIR}/errors.log" +cd "${ROOT}" +git ls-files | grep -v vendor | xargs misspell > "${ERROR_LOG}" +if [[ -s "${ERROR_LOG}" ]]; then + sed 's/^/error: /' "${ERROR_LOG}" # add 'error' to each line to highlight in e2e status + echo "Found spelling errors!" + RES=1 +fi +exit "${RES}" diff --git a/verify-subtree.sh b/verify-subtree.sh index f04a9fa26..aa72194a4 100755 --- a/verify-subtree.sh +++ b/verify-subtree.sh @@ -1,5 +1,5 @@ #! /bin/sh -e -# + # Copyright 2019 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); From 72d200ff0fae191a22692f2c19bd60564f4a8f4d Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 2 Aug 2021 21:51:09 +0200 Subject: [PATCH 2/2] csi-release-tools: introduce owner aliases This is needed for the OWNERS file in release-tools. --- OWNERS_ALIASES | 1 + 1 file changed, 1 insertion(+) create mode 120000 OWNERS_ALIASES diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES new file mode 120000 index 000000000..3f60d4615 --- /dev/null +++ b/OWNERS_ALIASES @@ -0,0 +1 @@ +release-tools/KUBERNETES_CSI_OWNERS_ALIASES \ No newline at end of file