Skip to content

Commit d0cc023

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request kubernetes-retired#5 from brahmaroutu/add_release_tools
Adding release tools for CI
2 parents 3ef85b0 + 7277b14 commit d0cc023

12 files changed

+2111
-0
lines changed

‎release-tools/.prow.sh

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#! /bin/bash -e
2+
#
3+
# This is for testing csi-release-tools itself in Prow. All other
4+
# repos use prow.sh for that, but as csi-release-tools isn't a normal
5+
# repo with some Go code in it, it has a custom Prow test script.
6+
7+
./verify-shellcheck.sh "$(pwd)"

‎release-tools/build.make

+194
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
# Copyright 2020 The Kubernetes Authors.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
.PHONY: build-% build container-% container push-% push clean test
16+
17+
# A space-separated list of all commands in the repository, must be
18+
# set in main Makefile of a repository.
19+
# CMDS=
20+
21+
# This is the default. It can be overridden in the main Makefile after
22+
# including build.make.
23+
REGISTRY_NAME=quay.io/k8scosi
24+
25+
# Can be set to -mod=vendor to ensure that the "vendor" directory is used.
26+
GOFLAGS_VENDOR=
27+
28+
# Revision that gets built into each binary via the main.version
29+
# string. Uses the `git describe` output based on the most recent
30+
# version tag with a short revision suffix or, if nothing has been
31+
# tagged yet, just the revision.
32+
#
33+
# Beware that tags may also be missing in shallow clones as done by
34+
# some CI systems (like TravisCI, which pulls only 50 commits).
35+
REV=$(shell git describe --long --tags --match='v*' --dirty 2>/dev/null || git rev-list -n1 HEAD)
36+
37+
# A space-separated list of image tags under which the current build is to be pushed.
38+
# Determined dynamically.
39+
IMAGE_TAGS=
40+
41+
# A "canary" image gets built if the current commit is the head of the remote "master" branch.
42+
# That branch does not exist when building some other branch in TravisCI.
43+
IMAGE_TAGS+=$(shell if [ "$$(git rev-list -n1 HEAD)" = "$$(git rev-list -n1 origin/master 2>/dev/null)" ]; then echo "canary"; fi)
44+
45+
# A "X.Y.Z-canary" image gets built if the current commit is the head of a "origin/release-X.Y.Z" branch.
46+
# The actual suffix does not matter, only the "release-" prefix is checked.
47+
IMAGE_TAGS+=$(shell git branch -r --points-at=HEAD | grep 'origin/release-' | grep -v -e ' -> ' | sed -e 's;.*/release-\(.*\);\1-canary;')
48+
49+
# A release image "vX.Y.Z" gets built if there is a tag of that format for the current commit.
50+
# --abbrev=0 suppresses long format, only showing the closest tag.
51+
IMAGE_TAGS+=$(shell tagged="$$(git describe --tags --match='v*' --abbrev=0)"; if [ "$$tagged" ] && [ "$$(git rev-list -n1 HEAD)" = "$$(git rev-list -n1 $$tagged)" ]; then echo $$tagged; fi)
52+
53+
# Images are named after the command contained in them.
54+
IMAGE_NAME=$(REGISTRY_NAME)/$*
55+
56+
ifdef V
57+
# Adding "-alsologtostderr" assumes that all test binaries contain glog. This is not guaranteed.
58+
TESTARGS = -v -args -alsologtostderr -v 5
59+
else
60+
TESTARGS =
61+
endif
62+
63+
ARCH := $(if $(GOARCH),$(GOARCH),$(shell go env GOARCH))
64+
65+
# Specific packages can be excluded from each of the tests below by setting the *_FILTER_CMD variables
66+
# to something like "| grep -v 'github.com/kubernetes-csi/project/pkg/foobar'". See usage below.
67+
68+
build-%: check-go-version-go
69+
mkdir -p bin
70+
CGO_ENABLED=0 GOOS=linux go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$*
71+
if [ "$$ARCH" = "amd64" ]; then \
72+
CGO_ENABLED=0 GOOS=windows go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*.exe ./cmd/$* ; \
73+
CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*-ppc64le ./cmd/$* ; \
74+
fi
75+
76+
container-%: build-%
77+
docker build -t $*:latest -f $(shell if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi) --label revision=$(REV) .
78+
79+
push-%: container-%
80+
set -ex; \
81+
push_image () { \
82+
docker tag $*:latest $(IMAGE_NAME):$$tag; \
83+
docker push $(IMAGE_NAME):$$tag; \
84+
}; \
85+
for tag in $(IMAGE_TAGS); do \
86+
if [ "$$tag" = "canary" ] || echo "$$tag" | grep -q -e '-canary$$'; then \
87+
: "creating or overwriting canary image"; \
88+
push_image; \
89+
elif docker pull $(IMAGE_NAME):$$tag 2>&1 | tee /dev/stderr | grep -q "manifest for $(IMAGE_NAME):$$tag not found"; then \
90+
: "creating release image"; \
91+
push_image; \
92+
else \
93+
: "release image $(IMAGE_NAME):$$tag already exists, skipping push"; \
94+
fi; \
95+
done
96+
97+
build: $(CMDS:%=build-%)
98+
container: $(CMDS:%=container-%)
99+
push: $(CMDS:%=push-%)
100+
101+
clean:
102+
-rm -rf bin
103+
104+
test: check-go-version-go
105+
106+
.PHONY: test-go
107+
test: test-go
108+
test-go:
109+
@ echo; echo "### $@:"
110+
go test $(GOFLAGS_VENDOR) `go list $(GOFLAGS_VENDOR) ./... | grep -v -e 'vendor' -e '/test/e2e$$' $(TEST_GO_FILTER_CMD)` $(TESTARGS)
111+
112+
.PHONY: test-vet
113+
test: test-vet
114+
test-vet:
115+
@ echo; echo "### $@:"
116+
go vet $(GOFLAGS_VENDOR) `go list $(GOFLAGS_VENDOR) ./... | grep -v vendor $(TEST_VET_FILTER_CMD)`
117+
118+
.PHONY: test-fmt
119+
test: test-fmt
120+
test-fmt:
121+
@ echo; echo "### $@:"
122+
files=$$(find . -name '*.go' | grep -v './vendor' $(TEST_FMT_FILTER_CMD)); \
123+
if [ $$(gofmt -d $$files | wc -l) -ne 0 ]; then \
124+
echo "formatting errors:"; \
125+
gofmt -d $$files; \
126+
false; \
127+
fi
128+
129+
# This test only runs when dep >= 0.5 is installed, which is the case for the CI setup.
130+
# When using 'go mod', we allow the test to be skipped in the Prow CI under some special
131+
# circumstances, because it depends on accessing all remote repos and thus
132+
# running it all the time would defeat the purpose of vendoring:
133+
# - not handling a PR or
134+
# - the fabricated merge commit leaves go.mod, go.sum and vendor dir unchanged
135+
# - release-tools also didn't change (changing rules or Go version might lead to
136+
# a different result and thus must be tested)
137+
# - import statements not changed (because if they change, go.mod might have to be updated)
138+
#
139+
# "git diff" is intelligent enough to annotate changes inside the "import" block in
140+
# the start of the diff hunk:
141+
#
142+
# diff --git a/rpc/common.go b/rpc/common.go
143+
# index bb4a5c4..5fa4271 100644
144+
# --- a/rpc/common.go
145+
# +++ b/rpc/common.go
146+
# @@ -21,7 +21,6 @@ import (
147+
# "fmt"
148+
# "time"
149+
#
150+
# - "google.golang.org/grpc"
151+
# "google.golang.org/grpc/codes"
152+
# "google.golang.org/grpc/status"
153+
#
154+
# We rely on that to find such changes.
155+
#
156+
# Vendoring is optional when using go.mod.
157+
.PHONY: test-vendor
158+
test: test-vendor
159+
test-vendor:
160+
@ echo; echo "### $@:"
161+
@ ./release-tools/verify-vendor.sh
162+
163+
.PHONY: test-subtree
164+
test: test-subtree
165+
test-subtree:
166+
@ echo; echo "### $@:"
167+
./release-tools/verify-subtree.sh release-tools
168+
169+
# Components can extend the set of directories which must pass shellcheck.
170+
# The default is to check only the release-tools directory itself.
171+
TEST_SHELLCHECK_DIRS=release-tools
172+
.PHONY: test-shellcheck
173+
test: test-shellcheck
174+
test-shellcheck:
175+
@ echo; echo "### $@:"
176+
@ ret=0; \
177+
if ! command -v docker; then \
178+
echo "skipped, no Docker"; \
179+
exit 0; \
180+
fi; \
181+
for dir in $(abspath $(TEST_SHELLCHECK_DIRS)); do \
182+
echo; \
183+
echo "$$dir:"; \
184+
./release-tools/verify-shellcheck.sh "$$dir" || ret=1; \
185+
done; \
186+
exit $$ret
187+
188+
# Targets in the makefile can depend on check-go-version-<path to go binary>
189+
# to trigger a warning if the x.y version of that binary does not match
190+
# what the project uses. Make ensures that this is only checked once per
191+
# invocation.
192+
.PHONY: check-go-version-%
193+
check-go-version-%:
194+
./release-tools/verify-go-version.sh "$*"

‎release-tools/filter-junit.go

+133
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
/*
2+
Copyright 2020 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
/*
18+
* This command filters a JUnit file such that only tests with a name
19+
* matching a regular expression are passed through. By concatenating
20+
* multiple input files it is possible to merge them into a single file.
21+
*/
22+
package main
23+
24+
import (
25+
"encoding/xml"
26+
"flag"
27+
"io/ioutil"
28+
"os"
29+
"regexp"
30+
)
31+
32+
var (
33+
output = flag.String("o", "-", "junit file to write, - for stdout")
34+
tests = flag.String("t", "", "regular expression matching the test names that are to be included in the output")
35+
)
36+
37+
/*
38+
* TestSuite represents a JUnit file. Due to how encoding/xml works, we have
39+
* represent all fields that we want to be passed through. It's therefore
40+
* not a complete solution, but good enough for Ginkgo + Spyglass.
41+
*/
42+
type TestSuite struct {
43+
XMLName string `xml:"testsuite"`
44+
TestCases []TestCase `xml:"testcase"`
45+
}
46+
47+
type TestCase struct {
48+
Name string `xml:"name,attr"`
49+
Time string `xml:"time,attr"`
50+
SystemOut string `xml:"system-out,omitempty"`
51+
Failure string `xml:"failure,omitempty"`
52+
Skipped SkipReason `xml:"skipped,omitempty"`
53+
}
54+
55+
// SkipReason deals with the special <skipped></skipped>:
56+
// if present, we must re-encode it, even if empty.
57+
type SkipReason string
58+
59+
func (s *SkipReason) UnmarshalText(text []byte) error {
60+
*s = SkipReason(text)
61+
if *s == "" {
62+
*s = " "
63+
}
64+
return nil
65+
}
66+
67+
func (s SkipReason) MarshalText() ([]byte, error) {
68+
if s == " " {
69+
return []byte{}, nil
70+
}
71+
return []byte(s), nil
72+
}
73+
74+
func main() {
75+
var junit TestSuite
76+
var data []byte
77+
78+
flag.Parse()
79+
80+
re := regexp.MustCompile(*tests)
81+
82+
// Read all input files.
83+
for _, input := range flag.Args() {
84+
if input == "-" {
85+
if _, err := os.Stdin.Read(data); err != nil {
86+
panic(err)
87+
}
88+
} else {
89+
var err error
90+
data, err = ioutil.ReadFile(input)
91+
if err != nil {
92+
panic(err)
93+
}
94+
}
95+
if err := xml.Unmarshal(data, &junit); err != nil {
96+
panic(err)
97+
}
98+
}
99+
100+
// Keep only matching testcases. Testcases skipped in all test runs are only stored once.
101+
filtered := map[string]TestCase{}
102+
for _, testcase := range junit.TestCases {
103+
if !re.MatchString(testcase.Name) {
104+
continue
105+
}
106+
entry, ok := filtered[testcase.Name]
107+
if !ok || // not present yet
108+
entry.Skipped != "" && testcase.Skipped == "" { // replaced skipped test with real test run
109+
filtered[testcase.Name] = testcase
110+
}
111+
}
112+
junit.TestCases = nil
113+
for _, testcase := range filtered {
114+
junit.TestCases = append(junit.TestCases, testcase)
115+
}
116+
117+
// Re-encode.
118+
data, err := xml.MarshalIndent(junit, "", " ")
119+
if err != nil {
120+
panic(err)
121+
}
122+
123+
// Write to output.
124+
if *output == "-" {
125+
if _, err := os.Stdout.Write(data); err != nil {
126+
panic(err)
127+
}
128+
} else {
129+
if err := ioutil.WriteFile(*output, data, 0644); err != nil {
130+
panic(err)
131+
}
132+
}
133+
}

0 commit comments

Comments
 (0)