Skip to content

Commit 8e84048

Browse files
Maisem Alimaisem
Maisem Ali
authored and
maisem
committed
cmd/testwrapper: only retry flaky failed tests
Redo the testwrapper to track and only retry flaky tests instead of retrying the entire pkg. It also fails early if a non-flaky test fails. This also makes it so that the go test caches are used. Fixes tailscale#7975 Signed-off-by: Maisem Ali <[email protected]>
1 parent 2cf6e12 commit 8e84048

File tree

4 files changed

+232
-55
lines changed

4 files changed

+232
-55
lines changed

‎.github/workflows/test.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -90,11 +90,11 @@ jobs:
9090
- name: build test wrapper
9191
run: ./tool/go build -o /tmp/testwrapper ./cmd/testwrapper
9292
- name: test all
93-
run: ./tool/go test ${{matrix.buildflags}} -exec=/tmp/testwrapper ./...
93+
run: PATH=$PWD/tool:$PATH /tmp/testwrapper ./... ${{matrix.buildflags}}
9494
env:
9595
GOARCH: ${{ matrix.goarch }}
9696
- name: bench all
97-
run: ./tool/go test ${{matrix.buildflags}} -exec=/tmp/testwrapper -test.bench=. -test.benchtime=1x -test.run=^$ ./...
97+
run: PATH=$PWD/tool:$PATH /tmp/testwrapper ./... ${{matrix.buildflags}} -bench=. -benchtime=1x -run=^$
9898
env:
9999
GOARCH: ${{ matrix.goarch }}
100100
- name: check that no tracked files changed

‎cmd/testwrapper/flakytest/flakytest.go

Lines changed: 11 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -7,16 +7,20 @@
77
package flakytest
88

99
import (
10+
"fmt"
1011
"os"
1112
"regexp"
1213
"testing"
1314
)
1415

15-
// InTestWrapper returns whether or not this binary is running under our test
16-
// wrapper.
17-
func InTestWrapper() bool {
18-
return os.Getenv("TS_IN_TESTWRAPPER") != ""
19-
}
16+
// FlakyTestLogMessage is a sentinel value that is printed to stderr when a
17+
// flaky test is marked. This is used by cmd/testwrapper to detect flaky tests
18+
// and retry them.
19+
const FlakyTestLogMessage = "flakytest: this is a known flaky test"
20+
21+
// FlakeAttemptEnv is an environment variable that is set by cmd/testwrapper
22+
// when a flaky test is retried. It contains the attempt number, starting at 1.
23+
const FlakeAttemptEnv = "TS_TESTWRAPPER_ATTEMPT"
2024

2125
var issueRegexp = regexp.MustCompile(`\Ahttps://github\.com/tailscale/[a-zA-Z0-9_.-]+/issues/\d+\z`)
2226

@@ -30,16 +34,6 @@ func Mark(t testing.TB, issue string) {
3034
t.Fatalf("bad issue format: %q", issue)
3135
}
3236

33-
if !InTestWrapper() {
34-
return
35-
}
36-
37-
t.Cleanup(func() {
38-
if t.Failed() {
39-
t.Logf("flakytest: signaling test wrapper to retry test")
40-
41-
// Signal to test wrapper that we should restart.
42-
os.Exit(123)
43-
}
44-
})
37+
fmt.Fprintln(os.Stderr, FlakyTestLogMessage) // sentinel value for testwrapper
38+
t.Logf("flakytest: issue tracking this flaky test: %s", issue)
4539
}

‎cmd/testwrapper/flakytest/flakytest_test.go

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,10 @@
33

44
package flakytest
55

6-
import "testing"
6+
import (
7+
"os"
8+
"testing"
9+
)
710

811
func TestIssueFormat(t *testing.T) {
912
testCases := []struct {
@@ -24,3 +27,14 @@ func TestIssueFormat(t *testing.T) {
2427
}
2528
}
2629
}
30+
31+
func TestFlakeRun(t *testing.T) {
32+
Mark(t, "https://github.com/tailscale/tailscale/issues/0") // random issue
33+
e := os.Getenv(FlakeAttemptEnv)
34+
if e == "" {
35+
t.Skip("not running in testwrapper")
36+
}
37+
if e == "1" {
38+
t.Fatal("failing on purpose")
39+
}
40+
}

‎cmd/testwrapper/testwrapper.go

Lines changed: 204 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,62 +1,231 @@
11
// Copyright (c) Tailscale Inc & AUTHORS
22
// SPDX-License-Identifier: BSD-3-Clause
33

4-
// testwrapper is a wrapper for retrying flaky tests, using the -exec flag of
5-
// 'go test'. Tests that are flaky can use the 'flakytest' subpackage to mark
6-
// themselves as flaky and be retried on failure.
4+
// testwrapper is a wrapper for retrying flaky tests. It is an alternative to
5+
// `go test` and re-runs failed marked flaky tests (using the flakytest pkg). It
6+
// takes different arguments than go test and requires the first positional
7+
// argument to be the pattern to test.
78
package main
89

910
import (
11+
"bytes"
1012
"context"
13+
"encoding/json"
1114
"errors"
15+
"flag"
16+
"fmt"
17+
"io"
1218
"log"
1319
"os"
1420
"os/exec"
15-
)
21+
"sort"
22+
"strings"
23+
"time"
1624

17-
const (
18-
retryStatus = 123
19-
maxIterations = 3
25+
"golang.org/x/exp/maps"
26+
"tailscale.com/cmd/testwrapper/flakytest"
2027
)
2128

22-
func main() {
23-
ctx := context.Background()
24-
debug := os.Getenv("TS_TESTWRAPPER_DEBUG") != ""
29+
const maxAttempts = 3
30+
31+
type testAttempt struct {
32+
name testName
33+
outcome string // "pass", "fail", "skip"
34+
logs bytes.Buffer
35+
isMarkedFlaky bool // set if the test is marked as flaky
36+
}
37+
38+
type testName struct {
39+
pkg string // "tailscale.com/types/key"
40+
name string // "TestFoo"
41+
}
42+
43+
type packageTests struct {
44+
// pattern is the package pattern to run.
45+
// Must be a single pattern, not a list of patterns.
46+
pattern string // "./...", "./types/key"
47+
// tests is a list of tests to run. If empty, all tests in the package are
48+
// run.
49+
tests []string // ["TestFoo", "TestBar"]
50+
}
51+
52+
type goTestOutput struct {
53+
Time time.Time
54+
Action string
55+
Package string
56+
Test string
57+
Output string
58+
}
59+
60+
var debug = os.Getenv("TS_TESTWRAPPER_DEBUG") != ""
2561

26-
log.SetPrefix("testwrapper: ")
27-
if !debug {
28-
log.SetFlags(0)
62+
func runTests(ctx context.Context, attempt int, pt *packageTests, otherArgs []string) []*testAttempt {
63+
args := []string{"test", "-json", pt.pattern}
64+
args = append(args, otherArgs...)
65+
if len(pt.tests) > 0 {
66+
runArg := strings.Join(pt.tests, "|")
67+
args = append(args, "-run", runArg)
2968
}
69+
if debug {
70+
fmt.Println("running", strings.Join(args, " "))
71+
}
72+
cmd := exec.CommandContext(ctx, "go", args...)
73+
r, err := cmd.StdoutPipe()
74+
if err != nil {
75+
log.Printf("error creating stdout pipe: %v", err)
76+
}
77+
cmd.Stderr = os.Stderr
78+
79+
cmd.Env = os.Environ()
80+
cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", flakytest.FlakeAttemptEnv, attempt))
3081

31-
for i := 1; i <= maxIterations; i++ {
32-
if i > 1 {
33-
log.Printf("retrying flaky tests (%d of %d)", i, maxIterations)
82+
if err := cmd.Start(); err != nil {
83+
log.Printf("error starting test: %v", err)
84+
os.Exit(1)
85+
}
86+
done := make(chan struct{})
87+
go func() {
88+
defer close(done)
89+
cmd.Wait()
90+
}()
91+
92+
jd := json.NewDecoder(r)
93+
resultMap := make(map[testName]*testAttempt)
94+
var out []*testAttempt
95+
for {
96+
var goOutput goTestOutput
97+
if err := jd.Decode(&goOutput); err != nil {
98+
if errors.Is(err, io.EOF) || errors.Is(err, os.ErrClosed) {
99+
break
100+
}
101+
panic(err)
34102
}
35-
cmd := exec.CommandContext(ctx, os.Args[1], os.Args[2:]...)
36-
cmd.Stdout = os.Stdout
37-
cmd.Stderr = os.Stderr
38-
cmd.Env = append(os.Environ(), "TS_IN_TESTWRAPPER=1")
39-
err := cmd.Run()
40-
if err == nil {
41-
return
103+
if goOutput.Test == "" {
104+
continue
105+
}
106+
name := testName{
107+
pkg: goOutput.Package,
108+
name: goOutput.Test,
109+
}
110+
if test, _, isSubtest := strings.Cut(goOutput.Test, "/"); isSubtest {
111+
name.name = test
112+
if goOutput.Action == "output" {
113+
resultMap[name].logs.WriteString(goOutput.Output)
114+
}
115+
continue
116+
}
117+
switch goOutput.Action {
118+
case "start":
119+
// ignore
120+
case "run":
121+
resultMap[name] = &testAttempt{
122+
name: name,
123+
}
124+
case "skip", "pass", "fail":
125+
resultMap[name].outcome = goOutput.Action
126+
out = append(out, resultMap[name])
127+
case "output":
128+
if strings.TrimSpace(goOutput.Output) == flakytest.FlakyTestLogMessage {
129+
resultMap[name].isMarkedFlaky = true
130+
} else {
131+
resultMap[name].logs.WriteString(goOutput.Output)
132+
}
42133
}
134+
}
135+
<-done
136+
return out
137+
}
138+
139+
func main() {
140+
ctx := context.Background()
141+
142+
// We only need to parse the -v flag to figure out whether to print the logs
143+
// for a test. We don't need to parse any other flags, so we just use the
144+
// flag package to parse the -v flag and then pass the rest of the args
145+
// through to 'go test'.
146+
// We run `go test -json` which returns the same information as `go test -v`,
147+
// but in a machine-readable format. So this flag is only for testwrapper's
148+
// output.
149+
v := flag.Bool("v", false, "verbose")
150+
151+
flag.Usage = func() {
152+
fmt.Println("usage: testwrapper [testwrapper-flags] [pattern] [build/test flags & test binary flags]")
153+
fmt.Println()
154+
fmt.Println("testwrapper-flags:")
155+
flag.CommandLine.PrintDefaults()
156+
fmt.Println()
157+
fmt.Println("examples:")
158+
fmt.Println("\ttestwrapper -v ./... -count=1")
159+
fmt.Println("\ttestwrapper ./pkg/foo -run TestBar -count=1")
160+
fmt.Println()
161+
fmt.Println("Unlike 'go test', testwrapper requires a package pattern as the first positional argument and only supports a single pattern.")
162+
}
163+
flag.Parse()
164+
165+
args := flag.Args()
166+
if len(args) < 1 || strings.HasPrefix(args[0], "-") {
167+
fmt.Println("no pattern specified")
168+
flag.Usage()
169+
os.Exit(1)
170+
} else if len(args) > 1 && !strings.HasPrefix(args[1], "-") {
171+
fmt.Println("expected single pattern")
172+
flag.Usage()
173+
os.Exit(1)
174+
}
175+
pattern, otherArgs := args[0], args[1:]
176+
177+
toRun := []*packageTests{ // packages still to test
178+
{pattern: pattern},
179+
}
43180

44-
var exitErr *exec.ExitError
45-
if !errors.As(err, &exitErr) {
46-
if debug {
47-
log.Printf("error isn't an ExitError")
181+
pkgAttempts := make(map[string]int) // tracks how many times we've tried a package
182+
183+
attempt := 0
184+
for len(toRun) > 0 {
185+
attempt++
186+
var pt *packageTests
187+
pt, toRun = toRun[0], toRun[1:]
188+
189+
toRetry := make(map[string][]string) // pkg -> tests to retry
190+
191+
failed := false
192+
for _, tr := range runTests(ctx, attempt, pt, otherArgs) {
193+
if *v || tr.outcome == "fail" {
194+
io.Copy(os.Stderr, &tr.logs)
48195
}
196+
if tr.outcome != "fail" {
197+
continue
198+
}
199+
if tr.isMarkedFlaky {
200+
toRetry[tr.name.pkg] = append(toRetry[tr.name.pkg], tr.name.name)
201+
} else {
202+
failed = true
203+
}
204+
}
205+
if failed {
49206
os.Exit(1)
50207
}
51-
52-
if code := exitErr.ExitCode(); code != retryStatus {
53-
if debug {
54-
log.Printf("code (%d) != retryStatus (%d)", code, retryStatus)
208+
pkgs := maps.Keys(toRetry)
209+
sort.Strings(pkgs)
210+
for _, pkg := range pkgs {
211+
tests := toRetry[pkg]
212+
sort.Strings(tests)
213+
pkgAttempts[pkg]++
214+
if pkgAttempts[pkg] >= maxAttempts {
215+
fmt.Println("Too many attempts for flaky tests:", pkg, tests)
216+
continue
55217
}
56-
os.Exit(code)
218+
fmt.Println("\nRetrying flaky tests:", pkg, tests)
219+
toRun = append(toRun, &packageTests{
220+
pattern: pkg,
221+
tests: tests,
222+
})
57223
}
58224
}
59-
60-
log.Printf("test did not pass in %d iterations", maxIterations)
61-
os.Exit(1)
225+
for _, a := range pkgAttempts {
226+
if a >= maxAttempts {
227+
os.Exit(1)
228+
}
229+
}
230+
fmt.Println("PASS")
62231
}

0 commit comments

Comments
 (0)