Skip to content

Commit e39e8fb

Browse files
jirfagjirfag
committed
update x/tools
Significantly improve CPU and memory usage when not using SSA-powered linters. Improve readability of go/packages errors. Improve debugging capabilities and write doc about debugging.
1 parent d2b1eea commit e39e8fb

File tree

36 files changed

+1639
-568
lines changed

36 files changed

+1639
-568
lines changed

‎.travis.yml

+2-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,8 @@ go:
55
- 1.12.x
66

77
before_script:
8-
- go get github.com/valyala/quicktemplate
8+
- go get github.com/valyala/quicktemplate # for tests
9+
- go get github.com/pkg/errors # for tests
910

1011
script: make check_generated test
1112

‎Makefile

+2-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,8 @@ test_linters:
3030

3131
# Maintenance
3232

33-
generate: docs/demo.svg README.md install.sh pkg/logutils/log_mock.go vendor
33+
generate: README.md install.sh pkg/logutils/log_mock.go vendor
34+
generate_svg: docs/demo.svg
3435
maintainer-clean: clean
3536
rm -f docs/demo.svg README.md install.sh pkg/logutils/log_mock.go
3637
rm -rf vendor

‎README.md

+15
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ Sponsored by [GolangCI.com](https://golangci.com): SaaS service for running lint
2828
* [FAQ](#faq)
2929
* [Thanks](#thanks)
3030
* [Changelog](#changelog)
31+
* [Debug](#debug)
3132
* [Future Plans](#future-plans)
3233
* [Contact Information](#contact-information)
3334

@@ -1140,6 +1141,20 @@ There is the most valuable changes log:
11401141
1. Support GitHub Releases
11411142
2. Installation via Homebrew and Docker
11421143
1144+
## Debug
1145+
1146+
You can see a verbose output of linter by using `-v` option.
1147+
1148+
If you would like to see more detailed logs you can set environment variable `GL_DEBUG` to debug `golangci-lint`.
1149+
It's value is a list of debug tags. For example, `GL_DEBUG=loader,gocritic golangci-lint run`.
1150+
Existing debug tags:
1151+
1152+
1. `gocritic` - debug `go-critic` linter;
1153+
2. `env` - debug `go env` command;
1154+
3. `loader` - debug packages loading (including `go/packages` internal debugging);
1155+
4. `autogen_exclude` - debug a filter excluding autogenerated source code;
1156+
5. `nolint` - debug a filter excluding issues by `//nolint` comments.
1157+
11431158
## Future Plans
11441159
11451160
1. Upstream all changes of forked linters.

‎README.tmpl.md

+15
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ Sponsored by [GolangCI.com](https://golangci.com): SaaS service for running lint
2828
* [FAQ](#faq)
2929
* [Thanks](#thanks)
3030
* [Changelog](#changelog)
31+
* [Debug](#debug)
3132
* [Future Plans](#future-plans)
3233
* [Contact Information](#contact-information)
3334

@@ -668,6 +669,20 @@ There is the most valuable changes log:
668669
1. Support GitHub Releases
669670
2. Installation via Homebrew and Docker
670671

672+
## Debug
673+
674+
You can see a verbose output of linter by using `-v` option.
675+
676+
If you would like to see more detailed logs you can set environment variable `GL_DEBUG` to debug `golangci-lint`.
677+
It's value is a list of debug tags. For example, `GL_DEBUG=loader,gocritic golangci-lint run`.
678+
Existing debug tags:
679+
680+
1. `gocritic` - debug `go-critic` linter;
681+
2. `env` - debug `go env` command;
682+
3. `loader` - debug packages loading (including `go/packages` internal debugging);
683+
4. `autogen_exclude` - debug a filter excluding autogenerated source code;
684+
5. `nolint` - debug a filter excluding issues by `//nolint` comments.
685+
671686
## Future Plans
672687

673688
1. Upstream all changes of forked linters.

‎go.mod

+5-3
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,6 @@ require (
5252
github.com/timakin/bodyclose v0.0.0-00010101000000-000000000000
5353
github.com/valyala/quicktemplate v1.1.1
5454
golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a // indirect
55-
golang.org/x/net v0.0.0-20190313220215-9f648a60d977 // indirect
5655
golang.org/x/sys v0.0.0-20190312061237-fead79001313 // indirect
5756
golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd
5857
gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect
@@ -63,5 +62,8 @@ require (
6362
mvdan.cc/unparam v0.0.0-20190124213536-fbb59629db34
6463
)
6564

66-
// https://github.com/timakin/bodyclose/pull/17
67-
replace github.com/timakin/bodyclose => github.com/golangci/bodyclose v0.0.0-20190713050349-65da19158fa2
65+
replace (
66+
// https://github.com/timakin/bodyclose/pull/17
67+
github.com/timakin/bodyclose => github.com/golangci/bodyclose v0.0.0-20190713050349-65da19158fa2
68+
golang.org/x/tools => github.com/golangci/tools v0.0.0-20190713050349-979bdb7f8cc8
69+
)

‎go.sum

+5-13
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,8 @@ github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21 h1:leSNB7iYzLYSS
8181
github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI=
8282
github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0 h1:HVfrLniijszjS1aiNg8JbBMO2+E1WIQ+j/gL4SQqGPg=
8383
github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
84+
github.com/golangci/tools v0.0.0-20190713050349-979bdb7f8cc8 h1:rv5pCF5e6hFuSWEDuP3R+r8l0n/srMta+VWVEskASSQ=
85+
github.com/golangci/tools v0.0.0-20190713050349-979bdb7f8cc8/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
8486
github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSWhqsYNgcuWO2kFlpdOZbP0+yRjmvPGys=
8587
github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ=
8688
github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
@@ -177,9 +179,8 @@ golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73r
177179
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA=
178180
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
179181
golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
180-
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
181-
golang.org/x/net v0.0.0-20190313220215-9f648a60d977 h1:actzWV6iWn3GLqN8dZjzsB+CLt+gaV2+wsxroxiQI8I=
182-
golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
182+
golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
183+
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
183184
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
184185
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
185186
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
@@ -193,16 +194,7 @@ golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7w
193194
golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
194195
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
195196
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
196-
golang.org/x/tools v0.0.0-20170915040203-e531a2a1c15f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
197-
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
198-
golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
199-
golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
200-
golang.org/x/tools v0.0.0-20190121143147-24cd39ecf745/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
201-
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
202-
golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
203-
golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
204-
golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd h1:7E3PabyysDSEjnaANKBgums/hyvMI/HoHQ50qZEzTrg=
205-
golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
197+
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
206198
gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo=
207199
gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
208200
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

‎pkg/golinters/goanalysis/linter.go

+4-4
Original file line numberDiff line numberDiff line change
@@ -124,13 +124,13 @@ func (lnt Linter) Run(ctx context.Context, lintCtx *linter.Context) ([]result.Is
124124
}
125125

126126
var issues []result.Issue
127-
for _, diag := range diags {
128-
i := result.Issue{
127+
for i := range diags {
128+
diag := &diags[i]
129+
issues = append(issues, result.Issue{
129130
FromLinter: lnt.Name(),
130131
Text: fmt.Sprintf("%s: %s", diag.AnalyzerName, diag.Message),
131132
Pos: diag.Position,
132-
}
133-
issues = append(issues, i)
133+
})
134134
}
135135

136136
return issues, nil

‎pkg/lint/load.go

+64-47
Original file line numberDiff line numberDiff line change
@@ -148,39 +148,19 @@ func (cl ContextLoader) buildSSAProgram(pkgs []*packages.Package) *ssa.Program {
148148
}
149149

150150
func (cl ContextLoader) findLoadMode(linters []*linter.Config) packages.LoadMode {
151-
maxLoadMode := packages.LoadFiles
151+
//TODO: specify them in linters: need more fine-grained control.
152+
// e.g. NeedTypesSizes is needed only for go vet
153+
loadMode := packages.NeedName | packages.NeedFiles | packages.NeedCompiledGoFiles
152154
for _, lc := range linters {
153-
curLoadMode := packages.LoadFiles
154155
if lc.NeedsTypeInfo {
155-
curLoadMode = packages.LoadSyntax
156+
loadMode |= packages.NeedImports | packages.NeedTypes | packages.NeedTypesSizes | packages.NeedTypesInfo | packages.NeedSyntax
156157
}
157158
if lc.NeedsSSARepr {
158-
curLoadMode = packages.LoadAllSyntax
159-
}
160-
if curLoadMode > maxLoadMode {
161-
maxLoadMode = curLoadMode
159+
loadMode |= packages.NeedDeps
162160
}
163161
}
164162

165-
return maxLoadMode
166-
}
167-
168-
func stringifyLoadMode(mode packages.LoadMode) string {
169-
switch mode {
170-
case packages.LoadFiles:
171-
return "load files"
172-
case packages.LoadImports:
173-
return "load imports"
174-
case packages.LoadTypes:
175-
return "load types"
176-
case packages.LoadSyntax:
177-
return "load types and syntax"
178-
}
179-
// it may be an alias, and may be not
180-
if mode == packages.LoadAllSyntax {
181-
return "load deps types and syntax"
182-
}
183-
return "unknown"
163+
return loadMode
184164
}
185165

186166
func (cl ContextLoader) buildArgs() []string {
@@ -231,6 +211,58 @@ func (cl ContextLoader) makeBuildFlags() ([]string, error) {
231211
return buildFlags, nil
232212
}
233213

214+
func stringifyLoadMode(mode packages.LoadMode) string {
215+
m := map[packages.LoadMode]string{
216+
packages.NeedCompiledGoFiles: "compiled_files",
217+
packages.NeedDeps: "deps",
218+
packages.NeedExportsFile: "exports_file",
219+
packages.NeedFiles: "files",
220+
packages.NeedImports: "imports",
221+
packages.NeedName: "name",
222+
packages.NeedSyntax: "syntax",
223+
packages.NeedTypes: "types",
224+
packages.NeedTypesInfo: "types_info",
225+
packages.NeedTypesSizes: "types_sizes",
226+
}
227+
228+
var flags []string
229+
for flag, flagStr := range m {
230+
if mode&flag != 0 {
231+
flags = append(flags, flagStr)
232+
}
233+
}
234+
235+
return fmt.Sprintf("%d (%s)", mode, strings.Join(flags, "|"))
236+
}
237+
238+
func (cl ContextLoader) debugPrintLoadedPackages(pkgs []*packages.Package) {
239+
cl.debugf("loaded %d pkgs", len(pkgs))
240+
for i, pkg := range pkgs {
241+
var syntaxFiles []string
242+
for _, sf := range pkg.Syntax {
243+
syntaxFiles = append(syntaxFiles, pkg.Fset.Position(sf.Pos()).Filename)
244+
}
245+
cl.debugf("Loaded pkg #%d: ID=%s GoFiles=%s CompiledGoFiles=%s Syntax=%s",
246+
i, pkg.ID, pkg.GoFiles, pkg.CompiledGoFiles, syntaxFiles)
247+
}
248+
}
249+
250+
func (cl ContextLoader) parseLoadedPackagesErrors(pkgs []*packages.Package) error {
251+
for _, pkg := range pkgs {
252+
for _, err := range pkg.Errors {
253+
if strings.Contains(err.Msg, "no Go files") {
254+
return errors.Wrapf(exitcodes.ErrNoGoFiles, "package %s", pkg.PkgPath)
255+
}
256+
if strings.Contains(err.Msg, "cannot find package") {
257+
// when analyzing not existing directory
258+
return errors.Wrap(exitcodes.ErrFailure, err.Msg)
259+
}
260+
}
261+
}
262+
263+
return nil
264+
}
265+
234266
func (cl ContextLoader) loadPackages(ctx context.Context, loadMode packages.LoadMode) ([]*packages.Package, error) {
235267
defer func(startedAt time.Time) {
236268
cl.log.Infof("Go packages loading at mode %s took %s", stringifyLoadMode(loadMode), time.Since(startedAt))
@@ -248,6 +280,7 @@ func (cl ContextLoader) loadPackages(ctx context.Context, loadMode packages.Load
248280
Tests: cl.cfg.Run.AnalyzeTests,
249281
Context: ctx,
250282
BuildFlags: buildFlags,
283+
Logf: cl.debugf,
251284
//TODO: use fset, parsefile, overlay
252285
}
253286

@@ -257,26 +290,10 @@ func (cl ContextLoader) loadPackages(ctx context.Context, loadMode packages.Load
257290
if err != nil {
258291
return nil, errors.Wrap(err, "failed to load program with go/packages")
259292
}
260-
cl.debugf("loaded %d pkgs", len(pkgs))
261-
for i, pkg := range pkgs {
262-
var syntaxFiles []string
263-
for _, sf := range pkg.Syntax {
264-
syntaxFiles = append(syntaxFiles, pkg.Fset.Position(sf.Pos()).Filename)
265-
}
266-
cl.debugf("Loaded pkg #%d: ID=%s GoFiles=%s CompiledGoFiles=%s Syntax=%s",
267-
i, pkg.ID, pkg.GoFiles, pkg.CompiledGoFiles, syntaxFiles)
268-
}
293+
cl.debugPrintLoadedPackages(pkgs)
269294

270-
for _, pkg := range pkgs {
271-
for _, err := range pkg.Errors {
272-
if strings.Contains(err.Msg, "no Go files") {
273-
return nil, errors.Wrapf(exitcodes.ErrNoGoFiles, "package %s", pkg.PkgPath)
274-
}
275-
if strings.Contains(err.Msg, "cannot find package") {
276-
// when analyzing not existing directory
277-
return nil, errors.Wrap(exitcodes.ErrFailure, err.Msg)
278-
}
279-
}
295+
if err := cl.parseLoadedPackagesErrors(pkgs); err != nil {
296+
return nil, err
280297
}
281298

282299
return cl.filterPackages(pkgs), nil
@@ -341,12 +358,12 @@ func (cl ContextLoader) Load(ctx context.Context, linters []*linter.Config) (*li
341358
}
342359

343360
var prog *loader.Program
344-
if loadMode >= packages.LoadSyntax {
361+
if loadMode&packages.NeedTypes != 0 {
345362
prog = cl.makeFakeLoaderProgram(pkgs)
346363
}
347364

348365
var ssaProg *ssa.Program
349-
if loadMode == packages.LoadAllSyntax {
366+
if loadMode&packages.NeedDeps != 0 {
350367
ssaProg = cl.buildSSAProgram(pkgs)
351368
}
352369

‎vendor/golang.org/x/tools/go/analysis/analysis.go

+13-13
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)