Skip to content

Commit 339711c

Browse files
jirfagjirfag
committed
WIP: dramatically reduce memory usage
Run all linters per package. It allows unloading package data when it's processed. It dramatically reduces memory (and CPU because of GC) usage. Relates: #337
1 parent d63d235 commit 339711c

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

63 files changed

+2375
-2187
lines changed

‎cmd/golangci-lint/mod_version.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import (
1010
//nolint:gochecknoinits
1111
func init() {
1212
if info, available := debug.ReadBuildInfo(); available {
13-
if date == "" && info.Main.Version != "(devel)" {
13+
if date == "" {
1414
version = info.Main.Version
1515
commit = fmt.Sprintf("(unknown, mod sum: %q)", info.Main.Sum)
1616
date = "(unknown)"

‎pkg/commands/run.go

+8-22
Original file line numberDiff line numberDiff line change
@@ -265,7 +265,7 @@ func fixSlicesFlags(fs *pflag.FlagSet) {
265265
})
266266
}
267267

268-
func (e *Executor) runAnalysis(ctx context.Context, args []string) (<-chan result.Issue, error) {
268+
func (e *Executor) runAnalysis(ctx context.Context, args []string) ([]result.Issue, error) {
269269
e.cfg.Run.Args = args
270270

271271
enabledLinters, err := e.EnabledLintersSet.Get(true)
@@ -296,9 +296,9 @@ func (e *Executor) runAnalysis(ctx context.Context, args []string) (<-chan resul
296296
return nil, err
297297
}
298298

299-
issuesCh := runner.Run(ctx, enabledLinters, lintCtx)
299+
issues := runner.Run(ctx, enabledLinters, lintCtx)
300300
fixer := processors.NewFixer(e.cfg, e.log, e.fileCache)
301-
return fixer.Process(issuesCh), nil
301+
return fixer.Process(issues), nil
302302
}
303303

304304
func (e *Executor) setOutputToDevNull() (savedStdout, savedStderr *os.File) {
@@ -313,24 +313,10 @@ func (e *Executor) setOutputToDevNull() (savedStdout, savedStderr *os.File) {
313313
return
314314
}
315315

316-
func (e *Executor) setExitCodeIfIssuesFound(issues <-chan result.Issue) <-chan result.Issue {
317-
resCh := make(chan result.Issue, 1024)
318-
319-
go func() {
320-
issuesFound := false
321-
for i := range issues {
322-
issuesFound = true
323-
resCh <- i
324-
}
325-
326-
if issuesFound {
327-
e.exitCode = e.cfg.Run.ExitCodeIfIssuesFound
328-
}
329-
330-
close(resCh)
331-
}()
332-
333-
return resCh
316+
func (e *Executor) setExitCodeIfIssuesFound(issues []result.Issue) {
317+
if len(issues) != 0 {
318+
e.exitCode = e.cfg.Run.ExitCodeIfIssuesFound
319+
}
334320
}
335321

336322
func (e *Executor) runAndPrint(ctx context.Context, args []string) error {
@@ -357,7 +343,7 @@ func (e *Executor) runAndPrint(ctx context.Context, args []string) error {
357343
return err
358344
}
359345

360-
issues = e.setExitCodeIfIssuesFound(issues)
346+
e.setExitCodeIfIssuesFound(issues)
361347

362348
if err = p.Print(ctx, issues); err != nil {
363349
return fmt.Errorf("can't print %d issues: %s", len(issues), err)

‎pkg/golinters/deadcode.go

+39-29
Original file line numberDiff line numberDiff line change
@@ -1,42 +1,52 @@
11
package golinters
22

33
import (
4-
"context"
54
"fmt"
5+
"sync"
66

77
deadcodeAPI "github.com/golangci/go-misc/deadcode"
8+
"golang.org/x/tools/go/analysis"
89

10+
"github.com/golangci/golangci-lint/pkg/golinters/goanalysis"
911
"github.com/golangci/golangci-lint/pkg/lint/linter"
1012
"github.com/golangci/golangci-lint/pkg/result"
1113
)
1214

13-
type Deadcode struct{}
14-
15-
func (Deadcode) Name() string {
16-
return "deadcode"
17-
}
18-
19-
func (Deadcode) Desc() string {
20-
return "Finds unused code"
21-
}
22-
23-
func (d Deadcode) Run(ctx context.Context, lintCtx *linter.Context) ([]result.Issue, error) {
24-
issues, err := deadcodeAPI.Run(lintCtx.Program)
25-
if err != nil {
26-
return nil, err
27-
}
28-
29-
if len(issues) == 0 {
30-
return nil, nil
31-
}
32-
33-
res := make([]result.Issue, 0, len(issues))
34-
for _, i := range issues {
35-
res = append(res, result.Issue{
36-
Pos: i.Pos,
37-
Text: fmt.Sprintf("%s is unused", formatCode(i.UnusedIdentName, lintCtx.Cfg)),
38-
FromLinter: d.Name(),
39-
})
15+
func NewDeadcode() *goanalysis.Linter {
16+
const linterName = "deadcode"
17+
var mu sync.Mutex
18+
var resIssues []result.Issue
19+
20+
analyzer := &analysis.Analyzer{
21+
Name: goanalysis.TheOnlyAnalyzerName,
22+
Doc: goanalysis.TheOnlyanalyzerDoc,
23+
Run: func(pass *analysis.Pass) (interface{}, error) {
24+
prog := goanalysis.MakeFakeLoaderProgram(pass)
25+
issues, err := deadcodeAPI.Run(prog)
26+
if err != nil {
27+
return nil, err
28+
}
29+
res := make([]result.Issue, 0, len(issues))
30+
for _, i := range issues {
31+
res = append(res, result.Issue{
32+
Pos: i.Pos,
33+
Text: fmt.Sprintf("%s is unused", formatCode(i.UnusedIdentName, nil)),
34+
FromLinter: linterName,
35+
})
36+
}
37+
mu.Lock()
38+
resIssues = append(resIssues, res...)
39+
mu.Unlock()
40+
41+
return nil, nil
42+
},
4043
}
41-
return res, nil
44+
return goanalysis.NewLinter(
45+
linterName,
46+
"Finds unused code",
47+
[]*analysis.Analyzer{analyzer},
48+
nil,
49+
).WithIssuesReporter(func(*linter.Context) []result.Issue {
50+
return resIssues
51+
}).WithLoadMode(goanalysis.LoadModeTypesInfo)
4252
}

‎pkg/golinters/depguard.go

+65-41
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,21 @@
11
package golinters
22

33
import (
4-
"context"
54
"fmt"
65
"strings"
6+
"sync"
7+
8+
"golang.org/x/tools/go/analysis"
9+
"golang.org/x/tools/go/loader"
10+
11+
"github.com/golangci/golangci-lint/pkg/golinters/goanalysis"
712

813
depguardAPI "github.com/OpenPeeDeeP/depguard"
914

1015
"github.com/golangci/golangci-lint/pkg/lint/linter"
1116
"github.com/golangci/golangci-lint/pkg/result"
1217
)
1318

14-
type Depguard struct{}
15-
16-
func (Depguard) Name() string {
17-
return "depguard"
18-
}
19-
2019
func setDepguardListType(dg *depguardAPI.Depguard, lintCtx *linter.Context) error {
2120
listType := lintCtx.Settings().Depguard.ListType
2221
var found bool
@@ -49,42 +48,67 @@ func setupDepguardPackages(dg *depguardAPI.Depguard, lintCtx *linter.Context) {
4948
}
5049
}
5150

52-
func (Depguard) Desc() string {
53-
return "Go linter that checks if package imports are in a list of acceptable packages"
54-
}
51+
func NewDepguard() *goanalysis.Linter {
52+
const linterName = "depguard"
53+
var mu sync.Mutex
54+
var resIssues []result.Issue
5555

56-
func (d Depguard) Run(ctx context.Context, lintCtx *linter.Context) ([]result.Issue, error) {
57-
dg := &depguardAPI.Depguard{
58-
Packages: lintCtx.Settings().Depguard.Packages,
59-
IncludeGoRoot: lintCtx.Settings().Depguard.IncludeGoRoot,
60-
}
61-
if err := setDepguardListType(dg, lintCtx); err != nil {
62-
return nil, err
56+
analyzer := &analysis.Analyzer{
57+
Name: goanalysis.TheOnlyAnalyzerName,
58+
Doc: goanalysis.TheOnlyanalyzerDoc,
6359
}
64-
setupDepguardPackages(dg, lintCtx)
60+
return goanalysis.NewLinter(
61+
linterName,
62+
"Go linter that checks if package imports are in a list of acceptable packages",
63+
[]*analysis.Analyzer{analyzer},
64+
nil,
65+
).WithContextSetter(func(lintCtx *linter.Context) {
66+
dgSettings := &lintCtx.Settings().Depguard
67+
analyzer.Run = func(pass *analysis.Pass) (interface{}, error) {
68+
prog := goanalysis.MakeFakeLoaderProgram(pass)
69+
dg := &depguardAPI.Depguard{
70+
Packages: dgSettings.Packages,
71+
IncludeGoRoot: dgSettings.IncludeGoRoot,
72+
}
73+
if err := setDepguardListType(dg, lintCtx); err != nil {
74+
return nil, err
75+
}
76+
setupDepguardPackages(dg, lintCtx)
6577

66-
issues, err := dg.Run(lintCtx.LoaderConfig, lintCtx.Program)
67-
if err != nil {
68-
return nil, err
69-
}
70-
if len(issues) == 0 {
71-
return nil, nil
72-
}
73-
msgSuffix := "is in the blacklist"
74-
if dg.ListType == depguardAPI.LTWhitelist {
75-
msgSuffix = "is not in the whitelist"
76-
}
77-
res := make([]result.Issue, 0, len(issues))
78-
for _, i := range issues {
79-
userSuppliedMsgSuffix := lintCtx.Settings().Depguard.PackagesWithErrorMessage[i.PackageName]
80-
if userSuppliedMsgSuffix != "" {
81-
userSuppliedMsgSuffix = ": " + userSuppliedMsgSuffix
78+
loadConfig := &loader.Config{
79+
Cwd: "", // fallbacked to os.Getcwd
80+
Build: nil, //fallbacked to build.Default
81+
}
82+
issues, err := dg.Run(loadConfig, prog)
83+
if err != nil {
84+
return nil, err
85+
}
86+
if len(issues) == 0 {
87+
return nil, nil
88+
}
89+
msgSuffix := "is in the blacklist"
90+
if dg.ListType == depguardAPI.LTWhitelist {
91+
msgSuffix = "is not in the whitelist"
92+
}
93+
res := make([]result.Issue, 0, len(issues))
94+
for _, i := range issues {
95+
userSuppliedMsgSuffix := dgSettings.PackagesWithErrorMessage[i.PackageName]
96+
if userSuppliedMsgSuffix != "" {
97+
userSuppliedMsgSuffix = ": " + userSuppliedMsgSuffix
98+
}
99+
res = append(res, result.Issue{
100+
Pos: i.Position,
101+
Text: fmt.Sprintf("%s %s%s", formatCode(i.PackageName, lintCtx.Cfg), msgSuffix, userSuppliedMsgSuffix),
102+
FromLinter: linterName,
103+
})
104+
}
105+
mu.Lock()
106+
resIssues = append(resIssues, res...)
107+
mu.Unlock()
108+
109+
return nil, nil
82110
}
83-
res = append(res, result.Issue{
84-
Pos: i.Position,
85-
Text: fmt.Sprintf("%s %s%s", formatCode(i.PackageName, lintCtx.Cfg), msgSuffix, userSuppliedMsgSuffix),
86-
FromLinter: d.Name(),
87-
})
88-
}
89-
return res, nil
111+
}).WithIssuesReporter(func(*linter.Context) []result.Issue {
112+
return resIssues
113+
}).WithLoadMode(goanalysis.LoadModeTypesInfo)
90114
}

‎pkg/golinters/dogsled.go

+37-20
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,54 @@
11
package golinters
22

33
import (
4-
"context"
54
"fmt"
65
"go/ast"
76
"go/token"
7+
"sync"
88

9+
"golang.org/x/tools/go/analysis"
10+
11+
"github.com/golangci/golangci-lint/pkg/golinters/goanalysis"
912
"github.com/golangci/golangci-lint/pkg/lint/linter"
1013
"github.com/golangci/golangci-lint/pkg/result"
1114
)
1215

13-
type Dogsled struct{}
14-
15-
func (Dogsled) Name() string {
16-
return "dogsled"
17-
}
16+
const dogsledLinterName = "dogsled"
1817

19-
func (Dogsled) Desc() string {
20-
return "Checks assignments with too many blank identifiers (e.g. x, _, _, _, := f())"
21-
}
18+
func NewDogsled() *goanalysis.Linter {
19+
var mu sync.Mutex
20+
var resIssues []result.Issue
2221

23-
func (d Dogsled) Run(ctx context.Context, lintCtx *linter.Context) ([]result.Issue, error) {
24-
var res []result.Issue
25-
for _, f := range lintCtx.ASTCache.GetAllValidFiles() {
26-
v := returnsVisitor{
27-
maxBlanks: lintCtx.Settings().Dogsled.MaxBlankIdentifiers,
28-
f: f.Fset,
29-
}
30-
ast.Walk(&v, f.F)
31-
res = append(res, v.issues...)
22+
analyzer := &analysis.Analyzer{
23+
Name: goanalysis.TheOnlyAnalyzerName,
24+
Doc: goanalysis.TheOnlyanalyzerDoc,
3225
}
26+
return goanalysis.NewLinter(
27+
dogsledLinterName,
28+
"Checks assignments with too many blank identifiers (e.g. x, _, _, _, := f())",
29+
[]*analysis.Analyzer{analyzer},
30+
nil,
31+
).WithContextSetter(func(lintCtx *linter.Context) {
32+
analyzer.Run = func(pass *analysis.Pass) (interface{}, error) {
33+
var pkgIssues []result.Issue
34+
for _, f := range pass.Files {
35+
v := returnsVisitor{
36+
maxBlanks: lintCtx.Settings().Dogsled.MaxBlankIdentifiers,
37+
f: pass.Fset,
38+
}
39+
ast.Walk(&v, f)
40+
pkgIssues = append(pkgIssues, v.issues...)
41+
}
3342

34-
return res, nil
43+
mu.Lock()
44+
resIssues = append(resIssues, pkgIssues...)
45+
mu.Unlock()
46+
47+
return nil, nil
48+
}
49+
}).WithIssuesReporter(func(*linter.Context) []result.Issue {
50+
return resIssues
51+
}).WithLoadMode(goanalysis.LoadModeSyntax)
3552
}
3653

3754
type returnsVisitor struct {
@@ -68,7 +85,7 @@ func (v *returnsVisitor) Visit(node ast.Node) ast.Visitor {
6885

6986
if numBlank > v.maxBlanks {
7087
v.issues = append(v.issues, result.Issue{
71-
FromLinter: Dogsled{}.Name(),
88+
FromLinter: dogsledLinterName,
7289
Text: fmt.Sprintf("declaration has %v blank identifiers", numBlank),
7390
Pos: v.f.Position(assgnStmt.Pos()),
7491
})

0 commit comments

Comments
 (0)