improve benchmark

Gopkg.toml

@@ -73,3 +73,7 @@
 [[constraint]]
   name = "github.com/spf13/viper"
   version = "1.0.2"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/mitchellh/go-ps"

README.md

@@ -1,7 +1,7 @@
 # GolangCI-Lint
 [![Build Status](https://travis-ci.com/golangci/golangci-lint.svg?branch=master)](https://travis-ci.com/golangci/golangci-lint)
 
-GolangCI-Lint is a linters aggregator. It is [fast](#performance) (2-6 times faster than gometalinter), [easy to integrate and use](#issues-options), has [nice output](quick-start) and has minimum count of false positives.
+GolangCI-Lint is a linters aggregator. It is [fast](#performance) (2-7 times faster than gometalinter), [easy to integrate and use](#issues-options), has [nice output](quick-start) and has minimum count of false positives.
 
 Sponsored by [GolangCI.com](https://golangci.com): SaaS service for running linters on Github pull requests. Free for Open Source.
 
@@ -94,7 +94,7 @@ $ golangci-lint run --disable-all -E errcheck
 # Comparison
 ## `golangci-lint` vs `gometalinter`
 GolangCI-Lint was created to fix next issues with `gometalinter`:
-1. Slow work: `gometalinter` usually works for minutes in average projects. GolangCI-Lint works [2-6x times faster](#performance) by [reusing work](#internals).
+1. Slow work: `gometalinter` usually works for minutes in average projects. GolangCI-Lint works [2-7x times faster](#performance) by [reusing work](#internals).
 2. Huge memory consumption: parallel linters don't share the same program representation and can eat `n` times more memory (`n` - concurrency). GolangCI-Lint fixes it by sharing representation.
 3. Can't set honest concurrency: if you set it to `n` it can take `n+x` threads because of forced threads in specific linters. `gometalinter` can't do anything about it, because it runs linters as black-boxes in forked processes. In GolangCI-Lint we run all linters in one process and fully control them. Configured concurrency will be honest.
 This issue is important because often you'd like to set concurrency to CPUs count minus one to save one CPU for example for IDE. It concurrency isn't correct you will have troubles using IDE while analyzing code.
@@ -112,7 +112,9 @@ This issue is important because often you'd like to set concurrency to CPUs coun
 # Performance
 Benchmarks were executed on MacBook Pro (Retina, 13-inch, Late 2013), 2,4 GHz Intel Core i5, 8 GB 1600 MHz DDR3. It has 4 cores and concurrency for linters was default: number of cores. Benchmark runs and measures timings automatically, it's code is [here](https://github.com/golangci/golangci-lint/blob/master/pkg/enabled_linters_test.go) (`BenchmarkWithGometalinter`).
 
-## Default Mode
+We measure peak memory usage (RSS) by measurement of processes RSS every 5 ms.
+
+## Comparison with gometalinter
 We compare golangci-lint and gometalinter in default mode, but explicitly specify all linters to enable because of small differences in default configuration.
 ```bash
 $ golangci-lint run --no-config --issues-exit-code=0 --deadline=30m \
@@ -127,38 +129,20 @@ $ gometalinter --deadline=30m --vendor --cyclo-over=30 --dupl-threshold=150 \
 	./...
 ```
 
-| Repository | GolangCI Lint Time | GolangCI Is Faster In |
-| ---------- | ------------------ | --------------------- |
-| self repo, 4.4 kLoC         | 9.1s  | **6.6x** |
-| gometalinter repo, 3.8 kLoC | 5.1s  | **4.9x** |
-| hugo, 69 kLoC               | 12.4s | **5.8x** |
-| go source, 1300 kLoC        | 3m15s | **1.8x** |
-
-On average golangci-lint is 4.8 times faster than gometalinter. Maximum difference is in
-self repo: 6.6 times faster, minimum difference is in go source code repo: 1.8 times faster.
-
-## Fast Mode
-We compare golangci-lint and gometalinter in fast mode (`--fast`), but don't use option `--fast` because it differs a little.
-Instead we configure common linters from this option.
-```bash
-$ golangci-lint run --no-config --issues-exit-code=0 --deadline=30m \
-	--disable-all --enable=govet --enable=dupl --enable=goconst --enable=gocyclo --enable=golint --enable=ineffassign
-$ gometalinter --deadline=30m --vendor --cyclo-over=30 --dupl-threshold=150 \
-	--exclude=<defaul golangci-lint excludes> --skip=testdata --skip=builtin \
-	--disable-all --enable=vet --enable=vetshadow -enable=dupl --enable=goconst --enable=gocyclo --enable=golint --enable=ineffassign \
-	./...
-```
-
-| Repository | GolangCI Lint Time | GolangCI Is Faster In |
-| ---------- | ------------------ | --------------------- |
-| self repo, 4.4 kLoC         | 0.4s  | **3.1x**  |
-| gometalinter repo, 3.8 kLoC | 0.2s  | **1.9x**  |
-| hugo, 69 kLoC               | 1.6s  | **4x**    |
-| go source, 1300 kLoC        | 35.4s | **1.2x**  |
+| Repository | GolangCI Time | GolangCI Is Faster than Gometalinter | GolangCI Memory | GolangCI eats less memory than Gometalinter |
+| ---------- | ------------- | ------------------------------------ | --------------- | ------------------------------------------- |
+| gometalinter repo, 4 kLoC   | 6s    | **6.4x** | 0.7GB | 1.5x |
+| self repo, 4 kLoC           | 12s   | **7.5x** | 1.2GB | 1.7x |
+| beego, 50 kLoC              | 10s   | **4.2x** | 1.4GB | 1.1x |
+| hugo, 70 kLoC               | 15s   | **6.1x** | 1.6GB | 1.8x |
+| consul, 127 kLoC            | 58s   | **4x**   | 2.7GB | 1.7x |
+| terraform, 190 kLoC         | 2m13s | **1.6x** | 4.8GB | 1x   |
+| go-ethereum, 250 kLoC       | 33s   | **5x**   | 3.6GB | 1x   |
+| go source, 1300 kLoC        | 2m45s | **2x**   | 4.7GB | 1x   |
 
-On average golangci-lint is 2.5 times faster than gometalinter. Maximum difference is in
-self repo: 3.1 times faster, minimum difference is in go source code repo: 20% faster.
 
+**On average golangci-lint is 4.6 times faster** than gometalinter. Maximum difference is in
+self repo: **7.5 times faster**, minimum difference is in terraform source code repo: 1.8 times faster.
 
 # Supported Linters
 To see a list of supported linters and which linters are enabled/disabled by default execute command

pkg/enabled_linters_test.go

@@ -14,7 +14,8 @@ import (
 	"time"
 
 	"github.com/golangci/golangci-lint/pkg/config"
-	"github.com/shirou/gopsutil/mem"
+	gops "github.com/mitchellh/go-ps"
+	"github.com/shirou/gopsutil/process"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 )
@@ -125,18 +126,6 @@ func getBenchLintersArgs() []string {
 	}, getBenchLintersArgsNoMegacheck()...)
 }
 
-func getBenchFastLintersArgs() []string {
-	return []string{
-		"--enable=dupl",
-		"--enable=goconst",
-		"--enable=gocyclo",
-		"--enable=golint",
-		"--enable=ineffassign",
-		// don't add gas because gometalinter uses old, fast and not working for me version of it.
-		// golangci-lint uses new and slower version of it.
-	}
-}
-
 func getGometalinterCommonArgs() []string {
 	return []string{
 		"--deadline=30m",
@@ -152,19 +141,25 @@ func getGometalinterCommonArgs() []string {
 	}
 }
 
+func printCommand(cmd string, args ...string) {
+	if os.Getenv("PRINT_CMD") != "1" {
+		return
+	}
+	quotedArgs := []string{}
+	for _, a := range args {
+		quotedArgs = append(quotedArgs, strconv.Quote(a))
+	}
+
+	logrus.Warnf("%s %s", cmd, strings.Join(quotedArgs, " "))
+}
+
 func runGometalinter(b *testing.B) {
 	args := []string{}
 	args = append(args, getGometalinterCommonArgs()...)
 	args = append(args, getBenchLintersArgs()...)
 	args = append(args, "./...")
-	_ = exec.Command("gometalinter", args...).Run()
-}
 
-func runGometalinterFast(b *testing.B) {
-	args := []string{}
-	args = append(args, getGometalinterCommonArgs()...)
-	args = append(args, getBenchFastLintersArgs()...)
-	args = append(args, "./...")
+	printCommand("gometalinter", args...)
 	_ = exec.Command("gometalinter", args...).Run()
 }
 
@@ -175,15 +170,7 @@ func getGolangciLintCommonArgs() []string {
 func runGolangciLint(b *testing.B) {
 	args := getGolangciLintCommonArgs()
 	args = append(args, getBenchLintersArgs()...)
-	out, err := exec.Command("golangci-lint", args...).CombinedOutput()
-	if err != nil {
-		b.Fatalf("can't run golangci-lint: %s, %s", err, out)
-	}
-}
-
-func runGolangciLintFast(b *testing.B) {
-	args := getGolangciLintCommonArgs()
-	args = append(args, getBenchFastLintersArgs()...)
+	printCommand("golangci-lint", args...)
 	out, err := exec.Command("golangci-lint", args...).CombinedOutput()
 	if err != nil {
 		b.Fatalf("can't run golangci-lint: %s, %s", err, out)
@@ -206,20 +193,53 @@ func getGoLinesTotalCount(b *testing.B) int {
 	return n
 }
 
-func getUsedMemoryMb(b *testing.B) int {
-	v, err := mem.VirtualMemory()
+func getLinterMemoryMB(b *testing.B, progName string) (int, error) {
+	processes, err := gops.Processes()
 	if err != nil {
-		b.Fatalf("can't get usedmemory: %s", err)
+		b.Fatalf("Can't get processes: %s", err)
+	}
+
+	var progPID int
+	for _, p := range processes {
+		if p.Executable() == progName {
+			progPID = p.Pid()
+			break
+		}
 	}
+	if progPID == 0 {
+		return 0, fmt.Errorf("no process")
+	}
+
+	allProgPIDs := []int{progPID}
+	for _, p := range processes {
+		if p.PPid() == progPID {
+			allProgPIDs = append(allProgPIDs, p.Pid())
+		}
+	}
+
+	var totalProgMemBytes uint64
+	for _, pid := range allProgPIDs {
+		p, err := process.NewProcess(int32(pid))
+		if err != nil {
+			continue // subprocess could die
+		}
+
+		mi, err := p.MemoryInfo()
+		if err != nil {
+			continue
+		}
 
-	return int(v.Used / 1024 / 1024)
+		totalProgMemBytes += mi.RSS
+	}
+
+	return int(totalProgMemBytes / 1024 / 1024), nil
 }
 
-func trackPeakMemoryUsage(b *testing.B, doneCh chan struct{}) chan int {
+func trackPeakMemoryUsage(b *testing.B, doneCh <-chan struct{}, progName string) chan int {
 	resCh := make(chan int)
 	go func() {
 		var peakUsedMemMB int
-		t := time.NewTicker(time.Millisecond * 50)
+		t := time.NewTicker(time.Millisecond * 5)
 		defer t.Stop()
 
 		for {
@@ -231,7 +251,10 @@ func trackPeakMemoryUsage(b *testing.B, doneCh chan struct{}) chan int {
 			case <-t.C:
 			}
 
-			m := getUsedMemoryMb(b)
+			m, err := getLinterMemoryMB(b, progName)
+			if err != nil {
+				continue
+			}
 			if m > peakUsedMemMB {
 				peakUsedMemMB = m
 			}
@@ -240,24 +263,38 @@ func trackPeakMemoryUsage(b *testing.B, doneCh chan struct{}) chan int {
 	return resCh
 }
 
-func runBench(b *testing.B, run func(*testing.B), format string, args ...interface{}) {
-	startUsedMemMB := getUsedMemoryMb(b)
+type runResult struct {
+	peakMemMB int
+	duration  time.Duration
+}
+
+func compare(b *testing.B, gometalinterRun, golangciLintRun func(*testing.B), repoName, mode string, kLOC int) { // nolint
+	gometalinterRes := runOne(b, gometalinterRun, "gometalinter")
+	golangciLintRes := runOne(b, golangciLintRun, "golangci-lint")
+
+	if mode != "" {
+		mode = " " + mode
+	}
+	logrus.Warnf("%s (%d kLoC): golangci-lint%s: time: %s, %.1f times faster; memory: %dMB, %.1f times less",
+		repoName, kLOC, mode,
+		golangciLintRes.duration, gometalinterRes.duration.Seconds()/golangciLintRes.duration.Seconds(),
+		golangciLintRes.peakMemMB, float64(gometalinterRes.peakMemMB)/float64(golangciLintRes.peakMemMB),
+	)
+}
+
+func runOne(b *testing.B, run func(*testing.B), progName string) *runResult {
 	doneCh := make(chan struct{})
-	peakMemCh := trackPeakMemoryUsage(b, doneCh)
-	name := fmt.Sprintf(format, args...)
-	b.Run(name, func(b *testing.B) {
-		for i := 0; i < b.N; i++ {
-			run(b)
-		}
-	})
+	peakMemCh := trackPeakMemoryUsage(b, doneCh, progName)
+	startedAt := time.Now()
+	run(b)
+	duration := time.Since(startedAt)
 	close(doneCh)
+
 	peakUsedMemMB := <-peakMemCh
-	var linterPeakMemUsage int
-	if peakUsedMemMB > startUsedMemMB {
-		linterPeakMemUsage = peakUsedMemMB - startUsedMemMB
+	return &runResult{
+		peakMemMB: peakUsedMemMB,
+		duration:  duration,
 	}
-	logrus.Warnf("%s: start used mem is %dMB, peak used mem is %dMB, linter peak mem usage is %dMB",
-		name, startUsedMemMB, peakUsedMemMB, linterPeakMemUsage)
 }
 
 func BenchmarkWithGometalinter(b *testing.B) {
@@ -280,6 +317,22 @@ func BenchmarkWithGometalinter(b *testing.B) {
 			name:    "hugo",
 			prepare: prepareGithubProject("gohugoio", "hugo"),
 		},
+		{
+			name:    "go-ethereum",
+			prepare: prepareGithubProject("ethereum", "go-ethereum"),
+		},
+		{
+			name:    "beego",
+			prepare: prepareGithubProject("astaxie", "beego"),
+		},
+		{
+			name:    "terraform",
+			prepare: prepareGithubProject("hashicorp", "terraform"),
+		},
+		{
+			name:    "consul",
+			prepare: prepareGithubProject("hashicorp", "consul"),
+		},
 		{
 			name:    "go source code",
 			prepare: prepareGoSource,
@@ -289,10 +342,6 @@ func BenchmarkWithGometalinter(b *testing.B) {
 		bc.prepare(b)
 		lc := getGoLinesTotalCount(b)
 
-		runBench(b, runGometalinterFast, "%s/gometalinter --fast (%d lines of code)", bc.name, lc)
-		runBench(b, runGolangciLintFast, "%s/golangci-lint fast (%d lines of code)", bc.name, lc)
-
-		runBench(b, runGometalinter, "%s/gometalinter (%d lines of code)", bc.name, lc)
-		runBench(b, runGolangciLint, "%s/golangci-lint (%d lines of code)", bc.name, lc)
+		compare(b, runGometalinter, runGolangciLint, bc.name, "", lc/1000)
 	}
 }