dev: add targets to benchmark a linter (#4761)

Author: ldez

Makefile

@@ -60,6 +60,26 @@ fast_check_generated:
 	git checkout -- go.mod go.sum # can differ between go1.16 and go1.17
 	git diff --exit-code # check no changes
 
+# Benchmark
+
+# Benchmark with a local version
+# LINTER=gosec VERSION=v1.59.0 make bench_local
+bench_local: hyperfine
+	@:$(call check_defined, LINTER VERSION, 'missing parameter(s)')
+	@./scripts/bench/bench_local.sh $(LINTER) $(VERSION)
+.PHONY: bench_local
+
+# Benchmark between 2 existing versions
+# make bench_version LINTER=gosec VERSION_OLD=v1.58.2 VERSION_NEW=v1.59.0
+bench_version: hyperfine
+	@:$(call check_defined, LINTER VERSION_OLD VERSION_NEW, 'missing parameter(s)')
+	@./scripts/bench/bench_version.sh $(LINTER) $(VERSION_OLD) $(VERSION_NEW)
+.PHONY: bench_version
+
+hyperfine:
+	@which hyperfine > /dev/null || (echo "Please install hyperfine https://github.com/sharkdp/hyperfine#installation" && exit 1)
+.PHONY: hyperfine
+
 # Non-PHONY targets (real files)
 
 $(BINARY): FORCE
@@ -102,3 +122,19 @@ website_dump_info:
 update_contributors_list:
 	cd .github/contributors && npm run all
 
+# Functions
+
+# Check that given variables are set and all have non-empty values,
+# die with an error otherwise.
+#
+# Params:
+#   1. Variable name(s) to test.
+#   2. (optional) Error message to print.
+#
+# https://stackoverflow.com/a/10858332/8228109
+check_defined = \
+    $(strip $(foreach 1,$1, \
+        $(call __check_defined,$1,$(strip $(value 2)))))
+__check_defined = \
+    $(if $(value $1),, \
+      $(error Undefined $1$(if $2, ($2))))

scripts/bench/bench_local.sh

@@ -0,0 +1,52 @@
+#!/bin/bash -e
+
+# Benchmark with a local version
+# Usage: ./scripts/bench/bench_local.sh gosec v1.59.0
+
+# ex: gosec
+LINTER=$1
+
+# ex: v1.59.0
+VERSION=$2
+
+
+if [ -z "$LINTER" ] || [ -z "$VERSION" ]; then
+  cat <<-EOF
+Missing required arguments!
+
+Usage:   $0 <linter> <old version> <new version>
+Example: $0 gosec v1.58.1 v1.58.2
+EOF
+
+  exit 1
+fi
+
+EOF
+
+## Clean
+
+function cleanBinaries() {
+  echo "Clean binaries"
+  rm "./golangci-lint-${VERSION}"
+  rm ./golangci-lint
+}
+
+trap cleanBinaries EXIT
+
+## Download version
+
+curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b "./temp-${VERSION}" "${VERSION}"
+
+mv "temp-${VERSION}/golangci-lint" "./golangci-lint-${VERSION}"
+rm -rf "temp-${VERSION}"
+
+## Build local version
+
+make build
+
+## Run
+
+hyperfine \
+--prepare './golangci-lint cache clean' "./golangci-lint run --issues-exit-code 0 --print-issued-lines=false --enable-only ${LINTER}" \
+--prepare "./golangci-lint-${VERSION} cache clean" "./golangci-lint-${VERSION} run --issues-exit-code 0 --print-issued-lines=false --enable-only ${LINTER}"
+

scripts/bench/bench_version.sh

@@ -0,0 +1,60 @@
+#!/bin/bash -e
+
+# Benchmark between 2 existing versions
+# Usage: ./scripts/bench/bench_version.sh gosec v1.58.1 v1.58.2
+
+# ex: gosec
+LINTER="$1"
+
+# ex: v1.58.1
+VERSION_OLD="$2"
+# ex: v1.58.2
+VERSION_NEW="$3"
+
+if [ -z "$LINTER" ] || [ -z "$VERSION_OLD" ] || [ -z "$VERSION_NEW" ]; then
+  cat <<-EOF
+Missing required arguments!
+
+Usage:   $0 <linter> <old version> <new version>
+Example: $0 gosec v1.58.1 v1.58.2
+EOF
+
+  exit 1
+fi
+
+EOF
+
+## Clean
+
+function cleanBinaries() {
+  echo "Clean binaries"
+  rm "./golangci-lint-${VERSION_OLD}"
+  rm "./golangci-lint-${VERSION_NEW}"
+}
+
+trap cleanBinaries EXIT
+
+## Install
+
+function install() {
+  local VERSION=$1
+
+  curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b "./temp-${VERSION}" "${VERSION}"
+
+  mv "temp-${VERSION}/golangci-lint" "./golangci-lint-${VERSION}"
+  rm -rf "temp-${VERSION}"
+}
+
+## VERSION_OLD
+
+install "${VERSION_OLD}"
+
+## VERSION_NEW
+
+install "${VERSION_NEW}"
+
+## Run
+
+hyperfine \
+--prepare "./golangci-lint-${VERSION_OLD} cache clean" "./golangci-lint-${VERSION_OLD} run --issues-exit-code 0 --print-issued-lines=false --enable-only ${LINTER}" \
+--prepare "./golangci-lint-${VERSION_NEW} cache clean" "./golangci-lint-${VERSION_NEW} run --issues-exit-code 0 --print-issued-lines=false --enable-only ${LINTER}"

scripts/bench/readme.md

@@ -0,0 +1,15 @@
+# Benchmarks
+
+The script use [Hyperfine](https://github.com/sharkdp/hyperfine) to benchmark the command line of golangci-lint.
+
+## Benchmark one linter: with a local version
+
+```bash
+make bench_local LINTER=gosec VERSION=v1.59.0
+```
+
+## Benchmark one linter: between 2 existing versions
+
+```bash
+make bench_version LINTER=gosec VERSION_OLD=v1.58.1 VERSION_NEW=v1.59.0 
+```