Merge pull request #170 from per1234/indexing-mode

Add a Library Manager configuration for linting while indexing releases

Author: per1234

Diff for: docs/index.md

@@ -65,6 +65,8 @@ The `--report-file` flag causes `arduino-lint` to write the JSON output to the s
 Additional configuration options intended for internal use or development can be set via environment variables:
 
 - `ARDUINO_LINT_OFFICIAL` - Set to `"true"` to run the checks that only apply to official Arduino projects.
+- `ARDUINO_LINT_LIBRARY_MANAGER_INDEXING` - Set to `"true"` to run the checks that apply when adding releases to the
+  Library Manager index.
 - `ARDUINO_LINT_LOG_LEVEL` - Messages with this level and above will be logged.
   - Supported values: `trace`, `debug`, `info`, `warn`, `error`, `fatal`, `panic`
 - `ARDUINO_LINT_LOG_FORMAT` - The output format for the logs.

Diff for: go.mod

@@ -48,7 +48,7 @@ require (
 	go.bug.st/relaxed-semver v0.0.0-20190922224835-391e10178d18
 	go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.14.0 // indirect
 	golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9 // indirect
-	golang.org/x/mod v0.4.1 // indirect
+	golang.org/x/mod v0.4.2 // indirect
 	golang.org/x/net v0.0.0-20201209123823-ac852fbbde11 // indirect
 	golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d // indirect
 	golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf // indirect

Diff for: go.sum

@@ -1182,6 +1182,7 @@ golang.org/x/lint v0.0.0-20190409202823-959b441ac422 h1:QzoH/1pFpZguR8NrRHLcO6jK
 golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20200130185559-910be7a94367 h1:0IiAsCRByjO2QjX7ZPkw5oU9x+n1YqRL802rjC0c3Aw=
 golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
@@ -1191,10 +1192,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.0 h1:8pl+sMODzuvGJkmj2W4kZihvVb5mKm8pB/X44PIQHv8=
-golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.1 h1:Kvvh58BN8Y9/lBi7hTekvtMpm07eUZ0ck5pRHpsMWrY=
-golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180406214816-61147c48b25b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180816102801-aaf60122140d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=

Diff for: internal/configuration/configuration.go

@@ -51,12 +51,24 @@ func Initialize(flags *pflag.FlagSet, projectPaths []string) error {
 
 	libraryManagerModeString, _ := flags.GetString("library-manager")
 	if libraryManagerModeString != "" {
-		customRuleModes[rulemode.LibraryManagerSubmission], customRuleModes[rulemode.LibraryManagerIndexed], err = rulemode.LibraryManagerModeFromString(libraryManagerModeString)
+		customRuleModes[rulemode.LibraryManagerSubmission], customRuleModes[rulemode.LibraryManagerIndexed], customRuleModes[rulemode.LibraryManagerIndexing], err = rulemode.LibraryManagerModeFromString(libraryManagerModeString)
 		if err != nil {
 			return fmt.Errorf("--library-manager flag value %s not valid", libraryManagerModeString)
 		}
 	}
 
+	if libraryManagerModeString, ok := os.LookupEnv("ARDUINO_LINT_LIBRARY_MANAGER_INDEXING"); ok {
+		indexing, err := strconv.ParseBool(libraryManagerModeString)
+		if err != nil {
+			return fmt.Errorf("ARDUINO_LINT_LIBRARY_MANAGER_INDEXING environment variable value %s not valid", libraryManagerModeString)
+		}
+		if indexing {
+			customRuleModes[rulemode.LibraryManagerSubmission] = false
+			customRuleModes[rulemode.LibraryManagerIndexed] = false
+			customRuleModes[rulemode.LibraryManagerIndexing] = true
+		}
+	}
+
 	if logFormatString, ok := os.LookupEnv("ARDUINO_LINT_LOG_FORMAT"); ok {
 		logFormat, err := logFormatFromString(logFormatString)
 		if err != nil {
@@ -124,6 +136,7 @@ func Initialize(flags *pflag.FlagSet, projectPaths []string) error {
 		"output format":                   OutputFormat(),
 		"Library Manager submission mode": customRuleModes[rulemode.LibraryManagerSubmission],
 		"Library Manager update mode":     customRuleModes[rulemode.LibraryManagerIndexed],
+		"Library Manager indexing mode":   customRuleModes[rulemode.LibraryManagerIndexing],
 		"log level":                       logrus.GetLevel().String(),
 		"superproject type filter":        SuperprojectTypeFilter(),
 		"recursive":                       Recursive(),

Diff for: internal/configuration/configuration_test.go

@@ -105,6 +105,16 @@ func TestInitializeLibraryManager(t *testing.T) {
 	assert.Nil(t, Initialize(flags, projectPaths))
 	assert.False(t, customRuleModes[rulemode.LibraryManagerSubmission])
 	assert.False(t, customRuleModes[rulemode.LibraryManagerIndexed])
+
+	os.Setenv("ARDUINO_LINT_LIBRARY_MANAGER_INDEXING", "foo")
+	assert.Error(t, Initialize(flags, projectPaths))
+
+	flags.Set("library-manager", "")
+	os.Setenv("ARDUINO_LINT_LIBRARY_MANAGER_INDEXING", "true")
+	assert.Nil(t, Initialize(flags, projectPaths))
+	assert.False(t, customRuleModes[rulemode.LibraryManagerSubmission])
+	assert.False(t, customRuleModes[rulemode.LibraryManagerIndexed])
+	assert.True(t, customRuleModes[rulemode.LibraryManagerIndexing])
 }
 
 func TestInitializeLogFormat(t *testing.T) {

Diff for: internal/configuration/defaults.go

@@ -33,6 +33,7 @@ var defaultRuleModes = map[projecttype.Type]map[rulemode.Type]bool{
 		rulemode.Permissive:               false,
 		rulemode.LibraryManagerSubmission: false,
 		rulemode.LibraryManagerIndexed:    false,
+		rulemode.LibraryManagerIndexing:   false,
 		rulemode.Official:                 false,
 	},
 	projecttype.Library: {
@@ -41,6 +42,7 @@ var defaultRuleModes = map[projecttype.Type]map[rulemode.Type]bool{
 		rulemode.Permissive:               false,
 		rulemode.LibraryManagerSubmission: true,
 		rulemode.LibraryManagerIndexed:    false,
+		rulemode.LibraryManagerIndexing:   false,
 		rulemode.Official:                 false,
 	},
 	projecttype.Platform: {
@@ -49,6 +51,7 @@ var defaultRuleModes = map[projecttype.Type]map[rulemode.Type]bool{
 		rulemode.Permissive:               false,
 		rulemode.LibraryManagerSubmission: false,
 		rulemode.LibraryManagerIndexed:    false,
+		rulemode.LibraryManagerIndexing:   false,
 		rulemode.Official:                 false,
 	},
 	projecttype.PackageIndex: {
@@ -57,6 +60,7 @@ var defaultRuleModes = map[projecttype.Type]map[rulemode.Type]bool{
 		rulemode.Permissive:               false,
 		rulemode.LibraryManagerSubmission: false,
 		rulemode.LibraryManagerIndexed:    false,
+		rulemode.LibraryManagerIndexing:   false,
 		rulemode.Official:                 false,
 	},
 }

Diff for: internal/configuration/rulemode/rulemode.go

@@ -34,6 +34,7 @@ const (
 	Permissive                           // permissive
 	LibraryManagerSubmission             // submit
 	LibraryManagerIndexed                // update
+	LibraryManagerIndexing               // ARDUINO_LINT_LIBRARY_MANAGER_INDEXING
 	Official                             // ARDUINO_LINT_OFFICIAL
 	Default                              // default
 )
@@ -47,6 +48,7 @@ var Types = map[Type]struct{}{
 	Permissive:               empty,
 	LibraryManagerSubmission: empty,
 	LibraryManagerIndexed:    empty,
+	LibraryManagerIndexing:   empty,
 	Official:                 empty,
 	Default:                  empty,
 }
@@ -66,16 +68,16 @@ func ComplianceModeFromString(complianceModeString string) (bool, bool, bool, er
 }
 
 // LibraryManagerModeFromString parses the --library-manager flag value and returns the corresponding rule mode settings.
-func LibraryManagerModeFromString(libraryManagerModeString string) (bool, bool, error) {
+func LibraryManagerModeFromString(libraryManagerModeString string) (bool, bool, bool, error) {
 	switch strings.ToLower(libraryManagerModeString) {
 	case LibraryManagerSubmission.String():
-		return true, false, nil
+		return true, false, false, nil
 	case LibraryManagerIndexed.String():
-		return false, true, nil
+		return false, true, false, nil
 	case "false":
-		return false, false, nil
+		return false, false, false, nil
 	default:
-		return false, false, fmt.Errorf("No matching Library Manager mode for string %s", libraryManagerModeString)
+		return false, false, false, fmt.Errorf("No matching Library Manager mode for string %s", libraryManagerModeString)
 	}
 }
 
@@ -110,7 +112,7 @@ func Compliance(ruleModes map[Type]bool) string {
 // LibraryManager returns the string identifier for the Library Manager configuration mode.
 func LibraryManager(ruleModes map[Type]bool) string {
 	for key, value := range ruleModes {
-		if value && (key == LibraryManagerSubmission || key == LibraryManagerIndexed) {
+		if value && (key == LibraryManagerSubmission || key == LibraryManagerIndexed || key == LibraryManagerIndexing) {
 			return key.String()
 		}
 	}

Diff for: internal/configuration/rulemode/rulemode_test.go

@@ -40,11 +40,13 @@ func TestMode(t *testing.T) {
 		projecttype.Sketch: {
 			LibraryManagerSubmission: false,
 			LibraryManagerIndexed:    false,
+			LibraryManagerIndexing:   false,
 			Official:                 false,
 		},
 		projecttype.Library: {
 			LibraryManagerSubmission: true,
 			LibraryManagerIndexed:    false,
+			LibraryManagerIndexing:   false,
 			Official:                 false,
 		},
 	}

Diff for: internal/configuration/rulemode/type_string.go

@@ -22,6 +22,8 @@ import (
 	"os"
 
 	"github.com/arduino/arduino-cli/arduino/libraries"
+	"github.com/arduino/arduino-lint/internal/configuration"
+	"github.com/arduino/arduino-lint/internal/configuration/rulemode"
 	"github.com/arduino/arduino-lint/internal/project"
 	"github.com/arduino/arduino-lint/internal/project/library/libraryproperties"
 	"github.com/arduino/arduino-lint/internal/result/feedback"
@@ -56,7 +58,8 @@ func InitializeForLibrary(project project.Type) {
 		}
 	}
 
-	if libraryManagerIndex == nil { // Only download the Library Manager index once.
+	// Download the Library Manager index if needed.
+	if !configuration.RuleModes(project.SuperprojectType)[rulemode.LibraryManagerIndexing] && libraryManagerIndex == nil {
 		url := "http://downloads.arduino.cc/libraries/library_index.json"
 		httpResponse, err := http.Get(url)
 		if err != nil {

Diff for: internal/project/projectdata/library.go

@@ -134,7 +134,7 @@ var configurations = []Type{
 		EnableModes:      []rulemode.Type{rulemode.Default},
 		InfoModes:        nil,
 		WarningModes:     []rulemode.Type{rulemode.Default},
-		ErrorModes:       []rulemode.Type{rulemode.LibraryManagerSubmission, rulemode.LibraryManagerIndexed},
+		ErrorModes:       []rulemode.Type{rulemode.LibraryManagerSubmission, rulemode.LibraryManagerIndexed, rulemode.LibraryManagerIndexing},
 		RuleFunction:     rulefunction.LibraryContainsSymlinks,
 	},
 	{
@@ -150,7 +150,7 @@ var configurations = []Type{
 		EnableModes:      []rulemode.Type{rulemode.Default},
 		InfoModes:        nil,
 		WarningModes:     []rulemode.Type{rulemode.Default},
-		ErrorModes:       []rulemode.Type{rulemode.LibraryManagerSubmission, rulemode.LibraryManagerIndexed, rulemode.Strict},
+		ErrorModes:       []rulemode.Type{rulemode.LibraryManagerSubmission, rulemode.LibraryManagerIndexed, rulemode.LibraryManagerIndexing, rulemode.Strict},
 		RuleFunction:     rulefunction.LibraryHasDotDevelopmentFile,
 	},
 	{
@@ -163,7 +163,7 @@ var configurations = []Type{
 		Description:      "",
 		MessageTemplate:  ".exe file(s) found: {{.}}. Presence of these files blocks addition to the Library Manager index.",
 		DisableModes:     []rulemode.Type{rulemode.Default},
-		EnableModes:      []rulemode.Type{rulemode.LibraryManagerSubmission, rulemode.LibraryManagerIndexed},
+		EnableModes:      []rulemode.Type{rulemode.LibraryManagerSubmission, rulemode.LibraryManagerIndexed, rulemode.LibraryManagerIndexing},
 		InfoModes:        nil,
 		WarningModes:     nil,
 		ErrorModes:       []rulemode.Type{rulemode.Default},
@@ -262,7 +262,7 @@ var configurations = []Type{
 		EnableModes:      []rulemode.Type{rulemode.Default},
 		InfoModes:        nil,
 		WarningModes:     []rulemode.Type{rulemode.Default},
-		ErrorModes:       []rulemode.Type{rulemode.LibraryManagerSubmission, rulemode.LibraryManagerIndexed, rulemode.Strict},
+		ErrorModes:       []rulemode.Type{rulemode.LibraryManagerSubmission, rulemode.LibraryManagerIndexed, rulemode.LibraryManagerIndexing, rulemode.Strict},
 		RuleFunction:     rulefunction.LibraryPropertiesMissing,
 	},
 	{

Diff for: internal/rule/ruleconfiguration/ruleconfiguration.go

@@ -95,6 +95,20 @@ def test_library_manager_invalid(run_command):
     assert not result.ok
 
 
+def test_library_manager_indexing(run_command):
+    result = run_command(
+        cmd=[test_data_path.joinpath("library-manager", "Update")],
+        custom_env={"ARDUINO_LINT_LIBRARY_MANAGER_INDEXING": "true"},
+    )
+    assert result.ok
+
+    result = run_command(
+        cmd=[test_data_path.joinpath("library-manager", "Update")],
+        custom_env={"ARDUINO_LINT_LIBRARY_MANAGER_INDEXING": "foo"},
+    )
+    assert not result.ok
+
+
 @pytest.mark.parametrize(
     "project_folder, expected_exit_statuses",
     [