From 5acadeaa8a8fb142d4374afa779deba7fcd09a70 Mon Sep 17 00:00:00 2001 From: per1234 Date: Tue, 11 May 2021 13:47:22 -0700 Subject: [PATCH] Add a Library Manager configuration for linting while indexing releases Some of Arduino Lint's rules are based on the data in the published Arduino Library Manager index. These allow library maintainers to check compliance with the Library Manager submission or update requirements. Work is now in progress to use Arduino Lint to validate library releases during the generation of the index. In this usage, data from the published index is not relevant, meaning the results from those rules are inaccurate and so downloading the index for the sake of them is a waste. So another configuration mode is needed to to allow the index-based rules to be disabled. Since this mode is of no use to normal users of Arduino Lint, it is enabled via an environment variable so that no unnecessary complexity is added to the standard UI of the tool. The environment variable name is `ARDUINO_LINT_LIBRARY_MANAGER_INDEXING`. --- docs/index.md | 2 ++ go.mod | 2 +- go.sum | 7 +++---- internal/configuration/configuration.go | 15 ++++++++++++++- internal/configuration/configuration_test.go | 10 ++++++++++ internal/configuration/defaults.go | 4 ++++ internal/configuration/rulemode/rulemode.go | 14 ++++++++------ internal/configuration/rulemode/rulemode_test.go | 2 ++ internal/configuration/rulemode/type_string.go | 9 +++++---- internal/project/projectdata/library.go | 5 ++++- .../rule/ruleconfiguration/ruleconfiguration.go | 8 ++++---- test/test_all.py | 14 ++++++++++++++ 12 files changed, 71 insertions(+), 21 deletions(-) diff --git a/docs/index.md b/docs/index.md index cd531dc0..5c9eedca 100644 --- a/docs/index.md +++ b/docs/index.md @@ -65,6 +65,8 @@ The `--report-file` flag causes `arduino-lint` to write the JSON output to the s Additional configuration options intended for internal use or development can be set via environment variables: - `ARDUINO_LINT_OFFICIAL` - Set to `"true"` to run the checks that only apply to official Arduino projects. +- `ARDUINO_LINT_LIBRARY_MANAGER_INDEXING` - Set to `"true"` to run the checks that apply when adding releases to the + Library Manager index. - `ARDUINO_LINT_LOG_LEVEL` - Messages with this level and above will be logged. - Supported values: `trace`, `debug`, `info`, `warn`, `error`, `fatal`, `panic` - `ARDUINO_LINT_LOG_FORMAT` - The output format for the logs. diff --git a/go.mod b/go.mod index 74c09cf5..d53228e5 100644 --- a/go.mod +++ b/go.mod @@ -48,7 +48,7 @@ require ( go.bug.st/relaxed-semver v0.0.0-20190922224835-391e10178d18 go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.14.0 // indirect golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9 // indirect - golang.org/x/mod v0.4.1 // indirect + golang.org/x/mod v0.4.2 // indirect golang.org/x/net v0.0.0-20201209123823-ac852fbbde11 // indirect golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d // indirect golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf // indirect diff --git a/go.sum b/go.sum index 4e09516e..3eb7df8d 100644 --- a/go.sum +++ b/go.sum @@ -1182,6 +1182,7 @@ golang.org/x/lint v0.0.0-20190409202823-959b441ac422 h1:QzoH/1pFpZguR8NrRHLcO6jK golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367 h1:0IiAsCRByjO2QjX7ZPkw5oU9x+n1YqRL802rjC0c3Aw= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -1191,10 +1192,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0 h1:8pl+sMODzuvGJkmj2W4kZihvVb5mKm8pB/X44PIQHv8= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1 h1:Kvvh58BN8Y9/lBi7hTekvtMpm07eUZ0ck5pRHpsMWrY= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180406214816-61147c48b25b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180816102801-aaf60122140d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/internal/configuration/configuration.go b/internal/configuration/configuration.go index a1eb4836..e77be156 100644 --- a/internal/configuration/configuration.go +++ b/internal/configuration/configuration.go @@ -51,12 +51,24 @@ func Initialize(flags *pflag.FlagSet, projectPaths []string) error { libraryManagerModeString, _ := flags.GetString("library-manager") if libraryManagerModeString != "" { - customRuleModes[rulemode.LibraryManagerSubmission], customRuleModes[rulemode.LibraryManagerIndexed], err = rulemode.LibraryManagerModeFromString(libraryManagerModeString) + customRuleModes[rulemode.LibraryManagerSubmission], customRuleModes[rulemode.LibraryManagerIndexed], customRuleModes[rulemode.LibraryManagerIndexing], err = rulemode.LibraryManagerModeFromString(libraryManagerModeString) if err != nil { return fmt.Errorf("--library-manager flag value %s not valid", libraryManagerModeString) } } + if libraryManagerModeString, ok := os.LookupEnv("ARDUINO_LINT_LIBRARY_MANAGER_INDEXING"); ok { + indexing, err := strconv.ParseBool(libraryManagerModeString) + if err != nil { + return fmt.Errorf("ARDUINO_LINT_LIBRARY_MANAGER_INDEXING environment variable value %s not valid", libraryManagerModeString) + } + if indexing { + customRuleModes[rulemode.LibraryManagerSubmission] = false + customRuleModes[rulemode.LibraryManagerIndexed] = false + customRuleModes[rulemode.LibraryManagerIndexing] = true + } + } + if logFormatString, ok := os.LookupEnv("ARDUINO_LINT_LOG_FORMAT"); ok { logFormat, err := logFormatFromString(logFormatString) if err != nil { @@ -124,6 +136,7 @@ func Initialize(flags *pflag.FlagSet, projectPaths []string) error { "output format": OutputFormat(), "Library Manager submission mode": customRuleModes[rulemode.LibraryManagerSubmission], "Library Manager update mode": customRuleModes[rulemode.LibraryManagerIndexed], + "Library Manager indexing mode": customRuleModes[rulemode.LibraryManagerIndexing], "log level": logrus.GetLevel().String(), "superproject type filter": SuperprojectTypeFilter(), "recursive": Recursive(), diff --git a/internal/configuration/configuration_test.go b/internal/configuration/configuration_test.go index 93d96d30..10d03d0b 100644 --- a/internal/configuration/configuration_test.go +++ b/internal/configuration/configuration_test.go @@ -105,6 +105,16 @@ func TestInitializeLibraryManager(t *testing.T) { assert.Nil(t, Initialize(flags, projectPaths)) assert.False(t, customRuleModes[rulemode.LibraryManagerSubmission]) assert.False(t, customRuleModes[rulemode.LibraryManagerIndexed]) + + os.Setenv("ARDUINO_LINT_LIBRARY_MANAGER_INDEXING", "foo") + assert.Error(t, Initialize(flags, projectPaths)) + + flags.Set("library-manager", "") + os.Setenv("ARDUINO_LINT_LIBRARY_MANAGER_INDEXING", "true") + assert.Nil(t, Initialize(flags, projectPaths)) + assert.False(t, customRuleModes[rulemode.LibraryManagerSubmission]) + assert.False(t, customRuleModes[rulemode.LibraryManagerIndexed]) + assert.True(t, customRuleModes[rulemode.LibraryManagerIndexing]) } func TestInitializeLogFormat(t *testing.T) { diff --git a/internal/configuration/defaults.go b/internal/configuration/defaults.go index 5e8d197c..cefbead4 100644 --- a/internal/configuration/defaults.go +++ b/internal/configuration/defaults.go @@ -33,6 +33,7 @@ var defaultRuleModes = map[projecttype.Type]map[rulemode.Type]bool{ rulemode.Permissive: false, rulemode.LibraryManagerSubmission: false, rulemode.LibraryManagerIndexed: false, + rulemode.LibraryManagerIndexing: false, rulemode.Official: false, }, projecttype.Library: { @@ -41,6 +42,7 @@ var defaultRuleModes = map[projecttype.Type]map[rulemode.Type]bool{ rulemode.Permissive: false, rulemode.LibraryManagerSubmission: true, rulemode.LibraryManagerIndexed: false, + rulemode.LibraryManagerIndexing: false, rulemode.Official: false, }, projecttype.Platform: { @@ -49,6 +51,7 @@ var defaultRuleModes = map[projecttype.Type]map[rulemode.Type]bool{ rulemode.Permissive: false, rulemode.LibraryManagerSubmission: false, rulemode.LibraryManagerIndexed: false, + rulemode.LibraryManagerIndexing: false, rulemode.Official: false, }, projecttype.PackageIndex: { @@ -57,6 +60,7 @@ var defaultRuleModes = map[projecttype.Type]map[rulemode.Type]bool{ rulemode.Permissive: false, rulemode.LibraryManagerSubmission: false, rulemode.LibraryManagerIndexed: false, + rulemode.LibraryManagerIndexing: false, rulemode.Official: false, }, } diff --git a/internal/configuration/rulemode/rulemode.go b/internal/configuration/rulemode/rulemode.go index da5f8ca1..06addc7c 100644 --- a/internal/configuration/rulemode/rulemode.go +++ b/internal/configuration/rulemode/rulemode.go @@ -34,6 +34,7 @@ const ( Permissive // permissive LibraryManagerSubmission // submit LibraryManagerIndexed // update + LibraryManagerIndexing // ARDUINO_LINT_LIBRARY_MANAGER_INDEXING Official // ARDUINO_LINT_OFFICIAL Default // default ) @@ -47,6 +48,7 @@ var Types = map[Type]struct{}{ Permissive: empty, LibraryManagerSubmission: empty, LibraryManagerIndexed: empty, + LibraryManagerIndexing: empty, Official: empty, Default: empty, } @@ -66,16 +68,16 @@ func ComplianceModeFromString(complianceModeString string) (bool, bool, bool, er } // LibraryManagerModeFromString parses the --library-manager flag value and returns the corresponding rule mode settings. -func LibraryManagerModeFromString(libraryManagerModeString string) (bool, bool, error) { +func LibraryManagerModeFromString(libraryManagerModeString string) (bool, bool, bool, error) { switch strings.ToLower(libraryManagerModeString) { case LibraryManagerSubmission.String(): - return true, false, nil + return true, false, false, nil case LibraryManagerIndexed.String(): - return false, true, nil + return false, true, false, nil case "false": - return false, false, nil + return false, false, false, nil default: - return false, false, fmt.Errorf("No matching Library Manager mode for string %s", libraryManagerModeString) + return false, false, false, fmt.Errorf("No matching Library Manager mode for string %s", libraryManagerModeString) } } @@ -110,7 +112,7 @@ func Compliance(ruleModes map[Type]bool) string { // LibraryManager returns the string identifier for the Library Manager configuration mode. func LibraryManager(ruleModes map[Type]bool) string { for key, value := range ruleModes { - if value && (key == LibraryManagerSubmission || key == LibraryManagerIndexed) { + if value && (key == LibraryManagerSubmission || key == LibraryManagerIndexed || key == LibraryManagerIndexing) { return key.String() } } diff --git a/internal/configuration/rulemode/rulemode_test.go b/internal/configuration/rulemode/rulemode_test.go index ac795be1..7c44461f 100644 --- a/internal/configuration/rulemode/rulemode_test.go +++ b/internal/configuration/rulemode/rulemode_test.go @@ -40,11 +40,13 @@ func TestMode(t *testing.T) { projecttype.Sketch: { LibraryManagerSubmission: false, LibraryManagerIndexed: false, + LibraryManagerIndexing: false, Official: false, }, projecttype.Library: { LibraryManagerSubmission: true, LibraryManagerIndexed: false, + LibraryManagerIndexing: false, Official: false, }, } diff --git a/internal/configuration/rulemode/type_string.go b/internal/configuration/rulemode/type_string.go index f501d35f..4f89c21a 100644 --- a/internal/configuration/rulemode/type_string.go +++ b/internal/configuration/rulemode/type_string.go @@ -13,13 +13,14 @@ func _() { _ = x[Permissive-2] _ = x[LibraryManagerSubmission-3] _ = x[LibraryManagerIndexed-4] - _ = x[Official-5] - _ = x[Default-6] + _ = x[LibraryManagerIndexing-5] + _ = x[Official-6] + _ = x[Default-7] } -const _Type_name = "strictspecificationpermissivesubmitupdateARDUINO_LINT_OFFICIALdefault" +const _Type_name = "strictspecificationpermissivesubmitupdateARDUINO_LINT_LIBRARY_MANAGER_INDEXINGARDUINO_LINT_OFFICIALdefault" -var _Type_index = [...]uint8{0, 6, 19, 29, 35, 41, 62, 69} +var _Type_index = [...]uint8{0, 6, 19, 29, 35, 41, 78, 99, 106} func (i Type) String() string { if i < 0 || i >= Type(len(_Type_index)-1) { diff --git a/internal/project/projectdata/library.go b/internal/project/projectdata/library.go index 3676e286..211ff3db 100644 --- a/internal/project/projectdata/library.go +++ b/internal/project/projectdata/library.go @@ -22,6 +22,8 @@ import ( "os" "github.com/arduino/arduino-cli/arduino/libraries" + "github.com/arduino/arduino-lint/internal/configuration" + "github.com/arduino/arduino-lint/internal/configuration/rulemode" "github.com/arduino/arduino-lint/internal/project" "github.com/arduino/arduino-lint/internal/project/library/libraryproperties" "github.com/arduino/arduino-lint/internal/result/feedback" @@ -56,7 +58,8 @@ func InitializeForLibrary(project project.Type) { } } - if libraryManagerIndex == nil { // Only download the Library Manager index once. + // Download the Library Manager index if needed. + if !configuration.RuleModes(project.SuperprojectType)[rulemode.LibraryManagerIndexing] && libraryManagerIndex == nil { url := "http://downloads.arduino.cc/libraries/library_index.json" httpResponse, err := http.Get(url) if err != nil { diff --git a/internal/rule/ruleconfiguration/ruleconfiguration.go b/internal/rule/ruleconfiguration/ruleconfiguration.go index 731baa87..4fb9d95f 100644 --- a/internal/rule/ruleconfiguration/ruleconfiguration.go +++ b/internal/rule/ruleconfiguration/ruleconfiguration.go @@ -134,7 +134,7 @@ var configurations = []Type{ EnableModes: []rulemode.Type{rulemode.Default}, InfoModes: nil, WarningModes: []rulemode.Type{rulemode.Default}, - ErrorModes: []rulemode.Type{rulemode.LibraryManagerSubmission, rulemode.LibraryManagerIndexed}, + ErrorModes: []rulemode.Type{rulemode.LibraryManagerSubmission, rulemode.LibraryManagerIndexed, rulemode.LibraryManagerIndexing}, RuleFunction: rulefunction.LibraryContainsSymlinks, }, { @@ -150,7 +150,7 @@ var configurations = []Type{ EnableModes: []rulemode.Type{rulemode.Default}, InfoModes: nil, WarningModes: []rulemode.Type{rulemode.Default}, - ErrorModes: []rulemode.Type{rulemode.LibraryManagerSubmission, rulemode.LibraryManagerIndexed, rulemode.Strict}, + ErrorModes: []rulemode.Type{rulemode.LibraryManagerSubmission, rulemode.LibraryManagerIndexed, rulemode.LibraryManagerIndexing, rulemode.Strict}, RuleFunction: rulefunction.LibraryHasDotDevelopmentFile, }, { @@ -163,7 +163,7 @@ var configurations = []Type{ Description: "", MessageTemplate: ".exe file(s) found: {{.}}. Presence of these files blocks addition to the Library Manager index.", DisableModes: []rulemode.Type{rulemode.Default}, - EnableModes: []rulemode.Type{rulemode.LibraryManagerSubmission, rulemode.LibraryManagerIndexed}, + EnableModes: []rulemode.Type{rulemode.LibraryManagerSubmission, rulemode.LibraryManagerIndexed, rulemode.LibraryManagerIndexing}, InfoModes: nil, WarningModes: nil, ErrorModes: []rulemode.Type{rulemode.Default}, @@ -262,7 +262,7 @@ var configurations = []Type{ EnableModes: []rulemode.Type{rulemode.Default}, InfoModes: nil, WarningModes: []rulemode.Type{rulemode.Default}, - ErrorModes: []rulemode.Type{rulemode.LibraryManagerSubmission, rulemode.LibraryManagerIndexed, rulemode.Strict}, + ErrorModes: []rulemode.Type{rulemode.LibraryManagerSubmission, rulemode.LibraryManagerIndexed, rulemode.LibraryManagerIndexing, rulemode.Strict}, RuleFunction: rulefunction.LibraryPropertiesMissing, }, { diff --git a/test/test_all.py b/test/test_all.py index 026b03af..97260223 100644 --- a/test/test_all.py +++ b/test/test_all.py @@ -95,6 +95,20 @@ def test_library_manager_invalid(run_command): assert not result.ok +def test_library_manager_indexing(run_command): + result = run_command( + cmd=[test_data_path.joinpath("library-manager", "Update")], + custom_env={"ARDUINO_LINT_LIBRARY_MANAGER_INDEXING": "true"}, + ) + assert result.ok + + result = run_command( + cmd=[test_data_path.joinpath("library-manager", "Update")], + custom_env={"ARDUINO_LINT_LIBRARY_MANAGER_INDEXING": "foo"}, + ) + assert not result.ok + + @pytest.mark.parametrize( "project_folder, expected_exit_statuses", [