Change --permissive flag to --compliance

The initial design of the tool was based on the idea of simply enforcing the Arduino project specifications. It soon
became clear that, due to the legacy of not strictly enforcing specification compliance on Library Manager libraries
where the non-compliance doesn't result in a significant problem, there was a need for a "permissive" mode. In addition,
the need was found for a "strict" mode, in which the tool will advocate best practices above and beyond the
specifications.

For this purpose, the `--permissive` flag was added. However, along the way the original concept of enforcing the
specifications was lost. Permissive mode is an unfortunate necessity. Strict mode is great, but may not be to some
people's taste. So a configuration mode where the tool simply check specification compliance remains useful, and is a
reasonable default.

Author: per1234

Diff for: check/check_test.go

@@ -35,25 +35,25 @@ func Test_shouldRun(t *testing.T) {
 		disableModes          []checkmode.Type
 		enableModes           []checkmode.Type
 		libraryManagerSetting string
-		permissiveSetting     string
+		complianceSetting     string
 		shouldRunAssertion    assert.BoolAssertionFunc
 		errorAssertion        assert.ErrorAssertionFunc
 	}{
-		{"Project type mismatch", projecttype.Library, projecttype.Sketch, []checkmode.Type{}, []checkmode.Type{}, "false", "false", assert.False, assert.NoError},
-		{"Disable mode match", projecttype.Library, projecttype.Library, []checkmode.Type{checkmode.LibraryManagerSubmission}, []checkmode.Type{}, "submit", "false", assert.False, assert.NoError},
-		{"Enable mode match", projecttype.Library, projecttype.Library, []checkmode.Type{}, []checkmode.Type{checkmode.LibraryManagerSubmission}, "submit", "false", assert.True, assert.NoError},
-		{"Disable mode default", projecttype.Library, projecttype.Library, []checkmode.Type{checkmode.Default}, []checkmode.Type{checkmode.LibraryManagerSubmission}, "update", "false", assert.False, assert.NoError},
-		{"Disable mode default override", projecttype.Library, projecttype.Library, []checkmode.Type{checkmode.Default}, []checkmode.Type{checkmode.LibraryManagerSubmission}, "submit", "false", assert.True, assert.NoError},
-		{"Enable mode default", projecttype.Library, projecttype.Library, []checkmode.Type{checkmode.LibraryManagerSubmission}, []checkmode.Type{checkmode.Default}, "update", "false", assert.True, assert.NoError},
-		{"Disable mode default override", projecttype.Library, projecttype.Library, []checkmode.Type{checkmode.LibraryManagerSubmission}, []checkmode.Type{checkmode.Default}, "submit", "false", assert.False, assert.NoError},
-		{"Unable to resolve", projecttype.Library, projecttype.Library, []checkmode.Type{checkmode.LibraryManagerSubmission}, []checkmode.Type{checkmode.LibraryManagerIndexed}, "false", "false", assert.False, assert.Error},
+		{"Project type mismatch", projecttype.Library, projecttype.Sketch, []checkmode.Type{}, []checkmode.Type{}, "false", "specification", assert.False, assert.NoError},
+		{"Disable mode match", projecttype.Library, projecttype.Library, []checkmode.Type{checkmode.LibraryManagerSubmission}, []checkmode.Type{}, "submit", "specification", assert.False, assert.NoError},
+		{"Enable mode match", projecttype.Library, projecttype.Library, []checkmode.Type{}, []checkmode.Type{checkmode.LibraryManagerSubmission}, "submit", "specification", assert.True, assert.NoError},
+		{"Disable mode default", projecttype.Library, projecttype.Library, []checkmode.Type{checkmode.Default}, []checkmode.Type{checkmode.LibraryManagerSubmission}, "update", "specification", assert.False, assert.NoError},
+		{"Disable mode default override", projecttype.Library, projecttype.Library, []checkmode.Type{checkmode.Default}, []checkmode.Type{checkmode.LibraryManagerSubmission}, "submit", "specification", assert.True, assert.NoError},
+		{"Enable mode default", projecttype.Library, projecttype.Library, []checkmode.Type{checkmode.LibraryManagerSubmission}, []checkmode.Type{checkmode.Default}, "update", "specification", assert.True, assert.NoError},
+		{"Disable mode default override", projecttype.Library, projecttype.Library, []checkmode.Type{checkmode.LibraryManagerSubmission}, []checkmode.Type{checkmode.Default}, "submit", "specification", assert.False, assert.NoError},
+		{"Unable to resolve", projecttype.Library, projecttype.Library, []checkmode.Type{checkmode.LibraryManagerSubmission}, []checkmode.Type{checkmode.LibraryManagerIndexed}, "false", "specification", assert.False, assert.Error},
 	}
 
 	flags := test.ConfigurationFlags()
 
 	for _, testTable := range testTables {
 		flags.Set("library-manager", testTable.libraryManagerSetting)
-		flags.Set("permissive", testTable.permissiveSetting)
+		flags.Set("compliance", testTable.complianceSetting)
 
 		configuration.Initialize(flags, []string{"/foo"})
 

Diff for: cli/cli.go

@@ -31,11 +31,11 @@ func Root() *cobra.Command {
 		Run:                   command.ArduinoCheck,
 	}
 
+	rootCommand.PersistentFlags().String("compliance", "specification", "Configure how strict the tool is. Can be {strict|specification|permissive}")
 	rootCommand.PersistentFlags().String("format", "text", "The output format can be {text|json}.")
 	rootCommand.PersistentFlags().String("library-manager", "", "Configure the checks for libraries in the Arduino Library Manager index. Can be {submit|update|false}.\nsubmit: Also run additional checks required to pass before a library is accepted for inclusion in the index.\nupdate: Also run additional checks required to pass before new releases of a library already in the index are accepted.\nfalse: Don't run any Library Manager-specific checks.")
 	rootCommand.PersistentFlags().String("log-format", "text", "The output format for the logs, can be {text|json}.")
 	rootCommand.PersistentFlags().String("log-level", "panic", "Messages with this level and above will be logged. Valid levels are: trace, debug, info, warn, error, fatal, panic")
-	rootCommand.PersistentFlags().Bool("permissive", false, "Only fail when critical issues are detected.")
 	rootCommand.PersistentFlags().String("project-type", "all", "Only check projects of the specified type and their subprojects. Can be {sketch|library|all}.")
 	rootCommand.PersistentFlags().Bool("recursive", true, "Search path recursively for Arduino projects to check. Can be {true|false}.")
 	rootCommand.PersistentFlags().String("report-file", "", "Save a report on the checks to this file.")

Diff for: configuration/checkmode/checkmode.go

@@ -29,14 +29,30 @@ type Type int
 
 //go:generate stringer -type=Type -linecomment
 const (
-	Permissive               Type = iota // --permissive
+	Strict                   Type = iota // strict
+	Specification                        // specification
+	Permissive                           // permissive
 	LibraryManagerSubmission             // --library-manager=submit
 	LibraryManagerIndexed                // --library-manager=update
 	Official                             // ARDUINO_CHECK_OFFICIAL
 	All                                  // always
 	Default                              // default
 )
 
+// ComplianceModeFromString parses the --compliance flag value and returns the corresponding check mode settings.
+func ComplianceModeFromString(complianceModeString string) (bool, bool, bool, error) {
+	switch strings.ToLower(complianceModeString) {
+	case "strict":
+		return true, false, false, nil
+	case "specification":
+		return false, true, false, nil
+	case "permissive":
+		return false, false, true, nil
+	default:
+		return false, false, false, fmt.Errorf("No matching compliance mode for string %s", complianceModeString)
+	}
+}
+
 // LibraryManagerModeFromString parses the --library-manager flag value and returns the corresponding check mode settings.
 func LibraryManagerModeFromString(libraryManagerModeString string) (bool, bool, error) {
 	switch strings.ToLower(libraryManagerModeString) {

Diff for: configuration/checkmode/type_string.go

@@ -33,6 +33,15 @@ import (
 // Initialize sets up the tool configuration according to defaults and user-specified options.
 func Initialize(flags *pflag.FlagSet, projectPaths []string) error {
 	var err error
+
+	complianceString, _ := flags.GetString("compliance")
+	if complianceString != "" {
+		customCheckModes[checkmode.Strict], customCheckModes[checkmode.Specification], customCheckModes[checkmode.Permissive], err = checkmode.ComplianceModeFromString(complianceString)
+		if err != nil {
+			return fmt.Errorf("--compliance flag value %s not valid", complianceString)
+		}
+	}
+
 	outputFormatString, _ := flags.GetString("format")
 	outputFormat, err = outputformat.FromString(outputFormatString)
 	if err != nil {
@@ -61,8 +70,6 @@ func Initialize(flags *pflag.FlagSet, projectPaths []string) error {
 	}
 	logrus.SetLevel(logLevel)
 
-	customCheckModes[checkmode.Permissive], _ = flags.GetBool("permissive")
-
 	superprojectTypeFilterString, _ := flags.GetString("project-type")
 	superprojectTypeFilter, err = projecttype.FromString(superprojectTypeFilterString)
 	if err != nil {
@@ -103,12 +110,14 @@ func Initialize(flags *pflag.FlagSet, projectPaths []string) error {
 	}
 
 	logrus.WithFields(logrus.Fields{
+		"compliance strict mode":          customCheckModes[checkmode.Strict],
+		"compliance specification mode":   customCheckModes[checkmode.Specification],
+		"compliance permissive mode":      customCheckModes[checkmode.Permissive],
 		"output format":                   OutputFormat(),
 		"Library Manager submission mode": customCheckModes[checkmode.LibraryManagerSubmission],
 		"Library Manager update mode":     customCheckModes[checkmode.LibraryManagerIndexed],
 		"log format":                      logFormatString,
 		"log level":                       logrus.GetLevel().String(),
-		"permissive":                      customCheckModes[checkmode.Permissive],
 		"superproject type filter":        SuperprojectTypeFilter(),
 		"recursive":                       Recursive(),
 		"report file":                     ReportFilePath(),

Diff for: configuration/configuration.go

@@ -38,6 +38,31 @@ func init() {
 	projectPaths = []string{projectPath}
 }
 
+func TestInitializeCompliance(t *testing.T) {
+	flags := test.ConfigurationFlags()
+
+	flags.Set("compliance", "foo")
+	assert.Error(t, Initialize(flags, projectPaths))
+
+	flags.Set("compliance", "strict")
+	assert.Nil(t, Initialize(flags, projectPaths))
+	assert.True(t, customCheckModes[checkmode.Strict])
+	assert.False(t, customCheckModes[checkmode.Specification])
+	assert.False(t, customCheckModes[checkmode.Permissive])
+
+	flags.Set("compliance", "specification")
+	assert.Nil(t, Initialize(flags, projectPaths))
+	assert.False(t, customCheckModes[checkmode.Strict])
+	assert.True(t, customCheckModes[checkmode.Specification])
+	assert.False(t, customCheckModes[checkmode.Permissive])
+
+	flags.Set("compliance", "permissive")
+	assert.Nil(t, Initialize(flags, projectPaths))
+	assert.False(t, customCheckModes[checkmode.Strict])
+	assert.False(t, customCheckModes[checkmode.Specification])
+	assert.True(t, customCheckModes[checkmode.Permissive])
+}
+
 func TestInitializeFormat(t *testing.T) {
 	flags := test.ConfigurationFlags()
 	flags.Set("format", "foo")
@@ -94,18 +119,6 @@ func TestInitializeLogFormat(t *testing.T) {
 	assert.Nil(t, Initialize(flags, projectPaths))
 }
 
-func TestInitializePermissive(t *testing.T) {
-	flags := test.ConfigurationFlags()
-
-	flags.Set("permissive", "true")
-	assert.Nil(t, Initialize(flags, projectPaths))
-	assert.True(t, customCheckModes[checkmode.Permissive])
-
-	flags.Set("permissive", "false")
-	assert.Nil(t, Initialize(flags, projectPaths))
-	assert.False(t, customCheckModes[checkmode.Permissive])
-}
-
 func TestInitializeProjectType(t *testing.T) {
 	flags := test.ConfigurationFlags()
 

Diff for: configuration/configuration_test.go

@@ -21,11 +21,11 @@ import "github.com/spf13/pflag"
 // ConfigurationFlags returns a set of the flags used for command line configuration of arduino-check.
 func ConfigurationFlags() *pflag.FlagSet {
 	flags := pflag.NewFlagSet("", pflag.ExitOnError)
+	flags.String("compliance", "specification", "")
 	flags.String("format", "text", "")
 	flags.String("library-manager", "", "")
 	flags.String("log-format", "text", "")
 	flags.String("log-level", "panic", "")
-	flags.Bool("permissive", false, "")
 	flags.String("project-type", "all", "")
 	flags.Bool("recursive", true, "")
 	flags.String("report-file", "", "")