Merge pull request #105 from arduino/per1234/embed-schemas

Embed JSON schemas in code

Author: per1234

Diff for: .github/workflows/test.yml

@@ -9,6 +9,7 @@ on:
       - "go.sum"
       - "**/*.go"
       - "**/testdata/**"
+      - "etc/schemas/**/*.json"
   pull_request:
     paths:
       - ".github/workflows/test.yml"
@@ -17,6 +18,7 @@ on:
       - "go.sum"
       - "**/*.go"
       - "**/testdata/**"
+      - "etc/schemas/**/*.json"
 
 jobs:
   test-go:
@@ -44,6 +46,12 @@ jobs:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           version: 3.x
 
+      - name: Generate code
+        run: task go:generate
+
+      - name: Check for forgotten code generation
+        run: git diff --color --exit-code
+
       - name: Build
         run: task build
 

Diff for: .prettierignore

@@ -5,7 +5,7 @@
 .ionide/
 
 # Test files
-/check/checkdata/schema/testdata/invalid-schema.json
+/check/checkdata/schema/testdata/input/invalid-schema.json
 /check/checkdata/testdata/packageindexes/invalid-JSON/package_foo_index.json
 /check/checkfunctions/testdata/packageindexes/invalid-JSON/package_foo_index.json
 /check/checkfunctions/testdata/sketches/InvalidJSONMetadataFile/sketch.json

Diff for: Taskfile.yml

@@ -12,6 +12,16 @@ tasks:
       - task: go:test-unit
       - task: schema:compile
 
+  go:generate:
+    desc: Generate Go code
+    cmds:
+      - go get -u "github.com/go-bindata/go-bindata/[email protected]"
+      - go-bindata -nocompress -nometadata -o "./check/checkdata/schema/schemadata/bindata.go" --pkg schemadata --prefix "./etc/schemas/" "./etc/schemas/"
+      - go-bindata -nocompress -nometadata -o "./check/checkdata/schema/testdata/bindata.go" --pkg testdata --prefix "./check/checkdata/schema/testdata/input/" "./check/checkdata/schema/testdata/input/"
+      - go get -u golang.org/x/tools/cmd/[email protected]
+      - go generate ./...
+      - task: go:format
+
   go:test-unit:
     desc: Run unit tests
     cmds:
@@ -76,7 +86,7 @@ tasks:
   go:format:
     desc: Format Go code
     cmds:
-      - go fmt {{ default .DEFAULT_PACKAGES .PACKAGES }}
+      - gofmt -l -w {{ default .DEFAULT_PATHS .PATHS }}
 
   docs:check:
     desc: Lint and check formatting of documentation files

Diff for: check/check.go

@@ -34,7 +34,7 @@ import (
 func RunChecks(project project.Type) {
 	feedback.Printf("\nChecking %s in %s\n", project.ProjectType, project.Path)
 
-	checkdata.Initialize(project, configuration.SchemasPath())
+	checkdata.Initialize(project)
 
 	for _, checkConfiguration := range checkconfigurations.Configurations() {
 		runCheck, err := shouldRun(checkConfiguration, project)

Diff for: check/checkdata/checkdata.go

@@ -27,15 +27,15 @@ import (
 )
 
 // Initialize gathers the check data for the specified project.
-func Initialize(project project.Type, schemasPath *paths.Path) {
+func Initialize(project project.Type) {
 	superprojectType = project.SuperprojectType
 	projectType = project.ProjectType
 	projectPath = project.Path
 	switch project.ProjectType {
 	case projecttype.Sketch:
 		InitializeForSketch(project)
 	case projecttype.Library:
-		InitializeForLibrary(project, schemasPath)
+		InitializeForLibrary(project)
 	case projecttype.Platform:
 		InitializeForPlatform(project)
 	case projecttype.PackageIndex:

Diff for: check/checkdata/library.go

@@ -21,20 +21,19 @@ import (
 	"net/http"
 	"os"
 
+	"github.com/arduino/arduino-check/check/checkdata/schema"
 	"github.com/arduino/arduino-check/check/checkdata/schema/compliancelevel"
 	"github.com/arduino/arduino-check/project"
 	"github.com/arduino/arduino-check/project/library/libraryproperties"
 	"github.com/arduino/arduino-check/result/feedback"
 	"github.com/arduino/arduino-cli/arduino/libraries"
-	"github.com/arduino/go-paths-helper"
 	"github.com/arduino/go-properties-orderedmap"
 	"github.com/client9/misspell"
-	"github.com/ory/jsonschema/v3"
 	"github.com/sirupsen/logrus"
 )
 
 // Initialize gathers the library check data for the specified project.
-func InitializeForLibrary(project project.Type, schemasPath *paths.Path) {
+func InitializeForLibrary(project project.Type) {
 	var err error
 
 	libraryProperties, libraryPropertiesLoadError = libraryproperties.Properties(project.Path)
@@ -43,7 +42,7 @@ func InitializeForLibrary(project project.Type, schemasPath *paths.Path) {
 		// TODO: can I even do this?
 		libraryPropertiesSchemaValidationResult = nil
 	} else {
-		libraryPropertiesSchemaValidationResult = libraryproperties.Validate(libraryProperties, schemasPath)
+		libraryPropertiesSchemaValidationResult = libraryproperties.Validate(libraryProperties)
 	}
 
 	loadedLibrary, err = libraries.Load(project.Path, libraries.User)
@@ -98,10 +97,10 @@ func LibraryProperties() *properties.Map {
 	return libraryProperties
 }
 
-var libraryPropertiesSchemaValidationResult map[compliancelevel.Type]*jsonschema.ValidationError
+var libraryPropertiesSchemaValidationResult map[compliancelevel.Type]schema.ValidationResult
 
 // LibraryPropertiesSchemaValidationResult returns the result of validating library.properties against the JSON schema.
-func LibraryPropertiesSchemaValidationResult() map[compliancelevel.Type]*jsonschema.ValidationError {
+func LibraryPropertiesSchemaValidationResult() map[compliancelevel.Type]schema.ValidationResult {
 	return libraryPropertiesSchemaValidationResult
 }
 

Diff for: check/checkdata/packageindex_test.go

@@ -53,7 +53,7 @@ func TestInitializeForPackageIndex(t *testing.T) {
 			ProjectType:      projecttype.PackageIndex,
 			SuperprojectType: projecttype.PackageIndex,
 		}
-		Initialize(testProject, nil)
+		Initialize(testProject)
 
 		testTable.packageIndexLoadErrorAssertion(t, PackageIndexLoadError(), testTable.testName)
 		if PackageIndexLoadError() == nil {

Diff for: check/checkdata/platform_test.go

@@ -53,7 +53,7 @@ func TestInitializeForPlatform(t *testing.T) {
 			ProjectType:      projecttype.Platform,
 			SuperprojectType: projecttype.Platform,
 		}
-		Initialize(testProject, nil)
+		Initialize(testProject)
 
 		testTable.boardsTxtLoadErrorAssertion(t, BoardsTxtLoadError(), testTable.testName)
 		if BoardsTxtLoadError() == nil {

Diff for: check/checkdata/schema/parsevalidationresult.go

@@ -20,39 +20,38 @@ import (
 	"fmt"
 	"regexp"
 
-	"github.com/arduino/go-paths-helper"
 	"github.com/ory/jsonschema/v3"
 	"github.com/sirupsen/logrus"
 )
 
 // RequiredPropertyMissing returns whether the given required property is missing from the document.
-func RequiredPropertyMissing(propertyName string, validationResult *jsonschema.ValidationError, schemasPath *paths.Path) bool {
-	return ValidationErrorMatch("#", "/required$", "", "^#/"+propertyName+"$", validationResult, schemasPath)
+func RequiredPropertyMissing(propertyName string, validationResult ValidationResult) bool {
+	return ValidationErrorMatch("#", "/required$", "", "^#/"+propertyName+"$", validationResult)
 }
 
 // PropertyPatternMismatch returns whether the given property did not match the regular expression defined in the JSON schema.
-func PropertyPatternMismatch(propertyName string, validationResult *jsonschema.ValidationError, schemasPath *paths.Path) bool {
-	return ValidationErrorMatch("#/"+propertyName, "/pattern$", "", "", validationResult, schemasPath)
+func PropertyPatternMismatch(propertyName string, validationResult ValidationResult) bool {
+	return ValidationErrorMatch("#/"+propertyName, "/pattern$", "", "", validationResult)
 }
 
 // PropertyLessThanMinLength returns whether the given property is less than the minimum length allowed by the schema.
-func PropertyLessThanMinLength(propertyName string, validationResult *jsonschema.ValidationError, schemasPath *paths.Path) bool {
-	return ValidationErrorMatch("^#/"+propertyName+"$", "/minLength$", "", "", validationResult, schemasPath)
+func PropertyLessThanMinLength(propertyName string, validationResult ValidationResult) bool {
+	return ValidationErrorMatch("^#/"+propertyName+"$", "/minLength$", "", "", validationResult)
 }
 
 // PropertyGreaterThanMaxLength returns whether the given property is greater than the maximum length allowed by the schema.
-func PropertyGreaterThanMaxLength(propertyName string, validationResult *jsonschema.ValidationError, schemasPath *paths.Path) bool {
-	return ValidationErrorMatch("^#/"+propertyName+"$", "/maxLength$", "", "", validationResult, schemasPath)
+func PropertyGreaterThanMaxLength(propertyName string, validationResult ValidationResult) bool {
+	return ValidationErrorMatch("^#/"+propertyName+"$", "/maxLength$", "", "", validationResult)
 }
 
 // PropertyEnumMismatch returns whether the given property does not match any of the items in the enum array.
-func PropertyEnumMismatch(propertyName string, validationResult *jsonschema.ValidationError, schemasPath *paths.Path) bool {
-	return ValidationErrorMatch("#/"+propertyName, "/enum$", "", "", validationResult, schemasPath)
+func PropertyEnumMismatch(propertyName string, validationResult ValidationResult) bool {
+	return ValidationErrorMatch("#/"+propertyName, "/enum$", "", "", validationResult)
 }
 
 // MisspelledOptionalPropertyFound returns whether a misspelled optional property was found.
-func MisspelledOptionalPropertyFound(validationResult *jsonschema.ValidationError, schemasPath *paths.Path) bool {
-	return ValidationErrorMatch("#/", "/misspelledOptionalProperties/", "", "", validationResult, schemasPath)
+func MisspelledOptionalPropertyFound(validationResult ValidationResult) bool {
+	return ValidationErrorMatch("#/", "/misspelledOptionalProperties/", "", "", validationResult)
 }
 
 // ValidationErrorMatch returns whether the given query matches against the JSON schema validation error.
@@ -62,10 +61,9 @@ func ValidationErrorMatch(
 	schemaPointerQuery,
 	schemaPointerValueQuery,
 	failureContextQuery string,
-	validationResult *jsonschema.ValidationError,
-	schemasPath *paths.Path,
+	validationResult ValidationResult,
 ) bool {
-	if validationResult == nil {
+	if validationResult.Result == nil {
 		// No error, so nothing to match.
 		logrus.Trace("Schema validation passed. No match is possible.")
 		return false
@@ -82,28 +80,27 @@ func ValidationErrorMatch(
 		schemaPointerValueRegexp,
 		failureContextRegexp,
 		validationResult,
-		schemasPath)
+	)
 }
 
 func validationErrorMatch(
 	instancePointerRegexp,
 	schemaPointerRegexp,
 	schemaPointerValueRegexp,
 	failureContextRegexp *regexp.Regexp,
-	validationError *jsonschema.ValidationError,
-	schemasPath *paths.Path,
+	validationError ValidationResult,
 ) bool {
 	logrus.Trace("--------Checking schema validation failure match--------")
-	logrus.Tracef("Checking instance pointer: %s match with regexp: %s", validationError.InstancePtr, instancePointerRegexp)
-	if instancePointerRegexp.MatchString(validationError.InstancePtr) {
+	logrus.Tracef("Checking instance pointer: %s match with regexp: %s", validationError.Result.InstancePtr, instancePointerRegexp)
+	if instancePointerRegexp.MatchString(validationError.Result.InstancePtr) {
 		logrus.Tracef("Matched!")
-		matchedSchemaPointer := validationErrorSchemaPointerMatch(schemaPointerRegexp, validationError, schemasPath)
+		matchedSchemaPointer := validationErrorSchemaPointerMatch(schemaPointerRegexp, validationError)
 		if matchedSchemaPointer != "" {
 			logrus.Tracef("Matched!")
-			if validationErrorSchemaPointerValueMatch(schemaPointerValueRegexp, validationError.SchemaURL, matchedSchemaPointer, schemasPath) {
+			if validationErrorSchemaPointerValueMatch(schemaPointerValueRegexp, validationError, matchedSchemaPointer) {
 				logrus.Tracef("Matched!")
-				logrus.Tracef("Checking failure context: %v match with regexp: %s", validationError.Context, failureContextRegexp)
-				if validationErrorContextMatch(failureContextRegexp, validationError) {
+				logrus.Tracef("Checking failure context: %v match with regexp: %s", validationError.Result.Context, failureContextRegexp)
+				if validationErrorContextMatch(failureContextRegexp, validationError.Result) {
 					logrus.Tracef("Matched!")
 					return true
 				}
@@ -112,14 +109,16 @@ func validationErrorMatch(
 	}
 
 	// Recursively check all causes for a match.
-	for _, validationErrorCause := range validationError.Causes {
+	for _, validationErrorCause := range validationError.Result.Causes {
 		if validationErrorMatch(
 			instancePointerRegexp,
 			schemaPointerRegexp,
 			schemaPointerValueRegexp,
 			failureContextRegexp,
-			validationErrorCause,
-			schemasPath,
+			ValidationResult{
+				Result:     validationErrorCause,
+				dataLoader: validationError.dataLoader,
+			},
 		) {
 			return true
 		}
@@ -131,18 +130,17 @@ func validationErrorMatch(
 // validationErrorSchemaPointerMatch matches the JSON schema pointer related to the validation failure against a regular expression.
 func validationErrorSchemaPointerMatch(
 	schemaPointerRegexp *regexp.Regexp,
-	validationError *jsonschema.ValidationError,
-	schemasPath *paths.Path,
+	validationError ValidationResult,
 ) string {
-	logrus.Tracef("Checking schema pointer: %s match with regexp: %s", validationError.SchemaPtr, schemaPointerRegexp)
-	if schemaPointerRegexp.MatchString(validationError.SchemaPtr) {
-		return validationError.SchemaPtr
+	logrus.Tracef("Checking schema pointer: %s match with regexp: %s", validationError.Result.SchemaPtr, schemaPointerRegexp)
+	if schemaPointerRegexp.MatchString(validationError.Result.SchemaPtr) {
+		return validationError.Result.SchemaPtr
 	}
 
 	// The schema validator does not provide full pointer past logic inversion keywords to the lowest level keywords related to the validation error cause.
 	// Therefore, the sub-keywords must be checked for matches in order to be able to interpret the exact cause of the failure.
-	if regexp.MustCompile("(/not)|(/oneOf)$").MatchString(validationError.SchemaPtr) {
-		return validationErrorSchemaSubPointerMatch(schemaPointerRegexp, validationError.SchemaPtr, validationErrorSchemaPointerValue(validationError, schemasPath))
+	if regexp.MustCompile("(/not)|(/oneOf)$").MatchString(validationError.Result.SchemaPtr) {
+		return validationErrorSchemaSubPointerMatch(schemaPointerRegexp, validationError.Result.SchemaPtr, validationErrorSchemaPointerValue(validationError))
 	}
 
 	return ""
@@ -184,11 +182,10 @@ func validationErrorSchemaSubPointerMatch(schemaPointerRegexp *regexp.Regexp, pa
 // it matches against the given regular expression.
 func validationErrorSchemaPointerValueMatch(
 	schemaPointerValueRegexp *regexp.Regexp,
-	schemaURL,
+	validationError ValidationResult,
 	schemaPointer string,
-	schemasPath *paths.Path,
 ) bool {
-	marshalledSchemaPointerValue, err := json.Marshal(schemaPointerValue(schemaURL, schemaPointer, schemasPath))
+	marshalledSchemaPointerValue, err := json.Marshal(schemaPointerValue(validationError.Result.SchemaURL, schemaPointer, validationError.dataLoader))
 	if err != nil {
 		panic(err)
 	}