From e9f5400200bd07cbb48366729090a9b6f743e044 Mon Sep 17 00:00:00 2001 From: Greg Dennis Date: Wed, 23 Nov 2022 18:25:42 +1300 Subject: [PATCH 01/20] initial run at creating output tests --- output-tests/README.md | 26 ++++++++ output-tests/content/draft-next/general.json | 43 +++++++++++++ output-tests/content/draft-next/readOnly.json | 41 ++++++++++++ output-tests/content/draft-next/type.json | 42 +++++++++++++ .../content/draft2019-09/general.json | 34 ++++++++++ .../content/draft2019-09/readOnly.json | 38 +++++++++++ output-tests/content/draft2019-09/type.json | 63 +++++++++++++++++++ 7 files changed, 287 insertions(+) create mode 100644 output-tests/README.md create mode 100644 output-tests/content/draft-next/general.json create mode 100644 output-tests/content/draft-next/readOnly.json create mode 100644 output-tests/content/draft-next/type.json create mode 100644 output-tests/content/draft2019-09/general.json create mode 100644 output-tests/content/draft2019-09/readOnly.json create mode 100644 output-tests/content/draft2019-09/type.json diff --git a/output-tests/README.md b/output-tests/README.md new file mode 100644 index 00000000..11c3a2b6 --- /dev/null +++ b/output-tests/README.md @@ -0,0 +1,26 @@ +These tests are intended to validate that implementations are correctly generating output in accordance with the specification. + +Output was initially specified with draft 2019-09. It remained unchanged for draft 2020-12 , but will receive an update with the next release. + +## Organization + +The tests are organized into two categories: content and structure. + +Content tests verify that the keywords are producing the correct annotations. Since there are no requirements on the content of error messages, there's not much that can be verified for them, if they're even generated. These tests need to extensively cover the annotation behaviors of each keyword. The expected output format for these tests is `basic`. + +Structure tests verify that the structures of the various formats (i.e. `flag`, `basic`, `detailed`, `verbose`) are correct. These tests don't need to cover each keyword; rather they need to sufficiently cover the various aspects of building the output structures by using whatever keywords are necessary to do so. + +## Test Files + +The content of a test file is the same as the validation tests in `tests/`, however an `output` property has been added to each test case. + +The `output` property itself has a property for each of the output formats where the value is a schema that will successfully validate for compliant output. For the content tests, only `basic` needs to be present. + +## Contributing + +Of course, first and foremost, follow the [Contributing guide](/CONTRIBUTING.md). + +When writing test cases, try to keep output validation schemas targeted to verify a single requirement. Where possible (and where it makes sense), create multiple tests to cover multiple requirements. This will help keep the output validation schemas small and increase readability. (It also increases your test count. 😉) + +For the content tests, there is also a _general.json_ file that contains tests that do not necessarily pertain to any single keyword. + diff --git a/output-tests/content/draft-next/general.json b/output-tests/content/draft-next/general.json new file mode 100644 index 00000000..406ed2b3 --- /dev/null +++ b/output-tests/content/draft-next/general.json @@ -0,0 +1,43 @@ +[ + { + "description": "failed validation produces no annotations", + "schema": { + "$schema": "https://json-schema.org/draft/next/schema", + "$id": "https://json-schema.org/tests/content/draft-next/general/1", + "type": "string", + "readOnly": true + }, + "tests": [ + { + "description": "dropped annotations MAY appear in droppedAnnotations", + "data": 1, + "output": { + "basic": { + "type": "object", + "properties": { + "details": { + "contains": { + "type": "object", + "properties": { + "evaluationPath": {"const": ""}, + "schemaLocation": {"const": "https://json-schema.org/tests/content/draft-next/general/1"}, + "instanceLocation": {"const": ""}, + "annotations": false, + "droppedAnnotations": { + "properties": { + "readOnly": {"const": true} + }, + "required": ["readOnly"] + } + }, + "required": ["evaluationPath", "schemaLocation", "instanceLocation"] + } + } + }, + "required": ["details"] + } + } + } + ] + } +] \ No newline at end of file diff --git a/output-tests/content/draft-next/readOnly.json b/output-tests/content/draft-next/readOnly.json new file mode 100644 index 00000000..0e4acbee --- /dev/null +++ b/output-tests/content/draft-next/readOnly.json @@ -0,0 +1,41 @@ +[ + { + "description": "readOnly generates its value as an annotation", + "schema": { + "$schema": "https://json-schema.org/draft/next/schema", + "$id": "https://json-schema.org/tests/content/draft-next/readOnly/1", + "readOnly": true + }, + "tests": [ + { + "description": "readOnly is true", + "data": 1, + "output": { + "basic": { + "type": "object", + "properties": { + "details": { + "contains": { + "type": "object", + "properties": { + "evaluationPath": {"const": ""}, + "schemaLocation": {"const": "https://json-schema.org/tests/content/draft-next/readOnly/1"}, + "instanceLocation": {"const": ""}, + "annotations": { + "properties": { + "readOnly": {"const": true} + }, + "required": ["readOnly"] + } + }, + "required": ["evaluationPath", "schemaLocation", "instanceLocation", "annotations"] + } + } + }, + "required": ["details"] + } + } + } + ] + } +] \ No newline at end of file diff --git a/output-tests/content/draft-next/type.json b/output-tests/content/draft-next/type.json new file mode 100644 index 00000000..06659aab --- /dev/null +++ b/output-tests/content/draft-next/type.json @@ -0,0 +1,42 @@ +[ + { + "description": "incorrect type", + "schema": { + "$schema": "https://json-schema.org/draft/next/schema", + "$id": "https://json-schema.org/tests/content/draft-next/type/1", + "type": "string" + }, + "tests": [ + { + "description": "incorrect type may be reported", + "data": 1, + "output": { + "basic": { + "type": "object", + "properties": { + "details": { + "contains": { + "type": "object", + "properties": { + "evaluationPath": {"const": ""}, + "schemaLocation": {"const": "https://json-schema.org/tests/content/draft-next/type/1"}, + "instanceLocation": {"const": ""}, + "annotations": false, + "errors": { + "properties": { + "type": {"type": "string"} + }, + "required": ["type"] + } + }, + "required": ["evaluationPath", "schemaLocation", "instanceLocation"] + } + } + }, + "required": ["details"] + } + } + } + ] + } +] \ No newline at end of file diff --git a/output-tests/content/draft2019-09/general.json b/output-tests/content/draft2019-09/general.json new file mode 100644 index 00000000..3c764f01 --- /dev/null +++ b/output-tests/content/draft2019-09/general.json @@ -0,0 +1,34 @@ +[ + { + "description": "failed validation produces no annotations", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://json-schema.org/tests/content/draft2019-09/general/1", + "type": "string", + "readOnly": true + }, + "tests": [ + { + "description": "readOnly annotation is dropped", + "data": 1, + "output": { + "basic": { + "type": "object", + "properties": { + "errors": { + "type": "array", + "items": { + "properties": { + "annotation": false + } + } + }, + "annotations": false + }, + "required": ["errors"] + } + } + } + ] + } +] \ No newline at end of file diff --git a/output-tests/content/draft2019-09/readOnly.json b/output-tests/content/draft2019-09/readOnly.json new file mode 100644 index 00000000..7a054235 --- /dev/null +++ b/output-tests/content/draft2019-09/readOnly.json @@ -0,0 +1,38 @@ +[ + { + "description": "readOnly generates its value as an annotation", + "schema": { + "$schema": "https://json-schema.org/draft/2019-09/schema", + "$id": "https://json-schema.org/tests/content/draft2019-09/readOnly/1", + "readOnly": true + }, + "tests": [ + { + "description": "readOnly is true", + "data": 1, + "output": { + "basic": { + "type": "object", + "properties": { + "annotations": { + "type": "array", + "contains": { + "type": "object", + "properties": { + "keywordLocation": {"const": "/readOnly"}, + "absoluteKeywordLocation": {"const": "https://json-schema.org/tests/content/draft2019-09/readOnly/1#/readOnly"}, + "instanceLocation": {"const": ""}, + "annotation": {"const": true} + }, + "required": ["keywordLocation", "absoluteKeywordLocation", "instanceLocation", "annotation"] + } + }, + "errors": false + }, + "required": ["annotations"] + } + } + } + ] + } +] \ No newline at end of file diff --git a/output-tests/content/draft2019-09/type.json b/output-tests/content/draft2019-09/type.json new file mode 100644 index 00000000..b1d2fe92 --- /dev/null +++ b/output-tests/content/draft2019-09/type.json @@ -0,0 +1,63 @@ +[ + { + "description": "validating type", + "schema": { + "$schema": "https://json-schema.org/draft/2019-09/schema", + "$id": "https://json-schema.org/tests/content/draft2019-09/type/1", + "type": "string" + }, + "tests": [ + { + "description": "incorrect type must be reported, but a message is not required", + "data": 1, + "output": { + "basic": { + "type": "object", + "properties": { + "errors": { + "contains": { + "type": "object", + "properties": { + "keywordLocation": {"const": "/type"}, + "absoluteKeywordLocation": {"const": "https://json-schema.org/tests/content/draft2019-09/type/1#/type"}, + "instanceLocation": {"const": ""}, + "annotation": false, + "error": {"type": "string"} + }, + "required": ["keywordLocation", "absoluteKeywordLocation", "instanceLocation"] + } + } + }, + "required": ["errors"] + } + } + }, + { + "description": "correct type yields an output unit", + "data": 1, + "output": { + "basic": { + "type": "object", + "properties": { + "annotations": { + "contains": { + "type": "object", + "properties": { + "valid": {"const": true}, + "keywordLocation": {"const": "/type"}, + "absoluteKeywordLocation": {"const": "https://json-schema.org/tests/content/draft2019-09/type/1#/type"}, + "instanceLocation": {"const": ""}, + "annotation": false, + "error": false + }, + "required": ["keywordLocation", "absoluteKeywordLocation", "instanceLocation"] + } + } + }, + "required": ["annotations"] + } + } + } + ] + } +] \ No newline at end of file From 87c184d200ececc363c7804236ebe8cee8d65cd0 Mon Sep 17 00:00:00 2001 From: Greg Dennis Date: Thu, 24 Nov 2022 09:09:59 +1300 Subject: [PATCH 02/20] add .editorconfig; rearrange folders --- .editorconfig | 1 + .../{content/draft-next => draft-next/content}/general.json | 2 +- .../{content/draft-next => draft-next/content}/readOnly.json | 2 +- .../{content/draft-next => draft-next/content}/type.json | 2 +- .../{content/draft2019-09 => draft2019-09/content}/general.json | 2 +- .../draft2019-09 => draft2019-09/content}/readOnly.json | 2 +- .../{content/draft2019-09 => draft2019-09/content}/type.json | 2 +- 7 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 .editorconfig rename output-tests/{content/draft-next => draft-next/content}/general.json (99%) rename output-tests/{content/draft-next => draft-next/content}/readOnly.json (99%) rename output-tests/{content/draft-next => draft-next/content}/type.json (99%) rename output-tests/{content/draft2019-09 => draft2019-09/content}/general.json (99%) rename output-tests/{content/draft2019-09 => draft2019-09/content}/readOnly.json (99%) rename output-tests/{content/draft2019-09 => draft2019-09/content}/type.json (99%) diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..59e6583c --- /dev/null +++ b/.editorconfig @@ -0,0 +1 @@ +insert_final_newline = true \ No newline at end of file diff --git a/output-tests/content/draft-next/general.json b/output-tests/draft-next/content/general.json similarity index 99% rename from output-tests/content/draft-next/general.json rename to output-tests/draft-next/content/general.json index 406ed2b3..6001dec2 100644 --- a/output-tests/content/draft-next/general.json +++ b/output-tests/draft-next/content/general.json @@ -40,4 +40,4 @@ } ] } -] \ No newline at end of file +] diff --git a/output-tests/content/draft-next/readOnly.json b/output-tests/draft-next/content/readOnly.json similarity index 99% rename from output-tests/content/draft-next/readOnly.json rename to output-tests/draft-next/content/readOnly.json index 0e4acbee..5b4b7818 100644 --- a/output-tests/content/draft-next/readOnly.json +++ b/output-tests/draft-next/content/readOnly.json @@ -38,4 +38,4 @@ } ] } -] \ No newline at end of file +] diff --git a/output-tests/content/draft-next/type.json b/output-tests/draft-next/content/type.json similarity index 99% rename from output-tests/content/draft-next/type.json rename to output-tests/draft-next/content/type.json index 06659aab..de8d5c17 100644 --- a/output-tests/content/draft-next/type.json +++ b/output-tests/draft-next/content/type.json @@ -39,4 +39,4 @@ } ] } -] \ No newline at end of file +] diff --git a/output-tests/content/draft2019-09/general.json b/output-tests/draft2019-09/content/general.json similarity index 99% rename from output-tests/content/draft2019-09/general.json rename to output-tests/draft2019-09/content/general.json index 3c764f01..4844a57c 100644 --- a/output-tests/content/draft2019-09/general.json +++ b/output-tests/draft2019-09/content/general.json @@ -31,4 +31,4 @@ } ] } -] \ No newline at end of file +] diff --git a/output-tests/content/draft2019-09/readOnly.json b/output-tests/draft2019-09/content/readOnly.json similarity index 99% rename from output-tests/content/draft2019-09/readOnly.json rename to output-tests/draft2019-09/content/readOnly.json index 7a054235..a6617607 100644 --- a/output-tests/content/draft2019-09/readOnly.json +++ b/output-tests/draft2019-09/content/readOnly.json @@ -35,4 +35,4 @@ } ] } -] \ No newline at end of file +] diff --git a/output-tests/content/draft2019-09/type.json b/output-tests/draft2019-09/content/type.json similarity index 99% rename from output-tests/content/draft2019-09/type.json rename to output-tests/draft2019-09/content/type.json index b1d2fe92..33f9fb65 100644 --- a/output-tests/content/draft2019-09/type.json +++ b/output-tests/draft2019-09/content/type.json @@ -60,4 +60,4 @@ } ] } -] \ No newline at end of file +] From fe25ba95419cbe2ab0140678650f497a170f9e2d Mon Sep 17 00:00:00 2001 From: Greg Dennis Date: Thu, 24 Nov 2022 09:17:51 +1300 Subject: [PATCH 03/20] updated readme; fixed $schema typo; tests for 2020-12 --- output-tests/README.md | 8 ++- .../draft2019-09/content/general.json | 2 +- .../draft2020-12/content/general.json | 34 ++++++++++ .../draft2020-12/content/readOnly.json | 38 +++++++++++ output-tests/draft2020-12/content/type.json | 63 +++++++++++++++++++ 5 files changed, 141 insertions(+), 4 deletions(-) create mode 100644 output-tests/draft2020-12/content/general.json create mode 100644 output-tests/draft2020-12/content/readOnly.json create mode 100644 output-tests/draft2020-12/content/type.json diff --git a/output-tests/README.md b/output-tests/README.md index 11c3a2b6..a59160f0 100644 --- a/output-tests/README.md +++ b/output-tests/README.md @@ -1,12 +1,14 @@ These tests are intended to validate that implementations are correctly generating output in accordance with the specification. -Output was initially specified with draft 2019-09. It remained unchanged for draft 2020-12 , but will receive an update with the next release. +Output was initially specified with draft 2019-09. It remained unchanged for draft 2020-12, but will receive an update with the next release. + +***NOTE** Although the formats didn't change between 2019-09 and 2020-12, the tests are replicated for 2020-12 because the `$schema` is different and implementations may (but shouldn't) produce different output.* ## Organization -The tests are organized into two categories: content and structure. +The tests are organized by specification release and then into two categories: content and structure. -Content tests verify that the keywords are producing the correct annotations. Since there are no requirements on the content of error messages, there's not much that can be verified for them, if they're even generated. These tests need to extensively cover the annotation behaviors of each keyword. The expected output format for these tests is `basic`. +Content tests verify that the keywords are producing the correct annotations and/or error messages. Since there are no requirements on the content of error messages, there's not much that can be verified for them, but it is possible to identify when a error message _could_ be present. Primarily, these tests need to extensively cover the annotation behaviors of each keyword. The only output format needed for these tests is `basic`. Structure tests verify that the structures of the various formats (i.e. `flag`, `basic`, `detailed`, `verbose`) are correct. These tests don't need to cover each keyword; rather they need to sufficiently cover the various aspects of building the output structures by using whatever keywords are necessary to do so. diff --git a/output-tests/draft2019-09/content/general.json b/output-tests/draft2019-09/content/general.json index 4844a57c..9eedbbfe 100644 --- a/output-tests/draft2019-09/content/general.json +++ b/output-tests/draft2019-09/content/general.json @@ -2,7 +2,7 @@ { "description": "failed validation produces no annotations", "schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", + "$schema": "https://json-schema.org/draft/2019-09/schema", "$id": "https://json-schema.org/tests/content/draft2019-09/general/1", "type": "string", "readOnly": true diff --git a/output-tests/draft2020-12/content/general.json b/output-tests/draft2020-12/content/general.json new file mode 100644 index 00000000..6d9d4e49 --- /dev/null +++ b/output-tests/draft2020-12/content/general.json @@ -0,0 +1,34 @@ +[ + { + "description": "failed validation produces no annotations", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://json-schema.org/tests/content/draft2020-12/general/1", + "type": "string", + "readOnly": true + }, + "tests": [ + { + "description": "readOnly annotation is dropped", + "data": 1, + "output": { + "basic": { + "type": "object", + "properties": { + "errors": { + "type": "array", + "items": { + "properties": { + "annotation": false + } + } + }, + "annotations": false + }, + "required": ["errors"] + } + } + } + ] + } +] diff --git a/output-tests/draft2020-12/content/readOnly.json b/output-tests/draft2020-12/content/readOnly.json new file mode 100644 index 00000000..cfb07a2a --- /dev/null +++ b/output-tests/draft2020-12/content/readOnly.json @@ -0,0 +1,38 @@ +[ + { + "description": "readOnly generates its value as an annotation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://json-schema.org/tests/content/draft2020-12/readOnly/1", + "readOnly": true + }, + "tests": [ + { + "description": "readOnly is true", + "data": 1, + "output": { + "basic": { + "type": "object", + "properties": { + "annotations": { + "type": "array", + "contains": { + "type": "object", + "properties": { + "keywordLocation": {"const": "/readOnly"}, + "absoluteKeywordLocation": {"const": "https://json-schema.org/tests/content/draft2020-12/readOnly/1#/readOnly"}, + "instanceLocation": {"const": ""}, + "annotation": {"const": true} + }, + "required": ["keywordLocation", "absoluteKeywordLocation", "instanceLocation", "annotation"] + } + }, + "errors": false + }, + "required": ["annotations"] + } + } + } + ] + } +] diff --git a/output-tests/draft2020-12/content/type.json b/output-tests/draft2020-12/content/type.json new file mode 100644 index 00000000..02daebb6 --- /dev/null +++ b/output-tests/draft2020-12/content/type.json @@ -0,0 +1,63 @@ +[ + { + "description": "validating type", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://json-schema.org/tests/content/draft2020-12/type/1", + "type": "string" + }, + "tests": [ + { + "description": "incorrect type must be reported, but a message is not required", + "data": 1, + "output": { + "basic": { + "type": "object", + "properties": { + "errors": { + "contains": { + "type": "object", + "properties": { + "keywordLocation": {"const": "/type"}, + "absoluteKeywordLocation": {"const": "https://json-schema.org/tests/content/draft2020-12/type/1#/type"}, + "instanceLocation": {"const": ""}, + "annotation": false, + "error": {"type": "string"} + }, + "required": ["keywordLocation", "absoluteKeywordLocation", "instanceLocation"] + } + } + }, + "required": ["errors"] + } + } + }, + { + "description": "correct type yields an output unit", + "data": 1, + "output": { + "basic": { + "type": "object", + "properties": { + "annotations": { + "contains": { + "type": "object", + "properties": { + "valid": {"const": true}, + "keywordLocation": {"const": "/type"}, + "absoluteKeywordLocation": {"const": "https://json-schema.org/tests/content/draft2020-12/type/1#/type"}, + "instanceLocation": {"const": ""}, + "annotation": false, + "error": false + }, + "required": ["keywordLocation", "absoluteKeywordLocation", "instanceLocation"] + } + } + }, + "required": ["annotations"] + } + } + } + ] + } +] From 147df4867186dfdb3762a823de01af05d6e90339 Mon Sep 17 00:00:00 2001 From: Greg Dennis Date: Thu, 24 Nov 2022 09:27:44 +1300 Subject: [PATCH 04/20] add output schemas --- output-tests/draft-next/output-schema.json | 95 +++++++++++++++++++ output-tests/draft2019-09/output-schema.json | 96 ++++++++++++++++++++ output-tests/draft2020-12/output-schema.json | 96 ++++++++++++++++++++ 3 files changed, 287 insertions(+) create mode 100644 output-tests/draft-next/output-schema.json create mode 100644 output-tests/draft2019-09/output-schema.json create mode 100644 output-tests/draft2020-12/output-schema.json diff --git a/output-tests/draft-next/output-schema.json b/output-tests/draft-next/output-schema.json new file mode 100644 index 00000000..ea24e42d --- /dev/null +++ b/output-tests/draft-next/output-schema.json @@ -0,0 +1,95 @@ +{ + "$schema": "https://json-schema.org/draft/next/schema", + "$id": "https://json-schema.org/draft/next/output/schema", + "description": "A schema that validates the minimum requirements for validation output", + + "anyOf": [ + { "$ref": "#/$defs/flag" }, + { "$ref": "#/$defs/basic" }, + { "$ref": "#/$defs/hierarchical" } + ], + "$defs": { + "outputUnit":{ + "properties": { + "valid": { "type": "boolean" }, + "evaluationPath": { + "type": "string", + "format": "json-pointer" + }, + "schemaLocation": { + "type": "string", + "format": "uri" + }, + "instanceLocation": { + "type": "string", + "format": "json-pointer" + }, + "details": { + "$ref": "#/$defs/outputUnitArray" + }, + "annotations": { + "type": "object", + "additionalProperties": true + }, + "droppedAnnotations": { + "type": "object", + "additionalProperties": true + }, + "errors": { + "type": "object", + "additionalProperties": { "type": "string" } + } + }, + "required": [ "valid", "evaluationPath", "schemaLocation", "instanceLocation" ], + "allOf": [ + { + "if": { + "anyOf": [ + { + "required": [ "errors" ] + }, + { + "required": [ "droppedAnnotations" ] + } + ] + }, + "then": { + "properties": { + "valid": { "const": false } + } + } + }, + { + "if": { + "required": [ "annotations" ] + }, + "then": { + "properties": { + "valid": { "const": true } + } + } + } + ] + }, + "outputUnitArray": { + "type": "array", + "items": { "$ref": "#/$defs/outputUnit" } + }, + "flag": { + "properties": { + "valid": { "type": "boolean" } + }, + "required": [ "valid" ] + }, + "basic": { + "properties": { + "valid": { "type": "boolean" }, + "details": { + "$ref": "#/$defs/outputUnitArray" + } + }, + "required": [ "valid", "details" ] + }, + "hierarchical": { "$ref": "#/$defs/outputUnit" } + } +} \ No newline at end of file diff --git a/output-tests/draft2019-09/output-schema.json b/output-tests/draft2019-09/output-schema.json new file mode 100644 index 00000000..f705fd54 --- /dev/null +++ b/output-tests/draft2019-09/output-schema.json @@ -0,0 +1,96 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09/schema", + "$id": "https://json-schema.org/draft/2019-09/output/schema", + "description": "A schema that validates the minimum requirements for validation output", + + "anyOf": [ + { "$ref": "#/$defs/flag" }, + { "$ref": "#/$defs/basic" }, + { "$ref": "#/$defs/detailed" }, + { "$ref": "#/$defs/verbose" } + ], + "$defs": { + "outputUnit":{ + "properties": { + "valid": { "type": "boolean" }, + "keywordLocation": { + "type": "string", + "format": "json-pointer" + }, + "absoluteKeywordLocation": { + "type": "string", + "format": "uri" + }, + "instanceLocation": { + "type": "string", + "format": "json-pointer" + }, + "error": { + "type": "string" + }, + "errors": { + "$ref": "#/$defs/outputUnitArray" + }, + "annotations": { + "$ref": "#/$defs/outputUnitArray" + } + }, + "required": [ "valid", "keywordLocation", "instanceLocation" ], + "allOf": [ + { + "if": { + "properties": { + "valid": { "const": false } + } + }, + "then": { + "anyOf": [ + { + "required": [ "error" ] + }, + { + "required": [ "errors" ] + } + ] + } + }, + { + "if": { + "anyOf": [ + { + "properties": { + "keywordLocation": { + "pattern": "/\\$ref/" + } + } + }, + { + "properties": { + "keywordLocation": { + "pattern": "/\\$dynamicRef/" + } + } + } + ] + }, + "then": { + "required": [ "absoluteKeywordLocation" ] + } + } + ] + }, + "outputUnitArray": { + "type": "array", + "items": { "$ref": "#/$defs/outputUnit" } + }, + "flag": { + "properties": { + "valid": { "type": "boolean" } + }, + "required": [ "valid" ] + }, + "basic": { "$ref": "#/$defs/outputUnit" }, + "detailed": { "$ref": "#/$defs/outputUnit" }, + "verbose": { "$ref": "#/$defs/outputUnit" } + } +} \ No newline at end of file diff --git a/output-tests/draft2020-12/output-schema.json b/output-tests/draft2020-12/output-schema.json new file mode 100644 index 00000000..f63385fb --- /dev/null +++ b/output-tests/draft2020-12/output-schema.json @@ -0,0 +1,96 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://json-schema.org/draft/2020-12/output/schema", + "description": "A schema that validates the minimum requirements for validation output", + + "anyOf": [ + { "$ref": "#/$defs/flag" }, + { "$ref": "#/$defs/basic" }, + { "$ref": "#/$defs/detailed" }, + { "$ref": "#/$defs/verbose" } + ], + "$defs": { + "outputUnit":{ + "properties": { + "valid": { "type": "boolean" }, + "keywordLocation": { + "type": "string", + "format": "json-pointer" + }, + "absoluteKeywordLocation": { + "type": "string", + "format": "uri" + }, + "instanceLocation": { + "type": "string", + "format": "json-pointer" + }, + "error": { + "type": "string" + }, + "errors": { + "$ref": "#/$defs/outputUnitArray" + }, + "annotations": { + "$ref": "#/$defs/outputUnitArray" + } + }, + "required": [ "valid", "keywordLocation", "instanceLocation" ], + "allOf": [ + { + "if": { + "properties": { + "valid": { "const": false } + } + }, + "then": { + "anyOf": [ + { + "required": [ "error" ] + }, + { + "required": [ "errors" ] + } + ] + } + }, + { + "if": { + "anyOf": [ + { + "properties": { + "keywordLocation": { + "pattern": "/\\$ref/" + } + } + }, + { + "properties": { + "keywordLocation": { + "pattern": "/\\$dynamicRef/" + } + } + } + ] + }, + "then": { + "required": [ "absoluteKeywordLocation" ] + } + } + ] + }, + "outputUnitArray": { + "type": "array", + "items": { "$ref": "#/$defs/outputUnit" } + }, + "flag": { + "properties": { + "valid": { "type": "boolean" } + }, + "required": [ "valid" ] + }, + "basic": { "$ref": "#/$defs/outputUnit" }, + "detailed": { "$ref": "#/$defs/outputUnit" }, + "verbose": { "$ref": "#/$defs/outputUnit" } + } +} \ No newline at end of file From efeee1d4edaaf3b53e0b192fc83c2b51ddf80ca8 Mon Sep 17 00:00:00 2001 From: Greg Dennis Date: Thu, 24 Nov 2022 09:35:07 +1300 Subject: [PATCH 05/20] update tests to reference output schemas --- output-tests/draft-next/content/general.json | 3 +-- output-tests/draft-next/content/readOnly.json | 3 +-- output-tests/draft-next/content/type.json | 8 ++------ output-tests/draft2019-09/content/general.json | 3 +-- output-tests/draft2019-09/content/readOnly.json | 3 +-- output-tests/draft2019-09/content/type.json | 9 +++------ output-tests/draft2020-12/content/general.json | 3 +-- output-tests/draft2020-12/content/readOnly.json | 4 +--- output-tests/draft2020-12/content/type.json | 9 +++------ 9 files changed, 14 insertions(+), 31 deletions(-) diff --git a/output-tests/draft-next/content/general.json b/output-tests/draft-next/content/general.json index 6001dec2..7c3cfe07 100644 --- a/output-tests/draft-next/content/general.json +++ b/output-tests/draft-next/content/general.json @@ -13,11 +13,10 @@ "data": 1, "output": { "basic": { - "type": "object", + "$ref": "/draft/next/output/schema", "properties": { "details": { "contains": { - "type": "object", "properties": { "evaluationPath": {"const": ""}, "schemaLocation": {"const": "https://json-schema.org/tests/content/draft-next/general/1"}, diff --git a/output-tests/draft-next/content/readOnly.json b/output-tests/draft-next/content/readOnly.json index 5b4b7818..de51e935 100644 --- a/output-tests/draft-next/content/readOnly.json +++ b/output-tests/draft-next/content/readOnly.json @@ -12,11 +12,10 @@ "data": 1, "output": { "basic": { - "type": "object", + "$ref": "/draft/next/output/schema", "properties": { "details": { "contains": { - "type": "object", "properties": { "evaluationPath": {"const": ""}, "schemaLocation": {"const": "https://json-schema.org/tests/content/draft-next/readOnly/1"}, diff --git a/output-tests/draft-next/content/type.json b/output-tests/draft-next/content/type.json index de8d5c17..2d6c2529 100644 --- a/output-tests/draft-next/content/type.json +++ b/output-tests/draft-next/content/type.json @@ -8,24 +8,20 @@ }, "tests": [ { - "description": "incorrect type may be reported", + "description": "incorrect type must be reported, but a message is not required", "data": 1, "output": { "basic": { - "type": "object", + "$ref": "/draft/next/output/schema", "properties": { "details": { "contains": { - "type": "object", "properties": { "evaluationPath": {"const": ""}, "schemaLocation": {"const": "https://json-schema.org/tests/content/draft-next/type/1"}, "instanceLocation": {"const": ""}, "annotations": false, "errors": { - "properties": { - "type": {"type": "string"} - }, "required": ["type"] } }, diff --git a/output-tests/draft2019-09/content/general.json b/output-tests/draft2019-09/content/general.json index 9eedbbfe..96b7df9a 100644 --- a/output-tests/draft2019-09/content/general.json +++ b/output-tests/draft2019-09/content/general.json @@ -13,10 +13,9 @@ "data": 1, "output": { "basic": { - "type": "object", + "$ref": "/draft/2019-09/output/schema", "properties": { "errors": { - "type": "array", "items": { "properties": { "annotation": false diff --git a/output-tests/draft2019-09/content/readOnly.json b/output-tests/draft2019-09/content/readOnly.json index a6617607..c284d3d3 100644 --- a/output-tests/draft2019-09/content/readOnly.json +++ b/output-tests/draft2019-09/content/readOnly.json @@ -12,10 +12,9 @@ "data": 1, "output": { "basic": { - "type": "object", + "$ref": "/draft/2019-09/output/schema", "properties": { "annotations": { - "type": "array", "contains": { "type": "object", "properties": { diff --git a/output-tests/draft2019-09/content/type.json b/output-tests/draft2019-09/content/type.json index 33f9fb65..685b4c7f 100644 --- a/output-tests/draft2019-09/content/type.json +++ b/output-tests/draft2019-09/content/type.json @@ -12,17 +12,15 @@ "data": 1, "output": { "basic": { - "type": "object", + "$ref": "/draft/2019-09/output/schema", "properties": { "errors": { "contains": { - "type": "object", "properties": { "keywordLocation": {"const": "/type"}, "absoluteKeywordLocation": {"const": "https://json-schema.org/tests/content/draft2019-09/type/1#/type"}, "instanceLocation": {"const": ""}, - "annotation": false, - "error": {"type": "string"} + "annotation": false }, "required": ["keywordLocation", "absoluteKeywordLocation", "instanceLocation"] } @@ -37,11 +35,10 @@ "data": 1, "output": { "basic": { - "type": "object", + "$ref": "/draft/2019-09/output/schema", "properties": { "annotations": { "contains": { - "type": "object", "properties": { "valid": {"const": true}, "keywordLocation": {"const": "/type"}, diff --git a/output-tests/draft2020-12/content/general.json b/output-tests/draft2020-12/content/general.json index 6d9d4e49..8662da30 100644 --- a/output-tests/draft2020-12/content/general.json +++ b/output-tests/draft2020-12/content/general.json @@ -13,10 +13,9 @@ "data": 1, "output": { "basic": { - "type": "object", + "$ref": "/draft/2020-12/output/schema", "properties": { "errors": { - "type": "array", "items": { "properties": { "annotation": false diff --git a/output-tests/draft2020-12/content/readOnly.json b/output-tests/draft2020-12/content/readOnly.json index cfb07a2a..e4b9c3fd 100644 --- a/output-tests/draft2020-12/content/readOnly.json +++ b/output-tests/draft2020-12/content/readOnly.json @@ -12,12 +12,10 @@ "data": 1, "output": { "basic": { - "type": "object", + "$ref": "/draft/2020-12/output/schema", "properties": { "annotations": { - "type": "array", "contains": { - "type": "object", "properties": { "keywordLocation": {"const": "/readOnly"}, "absoluteKeywordLocation": {"const": "https://json-schema.org/tests/content/draft2020-12/readOnly/1#/readOnly"}, diff --git a/output-tests/draft2020-12/content/type.json b/output-tests/draft2020-12/content/type.json index 02daebb6..081a21da 100644 --- a/output-tests/draft2020-12/content/type.json +++ b/output-tests/draft2020-12/content/type.json @@ -12,17 +12,15 @@ "data": 1, "output": { "basic": { - "type": "object", + "$ref": "/draft/2020-12/output/schema", "properties": { "errors": { "contains": { - "type": "object", "properties": { "keywordLocation": {"const": "/type"}, "absoluteKeywordLocation": {"const": "https://json-schema.org/tests/content/draft2020-12/type/1#/type"}, "instanceLocation": {"const": ""}, - "annotation": false, - "error": {"type": "string"} + "annotation": false }, "required": ["keywordLocation", "absoluteKeywordLocation", "instanceLocation"] } @@ -37,11 +35,10 @@ "data": 1, "output": { "basic": { - "type": "object", + "$ref": "/draft/2020-12/output/schema", "properties": { "annotations": { "contains": { - "type": "object", "properties": { "valid": {"const": true}, "keywordLocation": {"const": "/type"}, From 46bc7ace4f986cd9243eab1cda3d46d701f1fdfa Mon Sep 17 00:00:00 2001 From: Greg Dennis Date: Thu, 24 Nov 2022 09:55:49 +1300 Subject: [PATCH 06/20] add readme note about output-schema.json files --- output-tests/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/output-tests/README.md b/output-tests/README.md index a59160f0..accfffb7 100644 --- a/output-tests/README.md +++ b/output-tests/README.md @@ -12,6 +12,8 @@ Content tests verify that the keywords are producing the correct annotations and Structure tests verify that the structures of the various formats (i.e. `flag`, `basic`, `detailed`, `verbose`) are correct. These tests don't need to cover each keyword; rather they need to sufficiently cover the various aspects of building the output structures by using whatever keywords are necessary to do so. +In each release folder, you'll also find an _output-schema.json_ file that contains the schema from the specification repo that describes output for that release. This schema will need to be loaded as the tests reference it. + ## Test Files The content of a test file is the same as the validation tests in `tests/`, however an `output` property has been added to each test case. From daf42a748e5771900accdb207b865ef2408344c1 Mon Sep 17 00:00:00 2001 From: Greg Dennis Date: Fri, 25 Nov 2022 10:21:47 +1300 Subject: [PATCH 07/20] attempt to update ci for output tests --- bin/jsonschema_suite | 41 +++++++++++++++++++++++++++++++++++------ output-test-schema.json | 36 ++++++++++++++++++++++++++++++++++++ test-schema.json | 10 +++++++--- 3 files changed, 78 insertions(+), 9 deletions(-) create mode 100644 output-test-schema.json diff --git a/bin/jsonschema_suite b/bin/jsonschema_suite index 8cc28508..9f7ba657 100755 --- a/bin/jsonschema_suite +++ b/bin/jsonschema_suite @@ -30,11 +30,13 @@ else: ROOT_DIR = Path(__file__).parent.parent SUITE_ROOT_DIR = ROOT_DIR / "tests" +OUTPUT_ROOT_DIR = ROOT_DIR / "output-tests" REMOTES_DIR = ROOT_DIR / "remotes" REMOTES_BASE_URL = "http://localhost:1234/" TESTSUITE_SCHEMA = json.loads((ROOT_DIR / "test-schema.json").read_text()) +OUTPUTTESTSUITE_SCHEMA = json.loads((ROOT_DIR / "output-test-schema.json").read_text()) def files(paths): @@ -88,12 +90,17 @@ class SanityTests(unittest.TestCase): @classmethod def setUpClass(cls): print(f"Looking for tests in {SUITE_ROOT_DIR}") + print(f"Looking for output tests in {OUTPUT_ROOT_DIR}") print(f"Looking for remotes in {REMOTES_DIR}") cls.test_files = list(collect(SUITE_ROOT_DIR)) assert cls.test_files, "Didn't find the test files!" print(f"Found {len(cls.test_files)} test files") + cls.output_test_files = list(collect(OUTPUT_ROOT_DIR)) + assert cls.remote_files, "Didn't find the output test files!" + print(f"Found {len(cls.remote_files)} output test files") + cls.remote_files = list(collect(REMOTES_DIR)) assert cls.remote_files, "Didn't find the remote files!" print(f"Found {len(cls.remote_files)} remote files") @@ -142,6 +149,17 @@ class SanityTests(unittest.TestCase): except ValueError as error: self.fail(f"{path} contains invalid JSON ({error})") + def test_all_output_test_files_are_valid_json(self): + """ + All test files contain valid JSON. + """ + for path in self.output_test_files: + with self.subTest(path=path): + try: + json.loads(path.read_text()) + except ValueError as error: + self.fail(f"{path} contains invalid JSON ({error})") + def test_all_remote_files_are_valid_json(self): """ All remote files contain valid JSON. @@ -157,7 +175,7 @@ class SanityTests(unittest.TestCase): """ All cases have reasonably long descriptions. """ - for case in cases(self.test_files): + for case in cases(self.test_files + self.output_test_files): with self.subTest(description=case["description"]): self.assertLess( len(case["description"]), @@ -169,7 +187,7 @@ class SanityTests(unittest.TestCase): """ All tests have reasonably long descriptions. """ - for count, test in enumerate(tests(self.test_files)): + for count, test in enumerate(tests(self.test_files + self.output_test_files)): with self.subTest(description=test["description"]): self.assertLess( len(test["description"]), @@ -182,7 +200,7 @@ class SanityTests(unittest.TestCase): """ All cases have unique descriptions in their files. """ - for path, cases in files(self.test_files): + for path, cases in files(self.test_files + self.output_test_files): with self.subTest(path=path): self.assertUnique(case["description"] for case in cases) @@ -190,7 +208,7 @@ class SanityTests(unittest.TestCase): """ All test cases have unique test descriptions in their tests. """ - for count, case in enumerate(cases(self.test_files)): + for count, case in enumerate(cases(self.test_files + self.output_test_files)): with self.subTest(description=case["description"]): self.assertUnique( test["description"] for test in case["tests"] @@ -198,12 +216,12 @@ class SanityTests(unittest.TestCase): print(f"Found {count} test cases.") def test_case_descriptions_do_not_use_modal_verbs(self): - for case in cases(self.test_files): + for case in cases(self.test_files + self.output_test_files): with self.subTest(description=case["description"]): self.assertFollowsDescriptionStyle(case["description"]) def test_test_descriptions_do_not_use_modal_verbs(self): - for test in tests(self.test_files): + for test in tests(self.test_files + self.output_test_files): with self.subTest(description=test["description"]): self.assertFollowsDescriptionStyle(test["description"]) @@ -244,6 +262,17 @@ class SanityTests(unittest.TestCase): validator.validate(cases) except jsonschema.ValidationError as error: self.fail(str(error)) + """ + All output test files are valid under output-test-schema.json. + """ + Validator = jsonschema.validators.validator_for(OUTPUTTESTSUITE_SCHEMA) + validator = Validator(OUTPUTTESTSUITE_SCHEMA) + for path, cases in files(self.output_test_files): + with self.subTest(path=path): + try: + validator.validate(cases) + except jsonschema.ValidationError as error: + self.fail(str(error)) def main(arguments): diff --git a/output-test-schema.json b/output-test-schema.json new file mode 100644 index 00000000..cda77b1b --- /dev/null +++ b/output-test-schema.json @@ -0,0 +1,36 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://json-schema.org/tests/output-test-schema", + "description": "A schema for files contained within this suite", + + "$ref": "test-schema", + + "$defs": { + "test": { + "$dynamicAnchor": "foo", + "description": "A single output test", + + "allOf": [ + { "$ref": "test-schema#/$defs/test" }, + { + "properties": { + "output": { + "description": "schemas that are used to verify output", + "type": "object", + "properties": { + "flag": { "$ref": "https://json-schema.org/draft/2020-12/schema" }, + "basic": { "$ref": "https://json-schema.org/draft/2020-12/schema" }, + "detailed": { "$ref": "https://json-schema.org/draft/2020-12/schema" }, + "verbose": { "$ref": "https://json-schema.org/draft/2020-12/schema" }, + "list": { "$ref": "https://json-schema.org/draft/2020-12/schema" }, + "hierarchy": { "$ref": "https://json-schema.org/draft/2020-12/schema" } + }, + "minProperties": 1, + "additionalProperties": false + } + } + } + ] + } + } +} diff --git a/test-schema.json b/test-schema.json index 5d250317..c8e90ef1 100644 --- a/test-schema.json +++ b/test-schema.json @@ -1,5 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://json-schema.org/tests/test-schema", "description": "A schema for files contained within this suite", "type": "array", @@ -24,7 +25,10 @@ "tests": { "description": "A set of related tests all using the same schema", "type": "array", - "items": { "$ref": "#/$defs/test" }, + "items": { + "$dynamicRef": "#foo", + "unevaluatedProperties": false + }, "minItems": 1 } }, @@ -33,6 +37,7 @@ "$defs": { "test": { + "$dynamicAnchor": "foo", "description": "A single test", "type": "object", @@ -53,8 +58,7 @@ "description": "Whether the validation process of this instance should consider the instance valid or not", "type": "boolean" } - }, - "additionalProperties": false + } } } } From d263bf01424addb20b223e0cda815275ceeeb0f1 Mon Sep 17 00:00:00 2001 From: Greg Dennis Date: Tue, 29 Nov 2022 09:28:44 +1300 Subject: [PATCH 08/20] add blank lines at the end of all the files --- .editorconfig | 2 +- output-tests/draft-next/output-schema.json | 2 +- output-tests/draft2019-09/output-schema.json | 2 +- output-tests/draft2020-12/output-schema.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.editorconfig b/.editorconfig index 59e6583c..6db6a5bf 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1 +1 @@ -insert_final_newline = true \ No newline at end of file +insert_final_newline = true diff --git a/output-tests/draft-next/output-schema.json b/output-tests/draft-next/output-schema.json index ea24e42d..26286fa4 100644 --- a/output-tests/draft-next/output-schema.json +++ b/output-tests/draft-next/output-schema.json @@ -92,4 +92,4 @@ }, "hierarchical": { "$ref": "#/$defs/outputUnit" } } -} \ No newline at end of file +} diff --git a/output-tests/draft2019-09/output-schema.json b/output-tests/draft2019-09/output-schema.json index f705fd54..0a65f20f 100644 --- a/output-tests/draft2019-09/output-schema.json +++ b/output-tests/draft2019-09/output-schema.json @@ -93,4 +93,4 @@ "detailed": { "$ref": "#/$defs/outputUnit" }, "verbose": { "$ref": "#/$defs/outputUnit" } } -} \ No newline at end of file +} diff --git a/output-tests/draft2020-12/output-schema.json b/output-tests/draft2020-12/output-schema.json index f63385fb..1eef288a 100644 --- a/output-tests/draft2020-12/output-schema.json +++ b/output-tests/draft2020-12/output-schema.json @@ -93,4 +93,4 @@ "detailed": { "$ref": "#/$defs/outputUnit" }, "verbose": { "$ref": "#/$defs/outputUnit" } } -} \ No newline at end of file +} From 691d039c132c6f067f117910ae4d312a265a2e1a Mon Sep 17 00:00:00 2001 From: Greg Dennis Date: Tue, 29 Nov 2022 09:31:17 +1300 Subject: [PATCH 09/20] update $dynamic* value in schemas --- output-test-schema.json | 2 +- test-schema.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/output-test-schema.json b/output-test-schema.json index cda77b1b..232d3c08 100644 --- a/output-test-schema.json +++ b/output-test-schema.json @@ -7,7 +7,7 @@ "$defs": { "test": { - "$dynamicAnchor": "foo", + "$dynamicAnchor": "test", "description": "A single output test", "allOf": [ diff --git a/test-schema.json b/test-schema.json index c8e90ef1..926d895b 100644 --- a/test-schema.json +++ b/test-schema.json @@ -26,7 +26,7 @@ "description": "A set of related tests all using the same schema", "type": "array", "items": { - "$dynamicRef": "#foo", + "$dynamicRef": "#test", "unevaluatedProperties": false }, "minItems": 1 @@ -37,7 +37,7 @@ "$defs": { "test": { - "$dynamicAnchor": "foo", + "$dynamicAnchor": "test", "description": "A single test", "type": "object", From b40a6cacbe087a8e63f5449e93dbe6887b1e5082 Mon Sep 17 00:00:00 2001 From: Greg Dennis Date: Tue, 29 Nov 2022 10:14:11 +1300 Subject: [PATCH 10/20] Update bin/jsonschema_suite Co-authored-by: Julian Berman --- bin/jsonschema_suite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/jsonschema_suite b/bin/jsonschema_suite index 9f7ba657..234682ac 100755 --- a/bin/jsonschema_suite +++ b/bin/jsonschema_suite @@ -98,7 +98,7 @@ class SanityTests(unittest.TestCase): print(f"Found {len(cls.test_files)} test files") cls.output_test_files = list(collect(OUTPUT_ROOT_DIR)) - assert cls.remote_files, "Didn't find the output test files!" + assert cls.output_test_files, "Didn't find the output test files!" print(f"Found {len(cls.remote_files)} output test files") cls.remote_files = list(collect(REMOTES_DIR)) From b1267590ce13ba6736e91d5f92e740a7ca83ad7b Mon Sep 17 00:00:00 2001 From: Greg Dennis Date: Tue, 29 Nov 2022 10:14:21 +1300 Subject: [PATCH 11/20] Update bin/jsonschema_suite Co-authored-by: Julian Berman --- bin/jsonschema_suite | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/jsonschema_suite b/bin/jsonschema_suite index 234682ac..33416470 100755 --- a/bin/jsonschema_suite +++ b/bin/jsonschema_suite @@ -99,7 +99,7 @@ class SanityTests(unittest.TestCase): cls.output_test_files = list(collect(OUTPUT_ROOT_DIR)) assert cls.output_test_files, "Didn't find the output test files!" - print(f"Found {len(cls.remote_files)} output test files") + print(f"Found {len(cls.output_test_files)} output test files") cls.remote_files = list(collect(REMOTES_DIR)) assert cls.remote_files, "Didn't find the remote files!" From 3413863a714137f8b5161a803b4837a4f8305b5e Mon Sep 17 00:00:00 2001 From: Julian Berman Date: Tue, 29 Nov 2022 11:18:08 -0500 Subject: [PATCH 12/20] Inline the relevant parts of the test schema to output tests. Output tests aren't actually overlapping with the normal test schema, they don't contain the 'valid' property (which is required in normal tests). --- output-test-schema.json | 76 +++++++++++++++++++++++++++++------------ test-schema.json | 9 ++--- 2 files changed, 58 insertions(+), 27 deletions(-) diff --git a/output-test-schema.json b/output-test-schema.json index 232d3c08..02c51ef5 100644 --- a/output-test-schema.json +++ b/output-test-schema.json @@ -3,34 +3,68 @@ "$id": "https://json-schema.org/tests/output-test-schema", "description": "A schema for files contained within this suite", - "$ref": "test-schema", + "type": "array", + "minItems": 1, + "items": { + "description": "An individual test case, containing multiple tests of a single schema's behavior", + + "type": "object", + "required": [ "description", "schema", "tests" ], + "properties": { + "description": { + "description": "The test case description", + "type": "string" + }, + "comment": { + "description": "Any additional comments about the test case", + "type": "string" + }, + "schema": { + "description": "A valid JSON Schema (one written for the corresponding version directory that the file sits within)." + }, + "tests": { + "description": "A set of related tests all using the same schema", + "type": "array", + "items": { "$ref": "#/$defs/test" }, + "minItems": 1 + } + }, + "additionalProperties": false + }, "$defs": { "test": { - "$dynamicAnchor": "test", "description": "A single output test", - "allOf": [ - { "$ref": "test-schema#/$defs/test" }, - { + "type": "object", + "required": [ "description", "data", "output" ], + "properties": { + "description": { + "description": "The test description, briefly explaining which behavior it exercises", + "type": "string" + }, + "comment": { + "description": "Any additional comments about the test", + "type": "string" + }, + "data": { + "description": "The instance which should be validated against the schema in \"schema\"." + }, + "output": { + "description": "schemas that are used to verify output", + "type": "object", "properties": { - "output": { - "description": "schemas that are used to verify output", - "type": "object", - "properties": { - "flag": { "$ref": "https://json-schema.org/draft/2020-12/schema" }, - "basic": { "$ref": "https://json-schema.org/draft/2020-12/schema" }, - "detailed": { "$ref": "https://json-schema.org/draft/2020-12/schema" }, - "verbose": { "$ref": "https://json-schema.org/draft/2020-12/schema" }, - "list": { "$ref": "https://json-schema.org/draft/2020-12/schema" }, - "hierarchy": { "$ref": "https://json-schema.org/draft/2020-12/schema" } - }, - "minProperties": 1, - "additionalProperties": false - } - } + "flag": { "$ref": "https://json-schema.org/draft/2020-12/schema" }, + "basic": { "$ref": "https://json-schema.org/draft/2020-12/schema" }, + "detailed": { "$ref": "https://json-schema.org/draft/2020-12/schema" }, + "verbose": { "$ref": "https://json-schema.org/draft/2020-12/schema" }, + "list": { "$ref": "https://json-schema.org/draft/2020-12/schema" }, + "hierarchy": { "$ref": "https://json-schema.org/draft/2020-12/schema" } + }, + "minProperties": 1, + "additionalProperties": false } - ] + } } } } diff --git a/test-schema.json b/test-schema.json index 926d895b..83393162 100644 --- a/test-schema.json +++ b/test-schema.json @@ -25,10 +25,7 @@ "tests": { "description": "A set of related tests all using the same schema", "type": "array", - "items": { - "$dynamicRef": "#test", - "unevaluatedProperties": false - }, + "items": { "$ref": "#/$defs/test" }, "minItems": 1 } }, @@ -37,7 +34,6 @@ "$defs": { "test": { - "$dynamicAnchor": "test", "description": "A single test", "type": "object", @@ -58,7 +54,8 @@ "description": "Whether the validation process of this instance should consider the instance valid or not", "type": "boolean" } - } + }, + "additionalProperties": false } } } From a8b280571bd1135a034d0e5ef23343e5916a73f4 Mon Sep 17 00:00:00 2001 From: Julian Berman Date: Tue, 29 Nov 2022 11:19:05 -0500 Subject: [PATCH 13/20] Minor style tweaks. --- bin/jsonschema_suite | 36 ++++++++---------------------------- 1 file changed, 8 insertions(+), 28 deletions(-) diff --git a/bin/jsonschema_suite b/bin/jsonschema_suite index 33416470..dae6c9b4 100755 --- a/bin/jsonschema_suite +++ b/bin/jsonschema_suite @@ -35,8 +35,10 @@ OUTPUT_ROOT_DIR = ROOT_DIR / "output-tests" REMOTES_DIR = ROOT_DIR / "remotes" REMOTES_BASE_URL = "http://localhost:1234/" -TESTSUITE_SCHEMA = json.loads((ROOT_DIR / "test-schema.json").read_text()) -OUTPUTTESTSUITE_SCHEMA = json.loads((ROOT_DIR / "output-test-schema.json").read_text()) +TEST_SCHEMA = json.loads(ROOT_DIR.joinpath("test-schema.json").read_text()) +OUTPUT_TEST_SCHEMA = json.loads( + ROOT_DIR.joinpath("output-test-schema.json").read_text(), +) def files(paths): @@ -69,7 +71,7 @@ def collect(root_dir): """ All of the test file paths within the given root directory, recursively. """ - return root_dir.glob("**/*.json") + return root_dir.rglob("*.json") def url_for_path(path): @@ -138,33 +140,11 @@ class SanityTests(unittest.TestCase): self.assertNotRegex(description, r"\bshould\b", message) self.assertNotRegex(description, r"(?i)\btest(s)? that\b", message) - def test_all_test_files_are_valid_json(self): - """ - All test files contain valid JSON. - """ - for path in self.test_files: - with self.subTest(path=path): - try: - json.loads(path.read_text()) - except ValueError as error: - self.fail(f"{path} contains invalid JSON ({error})") - - def test_all_output_test_files_are_valid_json(self): - """ - All test files contain valid JSON. - """ - for path in self.output_test_files: - with self.subTest(path=path): - try: - json.loads(path.read_text()) - except ValueError as error: - self.fail(f"{path} contains invalid JSON ({error})") - - def test_all_remote_files_are_valid_json(self): + def test_all_json_files_are_valid(self): """ - All remote files contain valid JSON. + All files (tests, output tests, remotes, etc.) contain valid JSON. """ - for path in self.remote_files: + for path in collect(ROOT_DIR): with self.subTest(path=path): try: json.loads(path.read_text()) From f5197e0872e5113e52157062f8037cef8700e957 Mon Sep 17 00:00:00 2001 From: Julian Berman Date: Tue, 29 Nov 2022 11:19:34 -0500 Subject: [PATCH 14/20] Fix the output check to ignore output-schema.json. It doesn't contain tests, so it's indeed not valid under the output schema. --- bin/jsonschema_suite | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/bin/jsonschema_suite b/bin/jsonschema_suite index dae6c9b4..cd7e548d 100755 --- a/bin/jsonschema_suite +++ b/bin/jsonschema_suite @@ -99,7 +99,10 @@ class SanityTests(unittest.TestCase): assert cls.test_files, "Didn't find the test files!" print(f"Found {len(cls.test_files)} test files") - cls.output_test_files = list(collect(OUTPUT_ROOT_DIR)) + cls.output_test_files = [ + each for each in collect(OUTPUT_ROOT_DIR) + if each.name != "output-schema.json" + ] assert cls.output_test_files, "Didn't find the output test files!" print(f"Found {len(cls.output_test_files)} output test files") @@ -234,19 +237,22 @@ class SanityTests(unittest.TestCase): """ All test files are valid under test-schema.json. """ - Validator = jsonschema.validators.validator_for(TESTSUITE_SCHEMA) - validator = Validator(TESTSUITE_SCHEMA) + Validator = jsonschema.validators.validator_for(TEST_SCHEMA) + validator = Validator(TEST_SCHEMA) for path, cases in files(self.test_files): with self.subTest(path=path): try: validator.validate(cases) except jsonschema.ValidationError as error: self.fail(str(error)) + + @unittest.skipIf(jsonschema is None, "Validation library not present!") + def test_output_suites_are_valid(self): """ All output test files are valid under output-test-schema.json. """ - Validator = jsonschema.validators.validator_for(OUTPUTTESTSUITE_SCHEMA) - validator = Validator(OUTPUTTESTSUITE_SCHEMA) + Validator = jsonschema.validators.validator_for(OUTPUT_TEST_SCHEMA) + validator = Validator(OUTPUT_TEST_SCHEMA) for path, cases in files(self.output_test_files): with self.subTest(path=path): try: From 930e87e7710179e9a24bcb36d2d44b6ff5a28bf7 Mon Sep 17 00:00:00 2001 From: Greg Dennis Date: Wed, 30 Nov 2022 16:02:05 +1300 Subject: [PATCH 15/20] add clarification on no changes between 2019 and 2020 --- output-tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/output-tests/README.md b/output-tests/README.md index accfffb7..63a8fdad 100644 --- a/output-tests/README.md +++ b/output-tests/README.md @@ -1,6 +1,6 @@ These tests are intended to validate that implementations are correctly generating output in accordance with the specification. -Output was initially specified with draft 2019-09. It remained unchanged for draft 2020-12, but will receive an update with the next release. +Output was initially specified with draft 2019-09. It remained largely unchanged for draft 2020-12, but will receive an update with the next release. ***NOTE** Although the formats didn't change between 2019-09 and 2020-12, the tests are replicated for 2020-12 because the `$schema` is different and implementations may (but shouldn't) produce different output.* From 1a860cfb2087d040a57cf2d3b3990a49d4b31fa6 Mon Sep 17 00:00:00 2001 From: Greg Dennis Date: Fri, 2 Dec 2022 13:42:02 +1300 Subject: [PATCH 16/20] added $id to all output schemas; reindexed test cases to 0 (instead of 1) --- output-tests/draft-next/content/general.json | 7 ++++--- output-tests/draft-next/content/readOnly.json | 7 ++++--- output-tests/draft-next/content/type.json | 7 ++++--- output-tests/draft2019-09/content/general.json | 3 ++- output-tests/draft2019-09/content/readOnly.json | 5 +++-- output-tests/draft2019-09/content/type.json | 10 ++++++---- output-tests/draft2020-12/content/general.json | 3 ++- output-tests/draft2020-12/content/readOnly.json | 5 +++-- output-tests/draft2020-12/content/type.json | 10 ++++++---- 9 files changed, 34 insertions(+), 23 deletions(-) diff --git a/output-tests/draft-next/content/general.json b/output-tests/draft-next/content/general.json index 7c3cfe07..27082ed6 100644 --- a/output-tests/draft-next/content/general.json +++ b/output-tests/draft-next/content/general.json @@ -3,7 +3,7 @@ "description": "failed validation produces no annotations", "schema": { "$schema": "https://json-schema.org/draft/next/schema", - "$id": "https://json-schema.org/tests/content/draft-next/general/1", + "$id": "https://json-schema.org/tests/content/draft-next/general/0", "type": "string", "readOnly": true }, @@ -12,14 +12,15 @@ "description": "dropped annotations MAY appear in droppedAnnotations", "data": 1, "output": { - "basic": { + "list": { + "$id": "https://json-schema.org/tests/content/draft-next/general/0/tests/0/basic", "$ref": "/draft/next/output/schema", "properties": { "details": { "contains": { "properties": { "evaluationPath": {"const": ""}, - "schemaLocation": {"const": "https://json-schema.org/tests/content/draft-next/general/1"}, + "schemaLocation": {"const": "https://json-schema.org/tests/content/draft-next/general/0"}, "instanceLocation": {"const": ""}, "annotations": false, "droppedAnnotations": { diff --git a/output-tests/draft-next/content/readOnly.json b/output-tests/draft-next/content/readOnly.json index de51e935..d387d932 100644 --- a/output-tests/draft-next/content/readOnly.json +++ b/output-tests/draft-next/content/readOnly.json @@ -3,7 +3,7 @@ "description": "readOnly generates its value as an annotation", "schema": { "$schema": "https://json-schema.org/draft/next/schema", - "$id": "https://json-schema.org/tests/content/draft-next/readOnly/1", + "$id": "https://json-schema.org/tests/content/draft-next/readOnly/0", "readOnly": true }, "tests": [ @@ -11,14 +11,15 @@ "description": "readOnly is true", "data": 1, "output": { - "basic": { + "list": { + "$id": "https://json-schema.org/tests/content/draft-next/readOnly/0/tests/0/basic", "$ref": "/draft/next/output/schema", "properties": { "details": { "contains": { "properties": { "evaluationPath": {"const": ""}, - "schemaLocation": {"const": "https://json-schema.org/tests/content/draft-next/readOnly/1"}, + "schemaLocation": {"const": "https://json-schema.org/tests/content/draft-next/readOnly/0"}, "instanceLocation": {"const": ""}, "annotations": { "properties": { diff --git a/output-tests/draft-next/content/type.json b/output-tests/draft-next/content/type.json index 2d6c2529..e17f1f53 100644 --- a/output-tests/draft-next/content/type.json +++ b/output-tests/draft-next/content/type.json @@ -3,7 +3,7 @@ "description": "incorrect type", "schema": { "$schema": "https://json-schema.org/draft/next/schema", - "$id": "https://json-schema.org/tests/content/draft-next/type/1", + "$id": "https://json-schema.org/tests/content/draft-next/type/0", "type": "string" }, "tests": [ @@ -11,14 +11,15 @@ "description": "incorrect type must be reported, but a message is not required", "data": 1, "output": { - "basic": { + "list": { + "$id": "https://json-schema.org/tests/content/draft-next/type/0/tests/0/basic", "$ref": "/draft/next/output/schema", "properties": { "details": { "contains": { "properties": { "evaluationPath": {"const": ""}, - "schemaLocation": {"const": "https://json-schema.org/tests/content/draft-next/type/1"}, + "schemaLocation": {"const": "https://json-schema.org/tests/content/draft-next/type/0"}, "instanceLocation": {"const": ""}, "annotations": false, "errors": { diff --git a/output-tests/draft2019-09/content/general.json b/output-tests/draft2019-09/content/general.json index 96b7df9a..91941700 100644 --- a/output-tests/draft2019-09/content/general.json +++ b/output-tests/draft2019-09/content/general.json @@ -3,7 +3,7 @@ "description": "failed validation produces no annotations", "schema": { "$schema": "https://json-schema.org/draft/2019-09/schema", - "$id": "https://json-schema.org/tests/content/draft2019-09/general/1", + "$id": "https://json-schema.org/tests/content/draft2019-09/general/0", "type": "string", "readOnly": true }, @@ -13,6 +13,7 @@ "data": 1, "output": { "basic": { + "$id": "https://json-schema.org/tests/content/draft2019-09/general/0/tests/0/basic", "$ref": "/draft/2019-09/output/schema", "properties": { "errors": { diff --git a/output-tests/draft2019-09/content/readOnly.json b/output-tests/draft2019-09/content/readOnly.json index c284d3d3..cd7b4fac 100644 --- a/output-tests/draft2019-09/content/readOnly.json +++ b/output-tests/draft2019-09/content/readOnly.json @@ -3,7 +3,7 @@ "description": "readOnly generates its value as an annotation", "schema": { "$schema": "https://json-schema.org/draft/2019-09/schema", - "$id": "https://json-schema.org/tests/content/draft2019-09/readOnly/1", + "$id": "https://json-schema.org/tests/content/draft2019-09/readOnly/0", "readOnly": true }, "tests": [ @@ -12,6 +12,7 @@ "data": 1, "output": { "basic": { + "$id": "https://json-schema.org/tests/content/draft2019-09/readOnly/0/tests/0/basic", "$ref": "/draft/2019-09/output/schema", "properties": { "annotations": { @@ -19,7 +20,7 @@ "type": "object", "properties": { "keywordLocation": {"const": "/readOnly"}, - "absoluteKeywordLocation": {"const": "https://json-schema.org/tests/content/draft2019-09/readOnly/1#/readOnly"}, + "absoluteKeywordLocation": {"const": "https://json-schema.org/tests/content/draft2019-09/readOnly/0#/readOnly"}, "instanceLocation": {"const": ""}, "annotation": {"const": true} }, diff --git a/output-tests/draft2019-09/content/type.json b/output-tests/draft2019-09/content/type.json index 685b4c7f..f78f0473 100644 --- a/output-tests/draft2019-09/content/type.json +++ b/output-tests/draft2019-09/content/type.json @@ -3,7 +3,7 @@ "description": "validating type", "schema": { "$schema": "https://json-schema.org/draft/2019-09/schema", - "$id": "https://json-schema.org/tests/content/draft2019-09/type/1", + "$id": "https://json-schema.org/tests/content/draft2019-09/type/0", "type": "string" }, "tests": [ @@ -12,13 +12,14 @@ "data": 1, "output": { "basic": { + "$id": "https://json-schema.org/tests/content/draft2019-09/type/0/tests/0/basic", "$ref": "/draft/2019-09/output/schema", "properties": { "errors": { "contains": { "properties": { "keywordLocation": {"const": "/type"}, - "absoluteKeywordLocation": {"const": "https://json-schema.org/tests/content/draft2019-09/type/1#/type"}, + "absoluteKeywordLocation": {"const": "https://json-schema.org/tests/content/draft2019-09/type/0#/type"}, "instanceLocation": {"const": ""}, "annotation": false }, @@ -32,9 +33,10 @@ }, { "description": "correct type yields an output unit", - "data": 1, + "data": "a string", "output": { "basic": { + "$id": "https://json-schema.org/tests/content/draft2019-09/type/0/tests/1/basic", "$ref": "/draft/2019-09/output/schema", "properties": { "annotations": { @@ -42,7 +44,7 @@ "properties": { "valid": {"const": true}, "keywordLocation": {"const": "/type"}, - "absoluteKeywordLocation": {"const": "https://json-schema.org/tests/content/draft2019-09/type/1#/type"}, + "absoluteKeywordLocation": {"const": "https://json-schema.org/tests/content/draft2019-09/type/0#/type"}, "instanceLocation": {"const": ""}, "annotation": false, "error": false diff --git a/output-tests/draft2020-12/content/general.json b/output-tests/draft2020-12/content/general.json index 8662da30..1f2b370c 100644 --- a/output-tests/draft2020-12/content/general.json +++ b/output-tests/draft2020-12/content/general.json @@ -3,7 +3,7 @@ "description": "failed validation produces no annotations", "schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://json-schema.org/tests/content/draft2020-12/general/1", + "$id": "https://json-schema.org/tests/content/draft2020-12/general/0", "type": "string", "readOnly": true }, @@ -13,6 +13,7 @@ "data": 1, "output": { "basic": { + "$id": "https://json-schema.org/tests/content/draft2020-12/general/0/tests/0/basic", "$ref": "/draft/2020-12/output/schema", "properties": { "errors": { diff --git a/output-tests/draft2020-12/content/readOnly.json b/output-tests/draft2020-12/content/readOnly.json index e4b9c3fd..67164464 100644 --- a/output-tests/draft2020-12/content/readOnly.json +++ b/output-tests/draft2020-12/content/readOnly.json @@ -3,7 +3,7 @@ "description": "readOnly generates its value as an annotation", "schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://json-schema.org/tests/content/draft2020-12/readOnly/1", + "$id": "https://json-schema.org/tests/content/draft2020-12/readOnly/0", "readOnly": true }, "tests": [ @@ -12,13 +12,14 @@ "data": 1, "output": { "basic": { + "$id": "https://json-schema.org/tests/content/draft2020-12/readOnly/0/tests/0/basic", "$ref": "/draft/2020-12/output/schema", "properties": { "annotations": { "contains": { "properties": { "keywordLocation": {"const": "/readOnly"}, - "absoluteKeywordLocation": {"const": "https://json-schema.org/tests/content/draft2020-12/readOnly/1#/readOnly"}, + "absoluteKeywordLocation": {"const": "https://json-schema.org/tests/content/draft2020-12/readOnly/0#/readOnly"}, "instanceLocation": {"const": ""}, "annotation": {"const": true} }, diff --git a/output-tests/draft2020-12/content/type.json b/output-tests/draft2020-12/content/type.json index 081a21da..2d8cf010 100644 --- a/output-tests/draft2020-12/content/type.json +++ b/output-tests/draft2020-12/content/type.json @@ -3,7 +3,7 @@ "description": "validating type", "schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://json-schema.org/tests/content/draft2020-12/type/1", + "$id": "https://json-schema.org/tests/content/draft2020-12/type/0", "type": "string" }, "tests": [ @@ -12,13 +12,14 @@ "data": 1, "output": { "basic": { + "$id": "https://json-schema.org/tests/content/draft2020-12/type/0/tests/0/basic", "$ref": "/draft/2020-12/output/schema", "properties": { "errors": { "contains": { "properties": { "keywordLocation": {"const": "/type"}, - "absoluteKeywordLocation": {"const": "https://json-schema.org/tests/content/draft2020-12/type/1#/type"}, + "absoluteKeywordLocation": {"const": "https://json-schema.org/tests/content/draft2020-12/type/0#/type"}, "instanceLocation": {"const": ""}, "annotation": false }, @@ -32,9 +33,10 @@ }, { "description": "correct type yields an output unit", - "data": 1, + "data": "a string", "output": { "basic": { + "$id": "https://json-schema.org/tests/content/draft2020-12/type/0/tests/1/basic", "$ref": "/draft/2020-12/output/schema", "properties": { "annotations": { @@ -42,7 +44,7 @@ "properties": { "valid": {"const": true}, "keywordLocation": {"const": "/type"}, - "absoluteKeywordLocation": {"const": "https://json-schema.org/tests/content/draft2020-12/type/1#/type"}, + "absoluteKeywordLocation": {"const": "https://json-schema.org/tests/content/draft2020-12/type/0#/type"}, "instanceLocation": {"const": ""}, "annotation": false, "error": false From c88355294d1db8aa299c15968866831948421797 Mon Sep 17 00:00:00 2001 From: Greg Dennis Date: Fri, 2 Dec 2022 14:10:15 +1300 Subject: [PATCH 17/20] absoluteKeywordLocation is not required when there is no `$ref` --- output-tests/draft2019-09/content/readOnly.json | 2 +- output-tests/draft2019-09/content/type.json | 4 ++-- output-tests/draft2020-12/content/readOnly.json | 2 +- output-tests/draft2020-12/content/type.json | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/output-tests/draft2019-09/content/readOnly.json b/output-tests/draft2019-09/content/readOnly.json index cd7b4fac..62db1a83 100644 --- a/output-tests/draft2019-09/content/readOnly.json +++ b/output-tests/draft2019-09/content/readOnly.json @@ -24,7 +24,7 @@ "instanceLocation": {"const": ""}, "annotation": {"const": true} }, - "required": ["keywordLocation", "absoluteKeywordLocation", "instanceLocation", "annotation"] + "required": ["keywordLocation", "instanceLocation", "annotation"] } }, "errors": false diff --git a/output-tests/draft2019-09/content/type.json b/output-tests/draft2019-09/content/type.json index f78f0473..7462945c 100644 --- a/output-tests/draft2019-09/content/type.json +++ b/output-tests/draft2019-09/content/type.json @@ -23,7 +23,7 @@ "instanceLocation": {"const": ""}, "annotation": false }, - "required": ["keywordLocation", "absoluteKeywordLocation", "instanceLocation"] + "required": ["keywordLocation", "instanceLocation"] } } }, @@ -49,7 +49,7 @@ "annotation": false, "error": false }, - "required": ["keywordLocation", "absoluteKeywordLocation", "instanceLocation"] + "required": ["keywordLocation", "instanceLocation"] } } }, diff --git a/output-tests/draft2020-12/content/readOnly.json b/output-tests/draft2020-12/content/readOnly.json index 67164464..9baf48de 100644 --- a/output-tests/draft2020-12/content/readOnly.json +++ b/output-tests/draft2020-12/content/readOnly.json @@ -23,7 +23,7 @@ "instanceLocation": {"const": ""}, "annotation": {"const": true} }, - "required": ["keywordLocation", "absoluteKeywordLocation", "instanceLocation", "annotation"] + "required": ["keywordLocation", "instanceLocation", "annotation"] } }, "errors": false diff --git a/output-tests/draft2020-12/content/type.json b/output-tests/draft2020-12/content/type.json index 2d8cf010..165c9b5f 100644 --- a/output-tests/draft2020-12/content/type.json +++ b/output-tests/draft2020-12/content/type.json @@ -23,7 +23,7 @@ "instanceLocation": {"const": ""}, "annotation": false }, - "required": ["keywordLocation", "absoluteKeywordLocation", "instanceLocation"] + "required": ["keywordLocation", "instanceLocation"] } } }, @@ -49,7 +49,7 @@ "annotation": false, "error": false }, - "required": ["keywordLocation", "absoluteKeywordLocation", "instanceLocation"] + "required": ["keywordLocation", "instanceLocation"] } } }, From 8ee43236dce4c70cb34ad90d54f82e7aa15bf267 Mon Sep 17 00:00:00 2001 From: Greg Dennis Date: Mon, 5 Dec 2022 10:28:50 +1300 Subject: [PATCH 18/20] add some more detail in readme; add redundant keyword to some tests --- output-tests/README.md | 39 +++++++++++++++++++-- output-tests/draft2019-09/content/type.json | 3 +- output-tests/draft2020-12/content/type.json | 3 +- 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/output-tests/README.md b/output-tests/README.md index 63a8fdad..0cd7c87b 100644 --- a/output-tests/README.md +++ b/output-tests/README.md @@ -8,9 +8,9 @@ Output was initially specified with draft 2019-09. It remained largely unchange The tests are organized by specification release and then into two categories: content and structure. -Content tests verify that the keywords are producing the correct annotations and/or error messages. Since there are no requirements on the content of error messages, there's not much that can be verified for them, but it is possible to identify when a error message _could_ be present. Primarily, these tests need to extensively cover the annotation behaviors of each keyword. The only output format needed for these tests is `basic`. +Content tests verify that the keywords are producing the correct annotations and/or error messages. Since there are no requirements on the content of error messages, there's not much that can be verified for them, but it is possible to identify when a error message _could_ be present. Primarily, these tests need to extensively cover the annotation behaviors of each keyword. The only output format needed for these tests is `basic` for 2019-09/2020/12 and `list` for later versions. -Structure tests verify that the structures of the various formats (i.e. `flag`, `basic`, `detailed`, `verbose`) are correct. These tests don't need to cover each keyword; rather they need to sufficiently cover the various aspects of building the output structures by using whatever keywords are necessary to do so. +Structure tests verify that the structures of the various formats (i.e. `flag`, `basic`, `detailed`, `verbose` for 2019/2020 and `flag`, `list`, `hierarchical` for later versions) are correct. These tests don't need to cover each keyword; rather they need to sufficiently cover the various aspects of building the output structures by using whatever keywords are necessary to do so. In each release folder, you'll also find an _output-schema.json_ file that contains the schema from the specification repo that describes output for that release. This schema will need to be loaded as the tests reference it. @@ -18,7 +18,40 @@ In each release folder, you'll also find an _output-schema.json_ file that conta The content of a test file is the same as the validation tests in `tests/`, however an `output` property has been added to each test case. -The `output` property itself has a property for each of the output formats where the value is a schema that will successfully validate for compliant output. For the content tests, only `basic` needs to be present. +The `output` property itself has a property for each of the output formats where the value is a schema that will successfully validate for compliant output. For the content tests, only `basic`/`list` needs to be present. + +## Other notes + +### Ambiguity around 2020-09/2020-12 `basic` + +The 2019-09/2020-12 specs don't define the structure of `basic` very thoroughly. Specifically there is a nuance where if the list contains a single output node, there are two possible structures, given the text: + +- the output node for the root schema appears in the list with a containing node that just has a `valid` property + ```json + { + "valid": false, + "errors": [ + { + "valid": false, + "keywordLocation": "", + "absoluteKeywordLocation": "https://json-schema.org/tests/content/draft2019-09/general/0", + "instanceLocation": "" + } + ] + } + ``` +- the entire structure is collapsed to just the root output node as `detailed` would do. + ```json + { + "valid": false, + "keywordLocation": "", + "absoluteKeywordLocation": "https://json-schema.org/tests/content/draft2019-09/general/0", + "instanceLocation": "" + } + ``` +As the Test Suite should not prefer one interpretation over another, these cases need to be tested another way. + +A simple solution (though there are likely others) is to force a second output unit by adding an `"anyOf": [ true ]`. This has no impact on the validation result while adding superfluous structure to the output that avoids the above ambiguous scenario. The test schema should still be targeted on what's being tested and ignore any output units generated by this extra keyword. ## Contributing diff --git a/output-tests/draft2019-09/content/type.json b/output-tests/draft2019-09/content/type.json index 7462945c..cff77a74 100644 --- a/output-tests/draft2019-09/content/type.json +++ b/output-tests/draft2019-09/content/type.json @@ -4,7 +4,8 @@ "schema": { "$schema": "https://json-schema.org/draft/2019-09/schema", "$id": "https://json-schema.org/tests/content/draft2019-09/type/0", - "type": "string" + "type": "string", + "anyOf": [ true ] }, "tests": [ { diff --git a/output-tests/draft2020-12/content/type.json b/output-tests/draft2020-12/content/type.json index 165c9b5f..710475b2 100644 --- a/output-tests/draft2020-12/content/type.json +++ b/output-tests/draft2020-12/content/type.json @@ -4,7 +4,8 @@ "schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://json-schema.org/tests/content/draft2020-12/type/0", - "type": "string" + "type": "string", + "anyOf": [ true ] }, "tests": [ { From c2644010730878886def168e41a0e4d3bec22f63 Mon Sep 17 00:00:00 2001 From: Julian Berman Date: Tue, 29 Nov 2022 11:22:25 -0500 Subject: [PATCH 19/20] Blacked. --- bin/jsonschema_suite | 41 +++++++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/bin/jsonschema_suite b/bin/jsonschema_suite index cd7e548d..3e3bd808 100755 --- a/bin/jsonschema_suite +++ b/bin/jsonschema_suite @@ -84,7 +84,7 @@ def url_for_path(path): return urljoin( REMOTES_BASE_URL, - str(path.relative_to(REMOTES_DIR)).replace("\\", "/") # Windows... + str(path.relative_to(REMOTES_DIR)).replace("\\", "/"), # Windows... ) @@ -100,7 +100,8 @@ class SanityTests(unittest.TestCase): print(f"Found {len(cls.test_files)} test files") cls.output_test_files = [ - each for each in collect(OUTPUT_ROOT_DIR) + each + for each in collect(OUTPUT_ROOT_DIR) if each.name != "output-schema.json" ] assert cls.output_test_files, "Didn't find the output test files!" @@ -163,19 +164,21 @@ class SanityTests(unittest.TestCase): self.assertLess( len(case["description"]), 150, - "Description is too long (keep it to less than 150 chars)." + "Description is too long (keep it to less than 150 chars).", ) def test_all_test_descriptions_have_reasonable_length(self): """ All tests have reasonably long descriptions. """ - for count, test in enumerate(tests(self.test_files + self.output_test_files)): + for count, test in enumerate( + tests(self.test_files + self.output_test_files) + ): with self.subTest(description=test["description"]): self.assertLess( len(test["description"]), 70, - "Description is too long (keep it to less than 70 chars)." + "Description is too long (keep it to less than 70 chars).", ) print(f"Found {count} tests.") @@ -191,7 +194,9 @@ class SanityTests(unittest.TestCase): """ All test cases have unique test descriptions in their tests. """ - for count, case in enumerate(cases(self.test_files + self.output_test_files)): + for count, case in enumerate( + cases(self.test_files + self.output_test_files) + ): with self.subTest(description=case["description"]): self.assertUnique( test["description"] for test in case["tests"] @@ -226,7 +231,7 @@ class SanityTests(unittest.TestCase): Validator.check_schema(case["schema"]) except jsonschema.SchemaError: self.fail( - "Found an invalid schema." + "Found an invalid schema. " "See the traceback for details on why." ) else: @@ -292,7 +297,9 @@ def main(arguments): try: import flask except ImportError: - print(textwrap.dedent(""" + print( + textwrap.dedent( + """ The Flask library is required to serve the remote schemas. You can install it by running `pip install Flask`. @@ -300,7 +307,11 @@ def main(arguments): Alternatively, see the `jsonschema_suite remotes` or `jsonschema_suite dump_remotes` commands to create static files that can be served with your own web server. - """.strip("\n"))) + """.strip( + "\n" + ) + ) + ) sys.exit(1) app = flask.Flask(__name__) @@ -324,7 +335,7 @@ check = subparsers.add_parser("check", help="Sanity check the test suite.") flatten = subparsers.add_parser( "flatten", - help="Output a flattened file containing a selected version's test cases." + help="Output a flattened file containing a selected version's test cases.", ) flatten.add_argument( "--randomize", @@ -332,17 +343,19 @@ flatten.add_argument( help="Randomize the order of the outputted cases.", ) flatten.add_argument( - "version", help="The directory containing the version to output", + "version", + help="The directory containing the version to output", ) remotes = subparsers.add_parser( "remotes", help="Output the expected URLs and their associated schemas for remote " - "ref tests as a JSON object." + "ref tests as a JSON object.", ) dump_remotes = subparsers.add_parser( - "dump_remotes", help="Dump the remote ref schemas into a file tree", + "dump_remotes", + help="Dump the remote ref schemas into a file tree", ) dump_remotes.add_argument( "--update", @@ -358,7 +371,7 @@ dump_remotes.add_argument( serve = subparsers.add_parser( "serve", - help="Start a webserver to serve schemas used by remote ref tests." + help="Start a webserver to serve schemas used by remote ref tests.", ) if __name__ == "__main__": From b538fe75abc1d735ddbdcb7d38678c4539d77cd7 Mon Sep 17 00:00:00 2001 From: Julian Berman Date: Tue, 6 Dec 2022 10:04:28 -0500 Subject: [PATCH 20/20] Bump the validator version used for suite sanity checks. And fix skipping regex format validation and the case where dynamicRef blows up. (The latter hopefully being temporary, but just so the PR passes) --- bin/jsonschema_suite | 12 +++++++++++- tox.ini | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/bin/jsonschema_suite b/bin/jsonschema_suite index 3e3bd808..33d4c565 100755 --- a/bin/jsonschema_suite +++ b/bin/jsonschema_suite @@ -224,11 +224,18 @@ class SanityTests(unittest.TestCase): Validator = VALIDATORS.get(version.name) if Validator is not None: + # Valid (optional test) schemas contain regexes which + # aren't valid Python regexes, so skip checking it + Validator.FORMAT_CHECKER.checkers.pop("regex", None) + test_files = collect(version) for case in cases(test_files): with self.subTest(case=case): try: - Validator.check_schema(case["schema"]) + Validator.check_schema( + case["schema"], + format_checker=Validator.FORMAT_CHECKER, + ) except jsonschema.SchemaError: self.fail( "Found an invalid schema. " @@ -262,6 +269,9 @@ class SanityTests(unittest.TestCase): with self.subTest(path=path): try: validator.validate(cases) + except jsonschema.exceptions.RefResolutionError as error: + # python-jsonschema/jsonschema#884 + pass except jsonschema.ValidationError as error: self.fail(str(error)) diff --git a/tox.ini b/tox.ini index ec180a91..7ca9de98 100644 --- a/tox.ini +++ b/tox.ini @@ -5,5 +5,5 @@ skipsdist = True [testenv:sanity] # used just for validating the structure of the test case files themselves -deps = jsonschema==4.6.1 +deps = jsonschema==4.17.3 commands = {envpython} bin/jsonschema_suite check