And now push the smoke test down onto Implementation.

Author: Julian

bowtie/_cli.py

@@ -715,28 +715,6 @@ async def smoke(
     for implementation in implementations:
         echo(f"Testing {implementation.name!r}...\n", file=sys.stderr)
 
-        # FIXME: All dialects / and/or newest dialect with proper sort
-        dialect = max(implementation.info().dialects, key=str)
-        runner = await implementation.start_speaking(dialect)
-
-        cases = [
-            TestCase(
-                description="allow-everything schema",
-                schema={"$schema": str(dialect)},
-                tests=[
-                    Test(description="First", instance=1, valid=True),
-                    Test(description="Second", instance="foo", valid=True),
-                ],
-            ),
-            TestCase(
-                description="allow-nothing schema",
-                schema={"$schema": str(dialect), "not": {}},
-                tests=[
-                    Test(description="First", instance=12, valid=False),
-                ],
-            ),
-        ]
-
         match format:
             case "json":
                 serializable: list[dict[str, Any]] = []
@@ -758,8 +736,7 @@ def see(seq_case: SeqCase, response: SeqResult):
                     )
                     echo(f"  · {seq_case.case.description}: {signs}")
 
-        for seq_case in SeqCase.for_cases(cases):
-            result = await seq_case.run(runner=runner)
+        async for seq_case, result in implementation.smoke():
             if result.unsuccessful().causes_stop:
                 exit_code |= _EX_DATAERR
             see(seq_case, result)

bowtie/_commands.py

@@ -146,7 +146,7 @@ class SeqCase:
     case: TestCase
 
     @classmethod
-    def for_cases(cls, cases: Iterable[TestCase]):
+    def for_cases(cls, cases: Iterable[TestCase]) -> Iterable[SeqCase]:
         return (cls(seq=i, case=case) for i, case in enumerate(cases))
 
     def run(self, runner: DialectRunner) -> Awaitable[SeqResult]:

bowtie/_core.py

@@ -436,3 +436,53 @@ async def _send(self, cmd: _commands.Command[Any], retry: int = 3) -> Any:
                 )
                 break
         return INVALID
+
+    async def smoke(
+        self,
+    ) -> AsyncIterator[tuple[_commands.SeqCase, _commands.SeqResult],]:
+        """
+        Smoke test this implementation.
+        """
+        # FIXME: All dialects / and/or newest dialect with proper sort
+        dialect = max(self.info().dialects, key=str)
+        runner = await self.start_speaking(dialect)
+
+        instances = [
+            ("nil", None),
+            ("boolean", True),
+            ("integer", 37),
+            ("number", 37.37),
+            ("string", "37"),
+            ("array", [37]),
+            ("object", {"foo": 37}),
+        ]
+        cases = [
+            _commands.TestCase(
+                description="allow-everything",
+                schema={"$schema": str(dialect)},
+                tests=[
+                    _commands.Test(
+                        description=json_type,
+                        instance=instance,
+                        valid=True,
+                    )
+                    for json_type, instance in instances
+                ],
+            ),
+            _commands.TestCase(
+                description="allow-nothing",
+                schema={"$schema": str(dialect), "not": {}},
+                tests=[
+                    _commands.Test(
+                        description=json_type,
+                        instance=instance,
+                        valid=False,
+                    )
+                    for json_type, instance in instances
+                ],
+            ),
+        ]
+
+        for seq_case in _commands.SeqCase.for_cases(cases):
+            result = await seq_case.run(runner=runner)
+            yield seq_case, result

tests/test_integration.py

@@ -601,8 +601,8 @@ async def test_smoke_pretty(envsonschema):
         dedent(stdout)
         == dedent(
             """
-            · allow-everything schema: ✗✗
-            · allow-nothing schema: ✓
+            · allow-everything: ✗✗✗✗✗✗✗
+            · allow-nothing: ✓✓✓✓✓✓✓
         """,
         ).lstrip("\n")
     ), stderr
@@ -625,27 +625,60 @@ async def test_smoke_json(envsonschema):
     assert jsonout == [
         {
             "case": {
-                "description": "allow-everything schema",
+                "description": "allow-everything",
                 "schema": {
                     "$schema": "https://json-schema.org/draft/2020-12/schema",
                 },
                 "tests": [
-                    {"description": "First", "instance": 1},
-                    {"description": "Second", "instance": "foo"},
+                    {"description": "nil", "instance": None},
+                    {"description": "boolean", "instance": True},
+                    {"description": "integer", "instance": 37},
+                    {"description": "number", "instance": 37.37},
+                    {"description": "string", "instance": "37"},
+                    {"description": "array", "instance": [37]},
+                    {"description": "object", "instance": {"foo": 37}},
+                ],
+            },
+            "result": {
+                "results": [
+                    {"valid": False},
+                    {"valid": False},
+                    {"valid": False},
+                    {"valid": False},
+                    {"valid": False},
+                    {"valid": False},
+                    {"valid": False},
                 ],
             },
-            "result": {"results": [{"valid": False}, {"valid": False}]},
         },
         {
             "case": {
-                "description": "allow-nothing schema",
+                "description": "allow-nothing",
                 "schema": {
                     "$schema": "https://json-schema.org/draft/2020-12/schema",
                     "not": {},
                 },
-                "tests": [{"description": "First", "instance": 12}],
+                "tests": [
+                    {"description": "nil", "instance": None},
+                    {"description": "boolean", "instance": True},
+                    {"description": "integer", "instance": 37},
+                    {"description": "number", "instance": 37.37},
+                    {"description": "string", "instance": "37"},
+                    {"description": "array", "instance": [37]},
+                    {"description": "object", "instance": {"foo": 37}},
+                ],
+            },
+            "result": {
+                "results": [
+                    {"valid": False},
+                    {"valid": False},
+                    {"valid": False},
+                    {"valid": False},
+                    {"valid": False},
+                    {"valid": False},
+                    {"valid": False},
+                ],
             },
-            "result": {"results": [{"valid": False}]},
         },
     ], stderr