Allow passing ParseOptions to inline tests (#16357)

ntBre · AlexWaygood · web-flow · commit 764aa0e6a1b1 · 2025-02-27T10:23:15.000-05:00
## Summary

This PR adds support for a pragma-style header for inline parser tests
containing JSON-serialized `ParseOptions`. For example,

```python
# parse_options: { "target-version": "3.9" }
match 2:
    case 1:
        pass
```

The line must start with `# parse_options: ` and then the rest of the
(trimmed) line is deserialized into `ParseOptions` used for parsing the
the test.

## Test Plan

Existing inline tests, plus two new inline tests for
`match-before-py310`.

---------

Co-authored-by: Alex Waygood &lt;alex.waygood@gmail.com&gt;
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/crates/ruff_python_parser/Cargo.toml b/crates/ruff_python_parser/Cargo.toml
@@ -29,10 +29,13 @@ unicode-normalization = { workspace = true }
 
 [dev-dependencies]
 ruff_annotate_snippets = { workspace = true }
+ruff_python_ast = { workspace = true, features = ["serde"] }
 ruff_source_file = { workspace = true }
 
 anyhow = { workspace = true }
 insta = { workspace = true, features = ["glob"] }
+serde = { workspace = true }
+serde_json = { workspace = true }
 walkdir = { workspace = true }
 
 [lints]
diff --git a/crates/ruff_python_parser/resources/inline/err/match_before_py310.py b/crates/ruff_python_parser/resources/inline/err/match_before_py310.py
@@ -0,0 +1,4 @@
+# parse_options: { "target-version": "3.9" }
+match 2:
+    case 1:
+        pass
diff --git a/crates/ruff_python_parser/resources/inline/ok/match_after_py310.py b/crates/ruff_python_parser/resources/inline/ok/match_after_py310.py
@@ -0,0 +1,4 @@
+# parse_options: { "target-version": "3.10" }
+match 2:
+    case 1:
+        pass
diff --git a/crates/ruff_python_parser/src/parser/statement.rs b/crates/ruff_python_parser/src/parser/statement.rs
@@ -2265,6 +2265,18 @@ impl<'src> Parser<'src> {
 
         let cases = self.parse_match_body();
 
+        // test_err match_before_py310
+        // # parse_options: { "target-version": "3.9" }
+        // match 2:
+        //     case 1:
+        //         pass
+
+        // test_ok match_after_py310
+        // # parse_options: { "target-version": "3.10" }
+        // match 2:
+        //     case 1:
+        //         pass
+
         if self.options.target_version < PythonVersion::PY310 {
             self.unsupported_syntax_errors.push(UnsupportedSyntaxError {
                 kind: UnsupportedSyntaxErrorKind::MatchBeforePy310,
diff --git a/crates/ruff_python_parser/tests/fixtures.rs b/crates/ruff_python_parser/tests/fixtures.rs
@@ -5,7 +5,7 @@ use std::path::Path;
 
 use ruff_annotate_snippets::{Level, Renderer, Snippet};
 use ruff_python_ast::visitor::source_order::{walk_module, SourceOrderVisitor, TraversalSignal};
-use ruff_python_ast::{AnyNodeRef, Mod};
+use ruff_python_ast::{AnyNodeRef, Mod, PythonVersion};
 use ruff_python_parser::{parse_unchecked, Mode, ParseErrorType, ParseOptions, Token};
 use ruff_source_file::{LineIndex, OneIndexed, SourceCode};
 use ruff_text_size::{Ranged, TextLen, TextRange, TextSize};
@@ -34,9 +34,14 @@ fn inline_err() {
 /// Snapshots the AST.
 fn test_valid_syntax(input_path: &Path) {
     let source = fs::read_to_string(input_path).expect("Expected test file to exist");
-    let parsed = parse_unchecked(&source, ParseOptions::from(Mode::Module));
+    let options = extract_options(&source).unwrap_or_else(|| {
+        ParseOptions::from(Mode::Module).with_target_version(PythonVersion::latest())
+    });
+    let parsed = parse_unchecked(&source, options);
 
-    if !parsed.is_valid() {
+    let is_valid = parsed.is_valid() && parsed.unsupported_syntax_errors().is_empty();
+
+    if !is_valid {
         let line_index = LineIndex::from_source_text(&source);
         let source_code = SourceCode::new(&source, &line_index);
 
@@ -55,6 +60,19 @@ fn test_valid_syntax(input_path: &Path) {
             .unwrap();
         }
 
+        for error in parsed.unsupported_syntax_errors() {
+            writeln!(
+                &mut message,
+                "{}\n",
+                CodeFrame {
+                    range: error.range,
+                    error: &ParseErrorType::OtherError(error.to_string()),
+                    source_code: &source_code,
+                }
+            )
+            .unwrap();
+        }
+
         panic!("{input_path:?}: {message}");
     }
 
@@ -78,10 +96,15 @@ fn test_valid_syntax(input_path: &Path) {
 /// Snapshots the AST and the error messages.
 fn test_invalid_syntax(input_path: &Path) {
     let source = fs::read_to_string(input_path).expect("Expected test file to exist");
-    let parsed = parse_unchecked(&source, ParseOptions::from(Mode::Module));
+    let options = extract_options(&source).unwrap_or_else(|| {
+        ParseOptions::from(Mode::Module).with_target_version(PythonVersion::latest())
+    });
+    let parsed = parse_unchecked(&source, options);
+
+    let is_valid = parsed.is_valid() && parsed.unsupported_syntax_errors().is_empty();
 
     assert!(
-        !parsed.is_valid(),
+        !is_valid,
         "{input_path:?}: Expected parser to generate at least one syntax error for a program containing syntax errors."
     );
 
@@ -92,11 +115,13 @@ fn test_invalid_syntax(input_path: &Path) {
     writeln!(&mut output, "## AST").unwrap();
     writeln!(&mut output, "\n```\n{:#?}\n```", parsed.syntax()).unwrap();
 
-    writeln!(&mut output, "## Errors\n").unwrap();
-
     let line_index = LineIndex::from_source_text(&source);
     let source_code = SourceCode::new(&source, &line_index);
 
+    if !parsed.errors().is_empty() {
+        writeln!(&mut output, "## Errors\n").unwrap();
+    }
+
     for error in parsed.errors() {
         writeln!(
             &mut output,
@@ -110,6 +135,23 @@ fn test_invalid_syntax(input_path: &Path) {
         .unwrap();
     }
 
+    if !parsed.unsupported_syntax_errors().is_empty() {
+        writeln!(&mut output, "## Unsupported Syntax Errors\n").unwrap();
+    }
+
+    for error in parsed.unsupported_syntax_errors() {
+        writeln!(
+            &mut output,
+            "{}\n",
+            CodeFrame {
+                range: error.range,
+                error: &ParseErrorType::OtherError(error.to_string()),
+                source_code: &source_code,
+            }
+        )
+        .unwrap();
+    }
+
     insta::with_settings!({
         omit_expression => true,
         input_file => input_path,
@@ -119,6 +161,53 @@ fn test_invalid_syntax(input_path: &Path) {
     });
 }
 
+/// Copy of [`ParseOptions`] for deriving [`Deserialize`] with serde as a dev-dependency.
+#[derive(serde::Deserialize)]
+#[serde(rename_all = "kebab-case")]
+struct JsonParseOptions {
+    #[serde(default)]
+    mode: JsonMode,
+    #[serde(default)]
+    target_version: PythonVersion,
+}
+
+/// Copy of [`Mode`] for deserialization.
+#[derive(Default, serde::Deserialize)]
+#[serde(rename_all = "kebab-case")]
+enum JsonMode {
+    #[default]
+    Module,
+    Expression,
+    ParenthesizedExpression,
+    Ipython,
+}
+
+impl From<JsonParseOptions> for ParseOptions {
+    fn from(value: JsonParseOptions) -> Self {
+        let mode = match value.mode {
+            JsonMode::Module => Mode::Module,
+            JsonMode::Expression => Mode::Expression,
+            JsonMode::ParenthesizedExpression => Mode::ParenthesizedExpression,
+            JsonMode::Ipython => Mode::Ipython,
+        };
+        Self::from(mode).with_target_version(value.target_version)
+    }
+}
+
+/// Extract [`ParseOptions`] from an initial pragma line, if present.
+///
+/// For example,
+///
+/// ```python
+/// # parse_options: { "target-version": "3.10" }
+/// def f(): ...
+fn extract_options(source: &str) -> Option<ParseOptions> {
+    let header = source.lines().next()?;
+    let (_label, options) = header.split_once("# parse_options: ")?;
+    let options: Option<JsonParseOptions> = serde_json::from_str(options.trim()).ok();
+    options.map(ParseOptions::from)
+}
+
 // Test that is intentionally ignored by default.
 // Use it for quickly debugging a parser issue.
 #[test]
diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@match_before_py310.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@match_before_py310.py.snap
@@ -0,0 +1,63 @@
+---
+source: crates/ruff_python_parser/tests/fixtures.rs
+input_file: crates/ruff_python_parser/resources/inline/err/match_before_py310.py
+---
+## AST
+
+```
+Module(
+    ModModule {
+        range: 0..79,
+        body: [
+            Match(
+                StmtMatch {
+                    range: 45..78,
+                    subject: NumberLiteral(
+                        ExprNumberLiteral {
+                            range: 51..52,
+                            value: Int(
+                                2,
+                            ),
+                        },
+                    ),
+                    cases: [
+                        MatchCase {
+                            range: 58..78,
+                            pattern: MatchValue(
+                                PatternMatchValue {
+                                    range: 63..64,
+                                    value: NumberLiteral(
+                                        ExprNumberLiteral {
+                                            range: 63..64,
+                                            value: Int(
+                                                1,
+                                            ),
+                                        },
+                                    ),
+                                },
+                            ),
+                            guard: None,
+                            body: [
+                                Pass(
+                                    StmtPass {
+                                        range: 74..78,
+                                    },
+                                ),
+                            ],
+                        },
+                    ],
+                },
+            ),
+        ],
+    },
+)
+```
+## Unsupported Syntax Errors
+
+  |
+1 | # parse_options: { "target-version": "3.9" }
+2 | match 2:
+  | ^^^^^ Syntax Error: Cannot use `match` statement on Python 3.9 (syntax was added in Python 3.10)
+3 |     case 1:
+4 |         pass
+  |
diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@match_after_py310.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@match_after_py310.py.snap
@@ -0,0 +1,54 @@
+---
+source: crates/ruff_python_parser/tests/fixtures.rs
+input_file: crates/ruff_python_parser/resources/inline/ok/match_after_py310.py
+---
+## AST
+
+```
+Module(
+    ModModule {
+        range: 0..80,
+        body: [
+            Match(
+                StmtMatch {
+                    range: 46..79,
+                    subject: NumberLiteral(
+                        ExprNumberLiteral {
+                            range: 52..53,
+                            value: Int(
+                                2,
+                            ),
+                        },
+                    ),
+                    cases: [
+                        MatchCase {
+                            range: 59..79,
+                            pattern: MatchValue(
+                                PatternMatchValue {
+                                    range: 64..65,
+                                    value: NumberLiteral(
+                                        ExprNumberLiteral {
+                                            range: 64..65,
+                                            value: Int(
+                                                1,
+                                            ),
+                                        },
+                                    ),
+                                },
+                            ),
+                            guard: None,
+                            body: [
+                                Pass(
+                                    StmtPass {
+                                        range: 75..79,
+                                    },
+                                ),
+                            ],
+                        },
+                    ],
+                },
+            ),
+        ],
+    },
+)
+```

-Original file line number
+Diff line change
@@ @@ -0,0 +1,4 @@ @@
 +# parse_options: { "target-version": "3.9" }
 +match 2:
 +    case 1:
 +        pass