Skip to content

Commit ed948ea

Browse files
dhruvmaniladhruvmanila
authored
Avoid moving back the lexer for triple-quoted fstring (#11939)
## Summary This PR avoids moving back the lexer for a triple-quoted f-string during the re-lexing phase. The reason this is a problem is that for a triple-quoted f-string the newlines are part of the f-string itself, specifically they'll be part of the `FStringMiddle` token. So, if we moved the lexer back, there would be a `Newline` token whose range would be in between an `FStringMiddle` token. This creates a panic in downstream usage. fixes: #11937 ## Test Plan Add test cases and validate the snapshots.
1 parent 22733cb commit ed948ea

10 files changed

+377
-156
lines changed

‎crates/ruff_python_parser/resources/invalid/re_lex_logical_token.py

+1-7
Original file line numberDiff line numberDiff line change
@@ -54,10 +54,4 @@ def bar():
5454

5555
if call(f"hello
5656
def bar():
57-
pass
58-
59-
60-
# There are trailing whitespace before the newline character but those whitespaces are
61-
# part of the comment token
62-
f"""hello {x # comment
63-
y = 1
57+
pass

‎crates/ruff_python_parser/resources/invalid/re_lexing/triple_quoted_fstring_1.py

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# There are trailing whitespace before the newline character but those whitespaces are
2+
# part of the comment token.
3+
# https://github.com/astral-sh/ruff/issues/11929
4+
5+
f"""hello {x # comment
6+
y = 1

‎crates/ruff_python_parser/resources/invalid/re_lexing/triple_quoted_fstring_2.py

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# The lexer can't be moved back for a triple-quoted f-string because the newlines are
2+
# part of the f-string itself.
3+
# https://github.com/astral-sh/ruff/issues/11937
4+
5+
f'''{foo:.3f
6+
'''

‎crates/ruff_python_parser/resources/invalid/re_lexing/triple_quoted_fstring_3.py

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# Here, the nesting level is 2 when the parser is trying to recover from an unclosed `{`
2+
# This test demonstrates that we need to reduce the nesting level when recovering from
3+
# within an f-string but the lexer shouldn't go back.
4+
5+
if call(f'''{x:.3f
6+
'''
7+
pass

‎crates/ruff_python_parser/src/lexer.rs

+11
Original file line numberDiff line numberDiff line change
@@ -1370,6 +1370,12 @@ impl<'src> Lexer<'src> {
13701370
// i.e., it recovered from an unclosed parenthesis (`(`, `[`, or `{`).
13711371
self.nesting -= 1;
13721372

1373+
// The lexer can't be moved back for a triple-quoted f-string because the newlines are
1374+
// part of the f-string itself, so there is no newline token to be emitted.
1375+
if self.current_flags.is_triple_quoted_fstring() {
1376+
return false;
1377+
}
1378+
13731379
let mut current_position = self.current_range().start();
13741380
let reverse_chars = self.source[..current_position.to_usize()].chars().rev();
13751381
let mut newline_position = None;
@@ -1578,6 +1584,11 @@ impl TokenFlags {
15781584
self.intersects(TokenFlags::F_STRING)
15791585
}
15801586

1587+
/// Returns `true` if the token is a triple-quoted f-string.
1588+
fn is_triple_quoted_fstring(self) -> bool {
1589+
self.contains(TokenFlags::F_STRING | TokenFlags::TRIPLE_QUOTED_STRING)
1590+
}
1591+
15811592
/// Returns `true` if the token is a raw string.
15821593
const fn is_raw_string(self) -> bool {
15831594
self.intersects(TokenFlags::RAW_STRING)

‎crates/ruff_python_parser/tests/snapshots/invalid_syntax@f_string_unclosed_lbrace.py.snap

+60-60
Original file line numberDiff line numberDiff line change
@@ -139,64 +139,74 @@ Module(
139139
),
140140
Expr(
141141
StmtExpr {
142-
range: 24..29,
142+
range: 24..37,
143143
value: FString(
144144
ExprFString {
145-
range: 24..29,
145+
range: 24..37,
146146
value: FStringValue {
147-
inner: Single(
148-
FString(
149-
FString {
150-
range: 24..29,
151-
elements: [
152-
Expression(
153-
FStringExpressionElement {
154-
range: 26..27,
155-
expression: Name(
156-
ExprName {
157-
range: 27..27,
158-
id: "",
159-
ctx: Invalid,
160-
},
161-
),
162-
debug_text: None,
163-
conversion: None,
164-
format_spec: None,
165-
},
166-
),
167-
],
168-
flags: FStringFlags {
169-
quote_style: Double,
170-
prefix: Regular,
171-
triple_quoted: false,
147+
inner: Concatenated(
148+
[
149+
FString(
150+
FString {
151+
range: 24..29,
152+
elements: [
153+
Expression(
154+
FStringExpressionElement {
155+
range: 26..27,
156+
expression: Name(
157+
ExprName {
158+
range: 27..27,
159+
id: "",
160+
ctx: Invalid,
161+
},
162+
),
163+
debug_text: None,
164+
conversion: None,
165+
format_spec: None,
166+
},
167+
),
168+
],
169+
flags: FStringFlags {
170+
quote_style: Double,
171+
prefix: Regular,
172+
triple_quoted: false,
173+
},
172174
},
173-
},
174-
),
175+
),
176+
FString(
177+
FString {
178+
range: 29..37,
179+
elements: [
180+
Expression(
181+
FStringExpressionElement {
182+
range: 33..34,
183+
expression: Name(
184+
ExprName {
185+
range: 34..34,
186+
id: "",
187+
ctx: Invalid,
188+
},
189+
),
190+
debug_text: None,
191+
conversion: None,
192+
format_spec: None,
193+
},
194+
),
195+
],
196+
flags: FStringFlags {
197+
quote_style: Double,
198+
prefix: Regular,
199+
triple_quoted: true,
200+
},
201+
},
202+
),
203+
],
175204
),
176205
},
177206
},
178207
),
179208
},
180209
),
181-
Expr(
182-
StmtExpr {
183-
range: 33..38,
184-
value: Set(
185-
ExprSet {
186-
range: 33..38,
187-
elts: [
188-
Name(
189-
ExprName {
190-
range: 34..34,
191-
id: "",
192-
ctx: Invalid,
193-
},
194-
),
195-
],
196-
},
197-
),
198-
},
199-
),
200210
],
201211
},
202212
)
@@ -318,29 +328,19 @@ Module(
318328
|
319329

320330

321-
|
322-
2 | f"{foo!r"
323-
3 | f"{foo="
324-
4 | f"{"
325-
| _____^
326-
5 | | f"""{"""
327-
| |_^ Syntax Error: Expected FStringEnd, found FStringMiddle
328-
|
329-
330-
331331
|
332332
3 | f"{foo="
333333
4 | f"{"
334334
5 | f"""{"""
335-
| ^^^ Syntax Error: Expected a statement
335+
| ^^^^ Syntax Error: Expected FStringEnd, found FStringStart
336336
|
337337

338338

339339
|
340340
3 | f"{foo="
341341
4 | f"{"
342342
5 | f"""{"""
343-
|______^
343+
| ^^^ Syntax Error: Expected an expression
344344
|
345345

346346

‎crates/ruff_python_parser/tests/snapshots/invalid_syntax@re_lex_logical_token.py.snap

+5-89
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ input_file: crates/ruff_python_parser/resources/invalid/re_lex_logical_token.py
77
```
88
Module(
99
ModModule {
10-
range: 0..1129,
10+
range: 0..979,
1111
body: [
1212
If(
1313
StmtIf {
@@ -670,53 +670,6 @@ Module(
670670
],
671671
},
672672
),
673-
Expr(
674-
StmtExpr {
675-
range: 1097..1109,
676-
value: FString(
677-
ExprFString {
678-
range: 1097..1109,
679-
value: FStringValue {
680-
inner: Single(
681-
FString(
682-
FString {
683-
range: 1097..1109,
684-
elements: [
685-
Literal(
686-
FStringLiteralElement {
687-
range: 1101..1107,
688-
value: "hello ",
689-
},
690-
),
691-
Expression(
692-
FStringExpressionElement {
693-
range: 1107..1109,
694-
expression: Name(
695-
ExprName {
696-
range: 1108..1109,
697-
id: "x",
698-
ctx: Load,
699-
},
700-
),
701-
debug_text: None,
702-
conversion: None,
703-
format_spec: None,
704-
},
705-
),
706-
],
707-
flags: FStringFlags {
708-
quote_style: Double,
709-
prefix: Regular,
710-
triple_quoted: true,
711-
},
712-
},
713-
),
714-
),
715-
},
716-
},
717-
),
718-
},
719-
),
720673
],
721674
},
722675
)
@@ -878,45 +831,8 @@ Module(
878831

879832

880833
|
881-
60 | # There are trailing whitespace before the newline character but those whitespaces are
882-
61 | # part of the comment token
883-
62 | f"""hello {x # comment
884-
| Syntax Error: Expected a statement
885-
63 | y = 1
886-
|
887-
888-
889-
|
890-
60 | # There are trailing whitespace before the newline character but those whitespaces are
891-
61 | # part of the comment token
892-
62 | f"""hello {x # comment
893-
| ___________________________^
894-
63 | | y = 1
895-
| |_____^ Syntax Error: f-string: unterminated triple-quoted string
896-
|
897-
898-
899-
|
900-
61 | # part of the comment token
901-
62 | f"""hello {x # comment
902-
63 | y = 1
903-
| ^ Syntax Error: f-string: expecting '}'
904-
|
905-
906-
907-
|
908-
60 | # There are trailing whitespace before the newline character but those whitespaces are
909-
61 | # part of the comment token
910-
62 | f"""hello {x # comment
911-
| ___________________________^
912-
63 | | y = 1
913-
| |_____^ Syntax Error: Expected FStringEnd, found Unknown
914-
|
915-
916-
917-
|
918-
61 | # part of the comment token
919-
62 | f"""hello {x # comment
920-
63 | y = 1
921-
| Syntax Error: Expected a statement
834+
55 | if call(f"hello
835+
56 | def bar():
836+
57 | pass
837+
| Syntax Error: Expected a statement
922838
|

0 commit comments

Comments
 (0)