Skip to content

Commit 23fd492

Browse files
Glyphackdcreager
Glyphack
and
dcreager
authored
Auto generate ast expression nodes (#16285)
## Summary Part of #15655 - Auto generate AST nodes using definitions in `ast.toml`. I added attributes similar to [`Field`](https://github.com/python/cpython/blob/main/Parser/asdl.py#L67) in ASDL to hold field information ## Test Plan Nothing outside the `ruff_python_ast` package should change. --------- Co-authored-by: Douglas Creager <[email protected]>
1 parent cc324ab commit 23fd492

File tree

6 files changed

+720
-361
lines changed

6 files changed

+720
-361
lines changed

‎crates/ruff_python_ast/ast.toml

Lines changed: 263 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -26,18 +26,36 @@
2626
# Controls the name of the AnyNodeRef::is_foo_bar method. The default is the
2727
# group name in snake_case.
2828
#
29-
# rustdoc:
30-
# A rustdoc comment that is added to the group's enums.
29+
# doc:
30+
# A doc comment that is added to the group's enums.
3131
#
3232
# The following syntax node options are available:
3333
#
34+
# doc:
35+
# A doc comment that is added to the syntax node struct.
36+
#
37+
# derives:
38+
# List of derives to add to the syntax node struct. Clone, Debug, PartialEq are added by default.
39+
#
40+
# fields:
41+
# List of fields in the syntax node struct. Each field is a table with the
42+
# following keys:
43+
# * name - The name of the field.
44+
# * type - The type of the field. This can be a simple type name, or a type
45+
# type field uses a special syntax to describe the rust type:
46+
# * `Expr` - A single expression.
47+
# * `Expr?` - Option type.
48+
# * `Expr*` - A vector of Expr.
49+
# * `&Expr*` - A boxed slice of Expr.
50+
# These properties cannot be nested, for example we cannot create a vector of option types.
51+
#
3452
# variant:
3553
# The name of the enum variant for this syntax node. Defaults to the node
3654
# name with the group prefix removed. (That is, `StmtIf` becomes `If`.)
3755

3856
[Mod]
3957
anynode_is_label = "module"
40-
rustdoc = "/// See also [mod](https://docs.python.org/3/library/ast.html#ast.mod)"
58+
doc = "See also [mod](https://docs.python.org/3/library/ast.html#ast.mod)"
4159

4260
[Mod.nodes]
4361
ModModule = {}
@@ -46,7 +64,7 @@ ModExpression = {}
4664
[Stmt]
4765
add_suffix_to_is_methods = true
4866
anynode_is_label = "statement"
49-
rustdoc = "/// See also [stmt](https://docs.python.org/3/library/ast.html#ast.stmt)"
67+
doc = "See also [stmt](https://docs.python.org/3/library/ast.html#ast.stmt)"
5068

5169
[Stmt.nodes]
5270
StmtFunctionDef = {}
@@ -78,44 +96,247 @@ StmtIpyEscapeCommand = {}
7896
[Expr]
7997
add_suffix_to_is_methods = true
8098
anynode_is_label = "expression"
81-
rustdoc = "/// See also [expr](https://docs.python.org/3/library/ast.html#ast.expr)"
82-
83-
[Expr.nodes]
84-
ExprBoolOp = {}
85-
ExprNamed = {}
86-
ExprBinOp = {}
87-
ExprUnaryOp = {}
88-
ExprLambda = {}
89-
ExprIf = {}
90-
ExprDict = {}
91-
ExprSet = {}
92-
ExprListComp = {}
93-
ExprSetComp = {}
94-
ExprDictComp = {}
95-
ExprGenerator = {}
96-
ExprAwait = {}
97-
ExprYield = {}
98-
ExprYieldFrom = {}
99-
ExprCompare = {}
100-
ExprCall = {}
101-
ExprFString = {}
102-
ExprStringLiteral = {}
103-
ExprBytesLiteral = {}
104-
ExprNumberLiteral = {}
105-
ExprBooleanLiteral = {}
106-
ExprNoneLiteral = {}
107-
ExprEllipsisLiteral = {}
108-
ExprAttribute = {}
109-
ExprSubscript = {}
110-
ExprStarred = {}
111-
ExprName = {}
112-
ExprList = {}
113-
ExprTuple = {}
114-
ExprSlice = {}
115-
ExprIpyEscapeCommand = {}
99+
doc = "See also [expr](https://docs.python.org/3/library/ast.html#ast.expr)"
100+
101+
[Expr.nodes.ExprBoolOp]
102+
doc = "See also [BoolOp](https://docs.python.org/3/library/ast.html#ast.BoolOp)"
103+
fields = [
104+
{ name = "op", type = "BoolOp" },
105+
{ name = "values", type = "Expr*" }
106+
]
107+
108+
[Expr.nodes.ExprNamed]
109+
doc = "See also [NamedExpr](https://docs.python.org/3/library/ast.html#ast.NamedExpr)"
110+
fields = [
111+
{ name = "target", type = "Expr" },
112+
{ name = "value", type = "Expr" }
113+
]
114+
115+
[Expr.nodes.ExprBinOp]
116+
doc = "See also [BinOp](https://docs.python.org/3/library/ast.html#ast.BinOp)"
117+
fields = [
118+
{ name = "left", type = "Expr" },
119+
{ name = "op", type = "Operator" },
120+
{ name = "right", type = "Expr" }
121+
]
122+
123+
[Expr.nodes.ExprUnaryOp]
124+
doc = "See also [UnaryOp](https://docs.python.org/3/library/ast.html#ast.UnaryOp)"
125+
fields = [
126+
{ name = "op", type = "UnaryOp" },
127+
{ name = "operand", type = "Expr" }
128+
]
129+
130+
[Expr.nodes.ExprLambda]
131+
doc = "See also [Lambda](https://docs.python.org/3/library/ast.html#ast.Lambda)"
132+
fields = [
133+
{ name = "parameters", type = "Box<crate::Parameters>?" },
134+
{ name = "body", type = "Expr" }
135+
]
136+
137+
[Expr.nodes.ExprIf]
138+
doc = "See also [IfExp](https://docs.python.org/3/library/ast.html#ast.IfExp)"
139+
fields = [
140+
{ name = "test", type = "Expr" },
141+
{ name = "body", type = "Expr" },
142+
{ name = "orelse", type = "Expr" }
143+
]
144+
145+
[Expr.nodes.ExprDict]
146+
doc = "See also [Dict](https://docs.python.org/3/library/ast.html#ast.Dict)"
147+
fields = [
148+
{ name = "items", type = "DictItem*" }
149+
]
150+
151+
[Expr.nodes.ExprSet]
152+
doc = "See also [Set](https://docs.python.org/3/library/ast.html#ast.Set)"
153+
fields = [
154+
{ name = "elts", type = "Expr*" }
155+
]
156+
157+
[Expr.nodes.ExprListComp]
158+
doc = "See also [ListComp](https://docs.python.org/3/library/ast.html#ast.ListComp)"
159+
fields = [
160+
{ name = "elt", type = "Expr" },
161+
{ name = "generators", type = "Comprehension*" }
162+
]
163+
164+
[Expr.nodes.ExprSetComp]
165+
doc = "See also [SetComp](https://docs.python.org/3/library/ast.html#ast.SetComp)"
166+
fields = [
167+
{ name = "elt", type = "Expr" },
168+
{ name = "generators", type = "Comprehension*" }
169+
]
170+
171+
[Expr.nodes.ExprDictComp]
172+
doc = "See also [DictComp](https://docs.python.org/3/library/ast.html#ast.DictComp)"
173+
fields = [
174+
{ name = "key", type = "Expr" },
175+
{ name = "value", type = "Expr" },
176+
{ name = "generators", type = "Comprehension*" }
177+
]
178+
179+
[Expr.nodes.ExprGenerator]
180+
doc = "See also [GeneratorExp](https://docs.python.org/3/library/ast.html#ast.GeneratorExp)"
181+
fields = [
182+
{ name = "elt", type = "Expr" },
183+
{ name = "generators", type = "Comprehension*" },
184+
{ name = "parenthesized", type = "bool" }
185+
]
186+
187+
[Expr.nodes.ExprAwait]
188+
doc = "See also [Await](https://docs.python.org/3/library/ast.html#ast.Await)"
189+
fields = [
190+
{ name = "value", type = "Expr" }
191+
]
192+
193+
[Expr.nodes.ExprYield]
194+
doc = "See also [Yield](https://docs.python.org/3/library/ast.html#ast.Yield)"
195+
fields = [
196+
{ name = "value", type = "Expr?" }
197+
]
198+
199+
[Expr.nodes.ExprYieldFrom]
200+
doc = "See also [YieldFrom](https://docs.python.org/3/library/ast.html#ast.YieldFrom)"
201+
fields = [
202+
{ name = "value", type = "Expr" }
203+
]
204+
205+
[Expr.nodes.ExprCompare]
206+
doc = "See also [Compare](https://docs.python.org/3/library/ast.html#ast.Compare)"
207+
fields = [
208+
{ name = "left", type = "Expr" },
209+
{ name = "ops", type = "&CmpOp*" },
210+
{ name = "comparators", type = "&Expr*" }
211+
]
212+
213+
[Expr.nodes.ExprCall]
214+
doc = "See also [Call](https://docs.python.org/3/library/ast.html#ast.Call)"
215+
fields = [
216+
{ name = "func", type = "Expr" },
217+
{ name = "arguments", type = "Arguments" }
218+
]
219+
220+
[Expr.nodes.ExprFString]
221+
doc = """An AST node that represents either a single-part f-string literal
222+
or an implicitly concatenated f-string literal.
223+
224+
This type differs from the original Python AST `JoinedStr` in that it
225+
doesn't join the implicitly concatenated parts into a single string. Instead,
226+
it keeps them separate and provide various methods to access the parts.
227+
228+
See also [JoinedStr](https://docs.python.org/3/library/ast.html#ast.JoinedStr)"""
229+
fields = [
230+
{ name = "value", type = "FStringValue" }
231+
]
232+
233+
[Expr.nodes.ExprStringLiteral]
234+
doc = """An AST node that represents either a single-part string literal
235+
or an implicitly concatenated string literal."""
236+
fields = [
237+
{ name = "value", type = "StringLiteralValue" }
238+
]
239+
240+
[Expr.nodes.ExprBytesLiteral]
241+
doc = """An AST node that represents either a single-part bytestring literal
242+
or an implicitly concatenated bytestring literal."""
243+
fields = [
244+
{ name = "value", type = "BytesLiteralValue" }
245+
]
246+
247+
[Expr.nodes.ExprNumberLiteral]
248+
fields = [
249+
{ name = "value", type = "Number" }
250+
]
251+
252+
[Expr.nodes.ExprBooleanLiteral]
253+
fields = [
254+
{ name = "value", type = "bool" }
255+
]
256+
derives = ["Default"]
257+
258+
[Expr.nodes.ExprNoneLiteral]
259+
fields = []
260+
derives = ["Default"]
261+
262+
[Expr.nodes.ExprEllipsisLiteral]
263+
fields = []
264+
derives = ["Default"]
265+
266+
[Expr.nodes.ExprAttribute]
267+
doc = "See also [Attribute](https://docs.python.org/3/library/ast.html#ast.Attribute)"
268+
fields = [
269+
{ name = "value", type = "Expr" },
270+
{ name = "attr", type = "Identifier" },
271+
{ name = "ctx", type = "ExprContext" }
272+
]
273+
274+
[Expr.nodes.ExprSubscript]
275+
doc = "See also [Subscript](https://docs.python.org/3/library/ast.html#ast.Subscript)"
276+
fields = [
277+
{ name = "value", type = "Expr" },
278+
{ name = "slice", type = "Expr" },
279+
{ name = "ctx", type = "ExprContext" }
280+
]
281+
282+
[Expr.nodes.ExprStarred]
283+
doc = "See also [Starred](https://docs.python.org/3/library/ast.html#ast.Starred)"
284+
fields = [
285+
{ name = "value", type = "Expr" },
286+
{ name = "ctx", type = "ExprContext" }
287+
]
288+
289+
[Expr.nodes.ExprName]
290+
doc = "See also [Name](https://docs.python.org/3/library/ast.html#ast.Name)"
291+
fields = [
292+
{ name = "id", type = "Name" },
293+
{ name = "ctx", type = "ExprContext" }
294+
]
295+
296+
[Expr.nodes.ExprList]
297+
doc = "See also [List](https://docs.python.org/3/library/ast.html#ast.List)"
298+
fields = [
299+
{ name = "elts", type = "Expr*" },
300+
{ name = "ctx", type = "ExprContext" }
301+
]
302+
303+
[Expr.nodes.ExprTuple]
304+
doc = "See also [Tuple](https://docs.python.org/3/library/ast.html#ast.Tuple)"
305+
fields = [
306+
{ name = "elts", type = "Expr*" },
307+
{ name = "ctx", type = "ExprContext" },
308+
{ name = "parenthesized", type = "bool" }
309+
]
310+
311+
[Expr.nodes.ExprSlice]
312+
doc = "See also [Slice](https://docs.python.org/3/library/ast.html#ast.Slice)"
313+
fields = [
314+
{ name = "lower", type = "Expr?" },
315+
{ name = "upper", type = "Expr?" },
316+
{ name = "step", type = "Expr?" }
317+
]
318+
319+
[Expr.nodes.ExprIpyEscapeCommand]
320+
# TODO: Remove the crate:: prefix once StmtIpyEscapeCommand is moved to generated.rs
321+
doc = """An AST node used to represent a IPython escape command at the expression level.
322+
323+
For example,
324+
```python
325+
dir = !pwd
326+
```
327+
328+
Here, the escape kind can only be `!` or `%` otherwise it is a syntax error.
329+
330+
For more information related to terminology and syntax of escape commands,
331+
see [`crate::StmtIpyEscapeCommand`]."""
332+
333+
fields = [
334+
{ name = "kind", type = "IpyEscapeKind" },
335+
{ name = "value", type = "Box<str>" }
336+
]
116337

117338
[ExceptHandler]
118-
rustdoc = "/// See also [excepthandler](https://docs.python.org/3/library/ast.html#ast.excepthandler)"
339+
doc = "See also [excepthandler](https://docs.python.org/3/library/ast.html#ast.excepthandler)"
119340

120341
[ExceptHandler.nodes]
121342
ExceptHandlerExceptHandler = {}
@@ -125,7 +346,7 @@ FStringExpressionElement = {variant = "Expression"}
125346
FStringLiteralElement = {variant = "Literal"}
126347

127348
[Pattern]
128-
rustdoc = "/// See also [pattern](https://docs.python.org/3/library/ast.html#ast.pattern)"
349+
doc = "See also [pattern](https://docs.python.org/3/library/ast.html#ast.pattern)"
129350

130351
[Pattern.nodes]
131352
PatternMatchValue = {}
@@ -138,7 +359,7 @@ PatternMatchAs = {}
138359
PatternMatchOr = {}
139360

140361
[TypeParam]
141-
rustdoc = "/// See also [type_param](https://docs.python.org/3/library/ast.html#ast.type_param)"
362+
doc = "See also [type_param](https://docs.python.org/3/library/ast.html#ast.type_param)"
142363

143364
[TypeParam.nodes]
144365
TypeParamTypeVar = {}

0 commit comments

Comments
 (0)