Skip to content
This repository was archived by the owner on May 28, 2025. It is now read-only.

Commit 7f12344

Browse files
borsbors
committed
Auto merge of rust-lang#14084 - Veykril:float-parse, r=Veykril
fix: Fix parsing of nested tuple field accesses in a cursed way This is absolutely terrible but seems to work. Macro fragment parsing comes next.
2 parents 57ea982 + a756c9a commit 7f12344

20 files changed

+447
-63
lines changed

‎crates/hir-def/src/item_tree.rs

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,8 @@ impl ItemTree {
111111
Some(node) => node,
112112
None => return Default::default(),
113113
};
114-
if never!(syntax.kind() == SyntaxKind::ERROR) {
114+
if never!(syntax.kind() == SyntaxKind::ERROR, "{:?} from {:?} {}", file_id, syntax, syntax)
115+
{
115116
// FIXME: not 100% sure why these crop up, but return an empty tree to avoid a panic
116117
return Default::default();
117118
}
@@ -133,7 +134,7 @@ impl ItemTree {
133134
ctx.lower_macro_stmts(stmts)
134135
},
135136
_ => {
136-
panic!("cannot create item tree from {syntax:?} {syntax}");
137+
panic!("cannot create item tree for file {file_id:?} from {syntax:?} {syntax}");
137138
},
138139
}
139140
};

‎crates/hir-def/src/macro_expansion_tests/mbe.rs

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,41 @@ fn#19 main#20(#21)#21 {#22
9797
"##]],
9898
);
9999
}
100+
#[test]
101+
fn float_field_acces_macro_input() {
102+
check(
103+
r#"
104+
macro_rules! foo {
105+
($expr:expr) => {
106+
fn foo() {
107+
$expr;
108+
}
109+
};
110+
}
111+
foo!(x .0.1);
112+
foo!(x .2. 3);
113+
foo!(x .4 .5);
114+
"#,
115+
expect![[r#"
116+
macro_rules! foo {
117+
($expr:expr) => {
118+
fn foo() {
119+
$expr;
120+
}
121+
};
122+
}
123+
fn foo() {
124+
(x.0.1);
125+
}
126+
fn foo() {
127+
(x.2.3);
128+
}
129+
fn foo() {
130+
(x.4.5);
131+
}
132+
"#]],
133+
);
134+
}
100135

101136
#[test]
102137
fn mbe_smoke_test() {

‎crates/hir-def/src/macro_expansion_tests/proc_macros.rs

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ macro_rules! id {
104104
$($t)*
105105
};
106106
}
107-
id /*+errors*/! {
107+
id! {
108108
#[proc_macros::identity]
109109
impl Foo for WrapBj {
110110
async fn foo(&self) {
@@ -113,18 +113,17 @@ id /*+errors*/! {
113113
}
114114
}
115115
"#,
116-
expect![[r##"
116+
expect![[r#"
117117
macro_rules! id {
118118
($($t:tt)*) => {
119119
$($t)*
120120
};
121121
}
122-
/* parse error: expected SEMICOLON */
123122
#[proc_macros::identity] impl Foo for WrapBj {
124123
async fn foo(&self ) {
125124
self .0.id().await ;
126125
}
127126
}
128-
"##]],
127+
"#]],
129128
);
130129
}

‎crates/mbe/src/syntax_bridge.rs

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,9 @@ pub fn token_tree_to_syntax_node(
9595
parser::Step::Token { kind, n_input_tokens: n_raw_tokens } => {
9696
tree_sink.token(kind, n_raw_tokens)
9797
}
98+
parser::Step::FloatSplit { ends_in_dot: has_pseudo_dot } => {
99+
tree_sink.float_split(has_pseudo_dot)
100+
}
98101
parser::Step::Enter { kind } => tree_sink.start_node(kind),
99102
parser::Step::Exit => tree_sink.finish_node(),
100103
parser::Step::Error { msg } => tree_sink.error(msg.to_string()),
@@ -796,6 +799,43 @@ fn delim_to_str(d: tt::DelimiterKind, closing: bool) -> Option<&'static str> {
796799
}
797800

798801
impl<'a> TtTreeSink<'a> {
802+
/// Parses a float literal as if it was a one to two name ref nodes with a dot inbetween.
803+
/// This occurs when a float literal is used as a field access.
804+
fn float_split(&mut self, has_pseudo_dot: bool) {
805+
let (text, _span) = match self.cursor.token_tree() {
806+
Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Literal(lit), _)) => {
807+
(lit.text.as_str(), lit.span)
808+
}
809+
_ => unreachable!(),
810+
};
811+
match text.split_once('.') {
812+
Some((left, right)) => {
813+
assert!(!left.is_empty());
814+
self.inner.start_node(SyntaxKind::NAME_REF);
815+
self.inner.token(SyntaxKind::INT_NUMBER, left);
816+
self.inner.finish_node();
817+
818+
// here we move the exit up, the original exit has been deleted in process
819+
self.inner.finish_node();
820+
821+
self.inner.token(SyntaxKind::DOT, ".");
822+
823+
if has_pseudo_dot {
824+
assert!(right.is_empty(), "{left}.{right}");
825+
} else {
826+
self.inner.start_node(SyntaxKind::NAME_REF);
827+
self.inner.token(SyntaxKind::INT_NUMBER, right);
828+
self.inner.finish_node();
829+
830+
// the parser creates an unbalanced start node, we are required to close it here
831+
self.inner.finish_node();
832+
}
833+
}
834+
None => unreachable!(),
835+
}
836+
self.cursor = self.cursor.bump();
837+
}
838+
799839
fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) {
800840
if kind == LIFETIME_IDENT {
801841
n_tokens = 2;

‎crates/mbe/src/to_parser_input.rs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,13 @@ pub(crate) fn to_parser_input(buffer: &TokenBuffer<'_>) -> parser::Input {
4545
.unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &lit));
4646

4747
res.push(kind);
48+
49+
if kind == FLOAT_NUMBER && !inner_text.ends_with('.') {
50+
// Tag the token as joint if it is float with a fractional part
51+
// we use this jointness to inform the parser about what token split
52+
// event to emit when we encounter a float literal in a field access
53+
res.was_joint();
54+
}
4855
}
4956
tt::Leaf::Ident(ident) => match ident.text.as_ref() {
5057
"_" => res.push(T![_]),

‎crates/mbe/src/tt_iter.rs

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,11 @@ impl<'a> TtIter<'a> {
150150
cursor = cursor.bump_subtree();
151151
}
152152
}
153+
parser::Step::FloatSplit { .. } => {
154+
// FIXME: We need to split the tree properly here, but mutating the token trees
155+
// in the buffer is somewhat tricky to pull off.
156+
cursor = cursor.bump_subtree();
157+
}
153158
parser::Step::Enter { .. } | parser::Step::Exit => (),
154159
parser::Step::Error { .. } => error = true,
155160
}
@@ -166,19 +171,18 @@ impl<'a> TtIter<'a> {
166171

167172
if cursor.is_root() {
168173
while curr != cursor {
169-
if let Some(token) = curr.token_tree() {
170-
res.push(token);
171-
}
174+
let Some(token) = curr.token_tree() else { break };
175+
res.push(token.cloned());
172176
curr = curr.bump();
173177
}
174178
}
179+
175180
self.inner = self.inner.as_slice()[res.len()..].iter();
176181
let res = match res.len() {
177-
1 => Some(res[0].cloned()),
178-
0 => None,
182+
0 | 1 => res.pop(),
179183
_ => Some(tt::TokenTree::Subtree(tt::Subtree {
180184
delimiter: tt::Delimiter::unspecified(),
181-
token_trees: res.into_iter().map(|it| it.cloned()).collect(),
185+
token_trees: res,
182186
})),
183187
};
184188
ExpandResult { value: res, err }

‎crates/parser/src/event.rs

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,13 @@ pub(crate) enum Event {
7474
kind: SyntaxKind,
7575
n_raw_tokens: u8,
7676
},
77-
77+
/// When we parse `foo.0.0` or `foo. 0. 0` the lexer will hand us a float literal
78+
/// instead of an integer literal followed by a dot as the lexer has no contextual knowledge.
79+
/// This event instructs whatever consumes the events to split the float literal into
80+
/// the corresponding parts.
81+
FloatSplitHack {
82+
ends_in_dot: bool,
83+
},
7884
Error {
7985
msg: String,
8086
},
@@ -125,6 +131,11 @@ pub(super) fn process(mut events: Vec<Event>) -> Output {
125131
Event::Token { kind, n_raw_tokens } => {
126132
res.token(kind, n_raw_tokens);
127133
}
134+
Event::FloatSplitHack { ends_in_dot } => {
135+
res.float_split_hack(ends_in_dot);
136+
let ev = mem::replace(&mut events[i + 1], Event::tombstone());
137+
assert!(matches!(ev, Event::Finish), "{ev:?}");
138+
}
128139
Event::Error { msg } => res.error(msg),
129140
}
130141
}

‎crates/parser/src/grammar/expressions.rs

Lines changed: 66 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -379,7 +379,7 @@ fn postfix_expr(
379379
// }
380380
T!['('] if allow_calls => call_expr(p, lhs),
381381
T!['['] if allow_calls => index_expr(p, lhs),
382-
T![.] => match postfix_dot_expr(p, lhs) {
382+
T![.] => match postfix_dot_expr::<false>(p, lhs) {
383383
Ok(it) => it,
384384
Err(it) => {
385385
lhs = it;
@@ -393,35 +393,44 @@ fn postfix_expr(
393393
block_like = BlockLike::NotBlock;
394394
}
395395
return (lhs, block_like);
396+
}
396397

397-
fn postfix_dot_expr(
398-
p: &mut Parser<'_>,
399-
lhs: CompletedMarker,
400-
) -> Result<CompletedMarker, CompletedMarker> {
398+
fn postfix_dot_expr<const FLOAT_RECOVERY: bool>(
399+
p: &mut Parser<'_>,
400+
lhs: CompletedMarker,
401+
) -> Result<CompletedMarker, CompletedMarker> {
402+
if !FLOAT_RECOVERY {
401403
assert!(p.at(T![.]));
402-
if p.nth(1) == IDENT && (p.nth(2) == T!['('] || p.nth_at(2, T![::])) {
403-
return Ok(method_call_expr(p, lhs));
404-
}
404+
}
405+
let nth1 = if FLOAT_RECOVERY { 0 } else { 1 };
406+
let nth2 = if FLOAT_RECOVERY { 1 } else { 2 };
405407

406-
// test await_expr
407-
// fn foo() {
408-
// x.await;
409-
// x.0.await;
410-
// x.0().await?.hello();
411-
// }
412-
if p.nth(1) == T![await] {
413-
let m = lhs.precede(p);
414-
p.bump(T![.]);
415-
p.bump(T![await]);
416-
return Ok(m.complete(p, AWAIT_EXPR));
417-
}
408+
if p.nth(nth1) == IDENT && (p.nth(nth2) == T!['('] || p.nth_at(nth2, T![::])) {
409+
return Ok(method_call_expr::<FLOAT_RECOVERY>(p, lhs));
410+
}
418411

419-
if p.at(T![..=]) || p.at(T![..]) {
420-
return Err(lhs);
412+
// test await_expr
413+
// fn foo() {
414+
// x.await;
415+
// x.0.await;
416+
// x.0().await?.hello();
417+
// x.0.0.await;
418+
// x.0. await;
419+
// }
420+
if p.nth(nth1) == T![await] {
421+
let m = lhs.precede(p);
422+
if !FLOAT_RECOVERY {
423+
p.bump(T![.]);
421424
}
425+
p.bump(T![await]);
426+
return Ok(m.complete(p, AWAIT_EXPR));
427+
}
422428

423-
Ok(field_expr(p, lhs))
429+
if p.at(T![..=]) || p.at(T![..]) {
430+
return Err(lhs);
424431
}
432+
433+
field_expr::<FLOAT_RECOVERY>(p, lhs)
425434
}
426435

427436
// test call_expr
@@ -455,11 +464,22 @@ fn index_expr(p: &mut Parser<'_>, lhs: CompletedMarker) -> CompletedMarker {
455464
// fn foo() {
456465
// x.foo();
457466
// y.bar::<T>(1, 2,);
467+
// x.0.0.call();
468+
// x.0. call();
458469
// }
459-
fn method_call_expr(p: &mut Parser<'_>, lhs: CompletedMarker) -> CompletedMarker {
460-
assert!(p.at(T![.]) && p.nth(1) == IDENT && (p.nth(2) == T!['('] || p.nth_at(2, T![::])));
470+
fn method_call_expr<const FLOAT_RECOVERY: bool>(
471+
p: &mut Parser<'_>,
472+
lhs: CompletedMarker,
473+
) -> CompletedMarker {
474+
if FLOAT_RECOVERY {
475+
assert!(p.nth(0) == IDENT && (p.nth(1) == T!['('] || p.nth_at(1, T![::])));
476+
} else {
477+
assert!(p.at(T![.]) && p.nth(1) == IDENT && (p.nth(2) == T!['('] || p.nth_at(2, T![::])));
478+
}
461479
let m = lhs.precede(p);
462-
p.bump_any();
480+
if !FLOAT_RECOVERY {
481+
p.bump(T![.]);
482+
}
463483
name_ref(p);
464484
generic_args::opt_generic_arg_list(p, true);
465485
if p.at(T!['(']) {
@@ -472,21 +492,35 @@ fn method_call_expr(p: &mut Parser<'_>, lhs: CompletedMarker) -> CompletedMarker
472492
// fn foo() {
473493
// x.foo;
474494
// x.0.bar;
495+
// x.0.1;
496+
// x.0. bar;
475497
// x.0();
476498
// }
477-
fn field_expr(p: &mut Parser<'_>, lhs: CompletedMarker) -> CompletedMarker {
478-
assert!(p.at(T![.]));
499+
fn field_expr<const FLOAT_RECOVERY: bool>(
500+
p: &mut Parser<'_>,
501+
lhs: CompletedMarker,
502+
) -> Result<CompletedMarker, CompletedMarker> {
503+
if !FLOAT_RECOVERY {
504+
assert!(p.at(T![.]));
505+
}
479506
let m = lhs.precede(p);
480-
p.bump(T![.]);
507+
if !FLOAT_RECOVERY {
508+
p.bump(T![.]);
509+
}
481510
if p.at(IDENT) || p.at(INT_NUMBER) {
482511
name_ref_or_index(p);
483512
} else if p.at(FLOAT_NUMBER) {
484-
// FIXME: How to recover and instead parse INT + T![.]?
485-
p.bump_any();
513+
return match p.split_float(m) {
514+
(true, m) => {
515+
let lhs = m.complete(p, FIELD_EXPR);
516+
postfix_dot_expr::<true>(p, lhs)
517+
}
518+
(false, m) => Ok(m.complete(p, FIELD_EXPR)),
519+
};
486520
} else {
487521
p.error("expected field name or number");
488522
}
489-
m.complete(p, FIELD_EXPR)
523+
Ok(m.complete(p, FIELD_EXPR))
490524
}
491525

492526
// test try_expr

‎crates/parser/src/lib.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,10 +102,14 @@ impl TopEntryPoint {
102102
match step {
103103
Step::Enter { .. } => depth += 1,
104104
Step::Exit => depth -= 1,
105+
Step::FloatSplit { ends_in_dot: has_pseudo_dot } => {
106+
depth -= 1 + !has_pseudo_dot as usize
107+
}
105108
Step::Token { .. } | Step::Error { .. } => (),
106109
}
107110
}
108111
assert!(!first, "no tree at all");
112+
assert_eq!(depth, 0, "unbalanced tree");
109113
}
110114

111115
res

0 commit comments

Comments
 (0)