Skip to content

Commit 601c13c

Browse files
borsbors
committed
Auto merge of rust-lang#78677 - Aaron1011:fix/capture-inner-attrs, r=petrochenkov
Use reparsed `TokenStream` if we captured any inner attributes Fixes rust-lang#78675 We now bail out of `prepend_attrs` if we ended up capturing any inner attributes (which can happen in several places, due to token capturing for `macro_rules!` arguments.
2 parents 5629309 + 22383b3 commit 601c13c

File tree

4 files changed

+132
-29
lines changed

4 files changed

+132
-29
lines changed

‎compiler/rustc_parse/src/lib.rs

+5-5
Original file line numberDiff line numberDiff line change
@@ -611,11 +611,11 @@ fn prepend_attrs(
611611
}
612612
let mut builder = tokenstream::TokenStreamBuilder::new();
613613
for attr in attrs {
614-
assert_eq!(
615-
attr.style,
616-
ast::AttrStyle::Outer,
617-
"inner attributes should prevent cached tokens from existing"
618-
);
614+
// FIXME: Correctly handle tokens for inner attributes.
615+
// For now, we fall back to reparsing the original AST node
616+
if attr.style == ast::AttrStyle::Inner {
617+
return None;
618+
}
619619
builder.push(
620620
attr.tokens
621621
.as_ref()

‎compiler/rustc_parse/src/parser/item.rs

+9-24
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ use crate::maybe_whole;
77
use rustc_ast::ptr::P;
88
use rustc_ast::token::{self, TokenKind};
99
use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree};
10-
use rustc_ast::{self as ast, AttrStyle, AttrVec, Attribute, DUMMY_NODE_ID};
10+
use rustc_ast::{self as ast, AttrVec, Attribute, DUMMY_NODE_ID};
1111
use rustc_ast::{AssocItem, AssocItemKind, ForeignItemKind, Item, ItemKind, Mod};
1212
use rustc_ast::{Async, Const, Defaultness, IsAuto, Mutability, Unsafe, UseTree, UseTreeKind};
1313
use rustc_ast::{BindingMode, Block, FnDecl, FnSig, Param, SelfKind};
@@ -127,34 +127,19 @@ impl<'a> Parser<'a> {
127127

128128
let (mut item, tokens) = if needs_tokens {
129129
let (item, tokens) = self.collect_tokens(parse_item)?;
130-
(item, Some(tokens))
130+
(item, tokens)
131131
} else {
132132
(parse_item(self)?, None)
133133
};
134-
135-
self.unclosed_delims.append(&mut unclosed_delims);
136-
137-
// Once we've parsed an item and recorded the tokens we got while
138-
// parsing we may want to store `tokens` into the item we're about to
139-
// return. Note, though, that we specifically didn't capture tokens
140-
// related to outer attributes. The `tokens` field here may later be
141-
// used with procedural macros to convert this item back into a token
142-
// stream, but during expansion we may be removing attributes as we go
143-
// along.
144-
//
145-
// If we've got inner attributes then the `tokens` we've got above holds
146-
// these inner attributes. If an inner attribute is expanded we won't
147-
// actually remove it from the token stream, so we'll just keep yielding
148-
// it (bad!). To work around this case for now we just avoid recording
149-
// `tokens` if we detect any inner attributes. This should help keep
150-
// expansion correct, but we should fix this bug one day!
151-
if let Some(tokens) = tokens {
152-
if let Some(item) = &mut item {
153-
if !item.attrs.iter().any(|attr| attr.style == AttrStyle::Inner) {
154-
item.tokens = tokens;
155-
}
134+
if let Some(item) = &mut item {
135+
// If we captured tokens during parsing (due to encountering an `NtItem`),
136+
// use those instead
137+
if item.tokens.is_none() {
138+
item.tokens = tokens;
156139
}
157140
}
141+
142+
self.unclosed_delims.append(&mut unclosed_delims);
158143
Ok(item)
159144
}
160145

‎src/test/ui/proc-macro/issue-78675-captured-inner-attrs.rs

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
// check-pass
2+
// edition:2018
3+
// compile-flags: -Z span-debug
4+
// aux-build:test-macros.rs
5+
6+
#![no_std] // Don't load unnecessary hygiene information from std
7+
extern crate std;
8+
9+
#[macro_use] extern crate test_macros;
10+
11+
macro_rules! foo {(
12+
#[fake_attr]
13+
$item:item
14+
) => (
15+
$item
16+
)}
17+
18+
macro_rules! outer {($item:item) => (
19+
print_bang! { // Identity proc-macro
20+
foo! {
21+
#[fake_attr]
22+
$item
23+
}
24+
}
25+
)}
26+
outer! {
27+
mod bar {
28+
//! Foo
29+
}
30+
}
31+
32+
fn main() {}

‎src/test/ui/proc-macro/issue-78675-captured-inner-attrs.stdout

+86
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
PRINT-BANG INPUT (DISPLAY): foo ! { #[fake_attr] mod bar {
2+
#![doc = r" Foo"]
3+
} }
4+
PRINT-BANG INPUT (DEBUG): TokenStream [
5+
Ident {
6+
ident: "foo",
7+
span: $DIR/issue-78675-captured-inner-attrs.rs:20:9: 20:12 (#4),
8+
},
9+
Punct {
10+
ch: '!',
11+
spacing: Alone,
12+
span: $DIR/issue-78675-captured-inner-attrs.rs:20:12: 20:13 (#4),
13+
},
14+
Group {
15+
delimiter: Brace,
16+
stream: TokenStream [
17+
Punct {
18+
ch: '#',
19+
spacing: Alone,
20+
span: $DIR/issue-78675-captured-inner-attrs.rs:21:13: 21:14 (#4),
21+
},
22+
Group {
23+
delimiter: Bracket,
24+
stream: TokenStream [
25+
Ident {
26+
ident: "fake_attr",
27+
span: $DIR/issue-78675-captured-inner-attrs.rs:21:15: 21:24 (#4),
28+
},
29+
],
30+
span: $DIR/issue-78675-captured-inner-attrs.rs:21:14: 21:25 (#4),
31+
},
32+
Group {
33+
delimiter: None,
34+
stream: TokenStream [
35+
Ident {
36+
ident: "mod",
37+
span: $DIR/issue-78675-captured-inner-attrs.rs:22:13: 22:18 (#4),
38+
},
39+
Ident {
40+
ident: "bar",
41+
span: $DIR/issue-78675-captured-inner-attrs.rs:22:13: 22:18 (#4),
42+
},
43+
Group {
44+
delimiter: Brace,
45+
stream: TokenStream [
46+
Punct {
47+
ch: '#',
48+
spacing: Joint,
49+
span: $DIR/issue-78675-captured-inner-attrs.rs:22:13: 22:18 (#4),
50+
},
51+
Punct {
52+
ch: '!',
53+
spacing: Alone,
54+
span: $DIR/issue-78675-captured-inner-attrs.rs:22:13: 22:18 (#4),
55+
},
56+
Group {
57+
delimiter: Bracket,
58+
stream: TokenStream [
59+
Ident {
60+
ident: "doc",
61+
span: $DIR/issue-78675-captured-inner-attrs.rs:22:13: 22:18 (#4),
62+
},
63+
Punct {
64+
ch: '=',
65+
spacing: Alone,
66+
span: $DIR/issue-78675-captured-inner-attrs.rs:22:13: 22:18 (#4),
67+
},
68+
Literal {
69+
kind: StrRaw(0),
70+
symbol: " Foo",
71+
suffix: None,
72+
span: $DIR/issue-78675-captured-inner-attrs.rs:22:13: 22:18 (#4),
73+
},
74+
],
75+
span: $DIR/issue-78675-captured-inner-attrs.rs:22:13: 22:18 (#4),
76+
},
77+
],
78+
span: $DIR/issue-78675-captured-inner-attrs.rs:22:13: 22:18 (#4),
79+
},
80+
],
81+
span: $DIR/issue-78675-captured-inner-attrs.rs:22:13: 22:18 (#4),
82+
},
83+
],
84+
span: $DIR/issue-78675-captured-inner-attrs.rs:20:14: 23:10 (#4),
85+
},
86+
]

0 commit comments

Comments
 (0)