Skip to content

Commit b6d3be4

Browse files
borsbors
committed
Auto merge of #133436 - nnethercote:rm-NtVis-NtTy, r=petrochenkov
Remove `NtVis` and `NtTy` The next part of #124141. The first actual remove of `Nonterminal` variants. `NtVis` is a simple case that doesn't get much use, but `NtTy` is more complex. r? `@petrochenkov`
2 parents b87eda7 + 0f490b0 commit b6d3be4

20 files changed

+226
-73
lines changed

Diff for: ‎compiler/rustc_ast/src/ast_traits.rs

-4
Original file line numberDiff line numberDiff line change
@@ -203,10 +203,8 @@ impl HasTokens for Nonterminal {
203203
Nonterminal::NtStmt(stmt) => stmt.tokens(),
204204
Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens(),
205205
Nonterminal::NtPat(pat) => pat.tokens(),
206-
Nonterminal::NtTy(ty) => ty.tokens(),
207206
Nonterminal::NtMeta(attr_item) => attr_item.tokens(),
208207
Nonterminal::NtPath(path) => path.tokens(),
209-
Nonterminal::NtVis(vis) => vis.tokens(),
210208
Nonterminal::NtBlock(block) => block.tokens(),
211209
}
212210
}
@@ -216,10 +214,8 @@ impl HasTokens for Nonterminal {
216214
Nonterminal::NtStmt(stmt) => stmt.tokens_mut(),
217215
Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens_mut(),
218216
Nonterminal::NtPat(pat) => pat.tokens_mut(),
219-
Nonterminal::NtTy(ty) => ty.tokens_mut(),
220217
Nonterminal::NtMeta(attr_item) => attr_item.tokens_mut(),
221218
Nonterminal::NtPath(path) => path.tokens_mut(),
222-
Nonterminal::NtVis(vis) => vis.tokens_mut(),
223219
Nonterminal::NtBlock(block) => block.tokens_mut(),
224220
}
225221
}

Diff for: ‎compiler/rustc_ast/src/mut_visit.rs

-2
Original file line numberDiff line numberDiff line change
@@ -907,7 +907,6 @@ fn visit_nonterminal<T: MutVisitor>(vis: &mut T, nt: &mut token::Nonterminal) {
907907
}),
908908
token::NtPat(pat) => vis.visit_pat(pat),
909909
token::NtExpr(expr) => vis.visit_expr(expr),
910-
token::NtTy(ty) => vis.visit_ty(ty),
911910
token::NtLiteral(expr) => vis.visit_expr(expr),
912911
token::NtMeta(item) => {
913912
let AttrItem { unsafety: _, path, args, tokens } = item.deref_mut();
@@ -916,7 +915,6 @@ fn visit_nonterminal<T: MutVisitor>(vis: &mut T, nt: &mut token::Nonterminal) {
916915
visit_lazy_tts(vis, tokens);
917916
}
918917
token::NtPath(path) => vis.visit_path(path),
919-
token::NtVis(visib) => vis.visit_vis(visib),
920918
}
921919
}
922920

Diff for: ‎compiler/rustc_ast/src/token.rs

+16-14
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,9 @@ pub enum MetaVarKind {
8484
// This field is needed for `Token::can_begin_string_literal`.
8585
can_begin_string_literal: bool,
8686
},
87-
Ty,
87+
Ty {
88+
is_path: bool,
89+
},
8890
Ident,
8991
Lifetime,
9092
Literal,
@@ -104,7 +106,7 @@ impl fmt::Display for MetaVarKind {
104106
MetaVarKind::Pat(PatParam { inferred: false }) => sym::pat_param,
105107
MetaVarKind::Expr { kind: Expr2021 { inferred: true } | Expr, .. } => sym::expr,
106108
MetaVarKind::Expr { kind: Expr2021 { inferred: false }, .. } => sym::expr_2021,
107-
MetaVarKind::Ty => sym::ty,
109+
MetaVarKind::Ty { .. } => sym::ty,
108110
MetaVarKind::Ident => sym::ident,
109111
MetaVarKind::Lifetime => sym::lifetime,
110112
MetaVarKind::Literal => sym::literal,
@@ -659,15 +661,14 @@ impl Token {
659661
| NtMeta(..)
660662
| NtPat(..)
661663
| NtPath(..)
662-
| NtTy(..)
663664
),
664665
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
665666
MetaVarKind::Expr { .. } |
666667
MetaVarKind::Literal |
667668
MetaVarKind::Meta |
668669
MetaVarKind::Pat(_) |
669670
MetaVarKind::Path |
670-
MetaVarKind::Ty
671+
MetaVarKind::Ty { .. }
671672
))) => true,
672673
_ => false,
673674
}
@@ -688,9 +689,9 @@ impl Token {
688689
Lifetime(..) | // lifetime bound in trait object
689690
Lt | BinOp(Shl) | // associated path
690691
PathSep => true, // global path
691-
Interpolated(ref nt) => matches!(&**nt, NtTy(..) | NtPath(..)),
692+
Interpolated(ref nt) => matches!(&**nt, NtPath(..)),
692693
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
693-
MetaVarKind::Ty |
694+
MetaVarKind::Ty { .. } |
694695
MetaVarKind::Path
695696
))) => true,
696697
// For anonymous structs or unions, which only appear in specific positions
@@ -969,6 +970,15 @@ impl Token {
969970
}
970971
}
971972

973+
/// Is this an invisible open delimiter at the start of a token sequence
974+
/// from an expanded metavar?
975+
pub fn is_metavar_seq(&self) -> Option<MetaVarKind> {
976+
match self.kind {
977+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(kind))) => Some(kind),
978+
_ => None,
979+
}
980+
}
981+
972982
pub fn glue(&self, joint: &Token) -> Option<Token> {
973983
let kind = match self.kind {
974984
Eq => match joint.kind {
@@ -1067,12 +1077,10 @@ pub enum Nonterminal {
10671077
NtStmt(P<ast::Stmt>),
10681078
NtPat(P<ast::Pat>),
10691079
NtExpr(P<ast::Expr>),
1070-
NtTy(P<ast::Ty>),
10711080
NtLiteral(P<ast::Expr>),
10721081
/// Stuff inside brackets for attributes
10731082
NtMeta(P<ast::AttrItem>),
10741083
NtPath(P<ast::Path>),
1075-
NtVis(P<ast::Visibility>),
10761084
}
10771085

10781086
#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
@@ -1166,10 +1174,8 @@ impl Nonterminal {
11661174
NtStmt(stmt) => stmt.span,
11671175
NtPat(pat) => pat.span,
11681176
NtExpr(expr) | NtLiteral(expr) => expr.span,
1169-
NtTy(ty) => ty.span,
11701177
NtMeta(attr_item) => attr_item.span(),
11711178
NtPath(path) => path.span,
1172-
NtVis(vis) => vis.span,
11731179
}
11741180
}
11751181

@@ -1181,10 +1187,8 @@ impl Nonterminal {
11811187
NtPat(..) => "pattern",
11821188
NtExpr(..) => "expression",
11831189
NtLiteral(..) => "literal",
1184-
NtTy(..) => "type",
11851190
NtMeta(..) => "attribute",
11861191
NtPath(..) => "path",
1187-
NtVis(..) => "visibility",
11881192
}
11891193
}
11901194
}
@@ -1207,11 +1211,9 @@ impl fmt::Debug for Nonterminal {
12071211
NtStmt(..) => f.pad("NtStmt(..)"),
12081212
NtPat(..) => f.pad("NtPat(..)"),
12091213
NtExpr(..) => f.pad("NtExpr(..)"),
1210-
NtTy(..) => f.pad("NtTy(..)"),
12111214
NtLiteral(..) => f.pad("NtLiteral(..)"),
12121215
NtMeta(..) => f.pad("NtMeta(..)"),
12131216
NtPath(..) => f.pad("NtPath(..)"),
1214-
NtVis(..) => f.pad("NtVis(..)"),
12151217
}
12161218
}
12171219
}

Diff for: ‎compiler/rustc_ast/src/tokenstream.rs

-2
Original file line numberDiff line numberDiff line change
@@ -469,10 +469,8 @@ impl TokenStream {
469469
}
470470
Nonterminal::NtStmt(stmt) => TokenStream::from_ast(stmt),
471471
Nonterminal::NtPat(pat) => TokenStream::from_ast(pat),
472-
Nonterminal::NtTy(ty) => TokenStream::from_ast(ty),
473472
Nonterminal::NtMeta(attr) => TokenStream::from_ast(attr),
474473
Nonterminal::NtPath(path) => TokenStream::from_ast(path),
475-
Nonterminal::NtVis(vis) => TokenStream::from_ast(vis),
476474
Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => TokenStream::from_ast(expr),
477475
}
478476
}

Diff for: ‎compiler/rustc_expand/src/mbe/transcribe.rs

+39-2
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
11
use std::mem;
22
use std::sync::Arc;
33

4-
use rustc_ast::ExprKind;
54
use rustc_ast::mut_visit::{self, MutVisitor};
6-
use rustc_ast::token::{self, Delimiter, IdentIsRaw, Lit, LitKind, Nonterminal, Token, TokenKind};
5+
use rustc_ast::token::{
6+
self, Delimiter, IdentIsRaw, InvisibleOrigin, Lit, LitKind, MetaVarKind, Nonterminal, Token,
7+
TokenKind,
8+
};
79
use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree};
10+
use rustc_ast::{ExprKind, TyKind};
811
use rustc_data_structures::fx::FxHashMap;
912
use rustc_errors::{Diag, DiagCtxtHandle, PResult, pluralize};
1013
use rustc_parse::lexer::nfc_normalize;
@@ -274,6 +277,33 @@ pub(super) fn transcribe<'a>(
274277
// some of the unnecessary whitespace.
275278
let ident = MacroRulesNormalizedIdent::new(original_ident);
276279
if let Some(cur_matched) = lookup_cur_matched(ident, interp, &repeats) {
280+
// We wrap the tokens in invisible delimiters, unless they are already wrapped
281+
// in invisible delimiters with the same `MetaVarKind`. Because some proc
282+
// macros can't multiple layers of invisible delimiters of the same
283+
// `MetaVarKind`. This loses some span info, though it hopefully won't matter.
284+
let mut mk_delimited = |mv_kind, mut stream: TokenStream| {
285+
if stream.len() == 1 {
286+
let tree = stream.iter().next().unwrap();
287+
if let TokenTree::Delimited(_, _, delim, inner) = tree
288+
&& let Delimiter::Invisible(InvisibleOrigin::MetaVar(mvk)) = delim
289+
&& mv_kind == *mvk
290+
{
291+
stream = inner.clone();
292+
}
293+
}
294+
295+
// Emit as a token stream within `Delimiter::Invisible` to maintain
296+
// parsing priorities.
297+
marker.visit_span(&mut sp);
298+
// Both the open delim and close delim get the same span, which covers the
299+
// `$foo` in the decl macro RHS.
300+
TokenTree::Delimited(
301+
DelimSpan::from_single(sp),
302+
DelimSpacing::new(Spacing::Alone, Spacing::Alone),
303+
Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind)),
304+
stream,
305+
)
306+
};
277307
let tt = match cur_matched {
278308
MatchedSingle(ParseNtResult::Tt(tt)) => {
279309
// `tt`s are emitted into the output stream directly as "raw tokens",
@@ -292,6 +322,13 @@ pub(super) fn transcribe<'a>(
292322
let kind = token::NtLifetime(*ident, *is_raw);
293323
TokenTree::token_alone(kind, sp)
294324
}
325+
MatchedSingle(ParseNtResult::Ty(ty)) => {
326+
let is_path = matches!(&ty.kind, TyKind::Path(None, _path));
327+
mk_delimited(MetaVarKind::Ty { is_path }, TokenStream::from_ast(ty))
328+
}
329+
MatchedSingle(ParseNtResult::Vis(vis)) => {
330+
mk_delimited(MetaVarKind::Vis, TokenStream::from_ast(vis))
331+
}
295332
MatchedSingle(ParseNtResult::Nt(nt)) => {
296333
// Other variables are emitted into the output stream as groups with
297334
// `Delimiter::Invisible` to maintain parsing priorities.

Diff for: ‎compiler/rustc_parse/src/parser/mod.rs

+71-7
Original file line numberDiff line numberDiff line change
@@ -117,12 +117,15 @@ macro_rules! maybe_recover_from_interpolated_ty_qpath {
117117
($self: expr, $allow_qpath_recovery: expr) => {
118118
if $allow_qpath_recovery
119119
&& $self.may_recover()
120-
&& $self.look_ahead(1, |t| t == &token::PathSep)
121-
&& let token::Interpolated(nt) = &$self.token.kind
122-
&& let token::NtTy(ty) = &**nt
120+
&& let Some(mv_kind) = $self.token.is_metavar_seq()
121+
&& let token::MetaVarKind::Ty { .. } = mv_kind
122+
&& $self.check_noexpect_past_close_delim(&token::PathSep)
123123
{
124-
let ty = ty.clone();
125-
$self.bump();
124+
// Reparse the type, then move to recovery.
125+
let ty = $self
126+
.eat_metavar_seq(mv_kind, |this| this.parse_ty_no_question_mark_recover())
127+
.expect("metavar seq ty");
128+
126129
return $self.maybe_recover_from_bad_qpath_stage_2($self.prev_token.span, ty);
127130
}
128131
};
@@ -614,6 +617,24 @@ impl<'a> Parser<'a> {
614617
self.token == *tok
615618
}
616619

620+
// Check the first token after the delimiter that closes the current
621+
// delimited sequence. (Panics if used in the outermost token stream, which
622+
// has no delimiters.) It uses a clone of the relevant tree cursor to skip
623+
// past the entire `TokenTree::Delimited` in a single step, avoiding the
624+
// need for unbounded token lookahead.
625+
//
626+
// Primarily used when `self.token` matches
627+
// `OpenDelim(Delimiter::Invisible(_))`, to look ahead through the current
628+
// metavar expansion.
629+
fn check_noexpect_past_close_delim(&self, tok: &TokenKind) -> bool {
630+
let mut tree_cursor = self.token_cursor.stack.last().unwrap().clone();
631+
tree_cursor.bump();
632+
matches!(
633+
tree_cursor.curr(),
634+
Some(TokenTree::Token(token::Token { kind, .. }, _)) if kind == tok
635+
)
636+
}
637+
617638
/// Consumes a token 'tok' if it exists. Returns whether the given token was present.
618639
///
619640
/// the main purpose of this function is to reduce the cluttering of the suggestions list
@@ -721,6 +742,43 @@ impl<'a> Parser<'a> {
721742
if !self.eat_keyword(exp) { self.unexpected() } else { Ok(()) }
722743
}
723744

745+
/// Consume a sequence produced by a metavar expansion, if present.
746+
fn eat_metavar_seq<T>(
747+
&mut self,
748+
mv_kind: MetaVarKind,
749+
f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>,
750+
) -> Option<T> {
751+
self.eat_metavar_seq_with_matcher(|mvk| mvk == mv_kind, f)
752+
}
753+
754+
/// A slightly more general form of `eat_metavar_seq`, for use with the
755+
/// `MetaVarKind` variants that have parameters, where an exact match isn't
756+
/// desired.
757+
fn eat_metavar_seq_with_matcher<T>(
758+
&mut self,
759+
match_mv_kind: impl Fn(MetaVarKind) -> bool,
760+
mut f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>,
761+
) -> Option<T> {
762+
if let token::OpenDelim(delim) = self.token.kind
763+
&& let Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind)) = delim
764+
&& match_mv_kind(mv_kind)
765+
{
766+
self.bump();
767+
let res = f(self).expect("failed to reparse {mv_kind:?}");
768+
if let token::CloseDelim(delim) = self.token.kind
769+
&& let Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind)) = delim
770+
&& match_mv_kind(mv_kind)
771+
{
772+
self.bump();
773+
Some(res)
774+
} else {
775+
panic!("no close delim when reparsing {mv_kind:?}");
776+
}
777+
} else {
778+
None
779+
}
780+
}
781+
724782
/// Is the given keyword `kw` followed by a non-reserved identifier?
725783
fn is_kw_followed_by_ident(&self, kw: Symbol) -> bool {
726784
self.token.is_keyword(kw) && self.look_ahead(1, |t| t.is_ident() && !t.is_reserved_ident())
@@ -1455,7 +1513,11 @@ impl<'a> Parser<'a> {
14551513
/// so emit a proper diagnostic.
14561514
// Public for rustfmt usage.
14571515
pub fn parse_visibility(&mut self, fbt: FollowedByType) -> PResult<'a, Visibility> {
1458-
maybe_whole!(self, NtVis, |vis| vis.into_inner());
1516+
if let Some(vis) = self
1517+
.eat_metavar_seq(MetaVarKind::Vis, |this| this.parse_visibility(FollowedByType::Yes))
1518+
{
1519+
return Ok(vis);
1520+
}
14591521

14601522
if !self.eat_keyword(exp!(Pub)) {
14611523
// We need a span for our `Spanned<VisibilityKind>`, but there's inherently no
@@ -1683,7 +1745,9 @@ pub enum ParseNtResult {
16831745
Tt(TokenTree),
16841746
Ident(Ident, IdentIsRaw),
16851747
Lifetime(Ident, IdentIsRaw),
1748+
Ty(P<ast::Ty>),
1749+
Vis(P<ast::Visibility>),
16861750

1687-
/// This case will eventually be removed, along with `Token::Interpolate`.
1751+
/// This variant will eventually be removed, along with `Token::Interpolate`.
16881752
Nt(Arc<Nonterminal>),
16891753
}

0 commit comments

Comments
 (0)