Remove NtIdent and NtLifetime.

The extra span is now recorded in the new `TokenKind::NtIdent` and
`TokenKind::NtLifetime`. These both consist of a single token, and so
there's no operator precedence problems with inserting them directly
into the token stream.

The other way to do this would be to wrap the ident/lifetime in invisible
delimiters, but there's a lot of code that assumes an interpolated
ident/lifetime fits in a single token, and changing all that code to work with
invisible delimiters would have been a pain. (Maybe it could be done in a
follow-up.)

This change might not seem like much of a win, but it's a first step toward the
much bigger and long-desired removal of `Nonterminal` and
`TokenKind::Interpolated`. That change is big and complex enough that it's
worth doing this piece separately. (Indeed, this commit is based on part of a
late commit in #114647, a prior attempt at that big and complex change.)

Author: nnethercote

Diff for: compiler/rustc_ast/src/ast_traits.rs

@@ -240,7 +240,6 @@ impl HasTokens for Nonterminal {
             Nonterminal::NtPath(path) => path.tokens(),
             Nonterminal::NtVis(vis) => vis.tokens(),
             Nonterminal::NtBlock(block) => block.tokens(),
-            Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) => None,
         }
     }
     fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
@@ -254,7 +253,6 @@ impl HasTokens for Nonterminal {
             Nonterminal::NtPath(path) => path.tokens_mut(),
             Nonterminal::NtVis(vis) => vis.tokens_mut(),
             Nonterminal::NtBlock(block) => block.tokens_mut(),
-            Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) => None,
         }
     }
 }

Diff for: compiler/rustc_ast/src/mut_visit.rs

@@ -781,6 +781,12 @@ pub fn visit_token<T: MutVisitor>(t: &mut Token, vis: &mut T) {
             *span = ident.span;
             return; // Avoid visiting the span for the second time.
         }
+        token::NtIdent(ident, _is_raw) => {
+            vis.visit_ident(ident);
+        }
+        token::NtLifetime(ident) => {
+            vis.visit_ident(ident);
+        }
         token::Interpolated(nt) => {
             let nt = Lrc::make_mut(nt);
             visit_nonterminal(nt, vis);
@@ -832,8 +838,6 @@ fn visit_nonterminal<T: MutVisitor>(nt: &mut token::Nonterminal, vis: &mut T) {
         token::NtPat(pat) => vis.visit_pat(pat),
         token::NtExpr(expr) => vis.visit_expr(expr),
         token::NtTy(ty) => vis.visit_ty(ty),
-        token::NtIdent(ident, _is_raw) => vis.visit_ident(ident),
-        token::NtLifetime(ident) => vis.visit_ident(ident),
         token::NtLiteral(expr) => vis.visit_expr(expr),
         token::NtMeta(item) => {
             let AttrItem { path, args, tokens } = item.deref_mut();

Diff for: compiler/rustc_ast/src/token.rs

@@ -318,11 +318,20 @@ pub enum TokenKind {
     /// It's recommended to use `Token::(ident,uninterpolate,uninterpolated_span)` to
     /// treat regular and interpolated identifiers in the same way.
     Ident(Symbol, IdentIsRaw),
+    /// This identifier (and its span) is the identifier passed to the
+    /// declarative macro. The span in the surrounding `Token` is the span of
+    /// the `ident` metavariable in the macro's RHS.
+    NtIdent(Ident, IdentIsRaw),
+
     /// Lifetime identifier token.
     /// Do not forget about `NtLifetime` when you want to match on lifetime identifiers.
     /// It's recommended to use `Token::(lifetime,uninterpolate,uninterpolated_span)` to
     /// treat regular and interpolated lifetime identifiers in the same way.
     Lifetime(Symbol),
+    /// This identifier (and its span) is the lifetime passed to the
+    /// declarative macro. The span in the surrounding `Token` is the span of
+    /// the `lifetime` metavariable in the macro's RHS.
+    NtLifetime(Ident),
 
     /// An embedded AST node, as produced by a macro. This only exists for
     /// historical reasons. We'd like to get rid of it, for multiple reasons.
@@ -444,8 +453,9 @@ impl Token {
     /// Note that keywords are also identifiers, so they should use this
     /// if they keep spans or perform edition checks.
     pub fn uninterpolated_span(&self) -> Span {
-        match &self.kind {
-            Interpolated(nt) => nt.use_span(),
+        match self.kind {
+            NtIdent(ident, _) | NtLifetime(ident) => ident.span,
+            Interpolated(ref nt) => nt.use_span(),
             _ => self.span,
         }
     }
@@ -463,7 +473,7 @@ impl Token {
             }
 
             OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..)
-            | Lifetime(..) | Interpolated(..) | Eof => false,
+            | NtIdent(..) | Lifetime(..) | NtLifetime(..) | Interpolated(..) | Eof => false,
         }
     }
 
@@ -613,14 +623,9 @@ impl Token {
     /// into the regular identifier or lifetime token it refers to,
     /// otherwise returns the original token.
     pub fn uninterpolate(&self) -> Cow<'_, Token> {
-        match &self.kind {
-            Interpolated(nt) => match &**nt {
-                NtIdent(ident, is_raw) => {
-                    Cow::Owned(Token::new(Ident(ident.name, *is_raw), ident.span))
-                }
-                NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)),
-                _ => Cow::Borrowed(self),
-            },
+        match self.kind {
+            NtIdent(ident, is_raw) => Cow::Owned(Token::new(Ident(ident.name, is_raw), ident.span)),
+            NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)),
             _ => Cow::Borrowed(self),
         }
     }
@@ -629,12 +634,9 @@ impl Token {
     #[inline]
     pub fn ident(&self) -> Option<(Ident, IdentIsRaw)> {
         // We avoid using `Token::uninterpolate` here because it's slow.
-        match &self.kind {
-            &Ident(name, is_raw) => Some((Ident::new(name, self.span), is_raw)),
-            Interpolated(nt) => match &**nt {
-                NtIdent(ident, is_raw) => Some((*ident, *is_raw)),
-                _ => None,
-            },
+        match self.kind {
+            Ident(name, is_raw) => Some((Ident::new(name, self.span), is_raw)),
+            NtIdent(ident, is_raw) => Some((ident, is_raw)),
             _ => None,
         }
     }
@@ -643,12 +645,9 @@ impl Token {
     #[inline]
     pub fn lifetime(&self) -> Option<Ident> {
         // We avoid using `Token::uninterpolate` here because it's slow.
-        match &self.kind {
-            &Lifetime(name) => Some(Ident::new(name, self.span)),
-            Interpolated(nt) => match &**nt {
-                NtLifetime(ident) => Some(*ident),
-                _ => None,
-            },
+        match self.kind {
+            Lifetime(name) => Some(Ident::new(name, self.span)),
+            NtLifetime(ident) => Some(ident),
             _ => None,
         }
     }
@@ -837,8 +836,10 @@ impl Token {
 
             Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot
             | DotDotEq | Comma | Semi | PathSep | RArrow | LArrow | FatArrow | Pound | Dollar
-            | Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..)
-            | Lifetime(..) | Interpolated(..) | DocComment(..) | Eof => return None,
+            | Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) | NtIdent(..)
+            | Lifetime(..) | NtLifetime(..) | Interpolated(..) | DocComment(..) | Eof => {
+                return None;
+            }
         };
 
         Some(Token::new(kind, self.span.to(joint.span)))
@@ -861,9 +862,6 @@ pub enum Nonterminal {
     NtPat(P<ast::Pat>),
     NtExpr(P<ast::Expr>),
     NtTy(P<ast::Ty>),
-    /// The span is for the identifier argument passed to the macro.
-    NtIdent(Ident, IdentIsRaw),
-    NtLifetime(Ident),
     NtLiteral(P<ast::Expr>),
     /// Stuff inside brackets for attributes
     NtMeta(P<ast::AttrItem>),
@@ -958,7 +956,6 @@ impl Nonterminal {
             NtPat(pat) => pat.span,
             NtExpr(expr) | NtLiteral(expr) => expr.span,
             NtTy(ty) => ty.span,
-            NtIdent(ident, _) | NtLifetime(ident) => ident.span,
             NtMeta(attr_item) => attr_item.span(),
             NtPath(path) => path.span,
             NtVis(vis) => vis.span,
@@ -974,8 +971,6 @@ impl Nonterminal {
             NtExpr(..) => "expression",
             NtLiteral(..) => "literal",
             NtTy(..) => "type",
-            NtIdent(..) => "identifier",
-            NtLifetime(..) => "lifetime",
             NtMeta(..) => "attribute",
             NtPath(..) => "path",
             NtVis(..) => "visibility",
@@ -984,18 +979,12 @@ impl Nonterminal {
 }
 
 impl PartialEq for Nonterminal {
-    fn eq(&self, rhs: &Self) -> bool {
-        match (self, rhs) {
-            (NtIdent(ident_lhs, is_raw_lhs), NtIdent(ident_rhs, is_raw_rhs)) => {
-                ident_lhs == ident_rhs && is_raw_lhs == is_raw_rhs
-            }
-            (NtLifetime(ident_lhs), NtLifetime(ident_rhs)) => ident_lhs == ident_rhs,
-            // FIXME: Assume that all "complex" nonterminal are not equal, we can't compare them
-            // correctly based on data from AST. This will prevent them from matching each other
-            // in macros. The comparison will become possible only when each nonterminal has an
-            // attached token stream from which it was parsed.
-            _ => false,
-        }
+    fn eq(&self, _rhs: &Self) -> bool {
+        // FIXME: Assume that all nonterminals are not equal, we can't compare them
+        // correctly based on data from AST. This will prevent them from matching each other
+        // in macros. The comparison will become possible only when each nonterminal has an
+        // attached token stream from which it was parsed.
+        false
     }
 }
 
@@ -1008,12 +997,10 @@ impl fmt::Debug for Nonterminal {
             NtPat(..) => f.pad("NtPat(..)"),
             NtExpr(..) => f.pad("NtExpr(..)"),
             NtTy(..) => f.pad("NtTy(..)"),
-            NtIdent(..) => f.pad("NtIdent(..)"),
             NtLiteral(..) => f.pad("NtLiteral(..)"),
             NtMeta(..) => f.pad("NtMeta(..)"),
             NtPath(..) => f.pad("NtPath(..)"),
             NtVis(..) => f.pad("NtVis(..)"),
-            NtLifetime(..) => f.pad("NtLifetime(..)"),
         }
     }
 }

Diff for: compiler/rustc_ast/src/tokenstream.rs

@@ -466,12 +466,6 @@ impl TokenStream {
 
     pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream {
         match nt {
-            Nonterminal::NtIdent(ident, is_raw) => {
-                TokenStream::token_alone(token::Ident(ident.name, *is_raw), ident.span)
-            }
-            Nonterminal::NtLifetime(ident) => {
-                TokenStream::token_alone(token::Lifetime(ident.name), ident.span)
-            }
             Nonterminal::NtItem(item) => TokenStream::from_ast(item),
             Nonterminal::NtBlock(block) => TokenStream::from_ast(block),
             Nonterminal::NtStmt(stmt) if let StmtKind::Empty = stmt.kind => {
@@ -489,11 +483,17 @@ impl TokenStream {
     }
 
     fn flatten_token(token: &Token, spacing: Spacing) -> TokenTree {
-        match &token.kind {
-            token::Interpolated(nt) if let token::NtIdent(ident, is_raw) = &**nt => {
-                TokenTree::Token(Token::new(token::Ident(ident.name, *is_raw), ident.span), spacing)
+        match token.kind {
+            token::NtIdent(ident, is_raw) => {
+                TokenTree::Token(Token::new(token::Ident(ident.name, is_raw), ident.span), spacing)
             }
-            token::Interpolated(nt) => TokenTree::Delimited(
+            token::NtLifetime(ident) => TokenTree::Delimited(
+                DelimSpan::from_single(token.span),
+                DelimSpacing::new(Spacing::JointHidden, spacing),
+                Delimiter::Invisible,
+                TokenStream::token_alone(token::Lifetime(ident.name), ident.span),
+            ),
+            token::Interpolated(ref nt) => TokenTree::Delimited(
                 DelimSpan::from_single(token.span),
                 DelimSpacing::new(Spacing::JointHidden, spacing),
                 Delimiter::Invisible,
@@ -516,7 +516,10 @@ impl TokenStream {
     pub fn flattened(&self) -> TokenStream {
         fn can_skip(stream: &TokenStream) -> bool {
             stream.trees().all(|tree| match tree {
-                TokenTree::Token(token, _) => !matches!(token.kind, token::Interpolated(_)),
+                TokenTree::Token(token, _) => !matches!(
+                    token.kind,
+                    token::NtIdent(..) | token::NtLifetime(..) | token::Interpolated(..)
+                ),
                 TokenTree::Delimited(.., inner) => can_skip(inner),
             })
         }

Diff for: compiler/rustc_ast_pretty/src/pprust/state.rs

@@ -852,8 +852,6 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
             token::NtBlock(e) => self.block_to_string(e),
             token::NtStmt(e) => self.stmt_to_string(e),
             token::NtPat(e) => self.pat_to_string(e),
-            &token::NtIdent(e, is_raw) => IdentPrinter::for_ast_ident(e, is_raw.into()).to_string(),
-            token::NtLifetime(e) => e.to_string(),
             token::NtLiteral(e) => self.expr_to_string(e),
             token::NtVis(e) => self.vis_to_string(e),
         }
@@ -915,10 +913,14 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
             token::Literal(lit) => literal_to_string(lit).into(),
 
             /* Name components */
-            token::Ident(s, is_raw) => {
-                IdentPrinter::new(s, is_raw.into(), convert_dollar_crate).to_string().into()
+            token::Ident(name, is_raw) => {
+                IdentPrinter::new(name, is_raw.into(), convert_dollar_crate).to_string().into()
             }
-            token::Lifetime(s) => s.to_string().into(),
+            token::NtIdent(ident, is_raw) => {
+                IdentPrinter::for_ast_ident(ident, is_raw.into()).to_string().into()
+            }
+            token::Lifetime(name) => name.to_string().into(),
+            token::NtLifetime(ident) => ident.name.to_string().into(),
 
             /* Other */
             token::DocComment(comment_kind, attr_style, data) => {

Diff for: compiler/rustc_expand/src/config.rs

@@ -201,10 +201,17 @@ impl<'a> StripUnconfigured<'a> {
                     inner = self.configure_tokens(&inner);
                     Some(AttrTokenTree::Delimited(sp, spacing, delim, inner)).into_iter()
                 }
-                AttrTokenTree::Token(ref token, _)
-                    if let TokenKind::Interpolated(nt) = &token.kind =>
-                {
-                    panic!("Nonterminal should have been flattened at {:?}: {:?}", token.span, nt);
+                AttrTokenTree::Token(
+                    Token {
+                        kind:
+                            TokenKind::NtIdent(..)
+                            | TokenKind::NtLifetime(..)
+                            | TokenKind::Interpolated(..),
+                        ..
+                    },
+                    _,
+                ) => {
+                    panic!("Nonterminal should have been flattened: {:?}", tree);
                 }
                 AttrTokenTree::Token(token, spacing) => {
                     Some(AttrTokenTree::Token(token, spacing)).into_iter()

Diff for: compiler/rustc_expand/src/mbe/transcribe.rs

@@ -261,6 +261,16 @@ pub(super) fn transcribe<'a>(
                             // without wrapping them into groups.
                             maybe_use_metavar_location(cx, &stack, sp, tt, &mut marker)
                         }
+                        MatchedSingle(ParseNtResult::Ident(ident, is_raw)) => {
+                            marker.visit_span(&mut sp);
+                            let kind = token::NtIdent(*ident, *is_raw);
+                            TokenTree::token_alone(kind, sp)
+                        }
+                        MatchedSingle(ParseNtResult::Lifetime(ident)) => {
+                            marker.visit_span(&mut sp);
+                            let kind = token::NtLifetime(*ident);
+                            TokenTree::token_alone(kind, sp)
+                        }
                         MatchedSingle(ParseNtResult::Nt(nt)) => {
                             // Other variables are emitted into the output stream as groups with
                             // `Delimiter::Invisible` to maintain parsing priorities.

Diff for: compiler/rustc_expand/src/proc_macro_server.rs

@@ -220,13 +220,28 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
                 Ident(sym, is_raw) => {
                     trees.push(TokenTree::Ident(Ident { sym, is_raw: is_raw.into(), span }))
                 }
+                NtIdent(ident, is_raw) => trees.push(TokenTree::Ident(Ident {
+                    sym: ident.name,
+                    is_raw: is_raw.into(),
+                    span: ident.span,
+                })),
+
                 Lifetime(name) => {
                     let ident = symbol::Ident::new(name, span).without_first_quote();
                     trees.extend([
                         TokenTree::Punct(Punct { ch: b'\'', joint: true, span }),
                         TokenTree::Ident(Ident { sym: ident.name, is_raw: false, span }),
                     ]);
                 }
+                NtLifetime(ident) => {
+                    let stream = TokenStream::token_alone(token::Lifetime(ident.name), ident.span);
+                    trees.push(TokenTree::Group(Group {
+                        delimiter: pm::Delimiter::None,
+                        stream: Some(stream),
+                        span: DelimSpan::from_single(span),
+                    }))
+                }
+
                 Literal(token::Lit { kind, symbol, suffix }) => {
                     trees.push(TokenTree::Literal(self::Literal {
                         kind: FromInternal::from_internal(kind),
@@ -259,14 +274,6 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
                     }));
                 }
 
-                Interpolated(ref nt) if let NtIdent(ident, is_raw) = &**nt => {
-                    trees.push(TokenTree::Ident(Ident {
-                        sym: ident.name,
-                        is_raw: matches!(is_raw, IdentIsRaw::Yes),
-                        span: ident.span,
-                    }))
-                }
-
                 Interpolated(nt) => {
                     let stream = TokenStream::from_nonterminal_ast(&nt);
                     // A hack used to pass AST fragments to attribute and derive

Diff for: compiler/rustc_parse/src/parser/expr.rs

@@ -724,7 +724,9 @@ impl<'a> Parser<'a> {
     /// Returns the span of expr if it was not interpolated, or the span of the interpolated token.
     fn interpolated_or_expr_span(&self, expr: &Expr) -> Span {
         match self.prev_token.kind {
-            TokenKind::Interpolated(..) => self.prev_token.span,
+            TokenKind::NtIdent(..) | TokenKind::NtLifetime(..) | TokenKind::Interpolated(..) => {
+                self.prev_token.span
+            }
             _ => expr.span,
         }
     }

Diff for: compiler/rustc_parse/src/parser/mod.rs

@@ -407,6 +407,8 @@ pub(super) fn token_descr(token: &Token) -> String {
         (Some(TokenDescription::Keyword), _) => Some("keyword"),
         (Some(TokenDescription::ReservedKeyword), _) => Some("reserved keyword"),
         (Some(TokenDescription::DocComment), _) => Some("doc comment"),
+        (None, TokenKind::NtIdent(..)) => Some("identifier"),
+        (None, TokenKind::NtLifetime(..)) => Some("lifetime"),
         (None, TokenKind::Interpolated(node)) => Some(node.descr()),
         (None, _) => None,
     };
@@ -1633,5 +1635,9 @@ pub enum FlatToken {
 #[derive(Clone, Debug)]
 pub enum ParseNtResult {
     Tt(TokenTree),
+    Ident(Ident, IdentIsRaw),
+    Lifetime(Ident),
+
+    /// This case will eventually be removed, along with `Token::Interpolate`.
     Nt(Lrc<Nonterminal>),
 }