Remove TokenStream::flattened and InvisibleOrigin::FlattenToken.

They are no longer needed.

This does slightly worsen the error message for a single test, but that
test contains code that is so badly broken that I'm not worried about
it.

Author: nnethercote

Diff for: compiler/rustc_ast/src/token.rs

@@ -30,10 +30,6 @@ pub enum InvisibleOrigin {
     // Converted from `proc_macro::Delimiter` in
     // `proc_macro::Delimiter::to_internal`, i.e. returned by a proc macro.
     ProcMacro,
-
-    // Converted from `TokenKind::NtLifetime` in `TokenStream::flatten_token`.
-    // Treated similarly to `ProcMacro`.
-    FlattenToken,
 }
 
 impl PartialEq for InvisibleOrigin {
@@ -130,9 +126,7 @@ impl Delimiter {
         match self {
             Delimiter::Parenthesis | Delimiter::Bracket | Delimiter::Brace => false,
             Delimiter::Invisible(InvisibleOrigin::MetaVar(_)) => false,
-            Delimiter::Invisible(InvisibleOrigin::FlattenToken | InvisibleOrigin::ProcMacro) => {
-                true
-            }
+            Delimiter::Invisible(InvisibleOrigin::ProcMacro) => true,
         }
     }
 

Diff for: compiler/rustc_ast/src/tokenstream.rs

@@ -25,7 +25,7 @@ use rustc_span::{DUMMY_SP, Span, SpanDecoder, SpanEncoder, Symbol, sym};
 
 use crate::ast::AttrStyle;
 use crate::ast_traits::{HasAttrs, HasTokens};
-use crate::token::{self, Delimiter, InvisibleOrigin, Token, TokenKind};
+use crate::token::{self, Delimiter, Token, TokenKind};
 use crate::{AttrVec, Attribute};
 
 /// Part of a `TokenStream`.
@@ -471,48 +471,6 @@ impl TokenStream {
         TokenStream::new(tts)
     }
 
-    fn flatten_token(token: &Token, spacing: Spacing) -> TokenTree {
-        match token.kind {
-            token::NtIdent(ident, is_raw) => {
-                TokenTree::Token(Token::new(token::Ident(ident.name, is_raw), ident.span), spacing)
-            }
-            token::NtLifetime(ident, is_raw) => TokenTree::Delimited(
-                DelimSpan::from_single(token.span),
-                DelimSpacing::new(Spacing::JointHidden, spacing),
-                Delimiter::Invisible(InvisibleOrigin::FlattenToken),
-                TokenStream::token_alone(token::Lifetime(ident.name, is_raw), ident.span),
-            ),
-            _ => TokenTree::Token(*token, spacing),
-        }
-    }
-
-    fn flatten_token_tree(tree: &TokenTree) -> TokenTree {
-        match tree {
-            TokenTree::Token(token, spacing) => TokenStream::flatten_token(token, *spacing),
-            TokenTree::Delimited(span, spacing, delim, tts) => {
-                TokenTree::Delimited(*span, *spacing, *delim, tts.flattened())
-            }
-        }
-    }
-
-    #[must_use]
-    pub fn flattened(&self) -> TokenStream {
-        fn can_skip(stream: &TokenStream) -> bool {
-            stream.iter().all(|tree| match tree {
-                TokenTree::Token(token, _) => {
-                    !matches!(token.kind, token::NtIdent(..) | token::NtLifetime(..))
-                }
-                TokenTree::Delimited(.., inner) => can_skip(inner),
-            })
-        }
-
-        if can_skip(self) {
-            return self.clone();
-        }
-
-        self.iter().map(|tree| TokenStream::flatten_token_tree(tree)).collect()
-    }
-
     // If `vec` is not empty, try to glue `tt` onto its last token. The return
     // value indicates if gluing took place.
     fn try_glue_to_last(vec: &mut Vec<TokenTree>, tt: &TokenTree) -> bool {

Diff for: compiler/rustc_ast_lowering/src/lib.rs

@@ -917,7 +917,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
     }
 
     fn lower_delim_args(&self, args: &DelimArgs) -> DelimArgs {
-        DelimArgs { dspan: args.dspan, delim: args.delim, tokens: args.tokens.flattened() }
+        DelimArgs { dspan: args.dspan, delim: args.delim, tokens: args.tokens.clone() }
     }
 
     /// Lower an associated item constraint.

Diff for: compiler/rustc_attr_parsing/src/context.rs

@@ -320,7 +320,7 @@ impl<'sess> AttributeParser<'sess> {
             ast::AttrArgs::Delimited(args) => AttrArgs::Delimited(DelimArgs {
                 dspan: args.dspan,
                 delim: args.delim,
-                tokens: args.tokens.flattened(),
+                tokens: args.tokens.clone(),
             }),
             // This is an inert key-value attribute - it will never be visible to macros
             // after it gets lowered to HIR. Therefore, we can extract literals to handle

Diff for: compiler/rustc_builtin_macros/src/cfg_eval.rs

@@ -92,20 +92,16 @@ impl CfgEval<'_> {
         // the location of `#[cfg]` and `#[cfg_attr]` in the token stream. The tokenization
         // process is lossless, so this process is invisible to proc-macros.
 
-        // 'Flatten' all nonterminals (i.e. `TokenKind::Nt{Ident,Lifetime}`)
-        // to `None`-delimited groups containing the corresponding tokens. This
-        // is normally delayed until the proc-macro server actually needs to
-        // provide tokens to a proc-macro. We do this earlier, so that we can
-        // handle cases like:
+        // Interesting cases:
         //
         // ```rust
         // #[cfg_eval] #[cfg] $item
         //```
         //
         // where `$item` is `#[cfg_attr] struct Foo {}`. We want to make
         // sure to evaluate *all* `#[cfg]` and `#[cfg_attr]` attributes - the simplest
-        // way to do this is to do a single parse of a stream without any nonterminals.
-        let orig_tokens = annotatable.to_tokens().flattened();
+        // way to do this is to do a single parse of the token stream.
+        let orig_tokens = annotatable.to_tokens();
 
         // Re-parse the tokens, setting the `capture_cfg` flag to save extra information
         // to the captured `AttrTokenStream` (specifically, we capture

Diff for: compiler/rustc_expand/src/config.rs

@@ -237,12 +237,6 @@ impl<'a> StripUnconfigured<'a> {
                     inner = self.configure_tokens(&inner);
                     Some(AttrTokenTree::Delimited(sp, spacing, delim, inner))
                 }
-                AttrTokenTree::Token(
-                    Token { kind: TokenKind::NtIdent(..) | TokenKind::NtLifetime(..), .. },
-                    _,
-                ) => {
-                    panic!("Nonterminal should have been flattened: {:?}", tree);
-                }
                 AttrTokenTree::Token(
                     Token { kind: TokenKind::OpenDelim(_) | TokenKind::CloseDelim(_), .. },
                     _,

Diff for: tests/ui/macros/syntax-error-recovery.rs

@@ -5,14 +5,14 @@ macro_rules! values {
             $(
                 #[$attr]
                 $token $($inner)? = $value,
+                //~^ ERROR expected one of `!` or `::`, found `<eof>`
             )*
         }
     };
 }
-//~^^^^^ ERROR expected one of `(`, `,`, `=`, `{`, or `}`, found `ty` metavariable
+//~^^^^^^ ERROR expected one of `(`, `,`, `=`, `{`, or `}`, found `ty` metavariable
 //~| ERROR macro expansion ignores `ty` metavariable and any tokens following
 
 values!(STRING(1) as (String) => cfg(test),);
-//~^ ERROR expected one of `!` or `::`, found `<eof>`
 
 fn main() {}

Diff for: tests/ui/macros/syntax-error-recovery.stderr

@@ -22,10 +22,10 @@ LL | values!(STRING(1) as (String) => cfg(test),);
    = note: the usage of `values!` is likely invalid in item context
 
 error: expected one of `!` or `::`, found `<eof>`
-  --> $DIR/syntax-error-recovery.rs:15:9
+  --> $DIR/syntax-error-recovery.rs:7:17
    |
-LL | values!(STRING(1) as (String) => cfg(test),);
-   |         ^^^^^^ expected one of `!` or `::`
+LL |                 $token $($inner)? = $value,
+   |                 ^^^^^^ expected one of `!` or `::`
 
 error: aborting due to 3 previous errors