Auto merge of rust-lang#116649 - nnethercote:improve-print_tts-precursors, r=petrochenkov

Token cleanups

Some precursors to rust-lang#114571 that are worth merging even if the main part of rust-lang#114571 doesn't get merged.

r? `@petrochenkov`

Author: bors

Diff for: compiler/rustc_ast/src/attr/mod.rs

@@ -197,10 +197,10 @@ impl Attribute {
                 .unwrap_or_else(|| panic!("attribute is missing tokens: {self:?}"))
                 .to_attr_token_stream()
                 .to_tokenstream(),
-            &AttrKind::DocComment(comment_kind, data) => TokenStream::new(vec![TokenTree::Token(
-                Token::new(token::DocComment(comment_kind, self.style, data), self.span),
-                Spacing::Alone,
-            )]),
+            &AttrKind::DocComment(comment_kind, data) => TokenStream::token_alone(
+                token::DocComment(comment_kind, self.style, data),
+                self.span,
+            ),
         }
     }
 }

Diff for: compiler/rustc_ast/src/token.rs

@@ -404,7 +404,7 @@ impl Token {
         [DotDot, DotDotDot, DotDotEq].contains(&self.kind)
     }
 
-    pub fn is_op(&self) -> bool {
+    pub fn is_punct(&self) -> bool {
         match self.kind {
             Eq | Lt | Le | EqEq | Ne | Ge | Gt | AndAnd | OrOr | Not | Tilde | BinOp(_)
             | BinOpEq(_) | At | Dot | DotDot | DotDotDot | DotDotEq | Comma | Semi | Colon

Diff for: compiler/rustc_ast_pretty/src/pprust/state.rs

@@ -148,7 +148,7 @@ pub fn print_crate<'a>(
 
 /// This makes printed token streams look slightly nicer,
 /// and also addresses some specific regressions described in #63896 and #73345.
-fn tt_prepend_space(tt: &TokenTree, prev: &TokenTree) -> bool {
+fn space_between(prev: &TokenTree, curr: &TokenTree) -> bool {
     if let TokenTree::Token(token, _) = prev {
         // No space after these tokens, e.g. `x.y`, `$e`
         // (The carets point to `prev`.)       ^     ^
@@ -159,9 +159,9 @@ fn tt_prepend_space(tt: &TokenTree, prev: &TokenTree) -> bool {
             return comment_kind != CommentKind::Line;
         }
     }
-    match tt {
+    match curr {
         // No space before these tokens, e.g. `foo,`, `println!`, `x.y`
-        // (The carets point to `token`.)         ^           ^     ^
+        // (The carets point to `curr`.)          ^           ^     ^
         //
         // FIXME: having `Not` here works well for macro invocations like
         // `println!()`, but is bad when `!` means "logical not" or "the never
@@ -575,7 +575,7 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
         while let Some(tt) = iter.next() {
             self.print_tt(tt, convert_dollar_crate);
             if let Some(next) = iter.peek() {
-                if tt_prepend_space(next, tt) {
+                if space_between(tt, next) {
                     self.space();
                 }
             }

Diff for: compiler/rustc_parse/src/lexer/tokentrees.rs

@@ -55,16 +55,14 @@ impl<'a> TokenTreesReader<'a> {
                     let (this_spacing, next_tok) = loop {
                         let (next_tok, is_next_tok_preceded_by_whitespace) =
                             self.string_reader.next_token();
-                        if !is_next_tok_preceded_by_whitespace {
-                            if let Some(glued) = self.token.glue(&next_tok) {
-                                self.token = glued;
-                            } else {
-                                let this_spacing =
-                                    if next_tok.is_op() { Spacing::Joint } else { Spacing::Alone };
-                                break (this_spacing, next_tok);
-                            }
-                        } else {
+                        if is_next_tok_preceded_by_whitespace {
                             break (Spacing::Alone, next_tok);
+                        } else if let Some(glued) = self.token.glue(&next_tok) {
+                            self.token = glued;
+                        } else {
+                            let this_spacing =
+                                if next_tok.is_punct() { Spacing::Joint } else { Spacing::Alone };
+                            break (this_spacing, next_tok);
                         }
                     };
                     let this_tok = std::mem::replace(&mut self.token, next_tok);

Diff for: compiler/rustc_parse/src/parser/expr.rs

@@ -1592,7 +1592,7 @@ impl<'a> Parser<'a> {
         } else if !ate_colon
             && self.may_recover()
             && (matches!(self.token.kind, token::CloseDelim(_) | token::Comma)
-                || self.token.is_op())
+                || self.token.is_punct())
         {
             let (lit, _) =
                 self.recover_unclosed_char(label_.ident, Parser::mk_token_lit_char, |self_| {

Diff for: tests/ui/proc-macro/issue-75930-derive-cfg.rs

@@ -16,6 +16,36 @@ extern crate std;
 #[macro_use]
 extern crate test_macros;
 
+// Note: the expected output contains this sequence:
+// ```
+// Punct {
+//     ch: '<',
+//     spacing: Joint,
+//     span: $DIR/issue-75930-derive-cfg.rs:25:11: 25:12 (#0),
+// },
+// Ident {
+//     ident: "B",
+//     span: $DIR/issue-75930-derive-cfg.rs:25:29: 25:30 (#0),
+// },
+// ```
+// It's surprising to see a `Joint` token tree followed by an `Ident` token
+// tree, because `Joint` is supposed to only be used if the following token is
+// `Punct`.
+//
+// It is because of this code from below:
+// ```
+// struct Foo<#[cfg(FALSE)] A, B>
+// ```
+// When the token stream is formed during parsing, `<` is followed immediately
+// by `#`, which is punctuation, so it is marked `Joint`. But before being
+// passed to the proc macro it is rewritten to this:
+// ```
+// struct Foo<B>
+// ```
+// But the `Joint` marker on the `<` is not updated. Perhaps it should be
+// corrected before being passed to the proc macro? But a prior attempt to do
+// that kind of correction caused the problem seen in #76399, so maybe not.
+
 #[print_helper(a)] //~ WARN derive helper attribute is used before it is introduced
                    //~| WARN this was previously accepted
 #[cfg_attr(not(FALSE), allow(dead_code))]

Diff for: tests/ui/proc-macro/issue-75930-derive-cfg.stderr

@@ -1,5 +1,5 @@
 warning: derive helper attribute is used before it is introduced
-  --> $DIR/issue-75930-derive-cfg.rs:19:3
+  --> $DIR/issue-75930-derive-cfg.rs:49:3
    |
 LL | #[print_helper(a)]
    |   ^^^^^^^^^^^^
@@ -12,7 +12,7 @@ LL | #[derive(Print)]
    = note: `#[warn(legacy_derive_helpers)]` on by default
 
 warning: derive helper attribute is used before it is introduced
-  --> $DIR/issue-75930-derive-cfg.rs:19:3
+  --> $DIR/issue-75930-derive-cfg.rs:49:3
    |
 LL | #[print_helper(a)]
    |   ^^^^^^^^^^^^