Skip to content

Commit bb495d6

Browse files
committed
Remove NtBlock, Nonterminal, and TokenKind::Interpolated.
`NtBlock` is the last remaining variant of `Nonterminal`, so once it is gone then `Nonterminal` can be removed as well.
1 parent 70dab5a commit bb495d6

File tree

18 files changed

+108
-388
lines changed

18 files changed

+108
-388
lines changed

compiler/rustc_ast/src/ast_traits.rs

-14
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ use std::fmt;
66
use std::marker::PhantomData;
77

88
use crate::ptr::P;
9-
use crate::token::Nonterminal;
109
use crate::tokenstream::LazyAttrTokenStream;
1110
use crate::{
1211
Arm, AssocItem, AttrItem, AttrKind, AttrVec, Attribute, Block, Crate, Expr, ExprField,
@@ -206,19 +205,6 @@ impl HasTokens for Attribute {
206205
}
207206
}
208207

209-
impl HasTokens for Nonterminal {
210-
fn tokens(&self) -> Option<&LazyAttrTokenStream> {
211-
match self {
212-
Nonterminal::NtBlock(block) => block.tokens(),
213-
}
214-
}
215-
fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
216-
match self {
217-
Nonterminal::NtBlock(block) => block.tokens_mut(),
218-
}
219-
}
220-
}
221-
222208
/// A trait for AST nodes having (or not having) attributes.
223209
pub trait HasAttrs {
224210
/// This is `true` if this `HasAttrs` might support 'custom' (proc-macro) inner

compiler/rustc_ast/src/mut_visit.rs

+3-37
Original file line numberDiff line numberDiff line change
@@ -843,9 +843,9 @@ fn visit_lazy_tts<T: MutVisitor>(vis: &mut T, lazy_tts: &mut Option<LazyAttrToke
843843
visit_lazy_tts_opt_mut(vis, lazy_tts.as_mut());
844844
}
845845

846-
/// Applies ident visitor if it's an ident; applies other visits to interpolated nodes.
847-
/// In practice the ident part is not actually used by specific visitors right now,
848-
/// but there's a test below checking that it works.
846+
/// Applies ident visitor if it's an ident. In practice this is not actually
847+
/// used by specific visitors right now, but there's a test below checking that
848+
/// it works.
849849
// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
850850
pub fn visit_token<T: MutVisitor>(vis: &mut T, t: &mut Token) {
851851
let Token { kind, span } = t;
@@ -863,45 +863,11 @@ pub fn visit_token<T: MutVisitor>(vis: &mut T, t: &mut Token) {
863863
token::NtLifetime(ident, _is_raw) => {
864864
vis.visit_ident(ident);
865865
}
866-
token::Interpolated(nt) => {
867-
let nt = Arc::make_mut(nt);
868-
visit_nonterminal(vis, nt);
869-
}
870866
_ => {}
871867
}
872868
vis.visit_span(span);
873869
}
874870

875-
// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
876-
/// Applies the visitor to elements of interpolated nodes.
877-
//
878-
// N.B., this can occur only when applying a visitor to partially expanded
879-
// code, where parsed pieces have gotten implanted ito *other* macro
880-
// invocations. This is relevant for macro hygiene, but possibly not elsewhere.
881-
//
882-
// One problem here occurs because the types for flat_map_item, flat_map_stmt,
883-
// etc., allow the visitor to return *multiple* items; this is a problem for the
884-
// nodes here, because they insist on having exactly one piece. One solution
885-
// would be to mangle the MutVisitor trait to include one-to-many and
886-
// one-to-one versions of these entry points, but that would probably confuse a
887-
// lot of people and help very few. Instead, I'm just going to put in dynamic
888-
// checks. I think the performance impact of this will be pretty much
889-
// nonexistent. The danger is that someone will apply a `MutVisitor` to a
890-
// partially expanded node, and will be confused by the fact that their
891-
// `flat_map_item` or `flat_map_stmt` isn't getting called on `NtItem` or `NtStmt`
892-
// nodes. Hopefully they'll wind up reading this comment, and doing something
893-
// appropriate.
894-
//
895-
// BTW, design choice: I considered just changing the type of, e.g., `NtItem` to
896-
// contain multiple items, but decided against it when I looked at
897-
// `parse_item_or_view_item` and tried to figure out what I would do with
898-
// multiple items there....
899-
fn visit_nonterminal<T: MutVisitor>(vis: &mut T, nt: &mut token::Nonterminal) {
900-
match nt {
901-
token::NtBlock(block) => vis.visit_block(block),
902-
}
903-
}
904-
905871
// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
906872
fn visit_defaultness<T: MutVisitor>(vis: &mut T, defaultness: &mut Defaultness) {
907873
match defaultness {

compiler/rustc_ast/src/token.rs

+16-111
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,10 @@
11
use std::borrow::Cow;
22
use std::fmt;
3-
use std::sync::Arc;
43

54
pub use LitKind::*;
6-
pub use Nonterminal::*;
75
pub use NtExprKind::*;
86
pub use NtPatKind::*;
97
pub use TokenKind::*;
10-
use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
118
use rustc_macros::{Decodable, Encodable, HashStable_Generic};
129
use rustc_span::edition::Edition;
1310
use rustc_span::{DUMMY_SP, ErrorGuaranteed, Span, kw, sym};
@@ -16,7 +13,6 @@ use rustc_span::{DUMMY_SP, ErrorGuaranteed, Span, kw, sym};
1613
use rustc_span::{Ident, Symbol};
1714

1815
use crate::ast;
19-
use crate::ptr::P;
2016
use crate::util::case::Case;
2117

2218
#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
@@ -35,8 +31,8 @@ pub enum InvisibleOrigin {
3531
// `proc_macro::Delimiter::to_internal`, i.e. returned by a proc macro.
3632
ProcMacro,
3733

38-
// Converted from `TokenKind::Interpolated` in
39-
// `TokenStream::flatten_token`. Treated similarly to `ProcMacro`.
34+
// Converted from `TokenKind::NtLifetime` in `TokenStream::flatten_token`.
35+
// Treated similarly to `ProcMacro`.
4036
FlattenToken,
4137
}
4238

@@ -337,9 +333,7 @@ impl From<IdentIsRaw> for bool {
337333
}
338334
}
339335

340-
// SAFETY: due to the `Clone` impl below, all fields of all variants other than
341-
// `Interpolated` must impl `Copy`.
342-
#[derive(PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
336+
#[derive(Clone, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
343337
pub enum TokenKind {
344338
/* Expression-operator symbols. */
345339
/// `=`
@@ -468,21 +462,6 @@ pub enum TokenKind {
468462
/// the `lifetime` metavariable in the macro's RHS.
469463
NtLifetime(Ident, IdentIsRaw),
470464

471-
/// An embedded AST node, as produced by a macro. This only exists for
472-
/// historical reasons. We'd like to get rid of it, for multiple reasons.
473-
/// - It's conceptually very strange. Saying a token can contain an AST
474-
/// node is like saying, in natural language, that a word can contain a
475-
/// sentence.
476-
/// - It requires special handling in a bunch of places in the parser.
477-
/// - It prevents `Token` from implementing `Copy`.
478-
/// It adds complexity and likely slows things down. Please don't add new
479-
/// occurrences of this token kind!
480-
///
481-
/// The span in the surrounding `Token` is that of the metavariable in the
482-
/// macro's RHS. The span within the Nonterminal is that of the fragment
483-
/// passed to the macro at the call site.
484-
Interpolated(Arc<Nonterminal>),
485-
486465
/// A doc comment token.
487466
/// `Symbol` is the doc comment's data excluding its "quotes" (`///`, `/**`, etc)
488467
/// similarly to symbols in string literal tokens.
@@ -492,19 +471,6 @@ pub enum TokenKind {
492471
Eof,
493472
}
494473

495-
impl Clone for TokenKind {
496-
fn clone(&self) -> Self {
497-
// `TokenKind` would impl `Copy` if it weren't for `Interpolated`. So
498-
// for all other variants, this implementation of `clone` is just like
499-
// a copy. This is faster than the `derive(Clone)` version which has a
500-
// separate path for every variant.
501-
match self {
502-
Interpolated(nt) => Interpolated(Arc::clone(nt)),
503-
_ => unsafe { std::ptr::read(self) },
504-
}
505-
}
506-
}
507-
508474
#[derive(Clone, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
509475
pub struct Token {
510476
pub kind: TokenKind,
@@ -600,7 +566,7 @@ impl Token {
600566
| FatArrow | Pound | Dollar | Question | SingleQuote => true,
601567

602568
OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..)
603-
| NtIdent(..) | Lifetime(..) | NtLifetime(..) | Interpolated(..) | Eof => false,
569+
| NtIdent(..) | Lifetime(..) | NtLifetime(..) | Eof => false,
604570
}
605571
}
606572

@@ -631,7 +597,6 @@ impl Token {
631597
PathSep | // global path
632598
Lifetime(..) | // labeled loop
633599
Pound => true, // expression attributes
634-
Interpolated(ref nt) => matches!(&**nt, NtBlock(..)),
635600
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
636601
MetaVarKind::Block |
637602
MetaVarKind::Expr { .. } |
@@ -703,7 +668,6 @@ impl Token {
703668
match self.kind {
704669
OpenDelim(Delimiter::Brace) | Literal(..) | Minus => true,
705670
Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true,
706-
Interpolated(ref nt) => matches!(&**nt, NtBlock(..)),
707671
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
708672
MetaVarKind::Expr { .. } | MetaVarKind::Block | MetaVarKind::Literal,
709673
))) => true,
@@ -831,31 +795,20 @@ impl Token {
831795
/// Is this a pre-parsed expression dropped into the token stream
832796
/// (which happens while parsing the result of macro expansion)?
833797
pub fn is_metavar_expr(&self) -> bool {
834-
#[allow(irrefutable_let_patterns)] // FIXME: temporary
835-
if let Interpolated(nt) = &self.kind
836-
&& let NtBlock(_) = &**nt
837-
{
838-
true
839-
} else if matches!(
798+
matches!(
840799
self.is_metavar_seq(),
841-
Some(MetaVarKind::Expr { .. } | MetaVarKind::Literal | MetaVarKind::Path)
842-
) {
843-
true
844-
} else {
845-
matches!(self.is_metavar_seq(), Some(MetaVarKind::Path))
846-
}
800+
Some(
801+
MetaVarKind::Expr { .. }
802+
| MetaVarKind::Literal
803+
| MetaVarKind::Path
804+
| MetaVarKind::Block
805+
)
806+
)
847807
}
848808

849-
/// Is the token an interpolated block (`$b:block`)?
850-
pub fn is_whole_block(&self) -> bool {
851-
#[allow(irrefutable_let_patterns)] // FIXME: temporary
852-
if let Interpolated(nt) = &self.kind
853-
&& let NtBlock(..) = &**nt
854-
{
855-
return true;
856-
}
857-
858-
false
809+
/// Are we at a block from a metavar (`$b:block`)?
810+
pub fn is_metavar_block(&self) -> bool {
811+
matches!(self.is_metavar_seq(), Some(MetaVarKind::Block))
859812
}
860813

861814
/// Returns `true` if the token is either the `mut` or `const` keyword.
@@ -1024,7 +977,7 @@ impl Token {
1024977
| PercentEq | CaretEq | AndEq | OrEq | ShlEq | ShrEq | At | DotDotDot | DotDotEq
1025978
| Comma | Semi | PathSep | RArrow | LArrow | FatArrow | Pound | Dollar | Question
1026979
| OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) | NtIdent(..)
1027-
| Lifetime(..) | NtLifetime(..) | Interpolated(..) | DocComment(..) | Eof,
980+
| Lifetime(..) | NtLifetime(..) | DocComment(..) | Eof,
1028981
_,
1029982
) => {
1030983
return None;
@@ -1063,12 +1016,6 @@ pub enum NtExprKind {
10631016
Expr2021 { inferred: bool },
10641017
}
10651018

1066-
#[derive(Clone, Encodable, Decodable)]
1067-
/// For interpolation during macro expansion.
1068-
pub enum Nonterminal {
1069-
NtBlock(P<ast::Block>),
1070-
}
1071-
10721019
#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
10731020
pub enum NonterminalKind {
10741021
Item,
@@ -1152,47 +1099,6 @@ impl fmt::Display for NonterminalKind {
11521099
}
11531100
}
11541101

1155-
impl Nonterminal {
1156-
pub fn use_span(&self) -> Span {
1157-
match self {
1158-
NtBlock(block) => block.span,
1159-
}
1160-
}
1161-
1162-
pub fn descr(&self) -> &'static str {
1163-
match self {
1164-
NtBlock(..) => "block",
1165-
}
1166-
}
1167-
}
1168-
1169-
impl PartialEq for Nonterminal {
1170-
fn eq(&self, _rhs: &Self) -> bool {
1171-
// FIXME: Assume that all nonterminals are not equal, we can't compare them
1172-
// correctly based on data from AST. This will prevent them from matching each other
1173-
// in macros. The comparison will become possible only when each nonterminal has an
1174-
// attached token stream from which it was parsed.
1175-
false
1176-
}
1177-
}
1178-
1179-
impl fmt::Debug for Nonterminal {
1180-
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1181-
match *self {
1182-
NtBlock(..) => f.pad("NtBlock(..)"),
1183-
}
1184-
}
1185-
}
1186-
1187-
impl<CTX> HashStable<CTX> for Nonterminal
1188-
where
1189-
CTX: crate::HashStableContext,
1190-
{
1191-
fn hash_stable(&self, _hcx: &mut CTX, _hasher: &mut StableHasher) {
1192-
panic!("interpolated tokens should not be present in the HIR")
1193-
}
1194-
}
1195-
11961102
// Some types are used a lot. Make sure they don't unintentionally get bigger.
11971103
#[cfg(target_pointer_width = "64")]
11981104
mod size_asserts {
@@ -1202,7 +1108,6 @@ mod size_asserts {
12021108
// tidy-alphabetical-start
12031109
static_assert_size!(Lit, 12);
12041110
static_assert_size!(LitKind, 2);
1205-
static_assert_size!(Nonterminal, 8);
12061111
static_assert_size!(Token, 24);
12071112
static_assert_size!(TokenKind, 16);
12081113
// tidy-alphabetical-end

compiler/rustc_ast/src/tokenstream.rs

+4-22
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ use rustc_span::{DUMMY_SP, Span, SpanDecoder, SpanEncoder, Symbol, sym};
2525

2626
use crate::ast::AttrStyle;
2727
use crate::ast_traits::{HasAttrs, HasTokens};
28-
use crate::token::{self, Delimiter, InvisibleOrigin, Nonterminal, Token, TokenKind};
28+
use crate::token::{self, Delimiter, InvisibleOrigin, Token, TokenKind};
2929
use crate::{AttrVec, Attribute};
3030

3131
/// Part of a `TokenStream`.
@@ -305,11 +305,6 @@ pub struct AttrsTarget {
305305
}
306306

307307
/// A `TokenStream` is an abstract sequence of tokens, organized into [`TokenTree`]s.
308-
///
309-
/// The goal is for procedural macros to work with `TokenStream`s and `TokenTree`s
310-
/// instead of a representation of the abstract syntax tree.
311-
/// Today's `TokenTree`s can still contain AST via `token::Interpolated` for
312-
/// backwards compatibility.
313308
#[derive(Clone, Debug, Default, Encodable, Decodable)]
314309
pub struct TokenStream(pub(crate) Arc<Vec<TokenTree>>);
315310

@@ -476,12 +471,6 @@ impl TokenStream {
476471
TokenStream::new(tts)
477472
}
478473

479-
pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream {
480-
match nt {
481-
Nonterminal::NtBlock(block) => TokenStream::from_ast(block),
482-
}
483-
}
484-
485474
fn flatten_token(token: &Token, spacing: Spacing) -> TokenTree {
486475
match token.kind {
487476
token::NtIdent(ident, is_raw) => {
@@ -493,12 +482,6 @@ impl TokenStream {
493482
Delimiter::Invisible(InvisibleOrigin::FlattenToken),
494483
TokenStream::token_alone(token::Lifetime(ident.name, is_raw), ident.span),
495484
),
496-
token::Interpolated(ref nt) => TokenTree::Delimited(
497-
DelimSpan::from_single(token.span),
498-
DelimSpacing::new(Spacing::JointHidden, spacing),
499-
Delimiter::Invisible(InvisibleOrigin::FlattenToken),
500-
TokenStream::from_nonterminal_ast(&nt).flattened(),
501-
),
502485
_ => TokenTree::Token(token.clone(), spacing),
503486
}
504487
}
@@ -516,10 +499,9 @@ impl TokenStream {
516499
pub fn flattened(&self) -> TokenStream {
517500
fn can_skip(stream: &TokenStream) -> bool {
518501
stream.iter().all(|tree| match tree {
519-
TokenTree::Token(token, _) => !matches!(
520-
token.kind,
521-
token::NtIdent(..) | token::NtLifetime(..) | token::Interpolated(..)
522-
),
502+
TokenTree::Token(token, _) => {
503+
!matches!(token.kind, token::NtIdent(..) | token::NtLifetime(..))
504+
}
523505
TokenTree::Delimited(.., inner) => can_skip(inner),
524506
})
525507
}

compiler/rustc_ast_pretty/src/pprust/mod.rs

+1-5
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,10 @@ pub mod state;
55
use std::borrow::Cow;
66

77
use rustc_ast as ast;
8-
use rustc_ast::token::{Nonterminal, Token, TokenKind};
8+
use rustc_ast::token::{Token, TokenKind};
99
use rustc_ast::tokenstream::{TokenStream, TokenTree};
1010
pub use state::{AnnNode, Comments, PpAnn, PrintState, State, print_crate};
1111

12-
pub fn nonterminal_to_string(nt: &Nonterminal) -> String {
13-
State::new().nonterminal_to_string(nt)
14-
}
15-
1612
/// Print the token kind precisely, without converting `$crate` into its respective crate name.
1713
pub fn token_kind_to_string(tok: &TokenKind) -> Cow<'static, str> {
1814
State::new().token_kind_to_string(tok)

0 commit comments

Comments
 (0)