Skip to content
This repository was archived by the owner on May 28, 2025. It is now read-only.

Commit 1202bba

Browse files
borsbors
committed
Auto merge of rust-lang#99887 - nnethercote:rm-TreeAndSpacing, r=petrochenkov
Remove `TreeAndSpacing`. A `TokenStream` contains a `Lrc<Vec<(TokenTree, Spacing)>>`. But this is not quite right. `Spacing` makes sense for `TokenTree::Token`, but does not make sense for `TokenTree::Delimited`, because a `TokenTree::Delimited` cannot be joined with another `TokenTree`. This commit fixes this problem, by adding `Spacing` to `TokenTree::Token`, changing `TokenStream` to contain a `Lrc<Vec<TokenTree>>`, and removing the `TreeAndSpacing` typedef. The commit removes these two impls: - `impl From<TokenTree> for TokenStream` - `impl From<TokenTree> for TreeAndSpacing` These were useful, but also resulted in code with many `.into()` calls that was hard to read, particularly for anyone not highly familiar with the relevant types. This commit makes some other changes to compensate: - `TokenTree::token()` becomes `TokenTree::token_{alone,joint}()`. - `TokenStream::token_{alone,joint}()` are added. - `TokenStream::delimited` is added. This results in things like this: ```rust TokenTree::token(token::Semi, stmt.span).into() ``` changing to this: ```rust TokenStream::token_alone(token::Semi, stmt.span) ``` This makes the type of the result, and its spacing, clearer. These changes also simplifies `Cursor` and `CursorRef`, because they no longer need to distinguish between `next` and `next_with_spacing`. r? `@petrochenkov`
2 parents c907b6f + 332dffb commit 1202bba

File tree

23 files changed

+317
-307
lines changed

23 files changed

+317
-307
lines changed

‎compiler/rustc_ast/src/attr/mod.rs

Lines changed: 36 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ use crate::ast::{Path, PathSegment};
88
use crate::ptr::P;
99
use crate::token::{self, CommentKind, Delimiter, Token};
1010
use crate::tokenstream::{AttrAnnotatedTokenStream, AttrAnnotatedTokenTree};
11-
use crate::tokenstream::{DelimSpan, Spacing, TokenTree, TreeAndSpacing};
11+
use crate::tokenstream::{DelimSpan, Spacing, TokenTree};
1212
use crate::tokenstream::{LazyTokenStream, TokenStream};
1313
use crate::util::comments;
1414

@@ -388,20 +388,20 @@ pub fn list_contains_name(items: &[NestedMetaItem], name: Symbol) -> bool {
388388
}
389389

390390
impl MetaItem {
391-
fn token_trees_and_spacings(&self) -> Vec<TreeAndSpacing> {
391+
fn token_trees(&self) -> Vec<TokenTree> {
392392
let mut idents = vec![];
393393
let mut last_pos = BytePos(0_u32);
394394
for (i, segment) in self.path.segments.iter().enumerate() {
395395
let is_first = i == 0;
396396
if !is_first {
397397
let mod_sep_span =
398398
Span::new(last_pos, segment.ident.span.lo(), segment.ident.span.ctxt(), None);
399-
idents.push(TokenTree::token(token::ModSep, mod_sep_span).into());
399+
idents.push(TokenTree::token_alone(token::ModSep, mod_sep_span));
400400
}
401-
idents.push(TokenTree::Token(Token::from_ast_ident(segment.ident)).into());
401+
idents.push(TokenTree::Token(Token::from_ast_ident(segment.ident), Spacing::Alone));
402402
last_pos = segment.ident.span.hi();
403403
}
404-
idents.extend(self.kind.token_trees_and_spacings(self.span));
404+
idents.extend(self.kind.token_trees(self.span));
405405
idents
406406
}
407407

@@ -411,12 +411,13 @@ impl MetaItem {
411411
{
412412
// FIXME: Share code with `parse_path`.
413413
let path = match tokens.next().map(TokenTree::uninterpolate) {
414-
Some(TokenTree::Token(Token {
415-
kind: kind @ (token::Ident(..) | token::ModSep),
416-
span,
417-
})) => 'arm: {
414+
Some(TokenTree::Token(
415+
Token { kind: kind @ (token::Ident(..) | token::ModSep), span },
416+
_,
417+
)) => 'arm: {
418418
let mut segments = if let token::Ident(name, _) = kind {
419-
if let Some(TokenTree::Token(Token { kind: token::ModSep, .. })) = tokens.peek()
419+
if let Some(TokenTree::Token(Token { kind: token::ModSep, .. }, _)) =
420+
tokens.peek()
420421
{
421422
tokens.next();
422423
vec![PathSegment::from_ident(Ident::new(name, span))]
@@ -427,14 +428,15 @@ impl MetaItem {
427428
vec![PathSegment::path_root(span)]
428429
};
429430
loop {
430-
if let Some(TokenTree::Token(Token { kind: token::Ident(name, _), span })) =
431+
if let Some(TokenTree::Token(Token { kind: token::Ident(name, _), span }, _)) =
431432
tokens.next().map(TokenTree::uninterpolate)
432433
{
433434
segments.push(PathSegment::from_ident(Ident::new(name, span)));
434435
} else {
435436
return None;
436437
}
437-
if let Some(TokenTree::Token(Token { kind: token::ModSep, .. })) = tokens.peek()
438+
if let Some(TokenTree::Token(Token { kind: token::ModSep, .. }, _)) =
439+
tokens.peek()
438440
{
439441
tokens.next();
440442
} else {
@@ -444,7 +446,7 @@ impl MetaItem {
444446
let span = span.with_hi(segments.last().unwrap().ident.span.hi());
445447
Path { span, segments, tokens: None }
446448
}
447-
Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. })) => match *nt {
449+
Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. }, _)) => match *nt {
448450
token::Nonterminal::NtMeta(ref item) => return item.meta(item.path.span),
449451
token::Nonterminal::NtPath(ref path) => (**path).clone(),
450452
_ => return None,
@@ -491,9 +493,9 @@ impl MetaItemKind {
491493
let mut tts = Vec::new();
492494
for (i, item) in list.iter().enumerate() {
493495
if i > 0 {
494-
tts.push(TokenTree::token(token::Comma, span).into());
496+
tts.push(TokenTree::token_alone(token::Comma, span));
495497
}
496-
tts.extend(item.token_trees_and_spacings())
498+
tts.extend(item.token_trees())
497499
}
498500
MacArgs::Delimited(
499501
DelimSpan::from_single(span),
@@ -504,31 +506,28 @@ impl MetaItemKind {
504506
}
505507
}
506508

507-
fn token_trees_and_spacings(&self, span: Span) -> Vec<TreeAndSpacing> {
509+
fn token_trees(&self, span: Span) -> Vec<TokenTree> {
508510
match *self {
509511
MetaItemKind::Word => vec![],
510512
MetaItemKind::NameValue(ref lit) => {
511513
vec![
512-
TokenTree::token(token::Eq, span).into(),
513-
TokenTree::Token(lit.to_token()).into(),
514+
TokenTree::token_alone(token::Eq, span),
515+
TokenTree::Token(lit.to_token(), Spacing::Alone),
514516
]
515517
}
516518
MetaItemKind::List(ref list) => {
517519
let mut tokens = Vec::new();
518520
for (i, item) in list.iter().enumerate() {
519521
if i > 0 {
520-
tokens.push(TokenTree::token(token::Comma, span).into());
522+
tokens.push(TokenTree::token_alone(token::Comma, span));
521523
}
522-
tokens.extend(item.token_trees_and_spacings())
524+
tokens.extend(item.token_trees())
523525
}
524-
vec![
525-
TokenTree::Delimited(
526-
DelimSpan::from_single(span),
527-
Delimiter::Parenthesis,
528-
TokenStream::new(tokens),
529-
)
530-
.into(),
531-
]
526+
vec![TokenTree::Delimited(
527+
DelimSpan::from_single(span),
528+
Delimiter::Parenthesis,
529+
TokenStream::new(tokens),
530+
)]
532531
}
533532
}
534533
}
@@ -540,7 +539,7 @@ impl MetaItemKind {
540539
let item = NestedMetaItem::from_tokens(&mut tokens)?;
541540
result.push(item);
542541
match tokens.next() {
543-
None | Some(TokenTree::Token(Token { kind: token::Comma, .. })) => {}
542+
None | Some(TokenTree::Token(Token { kind: token::Comma, .. }, _)) => {}
544543
_ => return None,
545544
}
546545
}
@@ -554,7 +553,7 @@ impl MetaItemKind {
554553
Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => {
555554
MetaItemKind::name_value_from_tokens(&mut inner_tokens.into_trees())
556555
}
557-
Some(TokenTree::Token(token)) => {
556+
Some(TokenTree::Token(token, _)) => {
558557
Lit::from_token(&token).ok().map(MetaItemKind::NameValue)
559558
}
560559
_ => None,
@@ -586,7 +585,7 @@ impl MetaItemKind {
586585
MetaItemKind::list_from_tokens(inner_tokens)
587586
}
588587
Some(TokenTree::Delimited(..)) => None,
589-
Some(TokenTree::Token(Token { kind: token::Eq, .. })) => {
588+
Some(TokenTree::Token(Token { kind: token::Eq, .. }, _)) => {
590589
tokens.next();
591590
MetaItemKind::name_value_from_tokens(tokens)
592591
}
@@ -603,10 +602,12 @@ impl NestedMetaItem {
603602
}
604603
}
605604

606-
fn token_trees_and_spacings(&self) -> Vec<TreeAndSpacing> {
605+
fn token_trees(&self) -> Vec<TokenTree> {
607606
match *self {
608-
NestedMetaItem::MetaItem(ref item) => item.token_trees_and_spacings(),
609-
NestedMetaItem::Literal(ref lit) => vec![TokenTree::Token(lit.to_token()).into()],
607+
NestedMetaItem::MetaItem(ref item) => item.token_trees(),
608+
NestedMetaItem::Literal(ref lit) => {
609+
vec![TokenTree::Token(lit.to_token(), Spacing::Alone)]
610+
}
610611
}
611612
}
612613

@@ -615,7 +616,7 @@ impl NestedMetaItem {
615616
I: Iterator<Item = TokenTree>,
616617
{
617618
match tokens.peek() {
618-
Some(TokenTree::Token(token))
619+
Some(TokenTree::Token(token, _))
619620
if let Ok(lit) = Lit::from_token(token) =>
620621
{
621622
tokens.next();

‎compiler/rustc_ast/src/mut_visit.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -675,7 +675,7 @@ pub fn visit_attr_annotated_tt<T: MutVisitor>(tt: &mut AttrAnnotatedTokenTree, v
675675
// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
676676
pub fn visit_tt<T: MutVisitor>(tt: &mut TokenTree, vis: &mut T) {
677677
match tt {
678-
TokenTree::Token(token) => {
678+
TokenTree::Token(token, _) => {
679679
visit_token(token, vis);
680680
}
681681
TokenTree::Delimited(DelimSpan { open, close }, _delim, tts) => {
@@ -690,7 +690,7 @@ pub fn visit_tt<T: MutVisitor>(tt: &mut TokenTree, vis: &mut T) {
690690
pub fn visit_tts<T: MutVisitor>(TokenStream(tts): &mut TokenStream, vis: &mut T) {
691691
if T::VISIT_TOKENS && !tts.is_empty() {
692692
let tts = Lrc::make_mut(tts);
693-
visit_vec(tts, |(tree, _is_joint)| visit_tt(tree, vis));
693+
visit_vec(tts, |tree| visit_tt(tree, vis));
694694
}
695695
}
696696

0 commit comments

Comments
 (0)