Skip to content

Commit ff7d5ba

Browse files
nnethercotennethercote
committed
Move doc comment desugaring out of TokenCursor.
`TokenCursor` currently does doc comment desugaring on the fly, if the `desugar_doc_comment` field is set. This requires also modifying the token stream on the fly with `replace_prev_and_rewind`. This commit moves the doc comment desugaring out of `TokenCursor`, by introducing a new `TokenStream::desugar_doc_comment` method. This separation of desugaring and iterating makes the code nicer.
1 parent c70c8b7 commit ff7d5ba

File tree

2 files changed

+102
-76
lines changed

2 files changed

+102
-76
lines changed

‎compiler/rustc_ast/src/tokenstream.rs

+89-3
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
//! and a borrowed `TokenStream` is sufficient to build an owned `TokenStream` without taking
1414
//! ownership of the original.
1515
16-
use crate::ast::StmtKind;
16+
use crate::ast::{AttrStyle, StmtKind};
1717
use crate::ast_traits::{HasAttrs, HasSpan, HasTokens};
1818
use crate::token::{self, Delimiter, Nonterminal, Token, TokenKind};
1919
use crate::AttrVec;
@@ -22,11 +22,11 @@ use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
2222
use rustc_data_structures::sync::{self, Lrc};
2323
use rustc_macros::HashStable_Generic;
2424
use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
25-
use rustc_span::{Span, DUMMY_SP};
25+
use rustc_span::{sym, Span, Symbol, DUMMY_SP};
2626
use smallvec::{smallvec, SmallVec};
2727

2828
use std::borrow::Cow;
29-
use std::{fmt, iter, mem};
29+
use std::{cmp, fmt, iter, mem};
3030

3131
/// When the main Rust parser encounters a syntax-extension invocation, it
3232
/// parses the arguments to the invocation as a token tree. This is a very
@@ -566,6 +566,92 @@ impl TokenStream {
566566
pub fn chunks(&self, chunk_size: usize) -> core::slice::Chunks<'_, TokenTree> {
567567
self.0.chunks(chunk_size)
568568
}
569+
570+
/// Desugar doc comments like `/// foo` in the stream into `#[doc =
571+
/// r"foo"]`. Modifies the `TokenStream` via `Lrc::make_mut`, but as little
572+
/// as possible.
573+
pub fn desugar_doc_comments(&mut self) {
574+
if let Some(desugared_stream) = desugar_inner(self.clone()) {
575+
*self = desugared_stream;
576+
}
577+
578+
// The return value is `None` if nothing in `stream` changed.
579+
fn desugar_inner(mut stream: TokenStream) -> Option<TokenStream> {
580+
let mut i = 0;
581+
let mut modified = false;
582+
while let Some(tt) = stream.0.get(i) {
583+
match tt {
584+
&TokenTree::Token(
585+
Token { kind: token::DocComment(_, attr_style, data), span },
586+
_spacing,
587+
) => {
588+
let desugared = desugared_tts(attr_style, data, span);
589+
let desugared_len = desugared.len();
590+
Lrc::make_mut(&mut stream.0).splice(i..i + 1, desugared);
591+
modified = true;
592+
i += desugared_len;
593+
}
594+
595+
&TokenTree::Token(..) => i += 1,
596+
597+
&TokenTree::Delimited(sp, delim, ref delim_stream) => {
598+
if let Some(desugared_delim_stream) = desugar_inner(delim_stream.clone()) {
599+
let new_tt = TokenTree::Delimited(sp, delim, desugared_delim_stream);
600+
Lrc::make_mut(&mut stream.0)[i] = new_tt;
601+
modified = true;
602+
}
603+
i += 1;
604+
}
605+
}
606+
}
607+
if modified { Some(stream) } else { None }
608+
}
609+
610+
fn desugared_tts(attr_style: AttrStyle, data: Symbol, span: Span) -> Vec<TokenTree> {
611+
// Searches for the occurrences of `"#*` and returns the minimum number of `#`s
612+
// required to wrap the text. E.g.
613+
// - `abc d` is wrapped as `r"abc d"` (num_of_hashes = 0)
614+
// - `abc "d"` is wrapped as `r#"abc "d""#` (num_of_hashes = 1)
615+
// - `abc "##d##"` is wrapped as `r###"abc ##"d"##"###` (num_of_hashes = 3)
616+
let mut num_of_hashes = 0;
617+
let mut count = 0;
618+
for ch in data.as_str().chars() {
619+
count = match ch {
620+
'"' => 1,
621+
'#' if count > 0 => count + 1,
622+
_ => 0,
623+
};
624+
num_of_hashes = cmp::max(num_of_hashes, count);
625+
}
626+
627+
// `/// foo` becomes `doc = r"foo"`.
628+
let delim_span = DelimSpan::from_single(span);
629+
let body = TokenTree::Delimited(
630+
delim_span,
631+
Delimiter::Bracket,
632+
[
633+
TokenTree::token_alone(token::Ident(sym::doc, false), span),
634+
TokenTree::token_alone(token::Eq, span),
635+
TokenTree::token_alone(
636+
TokenKind::lit(token::StrRaw(num_of_hashes), data, None),
637+
span,
638+
),
639+
]
640+
.into_iter()
641+
.collect::<TokenStream>(),
642+
);
643+
644+
if attr_style == AttrStyle::Inner {
645+
vec![
646+
TokenTree::token_alone(token::Pound, span),
647+
TokenTree::token_alone(token::Not, span),
648+
body,
649+
]
650+
} else {
651+
vec![TokenTree::token_alone(token::Pound, span), body]
652+
}
653+
}
654+
}
569655
}
570656

571657
/// By-reference iterator over a [`TokenStream`], that produces `&TokenTree`

‎compiler/rustc_parse/src/parser/mod.rs

+13-73
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ use rustc_ast::tokenstream::{TokenStream, TokenTree, TokenTreeCursor};
2424
use rustc_ast::util::case::Case;
2525
use rustc_ast::AttrId;
2626
use rustc_ast::DUMMY_NODE_ID;
27-
use rustc_ast::{self as ast, AnonConst, AttrStyle, Const, DelimArgs, Extern};
27+
use rustc_ast::{self as ast, AnonConst, Const, DelimArgs, Extern};
2828
use rustc_ast::{Async, AttrArgs, AttrArgsEq, Expr, ExprKind, MacDelimiter, Mutability, StrLit};
2929
use rustc_ast::{HasAttrs, HasTokens, Unsafe, Visibility, VisibilityKind};
3030
use rustc_ast_pretty::pprust;
@@ -38,7 +38,7 @@ use rustc_session::parse::ParseSess;
3838
use rustc_span::source_map::{Span, DUMMY_SP};
3939
use rustc_span::symbol::{kw, sym, Ident, Symbol};
4040
use std::ops::Range;
41-
use std::{cmp, mem, slice};
41+
use std::{mem, slice};
4242
use thin_vec::ThinVec;
4343
use tracing::debug;
4444

@@ -224,11 +224,6 @@ struct TokenCursor {
224224
// because it's the outermost token stream which never has delimiters.
225225
stack: Vec<(TokenTreeCursor, Delimiter, DelimSpan)>,
226226

227-
// We need to desugar doc comments from `/// foo` form into `#[doc =
228-
// r"foo"]` form when parsing declarative macro inputs in `parse_tt`,
229-
// because some declarative macros look for `doc` attributes.
230-
desugar_doc_comments: bool,
231-
232227
// Counts the number of calls to `{,inlined_}next`.
233228
num_next_calls: usize,
234229

@@ -271,23 +266,11 @@ impl TokenCursor {
271266
if let Some(tree) = self.tree_cursor.next_ref() {
272267
match tree {
273268
&TokenTree::Token(ref token, spacing) => {
274-
match (self.desugar_doc_comments, token) {
275-
(
276-
true,
277-
&Token { kind: token::DocComment(_, attr_style, data), span },
278-
) => {
279-
let desugared = self.desugar(attr_style, data, span);
280-
self.tree_cursor.replace_prev_and_rewind(desugared);
281-
// Continue to get the first token of the desugared doc comment.
282-
}
283-
_ => {
284-
debug_assert!(!matches!(
285-
token.kind,
286-
token::OpenDelim(_) | token::CloseDelim(_)
287-
));
288-
return (token.clone(), spacing);
289-
}
290-
}
269+
debug_assert!(!matches!(
270+
token.kind,
271+
token::OpenDelim(_) | token::CloseDelim(_)
272+
));
273+
return (token.clone(), spacing);
291274
}
292275
&TokenTree::Delimited(sp, delim, ref tts) => {
293276
let trees = tts.clone().into_trees();
@@ -311,52 +294,6 @@ impl TokenCursor {
311294
}
312295
}
313296
}
314-
315-
// Desugar a doc comment into something like `#[doc = r"foo"]`.
316-
fn desugar(&mut self, attr_style: AttrStyle, data: Symbol, span: Span) -> Vec<TokenTree> {
317-
// Searches for the occurrences of `"#*` and returns the minimum number of `#`s
318-
// required to wrap the text. E.g.
319-
// - `abc d` is wrapped as `r"abc d"` (num_of_hashes = 0)
320-
// - `abc "d"` is wrapped as `r#"abc "d""#` (num_of_hashes = 1)
321-
// - `abc "##d##"` is wrapped as `r###"abc ##"d"##"###` (num_of_hashes = 3)
322-
let mut num_of_hashes = 0;
323-
let mut count = 0;
324-
for ch in data.as_str().chars() {
325-
count = match ch {
326-
'"' => 1,
327-
'#' if count > 0 => count + 1,
328-
_ => 0,
329-
};
330-
num_of_hashes = cmp::max(num_of_hashes, count);
331-
}
332-
333-
// `/// foo` becomes `doc = r"foo"`.
334-
let delim_span = DelimSpan::from_single(span);
335-
let body = TokenTree::Delimited(
336-
delim_span,
337-
Delimiter::Bracket,
338-
[
339-
TokenTree::token_alone(token::Ident(sym::doc, false), span),
340-
TokenTree::token_alone(token::Eq, span),
341-
TokenTree::token_alone(
342-
TokenKind::lit(token::StrRaw(num_of_hashes), data, None),
343-
span,
344-
),
345-
]
346-
.into_iter()
347-
.collect::<TokenStream>(),
348-
);
349-
350-
if attr_style == AttrStyle::Inner {
351-
vec![
352-
TokenTree::token_alone(token::Pound, span),
353-
TokenTree::token_alone(token::Not, span),
354-
body,
355-
]
356-
} else {
357-
vec![TokenTree::token_alone(token::Pound, span), body]
358-
}
359-
}
360297
}
361298

362299
#[derive(Debug, Clone, PartialEq)]
@@ -451,10 +388,14 @@ pub(super) fn token_descr(token: &Token) -> String {
451388
impl<'a> Parser<'a> {
452389
pub fn new(
453390
sess: &'a ParseSess,
454-
tokens: TokenStream,
391+
mut stream: TokenStream,
455392
desugar_doc_comments: bool,
456393
subparser_name: Option<&'static str>,
457394
) -> Self {
395+
if desugar_doc_comments {
396+
stream.desugar_doc_comments();
397+
}
398+
458399
let mut parser = Parser {
459400
sess,
460401
token: Token::dummy(),
@@ -464,10 +405,9 @@ impl<'a> Parser<'a> {
464405
restrictions: Restrictions::empty(),
465406
expected_tokens: Vec::new(),
466407
token_cursor: TokenCursor {
467-
tree_cursor: tokens.into_trees(),
408+
tree_cursor: stream.into_trees(),
468409
stack: Vec::new(),
469410
num_next_calls: 0,
470-
desugar_doc_comments,
471411
break_last_token: false,
472412
},
473413
unmatched_angle_bracket_count: 0,

0 commit comments

Comments
 (0)