Skip to content

Commit aa6fba9

Browse files
petrochenkovpetrochenkov
committed
syntax: Use Token in Parser 
1 parent c0c57ac commit aa6fba9

File tree

10 files changed

+126
-126
lines changed

10 files changed

+126
-126
lines changed

‎src/libsyntax/ext/tt/macro_parser.rs

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -273,7 +273,7 @@ pub enum ParseResult<T> {
273273
Success(T),
274274
/// Arm failed to match. If the second parameter is `token::Eof`, it indicates an unexpected
275275
/// end of macro invocation. Otherwise, it indicates that no rules expected the given token.
276-
Failure(syntax_pos::Span, TokenKind, &'static str),
276+
Failure(Token, &'static str),
277277
/// Fatal error (malformed macro?). Abort compilation.
278278
Error(syntax_pos::Span, String),
279279
}
@@ -701,7 +701,7 @@ pub fn parse(
701701
parser.span,
702702
) {
703703
Success(_) => {}
704-
Failure(sp, tok, t) => return Failure(sp, tok, t),
704+
Failure(token, msg) => return Failure(token, msg),
705705
Error(sp, msg) => return Error(sp, msg),
706706
}
707707

@@ -727,13 +727,13 @@ pub fn parse(
727727
"ambiguity: multiple successful parses".to_string(),
728728
);
729729
} else {
730+
let span = if parser.span.is_dummy() {
731+
parser.span
732+
} else {
733+
sess.source_map().next_point(parser.span)
734+
};
730735
return Failure(
731-
if parser.span.is_dummy() {
732-
parser.span
733-
} else {
734-
sess.source_map().next_point(parser.span)
735-
},
736-
token::Eof,
736+
Token { kind: token::Eof, span },
737737
"missing tokens in macro arguments",
738738
);
739739
}
@@ -771,7 +771,6 @@ pub fn parse(
771771
// then there is a syntax error.
772772
else if bb_items.is_empty() && next_items.is_empty() {
773773
return Failure(
774-
parser.span,
775774
parser.token.clone(),
776775
"no rules expected this token in macro call",
777776
);

‎src/libsyntax/ext/tt/macro_rules.rs

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -190,10 +190,10 @@ fn generic_extension<'cx>(cx: &'cx mut ExtCtxt<'_>,
190190
arm_span,
191191
})
192192
}
193-
Failure(sp, tok, t) => if sp.lo() >= best_fail_spot.lo() {
194-
best_fail_spot = sp;
195-
best_fail_tok = Some(tok);
196-
best_fail_text = Some(t);
193+
Failure(token, msg) => if token.span.lo() >= best_fail_spot.lo() {
194+
best_fail_spot = token.span;
195+
best_fail_tok = Some(token.kind);
196+
best_fail_text = Some(msg);
197197
},
198198
Error(err_sp, ref msg) => {
199199
cx.span_fatal(err_sp.substitute_dummy(sp), &msg[..])
@@ -288,11 +288,11 @@ pub fn compile(
288288

289289
let argument_map = match parse(sess, body.stream(), &argument_gram, None, true) {
290290
Success(m) => m,
291-
Failure(sp, tok, t) => {
292-
let s = parse_failure_msg(tok);
293-
let sp = sp.substitute_dummy(def.span);
291+
Failure(token, msg) => {
292+
let s = parse_failure_msg(token.kind);
293+
let sp = token.span.substitute_dummy(def.span);
294294
let mut err = sess.span_diagnostic.struct_span_fatal(sp, &s);
295-
err.span_label(sp, t);
295+
err.span_label(sp, msg);
296296
err.emit();
297297
FatalError.raise();
298298
}

‎src/libsyntax/parse/attr.rs

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ impl<'a> Parser<'a> {
2424
let mut just_parsed_doc_comment = false;
2525
loop {
2626
debug!("parse_outer_attributes: self.token={:?}", self.token);
27-
match self.token {
27+
match self.token.kind {
2828
token::Pound => {
2929
let inner_error_reason = if just_parsed_doc_comment {
3030
"an inner attribute is not permitted following an outer doc comment"
@@ -81,7 +81,7 @@ impl<'a> Parser<'a> {
8181
debug!("parse_attribute_with_inner_parse_policy: inner_parse_policy={:?} self.token={:?}",
8282
inner_parse_policy,
8383
self.token);
84-
let (span, path, tokens, style) = match self.token {
84+
let (span, path, tokens, style) = match self.token.kind {
8585
token::Pound => {
8686
let lo = self.span;
8787
self.bump();
@@ -140,7 +140,7 @@ impl<'a> Parser<'a> {
140140
/// PATH `=` TOKEN_TREE
141141
/// The delimiters or `=` are still put into the resulting token stream.
142142
crate fn parse_meta_item_unrestricted(&mut self) -> PResult<'a, (ast::Path, TokenStream)> {
143-
let meta = match self.token {
143+
let meta = match self.token.kind {
144144
token::Interpolated(ref nt) => match **nt {
145145
Nonterminal::NtMeta(ref meta) => Some(meta.clone()),
146146
_ => None,
@@ -159,7 +159,7 @@ impl<'a> Parser<'a> {
159159
} else if self.eat(&token::Eq) {
160160
let eq = TokenTree::token(self.prev_span, token::Eq);
161161
let mut is_interpolated_expr = false;
162-
if let token::Interpolated(nt) = &self.token {
162+
if let token::Interpolated(nt) = &self.token.kind {
163163
if let token::NtExpr(..) = **nt {
164164
is_interpolated_expr = true;
165165
}
@@ -188,7 +188,7 @@ impl<'a> Parser<'a> {
188188
crate fn parse_inner_attributes(&mut self) -> PResult<'a, Vec<ast::Attribute>> {
189189
let mut attrs: Vec<ast::Attribute> = vec![];
190190
loop {
191-
match self.token {
191+
match self.token.kind {
192192
token::Pound => {
193193
// Don't even try to parse if it's not an inner attribute.
194194
if !self.look_ahead(1, |t| t == &token::Not) {
@@ -236,7 +236,7 @@ impl<'a> Parser<'a> {
236236
/// meta_item : IDENT ( '=' UNSUFFIXED_LIT | '(' meta_item_inner? ')' )? ;
237237
/// meta_item_inner : (meta_item | UNSUFFIXED_LIT) (',' meta_item_inner)? ;
238238
pub fn parse_meta_item(&mut self) -> PResult<'a, ast::MetaItem> {
239-
let nt_meta = match self.token {
239+
let nt_meta = match self.token.kind {
240240
token::Interpolated(ref nt) => match **nt {
241241
token::NtMeta(ref e) => Some(e.clone()),
242242
_ => None,

‎src/libsyntax/parse/diagnostics.rs

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -201,7 +201,7 @@ impl<'a> Parser<'a> {
201201
self.span,
202202
&format!("expected identifier, found {}", self.this_token_descr()),
203203
);
204-
if let token::Ident(ident, false) = &self.token {
204+
if let token::Ident(ident, false) = &self.token.kind {
205205
if ident.is_raw_guess() {
206206
err.span_suggestion(
207207
self.span,
@@ -730,7 +730,7 @@ impl<'a> Parser<'a> {
730730
) -> PResult<'a, bool /* recovered */> {
731731
let token_str = pprust::token_to_string(t);
732732
let this_token_str = self.this_token_descr();
733-
let (prev_sp, sp) = match (&self.token, self.subparser_name) {
733+
let (prev_sp, sp) = match (&self.token.kind, self.subparser_name) {
734734
// Point at the end of the macro call when reaching end of macro arguments.
735735
(token::Eof, Some(_)) => {
736736
let sp = self.sess.source_map().next_point(self.span);
@@ -746,7 +746,7 @@ impl<'a> Parser<'a> {
746746
let msg = format!(
747747
"expected `{}`, found {}",
748748
token_str,
749-
match (&self.token, self.subparser_name) {
749+
match (&self.token.kind, self.subparser_name) {
750750
(token::Eof, Some(origin)) => format!("end of {}", origin),
751751
_ => this_token_str,
752752
},
@@ -989,7 +989,7 @@ impl<'a> Parser<'a> {
989989
break_on_semi, break_on_block);
990990
loop {
991991
debug!("recover_stmt_ loop {:?}", self.token);
992-
match self.token {
992+
match self.token.kind {
993993
token::OpenDelim(token::DelimToken::Brace) => {
994994
brace_depth += 1;
995995
self.bump();
@@ -1074,7 +1074,7 @@ impl<'a> Parser<'a> {
10741074
}
10751075

10761076
crate fn eat_incorrect_doc_comment(&mut self, applied_to: &str) {
1077-
if let token::DocComment(_) = self.token {
1077+
if let token::DocComment(_) = self.token.kind {
10781078
let mut err = self.diagnostic().struct_span_err(
10791079
self.span,
10801080
&format!("documentation comments cannot be applied to {}", applied_to),
@@ -1214,7 +1214,7 @@ impl<'a> Parser<'a> {
12141214
}
12151215

12161216
crate fn expected_expression_found(&self) -> DiagnosticBuilder<'a> {
1217-
let (span, msg) = match (&self.token, self.subparser_name) {
1217+
let (span, msg) = match (&self.token.kind, self.subparser_name) {
12181218
(&token::Eof, Some(origin)) => {
12191219
let sp = self.sess.source_map().next_point(self.span);
12201220
(sp, format!("expected expression, found end of {}", origin))

‎src/libsyntax/parse/literal.rs

Lines changed: 12 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
use crate::ast::{self, Ident, Lit, LitKind};
44
use crate::parse::parser::Parser;
55
use crate::parse::PResult;
6-
use crate::parse::token::{self, TokenKind};
6+
use crate::parse::token::{self, Token, TokenKind};
77
use crate::parse::unescape::{unescape_str, unescape_char, unescape_byte_str, unescape_byte};
88
use crate::print::pprust;
99
use crate::symbol::{kw, sym, Symbol};
@@ -272,44 +272,42 @@ impl<'a> Parser<'a> {
272272
if self.token == token::Dot {
273273
// Attempt to recover `.4` as `0.4`.
274274
recovered = self.look_ahead(1, |t| {
275-
if let token::Literal(token::Lit { kind: token::Integer, symbol, suffix }) = *t {
275+
if let token::Literal(token::Lit { kind: token::Integer, symbol, suffix }) = t.kind {
276276
let next_span = self.look_ahead_span(1);
277277
if self.span.hi() == next_span.lo() {
278278
let s = String::from("0.") + &symbol.as_str();
279-
let token = TokenKind::lit(token::Float, Symbol::intern(&s), suffix);
280-
return Some((token, self.span.to(next_span)));
279+
let kind = TokenKind::lit(token::Float, Symbol::intern(&s), suffix);
280+
return Some(Token { kind, span: self.span.to(next_span) });
281281
}
282282
}
283283
None
284284
});
285-
if let Some((ref token, span)) = recovered {
285+
if let Some(token) = &recovered {
286286
self.bump();
287287
self.diagnostic()
288-
.struct_span_err(span, "float literals must have an integer part")
288+
.struct_span_err(token.span, "float literals must have an integer part")
289289
.span_suggestion(
290-
span,
290+
token.span,
291291
"must have an integer part",
292-
pprust::token_to_string(&token),
292+
pprust::token_to_string(token),
293293
Applicability::MachineApplicable,
294294
)
295295
.emit();
296296
}
297297
}
298298

299-
let (token, span) = recovered.as_ref().map_or((&self.token, self.span),
300-
|(token, span)| (token, *span));
301-
302-
match Lit::from_token(token, span) {
299+
let token = recovered.as_ref().unwrap_or(&self.token);
300+
match Lit::from_token(token, token.span) {
303301
Ok(lit) => {
304302
self.bump();
305303
Ok(lit)
306304
}
307305
Err(LitError::NotLiteral) => {
308306
let msg = format!("unexpected token: {}", self.this_token_descr());
309-
Err(self.span_fatal(span, &msg))
307+
Err(self.span_fatal(token.span, &msg))
310308
}
311309
Err(err) => {
312-
let lit = token.expect_lit();
310+
let (lit, span) = (token.expect_lit(), token.span);
313311
self.bump();
314312
err.report(&self.sess.span_diagnostic, lit, span);
315313
let lit = token::Lit::new(token::Err, lit.symbol, lit.suffix);

‎src/libsyntax/parse/mod.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -239,7 +239,7 @@ fn maybe_source_file_to_parser(
239239
let mut parser = stream_to_parser(sess, stream, None);
240240
parser.unclosed_delims = unclosed_delims;
241241
if parser.token == token::Eof && parser.span.is_dummy() {
242-
parser.span = Span::new(end_pos, end_pos, parser.span.ctxt());
242+
parser.token.span = Span::new(end_pos, end_pos, parser.span.ctxt());
243243
}
244244

245245
Ok(parser)

0 commit comments

Comments
 (0)