Skip to content

Commit 1764b29

Browse files
borsbors
committed
Auto merge of #60679 - petrochenkov:lit2, r=matklad
Keep original literal tokens in AST The original literal tokens (`token::Lit`) are kept in AST until lowering to HIR. The tokens are kept together with their lowered "semantic" representation (`ast::LitKind`), so the size of `ast::Lit` is increased (this also increases the size of meta-item structs used for processing built-in attributes). However, the size of `ast::Expr` stays the same. The intent is to remove the "semantic" representation from AST eventually and keep literals as tokens until lowering to HIR (at least), and I'm going to work on that, but it would be good to land this sooner to unblock progress on the [lexer refactoring](#59706). Fixes a part of #43081 (literal tokens that are passed to proc macros are always precise, including hexadecimal numbers, strings with their original escaping, etc) Fixes a part of #60495 (everything except for proc macro API doesn't need escaping anymore) This also allows to eliminate a certain hack from the lexer (https://rust-lang.zulipchat.com/#narrow/stream/131828-t-compiler/topic/pretty-printing.20comments/near/165005357). cc @matklad
2 parents 0df1e57 + 83ed781 commit 1764b29

28 files changed

+671
-796
lines changed

Diff for: ‎src/librustc/hir/lowering.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -4100,7 +4100,7 @@ impl<'a> LoweringContext<'a> {
41004100
let ohs = P(self.lower_expr(ohs));
41014101
hir::ExprKind::Unary(op, ohs)
41024102
}
4103-
ExprKind::Lit(ref l) => hir::ExprKind::Lit((*l).clone()),
4103+
ExprKind::Lit(ref l) => hir::ExprKind::Lit(respan(l.span, l.node.clone())),
41044104
ExprKind::Cast(ref expr, ref ty) => {
41054105
let expr = P(self.lower_expr(expr));
41064106
hir::ExprKind::Cast(expr, self.lower_ty(ty, ImplTraitContext::disallowed()))

Diff for: ‎src/librustc/hir/mod.rs

+4-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ use syntax_pos::{Span, DUMMY_SP, symbol::InternedString};
2020
use syntax::source_map::Spanned;
2121
use rustc_target::spec::abi::Abi;
2222
use syntax::ast::{self, CrateSugar, Ident, Name, NodeId, AsmDialect};
23-
use syntax::ast::{Attribute, Label, Lit, StrStyle, FloatTy, IntTy, UintTy};
23+
use syntax::ast::{Attribute, Label, LitKind, StrStyle, FloatTy, IntTy, UintTy};
2424
use syntax::attr::{InlineAttr, OptimizeAttr};
2525
use syntax::ext::hygiene::SyntaxContext;
2626
use syntax::ptr::P;
@@ -1331,6 +1331,9 @@ impl BodyOwnerKind {
13311331
}
13321332
}
13331333

1334+
/// A literal.
1335+
pub type Lit = Spanned<LitKind>;
1336+
13341337
/// A constant (expression) that's not an item or associated item,
13351338
/// but needs its own `DefId` for type-checking, const-eval, etc.
13361339
/// These are usually found nested inside types (e.g., array lengths)

Diff for: ‎src/librustc/hir/print.rs

+14-36
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ use syntax::parse::ParseSess;
55
use syntax::parse::lexer::comments;
66
use syntax::print::pp::{self, Breaks};
77
use syntax::print::pp::Breaks::{Consistent, Inconsistent};
8-
use syntax::print::pprust::PrintState;
8+
use syntax::print::pprust::{self, PrintState};
99
use syntax::ptr::P;
1010
use syntax::symbol::keywords;
1111
use syntax::util::parser::{self, AssocOp, Fixity};
@@ -18,7 +18,6 @@ use crate::hir::{GenericParam, GenericParamKind, GenericArg};
1818
use std::borrow::Cow;
1919
use std::cell::Cell;
2020
use std::io::{self, Write, Read};
21-
use std::iter::Peekable;
2221
use std::vec;
2322

2423
pub enum AnnNode<'a> {
@@ -76,7 +75,6 @@ pub struct State<'a> {
7675
pub s: pp::Printer<'a>,
7776
cm: Option<&'a SourceMap>,
7877
comments: Option<Vec<comments::Comment>>,
79-
literals: Peekable<vec::IntoIter<comments::Literal>>,
8078
cur_cmnt: usize,
8179
boxes: Vec<pp::Breaks>,
8280
ann: &'a (dyn PpAnn + 'a),
@@ -98,14 +96,6 @@ impl<'a> PrintState<'a> for State<'a> {
9896
fn cur_cmnt(&mut self) -> &mut usize {
9997
&mut self.cur_cmnt
10098
}
101-
102-
fn cur_lit(&mut self) -> Option<&comments::Literal> {
103-
self.literals.peek()
104-
}
105-
106-
fn bump_lit(&mut self) -> Option<comments::Literal> {
107-
self.literals.next()
108-
}
10999
}
110100

111101
#[allow(non_upper_case_globals)]
@@ -116,18 +106,16 @@ pub const default_columns: usize = 78;
116106

117107

118108
/// Requires you to pass an input filename and reader so that
119-
/// it can scan the input text for comments and literals to
120-
/// copy forward.
109+
/// it can scan the input text for comments to copy forward.
121110
pub fn print_crate<'a>(cm: &'a SourceMap,
122111
sess: &ParseSess,
123112
krate: &hir::Crate,
124113
filename: FileName,
125114
input: &mut dyn Read,
126115
out: Box<dyn Write + 'a>,
127-
ann: &'a dyn PpAnn,
128-
is_expanded: bool)
116+
ann: &'a dyn PpAnn)
129117
-> io::Result<()> {
130-
let mut s = State::new_from_input(cm, sess, filename, input, out, ann, is_expanded);
118+
let mut s = State::new_from_input(cm, sess, filename, input, out, ann);
131119

132120
// When printing the AST, we sometimes need to inject `#[no_std]` here.
133121
// Since you can't compile the HIR, it's not necessary.
@@ -143,36 +131,21 @@ impl<'a> State<'a> {
143131
filename: FileName,
144132
input: &mut dyn Read,
145133
out: Box<dyn Write + 'a>,
146-
ann: &'a dyn PpAnn,
147-
is_expanded: bool)
134+
ann: &'a dyn PpAnn)
148135
-> State<'a> {
149-
let (cmnts, lits) = comments::gather_comments_and_literals(sess, filename, input);
150-
151-
State::new(cm,
152-
out,
153-
ann,
154-
Some(cmnts),
155-
// If the code is post expansion, don't use the table of
156-
// literals, since it doesn't correspond with the literals
157-
// in the AST anymore.
158-
if is_expanded {
159-
None
160-
} else {
161-
Some(lits)
162-
})
136+
let comments = comments::gather_comments(sess, filename, input);
137+
State::new(cm, out, ann, Some(comments))
163138
}
164139

165140
pub fn new(cm: &'a SourceMap,
166141
out: Box<dyn Write + 'a>,
167142
ann: &'a dyn PpAnn,
168-
comments: Option<Vec<comments::Comment>>,
169-
literals: Option<Vec<comments::Literal>>)
143+
comments: Option<Vec<comments::Comment>>)
170144
-> State<'a> {
171145
State {
172146
s: pp::mk_printer(out, default_columns),
173147
cm: Some(cm),
174148
comments,
175-
literals: literals.unwrap_or_default().into_iter().peekable(),
176149
cur_cmnt: 0,
177150
boxes: Vec::new(),
178151
ann,
@@ -189,7 +162,6 @@ pub fn to_string<F>(ann: &dyn PpAnn, f: F) -> String
189162
s: pp::mk_printer(Box::new(&mut wr), default_columns),
190163
cm: None,
191164
comments: None,
192-
literals: vec![].into_iter().peekable(),
193165
cur_cmnt: 0,
194166
boxes: Vec::new(),
195167
ann,
@@ -1276,6 +1248,12 @@ impl<'a> State<'a> {
12761248
self.print_expr_maybe_paren(expr, parser::PREC_PREFIX)
12771249
}
12781250

1251+
fn print_literal(&mut self, lit: &hir::Lit) -> io::Result<()> {
1252+
self.maybe_print_comment(lit.span.lo())?;
1253+
let (token, suffix) = lit.node.to_lit_token();
1254+
self.writer().word(pprust::literal_to_string(token, suffix))
1255+
}
1256+
12791257
pub fn print_expr(&mut self, expr: &hir::Expr) -> io::Result<()> {
12801258
self.maybe_print_comment(expr.span.lo())?;
12811259
self.print_outer_attributes(&expr.attrs)?;

Diff for: ‎src/librustc/ich/impls_syntax.rs

+24-17
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,13 @@ impl_stable_hash_for!(enum ::syntax::ast::LitIntType {
162162
Unsuffixed
163163
});
164164

165-
impl_stable_hash_for_spanned!(::syntax::ast::LitKind);
165+
impl_stable_hash_for!(struct ::syntax::ast::Lit {
166+
node,
167+
token,
168+
suffix,
169+
span
170+
});
171+
166172
impl_stable_hash_for!(enum ::syntax::ast::LitKind {
167173
Str(value, style),
168174
Err(value),
@@ -175,6 +181,8 @@ impl_stable_hash_for!(enum ::syntax::ast::LitKind {
175181
Bool(value)
176182
});
177183

184+
impl_stable_hash_for_spanned!(::syntax::ast::LitKind);
185+
178186
impl_stable_hash_for!(enum ::syntax::ast::IntTy { Isize, I8, I16, I32, I64, I128 });
179187
impl_stable_hash_for!(enum ::syntax::ast::UintTy { Usize, U8, U16, U32, U64, U128 });
180188
impl_stable_hash_for!(enum ::syntax::ast::FloatTy { F32, F64 });
@@ -280,6 +288,19 @@ for tokenstream::TokenStream {
280288
}
281289
}
282290

291+
impl_stable_hash_for!(enum token::Lit {
292+
Bool(val),
293+
Byte(val),
294+
Char(val),
295+
Err(val),
296+
Integer(val),
297+
Float(val),
298+
Str_(val),
299+
ByteStr(val),
300+
StrRaw(val, n),
301+
ByteStrRaw(val, n)
302+
});
303+
283304
fn hash_token<'a, 'gcx, W: StableHasherResult>(
284305
token: &token::Token,
285306
hcx: &mut StableHashingContext<'a>,
@@ -327,22 +348,8 @@ fn hash_token<'a, 'gcx, W: StableHasherResult>(
327348
token::Token::CloseDelim(delim_token) => {
328349
std_hash::Hash::hash(&delim_token, hasher);
329350
}
330-
token::Token::Literal(ref lit, ref opt_name) => {
331-
mem::discriminant(lit).hash_stable(hcx, hasher);
332-
match *lit {
333-
token::Lit::Byte(val) |
334-
token::Lit::Char(val) |
335-
token::Lit::Err(val) |
336-
token::Lit::Integer(val) |
337-
token::Lit::Float(val) |
338-
token::Lit::Str_(val) |
339-
token::Lit::ByteStr(val) => val.hash_stable(hcx, hasher),
340-
token::Lit::StrRaw(val, n) |
341-
token::Lit::ByteStrRaw(val, n) => {
342-
val.hash_stable(hcx, hasher);
343-
n.hash_stable(hcx, hasher);
344-
}
345-
};
351+
token::Token::Literal(lit, opt_name) => {
352+
lit.hash_stable(hcx, hasher);
346353
opt_name.hash_stable(hcx, hasher);
347354
}
348355

Diff for: ‎src/librustc_driver/pretty.rs

+2-4
Original file line numberDiff line numberDiff line change
@@ -805,8 +805,7 @@ pub fn print_after_hir_lowering<'tcx>(
805805
src_name,
806806
&mut rdr,
807807
box out,
808-
annotation.pp_ann(),
809-
true)
808+
annotation.pp_ann())
810809
})
811810
}
812811

@@ -829,8 +828,7 @@ pub fn print_after_hir_lowering<'tcx>(
829828
src_name,
830829
&mut rdr,
831830
box out,
832-
annotation.pp_ann(),
833-
true);
831+
annotation.pp_ann());
834832
for node_id in uii.all_matching_node_ids(hir_map) {
835833
let node = hir_map.get(node_id);
836834
pp_state.print_node(node)?;

Diff for: ‎src/librustc_lint/types.rs

+5-5
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ impl TypeLimits {
6262
/// Returns `true` iff the lint was overridden.
6363
fn lint_overflowing_range_endpoint<'a, 'tcx>(
6464
cx: &LateContext<'a, 'tcx>,
65-
lit: &ast::Lit,
65+
lit: &hir::Lit,
6666
lit_val: u128,
6767
max: u128,
6868
expr: &'tcx hir::Expr,
@@ -132,7 +132,7 @@ fn uint_ty_range(uint_ty: ast::UintTy) -> (u128, u128) {
132132
}
133133
}
134134

135-
fn get_bin_hex_repr(cx: &LateContext<'_, '_>, lit: &ast::Lit) -> Option<String> {
135+
fn get_bin_hex_repr(cx: &LateContext<'_, '_>, lit: &hir::Lit) -> Option<String> {
136136
let src = cx.sess().source_map().span_to_snippet(lit.span).ok()?;
137137
let firstch = src.chars().next()?;
138138

@@ -249,7 +249,7 @@ fn lint_int_literal<'a, 'tcx>(
249249
cx: &LateContext<'a, 'tcx>,
250250
type_limits: &TypeLimits,
251251
e: &'tcx hir::Expr,
252-
lit: &ast::Lit,
252+
lit: &hir::Lit,
253253
t: ast::IntTy,
254254
v: u128,
255255
) {
@@ -301,7 +301,7 @@ fn lint_int_literal<'a, 'tcx>(
301301
fn lint_uint_literal<'a, 'tcx>(
302302
cx: &LateContext<'a, 'tcx>,
303303
e: &'tcx hir::Expr,
304-
lit: &ast::Lit,
304+
lit: &hir::Lit,
305305
t: ast::UintTy,
306306
) {
307307
let uint_type = if let ast::UintTy::Usize = t {
@@ -363,7 +363,7 @@ fn lint_literal<'a, 'tcx>(
363363
cx: &LateContext<'a, 'tcx>,
364364
type_limits: &TypeLimits,
365365
e: &'tcx hir::Expr,
366-
lit: &ast::Lit,
366+
lit: &hir::Lit,
367367
) {
368368
match cx.tables.node_type(e.hir_id).sty {
369369
ty::Int(t) => {

Diff for: ‎src/librustc_typeck/check/mod.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -3083,7 +3083,7 @@ impl<'a, 'gcx, 'tcx> FnCtxt<'a, 'gcx, 'tcx> {
30833083

30843084
// AST fragment checking
30853085
fn check_lit(&self,
3086-
lit: &ast::Lit,
3086+
lit: &hir::Lit,
30873087
expected: Expectation<'tcx>)
30883088
-> Ty<'tcx>
30893089
{

Diff for: ‎src/librustdoc/clean/cfg.rs

+13-15
Original file line numberDiff line numberDiff line change
@@ -414,10 +414,11 @@ impl<'a> fmt::Display for Html<'a> {
414414
mod test {
415415
use super::Cfg;
416416

417-
use syntax::symbol::Symbol;
417+
use syntax_pos::DUMMY_SP;
418418
use syntax::ast::*;
419+
use syntax::attr;
419420
use syntax::source_map::dummy_spanned;
420-
use syntax_pos::DUMMY_SP;
421+
use syntax::symbol::Symbol;
421422
use syntax::with_globals;
422423

423424
fn word_cfg(s: &str) -> Cfg {
@@ -592,14 +593,10 @@ mod test {
592593
let mi = dummy_meta_item_word("all");
593594
assert_eq!(Cfg::parse(&mi), Ok(word_cfg("all")));
594595

595-
let mi = MetaItem {
596-
path: Path::from_ident(Ident::from_str("all")),
597-
node: MetaItemKind::NameValue(dummy_spanned(LitKind::Str(
598-
Symbol::intern("done"),
599-
StrStyle::Cooked,
600-
))),
601-
span: DUMMY_SP,
602-
};
596+
let mi = attr::mk_name_value_item_str(
597+
Ident::from_str("all"),
598+
dummy_spanned(Symbol::intern("done"))
599+
);
603600
assert_eq!(Cfg::parse(&mi), Ok(name_value_cfg("all", "done")));
604601

605602
let mi = dummy_meta_item_list!(all, [a, b]);
@@ -627,11 +624,12 @@ mod test {
627624
#[test]
628625
fn test_parse_err() {
629626
with_globals(|| {
630-
let mi = MetaItem {
631-
path: Path::from_ident(Ident::from_str("foo")),
632-
node: MetaItemKind::NameValue(dummy_spanned(LitKind::Bool(false))),
633-
span: DUMMY_SP,
634-
};
627+
let mi = attr::mk_name_value_item(
628+
DUMMY_SP,
629+
Ident::from_str("foo"),
630+
LitKind::Bool(false),
631+
DUMMY_SP,
632+
);
635633
assert!(Cfg::parse(&mi).is_err());
636634

637635
let mi = dummy_meta_item_list!(not, [a, b]);

Diff for: ‎src/librustdoc/html/highlight.rs

+2
Original file line numberDiff line numberDiff line change
@@ -318,6 +318,8 @@ impl<'a> Classifier<'a> {
318318

319319
// Number literals.
320320
token::Integer(..) | token::Float(..) => Class::Number,
321+
322+
token::Bool(..) => panic!("literal token contains `Lit::Bool`"),
321323
}
322324
}
323325

Diff for: ‎src/libsyntax/ast.rs

+14-2
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ pub use crate::symbol::{Ident, Symbol as Name};
66
pub use crate::util::parser::ExprPrecedence;
77

88
use crate::ext::hygiene::{Mark, SyntaxContext};
9+
use crate::parse::token;
910
use crate::print::pprust;
1011
use crate::ptr::P;
1112
use crate::source_map::{dummy_spanned, respan, Spanned};
@@ -1350,8 +1351,19 @@ pub enum StrStyle {
13501351
Raw(u16),
13511352
}
13521353

1353-
/// A literal.
1354-
pub type Lit = Spanned<LitKind>;
1354+
/// An AST literal.
1355+
#[derive(Clone, RustcEncodable, RustcDecodable, Debug)]
1356+
pub struct Lit {
1357+
/// The original literal token as written in source code.
1358+
pub token: token::Lit,
1359+
/// The original literal suffix as written in source code.
1360+
pub suffix: Option<Symbol>,
1361+
/// The "semantic" representation of the literal lowered from the original tokens.
1362+
/// Strings are unescaped, hexadecimal forms are eliminated, etc.
1363+
/// FIXME: Remove this and only create the semantic representation during lowering to HIR.
1364+
pub node: LitKind,
1365+
pub span: Span,
1366+
}
13551367

13561368
#[derive(Clone, RustcEncodable, RustcDecodable, Debug, Copy, Hash, PartialEq)]
13571369
pub enum LitIntType {

0 commit comments

Comments
 (0)