Skip to content

incr.comp.: Implement query result cache and use it to cache type checking tables. #46004

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 21 commits into from
Nov 17, 2017
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
8cbc022
incr.comp.: Include header when loading cache files in order to get t…
michaelwoerister Nov 13, 2017
c08e03a
incr.comp.: Add position() method to TyEncoder.
michaelwoerister Nov 13, 2017
bc96d9d
incr.comp.: Implement UseSpecializedXXcodable for DefIndex and DefId.
michaelwoerister Nov 13, 2017
9ac1026
incr.comp.: Properly use ty::codec::decode_cnum() in rustc_metadata::…
michaelwoerister Nov 13, 2017
3bd333c
incr.comp.: Add CacheEncoder for encoding query results into the incr…
michaelwoerister Nov 13, 2017
15db165
incr.comp.: Implement TyDecoder for on_disk_cache::CacheDecoder.
michaelwoerister Nov 14, 2017
bedb44c
incr.comp.: Allow for mapping from prev-session-CrateNums to current-…
michaelwoerister Nov 14, 2017
de0317e
incr.comp.: Encode DefPathTables for reconstructing DefIds.
michaelwoerister Nov 14, 2017
2087d5e
incr.comp.: Do some verification on data decoded from incr. comp. cache.
michaelwoerister Nov 14, 2017
4bfab89
incr.comp.: Store the query result index which records where query re…
michaelwoerister Nov 14, 2017
0b14383
incr.comp.: Add 'tcx to QueryDescription.
michaelwoerister Nov 14, 2017
2c1aedd
incr.comp.: Cache TypeckTables and add -Zincremental-queries flag.
michaelwoerister Nov 14, 2017
279b6df
incr.comp.: Refactor query cache serialization to be more re-usable.
michaelwoerister Nov 15, 2017
13582c6
incr.comp.: Add missing [input] annotation for DepNode::MaybeUnusedEx…
michaelwoerister Nov 15, 2017
2f50e62
incr.comp.: Only save and load query result cache when -Zincremental-…
michaelwoerister Nov 15, 2017
24e54dd
Introduce LocalDefId which provides a type-level guarantee that the D…
michaelwoerister Nov 16, 2017
2f44ef2
incr.comp.: Encode DefIds as DefPathHashes instead of recomputing tho…
michaelwoerister Nov 16, 2017
723028f
incr.comp.: Remove some code duplication around TyDecoder by factorin…
michaelwoerister Nov 16, 2017
cb1ff24
incr.comp.: Remove default serialization implementations for things i…
michaelwoerister Nov 16, 2017
4c4f7a3
Fix some tidy errors in ty::codec.
michaelwoerister Nov 16, 2017
0a1f6dd
Add doc comment for LocalDefId.
michaelwoerister Nov 16, 2017
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/librustc/ty/context.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1311,9 +1311,9 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
pub fn serialize_query_result_cache<E>(self,
encoder: &mut E)
-> Result<(), E::Error>
where E: ::rustc_serialize::Encoder
where E: ty::codec::TyEncoder
{
self.on_disk_query_result_cache.serialize(encoder)
self.on_disk_query_result_cache.serialize(self, encoder)
}

}
Expand Down
133 changes: 116 additions & 17 deletions src/librustc/ty/maps/on_disk_cache.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,17 @@ use rustc_data_structures::fx::FxHashMap;
use rustc_data_structures::indexed_vec::Idx;
use errors::Diagnostic;
use rustc_serialize::{Decodable, Decoder, Encodable, Encoder, opaque,
SpecializedDecoder};
SpecializedDecoder, SpecializedEncoder};
use session::Session;
use std::borrow::Cow;
use std::cell::RefCell;
use std::collections::BTreeMap;
use std::mem;
use syntax::codemap::{CodeMap, StableFilemapId};
use syntax_pos::{BytePos, Span, NO_EXPANSION, DUMMY_SP};
use ty;
use ty::codec::{self as ty_codec};
use ty::context::TyCtxt;

/// `OnDiskCache` provides an interface to incr. comp. data cached from the
/// previous compilation session. This data will eventually include the results
Expand All @@ -46,11 +49,7 @@ struct Header {
prev_filemap_starts: BTreeMap<BytePos, StableFilemapId>,
}

// This type is used only for (de-)serialization.
#[derive(RustcEncodable, RustcDecodable)]
struct Body {
diagnostics: Vec<(SerializedDepNodeIndex, Vec<Diagnostic>)>,
}
type EncodedPrevDiagnostics = Vec<(SerializedDepNodeIndex, Vec<Diagnostic>)>;

impl<'sess> OnDiskCache<'sess> {
/// Create a new OnDiskCache instance from the serialized data in `data`.
Expand All @@ -64,14 +63,21 @@ impl<'sess> OnDiskCache<'sess> {
let mut decoder = opaque::Decoder::new(&data[..], start_pos);
let header = Header::decode(&mut decoder).unwrap();

let prev_diagnostics: FxHashMap<_, _> = {
let prev_diagnostics = {
let mut decoder = CacheDecoder {
opaque: decoder,
codemap: sess.codemap(),
prev_filemap_starts: &header.prev_filemap_starts,
};
let body = Body::decode(&mut decoder).unwrap();
body.diagnostics.into_iter().collect()

let prev_diagnostics: FxHashMap<_, _> = {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do we have this new variable here? It doesn't serve an obvious purpose. =)

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It does in later commits where this block is used to limit the scope of the CacheDecoder.

let diagnostics = EncodedPrevDiagnostics::decode(&mut decoder)
.expect("Error while trying to decode prev. diagnostics \
from incr. comp. cache.");
diagnostics.into_iter().collect()
};

prev_diagnostics
};

OnDiskCache {
Expand All @@ -91,28 +97,38 @@ impl<'sess> OnDiskCache<'sess> {
}
}

pub fn serialize<'a, 'tcx, E>(&self,
encoder: &mut E)
-> Result<(), E::Error>
where E: Encoder
{
pub fn serialize<'a, 'gcx, 'lcx, E>(&self,
tcx: TyCtxt<'a, 'gcx, 'lcx>,
encoder: &mut E)
-> Result<(), E::Error>
where E: ty_codec::TyEncoder
{
// Serializing the DepGraph should not modify it:
let _in_ignore = tcx.dep_graph.in_ignore();

let mut encoder = CacheEncoder {
encoder,
type_shorthands: FxHashMap(),
predicate_shorthands: FxHashMap(),
};

let prev_filemap_starts: BTreeMap<_, _> = self
.codemap
.files()
.iter()
.map(|fm| (fm.start_pos, StableFilemapId::new(fm)))
.collect();

Header { prev_filemap_starts }.encode(encoder)?;
Header { prev_filemap_starts }.encode(&mut encoder)?;

let diagnostics: Vec<(SerializedDepNodeIndex, Vec<Diagnostic>)> =
let diagnostics: EncodedPrevDiagnostics =
self.current_diagnostics
.borrow()
.iter()
.map(|(k, v)| (SerializedDepNodeIndex::new(k.index()), v.clone()))
.collect();

Body { diagnostics }.encode(encoder)?;
diagnostics.encode(&mut encoder)?;

Ok(())
}
Expand Down Expand Up @@ -152,6 +168,9 @@ impl<'sess> OnDiskCache<'sess> {
}
}


//- DECODING -------------------------------------------------------------------

/// A decoder that can read the incr. comp. cache. It is similar to the one
/// we use for crate metadata decoding in that it can rebase spans and
/// eventually will also handle things that contain `Ty` instances.
Expand Down Expand Up @@ -229,3 +248,83 @@ impl<'a> SpecializedDecoder<Span> for CacheDecoder<'a> {
Ok(DUMMY_SP)
}
}


//- ENCODING -------------------------------------------------------------------

struct CacheEncoder<'enc, 'tcx, E>
where E: 'enc + ty_codec::TyEncoder
{
encoder: &'enc mut E,
type_shorthands: FxHashMap<ty::Ty<'tcx>, usize>,
predicate_shorthands: FxHashMap<ty::Predicate<'tcx>, usize>,
}

impl<'enc, 'tcx, E> ty_codec::TyEncoder for CacheEncoder<'enc, 'tcx, E>
where E: 'enc + ty_codec::TyEncoder
{
fn position(&self) -> usize {
self.encoder.position()
}
}

impl<'enc, 'tcx, E> SpecializedEncoder<ty::Ty<'tcx>> for CacheEncoder<'enc, 'tcx, E>
where E: 'enc + ty_codec::TyEncoder
{
fn specialized_encode(&mut self, ty: &ty::Ty<'tcx>) -> Result<(), Self::Error> {
ty_codec::encode_with_shorthand(self, ty,
|encoder| &mut encoder.type_shorthands)
}
}

impl<'enc, 'tcx, E> SpecializedEncoder<ty::GenericPredicates<'tcx>>
for CacheEncoder<'enc, 'tcx, E>
where E: 'enc + ty_codec::TyEncoder
{
fn specialized_encode(&mut self,
predicates: &ty::GenericPredicates<'tcx>)
-> Result<(), Self::Error> {
ty_codec::encode_predicates(self, predicates,
|encoder| &mut encoder.predicate_shorthands)
}
}

macro_rules! encoder_methods {
($($name:ident($ty:ty);)*) => {
$(fn $name(&mut self, value: $ty) -> Result<(), Self::Error> {
self.encoder.$name(value)
})*
}
}

impl<'enc, 'tcx, E> Encoder for CacheEncoder<'enc, 'tcx, E>
where E: 'enc + ty_codec::TyEncoder
{
type Error = E::Error;

fn emit_nil(&mut self) -> Result<(), Self::Error> {
Ok(())
}

encoder_methods! {
emit_usize(usize);
emit_u128(u128);
emit_u64(u64);
emit_u32(u32);
emit_u16(u16);
emit_u8(u8);

emit_isize(isize);
emit_i128(i128);
emit_i64(i64);
emit_i32(i32);
emit_i16(i16);
emit_i8(i8);

emit_bool(bool);
emit_f64(f64);
emit_f32(f32);
emit_char(char);
emit_str(&str);
}
}