Auto merge of #30140 - michaelwoerister:tls-encoding, r=nikomatsakis

With this commit, metadata encoding and decoding can make use of thread-local encoding and decoding contexts. These allow implementers of `serialize::Encodable` and `Decodable` to access information and
datastructures that would otherwise not be available to them. For example, we can automatically translate def-id and span information during decoding because the decoding context knows which crate the data is decoded from. Or it allows to make `ty::Ty` decodable because the context has access to the `ty::ctxt` that is needed for creating `ty::Ty` instances.

Some notes:
- `tls::with_encoding_context()` and `tls::with_decoding_context()` (as opposed to their unsafe versions) try to prevent the TLS data getting out-of-sync by making sure that the encoder/decoder passed in is actually the same as the one stored in the context. This should prevent accidentally reading from the wrong decoder.
- There are no real tests in this PR. I had a unit tests for some of the core aspects of the TLS implementation but it was kind of brittle, a lot of code for mocking `ty::ctxt`, `crate_metadata`, etc and did actually test not so much. The code will soon be tested by the first incremental compilation auto-tests that rely on MIR being properly serialized. However, if people think that some tests should be added before this can land, I'll try to provide some that make sense.

r? @nikomatsakis

Author: bors

mk/crates.mk

@@ -88,7 +88,7 @@ DEPS_test := std getopts serialize rbml term native:rust_test_helpers
 
 DEPS_syntax := std term serialize log fmt_macros arena libc rustc_bitflags
 
-DEPS_rustc := syntax flate arena serialize getopts rustc_front\
+DEPS_rustc := syntax flate arena serialize getopts rbml rustc_front\
               log graphviz rustc_llvm rustc_back rustc_data_structures
 DEPS_rustc_back := std syntax rustc_llvm rustc_front flate log libc
 DEPS_rustc_borrowck := rustc rustc_front log graphviz syntax

src/librbml/lib.rs

@@ -638,6 +638,14 @@ pub mod reader {
             self.pos = old_pos;
             Ok(result)
         }
+
+        pub fn position(&self) -> usize {
+            self.pos
+        }
+
+        pub fn advance(&mut self, bytes: usize) {
+            self.pos += bytes;
+        }
     }
 
     impl<'doc> serialize::Decoder for Decoder<'doc> {

src/librustc/lib.rs

@@ -59,6 +59,7 @@ extern crate fmt_macros;
 extern crate getopts;
 extern crate graphviz;
 extern crate libc;
+extern crate rbml;
 extern crate rustc_llvm;
 extern crate rustc_back;
 extern crate rustc_front;

src/librustc/middle/cstore.rs

@@ -407,3 +407,141 @@ impl<'tcx> CrateStore<'tcx> for DummyCrateStore {
                        krate: &hir::Crate) -> Vec<u8> { vec![] }
     fn metadata_encoding_version(&self) -> &[u8] { unimplemented!() }
 }
+
+
+/// Metadata encoding and decoding can make use of thread-local encoding and
+/// decoding contexts. These allow implementers of serialize::Encodable and
+/// Decodable to access information and datastructures that would otherwise not
+/// be available to them. For example, we can automatically translate def-id and
+/// span information during decoding because the decoding context knows which
+/// crate the data is decoded from. Or it allows to make ty::Ty decodable
+/// because the context has access to the ty::ctxt that is needed for creating
+/// ty::Ty instances.
+///
+/// Note, however, that this only works for RBML-based encoding and decoding at
+/// the moment.
+pub mod tls {
+    use rbml::writer::Encoder as RbmlEncoder;
+    use rbml::reader::Decoder as RbmlDecoder;
+    use serialize;
+    use std::mem;
+    use middle::ty::{self, Ty};
+    use middle::subst::Substs;
+    use middle::def_id::DefId;
+
+    pub trait EncodingContext<'tcx> {
+        fn tcx<'a>(&'a self) -> &'a ty::ctxt<'tcx>;
+        fn encode_ty(&self, rbml_w: &mut RbmlEncoder, t: Ty<'tcx>);
+        fn encode_substs(&self, rbml_w: &mut RbmlEncoder, substs: &Substs<'tcx>);
+    }
+
+    /// Marker type used for the scoped TLS slot.
+    /// The type context cannot be used directly because the scoped TLS
+    /// in libstd doesn't allow types generic over lifetimes.
+    struct TlsPayload;
+
+    scoped_thread_local!(static TLS_ENCODING: TlsPayload);
+
+    /// Execute f after pushing the given EncodingContext onto the TLS stack.
+    pub fn enter_encoding_context<'tcx, F, R>(ecx: &EncodingContext<'tcx>,
+                                              rbml_w: &mut RbmlEncoder,
+                                              f: F) -> R
+        where F: FnOnce(&EncodingContext<'tcx>, &mut RbmlEncoder) -> R
+    {
+        let tls_payload = (ecx as *const _, rbml_w as *mut _);
+        let tls_ptr = &tls_payload as *const _ as *const TlsPayload;
+        TLS_ENCODING.set(unsafe { &*tls_ptr }, || f(ecx, rbml_w))
+    }
+
+    /// Execute f with access to the thread-local encoding context and
+    /// rbml encoder. This function will panic if the encoder passed in and the
+    /// context encoder are not the same.
+    ///
+    /// Note that this method is 'practically' safe due to its checking that the
+    /// encoder passed in is the same as the one in TLS, but it would still be
+    /// possible to construct cases where the EncodingContext is exchanged
+    /// while the same encoder is used, thus working with a wrong context.
+    pub fn with_encoding_context<'tcx, E, F, R>(encoder: &mut E, f: F) -> R
+        where F: FnOnce(&EncodingContext<'tcx>, &mut RbmlEncoder) -> R,
+              E: serialize::Encoder
+    {
+        unsafe {
+            unsafe_with_encoding_context(|ecx, rbml_w| {
+                assert!(encoder as *mut _ as usize == rbml_w as *mut _ as usize);
+
+                let ecx: &EncodingContext<'tcx> = mem::transmute(ecx);
+
+                f(ecx, rbml_w)
+            })
+        }
+    }
+
+    /// Execute f with access to the thread-local encoding context and
+    /// rbml encoder.
+    pub unsafe fn unsafe_with_encoding_context<F, R>(f: F) -> R
+        where F: FnOnce(&EncodingContext, &mut RbmlEncoder) -> R
+    {
+        TLS_ENCODING.with(|tls| {
+            let tls_payload = (tls as *const TlsPayload)
+                                   as *mut (&EncodingContext, &mut RbmlEncoder);
+            f((*tls_payload).0, (*tls_payload).1)
+        })
+    }
+
+    pub trait DecodingContext<'tcx> {
+        fn tcx<'a>(&'a self) -> &'a ty::ctxt<'tcx>;
+        fn decode_ty(&self, rbml_r: &mut RbmlDecoder) -> ty::Ty<'tcx>;
+        fn decode_substs(&self, rbml_r: &mut RbmlDecoder) -> Substs<'tcx>;
+        fn translate_def_id(&self, def_id: DefId) -> DefId;
+    }
+
+    scoped_thread_local!(static TLS_DECODING: TlsPayload);
+
+    /// Execute f after pushing the given DecodingContext onto the TLS stack.
+    pub fn enter_decoding_context<'tcx, F, R>(dcx: &DecodingContext<'tcx>,
+                                              rbml_r: &mut RbmlDecoder,
+                                              f: F) -> R
+        where F: FnOnce(&DecodingContext<'tcx>, &mut RbmlDecoder) -> R
+    {
+        let tls_payload = (dcx as *const _, rbml_r as *mut _);
+        let tls_ptr = &tls_payload as *const _ as *const TlsPayload;
+        TLS_DECODING.set(unsafe { &*tls_ptr }, || f(dcx, rbml_r))
+    }
+
+    /// Execute f with access to the thread-local decoding context and
+    /// rbml decoder. This function will panic if the decoder passed in and the
+    /// context decoder are not the same.
+    ///
+    /// Note that this method is 'practically' safe due to its checking that the
+    /// decoder passed in is the same as the one in TLS, but it would still be
+    /// possible to construct cases where the DecodingContext is exchanged
+    /// while the same decoder is used, thus working with a wrong context.
+    pub fn with_decoding_context<'decoder, 'tcx, D, F, R>(d: &'decoder mut D, f: F) -> R
+        where D: serialize::Decoder,
+              F: FnOnce(&DecodingContext<'tcx>,
+                        &mut RbmlDecoder) -> R,
+              'tcx: 'decoder
+    {
+        unsafe {
+            unsafe_with_decoding_context(|dcx, rbml_r| {
+                assert!((d as *mut _ as usize) == (rbml_r as *mut _ as usize));
+
+                let dcx: &DecodingContext<'tcx> = mem::transmute(dcx);
+
+                f(dcx, rbml_r)
+            })
+        }
+    }
+
+    /// Execute f with access to the thread-local decoding context and
+    /// rbml decoder.
+    pub unsafe fn unsafe_with_decoding_context<F, R>(f: F) -> R
+        where F: FnOnce(&DecodingContext, &mut RbmlDecoder) -> R
+    {
+        TLS_DECODING.with(|tls| {
+            let tls_payload = (tls as *const TlsPayload)
+                                   as *mut (&DecodingContext, &mut RbmlDecoder);
+            f((*tls_payload).0, (*tls_payload).1)
+        })
+    }
+}

src/librustc/middle/subst.rs

@@ -13,9 +13,11 @@
 pub use self::ParamSpace::*;
 pub use self::RegionSubsts::*;
 
+use middle::cstore;
 use middle::ty::{self, Ty, HasTypeFlags, RegionEscape};
 use middle::ty::fold::{TypeFoldable, TypeFolder};
 
+use serialize::{Encodable, Encoder, Decodable, Decoder};
 use std::fmt;
 use std::iter::IntoIterator;
 use std::slice::Iter;
@@ -153,6 +155,35 @@ impl<'tcx> Substs<'tcx> {
     }
 }
 
+impl<'tcx> Encodable for Substs<'tcx> {
+
+    fn encode<S: Encoder>(&self, s: &mut S) -> Result<(), S::Error> {
+        cstore::tls::with_encoding_context(s, |ecx, rbml_w| {
+            ecx.encode_substs(rbml_w, self);
+            Ok(())
+        })
+    }
+}
+
+impl<'tcx> Decodable for Substs<'tcx> {
+    fn decode<D: Decoder>(d: &mut D) -> Result<Substs<'tcx>, D::Error> {
+        cstore::tls::with_decoding_context(d, |dcx, rbml_r| {
+            Ok(dcx.decode_substs(rbml_r))
+        })
+    }
+}
+
+impl<'tcx> Decodable for &'tcx Substs<'tcx> {
+    fn decode<D: Decoder>(d: &mut D) -> Result<&'tcx Substs<'tcx>, D::Error> {
+        let substs = cstore::tls::with_decoding_context(d, |dcx, rbml_r| {
+            let substs = dcx.decode_substs(rbml_r);
+            dcx.tcx().mk_substs(substs)
+        });
+
+        Ok(substs)
+    }
+}
+
 impl RegionSubsts {
     pub fn map<F>(self, op: F) -> RegionSubsts where
         F: FnOnce(VecPerParamSpace<ty::Region>) -> VecPerParamSpace<ty::Region>,

src/librustc/middle/ty/mod.rs

@@ -22,7 +22,7 @@ pub use self::LvaluePreference::*;
 use front::map as ast_map;
 use front::map::LinkedPath;
 use middle;
-use middle::cstore::{CrateStore, LOCAL_CRATE};
+use middle::cstore::{self, CrateStore, LOCAL_CRATE};
 use middle::def::{self, ExportMap};
 use middle::def_id::DefId;
 use middle::lang_items::{FnTraitLangItem, FnMutTraitLangItem, FnOnceTraitLangItem};
@@ -35,6 +35,7 @@ use util::common::memoized;
 use util::nodemap::{NodeMap, NodeSet, DefIdMap};
 use util::nodemap::FnvHashMap;
 
+use serialize::{Encodable, Encoder, Decodable, Decoder};
 use std::borrow::{Borrow, Cow};
 use std::cell::{Cell, RefCell};
 use std::hash::{Hash, Hasher};
@@ -479,6 +480,24 @@ impl<'tcx> Hash for TyS<'tcx> {
 
 pub type Ty<'tcx> = &'tcx TyS<'tcx>;
 
+impl<'tcx> Encodable for Ty<'tcx> {
+    fn encode<S: Encoder>(&self, s: &mut S) -> Result<(), S::Error> {
+        cstore::tls::with_encoding_context(s, |ecx, rbml_w| {
+            ecx.encode_ty(rbml_w, *self);
+            Ok(())
+        })
+    }
+}
+
+impl<'tcx> Decodable for Ty<'tcx> {
+    fn decode<D: Decoder>(d: &mut D) -> Result<Ty<'tcx>, D::Error> {
+        cstore::tls::with_decoding_context(d, |dcx, rbml_r| {
+            Ok(dcx.decode_ty(rbml_r))
+        })
+    }
+}
+
+
 /// Upvars do not get their own node-id. Instead, we use the pair of
 /// the original var id (that is, the root variable that is referenced
 /// by the upvar) and the id of the closure expression.
@@ -1529,6 +1548,23 @@ impl<'tcx, 'container> Hash for AdtDefData<'tcx, 'container> {
     }
 }
 
+impl<'tcx> Encodable for AdtDef<'tcx> {
+    fn encode<S: Encoder>(&self, s: &mut S) -> Result<(), S::Error> {
+        self.did.encode(s)
+    }
+}
+
+impl<'tcx> Decodable for AdtDef<'tcx> {
+    fn decode<D: Decoder>(d: &mut D) -> Result<AdtDef<'tcx>, D::Error> {
+        let def_id: DefId = try!{ Decodable::decode(d) };
+
+        cstore::tls::with_decoding_context(d, |dcx, _| {
+            let def_id = dcx.translate_def_id(def_id);
+            Ok(dcx.tcx().lookup_adt_def(def_id))
+        })
+    }
+}
+
 
 #[derive(Copy, Clone, Debug, Eq, PartialEq)]
 pub enum AdtKind { Struct, Enum }

src/librustc_metadata/astencode.rs

@@ -39,7 +39,6 @@ use middle::ty::{self, Ty};
 
 use syntax::{ast, ast_util, codemap};
 use syntax::ast::NodeIdAssigner;
-use syntax::codemap::Span;
 use syntax::ptr::P;
 
 use std::cell::Cell;
@@ -116,7 +115,7 @@ impl<'a, 'b, 'c, 'tcx> ast_map::FoldOps for &'a DecodeContext<'b, 'c, 'tcx> {
     fn new_def_id(&self, def_id: DefId) -> DefId {
         self.tr_def_id(def_id)
     }
-    fn new_span(&self, span: Span) -> Span {
+    fn new_span(&self, span: codemap::Span) -> codemap::Span {
         self.tr_span(span)
     }
 }
@@ -219,60 +218,12 @@ impl<'a, 'b, 'tcx> DecodeContext<'a, 'b, 'tcx> {
     }
 
     /// Translates a `Span` from an extern crate to the corresponding `Span`
-    /// within the local crate's codemap. `creader::import_codemap()` will
-    /// already have allocated any additionally needed FileMaps in the local
-    /// codemap as a side-effect of creating the crate_metadata's
-    /// `codemap_import_info`.
-    pub fn tr_span(&self, span: Span) -> Span {
-        let span = if span.lo > span.hi {
-            // Currently macro expansion sometimes produces invalid Span values
-            // where lo > hi. In order not to crash the compiler when trying to
-            // translate these values, let's transform them into something we
-            // can handle (and which will produce useful debug locations at
-            // least some of the time).
-            // This workaround is only necessary as long as macro expansion is
-            // not fixed. FIXME(#23480)
-            codemap::mk_sp(span.lo, span.lo)
-        } else {
-            span
-        };
-
-        let imported_filemaps = self.cdata.imported_filemaps(self.tcx.sess.codemap());
-        let filemap = {
-            // Optimize for the case that most spans within a translated item
-            // originate from the same filemap.
-            let last_filemap_index = self.last_filemap_index.get();
-            let last_filemap = &imported_filemaps[last_filemap_index];
-
-            if span.lo >= last_filemap.original_start_pos &&
-               span.lo <= last_filemap.original_end_pos &&
-               span.hi >= last_filemap.original_start_pos &&
-               span.hi <= last_filemap.original_end_pos {
-                last_filemap
-            } else {
-                let mut a = 0;
-                let mut b = imported_filemaps.len();
-
-                while b - a > 1 {
-                    let m = (a + b) / 2;
-                    if imported_filemaps[m].original_start_pos > span.lo {
-                        b = m;
-                    } else {
-                        a = m;
-                    }
-                }
-
-                self.last_filemap_index.set(a);
-                &imported_filemaps[a]
-            }
-        };
-
-        let lo = (span.lo - filemap.original_start_pos) +
-                  filemap.translated_filemap.start_pos;
-        let hi = (span.hi - filemap.original_start_pos) +
-                  filemap.translated_filemap.start_pos;
-
-        codemap::mk_sp(lo, hi)
+    /// within the local crate's codemap.
+    pub fn tr_span(&self, span: codemap::Span) -> codemap::Span {
+        decoder::translate_span(self.cdata,
+                                self.tcx.sess.codemap(),
+                                &self.last_filemap_index,
+                                span)
     }
 }
 
@@ -288,8 +239,8 @@ impl tr for Option<DefId> {
     }
 }
 
-impl tr for Span {
-    fn tr(&self, dcx: &DecodeContext) -> Span {
+impl tr for codemap::Span {
+    fn tr(&self, dcx: &DecodeContext) -> codemap::Span {
         dcx.tr_span(*self)
     }
 }