add a recursion limit for type representation

I could have added a check for explicit recursion, as irregular types
tend to cause selection errors, but I am not sufficiently sure that
cannot be bypassed.

Fixes #22919
Fixes #25639
Fixes #26548

Author: arielb1

src/librustc_trans/trans/context.rs

@@ -156,6 +156,9 @@ pub struct LocalCrateContext<'tcx> {
     /// contexts around the same size.
     n_llvm_insns: Cell<usize>,
 
+    /// Depth of the current type-of computation - used to bail out
+    type_of_depth: Cell<usize>,
+
     trait_cache: RefCell<FnvHashMap<ty::PolyTraitRef<'tcx>,
                                     traits::Vtable<'tcx, ()>>>,
 }
@@ -470,6 +473,7 @@ impl<'tcx> LocalCrateContext<'tcx> {
                 unwind_resume_hooked: Cell::new(false),
                 intrinsics: RefCell::new(FnvHashMap()),
                 n_llvm_insns: Cell::new(0),
+                type_of_depth: Cell::new(0),
                 trait_cache: RefCell::new(FnvHashMap()),
             };
 
@@ -774,6 +778,17 @@ impl<'b, 'tcx> CrateContext<'b, 'tcx> {
                     obj))
     }
 
+    pub fn enter_type_of(&self, ty: Ty<'tcx>) -> TypeOfDepthLock<'b, 'tcx> {
+        let current_depth = self.local.type_of_depth.get();
+        debug!("enter_type_of({:?}) at depth {:?}", ty, current_depth);
+        if current_depth > self.sess().recursion_limit.get() {
+            self.sess().fatal(
+                &format!("overflow representing the type `{}`", ty))
+        }
+        self.local.type_of_depth.set(current_depth + 1);
+        TypeOfDepthLock(self.local)
+    }
+
     pub fn check_overflow(&self) -> bool {
         self.shared.check_overflow
     }
@@ -790,6 +805,14 @@ impl<'b, 'tcx> CrateContext<'b, 'tcx> {
     }
 }
 
+pub struct TypeOfDepthLock<'a, 'tcx: 'a>(&'a LocalCrateContext<'tcx>);
+
+impl<'a, 'tcx> Drop for TypeOfDepthLock<'a, 'tcx> {
+    fn drop(&mut self) {
+        self.0.type_of_depth.set(self.0.type_of_depth.get() - 1);
+    }
+}
+
 /// Declare any llvm intrinsics that you might need
 fn declare_intrinsic(ccx: &CrateContext, key: &str) -> Option<ValueRef> {
     macro_rules! ifn {

src/librustc_trans/trans/type_of.rs

@@ -184,6 +184,8 @@ pub fn sizing_type_of<'a, 'tcx>(cx: &CrateContext<'a, 'tcx>, t: Ty<'tcx>) -> Typ
     }
 
     debug!("sizing_type_of {:?}", t);
+    let _recursion_lock = cx.enter_type_of(t);
+
     let llsizingty = match t.sty {
         _ if !type_is_sized(cx.tcx(), t) => {
             Type::struct_(cx, &[Type::i8p(cx), Type::i8p(cx)], false)

src/test/compile-fail/issue-26548.rs

@@ -0,0 +1,19 @@
+// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// error-pattern: overflow representing the type `S`
+
+trait Mirror { type It; }
+impl<T> Mirror for T { type It = Self; }
+struct S(Option<<S as Mirror>::It>);
+
+fn main() {
+    let _s = S(None);
+}