Remove segmented stack on windows

On Windows, for each function `__chkstk()`/`_alloca()` is (already)
inserted at prologue if the function takes 4096 or more bytes on stack.
The stack checker aborts (causes segmentation fault) if current stack
pointer is out of range.
With this check and stack guard page, we can detect stack overflow
without segmented stack support.

This also fixes some issue regarding `$fs:0x14` (we use it for
segmented stack support).

Author: klutzy

src/librustc/middle/trans/base.rs

@@ -417,15 +417,23 @@ pub fn set_always_inline(f: ValueRef) {
 }
 
 pub fn set_split_stack(f: ValueRef) {
-    "split-stack".with_c_str(|buf| {
-        unsafe { llvm::LLVMAddFunctionAttrString(f, lib::llvm::FunctionIndex as c_uint, buf); }
-    })
+    if !cfg!(windows) {
+        "split-stack".with_c_str(|buf| {
+            unsafe {
+                llvm::LLVMAddFunctionAttrString(f, lib::llvm::FunctionIndex as c_uint, buf);
+            }
+        })
+    }
 }
 
 pub fn unset_split_stack(f: ValueRef) {
-    "split-stack".with_c_str(|buf| {
-        unsafe { llvm::LLVMRemoveFunctionAttrString(f, lib::llvm::FunctionIndex as c_uint, buf); }
-    })
+    if !cfg!(windows) {
+        "split-stack".with_c_str(|buf| {
+            unsafe {
+                llvm::LLVMRemoveFunctionAttrString(f, lib::llvm::FunctionIndex as c_uint, buf);
+            }
+        })
+    }
 }
 
 // Double-check that we never ask LLVM to declare the same symbol twice. It

src/libstd/rand/os.rs

@@ -69,7 +69,7 @@ mod imp {
     use os;
     use rand::Rng;
     use result::{Ok, Err};
-    use rt::stack;
+    //use rt::stack;
     use self::libc::{c_ulong, DWORD, BYTE, LPCSTR, BOOL};
     use slice::MutableVector;
 
@@ -91,7 +91,7 @@ mod imp {
     static PROV_RSA_FULL: DWORD = 1;
     static CRYPT_SILENT: DWORD = 64;
     static CRYPT_VERIFYCONTEXT: DWORD = 0xF0000000;
-    static NTE_BAD_SIGNATURE: DWORD = 0x80090006;
+    //static NTE_BAD_SIGNATURE: DWORD = 0x80090006;
 
     #[allow(non_snake_case_functions)]
     extern "system" {
@@ -110,12 +110,13 @@ mod imp {
         /// Create a new `OsRng`.
         pub fn new() -> IoResult<OsRng> {
             let mut hcp = 0;
-            let mut ret = unsafe {
+            let ret = unsafe {
                 CryptAcquireContextA(&mut hcp, 0 as LPCSTR, 0 as LPCSTR,
                                      PROV_RSA_FULL,
                                      CRYPT_VERIFYCONTEXT | CRYPT_SILENT)
             };
 
+            // TODO remove this yah
             // FIXME #13259:
             // It turns out that if we can't acquire a context with the
             // NTE_BAD_SIGNATURE error code, the documentation states:
@@ -141,16 +142,16 @@ mod imp {
             // Again, I'm not sure why this is the fix, nor why we're getting
             // this error. All I can say is that this seems to allow libnative
             // to progress where it otherwise would be hindered. Who knew?
-            if ret == 0 && os::errno() as DWORD == NTE_BAD_SIGNATURE {
-                unsafe {
-                    let limit = stack::get_sp_limit();
-                    stack::record_sp_limit(0);
-                    ret = CryptAcquireContextA(&mut hcp, 0 as LPCSTR, 0 as LPCSTR,
-                                               PROV_RSA_FULL,
-                                               CRYPT_VERIFYCONTEXT | CRYPT_SILENT);
-                    stack::record_sp_limit(limit);
-                }
-            }
+            // if ret == 0 && os::errno() as DWORD == NTE_BAD_SIGNATURE {
+            //     unsafe {
+            //         let limit = stack::get_sp_limit();
+            //         stack::record_sp_limit(0);
+            //         ret = CryptAcquireContextA(&mut hcp, 0 as LPCSTR, 0 as LPCSTR,
+            //                                    PROV_RSA_FULL,
+            //                                    CRYPT_VERIFYCONTEXT | CRYPT_SILENT);
+            //         stack::record_sp_limit(limit);
+            //     }
+            // }
 
             if ret == 0 {
                 Err(IoError::last_error())

src/libstd/rt/stack.rs

@@ -162,11 +162,9 @@ pub unsafe fn record_sp_limit(limit: uint) {
         asm!("movq $0, %fs:112" :: "r"(limit) :: "volatile")
     }
     #[cfg(target_arch = "x86_64", target_os = "win32")] #[inline(always)]
-    unsafe fn target_record_sp_limit(limit: uint) {
-        // see: http://en.wikipedia.org/wiki/Win32_Thread_Information_Block
-        // store this inside of the "arbitrary data slot", but double the size
-        // because this is 64 bit instead of 32 bit
-        asm!("movq $0, %gs:0x28" :: "r"(limit) :: "volatile")
+    unsafe fn target_record_sp_limit(_limit: uint) {
+        // on windows, TODO explanation what is this
+        // asm!("movq $0, %gs:0x28" :: "r"(limit) :: "volatile")
     }
     #[cfg(target_arch = "x86_64", target_os = "freebsd")] #[inline(always)]
     unsafe fn target_record_sp_limit(limit: uint) {
@@ -185,10 +183,11 @@ pub unsafe fn record_sp_limit(limit: uint) {
         asm!("movl $0, %gs:48" :: "r"(limit) :: "volatile")
     }
     #[cfg(target_arch = "x86", target_os = "win32")] #[inline(always)]
-    unsafe fn target_record_sp_limit(limit: uint) {
+    unsafe fn target_record_sp_limit(_limit: uint) {
         // see: http://en.wikipedia.org/wiki/Win32_Thread_Information_Block
         // store this inside of the "arbitrary data slot"
-        asm!("movl $0, %fs:0x14" :: "r"(limit) :: "volatile")
+        //  TODO explanation
+        // asm!("movl $0, %fs:0x14" :: "r"(limit) :: "volatile")
     }
 
     // mips, arm - Some brave soul can port these to inline asm, but it's over
@@ -232,9 +231,10 @@ pub unsafe fn get_sp_limit() -> uint {
     }
     #[cfg(target_arch = "x86_64", target_os = "win32")] #[inline(always)]
     unsafe fn target_get_sp_limit() -> uint {
-        let limit;
-        asm!("movq %gs:0x28, $0" : "=r"(limit) ::: "volatile");
-        return limit;
+        return 0; // TODO
+        // let limit;
+        // asm!("movq %gs:0x28, $0" : "=r"(limit) ::: "volatile");
+        // return limit;
     }
     #[cfg(target_arch = "x86_64", target_os = "freebsd")] #[inline(always)]
     unsafe fn target_get_sp_limit() -> uint {
@@ -260,9 +260,10 @@ pub unsafe fn get_sp_limit() -> uint {
     }
     #[cfg(target_arch = "x86", target_os = "win32")] #[inline(always)]
     unsafe fn target_get_sp_limit() -> uint {
-        let limit;
-        asm!("movl %fs:0x14, $0" : "=r"(limit) ::: "volatile");
-        return limit;
+        return 0; // TODO
+        //let limit;
+        //asm!("movl %fs:0x14, $0" : "=r"(limit) ::: "volatile");
+        //return limit;
     }
 
     // mips, arm - Some brave soul can port these to inline asm, but it's over

src/test/run-pass/out-of-stack.rs

@@ -42,12 +42,16 @@ fn main() {
     } else {
         let silent = Command::new(args[0].as_slice()).arg("silent").output().unwrap();
         assert!(!silent.status.success());
-        let error = str::from_utf8_lossy(silent.error.as_slice());
-        assert!(error.as_slice().contains("has overflowed its stack"));
+        if !cfg!(windows) {
+            let error = str::from_utf8_lossy(silent.error.as_slice());
+            assert!(error.as_slice().contains("has overflowed its stack"));
+        }
 
         let loud = Command::new(args[0].as_slice()).arg("loud").output().unwrap();
         assert!(!loud.status.success());
-        let error = str::from_utf8_lossy(silent.error.as_slice());
-        assert!(error.as_slice().contains("has overflowed its stack"));
+        if !cfg!(windows) {
+            let error = str::from_utf8_lossy(silent.error.as_slice());
+            assert!(error.as_slice().contains("has overflowed its stack"));
+        }
     }
 }