run optimization and codegen on worker threads

Refactor the code in `llvm::back` that invokes LLVM optimization and codegen
passes so that it can be called from worker threads.  (Previously, it used
`&Session` extensively, and `Session` is not `Share`.)  The new code can handle
multiple compilation units, by compiling each unit to `crate.0.o`, `crate.1.o`,
etc., and linking together all the `crate.N.o` files into a single `crate.o`
using `ld -r`.  The later linking steps can then be run unchanged.

The new code preserves the behavior of `--emit`/`-o` when building a single
compilation unit.  With multiple compilation units, the `--emit=asm/ir/bc`
options produce multiple files, so combinations like `--emit=ir -o foo.ll` will
not actually produce `foo.ll` (they instead produce several `foo.N.ll` files).

The new code supports `-Z lto` only when using a single compilation unit.
Compiling with multiple compilation units and `-Z lto` will produce an error.
(I can't think of any good reason to do such a thing.)  Linking with `-Z lto`
against a library that was built as multiple compilation units will also fail,
because the rlib does not contain a `crate.bytecode.deflate` file.  This could
be supported in the future by linking together the `crate.N.bc` files produced
when compiling the library into a single `crate.bc`, or by making the LTO code
support multiple `crate.N.bytecode.deflate` files.

Author: spernsteiner

src/compiletest/runtest.rs

@@ -1577,10 +1577,6 @@ fn _arm_push_aux_shared_library(config: &Config, testfile: &Path) {
 
 // codegen tests (vs. clang)
 
-fn make_o_name(config: &Config, testfile: &Path) -> Path {
-    output_base_name(config, testfile).with_extension("o")
-}
-
 fn append_suffix_to_stem(p: &Path, suffix: &str) -> Path {
     if suffix.len() == 0 {
         (*p).clone()
@@ -1596,14 +1592,13 @@ fn compile_test_and_save_bitcode(config: &Config, props: &TestProps,
     // FIXME (#9639): This needs to handle non-utf8 paths
     let link_args = vec!("-L".to_string(),
                          aux_dir.as_str().unwrap().to_string());
-    let llvm_args = vec!("--emit=obj".to_string(),
-                         "--crate-type=lib".to_string(),
-                         "-C".to_string(),
-                         "save-temps".to_string());
+    let llvm_args = vec!("--emit=bc,obj".to_string(),
+                         "--crate-type=lib".to_string());
     let args = make_compile_args(config,
                                  props,
                                  link_args.append(llvm_args.as_slice()),
-                                 |a, b| ThisFile(make_o_name(a, b)), testfile);
+                                 |a, b| ThisDirectory(output_base_name(a, b).dir_path()),
+                                 testfile);
     compose_and_run_compiler(config, props, testfile, args, None)
 }
 

src/librustc/back/link.rs

@@ -662,51 +662,56 @@ fn link_rlib<'a>(sess: &'a Session,
             ab.add_file(&metadata).unwrap();
             remove(sess, &metadata);
 
-            // For LTO purposes, the bytecode of this library is also inserted
-            // into the archive.
-            //
-            // Note that we make sure that the bytecode filename in the archive
-            // is never exactly 16 bytes long by adding a 16 byte extension to
-            // it. This is to work around a bug in LLDB that would cause it to
-            // crash if the name of a file in an archive was exactly 16 bytes.
-            let bc_filename = obj_filename.with_extension("bc");
-            let bc_deflated_filename = obj_filename.with_extension("bytecode.deflate");
-
-            let bc_data = match fs::File::open(&bc_filename).read_to_end() {
-                Ok(buffer) => buffer,
-                Err(e) => sess.fatal(format!("failed to read bytecode: {}",
-                                             e).as_slice())
-            };
+            if sess.opts.cg.codegen_units == 1 {
+                // For LTO purposes, the bytecode of this library is also
+                // inserted into the archive.  We currently do this only when
+                // codegen_units == 1, so we don't have to deal with multiple
+                // bitcode files per crate.
+                //
+                // Note that we make sure that the bytecode filename in the
+                // archive is never exactly 16 bytes long by adding a 16 byte
+                // extension to it. This is to work around a bug in LLDB that
+                // would cause it to crash if the name of a file in an archive
+                // was exactly 16 bytes.
+                let bc_filename = obj_filename.with_extension("bc");
+                let bc_deflated_filename = obj_filename.with_extension("bytecode.deflate");
+
+                let bc_data = match fs::File::open(&bc_filename).read_to_end() {
+                    Ok(buffer) => buffer,
+                    Err(e) => sess.fatal(format!("failed to read bytecode: {}",
+                                                 e).as_slice())
+                };
 
-            let bc_data_deflated = match flate::deflate_bytes(bc_data.as_slice()) {
-                Some(compressed) => compressed,
-                None => sess.fatal(format!("failed to compress bytecode from {}",
-                                           bc_filename.display()).as_slice())
-            };
+                let bc_data_deflated = match flate::deflate_bytes(bc_data.as_slice()) {
+                    Some(compressed) => compressed,
+                    None => sess.fatal(format!("failed to compress bytecode from {}",
+                                               bc_filename.display()).as_slice())
+                };
 
-            let mut bc_file_deflated = match fs::File::create(&bc_deflated_filename) {
-                Ok(file) => file,
-                Err(e) => {
-                    sess.fatal(format!("failed to create compressed bytecode \
-                                        file: {}", e).as_slice())
-                }
-            };
+                let mut bc_file_deflated = match fs::File::create(&bc_deflated_filename) {
+                    Ok(file) => file,
+                    Err(e) => {
+                        sess.fatal(format!("failed to create compressed bytecode \
+                                            file: {}", e).as_slice())
+                    }
+                };
 
-            match write_rlib_bytecode_object_v1(&mut bc_file_deflated,
-                                                bc_data_deflated.as_slice()) {
-                Ok(()) => {}
-                Err(e) => {
-                    sess.err(format!("failed to write compressed bytecode: \
-                                      {}", e).as_slice());
-                    sess.abort_if_errors()
-                }
-            };
+                match write_rlib_bytecode_object_v1(&mut bc_file_deflated,
+                                                    bc_data_deflated.as_slice()) {
+                    Ok(()) => {}
+                    Err(e) => {
+                        sess.err(format!("failed to write compressed bytecode: \
+                                          {}", e).as_slice());
+                        sess.abort_if_errors()
+                    }
+                };
 
-            ab.add_file(&bc_deflated_filename).unwrap();
-            remove(sess, &bc_deflated_filename);
-            if !sess.opts.cg.save_temps &&
-               !sess.opts.output_types.contains(&OutputTypeBitcode) {
-                remove(sess, &bc_filename);
+                ab.add_file(&bc_deflated_filename).unwrap();
+                remove(sess, &bc_deflated_filename);
+                if !sess.opts.cg.save_temps &&
+                   !sess.opts.output_types.contains(&OutputTypeBitcode) {
+                    remove(sess, &bc_filename);
+                }
             }
         }
 

src/librustc/back/lto.rs

@@ -67,7 +67,14 @@ pub fn run(sess: &session::Session, llmod: ModuleRef,
                                   archive.read(format!("{}.bytecode.deflate",
                                                        file).as_slice())
                               });
-        let bc_encoded = bc_encoded.expect("missing compressed bytecode in archive!");
+        let bc_encoded = match bc_encoded {
+            Some(data) => data,
+            None => {
+                sess.fatal(format!("missing compressed bytecode in {} \
+                                    (perhaps it was compiled with -C codegen-units > 1)",
+                                   path.display()).as_slice());
+            },
+        };
         let bc_extractor = if is_versioned_bytecode_format(bc_encoded) {
             |_| {
                 // Read the version
@@ -120,7 +127,7 @@ pub fn run(sess: &session::Session, llmod: ModuleRef,
             if !llvm::LLVMRustLinkInExternalBitcode(llmod,
                                                     ptr as *const libc::c_char,
                                                     bc_decoded.len() as libc::size_t) {
-                write::llvm_err(sess,
+                write::llvm_err(sess.diagnostic().handler(),
                                 format!("failed to load bc of `{}`",
                                         name.as_slice()));
             }