From 3222439ef1a19804a0cb999bb057d0c0c32d696f Mon Sep 17 00:00:00 2001 From: Jeremy Fitzhardinge Date: Sat, 29 May 2021 13:51:13 -0700 Subject: [PATCH 1/3] Add test for pipelined builds This makes sure that building a crate with an rmeta dependency is viable as a linkable crate. --- src/test/run-make/pipelined/Makefile | 15 +++++++++++++++ src/test/run-make/pipelined/leaf.rs | 25 +++++++++++++++++++++++++ src/test/run-make/pipelined/main.rs | 9 +++++++++ src/test/run-make/pipelined/middle.rs | 22 ++++++++++++++++++++++ 4 files changed, 71 insertions(+) create mode 100644 src/test/run-make/pipelined/Makefile create mode 100644 src/test/run-make/pipelined/leaf.rs create mode 100644 src/test/run-make/pipelined/main.rs create mode 100644 src/test/run-make/pipelined/middle.rs diff --git a/src/test/run-make/pipelined/Makefile b/src/test/run-make/pipelined/Makefile new file mode 100644 index 0000000000000..84138a333ee95 --- /dev/null +++ b/src/test/run-make/pipelined/Makefile @@ -0,0 +1,15 @@ +-include ../../run-make-fulldeps/tools.mk + +O=$(TMPDIR) + +all: $O/main + $O/main + +$O/main: $O/libmiddle.rlib $O/libleaf.rlib + $(RUSTC) main.rs --extern middle=$O/libmiddle.rlib -Ldependency=$O + +$O/libmiddle.rlib: middle.rs $O/libleaf.rmeta + $(RUSTC) --emit link middle.rs --extern leaf=$O/libleaf.rmeta + +$O/libleaf.rlib $O/libleaf.rmeta: leaf.rs + $(RUSTC) --emit metadata,link leaf.rs diff --git a/src/test/run-make/pipelined/leaf.rs b/src/test/run-make/pipelined/leaf.rs new file mode 100644 index 0000000000000..29d83ea1be58f --- /dev/null +++ b/src/test/run-make/pipelined/leaf.rs @@ -0,0 +1,25 @@ +#![crate_type = "rlib"] + +pub static FOO: &str = "this is a static"; + +#[derive(Debug)] +pub struct Plain { + pub a: u32, + pub b: u32, +} + +#[derive(Debug)] +pub struct GenericStruct { + pub a: A, + pub b: Option, +} + +pub fn simple(a: u32, b: u32) -> u32 { + let c = a + b; + println!("simple {} + {} => {}", a, b, c); + c +} + +pub fn generic(d: D) { + println!("generically printing {:?}", d); +} diff --git a/src/test/run-make/pipelined/main.rs b/src/test/run-make/pipelined/main.rs new file mode 100644 index 0000000000000..88d29fcbb9635 --- /dev/null +++ b/src/test/run-make/pipelined/main.rs @@ -0,0 +1,9 @@ +#![crate_type = "bin"] + +fn main() { + middle::do_a_thing(); + + println!("middle::BAR {}", middle::BAR); + + assert_eq!(middle::simple(2, 3), 5); +} diff --git a/src/test/run-make/pipelined/middle.rs b/src/test/run-make/pipelined/middle.rs new file mode 100644 index 0000000000000..5ba99fb4c8ade --- /dev/null +++ b/src/test/run-make/pipelined/middle.rs @@ -0,0 +1,22 @@ +#![crate_type = "rlib"] + +extern crate leaf; + +pub static BAR: &str = leaf::FOO; + +pub use leaf::simple; + +pub fn do_a_thing() { + let s = leaf::GenericStruct { + a: "hello", + b: Some("world") + }; + + let u = leaf::GenericStruct { + a: leaf::Plain { a: 1, b: 2 }, + b: None + }; + + leaf::generic(s); + leaf::generic(u); +} From fbef30077525eebe9f185e5e401031e5fb7be4a0 Mon Sep 17 00:00:00 2001 From: Jeremy Fitzhardinge Date: Fri, 28 May 2021 11:46:50 -0700 Subject: [PATCH 2/3] metadata-link: add -Zmetadata-link flag This adds the -Zmetadata-link flag, whose intent is to make the output of `--emit metadata` equivalent to `--emit metadata,link`. The goal is to allow pipelined builds with separate invocations of rustc. This is desireable for two reasons: 1. The "artifact notification" system that rustc and cargo use to communicate is very cargo-specific - it doesn't fit well into other builds systems such as Buck or Bazel. In general its incompatible with any build system which only recognizes output artifacts once the compiler process has terminated. This is a particular problem when compilation is distributed. 2. The rmeta file could be cached independently from the rlib. For example, if one generates compilation-ready rmeta files as part of "check" build, then those can be directly used for a full compilation from cache, without having to regenerate them. This means the build turns into a highly parallelizable invocation of `rustc --emit link` commands using cached `rmeta` files as input. Initially this flag `-Zmetadata-link`, but ultimately I'd expect to stabilize it as `-Cmetadata-link` (or a better name). --- .../rustc_codegen_ssa/src/back/symbol_export.rs | 4 ++-- compiler/rustc_interface/src/tests.rs | 1 + compiler/rustc_metadata/src/rmeta/encoder.rs | 15 ++++++++------- compiler/rustc_session/src/options.rs | 2 ++ 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs index 14d6f0ba147b5..e96cde2b3b03d 100644 --- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs +++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs @@ -45,7 +45,7 @@ pub fn crates_export_threshold(crate_types: &[CrateType]) -> SymbolExportLevel { fn reachable_non_generics_provider(tcx: TyCtxt<'_>, cnum: CrateNum) -> DefIdMap { assert_eq!(cnum, LOCAL_CRATE); - if !tcx.sess.opts.output_types.should_codegen() { + if !tcx.sess.opts.output_types.should_codegen() && !tcx.sess.opts.debugging_opts.metadata_link { return Default::default(); } @@ -164,7 +164,7 @@ fn exported_symbols_provider_local( ) -> &'tcx [(ExportedSymbol<'tcx>, SymbolExportLevel)] { assert_eq!(cnum, LOCAL_CRATE); - if !tcx.sess.opts.output_types.should_codegen() { + if !tcx.sess.opts.output_types.should_codegen() && !tcx.sess.opts.debugging_opts.metadata_link { return &[]; } diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs index bea7d0fb81f95..d9276aff37d6a 100644 --- a/compiler/rustc_interface/src/tests.rs +++ b/compiler/rustc_interface/src/tests.rs @@ -706,6 +706,7 @@ fn test_debugging_options_tracking_hash() { tracked!(instrument_mcount, true); tracked!(link_only, true); tracked!(merge_functions, Some(MergeFunctions::Disabled)); + tracked!(metadata_link, true); tracked!(mir_emit_retag, true); tracked!(mir_opt_level, Some(4)); tracked!(mutable_noalias, Some(true)); diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 5c45e4130d2d7..488a1c538dc73 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -815,11 +815,14 @@ fn should_encode_stability(def_kind: DefKind) -> bool { /// /// Return a pair, resp. for CTFE and for LLVM. fn should_encode_mir(tcx: TyCtxt<'_>, def_id: LocalDefId) -> (bool, bool) { + let always_encode_mir = tcx.sess.opts.debugging_opts.always_encode_mir; + let metadata_link = tcx.sess.opts.debugging_opts.metadata_link; + match tcx.def_kind(def_id) { // Constructors DefKind::Ctor(_, _) => { - let mir_opt_base = tcx.sess.opts.output_types.should_codegen() - || tcx.sess.opts.debugging_opts.always_encode_mir; + let mir_opt_base = + tcx.sess.opts.output_types.should_codegen() || always_encode_mir || metadata_link; (true, mir_opt_base) } // Constants @@ -831,10 +834,9 @@ fn should_encode_mir(tcx: TyCtxt<'_>, def_id: LocalDefId) -> (bool, bool) { let generics = tcx.generics_of(def_id); let needs_inline = (generics.requires_monomorphization(tcx) || tcx.codegen_fn_attrs(def_id).requests_inline()) - && tcx.sess.opts.output_types.should_codegen(); + && (tcx.sess.opts.output_types.should_codegen() || metadata_link); // Only check the presence of the `const` modifier. let is_const_fn = tcx.is_const_fn_raw(def_id.to_def_id()); - let always_encode_mir = tcx.sess.opts.debugging_opts.always_encode_mir; (is_const_fn, needs_inline || always_encode_mir) } // Closures can't be const fn. @@ -842,8 +844,7 @@ fn should_encode_mir(tcx: TyCtxt<'_>, def_id: LocalDefId) -> (bool, bool) { let generics = tcx.generics_of(def_id); let needs_inline = (generics.requires_monomorphization(tcx) || tcx.codegen_fn_attrs(def_id).requests_inline()) - && tcx.sess.opts.output_types.should_codegen(); - let always_encode_mir = tcx.sess.opts.debugging_opts.always_encode_mir; + && (tcx.sess.opts.output_types.should_codegen() || metadata_link); (false, needs_inline || always_encode_mir) } // Generators require optimized MIR to compute layout. @@ -2020,7 +2021,7 @@ impl<'tcx, 'v> ItemLikeVisitor<'v> for ImplVisitor<'tcx> { /// Used to prefetch queries which will be needed later by metadata encoding. /// Only a subset of the queries are actually prefetched to keep this code smaller. fn prefetch_mir(tcx: TyCtxt<'_>) { - if !tcx.sess.opts.output_types.should_codegen() { + if !tcx.sess.opts.output_types.should_codegen() && !tcx.sess.opts.debugging_opts.metadata_link { // We won't emit MIR, so don't prefetch it. return; } diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index c9f95ed1224d0..fe97eeded2a9a 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -1121,6 +1121,8 @@ options! { the same values as the target option of the same name"), meta_stats: bool = (false, parse_bool, [UNTRACKED], "gather metadata statistics (default: no)"), + metadata_link: bool = (false, parse_bool, [TRACKED], + "always emit metadata that's equivalent to metadata,link"), mir_emit_retag: bool = (false, parse_bool, [TRACKED], "emit Retagging MIR statements, interpreted e.g., by miri; implies -Zmir-opt-level=0 \ (default: no)"), From 83b4d4da4597c38cc7a28b3f1d12a2cf40bc7c1d Mon Sep 17 00:00:00 2001 From: Jeremy Fitzhardinge Date: Sat, 29 May 2021 14:10:06 -0700 Subject: [PATCH 3/3] Add test for separate pipelined builds Make sure the resulting executable is 1) linkable, and 2) the same as built with `--emit metadata,link`. --- src/test/run-make/pipelined/Makefile | 30 +++++++++++++++++++++------ src/test/run-make/pipelined/leaf.rs | 5 +++++ src/test/run-make/pipelined/main.rs | 1 + src/test/run-make/pipelined/middle.rs | 1 + 4 files changed, 31 insertions(+), 6 deletions(-) diff --git a/src/test/run-make/pipelined/Makefile b/src/test/run-make/pipelined/Makefile index 84138a333ee95..cf851dc69951b 100644 --- a/src/test/run-make/pipelined/Makefile +++ b/src/test/run-make/pipelined/Makefile @@ -1,15 +1,33 @@ -include ../../run-make-fulldeps/tools.mk -O=$(TMPDIR) +O=$(TMPDIR)/together +S=$(TMPDIR)/separate -all: $O/main - $O/main +all: $O/main $S/main + # Should produce identical output + cmp $O/main $S/main +# Build with combined --emit metadata,link $O/main: $O/libmiddle.rlib $O/libleaf.rlib - $(RUSTC) main.rs --extern middle=$O/libmiddle.rlib -Ldependency=$O + $(BARE_RUSTC) --out-dir $O main.rs --extern middle=$O/libmiddle.rlib -Ldependency=$O + $O/main $O/libmiddle.rlib: middle.rs $O/libleaf.rmeta - $(RUSTC) --emit link middle.rs --extern leaf=$O/libleaf.rmeta + $(BARE_RUSTC) --out-dir $O --emit link middle.rs --extern leaf=$O/libleaf.rmeta $O/libleaf.rlib $O/libleaf.rmeta: leaf.rs - $(RUSTC) --emit metadata,link leaf.rs + $(BARE_RUSTC) --out-dir $O --emit metadata,link leaf.rs + +# Build with separate --emit metadata / --emit link +$S/main: $S/libmiddle.rlib $S/libleaf.rlib + $(BARE_RUSTC) --out-dir $S main.rs --extern middle=$S/libmiddle.rlib -Ldependency=$S + $S/main + +$S/libmiddle.rlib: middle.rs $S/libleaf.rmeta + $(BARE_RUSTC) --out-dir $S --emit link middle.rs --extern leaf=$S/libleaf.rmeta + +$S/libleaf.rlib: leaf.rs + $(BARE_RUSTC) --out-dir $S --emit link leaf.rs + +$S/libleaf.rmeta: leaf.rs + $(BARE_RUSTC) --out-dir $S --emit metadata -Zmetadata-link leaf.rs diff --git a/src/test/run-make/pipelined/leaf.rs b/src/test/run-make/pipelined/leaf.rs index 29d83ea1be58f..6ddcd63d59559 100644 --- a/src/test/run-make/pipelined/leaf.rs +++ b/src/test/run-make/pipelined/leaf.rs @@ -20,6 +20,11 @@ pub fn simple(a: u32, b: u32) -> u32 { c } +#[inline] +pub fn inlined(a: u32, b: u32) -> u32 { + a * b +} + pub fn generic(d: D) { println!("generically printing {:?}", d); } diff --git a/src/test/run-make/pipelined/main.rs b/src/test/run-make/pipelined/main.rs index 88d29fcbb9635..211ecb5672514 100644 --- a/src/test/run-make/pipelined/main.rs +++ b/src/test/run-make/pipelined/main.rs @@ -6,4 +6,5 @@ fn main() { println!("middle::BAR {}", middle::BAR); assert_eq!(middle::simple(2, 3), 5); + assert_eq!(middle::inlined(2, 3), 6); } diff --git a/src/test/run-make/pipelined/middle.rs b/src/test/run-make/pipelined/middle.rs index 5ba99fb4c8ade..444ace389b75d 100644 --- a/src/test/run-make/pipelined/middle.rs +++ b/src/test/run-make/pipelined/middle.rs @@ -5,6 +5,7 @@ extern crate leaf; pub static BAR: &str = leaf::FOO; pub use leaf::simple; +pub use leaf::inlined; pub fn do_a_thing() { let s = leaf::GenericStruct {