Skip to content

Commit 8128d0d

Browse files
nikicnikic
committed
Serialize modules into ThinBuffer after initial optimization
Instead of keeping all modules in memory until thin LTO and only serializing them then, serialize the module immediately after it finishes optimizing.
1 parent bc2db43 commit 8128d0d

File tree

4 files changed

+64
-49
lines changed

4 files changed

+64
-49
lines changed

‎src/librustc_codegen_llvm/back/lto.rs

+33-32
Original file line numberDiff line numberDiff line change
@@ -159,7 +159,7 @@ pub(crate) fn run_fat(cgcx: &CodegenContext<LlvmCodegenBackend>,
159159
/// lists, one of the modules that need optimization and another for modules that
160160
/// can simply be copied over from the incr. comp. cache.
161161
pub(crate) fn run_thin(cgcx: &CodegenContext<LlvmCodegenBackend>,
162-
modules: Vec<ModuleCodegen<ModuleLlvm>>,
162+
modules: Vec<(String, ThinBuffer)>,
163163
cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
164164
timeline: &mut Timeline)
165165
-> Result<(Vec<LtoModuleCodegen<LlvmCodegenBackend>>, Vec<WorkProduct>), FatalError>
@@ -182,6 +182,31 @@ pub(crate) fn run_thin(cgcx: &CodegenContext<LlvmCodegenBackend>,
182182
timeline)
183183
}
184184

185+
pub(crate) fn prepare_thin(
186+
cgcx: &CodegenContext<LlvmCodegenBackend>,
187+
module: ModuleCodegen<ModuleLlvm>
188+
) -> (String, ThinBuffer) {
189+
let name = module.name.clone();
190+
let buffer = ThinBuffer::new(module.module_llvm.llmod());
191+
192+
// We emit the module after having serialized it into a ThinBuffer
193+
// because only then it will contain the ThinLTO module summary.
194+
if let Some(ref incr_comp_session_dir) = cgcx.incr_comp_session_dir {
195+
if cgcx.config(module.kind).emit_pre_thin_lto_bc {
196+
let path = incr_comp_session_dir
197+
.join(pre_lto_bitcode_filename(&name));
198+
199+
fs::write(&path, buffer.data()).unwrap_or_else(|e| {
200+
panic!("Error writing pre-lto-bitcode file `{}`: {}",
201+
path.display(),
202+
e);
203+
});
204+
}
205+
}
206+
207+
(name, buffer)
208+
}
209+
185210
fn fat_lto(cgcx: &CodegenContext<LlvmCodegenBackend>,
186211
diag_handler: &Handler,
187212
mut modules: Vec<ModuleCodegen<ModuleLlvm>>,
@@ -341,7 +366,7 @@ impl Drop for Linker<'a> {
341366
/// they all go out of scope.
342367
fn thin_lto(cgcx: &CodegenContext<LlvmCodegenBackend>,
343368
diag_handler: &Handler,
344-
modules: Vec<ModuleCodegen<ModuleLlvm>>,
369+
modules: Vec<(String, ThinBuffer)>,
345370
serialized_modules: Vec<(SerializedModule<ModuleBuffer>, CString)>,
346371
cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
347372
symbol_white_list: &[*const libc::c_char],
@@ -361,41 +386,17 @@ fn thin_lto(cgcx: &CodegenContext<LlvmCodegenBackend>,
361386
let mut module_names = Vec::with_capacity(full_scope_len);
362387
let mut thin_modules = Vec::with_capacity(full_scope_len);
363388

364-
// FIXME: right now, like with fat LTO, we serialize all in-memory
365-
// modules before working with them and ThinLTO. We really
366-
// shouldn't do this, however, and instead figure out how to
367-
// extract a summary from an in-memory module and then merge that
368-
// into the global index. It turns out that this loop is by far
369-
// the most expensive portion of this small bit of global
370-
// analysis!
371-
for (i, module) in modules.into_iter().enumerate() {
372-
info!("local module: {} - {}", i, module.name);
373-
let name = CString::new(module.name.clone()).unwrap();
374-
let buffer = ThinBuffer::new(module.module_llvm.llmod());
375-
376-
// We emit the module after having serialized it into a ThinBuffer
377-
// because only then it will contain the ThinLTO module summary.
378-
if let Some(ref incr_comp_session_dir) = cgcx.incr_comp_session_dir {
379-
if cgcx.config(module.kind).emit_pre_thin_lto_bc {
380-
let path = incr_comp_session_dir
381-
.join(pre_lto_bitcode_filename(&module.name));
382-
383-
fs::write(&path, buffer.data()).unwrap_or_else(|e| {
384-
panic!("Error writing pre-lto-bitcode file `{}`: {}",
385-
path.display(),
386-
e);
387-
});
388-
}
389-
}
390-
389+
for (i, (name, buffer)) in modules.into_iter().enumerate() {
390+
info!("local module: {} - {}", i, name);
391+
let cname = CString::new(name.clone()).unwrap();
391392
thin_modules.push(llvm::ThinLTOModule {
392-
identifier: name.as_ptr(),
393+
identifier: cname.as_ptr(),
393394
data: buffer.data().as_ptr(),
394395
len: buffer.data().len(),
395396
});
396397
thin_buffers.push(buffer);
397-
module_names.push(name);
398-
timeline.record(&module.name);
398+
module_names.push(cname);
399+
timeline.record(&name);
399400
}
400401

401402
// FIXME: All upstream crates are deserialized internally in the

‎src/librustc_codegen_llvm/lib.rs

+7-1
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,7 @@ impl WriteBackendMethods for LlvmCodegenBackend {
185185
}
186186
fn run_thin_lto(
187187
cgcx: &CodegenContext<Self>,
188-
modules: Vec<ModuleCodegen<Self::Module>>,
188+
modules: Vec<(String, Self::ThinBuffer)>,
189189
cached_modules: Vec<(SerializedModule<Self::ModuleBuffer>, WorkProduct)>,
190190
timeline: &mut Timeline
191191
) -> Result<(Vec<LtoModuleCodegen<Self>>, Vec<WorkProduct>), FatalError> {
@@ -216,6 +216,12 @@ impl WriteBackendMethods for LlvmCodegenBackend {
216216
) -> Result<CompiledModule, FatalError> {
217217
back::write::codegen(cgcx, diag_handler, module, config, timeline)
218218
}
219+
fn prepare_thin(
220+
cgcx: &CodegenContext<Self>,
221+
module: ModuleCodegen<Self::Module>
222+
) -> (String, Self::ThinBuffer) {
223+
back::lto::prepare_thin(cgcx, module)
224+
}
219225
fn run_lto_pass_manager(
220226
cgcx: &CodegenContext<Self>,
221227
module: &ModuleCodegen<Self::Module>,

‎src/librustc_codegen_ssa/back/write.rs

+19-15
Original file line numberDiff line numberDiff line change
@@ -253,7 +253,7 @@ impl<B: WriteBackendMethods> CodegenContext<B> {
253253
fn generate_lto_work<B: ExtraBackendMethods>(
254254
cgcx: &CodegenContext<B>,
255255
needs_fat_lto: Vec<ModuleCodegen<B::Module>>,
256-
needs_thin_lto: Vec<ModuleCodegen<B::Module>>,
256+
needs_thin_lto: Vec<(String, B::ThinBuffer)>,
257257
import_only_modules: Vec<(SerializedModule<B::ModuleBuffer>, WorkProduct)>
258258
) -> Vec<(WorkItem<B>, u64)> {
259259
let mut timeline = cgcx.time_graph.as_ref().map(|tg| {
@@ -678,17 +678,17 @@ impl<B: WriteBackendMethods> WorkItem<B> {
678678
}
679679
}
680680

681-
enum WorkItemResult<M> {
681+
enum WorkItemResult<B: WriteBackendMethods> {
682682
Compiled(CompiledModule),
683-
NeedsFatLTO(ModuleCodegen<M>),
684-
NeedsThinLTO(ModuleCodegen<M>),
683+
NeedsFatLTO(ModuleCodegen<B::Module>),
684+
NeedsThinLTO(String, B::ThinBuffer),
685685
}
686686

687687
fn execute_work_item<B: ExtraBackendMethods>(
688688
cgcx: &CodegenContext<B>,
689689
work_item: WorkItem<B>,
690690
timeline: &mut Timeline
691-
) -> Result<WorkItemResult<B::Module>, FatalError> {
691+
) -> Result<WorkItemResult<B>, FatalError> {
692692
let module_config = cgcx.config(work_item.module_kind());
693693

694694
match work_item {
@@ -716,7 +716,7 @@ fn execute_optimize_work_item<B: ExtraBackendMethods>(
716716
module: ModuleCodegen<B::Module>,
717717
module_config: &ModuleConfig,
718718
timeline: &mut Timeline
719-
) -> Result<WorkItemResult<B::Module>, FatalError> {
719+
) -> Result<WorkItemResult<B>, FatalError> {
720720
let diag_handler = cgcx.create_diag_handler();
721721

722722
unsafe {
@@ -772,7 +772,10 @@ fn execute_optimize_work_item<B: ExtraBackendMethods>(
772772
};
773773
WorkItemResult::Compiled(module)
774774
}
775-
ComputedLtoType::Thin => WorkItemResult::NeedsThinLTO(module),
775+
ComputedLtoType::Thin => {
776+
let (name, thin_buffer) = B::prepare_thin(cgcx, module);
777+
WorkItemResult::NeedsThinLTO(name, thin_buffer)
778+
}
776779
ComputedLtoType::Fat => WorkItemResult::NeedsFatLTO(module),
777780
})
778781
}
@@ -782,7 +785,7 @@ fn execute_copy_from_cache_work_item<B: ExtraBackendMethods>(
782785
module: CachedModuleCodegen,
783786
module_config: &ModuleConfig,
784787
_: &mut Timeline
785-
) -> Result<WorkItemResult<B::Module>, FatalError> {
788+
) -> Result<WorkItemResult<B>, FatalError> {
786789
let incr_comp_session_dir = cgcx.incr_comp_session_dir
787790
.as_ref()
788791
.unwrap();
@@ -844,7 +847,7 @@ fn execute_lto_work_item<B: ExtraBackendMethods>(
844847
mut module: lto::LtoModuleCodegen<B>,
845848
module_config: &ModuleConfig,
846849
timeline: &mut Timeline
847-
) -> Result<WorkItemResult<B::Module>, FatalError> {
850+
) -> Result<WorkItemResult<B>, FatalError> {
848851
let diag_handler = cgcx.create_diag_handler();
849852

850853
unsafe {
@@ -861,7 +864,8 @@ pub enum Message<B: WriteBackendMethods> {
861864
worker_id: usize,
862865
},
863866
NeedsThinLTO {
864-
result: ModuleCodegen<B::Module>,
867+
name: String,
868+
thin_buffer: B::ThinBuffer,
865869
worker_id: usize,
866870
},
867871
Done {
@@ -1423,10 +1427,10 @@ fn start_executing_work<B: ExtraBackendMethods>(
14231427
free_worker(worker_id);
14241428
needs_fat_lto.push(result);
14251429
}
1426-
Message::NeedsThinLTO { result, worker_id } => {
1430+
Message::NeedsThinLTO { name, thin_buffer, worker_id } => {
14271431
assert!(!started_lto);
14281432
free_worker(worker_id);
1429-
needs_thin_lto.push(result);
1433+
needs_thin_lto.push((name, thin_buffer));
14301434
}
14311435
Message::AddImportOnlyModule { module_data, work_product } => {
14321436
assert!(!started_lto);
@@ -1514,7 +1518,7 @@ fn spawn_work<B: ExtraBackendMethods>(
15141518
// we exit.
15151519
struct Bomb<B: ExtraBackendMethods> {
15161520
coordinator_send: Sender<Box<dyn Any + Send>>,
1517-
result: Option<WorkItemResult<B::Module>>,
1521+
result: Option<WorkItemResult<B>>,
15181522
worker_id: usize,
15191523
}
15201524
impl<B: ExtraBackendMethods> Drop for Bomb<B> {
@@ -1527,8 +1531,8 @@ fn spawn_work<B: ExtraBackendMethods>(
15271531
Some(WorkItemResult::NeedsFatLTO(m)) => {
15281532
Message::NeedsFatLTO::<B> { result: m, worker_id }
15291533
}
1530-
Some(WorkItemResult::NeedsThinLTO(m)) => {
1531-
Message::NeedsThinLTO::<B> { result: m, worker_id }
1534+
Some(WorkItemResult::NeedsThinLTO(name, thin_buffer)) => {
1535+
Message::NeedsThinLTO::<B> { name, thin_buffer, worker_id }
15321536
}
15331537
None => Message::Done::<B> { result: Err(()), worker_id }
15341538
};

‎src/librustc_codegen_ssa/traits/write.rs

+5-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ pub trait WriteBackendMethods: 'static + Sized + Clone {
3636
/// can simply be copied over from the incr. comp. cache.
3737
fn run_thin_lto(
3838
cgcx: &CodegenContext<Self>,
39-
modules: Vec<ModuleCodegen<Self::Module>>,
39+
modules: Vec<(String, Self::ThinBuffer)>,
4040
cached_modules: Vec<(SerializedModule<Self::ModuleBuffer>, WorkProduct)>,
4141
timeline: &mut Timeline,
4242
) -> Result<(Vec<LtoModuleCodegen<Self>>, Vec<WorkProduct>), FatalError>;
@@ -60,6 +60,10 @@ pub trait WriteBackendMethods: 'static + Sized + Clone {
6060
config: &ModuleConfig,
6161
timeline: &mut Timeline,
6262
) -> Result<CompiledModule, FatalError>;
63+
fn prepare_thin(
64+
cgcx: &CodegenContext<Self>,
65+
module: ModuleCodegen<Self::Module>
66+
) -> (String, Self::ThinBuffer);
6367
fn run_lto_pass_manager(
6468
cgcx: &CodegenContext<Self>,
6569
llmod: &ModuleCodegen<Self::Module>,

0 commit comments

Comments
 (0)