Skip to content

Commit 30ff300

Browse files
dpaoliellodpaoliello
committed
Enable constant folding across crates (weak linkage + comdat)
1 parent b03864d commit 30ff300

File tree

17 files changed

+217
-40
lines changed

17 files changed

+217
-40
lines changed

‎compiler/rustc_codegen_llvm/src/common.rs

Lines changed: 41 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22
33
use crate::consts::const_alloc_to_llvm;
44
pub use crate::context::CodegenCx;
5+
use crate::debuginfo::metadata::{
6+
build_const_str_di_node, build_opaque_pointer_global_var_di_node,
7+
};
58
use crate::llvm::{self, BasicBlock, Bool, ConstantInt, False, OperandBundleDef, True};
69
use crate::type_::Type;
710
use crate::value::Value;
@@ -11,7 +14,9 @@ use rustc_codegen_ssa::traits::*;
1114
use rustc_data_structures::stable_hasher::{Hash128, HashStable, StableHasher};
1215
use rustc_hir::def_id::DefId;
1316
use rustc_middle::bug;
14-
use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, Scalar};
17+
use rustc_middle::mir::interpret::{
18+
ConstAllocation, ConstAllocationDebugHint, GlobalAlloc, Scalar,
19+
};
1520
use rustc_middle::ty::TyCtxt;
1621
use rustc_session::cstore::{DllCallingConvention, DllImport, PeImportNameType};
1722
use rustc_target::abi::{self, AddressSpace, HasDataLayout, Pointer};
@@ -20,6 +25,8 @@ use rustc_target::spec::Target;
2025
use libc::{c_char, c_uint};
2126
use std::fmt::Write;
2227

28+
const RUST_VERSION: &str = env!("CFG_RELEASE_NUM");
29+
2330
/*
2431
* A note on nomenclature of linking: "extern", "foreign", and "upcall".
2532
*
@@ -196,15 +203,23 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
196203
.from_key(s)
197204
.or_insert_with(|| {
198205
let sc = self.const_bytes(s.as_bytes());
199-
let sym = self.generate_local_symbol_name("str");
206+
let hash = self.tcx.with_stable_hashing_context(|mut hcx| {
207+
let mut hasher = StableHasher::new();
208+
s.hash_stable(&mut hcx, &mut hasher);
209+
hasher.finish::<Hash128>()
210+
});
211+
let sym = format!("__rust_{RUST_VERSION}_conststr_{hash:032x}");
200212
let g = self.define_global(&sym, self.val_ty(sc)).unwrap_or_else(|| {
201213
bug!("symbol `{}` is already defined", sym);
202214
});
203215
unsafe {
204216
llvm::LLVMSetInitializer(g, sc);
205217
llvm::LLVMSetGlobalConstant(g, True);
206-
llvm::LLVMRustSetLinkage(g, llvm::Linkage::InternalLinkage);
218+
llvm::LLVMRustSetLinkage(g, llvm::Linkage::LinkOnceODRLinkage);
219+
llvm::SetUniqueComdat(self.llmod, g);
220+
llvm::LLVMRustSetVisibility(g, llvm::Visibility::Hidden);
207221
}
222+
build_const_str_di_node(self, &sym, g);
208223
(s.to_owned(), g)
209224
})
210225
.1;
@@ -249,18 +264,38 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
249264
let (base_addr, base_addr_space) = match self.tcx.global_alloc(alloc_id) {
250265
GlobalAlloc::Memory(alloc) => {
251266
let init = const_alloc_to_llvm(self, alloc);
267+
let debug_hint = alloc.1;
252268
let alloc = alloc.inner();
253269
let value = match alloc.mutability {
254270
Mutability::Mut => self.static_addr_of_mut(init, alloc.align, None),
255-
_ => self.static_addr_of(init, alloc.align, None),
271+
_ => {
272+
let value = self.static_addr_of(init, alloc.align, None);
273+
llvm::set_linkage(value, llvm::Linkage::LinkOnceODRLinkage);
274+
llvm::set_visibility(value, llvm::Visibility::Hidden);
275+
value
276+
}
256277
};
257-
if !self.sess().fewer_names() && llvm::get_value_name(value).is_empty() {
278+
279+
if llvm::get_value_name(value).is_empty() {
280+
let name_prefix = match debug_hint {
281+
Some(ConstAllocationDebugHint::StrLiteral) => "str",
282+
Some(ConstAllocationDebugHint::CallerLocation) => "callerloc",
283+
Some(ConstAllocationDebugHint::TypeName) => "typename",
284+
Some(ConstAllocationDebugHint::VTable) => "vtable",
285+
None => "alloc",
286+
};
287+
258288
let hash = self.tcx.with_stable_hashing_context(|mut hcx| {
259289
let mut hasher = StableHasher::new();
260290
alloc.hash_stable(&mut hcx, &mut hasher);
261291
hasher.finish::<Hash128>()
262292
});
263-
llvm::set_value_name(value, format!("alloc_{hash:032x}").as_bytes());
293+
let name = format!("__rust_{RUST_VERSION}_{name_prefix}_{hash:032x}");
294+
llvm::set_value_name(value, name.as_bytes());
295+
build_opaque_pointer_global_var_di_node(self, &name, value);
296+
if alloc.mutability == Mutability::Not {
297+
llvm::SetUniqueComdat(self.llmod, value);
298+
}
264299
}
265300
(value, AddressSpace::DATA)
266301
}

‎compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1252,6 +1252,62 @@ pub fn build_global_var_di_node<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId, glo
12521252
}
12531253
}
12541254

1255+
/// Creates debug information for the given constant string.
1256+
///
1257+
/// Adds the created debuginfo nodes directly to the crate's IR.
1258+
pub fn build_const_str_di_node<'ll>(
1259+
cx: &CodegenCx<'ll, '_>,
1260+
var_name: &str,
1261+
const_str: &'ll Value,
1262+
) {
1263+
if cx.dbg_cx.is_none() {
1264+
return;
1265+
}
1266+
1267+
// Only create type information if full debuginfo is enabled
1268+
if cx.sess().opts.debuginfo != DebugInfo::Full {
1269+
return;
1270+
}
1271+
1272+
unsafe {
1273+
llvm::LLVMRustDIBuilderCreateConstStr(
1274+
DIB(cx),
1275+
var_name.as_ptr().cast(),
1276+
var_name.len(),
1277+
type_di_node(cx, cx.tcx.types.str_),
1278+
const_str,
1279+
);
1280+
}
1281+
}
1282+
1283+
/// Creates debug information for the given global variable which is a pointer to an
1284+
/// unknown or opaque type.
1285+
///
1286+
/// Adds the created debuginfo nodes directly to the crate's IR.
1287+
pub fn build_opaque_pointer_global_var_di_node<'ll>(
1288+
cx: &CodegenCx<'ll, '_>,
1289+
var_name: &str,
1290+
global: &'ll Value,
1291+
) {
1292+
if cx.dbg_cx.is_none() {
1293+
return;
1294+
}
1295+
1296+
// Only create type information if full debuginfo is enabled
1297+
if cx.sess().opts.debuginfo != DebugInfo::Full {
1298+
return;
1299+
}
1300+
1301+
unsafe {
1302+
llvm::LLVMRustDIBuilderCreateOpaquePointerStaticVariable(
1303+
DIB(cx),
1304+
var_name.as_ptr().cast(),
1305+
var_name.len(),
1306+
global,
1307+
);
1308+
}
1309+
}
1310+
12551311
/// Generates LLVM debuginfo for a vtable.
12561312
///
12571313
/// The vtable type looks like a struct with a field for each function pointer and super-trait

‎compiler/rustc_codegen_llvm/src/llvm/ffi.rs

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1948,6 +1948,21 @@ extern "C" {
19481948
AlignInBits: u32,
19491949
) -> &'a DIGlobalVariableExpression;
19501950

1951+
pub fn LLVMRustDIBuilderCreateConstStr<'a>(
1952+
Builder: &DIBuilder<'a>,
1953+
Name: *const c_char,
1954+
NameLen: size_t,
1955+
Ty: &'a DIType,
1956+
Val: &'a Value,
1957+
) -> &'a DIGlobalVariableExpression;
1958+
1959+
pub fn LLVMRustDIBuilderCreateOpaquePointerStaticVariable<'a>(
1960+
Builder: &DIBuilder<'a>,
1961+
Name: *const c_char,
1962+
NameLen: size_t,
1963+
Val: &'a Value,
1964+
) -> &'a DIGlobalVariableExpression;
1965+
19511966
pub fn LLVMRustDIBuilderCreateVariable<'a>(
19521967
Builder: &DIBuilder<'a>,
19531968
Tag: c_uint,

‎compiler/rustc_const_eval/src/const_eval/eval_queries.rs

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -177,9 +177,10 @@ pub(super) fn op_to_const<'tcx>(
177177
(ecx.tcx.global_alloc(alloc_id).unwrap_memory(), offset.bytes())
178178
}
179179
(None, _offset) => (
180-
ecx.tcx.mk_const_alloc(Allocation::from_bytes_byte_aligned_immutable(
181-
b"" as &[u8],
182-
)),
180+
ecx.tcx.mk_const_alloc(
181+
Allocation::from_bytes_byte_aligned_immutable(b"" as &[u8]),
182+
None,
183+
),
183184
0,
184185
),
185186
};

‎compiler/rustc_const_eval/src/interpret/intern.rs

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ use super::validity::RefTracking;
1818
use rustc_data_structures::fx::{FxIndexMap, FxIndexSet};
1919
use rustc_errors::ErrorGuaranteed;
2020
use rustc_hir as hir;
21-
use rustc_middle::mir::interpret::InterpResult;
21+
use rustc_middle::mir::interpret::{ConstAllocationDebugHint, InterpResult};
2222
use rustc_middle::ty::{self, layout::TyAndLayout, Ty};
2323

2424
use rustc_ast::Mutability;
@@ -104,11 +104,11 @@ fn intern_shallow<'rt, 'mir, 'tcx, M: CompileTimeMachine<'mir, 'tcx, const_eval:
104104
};
105105
// This match is just a canary for future changes to `MemoryKind`, which most likely need
106106
// changes in this function.
107-
match kind {
108-
MemoryKind::Stack
109-
| MemoryKind::Machine(const_eval::MemoryKind::Heap)
110-
| MemoryKind::CallerLocation => {}
111-
}
107+
let const_allocation_kind = match kind {
108+
MemoryKind::Stack | MemoryKind::Machine(const_eval::MemoryKind::Heap) => None,
109+
MemoryKind::CallerLocation => Some(ConstAllocationDebugHint::CallerLocation),
110+
};
111+
112112
// Set allocation mutability as appropriate. This is used by LLVM to put things into
113113
// read-only memory, and also by Miri when evaluating other globals that
114114
// access this one.
@@ -136,7 +136,7 @@ fn intern_shallow<'rt, 'mir, 'tcx, M: CompileTimeMachine<'mir, 'tcx, const_eval:
136136
};
137137
// link the alloc id to the actual allocation
138138
leftover_allocations.extend(alloc.provenance().ptrs().iter().map(|&(_, alloc_id)| alloc_id));
139-
let alloc = tcx.mk_const_alloc(alloc);
139+
let alloc = tcx.mk_const_alloc(alloc, const_allocation_kind);
140140
tcx.set_alloc_id_memory(alloc_id, alloc);
141141
None
142142
}
@@ -428,7 +428,7 @@ pub fn intern_const_alloc_recursive<
428428
alloc.mutability = Mutability::Not;
429429
}
430430
}
431-
let alloc = tcx.mk_const_alloc(alloc);
431+
let alloc = tcx.mk_const_alloc(alloc, None);
432432
tcx.set_alloc_id_memory(alloc_id, alloc);
433433
for &(_, alloc_id) in alloc.inner().provenance().ptrs().iter() {
434434
if leftover_allocations.insert(alloc_id) {
@@ -467,6 +467,6 @@ impl<'mir, 'tcx: 'mir, M: super::intern::CompileTimeMachine<'mir, 'tcx, !>>
467467
f(self, &dest.clone().into())?;
468468
let mut alloc = self.memory.alloc_map.remove(&dest.ptr.provenance.unwrap()).unwrap().1;
469469
alloc.mutability = Mutability::Not;
470-
Ok(self.tcx.mk_const_alloc(alloc))
470+
Ok(self.tcx.mk_const_alloc(alloc, None))
471471
}
472472
}

‎compiler/rustc_const_eval/src/interpret/intrinsics.rs

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@ use rustc_hir::def_id::DefId;
66
use rustc_middle::mir::{
77
self,
88
interpret::{
9-
Allocation, ConstAllocation, ConstValue, GlobalId, InterpResult, PointerArithmetic, Scalar,
9+
Allocation, ConstAllocation, ConstAllocationDebugHint, ConstValue, GlobalId, InterpResult,
10+
PointerArithmetic, Scalar,
1011
},
1112
BinOp, NonDivergingIntrinsic,
1213
};
@@ -47,7 +48,7 @@ fn numeric_intrinsic<Prov>(name: Symbol, bits: u128, kind: Primitive) -> Scalar<
4748
pub(crate) fn alloc_type_name<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> ConstAllocation<'tcx> {
4849
let path = crate::util::type_name(tcx, ty);
4950
let alloc = Allocation::from_bytes_byte_aligned_immutable(path.into_bytes());
50-
tcx.mk_const_alloc(alloc)
51+
tcx.mk_const_alloc(alloc, Some(ConstAllocationDebugHint::TypeName))
5152
}
5253

5354
/// The logic for all nullary intrinsics is implemented here. These intrinsics don't get evaluated

‎compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1035,6 +1035,50 @@ extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateStaticVariable(
10351035
return wrap(VarExpr);
10361036
}
10371037

1038+
extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateConstStr(
1039+
LLVMRustDIBuilderRef Builder,
1040+
const char *Name, size_t NameLen,
1041+
LLVMMetadataRef Ty, LLVMValueRef V) {
1042+
llvm::GlobalVariable *InitVal = cast<llvm::GlobalVariable>(unwrap(V));
1043+
1044+
llvm::DIGlobalVariableExpression *VarExpr = Builder->createGlobalVariableExpression(
1045+
/* context */ nullptr, StringRef(Name, NameLen),
1046+
/* linkageName */ StringRef(),
1047+
nullptr, 0, unwrapDI<DIType>(Ty), /* isLocalToUnit */ false,
1048+
/* isDefined */ true,
1049+
/* expr */ nullptr, /* decl */ nullptr,
1050+
/* templateParams */ nullptr,
1051+
/* alignInBits */ 0);
1052+
1053+
InitVal->setMetadata("dbg", VarExpr);
1054+
1055+
return wrap(VarExpr);
1056+
}
1057+
1058+
extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateOpaquePointerStaticVariable(
1059+
LLVMRustDIBuilderRef Builder,
1060+
const char *Name, size_t NameLen,
1061+
LLVMValueRef V) {
1062+
llvm::GlobalVariable *InitVal = cast<llvm::GlobalVariable>(unwrap(V));
1063+
1064+
DataLayout Layout = InitVal->getParent()->getDataLayout();
1065+
DIType *DebugType = Builder->createPointerType(nullptr, Layout.getTypeSizeInBits(
1066+
InitVal->getValueType()));
1067+
1068+
llvm::DIGlobalVariableExpression *VarExpr = Builder->createGlobalVariableExpression(
1069+
/* context */ nullptr, StringRef(Name, NameLen),
1070+
/* linkageName */ StringRef(),
1071+
nullptr, 0, DebugType, /* isLocalToUnit */ false,
1072+
/* isDefined */ true,
1073+
/* expr */ nullptr, /* decl */ nullptr,
1074+
/* templateParams */ nullptr,
1075+
/* alignInBits */ 0);
1076+
1077+
InitVal->setMetadata("dbg", VarExpr);
1078+
1079+
return wrap(VarExpr);
1080+
}
1081+
10381082
extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateVariable(
10391083
LLVMRustDIBuilderRef Builder, unsigned Tag, LLVMMetadataRef Scope,
10401084
const char *Name, size_t NameLen,

‎compiler/rustc_middle/src/mir/interpret/allocation.rs

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,10 @@ impl hash::Hash for Allocation {
150150
/// (`ConstAllocation`) are used quite a bit.
151151
#[derive(Copy, Clone, PartialEq, Eq, Hash, HashStable)]
152152
#[rustc_pass_by_value]
153-
pub struct ConstAllocation<'tcx>(pub Interned<'tcx, Allocation>);
153+
pub struct ConstAllocation<'tcx>(
154+
pub Interned<'tcx, Allocation>,
155+
pub Option<ConstAllocationDebugHint>,
156+
);
154157

155158
impl<'tcx> fmt::Debug for ConstAllocation<'tcx> {
156159
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
@@ -166,6 +169,20 @@ impl<'tcx> ConstAllocation<'tcx> {
166169
}
167170
}
168171

172+
/// Hint used to tag the name of a `ConstAllocation` in debug information so
173+
/// that data in the final binary can be attributed back to the allocator.
174+
#[derive(Copy, Clone, PartialEq, Eq, Hash, HashStable, Encodable, Decodable)]
175+
pub enum ConstAllocationDebugHint {
176+
/// A string literal.
177+
StrLiteral,
178+
/// Caller infromation (e.g., for panics).
179+
CallerLocation,
180+
/// The name of a type.
181+
TypeName,
182+
/// A vtable.
183+
VTable,
184+
}
185+
169186
/// We have our own error type that does not know about the `AllocId`; that information
170187
/// is added when converting to `InterpError`.
171188
#[derive(Debug)]

‎compiler/rustc_middle/src/mir/interpret/mod.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ pub use self::value::{get_slice_bytes, ConstAlloc, ConstValue, Scalar};
153153

154154
pub use self::allocation::{
155155
alloc_range, AllocBytes, AllocError, AllocRange, AllocResult, Allocation, ConstAllocation,
156-
InitChunk, InitChunkIter,
156+
ConstAllocationDebugHint, InitChunk, InitChunkIter,
157157
};
158158

159159
pub use self::pointer::{Pointer, PointerArithmetic, Provenance};

‎compiler/rustc_middle/src/ty/codec.rs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,7 @@ impl<'tcx, E: TyEncoder<I = TyCtxt<'tcx>>> Encodable<E> for ty::Const<'tcx> {
148148

149149
impl<'tcx, E: TyEncoder<I = TyCtxt<'tcx>>> Encodable<E> for ConstAllocation<'tcx> {
150150
fn encode(&self, e: &mut E) {
151+
self.1.encode(e);
151152
self.inner().encode(e)
152153
}
153154
}
@@ -356,7 +357,8 @@ impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D> for [ty::ValTre
356357

357358
impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> Decodable<D> for ConstAllocation<'tcx> {
358359
fn decode(decoder: &mut D) -> Self {
359-
decoder.interner().mk_const_alloc(Decodable::decode(decoder))
360+
let debug_hint = Decodable::decode(decoder);
361+
decoder.interner().mk_const_alloc(Decodable::decode(decoder), debug_hint)
360362
}
361363
}
362364

0 commit comments

Comments
 (0)