Skip to content

Commit 204bb38

Browse files
bjorn3bjorn3
committed
Make atomic instructions atomic using a global lock
1 parent 552853b commit 204bb38

File tree

5 files changed

+144
-2
lines changed

5 files changed

+144
-2
lines changed

‎src/atomic_shim.rs

+97
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
//! Atomic intrinsics are implemented using a global lock for now, as Cranelift doesn't support
2+
//! atomic operations yet.
3+
4+
// FIXME implement atomic instructions in Cranelift.
5+
6+
use crate::prelude::*;
7+
8+
#[no_mangle]
9+
pub static mut __cg_clif_global_atomic_mutex: libc::pthread_mutex_t = libc::PTHREAD_MUTEX_INITIALIZER;
10+
11+
pub fn init_global_lock(sess: &Session, module: &mut Module<impl Backend>, bcx: &mut FunctionBuilder<'_>) {
12+
if std::env::var("SHOULD_RUN").is_ok () {
13+
// When using JIT, dylibs won't find the __cg_clif_global_atomic_mutex data object defined here,
14+
// so instead define it in the cg_clif dylib.
15+
16+
return;
17+
}
18+
19+
let mut data_ctx = DataContext::new();
20+
data_ctx.define_zeroinit(1024); // 1024 bytes should be big enough on all platforms.
21+
let atomic_mutex = module.declare_data(
22+
"__cg_clif_global_atomic_mutex",
23+
Linkage::Export,
24+
true,
25+
false,
26+
Some(16),
27+
).unwrap();
28+
module.define_data(atomic_mutex, &data_ctx).unwrap();
29+
30+
let pthread_mutex_init = module.declare_function("pthread_mutex_init", Linkage::Import, &cranelift_codegen::ir::Signature {
31+
call_conv: module.target_config().default_call_conv,
32+
params: vec![
33+
AbiParam::new(module.target_config().pointer_type() /* *mut pthread_mutex_t */),
34+
AbiParam::new(module.target_config().pointer_type() /* *const pthread_mutex_attr_t */),
35+
],
36+
returns: vec![AbiParam::new(types::I32 /* c_int */)],
37+
}).unwrap();
38+
39+
let pthread_mutex_init = module.declare_func_in_func(pthread_mutex_init, bcx.func);
40+
41+
let atomic_mutex = module.declare_data_in_func(atomic_mutex, bcx.func);
42+
let atomic_mutex = bcx.ins().global_value(module.target_config().pointer_type(), atomic_mutex);
43+
44+
let nullptr = bcx.ins().iconst(module.target_config().pointer_type(), 0);
45+
46+
bcx.ins().call(pthread_mutex_init, &[atomic_mutex, nullptr]);
47+
}
48+
49+
pub fn lock_global_lock(fx: &mut FunctionCx<'_, '_, impl Backend>) {
50+
let atomic_mutex = fx.module.declare_data(
51+
"__cg_clif_global_atomic_mutex",
52+
Linkage::Import,
53+
true,
54+
false,
55+
None,
56+
).unwrap();
57+
58+
let pthread_mutex_lock = fx.module.declare_function("pthread_mutex_lock", Linkage::Import, &cranelift_codegen::ir::Signature {
59+
call_conv: fx.module.target_config().default_call_conv,
60+
params: vec![
61+
AbiParam::new(fx.module.target_config().pointer_type() /* *mut pthread_mutex_t */),
62+
],
63+
returns: vec![AbiParam::new(types::I32 /* c_int */)],
64+
}).unwrap();
65+
66+
let pthread_mutex_lock = fx.module.declare_func_in_func(pthread_mutex_lock, fx.bcx.func);
67+
68+
let atomic_mutex = fx.module.declare_data_in_func(atomic_mutex, fx.bcx.func);
69+
let atomic_mutex = fx.bcx.ins().global_value(fx.module.target_config().pointer_type(), atomic_mutex);
70+
71+
fx.bcx.ins().call(pthread_mutex_lock, &[atomic_mutex]);
72+
}
73+
74+
pub fn unlock_global_lock(fx: &mut FunctionCx<'_, '_, impl Backend>) {
75+
let atomic_mutex = fx.module.declare_data(
76+
"__cg_clif_global_atomic_mutex",
77+
Linkage::Import,
78+
true,
79+
false,
80+
None,
81+
).unwrap();
82+
83+
let pthread_mutex_unlock = fx.module.declare_function("pthread_mutex_unlock", Linkage::Import, &cranelift_codegen::ir::Signature {
84+
call_conv: fx.module.target_config().default_call_conv,
85+
params: vec![
86+
AbiParam::new(fx.module.target_config().pointer_type() /* *mut pthread_mutex_t */),
87+
],
88+
returns: vec![AbiParam::new(types::I32 /* c_int */)],
89+
}).unwrap();
90+
91+
let pthread_mutex_unlock = fx.module.declare_func_in_func(pthread_mutex_unlock, fx.bcx.func);
92+
93+
let atomic_mutex = fx.module.declare_data_in_func(atomic_mutex, fx.bcx.func);
94+
let atomic_mutex = fx.bcx.ins().global_value(fx.module.target_config().pointer_type(), atomic_mutex);
95+
96+
fx.bcx.ins().call(pthread_mutex_unlock, &[atomic_mutex]);
97+
}

‎src/driver.rs

+7
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,13 @@ pub fn codegen_crate(
3636
fn run_jit(tcx: TyCtxt<'_>) -> ! {
3737
use cranelift_simplejit::{SimpleJITBackend, SimpleJITBuilder};
3838

39+
// Rustc opens us without the RTLD_GLOBAL flag, so __cg_clif_global_atomic_mutex will not be
40+
// exported. We fix this by opening ourself again as global.
41+
// FIXME remove once atomic_shim is gone
42+
let cg_dylib = std::ffi::OsString::from(&tcx.sess.opts.debugging_opts.codegen_backend.as_ref().unwrap());
43+
std::mem::forget(libloading::os::unix::Library::open(Some(cg_dylib), libc::RTLD_NOW | libc::RTLD_GLOBAL).unwrap());
44+
45+
3946
let imported_symbols = load_imported_symbols_for_jit(tcx);
4047

4148
let mut jit_builder = SimpleJITBuilder::with_isa(

‎src/intrinsics/mod.rs

+36-2
Original file line numberDiff line numberDiff line change
@@ -104,14 +104,20 @@ macro call_intrinsic_match {
104104
}
105105

106106
macro atomic_binop_return_old($fx:expr, $op:ident<$T:ident>($ptr:ident, $src:ident) -> $ret:ident) {
107+
crate::atomic_shim::lock_global_lock($fx);
108+
107109
let clif_ty = $fx.clif_type($T).unwrap();
108110
let old = $fx.bcx.ins().load(clif_ty, MemFlags::new(), $ptr, 0);
109111
let new = $fx.bcx.ins().$op(old, $src);
110112
$fx.bcx.ins().store(MemFlags::new(), new, $ptr, 0);
111113
$ret.write_cvalue($fx, CValue::by_val(old, $fx.layout_of($T)));
114+
115+
crate::atomic_shim::unlock_global_lock($fx);
112116
}
113117

114118
macro atomic_minmax($fx:expr, $cc:expr, <$T:ident> ($ptr:ident, $src:ident) -> $ret:ident) {
119+
crate::atomic_shim::lock_global_lock($fx);
120+
115121
// Read old
116122
let clif_ty = $fx.clif_type($T).unwrap();
117123
let old = $fx.bcx.ins().load(clif_ty, MemFlags::new(), $ptr, 0);
@@ -125,6 +131,8 @@ macro atomic_minmax($fx:expr, $cc:expr, <$T:ident> ($ptr:ident, $src:ident) -> $
125131

126132
let ret_val = CValue::by_val(old, $ret.layout());
127133
$ret.write_cvalue($fx, ret_val);
134+
135+
crate::atomic_shim::unlock_global_lock($fx);
128136
}
129137

130138
fn lane_type_and_count<'tcx>(
@@ -845,19 +853,35 @@ pub fn codegen_intrinsic_call<'tcx>(
845853
ret.write_cvalue(fx, caller_location);
846854
};
847855

848-
_ if intrinsic.starts_with("atomic_fence"), () {};
849-
_ if intrinsic.starts_with("atomic_singlethreadfence"), () {};
856+
_ if intrinsic.starts_with("atomic_fence"), () {
857+
crate::atomic_shim::lock_global_lock(fx);
858+
crate::atomic_shim::unlock_global_lock(fx);
859+
};
860+
_ if intrinsic.starts_with("atomic_singlethreadfence"), () {
861+
crate::atomic_shim::lock_global_lock(fx);
862+
crate::atomic_shim::unlock_global_lock(fx);
863+
};
850864
_ if intrinsic.starts_with("atomic_load"), (c ptr) {
865+
crate::atomic_shim::lock_global_lock(fx);
866+
851867
let inner_layout =
852868
fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty);
853869
let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), inner_layout);
854870
ret.write_cvalue(fx, val);
871+
872+
crate::atomic_shim::unlock_global_lock(fx);
855873
};
856874
_ if intrinsic.starts_with("atomic_store"), (v ptr, c val) {
875+
crate::atomic_shim::lock_global_lock(fx);
876+
857877
let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout());
858878
dest.write_cvalue(fx, val);
879+
880+
crate::atomic_shim::unlock_global_lock(fx);
859881
};
860882
_ if intrinsic.starts_with("atomic_xchg"), <T> (v ptr, c src) {
883+
crate::atomic_shim::lock_global_lock(fx);
884+
861885
// Read old
862886
let clif_ty = fx.clif_type(T).unwrap();
863887
let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0);
@@ -866,8 +890,12 @@ pub fn codegen_intrinsic_call<'tcx>(
866890
// Write new
867891
let dest = CPlace::for_ptr(Pointer::new(ptr), src.layout());
868892
dest.write_cvalue(fx, src);
893+
894+
crate::atomic_shim::unlock_global_lock(fx);
869895
};
870896
_ if intrinsic.starts_with("atomic_cxchg"), <T> (v ptr, v test_old, v new) { // both atomic_cxchg_* and atomic_cxchgweak_*
897+
crate::atomic_shim::lock_global_lock(fx);
898+
871899
// Read old
872900
let clif_ty = fx.clif_type(T).unwrap();
873901
let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0);
@@ -881,6 +909,8 @@ pub fn codegen_intrinsic_call<'tcx>(
881909

882910
let ret_val = CValue::by_val_pair(old, fx.bcx.ins().bint(types::I8, is_eq), ret.layout());
883911
ret.write_cvalue(fx, ret_val);
912+
913+
crate::atomic_shim::unlock_global_lock(fx);
884914
};
885915

886916
_ if intrinsic.starts_with("atomic_xadd"), <T> (v ptr, v amount) {
@@ -893,12 +923,16 @@ pub fn codegen_intrinsic_call<'tcx>(
893923
atomic_binop_return_old! (fx, band<T>(ptr, src) -> ret);
894924
};
895925
_ if intrinsic.starts_with("atomic_nand"), <T> (v ptr, v src) {
926+
crate::atomic_shim::lock_global_lock(fx);
927+
896928
let clif_ty = fx.clif_type(T).unwrap();
897929
let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0);
898930
let and = fx.bcx.ins().band(old, src);
899931
let new = fx.bcx.ins().bnot(and);
900932
fx.bcx.ins().store(MemFlags::new(), new, ptr, 0);
901933
ret.write_cvalue(fx, CValue::by_val(old, fx.layout_of(T)));
934+
935+
crate::atomic_shim::unlock_global_lock(fx);
902936
};
903937
_ if intrinsic.starts_with("atomic_or"), <T> (v ptr, v src) {
904938
atomic_binop_return_old! (fx, bor<T>(ptr, src) -> ret);

‎src/lib.rs

+2
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
#![allow(intra_doc_link_resolution_failure)]
33

44
extern crate flate2;
5+
extern crate libc;
56
extern crate tempfile;
67
extern crate rustc;
78
extern crate rustc_codegen_ssa;
@@ -36,6 +37,7 @@ mod abi;
3637
mod allocator;
3738
mod analyze;
3839
mod archive;
40+
mod atomic_shim;
3941
mod base;
4042
mod backend;
4143
mod cast;

‎src/main_shim.rs

+2
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@ pub fn maybe_create_entry_wrapper(tcx: TyCtxt<'_>, module: &mut Module<impl Back
6767
let arg_argc = bcx.append_ebb_param(ebb, m.target_config().pointer_type());
6868
let arg_argv = bcx.append_ebb_param(ebb, m.target_config().pointer_type());
6969

70+
crate::atomic_shim::init_global_lock(tcx.sess, m, &mut bcx);
71+
7072
let main_func_ref = m.declare_func_in_func(main_func_id, &mut bcx.func);
7173

7274
let call_inst = if use_start_lang_item {

0 commit comments

Comments
 (0)