Skip to content

Commit 9b7150a

Browse files
committed
Use sharded maps for interning and queries
1 parent 241aad3 commit 9b7150a

File tree

7 files changed

+183
-108
lines changed

7 files changed

+183
-108
lines changed

src/librustc/ty/context.rs

Lines changed: 22 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -48,14 +48,14 @@ use crate::util::common::ErrorReported;
4848
use crate::util::nodemap::{DefIdMap, DefIdSet, ItemLocalMap, ItemLocalSet};
4949
use crate::util::nodemap::{FxHashMap, FxHashSet};
5050
use errors::DiagnosticBuilder;
51-
use rustc_data_structures::interner::HashInterner;
5251
use smallvec::SmallVec;
5352
use rustc_data_structures::stable_hasher::{HashStable, hash_stable_hashmap,
5453
StableHasher, StableHasherResult,
5554
StableVec};
5655
use arena::SyncDroplessArena;
5756
use rustc_data_structures::indexed_vec::{Idx, IndexVec};
5857
use rustc_data_structures::sync::{Lrc, Lock, WorkerLocal};
58+
use rustc_data_structures::sharded::ShardedHashMap;
5959
use std::any::Any;
6060
use std::borrow::Borrow;
6161
use std::cmp::Ordering;
@@ -93,7 +93,7 @@ impl AllArenas {
9393
}
9494
}
9595

96-
type InternedSet<'tcx, T> = Lock<FxHashMap<Interned<'tcx, T>, ()>>;
96+
type InternedSet<'tcx, T> = ShardedHashMap<Interned<'tcx, T>, ()>;
9797

9898
pub struct CtxtInterners<'tcx> {
9999
/// The arena that types, regions, etc are allocated from
@@ -147,7 +147,7 @@ impl<'tcx> CtxtInterners<'tcx> {
147147
// determine that all contents are in the global tcx.
148148
// See comments on Lift for why we can't use that.
149149
if flags.flags.intersects(ty::TypeFlags::KEEP_IN_LOCAL_TCX) {
150-
local.type_.borrow_mut().intern(st, |st| {
150+
local.type_.intern(st, |st| {
151151
let ty_struct = TyS {
152152
sty: st,
153153
flags: flags.flags,
@@ -171,7 +171,7 @@ impl<'tcx> CtxtInterners<'tcx> {
171171
Interned(local.arena.alloc(ty_struct))
172172
}).0
173173
} else {
174-
global.type_.borrow_mut().intern(st, |st| {
174+
global.type_.intern(st, |st| {
175175
let ty_struct = TyS {
176176
sty: st,
177177
flags: flags.flags,
@@ -964,7 +964,7 @@ impl<'tcx> CommonTypes<'tcx> {
964964
impl<'tcx> CommonLifetimes<'tcx> {
965965
fn new(interners: &CtxtInterners<'tcx>) -> CommonLifetimes<'tcx> {
966966
let mk = |r| {
967-
interners.region.borrow_mut().intern(r, |r| {
967+
interners.region.intern(r, |r| {
968968
Interned(interners.arena.alloc(r))
969969
}).0
970970
};
@@ -980,7 +980,7 @@ impl<'tcx> CommonLifetimes<'tcx> {
980980
impl<'tcx> CommonConsts<'tcx> {
981981
fn new(interners: &CtxtInterners<'tcx>, types: &CommonTypes<'tcx>) -> CommonConsts<'tcx> {
982982
let mk_const = |c| {
983-
interners.const_.borrow_mut().intern(c, |c| {
983+
interners.const_.intern(c, |c| {
984984
Interned(interners.arena.alloc(c))
985985
}).0
986986
};
@@ -1096,14 +1096,14 @@ pub struct GlobalCtxt<'tcx> {
10961096
/// Data layout specification for the current target.
10971097
pub data_layout: TargetDataLayout,
10981098

1099-
stability_interner: Lock<FxHashMap<&'tcx attr::Stability, ()>>,
1099+
stability_interner: ShardedHashMap<&'tcx attr::Stability, ()>,
11001100

11011101
/// Stores the value of constants (and deduplicates the actual memory)
1102-
allocation_interner: Lock<FxHashMap<&'tcx Allocation, ()>>,
1102+
allocation_interner: ShardedHashMap<&'tcx Allocation, ()>,
11031103

11041104
pub alloc_map: Lock<interpret::AllocMap<'tcx>>,
11051105

1106-
layout_interner: Lock<FxHashMap<&'tcx LayoutDetails, ()>>,
1106+
layout_interner: ShardedHashMap<&'tcx LayoutDetails, ()>,
11071107

11081108
/// A general purpose channel to throw data out the back towards LLVM worker
11091109
/// threads.
@@ -1148,7 +1148,7 @@ impl<'tcx> TyCtxt<'tcx> {
11481148
}
11491149

11501150
pub fn intern_const_alloc(self, alloc: Allocation) -> &'tcx Allocation {
1151-
self.allocation_interner.borrow_mut().intern(alloc, |alloc| {
1151+
self.allocation_interner.intern(alloc, |alloc| {
11521152
self.arena.alloc(alloc)
11531153
})
11541154
}
@@ -1162,13 +1162,13 @@ impl<'tcx> TyCtxt<'tcx> {
11621162
}
11631163

11641164
pub fn intern_stability(self, stab: attr::Stability) -> &'tcx attr::Stability {
1165-
self.stability_interner.borrow_mut().intern(stab, |stab| {
1165+
self.stability_interner.intern(stab, |stab| {
11661166
self.arena.alloc(stab)
11671167
})
11681168
}
11691169

11701170
pub fn intern_layout(self, layout: LayoutDetails) -> &'tcx LayoutDetails {
1171-
self.layout_interner.borrow_mut().intern(layout, |layout| {
1171+
self.layout_interner.intern(layout, |layout| {
11721172
self.arena.alloc(layout)
11731173
})
11741174
}
@@ -2110,7 +2110,9 @@ macro_rules! sty_debug_print {
21102110
};
21112111
$(let mut $variant = total;)*
21122112

2113-
for &Interned(t) in tcx.interners.type_.borrow().keys() {
2113+
let shards = tcx.interners.type_.lock_shards();
2114+
let types = shards.iter().flat_map(|shard| shard.keys());
2115+
for &Interned(t) in types {
21142116
let variant = match t.sty {
21152117
ty::Bool | ty::Char | ty::Int(..) | ty::Uint(..) |
21162118
ty::Float(..) | ty::Str | ty::Never => continue,
@@ -2161,11 +2163,11 @@ impl<'tcx> TyCtxt<'tcx> {
21612163
Generator, GeneratorWitness, Dynamic, Closure, Tuple, Bound,
21622164
Param, Infer, UnnormalizedProjection, Projection, Opaque, Foreign);
21632165

2164-
println!("InternalSubsts interner: #{}", self.interners.substs.borrow().len());
2165-
println!("Region interner: #{}", self.interners.region.borrow().len());
2166-
println!("Stability interner: #{}", self.stability_interner.borrow().len());
2167-
println!("Allocation interner: #{}", self.allocation_interner.borrow().len());
2168-
println!("Layout interner: #{}", self.layout_interner.borrow().len());
2166+
println!("InternalSubsts interner: #{}", self.interners.substs.len());
2167+
println!("Region interner: #{}", self.interners.region.len());
2168+
println!("Stability interner: #{}", self.stability_interner.len());
2169+
println!("Allocation interner: #{}", self.allocation_interner.len());
2170+
println!("Layout interner: #{}", self.layout_interner.len());
21692171
}
21702172
}
21712173

@@ -2298,7 +2300,7 @@ macro_rules! intern_method {
22982300
// determine that all contents are in the global tcx.
22992301
// See comments on Lift for why we can't use that.
23002302
if ($keep_in_local_tcx)(&v) {
2301-
self.interners.$name.borrow_mut().intern_ref(key, || {
2303+
self.interners.$name.intern_ref(key, || {
23022304
// Make sure we don't end up with inference
23032305
// types/regions in the global tcx.
23042306
if self.is_global() {
@@ -2310,7 +2312,7 @@ macro_rules! intern_method {
23102312
Interned($alloc_method(&self.interners.arena, v))
23112313
}).0
23122314
} else {
2313-
self.global_interners.$name.borrow_mut().intern_ref(key, || {
2315+
self.global_interners.$name.intern_ref(key, || {
23142316
Interned($alloc_method(&self.global_interners.arena, v))
23152317
}).0
23162318
}

src/librustc/ty/query/config.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ use crate::util::profiling::ProfileCategory;
1111
use std::borrow::Cow;
1212
use std::hash::Hash;
1313
use std::fmt::Debug;
14-
use rustc_data_structures::sync::Lock;
14+
use rustc_data_structures::sharded::Sharded;
1515
use rustc_data_structures::fingerprint::Fingerprint;
1616
use crate::ich::StableHashingContext;
1717

@@ -31,7 +31,7 @@ pub(crate) trait QueryAccessors<'tcx>: QueryConfig<'tcx> {
3131
fn query(key: Self::Key) -> Query<'tcx>;
3232

3333
// Don't use this method to access query results, instead use the methods on TyCtxt
34-
fn query_cache<'a>(tcx: TyCtxt<'tcx>) -> &'a Lock<QueryCache<'tcx, Self>>;
34+
fn query_cache<'a>(tcx: TyCtxt<'tcx>) -> &'a Sharded<QueryCache<'tcx, Self>>;
3535

3636
fn to_dep_node(tcx: TyCtxt<'tcx>, key: &Self::Key) -> DepNode;
3737

src/librustc/ty/query/on_disk_cache.rs

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -224,9 +224,9 @@ impl<'sess> OnDiskCache<'sess> {
224224

225225
// const eval is special, it only encodes successfully evaluated constants
226226
use crate::ty::query::QueryAccessors;
227-
let cache = const_eval::query_cache(tcx).borrow();
228-
assert!(cache.active.is_empty());
229-
for (key, entry) in cache.results.iter() {
227+
let shards = const_eval::query_cache(tcx).lock_shards();
228+
assert!(shards.iter().all(|shard| shard.active.is_empty()));
229+
for (key, entry) in shards.iter().flat_map(|shard| shard.results.iter()) {
230230
use crate::ty::query::config::QueryDescription;
231231
if const_eval::cache_on_disk(tcx, key.clone()) {
232232
if let Ok(ref value) = entry.value {
@@ -1087,9 +1087,9 @@ where
10871087
unsafe { ::std::intrinsics::type_name::<Q>() });
10881088

10891089
time_ext(tcx.sess.time_extended(), Some(tcx.sess), desc, || {
1090-
let map = Q::query_cache(tcx).borrow();
1091-
assert!(map.active.is_empty());
1092-
for (key, entry) in map.results.iter() {
1090+
let shards = Q::query_cache(tcx).lock_shards();
1091+
assert!(shards.iter().all(|shard| shard.active.is_empty()));
1092+
for (key, entry) in shards.iter().flat_map(|shard| shard.results.iter()) {
10931093
if Q::cache_on_disk(tcx, key.clone()) {
10941094
let dep_node = SerializedDepNodeIndex::new(entry.index.index());
10951095

src/librustc/ty/query/plumbing.rs

Lines changed: 23 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ use errors::Diagnostic;
1717
use errors::FatalError;
1818
use rustc_data_structures::fx::{FxHashMap};
1919
use rustc_data_structures::sync::{Lrc, Lock};
20+
use rustc_data_structures::sharded::Sharded;
2021
use rustc_data_structures::thin_vec::ThinVec;
2122
#[cfg(not(parallel_compiler))]
2223
use rustc_data_structures::cold_path;
@@ -90,7 +91,7 @@ macro_rules! profq_query_msg {
9091
/// A type representing the responsibility to execute the job in the `job` field.
9192
/// This will poison the relevant query if dropped.
9293
pub(super) struct JobOwner<'a, 'tcx, Q: QueryDescription<'tcx> + 'a> {
93-
cache: &'a Lock<QueryCache<'tcx, Q>>,
94+
cache: &'a Sharded<QueryCache<'tcx, Q>>,
9495
key: Q::Key,
9596
job: Lrc<QueryJob<'tcx>>,
9697
}
@@ -107,7 +108,7 @@ impl<'a, 'tcx, Q: QueryDescription<'tcx>> JobOwner<'a, 'tcx, Q> {
107108
pub(super) fn try_get(tcx: TyCtxt<'tcx>, span: Span, key: &Q::Key) -> TryGetJob<'a, 'tcx, Q> {
108109
let cache = Q::query_cache(tcx);
109110
loop {
110-
let mut lock = cache.borrow_mut();
111+
let mut lock = cache.get_shard_by_value(key).lock();
111112
if let Some(value) = lock.results.get(key) {
112113
profq_msg!(tcx, ProfileQueriesMsg::CacheHit);
113114
tcx.sess.profiler(|p| p.record_query_hit(Q::NAME));
@@ -191,7 +192,7 @@ impl<'a, 'tcx, Q: QueryDescription<'tcx>> JobOwner<'a, 'tcx, Q> {
191192

192193
let value = QueryValue::new(result.clone(), dep_node_index);
193194
{
194-
let mut lock = cache.borrow_mut();
195+
let mut lock = cache.get_shard_by_value(&key).lock();
195196
lock.active.remove(&key);
196197
lock.results.insert(key, value);
197198
}
@@ -215,7 +216,8 @@ impl<'a, 'tcx, Q: QueryDescription<'tcx>> Drop for JobOwner<'a, 'tcx, Q> {
215216
#[cold]
216217
fn drop(&mut self) {
217218
// Poison the query so jobs waiting on it panic
218-
self.cache.borrow_mut().active.insert(self.key.clone(), QueryResult::Poisoned);
219+
let shard = self.cache.get_shard_by_value(&self.key);
220+
shard.lock().active.insert(self.key.clone(), QueryResult::Poisoned);
219221
// Also signal the completion of the job, so waiters
220222
// will continue execution
221223
self.job.signal_complete();
@@ -683,7 +685,7 @@ macro_rules! define_queries_inner {
683685
use std::mem;
684686
#[cfg(parallel_compiler)]
685687
use ty::query::job::QueryResult;
686-
use rustc_data_structures::sync::Lock;
688+
use rustc_data_structures::sharded::Sharded;
687689
use crate::{
688690
rustc_data_structures::stable_hasher::HashStable,
689691
rustc_data_structures::stable_hasher::StableHasherResult,
@@ -715,18 +717,17 @@ macro_rules! define_queries_inner {
715717
pub fn collect_active_jobs(&self) -> Vec<Lrc<QueryJob<$tcx>>> {
716718
let mut jobs = Vec::new();
717719

718-
// We use try_lock here since we are only called from the
720+
// We use try_lock_shards here since we are only called from the
719721
// deadlock handler, and this shouldn't be locked.
720722
$(
721-
jobs.extend(
722-
self.$name.try_lock().unwrap().active.values().filter_map(|v|
723-
if let QueryResult::Started(ref job) = *v {
724-
Some(job.clone())
725-
} else {
726-
None
727-
}
728-
)
729-
);
723+
let shards = self.$name.try_lock_shards().unwrap();
724+
jobs.extend(shards.iter().flat_map(|shard| shard.active.values().filter_map(|v|
725+
if let QueryResult::Started(ref job) = *v {
726+
Some(job.clone())
727+
} else {
728+
None
729+
}
730+
)));
730731
)*
731732

732733
jobs
@@ -748,26 +749,27 @@ macro_rules! define_queries_inner {
748749

749750
fn stats<'tcx, Q: QueryConfig<'tcx>>(
750751
name: &'static str,
751-
map: &QueryCache<'tcx, Q>
752+
map: &Sharded<QueryCache<'tcx, Q>>,
752753
) -> QueryStats {
754+
let map = map.lock_shards();
753755
QueryStats {
754756
name,
755757
#[cfg(debug_assertions)]
756-
cache_hits: map.cache_hits,
758+
cache_hits: map.iter().map(|shard| shard.cache_hits).sum(),
757759
#[cfg(not(debug_assertions))]
758760
cache_hits: 0,
759761
key_size: mem::size_of::<Q::Key>(),
760762
key_type: unsafe { type_name::<Q::Key>() },
761763
value_size: mem::size_of::<Q::Value>(),
762764
value_type: unsafe { type_name::<Q::Value>() },
763-
entry_count: map.results.len(),
765+
entry_count: map.iter().map(|shard| shard.results.len()).sum(),
764766
}
765767
}
766768

767769
$(
768770
queries.push(stats::<queries::$name<'_>>(
769771
stringify!($name),
770-
&*self.$name.lock()
772+
&self.$name,
771773
));
772774
)*
773775

@@ -939,7 +941,7 @@ macro_rules! define_queries_inner {
939941
}
940942

941943
#[inline(always)]
942-
fn query_cache<'a>(tcx: TyCtxt<$tcx>) -> &'a Lock<QueryCache<$tcx, Self>> {
944+
fn query_cache<'a>(tcx: TyCtxt<$tcx>) -> &'a Sharded<QueryCache<$tcx, Self>> {
943945
&tcx.queries.$name
944946
}
945947

@@ -1066,7 +1068,7 @@ macro_rules! define_queries_struct {
10661068
providers: IndexVec<CrateNum, Providers<$tcx>>,
10671069
fallback_extern_providers: Box<Providers<$tcx>>,
10681070

1069-
$($(#[$attr])* $name: Lock<QueryCache<$tcx, queries::$name<$tcx>>>,)*
1071+
$($(#[$attr])* $name: Sharded<QueryCache<$tcx, queries::$name<$tcx>>>,)*
10701072
}
10711073
};
10721074
}

src/librustc_data_structures/interner.rs

Lines changed: 0 additions & 58 deletions
This file was deleted.

0 commit comments

Comments
 (0)