Skip to content

Commit 2ff0b3a

Browse files
SparrowLiiSparrowLii
committed
make sharded query caches only for multiple threads
1 parent b1b244d commit 2ff0b3a

File tree

8 files changed

+275
-49
lines changed

8 files changed

+275
-49
lines changed

‎compiler/rustc_interface/src/passes.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -731,6 +731,7 @@ pub fn create_global_ctxt<'tcx>(
731731
extern_providers,
732732
query_result_on_disk_cache,
733733
incremental,
734+
!rustc_data_structures::sync::is_dyn_thread_safe(),
734735
),
735736
)
736737
})

‎compiler/rustc_middle/src/query/plumbing.rs

Lines changed: 64 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ use crate::dep_graph::DepKind;
33
use crate::query::on_disk_cache::CacheEncoder;
44
use crate::query::on_disk_cache::EncodedDepNodeIndex;
55
use crate::query::on_disk_cache::OnDiskCache;
6+
#[cfg(parallel_compiler)]
7+
use crate::query::MtQueryCaches;
68
use crate::query::{
79
DynamicQueries, ExternProviders, Providers, QueryArenas, QueryCaches, QueryEngine, QueryStates,
810
};
@@ -20,6 +22,8 @@ pub(crate) use rustc_query_system::query::QueryJobId;
2022
use rustc_query_system::query::*;
2123
use rustc_query_system::HandleCycleError;
2224
use rustc_span::{ErrorGuaranteed, Span, DUMMY_SP};
25+
#[cfg(not(parallel_compiler))]
26+
use std::marker::PhantomData;
2327
use std::ops::Deref;
2428

2529
pub struct QueryKeyStringCache {
@@ -32,13 +36,17 @@ impl QueryKeyStringCache {
3236
}
3337
}
3438

35-
pub struct DynamicQuery<'tcx, C: QueryCache> {
39+
pub struct DynamicQuery<'tcx, C: QueryCache, C2: QueryCache<Key = C::Key, Value = C::Value>> {
3640
pub name: &'static str,
3741
pub eval_always: bool,
3842
pub dep_kind: DepKind,
3943
pub handle_cycle_error: HandleCycleError,
4044
pub query_state: FieldOffset<QueryStates<'tcx>, QueryState<C::Key, DepKind>>,
4145
pub query_cache: FieldOffset<QueryCaches<'tcx>, C>,
46+
#[cfg(not(parallel_compiler))]
47+
pub mt_query_cache: PhantomData<C2>,
48+
#[cfg(parallel_compiler)]
49+
pub mt_query_cache: FieldOffset<MtQueryCaches<'tcx>, C2>,
4250
pub cache_on_disk: fn(tcx: TyCtxt<'tcx>, key: &C::Key) -> bool,
4351
pub execute_query: fn(tcx: TyCtxt<'tcx>, k: C::Key) -> C::Value,
4452
pub compute: fn(tcx: TyCtxt<'tcx>, key: C::Key) -> C::Value,
@@ -73,6 +81,10 @@ pub struct QuerySystem<'tcx> {
7381
pub states: QueryStates<'tcx>,
7482
pub arenas: QueryArenas<'tcx>,
7583
pub caches: QueryCaches<'tcx>,
84+
#[cfg(parallel_compiler)]
85+
pub single_thread: bool,
86+
#[cfg(parallel_compiler)]
87+
pub mt_caches: MtQueryCaches<'tcx>,
7688
pub dynamic_queries: DynamicQueries<'tcx>,
7789

7890
/// This provides access to the incremental compilation on-disk cache for query results.
@@ -139,6 +151,36 @@ impl<'tcx> TyCtxt<'tcx> {
139151
}
140152
}
141153

154+
#[macro_export]
155+
#[cfg(not(parallel_compiler))]
156+
macro_rules! with_query_caches {
157+
($func: ident($($params: expr,)* :$tcx: expr, $name:ident, $($rest: expr,)*)) => {
158+
$func($($params,)* &$tcx.query_system.caches.$name, $($rest,)*)
159+
};
160+
($tcx: expr, $name:ident, $func: ident($($params: expr,)*)) => {
161+
$tcx.query_system.caches.$name.$func($($params,)*)
162+
}
163+
}
164+
165+
#[macro_export]
166+
#[cfg(parallel_compiler)]
167+
macro_rules! with_query_caches {
168+
($func: ident($($params: expr,)* :$tcx: expr, $name:ident, $($rest: expr,)*)) => {
169+
if $tcx.query_system.single_thread {
170+
$func($($params,)* &$tcx.query_system.caches.$name, $($rest,)*)
171+
} else {
172+
$func($($params,)* &$tcx.query_system.mt_caches.$name, $($rest,)*)
173+
}
174+
};
175+
($tcx: expr, $name:ident, $func: ident($($params: expr,)*)) => {
176+
if $tcx.query_system.single_thread {
177+
$tcx.query_system.caches.$name.$func($($params,)*)
178+
} else {
179+
$tcx.query_system.mt_caches.$name.$func($($params,)*)
180+
}
181+
}
182+
}
183+
142184
#[inline]
143185
pub fn query_get_at<'tcx, Cache>(
144186
tcx: TyCtxt<'tcx>,
@@ -287,6 +329,10 @@ macro_rules! define_callbacks {
287329
<$($K)* as keys::Key>::CacheSelector as CacheSelector<'tcx, Erase<$V>>
288330
>::Cache;
289331

332+
pub type MtStorage<'tcx> = <
333+
<$($K)* as keys::Key>::CacheSelector as CacheSelector<'tcx, Erase<$V>>
334+
>::MtCache;
335+
290336
// Ensure that keys grow no larger than 64 bytes
291337
#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
292338
const _: () = {
@@ -340,31 +386,36 @@ macro_rules! define_callbacks {
340386
$($(#[$attr])* pub $name: queries::$name::Storage<'tcx>,)*
341387
}
342388

389+
#[derive(Default)]
390+
pub struct MtQueryCaches<'tcx> {
391+
$($(#[$attr])* pub $name: queries::$name::MtStorage<'tcx>,)*
392+
}
393+
343394
impl<'tcx> TyCtxtEnsure<'tcx> {
344395
$($(#[$attr])*
345396
#[inline(always)]
346397
pub fn $name(self, key: query_helper_param_ty!($($K)*)) {
347-
query_ensure(
398+
with_query_caches!(query_ensure(
348399
self.tcx,
349400
self.tcx.query_system.fns.engine.$name,
350-
&self.tcx.query_system.caches.$name,
401+
:self.tcx, $name,
351402
key.into_query_param(),
352403
false,
353-
);
404+
));
354405
})*
355406
}
356407

357408
impl<'tcx> TyCtxtEnsureWithValue<'tcx> {
358409
$($(#[$attr])*
359410
#[inline(always)]
360411
pub fn $name(self, key: query_helper_param_ty!($($K)*)) {
361-
query_ensure(
412+
with_query_caches!(query_ensure(
362413
self.tcx,
363414
self.tcx.query_system.fns.engine.$name,
364-
&self.tcx.query_system.caches.$name,
415+
:self.tcx, $name,
365416
key.into_query_param(),
366417
true,
367-
);
418+
));
368419
})*
369420
}
370421

@@ -383,19 +434,19 @@ macro_rules! define_callbacks {
383434
#[inline(always)]
384435
pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> $V
385436
{
386-
restore::<$V>(query_get_at(
437+
restore::<$V>(with_query_caches!(query_get_at(
387438
self.tcx,
388439
self.tcx.query_system.fns.engine.$name,
389-
&self.tcx.query_system.caches.$name,
440+
:self.tcx, $name,
390441
self.span,
391442
key.into_query_param(),
392-
))
443+
)))
393444
})*
394445
}
395446

396447
pub struct DynamicQueries<'tcx> {
397448
$(
398-
pub $name: DynamicQuery<'tcx, queries::$name::Storage<'tcx>>,
449+
pub $name: DynamicQuery<'tcx, queries::$name::Storage<'tcx>, queries::$name::MtStorage<'tcx>>,
399450
)*
400451
}
401452

@@ -485,10 +536,9 @@ macro_rules! define_feedable {
485536
let tcx = self.tcx;
486537
let erased = queries::$name::provided_to_erased(tcx, value);
487538
let value = restore::<$V>(erased);
488-
let cache = &tcx.query_system.caches.$name;
489539

490540
let hasher: Option<fn(&mut StableHashingContext<'_>, &_) -> _> = hash_result!([$($modifiers)*]);
491-
match try_get_cached(tcx, cache, &key) {
541+
match with_query_caches!(try_get_cached(tcx, :tcx, $name, &key,)) {
492542
Some(old) => {
493543
let old = restore::<$V>(old);
494544
if let Some(hasher) = hasher {
@@ -524,7 +574,7 @@ macro_rules! define_feedable {
524574
&value,
525575
hash_result!([$($modifiers)*]),
526576
);
527-
cache.complete(key, erased, dep_node_index);
577+
with_query_caches!(tcx, $name, complete(key, erased, dep_node_index,));
528578
}
529579
}
530580
}

‎compiler/rustc_query_impl/src/lib.rs

Lines changed: 30 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ use rustc_middle::query::plumbing::{
2929
DynamicQuery, QueryKeyStringCache, QuerySystem, QuerySystemFns,
3030
};
3131
use rustc_middle::query::AsLocalKey;
32+
#[cfg(parallel_compiler)]
33+
use rustc_middle::query::MtQueryCaches;
3234
use rustc_middle::query::{
3335
queries, DynamicQueries, ExternProviders, Providers, QueryCaches, QueryEngine, QueryStates,
3436
};
@@ -53,33 +55,40 @@ pub use self::profiling_support::alloc_self_profile_query_strings;
5355
struct DynamicConfig<
5456
'tcx,
5557
C: QueryCache,
58+
C2: QueryCache<Key = C::Key, Value = C::Value>,
5659
const ANON: bool,
5760
const DEPTH_LIMIT: bool,
5861
const FEEDABLE: bool,
5962
> {
60-
dynamic: &'tcx DynamicQuery<'tcx, C>,
63+
dynamic: &'tcx DynamicQuery<'tcx, C, C2>,
6164
}
6265

63-
impl<'tcx, C: QueryCache, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool> Copy
64-
for DynamicConfig<'tcx, C, ANON, DEPTH_LIMIT, FEEDABLE>
66+
impl<'tcx, C: QueryCache, C2, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool> Copy
67+
for DynamicConfig<'tcx, C, C2, ANON, DEPTH_LIMIT, FEEDABLE>
68+
where
69+
C2: QueryCache<Key = C::Key, Value = C::Value>,
6570
{
6671
}
67-
impl<'tcx, C: QueryCache, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool> Clone
68-
for DynamicConfig<'tcx, C, ANON, DEPTH_LIMIT, FEEDABLE>
72+
impl<'tcx, C: QueryCache, C2, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool> Clone
73+
for DynamicConfig<'tcx, C, C2, ANON, DEPTH_LIMIT, FEEDABLE>
74+
where
75+
C2: QueryCache<Key = C::Key, Value = C::Value>,
6976
{
7077
fn clone(&self) -> Self {
7178
DynamicConfig { dynamic: self.dynamic }
7279
}
7380
}
7481

75-
impl<'tcx, C: QueryCache, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool>
76-
QueryConfig<QueryCtxt<'tcx>> for DynamicConfig<'tcx, C, ANON, DEPTH_LIMIT, FEEDABLE>
82+
impl<'tcx, C: QueryCache, C2, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool>
83+
QueryConfig<QueryCtxt<'tcx>> for DynamicConfig<'tcx, C, C2, ANON, DEPTH_LIMIT, FEEDABLE>
7784
where
85+
C2: QueryCache<Key = C::Key, Value = C::Value>,
7886
for<'a> C::Key: HashStable<StableHashingContext<'a>>,
7987
{
8088
type Key = C::Key;
8189
type Value = C::Value;
8290
type Cache = C;
91+
type MtCache = C2;
8392

8493
#[inline(always)]
8594
fn name(self) -> &'static str {
@@ -107,6 +116,15 @@ where
107116
self.dynamic.query_cache.apply(&qcx.tcx.query_system.caches)
108117
}
109118

119+
#[inline(always)]
120+
#[cfg(parallel_compiler)]
121+
fn mt_query_cache<'a>(self, qcx: QueryCtxt<'tcx>) -> &'a Self::MtCache
122+
where
123+
'tcx: 'a,
124+
{
125+
self.dynamic.mt_query_cache.apply(&qcx.tcx.query_system.mt_caches)
126+
}
127+
110128
#[inline(always)]
111129
fn execute_query(self, tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value {
112130
(self.dynamic.execute_query)(tcx, key)
@@ -208,11 +226,16 @@ pub fn query_system<'tcx>(
208226
extern_providers: ExternProviders,
209227
on_disk_cache: Option<OnDiskCache<'tcx>>,
210228
incremental: bool,
229+
_single_thread: bool,
211230
) -> QuerySystem<'tcx> {
212231
QuerySystem {
213232
states: Default::default(),
214233
arenas: Default::default(),
215234
caches: Default::default(),
235+
#[cfg(parallel_compiler)]
236+
single_thread: _single_thread,
237+
#[cfg(parallel_compiler)]
238+
mt_caches: Default::default(),
216239
dynamic_queries: dynamic_queries(),
217240
on_disk_cache,
218241
fns: QuerySystemFns {

‎compiler/rustc_query_impl/src/plumbing.rs

Lines changed: 33 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ use rustc_query_system::query::{
2323
force_query, QueryCache, QueryConfig, QueryContext, QueryJobId, QueryMap, QuerySideEffects,
2424
QueryStackFrame,
2525
};
26-
use rustc_query_system::{LayoutOfDepth, QueryOverflow};
26+
use rustc_query_system::{with_qcx_query_caches, LayoutOfDepth, QueryOverflow};
2727
use rustc_serialize::Decodable;
2828
use rustc_serialize::Encodable;
2929
use rustc_session::Limit;
@@ -73,6 +73,12 @@ impl QueryContext for QueryCtxt<'_> {
7373
)
7474
}
7575

76+
#[inline]
77+
#[cfg(parallel_compiler)]
78+
fn single_thread(self) -> bool {
79+
self.query_system.single_thread
80+
}
81+
7682
#[inline]
7783
fn current_query_job(self) -> Option<QueryJobId> {
7884
tls::with_related_context(self.tcx, |icx| icx.query)
@@ -352,19 +358,22 @@ pub(crate) fn encode_query_results<'a, 'tcx, Q>(
352358
qcx.profiler().verbose_generic_activity_with_arg("encode_query_results_for", query.name());
353359

354360
assert!(query.query_state(qcx).all_inactive());
355-
let cache = query.query_cache(qcx);
356-
cache.iter(&mut |key, value, dep_node| {
357-
if query.cache_on_disk(qcx.tcx, &key) {
358-
let dep_node = SerializedDepNodeIndex::new(dep_node.index());
359-
360-
// Record position of the cache entry.
361-
query_result_index.push((dep_node, AbsoluteBytePos::new(encoder.position())));
362-
363-
// Encode the type check tables with the `SerializedDepNodeIndex`
364-
// as tag.
365-
encoder.encode_tagged(dep_node, &Q::restore(*value));
366-
}
367-
});
361+
with_qcx_query_caches!(
362+
query,
363+
qcx,
364+
iter(&mut |key, value, dep_node| {
365+
if query.cache_on_disk(qcx.tcx, &key) {
366+
let dep_node = SerializedDepNodeIndex::new(dep_node.index());
367+
368+
// Record position of the cache entry.
369+
query_result_index.push((dep_node, AbsoluteBytePos::new(encoder.position())));
370+
371+
// Encode the type check tables with the `SerializedDepNodeIndex`
372+
// as tag.
373+
encoder.encode_tagged(dep_node, &Q::restore(*value));
374+
}
375+
})
376+
);
368377
}
369378

370379
fn try_load_from_on_disk_cache<'tcx, Q>(query: Q, tcx: TyCtxt<'tcx>, dep_node: DepNode)
@@ -562,14 +571,18 @@ macro_rules! define_queries {
562571
}
563572
}
564573

565-
pub fn dynamic_query<'tcx>() -> DynamicQuery<'tcx, queries::$name::Storage<'tcx>> {
574+
pub fn dynamic_query<'tcx>() -> DynamicQuery<'tcx, queries::$name::Storage<'tcx>, queries::$name::MtStorage<'tcx>> {
566575
DynamicQuery {
567576
name: stringify!($name),
568577
eval_always: is_eval_always!([$($modifiers)*]),
569578
dep_kind: dep_graph::DepKind::$name,
570579
handle_cycle_error: handle_cycle_error!([$($modifiers)*]),
571580
query_state: offset_of!(QueryStates<'tcx> => $name),
572581
query_cache: offset_of!(QueryCaches<'tcx> => $name),
582+
#[cfg(not(parallel_compiler))]
583+
mt_query_cache: PhantomData,
584+
#[cfg(parallel_compiler)]
585+
mt_query_cache: offset_of!(MtQueryCaches<'tcx> => $name),
573586
cache_on_disk: |tcx, key| ::rustc_middle::query::cached::$name(tcx, key),
574587
execute_query: |tcx, key| erase(tcx.$name(key)),
575588
compute: |tcx, key| {
@@ -632,6 +645,7 @@ macro_rules! define_queries {
632645
type Config = DynamicConfig<
633646
'tcx,
634647
queries::$name::Storage<'tcx>,
648+
queries::$name::MtStorage<'tcx>,
635649
{ is_anon!([$($modifiers)*]) },
636650
{ depth_limit!([$($modifiers)*]) },
637651
{ feedable!([$($modifiers)*]) },
@@ -664,12 +678,13 @@ macro_rules! define_queries {
664678
}
665679

666680
pub fn alloc_self_profile_query_strings<'tcx>(tcx: TyCtxt<'tcx>, string_cache: &mut QueryKeyStringCache) {
667-
$crate::profiling_support::alloc_self_profile_query_strings_for_query_cache(
681+
use $crate::profiling_support::alloc_self_profile_query_strings_for_query_cache;
682+
with_query_caches!(alloc_self_profile_query_strings_for_query_cache(
668683
tcx,
669684
stringify!($name),
670-
&tcx.query_system.caches.$name,
685+
:tcx, $name,
671686
string_cache,
672-
)
687+
))
673688
}
674689

675690
item_if_cached! { [$($modifiers)*] {

0 commit comments

Comments
 (0)