Skip to content

make query caches use Sharded only for multiple threads #115162

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

make query caches use Sharded only for multiple threads #115162

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
16b9941
make sharded query caches only for multiple threads
SparrowLii Aug 24, 2023
cd6850c
use `RefCell` for query caches in single thread
SparrowLii Aug 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions compiler/rustc_interface/src/passes.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -731,6 +731,7 @@ pub fn create_global_ctxt<'tcx>(
extern_providers,
query_result_on_disk_cache,
incremental,
!rustc_data_structures::sync::is_dyn_thread_safe(),
),
)
})
Expand Down
78 changes: 64 additions & 14 deletions compiler/rustc_middle/src/query/plumbing.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ use crate::dep_graph::DepKind;
use crate::query::on_disk_cache::CacheEncoder;
use crate::query::on_disk_cache::EncodedDepNodeIndex;
use crate::query::on_disk_cache::OnDiskCache;
#[cfg(parallel_compiler)]
use crate::query::MtQueryCaches;
use crate::query::{
DynamicQueries, ExternProviders, Providers, QueryArenas, QueryCaches, QueryEngine, QueryStates,
};
Expand All @@ -20,6 +22,8 @@ pub(crate) use rustc_query_system::query::QueryJobId;
use rustc_query_system::query::*;
use rustc_query_system::HandleCycleError;
use rustc_span::{Span, DUMMY_SP};
#[cfg(not(parallel_compiler))]
use std::marker::PhantomData;
use std::ops::Deref;

pub struct QueryKeyStringCache {
Expand All @@ -32,13 +36,17 @@ impl QueryKeyStringCache {
}
}

pub struct DynamicQuery<'tcx, C: QueryCache> {
pub struct DynamicQuery<'tcx, C: QueryCache, C2: QueryCache<Key = C::Key, Value = C::Value>> {
pub name: &'static str,
pub eval_always: bool,
pub dep_kind: DepKind,
pub handle_cycle_error: HandleCycleError,
pub query_state: FieldOffset<QueryStates<'tcx>, QueryState<C::Key, DepKind>>,
pub query_cache: FieldOffset<QueryCaches<'tcx>, C>,
#[cfg(not(parallel_compiler))]
pub mt_query_cache: PhantomData<C2>,
#[cfg(parallel_compiler)]
pub mt_query_cache: FieldOffset<MtQueryCaches<'tcx>, C2>,
pub cache_on_disk: fn(tcx: TyCtxt<'tcx>, key: &C::Key) -> bool,
pub execute_query: fn(tcx: TyCtxt<'tcx>, k: C::Key) -> C::Value,
pub compute: fn(tcx: TyCtxt<'tcx>, key: C::Key) -> C::Value,
Expand Down Expand Up @@ -72,6 +80,10 @@ pub struct QuerySystem<'tcx> {
pub states: QueryStates<'tcx>,
pub arenas: QueryArenas<'tcx>,
pub caches: QueryCaches<'tcx>,
#[cfg(parallel_compiler)]
pub single_thread: bool,
#[cfg(parallel_compiler)]
pub mt_caches: MtQueryCaches<'tcx>,
pub dynamic_queries: DynamicQueries<'tcx>,

/// This provides access to the incremental compilation on-disk cache for query results.
Expand Down Expand Up @@ -138,6 +150,36 @@ impl<'tcx> TyCtxt<'tcx> {
}
}

#[macro_export]
#[cfg(not(parallel_compiler))]
macro_rules! with_query_caches {
($func: ident($($params: expr,)* :$tcx: expr, $name:ident, $($rest: expr,)*)) => {
$func($($params,)* &$tcx.query_system.caches.$name, $($rest,)*)
};
($tcx: expr, $name:ident, $func: ident($($params: expr,)*)) => {
$tcx.query_system.caches.$name.$func($($params,)*)
}
}

#[macro_export]
#[cfg(parallel_compiler)]
macro_rules! with_query_caches {
($func: ident($($params: expr,)* :$tcx: expr, $name:ident, $($rest: expr,)*)) => {
if $tcx.query_system.single_thread {
$func($($params,)* &$tcx.query_system.caches.$name, $($rest,)*)
} else {
$func($($params,)* &$tcx.query_system.mt_caches.$name, $($rest,)*)
}
};
($tcx: expr, $name:ident, $func: ident($($params: expr,)*)) => {
if $tcx.query_system.single_thread {
$tcx.query_system.caches.$name.$func($($params,)*)
} else {
$tcx.query_system.mt_caches.$name.$func($($params,)*)
}
}
}

#[inline]
pub fn query_get_at<'tcx, Cache>(
tcx: TyCtxt<'tcx>,
Expand Down Expand Up @@ -286,6 +328,10 @@ macro_rules! define_callbacks {
<$($K)* as keys::Key>::CacheSelector as CacheSelector<'tcx, Erase<$V>>
>::Cache;

pub type MtStorage<'tcx> = <
<$($K)* as keys::Key>::CacheSelector as CacheSelector<'tcx, Erase<$V>>
>::MtCache;

// Ensure that keys grow no larger than 64 bytes
#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
const _: () = {
Expand Down Expand Up @@ -339,31 +385,36 @@ macro_rules! define_callbacks {
$($(#[$attr])* pub $name: queries::$name::Storage<'tcx>,)*
}

#[derive(Default)]
pub struct MtQueryCaches<'tcx> {
$($(#[$attr])* pub $name: queries::$name::MtStorage<'tcx>,)*
}

impl<'tcx> TyCtxtEnsure<'tcx> {
$($(#[$attr])*
#[inline(always)]
pub fn $name(self, key: query_helper_param_ty!($($K)*)) {
query_ensure(
with_query_caches!(query_ensure(
self.tcx,
self.tcx.query_system.fns.engine.$name,
&self.tcx.query_system.caches.$name,
:self.tcx, $name,
key.into_query_param(),
false,
);
));
})*
}

impl<'tcx> TyCtxtEnsureWithValue<'tcx> {
$($(#[$attr])*
#[inline(always)]
pub fn $name(self, key: query_helper_param_ty!($($K)*)) {
query_ensure(
with_query_caches!(query_ensure(
self.tcx,
self.tcx.query_system.fns.engine.$name,
&self.tcx.query_system.caches.$name,
:self.tcx, $name,
key.into_query_param(),
true,
);
));
})*
}

Expand All @@ -382,19 +433,19 @@ macro_rules! define_callbacks {
#[inline(always)]
pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> $V
{
restore::<$V>(query_get_at(
restore::<$V>(with_query_caches!(query_get_at(
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This change is probably a bit costly as is doubles the size of the query accessor.

Copy link
Member Author

@SparrowLii SparrowLii Aug 25, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yea, I added likely hint here, maybe helps a lillte

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That doesn't really affect the code size.

self.tcx,
self.tcx.query_system.fns.engine.$name,
&self.tcx.query_system.caches.$name,
:self.tcx, $name,
self.span,
key.into_query_param(),
))
)))
})*
}

pub struct DynamicQueries<'tcx> {
$(
pub $name: DynamicQuery<'tcx, queries::$name::Storage<'tcx>>,
pub $name: DynamicQuery<'tcx, queries::$name::Storage<'tcx>, queries::$name::MtStorage<'tcx>>,
)*
}

Expand Down Expand Up @@ -484,10 +535,9 @@ macro_rules! define_feedable {
let tcx = self.tcx;
let erased = queries::$name::provided_to_erased(tcx, value);
let value = restore::<$V>(erased);
let cache = &tcx.query_system.caches.$name;

let hasher: Option<fn(&mut StableHashingContext<'_>, &_) -> _> = hash_result!([$($modifiers)*]);
match try_get_cached(tcx, cache, &key) {
match with_query_caches!(try_get_cached(tcx, :tcx, $name, &key,)) {
Some(old) => {
let old = restore::<$V>(old);
if let Some(hasher) = hasher {
Expand Down Expand Up @@ -523,7 +573,7 @@ macro_rules! define_feedable {
&value,
hash_result!([$($modifiers)*]),
);
cache.complete(key, erased, dep_node_index);
with_query_caches!(tcx, $name, complete(key, erased, dep_node_index,));
}
}
}
Expand Down
37 changes: 30 additions & 7 deletions compiler/rustc_query_impl/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ use rustc_middle::query::plumbing::{
DynamicQuery, QueryKeyStringCache, QuerySystem, QuerySystemFns,
};
use rustc_middle::query::AsLocalKey;
#[cfg(parallel_compiler)]
use rustc_middle::query::MtQueryCaches;
use rustc_middle::query::{
queries, DynamicQueries, ExternProviders, Providers, QueryCaches, QueryEngine, QueryStates,
};
Expand All @@ -53,33 +55,40 @@ pub use self::profiling_support::alloc_self_profile_query_strings;
struct DynamicConfig<
'tcx,
C: QueryCache,
C2: QueryCache<Key = C::Key, Value = C::Value>,
const ANON: bool,
const DEPTH_LIMIT: bool,
const FEEDABLE: bool,
> {
dynamic: &'tcx DynamicQuery<'tcx, C>,
dynamic: &'tcx DynamicQuery<'tcx, C, C2>,
}

impl<'tcx, C: QueryCache, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool> Copy
for DynamicConfig<'tcx, C, ANON, DEPTH_LIMIT, FEEDABLE>
impl<'tcx, C: QueryCache, C2, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool> Copy
for DynamicConfig<'tcx, C, C2, ANON, DEPTH_LIMIT, FEEDABLE>
where
C2: QueryCache<Key = C::Key, Value = C::Value>,
{
}
impl<'tcx, C: QueryCache, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool> Clone
for DynamicConfig<'tcx, C, ANON, DEPTH_LIMIT, FEEDABLE>
impl<'tcx, C: QueryCache, C2, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool> Clone
for DynamicConfig<'tcx, C, C2, ANON, DEPTH_LIMIT, FEEDABLE>
where
C2: QueryCache<Key = C::Key, Value = C::Value>,
{
fn clone(&self) -> Self {
DynamicConfig { dynamic: self.dynamic }
}
}

impl<'tcx, C: QueryCache, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool>
QueryConfig<QueryCtxt<'tcx>> for DynamicConfig<'tcx, C, ANON, DEPTH_LIMIT, FEEDABLE>
impl<'tcx, C: QueryCache, C2, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool>
QueryConfig<QueryCtxt<'tcx>> for DynamicConfig<'tcx, C, C2, ANON, DEPTH_LIMIT, FEEDABLE>
where
C2: QueryCache<Key = C::Key, Value = C::Value>,
for<'a> C::Key: HashStable<StableHashingContext<'a>>,
{
type Key = C::Key;
type Value = C::Value;
type Cache = C;
type MtCache = C2;

#[inline(always)]
fn name(self) -> &'static str {
Expand Down Expand Up @@ -107,6 +116,15 @@ where
self.dynamic.query_cache.apply(&qcx.tcx.query_system.caches)
}

#[inline(always)]
#[cfg(parallel_compiler)]
fn mt_query_cache<'a>(self, qcx: QueryCtxt<'tcx>) -> &'a Self::MtCache
where
'tcx: 'a,
{
self.dynamic.mt_query_cache.apply(&qcx.tcx.query_system.mt_caches)
}

#[inline(always)]
fn execute_query(self, tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value {
(self.dynamic.execute_query)(tcx, key)
Expand Down Expand Up @@ -207,11 +225,16 @@ pub fn query_system<'tcx>(
extern_providers: ExternProviders,
on_disk_cache: Option<OnDiskCache<'tcx>>,
incremental: bool,
_single_thread: bool,
) -> QuerySystem<'tcx> {
QuerySystem {
states: Default::default(),
arenas: Default::default(),
caches: Default::default(),
#[cfg(parallel_compiler)]
single_thread: _single_thread,
#[cfg(parallel_compiler)]
mt_caches: Default::default(),
dynamic_queries: dynamic_queries(),
on_disk_cache,
fns: QuerySystemFns {
Expand Down
51 changes: 33 additions & 18 deletions compiler/rustc_query_impl/src/plumbing.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ use rustc_query_system::query::{
force_query, QueryCache, QueryConfig, QueryContext, QueryJobId, QueryMap, QuerySideEffects,
QueryStackFrame,
};
use rustc_query_system::{LayoutOfDepth, QueryOverflow};
use rustc_query_system::{with_qcx_query_caches, LayoutOfDepth, QueryOverflow};
use rustc_serialize::Decodable;
use rustc_serialize::Encodable;
use rustc_session::Limit;
Expand Down Expand Up @@ -73,6 +73,12 @@ impl QueryContext for QueryCtxt<'_> {
)
}

#[inline]
#[cfg(parallel_compiler)]
fn single_thread(self) -> bool {
self.query_system.single_thread
}

#[inline]
fn current_query_job(self) -> Option<QueryJobId> {
tls::with_related_context(self.tcx, |icx| icx.query)
Expand Down Expand Up @@ -352,19 +358,22 @@ pub(crate) fn encode_query_results<'a, 'tcx, Q>(
qcx.profiler().verbose_generic_activity_with_arg("encode_query_results_for", query.name());

assert!(query.query_state(qcx).all_inactive());
let cache = query.query_cache(qcx);
cache.iter(&mut |key, value, dep_node| {
if query.cache_on_disk(qcx.tcx, &key) {
let dep_node = SerializedDepNodeIndex::new(dep_node.index());

// Record position of the cache entry.
query_result_index.push((dep_node, AbsoluteBytePos::new(encoder.position())));

// Encode the type check tables with the `SerializedDepNodeIndex`
// as tag.
encoder.encode_tagged(dep_node, &Q::restore(*value));
}
});
with_qcx_query_caches!(
query,
qcx,
iter(&mut |key, value, dep_node| {
if query.cache_on_disk(qcx.tcx, &key) {
let dep_node = SerializedDepNodeIndex::new(dep_node.index());

// Record position of the cache entry.
query_result_index.push((dep_node, AbsoluteBytePos::new(encoder.position())));

// Encode the type check tables with the `SerializedDepNodeIndex`
// as tag.
encoder.encode_tagged(dep_node, &Q::restore(*value));
}
})
);
}

fn try_load_from_on_disk_cache<'tcx, Q>(query: Q, tcx: TyCtxt<'tcx>, dep_node: DepNode)
Expand Down Expand Up @@ -562,14 +571,18 @@ macro_rules! define_queries {
}
}

pub fn dynamic_query<'tcx>() -> DynamicQuery<'tcx, queries::$name::Storage<'tcx>> {
pub fn dynamic_query<'tcx>() -> DynamicQuery<'tcx, queries::$name::Storage<'tcx>, queries::$name::MtStorage<'tcx>> {
DynamicQuery {
name: stringify!($name),
eval_always: is_eval_always!([$($modifiers)*]),
dep_kind: dep_graph::DepKind::$name,
handle_cycle_error: handle_cycle_error!([$($modifiers)*]),
query_state: offset_of!(QueryStates<'tcx> => $name),
query_cache: offset_of!(QueryCaches<'tcx> => $name),
#[cfg(not(parallel_compiler))]
mt_query_cache: PhantomData,
#[cfg(parallel_compiler)]
mt_query_cache: offset_of!(MtQueryCaches<'tcx> => $name),
cache_on_disk: |tcx, key| ::rustc_middle::query::cached::$name(tcx, key),
execute_query: |tcx, key| erase(tcx.$name(key)),
compute: |tcx, key| {
Expand Down Expand Up @@ -632,6 +645,7 @@ macro_rules! define_queries {
type Config = DynamicConfig<
'tcx,
queries::$name::Storage<'tcx>,
queries::$name::MtStorage<'tcx>,
{ is_anon!([$($modifiers)*]) },
{ depth_limit!([$($modifiers)*]) },
{ feedable!([$($modifiers)*]) },
Expand Down Expand Up @@ -664,12 +678,13 @@ macro_rules! define_queries {
}

pub fn alloc_self_profile_query_strings<'tcx>(tcx: TyCtxt<'tcx>, string_cache: &mut QueryKeyStringCache) {
$crate::profiling_support::alloc_self_profile_query_strings_for_query_cache(
use $crate::profiling_support::alloc_self_profile_query_strings_for_query_cache;
with_query_caches!(alloc_self_profile_query_strings_for_query_cache(
tcx,
stringify!($name),
&tcx.query_system.caches.$name,
:tcx, $name,
string_cache,
)
))
}

item_if_cached! { [$($modifiers)*] {
Expand Down
Loading