Skip to content

Commit 6c5cf74

Browse files
committed
Auto merge of #120003 - Mark-Simulacrum:opt-promoted, r=<try>
perf: Don't track specific live points for promoteds We don't query this information out of the promoted (it's basically a single "unit" regardless of the complexity within it) and this saves on re-initializing the SparseIntervalMatrix's backing IndexVec with mostly empty rows for all of the leading regions in the function. Typical promoteds will only contain a few regions that need up be uplifted, while the parent function can have thousands. For a simple function repeating println!("Hello world"); 50,000 times this reduces compile times from 90 to 15 seconds in debug mode. The previous implementations re-initialization led to an overall roughly n^2 runtime as each promoted initialized slots for ~n regions, now we scale closer to linearly (5000 hello worlds takes 1.1 seconds). cc #50994
2 parents 67e7b84 + 5d68fa6 commit 6c5cf74

File tree

2 files changed

+63
-16
lines changed

2 files changed

+63
-16
lines changed

compiler/rustc_borrowck/src/region_infer/values.rs

+55-13
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ use rustc_index::Idx;
88
use rustc_index::IndexVec;
99
use rustc_middle::mir::{BasicBlock, Body, Location};
1010
use rustc_middle::ty::{self, RegionVid};
11+
use std::collections::BTreeSet;
1112
use std::fmt::Debug;
1213
use std::rc::Rc;
1314

@@ -125,8 +126,13 @@ pub(crate) struct LivenessValues {
125126
/// The map from locations to points.
126127
elements: Rc<RegionValueElements>,
127128

129+
live_regions: BTreeSet<RegionVid>,
130+
128131
/// For each region: the points where it is live.
129-
points: SparseIntervalMatrix<RegionVid, PointIndex>,
132+
///
133+
/// This is not initialized for promoteds, because we don't care *where* within a promoted a
134+
/// region is live, only that it is.
135+
points: Option<SparseIntervalMatrix<RegionVid, PointIndex>>,
130136

131137
/// When using `-Zpolonius=next`, for each point: the loans flowing into the live regions at
132138
/// that point.
@@ -155,24 +161,38 @@ impl LiveLoans {
155161

156162
impl LivenessValues {
157163
/// Create an empty map of regions to locations where they're live.
158-
pub(crate) fn new(elements: Rc<RegionValueElements>) -> Self {
164+
pub(crate) fn with_specific_points(elements: Rc<RegionValueElements>) -> Self {
159165
LivenessValues {
160-
points: SparseIntervalMatrix::new(elements.num_points),
166+
live_regions: BTreeSet::new(),
167+
points: Some(SparseIntervalMatrix::new(elements.num_points)),
161168
elements,
162169
loans: None,
163170
}
164171
}
165172

173+
/// Create an empty map of regions to locations where they're live.
174+
///
175+
/// Unlike `with_specific_points`, does not track exact locations where something is live, only
176+
/// which regions are live.
177+
pub(crate) fn without_specific_points(elements: Rc<RegionValueElements>) -> Self {
178+
LivenessValues { live_regions: BTreeSet::new(), points: None, elements, loans: None }
179+
}
180+
166181
/// Iterate through each region that has a value in this set.
167-
pub(crate) fn regions(&self) -> impl Iterator<Item = RegionVid> {
168-
self.points.rows()
182+
pub(crate) fn regions(&self) -> impl Iterator<Item = RegionVid> + '_ {
183+
self.live_regions.iter().copied()
169184
}
170185

171186
/// Records `region` as being live at the given `location`.
172187
pub(crate) fn add_location(&mut self, region: RegionVid, location: Location) {
173-
debug!("LivenessValues::add_location(region={:?}, location={:?})", region, location);
174188
let point = self.elements.point_from_location(location);
175-
self.points.insert(region, point);
189+
debug!("LivenessValues::add_location(region={:?}, location={:?})", region, location);
190+
if self.elements.point_in_range(point) {
191+
self.live_regions.insert(region);
192+
}
193+
if let Some(points) = &mut self.points {
194+
points.insert(region, point);
195+
}
176196

177197
// When available, record the loans flowing into this region as live at the given point.
178198
if let Some(loans) = self.loans.as_mut() {
@@ -185,7 +205,12 @@ impl LivenessValues {
185205
/// Records `region` as being live at all the given `points`.
186206
pub(crate) fn add_points(&mut self, region: RegionVid, points: &IntervalSet<PointIndex>) {
187207
debug!("LivenessValues::add_points(region={:?}, points={:?})", region, points);
188-
self.points.union_row(region, points);
208+
if points.iter().any(|point| self.elements.point_in_range(point)) {
209+
self.live_regions.insert(region);
210+
}
211+
if let Some(this) = &mut self.points {
212+
this.union_row(region, points);
213+
}
189214

190215
// When available, record the loans flowing into this region as live at the given points.
191216
if let Some(loans) = self.loans.as_mut() {
@@ -201,23 +226,37 @@ impl LivenessValues {
201226

202227
/// Records `region` as being live at all the control-flow points.
203228
pub(crate) fn add_all_points(&mut self, region: RegionVid) {
204-
self.points.insert_all_into_row(region);
229+
if let Some(points) = &mut self.points {
230+
points.insert_all_into_row(region);
231+
}
232+
self.live_regions.insert(region);
205233
}
206234

207235
/// Returns whether `region` is marked live at the given `location`.
208236
pub(crate) fn is_live_at(&self, region: RegionVid, location: Location) -> bool {
209237
let point = self.elements.point_from_location(location);
210-
self.points.row(region).is_some_and(|r| r.contains(point))
238+
if let Some(points) = &self.points {
239+
points.row(region).is_some_and(|r| r.contains(point))
240+
} else {
241+
unreachable!(
242+
"Should be using LivenessValues::with_specific_points to ask whether live at a location"
243+
)
244+
}
211245
}
212246

213247
/// Returns whether `region` is marked live at any location.
214248
pub(crate) fn is_live_anywhere(&self, region: RegionVid) -> bool {
215-
self.live_points(region).next().is_some()
249+
self.live_regions.contains(&region)
216250
}
217251

218252
/// Returns an iterator of all the points where `region` is live.
219253
fn live_points(&self, region: RegionVid) -> impl Iterator<Item = PointIndex> + '_ {
220-
self.points
254+
let Some(points) = &self.points else {
255+
unreachable!(
256+
"Should be using LivenessValues::with_specific_points to ask whether live at a location"
257+
)
258+
};
259+
points
221260
.row(region)
222261
.into_iter()
223262
.flat_map(|set| set.iter())
@@ -372,7 +411,10 @@ impl<N: Idx> RegionValues<N> {
372411
/// elements for the region `from` from `values` and add them to
373412
/// the region `to` in `self`.
374413
pub(crate) fn merge_liveness(&mut self, to: N, from: RegionVid, values: &LivenessValues) {
375-
if let Some(set) = values.points.row(from) {
414+
let Some(value_points) = &values.points else {
415+
panic!("LivenessValues must track specific points for use in merge_liveness");
416+
};
417+
if let Some(set) = value_points.row(from) {
376418
self.points.union_row(to, set);
377419
}
378420
}

compiler/rustc_borrowck/src/type_check/mod.rs

+8-3
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,7 @@ pub(crate) fn type_check<'mir, 'tcx>(
141141
let mut constraints = MirTypeckRegionConstraints {
142142
placeholder_indices: PlaceholderIndices::default(),
143143
placeholder_index_to_region: IndexVec::default(),
144-
liveness_constraints: LivenessValues::new(elements.clone()),
144+
liveness_constraints: LivenessValues::with_specific_points(elements.clone()),
145145
outlives_constraints: OutlivesConstraintSet::default(),
146146
member_constraints: MemberConstraintSet::default(),
147147
type_tests: Vec::default(),
@@ -544,8 +544,13 @@ impl<'a, 'b, 'tcx> TypeVerifier<'a, 'b, 'tcx> {
544544
// modify their locations.
545545
let all_facts = &mut None;
546546
let mut constraints = Default::default();
547-
let mut liveness_constraints =
548-
LivenessValues::new(Rc::new(RegionValueElements::new(promoted_body)));
547+
// We care which regions are used in the promoted, but not where in the promoted those
548+
// regions are used. So we skip tracking the specific points - that can be expensive, since
549+
// each promoted has its own LivenessValues but the regions are indexed globally within the
550+
// body. As a result we'd spend lots of time re-initializing the values.
551+
let mut liveness_constraints = LivenessValues::without_specific_points(Rc::new(
552+
RegionValueElements::new(promoted_body),
553+
));
549554
// Don't try to add borrow_region facts for the promoted MIR
550555

551556
let mut swap_constraints = |this: &mut Self| {

0 commit comments

Comments
 (0)