Skip to content

Commit d6fe24c

Browse files
jonas-schievinkjonas-schievink
committed
Record intra-statement/terminator conflicts
Some MIR statements and terminators have an (undocumented...) invariant that some of their input and outputs must not overlap. This records conflicts between locals used in these positions.
1 parent 8575026 commit d6fe24c

File tree

1 file changed

+206
-22
lines changed

1 file changed

+206
-22
lines changed

‎src/librustc_mir/transform/dest_prop.rs

Lines changed: 206 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -109,8 +109,8 @@ use rustc_index::{
109109
use rustc_middle::mir::tcx::PlaceTy;
110110
use rustc_middle::mir::visit::{MutVisitor, PlaceContext, Visitor};
111111
use rustc_middle::mir::{
112-
traversal, Body, Local, LocalKind, Location, Operand, Place, PlaceElem, Rvalue, Statement,
113-
StatementKind, Terminator, TerminatorKind,
112+
traversal, Body, InlineAsmOperand, Local, LocalKind, Location, Operand, Place, PlaceElem,
113+
Rvalue, Statement, StatementKind, Terminator, TerminatorKind,
114114
};
115115
use rustc_middle::ty::{self, Ty, TyCtxt};
116116

@@ -397,7 +397,9 @@ impl<'tcx> MutVisitor<'tcx> for Replacer<'tcx> {
397397
}
398398
}
399399

400-
struct Conflicts {
400+
struct Conflicts<'a> {
401+
relevant_locals: &'a BitSet<Local>,
402+
401403
/// The conflict matrix. It is always symmetric and the adjacency matrix of the corresponding
402404
/// conflict graph.
403405
matrix: BitMatrix<Local, Local>,
@@ -406,30 +408,21 @@ struct Conflicts {
406408
unify_cache: BitSet<Local>,
407409
}
408410

409-
impl Conflicts {
411+
impl Conflicts<'a> {
410412
fn build<'tcx>(
411413
tcx: TyCtxt<'tcx>,
412414
body: &'_ Body<'tcx>,
413415
source: MirSource<'tcx>,
414-
relevant_locals: &BitSet<Local>,
416+
relevant_locals: &'a BitSet<Local>,
415417
) -> Self {
416418
// We don't have to look out for locals that have their address taken, since
417419
// `find_candidates` already takes care of that.
418420

419-
let mut conflicts = BitMatrix::from_row_n(
421+
let conflicts = BitMatrix::from_row_n(
420422
&BitSet::new_empty(body.local_decls.len()),
421423
body.local_decls.len(),
422424
);
423425

424-
let mut record_conflicts = |new_conflicts: &mut BitSet<_>| {
425-
// Remove all locals that are not candidates.
426-
new_conflicts.intersect(relevant_locals);
427-
428-
for local in new_conflicts.iter() {
429-
conflicts.union_row_with(&new_conflicts, local);
430-
}
431-
};
432-
433426
let def_id = source.def_id();
434427
let mut init = MaybeInitializedLocals
435428
.into_engine(tcx, body, def_id)
@@ -480,6 +473,12 @@ impl Conflicts {
480473
},
481474
);
482475

476+
let mut this = Self {
477+
relevant_locals,
478+
matrix: conflicts,
479+
unify_cache: BitSet::new_empty(body.local_decls.len()),
480+
};
481+
483482
let mut live_and_init_locals = Vec::new();
484483

485484
// Visit only reachable basic blocks. The exact order is not important.
@@ -497,14 +496,22 @@ impl Conflicts {
497496
BitSet::new_empty(body.local_decls.len())
498497
});
499498

500-
// First, go forwards for `MaybeInitializedLocals`.
501-
for statement_index in 0..=data.statements.len() {
502-
let loc = Location { block, statement_index };
499+
// First, go forwards for `MaybeInitializedLocals` and apply intra-statement/terminator
500+
// conflicts.
501+
for (i, statement) in data.statements.iter().enumerate() {
502+
this.record_statement_conflicts(statement);
503+
504+
let loc = Location { block, statement_index: i };
503505
init.seek_before_primary_effect(loc);
504506

505-
live_and_init_locals[statement_index].overwrite(init.get());
507+
live_and_init_locals[i].overwrite(init.get());
506508
}
507509

510+
this.record_terminator_conflicts(data.terminator());
511+
let term_loc = Location { block, statement_index: data.statements.len() };
512+
init.seek_before_primary_effect(term_loc);
513+
live_and_init_locals[term_loc.statement_index].overwrite(init.get());
514+
508515
// Now, go backwards and union with the liveness results.
509516
for statement_index in (0..=data.statements.len()).rev() {
510517
let loc = Location { block, statement_index };
@@ -514,7 +521,7 @@ impl Conflicts {
514521

515522
trace!("record conflicts at {:?}", loc);
516523

517-
record_conflicts(&mut live_and_init_locals[statement_index]);
524+
this.record_conflicts(&mut live_and_init_locals[statement_index]);
518525
}
519526

520527
init.seek_to_block_end(block);
@@ -523,10 +530,187 @@ impl Conflicts {
523530
conflicts.intersect(live.get());
524531
trace!("record conflicts at end of {:?}", block);
525532

526-
record_conflicts(&mut conflicts);
533+
this.record_conflicts(&mut conflicts);
534+
}
535+
536+
this
537+
}
538+
539+
fn record_conflicts(&mut self, new_conflicts: &mut BitSet<Local>) {
540+
// Remove all locals that are not candidates.
541+
new_conflicts.intersect(self.relevant_locals);
542+
543+
for local in new_conflicts.iter() {
544+
self.matrix.union_row_with(&new_conflicts, local);
545+
}
546+
}
547+
548+
/// Records locals that must not overlap during the evaluation of `stmt`. These locals conflict
549+
/// and must not be merged.
550+
fn record_statement_conflicts(&mut self, stmt: &Statement<'_>) {
551+
match &stmt.kind {
552+
// While the left and right sides of an assignment must not overlap, we do not mark
553+
// conflicts here as that would make this optimization useless. When we optimize, we
554+
// eliminate the resulting self-assignments automatically.
555+
StatementKind::Assign(_) => {}
556+
557+
StatementKind::LlvmInlineAsm(asm) => {
558+
// Inputs and outputs must not overlap.
559+
for (_, input) in &*asm.inputs {
560+
if let Some(in_place) = input.place() {
561+
if !in_place.is_indirect() {
562+
for out_place in &*asm.outputs {
563+
if !out_place.is_indirect() && !in_place.is_indirect() {
564+
self.matrix.insert(in_place.local, out_place.local);
565+
self.matrix.insert(out_place.local, in_place.local);
566+
}
567+
}
568+
}
569+
}
570+
}
571+
}
572+
573+
StatementKind::SetDiscriminant { .. }
574+
| StatementKind::StorageLive(_)
575+
| StatementKind::StorageDead(_)
576+
| StatementKind::Retag(_, _)
577+
| StatementKind::FakeRead(_, _)
578+
| StatementKind::AscribeUserType(_, _)
579+
| StatementKind::Nop => {}
527580
}
581+
}
528582

529-
Self { matrix: conflicts, unify_cache: BitSet::new_empty(body.local_decls.len()) }
583+
fn record_terminator_conflicts(&mut self, term: &Terminator<'_>) {
584+
match &term.kind {
585+
TerminatorKind::DropAndReplace { location, value, target: _, unwind: _ } => {
586+
if let Some(place) = value.place() {
587+
if !place.is_indirect() && !location.is_indirect() {
588+
self.matrix.insert(place.local, location.local);
589+
self.matrix.insert(location.local, place.local);
590+
}
591+
}
592+
}
593+
TerminatorKind::Yield { value, resume: _, resume_arg, drop: _ } => {
594+
if let Some(place) = value.place() {
595+
if !place.is_indirect() && !resume_arg.is_indirect() {
596+
self.matrix.insert(place.local, resume_arg.local);
597+
self.matrix.insert(resume_arg.local, place.local);
598+
}
599+
}
600+
}
601+
TerminatorKind::Call {
602+
func,
603+
args,
604+
destination: Some((dest_place, _)),
605+
cleanup: _,
606+
from_hir_call: _,
607+
} => {
608+
// No arguments may overlap with the destination.
609+
for arg in args.iter().chain(Some(func)) {
610+
if let Some(place) = arg.place() {
611+
if !place.is_indirect() && !dest_place.is_indirect() {
612+
self.matrix.insert(dest_place.local, place.local);
613+
self.matrix.insert(place.local, dest_place.local);
614+
}
615+
}
616+
}
617+
}
618+
TerminatorKind::InlineAsm {
619+
template: _,
620+
operands,
621+
options: _,
622+
line_spans: _,
623+
destination: _,
624+
} => {
625+
// The intended semantics here aren't documented, we just assume that nothing that
626+
// could be written to by the assembly may overlap with any other operands.
627+
for op in operands {
628+
match op {
629+
InlineAsmOperand::Out { reg: _, late: _, place: Some(dest_place) }
630+
| InlineAsmOperand::InOut {
631+
reg: _,
632+
late: _,
633+
in_value: _,
634+
out_place: Some(dest_place),
635+
} => {
636+
// For output place `place`, add all places accessed by the inline asm.
637+
for op in operands {
638+
match op {
639+
InlineAsmOperand::In { reg: _, value } => {
640+
if let Some(p) = value.place() {
641+
if !p.is_indirect() && !dest_place.is_indirect() {
642+
self.matrix.insert(p.local, dest_place.local);
643+
self.matrix.insert(dest_place.local, p.local);
644+
}
645+
}
646+
}
647+
InlineAsmOperand::Out {
648+
reg: _,
649+
late: _,
650+
place: Some(place),
651+
} => {
652+
if !place.is_indirect() && !dest_place.is_indirect() {
653+
self.matrix.insert(place.local, dest_place.local);
654+
self.matrix.insert(dest_place.local, place.local);
655+
}
656+
}
657+
InlineAsmOperand::InOut {
658+
reg: _,
659+
late: _,
660+
in_value,
661+
out_place,
662+
} => {
663+
if let Some(place) = in_value.place() {
664+
if !place.is_indirect() && !dest_place.is_indirect() {
665+
self.matrix.insert(place.local, dest_place.local);
666+
self.matrix.insert(dest_place.local, place.local);
667+
}
668+
}
669+
670+
if let Some(place) = out_place {
671+
if !place.is_indirect() && !dest_place.is_indirect() {
672+
self.matrix.insert(place.local, dest_place.local);
673+
self.matrix.insert(dest_place.local, place.local);
674+
}
675+
}
676+
}
677+
InlineAsmOperand::Out { reg: _, late: _, place: None }
678+
| InlineAsmOperand::Const { value: _ }
679+
| InlineAsmOperand::SymFn { value: _ }
680+
| InlineAsmOperand::SymStatic { value: _ } => {}
681+
}
682+
}
683+
}
684+
InlineAsmOperand::Const { value } => {
685+
assert!(value.place().is_none());
686+
}
687+
InlineAsmOperand::InOut {
688+
reg: _,
689+
late: _,
690+
in_value: _,
691+
out_place: None,
692+
}
693+
| InlineAsmOperand::In { reg: _, value: _ }
694+
| InlineAsmOperand::Out { reg: _, late: _, place: None }
695+
| InlineAsmOperand::SymFn { value: _ }
696+
| InlineAsmOperand::SymStatic { value: _ } => {}
697+
}
698+
}
699+
}
700+
701+
TerminatorKind::Goto { .. }
702+
| TerminatorKind::Call { destination: None, .. }
703+
| TerminatorKind::SwitchInt { .. }
704+
| TerminatorKind::Resume
705+
| TerminatorKind::Abort
706+
| TerminatorKind::Return
707+
| TerminatorKind::Unreachable
708+
| TerminatorKind::Drop { .. }
709+
| TerminatorKind::Assert { .. }
710+
| TerminatorKind::GeneratorDrop
711+
| TerminatorKind::FalseEdges { .. }
712+
| TerminatorKind::FalseUnwind { .. } => {}
713+
}
530714
}
531715

532716
fn contains(&self, a: Local, b: Local) -> bool {

0 commit comments

Comments
 (0)