@@ -5,7 +5,6 @@ use rustc_data_structures::fx::FxHashMap;
5
5
use rustc_data_structures:: graph:: DirectedGraph ;
6
6
use rustc_index:: IndexVec ;
7
7
use rustc_index:: bit_set:: BitSet ;
8
- use rustc_middle:: bug;
9
8
use rustc_middle:: mir:: coverage:: { CounterId , CovTerm , Expression , ExpressionId , Op } ;
10
9
use tracing:: { debug, debug_span, instrument} ;
11
10
@@ -58,13 +57,13 @@ pub(super) struct CoverageCounters {
58
57
counter_increment_sites : IndexVec < CounterId , CounterIncrementSite > ,
59
58
60
59
/// Coverage counters/expressions that are associated with individual BCBs.
61
- bcb_counters : IndexVec < BasicCoverageBlock , Option < BcbCounter > > ,
60
+ node_counters : IndexVec < BasicCoverageBlock , Option < BcbCounter > > ,
62
61
/// Coverage counters/expressions that are associated with the control-flow
63
62
/// edge between two BCBs.
64
63
///
65
64
/// We currently don't iterate over this map, but if we do in the future,
66
65
/// switch it back to `FxIndexMap` to avoid query stability hazards.
67
- bcb_edge_counters : FxHashMap < ( BasicCoverageBlock , BasicCoverageBlock ) , BcbCounter > ,
66
+ edge_counters : FxHashMap < ( BasicCoverageBlock , BasicCoverageBlock ) , BcbCounter > ,
68
67
69
68
/// Table of expression data, associating each expression ID with its
70
69
/// corresponding operator (+ or -) and its LHS/RHS operands.
@@ -78,20 +77,20 @@ impl CoverageCounters {
78
77
/// Ensures that each BCB node needing a counter has one, by creating physical
79
78
/// counters or counter expressions for nodes and edges as required.
80
79
pub ( super ) fn make_bcb_counters (
81
- basic_coverage_blocks : & CoverageGraph ,
80
+ graph : & CoverageGraph ,
82
81
bcb_needs_counter : & BitSet < BasicCoverageBlock > ,
83
82
) -> Self {
84
- let mut counters = MakeBcbCounters :: new ( basic_coverage_blocks , bcb_needs_counter) ;
85
- counters . make_bcb_counters ( ) ;
83
+ let mut builder = CountersBuilder :: new ( graph , bcb_needs_counter) ;
84
+ builder . make_bcb_counters ( ) ;
86
85
87
- counters . coverage_counters
86
+ builder . counters
88
87
}
89
88
90
89
fn with_num_bcbs ( num_bcbs : usize ) -> Self {
91
90
Self {
92
91
counter_increment_sites : IndexVec :: new ( ) ,
93
- bcb_counters : IndexVec :: from_elem_n ( None , num_bcbs) ,
94
- bcb_edge_counters : FxHashMap :: default ( ) ,
92
+ node_counters : IndexVec :: from_elem_n ( None , num_bcbs) ,
93
+ edge_counters : FxHashMap :: default ( ) ,
95
94
expressions : IndexVec :: new ( ) ,
96
95
expressions_memo : FxHashMap :: default ( ) ,
97
96
}
@@ -187,35 +186,31 @@ impl CoverageCounters {
187
186
self . counter_increment_sites . len ( )
188
187
}
189
188
190
- fn set_bcb_counter ( & mut self , bcb : BasicCoverageBlock , counter_kind : BcbCounter ) -> BcbCounter {
191
- if let Some ( replaced) = self . bcb_counters [ bcb] . replace ( counter_kind) {
192
- bug ! (
193
- "attempt to set a BasicCoverageBlock coverage counter more than once; \
194
- {bcb:?} already had counter {replaced:?}",
195
- ) ;
196
- } else {
197
- counter_kind
198
- }
189
+ fn set_node_counter ( & mut self , bcb : BasicCoverageBlock , counter : BcbCounter ) -> BcbCounter {
190
+ let existing = self . node_counters [ bcb] . replace ( counter) ;
191
+ assert ! (
192
+ existing. is_none( ) ,
193
+ "node {bcb:?} already has a counter: {existing:?} => {counter:?}"
194
+ ) ;
195
+ counter
199
196
}
200
197
201
- fn set_bcb_edge_counter (
198
+ fn set_edge_counter (
202
199
& mut self ,
203
200
from_bcb : BasicCoverageBlock ,
204
201
to_bcb : BasicCoverageBlock ,
205
- counter_kind : BcbCounter ,
202
+ counter : BcbCounter ,
206
203
) -> BcbCounter {
207
- if let Some ( replaced) = self . bcb_edge_counters . insert ( ( from_bcb, to_bcb) , counter_kind) {
208
- bug ! (
209
- "attempt to set an edge counter more than once; from_bcb: \
210
- {from_bcb:?} already had counter {replaced:?}",
211
- ) ;
212
- } else {
213
- counter_kind
214
- }
204
+ let existing = self . edge_counters . insert ( ( from_bcb, to_bcb) , counter) ;
205
+ assert ! (
206
+ existing. is_none( ) ,
207
+ "edge ({from_bcb:?} -> {to_bcb:?}) already has a counter: {existing:?} => {counter:?}"
208
+ ) ;
209
+ counter
215
210
}
216
211
217
212
pub ( super ) fn term_for_bcb ( & self , bcb : BasicCoverageBlock ) -> Option < CovTerm > {
218
- self . bcb_counters [ bcb] . map ( |counter| counter. as_term ( ) )
213
+ self . node_counters [ bcb] . map ( |counter| counter. as_term ( ) )
219
214
}
220
215
221
216
/// Returns an iterator over all the nodes/edges in the coverage graph that
@@ -232,7 +227,7 @@ impl CoverageCounters {
232
227
pub ( super ) fn bcb_nodes_with_coverage_expressions (
233
228
& self ,
234
229
) -> impl Iterator < Item = ( BasicCoverageBlock , ExpressionId ) > + Captures < ' _ > {
235
- self . bcb_counters . iter_enumerated ( ) . filter_map ( |( bcb, & counter_kind ) | match counter_kind {
230
+ self . node_counters . iter_enumerated ( ) . filter_map ( |( bcb, & counter ) | match counter {
236
231
// Yield the BCB along with its associated expression ID.
237
232
Some ( BcbCounter :: Expression { id } ) => Some ( ( bcb, id) ) ,
238
233
// This BCB is associated with a counter or nothing, so skip it.
@@ -259,22 +254,20 @@ impl CoverageCounters {
259
254
}
260
255
}
261
256
262
- /// Helper struct that allows counter creation to inspect the BCB graph.
263
- struct MakeBcbCounters < ' a > {
264
- coverage_counters : CoverageCounters ,
265
- basic_coverage_blocks : & ' a CoverageGraph ,
257
+ /// Helper struct that allows counter creation to inspect the BCB graph, and
258
+ /// the set of nodes that need counters.
259
+ struct CountersBuilder < ' a > {
260
+ counters : CoverageCounters ,
261
+ graph : & ' a CoverageGraph ,
266
262
bcb_needs_counter : & ' a BitSet < BasicCoverageBlock > ,
267
263
}
268
264
269
- impl < ' a > MakeBcbCounters < ' a > {
270
- fn new (
271
- basic_coverage_blocks : & ' a CoverageGraph ,
272
- bcb_needs_counter : & ' a BitSet < BasicCoverageBlock > ,
273
- ) -> Self {
274
- assert_eq ! ( basic_coverage_blocks. num_nodes( ) , bcb_needs_counter. domain_size( ) ) ;
265
+ impl < ' a > CountersBuilder < ' a > {
266
+ fn new ( graph : & ' a CoverageGraph , bcb_needs_counter : & ' a BitSet < BasicCoverageBlock > ) -> Self {
267
+ assert_eq ! ( graph. num_nodes( ) , bcb_needs_counter. domain_size( ) ) ;
275
268
Self {
276
- coverage_counters : CoverageCounters :: with_num_bcbs ( basic_coverage_blocks . num_nodes ( ) ) ,
277
- basic_coverage_blocks ,
269
+ counters : CoverageCounters :: with_num_bcbs ( graph . num_nodes ( ) ) ,
270
+ graph ,
278
271
bcb_needs_counter,
279
272
}
280
273
}
@@ -289,7 +282,7 @@ impl<'a> MakeBcbCounters<'a> {
289
282
// nodes within the loop are visited before visiting any nodes outside
290
283
// the loop. It also keeps track of which loop(s) the traversal is
291
284
// currently inside.
292
- let mut traversal = TraverseCoverageGraphWithLoops :: new ( self . basic_coverage_blocks ) ;
285
+ let mut traversal = TraverseCoverageGraphWithLoops :: new ( self . graph ) ;
293
286
while let Some ( bcb) = traversal. next ( ) {
294
287
let _span = debug_span ! ( "traversal" , ?bcb) . entered ( ) ;
295
288
if self . bcb_needs_counter . contains ( bcb) {
@@ -316,26 +309,26 @@ impl<'a> MakeBcbCounters<'a> {
316
309
// We might also use that counter to compute one of the out-edge counters.
317
310
let node_counter = self . get_or_make_node_counter ( from_bcb) ;
318
311
319
- let successors = self . basic_coverage_blocks . successors [ from_bcb] . as_slice ( ) ;
312
+ let successors = self . graph . successors [ from_bcb] . as_slice ( ) ;
320
313
321
314
// If this node's out-edges won't sum to the node's counter,
322
315
// then there's no reason to create edge counters here.
323
- if !self . basic_coverage_blocks [ from_bcb] . is_out_summable {
316
+ if !self . graph [ from_bcb] . is_out_summable {
324
317
return ;
325
318
}
326
319
327
320
// When choosing which out-edge should be given a counter expression, ignore edges that
328
321
// already have counters, or could use the existing counter of their target node.
329
322
let out_edge_has_counter = |to_bcb| {
330
- if self . coverage_counters . bcb_edge_counters . contains_key ( & ( from_bcb, to_bcb) ) {
323
+ if self . counters . edge_counters . contains_key ( & ( from_bcb, to_bcb) ) {
331
324
return true ;
332
325
}
333
- self . basic_coverage_blocks . sole_predecessor ( to_bcb) == Some ( from_bcb)
334
- && self . coverage_counters . bcb_counters [ to_bcb] . is_some ( )
326
+ self . graph . sole_predecessor ( to_bcb) == Some ( from_bcb)
327
+ && self . counters . node_counters [ to_bcb] . is_some ( )
335
328
} ;
336
329
337
330
// Determine the set of out-edges that could benefit from being given an expression.
338
- let candidate_successors = self . basic_coverage_blocks . successors [ from_bcb]
331
+ let candidate_successors = self . graph . successors [ from_bcb]
339
332
. iter ( )
340
333
. copied ( )
341
334
. filter ( |& to_bcb| !out_edge_has_counter ( to_bcb) )
@@ -344,7 +337,7 @@ impl<'a> MakeBcbCounters<'a> {
344
337
345
338
// If there are out-edges without counters, choose one to be given an expression
346
339
// (computed from this node and the other out-edges) instead of a physical counter.
347
- let Some ( expression_to_bcb ) =
340
+ let Some ( target_bcb ) =
348
341
self . choose_out_edge_for_expression ( traversal, & candidate_successors)
349
342
else {
350
343
return ;
@@ -357,49 +350,44 @@ impl<'a> MakeBcbCounters<'a> {
357
350
. iter ( )
358
351
. copied ( )
359
352
// Skip the chosen edge, since we'll calculate its count from this sum.
360
- . filter ( |& to_bcb| to_bcb != expression_to_bcb )
353
+ . filter ( |& edge_target_bcb| edge_target_bcb != target_bcb )
361
354
. map ( |to_bcb| self . get_or_make_edge_counter ( from_bcb, to_bcb) )
362
355
. collect :: < Vec < _ > > ( ) ;
363
- let Some ( sum_of_all_other_out_edges) =
364
- self . coverage_counters . make_sum ( & other_out_edge_counters)
356
+ let Some ( sum_of_all_other_out_edges) = self . counters . make_sum ( & other_out_edge_counters)
365
357
else {
366
358
return ;
367
359
} ;
368
360
369
361
// Now create an expression for the chosen edge, by taking the counter
370
362
// for its source node and subtracting the sum of its sibling out-edges.
371
- let expression = self . coverage_counters . make_expression (
372
- node_counter,
373
- Op :: Subtract ,
374
- sum_of_all_other_out_edges,
375
- ) ;
363
+ let expression =
364
+ self . counters . make_expression ( node_counter, Op :: Subtract , sum_of_all_other_out_edges) ;
376
365
377
- debug ! ( "{expression_to_bcb :?} gets an expression: {expression:?}" ) ;
378
- self . coverage_counters . set_bcb_edge_counter ( from_bcb, expression_to_bcb , expression) ;
366
+ debug ! ( "{target_bcb :?} gets an expression: {expression:?}" ) ;
367
+ self . counters . set_edge_counter ( from_bcb, target_bcb , expression) ;
379
368
}
380
369
381
370
#[ instrument( level = "debug" , skip( self ) ) ]
382
371
fn get_or_make_node_counter ( & mut self , bcb : BasicCoverageBlock ) -> BcbCounter {
383
372
// If the BCB already has a counter, return it.
384
- if let Some ( counter_kind ) = self . coverage_counters . bcb_counters [ bcb] {
385
- debug ! ( "{bcb:?} already has a counter: {counter_kind :?}" ) ;
386
- return counter_kind ;
373
+ if let Some ( counter ) = self . counters . node_counters [ bcb] {
374
+ debug ! ( "{bcb:?} already has a counter: {counter :?}" ) ;
375
+ return counter ;
387
376
}
388
377
389
378
let counter = self . make_node_counter_inner ( bcb) ;
390
- self . coverage_counters . set_bcb_counter ( bcb, counter)
379
+ self . counters . set_node_counter ( bcb, counter)
391
380
}
392
381
393
382
fn make_node_counter_inner ( & mut self , bcb : BasicCoverageBlock ) -> BcbCounter {
394
383
// If the node's sole in-edge already has a counter, use that.
395
- if let Some ( sole_pred) = self . basic_coverage_blocks . sole_predecessor ( bcb)
396
- && let Some ( & edge_counter) =
397
- self . coverage_counters . bcb_edge_counters . get ( & ( sole_pred, bcb) )
384
+ if let Some ( sole_pred) = self . graph . sole_predecessor ( bcb)
385
+ && let Some ( & edge_counter) = self . counters . edge_counters . get ( & ( sole_pred, bcb) )
398
386
{
399
387
return edge_counter;
400
388
}
401
389
402
- let predecessors = self . basic_coverage_blocks . predecessors [ bcb] . as_slice ( ) ;
390
+ let predecessors = self . graph . predecessors [ bcb] . as_slice ( ) ;
403
391
404
392
// Handle cases where we can't compute a node's count from its in-edges:
405
393
// - START_BCB has no in-edges, so taking the sum would panic (or be wrong).
@@ -408,7 +396,7 @@ impl<'a> MakeBcbCounters<'a> {
408
396
// leading to infinite recursion.
409
397
if predecessors. len ( ) <= 1 || predecessors. contains ( & bcb) {
410
398
debug ! ( ?bcb, ?predecessors, "node has <=1 predecessors or is its own predecessor" ) ;
411
- let counter = self . coverage_counters . make_phys_node_counter ( bcb) ;
399
+ let counter = self . counters . make_phys_node_counter ( bcb) ;
412
400
debug ! ( ?bcb, ?counter, "node gets a physical counter" ) ;
413
401
return counter;
414
402
}
@@ -420,10 +408,8 @@ impl<'a> MakeBcbCounters<'a> {
420
408
. copied ( )
421
409
. map ( |from_bcb| self . get_or_make_edge_counter ( from_bcb, bcb) )
422
410
. collect :: < Vec < _ > > ( ) ;
423
- let sum_of_in_edges: BcbCounter = self
424
- . coverage_counters
425
- . make_sum ( & in_edge_counters)
426
- . expect ( "there must be at least one in-edge" ) ;
411
+ let sum_of_in_edges: BcbCounter =
412
+ self . counters . make_sum ( & in_edge_counters) . expect ( "there must be at least one in-edge" ) ;
427
413
428
414
debug ! ( "{bcb:?} gets a new counter (sum of predecessor counters): {sum_of_in_edges:?}" ) ;
429
415
sum_of_in_edges
@@ -436,15 +422,13 @@ impl<'a> MakeBcbCounters<'a> {
436
422
to_bcb : BasicCoverageBlock ,
437
423
) -> BcbCounter {
438
424
// If the edge already has a counter, return it.
439
- if let Some ( & counter_kind) =
440
- self . coverage_counters . bcb_edge_counters . get ( & ( from_bcb, to_bcb) )
441
- {
442
- debug ! ( "Edge {from_bcb:?}->{to_bcb:?} already has a counter: {counter_kind:?}" ) ;
443
- return counter_kind;
425
+ if let Some ( & counter) = self . counters . edge_counters . get ( & ( from_bcb, to_bcb) ) {
426
+ debug ! ( "Edge {from_bcb:?}->{to_bcb:?} already has a counter: {counter:?}" ) ;
427
+ return counter;
444
428
}
445
429
446
430
let counter = self . make_edge_counter_inner ( from_bcb, to_bcb) ;
447
- self . coverage_counters . set_bcb_edge_counter ( from_bcb, to_bcb, counter)
431
+ self . counters . set_edge_counter ( from_bcb, to_bcb, counter)
448
432
}
449
433
450
434
fn make_edge_counter_inner (
@@ -454,7 +438,7 @@ impl<'a> MakeBcbCounters<'a> {
454
438
) -> BcbCounter {
455
439
// If the target node has exactly one in-edge (i.e. this one), then just
456
440
// use the node's counter, since it will have the same value.
457
- if let Some ( sole_pred) = self . basic_coverage_blocks . sole_predecessor ( to_bcb) {
441
+ if let Some ( sole_pred) = self . graph . sole_predecessor ( to_bcb) {
458
442
assert_eq ! ( sole_pred, from_bcb) ;
459
443
// This call must take care not to invoke `get_or_make_edge` for
460
444
// this edge, since that would result in infinite recursion!
@@ -463,13 +447,13 @@ impl<'a> MakeBcbCounters<'a> {
463
447
464
448
// If the source node has exactly one out-edge (i.e. this one) and would have
465
449
// the same execution count as that edge, then just use the node's counter.
466
- if let Some ( simple_succ) = self . basic_coverage_blocks . simple_successor ( from_bcb) {
450
+ if let Some ( simple_succ) = self . graph . simple_successor ( from_bcb) {
467
451
assert_eq ! ( simple_succ, to_bcb) ;
468
452
return self . get_or_make_node_counter ( from_bcb) ;
469
453
}
470
454
471
455
// Make a new counter to count this edge.
472
- let counter = self . coverage_counters . make_phys_edge_counter ( from_bcb, to_bcb) ;
456
+ let counter = self . counters . make_phys_edge_counter ( from_bcb, to_bcb) ;
473
457
debug ! ( ?from_bcb, ?to_bcb, ?counter, "edge gets a physical counter" ) ;
474
458
counter
475
459
}
@@ -516,9 +500,9 @@ impl<'a> MakeBcbCounters<'a> {
516
500
for & target_bcb in candidate_successors {
517
501
// An edge is a reloop edge if its target dominates any BCB that has
518
502
// an edge back to the loop header. (Otherwise it's an exit edge.)
519
- let is_reloop_edge = reloop_bcbs. iter ( ) . any ( | & reloop_bcb| {
520
- self . basic_coverage_blocks . dominates ( target_bcb , reloop_bcb )
521
- } ) ;
503
+ let is_reloop_edge = reloop_bcbs
504
+ . iter ( )
505
+ . any ( | & reloop_bcb| self . graph . dominates ( target_bcb , reloop_bcb ) ) ;
522
506
if is_reloop_edge {
523
507
// We found a good out-edge to be given an expression.
524
508
return Some ( target_bcb) ;
0 commit comments