@@ -133,21 +133,20 @@ const QUEUE_LOCKED: usize = 4;
133
133
const SINGLE : usize = 8 ;
134
134
const MASK : usize = !( QUEUE_LOCKED | QUEUED | LOCKED ) ;
135
135
136
- /// Returns a closure that changes the state to the lock state corresponding to
137
- /// the lock mode indicated in `write`.
136
+ /// Marks the state as write-locked, if possible.
138
137
#[ inline]
139
- fn lock ( write : bool ) -> impl Fn ( State ) -> Option < State > {
140
- move | state| {
141
- if write {
142
- let state = state . wrapping_byte_add ( LOCKED ) ;
143
- if state . addr ( ) & LOCKED == LOCKED { Some ( state ) } else { None }
144
- } else {
145
- if state . addr ( ) & QUEUED == 0 && state . addr ( ) != LOCKED {
146
- Some ( invalid_mut ( state. addr ( ) . checked_add ( SINGLE ) ? | LOCKED ) )
147
- } else {
148
- None
149
- }
150
- }
138
+ fn write_lock ( state : State ) -> Option < State > {
139
+ let state = state . wrapping_byte_add ( LOCKED ) ;
140
+ if state . addr ( ) & LOCKED == LOCKED { Some ( state ) } else { None }
141
+ }
142
+
143
+ /// Marks the state as read-locked, if possible.
144
+ # [ inline ]
145
+ fn read_lock ( state : State ) -> Option < State > {
146
+ if state . addr ( ) & QUEUED == 0 && state . addr ( ) != LOCKED {
147
+ Some ( invalid_mut ( state . addr ( ) . checked_add ( SINGLE ) ? | LOCKED ) )
148
+ } else {
149
+ None
151
150
}
152
151
}
153
152
@@ -241,7 +240,7 @@ impl Drop for PanicGuard {
241
240
}
242
241
}
243
242
244
- /// Find the tail of the queue beginning with `head`, caching the result in `head` .
243
+ /// Add backlinks to the queue, returning the tail .
245
244
///
246
245
/// May be called from multiple threads at the same time, while the queue is not
247
246
/// modified (this happens when unlocking multiple readers).
@@ -252,7 +251,7 @@ impl Drop for PanicGuard {
252
251
/// last removal.
253
252
/// * The part of the queue starting with `head` must not be modified during this
254
253
/// call.
255
- unsafe fn find_tail ( head : NonNull < Node > ) -> NonNull < Node > {
254
+ unsafe fn add_backlinks_and_find_tail ( head : NonNull < Node > ) -> NonNull < Node > {
256
255
let mut current = head;
257
256
let tail = loop {
258
257
let c = unsafe { current. as_ref ( ) } ;
@@ -287,7 +286,7 @@ impl RwLock {
287
286
288
287
#[ inline]
289
288
pub fn try_read ( & self ) -> bool {
290
- self . state . fetch_update ( Acquire , Relaxed , lock ( false ) ) . is_ok ( )
289
+ self . state . fetch_update ( Acquire , Relaxed , read_lock ) . is_ok ( )
291
290
}
292
291
293
292
#[ inline]
@@ -316,7 +315,7 @@ impl RwLock {
316
315
317
316
#[ cold]
318
317
fn lock_contended ( & self , write : bool ) {
319
- let update = lock ( write) ;
318
+ let update = if write { write_lock } else { read_lock } ;
320
319
let mut node = Node :: new ( write) ;
321
320
let mut state = self . state . load ( Relaxed ) ;
322
321
let mut count = 0 ;
@@ -428,7 +427,7 @@ impl RwLock {
428
427
// all queue-lock owners will observe the set `LOCKED` bit. Because they
429
428
// do not modify the queue while there is a lock owner, the queue will
430
429
// not be removed from here.
431
- let tail = unsafe { find_tail ( to_node ( state) ) . as_ref ( ) } ;
430
+ let tail = unsafe { add_backlinks_and_find_tail ( to_node ( state) ) . as_ref ( ) } ;
432
431
// The lock count is stored in the `next` field of `tail`.
433
432
// Decrement it, making sure to observe all changes made to the queue
434
433
// by the other lock owners by using acquire-release ordering.
@@ -482,8 +481,7 @@ impl RwLock {
482
481
debug_assert_eq ! ( state. addr( ) & ( QUEUED | QUEUE_LOCKED ) , QUEUED | QUEUE_LOCKED ) ;
483
482
484
483
loop {
485
- // Find the last node in the linked list.
486
- let tail = unsafe { find_tail ( to_node ( state) ) } ;
484
+ let tail = unsafe { add_backlinks_and_find_tail ( to_node ( state) ) } ;
487
485
488
486
if state. addr ( ) & LOCKED == LOCKED {
489
487
// Another thread has locked the lock. Leave waking up waiters
0 commit comments