Merge branch 'conradludgate-rwlock-detached' (#323)

Author: xacrimon

src/iter.rs

@@ -1,7 +1,7 @@
 use super::mapref::multiple::{RefMulti, RefMutMulti};
-use crate::lock::{RwLockReadGuard, RwLockWriteGuard};
+use crate::lock::{RwLockReadGuardDetached, RwLockWriteGuardDetached};
 use crate::t::Map;
-use crate::{DashMap, HashMap};
+use crate::DashMap;
 use core::hash::{BuildHasher, Hash};
 use core::mem;
 use std::collections::hash_map::RandomState;
@@ -69,29 +69,13 @@ impl<K: Eq + Hash, V, S: BuildHasher + Clone> Iterator for OwningIter<K, V, S> {
     }
 }
 
-unsafe impl<K, V, S> Send for OwningIter<K, V, S>
-where
-    K: Eq + Hash + Send,
-    V: Send,
-    S: BuildHasher + Clone + Send,
-{
-}
-
-unsafe impl<K, V, S> Sync for OwningIter<K, V, S>
-where
-    K: Eq + Hash + Sync,
-    V: Sync,
-    S: BuildHasher + Clone + Sync,
-{
-}
-
 type GuardIter<'a, K, V> = (
-    Arc<RwLockReadGuard<'a, HashMap<K, V>>>,
+    Arc<RwLockReadGuardDetached<'a>>,
     hashbrown::raw::RawIter<(K, V)>,
 );
 
 type GuardIterMut<'a, K, V> = (
-    Arc<RwLockWriteGuard<'a, HashMap<K, V>>>,
+    Arc<RwLockWriteGuardDetached<'a>>,
     hashbrown::raw::RawIter<(K, V)>,
 );
 
@@ -119,25 +103,9 @@ impl<'i, K: Clone + Hash + Eq, V: Clone, S: Clone + BuildHasher> Clone for Iter<
     }
 }
 
-unsafe impl<'a, 'i, K, V, S, M> Send for Iter<'i, K, V, S, M>
-where
-    K: 'a + Eq + Hash + Send,
-    V: 'a + Send,
-    S: 'a + BuildHasher + Clone,
-    M: Map<'a, K, V, S>,
-{
-}
-
-unsafe impl<'a, 'i, K, V, S, M> Sync for Iter<'i, K, V, S, M>
-where
-    K: 'a + Eq + Hash + Sync,
-    V: 'a + Sync,
-    S: 'a + BuildHasher + Clone,
-    M: Map<'a, K, V, S>,
+impl<'a, K: Eq + Hash + 'a, V: 'a, S: 'a + BuildHasher + Clone, M: Map<'a, K, V, S>>
+    Iter<'a, K, V, S, M>
 {
-}
-
-impl<'a, K: Eq + Hash, V, S: 'a + BuildHasher + Clone, M: Map<'a, K, V, S>> Iter<'a, K, V, S, M> {
     pub(crate) fn new(map: &'a M) -> Self {
         Self {
             map,
@@ -148,7 +116,7 @@ impl<'a, K: Eq + Hash, V, S: 'a + BuildHasher + Clone, M: Map<'a, K, V, S>> Iter
     }
 }
 
-impl<'a, K: Eq + Hash, V, S: 'a + BuildHasher + Clone, M: Map<'a, K, V, S>> Iterator
+impl<'a, K: Eq + Hash + 'a, V: 'a, S: 'a + BuildHasher + Clone, M: Map<'a, K, V, S>> Iterator
     for Iter<'a, K, V, S, M>
 {
     type Item = RefMulti<'a, K, V>;
@@ -170,8 +138,11 @@ impl<'a, K: Eq + Hash, V, S: 'a + BuildHasher + Clone, M: Map<'a, K, V, S>> Iter
             }
 
             let guard = unsafe { self.map._yield_read_shard(self.shard_i) };
+            // SAFETY: we keep the guard alive with the shard iterator,
+            // and with any refs produced by the iterator
+            let (guard, shard) = unsafe { RwLockReadGuardDetached::detach_from(guard) };
 
-            let iter = unsafe { guard.iter() };
+            let iter = unsafe { shard.iter() };
 
             self.current = Some((Arc::new(guard), iter));
 
@@ -199,25 +170,7 @@ pub struct IterMut<'a, K, V, S = RandomState, M = DashMap<K, V, S>> {
     marker: PhantomData<S>,
 }
 
-unsafe impl<'a, 'i, K, V, S, M> Send for IterMut<'i, K, V, S, M>
-where
-    K: 'a + Eq + Hash + Send,
-    V: 'a + Send,
-    S: 'a + BuildHasher + Clone,
-    M: Map<'a, K, V, S>,
-{
-}
-
-unsafe impl<'a, 'i, K, V, S, M> Sync for IterMut<'i, K, V, S, M>
-where
-    K: 'a + Eq + Hash + Sync,
-    V: 'a + Sync,
-    S: 'a + BuildHasher + Clone,
-    M: Map<'a, K, V, S>,
-{
-}
-
-impl<'a, K: Eq + Hash, V, S: 'a + BuildHasher + Clone, M: Map<'a, K, V, S>>
+impl<'a, K: Eq + Hash + 'a, V: 'a, S: 'a + BuildHasher + Clone, M: Map<'a, K, V, S>>
     IterMut<'a, K, V, S, M>
 {
     pub(crate) fn new(map: &'a M) -> Self {
@@ -230,7 +183,7 @@ impl<'a, K: Eq + Hash, V, S: 'a + BuildHasher + Clone, M: Map<'a, K, V, S>>
     }
 }
 
-impl<'a, K: Eq + Hash, V, S: 'a + BuildHasher + Clone, M: Map<'a, K, V, S>> Iterator
+impl<'a, K: Eq + Hash + 'a, V: 'a, S: 'a + BuildHasher + Clone, M: Map<'a, K, V, S>> Iterator
     for IterMut<'a, K, V, S, M>
 {
     type Item = RefMutMulti<'a, K, V>;
@@ -253,7 +206,11 @@ impl<'a, K: Eq + Hash, V, S: 'a + BuildHasher + Clone, M: Map<'a, K, V, S>> Iter
 
             let guard = unsafe { self.map._yield_write_shard(self.shard_i) };
 
-            let iter = unsafe { guard.iter() };
+            // SAFETY: we keep the guard alive with the shard iterator,
+            // and with any refs produced by the iterator
+            let (guard, shard) = unsafe { RwLockWriteGuardDetached::detach_from(guard) };
+
+            let iter = unsafe { shard.iter() };
 
             self.current = Some((Arc::new(guard), iter));
 

src/iter_set.rs

@@ -20,47 +20,17 @@ impl<K: Eq + Hash, S: BuildHasher + Clone> Iterator for OwningIter<K, S> {
     }
 }
 
-unsafe impl<K, S> Send for OwningIter<K, S>
-where
-    K: Eq + Hash + Send,
-    S: BuildHasher + Clone + Send,
-{
-}
-
-unsafe impl<K, S> Sync for OwningIter<K, S>
-where
-    K: Eq + Hash + Sync,
-    S: BuildHasher + Clone + Sync,
-{
-}
-
 pub struct Iter<'a, K, S, M> {
     inner: crate::iter::Iter<'a, K, (), S, M>,
 }
 
-unsafe impl<'a, 'i, K, S, M> Send for Iter<'i, K, S, M>
-where
-    K: 'a + Eq + Hash + Send,
-    S: 'a + BuildHasher + Clone,
-    M: Map<'a, K, (), S>,
-{
-}
-
-unsafe impl<'a, 'i, K, S, M> Sync for Iter<'i, K, S, M>
-where
-    K: 'a + Eq + Hash + Sync,
-    S: 'a + BuildHasher + Clone,
-    M: Map<'a, K, (), S>,
-{
-}
-
-impl<'a, K: Eq + Hash, S: 'a + BuildHasher + Clone, M: Map<'a, K, (), S>> Iter<'a, K, S, M> {
+impl<'a, K: Eq + Hash + 'a, S: 'a + BuildHasher + Clone, M: Map<'a, K, (), S>> Iter<'a, K, S, M> {
     pub(crate) fn new(inner: crate::iter::Iter<'a, K, (), S, M>) -> Self {
         Self { inner }
     }
 }
 
-impl<'a, K: Eq + Hash, S: 'a + BuildHasher + Clone, M: Map<'a, K, (), S>> Iterator
+impl<'a, K: Eq + Hash + 'a, S: 'a + BuildHasher + Clone, M: Map<'a, K, (), S>> Iterator
     for Iter<'a, K, S, M>
 {
     type Item = RefMulti<'a, K>;

src/lib.rs

@@ -36,6 +36,7 @@ use core::iter::FromIterator;
 use core::ops::{BitAnd, BitOr, Shl, Shr, Sub};
 use crossbeam_utils::CachePadded;
 use iter::{Iter, IterMut, OwningIter};
+use lock::{RwLockReadGuardDetached, RwLockWriteGuardDetached};
 pub use mapref::entry::{Entry, OccupiedEntry, VacantEntry};
 use mapref::multiple::RefMulti;
 use mapref::one::{Ref, RefMut};
@@ -1028,11 +1029,13 @@ impl<'a, K: 'a + Eq + Hash, V: 'a, S: 'a + BuildHasher + Clone> Map<'a, K, V, S>
         let idx = self.determine_shard(hash as usize);
 
         let shard = unsafe { self._yield_read_shard(idx) };
+        // SAFETY: The data will not outlive the guard, since we pass the guard to `Ref`.
+        let (guard, shard) = unsafe { RwLockReadGuardDetached::detach_from(shard) };
 
         if let Some(bucket) = shard.find(hash, |(k, _v)| key == k.borrow()) {
             unsafe {
                 let (k, v) = bucket.as_ref();
-                Some(Ref::new(shard, k, v))
+                Some(Ref::new(guard, k, v))
             }
         } else {
             None
@@ -1049,11 +1052,13 @@ impl<'a, K: 'a + Eq + Hash, V: 'a, S: 'a + BuildHasher + Clone> Map<'a, K, V, S>
         let idx = self.determine_shard(hash as usize);
 
         let shard = unsafe { self._yield_write_shard(idx) };
+        // SAFETY: The data will not outlive the guard, since we pass the guard to `RefMut`.
+        let (guard, shard) = unsafe { RwLockWriteGuardDetached::detach_from(shard) };
 
         if let Some(bucket) = shard.find(hash, |(k, _v)| key == k.borrow()) {
             unsafe {
                 let (k, v) = bucket.as_mut();
-                Some(RefMut::new(shard, k, v))
+                Some(RefMut::new(guard, k, v))
             }
         } else {
             None
@@ -1073,11 +1078,13 @@ impl<'a, K: 'a + Eq + Hash, V: 'a, S: 'a + BuildHasher + Clone> Map<'a, K, V, S>
             Some(shard) => shard,
             None => return TryResult::Locked,
         };
+        // SAFETY: The data will not outlive the guard, since we pass the guard to `Ref`.
+        let (guard, shard) = unsafe { RwLockReadGuardDetached::detach_from(shard) };
 
         if let Some(bucket) = shard.find(hash, |(k, _v)| key == k.borrow()) {
             unsafe {
                 let (k, v) = bucket.as_ref();
-                TryResult::Present(Ref::new(shard, k, v))
+                TryResult::Present(Ref::new(guard, k, v))
             }
         } else {
             TryResult::Absent
@@ -1097,11 +1104,13 @@ impl<'a, K: 'a + Eq + Hash, V: 'a, S: 'a + BuildHasher + Clone> Map<'a, K, V, S>
             Some(shard) => shard,
             None => return TryResult::Locked,
         };
+        // SAFETY: The data will not outlive the guard, since we pass the guard to `RefMut`.
+        let (guard, shard) = unsafe { RwLockWriteGuardDetached::detach_from(shard) };
 
         if let Some(bucket) = shard.find(hash, |(k, _v)| key == k.borrow()) {
             unsafe {
                 let (k, v) = bucket.as_mut();
-                TryResult::Present(RefMut::new(shard, k, v))
+                TryResult::Present(RefMut::new(guard, k, v))
             }
         } else {
             TryResult::Absent
@@ -1174,7 +1183,9 @@ impl<'a, K: 'a + Eq + Hash, V: 'a, S: 'a + BuildHasher + Clone> Map<'a, K, V, S>
 
         let idx = self.determine_shard(hash as usize);
 
-        let mut shard = unsafe { self._yield_write_shard(idx) };
+        let shard = unsafe { self._yield_write_shard(idx) };
+        // SAFETY: The data will not outlive the guard, since we pass the guard to `Entry`.
+        let (guard, shard) = unsafe { RwLockWriteGuardDetached::detach_from(shard) };
 
         match shard.find_or_find_insert_slot(
             hash,
@@ -1185,8 +1196,8 @@ impl<'a, K: 'a + Eq + Hash, V: 'a, S: 'a + BuildHasher + Clone> Map<'a, K, V, S>
                 hasher.finish()
             },
         ) {
-            Ok(elem) => Entry::Occupied(unsafe { OccupiedEntry::new(shard, key, elem) }),
-            Err(slot) => Entry::Vacant(unsafe { VacantEntry::new(shard, key, hash, slot) }),
+            Ok(elem) => Entry::Occupied(unsafe { OccupiedEntry::new(guard, key, shard, elem) }),
+            Err(slot) => Entry::Vacant(unsafe { VacantEntry::new(guard, key, hash, shard, slot) }),
         }
     }
 
@@ -1195,10 +1206,12 @@ impl<'a, K: 'a + Eq + Hash, V: 'a, S: 'a + BuildHasher + Clone> Map<'a, K, V, S>
 
         let idx = self.determine_shard(hash as usize);
 
-        let mut shard = match unsafe { self._try_yield_write_shard(idx) } {
+        let shard = match unsafe { self._try_yield_write_shard(idx) } {
             Some(shard) => shard,
             None => return None,
         };
+        // SAFETY: The data will not outlive the guard, since we pass the guard to `Entry`.
+        let (guard, shard) = unsafe { RwLockWriteGuardDetached::detach_from(shard) };
 
         match shard.find_or_find_insert_slot(
             hash,
@@ -1210,10 +1223,10 @@ impl<'a, K: 'a + Eq + Hash, V: 'a, S: 'a + BuildHasher + Clone> Map<'a, K, V, S>
             },
         ) {
             Ok(elem) => Some(Entry::Occupied(unsafe {
-                OccupiedEntry::new(shard, key, elem)
+                OccupiedEntry::new(guard, key, shard, elem)
             })),
             Err(slot) => Some(Entry::Vacant(unsafe {
-                VacantEntry::new(shard, key, hash, slot)
+                VacantEntry::new(guard, key, hash, shard, slot)
             })),
         }
     }

src/lock.rs

@@ -4,6 +4,8 @@ use parking_lot_core::{ParkToken, SpinWait, UnparkToken};
 pub type RwLock<T> = lock_api::RwLock<RawRwLock, T>;
 pub type RwLockReadGuard<'a, T> = lock_api::RwLockReadGuard<'a, RawRwLock, T>;
 pub type RwLockWriteGuard<'a, T> = lock_api::RwLockWriteGuard<'a, RawRwLock, T>;
+pub(crate) type RwLockReadGuardDetached<'a> = crate::util::RwLockReadGuardDetached<'a, RawRwLock>;
+pub(crate) type RwLockWriteGuardDetached<'a> = crate::util::RwLockWriteGuardDetached<'a, RawRwLock>;
 
 const READERS_PARKED: usize = 0b0001;
 const WRITERS_PARKED: usize = 0b0010;
@@ -20,7 +22,7 @@ unsafe impl lock_api::RawRwLock for RawRwLock {
         state: AtomicUsize::new(0),
     };
 
-    type GuardMarker = lock_api::GuardNoSend;
+    type GuardMarker = lock_api::GuardSend;
 
     #[inline]
     fn try_lock_exclusive(&self) -> bool {