Rollup merge of #97647 - m-ou-se:lazy-box-locks, r=Amanieu

Lazily allocate and initialize pthread locks.

Lazily allocate and initialize pthread locks.

This allows {Mutex, Condvar, RwLock}::new() to be const, while still using the platform's native locks for features like priority inheritance and debug tooling. E.g. on macOS, we cannot directly use the (private) APIs that pthread's locks are implemented with, making it impossible for us to use anything other than pthread while still preserving priority inheritance, etc.

This PR doesn't yet make the public APIs const. That's for a separate PR with an FCP.

Tracking issue: https://github.com/rust-lang/rust/issues/93740
This commit is contained in:
Dylan DPC 2022-06-04 11:06:40 +02:00 committed by GitHub
commit e9ec02267a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
29 changed files with 184 additions and 123 deletions

View File

@ -70,9 +70,13 @@ pub unsafe fn wait_timeout(&self, mutex: &Mutex, dur: Duration) -> bool {
mutex.lock();
res == 0
}
}
pub unsafe fn destroy(&self) {
let _ = abi::sem_destroy(self.sem1);
let _ = abi::sem_destroy(self.sem2);
impl Drop for Condvar {
fn drop(&mut self) {
unsafe {
let _ = abi::sem_destroy(self.sem1);
let _ = abi::sem_destroy(self.sem2);
}
}
}

View File

@ -215,7 +215,4 @@ pub unsafe fn try_lock(&self) -> bool {
}
guard.locked
}
#[inline]
pub unsafe fn destroy(&self) {}
}

View File

@ -84,12 +84,6 @@ pub unsafe fn write_unlock(&self) {
// FIXME: should only wake up one of these some of the time
self.cond.notify_all();
}
#[inline]
pub unsafe fn destroy(&self) {
self.lock.destroy();
self.cond.destroy();
}
}
impl State {

View File

@ -117,8 +117,6 @@ pub unsafe fn wait_timeout(&self, mutex: &Mutex, dur: Duration) -> bool {
unsafe { mutex.lock() };
success
}
pub unsafe fn destroy(&self) {}
}
mod waiter_queue {

View File

@ -64,8 +64,10 @@ pub unsafe fn try_lock(&self) -> bool {
}
}
}
}
pub unsafe fn destroy(&self) {
impl Drop for Mutex {
fn drop(&mut self) {
if let Some(mtx) = self.mtx.get().map(|x| x.0) {
expect_success_aborting(unsafe { abi::del_mtx(mtx) }, &"del_mtx");
}

View File

@ -1,4 +1,5 @@
use crate::sys::locks::Mutex;
use crate::sys_common::lazy_box::{LazyBox, LazyInit};
use crate::time::Duration;
use super::waitqueue::{SpinMutex, WaitQueue, WaitVariable};
@ -7,16 +8,19 @@ pub struct Condvar {
inner: SpinMutex<WaitVariable<()>>,
}
pub type MovableCondvar = Box<Condvar>;
pub(crate) type MovableCondvar = LazyBox<Condvar>;
impl LazyInit for Condvar {
fn init() -> Box<Self> {
Box::new(Self::new())
}
}
impl Condvar {
pub const fn new() -> Condvar {
Condvar { inner: SpinMutex::new(WaitVariable::new(())) }
}
#[inline]
pub unsafe fn init(&mut self) {}
#[inline]
pub unsafe fn notify_one(&self) {
let _ = WaitQueue::notify_one(self.inner.lock());
@ -38,7 +42,4 @@ pub unsafe fn wait_timeout(&self, mutex: &Mutex, dur: Duration) -> bool {
unsafe { mutex.lock() };
success
}
#[inline]
pub unsafe fn destroy(&self) {}
}

View File

@ -1,11 +1,18 @@
use super::waitqueue::{try_lock_or_false, SpinMutex, WaitQueue, WaitVariable};
use crate::sys_common::lazy_box::{LazyBox, LazyInit};
pub struct Mutex {
inner: SpinMutex<WaitVariable<bool>>,
}
// not movable: see UnsafeList implementation
pub type MovableMutex = Box<Mutex>;
pub(crate) type MovableMutex = LazyBox<Mutex>;
impl LazyInit for Mutex {
fn init() -> Box<Self> {
Box::new(Self::new())
}
}
// Implementation according to “Operating Systems: Three Easy Pieces”, chapter 28
impl Mutex {
@ -52,7 +59,4 @@ pub unsafe fn try_lock(&self) -> bool {
true
}
}
#[inline]
pub unsafe fn destroy(&self) {}
}

View File

@ -2,6 +2,7 @@
mod tests;
use crate::num::NonZeroUsize;
use crate::sys_common::lazy_box::{LazyBox, LazyInit};
use super::waitqueue::{
try_lock_or_false, NotifiedTcs, SpinMutex, SpinMutexGuard, WaitQueue, WaitVariable,
@ -13,7 +14,13 @@ pub struct RwLock {
writer: SpinMutex<WaitVariable<bool>>,
}
pub type MovableRwLock = Box<RwLock>;
pub(crate) type MovableRwLock = LazyBox<RwLock>;
impl LazyInit for RwLock {
fn init() -> Box<Self> {
Box::new(Self::new())
}
}
// Check at compile time that RwLock size matches C definition (see test_c_rwlock_initializer below)
//
@ -168,9 +175,6 @@ unsafe fn unlock(&self) {
unsafe { self.__read_unlock(rguard, wguard) };
}
}
#[inline]
pub unsafe fn destroy(&self) {}
}
// The following functions are needed by libunwind. These symbols are named

View File

@ -82,9 +82,11 @@ pub unsafe fn write_unlock(&self) {
let rwl = self.raw();
expect_success_aborting(unsafe { abi::rwl_unl_rwl(rwl) }, &"rwl_unl_rwl");
}
}
impl Drop for RwLock {
#[inline]
pub unsafe fn destroy(&self) {
fn drop(&mut self) {
if let Some(rwl) = self.rwl.get().map(|x| x.0) {
expect_success_aborting(unsafe { abi::rwl_del_rwl(rwl) }, &"rwl_del_rwl");
}

View File

@ -24,9 +24,6 @@ pub const fn new() -> Self {
#[inline]
pub unsafe fn init(&mut self) {}
#[inline]
pub unsafe fn destroy(&self) {}
#[inline]
pub unsafe fn try_lock(&self) -> bool {
self.futex.compare_exchange(0, 1, Acquire, Relaxed).is_ok()
@ -118,12 +115,6 @@ pub const fn new() -> Self {
Self { futex: AtomicU32::new(0) }
}
#[inline]
pub unsafe fn init(&mut self) {}
#[inline]
pub unsafe fn destroy(&self) {}
// All the memory orderings here are `Relaxed`,
// because synchronization is done by unlocking and locking the mutex.

View File

@ -63,9 +63,6 @@ pub const fn new() -> Self {
Self { state: AtomicU32::new(0), writer_notify: AtomicU32::new(0) }
}
#[inline]
pub unsafe fn destroy(&self) {}
#[inline]
pub unsafe fn try_read(&self) -> bool {
self.state

View File

@ -9,14 +9,14 @@
))] {
mod futex;
mod futex_rwlock;
pub use futex::{Mutex, MovableMutex, Condvar, MovableCondvar};
pub use futex_rwlock::{RwLock, MovableRwLock};
pub(crate) use futex::{Mutex, MovableMutex, MovableCondvar};
pub(crate) use futex_rwlock::{RwLock, MovableRwLock};
} else {
mod pthread_mutex;
mod pthread_rwlock;
mod pthread_condvar;
pub use pthread_mutex::{Mutex, MovableMutex};
pub use pthread_rwlock::{RwLock, MovableRwLock};
pub use pthread_condvar::{Condvar, MovableCondvar};
pub(crate) use pthread_mutex::{Mutex, MovableMutex};
pub(crate) use pthread_rwlock::{RwLock, MovableRwLock};
pub(crate) use pthread_condvar::MovableCondvar;
}
}

View File

@ -1,12 +1,13 @@
use crate::cell::UnsafeCell;
use crate::sys::locks::{pthread_mutex, Mutex};
use crate::sys_common::lazy_box::{LazyBox, LazyInit};
use crate::time::Duration;
pub struct Condvar {
inner: UnsafeCell<libc::pthread_cond_t>,
}
pub type MovableCondvar = Box<Condvar>;
pub(crate) type MovableCondvar = LazyBox<Condvar>;
unsafe impl Send for Condvar {}
unsafe impl Sync for Condvar {}
@ -18,6 +19,14 @@ fn saturating_cast_to_time_t(value: u64) -> libc::time_t {
if value > <libc::time_t>::MAX as u64 { <libc::time_t>::MAX } else { value as libc::time_t }
}
impl LazyInit for Condvar {
fn init() -> Box<Self> {
let mut condvar = Box::new(Self::new());
unsafe { condvar.init() };
condvar
}
}
impl Condvar {
pub const fn new() -> Condvar {
// Might be moved and address is changing it is better to avoid
@ -32,14 +41,14 @@ pub const fn new() -> Condvar {
target_os = "android",
target_os = "redox"
))]
pub unsafe fn init(&mut self) {}
unsafe fn init(&mut self) {}
// NOTE: ESP-IDF's PTHREAD_COND_INITIALIZER support is not released yet
// So on that platform, init() should always be called
// Moreover, that platform does not have pthread_condattr_setclock support,
// hence that initialization should be skipped as well
#[cfg(target_os = "espidf")]
pub unsafe fn init(&mut self) {
unsafe fn init(&mut self) {
let r = libc::pthread_cond_init(self.inner.get(), crate::ptr::null());
assert_eq!(r, 0);
}
@ -52,7 +61,7 @@ pub unsafe fn init(&mut self) {
target_os = "redox",
target_os = "espidf"
)))]
pub unsafe fn init(&mut self) {
unsafe fn init(&mut self) {
use crate::mem::MaybeUninit;
let mut attr = MaybeUninit::<libc::pthread_condattr_t>::uninit();
let r = libc::pthread_condattr_init(attr.as_mut_ptr());
@ -179,14 +188,14 @@ pub unsafe fn wait_timeout(&self, mutex: &Mutex, mut dur: Duration) -> bool {
#[inline]
#[cfg(not(target_os = "dragonfly"))]
pub unsafe fn destroy(&self) {
unsafe fn destroy(&mut self) {
let r = libc::pthread_cond_destroy(self.inner.get());
debug_assert_eq!(r, 0);
}
#[inline]
#[cfg(target_os = "dragonfly")]
pub unsafe fn destroy(&self) {
unsafe fn destroy(&mut self) {
let r = libc::pthread_cond_destroy(self.inner.get());
// On DragonFly pthread_cond_destroy() returns EINVAL if called on
// a condvar that was just initialized with
@ -195,3 +204,10 @@ pub unsafe fn destroy(&self) {
debug_assert!(r == 0 || r == libc::EINVAL);
}
}
impl Drop for Condvar {
#[inline]
fn drop(&mut self) {
unsafe { self.destroy() };
}
}

View File

@ -1,12 +1,13 @@
use crate::cell::UnsafeCell;
use crate::mem::MaybeUninit;
use crate::sys::cvt_nz;
use crate::sys_common::lazy_box::{LazyBox, LazyInit};
pub struct Mutex {
inner: UnsafeCell<libc::pthread_mutex_t>,
}
pub type MovableMutex = Box<Mutex>;
pub(crate) type MovableMutex = LazyBox<Mutex>;
#[inline]
pub unsafe fn raw(m: &Mutex) -> *mut libc::pthread_mutex_t {
@ -16,6 +17,14 @@ pub unsafe fn raw(m: &Mutex) -> *mut libc::pthread_mutex_t {
unsafe impl Send for Mutex {}
unsafe impl Sync for Mutex {}
impl LazyInit for Mutex {
fn init() -> Box<Self> {
let mut mutex = Box::new(Self::new());
unsafe { mutex.init() };
mutex
}
}
impl Mutex {
pub const fn new() -> Mutex {
// Might be moved to a different address, so it is better to avoid
@ -73,13 +82,13 @@ pub unsafe fn try_lock(&self) -> bool {
}
#[inline]
#[cfg(not(target_os = "dragonfly"))]
pub unsafe fn destroy(&self) {
unsafe fn destroy(&mut self) {
let r = libc::pthread_mutex_destroy(self.inner.get());
debug_assert_eq!(r, 0);
}
#[inline]
#[cfg(target_os = "dragonfly")]
pub unsafe fn destroy(&self) {
unsafe fn destroy(&mut self) {
let r = libc::pthread_mutex_destroy(self.inner.get());
// On DragonFly pthread_mutex_destroy() returns EINVAL if called on a
// mutex that was just initialized with libc::PTHREAD_MUTEX_INITIALIZER.
@ -89,6 +98,13 @@ pub unsafe fn destroy(&self) {
}
}
impl Drop for Mutex {
#[inline]
fn drop(&mut self) {
unsafe { self.destroy() };
}
}
pub(super) struct PthreadMutexAttr<'a>(pub &'a mut MaybeUninit<libc::pthread_mutexattr_t>);
impl Drop for PthreadMutexAttr<'_> {

View File

@ -1,5 +1,6 @@
use crate::cell::UnsafeCell;
use crate::sync::atomic::{AtomicUsize, Ordering};
use crate::sys_common::lazy_box::{LazyBox, LazyInit};
pub struct RwLock {
inner: UnsafeCell<libc::pthread_rwlock_t>,
@ -7,11 +8,17 @@ pub struct RwLock {
num_readers: AtomicUsize,
}
pub type MovableRwLock = Box<RwLock>;
pub(crate) type MovableRwLock = LazyBox<RwLock>;
unsafe impl Send for RwLock {}
unsafe impl Sync for RwLock {}
impl LazyInit for RwLock {
fn init() -> Box<Self> {
Box::new(Self::new())
}
}
impl RwLock {
pub const fn new() -> RwLock {
RwLock {
@ -128,7 +135,7 @@ pub unsafe fn write_unlock(&self) {
self.raw_unlock();
}
#[inline]
pub unsafe fn destroy(&self) {
unsafe fn destroy(&mut self) {
let r = libc::pthread_rwlock_destroy(self.inner.get());
// On DragonFly pthread_rwlock_destroy() returns EINVAL if called on a
// rwlock that was just initialized with
@ -141,3 +148,10 @@ pub unsafe fn destroy(&self) {
}
}
}
impl Drop for RwLock {
#[inline]
fn drop(&mut self) {
unsafe { self.destroy() };
}
}

View File

@ -10,9 +10,6 @@ pub const fn new() -> Condvar {
Condvar {}
}
#[inline]
pub unsafe fn init(&mut self) {}
#[inline]
pub unsafe fn notify_one(&self) {}
@ -26,7 +23,4 @@ pub unsafe fn wait(&self, _mutex: &Mutex) {
pub unsafe fn wait_timeout(&self, _mutex: &Mutex, _dur: Duration) -> bool {
panic!("condvar wait not supported");
}
#[inline]
pub unsafe fn destroy(&self) {}
}

View File

@ -32,7 +32,4 @@ pub unsafe fn unlock(&self) {
pub unsafe fn try_lock(&self) -> bool {
self.locked.replace(true) == false
}
#[inline]
pub unsafe fn destroy(&self) {}
}

View File

@ -62,7 +62,4 @@ pub unsafe fn read_unlock(&self) {
pub unsafe fn write_unlock(&self) {
assert_eq!(self.mode.replace(0), -1);
}
#[inline]
pub unsafe fn destroy(&self) {}
}

View File

@ -54,8 +54,8 @@ pub mod locks {
#![allow(unsafe_op_in_unsafe_fn)]
mod futex;
mod futex_rwlock;
pub use futex::{Mutex, MovableMutex, Condvar, MovableCondvar};
pub use futex_rwlock::{RwLock, MovableRwLock};
pub(crate) use futex::{Mutex, MovableMutex, Condvar, MovableCondvar};
pub(crate) use futex_rwlock::{RwLock, MovableRwLock};
}
#[path = "atomics/futex.rs"]
pub mod futex;

View File

@ -18,9 +18,6 @@ pub const fn new() -> Condvar {
Condvar { inner: UnsafeCell::new(c::CONDITION_VARIABLE_INIT) }
}
#[inline]
pub unsafe fn init(&mut self) {}
#[inline]
pub unsafe fn wait(&self, mutex: &Mutex) {
let r = c::SleepConditionVariableSRW(self.inner.get(), mutex::raw(mutex), c::INFINITE, 0);
@ -51,8 +48,4 @@ pub unsafe fn notify_one(&self) {
pub unsafe fn notify_all(&self) {
c::WakeAllConditionVariable(self.inner.get())
}
pub unsafe fn destroy(&self) {
// ...
}
}

View File

@ -53,9 +53,4 @@ pub unsafe fn try_lock(&self) -> bool {
pub unsafe fn unlock(&self) {
c::ReleaseSRWLockExclusive(raw(self));
}
#[inline]
pub unsafe fn destroy(&self) {
// SRWLock does not need to be destroyed.
}
}

View File

@ -38,9 +38,4 @@ pub unsafe fn read_unlock(&self) {
pub unsafe fn write_unlock(&self) {
c::ReleaseSRWLockExclusive(self.inner.get())
}
#[inline]
pub unsafe fn destroy(&self) {
// ...
}
}

View File

@ -15,9 +15,7 @@ pub struct Condvar {
impl Condvar {
/// Creates a new condition variable for use.
pub fn new() -> Self {
let mut c = imp::MovableCondvar::from(imp::Condvar::new());
unsafe { c.init() };
Self { inner: c, check: CondvarCheck::new() }
Self { inner: imp::MovableCondvar::new(), check: CondvarCheck::new() }
}
/// Signals one waiter on this condition variable to wake up.
@ -55,9 +53,3 @@ pub unsafe fn wait_timeout(&self, mutex: &MovableMutex, dur: Duration) -> bool {
self.inner.wait_timeout(mutex.raw(), dur)
}
}
impl Drop for Condvar {
fn drop(&mut self) {
unsafe { self.inner.destroy() };
}
}

View File

@ -1,6 +1,7 @@
use crate::ptr;
use crate::sync::atomic::{AtomicPtr, Ordering};
use crate::sys::locks as imp;
use crate::sys_common::lazy_box::{LazyBox, LazyInit};
use crate::sys_common::mutex::MovableMutex;
pub trait CondvarCheck {
@ -9,7 +10,7 @@ pub trait CondvarCheck {
/// For boxed mutexes, a `Condvar` will check it's only ever used with the same
/// mutex, based on its (stable) address.
impl CondvarCheck for Box<imp::Mutex> {
impl<T: LazyInit> CondvarCheck for LazyBox<T> {
type Check = SameMutexCheck;
}

View File

@ -0,0 +1,77 @@
#![allow(dead_code)] // Only used on some platforms.
// This is used to wrap pthread {Mutex, Condvar, RwLock} in.
use crate::marker::PhantomData;
use crate::ops::{Deref, DerefMut};
use crate::ptr::null_mut;
use crate::sync::atomic::{
AtomicPtr,
Ordering::{AcqRel, Acquire},
};
pub(crate) struct LazyBox<T: LazyInit> {
ptr: AtomicPtr<T>,
_phantom: PhantomData<T>,
}
pub(crate) trait LazyInit {
/// This is called before the box is allocated, to provide the value to
/// move into the new box.
///
/// It might be called more than once per LazyBox, as multiple threads
/// might race to initialize it concurrently, each constructing and initializing
/// their own box. (All but one of them will be destroyed right after.)
fn init() -> Box<Self>;
}
impl<T: LazyInit> LazyBox<T> {
#[inline]
pub const fn new() -> Self {
Self { ptr: AtomicPtr::new(null_mut()), _phantom: PhantomData }
}
#[inline]
fn get_pointer(&self) -> *mut T {
let ptr = self.ptr.load(Acquire);
if ptr.is_null() { self.initialize() } else { ptr }
}
#[cold]
fn initialize(&self) -> *mut T {
let new_ptr = Box::into_raw(T::init());
match self.ptr.compare_exchange(null_mut(), new_ptr, AcqRel, Acquire) {
Ok(_) => new_ptr,
Err(ptr) => {
// Lost the race to another thread.
// Drop the box we created, and use the one from the other thread instead.
drop(unsafe { Box::from_raw(new_ptr) });
ptr
}
}
}
}
impl<T: LazyInit> Deref for LazyBox<T> {
type Target = T;
#[inline]
fn deref(&self) -> &T {
unsafe { &*self.get_pointer() }
}
}
impl<T: LazyInit> DerefMut for LazyBox<T> {
#[inline]
fn deref_mut(&mut self) -> &mut T {
unsafe { &mut *self.get_pointer() }
}
}
impl<T: LazyInit> Drop for LazyBox<T> {
fn drop(&mut self) {
let ptr = *self.ptr.get_mut();
if !ptr.is_null() {
drop(unsafe { Box::from_raw(ptr) });
}
}
}

View File

@ -24,6 +24,7 @@
pub mod condvar;
pub mod fs;
pub mod io;
pub mod lazy_box;
pub mod memchr;
pub mod mutex;
pub mod process;

View File

@ -61,9 +61,7 @@ unsafe impl Sync for MovableMutex {}
impl MovableMutex {
/// Creates a new mutex.
pub fn new() -> Self {
let mut mutex = imp::MovableMutex::from(imp::Mutex::new());
unsafe { mutex.init() };
Self(mutex)
Self(imp::MovableMutex::new())
}
pub(super) fn raw(&self) -> &imp::Mutex {
@ -92,9 +90,3 @@ pub unsafe fn raw_unlock(&self) {
self.0.unlock()
}
}
impl Drop for MovableMutex {
fn drop(&mut self) {
unsafe { self.0.destroy() };
}
}

View File

@ -168,13 +168,6 @@ unsafe fn increment_lock_count(&self) {
}
}
impl<T> Drop for ReentrantMutex<T> {
fn drop(&mut self) {
// Safety: We're the unique owner of this mutex and not going to use it afterwards.
unsafe { self.mutex.destroy() }
}
}
impl<T> Deref for ReentrantMutexGuard<'_, T> {
type Target = T;

View File

@ -74,7 +74,7 @@ fn drop(&mut self) {
impl MovableRwLock {
/// Creates a new reader-writer lock for use.
pub fn new() -> Self {
Self(imp::MovableRwLock::from(imp::RwLock::new()))
Self(imp::MovableRwLock::new())
}
/// Acquires shared access to the underlying lock, blocking the current
@ -126,9 +126,3 @@ pub unsafe fn write_unlock(&self) {
self.0.write_unlock()
}
}
impl Drop for MovableRwLock {
fn drop(&mut self) {
unsafe { self.0.destroy() };
}
}