std: simplify key-based thread locals

library/std/src/sys/thread_local/mod.rs

@@ -24,89 +24,6 @@
     }
 }
 
-// Not used by the fast-local TLS anymore.
-// FIXME(#110897): remove this.
-#[allow(unused)]
-mod lazy {
-    use crate::cell::UnsafeCell;
-    use crate::hint;
-    use crate::mem;
-
-    pub struct LazyKeyInner<T> {
-        inner: UnsafeCell<Option<T>>,
-    }
-
-    impl<T> LazyKeyInner<T> {
-        pub const fn new() -> LazyKeyInner<T> {
-            LazyKeyInner { inner: UnsafeCell::new(None) }
-        }
-
-        pub unsafe fn get(&self) -> Option<&'static T> {
-            // SAFETY: The caller must ensure no reference is ever handed out to
-            // the inner cell nor mutable reference to the Option<T> inside said
-            // cell. This make it safe to hand a reference, though the lifetime
-            // of 'static is itself unsafe, making the get method unsafe.
-            unsafe { (*self.inner.get()).as_ref() }
-        }
-
-        /// The caller must ensure that no reference is active: this method
-        /// needs unique access.
-        pub unsafe fn initialize<F: FnOnce() -> T>(&self, init: F) -> &'static T {
-            // Execute the initialization up front, *then* move it into our slot,
-            // just in case initialization fails.
-            let value = init();
-            let ptr = self.inner.get();
-
-            // SAFETY:
-            //
-            // note that this can in theory just be `*ptr = Some(value)`, but due to
-            // the compiler will currently codegen that pattern with something like:
-            //
-            //      ptr::drop_in_place(ptr)
-            //      ptr::write(ptr, Some(value))
-            //
-            // Due to this pattern it's possible for the destructor of the value in
-            // `ptr` (e.g., if this is being recursively initialized) to re-access
-            // TLS, in which case there will be a `&` and `&mut` pointer to the same
-            // value (an aliasing violation). To avoid setting the "I'm running a
-            // destructor" flag we just use `mem::replace` which should sequence the
-            // operations a little differently and make this safe to call.
-            //
-            // The precondition also ensures that we are the only one accessing
-            // `self` at the moment so replacing is fine.
-            unsafe {
-                let _ = mem::replace(&mut *ptr, Some(value));
-            }
-
-            // SAFETY: With the call to `mem::replace` it is guaranteed there is
-            // a `Some` behind `ptr`, not a `None` so `unreachable_unchecked`
-            // will never be reached.
-            unsafe {
-                // After storing `Some` we want to get a reference to the contents of
-                // what we just stored. While we could use `unwrap` here and it should
-                // always work it empirically doesn't seem to always get optimized away,
-                // which means that using something like `try_with` can pull in
-                // panicking code and cause a large size bloat.
-                match *ptr {
-                    Some(ref x) => x,
-                    None => hint::unreachable_unchecked(),
-                }
-            }
-        }
-
-        /// Watch out: unsynchronized internal mutability!
-        ///
-        /// # Safety
-        /// Causes UB if any reference to the value is used after this.
-        #[allow(unused)]
-        pub(crate) unsafe fn take(&self) -> Option<T> {
-            let mutable: *mut _ = UnsafeCell::get(&self.inner);
-            // SAFETY: That's the caller's problem.
-            unsafe { mutable.replace(None) }
-        }
-    }
-}
-
 /// Run a callback in a scenario which must not unwind (such as a `extern "C"
 /// fn` declared in a user crate). If the callback unwinds anyway, then
 /// `rtabort` with a message about thread local panicking on drop.

library/std/src/sys/thread_local/os_local.rs

@@ -1,7 +1,8 @@
-use super::lazy::LazyKeyInner;
+use super::abort_on_dtor_unwind;
 use crate::cell::Cell;
-use crate::sys_common::thread_local_key::StaticKey as OsStaticKey;
+use crate::marker::PhantomData;
-use crate::{fmt, marker, panic, ptr};
+use crate::sys_common::thread_local_key::StaticKey as OsKey;
+use crate::{panic, ptr};
 
 #[doc(hidden)]
 #[allow_internal_unstable(thread_local_internals)]
@@ -10,38 +11,9 @@
 #[rustc_macro_transparency = "semitransparent"]
 pub macro thread_local_inner {
     // used to generate the `LocalKey` value for const-initialized thread locals
-    (@key $t:ty, const $init:expr) => {{
+    (@key $t:ty, const $init:expr) => {
-        #[inline]
+        $crate::thread::local_impl::thread_local_inner!(@key $t, { const INIT_EXPR: $t = $init; INIT_EXPR })
-        #[deny(unsafe_op_in_unsafe_fn)]
+    },
-        unsafe fn __getit(
-            _init: $crate::option::Option<&mut $crate::option::Option<$t>>,
-        ) -> $crate::option::Option<&'static $t> {
-            const INIT_EXPR: $t = $init;
-
-            // On platforms without `#[thread_local]` we fall back to the
-            // same implementation as below for os thread locals.
-            #[inline]
-            const fn __init() -> $t { INIT_EXPR }
-            static __KEY: $crate::thread::local_impl::Key<$t> =
-                $crate::thread::local_impl::Key::new();
-            unsafe {
-                __KEY.get(move || {
-                    if let $crate::option::Option::Some(init) = _init {
-                        if let $crate::option::Option::Some(value) = init.take() {
-                            return value;
-                        } else if $crate::cfg!(debug_assertions) {
-                            $crate::unreachable!("missing initial value");
-                        }
-                    }
-                    __init()
-                })
-            }
-        }
-
-        unsafe {
-            $crate::thread::LocalKey::new(__getit)
-        }
-    }},
 
     // used to generate the `LocalKey` value for `thread_local!`
     (@key $t:ty, $init:expr) => {
@@ -55,20 +27,11 @@ fn __init() -> $t { $init }
             unsafe fn __getit(
                 init: $crate::option::Option<&mut $crate::option::Option<$t>>,
             ) -> $crate::option::Option<&'static $t> {
-                static __KEY: $crate::thread::local_impl::Key<$t> =
+                use $crate::thread::local_impl::Key;
-                    $crate::thread::local_impl::Key::new();
 
+                static __KEY: Key<$t> = Key::new();
                 unsafe {
-                    __KEY.get(move || {
+                    __KEY.get(init, __init)
-                        if let $crate::option::Option::Some(init) = init {
-                            if let $crate::option::Option::Some(value) = init.take() {
-                                return value;
-                            } else if $crate::cfg!(debug_assertions) {
-                                $crate::unreachable!("missing default value");
-                            }
-                        }
-                        __init()
-                    })
                 }
             }
 
@@ -85,78 +48,78 @@ unsafe fn __getit(
 
 /// Use a regular global static to store this key; the state provided will then be
 /// thread-local.
+#[allow(missing_debug_implementations)]
 pub struct Key<T> {
-    // OS-TLS key that we'll use to key off.
+    os: OsKey,
-    os: OsStaticKey,
+    marker: PhantomData<Cell<T>>,
-    marker: marker::PhantomData<Cell<T>>,
-}
-
-impl<T> fmt::Debug for Key<T> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_struct("Key").finish_non_exhaustive()
-    }
 }
 
 unsafe impl<T> Sync for Key<T> {}
 
 struct Value<T: 'static> {
-    inner: LazyKeyInner<T>,
+    value: T,
     key: &'static Key<T>,
 }
 
 impl<T: 'static> Key<T> {
     #[rustc_const_unstable(feature = "thread_local_internals", issue = "none")]
     pub const fn new() -> Key<T> {
-        Key { os: OsStaticKey::new(Some(destroy_value::<T>)), marker: marker::PhantomData }
+        Key { os: OsKey::new(Some(destroy_value::<T>)), marker: PhantomData }
     }
 
-    /// It is a requirement for the caller to ensure that no mutable
+    /// Get the value associated with this key, initializating it if necessary.
-    /// reference is active when this method is called.
+    ///
-    pub unsafe fn get(&'static self, init: impl FnOnce() -> T) -> Option<&'static T> {
+    /// # Safety
-        // SAFETY: See the documentation for this method.
+    /// * the returned reference must not be used after recursive initialization
+    /// or thread destruction occurs.
+    pub unsafe fn get(
+        &'static self,
+        i: Option<&mut Option<T>>,
+        f: impl FnOnce() -> T,
+    ) -> Option<&'static T> {
+        // SAFETY: (FIXME: get should actually be safe)
         let ptr = unsafe { self.os.get() as *mut Value<T> };
         if ptr.addr() > 1 {
             // SAFETY: the check ensured the pointer is safe (its destructor
             // is not running) + it is coming from a trusted source (self).
-            if let Some(ref value) = unsafe { (*ptr).inner.get() } {
+            unsafe { Some(&(*ptr).value) }
-                return Some(value);
+        } else {
-            }
+            // SAFETY: At this point we are sure we have no value and so
+            // initializing (or trying to) is safe.
+            unsafe { self.try_initialize(ptr, i, f) }
         }
-        // SAFETY: At this point we are sure we have no value and so
-        // initializing (or trying to) is safe.
-        unsafe { self.try_initialize(init) }
     }
 
-    // `try_initialize` is only called once per os thread local variable,
+    unsafe fn try_initialize(
-    // except in corner cases where thread_local dtors reference other
+        &'static self,
-    // thread_local's, or it is being recursively initialized.
+        ptr: *mut Value<T>,
-    unsafe fn try_initialize(&'static self, init: impl FnOnce() -> T) -> Option<&'static T> {
+        i: Option<&mut Option<T>>,
-        // SAFETY: No mutable references are ever handed out meaning getting
+        f: impl FnOnce() -> T,
-        // the value is ok.
+    ) -> Option<&'static T> {
-        let ptr = unsafe { self.os.get() as *mut Value<T> };
         if ptr.addr() == 1 {
             // destructor is running
             return None;
         }
 
-        let ptr = if ptr.is_null() {
+        let value = i.and_then(Option::take).unwrap_or_else(f);
-            // If the lookup returned null, we haven't initialized our own
+        let ptr = Box::into_raw(Box::new(Value { value, key: self }));
-            // local copy, so do that now.
+        // SAFETY: (FIXME: get should actually be safe)
-            let ptr = Box::into_raw(Box::new(Value { inner: LazyKeyInner::new(), key: self }));
+        let old = unsafe { self.os.get() as *mut Value<T> };
-            // SAFETY: At this point we are sure there is no value inside
+        // SAFETY: `ptr` is a correct pointer that can be destroyed by the key destructor.
-            // ptr so setting it will not affect anyone else.
+        unsafe {
-            unsafe {
+            self.os.set(ptr as *mut u8);
-                self.os.set(ptr as *mut u8);
+        }
-            }
+        if !old.is_null() {
-            ptr
+            // If the variable was recursively initialized, drop the old value.
-        } else {
+            // SAFETY: We cannot be inside a `LocalKey::with` scope, as the
-            // recursive initialization
+            // initializer has already returned and the next scope only starts
-            ptr
+            // after we return the pointer. Therefore, there can be no references
-        };
+            // to the old value.
+            drop(unsafe { Box::from_raw(old) });
+        }
 
-        // SAFETY: ptr has been ensured as non-NUL just above an so can be
+        // SAFETY: We just created this value above.
-        // dereferenced safely.
+        unsafe { Some(&(*ptr).value) }
-        unsafe { Some((*ptr).inner.initialize(init)) }
     }
 }
 
@@ -170,16 +133,11 @@ unsafe fn try_initialize(&'static self, init: impl FnOnce() -> T) -> Option<&'st
     //
     // Note that to prevent an infinite loop we reset it back to null right
     // before we return from the destructor ourselves.
-    //
+    abort_on_dtor_unwind(|| {
-    // Wrap the call in a catch to ensure unwinding is caught in the event
+        let ptr = unsafe { Box::from_raw(ptr as *mut Value<T>) };
-    // a panic takes place in a destructor.
-    if let Err(_) = panic::catch_unwind(|| unsafe {
-        let ptr = Box::from_raw(ptr as *mut Value<T>);
         let key = ptr.key;
-        key.os.set(ptr::without_provenance_mut(1));
+        unsafe { key.os.set(ptr::without_provenance_mut(1)) };
         drop(ptr);
-        key.os.set(ptr::null_mut());
+        unsafe { key.os.set(ptr::null_mut()) };
-    }) {
+    });
-        rtabort!("thread local panicked on drop");
-    }
 }