290 lines
10 KiB
Rust
290 lines
10 KiB
Rust
use super::key::{Key, LazyKey, get, set};
|
|
use super::{abort_on_dtor_unwind, guard};
|
|
use crate::alloc::{self, GlobalAlloc, Layout, System};
|
|
use crate::cell::Cell;
|
|
use crate::marker::PhantomData;
|
|
use crate::mem::ManuallyDrop;
|
|
use crate::ops::Deref;
|
|
use crate::panic::{AssertUnwindSafe, catch_unwind, resume_unwind};
|
|
use crate::ptr::{self, NonNull};
|
|
|
|
#[doc(hidden)]
|
|
#[allow_internal_unstable(thread_local_internals)]
|
|
#[allow_internal_unsafe]
|
|
#[unstable(feature = "thread_local_internals", issue = "none")]
|
|
#[rustc_macro_transparency = "semiopaque"]
|
|
pub macro thread_local_inner {
|
|
// NOTE: we cannot import `Storage` or `LocalKey` with a `use` because that can shadow user
|
|
// provided type or type alias with a matching name. Please update the shadowing test in
|
|
// `tests/thread.rs` if these types are renamed.
|
|
|
|
// used to generate the `LocalKey` value for `thread_local!`.
|
|
(@key $t:ty, $($(#[$($align_attr:tt)*])+)?, $init:expr) => {{
|
|
#[inline]
|
|
fn __rust_std_internal_init_fn() -> $t { $init }
|
|
|
|
// NOTE: this cannot import `LocalKey` or `Storage` with a `use` because that can shadow
|
|
// user provided type or type alias with a matching name. Please update the shadowing test
|
|
// in `tests/thread.rs` if these types are renamed.
|
|
unsafe {
|
|
$crate::thread::LocalKey::new(|__rust_std_internal_init| {
|
|
static __RUST_STD_INTERNAL_VAL: $crate::thread::local_impl::Storage<$t, {
|
|
$({
|
|
// Ensure that attributes have valid syntax
|
|
// and that the proper feature gate is enabled
|
|
$(#[$($align_attr)*])+
|
|
#[allow(unused)]
|
|
static DUMMY: () = ();
|
|
})?
|
|
|
|
#[allow(unused_mut)]
|
|
let mut final_align = $crate::thread::local_impl::value_align::<$t>();
|
|
$($($crate::thread::local_impl::thread_local_inner!(@align final_align, $($align_attr)*);)+)?
|
|
final_align
|
|
}>
|
|
= $crate::thread::local_impl::Storage::new();
|
|
__RUST_STD_INTERNAL_VAL.get(__rust_std_internal_init, __rust_std_internal_init_fn)
|
|
})
|
|
}
|
|
}},
|
|
|
|
// process a single `rustc_align_static` attribute
|
|
(@align $final_align:ident, rustc_align_static($($align:tt)*) $(, $($attr_rest:tt)+)?) => {
|
|
let new_align: $crate::primitive::usize = $($align)*;
|
|
if new_align > $final_align {
|
|
$final_align = new_align;
|
|
}
|
|
|
|
$($crate::thread::local_impl::thread_local_inner!(@align $final_align, $($attr_rest)+);)?
|
|
},
|
|
|
|
// process a single `cfg_attr` attribute
|
|
// by translating it into a `cfg`ed block and recursing.
|
|
// https://doc.rust-lang.org/reference/conditional-compilation.html#railroad-ConfigurationPredicate
|
|
|
|
(@align $final_align:ident, cfg_attr(true, $($cfg_rhs:tt)*) $(, $($attr_rest:tt)+)?) => {
|
|
#[cfg(true)]
|
|
{
|
|
$crate::thread::local_impl::thread_local_inner!(@align $final_align, $($cfg_rhs)*);
|
|
}
|
|
|
|
$($crate::thread::local_impl::thread_local_inner!(@align $final_align, $($attr_rest)+);)?
|
|
},
|
|
|
|
(@align $final_align:ident, cfg_attr(false, $($cfg_rhs:tt)*) $(, $($attr_rest:tt)+)?) => {
|
|
#[cfg(false)]
|
|
{
|
|
$crate::thread::local_impl::thread_local_inner!(@align $final_align, $($cfg_rhs)*);
|
|
}
|
|
|
|
$($crate::thread::local_impl::thread_local_inner!(@align $final_align, $($attr_rest)+);)?
|
|
},
|
|
|
|
(@align $final_align:ident, cfg_attr($cfg_pred:meta, $($cfg_rhs:tt)*) $(, $($attr_rest:tt)+)?) => {
|
|
#[cfg($cfg_pred)]
|
|
{
|
|
$crate::thread::local_impl::thread_local_inner!(@align $final_align, $($cfg_rhs)*);
|
|
}
|
|
|
|
$($crate::thread::local_impl::thread_local_inner!(@align $final_align, $($attr_rest)+);)?
|
|
},
|
|
}
|
|
|
|
/// Use a regular global static to store this key; the state provided will then be
|
|
/// thread-local.
|
|
/// INVARIANT: ALIGN must be a valid alignment, and no less than `value_align::<T>`.
|
|
#[allow(missing_debug_implementations)]
|
|
pub struct Storage<T, const ALIGN: usize> {
|
|
key: LazyKey,
|
|
marker: PhantomData<Cell<T>>,
|
|
}
|
|
|
|
unsafe impl<T, const ALIGN: usize> Sync for Storage<T, ALIGN> {}
|
|
|
|
#[repr(C)]
|
|
struct Value<T: 'static> {
|
|
// This field must be first, for correctness of `#[rustc_align_static]`
|
|
value: T,
|
|
// INVARIANT: if this value is stored under a TLS key, `key` must be that `key`.
|
|
key: Key,
|
|
}
|
|
|
|
pub const fn value_align<T: 'static>() -> usize {
|
|
crate::mem::align_of::<Value<T>>()
|
|
}
|
|
|
|
/// Equivalent to `Box<Value<T>, System>`, but potentially over-aligned.
|
|
struct AlignedSystemBox<T: 'static, const ALIGN: usize> {
|
|
ptr: NonNull<Value<T>>,
|
|
}
|
|
|
|
impl<T: 'static, const ALIGN: usize> AlignedSystemBox<T, ALIGN> {
|
|
#[inline]
|
|
fn new(v: Value<T>) -> Self {
|
|
let layout = Layout::new::<Value<T>>().align_to(ALIGN).unwrap();
|
|
|
|
// We use the System allocator here to avoid interfering with a potential
|
|
// Global allocator using thread-local storage.
|
|
let ptr: *mut Value<T> = (unsafe { System.alloc(layout) }).cast();
|
|
let Some(ptr) = NonNull::new(ptr) else {
|
|
alloc::handle_alloc_error(layout);
|
|
};
|
|
unsafe { ptr.write(v) };
|
|
Self { ptr }
|
|
}
|
|
|
|
#[inline]
|
|
fn into_raw(b: Self) -> *mut Value<T> {
|
|
let md = ManuallyDrop::new(b);
|
|
md.ptr.as_ptr()
|
|
}
|
|
|
|
#[inline]
|
|
unsafe fn from_raw(ptr: *mut Value<T>) -> Self {
|
|
Self { ptr: unsafe { NonNull::new_unchecked(ptr) } }
|
|
}
|
|
}
|
|
|
|
impl<T: 'static, const ALIGN: usize> Deref for AlignedSystemBox<T, ALIGN> {
|
|
type Target = Value<T>;
|
|
|
|
#[inline]
|
|
fn deref(&self) -> &Self::Target {
|
|
unsafe { &*(self.ptr.as_ptr()) }
|
|
}
|
|
}
|
|
|
|
impl<T: 'static, const ALIGN: usize> Drop for AlignedSystemBox<T, ALIGN> {
|
|
#[inline]
|
|
fn drop(&mut self) {
|
|
let layout = Layout::new::<Value<T>>().align_to(ALIGN).unwrap();
|
|
|
|
unsafe {
|
|
let unwind_result = catch_unwind(AssertUnwindSafe(|| self.ptr.drop_in_place()));
|
|
System.dealloc(self.ptr.as_ptr().cast(), layout);
|
|
if let Err(payload) = unwind_result {
|
|
resume_unwind(payload);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
impl<T: 'static, const ALIGN: usize> Storage<T, ALIGN> {
|
|
pub const fn new() -> Storage<T, ALIGN> {
|
|
Storage { key: LazyKey::new(Some(destroy_value::<T, ALIGN>)), marker: PhantomData }
|
|
}
|
|
|
|
/// Gets a pointer to the TLS value, potentially initializing it with the
|
|
/// provided parameters. If the TLS variable has been destroyed, a null
|
|
/// pointer is returned.
|
|
///
|
|
/// The resulting pointer may not be used after reentrant inialialization
|
|
/// or thread destruction has occurred.
|
|
pub fn get(&'static self, i: Option<&mut Option<T>>, f: impl FnOnce() -> T) -> *const T {
|
|
let key = self.key.force();
|
|
let ptr = unsafe { get(key) as *mut Value<T> };
|
|
if ptr.addr() > 1 {
|
|
// SAFETY: the check ensured the pointer is safe (its destructor
|
|
// is not running) + it is coming from a trusted source (self).
|
|
unsafe { &(*ptr).value }
|
|
} else {
|
|
// SAFETY: trivially correct.
|
|
unsafe { Self::try_initialize(key, ptr, i, f) }
|
|
}
|
|
}
|
|
|
|
/// # Safety
|
|
/// * `key` must be the result of calling `self.key.force()`
|
|
/// * `ptr` must be the current value associated with `key`.
|
|
unsafe fn try_initialize(
|
|
key: Key,
|
|
ptr: *mut Value<T>,
|
|
i: Option<&mut Option<T>>,
|
|
f: impl FnOnce() -> T,
|
|
) -> *const T {
|
|
if ptr.addr() == 1 {
|
|
// destructor is running
|
|
return ptr::null();
|
|
}
|
|
|
|
let value = AlignedSystemBox::<T, ALIGN>::new(Value {
|
|
value: i.and_then(Option::take).unwrap_or_else(f),
|
|
key,
|
|
});
|
|
let ptr = AlignedSystemBox::into_raw(value);
|
|
|
|
// SAFETY:
|
|
// * key came from a `LazyKey` and is thus correct.
|
|
// * `ptr` is a correct pointer that can be destroyed by the key destructor.
|
|
// * the value is stored under the key that it contains.
|
|
let old = unsafe {
|
|
let old = get(key) as *mut Value<T>;
|
|
set(key, ptr as *mut u8);
|
|
old
|
|
};
|
|
|
|
if !old.is_null() {
|
|
// If the variable was recursively initialized, drop the old value.
|
|
// SAFETY: We cannot be inside a `LocalKey::with` scope, as the
|
|
// initializer has already returned and the next scope only starts
|
|
// after we return the pointer. Therefore, there can be no references
|
|
// to the old value.
|
|
drop(unsafe { AlignedSystemBox::<T, ALIGN>::from_raw(old) });
|
|
}
|
|
|
|
// SAFETY: We just created this value above.
|
|
unsafe { &(*ptr).value }
|
|
}
|
|
}
|
|
|
|
unsafe extern "C" fn destroy_value<T: 'static, const ALIGN: usize>(ptr: *mut u8) {
|
|
// SAFETY:
|
|
//
|
|
// The OS TLS ensures that this key contains a null value when this
|
|
// destructor starts to run. We set it back to a sentinel value of 1 to
|
|
// ensure that any future calls to `get` for this thread will return
|
|
// `None`.
|
|
//
|
|
// Note that to prevent an infinite loop we reset it back to null right
|
|
// before we return from the destructor ourselves.
|
|
abort_on_dtor_unwind(|| {
|
|
let ptr = unsafe { AlignedSystemBox::<T, ALIGN>::from_raw(ptr as *mut Value<T>) };
|
|
let key = ptr.key;
|
|
// SAFETY: `key` is the TLS key `ptr` was stored under.
|
|
unsafe { set(key, ptr::without_provenance_mut(1)) };
|
|
drop(ptr);
|
|
// SAFETY: `key` is the TLS key `ptr` was stored under.
|
|
unsafe { set(key, ptr::null_mut()) };
|
|
// Make sure that the runtime cleanup will be performed
|
|
// after the next round of TLS destruction.
|
|
guard::enable();
|
|
});
|
|
}
|
|
|
|
#[rustc_macro_transparency = "semiopaque"]
|
|
pub(crate) macro local_pointer {
|
|
() => {},
|
|
($vis:vis static $name:ident; $($rest:tt)*) => {
|
|
$vis static $name: $crate::sys::thread_local::LocalPointer = $crate::sys::thread_local::LocalPointer::__new();
|
|
$crate::sys::thread_local::local_pointer! { $($rest)* }
|
|
},
|
|
}
|
|
|
|
pub(crate) struct LocalPointer {
|
|
key: LazyKey,
|
|
}
|
|
|
|
impl LocalPointer {
|
|
pub const fn __new() -> LocalPointer {
|
|
LocalPointer { key: LazyKey::new(None) }
|
|
}
|
|
|
|
pub fn get(&'static self) -> *mut () {
|
|
unsafe { get(self.key.force()) as *mut () }
|
|
}
|
|
|
|
pub fn set(&'static self, p: *mut ()) {
|
|
unsafe { set(self.key.force(), p as *mut u8) }
|
|
}
|
|
}
|