Skip to content

Commit

Permalink
AtomicCell: Use atomic-maybe-uninit
Browse files Browse the repository at this point in the history
  • Loading branch information
@taiki-e
taiki-e committed Dec 8, 2024
1 parent aea5fe1 commit 76e2b5a
Show file tree
Hide file tree
Showing 5 changed files with 183 additions and 81 deletions.
1 change: 1 addition & 0 deletions crossbeam-utils/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ default = ["std"]
std = []

[dependencies]
atomic-maybe-uninit = "0.3"

# Enable the use of loom for concurrency testing.
#
Expand Down
10 changes: 6 additions & 4 deletions crossbeam-utils/build.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ include!("build-common.rs");

fn main() {
println!("cargo:rerun-if-changed=no_atomic.rs");
println!("cargo:rustc-check-cfg=cfg(crossbeam_no_atomic,crossbeam_sanitize_thread)");
println!("cargo:rustc-check-cfg=cfg(crossbeam_no_atomic,crossbeam_sanitize_thread,crossbeam_atomic_cell_force_fallback)");

let target = match env::var("TARGET") {
Ok(target) => convert_custom_linux_target(target),
Expand All @@ -39,8 +39,10 @@ fn main() {
}

// `cfg(sanitize = "..")` is not stabilized.
let sanitize = env::var("CARGO_CFG_SANITIZE").unwrap_or_default();
if sanitize.contains("thread") {
println!("cargo:rustc-cfg=crossbeam_sanitize_thread");
if let Ok(sanitize) = env::var("CARGO_CFG_SANITIZE") {
if sanitize.contains("thread") {
println!("cargo:rustc-cfg=crossbeam_sanitize_thread");
}
println!("cargo:rustc-cfg=crossbeam_atomic_cell_force_fallback");
}
}
194 changes: 141 additions & 53 deletions crossbeam-utils/src/atomic/atomic_cell.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,8 @@ impl<T> AtomicCell<T> {
/// # Examples
///
/// ```
/// # // Always use fallback for now on environments that do not support inline assembly.
/// # if cfg!(any(miri, crossbeam_loom, crossbeam_atomic_cell_force_fallback)) { return; }
/// use crossbeam_utils::atomic::AtomicCell;
///
/// // This type is internally represented as `AtomicUsize` so we can just use atomic
Expand Down Expand Up @@ -307,21 +309,37 @@ macro_rules! atomic {
loop {
atomic!(@check, $t, AtomicUnit, $a, $atomic_op);

atomic!(@check, $t, atomic::AtomicU8, $a, $atomic_op);
atomic!(@check, $t, atomic::AtomicU16, $a, $atomic_op);
atomic!(@check, $t, atomic::AtomicU32, $a, $atomic_op);
#[cfg(target_has_atomic = "64")]
atomic!(@check, $t, atomic::AtomicU64, $a, $atomic_op);
// TODO: AtomicU128 is unstable
// atomic!(@check, $t, atomic::AtomicU128, $a, $atomic_op);
// Always use fallback for now on environments that do not support inline assembly.
#[cfg(not(any(
miri,
crossbeam_loom,
crossbeam_atomic_cell_force_fallback,
)))]
{
atomic_maybe_uninit::cfg_has_atomic_8! {
atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit<u8>, $a, $atomic_op);
}
atomic_maybe_uninit::cfg_has_atomic_16! {
atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit<u16>, $a, $atomic_op);
}
atomic_maybe_uninit::cfg_has_atomic_32! {
atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit<u32>, $a, $atomic_op);
}
atomic_maybe_uninit::cfg_has_atomic_64! {
atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit<u64>, $a, $atomic_op);
}
atomic_maybe_uninit::cfg_has_atomic_128! {
atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit<u128>, $a, $atomic_op);
}
}

break $fallback_op;
}
};
}

macro_rules! impl_arithmetic {
($t:ty, fallback, $example:tt) => {
($t:ty, fetch_update, $example:tt) => {
impl AtomicCell<$t> {
/// Increments the current value by `val` and returns the previous value.
///
Expand All @@ -339,11 +357,19 @@ macro_rules! impl_arithmetic {
/// ```
#[inline]
pub fn fetch_add(&self, val: $t) -> $t {
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = value.wrapping_add(val);
old
atomic! {
$t, _a,
{
self.fetch_update(|old| Some(old.wrapping_add(val))).unwrap()
},
{
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = value.wrapping_add(val);
old
}
}
}

/// Decrements the current value by `val` and returns the previous value.
Expand All @@ -362,11 +388,19 @@ macro_rules! impl_arithmetic {
/// ```
#[inline]
pub fn fetch_sub(&self, val: $t) -> $t {
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = value.wrapping_sub(val);
old
atomic! {
$t, _a,
{
self.fetch_update(|old| Some(old.wrapping_sub(val))).unwrap()
},
{
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = value.wrapping_sub(val);
old
}
}
}

/// Applies bitwise "and" to the current value and returns the previous value.
Expand All @@ -383,11 +417,19 @@ macro_rules! impl_arithmetic {
/// ```
#[inline]
pub fn fetch_and(&self, val: $t) -> $t {
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value &= val;
old
atomic! {
$t, _a,
{
self.fetch_update(|old| Some(old & val)).unwrap()
},
{
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value &= val;
old
}
}
}

/// Applies bitwise "nand" to the current value and returns the previous value.
Expand All @@ -404,11 +446,19 @@ macro_rules! impl_arithmetic {
/// ```
#[inline]
pub fn fetch_nand(&self, val: $t) -> $t {
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = !(old & val);
old
atomic! {
$t, _a,
{
self.fetch_update(|old| Some(!(old & val))).unwrap()
},
{
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = !(old & val);
old
}
}
}

/// Applies bitwise "or" to the current value and returns the previous value.
Expand All @@ -425,11 +475,19 @@ macro_rules! impl_arithmetic {
/// ```
#[inline]
pub fn fetch_or(&self, val: $t) -> $t {
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value |= val;
old
atomic! {
$t, _a,
{
self.fetch_update(|old| Some(old | val)).unwrap()
},
{
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value |= val;
old
}
}
}

/// Applies bitwise "xor" to the current value and returns the previous value.
Expand All @@ -446,11 +504,19 @@ macro_rules! impl_arithmetic {
/// ```
#[inline]
pub fn fetch_xor(&self, val: $t) -> $t {
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value ^= val;
old
atomic! {
$t, _a,
{
self.fetch_update(|old| Some(old ^ val)).unwrap()
},
{
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value ^= val;
old
}
}
}

/// Compares and sets the maximum of the current value and `val`,
Expand All @@ -468,11 +534,19 @@ macro_rules! impl_arithmetic {
/// ```
#[inline]
pub fn fetch_max(&self, val: $t) -> $t {
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = cmp::max(old, val);
old
atomic! {
$t, _a,
{
self.fetch_update(|old| Some(cmp::max(old, val))).unwrap()
},
{
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = cmp::max(old, val);
old
}
}
}

/// Compares and sets the minimum of the current value and `val`,
Expand All @@ -490,11 +564,19 @@ macro_rules! impl_arithmetic {
/// ```
#[inline]
pub fn fetch_min(&self, val: $t) -> $t {
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = cmp::min(old, val);
old
atomic! {
$t, _a,
{
self.fetch_update(|old| Some(cmp::min(old, val))).unwrap()
},
{
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = cmp::min(old, val);
old
}
}
}
}
};
Expand Down Expand Up @@ -754,23 +836,29 @@ impl_arithmetic!(i8, AtomicI8, "let a = AtomicCell::new(7i8);");
impl_arithmetic!(u16, AtomicU16, "let a = AtomicCell::new(7u16);");
impl_arithmetic!(i16, AtomicI16, "let a = AtomicCell::new(7i16);");

#[cfg(target_has_atomic = "32")]
impl_arithmetic!(u32, AtomicU32, "let a = AtomicCell::new(7u32);");
#[cfg(target_has_atomic = "32")]
impl_arithmetic!(i32, AtomicI32, "let a = AtomicCell::new(7i32);");
#[cfg(not(target_has_atomic = "32"))]
impl_arithmetic!(u32, fetch_update, "let a = AtomicCell::new(7u32);");
#[cfg(not(target_has_atomic = "32"))]
impl_arithmetic!(i32, fetch_update, "let a = AtomicCell::new(7i32);");

#[cfg(target_has_atomic = "64")]
impl_arithmetic!(u64, AtomicU64, "let a = AtomicCell::new(7u64);");
#[cfg(target_has_atomic = "64")]
impl_arithmetic!(i64, AtomicI64, "let a = AtomicCell::new(7i64);");
#[cfg(not(target_has_atomic = "64"))]
impl_arithmetic!(u64, fallback, "let a = AtomicCell::new(7u64);");
impl_arithmetic!(u64, fetch_update, "let a = AtomicCell::new(7u64);");
#[cfg(not(target_has_atomic = "64"))]
impl_arithmetic!(i64, fallback, "let a = AtomicCell::new(7i64);");
impl_arithmetic!(i64, fetch_update, "let a = AtomicCell::new(7i64);");

// TODO: AtomicU128 is unstable
// TODO: core::sync::atomic::AtomicU128 is unstable
// impl_arithmetic!(u128, AtomicU128, "let a = AtomicCell::new(7u128);");
// impl_arithmetic!(i128, AtomicI128, "let a = AtomicCell::new(7i128);");
impl_arithmetic!(u128, fallback, "let a = AtomicCell::new(7u128);");
impl_arithmetic!(i128, fallback, "let a = AtomicCell::new(7i128);");
impl_arithmetic!(u128, fetch_update, "let a = AtomicCell::new(7u128);");
impl_arithmetic!(i128, fetch_update, "let a = AtomicCell::new(7i128);");

impl_arithmetic!(usize, AtomicUsize, "let a = AtomicCell::new(7usize);");
impl_arithmetic!(isize, AtomicIsize, "let a = AtomicCell::new(7isize);");
Expand Down
12 changes: 5 additions & 7 deletions crossbeam-utils/src/atomic/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,12 @@
//! * [`AtomicCell`], a thread-safe mutable memory location.
//! * [`AtomicConsume`], for reading from primitive atomic types with "consume" ordering.
// We cannot provide AtomicCell under cfg(crossbeam_loom) because loom's atomic
// types have a different in-memory representation than the underlying type.
// TODO: The latest loom supports fences, so fallback using seqlock may be available.
#[cfg(target_has_atomic = "ptr")]
#[cfg(not(crossbeam_loom))]
atomic_maybe_uninit::cfg_has_atomic_cas! {
// Use "wide" sequence lock if the pointer width <= 32 for preventing its counter against wrap
// around.
//
Expand All @@ -18,15 +22,9 @@
)]
mod seq_lock;

#[cfg(target_has_atomic = "ptr")]
// We cannot provide AtomicCell under cfg(crossbeam_loom) because loom's atomic
// types have a different in-memory representation than the underlying type.
// TODO: The latest loom supports fences, so fallback using seqlock may be available.
#[cfg(not(crossbeam_loom))]
mod atomic_cell;
#[cfg(target_has_atomic = "ptr")]
#[cfg(not(crossbeam_loom))]
pub use atomic_cell::AtomicCell;
}

mod consume;
pub use consume::AtomicConsume;
Loading

0 comments on commit 76e2b5a

Please sign in to comment.