RwLock::read for UFT hits, kstats use AtomicU64

crates/illumos-sys-hdrs/src/lib.rs

@@ -12,7 +12,10 @@ pub mod kernel;
 #[cfg(feature = "kernel")]
 pub use kernel::*;
 
+use core::mem::transmute;
 use core::ptr;
+use core::sync::atomic::AtomicU64;
+use core::sync::atomic::Ordering;
 
 // The following are "C type" aliases for native Rust types so that
 // the native illumos structures may be defined almost verbatim to the
@@ -117,8 +120,9 @@ impl kstat_named_t {
         Self::default()
     }
 
+    #[inline]
     pub fn val_u64(&self) -> u64 {
-        unsafe { self.value._u64 }
+        self.value.as_u64().load(Ordering::Relaxed)
     }
 }
 
@@ -144,20 +148,48 @@ pub union KStatNamedValue {
 impl core::ops::AddAssign<u64> for KStatNamedValue {
     #[inline]
     fn add_assign(&mut self, other: u64) {
-        unsafe { self._u64 += other };
+        self.incr_u64(other);
     }
 }
 
 impl core::ops::SubAssign<u64> for KStatNamedValue {
     #[inline]
     fn sub_assign(&mut self, other: u64) {
-        unsafe { self._u64 -= other };
+        self.decr_u64(other);
     }
 }
 
 impl KStatNamedValue {
-    pub fn set_u64(&mut self, val: u64) {
-        self._u64 = val;
+    /// Validates at compile time whether ._u64 can be safely used as
+    /// an AtomicU64.
+    const KSTAT_ATOMIC_U64_SAFE: () = if align_of::<KStatNamedValue>() % 8 == 0
+    {
+    } else {
+        panic!("Platform does not meet u64 8B alignment for AtomicU64");
+    };
+
+    #[inline(always)]
+    #[allow(clippy::let_unit_value)]
+    pub fn as_u64(&self) -> &AtomicU64 {
+        _ = Self::KSTAT_ATOMIC_U64_SAFE;
+        // SAFETY: KStatNamedValue must have 8B alignment on target platform.
+        //         Validated by compile time check in `KSTAT_ATOMIC_U64_SAFE`.
+        unsafe { transmute::<&u64, &AtomicU64>(&self._u64) }
+    }
+
+    #[inline]
+    pub fn set_u64(&self, val: u64) {
+        core::hint::black_box(self.as_u64().store(val, Ordering::Relaxed));
+    }
+
+    #[inline]
+    pub fn incr_u64(&self, val: u64) {
+        core::hint::black_box(self.as_u64().fetch_add(val, Ordering::Relaxed));
+    }
+
+    #[inline]
+    pub fn decr_u64(&self, val: u64) {
+        core::hint::black_box(self.as_u64().fetch_sub(val, Ordering::Relaxed));
     }
 }
 

lib/opte/src/ddi/kstat.rs

@@ -22,6 +22,9 @@ cfg_if! {
             c_void, kstat_t, kstat_create, kstat_delete, kstat_install,
             kstat_named_init, kstat_named_t, KSTAT_STRLEN, KSTAT_TYPE_NAMED,
         };
+    } else {
+        use core::sync::atomic::AtomicU64;
+        use core::sync::atomic::Ordering;
     }
 }
 
@@ -221,35 +224,41 @@ impl KStatU64 {
         Self { inner: kstat_named_t::new() }
     }
 
-    pub fn set(&mut self, val: u64) {
+    pub fn set(&self, val: u64) {
         self.inner.value.set_u64(val);
     }
 
     pub fn val(&self) -> u64 {
         self.inner.val_u64()
     }
+
+    pub fn incr(&self, val: u64) {
+        self.inner.value.incr_u64(val);
+    }
+
+    pub fn decr(&self, val: u64) {
+        self.inner.value.decr_u64(val);
+    }
 }
 
-#[cfg(all(not(feature = "std"), not(test)))]
 impl core::ops::AddAssign<u64> for KStatU64 {
     #[inline]
     fn add_assign(&mut self, other: u64) {
-        self.inner.value += other;
+        self.incr(other);
     }
 }
 
-#[cfg(all(not(feature = "std"), not(test)))]
 impl core::ops::SubAssign<u64> for KStatU64 {
     #[inline]
     fn sub_assign(&mut self, other: u64) {
-        self.inner.value -= other;
+        self.decr(other);
     }
 }
 
 #[cfg(any(feature = "std", test))]
 #[derive(Default)]
 pub struct KStatU64 {
-    value: u64,
+    value: AtomicU64,
 }
 
 #[cfg(any(feature = "std", test))]
@@ -262,26 +271,20 @@ impl KStatU64 {
         Self::default()
     }
 
-    pub fn set(&mut self, val: u64) {
-        self.value = val;
+    pub fn set(&self, val: u64) {
+        self.value.store(val, Ordering::Relaxed);
     }
 
     pub fn val(&self) -> u64 {
-        self.value
+        self.value.load(Ordering::Relaxed)
     }
-}
 
-#[cfg(any(feature = "std", test))]
-impl core::ops::AddAssign<u64> for KStatU64 {
-    fn add_assign(&mut self, other: u64) {
-        self.value += other;
+    pub fn incr(&self, val: u64) {
+        self.value.fetch_add(val, Ordering::Relaxed);
     }
-}
 
-#[cfg(any(feature = "std", test))]
-impl core::ops::SubAssign<u64> for KStatU64 {
-    fn sub_assign(&mut self, other: u64) {
-        self.value -= other;
+    pub fn decr(&self, val: u64) {
+        self.value.fetch_sub(val, Ordering::Relaxed);
     }
 }
 

lib/opte/src/engine/layer.rs

@@ -751,7 +751,7 @@ impl Layer {
 
         // Unwrap: We know this is fine because the stat names are
         // generated from the LayerStats structure.
-        let mut stats = KStatNamed::new(
+        let stats = KStatNamed::new(
             "xde",
             &format!("{}_{}", port, name),
             LayerStats::new(),