rust/kernel/sync/barrier.rs

// SPDX-License-Identifier: GPL-2.0

//! Memory barriers.
//!
//! These primitives have the same semantics as their C counterparts: and the precise definitions
//! of semantics can be found at [`LKMM`].
//!
//! [`LKMM`]: srctree/tools/memory-model/

#![expect(private_bounds, reason = "sealed implementation")]

/// Memory barrier orderings.
///
/// The semantics of these orderings follows the [`LKMM`] definitions and rules.
///
/// - [`Read`] provides ordering between preceding load operations and succeeding load operations.
/// - [`Write`] provides ordering between preceding store operations and succeeding store
///   operations.
/// - [`Full`] provides ordering between all the preceding memory accesses and succeeding memory
///   accesses.
///
/// [`LKMM`]: srctree/tools/memory-model/
pub mod ordering {
    pub use crate::sync::atomic::ordering::Full;

    /// The annotation type for read-read barrier ordering.
    pub struct Read;

    /// The annotation type for write-write barrier ordering.
    pub struct Write;
}

pub use ordering::{
    Full,
    Read,
    Write, //
};

struct Smp;
struct Dma;

/// A compiler barrier.
///
/// A barrier that prevents compiler from reordering memory accesses across the barrier.
#[inline(always)]
pub(crate) fn barrier() {
    // By default, Rust inline asms are treated as being able to access any memory or flags, hence
    // it suffices as a compiler barrier.
    //
    // SAFETY: An empty asm block.
    unsafe { core::arch::asm!("") };
}

trait MemoryBarrier<Flavour = ()> {
    fn run();
}

macro_rules! define_barrier {
    ($([$flavour:ident])? $ordering:ident, $binding:ident) => {
        impl MemoryBarrier$(<$flavour>)? for $ordering {
            #[inline]
            fn run() {
                // SAFETY: barrier methods are safe to call.
                unsafe { bindings::$binding() };
            }
        }
    };
}

define_barrier!(Full, mb);
define_barrier!(Read, rmb);
define_barrier!(Write, wmb);
define_barrier!([Dma] Full, dma_mb);
define_barrier!([Dma] Read, dma_rmb);
define_barrier!([Dma] Write, dma_wmb);
define_barrier!([Smp] Full, smp_mb);
define_barrier!([Smp] Read, smp_rmb);
define_barrier!([Smp] Write, smp_wmb);

/// Memory barrier.
///
/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier.
///
/// The specific forms of reordering can be specified using the parameter.
/// - `mb(Read)` provides a read-read barrier.
/// - `mb(Write)` provides a write-write barrier.
/// - `mb(Full)` provides a full barrier.
///
/// # Examples
///
/// ```
/// # use kernel::sync::barrier::*;
/// mb(Read);
/// mb(Write);
/// mb(Full);
/// ```
#[inline]
#[doc(alias = "rmb")]
#[doc(alias = "wmb")]
pub fn mb<T: MemoryBarrier>(_: T) {
    T::run()
}

/// Memory barrier between CPUs.
///
/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier.
/// Does not prevent re-ordering with respect to other bus-mastering devices.
///
/// See [`mb`] for usage.
#[inline]
#[doc(alias = "smp_rmb")]
#[doc(alias = "smp_wmb")]
pub fn smp_mb<T: MemoryBarrier<Smp>>(_: T) {
    if cfg!(CONFIG_SMP) {
        T::run()
    } else {
        barrier()
    }
}

/// Memory barrier between local CPU and bus-mastering devices.
///
/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier.
/// Does not prevent re-ordering with respect to other CPUs.
///
/// See [`mb`] for usage.
#[inline]
#[doc(alias = "dma_rmb")]
#[doc(alias = "dma_wmb")]
pub fn dma_mb<T: MemoryBarrier<Dma>>(_: T) {
    T::run()
}