rust/kernel/block/mq/request.rs - linux - Git at Google

 // SPDX-License-Identifier: GPL-2.0

 //! This module provides a wrapper for the C `struct request` type.
 //!
 //! C header: [`include/linux/blk-mq.h`](srctree/include/linux/blk-mq.h)

 use crate::{
     bindings,
     block::mq::Operations,
     error::Result,
     types::{ARef, AlwaysRefCounted, Opaque},
 };
 use core::{
     marker::PhantomData,
     ptr::{addr_of_mut, NonNull},
     sync::atomic::{AtomicU64, Ordering},
 };

 /// A wrapper around a blk-mq `struct request`. This represents an IO request.
 ///
 /// # Implementation details
 ///
 /// There are four states for a request that the Rust bindings care about:
 ///
 /// A) Request is owned by block layer (refcount 0)
 /// B) Request is owned by driver but with zero `ARef`s in existence
 ///    (refcount 1)
 /// C) Request is owned by driver with exactly one `ARef` in existence
 ///    (refcount 2)
 /// D) Request is owned by driver with more than one `ARef` in existence
 ///    (refcount > 2)
 ///
 ///
 /// We need to track A and B to ensure we fail tag to request conversions for
 /// requests that are not owned by the driver.
 ///
 /// We need to track C and D to ensure that it is safe to end the request and hand
 /// back ownership to the block layer.
 ///
 /// The states are tracked through the private `refcount` field of
 /// `RequestDataWrapper`. This structure lives in the private data area of the C
 /// `struct request`.
 ///
 /// # Invariants
 ///
 /// * `self.0` is a valid `struct request` created by the C portion of the kernel.
 /// * The private data area associated with this request must be an initialized
 ///   and valid `RequestDataWrapper<T>`.
 /// * `self` is reference counted by atomic modification of
 ///   self.wrapper_ref().refcount().
 ///
 #[repr(transparent)]
 pub struct Request<T: Operations>(Opaque<bindings::request>, PhantomData<T>);

 impl<T: Operations> Request<T> {
     /// Create an `ARef<Request>` from a `struct request` pointer.
     ///
     /// # Safety
     ///
     /// * The caller must own a refcount on `ptr` that is transferred to the
     ///   returned `ARef`.
     /// * The type invariants for `Request` must hold for the pointee of `ptr`.
     pub(crate) unsafe fn aref_from_raw(ptr: *mut bindings::request) -> ARef<Self> {
         // INVARIANT: By the safety requirements of this function, invariants are upheld.
         // SAFETY: By the safety requirement of this function, we own a
         // reference count that we can pass to `ARef`.
         unsafe { ARef::from_raw(NonNull::new_unchecked(ptr as *const Self as *mut Self)) }
     }

     /// Notify the block layer that a request is going to be processed now.
     ///
     /// The block layer uses this hook to do proper initializations such as
     /// starting the timeout timer. It is a requirement that block device
     /// drivers call this function when starting to process a request.
     ///
     /// # Safety
     ///
     /// The caller must have exclusive ownership of `self`, that is
     /// `self.wrapper_ref().refcount() == 2`.
     pub(crate) unsafe fn start_unchecked(this: &ARef<Self>) {
         // SAFETY: By type invariant, `self.0` is a valid `struct request` and
         // we have exclusive access.
         unsafe { bindings::blk_mq_start_request(this.0.get()) };
     }

     /// Try to take exclusive ownership of `this` by dropping the refcount to 0.
     /// This fails if `this` is not the only `ARef` pointing to the underlying
     /// `Request`.
     ///
     /// If the operation is successful, `Ok` is returned with a pointer to the
     /// C `struct request`. If the operation fails, `this` is returned in the
     /// `Err` variant.
     fn try_set_end(this: ARef<Self>) -> Result<*mut bindings::request, ARef<Self>> {
         // We can race with `TagSet::tag_to_rq`
         if let Err(_old) = this.wrapper_ref().refcount().compare_exchange(
             2,
             0,
             Ordering::Relaxed,
             Ordering::Relaxed,
         ) {
             return Err(this);
         }

         let request_ptr = this.0.get();
         core::mem::forget(this);

         Ok(request_ptr)
     }

     /// Notify the block layer that the request has been completed without errors.
     ///
     /// This function will return `Err` if `this` is not the only `ARef`
     /// referencing the request.
     pub fn end_ok(this: ARef<Self>) -> Result<(), ARef<Self>> {
         let request_ptr = Self::try_set_end(this)?;

         // SAFETY: By type invariant, `this.0` was a valid `struct request`. The
         // success of the call to `try_set_end` guarantees that there are no
         // `ARef`s pointing to this request. Therefore it is safe to hand it
         // back to the block layer.
         unsafe { bindings::blk_mq_end_request(request_ptr, bindings::BLK_STS_OK as _) };

         Ok(())
     }

     /// Return a pointer to the `RequestDataWrapper` stored in the private area
     /// of the request structure.
     ///
     /// # Safety
     ///
     /// - `this` must point to a valid allocation of size at least size of
     ///   `Self` plus size of `RequestDataWrapper`.
     pub(crate) unsafe fn wrapper_ptr(this: *mut Self) -> NonNull<RequestDataWrapper> {
         let request_ptr = this.cast::<bindings::request>();
         // SAFETY: By safety requirements for this function, `this` is a
         // valid allocation.
         let wrapper_ptr =
             unsafe { bindings::blk_mq_rq_to_pdu(request_ptr).cast::<RequestDataWrapper>() };
         // SAFETY: By C API contract, wrapper_ptr points to a valid allocation
         // and is not null.
         unsafe { NonNull::new_unchecked(wrapper_ptr) }
     }

     /// Return a reference to the `RequestDataWrapper` stored in the private
     /// area of the request structure.
     pub(crate) fn wrapper_ref(&self) -> &RequestDataWrapper {
         // SAFETY: By type invariant, `self.0` is a valid allocation. Further,
         // the private data associated with this request is initialized and
         // valid. The existence of `&self` guarantees that the private data is
         // valid as a shared reference.
         unsafe { Self::wrapper_ptr(self as *const Self as *mut Self).as_ref() }
     }
 }

 /// A wrapper around data stored in the private area of the C `struct request`.
 pub(crate) struct RequestDataWrapper {
     /// The Rust request refcount has the following states:
     ///
     /// - 0: The request is owned by C block layer.
     /// - 1: The request is owned by Rust abstractions but there are no ARef references to it.
     /// - 2+: There are `ARef` references to the request.
     refcount: AtomicU64,
 }

 impl RequestDataWrapper {
     /// Return a reference to the refcount of the request that is embedding
     /// `self`.
     pub(crate) fn refcount(&self) -> &AtomicU64 {
         &self.refcount
     }

     /// Return a pointer to the refcount of the request that is embedding the
     /// pointee of `this`.
     ///
     /// # Safety
     ///
     /// - `this` must point to a live allocation of at least the size of `Self`.
     pub(crate) unsafe fn refcount_ptr(this: *mut Self) -> *mut AtomicU64 {
         // SAFETY: Because of the safety requirements of this function, the
         // field projection is safe.
         unsafe { addr_of_mut!((*this).refcount) }
     }
 }

 // SAFETY: Exclusive access is thread-safe for `Request`. `Request` has no `&mut
 // self` methods and `&self` methods that mutate `self` are internally
 // synchronized.
 unsafe impl<T: Operations> Send for Request<T> {}

 // SAFETY: Shared access is thread-safe for `Request`. `&self` methods that
 // mutate `self` are internally synchronized`
 unsafe impl<T: Operations> Sync for Request<T> {}

 /// Store the result of `op(target.load())` in target, returning new value of
 /// target.
 fn atomic_relaxed_op_return(target: &AtomicU64, op: impl Fn(u64) -> u64) -> u64 {
     let old = target.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |x| Some(op(x)));

     // SAFETY: Because the operation passed to `fetch_update` above always
     // return `Some`, `old` will always be `Ok`.
     let old = unsafe { old.unwrap_unchecked() };

     op(old)
 }

 /// Store the result of `op(target.load)` in `target` if `target.load() !=
 /// pred`, returning true if the target was updated.
 fn atomic_relaxed_op_unless(target: &AtomicU64, op: impl Fn(u64) -> u64, pred: u64) -> bool {
     target
         .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |x| {
             if x == pred {
                 None
             } else {
                 Some(op(x))
             }
         })
         .is_ok()
 }

 // SAFETY: All instances of `Request<T>` are reference counted. This
 // implementation of `AlwaysRefCounted` ensure that increments to the ref count
 // keeps the object alive in memory at least until a matching reference count
 // decrement is executed.
 unsafe impl<T: Operations> AlwaysRefCounted for Request<T> {
     fn inc_ref(&self) {
         let refcount = &self.wrapper_ref().refcount();

         #[cfg_attr(not(CONFIG_DEBUG_MISC), allow(unused_variables))]
         let updated = atomic_relaxed_op_unless(refcount, |x| x + 1, 0);

         #[cfg(CONFIG_DEBUG_MISC)]
         if !updated {
             panic!("Request refcount zero on clone")
         }
     }

     unsafe fn dec_ref(obj: core::ptr::NonNull<Self>) {
         // SAFETY: The type invariants of `ARef` guarantee that `obj` is valid
         // for read.
         let wrapper_ptr = unsafe { Self::wrapper_ptr(obj.as_ptr()).as_ptr() };
         // SAFETY: The type invariant of `Request` guarantees that the private
         // data area is initialized and valid.
         let refcount = unsafe { &*RequestDataWrapper::refcount_ptr(wrapper_ptr) };

         #[cfg_attr(not(CONFIG_DEBUG_MISC), allow(unused_variables))]
         let new_refcount = atomic_relaxed_op_return(refcount, |x| x - 1);

         #[cfg(CONFIG_DEBUG_MISC)]
         if new_refcount == 0 {
             panic!("Request reached refcount zero in Rust abstractions");
         }
     }
 }
	// SPDX-License-Identifier: GPL-2.0

	//! This module provides a wrapper for the C `struct request` type.
	//!
	//! C header: [`include/linux/blk-mq.h`](srctree/include/linux/blk-mq.h)

	use crate::{
	bindings,
	block::mq::Operations,
	error::Result,
	types::{ARef, AlwaysRefCounted, Opaque},
	};
	use core::{
	marker::PhantomData,
	ptr::{addr_of_mut, NonNull},
	sync::atomic::{AtomicU64, Ordering},
	};

	/// A wrapper around a blk-mq `struct request`. This represents an IO request.
	///
	/// # Implementation details
	///
	/// There are four states for a request that the Rust bindings care about:
	///
	/// A) Request is owned by block layer (refcount 0)
	/// B) Request is owned by driver but with zero `ARef`s in existence
	/// (refcount 1)
	/// C) Request is owned by driver with exactly one `ARef` in existence
	/// (refcount 2)
	/// D) Request is owned by driver with more than one `ARef` in existence
	/// (refcount > 2)
	///
	///
	/// We need to track A and B to ensure we fail tag to request conversions for
	/// requests that are not owned by the driver.
	///
	/// We need to track C and D to ensure that it is safe to end the request and hand
	/// back ownership to the block layer.
	///
	/// The states are tracked through the private `refcount` field of
	/// `RequestDataWrapper`. This structure lives in the private data area of the C
	/// `struct request`.
	///
	/// # Invariants
	///
	/// * `self.0` is a valid `struct request` created by the C portion of the kernel.
	/// * The private data area associated with this request must be an initialized
	/// and valid `RequestDataWrapper<T>`.
	/// * `self` is reference counted by atomic modification of
	/// self.wrapper_ref().refcount().
	///
	#[repr(transparent)]
	pub struct Request<T: Operations>(Opaque<bindings::request>, PhantomData<T>);

	impl<T: Operations> Request<T> {
	/// Create an `ARef<Request>` from a `struct request` pointer.
	///
	/// # Safety
	///
	/// * The caller must own a refcount on `ptr` that is transferred to the
	/// returned `ARef`.
	/// * The type invariants for `Request` must hold for the pointee of `ptr`.
	pub(crate) unsafe fn aref_from_raw(ptr: *mut bindings::request) -> ARef<Self> {
	// INVARIANT: By the safety requirements of this function, invariants are upheld.
	// SAFETY: By the safety requirement of this function, we own a
	// reference count that we can pass to `ARef`.
	unsafe { ARef::from_raw(NonNull::new_unchecked(ptr as const Self as mut Self)) }
	}

	/// Notify the block layer that a request is going to be processed now.
	///
	/// The block layer uses this hook to do proper initializations such as
	/// starting the timeout timer. It is a requirement that block device
	/// drivers call this function when starting to process a request.
	///
	/// # Safety
	///
	/// The caller must have exclusive ownership of `self`, that is
	/// `self.wrapper_ref().refcount() == 2`.
	pub(crate) unsafe fn start_unchecked(this: &ARef<Self>) {
	// SAFETY: By type invariant, `self.0` is a valid `struct request` and
	// we have exclusive access.
	unsafe { bindings::blk_mq_start_request(this.0.get()) };
	}

	/// Try to take exclusive ownership of `this` by dropping the refcount to 0.
	/// This fails if `this` is not the only `ARef` pointing to the underlying
	/// `Request`.
	///
	/// If the operation is successful, `Ok` is returned with a pointer to the
	/// C `struct request`. If the operation fails, `this` is returned in the
	/// `Err` variant.
	fn try_set_end(this: ARef<Self>) -> Result<*mut bindings::request, ARef<Self>> {
	// We can race with `TagSet::tag_to_rq`
	if let Err(_old) = this.wrapper_ref().refcount().compare_exchange(
	2,
	0,
	Ordering::Relaxed,
	Ordering::Relaxed,
	) {
	return Err(this);
	}

	let request_ptr = this.0.get();
	core::mem::forget(this);

	Ok(request_ptr)
	}

	/// Notify the block layer that the request has been completed without errors.
	///
	/// This function will return `Err` if `this` is not the only `ARef`
	/// referencing the request.
	pub fn end_ok(this: ARef<Self>) -> Result<(), ARef<Self>> {
	let request_ptr = Self::try_set_end(this)?;

	// SAFETY: By type invariant, `this.0` was a valid `struct request`. The
	// success of the call to `try_set_end` guarantees that there are no
	// `ARef`s pointing to this request. Therefore it is safe to hand it
	// back to the block layer.
	unsafe { bindings::blk_mq_end_request(request_ptr, bindings::BLK_STS_OK as _) };

	Ok(())
	}

	/// Return a pointer to the `RequestDataWrapper` stored in the private area
	/// of the request structure.
	///
	/// # Safety
	///
	/// - `this` must point to a valid allocation of size at least size of
	/// `Self` plus size of `RequestDataWrapper`.
	pub(crate) unsafe fn wrapper_ptr(this: *mut Self) -> NonNull<RequestDataWrapper> {
	let request_ptr = this.cast::<bindings::request>();
	// SAFETY: By safety requirements for this function, `this` is a
	// valid allocation.
	let wrapper_ptr =
	unsafe { bindings::blk_mq_rq_to_pdu(request_ptr).cast::<RequestDataWrapper>() };
	// SAFETY: By C API contract, wrapper_ptr points to a valid allocation
	// and is not null.
	unsafe { NonNull::new_unchecked(wrapper_ptr) }
	}

	/// Return a reference to the `RequestDataWrapper` stored in the private
	/// area of the request structure.
	pub(crate) fn wrapper_ref(&self) -> &RequestDataWrapper {
	// SAFETY: By type invariant, `self.0` is a valid allocation. Further,
	// the private data associated with this request is initialized and
	// valid. The existence of `&self` guarantees that the private data is
	// valid as a shared reference.
	unsafe { Self::wrapper_ptr(self as const Self as mut Self).as_ref() }
	}
	}

	/// A wrapper around data stored in the private area of the C `struct request`.
	pub(crate) struct RequestDataWrapper {
	/// The Rust request refcount has the following states:
	///
	/// - 0: The request is owned by C block layer.
	/// - 1: The request is owned by Rust abstractions but there are no ARef references to it.
	/// - 2+: There are `ARef` references to the request.
	refcount: AtomicU64,
	}

	impl RequestDataWrapper {
	/// Return a reference to the refcount of the request that is embedding
	/// `self`.
	pub(crate) fn refcount(&self) -> &AtomicU64 {
	&self.refcount
	}

	/// Return a pointer to the refcount of the request that is embedding the
	/// pointee of `this`.
	///
	/// # Safety
	///
	/// - `this` must point to a live allocation of at least the size of `Self`.
	pub(crate) unsafe fn refcount_ptr(this: mut Self) -> mut AtomicU64 {
	// SAFETY: Because of the safety requirements of this function, the
	// field projection is safe.
	unsafe { addr_of_mut!((*this).refcount) }
	}
	}

	// SAFETY: Exclusive access is thread-safe for `Request`. `Request` has no `&mut
	// self` methods and `&self` methods that mutate `self` are internally
	// synchronized.
	unsafe impl<T: Operations> Send for Request<T> {}

	// SAFETY: Shared access is thread-safe for `Request`. `&self` methods that
	// mutate `self` are internally synchronized`
	unsafe impl<T: Operations> Sync for Request<T> {}

	/// Store the result of `op(target.load())` in target, returning new value of
	/// target.
	fn atomic_relaxed_op_return(target: &AtomicU64, op: impl Fn(u64) -> u64) -> u64 {
	let old = target.fetch_update(Ordering::Relaxed, Ordering::Relaxed, \|x\| Some(op(x)));

	// SAFETY: Because the operation passed to `fetch_update` above always
	// return `Some`, `old` will always be `Ok`.
	let old = unsafe { old.unwrap_unchecked() };

	op(old)
	}

	/// Store the result of `op(target.load)` in `target` if `target.load() !=
	/// pred`, returning true if the target was updated.
	fn atomic_relaxed_op_unless(target: &AtomicU64, op: impl Fn(u64) -> u64, pred: u64) -> bool {
	target
	.fetch_update(Ordering::Relaxed, Ordering::Relaxed, \|x\| {
	if x == pred {
	None
	} else {
	Some(op(x))
	}
	})
	.is_ok()
	}

	// SAFETY: All instances of `Request<T>` are reference counted. This
	// implementation of `AlwaysRefCounted` ensure that increments to the ref count
	// keeps the object alive in memory at least until a matching reference count
	// decrement is executed.
	unsafe impl<T: Operations> AlwaysRefCounted for Request<T> {
	fn inc_ref(&self) {
	let refcount = &self.wrapper_ref().refcount();

	#[cfg_attr(not(CONFIG_DEBUG_MISC), allow(unused_variables))]
	let updated = atomic_relaxed_op_unless(refcount, \|x\| x + 1, 0);

	#[cfg(CONFIG_DEBUG_MISC)]
	if !updated {
	panic!("Request refcount zero on clone")
	}
	}

	unsafe fn dec_ref(obj: core::ptr::NonNull<Self>) {
	// SAFETY: The type invariants of `ARef` guarantee that `obj` is valid
	// for read.
	let wrapper_ptr = unsafe { Self::wrapper_ptr(obj.as_ptr()).as_ptr() };
	// SAFETY: The type invariant of `Request` guarantees that the private
	// data area is initialized and valid.
	let refcount = unsafe { &*RequestDataWrapper::refcount_ptr(wrapper_ptr) };

	#[cfg_attr(not(CONFIG_DEBUG_MISC), allow(unused_variables))]
	let new_refcount = atomic_relaxed_op_return(refcount, \|x\| x - 1);

	#[cfg(CONFIG_DEBUG_MISC)]
	if new_refcount == 0 {
	panic!("Request reached refcount zero in Rust abstractions");
	}
	}
	}