rust/kernel/page.rs - linux - Git at Google

 // SPDX-License-Identifier: GPL-2.0

 //! Kernel page allocation and management.

 use crate::{
     alloc::{AllocError, Flags},
     bindings,
     error::code::*,
     error::Result,
     uaccess::UserSliceReader,
 };
 use core::ptr::{self, NonNull};

 /// A bitwise shift for the page size.
 pub const PAGE_SHIFT: usize = bindings::PAGE_SHIFT as usize;

 /// The number of bytes in a page.
 pub const PAGE_SIZE: usize = bindings::PAGE_SIZE;

 /// A bitmask that gives the page containing a given address.
 pub const PAGE_MASK: usize = !(PAGE_SIZE - 1);

 /// A pointer to a page that owns the page allocation.
 ///
 /// # Invariants
 ///
 /// The pointer is valid, and has ownership over the page.
 pub struct Page {
     page: NonNull<bindings::page>,
 }

 // SAFETY: Pages have no logic that relies on them staying on a given thread, so moving them across
 // threads is safe.
 unsafe impl Send for Page {}

 // SAFETY: Pages have no logic that relies on them not being accessed concurrently, so accessing
 // them concurrently is safe.
 unsafe impl Sync for Page {}

 impl Page {
     /// Allocates a new page.
     ///
     /// # Examples
     ///
     /// Allocate memory for a page.
     ///
     /// ```
     /// use kernel::page::Page;
     ///
     /// # fn dox() -> Result<(), kernel::alloc::AllocError> {
     /// let page = Page::alloc_page(GFP_KERNEL)?;
     /// # Ok(()) }
     /// ```
     ///
     /// Allocate memory for a page and zero its contents.
     ///
     /// ```
     /// use kernel::page::Page;
     ///
     /// # fn dox() -> Result<(), kernel::alloc::AllocError> {
     /// let page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?;
     /// # Ok(()) }
     /// ```
     pub fn alloc_page(flags: Flags) -> Result<Self, AllocError> {
         // SAFETY: Depending on the value of `gfp_flags`, this call may sleep. Other than that, it
         // is always safe to call this method.
         let page = unsafe { bindings::alloc_pages(flags.as_raw(), 0) };
         let page = NonNull::new(page).ok_or(AllocError)?;
         // INVARIANT: We just successfully allocated a page, so we now have ownership of the newly
         // allocated page. We transfer that ownership to the new `Page` object.
         Ok(Self { page })
     }

     /// Returns a raw pointer to the page.
     pub fn as_ptr(&self) -> *mut bindings::page {
         self.page.as_ptr()
     }

     /// Runs a piece of code with this page mapped to an address.
     ///
     /// The page is unmapped when this call returns.
     ///
     /// # Using the raw pointer
     ///
     /// It is up to the caller to use the provided raw pointer correctly. The pointer is valid for
     /// `PAGE_SIZE` bytes and for the duration in which the closure is called. The pointer might
     /// only be mapped on the current thread, and when that is the case, dereferencing it on other
     /// threads is UB. Other than that, the usual rules for dereferencing a raw pointer apply: don't
     /// cause data races, the memory may be uninitialized, and so on.
     ///
     /// If multiple threads map the same page at the same time, then they may reference with
     /// different addresses. However, even if the addresses are different, the underlying memory is
     /// still the same for these purposes (e.g., it's still a data race if they both write to the
     /// same underlying byte at the same time).
     fn with_page_mapped<T>(&self, f: impl FnOnce(*mut u8) -> T) -> T {
         // SAFETY: `page` is valid due to the type invariants on `Page`.
         let mapped_addr = unsafe { bindings::kmap_local_page(self.as_ptr()) };

         let res = f(mapped_addr.cast());

         // This unmaps the page mapped above.
         //
         // SAFETY: Since this API takes the user code as a closure, it can only be used in a manner
         // where the pages are unmapped in reverse order. This is as required by `kunmap_local`.
         //
         // In other words, if this call to `kunmap_local` happens when a different page should be
         // unmapped first, then there must necessarily be a call to `kmap_local_page` other than the
         // call just above in `with_page_mapped` that made that possible. In this case, it is the
         // unsafe block that wraps that other call that is incorrect.
         unsafe { bindings::kunmap_local(mapped_addr) };

         res
     }

     /// Runs a piece of code with a raw pointer to a slice of this page, with bounds checking.
     ///
     /// If `f` is called, then it will be called with a pointer that points at `off` bytes into the
     /// page, and the pointer will be valid for at least `len` bytes. The pointer is only valid on
     /// this task, as this method uses a local mapping.
     ///
     /// If `off` and `len` refers to a region outside of this page, then this method returns
     /// [`EINVAL`] and does not call `f`.
     ///
     /// # Using the raw pointer
     ///
     /// It is up to the caller to use the provided raw pointer correctly. The pointer is valid for
     /// `len` bytes and for the duration in which the closure is called. The pointer might only be
     /// mapped on the current thread, and when that is the case, dereferencing it on other threads
     /// is UB. Other than that, the usual rules for dereferencing a raw pointer apply: don't cause
     /// data races, the memory may be uninitialized, and so on.
     ///
     /// If multiple threads map the same page at the same time, then they may reference with
     /// different addresses. However, even if the addresses are different, the underlying memory is
     /// still the same for these purposes (e.g., it's still a data race if they both write to the
     /// same underlying byte at the same time).
     fn with_pointer_into_page<T>(
         &self,
         off: usize,
         len: usize,
         f: impl FnOnce(*mut u8) -> Result<T>,
     ) -> Result<T> {
         let bounds_ok = off <= PAGE_SIZE && len <= PAGE_SIZE && (off + len) <= PAGE_SIZE;

         if bounds_ok {
             self.with_page_mapped(move |page_addr| {
                 // SAFETY: The `off` integer is at most `PAGE_SIZE`, so this pointer offset will
                 // result in a pointer that is in bounds or one off the end of the page.
                 f(unsafe { page_addr.add(off) })
             })
         } else {
             Err(EINVAL)
         }
     }

     /// Maps the page and reads from it into the given buffer.
     ///
     /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes
     /// outside of the page, then this call returns [`EINVAL`].
     ///
     /// # Safety
     ///
     /// * Callers must ensure that `dst` is valid for writing `len` bytes.
     /// * Callers must ensure that this call does not race with a write to the same page that
     ///   overlaps with this read.
     pub unsafe fn read_raw(&self, dst: *mut u8, offset: usize, len: usize) -> Result {
         self.with_pointer_into_page(offset, len, move |src| {
             // SAFETY: If `with_pointer_into_page` calls into this closure, then
             // it has performed a bounds check and guarantees that `src` is
             // valid for `len` bytes.
             //
             // There caller guarantees that there is no data race.
             unsafe { ptr::copy_nonoverlapping(src, dst, len) };
             Ok(())
         })
     }

     /// Maps the page and writes into it from the given buffer.
     ///
     /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes
     /// outside of the page, then this call returns [`EINVAL`].
     ///
     /// # Safety
     ///
     /// * Callers must ensure that `src` is valid for reading `len` bytes.
     /// * Callers must ensure that this call does not race with a read or write to the same page
     ///   that overlaps with this write.
     pub unsafe fn write_raw(&self, src: *const u8, offset: usize, len: usize) -> Result {
         self.with_pointer_into_page(offset, len, move |dst| {
             // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a
             // bounds check and guarantees that `dst` is valid for `len` bytes.
             //
             // There caller guarantees that there is no data race.
             unsafe { ptr::copy_nonoverlapping(src, dst, len) };
             Ok(())
         })
     }

     /// Maps the page and zeroes the given slice.
     ///
     /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes
     /// outside of the page, then this call returns [`EINVAL`].
     ///
     /// # Safety
     ///
     /// Callers must ensure that this call does not race with a read or write to the same page that
     /// overlaps with this write.
     pub unsafe fn fill_zero_raw(&self, offset: usize, len: usize) -> Result {
         self.with_pointer_into_page(offset, len, move |dst| {
             // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a
             // bounds check and guarantees that `dst` is valid for `len` bytes.
             //
             // There caller guarantees that there is no data race.
             unsafe { ptr::write_bytes(dst, 0u8, len) };
             Ok(())
         })
     }

     /// Copies data from userspace into this page.
     ///
     /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes
     /// outside of the page, then this call returns [`EINVAL`].
     ///
     /// Like the other `UserSliceReader` methods, data races are allowed on the userspace address.
     /// However, they are not allowed on the page you are copying into.
     ///
     /// # Safety
     ///
     /// Callers must ensure that this call does not race with a read or write to the same page that
     /// overlaps with this write.
     pub unsafe fn copy_from_user_slice_raw(
         &self,
         reader: &mut UserSliceReader,
         offset: usize,
         len: usize,
     ) -> Result {
         self.with_pointer_into_page(offset, len, move |dst| {
             // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a
             // bounds check and guarantees that `dst` is valid for `len` bytes. Furthermore, we have
             // exclusive access to the slice since the caller guarantees that there are no races.
             reader.read_raw(unsafe { core::slice::from_raw_parts_mut(dst.cast(), len) })
         })
     }
 }

 impl Drop for Page {
     fn drop(&mut self) {
         // SAFETY: By the type invariants, we have ownership of the page and can free it.
         unsafe { bindings::__free_pages(self.page.as_ptr(), 0) };
     }
 }
	// SPDX-License-Identifier: GPL-2.0

	//! Kernel page allocation and management.

	use crate::{
	alloc::{AllocError, Flags},
	bindings,
	error::code::*,
	error::Result,
	uaccess::UserSliceReader,
	};
	use core::ptr::{self, NonNull};

	/// A bitwise shift for the page size.
	pub const PAGE_SHIFT: usize = bindings::PAGE_SHIFT as usize;

	/// The number of bytes in a page.
	pub const PAGE_SIZE: usize = bindings::PAGE_SIZE;

	/// A bitmask that gives the page containing a given address.
	pub const PAGE_MASK: usize = !(PAGE_SIZE - 1);

	/// A pointer to a page that owns the page allocation.
	///
	/// # Invariants
	///
	/// The pointer is valid, and has ownership over the page.
	pub struct Page {
	page: NonNull<bindings::page>,
	}

	// SAFETY: Pages have no logic that relies on them staying on a given thread, so moving them across
	// threads is safe.
	unsafe impl Send for Page {}

	// SAFETY: Pages have no logic that relies on them not being accessed concurrently, so accessing
	// them concurrently is safe.
	unsafe impl Sync for Page {}

	impl Page {
	/// Allocates a new page.
	///
	/// # Examples
	///
	/// Allocate memory for a page.
	///
	/// ```
	/// use kernel::page::Page;
	///
	/// # fn dox() -> Result<(), kernel::alloc::AllocError> {
	/// let page = Page::alloc_page(GFP_KERNEL)?;
	/// # Ok(()) }
	/// ```
	///
	/// Allocate memory for a page and zero its contents.
	///
	/// ```
	/// use kernel::page::Page;
	///
	/// # fn dox() -> Result<(), kernel::alloc::AllocError> {
	/// let page = Page::alloc_page(GFP_KERNEL \| __GFP_ZERO)?;
	/// # Ok(()) }
	/// ```
	pub fn alloc_page(flags: Flags) -> Result<Self, AllocError> {
	// SAFETY: Depending on the value of `gfp_flags`, this call may sleep. Other than that, it
	// is always safe to call this method.
	let page = unsafe { bindings::alloc_pages(flags.as_raw(), 0) };
	let page = NonNull::new(page).ok_or(AllocError)?;
	// INVARIANT: We just successfully allocated a page, so we now have ownership of the newly
	// allocated page. We transfer that ownership to the new `Page` object.
	Ok(Self { page })
	}

	/// Returns a raw pointer to the page.
	pub fn as_ptr(&self) -> *mut bindings::page {
	self.page.as_ptr()
	}

	/// Runs a piece of code with this page mapped to an address.
	///
	/// The page is unmapped when this call returns.
	///
	/// # Using the raw pointer
	///
	/// It is up to the caller to use the provided raw pointer correctly. The pointer is valid for
	/// `PAGE_SIZE` bytes and for the duration in which the closure is called. The pointer might
	/// only be mapped on the current thread, and when that is the case, dereferencing it on other
	/// threads is UB. Other than that, the usual rules for dereferencing a raw pointer apply: don't
	/// cause data races, the memory may be uninitialized, and so on.
	///
	/// If multiple threads map the same page at the same time, then they may reference with
	/// different addresses. However, even if the addresses are different, the underlying memory is
	/// still the same for these purposes (e.g., it's still a data race if they both write to the
	/// same underlying byte at the same time).
	fn with_page_mapped<T>(&self, f: impl FnOnce(*mut u8) -> T) -> T {
	// SAFETY: `page` is valid due to the type invariants on `Page`.
	let mapped_addr = unsafe { bindings::kmap_local_page(self.as_ptr()) };

	let res = f(mapped_addr.cast());

	// This unmaps the page mapped above.
	//
	// SAFETY: Since this API takes the user code as a closure, it can only be used in a manner
	// where the pages are unmapped in reverse order. This is as required by `kunmap_local`.
	//
	// In other words, if this call to `kunmap_local` happens when a different page should be
	// unmapped first, then there must necessarily be a call to `kmap_local_page` other than the
	// call just above in `with_page_mapped` that made that possible. In this case, it is the
	// unsafe block that wraps that other call that is incorrect.
	unsafe { bindings::kunmap_local(mapped_addr) };

	res
	}

	/// Runs a piece of code with a raw pointer to a slice of this page, with bounds checking.
	///
	/// If `f` is called, then it will be called with a pointer that points at `off` bytes into the
	/// page, and the pointer will be valid for at least `len` bytes. The pointer is only valid on
	/// this task, as this method uses a local mapping.
	///
	/// If `off` and `len` refers to a region outside of this page, then this method returns
	/// [`EINVAL`] and does not call `f`.
	///
	/// # Using the raw pointer
	///
	/// It is up to the caller to use the provided raw pointer correctly. The pointer is valid for
	/// `len` bytes and for the duration in which the closure is called. The pointer might only be
	/// mapped on the current thread, and when that is the case, dereferencing it on other threads
	/// is UB. Other than that, the usual rules for dereferencing a raw pointer apply: don't cause
	/// data races, the memory may be uninitialized, and so on.
	///
	/// If multiple threads map the same page at the same time, then they may reference with
	/// different addresses. However, even if the addresses are different, the underlying memory is
	/// still the same for these purposes (e.g., it's still a data race if they both write to the
	/// same underlying byte at the same time).
	fn with_pointer_into_page<T>(
	&self,
	off: usize,
	len: usize,
	f: impl FnOnce(*mut u8) -> Result<T>,
	) -> Result<T> {
	let bounds_ok = off <= PAGE_SIZE && len <= PAGE_SIZE && (off + len) <= PAGE_SIZE;

	if bounds_ok {
	self.with_page_mapped(move \|page_addr\| {
	// SAFETY: The `off` integer is at most `PAGE_SIZE`, so this pointer offset will
	// result in a pointer that is in bounds or one off the end of the page.
	f(unsafe { page_addr.add(off) })
	})
	} else {
	Err(EINVAL)
	}
	}

	/// Maps the page and reads from it into the given buffer.
	///
	/// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes
	/// outside of the page, then this call returns [`EINVAL`].
	///
	/// # Safety
	///
	/// * Callers must ensure that `dst` is valid for writing `len` bytes.
	/// * Callers must ensure that this call does not race with a write to the same page that
	/// overlaps with this read.
	pub unsafe fn read_raw(&self, dst: *mut u8, offset: usize, len: usize) -> Result {
	self.with_pointer_into_page(offset, len, move \|src\| {
	// SAFETY: If `with_pointer_into_page` calls into this closure, then
	// it has performed a bounds check and guarantees that `src` is
	// valid for `len` bytes.
	//
	// There caller guarantees that there is no data race.
	unsafe { ptr::copy_nonoverlapping(src, dst, len) };
	Ok(())
	})
	}

	/// Maps the page and writes into it from the given buffer.
	///
	/// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes
	/// outside of the page, then this call returns [`EINVAL`].
	///
	/// # Safety
	///
	/// * Callers must ensure that `src` is valid for reading `len` bytes.
	/// * Callers must ensure that this call does not race with a read or write to the same page
	/// that overlaps with this write.
	pub unsafe fn write_raw(&self, src: *const u8, offset: usize, len: usize) -> Result {
	self.with_pointer_into_page(offset, len, move \|dst\| {
	// SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a
	// bounds check and guarantees that `dst` is valid for `len` bytes.
	//
	// There caller guarantees that there is no data race.
	unsafe { ptr::copy_nonoverlapping(src, dst, len) };
	Ok(())
	})
	}

	/// Maps the page and zeroes the given slice.
	///
	/// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes
	/// outside of the page, then this call returns [`EINVAL`].
	///
	/// # Safety
	///
	/// Callers must ensure that this call does not race with a read or write to the same page that
	/// overlaps with this write.
	pub unsafe fn fill_zero_raw(&self, offset: usize, len: usize) -> Result {
	self.with_pointer_into_page(offset, len, move \|dst\| {
	// SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a
	// bounds check and guarantees that `dst` is valid for `len` bytes.
	//
	// There caller guarantees that there is no data race.
	unsafe { ptr::write_bytes(dst, 0u8, len) };
	Ok(())
	})
	}

	/// Copies data from userspace into this page.
	///
	/// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes
	/// outside of the page, then this call returns [`EINVAL`].
	///
	/// Like the other `UserSliceReader` methods, data races are allowed on the userspace address.
	/// However, they are not allowed on the page you are copying into.
	///
	/// # Safety
	///
	/// Callers must ensure that this call does not race with a read or write to the same page that
	/// overlaps with this write.
	pub unsafe fn copy_from_user_slice_raw(
	&self,
	reader: &mut UserSliceReader,
	offset: usize,
	len: usize,
	) -> Result {
	self.with_pointer_into_page(offset, len, move \|dst\| {
	// SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a
	// bounds check and guarantees that `dst` is valid for `len` bytes. Furthermore, we have
	// exclusive access to the slice since the caller guarantees that there are no races.
	reader.read_raw(unsafe { core::slice::from_raw_parts_mut(dst.cast(), len) })
	})
	}
	}

	impl Drop for Page {
	fn drop(&mut self) {
	// SAFETY: By the type invariants, we have ownership of the page and can free it.
	unsafe { bindings::__free_pages(self.page.as_ptr(), 0) };
	}
	}