--- /dev/null
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+
+#![cfg(CONFIG_DRM_GPUVM = "y")]
+
+//! DRM GPUVM in immediate mode
+//!
+//! Rust abstractions for using GPUVM in immediate mode. This is when the GPUVM state is updated
+//! during `run_job()`, i.e., in the DMA fence signalling critical path, to ensure that the GPUVM
+//! and the GPU's virtual address space has the same state at all times.
+//!
+//! C header: [`include/drm/drm_gpuvm.h`](srctree/include/drm/drm_gpuvm.h)
+
+use kernel::{
+ alloc::AllocError,
+ bindings,
+ drm,
+ drm::gem::IntoGEMObject,
+ prelude::*,
+ sync::aref::{
+ ARef,
+ AlwaysRefCounted, //
+ },
+ types::Opaque, //
+};
+
+use core::{
+ cell::UnsafeCell,
+ ops::{
+ Deref,
+ Range, //
+ },
+ ptr::NonNull, //
+};
+
+/// A DRM GPU VA manager.
+///
+/// This object is refcounted, but the locations of mapped ranges may only be accessed or changed
+/// via the special unique handle [`UniqueRefGpuVm`].
+///
+/// # Invariants
+///
+/// * Stored in an allocation managed by the refcount in `self.vm`.
+/// * Access to `data` and the gpuvm interval tree is controlled via the [`UniqueRefGpuVm`] type.
+/// * Does not contain any sparse `GpuVa` instances.
+#[pin_data]
+pub struct GpuVm<T: DriverGpuVm> {
+ #[pin]
+ vm: Opaque<bindings::drm_gpuvm>,
+ /// Accessed only through the [`UniqueRefGpuVm`] reference.
+ data: UnsafeCell<T>,
+}
+
+// SAFETY: The GPUVM api does not assume that it is tied to a specific thread. The destructor will
+// drop the `data` field, which is okay because it is guaranteed `Send` by the `DriverGpuVm` trait.
+unsafe impl<T: DriverGpuVm> Send for GpuVm<T> {}
+// SAFETY: The GPUVM api is designed to allow &self methods to be called in parallel.
+unsafe impl<T: DriverGpuVm> Sync for GpuVm<T> {}
+
+// SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`.
+unsafe impl<T: DriverGpuVm> AlwaysRefCounted for GpuVm<T> {
+ fn inc_ref(&self) {
+ // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`.
+ unsafe { bindings::drm_gpuvm_get(self.vm.get()) };
+ }
+
+ unsafe fn dec_ref(obj: NonNull<Self>) {
+ // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`.
+ unsafe { bindings::drm_gpuvm_put((*obj.as_ptr()).vm.get()) };
+ }
+}
+
+impl<T: DriverGpuVm> PartialEq for GpuVm<T> {
+ #[inline]
+ fn eq(&self, other: &Self) -> bool {
+ core::ptr::eq(self.as_raw(), other.as_raw())
+ }
+}
+impl<T: DriverGpuVm> Eq for GpuVm<T> {}
+
+impl<T: DriverGpuVm> GpuVm<T> {
+ const fn vtable() -> &'static bindings::drm_gpuvm_ops {
+ &bindings::drm_gpuvm_ops {
+ vm_free: Some(Self::vm_free),
+ op_alloc: None,
+ op_free: None,
+ vm_bo_alloc: None,
+ vm_bo_free: None,
+ vm_bo_validate: None,
+ sm_step_map: None,
+ sm_step_unmap: None,
+ sm_step_remap: None,
+ }
+ }
+
+ /// Creates a GPUVM instance.
+ #[expect(clippy::new_ret_no_self)]
+ pub fn new<E>(
+ name: &'static CStr,
+ dev: &drm::Device<T::Driver>,
+ r_obj: &T::Object,
+ range: Range<u64>,
+ reserve_range: Range<u64>,
+ data: T,
+ ) -> Result<UniqueRefGpuVm<T>, E>
+ where
+ E: From<AllocError>,
+ E: From<core::convert::Infallible>,
+ {
+ let obj = KBox::try_pin_init::<E>(
+ try_pin_init!(Self {
+ data: UnsafeCell::new(data),
+ vm <- Opaque::ffi_init(|vm| {
+ // SAFETY: These arguments are valid. `vm` is valid until refcount drops to
+ // zero. The `vm` is zeroed before calling this method by `__GFP_ZERO` flag
+ // below.
+ unsafe {
+ bindings::drm_gpuvm_init(
+ vm,
+ name.as_char_ptr(),
+ bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE
+ | bindings::drm_gpuvm_flags_DRM_GPUVM_RESV_PROTECTED,
+ dev.as_raw(),
+ r_obj.as_raw(),
+ range.start,
+ range.end - range.start,
+ reserve_range.start,
+ reserve_range.end - reserve_range.start,
+ const { Self::vtable() },
+ )
+ }
+ }),
+ }? E),
+ GFP_KERNEL | __GFP_ZERO,
+ )?;
+ // SAFETY: This transfers the initial refcount to the ARef.
+ let aref = unsafe {
+ ARef::from_raw(NonNull::new_unchecked(KBox::into_raw(
+ Pin::into_inner_unchecked(obj),
+ )))
+ };
+ // INVARIANT: This reference is unique.
+ Ok(UniqueRefGpuVm(aref))
+ }
+
+ /// Access this [`GpuVm`] from a raw pointer.
+ ///
+ /// # Safety
+ ///
+ /// The pointer must reference the `struct drm_gpuvm` in a valid [`GpuVm<T>`] that remains
+ /// valid for at least `'a`.
+ #[inline]
+ pub unsafe fn from_raw<'a>(ptr: *mut bindings::drm_gpuvm) -> &'a Self {
+ // SAFETY: Caller passes a pointer to the `drm_gpuvm` in a `GpuVm<T>`. Caller ensures the
+ // pointer is valid for 'a.
+ unsafe { &*kernel::container_of!(Opaque::cast_from(ptr), Self, vm) }
+ }
+
+ /// Returns a raw pointer to the embedded `struct drm_gpuvm`.
+ #[inline]
+ pub fn as_raw(&self) -> *mut bindings::drm_gpuvm {
+ self.vm.get()
+ }
+
+ /// The start of the VA space.
+ #[inline]
+ pub fn va_start(&self) -> u64 {
+ // SAFETY: The `mm_start` field is immutable.
+ unsafe { (*self.as_raw()).mm_start }
+ }
+
+ /// The length of the GPU's virtual address space.
+ #[inline]
+ pub fn va_length(&self) -> u64 {
+ // SAFETY: The `mm_range` field is immutable.
+ unsafe { (*self.as_raw()).mm_range }
+ }
+
+ /// Returns the range of the GPU virtual address space.
+ #[inline]
+ pub fn va_range(&self) -> Range<u64> {
+ let start = self.va_start();
+ // OVERFLOW: This reconstructs the Range<u64> passed to the constructor, so it won't fail.
+ let end = start + self.va_length();
+ Range { start, end }
+ }
+
+ /// Clean up buffer objects that are no longer used.
+ #[inline]
+ pub fn deferred_cleanup(&self) {
+ // SAFETY: This GPUVM uses immediate mode.
+ unsafe { bindings::drm_gpuvm_bo_deferred_cleanup(self.as_raw()) }
+ }
+
+ /// Check if this GEM object is an external object for this GPUVM.
+ #[inline]
+ pub fn is_extobj(&self, obj: &T::Object) -> bool {
+ // SAFETY: We may call this with any GPUVM and GEM object.
+ unsafe { bindings::drm_gpuvm_is_extobj(self.as_raw(), obj.as_raw()) }
+ }
+
+ /// Free this GPUVM.
+ ///
+ /// # Safety
+ ///
+ /// Called when refcount hits zero.
+ unsafe extern "C" fn vm_free(me: *mut bindings::drm_gpuvm) {
+ // SAFETY: Caller passes a pointer to the `drm_gpuvm` in a `GpuVm<T>`.
+ let me = unsafe { kernel::container_of!(Opaque::cast_from(me), Self, vm).cast_mut() };
+ // SAFETY: By type invariants we can free it when refcount hits zero.
+ drop(unsafe { KBox::from_raw(me) })
+ }
+}
+
+/// The manager for a GPUVM.
+pub trait DriverGpuVm: Sized + Send {
+ /// Parent `Driver` for this object.
+ type Driver: drm::Driver<Object = Self::Object>;
+
+ /// The kind of GEM object stored in this GPUVM.
+ type Object: IntoGEMObject;
+}
+
+/// The core of the DRM GPU VA manager.
+///
+/// This object is a unique reference to the VM that can access the interval tree and the Rust
+/// `data` field.
+///
+/// # Invariants
+///
+/// Each `GpuVm` instance has at most one `UniqueRefGpuVm` reference.
+pub struct UniqueRefGpuVm<T: DriverGpuVm>(ARef<GpuVm<T>>);
+
+// SAFETY: The GPUVM api is designed to allow &self methods to be called in parallel, and
+// concurrent access to `data` is safe due to the `T: Sync` requirement.
+unsafe impl<T: DriverGpuVm + Sync> Sync for UniqueRefGpuVm<T> {}
+
+impl<T: DriverGpuVm> UniqueRefGpuVm<T> {
+ /// Access the data owned by this `UniqueRefGpuVm` immutably.
+ #[inline]
+ pub fn data_ref(&self) -> &T {
+ // SAFETY: By the type invariants we may access `data`.
+ unsafe { &*self.0.data.get() }
+ }
+
+ /// Access the data owned by this `UniqueRefGpuVm` mutably.
+ #[inline]
+ pub fn data(&mut self) -> &mut T {
+ // SAFETY: By the type invariants we may access `data`.
+ unsafe { &mut *self.0.data.get() }
+ }
+}
+
+impl<T: DriverGpuVm> Deref for UniqueRefGpuVm<T> {
+ type Target = GpuVm<T>;
+
+ #[inline]
+ fn deref(&self) -> &GpuVm<T> {
+ &self.0
+ }
+}
projects / thirdparty / linux.git / commitdiff
