user_context: &'a mut T::SmContext<'ctx>,
}
+/// Adds an extra field to `SmData` for `sm_map()` callbacks.
+///
+/// # Invariants
+///
+/// `self.vm_bo.gpuvm() == self.sm_data.gpuvm`.
+#[repr(C)]
+struct SmMapData<'a, 'ctx, T: DriverGpuVm> {
+ sm_data: SmData<'a, 'ctx, T>,
+ vm_bo: &'a GpuVmBo<T>,
+}
+
+/// The argument for [`UniqueRefGpuVm::sm_map`].
+pub struct OpMapRequest<'a, 'ctx, T: DriverGpuVm> {
+ /// Address in GPU virtual address space.
+ pub addr: u64,
+ /// Length of mapping to create.
+ pub range: u64,
+ /// Offset in GEM object.
+ pub gem_offset: u64,
+ /// The GEM object to map.
+ pub vm_bo: &'a GpuVmBo<T>,
+ /// The user-provided context type.
+ pub context: &'a mut T::SmContext<'ctx>,
+}
+
+impl<'a, 'ctx, T: DriverGpuVm> OpMapRequest<'a, 'ctx, T> {
+ fn raw_request(&self) -> bindings::drm_gpuvm_map_req {
+ bindings::drm_gpuvm_map_req {
+ map: bindings::drm_gpuva_op_map {
+ va: bindings::drm_gpuva_op_map__bindgen_ty_1 {
+ addr: self.addr,
+ range: self.range,
+ },
+ gem: bindings::drm_gpuva_op_map__bindgen_ty_2 {
+ offset: self.gem_offset,
+ obj: self.vm_bo.obj().as_raw(),
+ },
+ },
+ }
+ }
+}
+
+/// Represents an `sm_step_map` operation that has not yet been completed.
+pub struct OpMap<'op, T: DriverGpuVm> {
+ op: &'op bindings::drm_gpuva_op_map,
+ // Since these abstractions are designed for immediate mode, the VM BO needs to be
+ // pre-allocated, so we always have it available when we reach this point.
+ vm_bo: &'op GpuVmBo<T>,
+ // This ensures that 'op is invariant, so that `OpMap<'long, T>` does not
+ // coerce to `OpMap<'short, T>`. This ensures that the user can't return
+ // the wrong `OpMapped` value.
+ _invariant: PhantomData<*mut &'op mut T>,
+}
+
+impl<'op, T: DriverGpuVm> OpMap<'op, T> {
+ /// The base address of the new mapping.
+ pub fn addr(&self) -> u64 {
+ self.op.va.addr
+ }
+
+ /// The length of the new mapping.
+ pub fn length(&self) -> u64 {
+ self.op.va.range
+ }
+
+ /// The offset within the [`drm_gem_object`](DriverGpuVm::Object).
+ pub fn gem_offset(&self) -> u64 {
+ self.op.gem.offset
+ }
+
+ /// The [`drm_gem_object`](DriverGpuVm::Object) to map.
+ pub fn obj(&self) -> &T::Object {
+ // SAFETY: The `obj` pointer is guaranteed to be valid.
+ unsafe { <T::Object as IntoGEMObject>::from_raw(self.op.gem.obj) }
+ }
+
+ /// The [`GpuVmBo`] that the new VA will be associated with.
+ pub fn vm_bo(&self) -> &GpuVmBo<T> {
+ self.vm_bo
+ }
+
+ /// Use the pre-allocated VA to carry out this map operation.
+ pub fn insert(self, va: GpuVaAlloc<T>, va_data: impl PinInit<T::VaData>) -> OpMapped<'op, T> {
+ let va = va.prepare(va_data);
+ // SAFETY: By the type invariants we may access the interval tree.
+ unsafe { bindings::drm_gpuva_map(self.vm_bo.gpuvm().as_raw(), va, self.op) };
+
+ let _gpuva_guard = self.vm_bo().lock_gpuva();
+ // SAFETY: The va is prepared for insertion, and we hold the GEM lock.
+ unsafe { bindings::drm_gpuva_link(va, self.vm_bo.as_raw()) };
+
+ OpMapped {
+ _invariant: self._invariant,
+ }
+ }
+}
+
+/// Represents a completed [`OpMap`] operation.
+pub struct OpMapped<'op, T> {
+ _invariant: PhantomData<*mut &'op mut T>,
+}
+
/// Represents an `sm_step_unmap` operation that has not yet been completed.
pub struct OpUnmap<'op, T: DriverGpuVm> {
op: &'op bindings::drm_gpuva_op_unmap,
}
impl<T: DriverGpuVm> UniqueRefGpuVm<T> {
+ /// Create a mapping, removing or remapping anything that overlaps.
+ ///
+ /// Internally calls the [`DriverGpuVm`] callbacks similar to [`Self::sm_unmap`], except that
+ /// the [`DriverGpuVm::sm_step_map`] is called once to create the requested mapping.
+ #[inline]
+ pub fn sm_map(&mut self, req: OpMapRequest<'_, '_, T>) -> Result {
+ if req.vm_bo.gpuvm() != &**self {
+ return Err(EINVAL);
+ }
+
+ let gpuvm = self.as_raw();
+ let raw_req = req.raw_request();
+ // INVARIANT: Checked above that `vm_bo.gpuvm() == self`.
+ let mut p = SmMapData {
+ sm_data: SmData {
+ gpuvm: self,
+ user_context: req.context,
+ },
+ vm_bo: req.vm_bo,
+ };
+ // SAFETY:
+ // * raw_request() creates a valid request.
+ // * The private data is valid to be interpreted as both SmData and SmMapData since the
+ // first field of SmMapData is SmData.
+ to_result(unsafe {
+ bindings::drm_gpuvm_sm_map(gpuvm, (&raw mut p).cast(), &raw const raw_req)
+ })
+ }
+
/// Remove any mappings in the given region.
///
/// Internally calls [`DriverGpuVm::sm_step_unmap`] for ranges entirely contained within the
};
// SAFETY:
// * raw_request() creates a valid request.
- // * The private data is valid to be interpreted as SmData.
+ // * The private data is a valid SmData.
to_result(unsafe { bindings::drm_gpuvm_sm_unmap(gpuvm, (&raw mut p).cast(), addr, length) })
}
}
impl<T: DriverGpuVm> GpuVm<T> {
/// # Safety
- /// Must be called from `sm_unmap` with a pointer to `SmData`.
+ /// Must be called from `sm_map` with a pointer to `SmMapData`.
+ pub(super) unsafe extern "C" fn sm_step_map(
+ op: *mut bindings::drm_gpuva_op,
+ p: *mut c_void,
+ ) -> c_int {
+ // SAFETY: If we reach `sm_step_map` then we were called from `sm_map` which always passes
+ // an `SmMapData` as private data.
+ let p = unsafe { &mut *p.cast::<SmMapData<'_, '_, T>>() };
+ let op = OpMap {
+ // SAFETY: sm_step_map is called with a map operation.
+ op: unsafe { &(*op).__bindgen_anon_1.map },
+ vm_bo: p.vm_bo,
+ _invariant: PhantomData,
+ };
+ match p
+ .sm_data
+ .gpuvm
+ .data()
+ .sm_step_map(op, p.sm_data.user_context)
+ {
+ Ok(OpMapped { .. }) => 0,
+ Err(err) => err.to_errno(),
+ }
+ }
+
+ /// # Safety
+ /// Must be called from `sm_map` or `sm_unmap` with a pointer to `SmMapData` or `SmData`.
pub(super) unsafe extern "C" fn sm_step_unmap(
op: *mut bindings::drm_gpuva_op,
p: *mut c_void,
) -> c_int {
- // SAFETY: The caller provides a pointer to `SmData`.
+ // SAFETY: The caller provides a pointer that can be treated as `SmData`.
let p = unsafe { &mut *p.cast::<SmData<'_, '_, T>>() };
let op = OpUnmap {
// SAFETY: sm_step_unmap is called with an unmap operation.
}
/// # Safety
- /// Must be called from `sm_unmap` with a pointer to `SmData`.
+ /// Must be called from `sm_map` or `sm_unmap` with a pointer to `SmMapData` or `SmData`.
pub(super) unsafe extern "C" fn sm_step_remap(
op: *mut bindings::drm_gpuva_op,
p: *mut c_void,
) -> c_int {
- // SAFETY: The caller provides a pointer to `SmData`.
+ // SAFETY: The caller provides a pointer that can be treated as `SmData`.
let p = unsafe { &mut *p.cast::<SmData<'_, '_, T>>() };
let op = OpRemap {
// SAFETY: sm_step_remap is called with a remap operation.
projects / thirdparty / kernel / linux.git / commitdiff
