dma_buf_put(dma_buf);
- return to_xdna_obj(gobj);
+ abo = to_xdna_obj(gobj);
+ abo->private_buffer = true;
+
+ return abo;
}
+static struct amdxdna_gem_obj *
+amdxdna_gem_create_cbuf_object(struct drm_device *dev, struct amdxdna_drm_create_bo *args)
+{
+ struct amdxdna_dev *xdna = to_xdna_dev(dev);
+ size_t size = PAGE_ALIGN(args->size);
+ struct drm_gem_object *gobj;
+ struct amdxdna_gem_obj *ret;
+ struct dma_buf *dma_buf;
+ u64 align;
+
+ if (!size) {
+ XDNA_ERR(xdna, "Invalid BO size 0x%llx", args->size);
+ return ERR_PTR(-EINVAL);
+ }
+
+ align = (args->type == AMDXDNA_BO_DEV_HEAP) ? xdna->dev_info->dev_mem_size : 0;
+ dma_buf = amdxdna_get_cbuf(dev, size, align);
+ if (IS_ERR(dma_buf))
+ return ERR_CAST(dma_buf);
+
+ gobj = amdxdna_gem_prime_import(dev, dma_buf);
+ if (IS_ERR(gobj))
+ ret = ERR_CAST(gobj);
+ else
+ ret = to_xdna_obj(gobj);
+
+ dma_buf_put(dma_buf);
+ return ret;
+}
+
struct drm_gem_object *
amdxdna_gem_prime_import(struct drm_device *dev, struct dma_buf *dma_buf)
{
.map_dma_buf = amdxdna_ubuf_map,
.unmap_dma_buf = amdxdna_ubuf_unmap,
.release = amdxdna_ubuf_release,
- .mmap = amdxdna_ubuf_mmap,
- .vmap = amdxdna_ubuf_vmap,
- .vunmap = amdxdna_ubuf_vunmap,
};
+static int readonly_va_entry(struct amdxdna_drm_va_entry *va_ent)
+{
+ struct mm_struct *mm = current->mm;
+ struct vm_area_struct *vma;
+ int ret;
+
+ mmap_read_lock(mm);
+
+ vma = find_vma(mm, va_ent->vaddr);
+ if (!vma ||
+ vma->vm_start > va_ent->vaddr ||
+ vma->vm_end - va_ent->vaddr < va_ent->len)
+ ret = -ENOENT;
+ else
+ ret = vma->vm_flags & VM_WRITE ? 0 : 1;
+
+ mmap_read_unlock(mm);
+ return ret;
+}
+
struct dma_buf *amdxdna_get_ubuf(struct drm_device *dev,
u32 num_entries, void __user *va_entries)
{
if ((sdp->sdp_header.HB3 & 0x3F) != 9)
return -EINVAL;
- as_sdp->length = sdp->sdp_header.HB3 & DP_ADAPTIVE_SYNC_SDP_LENGTH;
- as_sdp->mode = sdp->db[0] & DP_ADAPTIVE_SYNC_SDP_OPERATION_MODE;
+ as_sdp->length = sdp->sdp_header.HB3 & DP_AS_SDP_LENGTH_MASK;
+ as_sdp->mode = sdp->db[0] & DP_AS_SDP_OPERATION_MODE_MASK;
as_sdp->vtotal = (sdp->db[2] << 8) | sdp->db[1];
- as_sdp->target_rr = (u64)sdp->db[3] | ((u64)sdp->db[4] & 0x3);
+ as_sdp->target_rr = ((sdp->db[4] & 0x3) << 8) | sdp->db[3];
as_sdp->target_rr_divider = sdp->db[4] & 0x20 ? true : false;
return 0;
#include "intel_display_wa.h"
#include "intel_dmc.h"
#include "intel_dp.h"
+ #include "intel_dpcd.h"
#include "intel_dp_aux.h"
+#include "intel_dp_tunnel.h"
#include "intel_dsb.h"
#include "intel_frontbuffer.h"
#include "intel_hdmi.h"
/** @devfreq: Device frequency scaling management data. */
struct panthor_devfreq *devfreq;
- /** @reclaim.lock: Lock protecting all LRUs */
- struct mutex lock;
-
+ /** @reclaim: Reclaim related stuff */
+ struct {
+ /** @reclaim.shrinker: Shrinker instance */
+ struct shrinker *shrinker;
+
+ /**
+ * @reclaim.unused: BOs with unused pages
+ *
+ * Basically all buffers that got mmapped, vmapped or GPU mapped and
+ * then unmapped. There should be no contention on these buffers,
+ * making them ideal to reclaim.
+ */
+ struct drm_gem_lru unused;
+
+ /**
+ * @reclaim.mmapped: mmap()-ed buffers
+ *
+ * Those are relatively easy to reclaim since we don't need user
+ * agreement, we can simply teardown the mapping and let it fault on
+ * the next access.
+ */
+ struct drm_gem_lru mmapped;
+
+ /**
+ * @reclaim.gpu_mapped_shared: shared BO LRU list
+ *
+ * That's the most tricky BO type to reclaim, because it involves
+ * tearing down all mappings in all VMs where this BO is mapped,
+ * which increases the risk of contention and thus decreases the
+ * likeliness of success.
+ */
+ struct drm_gem_lru gpu_mapped_shared;
+
+ /**
+ * @reclaim.vms: VM LRU list
+ *
+ * VMs that have reclaimable BOs only mapped to a single VM are placed
+ * in this LRU. Reclaiming such BOs implies waiting for VM idleness
+ * (no in-flight GPU jobs targeting this VM), meaning we can't reclaim
+ * those if we're in a context where we can't block/sleep.
+ */
+ struct list_head vms;
+
+ /**
+ * @reclaim.gpu_mapped_count: Global counter of pages that are GPU mapped
+ *
+ * Allows us to get the number of reclaimable pages without walking
+ * the vms and gpu_mapped_shared LRUs.
+ */
+ long gpu_mapped_count;
+
+ /**
+ * @reclaim.retry_count: Number of times we ran the shrinker without being
+ * able to reclaim stuff
+ *
+ * Used to stop scanning GEMs when too many attempts were made
+ * without progress.
+ */
+ atomic_t retry_count;
+
+#ifdef CONFIG_DEBUG_FS
+ /**
+ * @reclaim.nr_pages_reclaimed_on_last_scan: Number of pages reclaimed on the last
+ * shrinker scan
+ */
+ unsigned long nr_pages_reclaimed_on_last_scan;
+#endif
+ } reclaim;
+
/** @unplug: Device unplug related fields. */
struct {
/** @lock: Lock used to serialize unplug operations. */
*
* for the flush+invalidate case.
*/
- dma_sync_single_for_device(dev->dev, paddr, len, DMA_TO_DEVICE);
+ dma_sync_single_for_device(dma_dev, paddr, len, DMA_TO_DEVICE);
if (type == DRM_PANTHOR_BO_SYNC_CPU_CACHE_FLUSH_AND_INVALIDATE)
- dma_sync_single_for_cpu(dev->dev, paddr, len, DMA_FROM_DEVICE);
+ dma_sync_single_for_cpu(dma_dev, paddr, len, DMA_FROM_DEVICE);
+ }
+
+ ret = 0;
+
+out_unlock:
+ dma_resv_unlock(bo->base.resv);
+ return ret;
+}
+
+/**
+ * panthor_kernel_bo_destroy() - Destroy a kernel buffer object
+ * @bo: Kernel buffer object to destroy. If NULL or an ERR_PTR(), the destruction
+ * is skipped.
+ */
+void panthor_kernel_bo_destroy(struct panthor_kernel_bo *bo)
+{
+ struct panthor_device *ptdev;
+ struct panthor_vm *vm;
+
+ if (IS_ERR_OR_NULL(bo))
+ return;
+
+ ptdev = container_of(bo->obj->dev, struct panthor_device, base);
+ vm = bo->vm;
+ panthor_kernel_bo_vunmap(bo);
+
+ drm_WARN_ON(bo->obj->dev,
+ to_panthor_bo(bo->obj)->exclusive_vm_root_gem != panthor_vm_root_gem(vm));
+ panthor_vm_unmap_range(vm, bo->va_node.start, bo->va_node.size);
+ panthor_vm_free_va(vm, &bo->va_node);
+ if (vm == panthor_fw_vm(ptdev))
+ panthor_gem_unpin(to_panthor_bo(bo->obj));
+ drm_gem_object_put(bo->obj);
+ panthor_vm_put(vm);
+ kfree(bo);
+}
+
+/**
+ * panthor_kernel_bo_create() - Create and map a GEM object to a VM
+ * @ptdev: Device.
+ * @vm: VM to map the GEM to.
+ * @size: Size of the buffer object.
+ * @bo_flags: Combination of drm_panthor_bo_flags flags.
+ * @vm_map_flags: Combination of drm_panthor_vm_bind_op_flags (only those
+ * that are related to map operations).
+ * @gpu_va: GPU address assigned when mapping to the VM.
+ * If gpu_va == PANTHOR_VM_KERNEL_AUTO_VA, the virtual address will be
+ * automatically allocated.
+ * @name: Descriptive label of the BO's contents
+ *
+ * Return: A valid pointer in case of success, an ERR_PTR() otherwise.
+ */
+struct panthor_kernel_bo *
+panthor_kernel_bo_create(struct panthor_device *ptdev, struct panthor_vm *vm,
+ size_t size, u32 bo_flags, u32 vm_map_flags,
+ u64 gpu_va, const char *name)
+{
+ struct panthor_kernel_bo *kbo;
+ struct panthor_gem_object *bo;
+ u32 debug_flags = PANTHOR_DEBUGFS_GEM_USAGE_FLAG_KERNEL;
+ int ret;
+
+ if (drm_WARN_ON(&ptdev->base, !vm))
+ return ERR_PTR(-EINVAL);
+
+ kbo = kzalloc_obj(*kbo);
+ if (!kbo)
+ return ERR_PTR(-ENOMEM);
+
+ if (vm == panthor_fw_vm(ptdev))
+ debug_flags |= PANTHOR_DEBUGFS_GEM_USAGE_FLAG_FW_MAPPED;
+
+ bo = panthor_gem_create(&ptdev->base, size, bo_flags, vm, debug_flags);
+ if (IS_ERR(bo)) {
+ ret = PTR_ERR(bo);
+ goto err_free_kbo;
+ }
+
+ kbo->obj = &bo->base;
+
+ if (vm == panthor_fw_vm(ptdev)) {
+ ret = panthor_gem_pin(bo);
+ if (ret)
+ goto err_put_obj;
+ }
+
+ panthor_gem_kernel_bo_set_label(kbo, name);
+
+ /* The system and GPU MMU page size might differ, which becomes a
+ * problem for FW sections that need to be mapped at explicit address
+ * since our PAGE_SIZE alignment might cover a VA range that's
+ * expected to be used for another section.
+ * Make sure we never map more than we need.
+ */
+ size = ALIGN(size, panthor_vm_page_size(vm));
+ ret = panthor_vm_alloc_va(vm, gpu_va, size, &kbo->va_node);
+ if (ret)
+ goto err_unpin;
+
+ ret = panthor_vm_map_bo_range(vm, bo, 0, size, kbo->va_node.start, vm_map_flags);
+ if (ret)
+ goto err_free_va;
+
+ kbo->vm = panthor_vm_get(vm);
+ return kbo;
+
+err_free_va:
+ panthor_vm_free_va(vm, &kbo->va_node);
+
+err_unpin:
+ if (vm == panthor_fw_vm(ptdev))
+ panthor_gem_unpin(bo);
+
+err_put_obj:
+ drm_gem_object_put(&bo->base);
+
+err_free_kbo:
+ kfree(kbo);
+ return ERR_PTR(ret);
+}
+
+static bool can_swap(void)
+{
+ return get_nr_swap_pages() > 0;
+}
+
+static bool can_block(struct shrink_control *sc)
+{
+ /* If direct reclaim is allowed, we can always block.
+ * If kswapd reclaim is allowed, we can block, but only if we're called
+ * by the kswapd thread.
+ */
+ return (sc->gfp_mask & __GFP_DIRECT_RECLAIM) ||
+ ((sc->gfp_mask & __GFP_KSWAPD_RECLAIM) && current_is_kswapd());
+}
+
+static unsigned long
+panthor_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
+{
+ struct panthor_device *ptdev = shrinker->private_data;
+ unsigned long count;
+
+ /* We currently don't have a flag to tell when the content of a
+ * BO can be discarded.
+ */
+ if (!can_swap())
+ return 0;
+
+ /* This is racy, but that's okay because the returned count is just a
+ * hint. That's also what MSM is doing (no atomic var, it's relying on
+ * the fact unsigned long access is usually atomic), so if it's good
+ * enough for them, it's good enough for us too.
+ */
+ count = ptdev->reclaim.unused.count;
+ count += ptdev->reclaim.mmapped.count;
+
+ if (can_block(sc))
+ count += ptdev->reclaim.gpu_mapped_count;
+
+ return count ? count : SHRINK_EMPTY;
+}
+
+static bool panthor_gem_try_evict_no_resv_wait(struct drm_gem_object *obj,
+ struct ww_acquire_ctx *ticket)
+{
+ /*
+ * Track last locked entry for unwinding locks in error and
+ * success paths
+ */
+ struct panthor_gem_object *bo = to_panthor_bo(obj);
+ struct drm_gpuvm_bo *vm_bo, *last_locked = NULL;
+ enum panthor_gem_reclaim_state old_state;
+ int ret = 0;
+
+ /* To avoid potential lock ordering issue between bo_gpuva and
+ * mapping->i_mmap_rwsem, unmap the pages from CPU side before
+ * acquring the bo_gpuva lock. As the bo_resv lock is held, CPU
+ * page fault handler won't be able to map in the pages whilst
+ * eviction is in progress.
+ */
+ drm_vma_node_unmap(&bo->base.vma_node, bo->base.dev->anon_inode->i_mapping);
+
+ /* We take this lock when walking the list to prevent
+ * insertion/deletion.
+ */
+ /* We can only trylock in that path, because
+ * - allocation might happen while some of these locks are held
+ * - lock ordering is different in other paths
+ * vm_resv -> bo_resv -> bo_gpuva
+ * vs
+ * bo_resv -> bo_gpuva -> vm_resv
+ *
+ * If we fail to lock that's fine, we back off and will get
+ * back to it later.
+ */
+ if (!mutex_trylock(&bo->base.gpuva.lock))
+ return false;
+
+ drm_gem_for_each_gpuvm_bo(vm_bo, obj) {
+ struct dma_resv *resv = drm_gpuvm_resv(vm_bo->vm);
+
+ if (resv == obj->resv)
+ continue;
+
+ if (!dma_resv_trylock(resv)) {
+ ret = -EDEADLK;
+ goto out_unlock;
+ }
+
+ last_locked = vm_bo;
+ }
+
+ /* Update the state before trying to evict the buffer, if the state was
+ * updated to something that's harder to reclaim (higher value in the
+ * enum), skip it (will be processed when the relevant LRU is).
+ */
+ panthor_gem_update_reclaim_state_locked(bo, &old_state);
+ if (old_state < bo->reclaim_state) {
+ ret = -EAGAIN;
+ goto out_unlock;
+ }
+
+ /* Couldn't teardown the GPU mappings? Skip. */
+ ret = panthor_vm_evict_bo_mappings_locked(bo);
+ if (ret)
+ goto out_unlock;
+
+ /* If everything went fine, evict the object. */
+ panthor_gem_evict_locked(bo);
+
+out_unlock:
+ if (last_locked) {
+ drm_gem_for_each_gpuvm_bo(vm_bo, obj) {
+ struct dma_resv *resv = drm_gpuvm_resv(vm_bo->vm);
+
+ if (resv == obj->resv)
+ continue;
+
+ dma_resv_unlock(resv);
+
+ if (last_locked == vm_bo)
+ break;
+ }
}
- freed += drm_gem_lru_scan(&ptdev->reclaim.unused,
+ mutex_unlock(&bo->base.gpuva.lock);
+
+ return ret == 0;
+}
+
+static bool panthor_gem_try_evict(struct drm_gem_object *obj,
+ struct ww_acquire_ctx *ticket)
+{
+ struct panthor_gem_object *bo = to_panthor_bo(obj);
+
+ /* Wait was too long, skip. */
+ if (dma_resv_wait_timeout(obj->resv, DMA_RESV_USAGE_BOOKKEEP, false, 10) <= 0)
+ return false;
+
+ return panthor_gem_try_evict_no_resv_wait(&bo->base, ticket);
+}
+
+static unsigned long
+panthor_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
+{
+ struct panthor_device *ptdev = shrinker->private_data;
+ unsigned long remaining = 0;
+ unsigned long freed = 0;
+
+ if (!can_swap())
+ goto out;
+
- freed += drm_gem_lru_scan(&ptdev->reclaim.mmapped,
++ freed += drm_gem_lru_scan(&ptdev->base, &ptdev->reclaim.unused,
+ sc->nr_to_scan - freed, &remaining,
+ panthor_gem_try_evict_no_resv_wait, NULL);
+ if (freed >= sc->nr_to_scan)
+ goto out;
+
- freed += drm_gem_lru_scan(&ptdev->reclaim.gpu_mapped_shared,
++ freed += drm_gem_lru_scan(&ptdev->base, &ptdev->reclaim.mmapped,
+ sc->nr_to_scan - freed, &remaining,
+ panthor_gem_try_evict_no_resv_wait, NULL);
+ if (freed >= sc->nr_to_scan)
+ goto out;
+
+ if (!can_block(sc))
+ goto out;
+
+ freed += panthor_mmu_reclaim_priv_bos(ptdev, sc->nr_to_scan - freed,
+ &remaining, panthor_gem_try_evict);
+ if (freed >= sc->nr_to_scan)
+ goto out;
+
- int ret;
-
- ret = drmm_mutex_init(&ptdev->base, &ptdev->reclaim.lock);
- if (ret)
- return ret;
++ freed += drm_gem_lru_scan(&ptdev->base, &ptdev->reclaim.gpu_mapped_shared,
+ sc->nr_to_scan - freed, &remaining,
+ panthor_gem_try_evict, NULL);
+
+out:
+#ifdef CONFIG_DEBUG_FS
+ /* This is racy, but that's okay, because this is just debugfs
+ * reporting and doesn't need to be accurate.
+ */
+ ptdev->reclaim.nr_pages_reclaimed_on_last_scan = freed;
+#endif
+
+ /* If there are things to reclaim, try a couple times before giving up. */
+ if (!freed && remaining > 0 &&
+ atomic_inc_return(&ptdev->reclaim.retry_count) < 2)
+ return 0;
+
+ atomic_set(&ptdev->reclaim.retry_count, 0);
+
+ if (freed)
+ return freed;
+
+ /* There's nothing left to reclaim, or the resources are contended. Give up now. */
+ return SHRINK_STOP;
+}
+
+int panthor_gem_shrinker_init(struct panthor_device *ptdev)
+{
+ struct shrinker *shrinker;
- drm_gem_lru_init(&ptdev->reclaim.unused, &ptdev->reclaim.lock);
- drm_gem_lru_init(&ptdev->reclaim.mmapped, &ptdev->reclaim.lock);
- drm_gem_lru_init(&ptdev->reclaim.gpu_mapped_shared, &ptdev->reclaim.lock);
+
+ INIT_LIST_HEAD(&ptdev->reclaim.vms);
- might_lock(&ptdev->reclaim.lock);
++ drm_gem_lru_init(&ptdev->reclaim.unused);
++ drm_gem_lru_init(&ptdev->reclaim.mmapped);
++ drm_gem_lru_init(&ptdev->reclaim.gpu_mapped_shared);
+ ptdev->reclaim.gpu_mapped_count = 0;
+
+ /* Teach lockdep about lock ordering wrt. shrinker: */
+ fs_reclaim_acquire(GFP_KERNEL);
++ might_lock(&ptdev->base.gem_lru_mutex);
+ fs_reclaim_release(GFP_KERNEL);
+
+ shrinker = shrinker_alloc(0, "drm-panthor-gem");
+ if (!shrinker)
+ return -ENOMEM;
+ shrinker->count_objects = panthor_gem_shrinker_count;
+ shrinker->scan_objects = panthor_gem_shrinker_scan;
+ shrinker->private_data = ptdev;
+ ptdev->reclaim.shrinker = shrinker;
+
+ shrinker_register(shrinker);
return 0;
}
if (refcount_inc_not_zero(&vm->as.active_cnt))
goto out_dev_exit;
- mutex_lock(&ptdev->reclaim.lock);
+ /* As soon as active is called, we place the VM at the end of the VM LRU.
+ * If something fails after that, the only downside is that this VM that
+ * never became active in the first place will be reclaimed last, but
+ * that's an acceptable trade-off.
+ */
- mutex_unlock(&ptdev->reclaim.lock);
++ mutex_lock(&ptdev->base.gem_lru_mutex);
+ if (vm->reclaim.lru.count)
+ list_move_tail(&vm->reclaim.lru_node, &ptdev->reclaim.vms);
++ mutex_unlock(&ptdev->base.gem_lru_mutex);
+
/* Make sure we don't race with lock/unlock_region() calls
* happening around VM bind operations.
*/
struct panthor_vm *vm = container_of(gpuvm, struct panthor_vm, base);
struct panthor_device *ptdev = vm->ptdev;
- mutex_lock(&ptdev->reclaim.lock);
++ mutex_lock(&ptdev->base.gem_lru_mutex);
+ list_del_init(&vm->reclaim.lru_node);
- mutex_unlock(&ptdev->reclaim.lock);
++ mutex_unlock(&ptdev->base.gem_lru_mutex);
+
mutex_lock(&vm->heaps.lock);
if (drm_WARN_ON(&ptdev->base, vm->heaps.pool))
panthor_heap_pool_destroy(vm->heaps.pool);
return 0;
}
- mutex_lock(&ptdev->reclaim.lock);
+void panthor_vm_update_bo_reclaim_lru_locked(struct panthor_gem_object *bo)
+{
+ struct panthor_device *ptdev = container_of(bo->base.dev, struct panthor_device, base);
+ struct panthor_vm *vm = NULL;
+ struct drm_gpuvm_bo *vm_bo;
+
+ dma_resv_assert_held(bo->base.resv);
+ lockdep_assert_held(&bo->base.gpuva.lock);
+
+ drm_gem_for_each_gpuvm_bo(vm_bo, &bo->base) {
+ if (vm_bo->evicted)
+ continue;
+
+ /* We're only supposed to have one non-evicted vm_bo in the list if we get
+ * there.
+ */
+ drm_WARN_ON(&ptdev->base, vm);
+ vm = container_of(vm_bo->vm, struct panthor_vm, base);
+
- mutex_unlock(&ptdev->reclaim.lock);
++ mutex_lock(&ptdev->base.gem_lru_mutex);
+ drm_gem_lru_move_tail_locked(&vm->reclaim.lru, &bo->base);
+ if (list_empty(&vm->reclaim.lru_node))
+ list_move(&vm->reclaim.lru_node, &ptdev->reclaim.vms);
++ mutex_unlock(&ptdev->base.gem_lru_mutex);
+ }
+}
+
+int panthor_vm_evict_bo_mappings_locked(struct panthor_gem_object *bo)
+{
+ struct drm_gpuvm_bo *vm_bo;
+ int ret = 0;
+
+ drm_gem_for_each_gpuvm_bo(vm_bo, &bo->base) {
+ struct panthor_vm *vm = container_of(vm_bo->vm, struct panthor_vm, base);
+ struct drm_gpuva *va;
+
+ if (!mutex_trylock(&vm->op_lock))
+ return -EDEADLK;
+
+ /* It can be that the vm_bo was already evicted but a new
+ * mapping pointing to this BO got created in the meantime,
+ * thus turning the vm_bo in partially evicted state. In that case
+ * we don't call drm_gpuvm_bo_evict() again because this would
+ * mess up with the internal gpuvm lists, but we do walk the
+ * VAs on this vm_bo to make sure the non-evicted ones are
+ * torn down.
+ */
+ if (!vm_bo->evicted)
+ drm_gpuvm_bo_evict(vm_bo, true);
+
+ drm_gpuvm_bo_for_each_va(va, vm_bo) {
+ struct panthor_vma *vma = container_of(va, struct panthor_vma, base);
+
+ if (vma->evicted)
+ continue;
+
+ /* If something fail in the middle of a VM_BO eviction, the VM_BO
+ * is considered fully evicted, but some of its VMAs might still be
+ * active. That's okay because the pages won't be released if this
+ * function returns an error.
+ *
+ * On the next job targeting this VM, the partially evicted VM_BO
+ * will be validated, causing all its evicted VMAs to be repopulated
+ * before the job runs. So no GPU fault expected.
+ */
+ ret = panthor_vm_lock_region(vm, va->va.addr, va->va.range);
+ if (ret)
+ break;
+
+ panthor_vm_unmap_pages(vm, va->va.addr, va->va.range);
+ panthor_vm_unlock_region(vm);
+ vma->evicted = true;
+ }
+
+ mutex_unlock(&vm->op_lock);
+
+ if (ret)
+ break;
+ }
+
+ return ret;
+}
+
+static struct panthor_vma *select_evicted_vma(struct drm_gpuvm_bo *vm_bo,
+ struct panthor_vm_op_ctx *op_ctx)
+{
+ struct panthor_vm *vm = container_of(vm_bo->vm, struct panthor_vm, base);
+ struct panthor_vma *first_evicted_vma = NULL;
+ struct drm_gpuva *va;
+
+ /* Take op_lock to protect against va insertion/removal. */
+ mutex_lock(&vm->op_lock);
+ drm_gpuvm_bo_for_each_va(va, vm_bo) {
+ struct panthor_vma *vma = container_of(va, struct panthor_vma, base);
+
+ if (vma->evicted) {
+ first_evicted_vma = vma;
+ panthor_vm_init_op_ctx(op_ctx, va->va.range, va->va.addr, vma->flags);
+ op_ctx->map.bo_offset = va->gem.offset;
+ break;
+ }
+ }
+ mutex_unlock(&vm->op_lock);
+
+ return first_evicted_vma;
+}
+
+static int remap_evicted_vma(struct drm_gpuvm_bo *vm_bo,
+ struct panthor_vma *evicted_vma,
+ struct panthor_vm_op_ctx *op_ctx)
+{
+ struct panthor_vm *vm = container_of(vm_bo->vm, struct panthor_vm, base);
+ struct panthor_gem_object *bo = to_panthor_bo(vm_bo->obj);
+ struct drm_gpuva *va;
+ bool found = false;
+ int ret;
+
+ ret = panthor_vm_op_ctx_prealloc_pts(op_ctx);
+ if (ret)
+ goto out_cleanup;
+
+ /* Take op_lock to protect against va insertion/removal. Note that the
+ * evicted_vma selection was done with the same lock held, but we had
+ * to release it so we can allocate PTs, because this very same lock
+ * is taken in a DMA-signalling path.
+ */
+ mutex_lock(&vm->op_lock);
+ drm_gpuvm_bo_for_each_va(va, vm_bo) {
+ struct panthor_vma *vma = container_of(va, struct panthor_vma, base);
+
+ if (vma != evicted_vma)
+ continue;
+
+ /* Because we had to release the lock between the evicted_vma selection
+ * and its repopulation, we can't rely solely on pointer equality (the
+ * VMA might have been freed and a new one allocated at the same address).
+ * If the evicted bit is still set, we're sure it's our VMA, because
+ * population/eviction is serialized with the BO resv lock.
+ */
+ if (vma->evicted)
+ found = true;
+
+ break;
+ }
+
+ if (found) {
+ vm->op_ctx = op_ctx;
+ ret = panthor_vm_lock_region(vm, evicted_vma->base.va.addr,
+ evicted_vma->base.va.range);
+ if (!ret) {
+ ret = panthor_vm_map_pages(vm, evicted_vma->base.va.addr,
+ flags_to_prot(evicted_vma->flags),
+ bo->dmap.sgt,
+ evicted_vma->base.gem.offset,
+ evicted_vma->base.va.range);
+ if (!ret)
+ evicted_vma->evicted = false;
+
+ panthor_vm_unlock_region(vm);
+ }
+
+ vm->op_ctx = NULL;
+ }
+
+ mutex_unlock(&vm->op_lock);
+
+out_cleanup:
+ panthor_vm_cleanup_op_ctx(op_ctx, vm);
+ return ret;
+}
+
+static int panthor_vm_restore_vmas(struct drm_gpuvm_bo *vm_bo)
+{
+ struct panthor_vm *vm = container_of(vm_bo->vm, struct panthor_vm, base);
+ struct panthor_gem_object *bo = to_panthor_bo(vm_bo->obj);
+ struct panthor_vm_op_ctx op_ctx;
+
+ if (drm_WARN_ON_ONCE(&vm->ptdev->base, !bo->dmap.sgt))
+ return -EINVAL;
+
+ for (struct panthor_vma *vma = select_evicted_vma(vm_bo, &op_ctx);
+ vma; vma = select_evicted_vma(vm_bo, &op_ctx)) {
+ int ret;
+
+ ret = remap_evicted_vma(vm_bo, vma, &op_ctx);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+
+static int panthor_vm_bo_validate(struct drm_gpuvm_bo *vm_bo,
+ struct drm_exec *exec)
+{
+ struct panthor_gem_object *bo = to_panthor_bo(vm_bo->obj);
+ int ret;
+
+ ret = panthor_gem_swapin_locked(bo);
+ if (ret)
+ return ret;
+
+ ret = panthor_vm_restore_vmas(vm_bo);
+ if (ret)
+ return ret;
+
+ drm_gpuvm_bo_evict(vm_bo, false);
+ mutex_lock(&bo->base.gpuva.lock);
+ panthor_gem_update_reclaim_state_locked(bo, NULL);
+ mutex_unlock(&bo->base.gpuva.lock);
+ return 0;
+}
+
static const struct drm_gpuvm_ops panthor_gpuvm_ops = {
.vm_free = panthor_vm_free,
.vm_bo_free = panthor_vm_bo_free,
vm->kernel_auto_va.start = auto_kernel_va_start;
vm->kernel_auto_va.end = vm->kernel_auto_va.start + auto_kernel_va_size - 1;
- drm_gem_lru_init(&vm->reclaim.lru, &ptdev->reclaim.lock);
++ drm_gem_lru_init(&vm->reclaim.lru);
+ INIT_LIST_HEAD(&vm->reclaim.lru_node);
INIT_LIST_HEAD(&vm->node);
INIT_LIST_HEAD(&vm->as.lru_node);
vm->as.id = -1;
if (ret)
return ret;
- return drm_gpuvm_prepare_objects(&vm->base, exec, slot_count);
+ ret = drm_gpuvm_prepare_objects(&vm->base, exec, slot_count);
+ if (ret)
+ return ret;
+
+ return drm_gpuvm_validate(&vm->base, exec);
+}
+
+unsigned long
+panthor_mmu_reclaim_priv_bos(struct panthor_device *ptdev,
+ unsigned int nr_to_scan, unsigned long *remaining,
+ bool (*shrink)(struct drm_gem_object *,
+ struct ww_acquire_ctx *))
+{
+ unsigned long freed = 0;
+ LIST_HEAD(remaining_vms);
+ LIST_HEAD(vms);
+
- mutex_lock(&ptdev->reclaim.lock);
++ mutex_lock(&ptdev->base.gem_lru_mutex);
+ list_splice_init(&ptdev->reclaim.vms, &vms);
+
+ while (freed < nr_to_scan) {
+ struct panthor_vm *vm;
+
+ vm = list_first_entry_or_null(&vms, typeof(*vm),
+ reclaim.lru_node);
+ if (!vm)
+ break;
+
+ if (!kref_get_unless_zero(&vm->base.kref)) {
+ list_del_init(&vm->reclaim.lru_node);
+ continue;
+ }
+
- mutex_unlock(&ptdev->reclaim.lock);
++ mutex_unlock(&ptdev->base.gem_lru_mutex);
+
- freed += drm_gem_lru_scan(&vm->reclaim.lru, nr_to_scan - freed,
++ freed += drm_gem_lru_scan(&ptdev->base, &vm->reclaim.lru,
++ nr_to_scan - freed,
+ remaining, shrink, NULL);
+
- mutex_lock(&ptdev->reclaim.lock);
++ mutex_lock(&ptdev->base.gem_lru_mutex);
+
+ /* If the VM is still in the temporary list, remove it so we
+ * can proceed with the next VM.
+ */
+ if (vm == list_first_entry_or_null(&vms, typeof(*vm), reclaim.lru_node)) {
+ list_del_init(&vm->reclaim.lru_node);
+
+ /* Keep the VM around if there are still things to
+ * reclaim, so we can preserve the LRU order when
+ * re-inserting in ptdev->reclaim.vms at the end.
+ */
+ if (vm->reclaim.lru.count > 0)
+ list_add_tail(&vm->reclaim.lru_node, &remaining_vms);
+ }
+
- mutex_unlock(&ptdev->reclaim.lock);
++ mutex_unlock(&ptdev->base.gem_lru_mutex);
+
+ panthor_vm_put(vm);
+
- mutex_lock(&ptdev->reclaim.lock);
++ mutex_lock(&ptdev->base.gem_lru_mutex);
+ }
+
+ /* Re-insert VMs with remaining data to reclaim at the beginning of
+ * the LRU. Note that any activeness change on the VM that happened
+ * while we were reclaiming would have moved the VM out of our
+ * temporary [remaining_]vms list, meaning anything we re-insert here
+ * preserves the LRU order.
+ */
+ list_splice_tail(&vms, &remaining_vms);
+ list_splice(&remaining_vms, &ptdev->reclaim.vms);
- mutex_unlock(&ptdev->reclaim.lock);
++ mutex_unlock(&ptdev->base.gem_lru_mutex);
+
+ return freed;
}
/**
projects / thirdparty / linux.git / commitdiff
