From: Simona Vetter Date: Thu, 28 May 2026 07:56:06 +0000 (+0200) Subject: Merge v7.1-rc5 into drm-next X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=bed29492d413349e5b13f21936655064cdb63c91;p=thirdparty%2Flinux.git Merge v7.1-rc5 into drm-next Boris Brezillion needs the gem lru fixes 379e8f1ca5e9 ("drm/gem: Make the GEM LRU lock part of drm_device") backmerged for drm-misc-next. That also means we need to sort out the rename conflict in panthor with the fixup patch from Boris from drm-tip. Signed-off-by: Simona Vetter --- bed29492d413349e5b13f21936655064cdb63c91 diff --cc drivers/accel/amdxdna/amdxdna_gem.c index 2dfdc56ba91d4,6e367ddb9e1be..00efa8abfeeac --- a/drivers/accel/amdxdna/amdxdna_gem.c +++ b/drivers/accel/amdxdna/amdxdna_gem.c @@@ -804,39 -715,12 +808,42 @@@ amdxdna_gem_create_ubuf_object(struct d dma_buf_put(dma_buf); - return to_xdna_obj(gobj); + abo = to_xdna_obj(gobj); + abo->private_buffer = true; + + return abo; } +static struct amdxdna_gem_obj * +amdxdna_gem_create_cbuf_object(struct drm_device *dev, struct amdxdna_drm_create_bo *args) +{ + struct amdxdna_dev *xdna = to_xdna_dev(dev); + size_t size = PAGE_ALIGN(args->size); + struct drm_gem_object *gobj; + struct amdxdna_gem_obj *ret; + struct dma_buf *dma_buf; + u64 align; + + if (!size) { + XDNA_ERR(xdna, "Invalid BO size 0x%llx", args->size); + return ERR_PTR(-EINVAL); + } + + align = (args->type == AMDXDNA_BO_DEV_HEAP) ? xdna->dev_info->dev_mem_size : 0; + dma_buf = amdxdna_get_cbuf(dev, size, align); + if (IS_ERR(dma_buf)) + return ERR_CAST(dma_buf); + + gobj = amdxdna_gem_prime_import(dev, dma_buf); + if (IS_ERR(gobj)) + ret = ERR_CAST(gobj); + else + ret = to_xdna_obj(gobj); + + dma_buf_put(dma_buf); + return ret; +} + struct drm_gem_object * amdxdna_gem_prime_import(struct drm_device *dev, struct dma_buf *dma_buf) { diff --cc drivers/accel/amdxdna/amdxdna_ubuf.c index 3769210c55ccd,85390e3cc9f98..bb60fb80467ef --- a/drivers/accel/amdxdna/amdxdna_ubuf.c +++ b/drivers/accel/amdxdna/amdxdna_ubuf.c @@@ -120,31 -73,8 +73,28 @@@ static const struct dma_buf_ops amdxdna .map_dma_buf = amdxdna_ubuf_map, .unmap_dma_buf = amdxdna_ubuf_unmap, .release = amdxdna_ubuf_release, - .mmap = amdxdna_ubuf_mmap, - .vmap = amdxdna_ubuf_vmap, - .vunmap = amdxdna_ubuf_vunmap, }; +static int readonly_va_entry(struct amdxdna_drm_va_entry *va_ent) +{ + struct mm_struct *mm = current->mm; + struct vm_area_struct *vma; + int ret; + + mmap_read_lock(mm); + + vma = find_vma(mm, va_ent->vaddr); + if (!vma || + vma->vm_start > va_ent->vaddr || + vma->vm_end - va_ent->vaddr < va_ent->len) + ret = -ENOENT; + else + ret = vma->vm_flags & VM_WRITE ? 0 : 1; + + mmap_read_unlock(mm); + return ret; +} + struct dma_buf *amdxdna_get_ubuf(struct drm_device *dev, u32 num_entries, void __user *va_entries) { diff --cc drivers/gpu/drm/i915/display/intel_dp.c index 61ccaf0a46b69,6ef2a0043cdaf..f01a6eed38395 --- a/drivers/gpu/drm/i915/display/intel_dp.c +++ b/drivers/gpu/drm/i915/display/intel_dp.c @@@ -5373,10 -5300,10 +5373,10 @@@ int intel_dp_as_sdp_unpack(struct drm_d if ((sdp->sdp_header.HB3 & 0x3F) != 9) return -EINVAL; - as_sdp->length = sdp->sdp_header.HB3 & DP_ADAPTIVE_SYNC_SDP_LENGTH; - as_sdp->mode = sdp->db[0] & DP_ADAPTIVE_SYNC_SDP_OPERATION_MODE; + as_sdp->length = sdp->sdp_header.HB3 & DP_AS_SDP_LENGTH_MASK; + as_sdp->mode = sdp->db[0] & DP_AS_SDP_OPERATION_MODE_MASK; as_sdp->vtotal = (sdp->db[2] << 8) | sdp->db[1]; - as_sdp->target_rr = (u64)sdp->db[3] | ((u64)sdp->db[4] & 0x3); + as_sdp->target_rr = ((sdp->db[4] & 0x3) << 8) | sdp->db[3]; as_sdp->target_rr_divider = sdp->db[4] & 0x20 ? true : false; return 0; diff --cc drivers/gpu/drm/i915/display/intel_psr.c index 892d209dce1b8,29904a037575f..bb1c0252837eb --- a/drivers/gpu/drm/i915/display/intel_psr.c +++ b/drivers/gpu/drm/i915/display/intel_psr.c @@@ -44,8 -43,8 +44,9 @@@ #include "intel_display_wa.h" #include "intel_dmc.h" #include "intel_dp.h" + #include "intel_dpcd.h" #include "intel_dp_aux.h" +#include "intel_dp_tunnel.h" #include "intel_dsb.h" #include "intel_frontbuffer.h" #include "intel_hdmi.h" diff --cc drivers/gpu/drm/panthor/panthor_device.h index 4e4607bca7ccd,b6696f73a5367..a412a50eec76c --- a/drivers/gpu/drm/panthor/panthor_device.h +++ b/drivers/gpu/drm/panthor/panthor_device.h @@@ -182,78 -178,6 +182,75 @@@ struct panthor_device /** @devfreq: Device frequency scaling management data. */ struct panthor_devfreq *devfreq; + /** @reclaim: Reclaim related stuff */ + struct { + /** @reclaim.shrinker: Shrinker instance */ + struct shrinker *shrinker; + - /** @reclaim.lock: Lock protecting all LRUs */ - struct mutex lock; - + /** + * @reclaim.unused: BOs with unused pages + * + * Basically all buffers that got mmapped, vmapped or GPU mapped and + * then unmapped. There should be no contention on these buffers, + * making them ideal to reclaim. + */ + struct drm_gem_lru unused; + + /** + * @reclaim.mmapped: mmap()-ed buffers + * + * Those are relatively easy to reclaim since we don't need user + * agreement, we can simply teardown the mapping and let it fault on + * the next access. + */ + struct drm_gem_lru mmapped; + + /** + * @reclaim.gpu_mapped_shared: shared BO LRU list + * + * That's the most tricky BO type to reclaim, because it involves + * tearing down all mappings in all VMs where this BO is mapped, + * which increases the risk of contention and thus decreases the + * likeliness of success. + */ + struct drm_gem_lru gpu_mapped_shared; + + /** + * @reclaim.vms: VM LRU list + * + * VMs that have reclaimable BOs only mapped to a single VM are placed + * in this LRU. Reclaiming such BOs implies waiting for VM idleness + * (no in-flight GPU jobs targeting this VM), meaning we can't reclaim + * those if we're in a context where we can't block/sleep. + */ + struct list_head vms; + + /** + * @reclaim.gpu_mapped_count: Global counter of pages that are GPU mapped + * + * Allows us to get the number of reclaimable pages without walking + * the vms and gpu_mapped_shared LRUs. + */ + long gpu_mapped_count; + + /** + * @reclaim.retry_count: Number of times we ran the shrinker without being + * able to reclaim stuff + * + * Used to stop scanning GEMs when too many attempts were made + * without progress. + */ + atomic_t retry_count; + +#ifdef CONFIG_DEBUG_FS + /** + * @reclaim.nr_pages_reclaimed_on_last_scan: Number of pages reclaimed on the last + * shrinker scan + */ + unsigned long nr_pages_reclaimed_on_last_scan; +#endif + } reclaim; + /** @unplug: Device unplug related fields. */ struct { /** @lock: Lock used to serialize unplug operations. */ diff --cc drivers/gpu/drm/panthor/panthor_gem.c index 13295d7a593df,cd49859da89b1..abe0c5bb1bca3 --- a/drivers/gpu/drm/panthor/panthor_gem.c +++ b/drivers/gpu/drm/panthor/panthor_gem.c @@@ -1221,356 -593,11 +1221,351 @@@ panthor_gem_sync(struct drm_gem_object * * for the flush+invalidate case. */ - dma_sync_single_for_device(dev->dev, paddr, len, DMA_TO_DEVICE); + dma_sync_single_for_device(dma_dev, paddr, len, DMA_TO_DEVICE); if (type == DRM_PANTHOR_BO_SYNC_CPU_CACHE_FLUSH_AND_INVALIDATE) - dma_sync_single_for_cpu(dev->dev, paddr, len, DMA_FROM_DEVICE); + dma_sync_single_for_cpu(dma_dev, paddr, len, DMA_FROM_DEVICE); + } + + ret = 0; + +out_unlock: + dma_resv_unlock(bo->base.resv); + return ret; +} + +/** + * panthor_kernel_bo_destroy() - Destroy a kernel buffer object + * @bo: Kernel buffer object to destroy. If NULL or an ERR_PTR(), the destruction + * is skipped. + */ +void panthor_kernel_bo_destroy(struct panthor_kernel_bo *bo) +{ + struct panthor_device *ptdev; + struct panthor_vm *vm; + + if (IS_ERR_OR_NULL(bo)) + return; + + ptdev = container_of(bo->obj->dev, struct panthor_device, base); + vm = bo->vm; + panthor_kernel_bo_vunmap(bo); + + drm_WARN_ON(bo->obj->dev, + to_panthor_bo(bo->obj)->exclusive_vm_root_gem != panthor_vm_root_gem(vm)); + panthor_vm_unmap_range(vm, bo->va_node.start, bo->va_node.size); + panthor_vm_free_va(vm, &bo->va_node); + if (vm == panthor_fw_vm(ptdev)) + panthor_gem_unpin(to_panthor_bo(bo->obj)); + drm_gem_object_put(bo->obj); + panthor_vm_put(vm); + kfree(bo); +} + +/** + * panthor_kernel_bo_create() - Create and map a GEM object to a VM + * @ptdev: Device. + * @vm: VM to map the GEM to. + * @size: Size of the buffer object. + * @bo_flags: Combination of drm_panthor_bo_flags flags. + * @vm_map_flags: Combination of drm_panthor_vm_bind_op_flags (only those + * that are related to map operations). + * @gpu_va: GPU address assigned when mapping to the VM. + * If gpu_va == PANTHOR_VM_KERNEL_AUTO_VA, the virtual address will be + * automatically allocated. + * @name: Descriptive label of the BO's contents + * + * Return: A valid pointer in case of success, an ERR_PTR() otherwise. + */ +struct panthor_kernel_bo * +panthor_kernel_bo_create(struct panthor_device *ptdev, struct panthor_vm *vm, + size_t size, u32 bo_flags, u32 vm_map_flags, + u64 gpu_va, const char *name) +{ + struct panthor_kernel_bo *kbo; + struct panthor_gem_object *bo; + u32 debug_flags = PANTHOR_DEBUGFS_GEM_USAGE_FLAG_KERNEL; + int ret; + + if (drm_WARN_ON(&ptdev->base, !vm)) + return ERR_PTR(-EINVAL); + + kbo = kzalloc_obj(*kbo); + if (!kbo) + return ERR_PTR(-ENOMEM); + + if (vm == panthor_fw_vm(ptdev)) + debug_flags |= PANTHOR_DEBUGFS_GEM_USAGE_FLAG_FW_MAPPED; + + bo = panthor_gem_create(&ptdev->base, size, bo_flags, vm, debug_flags); + if (IS_ERR(bo)) { + ret = PTR_ERR(bo); + goto err_free_kbo; + } + + kbo->obj = &bo->base; + + if (vm == panthor_fw_vm(ptdev)) { + ret = panthor_gem_pin(bo); + if (ret) + goto err_put_obj; + } + + panthor_gem_kernel_bo_set_label(kbo, name); + + /* The system and GPU MMU page size might differ, which becomes a + * problem for FW sections that need to be mapped at explicit address + * since our PAGE_SIZE alignment might cover a VA range that's + * expected to be used for another section. + * Make sure we never map more than we need. + */ + size = ALIGN(size, panthor_vm_page_size(vm)); + ret = panthor_vm_alloc_va(vm, gpu_va, size, &kbo->va_node); + if (ret) + goto err_unpin; + + ret = panthor_vm_map_bo_range(vm, bo, 0, size, kbo->va_node.start, vm_map_flags); + if (ret) + goto err_free_va; + + kbo->vm = panthor_vm_get(vm); + return kbo; + +err_free_va: + panthor_vm_free_va(vm, &kbo->va_node); + +err_unpin: + if (vm == panthor_fw_vm(ptdev)) + panthor_gem_unpin(bo); + +err_put_obj: + drm_gem_object_put(&bo->base); + +err_free_kbo: + kfree(kbo); + return ERR_PTR(ret); +} + +static bool can_swap(void) +{ + return get_nr_swap_pages() > 0; +} + +static bool can_block(struct shrink_control *sc) +{ + /* If direct reclaim is allowed, we can always block. + * If kswapd reclaim is allowed, we can block, but only if we're called + * by the kswapd thread. + */ + return (sc->gfp_mask & __GFP_DIRECT_RECLAIM) || + ((sc->gfp_mask & __GFP_KSWAPD_RECLAIM) && current_is_kswapd()); +} + +static unsigned long +panthor_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc) +{ + struct panthor_device *ptdev = shrinker->private_data; + unsigned long count; + + /* We currently don't have a flag to tell when the content of a + * BO can be discarded. + */ + if (!can_swap()) + return 0; + + /* This is racy, but that's okay because the returned count is just a + * hint. That's also what MSM is doing (no atomic var, it's relying on + * the fact unsigned long access is usually atomic), so if it's good + * enough for them, it's good enough for us too. + */ + count = ptdev->reclaim.unused.count; + count += ptdev->reclaim.mmapped.count; + + if (can_block(sc)) + count += ptdev->reclaim.gpu_mapped_count; + + return count ? count : SHRINK_EMPTY; +} + +static bool panthor_gem_try_evict_no_resv_wait(struct drm_gem_object *obj, + struct ww_acquire_ctx *ticket) +{ + /* + * Track last locked entry for unwinding locks in error and + * success paths + */ + struct panthor_gem_object *bo = to_panthor_bo(obj); + struct drm_gpuvm_bo *vm_bo, *last_locked = NULL; + enum panthor_gem_reclaim_state old_state; + int ret = 0; + + /* To avoid potential lock ordering issue between bo_gpuva and + * mapping->i_mmap_rwsem, unmap the pages from CPU side before + * acquring the bo_gpuva lock. As the bo_resv lock is held, CPU + * page fault handler won't be able to map in the pages whilst + * eviction is in progress. + */ + drm_vma_node_unmap(&bo->base.vma_node, bo->base.dev->anon_inode->i_mapping); + + /* We take this lock when walking the list to prevent + * insertion/deletion. + */ + /* We can only trylock in that path, because + * - allocation might happen while some of these locks are held + * - lock ordering is different in other paths + * vm_resv -> bo_resv -> bo_gpuva + * vs + * bo_resv -> bo_gpuva -> vm_resv + * + * If we fail to lock that's fine, we back off and will get + * back to it later. + */ + if (!mutex_trylock(&bo->base.gpuva.lock)) + return false; + + drm_gem_for_each_gpuvm_bo(vm_bo, obj) { + struct dma_resv *resv = drm_gpuvm_resv(vm_bo->vm); + + if (resv == obj->resv) + continue; + + if (!dma_resv_trylock(resv)) { + ret = -EDEADLK; + goto out_unlock; + } + + last_locked = vm_bo; + } + + /* Update the state before trying to evict the buffer, if the state was + * updated to something that's harder to reclaim (higher value in the + * enum), skip it (will be processed when the relevant LRU is). + */ + panthor_gem_update_reclaim_state_locked(bo, &old_state); + if (old_state < bo->reclaim_state) { + ret = -EAGAIN; + goto out_unlock; + } + + /* Couldn't teardown the GPU mappings? Skip. */ + ret = panthor_vm_evict_bo_mappings_locked(bo); + if (ret) + goto out_unlock; + + /* If everything went fine, evict the object. */ + panthor_gem_evict_locked(bo); + +out_unlock: + if (last_locked) { + drm_gem_for_each_gpuvm_bo(vm_bo, obj) { + struct dma_resv *resv = drm_gpuvm_resv(vm_bo->vm); + + if (resv == obj->resv) + continue; + + dma_resv_unlock(resv); + + if (last_locked == vm_bo) + break; + } } + mutex_unlock(&bo->base.gpuva.lock); + + return ret == 0; +} + +static bool panthor_gem_try_evict(struct drm_gem_object *obj, + struct ww_acquire_ctx *ticket) +{ + struct panthor_gem_object *bo = to_panthor_bo(obj); + + /* Wait was too long, skip. */ + if (dma_resv_wait_timeout(obj->resv, DMA_RESV_USAGE_BOOKKEEP, false, 10) <= 0) + return false; + + return panthor_gem_try_evict_no_resv_wait(&bo->base, ticket); +} + +static unsigned long +panthor_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) +{ + struct panthor_device *ptdev = shrinker->private_data; + unsigned long remaining = 0; + unsigned long freed = 0; + + if (!can_swap()) + goto out; + - freed += drm_gem_lru_scan(&ptdev->reclaim.unused, ++ freed += drm_gem_lru_scan(&ptdev->base, &ptdev->reclaim.unused, + sc->nr_to_scan - freed, &remaining, + panthor_gem_try_evict_no_resv_wait, NULL); + if (freed >= sc->nr_to_scan) + goto out; + - freed += drm_gem_lru_scan(&ptdev->reclaim.mmapped, ++ freed += drm_gem_lru_scan(&ptdev->base, &ptdev->reclaim.mmapped, + sc->nr_to_scan - freed, &remaining, + panthor_gem_try_evict_no_resv_wait, NULL); + if (freed >= sc->nr_to_scan) + goto out; + + if (!can_block(sc)) + goto out; + + freed += panthor_mmu_reclaim_priv_bos(ptdev, sc->nr_to_scan - freed, + &remaining, panthor_gem_try_evict); + if (freed >= sc->nr_to_scan) + goto out; + - freed += drm_gem_lru_scan(&ptdev->reclaim.gpu_mapped_shared, ++ freed += drm_gem_lru_scan(&ptdev->base, &ptdev->reclaim.gpu_mapped_shared, + sc->nr_to_scan - freed, &remaining, + panthor_gem_try_evict, NULL); + +out: +#ifdef CONFIG_DEBUG_FS + /* This is racy, but that's okay, because this is just debugfs + * reporting and doesn't need to be accurate. + */ + ptdev->reclaim.nr_pages_reclaimed_on_last_scan = freed; +#endif + + /* If there are things to reclaim, try a couple times before giving up. */ + if (!freed && remaining > 0 && + atomic_inc_return(&ptdev->reclaim.retry_count) < 2) + return 0; + + atomic_set(&ptdev->reclaim.retry_count, 0); + + if (freed) + return freed; + + /* There's nothing left to reclaim, or the resources are contended. Give up now. */ + return SHRINK_STOP; +} + +int panthor_gem_shrinker_init(struct panthor_device *ptdev) +{ + struct shrinker *shrinker; - int ret; - - ret = drmm_mutex_init(&ptdev->base, &ptdev->reclaim.lock); - if (ret) - return ret; + + INIT_LIST_HEAD(&ptdev->reclaim.vms); - drm_gem_lru_init(&ptdev->reclaim.unused, &ptdev->reclaim.lock); - drm_gem_lru_init(&ptdev->reclaim.mmapped, &ptdev->reclaim.lock); - drm_gem_lru_init(&ptdev->reclaim.gpu_mapped_shared, &ptdev->reclaim.lock); ++ drm_gem_lru_init(&ptdev->reclaim.unused); ++ drm_gem_lru_init(&ptdev->reclaim.mmapped); ++ drm_gem_lru_init(&ptdev->reclaim.gpu_mapped_shared); + ptdev->reclaim.gpu_mapped_count = 0; + + /* Teach lockdep about lock ordering wrt. shrinker: */ + fs_reclaim_acquire(GFP_KERNEL); - might_lock(&ptdev->reclaim.lock); ++ might_lock(&ptdev->base.gem_lru_mutex); + fs_reclaim_release(GFP_KERNEL); + + shrinker = shrinker_alloc(0, "drm-panthor-gem"); + if (!shrinker) + return -ENOMEM; + shrinker->count_objects = panthor_gem_shrinker_count; + shrinker->scan_objects = panthor_gem_shrinker_scan; + shrinker->private_data = ptdev; + ptdev->reclaim.shrinker = shrinker; + + shrinker_register(shrinker); return 0; } diff --cc drivers/gpu/drm/panthor/panthor_mmu.c index 452d0b6d46684,75d98dad7b1dd..9d45008505619 --- a/drivers/gpu/drm/panthor/panthor_mmu.c +++ b/drivers/gpu/drm/panthor/panthor_mmu.c @@@ -710,16 -688,6 +710,16 @@@ int panthor_vm_active(struct panthor_v if (refcount_inc_not_zero(&vm->as.active_cnt)) goto out_dev_exit; + /* As soon as active is called, we place the VM at the end of the VM LRU. + * If something fails after that, the only downside is that this VM that + * never became active in the first place will be reclaimed last, but + * that's an acceptable trade-off. + */ - mutex_lock(&ptdev->reclaim.lock); ++ mutex_lock(&ptdev->base.gem_lru_mutex); + if (vm->reclaim.lru.count) + list_move_tail(&vm->reclaim.lru_node, &ptdev->reclaim.vms); - mutex_unlock(&ptdev->reclaim.lock); ++ mutex_unlock(&ptdev->base.gem_lru_mutex); + /* Make sure we don't race with lock/unlock_region() calls * happening around VM bind operations. */ @@@ -1962,10 -1893,6 +1962,10 @@@ static void panthor_vm_free(struct drm_ struct panthor_vm *vm = container_of(gpuvm, struct panthor_vm, base); struct panthor_device *ptdev = vm->ptdev; - mutex_lock(&ptdev->reclaim.lock); ++ mutex_lock(&ptdev->base.gem_lru_mutex); + list_del_init(&vm->reclaim.lru_node); - mutex_unlock(&ptdev->reclaim.lock); ++ mutex_unlock(&ptdev->base.gem_lru_mutex); + mutex_lock(&vm->heaps.lock); if (drm_WARN_ON(&ptdev->base, vm->heaps.pool)) panthor_heap_pool_destroy(vm->heaps.pool); @@@ -2341,219 -2254,6 +2341,219 @@@ static int panthor_gpuva_sm_step_unmap( return 0; } +void panthor_vm_update_bo_reclaim_lru_locked(struct panthor_gem_object *bo) +{ + struct panthor_device *ptdev = container_of(bo->base.dev, struct panthor_device, base); + struct panthor_vm *vm = NULL; + struct drm_gpuvm_bo *vm_bo; + + dma_resv_assert_held(bo->base.resv); + lockdep_assert_held(&bo->base.gpuva.lock); + + drm_gem_for_each_gpuvm_bo(vm_bo, &bo->base) { + if (vm_bo->evicted) + continue; + + /* We're only supposed to have one non-evicted vm_bo in the list if we get + * there. + */ + drm_WARN_ON(&ptdev->base, vm); + vm = container_of(vm_bo->vm, struct panthor_vm, base); + - mutex_lock(&ptdev->reclaim.lock); ++ mutex_lock(&ptdev->base.gem_lru_mutex); + drm_gem_lru_move_tail_locked(&vm->reclaim.lru, &bo->base); + if (list_empty(&vm->reclaim.lru_node)) + list_move(&vm->reclaim.lru_node, &ptdev->reclaim.vms); - mutex_unlock(&ptdev->reclaim.lock); ++ mutex_unlock(&ptdev->base.gem_lru_mutex); + } +} + +int panthor_vm_evict_bo_mappings_locked(struct panthor_gem_object *bo) +{ + struct drm_gpuvm_bo *vm_bo; + int ret = 0; + + drm_gem_for_each_gpuvm_bo(vm_bo, &bo->base) { + struct panthor_vm *vm = container_of(vm_bo->vm, struct panthor_vm, base); + struct drm_gpuva *va; + + if (!mutex_trylock(&vm->op_lock)) + return -EDEADLK; + + /* It can be that the vm_bo was already evicted but a new + * mapping pointing to this BO got created in the meantime, + * thus turning the vm_bo in partially evicted state. In that case + * we don't call drm_gpuvm_bo_evict() again because this would + * mess up with the internal gpuvm lists, but we do walk the + * VAs on this vm_bo to make sure the non-evicted ones are + * torn down. + */ + if (!vm_bo->evicted) + drm_gpuvm_bo_evict(vm_bo, true); + + drm_gpuvm_bo_for_each_va(va, vm_bo) { + struct panthor_vma *vma = container_of(va, struct panthor_vma, base); + + if (vma->evicted) + continue; + + /* If something fail in the middle of a VM_BO eviction, the VM_BO + * is considered fully evicted, but some of its VMAs might still be + * active. That's okay because the pages won't be released if this + * function returns an error. + * + * On the next job targeting this VM, the partially evicted VM_BO + * will be validated, causing all its evicted VMAs to be repopulated + * before the job runs. So no GPU fault expected. + */ + ret = panthor_vm_lock_region(vm, va->va.addr, va->va.range); + if (ret) + break; + + panthor_vm_unmap_pages(vm, va->va.addr, va->va.range); + panthor_vm_unlock_region(vm); + vma->evicted = true; + } + + mutex_unlock(&vm->op_lock); + + if (ret) + break; + } + + return ret; +} + +static struct panthor_vma *select_evicted_vma(struct drm_gpuvm_bo *vm_bo, + struct panthor_vm_op_ctx *op_ctx) +{ + struct panthor_vm *vm = container_of(vm_bo->vm, struct panthor_vm, base); + struct panthor_vma *first_evicted_vma = NULL; + struct drm_gpuva *va; + + /* Take op_lock to protect against va insertion/removal. */ + mutex_lock(&vm->op_lock); + drm_gpuvm_bo_for_each_va(va, vm_bo) { + struct panthor_vma *vma = container_of(va, struct panthor_vma, base); + + if (vma->evicted) { + first_evicted_vma = vma; + panthor_vm_init_op_ctx(op_ctx, va->va.range, va->va.addr, vma->flags); + op_ctx->map.bo_offset = va->gem.offset; + break; + } + } + mutex_unlock(&vm->op_lock); + + return first_evicted_vma; +} + +static int remap_evicted_vma(struct drm_gpuvm_bo *vm_bo, + struct panthor_vma *evicted_vma, + struct panthor_vm_op_ctx *op_ctx) +{ + struct panthor_vm *vm = container_of(vm_bo->vm, struct panthor_vm, base); + struct panthor_gem_object *bo = to_panthor_bo(vm_bo->obj); + struct drm_gpuva *va; + bool found = false; + int ret; + + ret = panthor_vm_op_ctx_prealloc_pts(op_ctx); + if (ret) + goto out_cleanup; + + /* Take op_lock to protect against va insertion/removal. Note that the + * evicted_vma selection was done with the same lock held, but we had + * to release it so we can allocate PTs, because this very same lock + * is taken in a DMA-signalling path. + */ + mutex_lock(&vm->op_lock); + drm_gpuvm_bo_for_each_va(va, vm_bo) { + struct panthor_vma *vma = container_of(va, struct panthor_vma, base); + + if (vma != evicted_vma) + continue; + + /* Because we had to release the lock between the evicted_vma selection + * and its repopulation, we can't rely solely on pointer equality (the + * VMA might have been freed and a new one allocated at the same address). + * If the evicted bit is still set, we're sure it's our VMA, because + * population/eviction is serialized with the BO resv lock. + */ + if (vma->evicted) + found = true; + + break; + } + + if (found) { + vm->op_ctx = op_ctx; + ret = panthor_vm_lock_region(vm, evicted_vma->base.va.addr, + evicted_vma->base.va.range); + if (!ret) { + ret = panthor_vm_map_pages(vm, evicted_vma->base.va.addr, + flags_to_prot(evicted_vma->flags), + bo->dmap.sgt, + evicted_vma->base.gem.offset, + evicted_vma->base.va.range); + if (!ret) + evicted_vma->evicted = false; + + panthor_vm_unlock_region(vm); + } + + vm->op_ctx = NULL; + } + + mutex_unlock(&vm->op_lock); + +out_cleanup: + panthor_vm_cleanup_op_ctx(op_ctx, vm); + return ret; +} + +static int panthor_vm_restore_vmas(struct drm_gpuvm_bo *vm_bo) +{ + struct panthor_vm *vm = container_of(vm_bo->vm, struct panthor_vm, base); + struct panthor_gem_object *bo = to_panthor_bo(vm_bo->obj); + struct panthor_vm_op_ctx op_ctx; + + if (drm_WARN_ON_ONCE(&vm->ptdev->base, !bo->dmap.sgt)) + return -EINVAL; + + for (struct panthor_vma *vma = select_evicted_vma(vm_bo, &op_ctx); + vma; vma = select_evicted_vma(vm_bo, &op_ctx)) { + int ret; + + ret = remap_evicted_vma(vm_bo, vma, &op_ctx); + if (ret) + return ret; + } + + return 0; +} + +static int panthor_vm_bo_validate(struct drm_gpuvm_bo *vm_bo, + struct drm_exec *exec) +{ + struct panthor_gem_object *bo = to_panthor_bo(vm_bo->obj); + int ret; + + ret = panthor_gem_swapin_locked(bo); + if (ret) + return ret; + + ret = panthor_vm_restore_vmas(vm_bo); + if (ret) + return ret; + + drm_gpuvm_bo_evict(vm_bo, false); + mutex_lock(&bo->base.gpuva.lock); + panthor_gem_update_reclaim_state_locked(bo, NULL); + mutex_unlock(&bo->base.gpuva.lock); + return 0; +} + static const struct drm_gpuvm_ops panthor_gpuvm_ops = { .vm_free = panthor_vm_free, .vm_bo_free = panthor_vm_bo_free, @@@ -2774,8 -2474,6 +2774,8 @@@ panthor_vm_create(struct panthor_devic vm->kernel_auto_va.start = auto_kernel_va_start; vm->kernel_auto_va.end = vm->kernel_auto_va.start + auto_kernel_va_size - 1; - drm_gem_lru_init(&vm->reclaim.lru, &ptdev->reclaim.lock); ++ drm_gem_lru_init(&vm->reclaim.lru); + INIT_LIST_HEAD(&vm->reclaim.lru_node); INIT_LIST_HEAD(&vm->node); INIT_LIST_HEAD(&vm->as.lru_node); vm->as.id = -1; @@@ -3123,78 -2821,7 +3123,79 @@@ int panthor_vm_prepare_mapped_bos_resvs if (ret) return ret; - return drm_gpuvm_prepare_objects(&vm->base, exec, slot_count); + ret = drm_gpuvm_prepare_objects(&vm->base, exec, slot_count); + if (ret) + return ret; + + return drm_gpuvm_validate(&vm->base, exec); +} + +unsigned long +panthor_mmu_reclaim_priv_bos(struct panthor_device *ptdev, + unsigned int nr_to_scan, unsigned long *remaining, + bool (*shrink)(struct drm_gem_object *, + struct ww_acquire_ctx *)) +{ + unsigned long freed = 0; + LIST_HEAD(remaining_vms); + LIST_HEAD(vms); + - mutex_lock(&ptdev->reclaim.lock); ++ mutex_lock(&ptdev->base.gem_lru_mutex); + list_splice_init(&ptdev->reclaim.vms, &vms); + + while (freed < nr_to_scan) { + struct panthor_vm *vm; + + vm = list_first_entry_or_null(&vms, typeof(*vm), + reclaim.lru_node); + if (!vm) + break; + + if (!kref_get_unless_zero(&vm->base.kref)) { + list_del_init(&vm->reclaim.lru_node); + continue; + } + - mutex_unlock(&ptdev->reclaim.lock); ++ mutex_unlock(&ptdev->base.gem_lru_mutex); + - freed += drm_gem_lru_scan(&vm->reclaim.lru, nr_to_scan - freed, ++ freed += drm_gem_lru_scan(&ptdev->base, &vm->reclaim.lru, ++ nr_to_scan - freed, + remaining, shrink, NULL); + - mutex_lock(&ptdev->reclaim.lock); ++ mutex_lock(&ptdev->base.gem_lru_mutex); + + /* If the VM is still in the temporary list, remove it so we + * can proceed with the next VM. + */ + if (vm == list_first_entry_or_null(&vms, typeof(*vm), reclaim.lru_node)) { + list_del_init(&vm->reclaim.lru_node); + + /* Keep the VM around if there are still things to + * reclaim, so we can preserve the LRU order when + * re-inserting in ptdev->reclaim.vms at the end. + */ + if (vm->reclaim.lru.count > 0) + list_add_tail(&vm->reclaim.lru_node, &remaining_vms); + } + - mutex_unlock(&ptdev->reclaim.lock); ++ mutex_unlock(&ptdev->base.gem_lru_mutex); + + panthor_vm_put(vm); + - mutex_lock(&ptdev->reclaim.lock); ++ mutex_lock(&ptdev->base.gem_lru_mutex); + } + + /* Re-insert VMs with remaining data to reclaim at the beginning of + * the LRU. Note that any activeness change on the VM that happened + * while we were reclaiming would have moved the VM out of our + * temporary [remaining_]vms list, meaning anything we re-insert here + * preserves the LRU order. + */ + list_splice_tail(&vms, &remaining_vms); + list_splice(&remaining_vms, &ptdev->reclaim.vms); - mutex_unlock(&ptdev->reclaim.lock); ++ mutex_unlock(&ptdev->base.gem_lru_mutex); + + return freed; } /**