*
* for the flush+invalidate case.
*/
- dma_sync_single_for_device(dev->dev, paddr, len, DMA_TO_DEVICE);
+ dma_sync_single_for_device(dma_dev, paddr, len, DMA_TO_DEVICE);
if (type == DRM_PANTHOR_BO_SYNC_CPU_CACHE_FLUSH_AND_INVALIDATE)
- dma_sync_single_for_cpu(dev->dev, paddr, len, DMA_FROM_DEVICE);
+ dma_sync_single_for_cpu(dma_dev, paddr, len, DMA_FROM_DEVICE);
+ }
+
+ ret = 0;
+
+out_unlock:
+ dma_resv_unlock(bo->base.resv);
+ return ret;
+}
+
+/**
+ * panthor_kernel_bo_destroy() - Destroy a kernel buffer object
+ * @bo: Kernel buffer object to destroy. If NULL or an ERR_PTR(), the destruction
+ * is skipped.
+ */
+void panthor_kernel_bo_destroy(struct panthor_kernel_bo *bo)
+{
+ struct panthor_device *ptdev;
+ struct panthor_vm *vm;
+
+ if (IS_ERR_OR_NULL(bo))
+ return;
+
+ ptdev = container_of(bo->obj->dev, struct panthor_device, base);
+ vm = bo->vm;
+ panthor_kernel_bo_vunmap(bo);
+
+ drm_WARN_ON(bo->obj->dev,
+ to_panthor_bo(bo->obj)->exclusive_vm_root_gem != panthor_vm_root_gem(vm));
+ panthor_vm_unmap_range(vm, bo->va_node.start, bo->va_node.size);
+ panthor_vm_free_va(vm, &bo->va_node);
+ if (vm == panthor_fw_vm(ptdev))
+ panthor_gem_unpin(to_panthor_bo(bo->obj));
+ drm_gem_object_put(bo->obj);
+ panthor_vm_put(vm);
+ kfree(bo);
+}
+
+/**
+ * panthor_kernel_bo_create() - Create and map a GEM object to a VM
+ * @ptdev: Device.
- * @vm: VM to map the GEM to. If NULL, the kernel object is not GPU mapped.
++ * @vm: VM to map the GEM to.
+ * @size: Size of the buffer object.
+ * @bo_flags: Combination of drm_panthor_bo_flags flags.
+ * @vm_map_flags: Combination of drm_panthor_vm_bind_op_flags (only those
+ * that are related to map operations).
+ * @gpu_va: GPU address assigned when mapping to the VM.
+ * If gpu_va == PANTHOR_VM_KERNEL_AUTO_VA, the virtual address will be
+ * automatically allocated.
+ * @name: Descriptive label of the BO's contents
+ *
+ * Return: A valid pointer in case of success, an ERR_PTR() otherwise.
+ */
+struct panthor_kernel_bo *
+panthor_kernel_bo_create(struct panthor_device *ptdev, struct panthor_vm *vm,
+ size_t size, u32 bo_flags, u32 vm_map_flags,
+ u64 gpu_va, const char *name)
+{
+ struct panthor_kernel_bo *kbo;
+ struct panthor_gem_object *bo;
+ u32 debug_flags = PANTHOR_DEBUGFS_GEM_USAGE_FLAG_KERNEL;
+ int ret;
+
+ if (drm_WARN_ON(&ptdev->base, !vm))
+ return ERR_PTR(-EINVAL);
+
+ kbo = kzalloc_obj(*kbo);
+ if (!kbo)
+ return ERR_PTR(-ENOMEM);
+
+ if (vm == panthor_fw_vm(ptdev))
+ debug_flags |= PANTHOR_DEBUGFS_GEM_USAGE_FLAG_FW_MAPPED;
+
+ bo = panthor_gem_create(&ptdev->base, size, bo_flags, vm, debug_flags);
+ if (IS_ERR(bo)) {
+ ret = PTR_ERR(bo);
+ goto err_free_kbo;
+ }
+
+ kbo->obj = &bo->base;
+
+ if (vm == panthor_fw_vm(ptdev)) {
+ ret = panthor_gem_pin(bo);
+ if (ret)
+ goto err_put_obj;
+ }
+
+ panthor_gem_kernel_bo_set_label(kbo, name);
+
+ /* The system and GPU MMU page size might differ, which becomes a
+ * problem for FW sections that need to be mapped at explicit address
+ * since our PAGE_SIZE alignment might cover a VA range that's
+ * expected to be used for another section.
+ * Make sure we never map more than we need.
+ */
+ size = ALIGN(size, panthor_vm_page_size(vm));
+ ret = panthor_vm_alloc_va(vm, gpu_va, size, &kbo->va_node);
+ if (ret)
+ goto err_unpin;
+
+ ret = panthor_vm_map_bo_range(vm, bo, 0, size, kbo->va_node.start, vm_map_flags);
+ if (ret)
+ goto err_free_va;
+
+ kbo->vm = panthor_vm_get(vm);
+ return kbo;
+
+err_free_va:
+ panthor_vm_free_va(vm, &kbo->va_node);
+
+err_unpin:
+ if (vm == panthor_fw_vm(ptdev))
+ panthor_gem_unpin(bo);
+
+err_put_obj:
+ drm_gem_object_put(&bo->base);
+
+err_free_kbo:
+ kfree(kbo);
+ return ERR_PTR(ret);
+}
+
+static bool can_swap(void)
+{
+ return get_nr_swap_pages() > 0;
+}
+
+static bool can_block(struct shrink_control *sc)
+{
+ /* If direct reclaim is allowed, we can always block.
+ * If kswapd reclaim is allowed, we can block, but only if we're called
+ * by the kswapd thread.
+ */
+ return (sc->gfp_mask & __GFP_DIRECT_RECLAIM) ||
+ ((sc->gfp_mask & __GFP_KSWAPD_RECLAIM) && current_is_kswapd());
+}
+
+static unsigned long
+panthor_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
+{
+ struct panthor_device *ptdev = shrinker->private_data;
+ unsigned long count;
+
+ /* We currently don't have a flag to tell when the content of a
+ * BO can be discarded.
+ */
+ if (!can_swap())
+ return 0;
+
+ /* This is racy, but that's okay because the returned count is just a
+ * hint. That's also what MSM is doing (no atomic var, it's relying on
+ * the fact unsigned long access is usually atomic), so if it's good
+ * enough for them, it's good enough for us too.
+ */
+ count = ptdev->reclaim.unused.count;
+ count += ptdev->reclaim.mmapped.count;
+
+ if (can_block(sc))
+ count += ptdev->reclaim.gpu_mapped_count;
+
+ return count ? count : SHRINK_EMPTY;
+}
+
+static bool panthor_gem_try_evict_no_resv_wait(struct drm_gem_object *obj,
+ struct ww_acquire_ctx *ticket)
+{
+ /*
+ * Track last locked entry for unwinding locks in error and
+ * success paths
+ */
+ struct panthor_gem_object *bo = to_panthor_bo(obj);
+ struct drm_gpuvm_bo *vm_bo, *last_locked = NULL;
+ enum panthor_gem_reclaim_state old_state;
+ int ret = 0;
+
+ /* To avoid potential lock ordering issue between bo_gpuva and
+ * mapping->i_mmap_rwsem, unmap the pages from CPU side before
+ * acquring the bo_gpuva lock. As the bo_resv lock is held, CPU
+ * page fault handler won't be able to map in the pages whilst
+ * eviction is in progress.
+ */
+ drm_vma_node_unmap(&bo->base.vma_node, bo->base.dev->anon_inode->i_mapping);
+
+ /* We take this lock when walking the list to prevent
+ * insertion/deletion.
+ */
+ /* We can only trylock in that path, because
+ * - allocation might happen while some of these locks are held
+ * - lock ordering is different in other paths
+ * vm_resv -> bo_resv -> bo_gpuva
+ * vs
+ * bo_resv -> bo_gpuva -> vm_resv
+ *
+ * If we fail to lock that's fine, we back off and will get
+ * back to it later.
+ */
+ if (!mutex_trylock(&bo->base.gpuva.lock))
+ return false;
+
+ drm_gem_for_each_gpuvm_bo(vm_bo, obj) {
+ struct dma_resv *resv = drm_gpuvm_resv(vm_bo->vm);
+
+ if (resv == obj->resv)
+ continue;
+
+ if (!dma_resv_trylock(resv)) {
+ ret = -EDEADLK;
+ goto out_unlock;
+ }
+
+ last_locked = vm_bo;
+ }
+
+ /* Update the state before trying to evict the buffer, if the state was
+ * updated to something that's harder to reclaim (higher value in the
+ * enum), skip it (will be processed when the relevant LRU is).
+ */
+ panthor_gem_update_reclaim_state_locked(bo, &old_state);
+ if (old_state < bo->reclaim_state) {
+ ret = -EAGAIN;
+ goto out_unlock;
+ }
+
+ /* Couldn't teardown the GPU mappings? Skip. */
+ ret = panthor_vm_evict_bo_mappings_locked(bo);
+ if (ret)
+ goto out_unlock;
+
+ /* If everything went fine, evict the object. */
+ panthor_gem_evict_locked(bo);
+
+out_unlock:
+ if (last_locked) {
+ drm_gem_for_each_gpuvm_bo(vm_bo, obj) {
+ struct dma_resv *resv = drm_gpuvm_resv(vm_bo->vm);
+
+ if (resv == obj->resv)
+ continue;
+
+ dma_resv_unlock(resv);
+
+ if (last_locked == vm_bo)
+ break;
+ }
}
+ mutex_unlock(&bo->base.gpuva.lock);
+
+ return ret == 0;
+}
+
+static bool panthor_gem_try_evict(struct drm_gem_object *obj,
+ struct ww_acquire_ctx *ticket)
+{
+ struct panthor_gem_object *bo = to_panthor_bo(obj);
+
+ /* Wait was too long, skip. */
+ if (dma_resv_wait_timeout(obj->resv, DMA_RESV_USAGE_BOOKKEEP, false, 10) <= 0)
+ return false;
+
+ return panthor_gem_try_evict_no_resv_wait(&bo->base, ticket);
+}
+
+static unsigned long
+panthor_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
+{
+ struct panthor_device *ptdev = shrinker->private_data;
+ unsigned long remaining = 0;
+ unsigned long freed = 0;
+
+ if (!can_swap())
+ goto out;
+
+ freed += drm_gem_lru_scan(&ptdev->reclaim.unused,
+ sc->nr_to_scan - freed, &remaining,
+ panthor_gem_try_evict_no_resv_wait, NULL);
+ if (freed >= sc->nr_to_scan)
+ goto out;
+
+ freed += drm_gem_lru_scan(&ptdev->reclaim.mmapped,
+ sc->nr_to_scan - freed, &remaining,
+ panthor_gem_try_evict_no_resv_wait, NULL);
+ if (freed >= sc->nr_to_scan)
+ goto out;
+
+ if (!can_block(sc))
+ goto out;
+
+ freed += panthor_mmu_reclaim_priv_bos(ptdev, sc->nr_to_scan - freed,
+ &remaining, panthor_gem_try_evict);
+ if (freed >= sc->nr_to_scan)
+ goto out;
+
+ freed += drm_gem_lru_scan(&ptdev->reclaim.gpu_mapped_shared,
+ sc->nr_to_scan - freed, &remaining,
+ panthor_gem_try_evict, NULL);
+
+out:
+#ifdef CONFIG_DEBUG_FS
+ /* This is racy, but that's okay, because this is just debugfs
+ * reporting and doesn't need to be accurate.
+ */
+ ptdev->reclaim.nr_pages_reclaimed_on_last_scan = freed;
+#endif
+
+ /* If there are things to reclaim, try a couple times before giving up. */
+ if (!freed && remaining > 0 &&
+ atomic_inc_return(&ptdev->reclaim.retry_count) < 2)
+ return 0;
+
+ atomic_set(&ptdev->reclaim.retry_count, 0);
+
+ if (freed)
+ return freed;
+
+ /* There's nothing left to reclaim, or the resources are contended. Give up now. */
+ return SHRINK_STOP;
+}
+
+int panthor_gem_shrinker_init(struct panthor_device *ptdev)
+{
+ struct shrinker *shrinker;
+ int ret;
+
+ ret = drmm_mutex_init(&ptdev->base, &ptdev->reclaim.lock);
+ if (ret)
+ return ret;
+
+ INIT_LIST_HEAD(&ptdev->reclaim.vms);
+ drm_gem_lru_init(&ptdev->reclaim.unused, &ptdev->reclaim.lock);
+ drm_gem_lru_init(&ptdev->reclaim.mmapped, &ptdev->reclaim.lock);
+ drm_gem_lru_init(&ptdev->reclaim.gpu_mapped_shared, &ptdev->reclaim.lock);
+ ptdev->reclaim.gpu_mapped_count = 0;
+
+ /* Teach lockdep about lock ordering wrt. shrinker: */
+ fs_reclaim_acquire(GFP_KERNEL);
+ might_lock(&ptdev->reclaim.lock);
+ fs_reclaim_release(GFP_KERNEL);
+
+ shrinker = shrinker_alloc(0, "drm-panthor-gem");
+ if (!shrinker)
+ return -ENOMEM;
+ shrinker->count_objects = panthor_gem_shrinker_count;
+ shrinker->scan_objects = panthor_gem_shrinker_scan;
+ shrinker->private_data = ptdev;
+ ptdev->reclaim.shrinker = shrinker;
+
+ shrinker_register(shrinker);
return 0;
}
projects / thirdparty / kernel / linux.git / commitdiff
