#include <linux/circ_buf.h>
+#include <drm/drm_exec.h>
#include <drm/drm_managed.h>
+#include "xe_bo.h"
#include "xe_device.h"
+#include "xe_gt_printk.h"
#include "xe_gt_types.h"
+#include "xe_gt_stats.h"
+#include "xe_hw_engine.h"
#include "xe_pagefault.h"
#include "xe_pagefault_types.h"
+#include "xe_svm.h"
+#include "xe_trace_bo.h"
+#include "xe_vm.h"
/**
* DOC: Xe page faults
return roundup_pow_of_two(sizeof(struct xe_pagefault));
}
+static int xe_pagefault_begin(struct drm_exec *exec, struct xe_vma *vma,
+ struct xe_vram_region *vram, bool need_vram_move)
+{
+ struct xe_bo *bo = xe_vma_bo(vma);
+ struct xe_vm *vm = xe_vma_vm(vma);
+ int err;
+
+ err = xe_vm_lock_vma(exec, vma);
+ if (err)
+ return err;
+
+ if (!bo)
+ return 0;
+
+ return need_vram_move ? xe_bo_migrate(bo, vram->placement, NULL, exec) :
+ xe_bo_validate(bo, vm, true, exec);
+}
+
+static int xe_pagefault_handle_vma(struct xe_gt *gt, struct xe_vma *vma,
+ bool atomic)
+{
+ struct xe_vm *vm = xe_vma_vm(vma);
+ struct xe_tile *tile = gt_to_tile(gt);
+ struct xe_validation_ctx ctx;
+ struct drm_exec exec;
+ struct dma_fence *fence;
+ int err, needs_vram;
+
+ lockdep_assert_held_write(&vm->lock);
+
+ needs_vram = xe_vma_need_vram_for_atomic(vm->xe, vma, atomic);
+ if (needs_vram < 0 || (needs_vram && xe_vma_is_userptr(vma)))
+ return needs_vram < 0 ? needs_vram : -EACCES;
+
+ xe_gt_stats_incr(gt, XE_GT_STATS_ID_VMA_PAGEFAULT_COUNT, 1);
+ xe_gt_stats_incr(gt, XE_GT_STATS_ID_VMA_PAGEFAULT_KB,
+ xe_vma_size(vma) / SZ_1K);
+
+ trace_xe_vma_pagefault(vma);
+
+ /* Check if VMA is valid, opportunistic check only */
+ if (xe_vm_has_valid_gpu_mapping(tile, vma->tile_present,
+ vma->tile_invalidated) && !atomic)
+ return 0;
+
+retry_userptr:
+ if (xe_vma_is_userptr(vma) &&
+ xe_vma_userptr_check_repin(to_userptr_vma(vma))) {
+ struct xe_userptr_vma *uvma = to_userptr_vma(vma);
+
+ err = xe_vma_userptr_pin_pages(uvma);
+ if (err)
+ return err;
+ }
+
+ /* Lock VM and BOs dma-resv */
+ xe_validation_ctx_init(&ctx, &vm->xe->val, &exec, (struct xe_val_flags) {});
+ drm_exec_init(&exec, 0, 0);
+ drm_exec_until_all_locked(&exec) {
+ err = xe_pagefault_begin(&exec, vma, tile->mem.vram,
+ needs_vram == 1);
+ drm_exec_retry_on_contention(&exec);
+ xe_validation_retry_on_oom(&ctx, &err);
+ if (err)
+ goto unlock_dma_resv;
+
+ /* Bind VMA only to the GT that has faulted */
+ trace_xe_vma_pf_bind(vma);
+ xe_vm_set_validation_exec(vm, &exec);
+ fence = xe_vma_rebind(vm, vma, BIT(tile->id));
+ xe_vm_set_validation_exec(vm, NULL);
+ if (IS_ERR(fence)) {
+ err = PTR_ERR(fence);
+ xe_validation_retry_on_oom(&ctx, &err);
+ goto unlock_dma_resv;
+ }
+ }
+
+ dma_fence_wait(fence, false);
+ dma_fence_put(fence);
+
+unlock_dma_resv:
+ xe_validation_ctx_fini(&ctx);
+ if (err == -EAGAIN)
+ goto retry_userptr;
+
+ return err;
+}
+
+static bool
+xe_pagefault_access_is_atomic(enum xe_pagefault_access_type access_type)
+{
+ return access_type == XE_PAGEFAULT_ACCESS_TYPE_ATOMIC;
+}
+
+static struct xe_vm *xe_pagefault_asid_to_vm(struct xe_device *xe, u32 asid)
+{
+ struct xe_vm *vm;
+
+ down_read(&xe->usm.lock);
+ vm = xa_load(&xe->usm.asid_to_vm, asid);
+ if (vm && xe_vm_in_fault_mode(vm))
+ xe_vm_get(vm);
+ else
+ vm = ERR_PTR(-EINVAL);
+ up_read(&xe->usm.lock);
+
+ return vm;
+}
+
+static int xe_pagefault_service(struct xe_pagefault *pf)
+{
+ struct xe_gt *gt = pf->gt;
+ struct xe_device *xe = gt_to_xe(gt);
+ struct xe_vm *vm;
+ struct xe_vma *vma = NULL;
+ int err;
+ bool atomic;
+
+ /* Producer flagged this fault to be nacked */
+ if (pf->consumer.fault_level == XE_PAGEFAULT_LEVEL_NACK)
+ return -EFAULT;
+
+ vm = xe_pagefault_asid_to_vm(xe, pf->consumer.asid);
+ if (IS_ERR(vm))
+ return PTR_ERR(vm);
+
+ /*
+ * TODO: Change to read lock? Using write lock for simplicity.
+ */
+ down_write(&vm->lock);
+
+ if (xe_vm_is_closed(vm)) {
+ err = -ENOENT;
+ goto unlock_vm;
+ }
+
+ vma = xe_vm_find_vma_by_addr(vm, pf->consumer.page_addr);
+ if (!vma) {
+ err = -EINVAL;
+ goto unlock_vm;
+ }
+
+ atomic = xe_pagefault_access_is_atomic(pf->consumer.access_type);
+
+ if (xe_vma_is_cpu_addr_mirror(vma))
+ err = xe_svm_handle_pagefault(vm, vma, gt,
+ pf->consumer.page_addr, atomic);
+ else
+ err = xe_pagefault_handle_vma(gt, vma, atomic);
+
+unlock_vm:
+ if (!err)
+ vm->usm.last_fault_vma = vma;
+ up_write(&vm->lock);
+ xe_vm_put(vm);
+
+ return err;
+}
+
+static bool xe_pagefault_queue_pop(struct xe_pagefault_queue *pf_queue,
+ struct xe_pagefault *pf)
+{
+ bool found_fault = false;
+
+ spin_lock_irq(&pf_queue->lock);
+ if (pf_queue->tail != pf_queue->head) {
+ memcpy(pf, pf_queue->data + pf_queue->tail, sizeof(*pf));
+ pf_queue->tail = (pf_queue->tail + xe_pagefault_entry_size()) %
+ pf_queue->size;
+ found_fault = true;
+ }
+ spin_unlock_irq(&pf_queue->lock);
+
+ return found_fault;
+}
+
+static void xe_pagefault_print(struct xe_pagefault *pf)
+{
+ xe_gt_dbg(pf->gt, "\n\tASID: %d\n"
+ "\tFaulted Address: 0x%08x%08x\n"
+ "\tFaultType: %d\n"
+ "\tAccessType: %d\n"
+ "\tFaultLevel: %d\n"
+ "\tEngineClass: %d %s\n"
+ "\tEngineInstance: %d\n",
+ pf->consumer.asid,
+ upper_32_bits(pf->consumer.page_addr),
+ lower_32_bits(pf->consumer.page_addr),
+ pf->consumer.fault_type,
+ pf->consumer.access_type,
+ pf->consumer.fault_level,
+ pf->consumer.engine_class,
+ xe_hw_engine_class_to_str(pf->consumer.engine_class),
+ pf->consumer.engine_instance);
+}
+
static void xe_pagefault_queue_work(struct work_struct *w)
{
- /* TODO: Implement */
+ struct xe_pagefault_queue *pf_queue =
+ container_of(w, typeof(*pf_queue), worker);
+ struct xe_pagefault pf;
+ unsigned long threshold;
+
+#define USM_QUEUE_MAX_RUNTIME_MS 20
+ threshold = jiffies + msecs_to_jiffies(USM_QUEUE_MAX_RUNTIME_MS);
+
+ while (xe_pagefault_queue_pop(pf_queue, &pf)) {
+ int err;
+
+ if (!pf.gt) /* Fault squashed during reset */
+ continue;
+
+ err = xe_pagefault_service(&pf);
+ if (err) {
+ xe_pagefault_print(&pf);
+ xe_gt_dbg(pf.gt, "Fault response: Unsuccessful %pe\n",
+ ERR_PTR(err));
+ }
+
+ pf.producer.ops->ack_fault(&pf, err);
+
+ if (time_after(jiffies, threshold)) {
+ queue_work(gt_to_xe(pf.gt)->usm.pf_wq, w);
+ break;
+ }
+ }
+#undef USM_QUEUE_MAX_RUNTIME_MS
}
static int xe_pagefault_queue_init(struct xe_device *xe,
projects / thirdparty / kernel / linux.git / commitdiff
