xe_gt_pagefault.o \
xe_gt_sysfs.o \
xe_gt_throttle.o \
- xe_gt_tlb_inval.o \
- xe_gt_tlb_inval_job.o \
xe_gt_topology.o \
xe_guc.o \
xe_guc_ads.o \
xe_sync.o \
xe_tile.o \
xe_tile_sysfs.o \
+ xe_tlb_inval.o \
+ xe_tlb_inval_job.o \
xe_trace.o \
xe_trace_bo.o \
xe_trace_guc.o \
#include "xe_device.h"
#include "xe_gt.h"
#include "xe_gt_printk.h"
-#include "xe_gt_tlb_inval.h"
#include "xe_map.h"
#include "xe_mmio.h"
#include "xe_pm.h"
#include "xe_res_cursor.h"
#include "xe_sriov.h"
#include "xe_tile_sriov_vf.h"
+#include "xe_tlb_inval.h"
#include "xe_wa.h"
#include "xe_wopcm.h"
if (!gt)
return;
- err = xe_gt_tlb_inval_ggtt(gt);
+ err = xe_tlb_inval_ggtt(>->tlb_inval);
xe_gt_WARN(gt, err, "Failed to invalidate GGTT (%pe)", ERR_PTR(err));
}
#include "xe_gt_sriov_pf.h"
#include "xe_gt_sriov_vf.h"
#include "xe_gt_sysfs.h"
-#include "xe_gt_tlb_inval.h"
#include "xe_gt_topology.h"
#include "xe_guc_exec_queue_types.h"
#include "xe_guc_pc.h"
#include "xe_sa.h"
#include "xe_sched_job.h"
#include "xe_sriov.h"
+#include "xe_tlb_inval.h"
#include "xe_tuning.h"
#include "xe_uc.h"
#include "xe_uc_fw.h"
xe_uc_stop(>->uc);
- xe_gt_tlb_inval_reset(gt);
+ xe_tlb_inval_reset(>->tlb_inval);
err = do_gt_reset(gt);
if (err)
xe_gt_assert(gt, gt_to_xe(gt)->wedged.mode);
xe_uc_declare_wedged(>->uc);
- xe_gt_tlb_inval_reset(gt);
+ xe_tlb_inval_reset(>->tlb_inval);
}
+++ /dev/null
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2023 Intel Corporation
- */
-
-#ifndef _XE_GT_TLB_INVAL_H_
-#define _XE_GT_TLB_INVAL_H_
-
-#include <linux/types.h>
-
-#include "xe_gt_tlb_inval_types.h"
-
-struct xe_gt;
-struct xe_guc;
-struct xe_vm;
-struct xe_vma;
-
-int xe_gt_tlb_inval_init_early(struct xe_gt *gt);
-void xe_gt_tlb_inval_fini(struct xe_gt *gt);
-
-void xe_gt_tlb_inval_reset(struct xe_gt *gt);
-int xe_gt_tlb_inval_ggtt(struct xe_gt *gt);
-void xe_gt_tlb_inval_vm(struct xe_gt *gt, struct xe_vm *vm);
-int xe_gt_tlb_inval_all(struct xe_gt *gt, struct xe_gt_tlb_inval_fence *fence);
-int xe_gt_tlb_inval_range(struct xe_gt *gt,
- struct xe_gt_tlb_inval_fence *fence,
- u64 start, u64 end, u32 asid);
-int xe_guc_tlb_inval_done_handler(struct xe_guc *guc, u32 *msg, u32 len);
-
-void xe_gt_tlb_inval_fence_init(struct xe_gt *gt,
- struct xe_gt_tlb_inval_fence *fence,
- bool stack);
-void xe_gt_tlb_inval_fence_signal(struct xe_gt_tlb_inval_fence *fence);
-
-static inline void
-xe_gt_tlb_inval_fence_wait(struct xe_gt_tlb_inval_fence *fence)
-{
- dma_fence_wait(&fence->base, false);
-}
-
-#endif /* _XE_GT_TLB_INVAL_ */
+++ /dev/null
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2025 Intel Corporation
- */
-
-#ifndef _XE_GT_TLB_INVAL_JOB_H_
-#define _XE_GT_TLB_INVAL_JOB_H_
-
-#include <linux/types.h>
-
-struct dma_fence;
-struct drm_sched_job;
-struct kref;
-struct xe_exec_queue;
-struct xe_gt;
-struct xe_gt_tlb_inval_job;
-struct xe_migrate;
-
-struct xe_gt_tlb_inval_job *xe_gt_tlb_inval_job_create(struct xe_exec_queue *q,
- struct xe_gt *gt,
- u64 start, u64 end,
- u32 asid);
-
-int xe_gt_tlb_inval_job_alloc_dep(struct xe_gt_tlb_inval_job *job);
-
-struct dma_fence *xe_gt_tlb_inval_job_push(struct xe_gt_tlb_inval_job *job,
- struct xe_migrate *m,
- struct dma_fence *fence);
-
-void xe_gt_tlb_inval_job_get(struct xe_gt_tlb_inval_job *job);
-
-void xe_gt_tlb_inval_job_put(struct xe_gt_tlb_inval_job *job);
-
-#endif
#include "xe_gt_sriov_pf_types.h"
#include "xe_gt_sriov_vf_types.h"
#include "xe_gt_stats_types.h"
-#include "xe_gt_tlb_inval_types.h"
#include "xe_hw_engine_types.h"
#include "xe_hw_fence_types.h"
#include "xe_oa_types.h"
#include "xe_reg_sr_types.h"
#include "xe_sa_types.h"
+#include "xe_tlb_inval_types.h"
#include "xe_uc_types.h"
struct xe_exec_queue_ops;
#include "xe_gt_sriov_pf_control.h"
#include "xe_gt_sriov_pf_monitor.h"
#include "xe_gt_sriov_printk.h"
-#include "xe_gt_tlb_inval.h"
#include "xe_guc.h"
#include "xe_guc_log.h"
#include "xe_guc_relay.h"
#include "xe_guc_submit.h"
#include "xe_map.h"
#include "xe_pm.h"
+#include "xe_tlb_inval.h"
#include "xe_trace_guc.h"
static void receive_g2h(struct xe_guc_ct *ct);
#include "xe_assert.h"
#include "xe_bo.h"
-#include "xe_gt_tlb_inval.h"
+#include "xe_tlb_inval.h"
#include "xe_lmtt.h"
#include "xe_map.h"
#include "xe_mmio.h"
static int lmtt_invalidate_hw(struct xe_lmtt *lmtt)
{
- struct xe_gt_tlb_inval_fence fences[XE_MAX_GT_PER_TILE];
- struct xe_gt_tlb_inval_fence *fence = fences;
+ struct xe_tlb_inval_fence fences[XE_MAX_GT_PER_TILE];
+ struct xe_tlb_inval_fence *fence = fences;
struct xe_tile *tile = lmtt_to_tile(lmtt);
struct xe_gt *gt;
int result = 0;
u8 id;
for_each_gt_on_tile(gt, tile, id) {
- xe_gt_tlb_inval_fence_init(gt, fence, true);
- err = xe_gt_tlb_inval_all(gt, fence);
+ xe_tlb_inval_fence_init(>->tlb_inval, fence, true);
+ err = xe_tlb_inval_all(>->tlb_inval, fence);
result = result ?: err;
fence++;
}
*/
fence = fences;
for_each_gt_on_tile(gt, tile, id)
- xe_gt_tlb_inval_fence_wait(fence++);
+ xe_tlb_inval_fence_wait(fence++);
return result;
}
struct xe_bo;
struct xe_gt;
-struct xe_gt_tlb_inval_job;
+struct xe_tlb_inval_job;
struct xe_exec_queue;
struct xe_migrate;
struct xe_migrate_pt_update;
/** @job: The job if a GPU page-table update. NULL otherwise */
struct xe_sched_job *job;
/**
- * @ijob: The GT TLB invalidation job for primary tile. NULL otherwise
+ * @ijob: The TLB invalidation job for primary GT. NULL otherwise
*/
- struct xe_gt_tlb_inval_job *ijob;
+ struct xe_tlb_inval_job *ijob;
/**
- * @mjob: The GT TLB invalidation job for media tile. NULL otherwise
+ * @mjob: The TLB invalidation job for media GT. NULL otherwise
*/
- struct xe_gt_tlb_inval_job *mjob;
+ struct xe_tlb_inval_job *mjob;
/** @tile_id: Tile ID of the update */
u8 tile_id;
};
#include "xe_drm_client.h"
#include "xe_exec_queue.h"
#include "xe_gt.h"
-#include "xe_gt_tlb_inval_job.h"
+#include "xe_tlb_inval_job.h"
#include "xe_migrate.h"
#include "xe_pt_types.h"
#include "xe_pt_walk.h"
#include "xe_sched_job.h"
#include "xe_sync.h"
#include "xe_svm.h"
+#include "xe_tlb_inval_job.h"
#include "xe_trace.h"
#include "xe_ttm_stolen_mgr.h"
#include "xe_vm.h"
}
static int xe_pt_vm_dependencies(struct xe_sched_job *job,
- struct xe_gt_tlb_inval_job *ijob,
- struct xe_gt_tlb_inval_job *mjob,
+ struct xe_tlb_inval_job *ijob,
+ struct xe_tlb_inval_job *mjob,
struct xe_vm *vm,
struct xe_vma_ops *vops,
struct xe_vm_pgtable_update_ops *pt_update_ops,
if (job) {
if (ijob) {
- err = xe_gt_tlb_inval_job_alloc_dep(ijob);
+ err = xe_tlb_inval_job_alloc_dep(ijob);
if (err)
return err;
}
if (mjob) {
- err = xe_gt_tlb_inval_job_alloc_dep(mjob);
+ err = xe_tlb_inval_job_alloc_dep(mjob);
if (err)
return err;
}
static const struct xe_migrate_pt_update_ops svm_migrate_ops;
#endif
+static struct xe_dep_scheduler *to_dep_scheduler(struct xe_exec_queue *q,
+ struct xe_gt *gt)
+{
+ if (xe_gt_is_media_type(gt))
+ return q->tlb_inval[XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT].dep_scheduler;
+
+ return q->tlb_inval[XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT].dep_scheduler;
+}
+
/**
* xe_pt_update_ops_run() - Run PT update operations
* @tile: Tile of PT update operations
struct xe_vm_pgtable_update_ops *pt_update_ops =
&vops->pt_update_ops[tile->id];
struct dma_fence *fence, *ifence, *mfence;
- struct xe_gt_tlb_inval_job *ijob = NULL, *mjob = NULL;
+ struct xe_tlb_inval_job *ijob = NULL, *mjob = NULL;
struct dma_fence **fences = NULL;
struct dma_fence_array *cf = NULL;
struct xe_range_fence *rfence;
#endif
if (pt_update_ops->needs_invalidation) {
- ijob = xe_gt_tlb_inval_job_create(pt_update_ops->q,
- tile->primary_gt,
- pt_update_ops->start,
- pt_update_ops->last,
- vm->usm.asid);
+ struct xe_exec_queue *q = pt_update_ops->q;
+ struct xe_dep_scheduler *dep_scheduler =
+ to_dep_scheduler(q, tile->primary_gt);
+
+ ijob = xe_tlb_inval_job_create(q, &tile->primary_gt->tlb_inval,
+ dep_scheduler,
+ pt_update_ops->start,
+ pt_update_ops->last,
+ vm->usm.asid);
if (IS_ERR(ijob)) {
err = PTR_ERR(ijob);
goto kill_vm_tile1;
update.ijob = ijob;
if (tile->media_gt) {
- mjob = xe_gt_tlb_inval_job_create(pt_update_ops->q,
- tile->media_gt,
- pt_update_ops->start,
- pt_update_ops->last,
- vm->usm.asid);
+ dep_scheduler = to_dep_scheduler(q, tile->media_gt);
+
+ mjob = xe_tlb_inval_job_create(q,
+ &tile->media_gt->tlb_inval,
+ dep_scheduler,
+ pt_update_ops->start,
+ pt_update_ops->last,
+ vm->usm.asid);
if (IS_ERR(mjob)) {
err = PTR_ERR(mjob);
goto free_ijob;
if (ijob) {
struct dma_fence *__fence;
- ifence = xe_gt_tlb_inval_job_push(ijob, tile->migrate, fence);
+ ifence = xe_tlb_inval_job_push(ijob, tile->migrate, fence);
__fence = ifence;
if (mjob) {
fences[0] = ifence;
- mfence = xe_gt_tlb_inval_job_push(mjob, tile->migrate,
- fence);
+ mfence = xe_tlb_inval_job_push(mjob, tile->migrate,
+ fence);
fences[1] = mfence;
dma_fence_array_init(cf, 2, fences,
if (pt_update_ops->needs_userptr_lock)
up_read(&vm->userptr.notifier_lock);
- xe_gt_tlb_inval_job_put(mjob);
- xe_gt_tlb_inval_job_put(ijob);
+ xe_tlb_inval_job_put(mjob);
+ xe_tlb_inval_job_put(ijob);
return fence;
free_ijob:
kfree(cf);
kfree(fences);
- xe_gt_tlb_inval_job_put(mjob);
- xe_gt_tlb_inval_job_put(ijob);
+ xe_tlb_inval_job_put(mjob);
+ xe_tlb_inval_job_put(ijob);
kill_vm_tile1:
if (err != -EAGAIN && err != -ENODATA && tile->id)
xe_vm_kill(vops->vm, false);
#include "xe_bo.h"
#include "xe_gt_stats.h"
-#include "xe_gt_tlb_inval.h"
#include "xe_migrate.h"
#include "xe_module.h"
#include "xe_pm.h"
#include "xe_guc.h"
#include "xe_guc_ct.h"
#include "xe_gt_stats.h"
-#include "xe_gt_tlb_inval.h"
+#include "xe_tlb_inval.h"
#include "xe_mmio.h"
#include "xe_pm.h"
#include "xe_sriov.h"
return hw_tlb_timeout + 2 * delay;
}
-static void xe_gt_tlb_inval_fence_fini(struct xe_gt_tlb_inval_fence *fence)
+static void xe_tlb_inval_fence_fini(struct xe_tlb_inval_fence *fence)
{
- if (WARN_ON_ONCE(!fence->gt))
+ struct xe_gt *gt;
+
+ if (WARN_ON_ONCE(!fence->tlb_inval))
return;
- xe_pm_runtime_put(gt_to_xe(fence->gt));
- fence->gt = NULL; /* fini() should be called once */
+ gt = fence->tlb_inval->private;
+
+ xe_pm_runtime_put(gt_to_xe(gt));
+ fence->tlb_inval = NULL; /* fini() should be called once */
}
static void
-__inval_fence_signal(struct xe_device *xe, struct xe_gt_tlb_inval_fence *fence)
+__inval_fence_signal(struct xe_device *xe, struct xe_tlb_inval_fence *fence)
{
bool stack = test_bit(FENCE_STACK_BIT, &fence->base.flags);
- trace_xe_gt_tlb_inval_fence_signal(xe, fence);
- xe_gt_tlb_inval_fence_fini(fence);
+ trace_xe_tlb_inval_fence_signal(xe, fence);
+ xe_tlb_inval_fence_fini(fence);
dma_fence_signal(&fence->base);
if (!stack)
dma_fence_put(&fence->base);
}
static void
-inval_fence_signal(struct xe_device *xe, struct xe_gt_tlb_inval_fence *fence)
+inval_fence_signal(struct xe_device *xe, struct xe_tlb_inval_fence *fence)
{
list_del(&fence->link);
__inval_fence_signal(xe, fence);
}
-void xe_gt_tlb_inval_fence_signal(struct xe_gt_tlb_inval_fence *fence)
+void xe_tlb_inval_fence_signal(struct xe_tlb_inval_fence *fence)
{
- if (WARN_ON_ONCE(!fence->gt))
+ struct xe_gt *gt;
+
+ if (WARN_ON_ONCE(!fence->tlb_inval))
return;
- __inval_fence_signal(gt_to_xe(fence->gt), fence);
+ gt = fence->tlb_inval->private;
+ __inval_fence_signal(gt_to_xe(gt), fence);
}
static void xe_gt_tlb_fence_timeout(struct work_struct *work)
struct xe_gt *gt = container_of(work, struct xe_gt,
tlb_inval.fence_tdr.work);
struct xe_device *xe = gt_to_xe(gt);
- struct xe_gt_tlb_inval_fence *fence, *next;
+ struct xe_tlb_inval_fence *fence, *next;
LNL_FLUSH_WORK(>->uc.guc.ct.g2h_worker);
if (msecs_to_jiffies(since_inval_ms) < tlb_timeout_jiffies(gt))
break;
- trace_xe_gt_tlb_inval_fence_timeout(xe, fence);
+ trace_xe_tlb_inval_fence_timeout(xe, fence);
xe_gt_err(gt, "TLB invalidation fence timeout, seqno=%d recv=%d",
fence->seqno, gt->tlb_inval.seqno_recv);
* xe_gt_tlb_inval_init_early - Initialize GT TLB invalidation state
* @gt: GT structure
*
- * Initialize GT TLB invalidation state, purely software initialization, should
+ * Initialize TLB invalidation state, purely software initialization, should
* be called once during driver load.
*
* Return: 0 on success, negative error code on error.
struct xe_device *xe = gt_to_xe(gt);
int err;
+ gt->tlb_inval.private = gt;
gt->tlb_inval.seqno = 1;
INIT_LIST_HEAD(>->tlb_inval.pending_fences);
spin_lock_init(>->tlb_inval.pending_lock);
}
/**
- * xe_gt_tlb_inval_reset - Initialize GT TLB invalidation reset
- * @gt: GT structure
+ * xe_tlb_inval_reset - Initialize TLB invalidation reset
+ * @tlb_inval: TLB invalidation client
*
* Signal any pending invalidation fences, should be called during a GT reset
*/
-void xe_gt_tlb_inval_reset(struct xe_gt *gt)
+void xe_tlb_inval_reset(struct xe_tlb_inval *tlb_inval)
{
- struct xe_gt_tlb_inval_fence *fence, *next;
+ struct xe_gt *gt = tlb_inval->private;
+ struct xe_tlb_inval_fence *fence, *next;
int pending_seqno;
/*
}
static int send_tlb_inval(struct xe_guc *guc,
- struct xe_gt_tlb_inval_fence *fence,
+ struct xe_tlb_inval_fence *fence,
u32 *action, int len)
{
struct xe_gt *gt = guc_to_gt(guc);
mutex_lock(>->tlb_inval.seqno_lock);
seqno = gt->tlb_inval.seqno;
fence->seqno = seqno;
- trace_xe_gt_tlb_inval_fence_send(xe, fence);
+ trace_xe_tlb_inval_fence_send(xe, fence);
action[1] = seqno;
ret = xe_guc_ct_send(&guc->ct, action, len,
G2H_LEN_DW_TLB_INVALIDATE, 1);
XE_GUC_TLB_INVAL_FLUSH_CACHE)
/**
- * xe_gt_tlb_inval_guc - Issue a TLB invalidation on this GT for the GuC
+ * xe_tlb_inval_guc - Issue a TLB invalidation on this GT for the GuC
* @gt: GT structure
* @fence: invalidation fence which will be signal on TLB invalidation
* completion
*
* Return: 0 on success, negative error code on error
*/
-static int xe_gt_tlb_inval_guc(struct xe_gt *gt,
- struct xe_gt_tlb_inval_fence *fence)
+static int xe_tlb_inval_guc(struct xe_gt *gt,
+ struct xe_tlb_inval_fence *fence)
{
u32 action[] = {
XE_GUC_ACTION_TLB_INVALIDATION,
}
/**
- * xe_gt_tlb_inval_ggtt - Issue a TLB invalidation on this GT for the GGTT
- * @gt: GT structure
+ * xe_tlb_inval_ggtt - Issue a TLB invalidation on this GT for the GGTT
+ * @tlb_inval: TLB invalidation client
*
* Issue a TLB invalidation for the GGTT. Completion of TLB invalidation is
* synchronous.
*
* Return: 0 on success, negative error code on error
*/
-int xe_gt_tlb_inval_ggtt(struct xe_gt *gt)
+int xe_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval)
{
+ struct xe_gt *gt = tlb_inval->private;
struct xe_device *xe = gt_to_xe(gt);
unsigned int fw_ref;
if (xe_guc_ct_enabled(>->uc.guc.ct) &&
gt->uc.guc.submission_state.enabled) {
- struct xe_gt_tlb_inval_fence fence;
+ struct xe_tlb_inval_fence fence;
int ret;
- xe_gt_tlb_inval_fence_init(gt, &fence, true);
- ret = xe_gt_tlb_inval_guc(gt, &fence);
+ xe_tlb_inval_fence_init(tlb_inval, &fence, true);
+ ret = xe_tlb_inval_guc(gt, &fence);
if (ret)
return ret;
- xe_gt_tlb_inval_fence_wait(&fence);
+ xe_tlb_inval_fence_wait(&fence);
} else if (xe_device_uc_enabled(xe) && !xe_device_wedged(xe)) {
struct xe_mmio *mmio = >->mmio;
return 0;
}
-static int send_tlb_inval_all(struct xe_gt *gt,
- struct xe_gt_tlb_inval_fence *fence)
+static int send_tlb_inval_all(struct xe_tlb_inval *tlb_inval,
+ struct xe_tlb_inval_fence *fence)
{
u32 action[] = {
XE_GUC_ACTION_TLB_INVALIDATION_ALL,
0, /* seqno, replaced in send_tlb_inval */
MAKE_INVAL_OP(XE_GUC_TLB_INVAL_FULL),
};
+ struct xe_gt *gt = tlb_inval->private;
+
+ xe_gt_assert(gt, fence);
return send_tlb_inval(>->uc.guc, fence, action, ARRAY_SIZE(action));
}
/**
* xe_gt_tlb_invalidation_all - Invalidate all TLBs across PF and all VFs.
* @gt: the &xe_gt structure
- * @fence: the &xe_gt_tlb_inval_fence to be signaled on completion
+ * @fence: the &xe_tlb_inval_fence to be signaled on completion
*
* Send a request to invalidate all TLBs across PF and all VFs.
*
* Return: 0 on success, negative error code on error
*/
-int xe_gt_tlb_inval_all(struct xe_gt *gt, struct xe_gt_tlb_inval_fence *fence)
+int xe_tlb_inval_all(struct xe_tlb_inval *tlb_inval,
+ struct xe_tlb_inval_fence *fence)
{
+ struct xe_gt *gt = tlb_inval->private;
int err;
- xe_gt_assert(gt, gt == fence->gt);
-
- err = send_tlb_inval_all(gt, fence);
+ err = send_tlb_inval_all(tlb_inval, fence);
if (err)
xe_gt_err(gt, "TLB invalidation request failed (%pe)", ERR_PTR(err));
#define MAX_RANGE_TLB_INVALIDATION_LENGTH (rounddown_pow_of_two(ULONG_MAX))
/**
- * xe_gt_tlb_inval_range - Issue a TLB invalidation on this GT for an address range
- *
- * @gt: GT structure
+ * xe_tlb_inval_range - Issue a TLB invalidation on this GT for an address range
+ * @tlb_inval: TLB invalidation client
* @fence: invalidation fence which will be signal on TLB invalidation
* completion
* @start: start address
*
* Return: Negative error code on error, 0 on success
*/
-int xe_gt_tlb_inval_range(struct xe_gt *gt, struct xe_gt_tlb_inval_fence *fence,
- u64 start, u64 end, u32 asid)
+int xe_tlb_inval_range(struct xe_tlb_inval *tlb_inval,
+ struct xe_tlb_inval_fence *fence, u64 start, u64 end,
+ u32 asid)
{
+ struct xe_gt *gt = tlb_inval->private;
struct xe_device *xe = gt_to_xe(gt);
#define MAX_TLB_INVALIDATION_LEN 7
u32 action[MAX_TLB_INVALIDATION_LEN];
}
/**
- * xe_gt_tlb_inval_vm - Issue a TLB invalidation on this GT for a VM
- * @gt: graphics tile
+ * xe_tlb_inval_vm - Issue a TLB invalidation on this GT for a VM
+ * @tlb_inval: TLB invalidation client
* @vm: VM to invalidate
*
* Invalidate entire VM's address space
*/
-void xe_gt_tlb_inval_vm(struct xe_gt *gt, struct xe_vm *vm)
+void xe_tlb_inval_vm(struct xe_tlb_inval *tlb_inval, struct xe_vm *vm)
{
- struct xe_gt_tlb_inval_fence fence;
+ struct xe_tlb_inval_fence fence;
u64 range = 1ull << vm->xe->info.va_bits;
int ret;
- xe_gt_tlb_inval_fence_init(gt, &fence, true);
+ xe_tlb_inval_fence_init(tlb_inval, &fence, true);
- ret = xe_gt_tlb_inval_range(gt, &fence, 0, range, vm->usm.asid);
+ ret = xe_tlb_inval_range(tlb_inval, &fence, 0, range, vm->usm.asid);
if (ret < 0)
return;
- xe_gt_tlb_inval_fence_wait(&fence);
+ xe_tlb_inval_fence_wait(&fence);
}
/**
- * xe_gt_tlb_inval_done_handler - GT TLB invalidation done handler
+ * xe_tlb_inval_done_handler - TLB invalidation done handler
* @gt: gt
* @seqno: seqno of invalidation that is done
*
- * Update recv seqno, signal any GT TLB invalidation fences, and restart TDR
+ * Update recv seqno, signal any TLB invalidation fences, and restart TDR
*/
-static void xe_gt_tlb_inval_done_handler(struct xe_gt *gt, int seqno)
+static void xe_tlb_inval_done_handler(struct xe_gt *gt, int seqno)
{
struct xe_device *xe = gt_to_xe(gt);
- struct xe_gt_tlb_inval_fence *fence, *next;
+ struct xe_tlb_inval_fence *fence, *next;
unsigned long flags;
/*
list_for_each_entry_safe(fence, next,
>->tlb_inval.pending_fences, link) {
- trace_xe_gt_tlb_inval_fence_recv(xe, fence);
+ trace_xe_tlb_inval_fence_recv(xe, fence);
if (!tlb_inval_seqno_past(gt, fence->seqno))
break;
if (unlikely(len != 1))
return -EPROTO;
- xe_gt_tlb_inval_done_handler(gt, msg[0]);
+ xe_tlb_inval_done_handler(gt, msg[0]);
return 0;
}
};
/**
- * xe_gt_tlb_inval_fence_init - Initialize TLB invalidation fence
- * @gt: GT
+ * xe_tlb_inval_fence_init - Initialize TLB invalidation fence
+ * @tlb_inval: TLB invalidation client
* @fence: TLB invalidation fence to initialize
* @stack: fence is stack variable
*
- * Initialize TLB invalidation fence for use. xe_gt_tlb_inval_fence_fini
+ * Initialize TLB invalidation fence for use. xe_tlb_inval_fence_fini
* will be automatically called when fence is signalled (all fences must signal),
* even on error.
*/
-void xe_gt_tlb_inval_fence_init(struct xe_gt *gt,
- struct xe_gt_tlb_inval_fence *fence,
- bool stack)
+void xe_tlb_inval_fence_init(struct xe_tlb_inval *tlb_inval,
+ struct xe_tlb_inval_fence *fence,
+ bool stack)
{
+ struct xe_gt *gt = tlb_inval->private;
+
xe_pm_runtime_get_noresume(gt_to_xe(gt));
spin_lock_irq(>->tlb_inval.lock);
set_bit(FENCE_STACK_BIT, &fence->base.flags);
else
dma_fence_get(&fence->base);
- fence->gt = gt;
+ fence->tlb_inval = tlb_inval;
}
--- /dev/null
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2025 Intel Corporation
+ */
+
+#ifndef _XE_TLB_INVAL_H_
+#define _XE_TLB_INVAL_H_
+
+#include <linux/types.h>
+
+#include "xe_tlb_inval_types.h"
+
+struct xe_gt;
+struct xe_guc;
+struct xe_vm;
+
+int xe_gt_tlb_inval_init_early(struct xe_gt *gt);
+void xe_gt_tlb_inval_fini(struct xe_gt *gt);
+
+void xe_tlb_inval_reset(struct xe_tlb_inval *tlb_inval);
+int xe_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval);
+void xe_tlb_inval_vm(struct xe_tlb_inval *tlb_inval, struct xe_vm *vm);
+int xe_tlb_inval_all(struct xe_tlb_inval *tlb_inval,
+ struct xe_tlb_inval_fence *fence);
+int xe_tlb_inval_range(struct xe_tlb_inval *tlb_inval,
+ struct xe_tlb_inval_fence *fence,
+ u64 start, u64 end, u32 asid);
+int xe_guc_tlb_inval_done_handler(struct xe_guc *guc, u32 *msg, u32 len);
+
+void xe_tlb_inval_fence_init(struct xe_tlb_inval *tlb_inval,
+ struct xe_tlb_inval_fence *fence,
+ bool stack);
+void xe_tlb_inval_fence_signal(struct xe_tlb_inval_fence *fence);
+
+static inline void
+xe_tlb_inval_fence_wait(struct xe_tlb_inval_fence *fence)
+{
+ dma_fence_wait(&fence->base, false);
+}
+
+#endif /* _XE_TLB_INVAL_ */
* Copyright © 2025 Intel Corporation
*/
+#include "xe_assert.h"
#include "xe_dep_job_types.h"
#include "xe_dep_scheduler.h"
#include "xe_exec_queue.h"
-#include "xe_gt.h"
-#include "xe_gt_tlb_inval.h"
-#include "xe_gt_tlb_inval_job.h"
+#include "xe_gt_types.h"
+#include "xe_tlb_inval.h"
+#include "xe_tlb_inval_job.h"
#include "xe_migrate.h"
#include "xe_pm.h"
-/** struct xe_gt_tlb_inval_job - GT TLB invalidation job */
-struct xe_gt_tlb_inval_job {
+/** struct xe_tlb_inval_job - TLB invalidation job */
+struct xe_tlb_inval_job {
/** @dep: base generic dependency Xe job */
struct xe_dep_job dep;
- /** @gt: GT to invalidate */
- struct xe_gt *gt;
+ /** @tlb_inval: TLB invalidation client */
+ struct xe_tlb_inval *tlb_inval;
/** @q: exec queue issuing the invalidate */
struct xe_exec_queue *q;
/** @refcount: ref count of this job */
bool fence_armed;
};
-static struct dma_fence *xe_gt_tlb_inval_job_run(struct xe_dep_job *dep_job)
+static struct dma_fence *xe_tlb_inval_job_run(struct xe_dep_job *dep_job)
{
- struct xe_gt_tlb_inval_job *job =
+ struct xe_tlb_inval_job *job =
container_of(dep_job, typeof(*job), dep);
- struct xe_gt_tlb_inval_fence *ifence =
+ struct xe_tlb_inval_fence *ifence =
container_of(job->fence, typeof(*ifence), base);
- xe_gt_tlb_inval_range(job->gt, ifence, job->start,
- job->end, job->asid);
+ xe_tlb_inval_range(job->tlb_inval, ifence, job->start,
+ job->end, job->asid);
return job->fence;
}
-static void xe_gt_tlb_inval_job_free(struct xe_dep_job *dep_job)
+static void xe_tlb_inval_job_free(struct xe_dep_job *dep_job)
{
- struct xe_gt_tlb_inval_job *job =
+ struct xe_tlb_inval_job *job =
container_of(dep_job, typeof(*job), dep);
- /* Pairs with get in xe_gt_tlb_inval_job_push */
- xe_gt_tlb_inval_job_put(job);
+ /* Pairs with get in xe_tlb_inval_job_push */
+ xe_tlb_inval_job_put(job);
}
static const struct xe_dep_job_ops dep_job_ops = {
- .run_job = xe_gt_tlb_inval_job_run,
- .free_job = xe_gt_tlb_inval_job_free,
+ .run_job = xe_tlb_inval_job_run,
+ .free_job = xe_tlb_inval_job_free,
};
-static int xe_gt_tlb_inval_context(struct xe_gt *gt)
-{
- return xe_gt_is_media_type(gt) ? XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT :
- XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT;
-}
-
/**
- * xe_gt_tlb_inval_job_create() - GT TLB invalidation job create
- * @gt: GT to invalidate
+ * xe_tlb_inval_job_create() - TLB invalidation job create
* @q: exec queue issuing the invalidate
+ * @tlb_inval: TLB invalidation client
+ * @dep_scheduler: Dependency scheduler for job
* @start: Start address to invalidate
* @end: End address to invalidate
* @asid: Address space ID to invalidate
*
- * Create a GT TLB invalidation job and initialize internal fields. The caller is
+ * Create a TLB invalidation job and initialize internal fields. The caller is
* responsible for releasing the creation reference.
*
- * Return: GT TLB invalidation job object on success, ERR_PTR failure
+ * Return: TLB invalidation job object on success, ERR_PTR failure
*/
-struct xe_gt_tlb_inval_job *xe_gt_tlb_inval_job_create(struct xe_exec_queue *q,
- struct xe_gt *gt,
- u64 start, u64 end,
- u32 asid)
+struct xe_tlb_inval_job *
+xe_tlb_inval_job_create(struct xe_exec_queue *q, struct xe_tlb_inval *tlb_inval,
+ struct xe_dep_scheduler *dep_scheduler, u64 start,
+ u64 end, u32 asid)
{
- struct xe_gt_tlb_inval_job *job;
- struct xe_dep_scheduler *dep_scheduler =
- q->tlb_inval[xe_gt_tlb_inval_context(gt)].dep_scheduler;
+ struct xe_tlb_inval_job *job;
struct drm_sched_entity *entity =
xe_dep_scheduler_entity(dep_scheduler);
- struct xe_gt_tlb_inval_fence *ifence;
+ struct xe_tlb_inval_fence *ifence;
int err;
job = kmalloc(sizeof(*job), GFP_KERNEL);
return ERR_PTR(-ENOMEM);
job->q = q;
- job->gt = gt;
+ job->tlb_inval = tlb_inval;
job->start = start;
job->end = end;
job->asid = asid;
job->fence_armed = false;
job->dep.ops = &dep_job_ops;
kref_init(&job->refcount);
- xe_exec_queue_get(q); /* Pairs with put in xe_gt_tlb_inval_job_destroy */
+ xe_exec_queue_get(q); /* Pairs with put in xe_tlb_inval_job_destroy */
ifence = kmalloc(sizeof(*ifence), GFP_KERNEL);
if (!ifence) {
if (err)
goto err_fence;
- /* Pairs with put in xe_gt_tlb_inval_job_destroy */
- xe_pm_runtime_get_noresume(gt_to_xe(job->gt));
+ /* Pairs with put in xe_tlb_inval_job_destroy */
+ xe_pm_runtime_get_noresume(gt_to_xe(q->gt));
return job;
return ERR_PTR(err);
}
-static void xe_gt_tlb_inval_job_destroy(struct kref *ref)
+static void xe_tlb_inval_job_destroy(struct kref *ref)
{
- struct xe_gt_tlb_inval_job *job = container_of(ref, typeof(*job),
- refcount);
- struct xe_gt_tlb_inval_fence *ifence =
+ struct xe_tlb_inval_job *job = container_of(ref, typeof(*job),
+ refcount);
+ struct xe_tlb_inval_fence *ifence =
container_of(job->fence, typeof(*ifence), base);
- struct xe_device *xe = gt_to_xe(job->gt);
struct xe_exec_queue *q = job->q;
+ struct xe_device *xe = gt_to_xe(q->gt);
if (!job->fence_armed)
kfree(ifence);
else
- /* Ref from xe_gt_tlb_inval_fence_init */
+ /* Ref from xe_tlb_inval_fence_init */
dma_fence_put(job->fence);
drm_sched_job_cleanup(&job->dep.drm);
kfree(job);
- xe_exec_queue_put(q); /* Pairs with get from xe_gt_tlb_inval_job_create */
- xe_pm_runtime_put(xe); /* Pairs with get from xe_gt_tlb_inval_job_create */
+ xe_exec_queue_put(q); /* Pairs with get from xe_tlb_inval_job_create */
+ xe_pm_runtime_put(xe); /* Pairs with get from xe_tlb_inval_job_create */
}
/**
- * xe_gt_tlb_inval_alloc_dep() - GT TLB invalidation job alloc dependency
- * @job: GT TLB invalidation job to alloc dependency for
+ * xe_tlb_inval_alloc_dep() - TLB invalidation job alloc dependency
+ * @job: TLB invalidation job to alloc dependency for
*
- * Allocate storage for a dependency in the GT TLB invalidation fence. This
+ * Allocate storage for a dependency in the TLB invalidation fence. This
* function should be called at most once per job and must be paired with
- * xe_gt_tlb_inval_job_push being called with a real fence.
+ * xe_tlb_inval_job_push being called with a real fence.
*
* Return: 0 on success, -errno on failure
*/
-int xe_gt_tlb_inval_job_alloc_dep(struct xe_gt_tlb_inval_job *job)
+int xe_tlb_inval_job_alloc_dep(struct xe_tlb_inval_job *job)
{
- xe_assert(gt_to_xe(job->gt), !xa_load(&job->dep.drm.dependencies, 0));
+ xe_assert(gt_to_xe(job->q->gt), !xa_load(&job->dep.drm.dependencies, 0));
might_alloc(GFP_KERNEL);
return drm_sched_job_add_dependency(&job->dep.drm,
}
/**
- * xe_gt_tlb_inval_job_push() - GT TLB invalidation job push
- * @job: GT TLB invalidation job to push
+ * xe_tlb_inval_job_push() - TLB invalidation job push
+ * @job: TLB invalidation job to push
* @m: The migration object being used
- * @fence: Dependency for GT TLB invalidation job
+ * @fence: Dependency for TLB invalidation job
*
- * Pushes a GT TLB invalidation job for execution, using @fence as a dependency.
- * Storage for @fence must be preallocated with xe_gt_tlb_inval_job_alloc_dep
+ * Pushes a TLB invalidation job for execution, using @fence as a dependency.
+ * Storage for @fence must be preallocated with xe_tlb_inval_job_alloc_dep
* prior to this call if @fence is not signaled. Takes a reference to the job’s
* finished fence, which the caller is responsible for releasing, and return it
* to the caller. This function is safe to be called in the path of reclaim.
*
* Return: Job's finished fence on success, cannot fail
*/
-struct dma_fence *xe_gt_tlb_inval_job_push(struct xe_gt_tlb_inval_job *job,
- struct xe_migrate *m,
- struct dma_fence *fence)
+struct dma_fence *xe_tlb_inval_job_push(struct xe_tlb_inval_job *job,
+ struct xe_migrate *m,
+ struct dma_fence *fence)
{
- struct xe_gt_tlb_inval_fence *ifence =
+ struct xe_tlb_inval_fence *ifence =
container_of(job->fence, typeof(*ifence), base);
if (!dma_fence_is_signaled(fence)) {
/*
* Can be in path of reclaim, hence the preallocation of fence
- * storage in xe_gt_tlb_inval_job_alloc_dep. Verify caller did
+ * storage in xe_tlb_inval_job_alloc_dep. Verify caller did
* this correctly.
*/
- xe_assert(gt_to_xe(job->gt),
+ xe_assert(gt_to_xe(job->q->gt),
xa_load(&job->dep.drm.dependencies, 0) ==
dma_fence_get_stub());
dma_fence_get(fence); /* ref released once dependency processed by scheduler */
ptr = xa_store(&job->dep.drm.dependencies, 0, fence,
GFP_ATOMIC);
- xe_assert(gt_to_xe(job->gt), !xa_is_err(ptr));
+ xe_assert(gt_to_xe(job->q->gt), !xa_is_err(ptr));
}
- xe_gt_tlb_inval_job_get(job); /* Pairs with put in free_job */
+ xe_tlb_inval_job_get(job); /* Pairs with put in free_job */
job->fence_armed = true;
/*
*/
xe_migrate_job_lock(m, job->q);
- /* Creation ref pairs with put in xe_gt_tlb_inval_job_destroy */
- xe_gt_tlb_inval_fence_init(job->gt, ifence, false);
+ /* Creation ref pairs with put in xe_tlb_inval_job_destroy */
+ xe_tlb_inval_fence_init(job->tlb_inval, ifence, false);
dma_fence_get(job->fence); /* Pairs with put in DRM scheduler */
drm_sched_job_arm(&job->dep.drm);
/*
* Not using job->fence, as it has its own dma-fence context, which does
- * not allow GT TLB invalidation fences on the same queue, GT tuple to
+ * not allow TLB invalidation fences on the same queue, GT tuple to
* be squashed in dma-resv/DRM scheduler. Instead, we use the DRM scheduler
* context and job's finished fence, which enables squashing.
*/
}
/**
- * xe_gt_tlb_inval_job_get() - Get a reference to GT TLB invalidation job
- * @job: GT TLB invalidation job object
+ * xe_tlb_inval_job_get() - Get a reference to TLB invalidation job
+ * @job: TLB invalidation job object
*
- * Increment the GT TLB invalidation job's reference count
+ * Increment the TLB invalidation job's reference count
*/
-void xe_gt_tlb_inval_job_get(struct xe_gt_tlb_inval_job *job)
+void xe_tlb_inval_job_get(struct xe_tlb_inval_job *job)
{
kref_get(&job->refcount);
}
/**
- * xe_gt_tlb_inval_job_put() - Put a reference to GT TLB invalidation job
- * @job: GT TLB invalidation job object
+ * xe_tlb_inval_job_put() - Put a reference to TLB invalidation job
+ * @job: TLB invalidation job object
*
- * Decrement the GT TLB invalidation job's reference count, call
- * xe_gt_tlb_inval_job_destroy when reference count == 0. Skips decrement if
+ * Decrement the TLB invalidation job's reference count, call
+ * xe_tlb_inval_job_destroy when reference count == 0. Skips decrement if
* input @job is NULL or IS_ERR.
*/
-void xe_gt_tlb_inval_job_put(struct xe_gt_tlb_inval_job *job)
+void xe_tlb_inval_job_put(struct xe_tlb_inval_job *job)
{
if (!IS_ERR_OR_NULL(job))
- kref_put(&job->refcount, xe_gt_tlb_inval_job_destroy);
+ kref_put(&job->refcount, xe_tlb_inval_job_destroy);
}
--- /dev/null
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2025 Intel Corporation
+ */
+
+#ifndef _XE_TLB_INVAL_JOB_H_
+#define _XE_TLB_INVAL_JOB_H_
+
+#include <linux/types.h>
+
+struct dma_fence;
+struct xe_dep_scheduler;
+struct xe_exec_queue;
+struct xe_tlb_inval;
+struct xe_tlb_inval_job;
+struct xe_migrate;
+
+struct xe_tlb_inval_job *
+xe_tlb_inval_job_create(struct xe_exec_queue *q, struct xe_tlb_inval *tlb_inval,
+ struct xe_dep_scheduler *dep_scheduler,
+ u64 start, u64 end, u32 asid);
+
+int xe_tlb_inval_job_alloc_dep(struct xe_tlb_inval_job *job);
+
+struct dma_fence *xe_tlb_inval_job_push(struct xe_tlb_inval_job *job,
+ struct xe_migrate *m,
+ struct dma_fence *fence);
+
+void xe_tlb_inval_job_get(struct xe_tlb_inval_job *job);
+
+void xe_tlb_inval_job_put(struct xe_tlb_inval_job *job);
+
+#endif
* Copyright © 2023 Intel Corporation
*/
-#ifndef _XE_GT_TLB_INVAL_TYPES_H_
-#define _XE_GT_TLB_INVAL_TYPES_H_
+#ifndef _XE_TLB_INVAL_TYPES_H_
+#define _XE_TLB_INVAL_TYPES_H_
#include <linux/workqueue.h>
#include <linux/dma-fence.h>
-struct xe_gt;
-
/** struct xe_tlb_inval - TLB invalidation client */
struct xe_tlb_inval {
+ /** @private: Backend private pointer */
+ void *private;
/** @tlb_inval.seqno: TLB invalidation seqno, protected by CT lock */
#define TLB_INVALIDATION_SEQNO_MAX 0x100000
int seqno;
/** @tlb_invalidation.seqno_lock: protects @tlb_invalidation.seqno */
struct mutex seqno_lock;
/**
- * @tlb_inval.seqno_recv: last received TLB invalidation seqno,
- * protected by CT lock
+ * @seqno_recv: last received TLB invalidation seqno, protected by
+ * CT lock
*/
int seqno_recv;
/**
- * @tlb_inval.pending_fences: list of pending fences waiting TLB
- * invaliations, protected by CT lock
+ * @pending_fences: list of pending fences waiting TLB invaliations,
+ * protected CT lock
*/
struct list_head pending_fences;
/**
- * @tlb_inval.pending_lock: protects @tlb_inval.pending_fences
- * and updating @tlb_inval.seqno_recv.
+ * @pending_lock: protects @pending_fences and updating @seqno_recv.
*/
spinlock_t pending_lock;
/**
- * @tlb_inval.fence_tdr: schedules a delayed call to
- * xe_gt_tlb_fence_timeout after the timeut interval is over.
+ * @fence_tdr: schedules a delayed call to xe_tlb_fence_timeout after
+ * the timeout interval is over.
*/
struct delayed_work fence_tdr;
- /** @wtlb_invalidation.wq: schedules GT TLB invalidation jobs */
+ /** @job_wq: schedules TLB invalidation jobs */
struct workqueue_struct *job_wq;
/** @tlb_inval.lock: protects TLB invalidation fences */
spinlock_t lock;
};
/**
- * struct xe_gt_tlb_inval_fence - XE GT TLB invalidation fence
+ * struct xe_tlb_inval_fence - TLB invalidation fence
*
- * Optionally passed to xe_gt_tlb_inval and will be signaled upon TLB
+ * Optionally passed to xe_tlb_inval* functions and will be signaled upon TLB
* invalidation completion.
*/
-struct xe_gt_tlb_inval_fence {
+struct xe_tlb_inval_fence {
/** @base: dma fence base */
struct dma_fence base;
- /** @gt: GT which fence belong to */
- struct xe_gt *gt;
+ /** @tlb_inval: TLB invalidation client which fence belong to */
+ struct xe_tlb_inval *tlb_inval;
/** @link: link into list of pending tlb fences */
struct list_head link;
/** @seqno: seqno of TLB invalidation to signal fence one */
#include "xe_exec_queue_types.h"
#include "xe_gpu_scheduler_types.h"
-#include "xe_gt_tlb_inval_types.h"
#include "xe_gt_types.h"
#include "xe_guc_exec_queue_types.h"
#include "xe_sched_job.h"
+#include "xe_tlb_inval_types.h"
#include "xe_vm.h"
#define __dev_name_xe(xe) dev_name((xe)->drm.dev)
#define __dev_name_gt(gt) __dev_name_xe(gt_to_xe((gt)))
#define __dev_name_eq(q) __dev_name_gt((q)->gt)
-DECLARE_EVENT_CLASS(xe_gt_tlb_inval_fence,
- TP_PROTO(struct xe_device *xe, struct xe_gt_tlb_inval_fence *fence),
+DECLARE_EVENT_CLASS(xe_tlb_inval_fence,
+ TP_PROTO(struct xe_device *xe, struct xe_tlb_inval_fence *fence),
TP_ARGS(xe, fence),
TP_STRUCT__entry(
__string(dev, __dev_name_xe(xe))
- __field(struct xe_gt_tlb_inval_fence *, fence)
+ __field(struct xe_tlb_inval_fence *, fence)
__field(int, seqno)
),
__get_str(dev), __entry->fence, __entry->seqno)
);
-DEFINE_EVENT(xe_gt_tlb_inval_fence, xe_gt_tlb_inval_fence_send,
- TP_PROTO(struct xe_device *xe, struct xe_gt_tlb_inval_fence *fence),
+DEFINE_EVENT(xe_tlb_inval_fence, xe_tlb_inval_fence_send,
+ TP_PROTO(struct xe_device *xe, struct xe_tlb_inval_fence *fence),
TP_ARGS(xe, fence)
);
-DEFINE_EVENT(xe_gt_tlb_inval_fence, xe_gt_tlb_inval_fence_recv,
- TP_PROTO(struct xe_device *xe, struct xe_gt_tlb_inval_fence *fence),
+DEFINE_EVENT(xe_tlb_inval_fence, xe_tlb_inval_fence_recv,
+ TP_PROTO(struct xe_device *xe, struct xe_tlb_inval_fence *fence),
TP_ARGS(xe, fence)
);
-DEFINE_EVENT(xe_gt_tlb_inval_fence, xe_gt_tlb_inval_fence_signal,
- TP_PROTO(struct xe_device *xe, struct xe_gt_tlb_inval_fence *fence),
+DEFINE_EVENT(xe_tlb_inval_fence, xe_tlb_inval_fence_signal,
+ TP_PROTO(struct xe_device *xe, struct xe_tlb_inval_fence *fence),
TP_ARGS(xe, fence)
);
-DEFINE_EVENT(xe_gt_tlb_inval_fence, xe_gt_tlb_inval_fence_timeout,
- TP_PROTO(struct xe_device *xe, struct xe_gt_tlb_inval_fence *fence),
+DEFINE_EVENT(xe_tlb_inval_fence, xe_tlb_inval_fence_timeout,
+ TP_PROTO(struct xe_device *xe, struct xe_tlb_inval_fence *fence),
TP_ARGS(xe, fence)
);
#include "xe_drm_client.h"
#include "xe_exec_queue.h"
#include "xe_gt_pagefault.h"
-#include "xe_gt_tlb_inval.h"
#include "xe_migrate.h"
#include "xe_pat.h"
#include "xe_pm.h"
#include "xe_svm.h"
#include "xe_sync.h"
#include "xe_tile.h"
+#include "xe_tlb_inval.h"
#include "xe_trace_bo.h"
#include "xe_wa.h"
#include "xe_hmm.h"
xe_pt_clear(xe, vm->pt_root[id]);
for_each_gt(gt, xe, id)
- xe_gt_tlb_inval_vm(gt, vm);
+ xe_tlb_inval_vm(>->tlb_inval, vm);
}
}
int xe_vm_range_tilemask_tlb_inval(struct xe_vm *vm, u64 start,
u64 end, u8 tile_mask)
{
- struct xe_gt_tlb_inval_fence
+ struct xe_tlb_inval_fence
fence[XE_MAX_TILES_PER_DEVICE * XE_MAX_GT_PER_TILE];
struct xe_tile *tile;
u32 fence_id = 0;
if (!(tile_mask & BIT(id)))
continue;
- xe_gt_tlb_inval_fence_init(tile->primary_gt,
- &fence[fence_id], true);
+ xe_tlb_inval_fence_init(&tile->primary_gt->tlb_inval,
+ &fence[fence_id], true);
- err = xe_gt_tlb_inval_range(tile->primary_gt, &fence[fence_id],
- start, end, vm->usm.asid);
+ err = xe_tlb_inval_range(&tile->primary_gt->tlb_inval,
+ &fence[fence_id], start, end,
+ vm->usm.asid);
if (err)
goto wait;
++fence_id;
if (!tile->media_gt)
continue;
- xe_gt_tlb_inval_fence_init(tile->media_gt,
- &fence[fence_id], true);
+ xe_tlb_inval_fence_init(&tile->media_gt->tlb_inval,
+ &fence[fence_id], true);
- err = xe_gt_tlb_inval_range(tile->media_gt, &fence[fence_id],
- start, end, vm->usm.asid);
+ err = xe_tlb_inval_range(&tile->media_gt->tlb_inval,
+ &fence[fence_id], start, end,
+ vm->usm.asid);
if (err)
goto wait;
++fence_id;
wait:
for (id = 0; id < fence_id; ++id)
- xe_gt_tlb_inval_fence_wait(&fence[id]);
+ xe_tlb_inval_fence_wait(&fence[id]);
return err;
}
projects / thirdparty / kernel / stable.git / commitdiff
