Boris needs 7.0-rc6 for a shmem helper fix.
Signed-off-by: Maxime Ripard <mripard@kernel.org>
{
struct vm_area_struct *vma = vmf->vma;
struct drm_gem_object *obj = vma->vm_private_data;
+ struct drm_device *dev = obj->dev;
struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
loff_t num_pages = obj->size >> PAGE_SHIFT;
- vm_fault_t ret;
+ vm_fault_t ret = VM_FAULT_SIGBUS;
struct page **pages = shmem->pages;
- pgoff_t page_offset;
+ pgoff_t page_offset = vmf->pgoff - vma->vm_pgoff; /* page offset within VMA */
+ struct page *page;
+ struct folio *folio;
unsigned long pfn;
- /* Offset to faulty address in the VMA. */
- page_offset = vmf->pgoff - vma->vm_pgoff;
+ if (order && order != PMD_ORDER)
+ return VM_FAULT_FALLBACK;
+
+ dma_resv_lock(obj->resv, NULL);
- dma_resv_lock(shmem->base.resv, NULL);
+ if (page_offset >= num_pages || drm_WARN_ON_ONCE(dev, !shmem->pages) ||
+ shmem->madv < 0)
+ goto out;
- if (page_offset >= num_pages ||
- drm_WARN_ON_ONCE(obj->dev, !shmem->pages) ||
- shmem->madv < 0) {
- ret = VM_FAULT_SIGBUS;
+ page = pages[page_offset];
+ if (drm_WARN_ON_ONCE(dev, !page))
goto out;
- }
+ folio = page_folio(page);
+
+ pfn = page_to_pfn(page);
- if (folio_test_pmd_mappable(folio))
- ret = drm_gem_shmem_try_insert_pfn_pmd(vmf, pfn);
- if (ret != VM_FAULT_NOPAGE)
- ret = vmf_insert_pfn(vma, vmf->address, pfn);
-
- pfn = page_to_pfn(pages[page_offset]);
+ ret = try_insert_pfn(vmf, order, pfn);
+ if (ret == VM_FAULT_NOPAGE)
+ folio_mark_accessed(folio);
- out:
- dma_resv_unlock(shmem->base.resv);
+out:
+ dma_resv_unlock(obj->resv);
return ret;
}
drm_gem_vm_close(vma);
}
+static vm_fault_t drm_gem_shmem_pfn_mkwrite(struct vm_fault *vmf)
+{
+ struct vm_area_struct *vma = vmf->vma;
+ struct drm_gem_object *obj = vma->vm_private_data;
+ struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
+ loff_t num_pages = obj->size >> PAGE_SHIFT;
+ pgoff_t page_offset = vmf->pgoff - vma->vm_pgoff; /* page offset within VMA */
+
+ if (drm_WARN_ON(obj->dev, !shmem->pages || page_offset >= num_pages))
+ return VM_FAULT_SIGBUS;
+
+ file_update_time(vma->vm_file);
+
+ folio_mark_dirty(page_folio(shmem->pages[page_offset]));
+
+ return 0;
+}
+
const struct vm_operations_struct drm_gem_shmem_vm_ops = {
.fault = drm_gem_shmem_fault,
+ #ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP
+ .huge_fault = drm_gem_shmem_any_fault,
+ #endif
.open = drm_gem_shmem_vm_open,
.close = drm_gem_shmem_vm_close,
+ .pfn_mkwrite = drm_gem_shmem_pfn_mkwrite,
};
EXPORT_SYMBOL_GPL(drm_gem_shmem_vm_ops);
#define ENABLE_SMP_LD_RENDER_SURFACE_CONTROL REG_BIT(44 - 32)
#define FORCE_SLM_FENCE_SCOPE_TO_TILE REG_BIT(42 - 32)
#define FORCE_UGM_FENCE_SCOPE_TO_TILE REG_BIT(41 - 32)
+ #define L3_128B_256B_WRT_DIS REG_BIT(40 - 32)
#define MAXREQS_PER_BANK REG_GENMASK(39 - 32, 37 - 32)
#define DISABLE_128B_EVICTION_COMMAND_UDW REG_BIT(36 - 32)
+#define LSCFE_SAME_ADDRESS_ATOMICS_COALESCING_DISABLE REG_BIT(35 - 32)
+
+#define ROW_CHICKEN5 XE_REG_MCR(0xe7f0)
+#define CPSS_AWARE_DIS REG_BIT(3)
#define SARB_CHICKEN1 XE_REG_MCR(0xe90c)
#define COMP_CKN_IN REG_GENMASK(30, 29)
* give us the correct placement for free.
*/
++#define XE_GGTT_FLAGS_64K BIT(0)
++#define XE_GGTT_FLAGS_ONLINE BIT(1)
++
/**
* struct xe_ggtt_node - A node in GGTT.
*
bool invalidate_on_remove;
};
+/**
+ * struct xe_ggtt_pt_ops - GGTT Page table operations
+ * Which can vary from platform to platform.
+ */
+struct xe_ggtt_pt_ops {
+ /** @pte_encode_flags: Encode PTE flags for a given BO */
+ u64 (*pte_encode_flags)(struct xe_bo *bo, u16 pat_index);
+
+ /** @ggtt_set_pte: Directly write into GGTT's PTE */
+ xe_ggtt_set_pte_fn ggtt_set_pte;
+
+ /** @ggtt_get_pte: Directly read from GGTT's PTE */
+ u64 (*ggtt_get_pte)(struct xe_ggtt *ggtt, u64 addr);
+};
+
+/**
+ * struct xe_ggtt - Main GGTT struct
+ *
+ * In general, each tile can contains its own Global Graphics Translation Table
+ * (GGTT) instance.
+ */
+struct xe_ggtt {
+ /** @tile: Back pointer to tile where this GGTT belongs */
+ struct xe_tile *tile;
+ /** @start: Start offset of GGTT */
+ u64 start;
+ /** @size: Total usable size of this GGTT */
+ u64 size;
+
+#define XE_GGTT_FLAGS_64K BIT(0)
+ /**
+ * @flags: Flags for this GGTT
+ * Acceptable flags:
+ * - %XE_GGTT_FLAGS_64K - if PTE size is 64K. Otherwise, regular is 4K.
++ * - %XE_GGTT_FLAGS_ONLINE - is GGTT online, protected by ggtt->lock
++ * after init
+ */
+ unsigned int flags;
+ /** @scratch: Internal object allocation used as a scratch page */
+ struct xe_bo *scratch;
+ /** @lock: Mutex lock to protect GGTT data */
+ struct mutex lock;
+ /**
+ * @gsm: The iomem pointer to the actual location of the translation
+ * table located in the GSM for easy PTE manipulation
+ */
+ u64 __iomem *gsm;
+ /** @pt_ops: Page Table operations per platform */
+ const struct xe_ggtt_pt_ops *pt_ops;
+ /** @mm: The memory manager used to manage individual GGTT allocations */
+ struct drm_mm mm;
+ /** @access_count: counts GGTT writes */
+ unsigned int access_count;
+ /** @wq: Dedicated unordered work queue to process node removals */
+ struct workqueue_struct *wq;
+};
+
static u64 xelp_ggtt_pte_flags(struct xe_bo *bo, u16 pat_index)
{
u64 pte = XE_PAGE_PRESENT;
if (err)
return err;
- err = devm_add_action_or_reset(xe->drm.dev, dev_fini_ggtt, ggtt);
- if (err)
- return err;
-
- if (IS_SRIOV_VF(xe)) {
- err = xe_tile_sriov_vf_prepare_ggtt(ggtt->tile);
- if (err)
- return err;
- }
-
- return 0;
+ ggtt->flags |= XE_GGTT_FLAGS_ONLINE;
+ return devm_add_action_or_reset(xe->drm.dev, dev_fini_ggtt, ggtt);
}
ALLOW_ERROR_INJECTION(xe_ggtt_init_early, ERRNO); /* See xe_pci_probe() */
static void ggtt_node_remove(struct xe_ggtt_node *node)
{
struct xe_ggtt *ggtt = node->ggtt;
- struct xe_device *xe = tile_to_xe(ggtt->tile);
bool bound;
- int idx;
-
- bound = drm_dev_enter(&xe->drm, &idx);
mutex_lock(&ggtt->lock);
+ bound = ggtt->flags & XE_GGTT_FLAGS_ONLINE;
if (bound)
- xe_ggtt_clear(ggtt, node->base.start, node->base.size);
+ xe_ggtt_clear(ggtt, xe_ggtt_node_addr(node), xe_ggtt_node_size(node));
drm_mm_remove_node(&node->base);
node->base.size = 0;
mutex_unlock(&ggtt->lock);
if (node->invalidate_on_remove)
xe_ggtt_invalidate(ggtt);
- drm_dev_exit(idx);
-
free_node:
- xe_ggtt_node_fini(node);
+ ggtt_node_fini(node);
}
static void ggtt_node_remove_work_func(struct work_struct *work)
#include "xe_gt_printk.h"
#include "xe_gt_sysfs.h"
#include "xe_mmio.h"
+ #include "xe_pm.h"
#include "xe_sriov.h"
+#include "xe_sriov_pf.h"
static void __xe_gt_apply_ccs_mode(struct xe_gt *gt, u32 num_engines)
{
return -EBUSY;
}
- if (gt->ccs_mode != num_engines) {
- xe_gt_info(gt, "Setting compute mode to %d\n", num_engines);
- gt->ccs_mode = num_engines;
- xe_gt_record_user_engines(gt);
- guard(xe_pm_runtime)(xe);
- xe_gt_reset(gt);
+ if (gt->ccs_mode == num_engines)
+ return count;
+
+ /*
+ * Changing default CCS mode is only allowed when there
+ * are no VFs. Try to lockdown PF to find out.
+ */
+ if (gt_ccs_mode_default(gt) && IS_SRIOV_PF(xe)) {
+ ret = xe_sriov_pf_lockdown(xe);
+ if (ret) {
+ xe_gt_dbg(gt, "Can't change CCS Mode: VFs are enabled\n");
+ return ret;
+ }
}
- mutex_unlock(&xe->drm.filelist_mutex);
+ xe_gt_info(gt, "Setting compute mode to %d\n", num_engines);
+ gt->ccs_mode = num_engines;
+ xe_gt_record_user_engines(gt);
++ guard(xe_pm_runtime)(xe);
+ xe_gt_reset(gt);
+
+ /* We may end PF lockdown once CCS mode is default again */
+ if (gt_ccs_mode_default(gt) && IS_SRIOV_PF(xe))
+ xe_sriov_pf_end_lockdown(xe);
return count;
}