#include "amdgpu_dma_buf.h"
#include <uapi/linux/kfd_ioctl.h>
#include "amdgpu_xgmi.h"
+#include "kfd_priv.h"
#include "kfd_smi_events.h"
#include <drm/ttm/ttm_tt.h>
#include "amdgpu_amdkfd.h"
#include "kfd_smi_events.h"
-static bool cik_event_interrupt_isr(struct kfd_dev *dev,
+static bool cik_event_interrupt_isr(struct kfd_node *dev,
const uint32_t *ih_ring_entry,
uint32_t *patched_ihre,
bool *patched_flag)
!amdgpu_no_queue_eviction_on_vm_fault);
}
-static void cik_event_interrupt_wq(struct kfd_dev *dev,
+static void cik_event_interrupt_wq(struct kfd_node *dev,
const uint32_t *ih_ring_entry)
{
const struct cik_ih_ring_entry *ihre =
void *data)
{
struct kfd_ioctl_create_queue_args *args = data;
- struct kfd_dev *dev;
+ struct kfd_node *dev;
int err = 0;
unsigned int queue_id;
struct kfd_process_device *pdd;
}
if (!pdd->doorbell_index &&
- kfd_alloc_process_doorbells(dev, &pdd->doorbell_index) < 0) {
+ kfd_alloc_process_doorbells(dev->kfd, &pdd->doorbell_index) < 0) {
err = -ENOMEM;
goto err_alloc_doorbells;
}
/* Starting with GFX11, wptr BOs must be mapped to GART for MES to determine work
* on unmapped queues for usermode queue oversubscription (no aggregated doorbell)
*/
- if (dev->shared_resources.enable_mes &&
+ if (dev->kfd->shared_resources.enable_mes &&
((dev->adev->mes.sched_version & AMDGPU_MES_API_VERSION_MASK)
>> AMDGPU_MES_API_VERSION_SHIFT) >= 2) {
struct amdgpu_bo_va_mapping *wptr_mapping;
{
struct kfd_ioctl_set_scratch_backing_va_args *args = data;
struct kfd_process_device *pdd;
- struct kfd_dev *dev;
+ struct kfd_node *dev;
long err;
mutex_lock(&p->mutex);
return ret;
}
-bool kfd_dev_is_large_bar(struct kfd_dev *dev)
+bool kfd_dev_is_large_bar(struct kfd_node *dev)
{
if (debug_largebar) {
pr_debug("Simulate large-bar allocation on non large-bar machine\n");
return true;
}
- if (dev->use_iommu_v2)
+ if (dev->kfd->use_iommu_v2)
return false;
- if (dev->local_mem_info.local_mem_size_private == 0 &&
- dev->local_mem_info.local_mem_size_public > 0)
+ if (dev->kfd->local_mem_info.local_mem_size_private == 0 &&
+ dev->kfd->local_mem_info.local_mem_size_public > 0)
return true;
return false;
}
struct kfd_ioctl_alloc_memory_of_gpu_args *args = data;
struct kfd_process_device *pdd;
void *mem;
- struct kfd_dev *dev;
+ struct kfd_node *dev;
int idr_handle;
long err;
uint64_t offset = args->mmap_offset;
}
if (flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) {
- if (args->size != kfd_doorbell_process_slice(dev)) {
+ if (args->size != kfd_doorbell_process_slice(dev->kfd)) {
err = -EINVAL;
goto err_unlock;
}
struct kfd_ioctl_map_memory_to_gpu_args *args = data;
struct kfd_process_device *pdd, *peer_pdd;
void *mem;
- struct kfd_dev *dev;
+ struct kfd_node *dev;
long err = 0;
int i;
uint32_t *devices_arr = NULL;
args->n_success = i+1;
}
- flush_tlb = kfd_flush_tlb_after_unmap(pdd->dev);
+ flush_tlb = kfd_flush_tlb_after_unmap(pdd->dev->kfd);
if (flush_tlb) {
err = amdgpu_amdkfd_gpuvm_sync_memory(pdd->dev->adev,
(struct kgd_mem *) mem, true);
int retval;
struct kfd_ioctl_alloc_queue_gws_args *args = data;
struct queue *q;
- struct kfd_dev *dev;
+ struct kfd_node *dev;
mutex_lock(&p->mutex);
q = pqm_get_user_queue(&p->pqm, args->queue_id);
struct kfd_process *p, void *data)
{
struct kfd_ioctl_get_dmabuf_info_args *args = data;
- struct kfd_dev *dev = NULL;
+ struct kfd_node *dev = NULL;
struct amdgpu_device *dmabuf_adev;
void *metadata_buffer = NULL;
uint32_t flags;
struct kfd_ioctl_export_dmabuf_args *args = data;
struct kfd_process_device *pdd;
struct dma_buf *dmabuf;
- struct kfd_dev *dev;
+ struct kfd_node *dev;
void *mem;
int ret = 0;
}
for (i = 0; i < args->num_devices; i++) {
- struct kfd_dev *dev;
+ struct kfd_node *dev;
struct kfd_process_device *pdd;
struct file *drm_file;
}
if (!pdd->doorbell_index &&
- kfd_alloc_process_doorbells(pdd->dev, &pdd->doorbell_index) < 0) {
+ kfd_alloc_process_doorbells(pdd->dev->kfd, &pdd->doorbell_index) < 0) {
ret = -ENOMEM;
goto exit;
}
u64 offset;
if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) {
- if (bo_bucket->size != kfd_doorbell_process_slice(pdd->dev))
+ if (bo_bucket->size !=
+ kfd_doorbell_process_slice(pdd->dev->kfd))
return -EINVAL;
offset = kfd_get_process_doorbells(pdd);
/* now map these BOs to GPU/s */
for (j = 0; j < p->n_pdds; j++) {
- struct kfd_dev *peer;
+ struct kfd_node *peer;
struct kfd_process_device *peer_pdd;
if (!bo_priv->mapped_gpuids[j])
return retcode;
}
-static int kfd_mmio_mmap(struct kfd_dev *dev, struct kfd_process *process,
+static int kfd_mmio_mmap(struct kfd_node *dev, struct kfd_process *process,
struct vm_area_struct *vma)
{
phys_addr_t address;
static int kfd_mmap(struct file *filp, struct vm_area_struct *vma)
{
struct kfd_process *process;
- struct kfd_dev *dev = NULL;
+ struct kfd_node *dev = NULL;
unsigned long mmap_offset;
unsigned int gpu_id;
return i;
}
-int kfd_get_gpu_cache_info(struct kfd_dev *kdev, struct kfd_gpu_cache_info **pcache_info)
+int kfd_get_gpu_cache_info(struct kfd_node *kdev, struct kfd_gpu_cache_info **pcache_info)
{
int num_of_cache_types = 0;
case IP_VERSION(11, 0, 3):
case IP_VERSION(11, 0, 4):
num_of_cache_types =
- kfd_fill_gpu_cache_info_from_gfx_config(kdev, *pcache_info);
+ kfd_fill_gpu_cache_info_from_gfx_config(kdev->kfd, *pcache_info);
break;
default:
*pcache_info = dummy_cache_info;
}
static int kfd_fill_gpu_memory_affinity(int *avail_size,
- struct kfd_dev *kdev, uint8_t type, uint64_t size,
+ struct kfd_node *kdev, uint8_t type, uint64_t size,
struct crat_subtype_memory *sub_type_hdr,
uint32_t proximity_domain,
const struct kfd_local_mem_info *local_mem_info)
}
#ifdef CONFIG_ACPI_NUMA
-static void kfd_find_numa_node_in_srat(struct kfd_dev *kdev)
+static void kfd_find_numa_node_in_srat(struct kfd_node *kdev)
{
struct acpi_table_header *table_header = NULL;
struct acpi_subtable_header *sub_header = NULL;
* Return 0 if successful else return -ve value
*/
static int kfd_fill_gpu_direct_io_link_to_cpu(int *avail_size,
- struct kfd_dev *kdev,
+ struct kfd_node *kdev,
struct crat_subtype_iolink *sub_type_hdr,
uint32_t proximity_domain)
{
}
static int kfd_fill_gpu_xgmi_link_to_gpu(int *avail_size,
- struct kfd_dev *kdev,
- struct kfd_dev *peer_kdev,
+ struct kfd_node *kdev,
+ struct kfd_node *peer_kdev,
struct crat_subtype_iolink *sub_type_hdr,
uint32_t proximity_domain_from,
uint32_t proximity_domain_to)
* [OUT] actual size of data filled in crat_image
*/
static int kfd_create_vcrat_image_gpu(void *pcrat_image,
- size_t *size, struct kfd_dev *kdev,
+ size_t *size, struct kfd_node *kdev,
uint32_t proximity_domain)
{
struct crat_header *crat_table = (struct crat_header *)pcrat_image;
/* Check if this node supports IOMMU. During parsing this flag will
* translate to HSA_CAP_ATS_PRESENT
*/
- if (!kfd_iommu_check_device(kdev))
+ if (!kfd_iommu_check_device(kdev->kfd))
cu->hsa_capability |= CRAT_CU_FLAGS_IOMMU_PRESENT;
crat_table->length += sub_type_hdr->length;
* report the total FB size (public+private) as a single
* private heap.
*/
- local_mem_info = kdev->local_mem_info;
+ local_mem_info = kdev->kfd->local_mem_info;
sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
sub_type_hdr->length);
* (from other GPU to this GPU) will be added
* in kfd_parse_subtype_iolink.
*/
- if (kdev->hive_id) {
+ if (kdev->kfd->hive_id) {
for (nid = 0; nid < proximity_domain; ++nid) {
peer_dev = kfd_topology_device_by_proximity_domain_no_lock(nid);
if (!peer_dev->gpu)
continue;
- if (peer_dev->gpu->hive_id != kdev->hive_id)
+ if (peer_dev->gpu->kfd->hive_id != kdev->kfd->hive_id)
continue;
sub_type_hdr = (typeof(sub_type_hdr))(
(char *)sub_type_hdr +
* (COMPUTE_UNIT_CPU | COMPUTE_UNIT_GPU) - Create VCRAT for APU
* -- this option is not currently implemented.
* The assumption is that all AMD APUs will have CRAT
- * @kdev: Valid kfd_device required if flags contain COMPUTE_UNIT_GPU
+ * @kdev: Valid kfd_node required if flags contain COMPUTE_UNIT_GPU
*
* Return 0 if successful else return -ve value
*/
int kfd_create_crat_image_virtual(void **crat_image, size_t *size,
- int flags, struct kfd_dev *kdev,
+ int flags, struct kfd_node *kdev,
uint32_t proximity_domain)
{
void *pcrat_image = NULL;
#pragma pack()
-struct kfd_dev;
+struct kfd_node;
/* Static table to describe GPU Cache information */
struct kfd_gpu_cache_info {
*/
uint32_t num_cu_shared;
};
-int kfd_get_gpu_cache_info(struct kfd_dev *kdev, struct kfd_gpu_cache_info **pcache_info);
+int kfd_get_gpu_cache_info(struct kfd_node *kdev, struct kfd_gpu_cache_info **pcache_info);
int kfd_create_crat_image_acpi(void **crat_image, size_t *size);
void kfd_destroy_crat_image(void *crat_image);
int kfd_parse_crat_table(void *crat_image, struct list_head *device_list,
uint32_t proximity_domain);
int kfd_create_crat_image_virtual(void **crat_image, size_t *size,
- int flags, struct kfd_dev *kdev,
+ int flags, struct kfd_node *kdev,
uint32_t proximity_domain);
#endif /* KFD_CRAT_H_INCLUDED */
static ssize_t kfd_debugfs_hang_hws_write(struct file *file,
const char __user *user_buf, size_t size, loff_t *ppos)
{
- struct kfd_dev *dev;
+ struct kfd_node *dev;
char tmp[16];
uint32_t gpu_id;
int ret = -EINVAL;
static void kfd_gtt_sa_fini(struct kfd_dev *kfd);
static int kfd_resume_iommu(struct kfd_dev *kfd);
-static int kfd_resume(struct kfd_dev *kfd);
+static int kfd_resume(struct kfd_node *kfd);
static void kfd_device_info_set_sdma_info(struct kfd_dev *kfd)
{
memset(&kfd->doorbell_available_index, 0,
sizeof(kfd->doorbell_available_index));
- atomic_set(&kfd->sram_ecc_flag, 0);
-
ida_init(&kfd->doorbell_ida);
return kfd;
}
}
-static int kfd_gws_init(struct kfd_dev *kfd)
+static int kfd_gws_init(struct kfd_node *node)
{
int ret = 0;
+ struct kfd_dev *kfd = node->kfd;
- if (kfd->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS)
+ if (node->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS)
return 0;
- if (hws_gws_support || (KFD_IS_SOC15(kfd) &&
- ((KFD_GC_VERSION(kfd) == IP_VERSION(9, 0, 1)
+ if (hws_gws_support || (KFD_IS_SOC15(node) &&
+ ((KFD_GC_VERSION(node) == IP_VERSION(9, 0, 1)
&& kfd->mec2_fw_version >= 0x81b3) ||
- (KFD_GC_VERSION(kfd) <= IP_VERSION(9, 4, 0)
+ (KFD_GC_VERSION(node) <= IP_VERSION(9, 4, 0)
&& kfd->mec2_fw_version >= 0x1b3) ||
- (KFD_GC_VERSION(kfd) == IP_VERSION(9, 4, 1)
+ (KFD_GC_VERSION(node) == IP_VERSION(9, 4, 1)
&& kfd->mec2_fw_version >= 0x30) ||
- (KFD_GC_VERSION(kfd) == IP_VERSION(9, 4, 2)
+ (KFD_GC_VERSION(node) == IP_VERSION(9, 4, 2)
&& kfd->mec2_fw_version >= 0x28) ||
- (KFD_GC_VERSION(kfd) >= IP_VERSION(10, 3, 0)
- && KFD_GC_VERSION(kfd) < IP_VERSION(11, 0, 0)
+ (KFD_GC_VERSION(node) >= IP_VERSION(10, 3, 0)
+ && KFD_GC_VERSION(node) < IP_VERSION(11, 0, 0)
&& kfd->mec2_fw_version >= 0x6b))))
- ret = amdgpu_amdkfd_alloc_gws(kfd->adev,
- kfd->adev->gds.gws_size, &kfd->gws);
+ ret = amdgpu_amdkfd_alloc_gws(node->adev,
+ node->adev->gds.gws_size, &node->gws);
return ret;
}
-static void kfd_smi_init(struct kfd_dev *dev)
+static void kfd_smi_init(struct kfd_node *dev)
{
INIT_LIST_HEAD(&dev->smi_clients);
spin_lock_init(&dev->smi_lock);
}
+static int kfd_init_node(struct kfd_node *node)
+{
+ int err = -1;
+
+ if (kfd_interrupt_init(node)) {
+ dev_err(kfd_device, "Error initializing interrupts\n");
+ goto kfd_interrupt_error;
+ }
+
+ node->dqm = device_queue_manager_init(node);
+ if (!node->dqm) {
+ dev_err(kfd_device, "Error initializing queue manager\n");
+ goto device_queue_manager_error;
+ }
+
+ if (kfd_gws_init(node)) {
+ dev_err(kfd_device, "Could not allocate %d gws\n",
+ node->adev->gds.gws_size);
+ goto gws_error;
+ }
+
+ if (kfd_resume(node))
+ goto kfd_resume_error;
+
+ if (kfd_topology_add_device(node)) {
+ dev_err(kfd_device, "Error adding device to topology\n");
+ goto kfd_topology_add_device_error;
+ }
+
+ kfd_smi_init(node);
+
+ return 0;
+
+kfd_topology_add_device_error:
+kfd_resume_error:
+gws_error:
+ device_queue_manager_uninit(node->dqm);
+device_queue_manager_error:
+ kfd_interrupt_exit(node);
+kfd_interrupt_error:
+ if (node->gws)
+ amdgpu_amdkfd_free_gws(node->adev, node->gws);
+
+ /* Cleanup the node memory here */
+ kfree(node);
+ return err;
+}
+
+static void kfd_cleanup_node(struct kfd_dev *kfd)
+{
+ struct kfd_node *knode = kfd->node;
+
+ device_queue_manager_uninit(knode->dqm);
+ kfd_interrupt_exit(knode);
+ kfd_topology_remove_device(knode);
+ if (knode->gws)
+ amdgpu_amdkfd_free_gws(knode->adev, knode->gws);
+ kfree(knode);
+ kfd->node = NULL;
+}
+
bool kgd2kfd_device_init(struct kfd_dev *kfd,
const struct kgd2kfd_shared_resources *gpu_resources)
{
unsigned int size, map_process_packet_size;
+ struct kfd_node *node;
+ uint32_t first_vmid_kfd, last_vmid_kfd, vmid_num_kfd;
+ unsigned int max_proc_per_quantum;
kfd->mec_fw_version = amdgpu_amdkfd_get_fw_version(kfd->adev,
KGD_ENGINE_MEC1);
KGD_ENGINE_SDMA1);
kfd->shared_resources = *gpu_resources;
- kfd->vm_info.first_vmid_kfd = ffs(gpu_resources->compute_vmid_bitmap)-1;
- kfd->vm_info.last_vmid_kfd = fls(gpu_resources->compute_vmid_bitmap)-1;
- kfd->vm_info.vmid_num_kfd = kfd->vm_info.last_vmid_kfd
- - kfd->vm_info.first_vmid_kfd + 1;
+ first_vmid_kfd = ffs(gpu_resources->compute_vmid_bitmap)-1;
+ last_vmid_kfd = fls(gpu_resources->compute_vmid_bitmap)-1;
+ vmid_num_kfd = last_vmid_kfd - first_vmid_kfd + 1;
/* Allow BIF to recode atomics to PCIe 3.0 AtomicOps.
* 32 and 64-bit requests are possible and must be
/* Verify module parameters regarding mapped process number*/
if (hws_max_conc_proc >= 0)
- kfd->max_proc_per_quantum = min((u32)hws_max_conc_proc, kfd->vm_info.vmid_num_kfd);
+ max_proc_per_quantum = min((u32)hws_max_conc_proc, vmid_num_kfd);
else
- kfd->max_proc_per_quantum = kfd->vm_info.vmid_num_kfd;
+ max_proc_per_quantum = vmid_num_kfd;
/* calculate max size of mqds needed for queues */
size = max_num_of_queues_per_device *
kfd->noretry = kfd->adev->gmc.noretry;
- if (kfd_interrupt_init(kfd)) {
- dev_err(kfd_device, "Error initializing interrupts\n");
- goto kfd_interrupt_error;
- }
-
- kfd->dqm = device_queue_manager_init(kfd);
- if (!kfd->dqm) {
- dev_err(kfd_device, "Error initializing queue manager\n");
- goto device_queue_manager_error;
- }
-
- /* If supported on this device, allocate global GWS that is shared
- * by all KFD processes
- */
- if (kfd_gws_init(kfd)) {
- dev_err(kfd_device, "Could not allocate %d gws\n",
- kfd->adev->gds.gws_size);
- goto gws_error;
- }
-
/* If CRAT is broken, won't set iommu enabled */
kfd_double_confirm_iommu_support(kfd);
svm_migrate_init(kfd->adev);
- if (kfd_resume_iommu(kfd))
- goto device_iommu_error;
-
- if (kfd_resume(kfd))
- goto kfd_resume_error;
-
- amdgpu_amdkfd_get_local_mem_info(kfd->adev, &kfd->local_mem_info);
+ /* Allocate the KFD node */
+ node = kzalloc(sizeof(struct kfd_node), GFP_KERNEL);
+ if (!node) {
+ dev_err(kfd_device, "Error allocating KFD node\n");
+ goto node_alloc_error;
+ }
- if (kfd_topology_add_device(kfd)) {
- dev_err(kfd_device, "Error adding device to topology\n");
- goto kfd_topology_add_device_error;
+ node->adev = kfd->adev;
+ node->kfd = kfd;
+ node->kfd2kgd = kfd->kfd2kgd;
+ node->vm_info.vmid_num_kfd = vmid_num_kfd;
+ node->vm_info.first_vmid_kfd = first_vmid_kfd;
+ node->vm_info.last_vmid_kfd = last_vmid_kfd;
+ node->max_proc_per_quantum = max_proc_per_quantum;
+ atomic_set(&node->sram_ecc_flag, 0);
+
+ /* Initialize the KFD node */
+ if (kfd_init_node(node)) {
+ dev_err(kfd_device, "Error initializing KFD node\n");
+ goto node_init_error;
}
+ kfd->node = node;
- kfd_smi_init(kfd);
+ if (kfd_resume_iommu(kfd))
+ goto kfd_resume_iommu_error;
+
+ amdgpu_amdkfd_get_local_mem_info(kfd->adev, &kfd->local_mem_info);
kfd->init_complete = true;
dev_info(kfd_device, "added device %x:%x\n", kfd->adev->pdev->vendor,
kfd->adev->pdev->device);
pr_debug("Starting kfd with the following scheduling policy %d\n",
- kfd->dqm->sched_policy);
+ node->dqm->sched_policy);
goto out;
-kfd_topology_add_device_error:
-kfd_resume_error:
+kfd_resume_iommu_error:
+ kfd_cleanup_node(kfd);
+node_init_error:
+node_alloc_error:
device_iommu_error:
-gws_error:
- device_queue_manager_uninit(kfd->dqm);
-device_queue_manager_error:
- kfd_interrupt_exit(kfd);
-kfd_interrupt_error:
kfd_doorbell_fini(kfd);
kfd_doorbell_error:
kfd_gtt_sa_fini(kfd);
kfd_gtt_sa_init_error:
amdgpu_amdkfd_free_gtt_mem(kfd->adev, kfd->gtt_mem);
alloc_gtt_mem_failure:
- if (kfd->gws)
- amdgpu_amdkfd_free_gws(kfd->adev, kfd->gws);
dev_err(kfd_device,
"device %x:%x NOT added due to errors\n",
kfd->adev->pdev->vendor, kfd->adev->pdev->device);
void kgd2kfd_device_exit(struct kfd_dev *kfd)
{
if (kfd->init_complete) {
- device_queue_manager_uninit(kfd->dqm);
- kfd_interrupt_exit(kfd);
- kfd_topology_remove_device(kfd);
+ kfd_cleanup_node(kfd);
kfd_doorbell_fini(kfd);
ida_destroy(&kfd->doorbell_ida);
kfd_gtt_sa_fini(kfd);
amdgpu_amdkfd_free_gtt_mem(kfd->adev, kfd->gtt_mem);
- if (kfd->gws)
- amdgpu_amdkfd_free_gws(kfd->adev, kfd->gws);
}
kfree(kfd);
int kgd2kfd_pre_reset(struct kfd_dev *kfd)
{
+ struct kfd_node *node = kfd->node;
+
if (!kfd->init_complete)
return 0;
- kfd_smi_event_update_gpu_reset(kfd, false);
+ kfd_smi_event_update_gpu_reset(node, false);
- kfd->dqm->ops.pre_reset(kfd->dqm);
+ node->dqm->ops.pre_reset(node->dqm);
kgd2kfd_suspend(kfd, false);
- kfd_signal_reset_event(kfd);
+ kfd_signal_reset_event(node);
return 0;
}
int kgd2kfd_post_reset(struct kfd_dev *kfd)
{
int ret;
+ struct kfd_node *node = kfd->node;
if (!kfd->init_complete)
return 0;
- ret = kfd_resume(kfd);
+ ret = kfd_resume(node);
if (ret)
return ret;
atomic_dec(&kfd_locked);
- atomic_set(&kfd->sram_ecc_flag, 0);
+ atomic_set(&node->sram_ecc_flag, 0);
- kfd_smi_event_update_gpu_reset(kfd, true);
+ kfd_smi_event_update_gpu_reset(node, true);
return 0;
}
void kgd2kfd_suspend(struct kfd_dev *kfd, bool run_pm)
{
+ struct kfd_node *node = kfd->node;
+
if (!kfd->init_complete)
return;
kfd_suspend_all_processes();
}
- kfd->dqm->ops.stop(kfd->dqm);
+ node->dqm->ops.stop(node->dqm);
kfd_iommu_suspend(kfd);
}
int kgd2kfd_resume(struct kfd_dev *kfd, bool run_pm)
{
int ret, count;
+ struct kfd_node *node = kfd->node;
if (!kfd->init_complete)
return 0;
- ret = kfd_resume(kfd);
+ ret = kfd_resume(node);
if (ret)
return ret;
return err;
}
-static int kfd_resume(struct kfd_dev *kfd)
+static int kfd_resume(struct kfd_node *node)
{
int err = 0;
- err = kfd->dqm->ops.start(kfd->dqm);
+ err = node->dqm->ops.start(node->dqm);
if (err)
dev_err(kfd_device,
"Error starting queue manager for device %x:%x\n",
- kfd->adev->pdev->vendor, kfd->adev->pdev->device);
+ node->adev->pdev->vendor, node->adev->pdev->device);
return err;
}
uint32_t patched_ihre[KFD_MAX_RING_ENTRY_SIZE];
bool is_patched = false;
unsigned long flags;
+ struct kfd_node *node = kfd->node;
if (!kfd->init_complete)
return;
return;
}
- spin_lock_irqsave(&kfd->interrupt_lock, flags);
+ spin_lock_irqsave(&node->interrupt_lock, flags);
- if (kfd->interrupts_active
- && interrupt_is_wanted(kfd, ih_ring_entry,
+ if (node->interrupts_active
+ && interrupt_is_wanted(node, ih_ring_entry,
patched_ihre, &is_patched)
- && enqueue_ih_ring_entry(kfd,
+ && enqueue_ih_ring_entry(node,
is_patched ? patched_ihre : ih_ring_entry))
- kfd_queue_work(kfd->ih_wq, &kfd->interrupt_work);
+ kfd_queue_work(node->ih_wq, &node->interrupt_work);
- spin_unlock_irqrestore(&kfd->interrupt_lock, flags);
+ spin_unlock_irqrestore(&node->interrupt_lock, flags);
}
int kgd2kfd_quiesce_mm(struct mm_struct *mm, uint32_t trigger)
return (uint32_t *) ((uint64_t) start_addr + bit_num * chunk_size);
}
-int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size,
+int kfd_gtt_sa_allocate(struct kfd_node *node, unsigned int size,
struct kfd_mem_obj **mem_obj)
{
unsigned int found, start_search, cur_size;
+ struct kfd_dev *kfd = node->kfd;
if (size == 0)
return -EINVAL;
return -ENOMEM;
}
-int kfd_gtt_sa_free(struct kfd_dev *kfd, struct kfd_mem_obj *mem_obj)
+int kfd_gtt_sa_free(struct kfd_node *node, struct kfd_mem_obj *mem_obj)
{
+ struct kfd_dev *kfd = node->kfd;
+
/* Act like kfree when trying to free a NULL object */
if (!mem_obj)
return 0;
void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd)
{
if (kfd)
- atomic_inc(&kfd->sram_ecc_flag);
+ atomic_inc(&kfd->node->sram_ecc_flag);
}
-void kfd_inc_compute_active(struct kfd_dev *kfd)
+void kfd_inc_compute_active(struct kfd_node *node)
{
- if (atomic_inc_return(&kfd->compute_profile) == 1)
- amdgpu_amdkfd_set_compute_idle(kfd->adev, false);
+ if (atomic_inc_return(&node->kfd->compute_profile) == 1)
+ amdgpu_amdkfd_set_compute_idle(node->adev, false);
}
-void kfd_dec_compute_active(struct kfd_dev *kfd)
+void kfd_dec_compute_active(struct kfd_node *node)
{
- int count = atomic_dec_return(&kfd->compute_profile);
+ int count = atomic_dec_return(&node->kfd->compute_profile);
if (count == 0)
- amdgpu_amdkfd_set_compute_idle(kfd->adev, true);
+ amdgpu_amdkfd_set_compute_idle(node->adev, true);
WARN_ONCE(count < 0, "Compute profile ref. count error");
}
void kgd2kfd_smi_event_throttle(struct kfd_dev *kfd, uint64_t throttle_bitmask)
{
if (kfd && kfd->init_complete)
- kfd_smi_event_update_thermal_throttling(kfd, throttle_bitmask);
+ kfd_smi_event_update_thermal_throttling(kfd->node, throttle_bitmask);
}
/* kfd_get_num_sdma_engines returns the number of PCIe optimized SDMA and
* When the device has more than two engines, we reserve two for PCIe to enable
* full-duplex and the rest are used as XGMI.
*/
-unsigned int kfd_get_num_sdma_engines(struct kfd_dev *kdev)
+unsigned int kfd_get_num_sdma_engines(struct kfd_node *node)
{
/* If XGMI is not supported, all SDMA engines are PCIe */
- if (!kdev->adev->gmc.xgmi.supported)
- return kdev->adev->sdma.num_instances;
+ if (!node->adev->gmc.xgmi.supported)
+ return node->adev->sdma.num_instances;
- return min(kdev->adev->sdma.num_instances, 2);
+ return min(node->adev->sdma.num_instances, 2);
}
-unsigned int kfd_get_num_xgmi_sdma_engines(struct kfd_dev *kdev)
+unsigned int kfd_get_num_xgmi_sdma_engines(struct kfd_node *node)
{
/* After reserved for PCIe, the rest of engines are XGMI */
- return kdev->adev->sdma.num_instances - kfd_get_num_sdma_engines(kdev);
+ return node->adev->sdma.num_instances - kfd_get_num_sdma_engines(node);
}
#if defined(CONFIG_DEBUG_FS)
/* This function will send a package to HIQ to hang the HWS
* which will trigger a GPU reset and bring the HWS back to normal state
*/
-int kfd_debugfs_hang_hws(struct kfd_dev *dev)
+int kfd_debugfs_hang_hws(struct kfd_node *dev)
{
if (dev->dqm->sched_policy != KFD_SCHED_POLICY_HWS) {
pr_err("HWS is not enabled");
static bool is_pipe_enabled(struct device_queue_manager *dqm, int mec, int pipe)
{
int i;
- int pipe_offset = (mec * dqm->dev->shared_resources.num_pipe_per_mec
- + pipe) * dqm->dev->shared_resources.num_queue_per_pipe;
+ int pipe_offset = (mec * dqm->dev->kfd->shared_resources.num_pipe_per_mec
+ + pipe) * dqm->dev->kfd->shared_resources.num_queue_per_pipe;
/* queue is available for KFD usage if bit is 1 */
- for (i = 0; i < dqm->dev->shared_resources.num_queue_per_pipe; ++i)
+ for (i = 0; i < dqm->dev->kfd->shared_resources.num_queue_per_pipe; ++i)
if (test_bit(pipe_offset + i,
- dqm->dev->shared_resources.cp_queue_bitmap))
+ dqm->dev->kfd->shared_resources.cp_queue_bitmap))
return true;
return false;
}
unsigned int get_cp_queues_num(struct device_queue_manager *dqm)
{
- return bitmap_weight(dqm->dev->shared_resources.cp_queue_bitmap,
+ return bitmap_weight(dqm->dev->kfd->shared_resources.cp_queue_bitmap,
KGD_MAX_QUEUES);
}
unsigned int get_queues_per_pipe(struct device_queue_manager *dqm)
{
- return dqm->dev->shared_resources.num_queue_per_pipe;
+ return dqm->dev->kfd->shared_resources.num_queue_per_pipe;
}
unsigned int get_pipes_per_mec(struct device_queue_manager *dqm)
{
- return dqm->dev->shared_resources.num_pipe_per_mec;
+ return dqm->dev->kfd->shared_resources.num_pipe_per_mec;
}
static unsigned int get_num_all_sdma_engines(struct device_queue_manager *dqm)
unsigned int get_num_sdma_queues(struct device_queue_manager *dqm)
{
return kfd_get_num_sdma_engines(dqm->dev) *
- dqm->dev->device_info.num_sdma_queues_per_engine;
+ dqm->dev->kfd->device_info.num_sdma_queues_per_engine;
}
unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm)
{
return kfd_get_num_xgmi_sdma_engines(dqm->dev) *
- dqm->dev->device_info.num_sdma_queues_per_engine;
+ dqm->dev->kfd->device_info.num_sdma_queues_per_engine;
}
static inline uint64_t get_reserved_sdma_queues_bitmap(struct device_queue_manager *dqm)
{
- return dqm->dev->device_info.reserved_sdma_queues_bitmap;
+ return dqm->dev->kfd->device_info.reserved_sdma_queues_bitmap;
}
void program_sh_mem_settings(struct device_queue_manager *dqm,
struct queue *q,
uint32_t const *restore_id)
{
- struct kfd_dev *dev = qpd->dqm->dev;
+ struct kfd_node *dev = qpd->dqm->dev;
if (!KFD_IS_SOC15(dev)) {
/* On pre-SOC15 chips we need to use the queue ID to
* for a SDMA engine is 512.
*/
- uint32_t *idx_offset = dev->shared_resources.sdma_doorbell_idx;
+ uint32_t *idx_offset = dev->kfd->shared_resources.sdma_doorbell_idx;
uint32_t valid_id = idx_offset[q->properties.sdma_engine_id]
+ (q->properties.sdma_queue_id & 1)
* KFD_QUEUE_DOORBELL_MIRROR_OFFSET
}
q->properties.doorbell_off =
- kfd_get_doorbell_dw_offset_in_bar(dev, qpd_to_pdd(qpd),
+ kfd_get_doorbell_dw_offset_in_bar(dev->kfd, qpd_to_pdd(qpd),
q->doorbell_id);
return 0;
}
struct queue *q)
{
unsigned int old;
- struct kfd_dev *dev = qpd->dqm->dev;
+ struct kfd_node *dev = qpd->dqm->dev;
if (!KFD_IS_SOC15(dev) ||
q->properties.type == KFD_QUEUE_TYPE_SDMA ||
program_sh_mem_settings(dqm, qpd);
- if (KFD_IS_SOC15(dqm->dev) && dqm->dev->cwsr_enabled)
+ if (KFD_IS_SOC15(dqm->dev) && dqm->dev->kfd->cwsr_enabled)
program_trap_handler_settings(dqm, qpd);
/* qpd->page_table_base is set earlier when register_process()
return 0;
}
-static int flush_texture_cache_nocpsch(struct kfd_dev *kdev,
+static int flush_texture_cache_nocpsch(struct kfd_node *kdev,
struct qcm_process_device *qpd)
{
const struct packet_manager_funcs *pmf = qpd->dqm->packet_mgr.pmf;
#define SQ_IND_CMD_CMD_KILL 0x00000003
#define SQ_IND_CMD_MODE_BROADCAST 0x00000001
-static int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p)
+static int dbgdev_wave_reset_wavefronts(struct kfd_node *dev, struct kfd_process *p)
{
int status = 0;
unsigned int vmid;
/* Make sure the queue is unmapped before updating the MQD */
if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) {
- if (!dqm->dev->shared_resources.enable_mes)
+ if (!dqm->dev->kfd->shared_resources.enable_mes)
retval = unmap_queues_cpsch(dqm,
KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, false);
else if (prev_active)
}
retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
- (dqm->dev->cwsr_enabled ?
+ (dqm->dev->kfd->cwsr_enabled ?
KFD_PREEMPT_TYPE_WAVEFRONT_SAVE :
KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN),
KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
}
if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) {
- if (!dqm->dev->shared_resources.enable_mes)
+ if (!dqm->dev->kfd->shared_resources.enable_mes)
retval = map_queues_cpsch(dqm);
else if (q->properties.is_active)
retval = add_queue_mes(dqm, q, &pdd->qpd);
continue;
retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
- (dqm->dev->cwsr_enabled ?
+ (dqm->dev->kfd->cwsr_enabled ?
KFD_PREEMPT_TYPE_WAVEFRONT_SAVE :
KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN),
KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
q->properties.is_active = false;
decrement_queue_count(dqm, qpd, q);
- if (dqm->dev->shared_resources.enable_mes) {
+ if (dqm->dev->kfd->shared_resources.enable_mes) {
retval = remove_queue_mes(dqm, q, qpd);
if (retval) {
pr_err("Failed to evict queue %d\n",
}
}
pdd->last_evict_timestamp = get_jiffies_64();
- if (!dqm->dev->shared_resources.enable_mes)
+ if (!dqm->dev->kfd->shared_resources.enable_mes)
retval = execute_queues_cpsch(dqm,
qpd->is_debug ?
KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES :
q->properties.is_active = true;
increment_queue_count(dqm, &pdd->qpd, q);
- if (dqm->dev->shared_resources.enable_mes) {
+ if (dqm->dev->kfd->shared_resources.enable_mes) {
retval = add_queue_mes(dqm, q, qpd);
if (retval) {
pr_err("Failed to restore queue %d\n",
}
}
}
- if (!dqm->dev->shared_resources.enable_mes)
+ if (!dqm->dev->kfd->shared_resources.enable_mes)
retval = execute_queues_cpsch(dqm,
KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
qpd->evicted = 0;
for (queue = 0; queue < get_queues_per_pipe(dqm); queue++)
if (test_bit(pipe_offset + queue,
- dqm->dev->shared_resources.cp_queue_bitmap))
+ dqm->dev->kfd->shared_resources.cp_queue_bitmap))
dqm->allocated_queues[pipe] |= 1 << queue;
}
int i, mec;
struct scheduling_resources res;
- res.vmid_mask = dqm->dev->shared_resources.compute_vmid_bitmap;
+ res.vmid_mask = dqm->dev->kfd->shared_resources.compute_vmid_bitmap;
res.queue_mask = 0;
for (i = 0; i < KGD_MAX_QUEUES; ++i) {
- mec = (i / dqm->dev->shared_resources.num_queue_per_pipe)
- / dqm->dev->shared_resources.num_pipe_per_mec;
+ mec = (i / dqm->dev->kfd->shared_resources.num_queue_per_pipe)
+ / dqm->dev->kfd->shared_resources.num_pipe_per_mec;
- if (!test_bit(i, dqm->dev->shared_resources.cp_queue_bitmap))
+ if (!test_bit(i, dqm->dev->kfd->shared_resources.cp_queue_bitmap))
continue;
/* only acquire queues from the first MEC */
dqm_lock(dqm);
- if (!dqm->dev->shared_resources.enable_mes) {
+ if (!dqm->dev->kfd->shared_resources.enable_mes) {
retval = pm_init(&dqm->packet_mgr, dqm);
if (retval)
goto fail_packet_manager_init;
dqm->is_hws_hang = false;
dqm->is_resetting = false;
dqm->sched_running = true;
- if (!dqm->dev->shared_resources.enable_mes)
+ if (!dqm->dev->kfd->shared_resources.enable_mes)
execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
dqm_unlock(dqm);
return 0;
fail_allocate_vidmem:
fail_set_sched_resources:
- if (!dqm->dev->shared_resources.enable_mes)
+ if (!dqm->dev->kfd->shared_resources.enable_mes)
pm_uninit(&dqm->packet_mgr, false);
fail_packet_manager_init:
dqm_unlock(dqm);
}
if (!dqm->is_hws_hang) {
- if (!dqm->dev->shared_resources.enable_mes)
+ if (!dqm->dev->kfd->shared_resources.enable_mes)
unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, false);
else
remove_all_queues_mes(dqm);
hanging = dqm->is_hws_hang || dqm->is_resetting;
dqm->sched_running = false;
- if (!dqm->dev->shared_resources.enable_mes)
+ if (!dqm->dev->kfd->shared_resources.enable_mes)
pm_release_ib(&dqm->packet_mgr);
kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);
- if (!dqm->dev->shared_resources.enable_mes)
+ if (!dqm->dev->kfd->shared_resources.enable_mes)
pm_uninit(&dqm->packet_mgr, hanging);
dqm_unlock(dqm);
if (q->properties.is_active) {
increment_queue_count(dqm, qpd, q);
- if (!dqm->dev->shared_resources.enable_mes)
+ if (!dqm->dev->kfd->shared_resources.enable_mes)
retval = execute_queues_cpsch(dqm,
KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
else
list_del(&q->list);
qpd->queue_count--;
if (q->properties.is_active) {
- if (!dqm->dev->shared_resources.enable_mes) {
+ if (!dqm->dev->kfd->shared_resources.enable_mes) {
decrement_queue_count(dqm, qpd, q);
retval = execute_queues_cpsch(dqm,
KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_CP];
if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE ||
- q->properties.is_active || !q->device->cwsr_enabled ||
+ q->properties.is_active || !q->device->kfd->cwsr_enabled ||
!mqd_mgr->get_wave_state) {
dqm_unlock(dqm);
return -EINVAL;
dqm_lock(dqm);
- if (q->properties.is_active || !q->device->cwsr_enabled) {
+ if (q->properties.is_active || !q->device->kfd->cwsr_enabled) {
r = -EINVAL;
goto dqm_unlock;
}
if (q->properties.is_active) {
decrement_queue_count(dqm, qpd, q);
- if (dqm->dev->shared_resources.enable_mes) {
+ if (dqm->dev->kfd->shared_resources.enable_mes) {
retval = remove_queue_mes(dqm, q, qpd);
if (retval)
pr_err("Failed to remove queue %d\n",
}
}
- if (!dqm->dev->shared_resources.enable_mes)
+ if (!dqm->dev->kfd->shared_resources.enable_mes)
retval = execute_queues_cpsch(dqm, filter, 0);
if ((!dqm->is_hws_hang) && (retval || qpd->reset_wavefronts)) {
static int allocate_hiq_sdma_mqd(struct device_queue_manager *dqm)
{
int retval;
- struct kfd_dev *dev = dqm->dev;
+ struct kfd_node *dev = dqm->dev;
struct kfd_mem_obj *mem_obj = &dqm->hiq_sdma_mqd;
uint32_t size = dqm->mqd_mgrs[KFD_MQD_TYPE_SDMA]->mqd_size *
get_num_all_sdma_engines(dqm) *
- dev->device_info.num_sdma_queues_per_engine +
+ dev->kfd->device_info.num_sdma_queues_per_engine +
dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ]->mqd_size;
retval = amdgpu_amdkfd_alloc_gtt_mem(dev->adev, size,
return retval;
}
-struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
+struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev)
{
struct device_queue_manager *dqm;
if (init_mqd_managers(dqm))
goto out_free;
- if (!dev->shared_resources.enable_mes && allocate_hiq_sdma_mqd(dqm)) {
+ if (!dev->kfd->shared_resources.enable_mes && allocate_hiq_sdma_mqd(dqm)) {
pr_err("Failed to allocate hiq sdma mqd trunk buffer\n");
goto out_free;
}
return NULL;
}
-static void deallocate_hiq_sdma_mqd(struct kfd_dev *dev,
+static void deallocate_hiq_sdma_mqd(struct kfd_node *dev,
struct kfd_mem_obj *mqd)
{
WARN(!mqd, "No hiq sdma mqd trunk to free");
void device_queue_manager_uninit(struct device_queue_manager *dqm)
{
dqm->ops.uninitialize(dqm);
- if (!dqm->dev->shared_resources.enable_mes)
+ if (!dqm->dev->kfd->shared_resources.enable_mes)
deallocate_hiq_sdma_mqd(dqm->dev, &dqm->hiq_sdma_mqd);
kfree(dqm);
}
for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) {
if (!test_bit(pipe_offset + queue,
- dqm->dev->shared_resources.cp_queue_bitmap))
+ dqm->dev->kfd->shared_resources.cp_queue_bitmap))
continue;
r = dqm->dev->kfd2kgd->hqd_dump(
for (pipe = 0; pipe < get_num_all_sdma_engines(dqm); pipe++) {
for (queue = 0;
- queue < dqm->dev->device_info.num_sdma_queues_per_engine;
+ queue < dqm->dev->kfd->device_info.num_sdma_queues_per_engine;
queue++) {
r = dqm->dev->kfd2kgd->hqd_sdma_dump(
dqm->dev->adev, pipe, queue, &dump, &n_regs);
struct queue *q,
struct qcm_process_device *qpd);
struct mqd_manager * (*mqd_manager_init)(enum KFD_MQD_TYPE type,
- struct kfd_dev *dev);
+ struct kfd_node *dev);
};
/**
struct mqd_manager *mqd_mgrs[KFD_MQD_TYPE_MAX];
struct packet_manager packet_mgr;
- struct kfd_dev *dev;
+ struct kfd_node *dev;
struct mutex lock_hidden; /* use dqm_lock/unlock(dqm) */
struct list_head queues;
unsigned int saved_flags;
qpd->sh_mem_config = SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
- if (dqm->dev->noretry && !dqm->dev->use_iommu_v2)
+ if (dqm->dev->kfd->noretry && !dqm->dev->kfd->use_iommu_v2)
qpd->sh_mem_config |= 1 << SH_MEM_CONFIG__RETRY_DISABLE__SHIFT;
- if (KFD_GC_VERSION(dqm->dev) == IP_VERSION(9, 4, 3))
+ if (KFD_GC_VERSION(dqm->dev->kfd) == IP_VERSION(9, 4, 3))
qpd->sh_mem_config |=
(1 << SH_MEM_CONFIG__F8_MODE__SHIFT);
iounmap(kfd->doorbell_kernel_ptr);
}
-int kfd_doorbell_mmap(struct kfd_dev *dev, struct kfd_process *process,
+int kfd_doorbell_mmap(struct kfd_node *dev, struct kfd_process *process,
struct vm_area_struct *vma)
{
phys_addr_t address;
* For simplicitly we only allow mapping of the entire doorbell
* allocation of a single device & process.
*/
- if (vma->vm_end - vma->vm_start != kfd_doorbell_process_slice(dev))
+ if (vma->vm_end - vma->vm_start != kfd_doorbell_process_slice(dev->kfd))
return -EINVAL;
pdd = kfd_get_process_device_data(dev, process);
" vm_flags == 0x%04lX\n"
" size == 0x%04lX\n",
(unsigned long long) vma->vm_start, address, vma->vm_flags,
- kfd_doorbell_process_slice(dev));
+ kfd_doorbell_process_slice(dev->kfd));
return io_remap_pfn_range(vma,
vma->vm_start,
address >> PAGE_SHIFT,
- kfd_doorbell_process_slice(dev),
+ kfd_doorbell_process_slice(dev->kfd),
vma->vm_page_prot);
}
phys_addr_t kfd_get_process_doorbells(struct kfd_process_device *pdd)
{
if (!pdd->doorbell_index) {
- int r = kfd_alloc_process_doorbells(pdd->dev,
+ int r = kfd_alloc_process_doorbells(pdd->dev->kfd,
&pdd->doorbell_index);
if (r < 0)
return 0;
}
- return pdd->dev->doorbell_base +
- pdd->doorbell_index * kfd_doorbell_process_slice(pdd->dev);
+ return pdd->dev->kfd->doorbell_base +
+ pdd->doorbell_index * kfd_doorbell_process_slice(pdd->dev->kfd);
}
int kfd_alloc_process_doorbells(struct kfd_dev *kfd, unsigned int *doorbell_index)
int kfd_kmap_event_page(struct kfd_process *p, uint64_t event_page_offset)
{
- struct kfd_dev *kfd;
+ struct kfd_node *kfd;
struct kfd_process_device *pdd;
void *mem, *kern_addr;
uint64_t size;
}
#ifdef KFD_SUPPORT_IOMMU_V2
-void kfd_signal_iommu_event(struct kfd_dev *dev, u32 pasid,
+void kfd_signal_iommu_event(struct kfd_node *dev, u32 pasid,
unsigned long address, bool is_write_requested,
bool is_execute_requested)
{
kfd_unref_process(p);
}
-void kfd_signal_vm_fault_event(struct kfd_dev *dev, u32 pasid,
- struct kfd_vm_fault_info *info)
+void kfd_signal_vm_fault_event(struct kfd_node *dev, u32 pasid,
+ struct kfd_vm_fault_info *info)
{
struct kfd_event *ev;
uint32_t id;
kfd_unref_process(p);
}
-void kfd_signal_reset_event(struct kfd_dev *dev)
+void kfd_signal_reset_event(struct kfd_node *dev)
{
struct kfd_hsa_hw_exception_data hw_exception_data;
struct kfd_hsa_memory_exception_data memory_exception_data;
srcu_read_unlock(&kfd_processes_srcu, idx);
}
-void kfd_signal_poison_consumed_event(struct kfd_dev *dev, u32 pasid)
+void kfd_signal_poison_consumed_event(struct kfd_node *dev, u32 pasid)
{
struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
struct kfd_hsa_memory_exception_data memory_exception_data;
pdd->lds_base = MAKE_LDS_APP_BASE_VI();
pdd->lds_limit = MAKE_LDS_APP_LIMIT(pdd->lds_base);
- if (!pdd->dev->use_iommu_v2) {
+ if (!pdd->dev->kfd->use_iommu_v2) {
/* dGPUs: SVM aperture starting at 0
* with small reserved space for kernel.
* Set them to CANONICAL addresses.
*/
pdd->gpuvm_base = SVM_USER_BASE;
pdd->gpuvm_limit =
- pdd->dev->shared_resources.gpuvm_size - 1;
+ pdd->dev->kfd->shared_resources.gpuvm_size - 1;
} else {
/* set them to non CANONICAL addresses, and no SVM is
* allocated.
*/
pdd->gpuvm_base = MAKE_GPUVM_APP_BASE_VI(id + 1);
pdd->gpuvm_limit = MAKE_GPUVM_APP_LIMIT(pdd->gpuvm_base,
- pdd->dev->shared_resources.gpuvm_size);
+ pdd->dev->kfd->shared_resources.gpuvm_size);
}
pdd->scratch_base = MAKE_SCRATCH_APP_BASE_VI();
*/
pdd->gpuvm_base = SVM_USER_BASE;
pdd->gpuvm_limit =
- pdd->dev->shared_resources.gpuvm_size - 1;
+ pdd->dev->kfd->shared_resources.gpuvm_size - 1;
pdd->scratch_base = MAKE_SCRATCH_APP_BASE_V9();
pdd->scratch_limit = MAKE_SCRATCH_APP_LIMIT(pdd->scratch_base);
int kfd_init_apertures(struct kfd_process *process)
{
uint8_t id = 0;
- struct kfd_dev *dev;
+ struct kfd_node *dev;
struct kfd_process_device *pdd;
/*Iterating over all devices*/
}
}
- if (!dev->use_iommu_v2) {
+ if (!dev->kfd->use_iommu_v2) {
/* dGPUs: the reserved space for kernel
* before SVM
*/
REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_ERROR_CTXID1, WGP_ID));
}
-static void event_interrupt_poison_consumption_v11(struct kfd_dev *dev,
+static void event_interrupt_poison_consumption_v11(struct kfd_node *dev,
uint16_t pasid, uint16_t source_id)
{
int ret = -EINVAL;
amdgpu_amdkfd_ras_poison_consumption_handler(dev->adev, true);
}
-static bool event_interrupt_isr_v11(struct kfd_dev *dev,
+static bool event_interrupt_isr_v11(struct kfd_node *dev,
const uint32_t *ih_ring_entry,
uint32_t *patched_ihre,
bool *patched_flag)
!amdgpu_no_queue_eviction_on_vm_fault);
}
-static void event_interrupt_wq_v11(struct kfd_dev *dev,
+static void event_interrupt_wq_v11(struct kfd_node *dev,
const uint32_t *ih_ring_entry)
{
uint16_t source_id, client_id, ring_id, pasid, vmid;
#define KFD_SQ_INT_DATA__ERR_TYPE_MASK 0xF00000
#define KFD_SQ_INT_DATA__ERR_TYPE__SHIFT 20
-static void event_interrupt_poison_consumption_v9(struct kfd_dev *dev,
+static void event_interrupt_poison_consumption_v9(struct kfd_node *dev,
uint16_t pasid, uint16_t client_id)
{
int old_poison, ret = -EINVAL;
}
}
-static bool event_interrupt_isr_v9(struct kfd_dev *dev,
+static bool event_interrupt_isr_v9(struct kfd_node *dev,
const uint32_t *ih_ring_entry,
uint32_t *patched_ihre,
bool *patched_flag)
*patched_flag = true;
memcpy(patched_ihre, ih_ring_entry,
- dev->device_info.ih_ring_entry_size);
+ dev->kfd->device_info.ih_ring_entry_size);
pasid = dev->dqm->vmid_pasid[vmid];
uint32_t context_id =
SOC15_CONTEXT_ID0_FROM_IH_ENTRY(ih_ring_entry);
- if (context_id == 0 && context_id_expected(dev))
+ if (context_id == 0 && context_id_expected(dev->kfd))
return false;
}
!amdgpu_no_queue_eviction_on_vm_fault);
}
-static void event_interrupt_wq_v9(struct kfd_dev *dev,
+static void event_interrupt_wq_v9(struct kfd_node *dev,
const uint32_t *ih_ring_entry)
{
uint16_t source_id, client_id, pasid, vmid;
static void interrupt_wq(struct work_struct *);
-int kfd_interrupt_init(struct kfd_dev *kfd)
+int kfd_interrupt_init(struct kfd_node *node)
{
int r;
- r = kfifo_alloc(&kfd->ih_fifo,
- KFD_IH_NUM_ENTRIES * kfd->device_info.ih_ring_entry_size,
+ r = kfifo_alloc(&node->ih_fifo,
+ KFD_IH_NUM_ENTRIES * node->kfd->device_info.ih_ring_entry_size,
GFP_KERNEL);
if (r) {
- dev_err(kfd->adev->dev, "Failed to allocate IH fifo\n");
+ dev_err(node->adev->dev, "Failed to allocate IH fifo\n");
return r;
}
- kfd->ih_wq = alloc_workqueue("KFD IH", WQ_HIGHPRI, 1);
- if (unlikely(!kfd->ih_wq)) {
- kfifo_free(&kfd->ih_fifo);
- dev_err(kfd->adev->dev, "Failed to allocate KFD IH workqueue\n");
+ node->ih_wq = alloc_workqueue("KFD IH", WQ_HIGHPRI, 1);
+ if (unlikely(!node->ih_wq)) {
+ kfifo_free(&node->ih_fifo);
+ dev_err(node->adev->dev, "Failed to allocate KFD IH workqueue\n");
return -ENOMEM;
}
- spin_lock_init(&kfd->interrupt_lock);
+ spin_lock_init(&node->interrupt_lock);
- INIT_WORK(&kfd->interrupt_work, interrupt_wq);
+ INIT_WORK(&node->interrupt_work, interrupt_wq);
- kfd->interrupts_active = true;
+ node->interrupts_active = true;
/*
* After this function returns, the interrupt will be enabled. This
return 0;
}
-void kfd_interrupt_exit(struct kfd_dev *kfd)
+void kfd_interrupt_exit(struct kfd_node *node)
{
/*
* Stop the interrupt handler from writing to the ring and scheduling
*/
unsigned long flags;
- spin_lock_irqsave(&kfd->interrupt_lock, flags);
- kfd->interrupts_active = false;
- spin_unlock_irqrestore(&kfd->interrupt_lock, flags);
+ spin_lock_irqsave(&node->interrupt_lock, flags);
+ node->interrupts_active = false;
+ spin_unlock_irqrestore(&node->interrupt_lock, flags);
/*
* flush_work ensures that there are no outstanding
* work-queue items that will access interrupt_ring. New work items
* can't be created because we stopped interrupt handling above.
*/
- flush_workqueue(kfd->ih_wq);
+ flush_workqueue(node->ih_wq);
- kfifo_free(&kfd->ih_fifo);
+ kfifo_free(&node->ih_fifo);
}
/*
* Assumption: single reader/writer. This function is not re-entrant
*/
-bool enqueue_ih_ring_entry(struct kfd_dev *kfd, const void *ih_ring_entry)
+bool enqueue_ih_ring_entry(struct kfd_node *node, const void *ih_ring_entry)
{
int count;
- count = kfifo_in(&kfd->ih_fifo, ih_ring_entry,
- kfd->device_info.ih_ring_entry_size);
- if (count != kfd->device_info.ih_ring_entry_size) {
- dev_dbg_ratelimited(kfd->adev->dev,
+ count = kfifo_in(&node->ih_fifo, ih_ring_entry,
+ node->kfd->device_info.ih_ring_entry_size);
+ if (count != node->kfd->device_info.ih_ring_entry_size) {
+ dev_dbg_ratelimited(node->adev->dev,
"Interrupt ring overflow, dropping interrupt %d\n",
count);
return false;
/*
* Assumption: single reader/writer. This function is not re-entrant
*/
-static bool dequeue_ih_ring_entry(struct kfd_dev *kfd, void *ih_ring_entry)
+static bool dequeue_ih_ring_entry(struct kfd_node *node, void *ih_ring_entry)
{
int count;
- count = kfifo_out(&kfd->ih_fifo, ih_ring_entry,
- kfd->device_info.ih_ring_entry_size);
+ count = kfifo_out(&node->ih_fifo, ih_ring_entry,
+ node->kfd->device_info.ih_ring_entry_size);
- WARN_ON(count && count != kfd->device_info.ih_ring_entry_size);
+ WARN_ON(count && count != node->kfd->device_info.ih_ring_entry_size);
- return count == kfd->device_info.ih_ring_entry_size;
+ return count == node->kfd->device_info.ih_ring_entry_size;
}
static void interrupt_wq(struct work_struct *work)
{
- struct kfd_dev *dev = container_of(work, struct kfd_dev,
+ struct kfd_node *dev = container_of(work, struct kfd_node,
interrupt_work);
uint32_t ih_ring_entry[KFD_MAX_RING_ENTRY_SIZE];
unsigned long start_jiffies = jiffies;
- if (dev->device_info.ih_ring_entry_size > sizeof(ih_ring_entry)) {
+ if (dev->kfd->device_info.ih_ring_entry_size > sizeof(ih_ring_entry)) {
dev_err_once(dev->adev->dev, "Ring entry too small\n");
return;
}
while (dequeue_ih_ring_entry(dev, ih_ring_entry)) {
- dev->device_info.event_interrupt_class->interrupt_wq(dev,
+ dev->kfd->device_info.event_interrupt_class->interrupt_wq(dev,
ih_ring_entry);
if (time_is_before_jiffies(start_jiffies + HZ)) {
/* If we spent more than a second processing signals,
}
}
-bool interrupt_is_wanted(struct kfd_dev *dev,
+bool interrupt_is_wanted(struct kfd_node *dev,
const uint32_t *ih_ring_entry,
uint32_t *patched_ihre, bool *flag)
{
/* integer and bitwise OR so there is no boolean short-circuiting */
unsigned int wanted = 0;
- wanted |= dev->device_info.event_interrupt_class->interrupt_isr(dev,
+ wanted |= dev->kfd->device_info.event_interrupt_class->interrupt_isr(dev,
ih_ring_entry, patched_ihre, flag);
return wanted != 0;
*/
int kfd_iommu_bind_process_to_device(struct kfd_process_device *pdd)
{
- struct kfd_dev *dev = pdd->dev;
+ struct kfd_node *dev = pdd->dev;
struct kfd_process *p = pdd->process;
int err;
- if (!dev->use_iommu_v2 || pdd->bound == PDD_BOUND)
+ if (!dev->kfd->use_iommu_v2 || pdd->bound == PDD_BOUND)
return 0;
if (unlikely(pdd->bound == PDD_BOUND_SUSPENDED)) {
/* Callback for process shutdown invoked by the IOMMU driver */
static void iommu_pasid_shutdown_callback(struct pci_dev *pdev, u32 pasid)
{
- struct kfd_dev *dev = kfd_device_by_pci_dev(pdev);
+ struct kfd_node *dev = kfd_device_by_pci_dev(pdev);
struct kfd_process *p;
struct kfd_process_device *pdd;
static int iommu_invalid_ppr_cb(struct pci_dev *pdev, u32 pasid,
unsigned long address, u16 flags)
{
- struct kfd_dev *dev;
+ struct kfd_node *dev;
dev_warn_ratelimited(kfd_device,
"Invalid PPR device %x:%x.%x pasid 0x%x address 0x%lX flags 0x%X",
* Bind processes do the device that have been temporarily unbound
* (PDD_BOUND_SUSPENDED) in kfd_unbind_processes_from_device.
*/
-static int kfd_bind_processes_to_device(struct kfd_dev *kfd)
+static int kfd_bind_processes_to_device(struct kfd_node *knode)
{
struct kfd_process_device *pdd;
struct kfd_process *p;
hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
mutex_lock(&p->mutex);
- pdd = kfd_get_process_device_data(kfd, p);
+ pdd = kfd_get_process_device_data(knode, p);
if (WARN_ON(!pdd) || pdd->bound != PDD_BOUND_SUSPENDED) {
mutex_unlock(&p->mutex);
continue;
}
- err = amd_iommu_bind_pasid(kfd->adev->pdev, p->pasid,
+ err = amd_iommu_bind_pasid(knode->adev->pdev, p->pasid,
p->lead_thread);
if (err < 0) {
pr_err("Unexpected pasid 0x%x binding failure\n",
* processes will be restored to PDD_BOUND state in
* kfd_bind_processes_to_device.
*/
-static void kfd_unbind_processes_from_device(struct kfd_dev *kfd)
+static void kfd_unbind_processes_from_device(struct kfd_node *knode)
{
struct kfd_process_device *pdd;
struct kfd_process *p;
hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
mutex_lock(&p->mutex);
- pdd = kfd_get_process_device_data(kfd, p);
+ pdd = kfd_get_process_device_data(knode, p);
if (WARN_ON(!pdd)) {
mutex_unlock(&p->mutex);
if (!kfd->use_iommu_v2)
return;
- kfd_unbind_processes_from_device(kfd);
+ kfd_unbind_processes_from_device(kfd->node);
amd_iommu_set_invalidate_ctx_cb(kfd->adev->pdev, NULL);
amd_iommu_set_invalid_ppr_cb(kfd->adev->pdev, NULL);
amd_iommu_set_invalid_ppr_cb(kfd->adev->pdev,
iommu_invalid_ppr_cb);
- err = kfd_bind_processes_to_device(kfd);
+ err = kfd_bind_processes_to_device(kfd->node);
if (err) {
amd_iommu_set_invalidate_ctx_cb(kfd->adev->pdev, NULL);
amd_iommu_set_invalid_ppr_cb(kfd->adev->pdev, NULL);
/* Initialize a kernel queue, including allocations of GART memory
* needed for the queue.
*/
-static bool kq_initialize(struct kernel_queue *kq, struct kfd_dev *dev,
+static bool kq_initialize(struct kernel_queue *kq, struct kfd_node *dev,
enum kfd_queue_type type, unsigned int queue_size)
{
struct queue_properties prop;
if (!kq->mqd_mgr)
return false;
- prop.doorbell_ptr = kfd_get_kernel_doorbell(dev, &prop.doorbell_off);
+ prop.doorbell_ptr = kfd_get_kernel_doorbell(dev->kfd, &prop.doorbell_off);
if (!prop.doorbell_ptr) {
pr_err("Failed to initialize doorbell");
kq->rptr_kernel = kq->rptr_mem->cpu_ptr;
kq->rptr_gpu_addr = kq->rptr_mem->gpu_addr;
- retval = kfd_gtt_sa_allocate(dev, dev->device_info.doorbell_size,
+ retval = kfd_gtt_sa_allocate(dev, dev->kfd->device_info.doorbell_size,
&kq->wptr_mem);
if (retval != 0)
err_eop_allocate_vidmem:
kfd_gtt_sa_free(dev, kq->pq);
err_pq_allocate_vidmem:
- kfd_release_kernel_doorbell(dev, prop.doorbell_ptr);
+ kfd_release_kernel_doorbell(dev->kfd, prop.doorbell_ptr);
err_get_kernel_doorbell:
return false;
kfd_gtt_sa_free(kq->dev, kq->eop_mem);
kfd_gtt_sa_free(kq->dev, kq->pq);
- kfd_release_kernel_doorbell(kq->dev,
+ kfd_release_kernel_doorbell(kq->dev->kfd,
kq->queue->properties.doorbell_ptr);
uninit_queue(kq->queue);
}
}
pr_debug("\n");
#endif
- if (kq->dev->device_info.doorbell_size == 8) {
+ if (kq->dev->kfd->device_info.doorbell_size == 8) {
*kq->wptr64_kernel = kq->pending_wptr64;
write_kernel_doorbell64(kq->queue->properties.doorbell_ptr,
kq->pending_wptr64);
void kq_rollback_packet(struct kernel_queue *kq)
{
- if (kq->dev->device_info.doorbell_size == 8) {
+ if (kq->dev->kfd->device_info.doorbell_size == 8) {
kq->pending_wptr64 = *kq->wptr64_kernel;
kq->pending_wptr = *kq->wptr_kernel %
(kq->queue->properties.queue_size / 4);
}
}
-struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,
+struct kernel_queue *kernel_queue_init(struct kfd_node *dev,
enum kfd_queue_type type)
{
struct kernel_queue *kq;
}
/* FIXME: Can this test be removed? */
-static __attribute__((unused)) void test_kq(struct kfd_dev *dev)
+static __attribute__((unused)) void test_kq(struct kfd_node *dev)
{
struct kernel_queue *kq;
uint32_t *buffer, i;
struct kernel_queue {
/* data */
- struct kfd_dev *dev;
+ struct kfd_node *dev;
struct mqd_manager *mqd_mgr;
struct queue *queue;
uint64_t pending_wptr64;
kfd_smi_event_migration_start(adev->kfd.dev, p->lead_thread->pid,
start >> PAGE_SHIFT, end >> PAGE_SHIFT,
- 0, adev->kfd.dev->id, prange->prefetch_loc,
+ 0, adev->kfd.dev->node->id, prange->prefetch_loc,
prange->preferred_loc, trigger);
r = migrate_vma_setup(&migrate);
kfd_smi_event_migration_end(adev->kfd.dev, p->lead_thread->pid,
start >> PAGE_SHIFT, end >> PAGE_SHIFT,
- 0, adev->kfd.dev->id, trigger);
+ 0, adev->kfd.dev->node->id, trigger);
svm_range_dma_unmap(adev->dev, scratch, 0, npages);
svm_range_free_dma_mappings(prange);
kfd_smi_event_migration_start(adev->kfd.dev, p->lead_thread->pid,
start >> PAGE_SHIFT, end >> PAGE_SHIFT,
- adev->kfd.dev->id, 0, prange->prefetch_loc,
+ adev->kfd.dev->node->id, 0, prange->prefetch_loc,
prange->preferred_loc, trigger);
r = migrate_vma_setup(&migrate);
kfd_smi_event_migration_end(adev->kfd.dev, p->lead_thread->pid,
start >> PAGE_SHIFT, end >> PAGE_SHIFT,
- adev->kfd.dev->id, 0, trigger);
+ adev->kfd.dev->node->id, 0, trigger);
svm_range_dma_unmap(adev->dev, scratch, 0, npages);
KFD_PIPE_PRIORITY_CS_HIGH
};
-struct kfd_mem_obj *allocate_hiq_mqd(struct kfd_dev *dev, struct queue_properties *q)
+struct kfd_mem_obj *allocate_hiq_mqd(struct kfd_node *dev, struct queue_properties *q)
{
struct kfd_mem_obj *mqd_mem_obj = NULL;
return mqd_mem_obj;
}
-struct kfd_mem_obj *allocate_sdma_mqd(struct kfd_dev *dev,
+struct kfd_mem_obj *allocate_sdma_mqd(struct kfd_node *dev,
struct queue_properties *q)
{
struct kfd_mem_obj *mqd_mem_obj = NULL;
return NULL;
offset = (q->sdma_engine_id *
- dev->device_info.num_sdma_queues_per_engine +
+ dev->kfd->device_info.num_sdma_queues_per_engine +
q->sdma_queue_id) *
dev->dqm->mqd_mgrs[KFD_MQD_TYPE_SDMA]->mqd_size;
*/
extern int pipe_priority_map[];
struct mqd_manager {
- struct kfd_mem_obj* (*allocate_mqd)(struct kfd_dev *kfd,
+ struct kfd_mem_obj* (*allocate_mqd)(struct kfd_node *kfd,
struct queue_properties *q);
void (*init_mqd)(struct mqd_manager *mm, void **mqd,
uint32_t (*read_doorbell_id)(void *mqd);
struct mutex mqd_mutex;
- struct kfd_dev *dev;
+ struct kfd_node *dev;
uint32_t mqd_size;
};
-struct kfd_mem_obj *allocate_hiq_mqd(struct kfd_dev *dev,
+struct kfd_mem_obj *allocate_hiq_mqd(struct kfd_node *dev,
struct queue_properties *q);
-struct kfd_mem_obj *allocate_sdma_mqd(struct kfd_dev *dev,
+struct kfd_mem_obj *allocate_sdma_mqd(struct kfd_node *dev,
struct queue_properties *q);
void free_mqd_hiq_sdma(struct mqd_manager *mm, void *mqd,
struct kfd_mem_obj *mqd_mem_obj);
m->cp_hqd_queue_priority = q->priority;
}
-static struct kfd_mem_obj *allocate_mqd(struct kfd_dev *kfd,
+static struct kfd_mem_obj *allocate_mqd(struct kfd_node *kfd,
struct queue_properties *q)
{
struct kfd_mem_obj *mqd_mem_obj;
struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
- struct kfd_dev *dev)
+ struct kfd_node *dev)
{
struct mqd_manager *mqd;
}
struct mqd_manager *mqd_manager_init_cik_hawaii(enum KFD_MQD_TYPE type,
- struct kfd_dev *dev)
+ struct kfd_node *dev)
{
struct mqd_manager *mqd;
m->cp_hqd_queue_priority = q->priority;
}
-static struct kfd_mem_obj *allocate_mqd(struct kfd_dev *kfd,
+static struct kfd_mem_obj *allocate_mqd(struct kfd_node *kfd,
struct queue_properties *q)
{
struct kfd_mem_obj *mqd_mem_obj;
1 << CP_HQD_AQL_CONTROL__CONTROL0__SHIFT;
}
- if (mm->dev->cwsr_enabled) {
+ if (mm->dev->kfd->cwsr_enabled) {
m->cp_hqd_persistent_state |=
(1 << CP_HQD_PERSISTENT_STATE__QSWITCH_MODE__SHIFT);
m->cp_hqd_ctx_save_base_addr_lo =
m->cp_hqd_pq_doorbell_control |=
1 << CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_BIF_DROP__SHIFT;
}
- if (mm->dev->cwsr_enabled)
+ if (mm->dev->kfd->cwsr_enabled)
m->cp_hqd_ctx_save_control = 0;
update_cu_mask(mm, mqd, minfo);
#endif
struct mqd_manager *mqd_manager_init_v10(enum KFD_MQD_TYPE type,
- struct kfd_dev *dev)
+ struct kfd_node *dev)
{
struct mqd_manager *mqd;
m->cp_hqd_queue_priority = q->priority;
}
-static struct kfd_mem_obj *allocate_mqd(struct kfd_dev *kfd,
+static struct kfd_mem_obj *allocate_mqd(struct kfd_node *node,
struct queue_properties *q)
{
struct kfd_mem_obj *mqd_mem_obj;
* MES write to areas beyond MQD size. So allocate
* 1 PAGE_SIZE memory for MQD is MES is enabled.
*/
- if (kfd->shared_resources.enable_mes)
+ if (node->kfd->shared_resources.enable_mes)
size = PAGE_SIZE;
else
size = sizeof(struct v11_compute_mqd);
- if (kfd_gtt_sa_allocate(kfd, size, &mqd_mem_obj))
+ if (kfd_gtt_sa_allocate(node, size, &mqd_mem_obj))
return NULL;
return mqd_mem_obj;
m = (struct v11_compute_mqd *) mqd_mem_obj->cpu_ptr;
addr = mqd_mem_obj->gpu_addr;
- if (mm->dev->shared_resources.enable_mes)
+ if (mm->dev->kfd->shared_resources.enable_mes)
size = PAGE_SIZE;
else
size = sizeof(struct v11_compute_mqd);
1 << CP_HQD_AQL_CONTROL__CONTROL0__SHIFT;
}
- if (mm->dev->cwsr_enabled) {
+ if (mm->dev->kfd->cwsr_enabled) {
m->cp_hqd_persistent_state |=
(1 << CP_HQD_PERSISTENT_STATE__QSWITCH_MODE__SHIFT);
m->cp_hqd_ctx_save_base_addr_lo =
m->cp_hqd_pq_doorbell_control |=
1 << CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_BIF_DROP__SHIFT;
}
- if (mm->dev->cwsr_enabled)
+ if (mm->dev->kfd->cwsr_enabled)
m->cp_hqd_ctx_save_control = 0;
update_cu_mask(mm, mqd, minfo);
m = (struct v11_sdma_mqd *) mqd_mem_obj->cpu_ptr;
- if (mm->dev->shared_resources.enable_mes)
+ if (mm->dev->kfd->shared_resources.enable_mes)
size = PAGE_SIZE;
else
size = sizeof(struct v11_sdma_mqd);
#endif
struct mqd_manager *mqd_manager_init_v11(enum KFD_MQD_TYPE type,
- struct kfd_dev *dev)
+ struct kfd_node *dev)
{
struct mqd_manager *mqd;
* To allocate SDMA MQDs by generic functions
* when MES is enabled.
*/
- if (dev->shared_resources.enable_mes) {
+ if (dev->kfd->shared_resources.enable_mes) {
mqd->allocate_mqd = allocate_mqd;
mqd->free_mqd = kfd_free_mqd_cp;
}
m->cp_hqd_queue_priority = q->priority;
}
-static struct kfd_mem_obj *allocate_mqd(struct kfd_dev *kfd,
+static struct kfd_mem_obj *allocate_mqd(struct kfd_node *node,
struct queue_properties *q)
{
int retval;
* pass a special bo flag AMDGPU_GEM_CREATE_CP_MQD_GFX9 to instruct
* amdgpu memory functions to do so.
*/
- if (kfd->cwsr_enabled && (q->type == KFD_QUEUE_TYPE_COMPUTE)) {
+ if (node->kfd->cwsr_enabled && (q->type == KFD_QUEUE_TYPE_COMPUTE)) {
mqd_mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_KERNEL);
if (!mqd_mem_obj)
return NULL;
- retval = amdgpu_amdkfd_alloc_gtt_mem(kfd->adev,
+ retval = amdgpu_amdkfd_alloc_gtt_mem(node->adev,
ALIGN(q->ctl_stack_size, PAGE_SIZE) +
ALIGN(sizeof(struct v9_mqd), PAGE_SIZE),
&(mqd_mem_obj->gtt_mem),
return NULL;
}
} else {
- retval = kfd_gtt_sa_allocate(kfd, sizeof(struct v9_mqd),
+ retval = kfd_gtt_sa_allocate(node, sizeof(struct v9_mqd),
&mqd_mem_obj);
if (retval)
return NULL;
{
uint64_t addr;
struct v9_mqd *m;
- struct amdgpu_device *adev = (struct amdgpu_device *)mm->dev->adev;
m = (struct v9_mqd *) mqd_mem_obj->cpu_ptr;
addr = mqd_mem_obj->gpu_addr;
if (q->format == KFD_QUEUE_FORMAT_AQL) {
m->cp_hqd_aql_control =
1 << CP_HQD_AQL_CONTROL__CONTROL0__SHIFT;
- if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3)) {
+ if (KFD_GC_VERSION(mm->dev) == IP_VERSION(9, 4, 3)) {
/* On GC 9.4.3, DW 41 is re-purposed as
* compute_tg_chunk_size.
* TODO: review this setting when active CUs in the
}
} else {
/* PM4 queue */
- if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3)) {
+ if (KFD_GC_VERSION(mm->dev) == IP_VERSION(9, 4, 3)) {
m->compute_static_thread_mgmt_se6 = 0;
/* TODO: program pm4_target_xcc */
}
(1 << COMPUTE_PGM_RSRC2__TRAP_PRESENT__SHIFT);
}
- if (mm->dev->cwsr_enabled && q->ctx_save_restore_area_address) {
+ if (mm->dev->kfd->cwsr_enabled && q->ctx_save_restore_area_address) {
m->cp_hqd_persistent_state |=
(1 << CP_HQD_PERSISTENT_STATE__QSWITCH_MODE__SHIFT);
m->cp_hqd_ctx_save_base_addr_lo =
struct queue_properties *q,
struct mqd_update_info *minfo)
{
- struct amdgpu_device *adev = (struct amdgpu_device *)mm->dev->adev;
struct v9_mqd *m;
m = get_mqd(mqd);
2 << CP_HQD_PQ_CONTROL__SLOT_BASED_WPTR__SHIFT |
1 << CP_HQD_PQ_CONTROL__QUEUE_FULL_EN__SHIFT |
1 << CP_HQD_PQ_CONTROL__WPP_CLAMP_EN__SHIFT;
- if (adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 4, 3))
+ if (KFD_GC_VERSION(mm->dev) != IP_VERSION(9, 4, 3))
m->cp_hqd_pq_control |=
- CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK;
+ CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK;
m->cp_hqd_pq_doorbell_control |= 1 <<
CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_BIF_DROP__SHIFT;
}
- if (mm->dev->cwsr_enabled && q->ctx_save_restore_area_address)
+ if (mm->dev->kfd->cwsr_enabled && q->ctx_save_restore_area_address)
m->cp_hqd_ctx_save_control = 0;
update_cu_mask(mm, mqd, minfo);
#endif
struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type,
- struct kfd_dev *dev)
+ struct kfd_node *dev)
{
struct mqd_manager *mqd;
m->cp_hqd_queue_priority = q->priority;
}
-static struct kfd_mem_obj *allocate_mqd(struct kfd_dev *kfd,
+static struct kfd_mem_obj *allocate_mqd(struct kfd_node *kfd,
struct queue_properties *q)
{
struct kfd_mem_obj *mqd_mem_obj;
(1 << COMPUTE_PGM_RSRC2__TRAP_PRESENT__SHIFT);
}
- if (mm->dev->cwsr_enabled && q->ctx_save_restore_area_address) {
+ if (mm->dev->kfd->cwsr_enabled && q->ctx_save_restore_area_address) {
m->cp_hqd_persistent_state |=
(1 << CP_HQD_PERSISTENT_STATE__QSWITCH_MODE__SHIFT);
m->cp_hqd_ctx_save_base_addr_lo =
2 << CP_HQD_PQ_CONTROL__SLOT_BASED_WPTR__SHIFT;
}
- if (mm->dev->cwsr_enabled && q->ctx_save_restore_area_address)
+ if (mm->dev->kfd->cwsr_enabled && q->ctx_save_restore_area_address)
m->cp_hqd_ctx_save_control =
atc_bit << CP_HQD_CTX_SAVE_CONTROL__ATC__SHIFT |
mtype << CP_HQD_CTX_SAVE_CONTROL__MTYPE__SHIFT;
#endif
struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
- struct kfd_dev *dev)
+ struct kfd_node *dev)
{
struct mqd_manager *mqd;
}
struct mqd_manager *mqd_manager_init_vi_tonga(enum KFD_MQD_TYPE type,
- struct kfd_dev *dev)
+ struct kfd_node *dev)
{
struct mqd_manager *mqd;
unsigned int process_count, queue_count, compute_queue_count, gws_queue_count;
unsigned int map_queue_size;
unsigned int max_proc_per_quantum = 1;
- struct kfd_dev *dev = pm->dqm->dev;
+ struct kfd_node *dev = pm->dqm->dev;
process_count = pm->dqm->processes_count;
queue_count = pm->dqm->active_queue_count;
struct pm4_mes_runlist *packet;
int concurrent_proc_cnt = 0;
- struct kfd_dev *kfd = pm->dqm->dev;
+ struct kfd_node *kfd = pm->dqm->dev;
/* Determine the number of processes to map together to HW:
* it can not exceed the number of VMIDs available to the
case KFD_QUEUE_TYPE_SDMA:
case KFD_QUEUE_TYPE_SDMA_XGMI:
use_static = false; /* no static queues under SDMA */
- if (q->properties.sdma_engine_id < 2 && !pm_use_ext_eng(q->device))
+ if (q->properties.sdma_engine_id < 2 &&
+ !pm_use_ext_eng(q->device->kfd))
packet->bitfields2.engine_sel = q->properties.sdma_engine_id +
engine_sel__mes_map_queues__sdma0_vi;
else {
packet->header.u32All = pm_build_pm4_header(IT_UNMAP_QUEUES,
sizeof(struct pm4_mes_unmap_queues));
- packet->bitfields2.extended_engine_sel = pm_use_ext_eng(pm->dqm->dev) ?
+ packet->bitfields2.extended_engine_sel =
+ pm_use_ext_eng(pm->dqm->dev->kfd) ?
extended_engine_sel__mes_unmap_queues__sdma0_to_7_sel :
extended_engine_sel__mes_unmap_queues__legacy_engine_sel;
{
struct pm4_mes_runlist *packet;
int concurrent_proc_cnt = 0;
- struct kfd_dev *kfd = pm->dqm->dev;
+ struct kfd_node *kfd = pm->dqm->dev;
if (WARN_ON(!ib))
return -EFAULT;
((KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 2)) || \
(KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 3)))
+struct kfd_node;
+
struct kfd_event_interrupt_class {
- bool (*interrupt_isr)(struct kfd_dev *dev,
+ bool (*interrupt_isr)(struct kfd_node *dev,
const uint32_t *ih_ring_entry, uint32_t *patched_ihre,
bool *patched_flag);
- void (*interrupt_wq)(struct kfd_dev *dev,
+ void (*interrupt_wq)(struct kfd_node *dev,
const uint32_t *ih_ring_entry);
};
uint64_t reserved_sdma_queues_bitmap;
};
-unsigned int kfd_get_num_sdma_engines(struct kfd_dev *kdev);
-unsigned int kfd_get_num_xgmi_sdma_engines(struct kfd_dev *kdev);
+unsigned int kfd_get_num_sdma_engines(struct kfd_node *kdev);
+unsigned int kfd_get_num_xgmi_sdma_engines(struct kfd_node *kdev);
struct kfd_mem_obj {
uint32_t range_start;
uint32_t vmid_num_kfd;
};
+struct kfd_dev;
+
+struct kfd_node {
+ struct amdgpu_device *adev; /* Duplicated here along with keeping
+ * a copy in kfd_dev to save a hop
+ */
+ const struct kfd2kgd_calls *kfd2kgd; /* Duplicated here along with
+ * keeping a copy in kfd_dev to
+ * save a hop
+ */
+ struct kfd_vmid_info vm_info;
+ unsigned int id; /* topology stub index */
+ /* Interrupts */
+ struct kfifo ih_fifo;
+ struct workqueue_struct *ih_wq;
+ struct work_struct interrupt_work;
+ spinlock_t interrupt_lock;
+
+ /*
+ * Interrupts of interest to KFD are copied
+ * from the HW ring into a SW ring.
+ */
+ bool interrupts_active;
+
+ /* QCM Device instance */
+ struct device_queue_manager *dqm;
+
+ /* Global GWS resource shared between processes */
+ void *gws;
+ bool gws_debug_workaround;
+
+ /* Clients watching SMI events */
+ struct list_head smi_clients;
+ spinlock_t smi_lock;
+ uint32_t reset_seq_num;
+
+ /* SRAM ECC flag */
+ atomic_t sram_ecc_flag;
+
+ /*spm process id */
+ unsigned int spm_pasid;
+
+ /* Maximum process number mapped to HW scheduler */
+ unsigned int max_proc_per_quantum;
+
+ struct kfd_dev *kfd;
+};
+
struct kfd_dev {
struct amdgpu_device *adev;
struct kfd_device_info device_info;
- unsigned int id; /* topology stub index */
-
phys_addr_t doorbell_base; /* Start of actual doorbells used by
* KFD. It is aligned for mapping
* into user mode
*/
struct kgd2kfd_shared_resources shared_resources;
- struct kfd_vmid_info vm_info;
struct kfd_local_mem_info local_mem_info;
const struct kfd2kgd_calls *kfd2kgd;
unsigned int gtt_sa_chunk_size;
unsigned int gtt_sa_num_of_chunks;
- /* Interrupts */
- struct kfifo ih_fifo;
- struct workqueue_struct *ih_wq;
- struct work_struct interrupt_work;
- spinlock_t interrupt_lock;
-
- /* QCM Device instance */
- struct device_queue_manager *dqm;
-
bool init_complete;
- /*
- * Interrupts of interest to KFD are copied
- * from the HW ring into a SW ring.
- */
- bool interrupts_active;
/* Firmware versions */
uint16_t mec_fw_version;
uint16_t mec2_fw_version;
uint16_t sdma_fw_version;
- /* Maximum process number mapped to HW scheduler */
- unsigned int max_proc_per_quantum;
-
/* CWSR */
bool cwsr_enabled;
const void *cwsr_isa;
/* Use IOMMU v2 flag */
bool use_iommu_v2;
- /* SRAM ECC flag */
- atomic_t sram_ecc_flag;
-
/* Compute Profile ref. count */
atomic_t compute_profile;
- /* Global GWS resource shared between processes */
- void *gws;
-
- /* Clients watching SMI events */
- struct list_head smi_clients;
- spinlock_t smi_lock;
-
- uint32_t reset_seq_num;
-
struct ida doorbell_ida;
unsigned int max_doorbell_slices;
/* HMM page migration MEMORY_DEVICE_PRIVATE mapping */
struct dev_pagemap pgmap;
+
+ struct kfd_node *node;
};
enum kfd_mempool {
unsigned int doorbell_id;
struct kfd_process *process;
- struct kfd_dev *device;
+ struct kfd_node *device;
void *gws;
/* procfs */
/* Data that is per-process-per device. */
struct kfd_process_device {
/* The device that owns this data. */
- struct kfd_dev *dev;
+ struct kfd_node *dev;
/* The process that owns this kfd_process_device. */
struct kfd_process *process;
unsigned int cmd_drv;
const char *name;
};
-bool kfd_dev_is_large_bar(struct kfd_dev *dev);
+bool kfd_dev_is_large_bar(struct kfd_node *dev);
int kfd_process_create_wq(void);
void kfd_process_destroy_wq(void);
int kfd_process_device_init_vm(struct kfd_process_device *pdd,
struct file *drm_file);
-struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
+struct kfd_process_device *kfd_bind_process_to_device(struct kfd_node *dev,
struct kfd_process *p);
-struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
+struct kfd_process_device *kfd_get_process_device_data(struct kfd_node *dev,
struct kfd_process *p);
-struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
+struct kfd_process_device *kfd_create_process_device_data(struct kfd_node *dev,
struct kfd_process *p);
bool kfd_process_xnack_mode(struct kfd_process *p, bool supported);
-int kfd_reserved_mem_mmap(struct kfd_dev *dev, struct kfd_process *process,
+int kfd_reserved_mem_mmap(struct kfd_node *dev, struct kfd_process *process,
struct vm_area_struct *vma);
/* KFD process API for creating and translating handles */
size_t kfd_doorbell_process_slice(struct kfd_dev *kfd);
int kfd_doorbell_init(struct kfd_dev *kfd);
void kfd_doorbell_fini(struct kfd_dev *kfd);
-int kfd_doorbell_mmap(struct kfd_dev *dev, struct kfd_process *process,
+int kfd_doorbell_mmap(struct kfd_node *dev, struct kfd_process *process,
struct vm_area_struct *vma);
void __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd,
unsigned int *doorbell_off);
unsigned int doorbell_index);
/* GTT Sub-Allocator */
-int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size,
+int kfd_gtt_sa_allocate(struct kfd_node *node, unsigned int size,
struct kfd_mem_obj **mem_obj);
-int kfd_gtt_sa_free(struct kfd_dev *kfd, struct kfd_mem_obj *mem_obj);
+int kfd_gtt_sa_free(struct kfd_node *node, struct kfd_mem_obj *mem_obj);
extern struct device *kfd_device;
/* Topology */
int kfd_topology_init(void);
void kfd_topology_shutdown(void);
-int kfd_topology_add_device(struct kfd_dev *gpu);
-int kfd_topology_remove_device(struct kfd_dev *gpu);
+int kfd_topology_add_device(struct kfd_node *gpu);
+int kfd_topology_remove_device(struct kfd_node *gpu);
struct kfd_topology_device *kfd_topology_device_by_proximity_domain(
uint32_t proximity_domain);
struct kfd_topology_device *kfd_topology_device_by_proximity_domain_no_lock(
uint32_t proximity_domain);
struct kfd_topology_device *kfd_topology_device_by_id(uint32_t gpu_id);
-struct kfd_dev *kfd_device_by_id(uint32_t gpu_id);
-struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev);
-struct kfd_dev *kfd_device_by_adev(const struct amdgpu_device *adev);
-int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_dev **kdev);
+struct kfd_node *kfd_device_by_id(uint32_t gpu_id);
+struct kfd_node *kfd_device_by_pci_dev(const struct pci_dev *pdev);
+struct kfd_node *kfd_device_by_adev(const struct amdgpu_device *adev);
+int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_node **kdev);
int kfd_numa_node_to_apic_id(int numa_node_id);
void kfd_double_confirm_iommu_support(struct kfd_dev *gpu);
/* Interrupts */
-int kfd_interrupt_init(struct kfd_dev *dev);
-void kfd_interrupt_exit(struct kfd_dev *dev);
-bool enqueue_ih_ring_entry(struct kfd_dev *kfd, const void *ih_ring_entry);
-bool interrupt_is_wanted(struct kfd_dev *dev,
+int kfd_interrupt_init(struct kfd_node *dev);
+void kfd_interrupt_exit(struct kfd_node *dev);
+bool enqueue_ih_ring_entry(struct kfd_node *kfd, const void *ih_ring_entry);
+bool interrupt_is_wanted(struct kfd_node *dev,
const uint32_t *ih_ring_entry,
uint32_t *patched_ihre, bool *flag);
void print_queue(struct queue *q);
struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
- struct kfd_dev *dev);
+ struct kfd_node *dev);
struct mqd_manager *mqd_manager_init_cik_hawaii(enum KFD_MQD_TYPE type,
- struct kfd_dev *dev);
+ struct kfd_node *dev);
struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
- struct kfd_dev *dev);
+ struct kfd_node *dev);
struct mqd_manager *mqd_manager_init_vi_tonga(enum KFD_MQD_TYPE type,
- struct kfd_dev *dev);
+ struct kfd_node *dev);
struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type,
- struct kfd_dev *dev);
+ struct kfd_node *dev);
struct mqd_manager *mqd_manager_init_v10(enum KFD_MQD_TYPE type,
- struct kfd_dev *dev);
+ struct kfd_node *dev);
struct mqd_manager *mqd_manager_init_v11(enum KFD_MQD_TYPE type,
- struct kfd_dev *dev);
-struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev);
+ struct kfd_node *dev);
+struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev);
void device_queue_manager_uninit(struct device_queue_manager *dqm);
-struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,
+struct kernel_queue *kernel_queue_init(struct kfd_node *dev,
enum kfd_queue_type type);
void kernel_queue_uninit(struct kernel_queue *kq, bool hanging);
int kfd_dqm_evict_pasid(struct device_queue_manager *dqm, u32 pasid);
int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p);
void pqm_uninit(struct process_queue_manager *pqm);
int pqm_create_queue(struct process_queue_manager *pqm,
- struct kfd_dev *dev,
+ struct kfd_node *dev,
struct file *f,
struct queue_properties *properties,
unsigned int *qid,
uint32_t *wait_result);
void kfd_signal_event_interrupt(u32 pasid, uint32_t partial_id,
uint32_t valid_id_bits);
-void kfd_signal_iommu_event(struct kfd_dev *dev,
+void kfd_signal_iommu_event(struct kfd_node *dev,
u32 pasid, unsigned long address,
bool is_write_requested, bool is_execute_requested);
void kfd_signal_hw_exception_event(u32 pasid);
int kfd_get_num_events(struct kfd_process *p);
int kfd_event_destroy(struct kfd_process *p, uint32_t event_id);
-void kfd_signal_vm_fault_event(struct kfd_dev *dev, u32 pasid,
+void kfd_signal_vm_fault_event(struct kfd_node *dev, u32 pasid,
struct kfd_vm_fault_info *info);
-void kfd_signal_reset_event(struct kfd_dev *dev);
+void kfd_signal_reset_event(struct kfd_node *dev);
-void kfd_signal_poison_consumed_event(struct kfd_dev *dev, u32 pasid);
+void kfd_signal_poison_consumed_event(struct kfd_node *dev, u32 pasid);
void kfd_flush_tlb(struct kfd_process_device *pdd, enum TLB_FLUSH_TYPE type);
bool kfd_is_locked(void);
/* Compute profile */
-void kfd_inc_compute_active(struct kfd_dev *dev);
-void kfd_dec_compute_active(struct kfd_dev *dev);
+void kfd_inc_compute_active(struct kfd_node *dev);
+void kfd_dec_compute_active(struct kfd_node *dev);
/* Cgroup Support */
/* Check with device cgroup if @kfd device is accessible */
-static inline int kfd_devcgroup_check_permission(struct kfd_dev *kfd)
+static inline int kfd_devcgroup_check_permission(struct kfd_node *kfd)
{
#if defined(CONFIG_CGROUP_DEVICE) || defined(CONFIG_CGROUP_BPF)
struct drm_device *ddev = adev_to_drm(kfd->adev);
int kfd_debugfs_rls_by_device(struct seq_file *m, void *data);
int pm_debugfs_runlist(struct seq_file *m, void *data);
-int kfd_debugfs_hang_hws(struct kfd_dev *dev);
+int kfd_debugfs_hang_hws(struct kfd_node *dev);
int pm_debugfs_hang_hws(struct packet_manager *pm);
int dqm_debugfs_hang_hws(struct device_queue_manager *dqm);
int cu_cnt;
int wave_cnt;
int max_waves_per_cu;
- struct kfd_dev *dev = NULL;
+ struct kfd_node *dev = NULL;
struct kfd_process *proc = NULL;
struct kfd_process_device *pdd = NULL;
static void kfd_process_free_gpuvm(struct kgd_mem *mem,
struct kfd_process_device *pdd, void **kptr)
{
- struct kfd_dev *dev = pdd->dev;
+ struct kfd_node *dev = pdd->dev;
if (kptr && *kptr) {
amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(mem);
uint64_t gpu_va, uint32_t size,
uint32_t flags, struct kgd_mem **mem, void **kptr)
{
- struct kfd_dev *kdev = pdd->dev;
+ struct kfd_node *kdev = pdd->dev;
int err;
err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(kdev->adev, gpu_va, size,
static void kfd_process_kunmap_signal_bo(struct kfd_process *p)
{
struct kfd_process_device *pdd;
- struct kfd_dev *kdev;
+ struct kfd_node *kdev;
void *mem;
kdev = kfd_device_by_id(GET_GPU_ID(p->signal_handle));
bitmap_free(pdd->qpd.doorbell_bitmap);
idr_destroy(&pdd->alloc_idr);
- kfd_free_process_doorbells(pdd->dev, pdd->doorbell_index);
+ kfd_free_process_doorbells(pdd->dev->kfd, pdd->doorbell_index);
- if (pdd->dev->shared_resources.enable_mes)
+ if (pdd->dev->kfd->shared_resources.enable_mes)
amdgpu_amdkfd_free_gtt_mem(pdd->dev->adev,
pdd->proc_ctx_bo);
/*
int i;
for (i = 0; i < p->n_pdds; i++) {
- struct kfd_dev *dev = p->pdds[i]->dev;
+ struct kfd_node *dev = p->pdds[i]->dev;
struct qcm_process_device *qpd = &p->pdds[i]->qpd;
- if (!dev->cwsr_enabled || qpd->cwsr_kaddr || qpd->cwsr_base)
+ if (!dev->kfd->cwsr_enabled || qpd->cwsr_kaddr || qpd->cwsr_base)
continue;
offset = KFD_MMAP_TYPE_RESERVED_MEM | KFD_MMAP_GPU_ID(dev->id);
return err;
}
- memcpy(qpd->cwsr_kaddr, dev->cwsr_isa, dev->cwsr_isa_size);
+ memcpy(qpd->cwsr_kaddr, dev->kfd->cwsr_isa, dev->kfd->cwsr_isa_size);
qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)
{
- struct kfd_dev *dev = pdd->dev;
+ struct kfd_node *dev = pdd->dev;
struct qcm_process_device *qpd = &pdd->qpd;
uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT
| KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE
void *kaddr;
int ret;
- if (!dev->cwsr_enabled || qpd->cwsr_kaddr || !qpd->cwsr_base)
+ if (!dev->kfd->cwsr_enabled || qpd->cwsr_kaddr || !qpd->cwsr_base)
return 0;
/* cwsr_base is only set for dGPU */
qpd->cwsr_kaddr = kaddr;
qpd->tba_addr = qpd->cwsr_base;
- memcpy(qpd->cwsr_kaddr, dev->cwsr_isa, dev->cwsr_isa_size);
+ memcpy(qpd->cwsr_kaddr, dev->kfd->cwsr_isa, dev->kfd->cwsr_isa_size);
qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd)
{
- struct kfd_dev *dev = pdd->dev;
+ struct kfd_node *dev = pdd->dev;
struct qcm_process_device *qpd = &pdd->qpd;
- if (!dev->cwsr_enabled || !qpd->cwsr_kaddr || !qpd->cwsr_base)
+ if (!dev->kfd->cwsr_enabled || !qpd->cwsr_kaddr || !qpd->cwsr_base)
return;
kfd_process_free_gpuvm(qpd->cwsr_mem, pdd, &qpd->cwsr_kaddr);
* support retry.
*/
for (i = 0; i < p->n_pdds; i++) {
- struct kfd_dev *dev = p->pdds[i]->dev;
+ struct kfd_node *dev = p->pdds[i]->dev;
/* Only consider GFXv9 and higher GPUs. Older GPUs don't
* support the SVM APIs and don't need to be considered
if (KFD_GC_VERSION(dev) >= IP_VERSION(10, 1, 1))
return false;
- if (dev->noretry)
+ if (dev->kfd->noretry)
return false;
}
return 0;
}
-struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
+struct kfd_process_device *kfd_get_process_device_data(struct kfd_node *dev,
struct kfd_process *p)
{
int i;
return NULL;
}
-struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
+struct kfd_process_device *kfd_create_process_device_data(struct kfd_node *dev,
struct kfd_process *p)
{
struct kfd_process_device *pdd = NULL;
if (!pdd)
return NULL;
- if (init_doorbell_bitmap(&pdd->qpd, dev)) {
+ if (init_doorbell_bitmap(&pdd->qpd, dev->kfd)) {
pr_err("Failed to init doorbell for process\n");
goto err_free_pdd;
}
pdd->user_gpu_id = dev->id;
atomic64_set(&pdd->evict_duration_counter, 0);
- if (dev->shared_resources.enable_mes) {
+ if (dev->kfd->shared_resources.enable_mes) {
retval = amdgpu_amdkfd_alloc_gtt_mem(dev->adev,
AMDGPU_MES_PROC_CTX_SIZE,
&pdd->proc_ctx_bo,
struct amdgpu_fpriv *drv_priv;
struct amdgpu_vm *avm;
struct kfd_process *p;
- struct kfd_dev *dev;
+ struct kfd_node *dev;
int ret;
if (!drm_file)
*
* Assumes that the process lock is held.
*/
-struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
+struct kfd_process_device *kfd_bind_process_to_device(struct kfd_node *dev,
struct kfd_process *p)
{
struct kfd_process_device *pdd;
for (i = 0; i < p->n_pdds; i++) {
struct kfd_process_device *pdd = p->pdds[i];
- kfd_smi_event_queue_eviction(pdd->dev, p->lead_thread->pid,
+ kfd_smi_event_queue_eviction(pdd->dev->kfd, p->lead_thread->pid,
trigger);
r = pdd->dev->dqm->ops.evict_process_queues(pdd->dev->dqm,
if (n_evicted == 0)
break;
- kfd_smi_event_queue_restore(pdd->dev, p->lead_thread->pid);
+ kfd_smi_event_queue_restore(pdd->dev->kfd, p->lead_thread->pid);
if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
&pdd->qpd))
for (i = 0; i < p->n_pdds; i++) {
struct kfd_process_device *pdd = p->pdds[i];
- kfd_smi_event_queue_restore(pdd->dev, p->lead_thread->pid);
+ kfd_smi_event_queue_restore(pdd->dev->kfd, p->lead_thread->pid);
r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
&pdd->qpd);
return ret;
}
-int kfd_reserved_mem_mmap(struct kfd_dev *dev, struct kfd_process *process,
+int kfd_reserved_mem_mmap(struct kfd_node *dev, struct kfd_process *process,
struct vm_area_struct *vma)
{
struct kfd_process_device *pdd;
{
struct amdgpu_vm *vm = drm_priv_to_vm(pdd->drm_priv);
uint64_t tlb_seq = amdgpu_vm_tlb_seq(vm);
- struct kfd_dev *dev = pdd->dev;
+ struct kfd_node *dev = pdd->dev;
/*
* It can be that we race and lose here, but that is extremely unlikely
void kfd_process_dequeue_from_device(struct kfd_process_device *pdd)
{
- struct kfd_dev *dev = pdd->dev;
+ struct kfd_node *dev = pdd->dev;
if (pdd->already_dequeued)
return;
int pqm_set_gws(struct process_queue_manager *pqm, unsigned int qid,
void *gws)
{
- struct kfd_dev *dev = NULL;
+ struct kfd_node *dev = NULL;
struct process_queue_node *pqn;
struct kfd_process_device *pdd;
struct kgd_mem *mem = NULL;
}
static int init_user_queue(struct process_queue_manager *pqm,
- struct kfd_dev *dev, struct queue **q,
+ struct kfd_node *dev, struct queue **q,
struct queue_properties *q_properties,
struct file *f, struct amdgpu_bo *wptr_bo,
unsigned int qid)
(*q)->device = dev;
(*q)->process = pqm->process;
- if (dev->shared_resources.enable_mes) {
+ if (dev->kfd->shared_resources.enable_mes) {
retval = amdgpu_amdkfd_alloc_gtt_mem(dev->adev,
AMDGPU_MES_GANG_CTX_SIZE,
&(*q)->gang_ctx_bo,
}
int pqm_create_queue(struct process_queue_manager *pqm,
- struct kfd_dev *dev,
+ struct kfd_node *dev,
struct file *f,
struct queue_properties *properties,
unsigned int *qid,
* Hence we also check the type as well
*/
if ((pdd->qpd.is_debug) || (type == KFD_QUEUE_TYPE_DIQ))
- max_queues = dev->device_info.max_no_of_hqd/2;
+ max_queues = dev->kfd->device_info.max_no_of_hqd/2;
if (pdd->qpd.queue_count >= max_queues)
return -ENOSPC;
*/
*p_doorbell_offset_in_process =
(q->properties.doorbell_off * sizeof(uint32_t)) &
- (kfd_doorbell_process_slice(dev) - 1);
+ (kfd_doorbell_process_slice(dev->kfd) - 1);
pr_debug("PQM After DQM create queue\n");
struct process_queue_node *pqn;
struct kfd_process_device *pdd;
struct device_queue_manager *dqm;
- struct kfd_dev *dev;
+ struct kfd_node *dev;
int retval;
dqm = NULL;
pdd->qpd.num_gws = 0;
}
- if (dev->shared_resources.enable_mes) {
+ if (dev->kfd->shared_resources.enable_mes) {
amdgpu_amdkfd_free_gtt_mem(dev->adev,
pqn->q->gang_ctx_bo);
if (pqn->q->wptr_bo)
}
if (!pdd->doorbell_index &&
- kfd_alloc_process_doorbells(pdd->dev, &pdd->doorbell_index) < 0) {
+ kfd_alloc_process_doorbells(pdd->dev->kfd, &pdd->doorbell_index) < 0) {
ret = -ENOMEM;
goto exit;
}
wait_queue_head_t wait_queue;
/* events enabled */
uint64_t events;
- struct kfd_dev *dev;
+ struct kfd_node *dev;
spinlock_t lock;
struct rcu_head rcu;
pid_t pid;
static int kfd_smi_ev_release(struct inode *inode, struct file *filep)
{
struct kfd_smi_client *client = filep->private_data;
- struct kfd_dev *dev = client->dev;
+ struct kfd_node *dev = client->dev;
spin_lock(&dev->smi_lock);
list_del_rcu(&client->list);
return events & KFD_SMI_EVENT_MASK_FROM_INDEX(event);
}
-static void add_event_to_kfifo(pid_t pid, struct kfd_dev *dev,
+static void add_event_to_kfifo(pid_t pid, struct kfd_node *dev,
unsigned int smi_event, char *event_msg, int len)
{
struct kfd_smi_client *client;
}
__printf(4, 5)
-static void kfd_smi_event_add(pid_t pid, struct kfd_dev *dev,
+static void kfd_smi_event_add(pid_t pid, struct kfd_node *dev,
unsigned int event, char *fmt, ...)
{
char fifo_in[KFD_SMI_EVENT_MSG_SIZE];
add_event_to_kfifo(pid, dev, event, fifo_in, len);
}
-void kfd_smi_event_update_gpu_reset(struct kfd_dev *dev, bool post_reset)
+void kfd_smi_event_update_gpu_reset(struct kfd_node *dev, bool post_reset)
{
unsigned int event;
kfd_smi_event_add(0, dev, event, "%x\n", dev->reset_seq_num);
}
-void kfd_smi_event_update_thermal_throttling(struct kfd_dev *dev,
+void kfd_smi_event_update_thermal_throttling(struct kfd_node *dev,
uint64_t throttle_bitmask)
{
kfd_smi_event_add(0, dev, KFD_SMI_EVENT_THERMAL_THROTTLE, "%llx:%llx\n",
amdgpu_dpm_get_thermal_throttling_counter(dev->adev));
}
-void kfd_smi_event_update_vmfault(struct kfd_dev *dev, uint16_t pasid)
+void kfd_smi_event_update_vmfault(struct kfd_node *dev, uint16_t pasid)
{
struct amdgpu_task_info task_info;
unsigned long address, bool write_fault,
ktime_t ts)
{
- kfd_smi_event_add(pid, dev, KFD_SMI_EVENT_PAGE_FAULT_START,
+ kfd_smi_event_add(pid, dev->node, KFD_SMI_EVENT_PAGE_FAULT_START,
"%lld -%d @%lx(%x) %c\n", ktime_to_ns(ts), pid,
- address, dev->id, write_fault ? 'W' : 'R');
+ address, dev->node->id, write_fault ? 'W' : 'R');
}
void kfd_smi_event_page_fault_end(struct kfd_dev *dev, pid_t pid,
unsigned long address, bool migration)
{
- kfd_smi_event_add(pid, dev, KFD_SMI_EVENT_PAGE_FAULT_END,
+ kfd_smi_event_add(pid, dev->node, KFD_SMI_EVENT_PAGE_FAULT_END,
"%lld -%d @%lx(%x) %c\n", ktime_get_boottime_ns(),
- pid, address, dev->id, migration ? 'M' : 'U');
+ pid, address, dev->node->id, migration ? 'M' : 'U');
}
void kfd_smi_event_migration_start(struct kfd_dev *dev, pid_t pid,
uint32_t prefetch_loc, uint32_t preferred_loc,
uint32_t trigger)
{
- kfd_smi_event_add(pid, dev, KFD_SMI_EVENT_MIGRATE_START,
+ kfd_smi_event_add(pid, dev->node, KFD_SMI_EVENT_MIGRATE_START,
"%lld -%d @%lx(%lx) %x->%x %x:%x %d\n",
ktime_get_boottime_ns(), pid, start, end - start,
from, to, prefetch_loc, preferred_loc, trigger);
unsigned long start, unsigned long end,
uint32_t from, uint32_t to, uint32_t trigger)
{
- kfd_smi_event_add(pid, dev, KFD_SMI_EVENT_MIGRATE_END,
+ kfd_smi_event_add(pid, dev->node, KFD_SMI_EVENT_MIGRATE_END,
"%lld -%d @%lx(%lx) %x->%x %d\n",
ktime_get_boottime_ns(), pid, start, end - start,
from, to, trigger);
void kfd_smi_event_queue_eviction(struct kfd_dev *dev, pid_t pid,
uint32_t trigger)
{
- kfd_smi_event_add(pid, dev, KFD_SMI_EVENT_QUEUE_EVICTION,
+ kfd_smi_event_add(pid, dev->node, KFD_SMI_EVENT_QUEUE_EVICTION,
"%lld -%d %x %d\n", ktime_get_boottime_ns(), pid,
- dev->id, trigger);
+ dev->node->id, trigger);
}
void kfd_smi_event_queue_restore(struct kfd_dev *dev, pid_t pid)
{
- kfd_smi_event_add(pid, dev, KFD_SMI_EVENT_QUEUE_RESTORE,
+ kfd_smi_event_add(pid, dev->node, KFD_SMI_EVENT_QUEUE_RESTORE,
"%lld -%d %x\n", ktime_get_boottime_ns(), pid,
- dev->id);
+ dev->node->id);
}
void kfd_smi_event_queue_restore_rescheduled(struct mm_struct *mm)
unsigned long address, unsigned long last,
uint32_t trigger)
{
- kfd_smi_event_add(pid, dev, KFD_SMI_EVENT_UNMAP_FROM_GPU,
+ kfd_smi_event_add(pid, dev->node, KFD_SMI_EVENT_UNMAP_FROM_GPU,
"%lld -%d @%lx(%lx) %x %d\n", ktime_get_boottime_ns(),
- pid, address, last - address + 1, dev->id, trigger);
+ pid, address, last - address + 1, dev->node->id, trigger);
}
-int kfd_smi_event_open(struct kfd_dev *dev, uint32_t *fd)
+int kfd_smi_event_open(struct kfd_node *dev, uint32_t *fd)
{
struct kfd_smi_client *client;
int ret;
#ifndef KFD_SMI_EVENTS_H_INCLUDED
#define KFD_SMI_EVENTS_H_INCLUDED
-int kfd_smi_event_open(struct kfd_dev *dev, uint32_t *fd);
-void kfd_smi_event_update_vmfault(struct kfd_dev *dev, uint16_t pasid);
-void kfd_smi_event_update_thermal_throttling(struct kfd_dev *dev,
+int kfd_smi_event_open(struct kfd_node *dev, uint32_t *fd);
+void kfd_smi_event_update_vmfault(struct kfd_node *dev, uint16_t pasid);
+void kfd_smi_event_update_thermal_throttling(struct kfd_node *dev,
uint64_t throttle_bitmask);
-void kfd_smi_event_update_gpu_reset(struct kfd_dev *dev, bool post_reset);
+void kfd_smi_event_update_gpu_reset(struct kfd_node *dev, bool post_reset);
void kfd_smi_event_page_fault_start(struct kfd_dev *dev, pid_t pid,
unsigned long address, bool write_fault,
ktime_t ts);
return -EINVAL;
}
- kfd_smi_event_unmap_from_gpu(pdd->dev, p->lead_thread->pid,
+ kfd_smi_event_unmap_from_gpu(pdd->dev->kfd, p->lead_thread->pid,
start, last, trigger);
r = svm_range_unmap_from_gpu(pdd->dev->adev,
spin_lock_init(&svms->deferred_list_lock);
for (i = 0; i < p->n_pdds; i++)
- if (KFD_IS_SVM_API_SUPPORTED(p->pdds[i]->dev))
+ if (KFD_IS_SVM_API_SUPPORTED(p->pdds[i]->dev->kfd))
bitmap_set(svms->bitmap_supported, i, 1);
return 0;
return ret;
}
-struct kfd_dev *kfd_device_by_id(uint32_t gpu_id)
+struct kfd_node *kfd_device_by_id(uint32_t gpu_id)
{
struct kfd_topology_device *top_dev;
return top_dev->gpu;
}
-struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev)
+struct kfd_node *kfd_device_by_pci_dev(const struct pci_dev *pdev)
{
struct kfd_topology_device *top_dev;
- struct kfd_dev *device = NULL;
+ struct kfd_node *device = NULL;
down_read(&topology_lock);
return device;
}
-struct kfd_dev *kfd_device_by_adev(const struct amdgpu_device *adev)
+struct kfd_node *kfd_device_by_adev(const struct amdgpu_device *adev)
{
struct kfd_topology_device *top_dev;
- struct kfd_dev *device = NULL;
+ struct kfd_node *device = NULL;
down_read(&topology_lock);
if (dev->gpu) {
log_max_watch_addr =
- __ilog2_u32(dev->gpu->device_info.num_of_watch_points);
+ __ilog2_u32(dev->gpu->kfd->device_info.num_of_watch_points);
if (log_max_watch_addr) {
dev->node_props.capability |=
sysfs_show_64bit_prop(buffer, offs, "local_mem_size", 0ULL);
sysfs_show_32bit_prop(buffer, offs, "fw_version",
- dev->gpu->mec_fw_version);
+ dev->gpu->kfd->mec_fw_version);
sysfs_show_32bit_prop(buffer, offs, "capability",
dev->node_props.capability);
sysfs_show_32bit_prop(buffer, offs, "sdma_fw_version",
- dev->gpu->sdma_fw_version);
+ dev->gpu->kfd->sdma_fw_version);
sysfs_show_64bit_prop(buffer, offs, "unique_id",
dev->gpu->adev->unique_id);
up_write(&topology_lock);
}
-static uint32_t kfd_generate_gpu_id(struct kfd_dev *gpu)
+static uint32_t kfd_generate_gpu_id(struct kfd_node *gpu)
{
uint32_t hashout;
uint32_t buf[7];
if (!gpu)
return 0;
- local_mem_size = gpu->local_mem_info.local_mem_size_private +
- gpu->local_mem_info.local_mem_size_public;
+ local_mem_size = gpu->kfd->local_mem_info.local_mem_size_private +
+ gpu->kfd->local_mem_info.local_mem_size_public;
buf[0] = gpu->adev->pdev->devfn;
buf[1] = gpu->adev->pdev->subsystem_vendor |
(gpu->adev->pdev->subsystem_device << 16);
* list then return NULL. This means a new topology device has to
* be created for this GPU.
*/
-static struct kfd_topology_device *kfd_assign_gpu(struct kfd_dev *gpu)
+static struct kfd_topology_device *kfd_assign_gpu(struct kfd_node *gpu)
{
struct kfd_topology_device *dev;
struct kfd_topology_device *out_dev = NULL;
/* Discrete GPUs need their own topology device list
* entries. Don't assign them to CPU/APU nodes.
*/
- if (!gpu->use_iommu_v2 &&
+ if (!gpu->kfd->use_iommu_v2 &&
dev->node_props.cpu_cores_count)
continue;
CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT;
/* set gpu (dev) flags. */
} else {
- if (!dev->gpu->pci_atomic_requested ||
+ if (!dev->gpu->kfd->pci_atomic_requested ||
dev->gpu->adev->asic_type == CHIP_HAWAII)
link->flags |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT |
CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT;
if (dev == new_dev)
break;
if (!dev->gpu || !dev->gpu->adev ||
- (dev->gpu->hive_id &&
- dev->gpu->hive_id == new_dev->gpu->hive_id))
+ (dev->gpu->kfd->hive_id &&
+ dev->gpu->kfd->hive_id == new_dev->gpu->kfd->hive_id))
goto next;
/* check if node(s) is/are peer accessible in one direction or bi-direction */
return ret;
}
-
/* Helper function. See kfd_fill_gpu_cache_info for parameter description */
static int fill_in_l1_pcache(struct kfd_cache_properties **props_ext,
struct kfd_gpu_cache_info *pcache_info,
/* kfd_fill_cache_non_crat_info - Fill GPU cache info using kfd_gpu_cache_info
* tables
*/
-static void kfd_fill_cache_non_crat_info(struct kfd_topology_device *dev, struct kfd_dev *kdev)
+static void kfd_fill_cache_non_crat_info(struct kfd_topology_device *dev, struct kfd_node *kdev)
{
struct kfd_gpu_cache_info *pcache_info = NULL;
int i, j, k;
pr_debug("Added [%d] GPU cache entries\n", num_of_entries);
}
-static int kfd_topology_add_device_locked(struct kfd_dev *gpu, uint32_t gpu_id,
+static int kfd_topology_add_device_locked(struct kfd_node *gpu, uint32_t gpu_id,
struct kfd_topology_device **dev)
{
int proximity_domain = ++topology_crat_proximity_domain;
return res;
}
-int kfd_topology_add_device(struct kfd_dev *gpu)
+int kfd_topology_add_device(struct kfd_node *gpu)
{
uint32_t gpu_id;
struct kfd_topology_device *dev;
dev->node_props.simd_arrays_per_engine =
cu_info.num_shader_arrays_per_engine;
- dev->node_props.gfx_target_version = gpu->device_info.gfx_target_version;
+ dev->node_props.gfx_target_version =
+ gpu->kfd->device_info.gfx_target_version;
dev->node_props.vendor_id = gpu->adev->pdev->vendor;
dev->node_props.device_id = gpu->adev->pdev->device;
dev->node_props.capability |=
dev->node_props.max_engine_clk_ccompute =
cpufreq_quick_get_max(0) / 1000;
dev->node_props.drm_render_minor =
- gpu->shared_resources.drm_render_minor;
+ gpu->kfd->shared_resources.drm_render_minor;
- dev->node_props.hive_id = gpu->hive_id;
+ dev->node_props.hive_id = gpu->kfd->hive_id;
dev->node_props.num_sdma_engines = kfd_get_num_sdma_engines(gpu);
dev->node_props.num_sdma_xgmi_engines =
kfd_get_num_xgmi_sdma_engines(gpu);
dev->node_props.num_sdma_queues_per_engine =
- gpu->device_info.num_sdma_queues_per_engine -
- gpu->device_info.num_reserved_sdma_queues_per_engine;
+ gpu->kfd->device_info.num_sdma_queues_per_engine -
+ gpu->kfd->device_info.num_reserved_sdma_queues_per_engine;
dev->node_props.num_gws = (dev->gpu->gws &&
dev->gpu->dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) ?
dev->gpu->adev->gds.gws_size : 0;
* Overwrite ATS capability according to needs_iommu_device to fix
* potential missing corresponding bit in CRAT of BIOS.
*/
- if (dev->gpu->use_iommu_v2)
+ if (dev->gpu->kfd->use_iommu_v2)
dev->node_props.capability |= HSA_CAP_ATS_PRESENT;
else
dev->node_props.capability &= ~HSA_CAP_ATS_PRESENT;
}
}
-int kfd_topology_remove_device(struct kfd_dev *gpu)
+int kfd_topology_remove_device(struct kfd_node *gpu)
{
struct kfd_topology_device *dev, *tmp;
uint32_t gpu_id;
* Return - 0: On success (@kdev will be NULL for non GPU nodes)
* -1: If end of list
*/
-int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_dev **kdev)
+int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_node **kdev)
{
struct kfd_topology_device *top_dev;
uint32_t flags;
uint32_t width;
uint32_t mem_clk_max;
- struct kfd_dev *gpu;
+ struct kfd_node *gpu;
struct kobject *kobj;
struct attribute attr;
};
uint32_t cache_latency;
uint32_t cache_type;
uint8_t sibling_map[CACHE_SIBLINGMAP_SIZE];
- struct kfd_dev *gpu;
+ struct kfd_node *gpu;
struct kobject *kobj;
struct attribute attr;
uint32_t sibling_map_size;
uint32_t max_bandwidth;
uint32_t rec_transfer_size;
uint32_t flags;
- struct kfd_dev *gpu;
+ struct kfd_node *gpu;
struct kobject *kobj;
struct attribute attr;
};
struct list_head io_link_props;
struct list_head p2p_link_props;
struct list_head perf_props;
- struct kfd_dev *gpu;
+ struct kfd_node *gpu;
struct kobject *kobj_node;
struct kobject *kobj_mem;
struct kobject *kobj_cache;
projects / thirdparty / kernel / linux.git / commitdiff
