diff options
Diffstat (limited to 'drivers/gpu/drm/amd/amdkfd')
| -rw-r--r-- | drivers/gpu/drm/amd/amdkfd/kfd_device.c | 44 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c | 29 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 1 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/amdkfd/kfd_process.c | 9 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c | 35 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/amdkfd/kfd_topology.c | 4 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/amdkfd/soc15_int.h | 1 |
7 files changed, 110 insertions, 13 deletions
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c index 9d4abfd8b55e..0b3dc754e06b 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c @@ -138,9 +138,12 @@ static void kfd_device_info_set_event_interrupt_class(struct kfd_dev *kfd) case IP_VERSION(9, 4, 0): /* VEGA20 */ case IP_VERSION(9, 4, 1): /* ARCTURUS */ case IP_VERSION(9, 4, 2): /* ALDEBARAN */ - case IP_VERSION(9, 4, 3): /* GC 9.4.3 */ kfd->device_info.event_interrupt_class = &event_interrupt_class_v9; break; + case IP_VERSION(9, 4, 3): /* GC 9.4.3 */ + kfd->device_info.event_interrupt_class = + &event_interrupt_class_v9_4_3; + break; case IP_VERSION(10, 3, 1): /* VANGOGH */ case IP_VERSION(10, 3, 3): /* YELLOW_CARP */ case IP_VERSION(10, 3, 6): /* GC 10.3.6 */ @@ -518,6 +521,7 @@ static int kfd_gws_init(struct kfd_node *node) && kfd->mec2_fw_version >= 0x30) || (KFD_GC_VERSION(node) == IP_VERSION(9, 4, 2) && kfd->mec2_fw_version >= 0x28) || + (KFD_GC_VERSION(node) == IP_VERSION(9, 4, 3)) || (KFD_GC_VERSION(node) >= IP_VERSION(10, 3, 0) && KFD_GC_VERSION(node) < IP_VERSION(11, 0, 0) && kfd->mec2_fw_version >= 0x6b)))) @@ -598,6 +602,41 @@ static void kfd_cleanup_nodes(struct kfd_dev *kfd, unsigned int num_nodes) } } +static void kfd_setup_interrupt_bitmap(struct kfd_node *node, + unsigned int kfd_node_idx) +{ + struct amdgpu_device *adev = node->adev; + uint32_t xcc_mask = node->xcc_mask; + uint32_t xcc, mapped_xcc; + /* + * Interrupt bitmap is setup for processing interrupts from + * different XCDs and AIDs. + * Interrupt bitmap is defined as follows: + * 1. Bits 0-15 - correspond to the NodeId field. + * Each bit corresponds to NodeId number. For example, if + * a KFD node has interrupt bitmap set to 0x7, then this + * KFD node will process interrupts with NodeId = 0, 1 and 2 + * in the IH cookie. + * 2. Bits 16-31 - unused. + * + * Please note that the kfd_node_idx argument passed to this + * function is not related to NodeId field received in the + * IH cookie. + * + * In CPX mode, a KFD node will process an interrupt if: + * - the Node Id matches the corresponding bit set in + * Bits 0-15. + * - AND VMID reported in the interrupt lies within the + * VMID range of the node. + */ + for_each_inst(xcc, xcc_mask) { + mapped_xcc = GET_INST(GC, xcc); + node->interrupt_bitmap |= (mapped_xcc % 2 ? 5 : 3) << (4 * (mapped_xcc / 2)); + } + dev_info(kfd_device, "Node: %d, interrupt_bitmap: %x\n", kfd_node_idx, + node->interrupt_bitmap); +} + bool kgd2kfd_device_init(struct kfd_dev *kfd, const struct kgd2kfd_shared_resources *gpu_resources) { @@ -797,6 +836,9 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd, amdgpu_amdkfd_get_local_mem_info(kfd->adev, &node->local_mem_info, node->xcp); + if (KFD_GC_VERSION(kfd) == IP_VERSION(9, 4, 3)) + kfd_setup_interrupt_bitmap(node, i); + /* Initialize the KFD node */ if (kfd_init_node(node)) { dev_err(kfd_device, "Error initializing KFD node\n"); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c index d5c9f30552e3..f0731a6a5306 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c @@ -446,7 +446,36 @@ static void event_interrupt_wq_v9(struct kfd_node *dev, } } +static bool event_interrupt_isr_v9_4_3(struct kfd_node *node, + const uint32_t *ih_ring_entry, + uint32_t *patched_ihre, + bool *patched_flag) +{ + uint16_t node_id, vmid; + + /* + * For GFX 9.4.3, process the interrupt if: + * - NodeID field in IH entry matches the corresponding bit + * set in interrupt_bitmap Bits 0-15. + * OR + * - If partition mode is CPX and interrupt came from + * Node_id 0,4,8,12, then check if the Bit (16 + client id) + * is set in interrupt bitmap Bits 16-31. + */ + node_id = SOC15_NODEID_FROM_IH_ENTRY(ih_ring_entry); + vmid = SOC15_VMID_FROM_IH_ENTRY(ih_ring_entry); + if (kfd_irq_is_from_node(node, node_id, vmid)) + return event_interrupt_isr_v9(node, ih_ring_entry, + patched_ihre, patched_flag); + return false; +} + const struct kfd_event_interrupt_class event_interrupt_class_v9 = { .interrupt_isr = event_interrupt_isr_v9, .interrupt_wq = event_interrupt_wq_v9, }; + +const struct kfd_event_interrupt_class event_interrupt_class_v9_4_3 = { + .interrupt_isr = event_interrupt_isr_v9_4_3, + .interrupt_wq = event_interrupt_wq_v9, +}; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h index 7364a5d77c6e..d4c9ee3f9953 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h @@ -1444,6 +1444,7 @@ uint64_t kfd_get_number_elems(struct kfd_dev *kfd); /* Events */ extern const struct kfd_event_interrupt_class event_interrupt_class_cik; extern const struct kfd_event_interrupt_class event_interrupt_class_v9; +extern const struct kfd_event_interrupt_class event_interrupt_class_v9_4_3; extern const struct kfd_event_interrupt_class event_interrupt_class_v10; extern const struct kfd_event_interrupt_class event_interrupt_class_v11; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c index 3d3611705d41..a844e68211ac 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c @@ -2142,6 +2142,7 @@ void kfd_flush_tlb(struct kfd_process_device *pdd, enum TLB_FLUSH_TYPE type) int kfd_process_drain_interrupts(struct kfd_process_device *pdd) { uint32_t irq_drain_fence[8]; + uint8_t node_id = 0; int r = 0; if (!KFD_IS_SOC15(pdd->dev)) @@ -2154,6 +2155,14 @@ int kfd_process_drain_interrupts(struct kfd_process_device *pdd) KFD_IRQ_FENCE_CLIENTID; irq_drain_fence[3] = pdd->process->pasid; + /* + * For GFX 9.4.3, send the NodeId also in IH cookie DW[3] + */ + if (KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 4, 3)) { + node_id = ffs(pdd->dev->interrupt_bitmap) - 1; + irq_drain_fence[3] |= node_id << 16; + } + /* ensure stale irqs scheduled KFD interrupts and send drain fence. */ if (amdgpu_amdkfd_send_close_event_drain_irq(pdd->dev->adev, irq_drain_fence)) { diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c index 9ad1a2186a24..ba9d69054119 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c @@ -123,16 +123,24 @@ int pqm_set_gws(struct process_queue_manager *pqm, unsigned int qid, if (!gws && pdd->qpd.num_gws == 0) return -EINVAL; - if (gws) - ret = amdgpu_amdkfd_add_gws_to_process(pdd->process->kgd_process_info, - gws, &mem); - else - ret = amdgpu_amdkfd_remove_gws_from_process(pdd->process->kgd_process_info, - pqn->q->gws); - if (unlikely(ret)) - return ret; + if (KFD_GC_VERSION(dev) != IP_VERSION(9, 4, 3)) { + if (gws) + ret = amdgpu_amdkfd_add_gws_to_process(pdd->process->kgd_process_info, + gws, &mem); + else + ret = amdgpu_amdkfd_remove_gws_from_process(pdd->process->kgd_process_info, + pqn->q->gws); + if (unlikely(ret)) + return ret; + pqn->q->gws = mem; + } else { + /* + * Intentionally set GWS to a non-NULL value + * for GFX 9.4.3. + */ + pqn->q->gws = gws ? ERR_PTR(-ENOMEM) : NULL; + } - pqn->q->gws = mem; pdd->qpd.num_gws = gws ? dev->adev->gds.gws_size : 0; return pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm, @@ -164,7 +172,8 @@ void pqm_uninit(struct process_queue_manager *pqm) struct process_queue_node *pqn, *next; list_for_each_entry_safe(pqn, next, &pqm->queues, process_queue_list) { - if (pqn->q && pqn->q->gws) + if (pqn->q && pqn->q->gws && + KFD_GC_VERSION(pqn->q->device) != IP_VERSION(9, 4, 3)) amdgpu_amdkfd_remove_gws_from_process(pqm->process->kgd_process_info, pqn->q->gws); kfd_procfs_del_queue(pqn->q); @@ -446,8 +455,10 @@ int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid) } if (pqn->q->gws) { - amdgpu_amdkfd_remove_gws_from_process(pqm->process->kgd_process_info, - pqn->q->gws); + if (KFD_GC_VERSION(pqn->q->device) != IP_VERSION(9, 4, 3)) + amdgpu_amdkfd_remove_gws_from_process( + pqm->process->kgd_process_info, + pqn->q->gws); pdd->qpd.num_gws = 0; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c index 90b86a6ac7bd..61fc62f3e003 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c @@ -2107,6 +2107,10 @@ int kfd_topology_add_device(struct kfd_node *gpu) if (KFD_IS_SVM_API_SUPPORTED(dev->gpu->adev)) dev->node_props.capability |= HSA_CAP_SVMAPI_SUPPORTED; + if (dev->gpu->adev->gmc.is_app_apu || + dev->gpu->adev->gmc.xgmi.connected_to_cpu) + dev->node_props.capability |= HSA_CAP_FLAGS_COHERENTHOSTACCESS; + kfd_debug_print_topology(); kfd_notify_gpu_change(gpu_id, 1); diff --git a/drivers/gpu/drm/amd/amdkfd/soc15_int.h b/drivers/gpu/drm/amd/amdkfd/soc15_int.h index e3f3b0b93a59..10138676f27f 100644 --- a/drivers/gpu/drm/amd/amdkfd/soc15_int.h +++ b/drivers/gpu/drm/amd/amdkfd/soc15_int.h @@ -40,6 +40,7 @@ #define SOC15_VMID_FROM_IH_ENTRY(entry) (le32_to_cpu(entry[0]) >> 24 & 0xf) #define SOC15_VMID_TYPE_FROM_IH_ENTRY(entry) (le32_to_cpu(entry[0]) >> 31 & 0x1) #define SOC15_PASID_FROM_IH_ENTRY(entry) (le32_to_cpu(entry[3]) & 0xffff) +#define SOC15_NODEID_FROM_IH_ENTRY(entry) (le32_to_cpu(entry[3]) >> 16 & 0xff) #define SOC15_CONTEXT_ID0_FROM_IH_ENTRY(entry) (le32_to_cpu(entry[4])) #define SOC15_CONTEXT_ID1_FROM_IH_ENTRY(entry) (le32_to_cpu(entry[5])) #define SOC15_CONTEXT_ID2_FROM_IH_ENTRY(entry) (le32_to_cpu(entry[6])) |