diff options
Diffstat (limited to 'drivers/gpu/drm/amd/amdkfd/kfd_device.c')
| -rw-r--r-- | drivers/gpu/drm/amd/amdkfd/kfd_device.c | 182 |
1 files changed, 140 insertions, 42 deletions
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c index e9cfb80bd436..9a66ee661e57 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c @@ -57,6 +57,7 @@ extern const struct kfd2kgd_calls gfx_v10_kfd2kgd; extern const struct kfd2kgd_calls gfx_v10_3_kfd2kgd; extern const struct kfd2kgd_calls gfx_v11_kfd2kgd; extern const struct kfd2kgd_calls gfx_v12_kfd2kgd; +extern const struct kfd2kgd_calls gfx_v12_1_kfd2kgd; static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size, unsigned int chunk_size); @@ -94,6 +95,8 @@ static void kfd_device_info_set_sdma_info(struct kfd_dev *kfd) case IP_VERSION(5, 2, 2):/* NAVY_FLOUNDER */ case IP_VERSION(5, 2, 4):/* DIMGREY_CAVEFISH */ case IP_VERSION(5, 2, 5):/* BEIGE_GOBY */ + kfd->device_info.num_sdma_queues_per_engine = 8; + break; case IP_VERSION(6, 0, 0): case IP_VERSION(6, 0, 1): case IP_VERSION(6, 0, 2): @@ -102,9 +105,13 @@ static void kfd_device_info_set_sdma_info(struct kfd_dev *kfd) case IP_VERSION(6, 1, 1): case IP_VERSION(6, 1, 2): case IP_VERSION(6, 1, 3): + case IP_VERSION(6, 1, 4): case IP_VERSION(7, 0, 0): case IP_VERSION(7, 0, 1): + case IP_VERSION(7, 1, 0): kfd->device_info.num_sdma_queues_per_engine = 8; + /* Reserve 1 for paging and 1 for gfx */ + kfd->device_info.num_reserved_sdma_queues_per_engine = 2; break; default: dev_warn(kfd_device, @@ -112,30 +119,6 @@ static void kfd_device_info_set_sdma_info(struct kfd_dev *kfd) sdma_version); kfd->device_info.num_sdma_queues_per_engine = 8; } - - bitmap_zero(kfd->device_info.reserved_sdma_queues_bitmap, KFD_MAX_SDMA_QUEUES); - - switch (sdma_version) { - case IP_VERSION(6, 0, 0): - case IP_VERSION(6, 0, 1): - case IP_VERSION(6, 0, 2): - case IP_VERSION(6, 0, 3): - case IP_VERSION(6, 1, 0): - case IP_VERSION(6, 1, 1): - case IP_VERSION(6, 1, 2): - case IP_VERSION(6, 1, 3): - case IP_VERSION(7, 0, 0): - case IP_VERSION(7, 0, 1): - /* Reserve 1 for paging and 1 for gfx */ - kfd->device_info.num_reserved_sdma_queues_per_engine = 2; - /* BIT(0)=engine-0 queue-0; BIT(1)=engine-1 queue-0; BIT(2)=engine-0 queue-1; ... */ - bitmap_set(kfd->device_info.reserved_sdma_queues_bitmap, 0, - kfd->adev->sdma.num_instances * - kfd->device_info.num_reserved_sdma_queues_per_engine); - break; - default: - break; - } } static void kfd_device_info_set_event_interrupt_class(struct kfd_dev *kfd) @@ -183,6 +166,7 @@ static void kfd_device_info_set_event_interrupt_class(struct kfd_dev *kfd) case IP_VERSION(11, 5, 1): case IP_VERSION(11, 5, 2): case IP_VERSION(11, 5, 3): + case IP_VERSION(11, 5, 4): kfd->device_info.event_interrupt_class = &event_interrupt_class_v11; break; case IP_VERSION(12, 0, 0): @@ -190,6 +174,10 @@ static void kfd_device_info_set_event_interrupt_class(struct kfd_dev *kfd) /* GFX12_TODO: Change to v12 version. */ kfd->device_info.event_interrupt_class = &event_interrupt_class_v11; break; + case IP_VERSION(12, 1, 0): + kfd->device_info.event_interrupt_class = + &event_interrupt_class_v12_1; + break; default: dev_warn(kfd_device, "v9 event interrupt handler is set due to " "mismatch of gc ip block(GC_HWIP:0x%x).\n", gc_version); @@ -456,6 +444,10 @@ struct kfd_dev *kgd2kfd_probe(struct amdgpu_device *adev, bool vf) gfx_target_version = 110503; f2g = &gfx_v11_kfd2kgd; break; + case IP_VERSION(11, 5, 4): + gfx_target_version = 110504; + f2g = &gfx_v11_kfd2kgd; + break; case IP_VERSION(12, 0, 0): gfx_target_version = 120000; f2g = &gfx_v12_kfd2kgd; @@ -464,6 +456,10 @@ struct kfd_dev *kgd2kfd_probe(struct amdgpu_device *adev, bool vf) gfx_target_version = 120001; f2g = &gfx_v12_kfd2kgd; break; + case IP_VERSION(12, 1, 0): + gfx_target_version = 120500; + f2g = &gfx_v12_1_kfd2kgd; + break; default: break; } @@ -549,11 +545,16 @@ static void kfd_cwsr_init(struct kfd_dev *kfd) BUILD_BUG_ON(sizeof(cwsr_trap_gfx11_hex) > PAGE_SIZE); kfd->cwsr_isa = cwsr_trap_gfx11_hex; kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx11_hex); - } else { + } else if (KFD_GC_VERSION(kfd) < IP_VERSION(12, 1, 0)) { BUILD_BUG_ON(sizeof(cwsr_trap_gfx12_hex) > KFD_CWSR_TMA_OFFSET); kfd->cwsr_isa = cwsr_trap_gfx12_hex; kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx12_hex); + } else { + BUILD_BUG_ON(sizeof(cwsr_trap_gfx12_1_0_hex) + > KFD_CWSR_TMA_OFFSET); + kfd->cwsr_isa = cwsr_trap_gfx12_1_0_hex; + kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx12_1_0_hex); } kfd->cwsr_enabled = true; @@ -682,6 +683,7 @@ static void kfd_setup_interrupt_bitmap(struct kfd_node *node, struct amdgpu_device *adev = node->adev; uint32_t xcc_mask = node->xcc_mask; uint32_t xcc, mapped_xcc; + uint32_t bitmap; /* * Interrupt bitmap is setup for processing interrupts from * different XCDs and AIDs. @@ -703,9 +705,22 @@ static void kfd_setup_interrupt_bitmap(struct kfd_node *node, * - AND VMID reported in the interrupt lies within the * VMID range of the node. */ - for_each_inst(xcc, xcc_mask) { - mapped_xcc = GET_INST(GC, xcc); - node->interrupt_bitmap |= (mapped_xcc % 2 ? 5 : 3) << (4 * (mapped_xcc / 2)); + switch (KFD_GC_VERSION(node)) { + case IP_VERSION(12, 1, 0): + for_each_inst(xcc, xcc_mask) { + mapped_xcc = GET_INST(GC, xcc); + bitmap = 0x2 | (0x4 << (mapped_xcc % 4)); + if (mapped_xcc/4) + bitmap = bitmap << 8; + node->interrupt_bitmap |= bitmap; + } + break; + default: + for_each_inst(xcc, xcc_mask) { + mapped_xcc = GET_INST(GC, xcc); + node->interrupt_bitmap |= (mapped_xcc % 2 ? 5 : 3) << (4 * (mapped_xcc / 2)); + } + break; } dev_info(kfd_device, "Node: %d, interrupt_bitmap: %x\n", kfd_node_idx, node->interrupt_bitmap); @@ -768,7 +783,7 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd, * If the VMID range changes for multi-partition capable GPUs, then * this code MUST be revisited. */ - if (kfd->adev->xcp_mgr) { + if (kfd->adev->xcp_mgr && (KFD_GC_VERSION(kfd) != IP_VERSION(12, 1, 0))) { partition_mode = amdgpu_xcp_query_partition_mode(kfd->adev->xcp_mgr, AMDGPU_XCP_FL_LOCKED); if (partition_mode == AMDGPU_CPX_PARTITION_MODE && @@ -805,12 +820,13 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd, /* add another 512KB for all other allocations on gart (HPD, fences) */ size += 512 * 1024; - if (amdgpu_amdkfd_alloc_gtt_mem( - kfd->adev, size, &kfd->gtt_mem, + if (amdgpu_amdkfd_alloc_kernel_mem( + kfd->adev, size, AMDGPU_GEM_DOMAIN_GTT, + &kfd->gtt_mem, &kfd->gtt_start_gpu_addr, &kfd->gtt_start_cpu_ptr, false)) { dev_err(kfd_device, "Could not allocate %d bytes\n", size); - goto alloc_gtt_mem_failure; + goto alloc_kernel_mem_failure; } dev_info(kfd_device, "Allocated %d bytes on gart\n", size); @@ -875,7 +891,8 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd, } if (partition_mode == AMDGPU_CPX_PARTITION_MODE && - kfd->num_nodes != 1) { + kfd->num_nodes != 1 && + (KFD_GC_VERSION(kfd) != IP_VERSION(12, 1, 0))) { /* For multi-partition capable GPUs and CPX mode, first * XCD gets VMID range 4-9 and second XCD gets VMID * range 10-15. @@ -896,6 +913,7 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd, node->compute_vmid_bitmap = gpu_resources->compute_vmid_bitmap; } + node->max_proc_per_quantum = max_proc_per_quantum; atomic_set(&node->sram_ecc_flag, 0); @@ -934,8 +952,8 @@ node_alloc_error: kfd_doorbell_error: kfd_gtt_sa_fini(kfd); kfd_gtt_sa_init_error: - amdgpu_amdkfd_free_gtt_mem(kfd->adev, &kfd->gtt_mem); -alloc_gtt_mem_failure: + amdgpu_amdkfd_free_kernel_mem(kfd->adev, &kfd->gtt_mem); +alloc_kernel_mem_failure: dev_err(kfd_device, "device %x:%x NOT added due to errors\n", kfd->adev->pdev->vendor, kfd->adev->pdev->device); @@ -952,10 +970,13 @@ void kgd2kfd_device_exit(struct kfd_dev *kfd) kfd_doorbell_fini(kfd); ida_destroy(&kfd->doorbell_ida); kfd_gtt_sa_fini(kfd); - amdgpu_amdkfd_free_gtt_mem(kfd->adev, &kfd->gtt_mem); + amdgpu_amdkfd_free_kernel_mem(kfd->adev, &kfd->gtt_mem); } kfree(kfd); + + /* after remove a kfd device unlock kfd driver */ + kgd2kfd_unlock_kfd(NULL); } int kgd2kfd_pre_reset(struct kfd_dev *kfd, @@ -1059,7 +1080,7 @@ void kgd2kfd_suspend(struct kfd_dev *kfd, bool suspend_proc) int kgd2kfd_resume(struct kfd_dev *kfd, bool resume_proc) { - int ret, i; + int ret = 0, i; if (!kfd->init_complete) return 0; @@ -1202,12 +1223,13 @@ int kgd2kfd_resume_mm(struct mm_struct *mm) * prepare for safe eviction of KFD BOs that belong to the specified * process. * - * @mm: mm_struct that identifies the specified KFD process + * @mm: mm_struct that identifies a group of KFD processes + * @context_id: an id that identifies a specific KFD context in the above kfd process group * @fence: eviction fence attached to KFD process BOs * */ int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm, - struct dma_fence *fence) + u16 context_id, struct dma_fence *fence) { struct kfd_process *p; unsigned long active_time; @@ -1219,7 +1241,7 @@ int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm, if (dma_fence_is_signaled(fence)) return 0; - p = kfd_lookup_process_by_mm(mm); + p = kfd_lookup_process_by_id(mm, context_id); if (!p) return -ENODEV; @@ -1539,10 +1561,14 @@ out: return r; } +/* unlock a kfd dev or kfd driver */ void kgd2kfd_unlock_kfd(struct kfd_dev *kfd) { mutex_lock(&kfd_processes_mutex); - --kfd->kfd_dev_lock; + if (kfd) + --kfd->kfd_dev_lock; + else + --kfd_locked; mutex_unlock(&kfd_processes_mutex); } @@ -1681,6 +1707,10 @@ bool kgd2kfd_vmfault_fast_path(struct amdgpu_device *adev, struct amdgpu_iv_entr { struct kfd_process *p; u32 cam_index; + u32 src_data_idx; + + src_data_idx = (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0)) ? + 3 : 2; if (entry->ih == &adev->irq.ih_soft || entry->ih == &adev->irq.ih1) { p = kfd_lookup_process_by_pasid(entry->pasid, NULL); @@ -1689,7 +1719,8 @@ bool kgd2kfd_vmfault_fast_path(struct amdgpu_device *adev, struct amdgpu_iv_entr if (p->gpu_page_fault && !p->debug_trap_enabled) { if (retry_fault && adev->irq.retry_cam_enabled) { - cam_index = entry->src_data[2] & 0x3ff; + cam_index = entry->src_data[src_data_idx] & 0x3ff; + WDOORBELL32(adev->irq.retry_cam_doorbell_index, cam_index); } @@ -1706,6 +1737,73 @@ bool kgd2kfd_vmfault_fast_path(struct amdgpu_device *adev, struct amdgpu_iv_entr return false; } +/* check if there is kfd process still uses adev */ +static bool kgd2kfd_check_device_idle(struct amdgpu_device *adev) +{ + struct kfd_process *p; + struct hlist_node *p_temp; + unsigned int temp; + struct kfd_node *dev; + + mutex_lock(&kfd_processes_mutex); + + if (hash_empty(kfd_processes_table)) { + mutex_unlock(&kfd_processes_mutex); + return true; + } + + /* check if there is device still use adev */ + hash_for_each_safe(kfd_processes_table, temp, p_temp, p, kfd_processes) { + for (int i = 0; i < p->n_pdds; i++) { + dev = p->pdds[i]->dev; + if (dev->adev == adev) { + mutex_unlock(&kfd_processes_mutex); + return false; + } + } + } + + mutex_unlock(&kfd_processes_mutex); + + return true; +} + +/** kgd2kfd_teardown_processes - gracefully tear down existing + * kfd processes that use adev + * + * @adev: amdgpu_device where kfd processes run on and will be + * teardown + * + */ +void kgd2kfd_teardown_processes(struct amdgpu_device *adev) +{ + struct hlist_node *p_temp; + struct kfd_process *p; + struct kfd_node *dev; + unsigned int temp; + + mutex_lock(&kfd_processes_mutex); + + if (hash_empty(kfd_processes_table)) { + mutex_unlock(&kfd_processes_mutex); + return; + } + + hash_for_each_safe(kfd_processes_table, temp, p_temp, p, kfd_processes) { + for (int i = 0; i < p->n_pdds; i++) { + dev = p->pdds[i]->dev; + if (dev->adev == adev) + kfd_signal_process_terminate_event(p); + } + } + + mutex_unlock(&kfd_processes_mutex); + + /* wait all kfd processes use adev terminate */ + while (!kgd2kfd_check_device_idle(adev)) + cond_resched(); +} + #if defined(CONFIG_DEBUG_FS) /* This function will send a package to HIQ to hang the HWS |