diff options
Diffstat (limited to 'drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c')
| -rw-r--r-- | drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c | 637 |
1 files changed, 637 insertions, 0 deletions
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c new file mode 100644 index 000000000000..ef6e550ce7c3 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c @@ -0,0 +1,637 @@ +/* + * Copyright 2025 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#include "amdgpu.h" +#include "gmc_v12_1.h" +#include "soc15_common.h" +#include "soc_v1_0_enum.h" +#include "oss/osssys_7_1_0_offset.h" +#include "oss/osssys_7_1_0_sh_mask.h" +#include "ivsrcid/vmc/irqsrcs_vmc_1_0.h" + +static int gmc_v12_1_vm_fault_interrupt_state(struct amdgpu_device *adev, + struct amdgpu_irq_src *src, + unsigned int type, + enum amdgpu_interrupt_state state) +{ + struct amdgpu_vmhub *hub; + u32 tmp, reg, i, j; + + switch (state) { + case AMDGPU_IRQ_STATE_DISABLE: + for_each_set_bit(j, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) { + hub = &adev->vmhub[j]; + for (i = 0; i < 16; i++) { + reg = hub->vm_context0_cntl + i; + + /* This works because this interrupt is only + * enabled at init/resume and disabled in + * fini/suspend, so the overall state doesn't + * change over the course of suspend/resume. + */ + if (adev->in_s0ix && (j == AMDGPU_GFXHUB(0))) + continue; + + if (j >= AMDGPU_MMHUB0(0)) + tmp = RREG32_SOC15_IP(MMHUB, reg); + else + tmp = RREG32_XCC(reg, j); + + tmp &= ~hub->vm_cntx_cntl_vm_fault; + + if (j >= AMDGPU_MMHUB0(0)) + WREG32_SOC15_IP(MMHUB, reg, tmp); + else + WREG32_XCC(reg, tmp, j); + } + } + break; + case AMDGPU_IRQ_STATE_ENABLE: + for_each_set_bit(j, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) { + hub = &adev->vmhub[j]; + for (i = 0; i < 16; i++) { + reg = hub->vm_context0_cntl + i; + + /* This works because this interrupt is only + * enabled at init/resume and disabled in + * fini/suspend, so the overall state doesn't + * change over the course of suspend/resume. + */ + if (adev->in_s0ix && (j == AMDGPU_GFXHUB(0))) + continue; + + if (j >= AMDGPU_MMHUB0(0)) + tmp = RREG32_SOC15_IP(MMHUB, reg); + else + tmp = RREG32_XCC(reg, j); + + tmp |= hub->vm_cntx_cntl_vm_fault; + + if (j >= AMDGPU_MMHUB0(0)) + WREG32_SOC15_IP(MMHUB, reg, tmp); + else + WREG32_XCC(reg, tmp, j); + } + } + break; + default: + break; + } + + return 0; +} + +static int gmc_v12_1_process_interrupt(struct amdgpu_device *adev, + struct amdgpu_irq_src *source, + struct amdgpu_iv_entry *entry) +{ + struct amdgpu_task_info *task_info; + bool retry_fault = false, write_fault = false; + unsigned int vmhub, node_id; + struct amdgpu_vmhub *hub; + uint32_t cam_index = 0; + const char *hub_name; + int ret, xcc_id = 0; + uint32_t status = 0; + u64 addr; + + node_id = entry->node_id; + + addr = (u64)entry->src_data[0] << 12; + addr |= ((u64)entry->src_data[1] & 0x1fff) << 44; + + if (entry->src_id == UTCL2_1_0__SRCID__RETRY) { + retry_fault = true; + write_fault = !!(entry->src_data[1] & 0x200000); + } + + if (entry->client_id == SOC_V1_0_IH_CLIENTID_VMC) { + hub_name = "mmhub0"; + vmhub = AMDGPU_MMHUB0(node_id / 4); + } else { + hub_name = "gfxhub0"; + if (adev->gfx.funcs->ih_node_to_logical_xcc) { + xcc_id = adev->gfx.funcs->ih_node_to_logical_xcc(adev, + node_id); + if (xcc_id < 0) + xcc_id = 0; + } + vmhub = xcc_id; + } + + hub = &adev->vmhub[vmhub]; + + if (retry_fault) { + if (adev->irq.retry_cam_enabled) { + /* Delegate it to a different ring if the hardware hasn't + * already done it. + */ + if (entry->ih == &adev->irq.ih) { + amdgpu_irq_delegate(adev, entry, 8); + return 1; + } + + cam_index = entry->src_data[3] & 0x3ff; + + ret = amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id, + addr, entry->timestamp, write_fault); + WDOORBELL32(adev->irq.retry_cam_doorbell_index, cam_index); + if (ret) + return 1; + } else { + /* Process it onyl if it's the first fault for this address */ + if (entry->ih != &adev->irq.ih_soft && + amdgpu_gmc_filter_faults(adev, entry->ih, addr, entry->pasid, + entry->timestamp)) + return 1; + + /* Delegate it to a different ring if the hardware hasn't + * already done it. + */ + if (entry->ih == &adev->irq.ih) { + amdgpu_irq_delegate(adev, entry, 8); + return 1; + } + + /* Try to handle the recoverable page faults by filling page + * tables + */ + if (amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id, + addr, entry->timestamp, write_fault)) + return 1; + } + } + + if (kgd2kfd_vmfault_fast_path(adev, entry, retry_fault)) + return 1; + + if (!printk_ratelimit()) + return 0; + + dev_err(adev->dev, + "[%s] %s page fault (src_id:%u ring:%u vmid:%u pasid:%u)\n", hub_name, + retry_fault ? "retry" : "no-retry", + entry->src_id, entry->ring_id, entry->vmid, entry->pasid); + + task_info = amdgpu_vm_get_task_info_pasid(adev, entry->pasid); + if (task_info) { + amdgpu_vm_print_task_info(adev, task_info); + amdgpu_vm_put_task_info(task_info); + } + + dev_err(adev->dev, " in page starting at address 0x%016llx from IH client %d (%s)\n", + addr, entry->client_id, soc_v1_0_ih_clientid_name[entry->client_id]); + + if (amdgpu_sriov_vf(adev)) + return 0; + + /* + * Issue a dummy read to wait for the status register to + * be updated to avoid reading an incorrect value due to + * the new fast GRBM interface. + */ + if (entry->vmid_src == AMDGPU_GFXHUB(0)) + RREG32(hub->vm_l2_pro_fault_status); + + status = RREG32(hub->vm_l2_pro_fault_status); + + /* Only print L2 fault status if the status register could be read and + * contains useful information + */ + if (!status) + return 0; + + WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1); + + amdgpu_vm_update_fault_cache(adev, entry->pasid, addr, status, vmhub); + + hub->vmhub_funcs->print_l2_protection_fault_status(adev, status); + + return 0; +} + +static bool gmc_v12_1_get_vmid_pasid_mapping_info(struct amdgpu_device *adev, + uint8_t vmid, uint8_t inst, + uint16_t *p_pasid) +{ + uint16_t index; + + if (inst/4) + index = 0xA + inst%4; + else + index = 0x2 + inst%4; + + WREG32(SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_LUT_INDEX), index); + + *p_pasid = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT) + vmid) & 0xffff; + + return !!(*p_pasid); +} + +/* + * GART + * VMID 0 is the physical GPU addresses as used by the kernel. + * VMIDs 1-15 are used for userspace clients and are handled + * by the amdgpu vm/hsa code. + */ + +static void gmc_v12_1_flush_vm_hub(struct amdgpu_device *adev, uint32_t vmid, + unsigned int vmhub, uint32_t flush_type) +{ + struct amdgpu_vmhub *hub = &adev->vmhub[vmhub]; + u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid, flush_type); + u32 tmp; + /* Use register 17 for GART */ + const unsigned eng = 17; + unsigned int i; + unsigned char hub_ip = 0; + + hub_ip = (AMDGPU_IS_GFXHUB(vmhub)) ? + GC_HWIP : MMHUB_HWIP; + + spin_lock(&adev->gmc.invalidate_lock); + + WREG32_RLC_NO_KIQ(hub->vm_inv_eng0_req + hub->eng_distance * eng, inv_req, hub_ip); + + /* Wait for ACK with a delay.*/ + for (i = 0; i < adev->usec_timeout; i++) { + tmp = RREG32_RLC_NO_KIQ(hub->vm_inv_eng0_ack + + hub->eng_distance * eng, hub_ip); + tmp &= 1 << vmid; + if (tmp) + break; + + udelay(1); + } + + /* Issue additional private vm invalidation to MMHUB */ + if (!AMDGPU_IS_GFXHUB(vmhub) && + (hub->vm_l2_bank_select_reserved_cid2) && + !amdgpu_sriov_vf(adev)) { + inv_req = RREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2); + /* bit 25: RSERVED_CACHE_PRIVATE_INVALIDATION */ + inv_req |= (1 << 25); + /* Issue private invalidation */ + WREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2, inv_req); + /* Read back to ensure invalidation is done*/ + RREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2); + } + + spin_unlock(&adev->gmc.invalidate_lock); + + if (i < adev->usec_timeout) + return; + + dev_err(adev->dev, "Timeout waiting for VM flush ACK!\n"); +} + +/** + * gmc_v12_1_flush_gpu_tlb - gart tlb flush callback + * + * @adev: amdgpu_device pointer + * @vmid: vm instance to flush + * @vmhub: which hub to flush + * @flush_type: the flush type + * + * Flush the TLB for the requested page table. + */ +static void gmc_v12_1_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, + uint32_t vmhub, uint32_t flush_type) +{ + u32 inst; + + if (AMDGPU_IS_GFXHUB(vmhub) && + !adev->gfx.is_poweron) + return; + + if (vmhub >= AMDGPU_MMHUB0(0)) + inst = 0; + else + inst = vmhub; + + /* This is necessary for SRIOV as well as for GFXOFF to function + * properly under bare metal + */ + if (((adev->gfx.kiq[inst].ring.sched.ready || + adev->mes.ring[MES_PIPE_INST(inst, 0)].sched.ready) && + (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)))) { + struct amdgpu_vmhub *hub = &adev->vmhub[vmhub]; + const unsigned eng = 17; + u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid, flush_type); + u32 req = hub->vm_inv_eng0_req + hub->eng_distance * eng; + u32 ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng; + + amdgpu_gmc_fw_reg_write_reg_wait(adev, req, ack, inv_req, + 1 << vmid, inst); + return; + } + + mutex_lock(&adev->mman.gtt_window_lock); + gmc_v12_1_flush_vm_hub(adev, vmid, vmhub, 0); + mutex_unlock(&adev->mman.gtt_window_lock); + return; +} + +/** + * gmc_v12_1_flush_gpu_tlb_pasid - tlb flush via pasid + * + * @adev: amdgpu_device pointer + * @pasid: pasid to be flush + * @flush_type: the flush type + * @all_hub: flush all hubs + * @inst: is used to select which instance of KIQ to use for the invalidation + * + * Flush the TLB for the requested pasid. + */ +static void gmc_v12_1_flush_gpu_tlb_pasid(struct amdgpu_device *adev, + uint16_t pasid, uint32_t flush_type, + bool all_hub, uint32_t inst) +{ + uint16_t queried; + int vmid, i; + + if (adev->enable_uni_mes && adev->mes.ring[0].sched.ready && + (adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x6f) { + struct mes_inv_tlbs_pasid_input input = {0}; + input.xcc_id = inst; + input.pasid = pasid; + input.flush_type = flush_type; + + /* MES will invalidate hubs for the device(including slave xcc) from master, ignore request from slave */ + if (!amdgpu_gfx_is_master_xcc(adev, inst)) + return; + + input.hub_id = AMDGPU_GFXHUB(0); + adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input); + + if (all_hub) { + /* invalidate mm_hub */ + if (test_bit(AMDGPU_MMHUB1(0), adev->vmhubs_mask)) { + input.hub_id = AMDGPU_MMHUB0(0); + adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input); + } + if (test_bit(AMDGPU_MMHUB1(0), adev->vmhubs_mask)) { + input.hub_id = AMDGPU_MMHUB1(0); + adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input); + } + } + return; + } + + for (vmid = 1; vmid < 16; vmid++) { + bool valid; + + valid = gmc_v12_1_get_vmid_pasid_mapping_info(adev, vmid, inst, + &queried); + if (!valid || queried != pasid) + continue; + + if (all_hub) { + for_each_set_bit(i, adev->vmhubs_mask, + AMDGPU_MAX_VMHUBS) + gmc_v12_1_flush_gpu_tlb(adev, vmid, i, + flush_type); + } else { + gmc_v12_1_flush_gpu_tlb(adev, vmid, AMDGPU_GFXHUB(inst), + flush_type); + } + } +} + +static uint64_t gmc_v12_1_emit_flush_gpu_tlb(struct amdgpu_ring *ring, + unsigned vmid, uint64_t pd_addr) +{ + struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub]; + uint32_t req = hub->vmhub_funcs->get_invalidate_req(vmid, 0); + unsigned eng = ring->vm_inv_eng; + + amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 + + (hub->ctx_addr_distance * vmid), + lower_32_bits(pd_addr)); + + amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 + + (hub->ctx_addr_distance * vmid), + upper_32_bits(pd_addr)); + + amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req + + hub->eng_distance * eng, + hub->vm_inv_eng0_ack + + hub->eng_distance * eng, + req, 1 << vmid); + + return pd_addr; +} + +static void gmc_v12_1_emit_pasid_mapping(struct amdgpu_ring *ring, + unsigned vmid, unsigned pasid) +{ + struct amdgpu_device *adev = ring->adev; + uint32_t reg; + + if (ring->vm_hub == AMDGPU_GFXHUB(0)) + reg = SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT) + vmid; + else + reg = SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT_MM) + vmid; + + amdgpu_ring_emit_wreg(ring, reg, pasid); +} + +/* + * PTE format: + * 63 P + * 62:59 reserved + * 58 D + * 57 G + * 56 T + * 55:54 M + * 53:52 SW + * 51:48 reserved for future + * 47:12 4k physical page base address + * 11:7 fragment + * 6 write + * 5 read + * 4 exe + * 3 Z + * 2 snooped + * 1 system + * 0 valid + * + * PDE format: + * 63 P + * 62:58 block fragment size + * 57 reserved + * 56 A + * 55:54 M + * 53:52 reserved + * 51:48 reserved for future + * 47:6 physical base address of PD or PTE + * 5:3 reserved + * 2 C + * 1 system + * 0 valid + */ + +static void gmc_v12_1_get_vm_pde(struct amdgpu_device *adev, int level, + uint64_t *addr, uint64_t *flags) +{ + if (!(*flags & AMDGPU_PDE_PTE_GFX12) && !(*flags & AMDGPU_PTE_SYSTEM)) + *addr = adev->vm_manager.vram_base_offset + *addr - + adev->gmc.vram_start; + BUG_ON(*addr & 0xFFFF00000000003FULL); + + *flags |= AMDGPU_PTE_SNOOPED; + + if (!adev->gmc.translate_further) + return; + + if (level == AMDGPU_VM_PDB1) { + /* Set the block fragment size */ + if (!(*flags & AMDGPU_PDE_PTE_GFX12)) + *flags |= AMDGPU_PDE_BFS_GFX12(0x9); + + } else if (level == AMDGPU_VM_PDB0) { + if (*flags & AMDGPU_PDE_PTE_GFX12) + *flags &= ~AMDGPU_PDE_PTE_GFX12; + } +} + +static void gmc_v12_1_get_coherence_flags(struct amdgpu_device *adev, + struct amdgpu_bo *bo, + uint64_t *flags) +{ + struct amdgpu_device *bo_adev = amdgpu_ttm_adev(bo->tbo.bdev); + bool is_vram = bo->tbo.resource && + bo->tbo.resource->mem_type == TTM_PL_VRAM; + bool coherent = bo->flags & (AMDGPU_GEM_CREATE_COHERENT | + AMDGPU_GEM_CREATE_EXT_COHERENT); + bool ext_coherent = bo->flags & AMDGPU_GEM_CREATE_EXT_COHERENT; + uint32_t gc_ip_version = amdgpu_ip_version(adev, GC_HWIP, 0); + bool uncached = bo->flags & AMDGPU_GEM_CREATE_UNCACHED; + unsigned int mtype, mtype_local; + bool snoop = false; + bool is_local = false; + + switch (gc_ip_version) { + case IP_VERSION(12, 1, 0): + mtype_local = MTYPE_RW; + if (amdgpu_mtype_local == 1) { + DRM_INFO_ONCE("Using MTYPE_NC for local memory\n"); + mtype_local = MTYPE_NC; + } else if (amdgpu_mtype_local == 2) { + DRM_INFO_ONCE("MTYPE_CC not supported, using MTYPE_RW instead for local memory\n"); + } else { + DRM_INFO_ONCE("Using MTYPE_RW for local memory\n"); + } + + is_local = (is_vram && adev == bo_adev); + snoop = true; + if (uncached) { + mtype = MTYPE_UC; + } else if (ext_coherent) { + mtype = is_local ? mtype_local : MTYPE_UC; + } else { + if (is_local) + mtype = mtype_local; + else + mtype = MTYPE_NC; + } + break; + default: + if (uncached || coherent) + mtype = MTYPE_UC; + else + mtype = MTYPE_NC; + } + + if (mtype != MTYPE_NC) + *flags = AMDGPU_PTE_MTYPE_GFX12(*flags, mtype); + + if (is_local || adev->have_atomics_support) + *flags |= AMDGPU_PTE_BUS_ATOMICS; + + *flags |= snoop ? AMDGPU_PTE_SNOOPED : 0; +} + +static void gmc_v12_1_get_vm_pte(struct amdgpu_device *adev, + struct amdgpu_vm *vm, + struct amdgpu_bo *bo, + uint32_t vm_flags, + uint64_t *flags) +{ + if (vm_flags & AMDGPU_VM_PAGE_EXECUTABLE) + *flags |= AMDGPU_PTE_EXECUTABLE; + else + *flags &= ~AMDGPU_PTE_EXECUTABLE; + + switch (vm_flags & AMDGPU_VM_MTYPE_MASK) { + case AMDGPU_VM_MTYPE_DEFAULT: + *flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_NC); + break; + case AMDGPU_VM_MTYPE_NC: + default: + *flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_NC); + break; + case AMDGPU_VM_MTYPE_RW: + *flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_RW); + break; + case AMDGPU_VM_MTYPE_UC: + *flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_UC); + break; + } + + if ((*flags & AMDGPU_PTE_VALID) && bo) + gmc_v12_1_get_coherence_flags(adev, bo, flags); +} + +static const struct amdgpu_gmc_funcs gmc_v12_1_gmc_funcs = { + .flush_gpu_tlb = gmc_v12_1_flush_gpu_tlb, + .flush_gpu_tlb_pasid = gmc_v12_1_flush_gpu_tlb_pasid, + .emit_flush_gpu_tlb = gmc_v12_1_emit_flush_gpu_tlb, + .emit_pasid_mapping = gmc_v12_1_emit_pasid_mapping, + .get_vm_pde = gmc_v12_1_get_vm_pde, + .get_vm_pte = gmc_v12_1_get_vm_pte, + .query_mem_partition_mode = &amdgpu_gmc_query_memory_partition, + .request_mem_partition_mode = &amdgpu_gmc_request_memory_partition, +}; + +void gmc_v12_1_set_gmc_funcs(struct amdgpu_device *adev) +{ + adev->gmc.gmc_funcs = &gmc_v12_1_gmc_funcs; +} + +static const struct amdgpu_irq_src_funcs gmc_v12_1_irq_funcs = { + .set = gmc_v12_1_vm_fault_interrupt_state, + .process = gmc_v12_1_process_interrupt, +}; + +void gmc_v12_1_set_irq_funcs(struct amdgpu_device *adev) +{ + adev->gmc.vm_fault.num_types = 1; + adev->gmc.vm_fault.funcs = &gmc_v12_1_irq_funcs; +} + +void gmc_v12_1_init_vram_info(struct amdgpu_device *adev) +{ + /* TODO: query vram_info from ip discovery binary */ + adev->gmc.vram_type = AMDGPU_VRAM_TYPE_HBM4; + adev->gmc.vram_width = 384 * 64; +} |