diff options
Diffstat (limited to 'drivers/gpu/drm/xe/xe_device.c')
| -rw-r--r-- | drivers/gpu/drm/xe/xe_device.c | 255 |
1 files changed, 167 insertions, 88 deletions
diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index cf29e259861f..1581f4dab69d 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -8,7 +8,6 @@ #include <linux/aperture.h> #include <linux/delay.h> #include <linux/fault-inject.h> -#include <linux/iopoll.h> #include <linux/units.h> #include <drm/drm_atomic_helper.h> @@ -16,6 +15,7 @@ #include <drm/drm_gem_ttm_helper.h> #include <drm/drm_ioctl.h> #include <drm/drm_managed.h> +#include <drm/drm_pagemap_util.h> #include <drm/drm_print.h> #include <uapi/drm/xe_drm.h> @@ -35,7 +35,6 @@ #include "xe_exec_queue.h" #include "xe_force_wake.h" #include "xe_ggtt.h" -#include "xe_gsc_proxy.h" #include "xe_gt.h" #include "xe_gt_mcr.h" #include "xe_gt_printk.h" @@ -61,8 +60,10 @@ #include "xe_pxp.h" #include "xe_query.h" #include "xe_shrinker.h" +#include "xe_soc_remapper.h" #include "xe_survivability_mode.h" #include "xe_sriov.h" +#include "xe_svm.h" #include "xe_tile.h" #include "xe_ttm_stolen_mgr.h" #include "xe_ttm_sys_mgr.h" @@ -166,7 +167,7 @@ static void xe_file_close(struct drm_device *dev, struct drm_file *file) struct xe_exec_queue *q; unsigned long idx; - xe_pm_runtime_get(xe); + guard(xe_pm_runtime)(xe); /* * No need for exec_queue.lock here as there is no contention for it @@ -184,8 +185,6 @@ static void xe_file_close(struct drm_device *dev, struct drm_file *file) xe_vm_close_and_put(vm); xe_file_put(xef); - - xe_pm_runtime_put(xe); } static const struct drm_ioctl_desc xe_ioctls[] = { @@ -209,6 +208,8 @@ static const struct drm_ioctl_desc xe_ioctls[] = { DRM_IOCTL_DEF_DRV(XE_MADVISE, xe_vm_madvise_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(XE_VM_QUERY_MEM_RANGE_ATTRS, xe_vm_query_vmas_attrs_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_SET_PROPERTY, xe_exec_queue_set_property_ioctl, + DRM_RENDER_ALLOW), }; static long xe_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg) @@ -220,10 +221,10 @@ static long xe_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg) if (xe_device_wedged(xe)) return -ECANCELED; - ret = xe_pm_runtime_get_ioctl(xe); + ACQUIRE(xe_pm_runtime_ioctl, pm)(xe); + ret = ACQUIRE_ERR(xe_pm_runtime_ioctl, &pm); if (ret >= 0) ret = drm_ioctl(file, cmd, arg); - xe_pm_runtime_put(xe); return ret; } @@ -238,10 +239,10 @@ static long xe_drm_compat_ioctl(struct file *file, unsigned int cmd, unsigned lo if (xe_device_wedged(xe)) return -ECANCELED; - ret = xe_pm_runtime_get_ioctl(xe); + ACQUIRE(xe_pm_runtime_ioctl, pm)(xe); + ret = ACQUIRE_ERR(xe_pm_runtime_ioctl, &pm); if (ret >= 0) ret = drm_compat_ioctl(file, cmd, arg); - xe_pm_runtime_put(xe); return ret; } @@ -371,6 +372,20 @@ static const struct file_operations xe_driver_fops = { .fop_flags = FOP_UNSIGNED_OFFSET, }; +/** + * xe_is_xe_file() - Is the file an xe device file? + * @file: The file. + * + * Checks whether the file is opened against + * an xe device. + * + * Return: %true if an xe file, %false if not. + */ +bool xe_is_xe_file(const struct file *file) +{ + return file->f_op == &xe_driver_fops; +} + static struct drm_driver driver = { /* Don't use MTRRs here; the Xserver or userspace app should * deal with them for Intel hardware. @@ -455,6 +470,7 @@ struct xe_device *xe_device_create(struct pci_dev *pdev, xe->info.revid = pdev->revision; xe->info.force_execlist = xe_modparam.force_execlist; xe->atomic_svm_timeslice_ms = 5; + xe->min_run_period_lr_ms = 5; err = xe_irq_init(xe); if (err) @@ -466,6 +482,10 @@ struct xe_device *xe_device_create(struct pci_dev *pdev, init_rwsem(&xe->usm.lock); + err = xe_pagemap_shrinker_create(xe); + if (err) + goto err; + xa_init_flags(&xe->usm.asid_to_vm, XA_FLAGS_ALLOC); if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) { @@ -632,62 +652,14 @@ mask_err: return err; } -static int lmem_initializing(struct xe_device *xe) -{ - if (xe_mmio_read32(xe_root_tile_mmio(xe), GU_CNTL) & LMEM_INIT) - return 0; - - if (signal_pending(current)) - return -EINTR; - - return 1; -} - -static int wait_for_lmem_ready(struct xe_device *xe) +static void assert_lmem_ready(struct xe_device *xe) { - const unsigned long TIMEOUT_SEC = 60; - unsigned long prev_jiffies; - int initializing; - - if (!IS_DGFX(xe)) - return 0; - - if (IS_SRIOV_VF(xe)) - return 0; - - if (!lmem_initializing(xe)) - return 0; - - drm_dbg(&xe->drm, "Waiting for lmem initialization\n"); - prev_jiffies = jiffies; - - /* - * The boot firmware initializes local memory and - * assesses its health. If memory training fails, - * the punit will have been instructed to keep the GT powered - * down.we won't be able to communicate with it - * - * If the status check is done before punit updates the register, - * it can lead to the system being unusable. - * use a timeout and defer the probe to prevent this. - */ - poll_timeout_us(initializing = lmem_initializing(xe), - initializing <= 0, - 20 * USEC_PER_MSEC, TIMEOUT_SEC * USEC_PER_SEC, true); - if (initializing < 0) - return initializing; - - if (initializing) { - drm_dbg(&xe->drm, "lmem not initialized by firmware\n"); - return -EPROBE_DEFER; - } - - drm_dbg(&xe->drm, "lmem ready after %ums", - jiffies_to_msecs(jiffies - prev_jiffies)); + if (!IS_DGFX(xe) || IS_SRIOV_VF(xe)) + return; - return 0; + xe_assert(xe, xe_mmio_read32(xe_root_tile_mmio(xe), GU_CNTL) & + LMEM_INIT); } -ALLOW_ERROR_INJECTION(wait_for_lmem_ready, ERRNO); /* See xe_pci_probe() */ static void vf_update_device_info(struct xe_device *xe) { @@ -742,6 +714,11 @@ int xe_device_probe_early(struct xe_device *xe) if (IS_SRIOV_VF(xe)) vf_update_device_info(xe); + /* + * Check for pcode uncore_init status to confirm if the SoC + * initialization is complete. Until done, any MMIO or lmem access from + * the driver will be blocked + */ err = xe_pcode_probe_early(xe); if (err || xe_survivability_mode_is_requested(xe)) { int save_err = err; @@ -758,11 +735,17 @@ int xe_device_probe_early(struct xe_device *xe) return save_err; } - err = wait_for_lmem_ready(xe); - if (err) - return err; + /* + * Make sure the lmem is initialized and ready to use. xe_pcode_ready() + * is flagged after full initialization is complete. Assert if lmem is + * not initialized. + */ + assert_lmem_ready(xe); - xe->wedged.mode = xe_modparam.wedged_mode; + xe->wedged.mode = xe_device_validate_wedged_mode(xe, xe_modparam.wedged_mode) ? + XE_WEDGED_MODE_DEFAULT : xe_modparam.wedged_mode; + drm_dbg(&xe->drm, "wedged_mode: setting mode (%u) %s\n", + xe->wedged.mode, xe_wedged_mode_to_string(xe->wedged.mode)); err = xe_device_vram_alloc(xe); if (err) @@ -775,7 +758,6 @@ ALLOW_ERROR_INJECTION(xe_device_probe_early, ERRNO); /* See xe_pci_probe() */ static int probe_has_flat_ccs(struct xe_device *xe) { struct xe_gt *gt; - unsigned int fw_ref; u32 reg; /* Always enabled/disabled, no runtime check to do */ @@ -786,8 +768,8 @@ static int probe_has_flat_ccs(struct xe_device *xe) if (!gt) return 0; - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); - if (!fw_ref) + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) return -ETIMEDOUT; reg = xe_gt_mcr_unicast_read_any(gt, XE2_FLAT_CCS_BASE_RANGE_LOWER); @@ -797,11 +779,64 @@ static int probe_has_flat_ccs(struct xe_device *xe) drm_dbg(&xe->drm, "Flat CCS has been disabled in bios, May lead to performance impact"); - xe_force_wake_put(gt_to_fw(gt), fw_ref); - return 0; } +/* + * Detect if the driver is being run on pre-production hardware. We don't + * keep workarounds for pre-production hardware long term, so print an + * error and add taint if we're being loaded on a pre-production platform + * for which the pre-prod workarounds have already been removed. + * + * The general policy is that we'll remove any workarounds that only apply to + * pre-production hardware around the time force_probe restrictions are lifted + * for a platform of the next major IP generation (for example, Xe2 pre-prod + * workarounds should be removed around the time the first Xe3 platforms have + * force_probe lifted). + */ +static void detect_preproduction_hw(struct xe_device *xe) +{ + struct xe_gt *gt; + int id; + + /* + * SR-IOV VFs don't have access to the FUSE2 register, so we can't + * check pre-production status there. But the host OS will notice + * and report the pre-production status, which should be enough to + * help us catch mistaken use of pre-production hardware. + */ + if (IS_SRIOV_VF(xe)) + return; + + /* + * The "SW_CAP" fuse contains a bit indicating whether the device is a + * production or pre-production device. This fuse is reflected through + * the GT "FUSE2" register, even though the contents of the fuse are + * not GT-specific. Every GT's reflection of this fuse should show the + * same value, so we'll just use the first available GT for lookup. + */ + for_each_gt(gt, xe, id) + break; + + if (!gt) + return; + + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT)) { + xe_gt_err(gt, "Forcewake failure; cannot determine production/pre-production hw status.\n"); + return; + } + + if (xe_mmio_read32(>->mmio, FUSE2) & PRODUCTION_HW) + return; + + xe_info(xe, "Pre-production hardware detected.\n"); + if (!xe->info.has_pre_prod_wa) { + xe_err(xe, "Pre-production workarounds for this platform have already been removed.\n"); + add_taint(TAINT_MACHINE_CHECK, LOCKDEP_STILL_OK); + } +} + int xe_device_probe(struct xe_device *xe) { struct xe_tile *tile; @@ -911,6 +946,10 @@ int xe_device_probe(struct xe_device *xe) xe_nvm_init(xe); + err = xe_soc_remapper_init(xe); + if (err) + return err; + err = xe_heci_gsc_init(xe); if (err) return err; @@ -972,10 +1011,13 @@ int xe_device_probe(struct xe_device *xe) if (err) goto err_unregister_display; + detect_preproduction_hw(xe); + return devm_add_action_or_reset(xe->drm.dev, xe_device_sanitize, xe); err_unregister_display: xe_display_unregister(xe); + drm_dev_unregister(&xe->drm); return err; } @@ -984,8 +1026,6 @@ void xe_device_remove(struct xe_device *xe) { xe_display_unregister(xe); - xe_nvm_fini(xe); - drm_dev_unplug(&xe->drm); xe_bo_pci_dev_remove_all(xe); @@ -1034,7 +1074,6 @@ void xe_device_wmb(struct xe_device *xe) */ static void tdf_request_sync(struct xe_device *xe) { - unsigned int fw_ref; struct xe_gt *gt; u8 id; @@ -1042,8 +1081,8 @@ static void tdf_request_sync(struct xe_device *xe) if (xe_gt_is_media_type(gt)) continue; - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); - if (!fw_ref) + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) return; xe_mmio_write32(>->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST); @@ -1058,15 +1097,12 @@ static void tdf_request_sync(struct xe_device *xe) if (xe_mmio_wait32(>->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST, 0, 300, NULL, false)) xe_gt_err_once(gt, "TD flush timeout\n"); - - xe_force_wake_put(gt_to_fw(gt), fw_ref); } } void xe_device_l2_flush(struct xe_device *xe) { struct xe_gt *gt; - unsigned int fw_ref; gt = xe_root_mmio_gt(xe); if (!gt) @@ -1075,8 +1111,8 @@ void xe_device_l2_flush(struct xe_device *xe) if (!XE_GT_WA(gt, 16023588340)) return; - fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); - if (!fw_ref) + CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); + if (!fw_ref.domains) return; spin_lock(>->global_invl_lock); @@ -1086,8 +1122,6 @@ void xe_device_l2_flush(struct xe_device *xe) xe_gt_err_once(gt, "Global invalidation timeout\n"); spin_unlock(>->global_invl_lock); - - xe_force_wake_put(gt_to_fw(gt), fw_ref); } /** @@ -1193,10 +1227,10 @@ static void xe_device_wedged_fini(struct drm_device *drm, void *arg) * DOC: Xe Device Wedging * * Xe driver uses drm device wedged uevent as documented in Documentation/gpu/drm-uapi.rst. - * When device is in wedged state, every IOCTL will be blocked and GT cannot be - * used. Certain critical errors like gt reset failure, firmware failures can cause - * the device to be wedged. The default recovery method for a wedged state - * is rebind/bus-reset. + * When device is in wedged state, every IOCTL will be blocked and GT cannot + * be used. The conditions under which the driver declares the device wedged + * depend on the wedged mode configuration (see &enum xe_wedged_mode). The + * default recovery method for a wedged state is rebind/bus-reset. * * Another recovery method is vendor-specific. Below are the cases that send * ``WEDGED=vendor-specific`` recovery method in drm device wedged uevent. @@ -1261,7 +1295,7 @@ void xe_device_declare_wedged(struct xe_device *xe) struct xe_gt *gt; u8 id; - if (xe->wedged.mode == 0) { + if (xe->wedged.mode == XE_WEDGED_MODE_NEVER) { drm_dbg(&xe->drm, "Wedged mode is forcibly disabled\n"); return; } @@ -1295,3 +1329,48 @@ void xe_device_declare_wedged(struct xe_device *xe) drm_dev_wedged_event(&xe->drm, xe->wedged.method, NULL); } } + +/** + * xe_device_validate_wedged_mode - Check if given mode is supported + * @xe: the &xe_device + * @mode: requested mode to validate + * + * Check whether the provided wedged mode is supported. + * + * Return: 0 if mode is supported, error code otherwise. + */ +int xe_device_validate_wedged_mode(struct xe_device *xe, unsigned int mode) +{ + if (mode > XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET) { + drm_dbg(&xe->drm, "wedged_mode: invalid value (%u)\n", mode); + return -EINVAL; + } else if (mode == XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET && (IS_SRIOV_VF(xe) || + (IS_SRIOV_PF(xe) && !IS_ENABLED(CONFIG_DRM_XE_DEBUG)))) { + drm_dbg(&xe->drm, "wedged_mode: (%u) %s mode is not supported for %s\n", + mode, xe_wedged_mode_to_string(mode), + xe_sriov_mode_to_string(xe_device_sriov_mode(xe))); + return -EPERM; + } + + return 0; +} + +/** + * xe_wedged_mode_to_string - Convert enum value to string. + * @mode: the &xe_wedged_mode to convert + * + * Returns: wedged mode as a user friendly string. + */ +const char *xe_wedged_mode_to_string(enum xe_wedged_mode mode) +{ + switch (mode) { + case XE_WEDGED_MODE_NEVER: + return "never"; + case XE_WEDGED_MODE_UPON_CRITICAL_ERROR: + return "upon-critical-error"; + case XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET: + return "upon-any-hang-no-reset"; + default: + return "<invalid>"; + } +} |