diff options
Diffstat (limited to 'drivers/gpu/drm/drm_pagemap.c')
| -rw-r--r-- | drivers/gpu/drm/drm_pagemap.c | 566 |
1 files changed, 483 insertions, 83 deletions
diff --git a/drivers/gpu/drm/drm_pagemap.c b/drivers/gpu/drm/drm_pagemap.c index 704f2f945019..38eca94f01a1 100644 --- a/drivers/gpu/drm/drm_pagemap.c +++ b/drivers/gpu/drm/drm_pagemap.c @@ -9,6 +9,8 @@ #include <linux/pagemap.h> #include <drm/drm_drv.h> #include <drm/drm_pagemap.h> +#include <drm/drm_pagemap_util.h> +#include <drm/drm_print.h> /** * DOC: Overview @@ -63,7 +65,7 @@ * * @refcount: Reference count for the zdd * @devmem_allocation: device memory allocation - * @device_private_page_owner: Device private pages owner + * @dpagemap: Refcounted pointer to the underlying struct drm_pagemap. * * This structure serves as a generic wrapper installed in * page->zone_device_data. It provides infrastructure for looking up a device @@ -75,12 +77,12 @@ struct drm_pagemap_zdd { struct kref refcount; struct drm_pagemap_devmem *devmem_allocation; - void *device_private_page_owner; + struct drm_pagemap *dpagemap; }; /** * drm_pagemap_zdd_alloc() - Allocate a zdd structure. - * @device_private_page_owner: Device private pages owner + * @dpagemap: Pointer to the underlying struct drm_pagemap. * * This function allocates and initializes a new zdd structure. It sets up the * reference count and initializes the destroy work. @@ -88,7 +90,7 @@ struct drm_pagemap_zdd { * Return: Pointer to the allocated zdd on success, ERR_PTR() on failure. */ static struct drm_pagemap_zdd * -drm_pagemap_zdd_alloc(void *device_private_page_owner) +drm_pagemap_zdd_alloc(struct drm_pagemap *dpagemap) { struct drm_pagemap_zdd *zdd; @@ -98,7 +100,7 @@ drm_pagemap_zdd_alloc(void *device_private_page_owner) kref_init(&zdd->refcount); zdd->devmem_allocation = NULL; - zdd->device_private_page_owner = device_private_page_owner; + zdd->dpagemap = drm_pagemap_get(dpagemap); return zdd; } @@ -128,6 +130,7 @@ static void drm_pagemap_zdd_destroy(struct kref *ref) struct drm_pagemap_zdd *zdd = container_of(ref, struct drm_pagemap_zdd, refcount); struct drm_pagemap_devmem *devmem = zdd->devmem_allocation; + struct drm_pagemap *dpagemap = zdd->dpagemap; if (devmem) { complete_all(&devmem->detached); @@ -135,6 +138,7 @@ static void drm_pagemap_zdd_destroy(struct kref *ref) devmem->ops->devmem_release(devmem); } kfree(zdd); + drm_pagemap_put(dpagemap); } /** @@ -202,11 +206,13 @@ static void drm_pagemap_get_devmem_page(struct page *page, /** * drm_pagemap_migrate_map_pages() - Map migration pages for GPU SVM migration - * @dev: The device for which the pages are being mapped - * @pagemap_addr: Array to store DMA information corresponding to mapped pages - * @migrate_pfn: Array of migrate page frame numbers to map - * @npages: Number of pages to map + * @dev: The device performing the migration. + * @local_dpagemap: The drm_pagemap local to the migrating device. + * @pagemap_addr: Array to store DMA information corresponding to mapped pages. + * @migrate_pfn: Array of page frame numbers of system pages or peer pages to map. + * @npages: Number of system pages or peer pages to map. * @dir: Direction of data transfer (e.g., DMA_BIDIRECTIONAL) + * @mdetails: Details governing the migration behaviour. * * This function maps pages of memory for migration usage in GPU SVM. It * iterates over each page frame number provided in @migrate_pfn, maps the @@ -216,12 +222,14 @@ static void drm_pagemap_get_devmem_page(struct page *page, * Returns: 0 on success, -EFAULT if an error occurs during mapping. */ static int drm_pagemap_migrate_map_pages(struct device *dev, + struct drm_pagemap *local_dpagemap, struct drm_pagemap_addr *pagemap_addr, unsigned long *migrate_pfn, unsigned long npages, - enum dma_data_direction dir) + enum dma_data_direction dir, + const struct drm_pagemap_migrate_details *mdetails) { - unsigned long i; + unsigned long num_peer_pages = 0, num_local_pages = 0, i; for (i = 0; i < npages;) { struct page *page = migrate_pfn_to_page(migrate_pfn[i]); @@ -232,31 +240,58 @@ static int drm_pagemap_migrate_map_pages(struct device *dev, if (!page) goto next; - if (WARN_ON_ONCE(is_zone_device_page(page))) - return -EFAULT; - folio = page_folio(page); order = folio_order(folio); - dma_addr = dma_map_page(dev, page, 0, page_size(page), dir); - if (dma_mapping_error(dev, dma_addr)) - return -EFAULT; - - pagemap_addr[i] = - drm_pagemap_addr_encode(dma_addr, - DRM_INTERCONNECT_SYSTEM, - order, dir); + if (is_device_private_page(page)) { + struct drm_pagemap_zdd *zdd = page->zone_device_data; + struct drm_pagemap *dpagemap = zdd->dpagemap; + struct drm_pagemap_addr addr; + + if (dpagemap == local_dpagemap) { + if (!mdetails->can_migrate_same_pagemap) + goto next; + + num_local_pages += NR_PAGES(order); + } else { + num_peer_pages += NR_PAGES(order); + } + + addr = dpagemap->ops->device_map(dpagemap, dev, page, order, dir); + if (dma_mapping_error(dev, addr.addr)) + return -EFAULT; + + pagemap_addr[i] = addr; + } else { + dma_addr = dma_map_page(dev, page, 0, page_size(page), dir); + if (dma_mapping_error(dev, dma_addr)) + return -EFAULT; + + pagemap_addr[i] = + drm_pagemap_addr_encode(dma_addr, + DRM_INTERCONNECT_SYSTEM, + order, dir); + } next: i += NR_PAGES(order); } + if (num_peer_pages) + drm_dbg(local_dpagemap->drm, "Migrating %lu peer pages over interconnect.\n", + num_peer_pages); + if (num_local_pages) + drm_dbg(local_dpagemap->drm, "Migrating %lu local pages over interconnect.\n", + num_local_pages); + return 0; } /** * drm_pagemap_migrate_unmap_pages() - Unmap pages previously mapped for GPU SVM migration * @dev: The device for which the pages were mapped + * @migrate_pfn: Array of migrate pfns set up for the mapped pages. Used to + * determine the drm_pagemap of a peer device private page. * @pagemap_addr: Array of DMA information corresponding to mapped pages * @npages: Number of pages to unmap * @dir: Direction of data transfer (e.g., DMA_BIDIRECTIONAL) @@ -267,16 +302,27 @@ next: */ static void drm_pagemap_migrate_unmap_pages(struct device *dev, struct drm_pagemap_addr *pagemap_addr, + unsigned long *migrate_pfn, unsigned long npages, enum dma_data_direction dir) { unsigned long i; for (i = 0; i < npages;) { - if (!pagemap_addr[i].addr || dma_mapping_error(dev, pagemap_addr[i].addr)) + struct page *page = migrate_pfn_to_page(migrate_pfn[i]); + + if (!page || !pagemap_addr[i].addr || dma_mapping_error(dev, pagemap_addr[i].addr)) goto next; - dma_unmap_page(dev, pagemap_addr[i].addr, PAGE_SIZE << pagemap_addr[i].order, dir); + if (is_zone_device_page(page)) { + struct drm_pagemap_zdd *zdd = page->zone_device_data; + struct drm_pagemap *dpagemap = zdd->dpagemap; + + dpagemap->ops->device_unmap(dpagemap, dev, pagemap_addr[i]); + } else { + dma_unmap_page(dev, pagemap_addr[i].addr, + PAGE_SIZE << pagemap_addr[i].order, dir); + } next: i += NR_PAGES(pagemap_addr[i].order); @@ -289,18 +335,125 @@ npages_in_range(unsigned long start, unsigned long end) return (end - start) >> PAGE_SHIFT; } +static int +drm_pagemap_migrate_remote_to_local(struct drm_pagemap_devmem *devmem, + struct device *remote_device, + struct drm_pagemap *remote_dpagemap, + unsigned long local_pfns[], + struct page *remote_pages[], + struct drm_pagemap_addr pagemap_addr[], + unsigned long npages, + const struct drm_pagemap_devmem_ops *ops, + const struct drm_pagemap_migrate_details *mdetails) + +{ + int err = drm_pagemap_migrate_map_pages(remote_device, remote_dpagemap, + pagemap_addr, local_pfns, + npages, DMA_FROM_DEVICE, mdetails); + + if (err) + goto out; + + err = ops->copy_to_ram(remote_pages, pagemap_addr, npages, + devmem->pre_migrate_fence); +out: + drm_pagemap_migrate_unmap_pages(remote_device, pagemap_addr, local_pfns, + npages, DMA_FROM_DEVICE); + return err; +} + +static int +drm_pagemap_migrate_sys_to_dev(struct drm_pagemap_devmem *devmem, + unsigned long sys_pfns[], + struct page *local_pages[], + struct drm_pagemap_addr pagemap_addr[], + unsigned long npages, + const struct drm_pagemap_devmem_ops *ops, + const struct drm_pagemap_migrate_details *mdetails) +{ + int err = drm_pagemap_migrate_map_pages(devmem->dev, devmem->dpagemap, + pagemap_addr, sys_pfns, npages, + DMA_TO_DEVICE, mdetails); + + if (err) + goto out; + + err = ops->copy_to_devmem(local_pages, pagemap_addr, npages, + devmem->pre_migrate_fence); +out: + drm_pagemap_migrate_unmap_pages(devmem->dev, pagemap_addr, sys_pfns, npages, + DMA_TO_DEVICE); + return err; +} + +/** + * struct migrate_range_loc - Cursor into the loop over migrate_pfns for migrating to + * device. + * @start: The current loop index. + * @device: migrating device. + * @dpagemap: Pointer to struct drm_pagemap used by the migrating device. + * @ops: The copy ops to be used for the migrating device. + */ +struct migrate_range_loc { + unsigned long start; + struct device *device; + struct drm_pagemap *dpagemap; + const struct drm_pagemap_devmem_ops *ops; +}; + +static int drm_pagemap_migrate_range(struct drm_pagemap_devmem *devmem, + unsigned long src_pfns[], + unsigned long dst_pfns[], + struct page *pages[], + struct drm_pagemap_addr pagemap_addr[], + struct migrate_range_loc *last, + const struct migrate_range_loc *cur, + const struct drm_pagemap_migrate_details *mdetails) +{ + int ret = 0; + + if (cur->start == 0) + goto out; + + if (cur->start <= last->start) + return 0; + + if (cur->dpagemap == last->dpagemap && cur->ops == last->ops) + return 0; + + if (last->dpagemap) + ret = drm_pagemap_migrate_remote_to_local(devmem, + last->device, + last->dpagemap, + &dst_pfns[last->start], + &pages[last->start], + &pagemap_addr[last->start], + cur->start - last->start, + last->ops, mdetails); + + else + ret = drm_pagemap_migrate_sys_to_dev(devmem, + &src_pfns[last->start], + &pages[last->start], + &pagemap_addr[last->start], + cur->start - last->start, + last->ops, mdetails); + +out: + *last = *cur; + return ret; +} + /** * drm_pagemap_migrate_to_devmem() - Migrate a struct mm_struct range to device memory * @devmem_allocation: The device memory allocation to migrate to. * The caller should hold a reference to the device memory allocation, - * and the reference is consumed by this function unless it returns with + * and the reference is consumed by this function even if it returns with * an error. * @mm: Pointer to the struct mm_struct. * @start: Start of the virtual address range to migrate. * @end: End of the virtual address range to migrate. - * @timeslice_ms: The time requested for the migrated pagemap pages to - * be present in @mm before being allowed to be migrated back. - * @pgmap_owner: Not used currently, since only system memory is considered. + * @mdetails: Details to govern the migration. * * This function migrates the specified virtual address range to device memory. * It performs the necessary setup and invokes the driver-specific operations for @@ -318,17 +471,31 @@ npages_in_range(unsigned long start, unsigned long end) int drm_pagemap_migrate_to_devmem(struct drm_pagemap_devmem *devmem_allocation, struct mm_struct *mm, unsigned long start, unsigned long end, - unsigned long timeslice_ms, - void *pgmap_owner) + const struct drm_pagemap_migrate_details *mdetails) { const struct drm_pagemap_devmem_ops *ops = devmem_allocation->ops; + struct drm_pagemap *dpagemap = devmem_allocation->dpagemap; + struct dev_pagemap *pagemap = dpagemap->pagemap; struct migrate_vma migrate = { .start = start, .end = end, - .pgmap_owner = pgmap_owner, - .flags = MIGRATE_VMA_SELECT_SYSTEM, + .pgmap_owner = pagemap->owner, + /* + * FIXME: MIGRATE_VMA_SELECT_DEVICE_PRIVATE intermittently + * causes 'xe_exec_system_allocator --r *race*no*' to trigger aa + * engine reset and a hard hang due to getting stuck on a folio + * lock. This should work and needs to be root-caused. The only + * downside of not selecting MIGRATE_VMA_SELECT_DEVICE_PRIVATE + * is that device-to-device migrations won’t work; instead, + * memory will bounce through system memory. This path should be + * rare and only occur when the madvise attributes of memory are + * changed or atomics are being used. + */ + .flags = MIGRATE_VMA_SELECT_SYSTEM | MIGRATE_VMA_SELECT_DEVICE_COHERENT, }; unsigned long i, npages = npages_in_range(start, end); + unsigned long own_pages = 0, migrated_pages = 0; + struct migrate_range_loc cur, last = {.device = dpagemap->drm->dev, .ops = ops}; struct vm_area_struct *vas; struct drm_pagemap_zdd *zdd = NULL; struct page **pages; @@ -367,11 +534,13 @@ int drm_pagemap_migrate_to_devmem(struct drm_pagemap_devmem *devmem_allocation, pagemap_addr = buf + (2 * sizeof(*migrate.src) * npages); pages = buf + (2 * sizeof(*migrate.src) + sizeof(*pagemap_addr)) * npages; - zdd = drm_pagemap_zdd_alloc(pgmap_owner); + zdd = drm_pagemap_zdd_alloc(dpagemap); if (!zdd) { err = -ENOMEM; - goto err_free; + kvfree(buf); + goto err_out; } + zdd->devmem_allocation = devmem_allocation; /* Owns ref */ migrate.vma = vas; migrate.src = buf; @@ -382,58 +551,136 @@ int drm_pagemap_migrate_to_devmem(struct drm_pagemap_devmem *devmem_allocation, goto err_free; if (!migrate.cpages) { - err = -EFAULT; + /* No pages to migrate. Raced or unknown device pages. */ + err = -EBUSY; goto err_free; } if (migrate.cpages != npages) { + /* + * Some pages to migrate. But we want to migrate all or + * nothing. Raced or unknown device pages. + */ err = -EBUSY; - goto err_finalize; + goto err_aborted_migration; } - err = ops->populate_devmem_pfn(devmem_allocation, npages, migrate.dst); - if (err) - goto err_finalize; + /* Count device-private pages to migrate */ + for (i = 0; i < npages;) { + struct page *src_page = migrate_pfn_to_page(migrate.src[i]); + unsigned long nr_pages = src_page ? NR_PAGES(folio_order(page_folio(src_page))) : 1; - err = drm_pagemap_migrate_map_pages(devmem_allocation->dev, pagemap_addr, - migrate.src, npages, DMA_TO_DEVICE); + if (src_page && is_zone_device_page(src_page)) { + if (page_pgmap(src_page) == pagemap) + own_pages += nr_pages; + } + + i += nr_pages; + } + drm_dbg(dpagemap->drm, "Total pages %lu; Own pages: %lu.\n", + npages, own_pages); + if (own_pages == npages) { + err = 0; + drm_dbg(dpagemap->drm, "Migration wasn't necessary.\n"); + goto err_aborted_migration; + } else if (own_pages && !mdetails->can_migrate_same_pagemap) { + err = -EBUSY; + drm_dbg(dpagemap->drm, "Migration aborted due to fragmentation.\n"); + goto err_aborted_migration; + } + + err = ops->populate_devmem_pfn(devmem_allocation, npages, migrate.dst); if (err) - goto err_finalize; + goto err_aborted_migration; + + own_pages = 0; for (i = 0; i < npages; ++i) { struct page *page = pfn_to_page(migrate.dst[i]); + struct page *src_page = migrate_pfn_to_page(migrate.src[i]); + cur.start = i; - pages[i] = page; + pages[i] = NULL; + if (src_page && is_device_private_page(src_page)) { + struct drm_pagemap_zdd *src_zdd = src_page->zone_device_data; + + if (page_pgmap(src_page) == pagemap && + !mdetails->can_migrate_same_pagemap) { + migrate.dst[i] = 0; + own_pages++; + continue; + } + if (mdetails->source_peer_migrates) { + cur.dpagemap = src_zdd->dpagemap; + cur.ops = src_zdd->devmem_allocation->ops; + cur.device = cur.dpagemap->drm->dev; + pages[i] = src_page; + } + } + if (!pages[i]) { + cur.dpagemap = NULL; + cur.ops = ops; + cur.device = dpagemap->drm->dev; + pages[i] = page; + } migrate.dst[i] = migrate_pfn(migrate.dst[i]); drm_pagemap_get_devmem_page(page, zdd); - } - err = ops->copy_to_devmem(pages, pagemap_addr, npages, - devmem_allocation->pre_migrate_fence); + /* If we switched the migrating drm_pagemap, migrate previous pages now */ + err = drm_pagemap_migrate_range(devmem_allocation, migrate.src, migrate.dst, + pages, pagemap_addr, &last, &cur, + mdetails); + if (err) { + npages = i + 1; + goto err_finalize; + } + } + cur.start = npages; + cur.ops = NULL; /* Force migration */ + err = drm_pagemap_migrate_range(devmem_allocation, migrate.src, migrate.dst, + pages, pagemap_addr, &last, &cur, mdetails); if (err) goto err_finalize; + drm_WARN_ON(dpagemap->drm, !!own_pages); + dma_fence_put(devmem_allocation->pre_migrate_fence); devmem_allocation->pre_migrate_fence = NULL; /* Upon success bind devmem allocation to range and zdd */ devmem_allocation->timeslice_expiration = get_jiffies_64() + - msecs_to_jiffies(timeslice_ms); - zdd->devmem_allocation = devmem_allocation; /* Owns ref */ + msecs_to_jiffies(mdetails->timeslice_ms); err_finalize: if (err) drm_pagemap_migration_unlock_put_pages(npages, migrate.dst); +err_aborted_migration: migrate_vma_pages(&migrate); + + for (i = 0; !err && i < npages;) { + struct page *page = migrate_pfn_to_page(migrate.src[i]); + unsigned long nr_pages = page ? NR_PAGES(folio_order(page_folio(page))) : 1; + + if (migrate.src[i] & MIGRATE_PFN_MIGRATE) + migrated_pages += nr_pages; + + i += nr_pages; + } + + if (!err && migrated_pages < npages - own_pages) { + drm_dbg(dpagemap->drm, "Raced while finalizing migration.\n"); + err = -EBUSY; + } + migrate_vma_finalize(&migrate); - drm_pagemap_migrate_unmap_pages(devmem_allocation->dev, pagemap_addr, npages, - DMA_TO_DEVICE); err_free: - if (zdd) - drm_pagemap_zdd_put(zdd); + drm_pagemap_zdd_put(zdd); kvfree(buf); + return err; + err_out: + devmem_allocation->ops->devmem_release(devmem_allocation); return err; } EXPORT_SYMBOL_GPL(drm_pagemap_migrate_to_devmem); @@ -543,6 +790,157 @@ next_put: return -ENOMEM; } +static void drm_pagemap_dev_unhold_work(struct work_struct *work); +static LLIST_HEAD(drm_pagemap_unhold_list); +static DECLARE_WORK(drm_pagemap_work, drm_pagemap_dev_unhold_work); + +/** + * struct drm_pagemap_dev_hold - Struct to aid in drm_device release. + * @link: Link into drm_pagemap_unhold_list for deferred reference releases. + * @drm: drm device to put. + * + * When a struct drm_pagemap is released, we also need to release the + * reference it holds on the drm device. However, typically that needs + * to be done separately from a system-wide workqueue. + * Each time a struct drm_pagemap is initialized + * (or re-initialized if cached) therefore allocate a separate + * drm_pagemap_dev_hold item, from which we put the drm device and + * associated module. + */ +struct drm_pagemap_dev_hold { + struct llist_node link; + struct drm_device *drm; +}; + +static void drm_pagemap_release(struct kref *ref) +{ + struct drm_pagemap *dpagemap = container_of(ref, typeof(*dpagemap), ref); + struct drm_pagemap_dev_hold *dev_hold = dpagemap->dev_hold; + + /* + * We know the pagemap provider is alive at this point, since + * the struct drm_pagemap_dev_hold holds a reference to the + * pagemap provider drm_device and its module. + */ + dpagemap->dev_hold = NULL; + drm_pagemap_shrinker_add(dpagemap); + llist_add(&dev_hold->link, &drm_pagemap_unhold_list); + schedule_work(&drm_pagemap_work); + /* + * Here, either the provider device is still alive, since if called from + * page_free(), the caller is holding a reference on the dev_pagemap, + * or if called from drm_pagemap_put(), the direct caller is still alive. + * This ensures we can't race with THIS module unload. + */ +} + +static void drm_pagemap_dev_unhold_work(struct work_struct *work) +{ + struct llist_node *node = llist_del_all(&drm_pagemap_unhold_list); + struct drm_pagemap_dev_hold *dev_hold, *next; + + /* + * Deferred release of drm_pagemap provider device and module. + * THIS module is kept alive during the release by the + * flush_work() in the drm_pagemap_exit() function. + */ + llist_for_each_entry_safe(dev_hold, next, node, link) { + struct drm_device *drm = dev_hold->drm; + struct module *module = drm->driver->fops->owner; + + drm_dbg(drm, "Releasing reference on provider device and module.\n"); + drm_dev_put(drm); + module_put(module); + kfree(dev_hold); + } +} + +static struct drm_pagemap_dev_hold * +drm_pagemap_dev_hold(struct drm_pagemap *dpagemap) +{ + struct drm_pagemap_dev_hold *dev_hold; + struct drm_device *drm = dpagemap->drm; + + dev_hold = kzalloc(sizeof(*dev_hold), GFP_KERNEL); + if (!dev_hold) + return ERR_PTR(-ENOMEM); + + init_llist_node(&dev_hold->link); + dev_hold->drm = drm; + (void)try_module_get(drm->driver->fops->owner); + drm_dev_get(drm); + + return dev_hold; +} + +/** + * drm_pagemap_reinit() - Reinitialize a drm_pagemap + * @dpagemap: The drm_pagemap to reinitialize + * + * Reinitialize a drm_pagemap, for which drm_pagemap_release + * has already been called. This interface is intended for the + * situation where the driver caches a destroyed drm_pagemap. + * + * Return: 0 on success, negative error code on failure. + */ +int drm_pagemap_reinit(struct drm_pagemap *dpagemap) +{ + dpagemap->dev_hold = drm_pagemap_dev_hold(dpagemap); + if (IS_ERR(dpagemap->dev_hold)) + return PTR_ERR(dpagemap->dev_hold); + + kref_init(&dpagemap->ref); + return 0; +} +EXPORT_SYMBOL(drm_pagemap_reinit); + +/** + * drm_pagemap_init() - Initialize a pre-allocated drm_pagemap + * @dpagemap: The drm_pagemap to initialize. + * @pagemap: The associated dev_pagemap providing the device + * private pages. + * @drm: The drm device. The drm_pagemap holds a reference on the + * drm_device and the module owning the drm_device until + * drm_pagemap_release(). This facilitates drm_pagemap exporting. + * @ops: The drm_pagemap ops. + * + * Initialize and take an initial reference on a drm_pagemap. + * After successful return, use drm_pagemap_put() to destroy. + * + ** Return: 0 on success, negative error code on error. + */ +int drm_pagemap_init(struct drm_pagemap *dpagemap, + struct dev_pagemap *pagemap, + struct drm_device *drm, + const struct drm_pagemap_ops *ops) +{ + kref_init(&dpagemap->ref); + dpagemap->ops = ops; + dpagemap->pagemap = pagemap; + dpagemap->drm = drm; + dpagemap->cache = NULL; + INIT_LIST_HEAD(&dpagemap->shrink_link); + + return drm_pagemap_reinit(dpagemap); +} +EXPORT_SYMBOL(drm_pagemap_init); + +/** + * drm_pagemap_put() - Put a struct drm_pagemap reference + * @dpagemap: Pointer to a struct drm_pagemap object. + * + * Puts a struct drm_pagemap reference and frees the drm_pagemap object + * if the refount reaches zero. + */ +void drm_pagemap_put(struct drm_pagemap *dpagemap) +{ + if (likely(dpagemap)) { + drm_pagemap_shrinker_might_lock(dpagemap); + kref_put(&dpagemap->ref, drm_pagemap_release); + } +} +EXPORT_SYMBOL(drm_pagemap_put); + /** * drm_pagemap_evict_to_ram() - Evict GPU SVM range to RAM * @devmem_allocation: Pointer to the device memory allocation @@ -555,6 +953,7 @@ next_put: int drm_pagemap_evict_to_ram(struct drm_pagemap_devmem *devmem_allocation) { const struct drm_pagemap_devmem_ops *ops = devmem_allocation->ops; + struct drm_pagemap_migrate_details mdetails = {}; unsigned long npages, mpages = 0; struct page **pages; unsigned long *src, *dst; @@ -593,8 +992,10 @@ retry: if (err || !mpages) goto err_finalize; - err = drm_pagemap_migrate_map_pages(devmem_allocation->dev, pagemap_addr, - dst, npages, DMA_FROM_DEVICE); + err = drm_pagemap_migrate_map_pages(devmem_allocation->dev, + devmem_allocation->dpagemap, pagemap_addr, + dst, npages, DMA_FROM_DEVICE, + &mdetails); if (err) goto err_finalize; @@ -610,8 +1011,9 @@ err_finalize: drm_pagemap_migration_unlock_put_pages(npages, dst); migrate_device_pages(src, dst, npages); migrate_device_finalize(src, dst, npages); - drm_pagemap_migrate_unmap_pages(devmem_allocation->dev, pagemap_addr, npages, + drm_pagemap_migrate_unmap_pages(devmem_allocation->dev, pagemap_addr, dst, npages, DMA_FROM_DEVICE); + err_free: kvfree(buf); err_out: @@ -632,8 +1034,7 @@ EXPORT_SYMBOL_GPL(drm_pagemap_evict_to_ram); /** * __drm_pagemap_migrate_to_ram() - Migrate GPU SVM range to RAM (internal) * @vas: Pointer to the VM area structure - * @device_private_page_owner: Device private pages owner - * @page: Pointer to the page for fault handling (can be NULL) + * @page: Pointer to the page for fault handling. * @fault_addr: Fault address * @size: Size of migration * @@ -644,18 +1045,18 @@ EXPORT_SYMBOL_GPL(drm_pagemap_evict_to_ram); * Return: 0 on success, negative error code on failure. */ static int __drm_pagemap_migrate_to_ram(struct vm_area_struct *vas, - void *device_private_page_owner, struct page *page, unsigned long fault_addr, unsigned long size) { struct migrate_vma migrate = { .vma = vas, - .pgmap_owner = device_private_page_owner, + .pgmap_owner = page_pgmap(page)->owner, .flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE | MIGRATE_VMA_SELECT_DEVICE_COHERENT, .fault_page = page, }; + struct drm_pagemap_migrate_details mdetails = {}; struct drm_pagemap_zdd *zdd; const struct drm_pagemap_devmem_ops *ops; struct device *dev = NULL; @@ -666,12 +1067,9 @@ static int __drm_pagemap_migrate_to_ram(struct vm_area_struct *vas, void *buf; int i, err = 0; - if (page) { - zdd = page->zone_device_data; - if (time_before64(get_jiffies_64(), - zdd->devmem_allocation->timeslice_expiration)) - return 0; - } + zdd = page->zone_device_data; + if (time_before64(get_jiffies_64(), zdd->devmem_allocation->timeslice_expiration)) + return 0; start = ALIGN_DOWN(fault_addr, size); end = ALIGN(fault_addr + 1, size); @@ -707,19 +1105,6 @@ static int __drm_pagemap_migrate_to_ram(struct vm_area_struct *vas, if (!migrate.cpages) goto err_free; - if (!page) { - for (i = 0; i < npages; ++i) { - if (!(migrate.src[i] & MIGRATE_PFN_MIGRATE)) - continue; - - page = migrate_pfn_to_page(migrate.src[i]); - break; - } - - if (!page) - goto err_finalize; - } - zdd = page->zone_device_data; ops = zdd->devmem_allocation->ops; dev = zdd->devmem_allocation->dev; @@ -729,8 +1114,8 @@ static int __drm_pagemap_migrate_to_ram(struct vm_area_struct *vas, if (err) goto err_finalize; - err = drm_pagemap_migrate_map_pages(dev, pagemap_addr, migrate.dst, npages, - DMA_FROM_DEVICE); + err = drm_pagemap_migrate_map_pages(dev, zdd->dpagemap, pagemap_addr, migrate.dst, npages, + DMA_FROM_DEVICE, &mdetails); if (err) goto err_finalize; @@ -747,8 +1132,8 @@ err_finalize: migrate_vma_pages(&migrate); migrate_vma_finalize(&migrate); if (dev) - drm_pagemap_migrate_unmap_pages(dev, pagemap_addr, npages, - DMA_FROM_DEVICE); + drm_pagemap_migrate_unmap_pages(dev, pagemap_addr, migrate.dst, + npages, DMA_FROM_DEVICE); err_free: kvfree(buf); err_out: @@ -785,7 +1170,6 @@ static vm_fault_t drm_pagemap_migrate_to_ram(struct vm_fault *vmf) int err; err = __drm_pagemap_migrate_to_ram(vmf->vma, - zdd->device_private_page_owner, vmf->page, vmf->address, zdd->devmem_allocation->size); @@ -889,3 +1273,19 @@ int drm_pagemap_populate_mm(struct drm_pagemap *dpagemap, return err; } EXPORT_SYMBOL(drm_pagemap_populate_mm); + +void drm_pagemap_destroy(struct drm_pagemap *dpagemap, bool is_atomic_or_reclaim) +{ + if (dpagemap->ops->destroy) + dpagemap->ops->destroy(dpagemap, is_atomic_or_reclaim); + else + kfree(dpagemap); +} + +static void drm_pagemap_exit(void) +{ + flush_work(&drm_pagemap_work); + if (WARN_ON(!llist_empty(&drm_pagemap_unhold_list))) + disable_work_sync(&drm_pagemap_work); +} +module_exit(drm_pagemap_exit); |