diff options
author | Christian König <christian.koenig@amd.com> | 2020-10-24 14:17:49 +0300 |
---|---|---|
committer | Christian König <christian.koenig@amd.com> | 2020-10-29 17:57:57 +0300 |
commit | 256dd44bd897055571c131703afdd02b2e4f1f29 (patch) | |
tree | 9444549929a9a19c85620cf191e7f369cbf2a2ee /drivers/gpu/drm/ttm | |
parent | fbf1c39cab37590e9b8c5a19d3b129e7fe86fee3 (diff) | |
download | linux-256dd44bd897055571c131703afdd02b2e4f1f29.tar.xz |
drm/ttm: nuke old page allocator
Not used any more.
Signed-off-by: Christian König <christian.koenig@amd.com>
Reviewed-by: Dave Airlie <airlied@redhat.com>
Reviewed-by: Madhav Chauhan <madhav.chauhan@amd.com>
Tested-by: Huang Rui <ray.huang@amd.com>
Link: https://patchwork.freedesktop.org/patch/397087/?series=83051&rev=1
Diffstat (limited to 'drivers/gpu/drm/ttm')
-rw-r--r-- | drivers/gpu/drm/ttm/Makefile | 3 | ||||
-rw-r--r-- | drivers/gpu/drm/ttm/ttm_agp_backend.c | 1 | ||||
-rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo.c | 1 | ||||
-rw-r--r-- | drivers/gpu/drm/ttm/ttm_memory.c | 7 | ||||
-rw-r--r-- | drivers/gpu/drm/ttm/ttm_page_alloc.c | 1176 | ||||
-rw-r--r-- | drivers/gpu/drm/ttm/ttm_page_alloc_dma.c | 1226 | ||||
-rw-r--r-- | drivers/gpu/drm/ttm/ttm_set_memory.h | 84 | ||||
-rw-r--r-- | drivers/gpu/drm/ttm/ttm_tt.c | 4 |
8 files changed, 2 insertions, 2500 deletions
diff --git a/drivers/gpu/drm/ttm/Makefile b/drivers/gpu/drm/ttm/Makefile index 0096bacbcf32..b6f5f87b270f 100644 --- a/drivers/gpu/drm/ttm/Makefile +++ b/drivers/gpu/drm/ttm/Makefile @@ -4,9 +4,8 @@ ttm-y := ttm_memory.o ttm_tt.o ttm_bo.o \ ttm_bo_util.o ttm_bo_vm.o ttm_module.o \ - ttm_execbuf_util.o ttm_page_alloc.o ttm_range_manager.o \ + ttm_execbuf_util.o ttm_range_manager.o \ ttm_resource.o ttm_pool.o ttm-$(CONFIG_AGP) += ttm_agp_backend.o -ttm-$(CONFIG_DRM_TTM_DMA_PAGE_POOL) += ttm_page_alloc_dma.o obj-$(CONFIG_DRM_TTM) += ttm.o diff --git a/drivers/gpu/drm/ttm/ttm_agp_backend.c b/drivers/gpu/drm/ttm/ttm_agp_backend.c index 4f76c9287159..03c86628e4ac 100644 --- a/drivers/gpu/drm/ttm/ttm_agp_backend.c +++ b/drivers/gpu/drm/ttm/ttm_agp_backend.c @@ -34,7 +34,6 @@ #include <drm/ttm/ttm_module.h> #include <drm/ttm/ttm_bo_driver.h> -#include <drm/ttm/ttm_page_alloc.h> #include <drm/ttm/ttm_placement.h> #include <linux/agp_backend.h> #include <linux/module.h> diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index e9f91cfce5e9..530c9a4af09f 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -1333,7 +1333,6 @@ int ttm_bo_device_init(struct ttm_bo_device *bdev, INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue); INIT_LIST_HEAD(&bdev->ddestroy); bdev->dev_mapping = mapping; - bdev->need_dma32 = use_dma32; mutex_lock(&ttm_global_mutex); list_add_tail(&bdev->device_list, &glob->device_list); mutex_unlock(&ttm_global_mutex); diff --git a/drivers/gpu/drm/ttm/ttm_memory.c b/drivers/gpu/drm/ttm/ttm_memory.c index b15a91c90271..f9a90bfaa3c1 100644 --- a/drivers/gpu/drm/ttm/ttm_memory.c +++ b/drivers/gpu/drm/ttm/ttm_memory.c @@ -30,7 +30,6 @@ #include <drm/ttm/ttm_memory.h> #include <drm/ttm/ttm_module.h> -#include <drm/ttm/ttm_page_alloc.h> #include <linux/spinlock.h> #include <linux/sched.h> #include <linux/wait.h> @@ -452,9 +451,7 @@ int ttm_mem_global_init(struct ttm_mem_global *glob) pr_info("Zone %7s: Available graphics memory: %llu KiB\n", zone->name, (unsigned long long)zone->max_mem >> 10); } - ttm_page_alloc_init(glob, glob->zone_kernel->max_mem/(2*PAGE_SIZE)); - ttm_dma_page_alloc_init(glob, glob->zone_kernel->max_mem/(2*PAGE_SIZE)); - ttm_pool_mgr_init(glob->zone_kernel->max_mem / (2 * PAGE_SIZE)); + ttm_pool_mgr_init(glob->zone_kernel->max_mem/(2*PAGE_SIZE)); return 0; out_no_zone: ttm_mem_global_release(glob); @@ -467,8 +464,6 @@ void ttm_mem_global_release(struct ttm_mem_global *glob) unsigned int i; /* let the page allocator first stop the shrink work. */ - ttm_page_alloc_fini(); - ttm_dma_page_alloc_fini(); ttm_pool_mgr_fini(); flush_workqueue(glob->swap_queue); diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c deleted file mode 100644 index 29e6c29ad60e..000000000000 --- a/drivers/gpu/drm/ttm/ttm_page_alloc.c +++ /dev/null @@ -1,1176 +0,0 @@ -/* - * Copyright (c) Red Hat Inc. - - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sub license, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the - * next paragraph) shall be included in all copies or substantial portions - * of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - * - * Authors: Dave Airlie <airlied@redhat.com> - * Jerome Glisse <jglisse@redhat.com> - * Pauli Nieminen <suokkos@gmail.com> - */ - -/* simple list based uncached page pool - * - Pool collects resently freed pages for reuse - * - Use page->lru to keep a free list - * - doesn't track currently in use pages - */ - -#define pr_fmt(fmt) "[TTM] " fmt - -#include <linux/list.h> -#include <linux/spinlock.h> -#include <linux/highmem.h> -#include <linux/mm_types.h> -#include <linux/module.h> -#include <linux/mm.h> -#include <linux/seq_file.h> /* for seq_printf */ -#include <linux/slab.h> -#include <linux/dma-mapping.h> - -#include <linux/atomic.h> - -#include <drm/ttm/ttm_bo_driver.h> -#include <drm/ttm/ttm_page_alloc.h> - -#include "ttm_set_memory.h" - -#define NUM_PAGES_TO_ALLOC (PAGE_SIZE/sizeof(struct page *)) -#define SMALL_ALLOCATION 16 -#define FREE_ALL_PAGES (~0U) -/* times are in msecs */ -#define PAGE_FREE_INTERVAL 1000 - -/** - * struct ttm_page_pool - Pool to reuse recently allocated uc/wc pages. - * - * @lock: Protects the shared pool from concurrnet access. Must be used with - * irqsave/irqrestore variants because pool allocator maybe called from - * delayed work. - * @fill_lock: Prevent concurrent calls to fill. - * @list: Pool of free uc/wc pages for fast reuse. - * @gfp_flags: Flags to pass for alloc_page. - * @npages: Number of pages in pool. - */ -struct ttm_page_pool { - spinlock_t lock; - bool fill_lock; - struct list_head list; - gfp_t gfp_flags; - unsigned npages; - char *name; - unsigned long nfrees; - unsigned long nrefills; - unsigned int order; -}; - -/** - * Limits for the pool. They are handled without locks because only place where - * they may change is in sysfs store. They won't have immediate effect anyway - * so forcing serialization to access them is pointless. - */ - -struct ttm_pool_opts { - unsigned alloc_size; - unsigned max_size; - unsigned small; -}; - -#define NUM_POOLS 6 - -/** - * struct ttm_pool_manager - Holds memory pools for fst allocation - * - * Manager is read only object for pool code so it doesn't need locking. - * - * @free_interval: minimum number of jiffies between freeing pages from pool. - * @page_alloc_inited: reference counting for pool allocation. - * @work: Work that is used to shrink the pool. Work is only run when there is - * some pages to free. - * @small_allocation: Limit in number of pages what is small allocation. - * - * @pools: All pool objects in use. - **/ -struct ttm_pool_manager { - struct kobject kobj; - struct shrinker mm_shrink; - struct ttm_pool_opts options; - - union { - struct ttm_page_pool pools[NUM_POOLS]; - struct { - struct ttm_page_pool wc_pool; - struct ttm_page_pool uc_pool; - struct ttm_page_pool wc_pool_dma32; - struct ttm_page_pool uc_pool_dma32; - struct ttm_page_pool wc_pool_huge; - struct ttm_page_pool uc_pool_huge; - } ; - }; -}; - -static struct attribute ttm_page_pool_max = { - .name = "pool_max_size", - .mode = S_IRUGO | S_IWUSR -}; -static struct attribute ttm_page_pool_small = { - .name = "pool_small_allocation", - .mode = S_IRUGO | S_IWUSR -}; -static struct attribute ttm_page_pool_alloc_size = { - .name = "pool_allocation_size", - .mode = S_IRUGO | S_IWUSR -}; - -static struct attribute *ttm_pool_attrs[] = { - &ttm_page_pool_max, - &ttm_page_pool_small, - &ttm_page_pool_alloc_size, - NULL -}; - -static void ttm_pool_kobj_release(struct kobject *kobj) -{ - struct ttm_pool_manager *m = - container_of(kobj, struct ttm_pool_manager, kobj); - kfree(m); -} - -static ssize_t ttm_pool_store(struct kobject *kobj, - struct attribute *attr, const char *buffer, size_t size) -{ - struct ttm_pool_manager *m = - container_of(kobj, struct ttm_pool_manager, kobj); - int chars; - unsigned val; - chars = sscanf(buffer, "%u", &val); - if (chars == 0) - return size; - - /* Convert kb to number of pages */ - val = val / (PAGE_SIZE >> 10); - - if (attr == &ttm_page_pool_max) - m->options.max_size = val; - else if (attr == &ttm_page_pool_small) - m->options.small = val; - else if (attr == &ttm_page_pool_alloc_size) { - if (val > NUM_PAGES_TO_ALLOC*8) { - pr_err("Setting allocation size to %lu is not allowed. Recommended size is %lu\n", - NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7), - NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); - return size; - } else if (val > NUM_PAGES_TO_ALLOC) { - pr_warn("Setting allocation size to larger than %lu is not recommended\n", - NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); - } - m->options.alloc_size = val; - } - - return size; -} - -static ssize_t ttm_pool_show(struct kobject *kobj, - struct attribute *attr, char *buffer) -{ - struct ttm_pool_manager *m = - container_of(kobj, struct ttm_pool_manager, kobj); - unsigned val = 0; - - if (attr == &ttm_page_pool_max) - val = m->options.max_size; - else if (attr == &ttm_page_pool_small) - val = m->options.small; - else if (attr == &ttm_page_pool_alloc_size) - val = m->options.alloc_size; - - val = val * (PAGE_SIZE >> 10); - - return snprintf(buffer, PAGE_SIZE, "%u\n", val); -} - -static const struct sysfs_ops ttm_pool_sysfs_ops = { - .show = &ttm_pool_show, - .store = &ttm_pool_store, -}; - -static struct kobj_type ttm_pool_kobj_type = { - .release = &ttm_pool_kobj_release, - .sysfs_ops = &ttm_pool_sysfs_ops, - .default_attrs = ttm_pool_attrs, -}; - -static struct ttm_pool_manager *_manager; - -/** - * Select the right pool or requested caching state and ttm flags. */ -static struct ttm_page_pool *ttm_get_pool(int flags, bool huge, - enum ttm_caching cstate) -{ - int pool_index; - - if (cstate == ttm_cached) - return NULL; - - if (cstate == ttm_write_combined) - pool_index = 0x0; - else - pool_index = 0x1; - - if (flags & TTM_PAGE_FLAG_DMA32) { - if (huge) - return NULL; - pool_index |= 0x2; - - } else if (huge) { - pool_index |= 0x4; - } - - return &_manager->pools[pool_index]; -} - -/* set memory back to wb and free the pages. */ -static void ttm_pages_put(struct page *pages[], unsigned npages, - unsigned int order) -{ - unsigned int i, pages_nr = (1 << order); - - if (order == 0) { - if (ttm_set_pages_array_wb(pages, npages)) - pr_err("Failed to set %d pages to wb!\n", npages); - } - - for (i = 0; i < npages; ++i) { - if (order > 0) { - if (ttm_set_pages_wb(pages[i], pages_nr)) - pr_err("Failed to set %d pages to wb!\n", pages_nr); - } - __free_pages(pages[i], order); - } -} - -static void ttm_pool_update_free_locked(struct ttm_page_pool *pool, - unsigned freed_pages) -{ - pool->npages -= freed_pages; - pool->nfrees += freed_pages; -} - -/** - * Free pages from pool. - * - * To prevent hogging the ttm_swap process we only free NUM_PAGES_TO_ALLOC - * number of pages in one go. - * - * @pool: to free the pages from - * @free_all: If set to true will free all pages in pool - * @use_static: Safe to use static buffer - **/ -static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free, - bool use_static) -{ - static struct page *static_buf[NUM_PAGES_TO_ALLOC]; - unsigned long irq_flags; - struct page *p; - struct page **pages_to_free; - unsigned freed_pages = 0, - npages_to_free = nr_free; - - if (NUM_PAGES_TO_ALLOC < nr_free) - npages_to_free = NUM_PAGES_TO_ALLOC; - - if (use_static) - pages_to_free = static_buf; - else - pages_to_free = kmalloc_array(npages_to_free, - sizeof(struct page *), - GFP_KERNEL); - if (!pages_to_free) { - pr_debug("Failed to allocate memory for pool free operation\n"); - return 0; - } - -restart: - spin_lock_irqsave(&pool->lock, irq_flags); - - list_for_each_entry_reverse(p, &pool->list, lru) { - if (freed_pages >= npages_to_free) - break; - - pages_to_free[freed_pages++] = p; - /* We can only remove NUM_PAGES_TO_ALLOC at a time. */ - if (freed_pages >= NUM_PAGES_TO_ALLOC) { - /* remove range of pages from the pool */ - __list_del(p->lru.prev, &pool->list); - - ttm_pool_update_free_locked(pool, freed_pages); - /** - * Because changing page caching is costly - * we unlock the pool to prevent stalling. - */ - spin_unlock_irqrestore(&pool->lock, irq_flags); - - ttm_pages_put(pages_to_free, freed_pages, pool->order); - if (likely(nr_free != FREE_ALL_PAGES)) - nr_free -= freed_pages; - - if (NUM_PAGES_TO_ALLOC >= nr_free) - npages_to_free = nr_free; - else - npages_to_free = NUM_PAGES_TO_ALLOC; - - freed_pages = 0; - - /* free all so restart the processing */ - if (nr_free) - goto restart; - - /* Not allowed to fall through or break because - * following context is inside spinlock while we are - * outside here. - */ - goto out; - - } - } - - /* remove range of pages from the pool */ - if (freed_pages) { - __list_del(&p->lru, &pool->list); - - ttm_pool_update_free_locked(pool, freed_pages); - nr_free -= freed_pages; - } - - spin_unlock_irqrestore(&pool->lock, irq_flags); - - if (freed_pages) - ttm_pages_put(pages_to_free, freed_pages, pool->order); -out: - if (pages_to_free != static_buf) - kfree(pages_to_free); - return nr_free; -} - -/** - * Callback for mm to request pool to reduce number of page held. - * - * XXX: (dchinner) Deadlock warning! - * - * This code is crying out for a shrinker per pool.... - */ -static unsigned long -ttm_pool_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) -{ - static DEFINE_MUTEX(lock); - static unsigned start_pool; - unsigned i; - unsigned pool_offset; - struct ttm_page_pool *pool; - int shrink_pages = sc->nr_to_scan; - unsigned long freed = 0; - unsigned int nr_free_pool; - - if (!mutex_trylock(&lock)) - return SHRINK_STOP; - pool_offset = ++start_pool % NUM_POOLS; - /* select start pool in round robin fashion */ - for (i = 0; i < NUM_POOLS; ++i) { - unsigned nr_free = shrink_pages; - unsigned page_nr; - - if (shrink_pages == 0) - break; - - pool = &_manager->pools[(i + pool_offset)%NUM_POOLS]; - page_nr = (1 << pool->order); - /* OK to use static buffer since global mutex is held. */ - nr_free_pool = roundup(nr_free, page_nr) >> pool->order; - shrink_pages = ttm_page_pool_free(pool, nr_free_pool, true); - freed += (nr_free_pool - shrink_pages) << pool->order; - if (freed >= sc->nr_to_scan) - break; - shrink_pages <<= pool->order; - } - mutex_unlock(&lock); - return freed; -} - - -static unsigned long -ttm_pool_shrink_count(struct shrinker *shrink, struct shrink_control *sc) -{ - unsigned i; - unsigned long count = 0; - struct ttm_page_pool *pool; - - for (i = 0; i < NUM_POOLS; ++i) { - pool = &_manager->pools[i]; - count += (pool->npages << pool->order); - } - - return count; -} - -static int ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager) -{ - manager->mm_shrink.count_objects = ttm_pool_shrink_count; - manager->mm_shrink.scan_objects = ttm_pool_shrink_scan; - manager->mm_shrink.seeks = 1; - return register_shrinker(&manager->mm_shrink); -} - -static void ttm_pool_mm_shrink_fini(struct ttm_pool_manager *manager) -{ - unregister_shrinker(&manager->mm_shrink); -} - -static int ttm_set_pages_caching(struct page **pages, - enum ttm_caching cstate, unsigned cpages) -{ - int r = 0; - /* Set page caching */ - switch (cstate) { - case ttm_uncached: - r = ttm_set_pages_array_uc(pages, cpages); - if (r) - pr_err("Failed to set %d pages to uc!\n", cpages); - break; - case ttm_write_combined: - r = ttm_set_pages_array_wc(pages, cpages); - if (r) - pr_err("Failed to set %d pages to wc!\n", cpages); - break; - default: - break; - } - return r; -} - -/** - * Free pages the pages that failed to change the caching state. If there is - * any pages that have changed their caching state already put them to the - * pool. - */ -static void ttm_handle_caching_failure(struct page **failed_pages, - unsigned cpages) -{ - unsigned i; - - /* Failed pages have to be freed */ - for (i = 0; i < cpages; ++i) { - list_del(&failed_pages[i]->lru); - __free_page(failed_pages[i]); - } -} - -/** - * Allocate new pages with correct caching. - * - * This function is reentrant if caller updates count depending on number of - * pages returned in pages array. - */ -static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags, - int ttm_flags, enum ttm_caching cstate, - unsigned count, unsigned order) -{ - struct page **caching_array; - struct page *p; - int r = 0; - unsigned i, j, cpages; - unsigned npages = 1 << order; - unsigned max_cpages = min(count << order, (unsigned)NUM_PAGES_TO_ALLOC); - - /* allocate array for page caching change */ - caching_array = kmalloc_array(max_cpages, sizeof(struct page *), - GFP_KERNEL); - - if (!caching_array) { - pr_debug("Unable to allocate table for new pages\n"); - return -ENOMEM; - } - - for (i = 0, cpages = 0; i < count; ++i) { - p = alloc_pages(gfp_flags, order); - - if (!p) { - pr_debug("Unable to get page %u\n", i); - - /* store already allocated pages in the pool after - * setting the caching state */ - if (cpages) { - r = ttm_set_pages_caching(caching_array, - cstate, cpages); - if (r) - ttm_handle_caching_failure(caching_array, - cpages); - } - r = -ENOMEM; - goto out; - } - - list_add(&p->lru, pages); - -#ifdef CONFIG_HIGHMEM - /* gfp flags of highmem page should never be dma32 so we - * we should be fine in such case - */ - if (PageHighMem(p)) - continue; - -#endif - for (j = 0; j < npages; ++j) { - caching_array[cpages++] = p++; - if (cpages == max_cpages) { - - r = ttm_set_pages_caching(caching_array, - cstate, cpages); - if (r) { - ttm_handle_caching_failure(caching_array, - cpages); - goto out; - } - cpages = 0; - } - } - } - - if (cpages) { - r = ttm_set_pages_caching(caching_array, cstate, cpages); - if (r) - ttm_handle_caching_failure(caching_array, cpages); - } -out: - kfree(caching_array); - - return r; -} - -/** - * Fill the given pool if there aren't enough pages and the requested number of - * pages is small. - */ -static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, int ttm_flags, - enum ttm_caching cstate, - unsigned count, unsigned long *irq_flags) -{ - struct page *p; - int r; - unsigned cpages = 0; - /** - * Only allow one pool fill operation at a time. - * If pool doesn't have enough pages for the allocation new pages are - * allocated from outside of pool. - */ - if (pool->fill_lock) - return; - - pool->fill_lock = true; - - /* If allocation request is small and there are not enough - * pages in a pool we fill the pool up first. */ - if (count < _manager->options.small - && count > pool->npages) { - struct list_head new_pages; - unsigned alloc_size = _manager->options.alloc_size; - - /** - * Can't change page caching if in irqsave context. We have to - * drop the pool->lock. - */ - spin_unlock_irqrestore(&pool->lock, *irq_flags); - - INIT_LIST_HEAD(&new_pages); - r = ttm_alloc_new_pages(&new_pages, pool->gfp_flags, ttm_flags, - cstate, alloc_size, 0); - spin_lock_irqsave(&pool->lock, *irq_flags); - - if (!r) { - list_splice(&new_pages, &pool->list); - ++pool->nrefills; - pool->npages += alloc_size; - } else { - pr_debug("Failed to fill pool (%p)\n", pool); - /* If we have any pages left put them to the pool. */ - list_for_each_entry(p, &new_pages, lru) { - ++cpages; - } - list_splice(&new_pages, &pool->list); - pool->npages += cpages; - } - - } - pool->fill_lock = false; -} - -/** - * Allocate pages from the pool and put them on the return list. - * - * @return zero for success or negative error code. - */ -static int ttm_page_pool_get_pages(struct ttm_page_pool *pool, - struct list_head *pages, - int ttm_flags, - enum ttm_caching cstate, - unsigned count, unsigned order) -{ - unsigned long irq_flags; - struct list_head *p; - unsigned i; - int r = 0; - - spin_lock_irqsave(&pool->lock, irq_flags); - if (!order) - ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count, - &irq_flags); - - if (count >= pool->npages) { - /* take all pages from the pool */ - list_splice_init(&pool->list, pages); - count -= pool->npages; - pool->npages = 0; - goto out; - } - /* find the last pages to include for requested number of pages. Split - * pool to begin and halve it to reduce search space. */ - if (count <= pool->npages/2) { - i = 0; - list_for_each(p, &pool->list) { - if (++i == count) - break; - } - } else { - i = pool->npages + 1; - list_for_each_prev(p, &pool->list) { - if (--i == count) - break; - } - } - /* Cut 'count' number of pages from the pool */ - list_cut_position(pages, &pool->list, p); - pool->npages -= count; - count = 0; -out: - spin_unlock_irqrestore(&pool->lock, irq_flags); - - /* clear the pages coming from the pool if requested */ - if (ttm_flags & TTM_PAGE_FLAG_ZERO_ALLOC) { - struct page *page; - - list_for_each_entry(page, pages, lru) { - if (PageHighMem(page)) - clear_highpage(page); - else - clear_page(page_address(page)); - } - } - - /* If pool didn't have enough pages allocate new one. */ - if (count) { - gfp_t gfp_flags = pool->gfp_flags; - - /* set zero flag for page allocation if required */ - if (ttm_flags & TTM_PAGE_FLAG_ZERO_ALLOC) - gfp_flags |= __GFP_ZERO; - - if (ttm_flags & TTM_PAGE_FLAG_NO_RETRY) - gfp_flags |= __GFP_RETRY_MAYFAIL; - - /* ttm_alloc_new_pages doesn't reference pool so we can run - * multiple requests in parallel. - **/ - r = ttm_alloc_new_pages(pages, gfp_flags, ttm_flags, cstate, - count, order); - } - - return r; -} - -/* Put all pages in pages list to correct pool to wait for reuse */ -static void ttm_put_pages(struct page **pages, unsigned npages, int flags, - enum ttm_caching cstate) -{ - struct ttm_page_pool *pool = ttm_get_pool(flags, false, cstate); -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - struct ttm_page_pool *huge = ttm_get_pool(flags, true, cstate); -#endif - unsigned long irq_flags; - unsigned i; - - if (pool == NULL) { - /* No pool for this memory type so free the pages */ - i = 0; - while (i < npages) { -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - struct page *p = pages[i]; -#endif - unsigned order = 0, j; - - if (!pages[i]) { - ++i; - continue; - } - -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - if (!(flags & TTM_PAGE_FLAG_DMA32) && - (npages - i) >= HPAGE_PMD_NR) { - for (j = 1; j < HPAGE_PMD_NR; ++j) - if (++p != pages[i + j]) - break; - - if (j == HPAGE_PMD_NR) - order = HPAGE_PMD_ORDER; - } -#endif - - if (page_count(pages[i]) != 1) - pr_err("Erroneous page count. Leaking pages.\n"); - __free_pages(pages[i], order); - - j = 1 << order; - while (j) { - pages[i++] = NULL; - --j; - } - } - return; - } - - i = 0; -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - if (huge) { - unsigned max_size, n2free; - - spin_lock_irqsave(&huge->lock, irq_flags); - while ((npages - i) >= HPAGE_PMD_NR) { - struct page *p = pages[i]; - unsigned j; - - if (!p) - break; - - for (j = 1; j < HPAGE_PMD_NR; ++j) - if (++p != pages[i + j]) - break; - - if (j != HPAGE_PMD_NR) - break; - - list_add_tail(&pages[i]->lru, &huge->list); - - for (j = 0; j < HPAGE_PMD_NR; ++j) - pages[i++] = NULL; - huge->npages++; - } - - /* Check that we don't go over the pool limit */ - max_size = _manager->options.max_size; - max_size /= HPAGE_PMD_NR; - if (huge->npages > max_size) - n2free = huge->npages - max_size; - else - n2free = 0; - spin_unlock_irqrestore(&huge->lock, irq_flags); - if (n2free) - ttm_page_pool_free(huge, n2free, false); - } -#endif - - spin_lock_irqsave(&pool->lock, irq_flags); - while (i < npages) { - if (pages[i]) { - if (page_count(pages[i]) != 1) - pr_err("Erroneous page count. Leaking pages.\n"); - list_add_tail(&pages[i]->lru, &pool->list); - pages[i] = NULL; - pool->npages++; - } - ++i; - } - /* Check that we don't go over the pool limit */ - npages = 0; - if (pool->npages > _manager->options.max_size) { - npages = pool->npages - _manager->options.max_size; - /* free at least NUM_PAGES_TO_ALLOC number of pages - * to reduce calls to set_memory_wb */ - if (npages < NUM_PAGES_TO_ALLOC) - npages = NUM_PAGES_TO_ALLOC; - } - spin_unlock_irqrestore(&pool->lock, irq_flags); - if (npages) - ttm_page_pool_free(pool, npages, false); -} - -/* - * On success pages list will hold count number of correctly - * cached pages. - */ -static int ttm_get_pages(struct page **pages, unsigned npages, int flags, - enum ttm_caching cstate) -{ - struct ttm_page_pool *pool = ttm_get_pool(flags, false, cstate); -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - struct ttm_page_pool *huge = ttm_get_pool(flags, true, cstate); -#endif - struct list_head plist; - struct page *p = NULL; - unsigned count, first; - int r; - - /* No pool for cached pages */ - if (pool == NULL) { - gfp_t gfp_flags = GFP_USER; - unsigned i; -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - unsigned j; -#endif - - /* set zero flag for page allocation if required */ - if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) - gfp_flags |= __GFP_ZERO; - - if (flags & TTM_PAGE_FLAG_NO_RETRY) - gfp_flags |= __GFP_RETRY_MAYFAIL; - - if (flags & TTM_PAGE_FLAG_DMA32) - gfp_flags |= GFP_DMA32; - else - gfp_flags |= GFP_HIGHUSER; - - i = 0; -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - if (!(gfp_flags & GFP_DMA32)) { - while (npages >= HPAGE_PMD_NR) { - gfp_t huge_flags = gfp_flags; - - huge_flags |= GFP_TRANSHUGE_LIGHT | __GFP_NORETRY | - __GFP_KSWAPD_RECLAIM; - huge_flags &= ~__GFP_MOVABLE; - huge_flags &= ~__GFP_COMP; - p = alloc_pages(huge_flags, HPAGE_PMD_ORDER); - if (!p) - break; - - for (j = 0; j < HPAGE_PMD_NR; ++j) - pages[i++] = p++; - - npages -= HPAGE_PMD_NR; - } - } -#endif - - first = i; - while (npages) { - p = alloc_page(gfp_flags); - if (!p) { - pr_debug("Unable to allocate page\n"); - return -ENOMEM; - } - - /* Swap the pages if we detect consecutive order */ - if (i > first && pages[i - 1] == p - 1) - swap(p, pages[i - 1]); - - pages[i++] = p; - --npages; - } - return 0; - } - - count = 0; - -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - if (huge && npages >= HPAGE_PMD_NR) { - INIT_LIST_HEAD(&plist); - ttm_page_pool_get_pages(huge, &plist, flags, cstate, - npages / HPAGE_PMD_NR, - HPAGE_PMD_ORDER); - - list_for_each_entry(p, &plist, lru) { - unsigned j; - - for (j = 0; j < HPAGE_PMD_NR; ++j) - pages[count++] = &p[j]; - } - } -#endif - - INIT_LIST_HEAD(&plist); - r = ttm_page_pool_get_pages(pool, &plist, flags, cstate, - npages - count, 0); - - first = count; - list_for_each_entry(p, &plist, lru) { - struct page *tmp = p; - - /* Swap the pages if we detect consecutive order */ - if (count > first && pages[count - 1] == tmp - 1) - swap(tmp, pages[count - 1]); - pages[count++] = tmp; - } - - if (r) { - /* If there is any pages in the list put them back to - * the pool. - */ - pr_debug("Failed to allocate extra pages for large request\n"); - ttm_put_pages(pages, count, flags, cstate); - return r; - } - - return 0; -} - -static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, gfp_t flags, - char *name, unsigned int order) -{ - spin_lock_init(&pool->lock); - pool->fill_lock = false; - INIT_LIST_HEAD(&pool->list); - pool->npages = pool->nfrees = 0; - pool->gfp_flags = flags; - pool->name = name; - pool->order = order; -} - -int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) -{ - int ret; -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - unsigned order = HPAGE_PMD_ORDER; -#else - unsigned order = 0; -#endif - - WARN_ON(_manager); - - pr_info("Initializing pool allocator\n"); - - _manager = kzalloc(sizeof(*_manager), GFP_KERNEL); - if (!_manager) - return -ENOMEM; - - ttm_page_pool_init_locked(&_manager->wc_pool, GFP_HIGHUSER, "wc", 0); - - ttm_page_pool_init_locked(&_manager->uc_pool, GFP_HIGHUSER, "uc", 0); - - ttm_page_pool_init_locked(&_manager->wc_pool_dma32, - GFP_USER | GFP_DMA32, "wc dma", 0); - - ttm_page_pool_init_locked(&_manager->uc_pool_dma32, - GFP_USER | GFP_DMA32, "uc dma", 0); - - ttm_page_pool_init_locked(&_manager->wc_pool_huge, - (GFP_TRANSHUGE_LIGHT | __GFP_NORETRY | - __GFP_KSWAPD_RECLAIM) & - ~(__GFP_MOVABLE | __GFP_COMP), - "wc huge", order); - - ttm_page_pool_init_locked(&_manager->uc_pool_huge, - (GFP_TRANSHUGE_LIGHT | __GFP_NORETRY | - __GFP_KSWAPD_RECLAIM) & - ~(__GFP_MOVABLE | __GFP_COMP) - , "uc huge", order); - - _manager->options.max_size = max_pages; - _manager->options.small = SMALL_ALLOCATION; - _manager->options.alloc_size = NUM_PAGES_TO_ALLOC; - - ret = kobject_init_and_add(&_manager->kobj, &ttm_pool_kobj_type, - &glob->kobj, "pool"); - if (unlikely(ret != 0)) - goto error; - - ret = ttm_pool_mm_shrink_init(_manager); - if (unlikely(ret != 0)) - goto error; - return 0; - -error: - kobject_put(&_manager->kobj); - _manager = NULL; - return ret; -} - -void ttm_page_alloc_fini(void) -{ - int i; - - pr_info("Finalizing pool allocator\n"); - ttm_pool_mm_shrink_fini(_manager); - - /* OK to use static buffer since global mutex is no longer used. */ - for (i = 0; i < NUM_POOLS; ++i) - ttm_page_pool_free(&_manager->pools[i], FREE_ALL_PAGES, true); - - kobject_put(&_manager->kobj); - _manager = NULL; -} - -static void -ttm_pool_unpopulate_helper(struct ttm_tt *ttm, unsigned mem_count_update) -{ - struct ttm_mem_global *mem_glob = &ttm_mem_glob; - unsigned i; - - if (mem_count_update == 0) - goto put_pages; - - for (i = 0; i < mem_count_update; ++i) { - if (!ttm->pages[i]) - continue; - - ttm_mem_global_free_page(mem_glob, ttm->pages[i], PAGE_SIZE); - } - -put_pages: - ttm_put_pages(ttm->pages, ttm->num_pages, ttm->page_flags, - ttm->caching); -} - -int ttm_pool_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx) -{ - struct ttm_mem_global *mem_glob = &ttm_mem_glob; - unsigned i; - int ret; - - if (ttm_tt_is_populated(ttm)) - return 0; - - if (ttm_check_under_lowerlimit(mem_glob, ttm->num_pages, ctx)) - return -ENOMEM; - - ret = ttm_get_pages(ttm->pages, ttm->num_pages, ttm->page_flags, - ttm->caching); - if (unlikely(ret != 0)) { - ttm_pool_unpopulate_helper(ttm, 0); - return ret; - } - - for (i = 0; i < ttm->num_pages; ++i) { - ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i], - PAGE_SIZE, ctx); - if (unlikely(ret != 0)) { - ttm_pool_unpopulate_helper(ttm, i); - return -ENOMEM; - } - } - - return 0; -} -EXPORT_SYMBOL(ttm_pool_populate); - -void ttm_pool_unpopulate(struct ttm_tt *ttm) -{ - ttm_pool_unpopulate_helper(ttm, ttm->num_pages); -} -EXPORT_SYMBOL(ttm_pool_unpopulate); - -int ttm_populate_and_map_pages(struct device *dev, struct ttm_tt *tt, - struct ttm_operation_ctx *ctx) -{ - unsigned i, j; - int r; - - r = ttm_pool_populate(tt, ctx); - if (r) - return r; - - for (i = 0; i < tt->num_pages; ++i) { - struct page *p = tt->pages[i]; - size_t num_pages = 1; - - for (j = i + 1; j < tt->num_pages; ++j) { - if (++p != tt->pages[j]) - break; - - ++num_pages; - } - - tt->dma_address[i] = dma_map_page(dev, tt->pages[i], - 0, num_pages * PAGE_SIZE, - DMA_BIDIRECTIONAL); - if (dma_mapping_error(dev, tt->dma_address[i])) { - while (i--) { - dma_unmap_page(dev, tt->dma_address[i], - PAGE_SIZE, DMA_BIDIRECTIONAL); - tt->dma_address[i] = 0; - } - ttm_pool_unpopulate(tt); - return -EFAULT; - } - - for (j = 1; j < num_pages; ++j) { - tt->dma_address[i + 1] = tt->dma_address[i] + PAGE_SIZE; - ++i; - } - } - return 0; -} -EXPORT_SYMBOL(ttm_populate_and_map_pages); - -void ttm_unmap_and_unpopulate_pages(struct device *dev, struct ttm_tt *tt) -{ - unsigned i, j; - - for (i = 0; i < tt->num_pages;) { - struct page *p = tt->pages[i]; - size_t num_pages = 1; - - if (!tt->dma_address[i] || !tt->pages[i]) { - ++i; - continue; - } - - for (j = i + 1; j < tt->num_pages; ++j) { - if (++p != tt->pages[j]) - break; - - ++num_pages; - } - - dma_unmap_page(dev, tt->dma_address[i], num_pages * PAGE_SIZE, - DMA_BIDIRECTIONAL); - - i += num_pages; - } - ttm_pool_unpopulate(tt); -} -EXPORT_SYMBOL(ttm_unmap_and_unpopulate_pages); - -int ttm_page_alloc_debugfs(struct seq_file *m, void *data) -{ - struct ttm_page_pool *p; - unsigned i; - char *h[] = {"pool", "refills", "pages freed", "size"}; - if (!_manager) { - seq_printf(m, "No pool allocator running.\n"); - return 0; - } - seq_printf(m, "%7s %12s %13s %8s\n", - h[0], h[1], h[2], h[3]); - for (i = 0; i < NUM_POOLS; ++i) { - p = &_manager->pools[i]; - - seq_printf(m, "%7s %12ld %13ld %8d\n", - p->name, p->nrefills, - p->nfrees, p->npages); - } - return 0; -} -EXPORT_SYMBOL(ttm_page_alloc_debugfs); diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c deleted file mode 100644 index c0353c25efd6..000000000000 --- a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c +++ /dev/null @@ -1,1226 +0,0 @@ -/* - * Copyright 2011 (c) Oracle Corp. - - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sub license, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the - * next paragraph) shall be included in all copies or substantial portions - * of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - * - * Author: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com> - */ - -/* - * A simple DMA pool losely based on dmapool.c. It has certain advantages - * over the DMA pools: - * - Pool collects resently freed pages for reuse (and hooks up to - * the shrinker). - * - Tracks currently in use pages - * - Tracks whether the page is UC, WB or cached (and reverts to WB - * when freed). - */ - -#define pr_fmt(fmt) "[TTM] " fmt - -#include <linux/dma-mapping.h> -#include <linux/list.h> -#include <linux/seq_file.h> /* for seq_printf */ -#include <linux/slab.h> -#include <linux/spinlock.h> -#include <linux/highmem.h> -#include <linux/mm_types.h> -#include <linux/module.h> -#include <linux/mm.h> -#include <linux/atomic.h> -#include <linux/device.h> -#include <linux/kthread.h> -#include <drm/ttm/ttm_bo_driver.h> -#include <drm/ttm/ttm_page_alloc.h> - -#include "ttm_set_memory.h" - -#define NUM_PAGES_TO_ALLOC (PAGE_SIZE/sizeof(struct page *)) -#define SMALL_ALLOCATION 4 -#define FREE_ALL_PAGES (~0U) -#define VADDR_FLAG_HUGE_POOL 1UL -#define VADDR_FLAG_UPDATED_COUNT 2UL - -enum pool_type { - IS_UNDEFINED = 0, - IS_WC = 1 << 1, - IS_UC = 1 << 2, - IS_CACHED = 1 << 3, - IS_DMA32 = 1 << 4, - IS_HUGE = 1 << 5 -}; - -/* - * The pool structure. There are up to nine pools: - * - generic (not restricted to DMA32): - * - write combined, uncached, cached. - * - dma32 (up to 2^32 - so up 4GB): - * - write combined, uncached, cached. - * - huge (not restricted to DMA32): - * - write combined, uncached, cached. - * for each 'struct device'. The 'cached' is for pages that are actively used. - * The other ones can be shrunk by the shrinker API if neccessary. - * @pools: The 'struct device->dma_pools' link. - * @type: Type of the pool - * @lock: Protects the free_list from concurrnet access. Must be - * used with irqsave/irqrestore variants because pool allocator maybe called - * from delayed work. - * @free_list: Pool of pages that are free to be used. No order requirements. - * @dev: The device that is associated with these pools. - * @size: Size used during DMA allocation. - * @npages_free: Count of available pages for re-use. - * @npages_in_use: Count of pages that are in use. - * @nfrees: Stats when pool is shrinking. - * @nrefills: Stats when the pool is grown. - * @gfp_flags: Flags to pass for alloc_page. - * @name: Name of the pool. - * @dev_name: Name derieved from dev - similar to how dev_info works. - * Used during shutdown as the dev_info during release is unavailable. - */ -struct dma_pool { - struct list_head pools; /* The 'struct device->dma_pools link */ - enum pool_type type; - spinlock_t lock; - struct list_head free_list; - struct device *dev; - unsigned size; - unsigned npages_free; - unsigned npages_in_use; - unsigned long nfrees; /* Stats when shrunk. */ - unsigned long nrefills; /* Stats when grown. */ - gfp_t gfp_flags; - char name[13]; /* "cached dma32" */ - char dev_name[64]; /* Constructed from dev */ -}; - -/* - * The accounting page keeping track of the allocated page along with - * the DMA address. - * @page_list: The link to the 'page_list' in 'struct dma_pool'. - * @vaddr: The virtual address of the page and a flag if the page belongs to a - * huge pool - * @dma: The bus address of the page. If the page is not allocated - * via the DMA API, it will be -1. - */ -struct dma_page { - struct list_head page_list; - unsigned long vaddr; - struct page *p; - dma_addr_t dma; -}; - -/* - * Limits for the pool. They are handled without locks because only place where - * they may change is in sysfs store. They won't have immediate effect anyway - * so forcing serialization to access them is pointless. - */ - -struct ttm_pool_opts { - unsigned alloc_size; - unsigned max_size; - unsigned small; -}; - -/* - * Contains the list of all of the 'struct device' and their corresponding - * DMA pools. Guarded by _mutex->lock. - * @pools: The link to 'struct ttm_pool_manager->pools' - * @dev: The 'struct device' associated with the 'pool' - * @pool: The 'struct dma_pool' associated with the 'dev' - */ -struct device_pools { - struct list_head pools; - struct device *dev; - struct dma_pool *pool; -}; - -/* - * struct ttm_pool_manager - Holds memory pools for fast allocation - * - * @lock: Lock used when adding/removing from pools - * @pools: List of 'struct device' and 'struct dma_pool' tuples. - * @options: Limits for the pool. - * @npools: Total amount of pools in existence. - * @shrinker: The structure used by [un|]register_shrinker - */ -struct ttm_pool_manager { - struct mutex lock; - struct list_head pools; - struct ttm_pool_opts options; - unsigned npools; - struct shrinker mm_shrink; - struct kobject kobj; -}; - -static struct ttm_pool_manager *_manager; - -static struct attribute ttm_page_pool_max = { - .name = "pool_max_size", - .mode = S_IRUGO | S_IWUSR -}; -static struct attribute ttm_page_pool_small = { - .name = "pool_small_allocation", - .mode = S_IRUGO | S_IWUSR -}; -static struct attribute ttm_page_pool_alloc_size = { - .name = "pool_allocation_size", - .mode = S_IRUGO | S_IWUSR -}; - -static struct attribute *ttm_pool_attrs[] = { - &ttm_page_pool_max, - &ttm_page_pool_small, - &ttm_page_pool_alloc_size, - NULL -}; - -static void ttm_pool_kobj_release(struct kobject *kobj) -{ - struct ttm_pool_manager *m = - container_of(kobj, struct ttm_pool_manager, kobj); - kfree(m); -} - -static ssize_t ttm_pool_store(struct kobject *kobj, struct attribute *attr, - const char *buffer, size_t size) -{ - struct ttm_pool_manager *m = - container_of(kobj, struct ttm_pool_manager, kobj); - int chars; - unsigned val; - - chars = sscanf(buffer, "%u", &val); - if (chars == 0) - return size; - - /* Convert kb to number of pages */ - val = val / (PAGE_SIZE >> 10); - - if (attr == &ttm_page_pool_max) { - m->options.max_size = val; - } else if (attr == &ttm_page_pool_small) { - m->options.small = val; - } else if (attr == &ttm_page_pool_alloc_size) { - if (val > NUM_PAGES_TO_ALLOC*8) { - pr_err("Setting allocation size to %lu is not allowed. Recommended size is %lu\n", - NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7), - NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); - return size; - } else if (val > NUM_PAGES_TO_ALLOC) { - pr_warn("Setting allocation size to larger than %lu is not recommended\n", - NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); - } - m->options.alloc_size = val; - } - - return size; -} - -static ssize_t ttm_pool_show(struct kobject *kobj, struct attribute *attr, - char *buffer) -{ - struct ttm_pool_manager *m = - container_of(kobj, struct ttm_pool_manager, kobj); - unsigned val = 0; - - if (attr == &ttm_page_pool_max) - val = m->options.max_size; - else if (attr == &ttm_page_pool_small) - val = m->options.small; - else if (attr == &ttm_page_pool_alloc_size) - val = m->options.alloc_size; - - val = val * (PAGE_SIZE >> 10); - - return snprintf(buffer, PAGE_SIZE, "%u\n", val); -} - -static const struct sysfs_ops ttm_pool_sysfs_ops = { - .show = &ttm_pool_show, - .store = &ttm_pool_store, -}; - -static struct kobj_type ttm_pool_kobj_type = { - .release = &ttm_pool_kobj_release, - .sysfs_ops = &ttm_pool_sysfs_ops, - .default_attrs = ttm_pool_attrs, -}; - -static int ttm_set_pages_caching(struct dma_pool *pool, - struct page **pages, unsigned cpages) -{ - int r = 0; - /* Set page caching */ - if (pool->type & IS_UC) { - r = ttm_set_pages_array_uc(pages, cpages); - if (r) - pr_err("%s: Failed to set %d pages to uc!\n", - pool->dev_name, cpages); - } - if (pool->type & IS_WC) { - r = ttm_set_pages_array_wc(pages, cpages); - if (r) - pr_err("%s: Failed to set %d pages to wc!\n", - pool->dev_name, cpages); - } - return r; -} - -static void __ttm_dma_free_page(struct dma_pool *pool, struct dma_page *d_page) -{ - unsigned long attrs = 0; - dma_addr_t dma = d_page->dma; - d_page->vaddr &= ~VADDR_FLAG_HUGE_POOL; - if (pool->type & IS_HUGE) - attrs = DMA_ATTR_NO_WARN; - - dma_free_attrs(pool->dev, pool->size, (void *)d_page->vaddr, dma, attrs); - - kfree(d_page); - d_page = NULL; -} -static struct dma_page *__ttm_dma_alloc_page(struct dma_pool *pool) -{ - struct dma_page *d_page; - unsigned long attrs = 0; - void *vaddr; - - d_page = kmalloc(sizeof(struct dma_page), GFP_KERNEL); - if (!d_page) - return NULL; - - if (pool->type & IS_HUGE) - attrs = DMA_ATTR_NO_WARN; - - vaddr = dma_alloc_attrs(pool->dev, pool->size, &d_page->dma, - pool->gfp_flags, attrs); - if (vaddr) { - if (is_vmalloc_addr(vaddr)) - d_page->p = vmalloc_to_page(vaddr); - else - d_page->p = virt_to_page(vaddr); - d_page->vaddr = (unsigned long)vaddr; - if (pool->type & IS_HUGE) - d_page->vaddr |= VADDR_FLAG_HUGE_POOL; - } else { - kfree(d_page); - d_page = NULL; - } - return d_page; -} -static enum pool_type ttm_to_type(int flags, enum ttm_caching cstate) -{ - enum pool_type type = IS_UNDEFINED; - - if (flags & TTM_PAGE_FLAG_DMA32) - type |= IS_DMA32; - if (cstate == ttm_cached) - type |= IS_CACHED; - else if (cstate == ttm_uncached) - type |= IS_UC; - else - type |= IS_WC; - - return type; -} - -static void ttm_pool_update_free_locked(struct dma_pool *pool, - unsigned freed_pages) -{ - pool->npages_free -= freed_pages; - pool->nfrees += freed_pages; - -} - -/* set memory back to wb and free the pages. */ -static void ttm_dma_page_put(struct dma_pool *pool, struct dma_page *d_page) -{ - struct page *page = d_page->p; - unsigned num_pages; - - /* Don't set WB on WB page pool. */ - if (!(pool->type & IS_CACHED)) { - num_pages = pool->size / PAGE_SIZE; - if (ttm_set_pages_wb(page, num_pages)) - pr_err("%s: Failed to set %d pages to wb!\n", - pool->dev_name, num_pages); - } - - list_del(&d_page->page_list); - __ttm_dma_free_page(pool, d_page); -} - -static void ttm_dma_pages_put(struct dma_pool *pool, struct list_head *d_pages, - struct page *pages[], unsigned npages) -{ - struct dma_page *d_page, *tmp; - - if (pool->type & IS_HUGE) { - list_for_each_entry_safe(d_page, tmp, d_pages, page_list) - ttm_dma_page_put(pool, d_page); - - return; - } - - /* Don't set WB on WB page pool. */ - if (npages && !(pool->type & IS_CACHED) && - ttm_set_pages_array_wb(pages, npages)) - pr_err("%s: Failed to set %d pages to wb!\n", - pool->dev_name, npages); - - list_for_each_entry_safe(d_page, tmp, d_pages, page_list) { - list_del(&d_page->page_list); - __ttm_dma_free_page(pool, d_page); - } -} - -/* - * Free pages from pool. - * - * To prevent hogging the ttm_swap process we only free NUM_PAGES_TO_ALLOC - * number of pages in one go. - * - * @pool: to free the pages from - * @nr_free: If set to true will free all pages in pool - * @use_static: Safe to use static buffer - **/ -static unsigned ttm_dma_page_pool_free(struct dma_pool *pool, unsigned nr_free, - bool use_static) -{ - static struct page *static_buf[NUM_PAGES_TO_ALLOC]; - unsigned long irq_flags; - struct dma_page *dma_p, *tmp; - struct page **pages_to_free; - struct list_head d_pages; - unsigned freed_pages = 0, - npages_to_free = nr_free; - - if (NUM_PAGES_TO_ALLOC < nr_free) - npages_to_free = NUM_PAGES_TO_ALLOC; - - if (use_static) - pages_to_free = static_buf; - else - pages_to_free = kmalloc_array(npages_to_free, - sizeof(struct page *), - GFP_KERNEL); - - if (!pages_to_free) { - pr_debug("%s: Failed to allocate memory for pool free operation\n", - pool->dev_name); - return 0; - } - INIT_LIST_HEAD(&d_pages); -restart: - spin_lock_irqsave(&pool->lock, irq_flags); - - /* We picking the oldest ones off the list */ - list_for_each_entry_safe_reverse(dma_p, tmp, &pool->free_list, - page_list) { - if (freed_pages >= npages_to_free) - break; - - /* Move the dma_page from one list to another. */ - list_move(&dma_p->page_list, &d_pages); - - pages_to_free[freed_pages++] = dma_p->p; - /* We can only remove NUM_PAGES_TO_ALLOC at a time. */ - if (freed_pages >= NUM_PAGES_TO_ALLOC) { - - ttm_pool_update_free_locked(pool, freed_pages); - /** - * Because changing page caching is costly - * we unlock the pool to prevent stalling. - */ - spin_unlock_irqrestore(&pool->lock, irq_flags); - - ttm_dma_pages_put(pool, &d_pages, pages_to_free, - freed_pages); - - INIT_LIST_HEAD(&d_pages); - - if (likely(nr_free != FREE_ALL_PAGES)) - nr_free -= freed_pages; - - if (NUM_PAGES_TO_ALLOC >= nr_free) - npages_to_free = nr_free; - else - npages_to_free = NUM_PAGES_TO_ALLOC; - - freed_pages = 0; - - /* free all so restart the processing */ - if (nr_free) - goto restart; - - /* Not allowed to fall through or break because - * following context is inside spinlock while we are - * outside here. - */ - goto out; - - } - } - - /* remove range of pages from the pool */ - if (freed_pages) { - ttm_pool_update_free_locked(pool, freed_pages); - nr_free -= freed_pages; - } - - spin_unlock_irqrestore(&pool->lock, irq_flags); - - if (freed_pages) - ttm_dma_pages_put(pool, &d_pages, pages_to_free, freed_pages); -out: - if (pages_to_free != static_buf) - kfree(pages_to_free); - return nr_free; -} - -static void ttm_dma_free_pool(struct device *dev, enum pool_type type) -{ - struct device_pools *p; - struct dma_pool *pool; - - if (!dev) - return; - - mutex_lock(&_manager->lock); - list_for_each_entry_reverse(p, &_manager->pools, pools) { - if (p->dev != dev) - continue; - pool = p->pool; - if (pool->type != type) - continue; - - list_del(&p->pools); - kfree(p); - _manager->npools--; - break; - } - list_for_each_entry_reverse(pool, &dev->dma_pools, pools) { - if (pool->type != type) - continue; - /* Takes a spinlock.. */ - /* OK to use static buffer since global mutex is held. */ - ttm_dma_page_pool_free(pool, FREE_ALL_PAGES, true); - WARN_ON(((pool->npages_in_use + pool->npages_free) != 0)); - /* This code path is called after _all_ references to the - * struct device has been dropped - so nobody should be - * touching it. In case somebody is trying to _add_ we are - * guarded by the mutex. */ - list_del(&pool->pools); - kfree(pool); - break; - } - mutex_unlock(&_manager->lock); -} - -/* - * On free-ing of the 'struct device' this deconstructor is run. - * Albeit the pool might have already been freed earlier. - */ -static void ttm_dma_pool_release(struct device *dev, void *res) -{ - struct dma_pool *pool = *(struct dma_pool **)res; - - if (pool) - ttm_dma_free_pool(dev, pool->type); -} - -static int ttm_dma_pool_match(struct device *dev, void *res, void *match_data) -{ - return *(struct dma_pool **)res == match_data; -} - -static struct dma_pool *ttm_dma_pool_init(struct device *dev, gfp_t flags, - enum pool_type type) -{ - const char *n[] = {"wc", "uc", "cached", " dma32", "huge"}; - enum pool_type t[] = {IS_WC, IS_UC, IS_CACHED, IS_DMA32, IS_HUGE}; - struct device_pools *sec_pool = NULL; - struct dma_pool *pool = NULL, **ptr; - unsigned i; - int ret = -ENODEV; - char *p; - - if (!dev) - return NULL; - - ptr = devres_alloc(ttm_dma_pool_release, sizeof(*ptr), GFP_KERNEL); - if (!ptr) - return NULL; - - ret = -ENOMEM; - - pool = kmalloc_node(sizeof(struct dma_pool), GFP_KERNEL, - dev_to_node(dev)); - if (!pool) - goto err_mem; - - sec_pool = kmalloc_node(sizeof(struct device_pools), GFP_KERNEL, - dev_to_node(dev)); - if (!sec_pool) - goto err_mem; - - INIT_LIST_HEAD(&sec_pool->pools); - sec_pool->dev = dev; - sec_pool->pool = pool; - - INIT_LIST_HEAD(&pool->free_list); - INIT_LIST_HEAD(&pool->pools); - spin_lock_init(&pool->lock); - pool->dev = dev; - pool->npages_free = pool->npages_in_use = 0; - pool->nfrees = 0; - pool->gfp_flags = flags; - if (type & IS_HUGE) -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - pool->size = HPAGE_PMD_SIZE; -#else - BUG(); -#endif - else - pool->size = PAGE_SIZE; - pool->type = type; - pool->nrefills = 0; - p = pool->name; - for (i = 0; i < ARRAY_SIZE(t); i++) { - if (type & t[i]) { - p += scnprintf(p, sizeof(pool->name) - (p - pool->name), - "%s", n[i]); - } - } - *p = 0; - /* We copy the name for pr_ calls b/c when dma_pool_destroy is called - * - the kobj->name has already been deallocated.*/ - snprintf(pool->dev_name, sizeof(pool->dev_name), "%s %s", - dev_driver_string(dev), dev_name(dev)); - mutex_lock(&_manager->lock); - /* You can get the dma_pool from either the global: */ - list_add(&sec_pool->pools, &_manager->pools); - _manager->npools++; - /* or from 'struct device': */ - list_add(&pool->pools, &dev->dma_pools); - mutex_unlock(&_manager->lock); - - *ptr = pool; - devres_add(dev, ptr); - - return pool; -err_mem: - devres_free(ptr); - kfree(sec_pool); - kfree(pool); - return ERR_PTR(ret); -} - -static struct dma_pool *ttm_dma_find_pool(struct device *dev, - enum pool_type type) -{ - struct dma_pool *pool, *tmp; - - if (type == IS_UNDEFINED) - return NULL; - - /* NB: We iterate on the 'struct dev' which has no spinlock, but - * it does have a kref which we have taken. The kref is taken during - * graphic driver loading - in the drm_pci_init it calls either - * pci_dev_get or pci_register_driver which both end up taking a kref - * on 'struct device'. - * - * On teardown, the graphic drivers end up quiescing the TTM (put_pages) - * and calls the dev_res deconstructors: ttm_dma_pool_release. The nice - * thing is at that point of time there are no pages associated with the - * driver so this function will not be called. - */ - list_for_each_entry_safe(pool, tmp, &dev->dma_pools, pools) - if (pool->type == type) - return pool; - return NULL; -} - -/* - * Free pages the pages that failed to change the caching state. If there - * are pages that have changed their caching state already put them to the - * pool. - */ -static void ttm_dma_handle_caching_failure(struct dma_pool *pool, - struct list_head *d_pages, - struct page **failed_pages, - unsigned cpages) -{ - struct dma_page *d_page, *tmp; - struct page *p; - unsigned i = 0; - - p = failed_pages[0]; - if (!p) - return; - /* Find the failed page. */ - list_for_each_entry_safe(d_page, tmp, d_pages, page_list) { - if (d_page->p != p) - continue; - /* .. and then progress over the full list. */ - list_del(&d_page->page_list); - __ttm_dma_free_page(pool, d_page); - if (++i < cpages) - p = failed_pages[i]; - else - break; - } - -} - -/* - * Allocate 'count' pages, and put 'need' number of them on the - * 'pages' and as well on the 'dma_address' starting at 'dma_offset' offset. - * The full list of pages should also be on 'd_pages'. - * We return zero for success, and negative numbers as errors. - */ -static int ttm_dma_pool_alloc_new_pages(struct dma_pool *pool, - struct list_head *d_pages, - unsigned count) -{ - struct page **caching_array; - struct dma_page *dma_p; - struct page *p; - int r = 0; - unsigned i, j, npages, cpages; - unsigned max_cpages = min(count, - (unsigned)(PAGE_SIZE/sizeof(struct page *))); - - /* allocate array for page caching change */ - caching_array = kmalloc_array(max_cpages, sizeof(struct page *), - GFP_KERNEL); - - if (!caching_array) { - pr_debug("%s: Unable to allocate table for new pages\n", - pool->dev_name); - return -ENOMEM; - } - - if (count > 1) - pr_debug("%s: (%s:%d) Getting %d pages\n", - pool->dev_name, pool->name, current->pid, count); - - for (i = 0, cpages = 0; i < count; ++i) { - dma_p = __ttm_dma_alloc_page(pool); - if (!dma_p) { - pr_debug("%s: Unable to get page %u\n", - pool->dev_name, i); - - /* store already allocated pages in the pool after - * setting the caching state */ - if (cpages) { - r = ttm_set_pages_caching(pool, caching_array, - cpages); - if (r) - ttm_dma_handle_caching_failure( - pool, d_pages, caching_array, - cpages); - } - r = -ENOMEM; - goto out; - } - p = dma_p->p; - list_add(&dma_p->page_list, d_pages); - -#ifdef CONFIG_HIGHMEM - /* gfp flags of highmem page should never be dma32 so we - * we should be fine in such case - */ - if (PageHighMem(p)) - continue; -#endif - - npages = pool->size / PAGE_SIZE; - for (j = 0; j < npages; ++j) { - caching_array[cpages++] = p + j; - if (cpages == max_cpages) { - /* Note: Cannot hold the spinlock */ - r = ttm_set_pages_caching(pool, caching_array, - cpages); - if (r) { - ttm_dma_handle_caching_failure( - pool, d_pages, caching_array, - cpages); - goto out; - } - cpages = 0; - } - } - } - - if (cpages) { - r = ttm_set_pages_caching(pool, caching_array, cpages); - if (r) - ttm_dma_handle_caching_failure(pool, d_pages, - caching_array, cpages); - } -out: - kfree(caching_array); - return r; -} - -/* - * @return count of pages still required to fulfill the request. - */ -static int ttm_dma_page_pool_fill_locked(struct dma_pool *pool, - unsigned long *irq_flags) -{ - unsigned count = _manager->options.small; - int r = pool->npages_free; - - if (count > pool->npages_free) { - struct list_head d_pages; - - INIT_LIST_HEAD(&d_pages); - - spin_unlock_irqrestore(&pool->lock, *irq_flags); - - /* Returns how many more are neccessary to fulfill the - * request. */ - r = ttm_dma_pool_alloc_new_pages(pool, &d_pages, count); - - spin_lock_irqsave(&pool->lock, *irq_flags); - if (!r) { - /* Add the fresh to the end.. */ - list_splice(&d_pages, &pool->free_list); - ++pool->nrefills; - pool->npages_free += count; - r = count; - } else { - struct dma_page *d_page; - unsigned cpages = 0; - - pr_debug("%s: Failed to fill %s pool (r:%d)!\n", - pool->dev_name, pool->name, r); - - list_for_each_entry(d_page, &d_pages, page_list) { - cpages++; - } - list_splice_tail(&d_pages, &pool->free_list); - pool->npages_free += cpages; - r = cpages; - } - } - return r; -} - -/* - * The populate list is actually a stack (not that is matters as TTM - * allocates one page at a time. - * return dma_page pointer if success, otherwise NULL. - */ -static struct dma_page *ttm_dma_pool_get_pages(struct dma_pool *pool, - struct ttm_tt *ttm, - unsigned index) -{ - struct dma_page *d_page = NULL; - unsigned long irq_flags; - int count; - - spin_lock_irqsave(&pool->lock, irq_flags); - count = ttm_dma_page_pool_fill_locked(pool, &irq_flags); - if (count) { - d_page = list_first_entry(&pool->free_list, struct dma_page, page_list); - ttm->pages[index] = d_page->p; - ttm->dma_address[index] = d_page->dma; - list_move_tail(&d_page->page_list, &ttm->pages_list); - pool->npages_in_use += 1; - pool->npages_free -= 1; - } - spin_unlock_irqrestore(&pool->lock, irq_flags); - return d_page; -} - -static gfp_t ttm_dma_pool_gfp_flags(struct ttm_tt *ttm, bool huge) -{ - gfp_t gfp_flags; - - if (ttm->page_flags & TTM_PAGE_FLAG_DMA32) - gfp_flags = GFP_USER | GFP_DMA32; - else - gfp_flags = GFP_HIGHUSER; - if (ttm->page_flags & TTM_PAGE_FLAG_ZERO_ALLOC) - gfp_flags |= __GFP_ZERO; - - if (huge) { - gfp_flags |= GFP_TRANSHUGE_LIGHT | __GFP_NORETRY | - __GFP_KSWAPD_RECLAIM; - gfp_flags &= ~__GFP_MOVABLE; - gfp_flags &= ~__GFP_COMP; - } - - if (ttm->page_flags & TTM_PAGE_FLAG_NO_RETRY) - gfp_flags |= __GFP_RETRY_MAYFAIL; - - return gfp_flags; -} - -/* - * On success pages list will hold count number of correctly - * cached pages. On failure will hold the negative return value (-ENOMEM, etc). - */ -int ttm_dma_populate(struct ttm_tt *ttm, struct device *dev, - struct ttm_operation_ctx *ctx) -{ - struct ttm_mem_global *mem_glob = &ttm_mem_glob; - unsigned long num_pages = ttm->num_pages; - struct dma_pool *pool; - struct dma_page *d_page; - enum pool_type type; - unsigned i; - int ret; - - if (ttm_tt_is_populated(ttm)) - return 0; - - if (ttm_check_under_lowerlimit(mem_glob, num_pages, ctx)) - return -ENOMEM; - - INIT_LIST_HEAD(&ttm->pages_list); - i = 0; - - type = ttm_to_type(ttm->page_flags, ttm->caching); - -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - if (ttm->page_flags & TTM_PAGE_FLAG_DMA32) - goto skip_huge; - - pool = ttm_dma_find_pool(dev, type | IS_HUGE); - if (!pool) { - gfp_t gfp_flags = ttm_dma_pool_gfp_flags(ttm, true); - - pool = ttm_dma_pool_init(dev, gfp_flags, type | IS_HUGE); - if (IS_ERR_OR_NULL(pool)) - goto skip_huge; - } - - while (num_pages >= HPAGE_PMD_NR) { - unsigned j; - - d_page = ttm_dma_pool_get_pages(pool, ttm, i); - if (!d_page) - break; - - ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i], - pool->size, ctx); - if (unlikely(ret != 0)) { - ttm_dma_unpopulate(ttm, dev); - return -ENOMEM; - } - - d_page->vaddr |= VADDR_FLAG_UPDATED_COUNT; - for (j = i + 1; j < (i + HPAGE_PMD_NR); ++j) { - ttm->pages[j] = ttm->pages[j - 1] + 1; - ttm->dma_address[j] = ttm->dma_address[j - 1] + - PAGE_SIZE; - } - - i += HPAGE_PMD_NR; - num_pages -= HPAGE_PMD_NR; - } - -skip_huge: -#endif - - pool = ttm_dma_find_pool(dev, type); - if (!pool) { - gfp_t gfp_flags = ttm_dma_pool_gfp_flags(ttm, false); - - pool = ttm_dma_pool_init(dev, gfp_flags, type); - if (IS_ERR_OR_NULL(pool)) - return -ENOMEM; - } - - while (num_pages) { - d_page = ttm_dma_pool_get_pages(pool, ttm, i); - if (!d_page) { - ttm_dma_unpopulate(ttm, dev); - return -ENOMEM; - } - - ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i], - pool->size, ctx); - if (unlikely(ret != 0)) { - ttm_dma_unpopulate(ttm, dev); - return -ENOMEM; - } - - d_page->vaddr |= VADDR_FLAG_UPDATED_COUNT; - ++i; - --num_pages; - } - - return 0; -} -EXPORT_SYMBOL_GPL(ttm_dma_populate); - -/* Put all pages in pages list to correct pool to wait for reuse */ -void ttm_dma_unpopulate(struct ttm_tt *ttm, struct device *dev) -{ - struct ttm_mem_global *mem_glob = &ttm_mem_glob; - struct dma_pool *pool; - struct dma_page *d_page, *next; - enum pool_type type; - bool is_cached = false; - unsigned count, i, npages = 0; - unsigned long irq_flags; - - type = ttm_to_type(ttm->page_flags, ttm->caching); - -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - pool = ttm_dma_find_pool(dev, type | IS_HUGE); - if (pool) { - count = 0; - list_for_each_entry_safe(d_page, next, &ttm->pages_list, - page_list) { - if (!(d_page->vaddr & VADDR_FLAG_HUGE_POOL)) - continue; - - count++; - if (d_page->vaddr & VADDR_FLAG_UPDATED_COUNT) { - ttm_mem_global_free_page(mem_glob, d_page->p, - pool->size); - d_page->vaddr &= ~VADDR_FLAG_UPDATED_COUNT; - } - ttm_dma_page_put(pool, d_page); - } - - spin_lock_irqsave(&pool->lock, irq_flags); - pool->npages_in_use -= count; - pool->nfrees += count; - spin_unlock_irqrestore(&pool->lock, irq_flags); - } -#endif - - pool = ttm_dma_find_pool(dev, type); - if (!pool) - return; - - is_cached = (ttm_dma_find_pool(pool->dev, - ttm_to_type(ttm->page_flags, ttm_cached)) == pool); - - /* make sure pages array match list and count number of pages */ - count = 0; - list_for_each_entry_safe(d_page, next, &ttm->pages_list, - page_list) { - ttm->pages[count] = d_page->p; - count++; - - if (d_page->vaddr & VADDR_FLAG_UPDATED_COUNT) { - ttm_mem_global_free_page(mem_glob, d_page->p, - pool->size); - d_page->vaddr &= ~VADDR_FLAG_UPDATED_COUNT; - } - - if (is_cached) - ttm_dma_page_put(pool, d_page); - } - - spin_lock_irqsave(&pool->lock, irq_flags); - pool->npages_in_use -= count; - if (is_cached) { - pool->nfrees += count; - } else { - pool->npages_free += count; - list_splice(&ttm->pages_list, &pool->free_list); - /* - * Wait to have at at least NUM_PAGES_TO_ALLOC number of pages - * to free in order to minimize calls to set_memory_wb(). - */ - if (pool->npages_free >= (_manager->options.max_size + - NUM_PAGES_TO_ALLOC)) - npages = pool->npages_free - _manager->options.max_size; - } - spin_unlock_irqrestore(&pool->lock, irq_flags); - - INIT_LIST_HEAD(&ttm->pages_list); - for (i = 0; i < ttm->num_pages; i++) { - ttm->pages[i] = NULL; - ttm->dma_address[i] = 0; - } - - /* shrink pool if necessary (only on !is_cached pools)*/ - if (npages) - ttm_dma_page_pool_free(pool, npages, false); -} -EXPORT_SYMBOL_GPL(ttm_dma_unpopulate); - -/** - * Callback for mm to request pool to reduce number of page held. - * - * XXX: (dchinner) Deadlock warning! - * - * I'm getting sadder as I hear more pathetical whimpers about needing per-pool - * shrinkers - */ -static unsigned long -ttm_dma_pool_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) -{ - static unsigned start_pool; - unsigned idx = 0; - unsigned pool_offset; - unsigned shrink_pages = sc->nr_to_scan; - struct device_pools *p; - unsigned long freed = 0; - - if (list_empty(&_manager->pools)) - return SHRINK_STOP; - - if (!mutex_trylock(&_manager->lock)) - return SHRINK_STOP; - if (!_manager->npools) - goto out; - pool_offset = ++start_pool % _manager->npools; - list_for_each_entry(p, &_manager->pools, pools) { - unsigned nr_free; - - if (!p->dev) - continue; - if (shrink_pages == 0) - break; - /* Do it in round-robin fashion. */ - if (++idx < pool_offset) - continue; - nr_free = shrink_pages; - /* OK to use static buffer since global mutex is held. */ - shrink_pages = ttm_dma_page_pool_free(p->pool, nr_free, true); - freed += nr_free - shrink_pages; - - pr_debug("%s: (%s:%d) Asked to shrink %d, have %d more to go\n", - p->pool->dev_name, p->pool->name, current->pid, - nr_free, shrink_pages); - } -out: - mutex_unlock(&_manager->lock); - return freed; -} - -static unsigned long -ttm_dma_pool_shrink_count(struct shrinker *shrink, struct shrink_control *sc) -{ - struct device_pools *p; - unsigned long count = 0; - - if (!mutex_trylock(&_manager->lock)) - return 0; - list_for_each_entry(p, &_manager->pools, pools) - count += p->pool->npages_free; - mutex_unlock(&_manager->lock); - return count; -} - -static int ttm_dma_pool_mm_shrink_init(struct ttm_pool_manager *manager) -{ - manager->mm_shrink.count_objects = ttm_dma_pool_shrink_count; - manager->mm_shrink.scan_objects = &ttm_dma_pool_shrink_scan; - manager->mm_shrink.seeks = 1; - return register_shrinker(&manager->mm_shrink); -} - -static void ttm_dma_pool_mm_shrink_fini(struct ttm_pool_manager *manager) -{ - unregister_shrinker(&manager->mm_shrink); -} - -int ttm_dma_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) -{ - int ret; - - WARN_ON(_manager); - - pr_info("Initializing DMA pool allocator\n"); - - _manager = kzalloc(sizeof(*_manager), GFP_KERNEL); - if (!_manager) - return -ENOMEM; - - mutex_init(&_manager->lock); - INIT_LIST_HEAD(&_manager->pools); - - _manager->options.max_size = max_pages; - _manager->options.small = SMALL_ALLOCATION; - _manager->options.alloc_size = NUM_PAGES_TO_ALLOC; - - /* This takes care of auto-freeing the _manager */ - ret = kobject_init_and_add(&_manager->kobj, &ttm_pool_kobj_type, - &glob->kobj, "dma_pool"); - if (unlikely(ret != 0)) - goto error; - - ret = ttm_dma_pool_mm_shrink_init(_manager); - if (unlikely(ret != 0)) - goto error; - return 0; - -error: - kobject_put(&_manager->kobj); - _manager = NULL; - return ret; -} - -void ttm_dma_page_alloc_fini(void) -{ - struct device_pools *p, *t; - - pr_info("Finalizing DMA pool allocator\n"); - ttm_dma_pool_mm_shrink_fini(_manager); - - list_for_each_entry_safe_reverse(p, t, &_manager->pools, pools) { - dev_dbg(p->dev, "(%s:%d) Freeing.\n", p->pool->name, - current->pid); - WARN_ON(devres_destroy(p->dev, ttm_dma_pool_release, - ttm_dma_pool_match, p->pool)); - ttm_dma_free_pool(p->dev, p->pool->type); - } - kobject_put(&_manager->kobj); - _manager = NULL; -} - -int ttm_dma_page_alloc_debugfs(struct seq_file *m, void *data) -{ - struct device_pools *p; - struct dma_pool *pool = NULL; - - if (!_manager) { - seq_printf(m, "No pool allocator running.\n"); - return 0; - } - seq_printf(m, " pool refills pages freed inuse available name\n"); - mutex_lock(&_manager->lock); - list_for_each_entry(p, &_manager->pools, pools) { - struct device *dev = p->dev; - if (!dev) - continue; - pool = p->pool; - seq_printf(m, "%13s %12ld %13ld %8d %8d %8s\n", - pool->name, pool->nrefills, - pool->nfrees, pool->npages_in_use, - pool->npages_free, - pool->dev_name); - } - mutex_unlock(&_manager->lock); - return 0; -} -EXPORT_SYMBOL_GPL(ttm_dma_page_alloc_debugfs); diff --git a/drivers/gpu/drm/ttm/ttm_set_memory.h b/drivers/gpu/drm/ttm/ttm_set_memory.h deleted file mode 100644 index 2343c18a6133..000000000000 --- a/drivers/gpu/drm/ttm/ttm_set_memory.h +++ /dev/null @@ -1,84 +0,0 @@ -/************************************************************************** - * - * Copyright (c) 2018 Advanced Micro Devices, Inc. - * All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sub license, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice (including the - * next paragraph) shall be included in all copies or substantial portions - * of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, - * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE - * USE OR OTHER DEALINGS IN THE SOFTWARE. - * - **************************************************************************/ -/* - * Authors: Huang Rui <ray.huang@amd.com> - */ - -#ifndef TTM_SET_MEMORY -#define TTM_SET_MEMORY - -#include <linux/mm.h> - -#ifdef CONFIG_X86 - -#include <asm/set_memory.h> - -static inline int ttm_set_pages_array_wb(struct page **pages, int addrinarray) -{ - return set_pages_array_wb(pages, addrinarray); -} - -static inline int ttm_set_pages_array_wc(struct page **pages, int addrinarray) -{ - return set_pages_array_wc(pages, addrinarray); -} - -static inline int ttm_set_pages_array_uc(struct page **pages, int addrinarray) -{ - return set_pages_array_uc(pages, addrinarray); -} - -static inline int ttm_set_pages_wb(struct page *page, int numpages) -{ - return set_pages_wb(page, numpages); -} - -#else /* for CONFIG_X86 */ - -static inline int ttm_set_pages_array_wb(struct page **pages, int addrinarray) -{ - return 0; -} - -static inline int ttm_set_pages_array_wc(struct page **pages, int addrinarray) -{ - return 0; -} - -static inline int ttm_set_pages_array_uc(struct page **pages, int addrinarray) -{ - return 0; -} - -static inline int ttm_set_pages_wb(struct page *page, int numpages) -{ - return 0; -} - -#endif /* for CONFIG_X86 */ - -#endif diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c index 90054280cd8f..8861a74ac335 100644 --- a/drivers/gpu/drm/ttm/ttm_tt.c +++ b/drivers/gpu/drm/ttm/ttm_tt.c @@ -51,9 +51,6 @@ int ttm_tt_create(struct ttm_buffer_object *bo, bool zero_alloc) if (bo->ttm) return 0; - if (bdev->need_dma32) - page_flags |= TTM_PAGE_FLAG_DMA32; - if (bdev->no_retry) page_flags |= TTM_PAGE_FLAG_NO_RETRY; @@ -141,7 +138,6 @@ static void ttm_tt_init_fields(struct ttm_tt *ttm, ttm->dma_address = NULL; ttm->swap_storage = NULL; ttm->sg = bo->sg; - INIT_LIST_HEAD(&ttm->pages_list); ttm->caching = caching; } |