diff options
Diffstat (limited to 'drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm_bo.c')
-rw-r--r-- | drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm_bo.c | 1543 |
1 files changed, 1543 insertions, 0 deletions
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm_bo.c b/drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm_bo.c new file mode 100644 index 000000000000..40ac3582fb7a --- /dev/null +++ b/drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm_bo.c @@ -0,0 +1,1543 @@ +/* + * Support for Medifield PNW Camera Imaging ISP subsystem. + * + * Copyright (c) 2010 Intel Corporation. All Rights Reserved. + * + * Copyright (c) 2010 Silicon Hive www.siliconhive.com. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License version + * 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + * + */ +/* + * This file contains functions for buffer object structure management + */ +#include <linux/kernel.h> +#include <linux/types.h> +#include <linux/gfp.h> /* for GFP_ATOMIC */ +#include <linux/mm.h> +#include <linux/mm_types.h> +#include <linux/hugetlb.h> +#include <linux/highmem.h> +#include <linux/slab.h> /* for kmalloc */ +#include <linux/module.h> +#include <linux/moduleparam.h> +#include <linux/string.h> +#include <linux/list.h> +#include <linux/errno.h> +#include <asm/cacheflush.h> +#include <linux/io.h> +#include <asm/current.h> +#include <linux/sched/signal.h> +#include <linux/file.h> + +#include "atomisp_internal.h" +#include "hmm/hmm_common.h" +#include "hmm/hmm_pool.h" +#include "hmm/hmm_bo.h" + +static unsigned int order_to_nr(unsigned int order) +{ + return 1U << order; +} + +static unsigned int nr_to_order_bottom(unsigned int nr) +{ + return fls(nr) - 1; +} + +struct hmm_buffer_object *__bo_alloc(struct kmem_cache *bo_cache) +{ + struct hmm_buffer_object *bo; + + bo = kmem_cache_alloc(bo_cache, GFP_KERNEL); + if (!bo) + dev_err(atomisp_dev, "%s: failed!\n", __func__); + + return bo; +} + +static int __bo_init(struct hmm_bo_device *bdev, struct hmm_buffer_object *bo, + unsigned int pgnr) +{ + check_bodev_null_return(bdev, -EINVAL); + var_equal_return(hmm_bo_device_inited(bdev), 0, -EINVAL, + "hmm_bo_device not inited yet.\n"); + /* prevent zero size buffer object */ + if (pgnr == 0) { + dev_err(atomisp_dev, "0 size buffer is not allowed.\n"); + return -EINVAL; + } + + memset(bo, 0, sizeof(*bo)); + mutex_init(&bo->mutex); + + /* init the bo->list HEAD as an element of entire_bo_list */ + INIT_LIST_HEAD(&bo->list); + + bo->bdev = bdev; + bo->vmap_addr = NULL; + bo->status = HMM_BO_FREE; + bo->start = bdev->start; + bo->pgnr = pgnr; + bo->end = bo->start + pgnr_to_size(pgnr); + bo->prev = NULL; + bo->next = NULL; + + return 0; +} + +struct hmm_buffer_object *__bo_search_and_remove_from_free_rbtree( + struct rb_node *node, unsigned int pgnr) +{ + struct hmm_buffer_object *this, *ret_bo, *temp_bo; + + this = rb_entry(node, struct hmm_buffer_object, node); + if (this->pgnr == pgnr || + (this->pgnr > pgnr && this->node.rb_left == NULL)) { + goto remove_bo_and_return; + } else { + if (this->pgnr < pgnr) { + if (!this->node.rb_right) + return NULL; + ret_bo = __bo_search_and_remove_from_free_rbtree( + this->node.rb_right, pgnr); + } else { + ret_bo = __bo_search_and_remove_from_free_rbtree( + this->node.rb_left, pgnr); + } + if (!ret_bo) { + if (this->pgnr > pgnr) + goto remove_bo_and_return; + else + return NULL; + } + return ret_bo; + } + +remove_bo_and_return: + /* NOTE: All nodes on free rbtree have a 'prev' that points to NULL. + * 1. check if 'this->next' is NULL: + * yes: erase 'this' node and rebalance rbtree, return 'this'. + */ + if (this->next == NULL) { + rb_erase(&this->node, &this->bdev->free_rbtree); + return this; + } + /* NOTE: if 'this->next' is not NULL, always return 'this->next' bo. + * 2. check if 'this->next->next' is NULL: + * yes: change the related 'next/prev' pointer, + * return 'this->next' but the rbtree stays unchanged. + */ + temp_bo = this->next; + this->next = temp_bo->next; + if (temp_bo->next) + temp_bo->next->prev = this; + temp_bo->next = NULL; + temp_bo->prev = NULL; + return temp_bo; +} + +struct hmm_buffer_object *__bo_search_by_addr(struct rb_root *root, + ia_css_ptr start) +{ + struct rb_node *n = root->rb_node; + struct hmm_buffer_object *bo; + + do { + bo = rb_entry(n, struct hmm_buffer_object, node); + + if (bo->start > start) { + if (n->rb_left == NULL) + return NULL; + n = n->rb_left; + } else if (bo->start < start) { + if (n->rb_right == NULL) + return NULL; + n = n->rb_right; + } else { + return bo; + } + } while (n); + + return NULL; +} + +struct hmm_buffer_object *__bo_search_by_addr_in_range(struct rb_root *root, + unsigned int start) +{ + struct rb_node *n = root->rb_node; + struct hmm_buffer_object *bo; + + do { + bo = rb_entry(n, struct hmm_buffer_object, node); + + if (bo->start > start) { + if (n->rb_left == NULL) + return NULL; + n = n->rb_left; + } else { + if (bo->end > start) + return bo; + if (n->rb_right == NULL) + return NULL; + n = n->rb_right; + } + } while (n); + + return NULL; +} + +static void __bo_insert_to_free_rbtree(struct rb_root *root, + struct hmm_buffer_object *bo) +{ + struct rb_node **new = &(root->rb_node); + struct rb_node *parent = NULL; + struct hmm_buffer_object *this; + unsigned int pgnr = bo->pgnr; + + while (*new) { + parent = *new; + this = container_of(*new, struct hmm_buffer_object, node); + + if (pgnr < this->pgnr) { + new = &((*new)->rb_left); + } else if (pgnr > this->pgnr) { + new = &((*new)->rb_right); + } else { + bo->prev = this; + bo->next = this->next; + if (this->next) + this->next->prev = bo; + this->next = bo; + bo->status = (bo->status & ~HMM_BO_MASK) | HMM_BO_FREE; + return; + } + } + + bo->status = (bo->status & ~HMM_BO_MASK) | HMM_BO_FREE; + + rb_link_node(&bo->node, parent, new); + rb_insert_color(&bo->node, root); +} + +static void __bo_insert_to_alloc_rbtree(struct rb_root *root, + struct hmm_buffer_object *bo) +{ + struct rb_node **new = &(root->rb_node); + struct rb_node *parent = NULL; + struct hmm_buffer_object *this; + unsigned int start = bo->start; + + while (*new) { + parent = *new; + this = container_of(*new, struct hmm_buffer_object, node); + + if (start < this->start) + new = &((*new)->rb_left); + else + new = &((*new)->rb_right); + } + + kref_init(&bo->kref); + bo->status = (bo->status & ~HMM_BO_MASK) | HMM_BO_ALLOCED; + + rb_link_node(&bo->node, parent, new); + rb_insert_color(&bo->node, root); +} + +struct hmm_buffer_object *__bo_break_up(struct hmm_bo_device *bdev, + struct hmm_buffer_object *bo, + unsigned int pgnr) +{ + struct hmm_buffer_object *new_bo; + unsigned long flags; + int ret; + + new_bo = __bo_alloc(bdev->bo_cache); + if (!new_bo) { + dev_err(atomisp_dev, "%s: __bo_alloc failed!\n", __func__); + return NULL; + } + ret = __bo_init(bdev, new_bo, pgnr); + if (ret) { + dev_err(atomisp_dev, "%s: __bo_init failed!\n", __func__); + kmem_cache_free(bdev->bo_cache, new_bo); + return NULL; + } + + new_bo->start = bo->start; + new_bo->end = new_bo->start + pgnr_to_size(pgnr); + bo->start = new_bo->end; + bo->pgnr = bo->pgnr - pgnr; + + spin_lock_irqsave(&bdev->list_lock, flags); + list_add_tail(&new_bo->list, &bo->list); + spin_unlock_irqrestore(&bdev->list_lock, flags); + + return new_bo; +} + +static void __bo_take_off_handling(struct hmm_buffer_object *bo) +{ + struct hmm_bo_device *bdev = bo->bdev; + /* There are 4 situations when we take off a known bo from free rbtree: + * 1. if bo->next && bo->prev == NULL, bo is a rbtree node + * and does not have a linked list after bo, to take off this bo, + * we just need erase bo directly and rebalance the free rbtree + */ + if (bo->prev == NULL && bo->next == NULL) { + rb_erase(&bo->node, &bdev->free_rbtree); + /* 2. when bo->next != NULL && bo->prev == NULL, bo is a rbtree node, + * and has a linked list,to take off this bo we need erase bo + * first, then, insert bo->next into free rbtree and rebalance + * the free rbtree + */ + } else if (bo->prev == NULL && bo->next != NULL) { + bo->next->prev = NULL; + rb_erase(&bo->node, &bdev->free_rbtree); + __bo_insert_to_free_rbtree(&bdev->free_rbtree, bo->next); + bo->next = NULL; + /* 3. when bo->prev != NULL && bo->next == NULL, bo is not a rbtree + * node, bo is the last element of the linked list after rbtree + * node, to take off this bo, we just need set the "prev/next" + * pointers to NULL, the free rbtree stays unchaged + */ + } else if (bo->prev != NULL && bo->next == NULL) { + bo->prev->next = NULL; + bo->prev = NULL; + /* 4. when bo->prev != NULL && bo->next != NULL ,bo is not a rbtree + * node, bo is in the middle of the linked list after rbtree node, + * to take off this bo, we just set take the "prev/next" pointers + * to NULL, the free rbtree stays unchaged + */ + } else { + bo->next->prev = bo->prev; + bo->prev->next = bo->next; + bo->next = NULL; + bo->prev = NULL; + } +} + +struct hmm_buffer_object *__bo_merge(struct hmm_buffer_object *bo, + struct hmm_buffer_object *next_bo) +{ + struct hmm_bo_device *bdev; + unsigned long flags; + + bdev = bo->bdev; + next_bo->start = bo->start; + next_bo->pgnr = next_bo->pgnr + bo->pgnr; + + spin_lock_irqsave(&bdev->list_lock, flags); + list_del(&bo->list); + spin_unlock_irqrestore(&bdev->list_lock, flags); + + kmem_cache_free(bo->bdev->bo_cache, bo); + + return next_bo; +} + +/* + * hmm_bo_device functions. + */ +int hmm_bo_device_init(struct hmm_bo_device *bdev, + struct isp_mmu_client *mmu_driver, + unsigned int vaddr_start, + unsigned int size) +{ + struct hmm_buffer_object *bo; + unsigned long flags; + int ret; + + check_bodev_null_return(bdev, -EINVAL); + + ret = isp_mmu_init(&bdev->mmu, mmu_driver); + if (ret) { + dev_err(atomisp_dev, "isp_mmu_init failed.\n"); + return ret; + } + + bdev->start = vaddr_start; + bdev->pgnr = size_to_pgnr_ceil(size); + bdev->size = pgnr_to_size(bdev->pgnr); + + spin_lock_init(&bdev->list_lock); + mutex_init(&bdev->rbtree_mutex); + + bdev->flag = HMM_BO_DEVICE_INITED; + + INIT_LIST_HEAD(&bdev->entire_bo_list); + bdev->allocated_rbtree = RB_ROOT; + bdev->free_rbtree = RB_ROOT; + + bdev->bo_cache = kmem_cache_create("bo_cache", + sizeof(struct hmm_buffer_object), 0, 0, NULL); + if (!bdev->bo_cache) { + dev_err(atomisp_dev, "%s: create cache failed!\n", __func__); + isp_mmu_exit(&bdev->mmu); + return -ENOMEM; + } + + bo = __bo_alloc(bdev->bo_cache); + if (!bo) { + dev_err(atomisp_dev, "%s: __bo_alloc failed!\n", __func__); + isp_mmu_exit(&bdev->mmu); + return -ENOMEM; + } + + ret = __bo_init(bdev, bo, bdev->pgnr); + if (ret) { + dev_err(atomisp_dev, "%s: __bo_init failed!\n", __func__); + kmem_cache_free(bdev->bo_cache, bo); + isp_mmu_exit(&bdev->mmu); + return -EINVAL; + } + + spin_lock_irqsave(&bdev->list_lock, flags); + list_add_tail(&bo->list, &bdev->entire_bo_list); + spin_unlock_irqrestore(&bdev->list_lock, flags); + + __bo_insert_to_free_rbtree(&bdev->free_rbtree, bo); + + return 0; +} + +struct hmm_buffer_object *hmm_bo_alloc(struct hmm_bo_device *bdev, + unsigned int pgnr) +{ + struct hmm_buffer_object *bo, *new_bo; + struct rb_root *root = &bdev->free_rbtree; + + check_bodev_null_return(bdev, NULL); + var_equal_return(hmm_bo_device_inited(bdev), 0, NULL, + "hmm_bo_device not inited yet.\n"); + + if (pgnr == 0) { + dev_err(atomisp_dev, "0 size buffer is not allowed.\n"); + return NULL; + } + + mutex_lock(&bdev->rbtree_mutex); + bo = __bo_search_and_remove_from_free_rbtree(root->rb_node, pgnr); + if (!bo) { + mutex_unlock(&bdev->rbtree_mutex); + dev_err(atomisp_dev, "%s: Out of Memory! hmm_bo_alloc failed", + __func__); + return NULL; + } + + if (bo->pgnr > pgnr) { + new_bo = __bo_break_up(bdev, bo, pgnr); + if (!new_bo) { + mutex_unlock(&bdev->rbtree_mutex); + dev_err(atomisp_dev, "%s: __bo_break_up failed!\n", + __func__); + return NULL; + } + + __bo_insert_to_alloc_rbtree(&bdev->allocated_rbtree, new_bo); + __bo_insert_to_free_rbtree(&bdev->free_rbtree, bo); + + mutex_unlock(&bdev->rbtree_mutex); + return new_bo; + } + + __bo_insert_to_alloc_rbtree(&bdev->allocated_rbtree, bo); + + mutex_unlock(&bdev->rbtree_mutex); + return bo; +} + +void hmm_bo_release(struct hmm_buffer_object *bo) +{ + struct hmm_bo_device *bdev = bo->bdev; + struct hmm_buffer_object *next_bo, *prev_bo; + + mutex_lock(&bdev->rbtree_mutex); + + /* + * FIX ME: + * + * how to destroy the bo when it is stilled MMAPED? + * + * ideally, this will not happened as hmm_bo_release + * will only be called when kref reaches 0, and in mmap + * operation the hmm_bo_ref will eventually be called. + * so, if this happened, something goes wrong. + */ + if (bo->status & HMM_BO_MMAPED) { + mutex_unlock(&bdev->rbtree_mutex); + dev_dbg(atomisp_dev, "destroy bo which is MMAPED, do nothing\n"); + return; + } + + if (bo->status & HMM_BO_BINDED) { + dev_warn(atomisp_dev, "the bo is still binded, unbind it first...\n"); + hmm_bo_unbind(bo); + } + + if (bo->status & HMM_BO_PAGE_ALLOCED) { + dev_warn(atomisp_dev, "the pages is not freed, free pages first\n"); + hmm_bo_free_pages(bo); + } + if (bo->status & HMM_BO_VMAPED || bo->status & HMM_BO_VMAPED_CACHED) { + dev_warn(atomisp_dev, "the vunmap is not done, do it...\n"); + hmm_bo_vunmap(bo); + } + + rb_erase(&bo->node, &bdev->allocated_rbtree); + + prev_bo = list_entry(bo->list.prev, struct hmm_buffer_object, list); + next_bo = list_entry(bo->list.next, struct hmm_buffer_object, list); + + if (bo->list.prev != &bdev->entire_bo_list && + prev_bo->end == bo->start && + (prev_bo->status & HMM_BO_MASK) == HMM_BO_FREE) { + __bo_take_off_handling(prev_bo); + bo = __bo_merge(prev_bo, bo); + } + + if (bo->list.next != &bdev->entire_bo_list && + next_bo->start == bo->end && + (next_bo->status & HMM_BO_MASK) == HMM_BO_FREE) { + __bo_take_off_handling(next_bo); + bo = __bo_merge(bo, next_bo); + } + + __bo_insert_to_free_rbtree(&bdev->free_rbtree, bo); + + mutex_unlock(&bdev->rbtree_mutex); + return; +} + +void hmm_bo_device_exit(struct hmm_bo_device *bdev) +{ + struct hmm_buffer_object *bo; + unsigned long flags; + + dev_dbg(atomisp_dev, "%s: entering!\n", __func__); + + check_bodev_null_return_void(bdev); + + /* + * release all allocated bos even they a in use + * and all bos will be merged into a big bo + */ + while (!RB_EMPTY_ROOT(&bdev->allocated_rbtree)) + hmm_bo_release( + rbtree_node_to_hmm_bo(bdev->allocated_rbtree.rb_node)); + + dev_dbg(atomisp_dev, "%s: finished releasing all allocated bos!\n", + __func__); + + /* free all bos to release all ISP virtual memory */ + while (!list_empty(&bdev->entire_bo_list)) { + bo = list_to_hmm_bo(bdev->entire_bo_list.next); + + spin_lock_irqsave(&bdev->list_lock, flags); + list_del(&bo->list); + spin_unlock_irqrestore(&bdev->list_lock, flags); + + kmem_cache_free(bdev->bo_cache, bo); + } + + dev_dbg(atomisp_dev, "%s: finished to free all bos!\n", __func__); + + kmem_cache_destroy(bdev->bo_cache); + + isp_mmu_exit(&bdev->mmu); +} + +int hmm_bo_device_inited(struct hmm_bo_device *bdev) +{ + check_bodev_null_return(bdev, -EINVAL); + + return bdev->flag == HMM_BO_DEVICE_INITED; +} + +int hmm_bo_allocated(struct hmm_buffer_object *bo) +{ + check_bo_null_return(bo, 0); + + return bo->status & HMM_BO_ALLOCED; +} + +struct hmm_buffer_object *hmm_bo_device_search_start( + struct hmm_bo_device *bdev, ia_css_ptr vaddr) +{ + struct hmm_buffer_object *bo; + + check_bodev_null_return(bdev, NULL); + + mutex_lock(&bdev->rbtree_mutex); + bo = __bo_search_by_addr(&bdev->allocated_rbtree, vaddr); + if (!bo) { + mutex_unlock(&bdev->rbtree_mutex); + dev_err(atomisp_dev, "%s can not find bo with addr: 0x%x\n", + __func__, vaddr); + return NULL; + } + mutex_unlock(&bdev->rbtree_mutex); + + return bo; +} + +struct hmm_buffer_object *hmm_bo_device_search_in_range( + struct hmm_bo_device *bdev, unsigned int vaddr) +{ + struct hmm_buffer_object *bo; + + check_bodev_null_return(bdev, NULL); + + mutex_lock(&bdev->rbtree_mutex); + bo = __bo_search_by_addr_in_range(&bdev->allocated_rbtree, vaddr); + if (!bo) { + mutex_unlock(&bdev->rbtree_mutex); + dev_err(atomisp_dev, "%s can not find bo contain addr: 0x%x\n", + __func__, vaddr); + return NULL; + } + mutex_unlock(&bdev->rbtree_mutex); + + return bo; +} + +struct hmm_buffer_object *hmm_bo_device_search_vmap_start( + struct hmm_bo_device *bdev, const void *vaddr) +{ + struct list_head *pos; + struct hmm_buffer_object *bo; + unsigned long flags; + + check_bodev_null_return(bdev, NULL); + + spin_lock_irqsave(&bdev->list_lock, flags); + list_for_each(pos, &bdev->entire_bo_list) { + bo = list_to_hmm_bo(pos); + /* pass bo which has no vm_node allocated */ + if ((bo->status & HMM_BO_MASK) == HMM_BO_FREE) + continue; + if (bo->vmap_addr == vaddr) + goto found; + } + spin_unlock_irqrestore(&bdev->list_lock, flags); + return NULL; +found: + spin_unlock_irqrestore(&bdev->list_lock, flags); + return bo; + +} + + +static void free_private_bo_pages(struct hmm_buffer_object *bo, + struct hmm_pool *dypool, + struct hmm_pool *repool, + int free_pgnr) +{ + int i, ret; + + for (i = 0; i < free_pgnr; i++) { + switch (bo->page_obj[i].type) { + case HMM_PAGE_TYPE_RESERVED: + if (repool->pops + && repool->pops->pool_free_pages) { + repool->pops->pool_free_pages(repool->pool_info, + &bo->page_obj[i]); + hmm_mem_stat.res_cnt--; + } + break; + /* + * HMM_PAGE_TYPE_GENERAL indicates that pages are from system + * memory, so when free them, they should be put into dynamic + * pool. + */ + case HMM_PAGE_TYPE_DYNAMIC: + case HMM_PAGE_TYPE_GENERAL: + if (dypool->pops + && dypool->pops->pool_inited + && dypool->pops->pool_inited(dypool->pool_info)) { + if (dypool->pops->pool_free_pages) + dypool->pops->pool_free_pages( + dypool->pool_info, + &bo->page_obj[i]); + break; + } + + /* + * if dynamic memory pool doesn't exist, need to free + * pages to system directly. + */ + default: + ret = set_pages_wb(bo->page_obj[i].page, 1); + if (ret) + dev_err(atomisp_dev, + "set page to WB err ...ret = %d\n", + ret); + /* + W/A: set_pages_wb seldom return value = -EFAULT + indicate that address of page is not in valid + range(0xffff880000000000~0xffffc7ffffffffff) + then, _free_pages would panic; Do not know why page + address be valid,it maybe memory corruption by lowmemory + */ + if (!ret) { + __free_pages(bo->page_obj[i].page, 0); + hmm_mem_stat.sys_size--; + } + break; + } + } + + return; +} + +/*Allocate pages which will be used only by ISP*/ +static int alloc_private_pages(struct hmm_buffer_object *bo, + int from_highmem, + bool cached, + struct hmm_pool *dypool, + struct hmm_pool *repool) +{ + int ret; + unsigned int pgnr, order, blk_pgnr, alloc_pgnr; + struct page *pages; + gfp_t gfp = GFP_NOWAIT | __GFP_NOWARN; /* REVISIT: need __GFP_FS too? */ + int i, j; + int failure_number = 0; + bool reduce_order = false; + bool lack_mem = true; + + if (from_highmem) + gfp |= __GFP_HIGHMEM; + + pgnr = bo->pgnr; + + bo->page_obj = kmalloc(sizeof(struct hmm_page_object) * pgnr, + GFP_KERNEL); + if (unlikely(!bo->page_obj)) { + dev_err(atomisp_dev, "out of memory for bo->page_obj\n"); + return -ENOMEM; + } + + i = 0; + alloc_pgnr = 0; + + /* + * get physical pages from dynamic pages pool. + */ + if (dypool->pops && dypool->pops->pool_alloc_pages) { + alloc_pgnr = dypool->pops->pool_alloc_pages(dypool->pool_info, + bo->page_obj, pgnr, + cached); + hmm_mem_stat.dyc_size -= alloc_pgnr; + + if (alloc_pgnr == pgnr) + return 0; + } + + pgnr -= alloc_pgnr; + i += alloc_pgnr; + + /* + * get physical pages from reserved pages pool for atomisp. + */ + if (repool->pops && repool->pops->pool_alloc_pages) { + alloc_pgnr = repool->pops->pool_alloc_pages(repool->pool_info, + &bo->page_obj[i], pgnr, + cached); + hmm_mem_stat.res_cnt += alloc_pgnr; + if (alloc_pgnr == pgnr) + return 0; + } + + pgnr -= alloc_pgnr; + i += alloc_pgnr; + + while (pgnr) { + order = nr_to_order_bottom(pgnr); + /* + * if be short of memory, we will set order to 0 + * everytime. + */ + if (lack_mem) + order = HMM_MIN_ORDER; + else if (order > HMM_MAX_ORDER) + order = HMM_MAX_ORDER; +retry: + /* + * When order > HMM_MIN_ORDER, for performance reasons we don't + * want alloc_pages() to sleep. In case it fails and fallbacks + * to HMM_MIN_ORDER or in case the requested order is originally + * the minimum value, we can allow alloc_pages() to sleep for + * robustness purpose. + * + * REVISIT: why __GFP_FS is necessary? + */ + if (order == HMM_MIN_ORDER) { + gfp &= ~GFP_NOWAIT; + gfp |= __GFP_RECLAIM | __GFP_FS; + } + + pages = alloc_pages(gfp, order); + if (unlikely(!pages)) { + /* + * in low memory case, if allocation page fails, + * we turn to try if order=0 allocation could + * succeed. if order=0 fails too, that means there is + * no memory left. + */ + if (order == HMM_MIN_ORDER) { + dev_err(atomisp_dev, + "%s: cannot allocate pages\n", + __func__); + goto cleanup; + } + order = HMM_MIN_ORDER; + failure_number++; + reduce_order = true; + /* + * if fail two times continuously, we think be short + * of memory now. + */ + if (failure_number == 2) { + lack_mem = true; + failure_number = 0; + } + goto retry; + } else { + blk_pgnr = order_to_nr(order); + + if (!cached) { + /* + * set memory to uncacheable -- UC_MINUS + */ + ret = set_pages_uc(pages, blk_pgnr); + if (ret) { + dev_err(atomisp_dev, + "set page uncacheable" + "failed.\n"); + + __free_pages(pages, order); + + goto cleanup; + } + } + + for (j = 0; j < blk_pgnr; j++) { + bo->page_obj[i].page = pages + j; + bo->page_obj[i++].type = HMM_PAGE_TYPE_GENERAL; + } + + pgnr -= blk_pgnr; + hmm_mem_stat.sys_size += blk_pgnr; + + /* + * if order is not reduced this time, clear + * failure_number. + */ + if (reduce_order) + reduce_order = false; + else + failure_number = 0; + } + } + + return 0; +cleanup: + alloc_pgnr = i; + free_private_bo_pages(bo, dypool, repool, alloc_pgnr); + + kfree(bo->page_obj); + + return -ENOMEM; +} + +static void free_private_pages(struct hmm_buffer_object *bo, + struct hmm_pool *dypool, + struct hmm_pool *repool) +{ + free_private_bo_pages(bo, dypool, repool, bo->pgnr); + + kfree(bo->page_obj); +} + +/* + * Hacked from kernel function __get_user_pages in mm/memory.c + * + * Handle buffers allocated by other kernel space driver and mmaped into user + * space, function Ignore the VM_PFNMAP and VM_IO flag in VMA structure + * + * Get physical pages from user space virtual address and update into page list + */ +static int __get_pfnmap_pages(struct task_struct *tsk, struct mm_struct *mm, + unsigned long start, int nr_pages, + unsigned int gup_flags, struct page **pages, + struct vm_area_struct **vmas) +{ + int i, ret; + unsigned long vm_flags; + + if (nr_pages <= 0) + return 0; + + VM_BUG_ON(!!pages != !!(gup_flags & FOLL_GET)); + + /* + * Require read or write permissions. + * If FOLL_FORCE is set, we only require the "MAY" flags. + */ + vm_flags = (gup_flags & FOLL_WRITE) ? + (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD); + vm_flags &= (gup_flags & FOLL_FORCE) ? + (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE); + i = 0; + + do { + struct vm_area_struct *vma; + + vma = find_vma(mm, start); + if (!vma) { + dev_err(atomisp_dev, "find_vma failed\n"); + return i ? : -EFAULT; + } + + if (is_vm_hugetlb_page(vma)) { + /* + i = follow_hugetlb_page(mm, vma, pages, vmas, + &start, &nr_pages, i, gup_flags); + */ + continue; + } + + do { + struct page *page; + unsigned long pfn; + + /* + * If we have a pending SIGKILL, don't keep faulting + * pages and potentially allocating memory. + */ + if (unlikely(fatal_signal_pending(current))) { + dev_err(atomisp_dev, + "fatal_signal_pending in %s\n", + __func__); + return i ? i : -ERESTARTSYS; + } + + ret = follow_pfn(vma, start, &pfn); + if (ret) { + dev_err(atomisp_dev, "follow_pfn() failed\n"); + return i ? : -EFAULT; + } + + page = pfn_to_page(pfn); + if (IS_ERR(page)) + return i ? i : PTR_ERR(page); + if (pages) { + pages[i] = page; + get_page(page); + flush_anon_page(vma, page, start); + flush_dcache_page(page); + } + if (vmas) + vmas[i] = vma; + i++; + start += PAGE_SIZE; + nr_pages--; + } while (nr_pages && start < vma->vm_end); + } while (nr_pages); + + return i; +} + +static int get_pfnmap_pages(struct task_struct *tsk, struct mm_struct *mm, + unsigned long start, int nr_pages, int write, int force, + struct page **pages, struct vm_area_struct **vmas) +{ + int flags = FOLL_TOUCH; + + if (pages) + flags |= FOLL_GET; + if (write) + flags |= FOLL_WRITE; + if (force) + flags |= FOLL_FORCE; + + return __get_pfnmap_pages(tsk, mm, start, nr_pages, flags, pages, vmas); +} + +/* + * Convert user space virtual address into pages list + */ +static int alloc_user_pages(struct hmm_buffer_object *bo, + void *userptr, bool cached) +{ + int page_nr; + int i; + struct vm_area_struct *vma; + struct page **pages; + + pages = kmalloc(sizeof(struct page *) * bo->pgnr, GFP_KERNEL); + if (unlikely(!pages)) { + dev_err(atomisp_dev, "out of memory for pages...\n"); + return -ENOMEM; + } + + bo->page_obj = kmalloc(sizeof(struct hmm_page_object) * bo->pgnr, + GFP_KERNEL); + if (unlikely(!bo->page_obj)) { + dev_err(atomisp_dev, "out of memory for bo->page_obj...\n"); + kfree(pages); + return -ENOMEM; + } + + mutex_unlock(&bo->mutex); + down_read(¤t->mm->mmap_sem); + vma = find_vma(current->mm, (unsigned long)userptr); + up_read(¤t->mm->mmap_sem); + if (vma == NULL) { + dev_err(atomisp_dev, "find_vma failed\n"); + kfree(bo->page_obj); + kfree(pages); + mutex_lock(&bo->mutex); + return -EFAULT; + } + mutex_lock(&bo->mutex); + /* + * Handle frame buffer allocated in other kerenl space driver + * and map to user space + */ + if (vma->vm_flags & (VM_IO | VM_PFNMAP)) { + page_nr = get_pfnmap_pages(current, current->mm, + (unsigned long)userptr, + (int)(bo->pgnr), 1, 0, + pages, NULL); + bo->mem_type = HMM_BO_MEM_TYPE_PFN; + } else { + /*Handle frame buffer allocated in user space*/ + mutex_unlock(&bo->mutex); + down_read(¤t->mm->mmap_sem); + page_nr = get_user_pages((unsigned long)userptr, + (int)(bo->pgnr), 1, pages, NULL); + up_read(¤t->mm->mmap_sem); + mutex_lock(&bo->mutex); + bo->mem_type = HMM_BO_MEM_TYPE_USER; + } + + /* can be written by caller, not forced */ + if (page_nr != bo->pgnr) { + dev_err(atomisp_dev, + "get_user_pages err: bo->pgnr = %d, " + "pgnr actually pinned = %d.\n", + bo->pgnr, page_nr); + goto out_of_mem; + } + + for (i = 0; i < bo->pgnr; i++) { + bo->page_obj[i].page = pages[i]; + bo->page_obj[i].type = HMM_PAGE_TYPE_GENERAL; + } + hmm_mem_stat.usr_size += bo->pgnr; + kfree(pages); + + return 0; + +out_of_mem: + for (i = 0; i < page_nr; i++) + put_page(pages[i]); + kfree(pages); + kfree(bo->page_obj); + + return -ENOMEM; +} + +static void free_user_pages(struct hmm_buffer_object *bo) +{ + int i; + + for (i = 0; i < bo->pgnr; i++) + put_page(bo->page_obj[i].page); + hmm_mem_stat.usr_size -= bo->pgnr; + + kfree(bo->page_obj); +} + +/* + * allocate/free physical pages for the bo. + * + * type indicate where are the pages from. currently we have 3 types + * of memory: HMM_BO_PRIVATE, HMM_BO_USER, HMM_BO_SHARE. + * + * from_highmem is only valid when type is HMM_BO_PRIVATE, it will + * try to alloc memory from highmem if from_highmem is set. + * + * userptr is only valid when type is HMM_BO_USER, it indicates + * the start address from user space task. + * + * from_highmem and userptr will both be ignored when type is + * HMM_BO_SHARE. + */ +int hmm_bo_alloc_pages(struct hmm_buffer_object *bo, + enum hmm_bo_type type, int from_highmem, + void *userptr, bool cached) +{ + int ret = -EINVAL; + + check_bo_null_return(bo, -EINVAL); + + mutex_lock(&bo->mutex); + check_bo_status_no_goto(bo, HMM_BO_PAGE_ALLOCED, status_err); + + /* + * TO DO: + * add HMM_BO_USER type + */ + if (type == HMM_BO_PRIVATE) { + ret = alloc_private_pages(bo, from_highmem, + cached, &dynamic_pool, &reserved_pool); + } else if (type == HMM_BO_USER) { + ret = alloc_user_pages(bo, userptr, cached); + } else { + dev_err(atomisp_dev, "invalid buffer type.\n"); + ret = -EINVAL; + } + if (ret) + goto alloc_err; + + bo->type = type; + + bo->status |= HMM_BO_PAGE_ALLOCED; + + mutex_unlock(&bo->mutex); + + return 0; + +alloc_err: + mutex_unlock(&bo->mutex); + dev_err(atomisp_dev, "alloc pages err...\n"); + return ret; +status_err: + mutex_unlock(&bo->mutex); + dev_err(atomisp_dev, + "buffer object has already page allocated.\n"); + return -EINVAL; +} + +/* + * free physical pages of the bo. + */ +void hmm_bo_free_pages(struct hmm_buffer_object *bo) +{ + check_bo_null_return_void(bo); + + mutex_lock(&bo->mutex); + + check_bo_status_yes_goto(bo, HMM_BO_PAGE_ALLOCED, status_err2); + + /* clear the flag anyway. */ + bo->status &= (~HMM_BO_PAGE_ALLOCED); + + if (bo->type == HMM_BO_PRIVATE) + free_private_pages(bo, &dynamic_pool, &reserved_pool); + else if (bo->type == HMM_BO_USER) + free_user_pages(bo); + else + dev_err(atomisp_dev, "invalid buffer type.\n"); + mutex_unlock(&bo->mutex); + + return; + +status_err2: + mutex_unlock(&bo->mutex); + dev_err(atomisp_dev, + "buffer object not page allocated yet.\n"); +} + +int hmm_bo_page_allocated(struct hmm_buffer_object *bo) +{ + int ret; + + check_bo_null_return(bo, 0); + + ret = bo->status & HMM_BO_PAGE_ALLOCED; + + return ret; +} + +/* + * get physical page info of the bo. + */ +int hmm_bo_get_page_info(struct hmm_buffer_object *bo, + struct hmm_page_object **page_obj, int *pgnr) +{ + check_bo_null_return(bo, -EINVAL); + + mutex_lock(&bo->mutex); + + check_bo_status_yes_goto(bo, HMM_BO_PAGE_ALLOCED, status_err); + + *page_obj = bo->page_obj; + *pgnr = bo->pgnr; + + mutex_unlock(&bo->mutex); + + return 0; + +status_err: + dev_err(atomisp_dev, + "buffer object not page allocated yet.\n"); + mutex_unlock(&bo->mutex); + return -EINVAL; +} + +/* + * bind the physical pages to a virtual address space. + */ +int hmm_bo_bind(struct hmm_buffer_object *bo) +{ + int ret; + unsigned int virt; + struct hmm_bo_device *bdev; + unsigned int i; + + check_bo_null_return(bo, -EINVAL); + + mutex_lock(&bo->mutex); + + check_bo_status_yes_goto(bo, + HMM_BO_PAGE_ALLOCED | HMM_BO_ALLOCED, + status_err1); + + check_bo_status_no_goto(bo, HMM_BO_BINDED, status_err2); + + bdev = bo->bdev; + + virt = bo->start; + + for (i = 0; i < bo->pgnr; i++) { + ret = + isp_mmu_map(&bdev->mmu, virt, + page_to_phys(bo->page_obj[i].page), 1); + if (ret) + goto map_err; + virt += (1 << PAGE_SHIFT); + } + + /* + * flush TBL here. + * + * theoretically, we donot need to flush TLB as we didnot change + * any existed address mappings, but for Silicon Hive's MMU, its + * really a bug here. I guess when fetching PTEs (page table entity) + * to TLB, its MMU will fetch additional INVALID PTEs automatically + * for performance issue. EX, we only set up 1 page address mapping, + * meaning updating 1 PTE, but the MMU fetches 4 PTE at one time, + * so the additional 3 PTEs are invalid. + */ + if (bo->start != 0x0) + isp_mmu_flush_tlb_range(&bdev->mmu, bo->start, + (bo->pgnr << PAGE_SHIFT)); + + bo->status |= HMM_BO_BINDED; + + mutex_unlock(&bo->mutex); + + return 0; + +map_err: + /* unbind the physical pages with related virtual address space */ + virt = bo->start; + for ( ; i > 0; i--) { + isp_mmu_unmap(&bdev->mmu, virt, 1); + virt += pgnr_to_size(1); + } + + mutex_unlock(&bo->mutex); + dev_err(atomisp_dev, + "setup MMU address mapping failed.\n"); + return ret; + +status_err2: + mutex_unlock(&bo->mutex); + dev_err(atomisp_dev, "buffer object already binded.\n"); + return -EINVAL; +status_err1: + mutex_unlock(&bo->mutex); + dev_err(atomisp_dev, + "buffer object vm_node or page not allocated.\n"); + return -EINVAL; +} + +/* + * unbind the physical pages with related virtual address space. + */ +void hmm_bo_unbind(struct hmm_buffer_object *bo) +{ + unsigned int virt; + struct hmm_bo_device *bdev; + unsigned int i; + + check_bo_null_return_void(bo); + + mutex_lock(&bo->mutex); + + check_bo_status_yes_goto(bo, + HMM_BO_PAGE_ALLOCED | + HMM_BO_ALLOCED | + HMM_BO_BINDED, status_err); + + bdev = bo->bdev; + + virt = bo->start; + + for (i = 0; i < bo->pgnr; i++) { + isp_mmu_unmap(&bdev->mmu, virt, 1); + virt += pgnr_to_size(1); + } + + /* + * flush TLB as the address mapping has been removed and + * related TLBs should be invalidated. + */ + isp_mmu_flush_tlb_range(&bdev->mmu, bo->start, + (bo->pgnr << PAGE_SHIFT)); + + bo->status &= (~HMM_BO_BINDED); + + mutex_unlock(&bo->mutex); + + return; + +status_err: + mutex_unlock(&bo->mutex); + dev_err(atomisp_dev, + "buffer vm or page not allocated or not binded yet.\n"); +} + +int hmm_bo_binded(struct hmm_buffer_object *bo) +{ + int ret; + + check_bo_null_return(bo, 0); + + mutex_lock(&bo->mutex); + + ret = bo->status & HMM_BO_BINDED; + + mutex_unlock(&bo->mutex); + + return ret; +} + +void *hmm_bo_vmap(struct hmm_buffer_object *bo, bool cached) +{ + struct page **pages; + int i; + + check_bo_null_return(bo, NULL); + + mutex_lock(&bo->mutex); + if (((bo->status & HMM_BO_VMAPED) && !cached) || + ((bo->status & HMM_BO_VMAPED_CACHED) && cached)) { + mutex_unlock(&bo->mutex); + return bo->vmap_addr; + } + + /* cached status need to be changed, so vunmap first */ + if (bo->status & HMM_BO_VMAPED || bo->status & HMM_BO_VMAPED_CACHED) { + vunmap(bo->vmap_addr); + bo->vmap_addr = NULL; + bo->status &= ~(HMM_BO_VMAPED | HMM_BO_VMAPED_CACHED); + } + + pages = kmalloc(sizeof(*pages) * bo->pgnr, GFP_KERNEL); + if (unlikely(!pages)) { + mutex_unlock(&bo->mutex); + dev_err(atomisp_dev, "out of memory for pages...\n"); + return NULL; + } + + for (i = 0; i < bo->pgnr; i++) + pages[i] = bo->page_obj[i].page; + + bo->vmap_addr = vmap(pages, bo->pgnr, VM_MAP, + cached ? PAGE_KERNEL : PAGE_KERNEL_NOCACHE); + if (unlikely(!bo->vmap_addr)) { + kfree(pages); + mutex_unlock(&bo->mutex); + dev_err(atomisp_dev, "vmap failed...\n"); + return NULL; + } + bo->status |= (cached ? HMM_BO_VMAPED_CACHED : HMM_BO_VMAPED); + + kfree(pages); + + mutex_unlock(&bo->mutex); + return bo->vmap_addr; +} + +void hmm_bo_flush_vmap(struct hmm_buffer_object *bo) +{ + check_bo_null_return_void(bo); + + mutex_lock(&bo->mutex); + if (!(bo->status & HMM_BO_VMAPED_CACHED) || !bo->vmap_addr) { + mutex_unlock(&bo->mutex); + return; + } + + clflush_cache_range(bo->vmap_addr, bo->pgnr * PAGE_SIZE); + mutex_unlock(&bo->mutex); +} + +void hmm_bo_vunmap(struct hmm_buffer_object *bo) +{ + check_bo_null_return_void(bo); + + mutex_lock(&bo->mutex); + if (bo->status & HMM_BO_VMAPED || bo->status & HMM_BO_VMAPED_CACHED) { + vunmap(bo->vmap_addr); + bo->vmap_addr = NULL; + bo->status &= ~(HMM_BO_VMAPED | HMM_BO_VMAPED_CACHED); + } + + mutex_unlock(&bo->mutex); + return; +} + +void hmm_bo_ref(struct hmm_buffer_object *bo) +{ + check_bo_null_return_void(bo); + + kref_get(&bo->kref); +} + +static void kref_hmm_bo_release(struct kref *kref) +{ + if (!kref) + return; + + hmm_bo_release(kref_to_hmm_bo(kref)); +} + +void hmm_bo_unref(struct hmm_buffer_object *bo) +{ + check_bo_null_return_void(bo); + + kref_put(&bo->kref, kref_hmm_bo_release); +} + +static void hmm_bo_vm_open(struct vm_area_struct *vma) +{ + struct hmm_buffer_object *bo = + (struct hmm_buffer_object *)vma->vm_private_data; + + check_bo_null_return_void(bo); + + hmm_bo_ref(bo); + + mutex_lock(&bo->mutex); + + bo->status |= HMM_BO_MMAPED; + + bo->mmap_count++; + + mutex_unlock(&bo->mutex); +} + +static void hmm_bo_vm_close(struct vm_area_struct *vma) +{ + struct hmm_buffer_object *bo = + (struct hmm_buffer_object *)vma->vm_private_data; + + check_bo_null_return_void(bo); + + hmm_bo_unref(bo); + + mutex_lock(&bo->mutex); + + bo->mmap_count--; + + if (!bo->mmap_count) { + bo->status &= (~HMM_BO_MMAPED); + vma->vm_private_data = NULL; + } + + mutex_unlock(&bo->mutex); +} + +static const struct vm_operations_struct hmm_bo_vm_ops = { + .open = hmm_bo_vm_open, + .close = hmm_bo_vm_close, +}; + +/* + * mmap the bo to user space. + */ +int hmm_bo_mmap(struct vm_area_struct *vma, struct hmm_buffer_object *bo) +{ + unsigned int start, end; + unsigned int virt; + unsigned int pgnr, i; + unsigned int pfn; + + check_bo_null_return(bo, -EINVAL); + + check_bo_status_yes_goto(bo, HMM_BO_PAGE_ALLOCED, status_err); + + pgnr = bo->pgnr; + start = vma->vm_start; + end = vma->vm_end; + + /* + * check vma's virtual address space size and buffer object's size. + * must be the same. + */ + if ((start + pgnr_to_size(pgnr)) != end) { + dev_warn(atomisp_dev, + "vma's address space size not equal" + " to buffer object's size"); + return -EINVAL; + } + + virt = vma->vm_start; + for (i = 0; i < pgnr; i++) { + pfn = page_to_pfn(bo->page_obj[i].page); + if (remap_pfn_range(vma, virt, pfn, PAGE_SIZE, PAGE_SHARED)) { + dev_warn(atomisp_dev, + "remap_pfn_range failed:" + " virt = 0x%x, pfn = 0x%x," + " mapped_pgnr = %d\n", virt, pfn, 1); + return -EINVAL; + } + virt += PAGE_SIZE; + } + + vma->vm_private_data = bo; + + vma->vm_ops = &hmm_bo_vm_ops; + vma->vm_flags |= VM_IO|VM_DONTEXPAND|VM_DONTDUMP; + + /* + * call hmm_bo_vm_open explictly. + */ + hmm_bo_vm_open(vma); + + return 0; + +status_err: + dev_err(atomisp_dev, "buffer page not allocated yet.\n"); + return -EINVAL; +} |