diff options
Diffstat (limited to 'drivers/staging/gasket/gasket_page_table.c')
| -rw-r--r-- | drivers/staging/gasket/gasket_page_table.c | 1359 |
1 files changed, 0 insertions, 1359 deletions
diff --git a/drivers/staging/gasket/gasket_page_table.c b/drivers/staging/gasket/gasket_page_table.c deleted file mode 100644 index 6f6273c83822..000000000000 --- a/drivers/staging/gasket/gasket_page_table.c +++ /dev/null @@ -1,1359 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Implementation of Gasket page table support. - * - * Copyright (C) 2018 Google, Inc. - */ - -/* - * Implementation of Gasket page table support. - * - * This file assumes 4kB pages throughout; can be factored out when necessary. - * - * There is a configurable number of page table entries, as well as a - * configurable bit index for the extended address flag. Both of these are - * specified in gasket_page_table_init through the page_table_config parameter. - * - * The following example assumes: - * page_table_config->total_entries = 8192 - * page_table_config->extended_bit = 63 - * - * Address format: - * Simple addresses - those whose containing pages are directly placed in the - * device's address translation registers - are laid out as: - * [ 63 - 25: 0 | 24 - 12: page index | 11 - 0: page offset ] - * page index: The index of the containing page in the device's address - * translation registers. - * page offset: The index of the address into the containing page. - * - * Extended address - those whose containing pages are contained in a second- - * level page table whose address is present in the device's address translation - * registers - are laid out as: - * [ 63: flag | 62 - 34: 0 | 33 - 21: dev/level 0 index | - * 20 - 12: host/level 1 index | 11 - 0: page offset ] - * flag: Marker indicating that this is an extended address. Always 1. - * dev index: The index of the first-level page in the device's extended - * address translation registers. - * host index: The index of the containing page in the [host-resident] second- - * level page table. - * page offset: The index of the address into the containing [second-level] - * page. - */ -#include "gasket_page_table.h" - -#include <linux/device.h> -#include <linux/file.h> -#include <linux/init.h> -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/moduleparam.h> -#include <linux/pagemap.h> -#include <linux/vmalloc.h> - -#include "gasket_constants.h" -#include "gasket_core.h" - -/* Constants & utility macros */ -/* The number of pages that can be mapped into each second-level page table. */ -#define GASKET_PAGES_PER_SUBTABLE 512 - -/* The starting position of the page index in a simple virtual address. */ -#define GASKET_SIMPLE_PAGE_SHIFT 12 - -/* Flag indicating that a [device] slot is valid for use. */ -#define GASKET_VALID_SLOT_FLAG 1 - -/* - * The starting position of the level 0 page index (i.e., the entry in the - * device's extended address registers) in an extended address. - * Also can be thought of as (log2(PAGE_SIZE) + log2(PAGES_PER_SUBTABLE)), - * or (12 + 9). - */ -#define GASKET_EXTENDED_LVL0_SHIFT 21 - -/* - * Number of first level pages that Gasket chips support. Equivalent to - * log2(NUM_LVL0_PAGE_TABLES) - * - * At a maximum, allowing for a 34 bits address space (or 16GB) - * = GASKET_EXTENDED_LVL0_WIDTH + (log2(PAGE_SIZE) + log2(PAGES_PER_SUBTABLE) - * or, = 13 + 9 + 12 - */ -#define GASKET_EXTENDED_LVL0_WIDTH 13 - -/* - * The starting position of the level 1 page index (i.e., the entry in the - * host second-level/sub- table) in an extended address. - */ -#define GASKET_EXTENDED_LVL1_SHIFT 12 - -/* Type declarations */ -/* Valid states for a struct gasket_page_table_entry. */ -enum pte_status { - PTE_FREE, - PTE_INUSE, -}; - -/* - * Mapping metadata for a single page. - * - * In this file, host-side page table entries are referred to as that (or PTEs). - * Where device vs. host entries are differentiated, device-side or -visible - * entries are called "slots". A slot may be either an entry in the device's - * address translation table registers or an entry in a second-level page - * table ("subtable"). - * - * The full data in this structure is visible on the host [of course]. Only - * the address contained in dma_addr is communicated to the device; that points - * to the actual page mapped and described by this structure. - */ -struct gasket_page_table_entry { - /* The status of this entry/slot: free or in use. */ - enum pte_status status; - - /* - * Index for alignment into host vaddrs. - * When a user specifies a host address for a mapping, that address may - * not be page-aligned. Offset is the index into the containing page of - * the host address (i.e., host_vaddr & (PAGE_SIZE - 1)). - * This is necessary for translating between user-specified addresses - * and page-aligned addresses. - */ - int offset; - - /* Address of the page in DMA space. */ - dma_addr_t dma_addr; - - /* Linux page descriptor for the page described by this structure. */ - struct page *page; - - /* - * If this is an extended and first-level entry, sublevel points - * to the second-level entries underneath this entry. - */ - struct gasket_page_table_entry *sublevel; -}; - -/* - * Maintains virtual to physical address mapping for a coherent page that is - * allocated by this module for a given device. - * Note that coherent pages mappings virt mapping cannot be tracked by the - * Linux kernel, and coherent pages don't have a struct page associated, - * hence Linux kernel cannot perform a get_user_page_xx() on a phys address - * that was allocated coherent. - * This structure trivially implements this mechanism. - */ -struct gasket_coherent_page_entry { - /* Phys address, dma'able by the owner device */ - dma_addr_t paddr; - - /* Kernel virtual address */ - u64 user_virt; - - /* User virtual address that was mapped by the mmap kernel subsystem */ - u64 kernel_virt; - - /* - * Whether this page has been mapped into a user land process virtual - * space - */ - u32 in_use; -}; - -/* - * [Host-side] page table descriptor. - * - * This structure tracks the metadata necessary to manage both simple and - * extended page tables. - */ -struct gasket_page_table { - /* The config used to create this page table. */ - struct gasket_page_table_config config; - - /* The number of simple (single-level) entries in the page table. */ - uint num_simple_entries; - - /* The number of extended (two-level) entries in the page table. */ - uint num_extended_entries; - - /* Array of [host-side] page table entries. */ - struct gasket_page_table_entry *entries; - - /* Number of actively mapped kernel pages in this table. */ - uint num_active_pages; - - /* Device register: base of/first slot in the page table. */ - u64 __iomem *base_slot; - - /* Device register: holds the offset indicating the start of the - * extended address region of the device's address translation table. - */ - u64 __iomem *extended_offset_reg; - - /* Device structure for the underlying device. Only used for logging. */ - struct device *device; - - /* PCI system descriptor for the underlying device. */ - struct pci_dev *pci_dev; - - /* Location of the extended address bit for this Gasket device. */ - u64 extended_flag; - - /* Mutex to protect page table internals. */ - struct mutex mutex; - - /* Number of coherent pages accessible thru by this page table */ - int num_coherent_pages; - - /* - * List of coherent memory (physical) allocated for a device. - * - * This structure also remembers the user virtual mapping, this is - * hacky, but we need to do this because the kernel doesn't keep track - * of the user coherent pages (pfn pages), and virt to coherent page - * mapping. - * TODO: use find_vma() APIs to convert host address to vm_area, to - * dma_addr_t instead of storing user virtu address in - * gasket_coherent_page_entry - * - * Note that the user virtual mapping is created by the driver, in - * gasket_mmap function, so user_virt belongs in the driver anyhow. - */ - struct gasket_coherent_page_entry *coherent_pages; -}; - -/* See gasket_page_table.h for description. */ -int gasket_page_table_init(struct gasket_page_table **ppg_tbl, - const struct gasket_bar_data *bar_data, - const struct gasket_page_table_config *page_table_config, - struct device *device, struct pci_dev *pci_dev) -{ - ulong bytes; - struct gasket_page_table *pg_tbl; - ulong total_entries = page_table_config->total_entries; - - /* - * TODO: Verify config->total_entries against value read from the - * hardware register that contains the page table size. - */ - if (total_entries == ULONG_MAX) { - dev_dbg(device, - "Error reading page table size. Initializing page table with size 0\n"); - total_entries = 0; - } - - dev_dbg(device, - "Attempting to initialize page table of size 0x%lx\n", - total_entries); - - dev_dbg(device, - "Table has base reg 0x%x, extended offset reg 0x%x\n", - page_table_config->base_reg, - page_table_config->extended_reg); - - *ppg_tbl = kzalloc(sizeof(**ppg_tbl), GFP_KERNEL); - if (!*ppg_tbl) { - dev_dbg(device, "No memory for page table\n"); - return -ENOMEM; - } - - pg_tbl = *ppg_tbl; - bytes = total_entries * sizeof(struct gasket_page_table_entry); - if (bytes != 0) { - pg_tbl->entries = vzalloc(bytes); - if (!pg_tbl->entries) { - dev_dbg(device, - "No memory for address translation metadata\n"); - kfree(pg_tbl); - *ppg_tbl = NULL; - return -ENOMEM; - } - } - - mutex_init(&pg_tbl->mutex); - memcpy(&pg_tbl->config, page_table_config, sizeof(*page_table_config)); - if (pg_tbl->config.mode == GASKET_PAGE_TABLE_MODE_NORMAL || - pg_tbl->config.mode == GASKET_PAGE_TABLE_MODE_SIMPLE) { - pg_tbl->num_simple_entries = total_entries; - pg_tbl->num_extended_entries = 0; - pg_tbl->extended_flag = 1ull << page_table_config->extended_bit; - } else { - pg_tbl->num_simple_entries = 0; - pg_tbl->num_extended_entries = total_entries; - pg_tbl->extended_flag = 0; - } - pg_tbl->num_active_pages = 0; - pg_tbl->base_slot = - (u64 __iomem *)&bar_data->virt_base[page_table_config->base_reg]; - pg_tbl->extended_offset_reg = - (u64 __iomem *)&bar_data->virt_base[page_table_config->extended_reg]; - pg_tbl->device = get_device(device); - pg_tbl->pci_dev = pci_dev; - - dev_dbg(device, "Page table initialized successfully\n"); - - return 0; -} - -/* - * Check if a range of PTEs is free. - * The page table mutex must be held by the caller. - */ -static bool gasket_is_pte_range_free(struct gasket_page_table_entry *ptes, - uint num_entries) -{ - int i; - - for (i = 0; i < num_entries; i++) { - if (ptes[i].status != PTE_FREE) - return false; - } - - return true; -} - -/* - * Free a second level page [sub]table. - * The page table mutex must be held before this call. - */ -static void gasket_free_extended_subtable(struct gasket_page_table *pg_tbl, - struct gasket_page_table_entry *pte, - u64 __iomem *slot) -{ - /* Release the page table from the driver */ - pte->status = PTE_FREE; - - /* Release the page table from the device */ - writeq(0, slot); - - if (pte->dma_addr) - dma_unmap_page(pg_tbl->device, pte->dma_addr, PAGE_SIZE, - DMA_TO_DEVICE); - - vfree(pte->sublevel); - - if (pte->page) - free_page((ulong)page_address(pte->page)); - - memset(pte, 0, sizeof(struct gasket_page_table_entry)); -} - -/* - * Actually perform collection. - * The page table mutex must be held by the caller. - */ -static void -gasket_page_table_garbage_collect_nolock(struct gasket_page_table *pg_tbl) -{ - struct gasket_page_table_entry *pte; - u64 __iomem *slot; - - /* XXX FIX ME XXX -- more efficient to keep a usage count */ - /* rather than scanning the second level page tables */ - - for (pte = pg_tbl->entries + pg_tbl->num_simple_entries, - slot = pg_tbl->base_slot + pg_tbl->num_simple_entries; - pte < pg_tbl->entries + pg_tbl->config.total_entries; - pte++, slot++) { - if (pte->status == PTE_INUSE) { - if (gasket_is_pte_range_free(pte->sublevel, - GASKET_PAGES_PER_SUBTABLE)) - gasket_free_extended_subtable(pg_tbl, pte, - slot); - } - } -} - -/* See gasket_page_table.h for description. */ -void gasket_page_table_garbage_collect(struct gasket_page_table *pg_tbl) -{ - mutex_lock(&pg_tbl->mutex); - gasket_page_table_garbage_collect_nolock(pg_tbl); - mutex_unlock(&pg_tbl->mutex); -} - -/* See gasket_page_table.h for description. */ -void gasket_page_table_cleanup(struct gasket_page_table *pg_tbl) -{ - /* Deallocate free second-level tables. */ - gasket_page_table_garbage_collect(pg_tbl); - - /* TODO: Check that all PTEs have been freed? */ - - vfree(pg_tbl->entries); - pg_tbl->entries = NULL; - - put_device(pg_tbl->device); - kfree(pg_tbl); -} - -/* See gasket_page_table.h for description. */ -int gasket_page_table_partition(struct gasket_page_table *pg_tbl, - uint num_simple_entries) -{ - int i, start; - - mutex_lock(&pg_tbl->mutex); - if (num_simple_entries > pg_tbl->config.total_entries) { - mutex_unlock(&pg_tbl->mutex); - return -EINVAL; - } - - gasket_page_table_garbage_collect_nolock(pg_tbl); - - start = min(pg_tbl->num_simple_entries, num_simple_entries); - - for (i = start; i < pg_tbl->config.total_entries; i++) { - if (pg_tbl->entries[i].status != PTE_FREE) { - dev_err(pg_tbl->device, "entry %d is not free\n", i); - mutex_unlock(&pg_tbl->mutex); - return -EBUSY; - } - } - - pg_tbl->num_simple_entries = num_simple_entries; - pg_tbl->num_extended_entries = - pg_tbl->config.total_entries - num_simple_entries; - writeq(num_simple_entries, pg_tbl->extended_offset_reg); - - mutex_unlock(&pg_tbl->mutex); - return 0; -} -EXPORT_SYMBOL(gasket_page_table_partition); - -/* - * Return whether a host buffer was mapped as coherent memory. - * - * A Gasket page_table currently support one contiguous dma range, mapped to one - * contiguous virtual memory range. Check if the host_addr is within that range. - */ -static int is_coherent(struct gasket_page_table *pg_tbl, ulong host_addr) -{ - u64 min, max; - - /* whether the host address is within user virt range */ - if (!pg_tbl->coherent_pages) - return 0; - - min = (u64)pg_tbl->coherent_pages[0].user_virt; - max = min + PAGE_SIZE * pg_tbl->num_coherent_pages; - - return min <= host_addr && host_addr < max; -} - -/* Safely return a page to the OS. */ -static bool gasket_release_page(struct page *page) -{ - if (!page) - return false; - - if (!PageReserved(page)) - SetPageDirty(page); - unpin_user_page(page); - - return true; -} - -/* - * Get and map last level page table buffers. - * - * slots is the location(s) to write device-mapped page address. If this is a - * simple mapping, these will be address translation registers. If this is - * an extended mapping, these will be within a second-level page table - * allocated by the host and so must have their __iomem attribute casted away. - */ -static int gasket_perform_mapping(struct gasket_page_table *pg_tbl, - struct gasket_page_table_entry *ptes, - u64 __iomem *slots, ulong host_addr, - uint num_pages, int is_simple_mapping) -{ - int ret; - ulong offset; - struct page *page; - dma_addr_t dma_addr; - ulong page_addr; - int i; - - for (i = 0; i < num_pages; i++) { - page_addr = host_addr + i * PAGE_SIZE; - offset = page_addr & (PAGE_SIZE - 1); - if (is_coherent(pg_tbl, host_addr)) { - u64 off = - (u64)host_addr - - (u64)pg_tbl->coherent_pages[0].user_virt; - ptes[i].page = NULL; - ptes[i].offset = offset; - ptes[i].dma_addr = pg_tbl->coherent_pages[0].paddr + - off + i * PAGE_SIZE; - } else { - ret = pin_user_pages_fast(page_addr - offset, 1, - FOLL_WRITE, &page); - - if (ret <= 0) { - dev_err(pg_tbl->device, - "pin user pages failed for addr=0x%lx, offset=0x%lx [ret=%d]\n", - page_addr, offset, ret); - return ret ? ret : -ENOMEM; - } - ++pg_tbl->num_active_pages; - - ptes[i].page = page; - ptes[i].offset = offset; - - /* Map the page into DMA space. */ - ptes[i].dma_addr = - dma_map_page(pg_tbl->device, page, 0, PAGE_SIZE, - DMA_BIDIRECTIONAL); - - if (dma_mapping_error(pg_tbl->device, - ptes[i].dma_addr)) { - if (gasket_release_page(ptes[i].page)) - --pg_tbl->num_active_pages; - - memset(&ptes[i], 0, - sizeof(struct gasket_page_table_entry)); - return -EINVAL; - } - } - - /* Make the DMA-space address available to the device. */ - dma_addr = (ptes[i].dma_addr + offset) | GASKET_VALID_SLOT_FLAG; - - if (is_simple_mapping) { - writeq(dma_addr, &slots[i]); - } else { - ((u64 __force *)slots)[i] = dma_addr; - /* Extended page table vectors are in DRAM, - * and so need to be synced each time they are updated. - */ - dma_map_single(pg_tbl->device, - (void *)&((u64 __force *)slots)[i], - sizeof(u64), DMA_TO_DEVICE); - } - ptes[i].status = PTE_INUSE; - } - return 0; -} - -/* - * Return the index of the page for the address in the simple table. - * Does not perform validity checking. - */ -static int gasket_simple_page_idx(struct gasket_page_table *pg_tbl, - ulong dev_addr) -{ - return (dev_addr >> GASKET_SIMPLE_PAGE_SHIFT) & - (pg_tbl->config.total_entries - 1); -} - -/* - * Return the level 0 page index for the given address. - * Does not perform validity checking. - */ -static ulong gasket_extended_lvl0_page_idx(struct gasket_page_table *pg_tbl, - ulong dev_addr) -{ - return (dev_addr >> GASKET_EXTENDED_LVL0_SHIFT) & - (pg_tbl->config.total_entries - 1); -} - -/* - * Return the level 1 page index for the given address. - * Does not perform validity checking. - */ -static ulong gasket_extended_lvl1_page_idx(struct gasket_page_table *pg_tbl, - ulong dev_addr) -{ - return (dev_addr >> GASKET_EXTENDED_LVL1_SHIFT) & - (GASKET_PAGES_PER_SUBTABLE - 1); -} - -/* - * Allocate page table entries in a simple table. - * The page table mutex must be held by the caller. - */ -static int gasket_alloc_simple_entries(struct gasket_page_table *pg_tbl, - ulong dev_addr, uint num_pages) -{ - if (!gasket_is_pte_range_free(pg_tbl->entries + - gasket_simple_page_idx(pg_tbl, dev_addr), - num_pages)) - return -EBUSY; - - return 0; -} - -/* - * Unmap and release mapped pages. - * The page table mutex must be held by the caller. - */ -static void gasket_perform_unmapping(struct gasket_page_table *pg_tbl, - struct gasket_page_table_entry *ptes, - u64 __iomem *slots, uint num_pages, - int is_simple_mapping) -{ - int i; - /* - * For each page table entry and corresponding entry in the device's - * address translation table: - */ - for (i = 0; i < num_pages; i++) { - /* release the address from the device, */ - if (is_simple_mapping || ptes[i].status == PTE_INUSE) { - writeq(0, &slots[i]); - } else { - ((u64 __force *)slots)[i] = 0; - /* sync above PTE update before updating mappings */ - wmb(); - } - - /* release the address from the driver, */ - if (ptes[i].status == PTE_INUSE) { - if (ptes[i].page && ptes[i].dma_addr) { - dma_unmap_page(pg_tbl->device, ptes[i].dma_addr, - PAGE_SIZE, DMA_BIDIRECTIONAL); - } - if (gasket_release_page(ptes[i].page)) - --pg_tbl->num_active_pages; - } - - /* and clear the PTE. */ - memset(&ptes[i], 0, sizeof(struct gasket_page_table_entry)); - } -} - -/* - * Unmap and release pages mapped to simple addresses. - * The page table mutex must be held by the caller. - */ -static void gasket_unmap_simple_pages(struct gasket_page_table *pg_tbl, - ulong dev_addr, uint num_pages) -{ - uint slot = gasket_simple_page_idx(pg_tbl, dev_addr); - - gasket_perform_unmapping(pg_tbl, pg_tbl->entries + slot, - pg_tbl->base_slot + slot, num_pages, 1); -} - -/* - * Unmap and release buffers to extended addresses. - * The page table mutex must be held by the caller. - */ -static void gasket_unmap_extended_pages(struct gasket_page_table *pg_tbl, - ulong dev_addr, uint num_pages) -{ - uint slot_idx, remain, len; - struct gasket_page_table_entry *pte; - u64 __iomem *slot_base; - - remain = num_pages; - slot_idx = gasket_extended_lvl1_page_idx(pg_tbl, dev_addr); - pte = pg_tbl->entries + pg_tbl->num_simple_entries + - gasket_extended_lvl0_page_idx(pg_tbl, dev_addr); - - while (remain > 0) { - /* TODO: Add check to ensure pte remains valid? */ - len = min(remain, GASKET_PAGES_PER_SUBTABLE - slot_idx); - - if (pte->status == PTE_INUSE) { - slot_base = (u64 __iomem *)(page_address(pte->page) + - pte->offset); - gasket_perform_unmapping(pg_tbl, - pte->sublevel + slot_idx, - slot_base + slot_idx, len, 0); - } - - remain -= len; - slot_idx = 0; - pte++; - } -} - -/* Evaluates to nonzero if the specified virtual address is simple. */ -static inline bool gasket_addr_is_simple(struct gasket_page_table *pg_tbl, - ulong addr) -{ - return !((addr) & (pg_tbl)->extended_flag); -} - -/* - * Convert (simple, page, offset) into a device address. - * Examples: - * Simple page 0, offset 32: - * Input (1, 0, 32), Output 0x20 - * Simple page 1000, offset 511: - * Input (1, 1000, 511), Output 0x3E81FF - * Extended page 0, offset 32: - * Input (0, 0, 32), Output 0x8000000020 - * Extended page 1000, offset 511: - * Input (0, 1000, 511), Output 0x8003E81FF - */ -static ulong gasket_components_to_dev_address(struct gasket_page_table *pg_tbl, - int is_simple, uint page_index, - uint offset) -{ - ulong dev_addr = (page_index << GASKET_SIMPLE_PAGE_SHIFT) | offset; - - return is_simple ? dev_addr : (pg_tbl->extended_flag | dev_addr); -} - -/* - * Validity checking for simple addresses. - * - * Verify that address translation commutes (from address to/from page + offset) - * and that the requested page range starts and ends within the set of - * currently-partitioned simple pages. - */ -static bool gasket_is_simple_dev_addr_bad(struct gasket_page_table *pg_tbl, - ulong dev_addr, uint num_pages) -{ - ulong page_offset = dev_addr & (PAGE_SIZE - 1); - ulong page_index = - (dev_addr / PAGE_SIZE) & (pg_tbl->config.total_entries - 1); - - if (gasket_components_to_dev_address(pg_tbl, 1, page_index, - page_offset) != dev_addr) { - dev_err(pg_tbl->device, "address is invalid, 0x%lX\n", - dev_addr); - return true; - } - - if (page_index >= pg_tbl->num_simple_entries) { - dev_err(pg_tbl->device, - "starting slot at %lu is too large, max is < %u\n", - page_index, pg_tbl->num_simple_entries); - return true; - } - - if (page_index + num_pages > pg_tbl->num_simple_entries) { - dev_err(pg_tbl->device, - "ending slot at %lu is too large, max is <= %u\n", - page_index + num_pages, pg_tbl->num_simple_entries); - return true; - } - - return false; -} - -/* - * Validity checking for extended addresses. - * - * Verify that address translation commutes (from address to/from page + - * offset) and that the requested page range starts and ends within the set of - * currently-partitioned extended pages. - */ -static bool gasket_is_extended_dev_addr_bad(struct gasket_page_table *pg_tbl, - ulong dev_addr, uint num_pages) -{ - /* Starting byte index of dev_addr into the first mapped page */ - ulong page_offset = dev_addr & (PAGE_SIZE - 1); - ulong page_global_idx, page_lvl0_idx; - ulong num_lvl0_pages; - ulong addr; - - /* check if the device address is out of bound */ - addr = dev_addr & ~((pg_tbl)->extended_flag); - if (addr >> (GASKET_EXTENDED_LVL0_WIDTH + GASKET_EXTENDED_LVL0_SHIFT)) { - dev_err(pg_tbl->device, "device address out of bounds: 0x%lx\n", - dev_addr); - return true; - } - - /* Find the starting sub-page index in the space of all sub-pages. */ - page_global_idx = (dev_addr / PAGE_SIZE) & - (pg_tbl->config.total_entries * GASKET_PAGES_PER_SUBTABLE - 1); - - /* Find the starting level 0 index. */ - page_lvl0_idx = gasket_extended_lvl0_page_idx(pg_tbl, dev_addr); - - /* Get the count of affected level 0 pages. */ - num_lvl0_pages = DIV_ROUND_UP(num_pages, GASKET_PAGES_PER_SUBTABLE); - - if (gasket_components_to_dev_address(pg_tbl, 0, page_global_idx, - page_offset) != dev_addr) { - dev_err(pg_tbl->device, "address is invalid: 0x%lx\n", - dev_addr); - return true; - } - - if (page_lvl0_idx >= pg_tbl->num_extended_entries) { - dev_err(pg_tbl->device, - "starting level 0 slot at %lu is too large, max is < %u\n", - page_lvl0_idx, pg_tbl->num_extended_entries); - return true; - } - - if (page_lvl0_idx + num_lvl0_pages > pg_tbl->num_extended_entries) { - dev_err(pg_tbl->device, - "ending level 0 slot at %lu is too large, max is <= %u\n", - page_lvl0_idx + num_lvl0_pages, - pg_tbl->num_extended_entries); - return true; - } - - return false; -} - -/* - * Non-locking entry to unmapping routines. - * The page table mutex must be held by the caller. - */ -static void gasket_page_table_unmap_nolock(struct gasket_page_table *pg_tbl, - ulong dev_addr, uint num_pages) -{ - if (!num_pages) - return; - - if (gasket_addr_is_simple(pg_tbl, dev_addr)) - gasket_unmap_simple_pages(pg_tbl, dev_addr, num_pages); - else - gasket_unmap_extended_pages(pg_tbl, dev_addr, num_pages); -} - -/* - * Allocate and map pages to simple addresses. - * If there is an error, no pages are mapped. - */ -static int gasket_map_simple_pages(struct gasket_page_table *pg_tbl, - ulong host_addr, ulong dev_addr, - uint num_pages) -{ - int ret; - uint slot_idx = gasket_simple_page_idx(pg_tbl, dev_addr); - - ret = gasket_alloc_simple_entries(pg_tbl, dev_addr, num_pages); - if (ret) { - dev_err(pg_tbl->device, - "page table slots %u (@ 0x%lx) to %u are not available\n", - slot_idx, dev_addr, slot_idx + num_pages - 1); - return ret; - } - - ret = gasket_perform_mapping(pg_tbl, pg_tbl->entries + slot_idx, - pg_tbl->base_slot + slot_idx, host_addr, - num_pages, 1); - - if (ret) { - gasket_page_table_unmap_nolock(pg_tbl, dev_addr, num_pages); - dev_err(pg_tbl->device, "gasket_perform_mapping %d\n", ret); - } - return ret; -} - -/* - * Allocate a second level page table. - * The page table mutex must be held by the caller. - */ -static int gasket_alloc_extended_subtable(struct gasket_page_table *pg_tbl, - struct gasket_page_table_entry *pte, - u64 __iomem *slot) -{ - ulong page_addr, subtable_bytes; - dma_addr_t dma_addr; - - /* XXX FIX ME XXX this is inefficient for non-4K page sizes */ - - /* GFP_DMA flag must be passed to architectures for which - * part of the memory range is not considered DMA'able. - * This seems to be the case for Juno board with 4.5.0 Linaro kernel - */ - page_addr = get_zeroed_page(GFP_KERNEL | GFP_DMA); - if (!page_addr) - return -ENOMEM; - pte->page = virt_to_page((void *)page_addr); - pte->offset = 0; - - subtable_bytes = sizeof(struct gasket_page_table_entry) * - GASKET_PAGES_PER_SUBTABLE; - pte->sublevel = vzalloc(subtable_bytes); - if (!pte->sublevel) { - free_page(page_addr); - memset(pte, 0, sizeof(struct gasket_page_table_entry)); - return -ENOMEM; - } - - /* Map the page into DMA space. */ - pte->dma_addr = dma_map_page(pg_tbl->device, pte->page, 0, PAGE_SIZE, - DMA_TO_DEVICE); - if (dma_mapping_error(pg_tbl->device, pte->dma_addr)) { - free_page(page_addr); - vfree(pte->sublevel); - memset(pte, 0, sizeof(struct gasket_page_table_entry)); - return -ENOMEM; - } - - /* make the addresses available to the device */ - dma_addr = (pte->dma_addr + pte->offset) | GASKET_VALID_SLOT_FLAG; - writeq(dma_addr, slot); - - pte->status = PTE_INUSE; - - return 0; -} - -/* - * Allocate slots in an extended page table. Check to see if a range of page - * table slots are available. If necessary, memory is allocated for second level - * page tables. - * - * Note that memory for second level page tables is allocated as needed, but - * that memory is only freed on the final close of the device file, when the - * page tables are repartitioned, or the device is removed. If there is an - * error or if the full range of slots is not available, any memory - * allocated for second level page tables remains allocated until final close, - * repartition, or device removal. - * - * The page table mutex must be held by the caller. - */ -static int gasket_alloc_extended_entries(struct gasket_page_table *pg_tbl, - ulong dev_addr, uint num_entries) -{ - int ret = 0; - uint remain, subtable_slot_idx, len; - struct gasket_page_table_entry *pte; - u64 __iomem *slot; - - remain = num_entries; - subtable_slot_idx = gasket_extended_lvl1_page_idx(pg_tbl, dev_addr); - pte = pg_tbl->entries + pg_tbl->num_simple_entries + - gasket_extended_lvl0_page_idx(pg_tbl, dev_addr); - slot = pg_tbl->base_slot + pg_tbl->num_simple_entries + - gasket_extended_lvl0_page_idx(pg_tbl, dev_addr); - - while (remain > 0) { - len = min(remain, - GASKET_PAGES_PER_SUBTABLE - subtable_slot_idx); - - if (pte->status == PTE_FREE) { - ret = gasket_alloc_extended_subtable(pg_tbl, pte, slot); - if (ret) { - dev_err(pg_tbl->device, - "no memory for extended addr subtable\n"); - return ret; - } - } else { - if (!gasket_is_pte_range_free(pte->sublevel + - subtable_slot_idx, len)) - return -EBUSY; - } - - remain -= len; - subtable_slot_idx = 0; - pte++; - slot++; - } - - return 0; -} - -/* - * gasket_map_extended_pages - Get and map buffers to extended addresses. - * If there is an error, no pages are mapped. - */ -static int gasket_map_extended_pages(struct gasket_page_table *pg_tbl, - ulong host_addr, ulong dev_addr, - uint num_pages) -{ - int ret; - ulong dev_addr_end; - uint slot_idx, remain, len; - struct gasket_page_table_entry *pte; - u64 __iomem *slot_base; - - ret = gasket_alloc_extended_entries(pg_tbl, dev_addr, num_pages); - if (ret) { - dev_addr_end = dev_addr + (num_pages / PAGE_SIZE) - 1; - dev_err(pg_tbl->device, - "page table slots (%lu,%lu) (@ 0x%lx) to (%lu,%lu) are not available\n", - gasket_extended_lvl0_page_idx(pg_tbl, dev_addr), - dev_addr, - gasket_extended_lvl1_page_idx(pg_tbl, dev_addr), - gasket_extended_lvl0_page_idx(pg_tbl, dev_addr_end), - gasket_extended_lvl1_page_idx(pg_tbl, dev_addr_end)); - return ret; - } - - remain = num_pages; - slot_idx = gasket_extended_lvl1_page_idx(pg_tbl, dev_addr); - pte = pg_tbl->entries + pg_tbl->num_simple_entries + - gasket_extended_lvl0_page_idx(pg_tbl, dev_addr); - - while (remain > 0) { - len = min(remain, GASKET_PAGES_PER_SUBTABLE - slot_idx); - - slot_base = - (u64 __iomem *)(page_address(pte->page) + pte->offset); - ret = gasket_perform_mapping(pg_tbl, pte->sublevel + slot_idx, - slot_base + slot_idx, host_addr, - len, 0); - if (ret) { - gasket_page_table_unmap_nolock(pg_tbl, dev_addr, - num_pages); - return ret; - } - - remain -= len; - slot_idx = 0; - pte++; - host_addr += len * PAGE_SIZE; - } - - return 0; -} - -/* - * See gasket_page_table.h for general description. - * - * gasket_page_table_map calls either gasket_map_simple_pages() or - * gasket_map_extended_pages() to actually perform the mapping. - * - * The page table mutex is held for the entire operation. - */ -int gasket_page_table_map(struct gasket_page_table *pg_tbl, ulong host_addr, - ulong dev_addr, uint num_pages) -{ - int ret; - - if (!num_pages) - return 0; - - mutex_lock(&pg_tbl->mutex); - - if (gasket_addr_is_simple(pg_tbl, dev_addr)) { - ret = gasket_map_simple_pages(pg_tbl, host_addr, dev_addr, - num_pages); - } else { - ret = gasket_map_extended_pages(pg_tbl, host_addr, dev_addr, - num_pages); - } - - mutex_unlock(&pg_tbl->mutex); - return ret; -} -EXPORT_SYMBOL(gasket_page_table_map); - -/* - * See gasket_page_table.h for general description. - * - * gasket_page_table_unmap takes the page table lock and calls either - * gasket_unmap_simple_pages() or gasket_unmap_extended_pages() to - * actually unmap the pages from device space. - * - * The page table mutex is held for the entire operation. - */ -void gasket_page_table_unmap(struct gasket_page_table *pg_tbl, ulong dev_addr, - uint num_pages) -{ - if (!num_pages) - return; - - mutex_lock(&pg_tbl->mutex); - gasket_page_table_unmap_nolock(pg_tbl, dev_addr, num_pages); - mutex_unlock(&pg_tbl->mutex); -} -EXPORT_SYMBOL(gasket_page_table_unmap); - -static void gasket_page_table_unmap_all_nolock(struct gasket_page_table *pg_tbl) -{ - gasket_unmap_simple_pages(pg_tbl, - gasket_components_to_dev_address(pg_tbl, 1, 0, - 0), - pg_tbl->num_simple_entries); - gasket_unmap_extended_pages(pg_tbl, - gasket_components_to_dev_address(pg_tbl, 0, - 0, 0), - pg_tbl->num_extended_entries * - GASKET_PAGES_PER_SUBTABLE); -} - -/* See gasket_page_table.h for description. */ -void gasket_page_table_unmap_all(struct gasket_page_table *pg_tbl) -{ - mutex_lock(&pg_tbl->mutex); - gasket_page_table_unmap_all_nolock(pg_tbl); - mutex_unlock(&pg_tbl->mutex); -} -EXPORT_SYMBOL(gasket_page_table_unmap_all); - -/* See gasket_page_table.h for description. */ -void gasket_page_table_reset(struct gasket_page_table *pg_tbl) -{ - mutex_lock(&pg_tbl->mutex); - gasket_page_table_unmap_all_nolock(pg_tbl); - writeq(pg_tbl->config.total_entries, pg_tbl->extended_offset_reg); - mutex_unlock(&pg_tbl->mutex); -} - -/* See gasket_page_table.h for description. */ -int gasket_page_table_lookup_page(struct gasket_page_table *pg_tbl, - ulong dev_addr, struct page **ppage, - ulong *poffset) -{ - uint page_num; - struct gasket_page_table_entry *pte; - - mutex_lock(&pg_tbl->mutex); - if (gasket_addr_is_simple(pg_tbl, dev_addr)) { - page_num = gasket_simple_page_idx(pg_tbl, dev_addr); - if (page_num >= pg_tbl->num_simple_entries) - goto fail; - - pte = pg_tbl->entries + page_num; - if (pte->status != PTE_INUSE) - goto fail; - } else { - /* Find the level 0 entry, */ - page_num = gasket_extended_lvl0_page_idx(pg_tbl, dev_addr); - if (page_num >= pg_tbl->num_extended_entries) - goto fail; - - pte = pg_tbl->entries + pg_tbl->num_simple_entries + page_num; - if (pte->status != PTE_INUSE) - goto fail; - - /* and its contained level 1 entry. */ - page_num = gasket_extended_lvl1_page_idx(pg_tbl, dev_addr); - pte = pte->sublevel + page_num; - if (pte->status != PTE_INUSE) - goto fail; - } - - *ppage = pte->page; - *poffset = pte->offset; - mutex_unlock(&pg_tbl->mutex); - return 0; - -fail: - *ppage = NULL; - *poffset = 0; - mutex_unlock(&pg_tbl->mutex); - return -EINVAL; -} - -/* See gasket_page_table.h for description. */ -bool gasket_page_table_are_addrs_bad(struct gasket_page_table *pg_tbl, - ulong host_addr, ulong dev_addr, - ulong bytes) -{ - if (host_addr & (PAGE_SIZE - 1)) { - dev_err(pg_tbl->device, - "host mapping address 0x%lx must be page aligned\n", - host_addr); - return true; - } - - return gasket_page_table_is_dev_addr_bad(pg_tbl, dev_addr, bytes); -} -EXPORT_SYMBOL(gasket_page_table_are_addrs_bad); - -/* See gasket_page_table.h for description. */ -bool gasket_page_table_is_dev_addr_bad(struct gasket_page_table *pg_tbl, - ulong dev_addr, ulong bytes) -{ - uint num_pages = bytes / PAGE_SIZE; - - if (bytes & (PAGE_SIZE - 1)) { - dev_err(pg_tbl->device, - "mapping size 0x%lX must be page aligned\n", bytes); - return true; - } - - if (num_pages == 0) { - dev_err(pg_tbl->device, - "requested mapping is less than one page: %lu / %lu\n", - bytes, PAGE_SIZE); - return true; - } - - if (gasket_addr_is_simple(pg_tbl, dev_addr)) - return gasket_is_simple_dev_addr_bad(pg_tbl, dev_addr, - num_pages); - return gasket_is_extended_dev_addr_bad(pg_tbl, dev_addr, num_pages); -} -EXPORT_SYMBOL(gasket_page_table_is_dev_addr_bad); - -/* See gasket_page_table.h for description. */ -uint gasket_page_table_max_size(struct gasket_page_table *page_table) -{ - if (!page_table) - return 0; - return page_table->config.total_entries; -} -EXPORT_SYMBOL(gasket_page_table_max_size); - -/* See gasket_page_table.h for description. */ -uint gasket_page_table_num_entries(struct gasket_page_table *pg_tbl) -{ - if (!pg_tbl) - return 0; - return pg_tbl->num_simple_entries + pg_tbl->num_extended_entries; -} -EXPORT_SYMBOL(gasket_page_table_num_entries); - -/* See gasket_page_table.h for description. */ -uint gasket_page_table_num_simple_entries(struct gasket_page_table *pg_tbl) -{ - if (!pg_tbl) - return 0; - return pg_tbl->num_simple_entries; -} -EXPORT_SYMBOL(gasket_page_table_num_simple_entries); - -/* See gasket_page_table.h for description. */ -uint gasket_page_table_num_active_pages(struct gasket_page_table *pg_tbl) -{ - if (!pg_tbl) - return 0; - return pg_tbl->num_active_pages; -} -EXPORT_SYMBOL(gasket_page_table_num_active_pages); - -/* See gasket_page_table.h */ -int gasket_page_table_system_status(struct gasket_page_table *page_table) -{ - if (!page_table) - return GASKET_STATUS_LAMED; - - if (gasket_page_table_num_entries(page_table) == 0) { - dev_dbg(page_table->device, "Page table size is 0\n"); - return GASKET_STATUS_LAMED; - } - - return GASKET_STATUS_ALIVE; -} - -/* Record the host_addr to coherent dma memory mapping. */ -int gasket_set_user_virt(struct gasket_dev *gasket_dev, u64 size, - dma_addr_t dma_address, ulong vma) -{ - int j; - struct gasket_page_table *pg_tbl; - - unsigned int num_pages = size / PAGE_SIZE; - - /* - * TODO: for future chipset, better handling of the case where multiple - * page tables are supported on a given device - */ - pg_tbl = gasket_dev->page_table[0]; - if (!pg_tbl) { - dev_dbg(gasket_dev->dev, "%s: invalid page table index\n", - __func__); - return 0; - } - for (j = 0; j < num_pages; j++) { - pg_tbl->coherent_pages[j].user_virt = - (u64)vma + j * PAGE_SIZE; - } - return 0; -} - -/* Allocate a block of coherent memory. */ -int gasket_alloc_coherent_memory(struct gasket_dev *gasket_dev, u64 size, - dma_addr_t *dma_address, u64 index) -{ - dma_addr_t handle; - void *mem; - int j; - unsigned int num_pages = DIV_ROUND_UP(size, PAGE_SIZE); - const struct gasket_driver_desc *driver_desc = - gasket_get_driver_desc(gasket_dev); - - if (!gasket_dev->page_table[index]) - return -EFAULT; - - if (num_pages == 0) - return -EINVAL; - - mem = dma_alloc_coherent(gasket_get_device(gasket_dev), - num_pages * PAGE_SIZE, &handle, GFP_KERNEL); - if (!mem) - goto nomem; - - gasket_dev->page_table[index]->num_coherent_pages = num_pages; - - /* allocate the physical memory block */ - gasket_dev->page_table[index]->coherent_pages = - kcalloc(num_pages, - sizeof(*gasket_dev->page_table[index]->coherent_pages), - GFP_KERNEL); - if (!gasket_dev->page_table[index]->coherent_pages) - goto nomem; - - gasket_dev->coherent_buffer.length_bytes = - PAGE_SIZE * (num_pages); - gasket_dev->coherent_buffer.phys_base = handle; - gasket_dev->coherent_buffer.virt_base = mem; - - *dma_address = driver_desc->coherent_buffer_description.base; - for (j = 0; j < num_pages; j++) { - gasket_dev->page_table[index]->coherent_pages[j].paddr = - handle + j * PAGE_SIZE; - gasket_dev->page_table[index]->coherent_pages[j].kernel_virt = - (u64)mem + j * PAGE_SIZE; - } - - return 0; - -nomem: - if (mem) { - dma_free_coherent(gasket_get_device(gasket_dev), - num_pages * PAGE_SIZE, mem, handle); - gasket_dev->coherent_buffer.length_bytes = 0; - gasket_dev->coherent_buffer.virt_base = NULL; - gasket_dev->coherent_buffer.phys_base = 0; - } - - kfree(gasket_dev->page_table[index]->coherent_pages); - gasket_dev->page_table[index]->coherent_pages = NULL; - gasket_dev->page_table[index]->num_coherent_pages = 0; - return -ENOMEM; -} - -/* Free a block of coherent memory. */ -int gasket_free_coherent_memory(struct gasket_dev *gasket_dev, u64 size, - dma_addr_t dma_address, u64 index) -{ - const struct gasket_driver_desc *driver_desc; - - if (!gasket_dev->page_table[index]) - return -EFAULT; - - driver_desc = gasket_get_driver_desc(gasket_dev); - - if (driver_desc->coherent_buffer_description.base != dma_address) - return -EADDRNOTAVAIL; - - if (gasket_dev->coherent_buffer.length_bytes) { - dma_free_coherent(gasket_get_device(gasket_dev), - gasket_dev->coherent_buffer.length_bytes, - gasket_dev->coherent_buffer.virt_base, - gasket_dev->coherent_buffer.phys_base); - gasket_dev->coherent_buffer.length_bytes = 0; - gasket_dev->coherent_buffer.virt_base = NULL; - gasket_dev->coherent_buffer.phys_base = 0; - } - - kfree(gasket_dev->page_table[index]->coherent_pages); - gasket_dev->page_table[index]->coherent_pages = NULL; - gasket_dev->page_table[index]->num_coherent_pages = 0; - - return 0; -} - -/* Release all coherent memory. */ -void gasket_free_coherent_memory_all(struct gasket_dev *gasket_dev, u64 index) -{ - if (!gasket_dev->page_table[index]) - return; - - if (gasket_dev->coherent_buffer.length_bytes) { - dma_free_coherent(gasket_get_device(gasket_dev), - gasket_dev->coherent_buffer.length_bytes, - gasket_dev->coherent_buffer.virt_base, - gasket_dev->coherent_buffer.phys_base); - gasket_dev->coherent_buffer.length_bytes = 0; - gasket_dev->coherent_buffer.virt_base = NULL; - gasket_dev->coherent_buffer.phys_base = 0; - } -} |