diff options
Diffstat (limited to 'drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c')
-rw-r--r-- | drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c | 85 |
1 files changed, 61 insertions, 24 deletions
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c b/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c index 91bd258ecf1b..8ce78f42dfc0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c @@ -35,7 +35,6 @@ #include <linux/module.h> #include <linux/delay.h> #include <linux/mlx5/driver.h> -#include <linux/mlx5/cmd.h> #include "mlx5_core.h" #include "lib/eq.h" @@ -51,6 +50,7 @@ struct mlx5_pages_req { u8 ec_function; s32 npages; struct work_struct work; + u8 release_all; }; struct fw_page { @@ -136,8 +136,8 @@ static struct fw_page *find_fw_page(struct mlx5_core_dev *dev, u64 addr) static int mlx5_cmd_query_pages(struct mlx5_core_dev *dev, u16 *func_id, s32 *npages, int boot) { - u32 out[MLX5_ST_SZ_DW(query_pages_out)] = {0}; - u32 in[MLX5_ST_SZ_DW(query_pages_in)] = {0}; + u32 out[MLX5_ST_SZ_DW(query_pages_out)] = {}; + u32 in[MLX5_ST_SZ_DW(query_pages_in)] = {}; int err; MLX5_SET(query_pages_in, in, opcode, MLX5_CMD_OP_QUERY_PAGES); @@ -146,7 +146,7 @@ static int mlx5_cmd_query_pages(struct mlx5_core_dev *dev, u16 *func_id, MLX5_QUERY_PAGES_IN_OP_MOD_INIT_PAGES); MLX5_SET(query_pages_in, in, embedded_cpu_function, mlx5_core_is_ecpf(dev)); - err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); + err = mlx5_cmd_exec_inout(dev, query_pages, in, out); if (err) return err; @@ -182,25 +182,17 @@ static int alloc_4k(struct mlx5_core_dev *dev, u64 *addr) #define MLX5_U64_4K_PAGE_MASK ((~(u64)0U) << PAGE_SHIFT) -static void free_4k(struct mlx5_core_dev *dev, u64 addr) +static void free_fwp(struct mlx5_core_dev *dev, struct fw_page *fwp) { - struct fw_page *fwp; - int n; - - fwp = find_fw_page(dev, addr & MLX5_U64_4K_PAGE_MASK); - if (!fwp) { - mlx5_core_warn(dev, "page not found\n"); - return; - } + int n = (fwp->addr & ~MLX5_U64_4K_PAGE_MASK) >> MLX5_ADAPTER_PAGE_SHIFT; - n = (addr & ~MLX5_U64_4K_PAGE_MASK) >> MLX5_ADAPTER_PAGE_SHIFT; fwp->free_count++; set_bit(n, &fwp->bitmask); if (fwp->free_count == MLX5_NUM_4K_IN_PAGE) { rb_erase(&fwp->rb_node, &dev->priv.page_root); if (fwp->free_count != 1) list_del(&fwp->list); - dma_unmap_page(dev->device, addr & MLX5_U64_4K_PAGE_MASK, + dma_unmap_page(dev->device, fwp->addr & MLX5_U64_4K_PAGE_MASK, PAGE_SIZE, DMA_BIDIRECTIONAL); __free_page(fwp->page); kfree(fwp); @@ -209,6 +201,18 @@ static void free_4k(struct mlx5_core_dev *dev, u64 addr) } } +static void free_addr(struct mlx5_core_dev *dev, u64 addr) +{ + struct fw_page *fwp; + + fwp = find_fw_page(dev, addr & MLX5_U64_4K_PAGE_MASK); + if (!fwp) { + mlx5_core_warn_rl(dev, "page not found\n"); + return; + } + free_fwp(dev, fwp); +} + static int alloc_system_page(struct mlx5_core_dev *dev, u16 func_id) { struct device *device = dev->device; @@ -257,8 +261,7 @@ err_mapping: static void page_notify_fail(struct mlx5_core_dev *dev, u16 func_id, bool ec_function) { - u32 out[MLX5_ST_SZ_DW(manage_pages_out)] = {0}; - u32 in[MLX5_ST_SZ_DW(manage_pages_in)] = {0}; + u32 in[MLX5_ST_SZ_DW(manage_pages_in)] = {}; int err; MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES); @@ -266,7 +269,7 @@ static void page_notify_fail(struct mlx5_core_dev *dev, u16 func_id, MLX5_SET(manage_pages_in, in, function_id, func_id); MLX5_SET(manage_pages_in, in, embedded_cpu_function, ec_function); - err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); + err = mlx5_cmd_exec_in(dev, manage_pages, in); if (err) mlx5_core_warn(dev, "page notify failed func_id(%d) err(%d)\n", func_id, err); @@ -331,7 +334,7 @@ retry: out_4k: for (i--; i >= 0; i--) - free_4k(dev, MLX5_GET64(manage_pages_in, in, pas[i])); + free_addr(dev, MLX5_GET64(manage_pages_in, in, pas[i])); out_free: kvfree(in); if (notify_fail) @@ -339,6 +342,33 @@ out_free: return err; } +static void release_all_pages(struct mlx5_core_dev *dev, u32 func_id, + bool ec_function) +{ + struct rb_node *p; + int npages = 0; + + p = rb_first(&dev->priv.page_root); + while (p) { + struct fw_page *fwp = rb_entry(p, struct fw_page, rb_node); + + p = rb_next(p); + if (fwp->func_id != func_id) + continue; + free_fwp(dev, fwp); + npages++; + } + + dev->priv.fw_pages -= npages; + if (func_id) + dev->priv.vfs_pages -= npages; + else if (mlx5_core_is_ecpf(dev) && !ec_function) + dev->priv.peer_pf_pages -= npages; + + mlx5_core_dbg(dev, "npages %d, ec_function %d, func_id 0x%x\n", + npages, ec_function, func_id); +} + static int reclaim_pages_cmd(struct mlx5_core_dev *dev, u32 *in, int in_size, u32 *out, int out_size) { @@ -374,7 +404,7 @@ static int reclaim_pages(struct mlx5_core_dev *dev, u32 func_id, int npages, int *nclaimed, bool ec_function) { int outlen = MLX5_ST_SZ_BYTES(manage_pages_out); - u32 in[MLX5_ST_SZ_DW(manage_pages_in)] = {0}; + u32 in[MLX5_ST_SZ_DW(manage_pages_in)] = {}; int num_claimed; u32 *out; int err; @@ -410,7 +440,7 @@ static int reclaim_pages(struct mlx5_core_dev *dev, u32 func_id, int npages, } for (i = 0; i < num_claimed; i++) - free_4k(dev, MLX5_GET64(manage_pages_out, out, pas[i])); + free_addr(dev, MLX5_GET64(manage_pages_out, out, pas[i])); if (nclaimed) *nclaimed = num_claimed; @@ -432,7 +462,9 @@ static void pages_work_handler(struct work_struct *work) struct mlx5_core_dev *dev = req->dev; int err = 0; - if (req->npages < 0) + if (req->release_all) + release_all_pages(dev, req->func_id, req->ec_function); + else if (req->npages < 0) err = reclaim_pages(dev, req->func_id, -1 * req->npages, NULL, req->ec_function); else if (req->npages > 0) @@ -447,6 +479,7 @@ static void pages_work_handler(struct work_struct *work) enum { EC_FUNCTION_MASK = 0x8000, + RELEASE_ALL_PAGES_MASK = 0x4000, }; static int req_pages_handler(struct notifier_block *nb, @@ -457,6 +490,7 @@ static int req_pages_handler(struct notifier_block *nb, struct mlx5_priv *priv; struct mlx5_eqe *eqe; bool ec_function; + bool release_all; u16 func_id; s32 npages; @@ -467,8 +501,10 @@ static int req_pages_handler(struct notifier_block *nb, func_id = be16_to_cpu(eqe->data.req_pages.func_id); npages = be32_to_cpu(eqe->data.req_pages.num_pages); ec_function = be16_to_cpu(eqe->data.req_pages.ec_function) & EC_FUNCTION_MASK; - mlx5_core_dbg(dev, "page request for func 0x%x, npages %d\n", - func_id, npages); + release_all = be16_to_cpu(eqe->data.req_pages.ec_function) & + RELEASE_ALL_PAGES_MASK; + mlx5_core_dbg(dev, "page request for func 0x%x, npages %d, release_all %d\n", + func_id, npages, release_all); req = kzalloc(sizeof(*req), GFP_ATOMIC); if (!req) { mlx5_core_warn(dev, "failed to allocate pages request\n"); @@ -479,6 +515,7 @@ static int req_pages_handler(struct notifier_block *nb, req->func_id = func_id; req->npages = npages; req->ec_function = ec_function; + req->release_all = release_all; INIT_WORK(&req->work, pages_work_handler); queue_work(dev->priv.pg_wq, &req->work); return NOTIFY_OK; |