summaryrefslogtreecommitdiff
path: root/net/core/flow.c
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2017-09-07 00:45:08 +0300
committerLinus Torvalds <torvalds@linux-foundation.org>2017-09-07 00:45:08 +0300
commitaae3dbb4776e7916b6cd442d00159bea27a695c1 (patch)
treed074c5d783a81e7e2e084b1eba77f57459da7e37 /net/core/flow.c
parentec3604c7a5aae8953545b0d05495357009a960e5 (diff)
parent66bed8465a808400eb14562510e26c8818082cb8 (diff)
downloadlinux-aae3dbb4776e7916b6cd442d00159bea27a695c1.tar.xz
Merge git://git.kernel.org/pub/scm/linux/kernel/git/davem/net-next
Pull networking updates from David Miller: 1) Support ipv6 checksum offload in sunvnet driver, from Shannon Nelson. 2) Move to RB-tree instead of custom AVL code in inetpeer, from Eric Dumazet. 3) Allow generic XDP to work on virtual devices, from John Fastabend. 4) Add bpf device maps and XDP_REDIRECT, which can be used to build arbitrary switching frameworks using XDP. From John Fastabend. 5) Remove UFO offloads from the tree, gave us little other than bugs. 6) Remove the IPSEC flow cache, from Florian Westphal. 7) Support ipv6 route offload in mlxsw driver. 8) Support VF representors in bnxt_en, from Sathya Perla. 9) Add support for forward error correction modes to ethtool, from Vidya Sagar Ravipati. 10) Add time filter for packet scheduler action dumping, from Jamal Hadi Salim. 11) Extend the zerocopy sendmsg() used by virtio and tap to regular sockets via MSG_ZEROCOPY. From Willem de Bruijn. 12) Significantly rework value tracking in the BPF verifier, from Edward Cree. 13) Add new jump instructions to eBPF, from Daniel Borkmann. 14) Rework rtnetlink plumbing so that operations can be run without taking the RTNL semaphore. From Florian Westphal. 15) Support XDP in tap driver, from Jason Wang. 16) Add 32-bit eBPF JIT for ARM, from Shubham Bansal. 17) Add Huawei hinic ethernet driver. 18) Allow to report MD5 keys in TCP inet_diag dumps, from Ivan Delalande. * git://git.kernel.org/pub/scm/linux/kernel/git/davem/net-next: (1780 commits) i40e: point wb_desc at the nvm_wb_desc during i40e_read_nvm_aq i40e: avoid NVM acquire deadlock during NVM update drivers: net: xgene: Remove return statement from void function drivers: net: xgene: Configure tx/rx delay for ACPI drivers: net: xgene: Read tx/rx delay for ACPI rocker: fix kcalloc parameter order rds: Fix non-atomic operation on shared flag variable net: sched: don't use GFP_KERNEL under spin lock vhost_net: correctly check tx avail during rx busy polling net: mdio-mux: add mdio_mux parameter to mdio_mux_init() rxrpc: Make service connection lookup always check for retry net: stmmac: Delete dead code for MDIO registration gianfar: Fix Tx flow control deactivation cxgb4: Ignore MPS_TX_INT_CAUSE[Bubble] for T6 cxgb4: Fix pause frame count in t4_get_port_stats cxgb4: fix memory leak tun: rename generic_xdp to skb_xdp tun: reserve extra headroom only when XDP is set net: dsa: bcm_sf2: Configure IMP port TC2QOS mapping net: dsa: bcm_sf2: Advertise number of egress queues ...
Diffstat (limited to 'net/core/flow.c')
-rw-r--r--net/core/flow.c516
1 files changed, 0 insertions, 516 deletions
diff --git a/net/core/flow.c b/net/core/flow.c
deleted file mode 100644
index f7f5d1932a27..000000000000
--- a/net/core/flow.c
+++ /dev/null
@@ -1,516 +0,0 @@
-/* flow.c: Generic flow cache.
- *
- * Copyright (C) 2003 Alexey N. Kuznetsov (kuznet@ms2.inr.ac.ru)
- * Copyright (C) 2003 David S. Miller (davem@redhat.com)
- */
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/list.h>
-#include <linux/jhash.h>
-#include <linux/interrupt.h>
-#include <linux/mm.h>
-#include <linux/random.h>
-#include <linux/init.h>
-#include <linux/slab.h>
-#include <linux/smp.h>
-#include <linux/completion.h>
-#include <linux/percpu.h>
-#include <linux/bitops.h>
-#include <linux/notifier.h>
-#include <linux/cpu.h>
-#include <linux/cpumask.h>
-#include <linux/mutex.h>
-#include <net/flow.h>
-#include <linux/atomic.h>
-#include <linux/security.h>
-#include <net/net_namespace.h>
-
-struct flow_cache_entry {
- union {
- struct hlist_node hlist;
- struct list_head gc_list;
- } u;
- struct net *net;
- u16 family;
- u8 dir;
- u32 genid;
- struct flowi key;
- struct flow_cache_object *object;
-};
-
-struct flow_flush_info {
- struct flow_cache *cache;
- atomic_t cpuleft;
- struct completion completion;
-};
-
-static struct kmem_cache *flow_cachep __read_mostly;
-
-#define flow_cache_hash_size(cache) (1U << (cache)->hash_shift)
-#define FLOW_HASH_RND_PERIOD (10 * 60 * HZ)
-
-static void flow_cache_new_hashrnd(unsigned long arg)
-{
- struct flow_cache *fc = (void *) arg;
- int i;
-
- for_each_possible_cpu(i)
- per_cpu_ptr(fc->percpu, i)->hash_rnd_recalc = 1;
-
- fc->rnd_timer.expires = jiffies + FLOW_HASH_RND_PERIOD;
- add_timer(&fc->rnd_timer);
-}
-
-static int flow_entry_valid(struct flow_cache_entry *fle,
- struct netns_xfrm *xfrm)
-{
- if (atomic_read(&xfrm->flow_cache_genid) != fle->genid)
- return 0;
- if (fle->object && !fle->object->ops->check(fle->object))
- return 0;
- return 1;
-}
-
-static void flow_entry_kill(struct flow_cache_entry *fle,
- struct netns_xfrm *xfrm)
-{
- if (fle->object)
- fle->object->ops->delete(fle->object);
- kmem_cache_free(flow_cachep, fle);
-}
-
-static void flow_cache_gc_task(struct work_struct *work)
-{
- struct list_head gc_list;
- struct flow_cache_entry *fce, *n;
- struct netns_xfrm *xfrm = container_of(work, struct netns_xfrm,
- flow_cache_gc_work);
-
- INIT_LIST_HEAD(&gc_list);
- spin_lock_bh(&xfrm->flow_cache_gc_lock);
- list_splice_tail_init(&xfrm->flow_cache_gc_list, &gc_list);
- spin_unlock_bh(&xfrm->flow_cache_gc_lock);
-
- list_for_each_entry_safe(fce, n, &gc_list, u.gc_list) {
- flow_entry_kill(fce, xfrm);
- atomic_dec(&xfrm->flow_cache_gc_count);
- }
-}
-
-static void flow_cache_queue_garbage(struct flow_cache_percpu *fcp,
- unsigned int deleted,
- struct list_head *gc_list,
- struct netns_xfrm *xfrm)
-{
- if (deleted) {
- atomic_add(deleted, &xfrm->flow_cache_gc_count);
- fcp->hash_count -= deleted;
- spin_lock_bh(&xfrm->flow_cache_gc_lock);
- list_splice_tail(gc_list, &xfrm->flow_cache_gc_list);
- spin_unlock_bh(&xfrm->flow_cache_gc_lock);
- schedule_work(&xfrm->flow_cache_gc_work);
- }
-}
-
-static void __flow_cache_shrink(struct flow_cache *fc,
- struct flow_cache_percpu *fcp,
- unsigned int shrink_to)
-{
- struct flow_cache_entry *fle;
- struct hlist_node *tmp;
- LIST_HEAD(gc_list);
- unsigned int deleted = 0;
- struct netns_xfrm *xfrm = container_of(fc, struct netns_xfrm,
- flow_cache_global);
- unsigned int i;
-
- for (i = 0; i < flow_cache_hash_size(fc); i++) {
- unsigned int saved = 0;
-
- hlist_for_each_entry_safe(fle, tmp,
- &fcp->hash_table[i], u.hlist) {
- if (saved < shrink_to &&
- flow_entry_valid(fle, xfrm)) {
- saved++;
- } else {
- deleted++;
- hlist_del(&fle->u.hlist);
- list_add_tail(&fle->u.gc_list, &gc_list);
- }
- }
- }
-
- flow_cache_queue_garbage(fcp, deleted, &gc_list, xfrm);
-}
-
-static void flow_cache_shrink(struct flow_cache *fc,
- struct flow_cache_percpu *fcp)
-{
- unsigned int shrink_to = fc->low_watermark / flow_cache_hash_size(fc);
-
- __flow_cache_shrink(fc, fcp, shrink_to);
-}
-
-static void flow_new_hash_rnd(struct flow_cache *fc,
- struct flow_cache_percpu *fcp)
-{
- get_random_bytes(&fcp->hash_rnd, sizeof(u32));
- fcp->hash_rnd_recalc = 0;
- __flow_cache_shrink(fc, fcp, 0);
-}
-
-static u32 flow_hash_code(struct flow_cache *fc,
- struct flow_cache_percpu *fcp,
- const struct flowi *key,
- unsigned int keysize)
-{
- const u32 *k = (const u32 *) key;
- const u32 length = keysize * sizeof(flow_compare_t) / sizeof(u32);
-
- return jhash2(k, length, fcp->hash_rnd)
- & (flow_cache_hash_size(fc) - 1);
-}
-
-/* I hear what you're saying, use memcmp. But memcmp cannot make
- * important assumptions that we can here, such as alignment.
- */
-static int flow_key_compare(const struct flowi *key1, const struct flowi *key2,
- unsigned int keysize)
-{
- const flow_compare_t *k1, *k1_lim, *k2;
-
- k1 = (const flow_compare_t *) key1;
- k1_lim = k1 + keysize;
-
- k2 = (const flow_compare_t *) key2;
-
- do {
- if (*k1++ != *k2++)
- return 1;
- } while (k1 < k1_lim);
-
- return 0;
-}
-
-struct flow_cache_object *
-flow_cache_lookup(struct net *net, const struct flowi *key, u16 family, u8 dir,
- flow_resolve_t resolver, void *ctx)
-{
- struct flow_cache *fc = &net->xfrm.flow_cache_global;
- struct flow_cache_percpu *fcp;
- struct flow_cache_entry *fle, *tfle;
- struct flow_cache_object *flo;
- unsigned int keysize;
- unsigned int hash;
-
- local_bh_disable();
- fcp = this_cpu_ptr(fc->percpu);
-
- fle = NULL;
- flo = NULL;
-
- keysize = flow_key_size(family);
- if (!keysize)
- goto nocache;
-
- /* Packet really early in init? Making flow_cache_init a
- * pre-smp initcall would solve this. --RR */
- if (!fcp->hash_table)
- goto nocache;
-
- if (fcp->hash_rnd_recalc)
- flow_new_hash_rnd(fc, fcp);
-
- hash = flow_hash_code(fc, fcp, key, keysize);
- hlist_for_each_entry(tfle, &fcp->hash_table[hash], u.hlist) {
- if (tfle->net == net &&
- tfle->family == family &&
- tfle->dir == dir &&
- flow_key_compare(key, &tfle->key, keysize) == 0) {
- fle = tfle;
- break;
- }
- }
-
- if (unlikely(!fle)) {
- if (fcp->hash_count > fc->high_watermark)
- flow_cache_shrink(fc, fcp);
-
- if (atomic_read(&net->xfrm.flow_cache_gc_count) >
- 2 * num_online_cpus() * fc->high_watermark) {
- flo = ERR_PTR(-ENOBUFS);
- goto ret_object;
- }
-
- fle = kmem_cache_alloc(flow_cachep, GFP_ATOMIC);
- if (fle) {
- fle->net = net;
- fle->family = family;
- fle->dir = dir;
- memcpy(&fle->key, key, keysize * sizeof(flow_compare_t));
- fle->object = NULL;
- hlist_add_head(&fle->u.hlist, &fcp->hash_table[hash]);
- fcp->hash_count++;
- }
- } else if (likely(fle->genid == atomic_read(&net->xfrm.flow_cache_genid))) {
- flo = fle->object;
- if (!flo)
- goto ret_object;
- flo = flo->ops->get(flo);
- if (flo)
- goto ret_object;
- } else if (fle->object) {
- flo = fle->object;
- flo->ops->delete(flo);
- fle->object = NULL;
- }
-
-nocache:
- flo = NULL;
- if (fle) {
- flo = fle->object;
- fle->object = NULL;
- }
- flo = resolver(net, key, family, dir, flo, ctx);
- if (fle) {
- fle->genid = atomic_read(&net->xfrm.flow_cache_genid);
- if (!IS_ERR(flo))
- fle->object = flo;
- else
- fle->genid--;
- } else {
- if (!IS_ERR_OR_NULL(flo))
- flo->ops->delete(flo);
- }
-ret_object:
- local_bh_enable();
- return flo;
-}
-EXPORT_SYMBOL(flow_cache_lookup);
-
-static void flow_cache_flush_tasklet(unsigned long data)
-{
- struct flow_flush_info *info = (void *)data;
- struct flow_cache *fc = info->cache;
- struct flow_cache_percpu *fcp;
- struct flow_cache_entry *fle;
- struct hlist_node *tmp;
- LIST_HEAD(gc_list);
- unsigned int deleted = 0;
- struct netns_xfrm *xfrm = container_of(fc, struct netns_xfrm,
- flow_cache_global);
- unsigned int i;
-
- fcp = this_cpu_ptr(fc->percpu);
- for (i = 0; i < flow_cache_hash_size(fc); i++) {
- hlist_for_each_entry_safe(fle, tmp,
- &fcp->hash_table[i], u.hlist) {
- if (flow_entry_valid(fle, xfrm))
- continue;
-
- deleted++;
- hlist_del(&fle->u.hlist);
- list_add_tail(&fle->u.gc_list, &gc_list);
- }
- }
-
- flow_cache_queue_garbage(fcp, deleted, &gc_list, xfrm);
-
- if (atomic_dec_and_test(&info->cpuleft))
- complete(&info->completion);
-}
-
-/*
- * Return whether a cpu needs flushing. Conservatively, we assume
- * the presence of any entries means the core may require flushing,
- * since the flow_cache_ops.check() function may assume it's running
- * on the same core as the per-cpu cache component.
- */
-static int flow_cache_percpu_empty(struct flow_cache *fc, int cpu)
-{
- struct flow_cache_percpu *fcp;
- unsigned int i;
-
- fcp = per_cpu_ptr(fc->percpu, cpu);
- for (i = 0; i < flow_cache_hash_size(fc); i++)
- if (!hlist_empty(&fcp->hash_table[i]))
- return 0;
- return 1;
-}
-
-static void flow_cache_flush_per_cpu(void *data)
-{
- struct flow_flush_info *info = data;
- struct tasklet_struct *tasklet;
-
- tasklet = &this_cpu_ptr(info->cache->percpu)->flush_tasklet;
- tasklet->data = (unsigned long)info;
- tasklet_schedule(tasklet);
-}
-
-void flow_cache_flush(struct net *net)
-{
- struct flow_flush_info info;
- cpumask_var_t mask;
- int i, self;
-
- /* Track which cpus need flushing to avoid disturbing all cores. */
- if (!alloc_cpumask_var(&mask, GFP_KERNEL))
- return;
- cpumask_clear(mask);
-
- /* Don't want cpus going down or up during this. */
- get_online_cpus();
- mutex_lock(&net->xfrm.flow_flush_sem);
- info.cache = &net->xfrm.flow_cache_global;
- for_each_online_cpu(i)
- if (!flow_cache_percpu_empty(info.cache, i))
- cpumask_set_cpu(i, mask);
- atomic_set(&info.cpuleft, cpumask_weight(mask));
- if (atomic_read(&info.cpuleft) == 0)
- goto done;
-
- init_completion(&info.completion);
-
- local_bh_disable();
- self = cpumask_test_and_clear_cpu(smp_processor_id(), mask);
- on_each_cpu_mask(mask, flow_cache_flush_per_cpu, &info, 0);
- if (self)
- flow_cache_flush_tasklet((unsigned long)&info);
- local_bh_enable();
-
- wait_for_completion(&info.completion);
-
-done:
- mutex_unlock(&net->xfrm.flow_flush_sem);
- put_online_cpus();
- free_cpumask_var(mask);
-}
-
-static void flow_cache_flush_task(struct work_struct *work)
-{
- struct netns_xfrm *xfrm = container_of(work, struct netns_xfrm,
- flow_cache_flush_work);
- struct net *net = container_of(xfrm, struct net, xfrm);
-
- flow_cache_flush(net);
-}
-
-void flow_cache_flush_deferred(struct net *net)
-{
- schedule_work(&net->xfrm.flow_cache_flush_work);
-}
-
-static int flow_cache_cpu_prepare(struct flow_cache *fc, int cpu)
-{
- struct flow_cache_percpu *fcp = per_cpu_ptr(fc->percpu, cpu);
- unsigned int sz = sizeof(struct hlist_head) * flow_cache_hash_size(fc);
-
- if (!fcp->hash_table) {
- fcp->hash_table = kzalloc_node(sz, GFP_KERNEL, cpu_to_node(cpu));
- if (!fcp->hash_table) {
- pr_err("NET: failed to allocate flow cache sz %u\n", sz);
- return -ENOMEM;
- }
- fcp->hash_rnd_recalc = 1;
- fcp->hash_count = 0;
- tasklet_init(&fcp->flush_tasklet, flow_cache_flush_tasklet, 0);
- }
- return 0;
-}
-
-static int flow_cache_cpu_up_prep(unsigned int cpu, struct hlist_node *node)
-{
- struct flow_cache *fc = hlist_entry_safe(node, struct flow_cache, node);
-
- return flow_cache_cpu_prepare(fc, cpu);
-}
-
-static int flow_cache_cpu_dead(unsigned int cpu, struct hlist_node *node)
-{
- struct flow_cache *fc = hlist_entry_safe(node, struct flow_cache, node);
- struct flow_cache_percpu *fcp = per_cpu_ptr(fc->percpu, cpu);
-
- __flow_cache_shrink(fc, fcp, 0);
- return 0;
-}
-
-int flow_cache_init(struct net *net)
-{
- int i;
- struct flow_cache *fc = &net->xfrm.flow_cache_global;
-
- if (!flow_cachep)
- flow_cachep = kmem_cache_create("flow_cache",
- sizeof(struct flow_cache_entry),
- 0, SLAB_PANIC, NULL);
- spin_lock_init(&net->xfrm.flow_cache_gc_lock);
- INIT_LIST_HEAD(&net->xfrm.flow_cache_gc_list);
- INIT_WORK(&net->xfrm.flow_cache_gc_work, flow_cache_gc_task);
- INIT_WORK(&net->xfrm.flow_cache_flush_work, flow_cache_flush_task);
- mutex_init(&net->xfrm.flow_flush_sem);
- atomic_set(&net->xfrm.flow_cache_gc_count, 0);
-
- fc->hash_shift = 10;
- fc->low_watermark = 2 * flow_cache_hash_size(fc);
- fc->high_watermark = 4 * flow_cache_hash_size(fc);
-
- fc->percpu = alloc_percpu(struct flow_cache_percpu);
- if (!fc->percpu)
- return -ENOMEM;
-
- if (cpuhp_state_add_instance(CPUHP_NET_FLOW_PREPARE, &fc->node))
- goto err;
-
- setup_timer(&fc->rnd_timer, flow_cache_new_hashrnd,
- (unsigned long) fc);
- fc->rnd_timer.expires = jiffies + FLOW_HASH_RND_PERIOD;
- add_timer(&fc->rnd_timer);
-
- return 0;
-
-err:
- for_each_possible_cpu(i) {
- struct flow_cache_percpu *fcp = per_cpu_ptr(fc->percpu, i);
- kfree(fcp->hash_table);
- fcp->hash_table = NULL;
- }
-
- free_percpu(fc->percpu);
- fc->percpu = NULL;
-
- return -ENOMEM;
-}
-EXPORT_SYMBOL(flow_cache_init);
-
-void flow_cache_fini(struct net *net)
-{
- int i;
- struct flow_cache *fc = &net->xfrm.flow_cache_global;
-
- del_timer_sync(&fc->rnd_timer);
-
- cpuhp_state_remove_instance_nocalls(CPUHP_NET_FLOW_PREPARE, &fc->node);
-
- for_each_possible_cpu(i) {
- struct flow_cache_percpu *fcp = per_cpu_ptr(fc->percpu, i);
- kfree(fcp->hash_table);
- fcp->hash_table = NULL;
- }
-
- free_percpu(fc->percpu);
- fc->percpu = NULL;
-}
-EXPORT_SYMBOL(flow_cache_fini);
-
-void __init flow_cache_hp_init(void)
-{
- int ret;
-
- ret = cpuhp_setup_state_multi(CPUHP_NET_FLOW_PREPARE,
- "net/flow:prepare",
- flow_cache_cpu_up_prep,
- flow_cache_cpu_dead);
- WARN_ON(ret < 0);
-}