summaryrefslogtreecommitdiff
path: root/net/xfrm
diff options
context:
space:
mode:
Diffstat (limited to 'net/xfrm')
-rw-r--r--net/xfrm/xfrm_device.c9
-rw-r--r--net/xfrm/xfrm_input.c7
-rw-r--r--net/xfrm/xfrm_output.c3
-rw-r--r--net/xfrm/xfrm_policy.c435
-rw-r--r--net/xfrm/xfrm_state.c21
-rw-r--r--net/xfrm/xfrm_user.c16
6 files changed, 210 insertions, 281 deletions
diff --git a/net/xfrm/xfrm_device.c b/net/xfrm/xfrm_device.c
index 5f7e8bfa0c2d..acf00104ef31 100644
--- a/net/xfrm/xfrm_device.c
+++ b/net/xfrm/xfrm_device.c
@@ -63,7 +63,7 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x,
xfrm_address_t *daddr;
if (!x->type_offload)
- return 0;
+ return -EINVAL;
/* We don't yet support UDP encapsulation, TFC padding and ESN. */
if (x->encap || x->tfcpad || (x->props.flags & XFRM_STATE_ESN))
@@ -79,7 +79,8 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x,
daddr = &x->props.saddr;
}
- dst = __xfrm_dst_lookup(net, 0, 0, saddr, daddr, x->props.family);
+ dst = __xfrm_dst_lookup(net, 0, 0, saddr, daddr,
+ x->props.family, x->props.output_mark);
if (IS_ERR(dst))
return 0;
@@ -153,6 +154,7 @@ static int xfrm_dev_register(struct net_device *dev)
static int xfrm_dev_unregister(struct net_device *dev)
{
+ xfrm_policy_cache_flush();
return NOTIFY_DONE;
}
@@ -175,8 +177,7 @@ static int xfrm_dev_down(struct net_device *dev)
if (dev->features & NETIF_F_HW_ESP)
xfrm_dev_state_flush(dev_net(dev), dev, true);
- xfrm_garbage_collect(dev_net(dev));
-
+ xfrm_policy_cache_flush();
return NOTIFY_DONE;
}
diff --git a/net/xfrm/xfrm_input.c b/net/xfrm/xfrm_input.c
index 923205e279f7..2515cd2bc5db 100644
--- a/net/xfrm/xfrm_input.c
+++ b/net/xfrm/xfrm_input.c
@@ -247,6 +247,11 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type)
goto drop;
}
+ if (xo->status & CRYPTO_INVALID_PROTOCOL) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR);
+ goto drop;
+ }
+
XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
goto drop;
}
@@ -424,6 +429,7 @@ resume:
nf_reset(skb);
if (decaps) {
+ skb->sp->olen = 0;
skb_dst_drop(skb);
gro_cells_receive(&gro_cells, skb);
return 0;
@@ -434,6 +440,7 @@ resume:
err = x->inner_mode->afinfo->transport_finish(skb, xfrm_gro || async);
if (xfrm_gro) {
+ skb->sp->olen = 0;
skb_dst_drop(skb);
gro_cells_receive(&gro_cells, skb);
return err;
diff --git a/net/xfrm/xfrm_output.c b/net/xfrm/xfrm_output.c
index 8c0b6722aaa8..31a2e6d34dba 100644
--- a/net/xfrm/xfrm_output.c
+++ b/net/xfrm/xfrm_output.c
@@ -66,6 +66,9 @@ static int xfrm_output_one(struct sk_buff *skb, int err)
goto error_nolock;
}
+ if (x->props.output_mark)
+ skb->mark = x->props.output_mark;
+
err = x->outer_mode->output(x, skb);
if (err) {
XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEMODEERROR);
diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c
index 69b16ee327d9..f06253969972 100644
--- a/net/xfrm/xfrm_policy.c
+++ b/net/xfrm/xfrm_policy.c
@@ -24,6 +24,7 @@
#include <linux/netfilter.h>
#include <linux/module.h>
#include <linux/cache.h>
+#include <linux/cpu.h>
#include <linux/audit.h>
#include <net/dst.h>
#include <net/flow.h>
@@ -44,6 +45,8 @@ struct xfrm_flo {
u8 flags;
};
+static DEFINE_PER_CPU(struct xfrm_dst *, xfrm_last_dst);
+static struct work_struct *xfrm_pcpu_work __read_mostly;
static DEFINE_SPINLOCK(xfrm_policy_afinfo_lock);
static struct xfrm_policy_afinfo const __rcu *xfrm_policy_afinfo[AF_INET6 + 1]
__read_mostly;
@@ -119,7 +122,7 @@ static const struct xfrm_policy_afinfo *xfrm_policy_get_afinfo(unsigned short fa
struct dst_entry *__xfrm_dst_lookup(struct net *net, int tos, int oif,
const xfrm_address_t *saddr,
const xfrm_address_t *daddr,
- int family)
+ int family, u32 mark)
{
const struct xfrm_policy_afinfo *afinfo;
struct dst_entry *dst;
@@ -128,7 +131,7 @@ struct dst_entry *__xfrm_dst_lookup(struct net *net, int tos, int oif,
if (unlikely(afinfo == NULL))
return ERR_PTR(-EAFNOSUPPORT);
- dst = afinfo->dst_lookup(net, tos, oif, saddr, daddr);
+ dst = afinfo->dst_lookup(net, tos, oif, saddr, daddr, mark);
rcu_read_unlock();
@@ -140,7 +143,7 @@ static inline struct dst_entry *xfrm_dst_lookup(struct xfrm_state *x,
int tos, int oif,
xfrm_address_t *prev_saddr,
xfrm_address_t *prev_daddr,
- int family)
+ int family, u32 mark)
{
struct net *net = xs_net(x);
xfrm_address_t *saddr = &x->props.saddr;
@@ -156,7 +159,7 @@ static inline struct dst_entry *xfrm_dst_lookup(struct xfrm_state *x,
daddr = x->coaddr;
}
- dst = __xfrm_dst_lookup(net, tos, oif, saddr, daddr, family);
+ dst = __xfrm_dst_lookup(net, tos, oif, saddr, daddr, family, mark);
if (!IS_ERR(dst)) {
if (prev_saddr != saddr)
@@ -246,36 +249,6 @@ expired:
xfrm_pol_put(xp);
}
-static struct flow_cache_object *xfrm_policy_flo_get(struct flow_cache_object *flo)
-{
- struct xfrm_policy *pol = container_of(flo, struct xfrm_policy, flo);
-
- if (unlikely(pol->walk.dead))
- flo = NULL;
- else
- xfrm_pol_hold(pol);
-
- return flo;
-}
-
-static int xfrm_policy_flo_check(struct flow_cache_object *flo)
-{
- struct xfrm_policy *pol = container_of(flo, struct xfrm_policy, flo);
-
- return !pol->walk.dead;
-}
-
-static void xfrm_policy_flo_delete(struct flow_cache_object *flo)
-{
- xfrm_pol_put(container_of(flo, struct xfrm_policy, flo));
-}
-
-static const struct flow_cache_ops xfrm_policy_fc_ops = {
- .get = xfrm_policy_flo_get,
- .check = xfrm_policy_flo_check,
- .delete = xfrm_policy_flo_delete,
-};
-
/* Allocate xfrm_policy. Not used here, it is supposed to be used by pfkeyv2
* SPD calls.
*/
@@ -298,7 +271,6 @@ struct xfrm_policy *xfrm_policy_alloc(struct net *net, gfp_t gfp)
(unsigned long)policy);
setup_timer(&policy->polq.hold_timer, xfrm_policy_queue_process,
(unsigned long)policy);
- policy->flo.ops = &xfrm_policy_fc_ops;
}
return policy;
}
@@ -798,7 +770,6 @@ int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl)
else
hlist_add_head(&policy->bydst, chain);
__xfrm_policy_link(policy, dir);
- atomic_inc(&net->xfrm.flow_cache_genid);
/* After previous checking, family can either be AF_INET or AF_INET6 */
if (policy->family == AF_INET)
@@ -1004,6 +975,8 @@ int xfrm_policy_flush(struct net *net, u8 type, bool task_valid)
}
if (!cnt)
err = -ESRCH;
+ else
+ xfrm_policy_cache_flush();
out:
spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
return err;
@@ -1175,7 +1148,7 @@ fail:
}
static struct xfrm_policy *
-__xfrm_policy_lookup(struct net *net, const struct flowi *fl, u16 family, u8 dir)
+xfrm_policy_lookup(struct net *net, const struct flowi *fl, u16 family, u8 dir)
{
#ifdef CONFIG_XFRM_SUB_POLICY
struct xfrm_policy *pol;
@@ -1187,61 +1160,6 @@ __xfrm_policy_lookup(struct net *net, const struct flowi *fl, u16 family, u8 dir
return xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_MAIN, fl, family, dir);
}
-static int flow_to_policy_dir(int dir)
-{
- if (XFRM_POLICY_IN == FLOW_DIR_IN &&
- XFRM_POLICY_OUT == FLOW_DIR_OUT &&
- XFRM_POLICY_FWD == FLOW_DIR_FWD)
- return dir;
-
- switch (dir) {
- default:
- case FLOW_DIR_IN:
- return XFRM_POLICY_IN;
- case FLOW_DIR_OUT:
- return XFRM_POLICY_OUT;
- case FLOW_DIR_FWD:
- return XFRM_POLICY_FWD;
- }
-}
-
-static struct flow_cache_object *
-xfrm_policy_lookup(struct net *net, const struct flowi *fl, u16 family,
- u8 dir, struct flow_cache_object *old_obj, void *ctx)
-{
- struct xfrm_policy *pol;
-
- if (old_obj)
- xfrm_pol_put(container_of(old_obj, struct xfrm_policy, flo));
-
- pol = __xfrm_policy_lookup(net, fl, family, flow_to_policy_dir(dir));
- if (IS_ERR_OR_NULL(pol))
- return ERR_CAST(pol);
-
- /* Resolver returns two references:
- * one for cache and one for caller of flow_cache_lookup() */
- xfrm_pol_hold(pol);
-
- return &pol->flo;
-}
-
-static inline int policy_to_flow_dir(int dir)
-{
- if (XFRM_POLICY_IN == FLOW_DIR_IN &&
- XFRM_POLICY_OUT == FLOW_DIR_OUT &&
- XFRM_POLICY_FWD == FLOW_DIR_FWD)
- return dir;
- switch (dir) {
- default:
- case XFRM_POLICY_IN:
- return FLOW_DIR_IN;
- case XFRM_POLICY_OUT:
- return FLOW_DIR_OUT;
- case XFRM_POLICY_FWD:
- return FLOW_DIR_FWD;
- }
-}
-
static struct xfrm_policy *xfrm_sk_policy_lookup(const struct sock *sk, int dir,
const struct flowi *fl, u16 family)
{
@@ -1261,7 +1179,7 @@ static struct xfrm_policy *xfrm_sk_policy_lookup(const struct sock *sk, int dir,
}
err = security_xfrm_policy_lookup(pol->security,
fl->flowi_secid,
- policy_to_flow_dir(dir));
+ dir);
if (!err) {
if (!xfrm_pol_hold_rcu(pol))
goto again;
@@ -1422,14 +1340,14 @@ int __xfrm_sk_clone_policy(struct sock *sk, const struct sock *osk)
static int
xfrm_get_saddr(struct net *net, int oif, xfrm_address_t *local,
- xfrm_address_t *remote, unsigned short family)
+ xfrm_address_t *remote, unsigned short family, u32 mark)
{
int err;
const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
if (unlikely(afinfo == NULL))
return -EINVAL;
- err = afinfo->get_saddr(net, oif, local, remote);
+ err = afinfo->get_saddr(net, oif, local, remote, mark);
rcu_read_unlock();
return err;
}
@@ -1460,7 +1378,7 @@ xfrm_tmpl_resolve_one(struct xfrm_policy *policy, const struct flowi *fl,
if (xfrm_addr_any(local, tmpl->encap_family)) {
error = xfrm_get_saddr(net, fl->flowi_oif,
&tmp, remote,
- tmpl->encap_family);
+ tmpl->encap_family, 0);
if (error)
goto fail;
local = &tmp;
@@ -1545,58 +1463,6 @@ static int xfrm_get_tos(const struct flowi *fl, int family)
return tos;
}
-static struct flow_cache_object *xfrm_bundle_flo_get(struct flow_cache_object *flo)
-{
- struct xfrm_dst *xdst = container_of(flo, struct xfrm_dst, flo);
- struct dst_entry *dst = &xdst->u.dst;
-
- if (xdst->route == NULL) {
- /* Dummy bundle - if it has xfrms we were not
- * able to build bundle as template resolution failed.
- * It means we need to try again resolving. */
- if (xdst->num_xfrms > 0)
- return NULL;
- } else if (dst->flags & DST_XFRM_QUEUE) {
- return NULL;
- } else {
- /* Real bundle */
- if (stale_bundle(dst))
- return NULL;
- }
-
- dst_hold(dst);
- return flo;
-}
-
-static int xfrm_bundle_flo_check(struct flow_cache_object *flo)
-{
- struct xfrm_dst *xdst = container_of(flo, struct xfrm_dst, flo);
- struct dst_entry *dst = &xdst->u.dst;
-
- if (!xdst->route)
- return 0;
- if (stale_bundle(dst))
- return 0;
-
- return 1;
-}
-
-static void xfrm_bundle_flo_delete(struct flow_cache_object *flo)
-{
- struct xfrm_dst *xdst = container_of(flo, struct xfrm_dst, flo);
- struct dst_entry *dst = &xdst->u.dst;
-
- /* Mark DST_OBSOLETE_DEAD to fail the next xfrm_dst_check() */
- dst->obsolete = DST_OBSOLETE_DEAD;
- dst_release_immediate(dst);
-}
-
-static const struct flow_cache_ops xfrm_bundle_fc_ops = {
- .get = xfrm_bundle_flo_get,
- .check = xfrm_bundle_flo_check,
- .delete = xfrm_bundle_flo_delete,
-};
-
static inline struct xfrm_dst *xfrm_alloc_dst(struct net *net, int family)
{
const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
@@ -1624,7 +1490,6 @@ static inline struct xfrm_dst *xfrm_alloc_dst(struct net *net, int family)
struct dst_entry *dst = &xdst->u.dst;
memset(dst + 1, 0, sizeof(*xdst) - sizeof(*dst));
- xdst->flo.ops = &xfrm_bundle_fc_ops;
} else
xdst = ERR_PTR(-ENOBUFS);
@@ -1733,7 +1598,8 @@ static struct dst_entry *xfrm_bundle_create(struct xfrm_policy *policy,
if (xfrm[i]->props.mode != XFRM_MODE_TRANSPORT) {
family = xfrm[i]->props.family;
dst = xfrm_dst_lookup(xfrm[i], tos, fl->flowi_oif,
- &saddr, &daddr, family);
+ &saddr, &daddr, family,
+ xfrm[i]->props.output_mark);
err = PTR_ERR(dst);
if (IS_ERR(dst))
goto put_states;
@@ -1840,6 +1706,102 @@ static int xfrm_expand_policies(const struct flowi *fl, u16 family,
}
+static void xfrm_last_dst_update(struct xfrm_dst *xdst, struct xfrm_dst *old)
+{
+ this_cpu_write(xfrm_last_dst, xdst);
+ if (old)
+ dst_release(&old->u.dst);
+}
+
+static void __xfrm_pcpu_work_fn(void)
+{
+ struct xfrm_dst *old;
+
+ old = this_cpu_read(xfrm_last_dst);
+ if (old && !xfrm_bundle_ok(old))
+ xfrm_last_dst_update(NULL, old);
+}
+
+static void xfrm_pcpu_work_fn(struct work_struct *work)
+{
+ local_bh_disable();
+ rcu_read_lock();
+ __xfrm_pcpu_work_fn();
+ rcu_read_unlock();
+ local_bh_enable();
+}
+
+void xfrm_policy_cache_flush(void)
+{
+ struct xfrm_dst *old;
+ bool found = 0;
+ int cpu;
+
+ local_bh_disable();
+ rcu_read_lock();
+ for_each_possible_cpu(cpu) {
+ old = per_cpu(xfrm_last_dst, cpu);
+ if (old && !xfrm_bundle_ok(old)) {
+ if (smp_processor_id() == cpu) {
+ __xfrm_pcpu_work_fn();
+ continue;
+ }
+ found = true;
+ break;
+ }
+ }
+
+ rcu_read_unlock();
+ local_bh_enable();
+
+ if (!found)
+ return;
+
+ get_online_cpus();
+
+ for_each_possible_cpu(cpu) {
+ bool bundle_release;
+
+ rcu_read_lock();
+ old = per_cpu(xfrm_last_dst, cpu);
+ bundle_release = old && !xfrm_bundle_ok(old);
+ rcu_read_unlock();
+
+ if (!bundle_release)
+ continue;
+
+ if (cpu_online(cpu)) {
+ schedule_work_on(cpu, &xfrm_pcpu_work[cpu]);
+ continue;
+ }
+
+ rcu_read_lock();
+ old = per_cpu(xfrm_last_dst, cpu);
+ if (old && !xfrm_bundle_ok(old)) {
+ per_cpu(xfrm_last_dst, cpu) = NULL;
+ dst_release(&old->u.dst);
+ }
+ rcu_read_unlock();
+ }
+
+ put_online_cpus();
+}
+
+static bool xfrm_pol_dead(struct xfrm_dst *xdst)
+{
+ unsigned int num_pols = xdst->num_pols;
+ unsigned int pol_dead = 0, i;
+
+ for (i = 0; i < num_pols; i++)
+ pol_dead |= xdst->pols[i]->walk.dead;
+
+ /* Mark DST_OBSOLETE_DEAD to fail the next xfrm_dst_check() */
+ if (pol_dead)
+ xdst->u.dst.obsolete = DST_OBSOLETE_DEAD;
+
+ return pol_dead;
+}
+
static struct xfrm_dst *
xfrm_resolve_and_create_bundle(struct xfrm_policy **pols, int num_pols,
const struct flowi *fl, u16 family,
@@ -1847,10 +1809,23 @@ xfrm_resolve_and_create_bundle(struct xfrm_policy **pols, int num_pols,
{
struct net *net = xp_net(pols[0]);
struct xfrm_state *xfrm[XFRM_MAX_DEPTH];
+ struct xfrm_dst *xdst, *old;
struct dst_entry *dst;
- struct xfrm_dst *xdst;
int err;
+ xdst = this_cpu_read(xfrm_last_dst);
+ if (xdst &&
+ xdst->u.dst.dev == dst_orig->dev &&
+ xdst->num_pols == num_pols &&
+ !xfrm_pol_dead(xdst) &&
+ memcmp(xdst->pols, pols,
+ sizeof(struct xfrm_policy *) * num_pols) == 0 &&
+ xfrm_bundle_ok(xdst)) {
+ dst_hold(&xdst->u.dst);
+ return xdst;
+ }
+
+ old = xdst;
/* Try to instantiate a bundle */
err = xfrm_tmpl_resolve(pols, num_pols, fl, xfrm, family);
if (err <= 0) {
@@ -1871,6 +1846,9 @@ xfrm_resolve_and_create_bundle(struct xfrm_policy **pols, int num_pols,
memcpy(xdst->pols, pols, sizeof(struct xfrm_policy *) * num_pols);
xdst->policy_genid = atomic_read(&pols[0]->genid);
+ atomic_set(&xdst->u.dst.__refcnt, 2);
+ xfrm_last_dst_update(xdst, old);
+
return xdst;
}
@@ -2051,86 +2029,39 @@ free_dst:
goto out;
}
-static struct flow_cache_object *
-xfrm_bundle_lookup(struct net *net, const struct flowi *fl, u16 family, u8 dir,
- struct flow_cache_object *oldflo, void *ctx)
+static struct xfrm_dst *
+xfrm_bundle_lookup(struct net *net, const struct flowi *fl, u16 family, u8 dir, struct xfrm_flo *xflo)
{
- struct xfrm_flo *xflo = (struct xfrm_flo *)ctx;
struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
- struct xfrm_dst *xdst, *new_xdst;
- int num_pols = 0, num_xfrms = 0, i, err, pol_dead;
-
- /* Check if the policies from old bundle are usable */
- xdst = NULL;
- if (oldflo) {
- xdst = container_of(oldflo, struct xfrm_dst, flo);
- num_pols = xdst->num_pols;
- num_xfrms = xdst->num_xfrms;
- pol_dead = 0;
- for (i = 0; i < num_pols; i++) {
- pols[i] = xdst->pols[i];
- pol_dead |= pols[i]->walk.dead;
- }
- if (pol_dead) {
- /* Mark DST_OBSOLETE_DEAD to fail the next
- * xfrm_dst_check()
- */
- xdst->u.dst.obsolete = DST_OBSOLETE_DEAD;
- dst_release_immediate(&xdst->u.dst);
- xdst = NULL;
- num_pols = 0;
- num_xfrms = 0;
- oldflo = NULL;
- }
- }
+ int num_pols = 0, num_xfrms = 0, err;
+ struct xfrm_dst *xdst;
/* Resolve policies to use if we couldn't get them from
* previous cache entry */
- if (xdst == NULL) {
- num_pols = 1;
- pols[0] = __xfrm_policy_lookup(net, fl, family,
- flow_to_policy_dir(dir));
- err = xfrm_expand_policies(fl, family, pols,
+ num_pols = 1;
+ pols[0] = xfrm_policy_lookup(net, fl, family, dir);
+ err = xfrm_expand_policies(fl, family, pols,
&num_pols, &num_xfrms);
- if (err < 0)
- goto inc_error;
- if (num_pols == 0)
- return NULL;
- if (num_xfrms <= 0)
- goto make_dummy_bundle;
- }
+ if (err < 0)
+ goto inc_error;
+ if (num_pols == 0)
+ return NULL;
+ if (num_xfrms <= 0)
+ goto make_dummy_bundle;
- new_xdst = xfrm_resolve_and_create_bundle(pols, num_pols, fl, family,
+ xdst = xfrm_resolve_and_create_bundle(pols, num_pols, fl, family,
xflo->dst_orig);
- if (IS_ERR(new_xdst)) {
- err = PTR_ERR(new_xdst);
+ if (IS_ERR(xdst)) {
+ err = PTR_ERR(xdst);
if (err != -EAGAIN)
goto error;
- if (oldflo == NULL)
- goto make_dummy_bundle;
- dst_hold(&xdst->u.dst);
- return oldflo;
- } else if (new_xdst == NULL) {
+ goto make_dummy_bundle;
+ } else if (xdst == NULL) {
num_xfrms = 0;
- if (oldflo == NULL)
- goto make_dummy_bundle;
- xdst->num_xfrms = 0;
- dst_hold(&xdst->u.dst);
- return oldflo;
- }
-
- /* Kill the previous bundle */
- if (xdst) {
- /* The policies were stolen for newly generated bundle */
- xdst->num_pols = 0;
- /* Mark DST_OBSOLETE_DEAD to fail the next xfrm_dst_check() */
- xdst->u.dst.obsolete = DST_OBSOLETE_DEAD;
- dst_release_immediate(&xdst->u.dst);
+ goto make_dummy_bundle;
}
- /* We do need to return one reference for original caller */
- dst_hold(&new_xdst->u.dst);
- return &new_xdst->flo;
+ return xdst;
make_dummy_bundle:
/* We found policies, but there's no bundles to instantiate:
@@ -2146,17 +2077,12 @@ make_dummy_bundle:
memcpy(xdst->pols, pols, sizeof(struct xfrm_policy *) * num_pols);
dst_hold(&xdst->u.dst);
- return &xdst->flo;
+ return xdst;
inc_error:
XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLERROR);
error:
- if (xdst != NULL) {
- /* Mark DST_OBSOLETE_DEAD to fail the next xfrm_dst_check() */
- xdst->u.dst.obsolete = DST_OBSOLETE_DEAD;
- dst_release_immediate(&xdst->u.dst);
- } else
- xfrm_pols_put(pols, num_pols);
+ xfrm_pols_put(pols, num_pols);
return ERR_PTR(err);
}
@@ -2187,11 +2113,10 @@ struct dst_entry *xfrm_lookup(struct net *net, struct dst_entry *dst_orig,
const struct sock *sk, int flags)
{
struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
- struct flow_cache_object *flo;
struct xfrm_dst *xdst;
struct dst_entry *dst, *route;
u16 family = dst_orig->ops->family;
- u8 dir = policy_to_flow_dir(XFRM_POLICY_OUT);
+ u8 dir = XFRM_POLICY_OUT;
int i, err, num_pols, num_xfrms = 0, drop_pols = 0;
dst = NULL;
@@ -2241,15 +2166,13 @@ struct dst_entry *xfrm_lookup(struct net *net, struct dst_entry *dst_orig,
!net->xfrm.policy_count[XFRM_POLICY_OUT])
goto nopol;
- flo = flow_cache_lookup(net, fl, family, dir,
- xfrm_bundle_lookup, &xflo);
- if (flo == NULL)
+ xdst = xfrm_bundle_lookup(net, fl, family, dir, &xflo);
+ if (xdst == NULL)
goto nopol;
- if (IS_ERR(flo)) {
- err = PTR_ERR(flo);
+ if (IS_ERR(xdst)) {
+ err = PTR_ERR(xdst);
goto dropdst;
}
- xdst = container_of(flo, struct xfrm_dst, flo);
num_pols = xdst->num_pols;
num_xfrms = xdst->num_xfrms;
@@ -2448,12 +2371,10 @@ int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb,
int pi;
int reverse;
struct flowi fl;
- u8 fl_dir;
int xerr_idx = -1;
reverse = dir & ~XFRM_POLICY_MASK;
dir &= XFRM_POLICY_MASK;
- fl_dir = policy_to_flow_dir(dir);
if (__xfrm_decode_session(skb, &fl, family, reverse) < 0) {
XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
@@ -2485,16 +2406,8 @@ int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb,
}
}
- if (!pol) {
- struct flow_cache_object *flo;
-
- flo = flow_cache_lookup(net, &fl, family, fl_dir,
- xfrm_policy_lookup, NULL);
- if (IS_ERR_OR_NULL(flo))
- pol = ERR_CAST(flo);
- else
- pol = container_of(flo, struct xfrm_policy, flo);
- }
+ if (!pol)
+ pol = xfrm_policy_lookup(net, &fl, family, dir);
if (IS_ERR(pol)) {
XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
@@ -2640,11 +2553,9 @@ static struct dst_entry *xfrm_dst_check(struct dst_entry *dst, u32 cookie)
* notice. That's what we are validating here via the
* stale_bundle() check.
*
- * When an xdst is removed from flow cache, DST_OBSOLETE_DEAD will
- * be marked on it.
* When a dst is removed from the fib tree, DST_OBSOLETE_DEAD will
* be marked on it.
- * Both will force stable_bundle() to fail on any xdst bundle with
+ * This will force stale_bundle() to fail on any xdst bundle with
* this dst linked in it.
*/
if (dst->obsolete < 0 && !stale_bundle(dst))
@@ -2684,18 +2595,6 @@ static struct dst_entry *xfrm_negative_advice(struct dst_entry *dst)
return dst;
}
-void xfrm_garbage_collect(struct net *net)
-{
- flow_cache_flush(net);
-}
-EXPORT_SYMBOL(xfrm_garbage_collect);
-
-void xfrm_garbage_collect_deferred(struct net *net)
-{
- flow_cache_flush_deferred(net);
-}
-EXPORT_SYMBOL(xfrm_garbage_collect_deferred);
-
static void xfrm_init_pmtu(struct dst_entry *dst)
{
do {
@@ -3033,14 +2932,9 @@ static int __net_init xfrm_net_init(struct net *net)
rv = xfrm_sysctl_init(net);
if (rv < 0)
goto out_sysctl;
- rv = flow_cache_init(net);
- if (rv < 0)
- goto out;
return 0;
-out:
- xfrm_sysctl_fini(net);
out_sysctl:
xfrm_policy_fini(net);
out_policy:
@@ -3053,7 +2947,6 @@ out_statistics:
static void __net_exit xfrm_net_exit(struct net *net)
{
- flow_cache_fini(net);
xfrm_sysctl_fini(net);
xfrm_policy_fini(net);
xfrm_state_fini(net);
@@ -3067,7 +2960,15 @@ static struct pernet_operations __net_initdata xfrm_net_ops = {
void __init xfrm_init(void)
{
- flow_cache_hp_init();
+ int i;
+
+ xfrm_pcpu_work = kmalloc_array(NR_CPUS, sizeof(*xfrm_pcpu_work),
+ GFP_KERNEL);
+ BUG_ON(!xfrm_pcpu_work);
+
+ for (i = 0; i < NR_CPUS; i++)
+ INIT_WORK(&xfrm_pcpu_work[i], xfrm_pcpu_work_fn);
+
register_pernet_subsys(&xfrm_net_ops);
seqcount_init(&xfrm_policy_hash_generation);
xfrm_input_init();
diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c
index a792effdb0b5..0dab1cd79ce4 100644
--- a/net/xfrm/xfrm_state.c
+++ b/net/xfrm/xfrm_state.c
@@ -296,12 +296,14 @@ int xfrm_unregister_type_offload(const struct xfrm_type_offload *type,
}
EXPORT_SYMBOL(xfrm_unregister_type_offload);
-static const struct xfrm_type_offload *xfrm_get_type_offload(u8 proto, unsigned short family)
+static const struct xfrm_type_offload *
+xfrm_get_type_offload(u8 proto, unsigned short family, bool try_load)
{
struct xfrm_state_afinfo *afinfo;
const struct xfrm_type_offload **typemap;
const struct xfrm_type_offload *type;
+retry:
afinfo = xfrm_state_get_afinfo(family);
if (unlikely(afinfo == NULL))
return NULL;
@@ -311,6 +313,12 @@ static const struct xfrm_type_offload *xfrm_get_type_offload(u8 proto, unsigned
if ((type && !try_module_get(type->owner)))
type = NULL;
+ if (!type && try_load) {
+ request_module("xfrm-offload-%d-%d", family, proto);
+ try_load = 0;
+ goto retry;
+ }
+
rcu_read_unlock();
return type;
}
@@ -724,9 +732,10 @@ restart:
}
}
}
- if (cnt)
+ if (cnt) {
err = 0;
-
+ xfrm_policy_cache_flush();
+ }
out:
spin_unlock_bh(&net->xfrm.xfrm_state_lock);
return err;
@@ -2172,7 +2181,7 @@ int xfrm_state_mtu(struct xfrm_state *x, int mtu)
return mtu - x->props.header_len;
}
-int __xfrm_init_state(struct xfrm_state *x, bool init_replay)
+int __xfrm_init_state(struct xfrm_state *x, bool init_replay, bool offload)
{
struct xfrm_state_afinfo *afinfo;
struct xfrm_mode *inner_mode;
@@ -2237,7 +2246,7 @@ int __xfrm_init_state(struct xfrm_state *x, bool init_replay)
if (x->type == NULL)
goto error;
- x->type_offload = xfrm_get_type_offload(x->id.proto, family);
+ x->type_offload = xfrm_get_type_offload(x->id.proto, family, offload);
err = x->type->init_state(x);
if (err)
@@ -2265,7 +2274,7 @@ EXPORT_SYMBOL(__xfrm_init_state);
int xfrm_init_state(struct xfrm_state *x)
{
- return __xfrm_init_state(x, true);
+ return __xfrm_init_state(x, true, false);
}
EXPORT_SYMBOL(xfrm_init_state);
diff --git a/net/xfrm/xfrm_user.c b/net/xfrm/xfrm_user.c
index 9391ced05259..2bfbd9121e3b 100644
--- a/net/xfrm/xfrm_user.c
+++ b/net/xfrm/xfrm_user.c
@@ -584,7 +584,10 @@ static struct xfrm_state *xfrm_state_construct(struct net *net,
xfrm_mark_get(attrs, &x->mark);
- err = __xfrm_init_state(x, false);
+ if (attrs[XFRMA_OUTPUT_MARK])
+ x->props.output_mark = nla_get_u32(attrs[XFRMA_OUTPUT_MARK]);
+
+ err = __xfrm_init_state(x, false, attrs[XFRMA_OFFLOAD_DEV]);
if (err)
goto error;
@@ -897,6 +900,11 @@ static int copy_to_user_state_extra(struct xfrm_state *x,
ret = copy_user_offload(&x->xso, skb);
if (ret)
goto out;
+ if (x->props.output_mark) {
+ ret = nla_put_u32(skb, XFRMA_OUTPUT_MARK, x->props.output_mark);
+ if (ret)
+ goto out;
+ }
if (x->security)
ret = copy_sec_ctx(x->security, skb);
out:
@@ -1815,8 +1823,6 @@ static int xfrm_get_policy(struct sk_buff *skb, struct nlmsghdr *nlh,
out:
xfrm_pol_put(xp);
- if (delete && err == 0)
- xfrm_garbage_collect(net);
return err;
}
@@ -2028,7 +2034,6 @@ static int xfrm_flush_policy(struct sk_buff *skb, struct nlmsghdr *nlh,
return 0;
return err;
}
- xfrm_garbage_collect(net);
c.data.type = type;
c.event = nlh->nlmsg_type;
@@ -2458,6 +2463,7 @@ static const struct nla_policy xfrma_policy[XFRMA_MAX+1] = {
[XFRMA_PROTO] = { .type = NLA_U8 },
[XFRMA_ADDRESS_FILTER] = { .len = sizeof(struct xfrm_address_filter) },
[XFRMA_OFFLOAD_DEV] = { .len = sizeof(struct xfrm_user_offload) },
+ [XFRMA_OUTPUT_MARK] = { .len = NLA_U32 },
};
static const struct nla_policy xfrma_spd_policy[XFRMA_SPD_MAX+1] = {
@@ -2679,6 +2685,8 @@ static inline size_t xfrm_sa_len(struct xfrm_state *x)
l += nla_total_size(sizeof(x->props.extra_flags));
if (x->xso.dev)
l += nla_total_size(sizeof(x->xso));
+ if (x->props.output_mark)
+ l += nla_total_size(sizeof(x->props.output_mark));
/* Must count x->lastused as it may become non-zero behind our back. */
l += nla_total_size_64bit(sizeof(u64));
generated by cgit v1.2.3 (git 2.39.0) at 2024-10-02 19:27:56 +0300