ipsec: Put dumpers on the dump list
Herbert Xu came up with the idea and the original patch to make
xfrm_state dump list contain also dumpers:
As it is we go to extraordinary lengths to ensure that states
don't go away while dumpers go to sleep. It's much easier if
we just put the dumpers themselves on the list since they can't
go away while they're going.
I've also changed the order of addition on new states to prevent
a never-ending dump.
Timo Teräs improved the patch to apply cleanly to latest tree,
modified iteration code to be more readable by using a common
struct for entries in the list, implemented the same idea for
xfrm_policy dumping and moved the af_key specific "last" entry
caching to af_key.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
Signed-off-by: Timo Teras <timo.teras@iki.fi>
Signed-off-by: David S. Miller <davem@davemloft.net>
diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c
index ef9ccbc..b7ec080 100644
--- a/net/xfrm/xfrm_policy.c
+++ b/net/xfrm/xfrm_policy.c
@@ -46,7 +46,7 @@
static DEFINE_RWLOCK(xfrm_policy_lock);
-static struct list_head xfrm_policy_bytype[XFRM_POLICY_TYPE_MAX];
+static struct list_head xfrm_policy_all;
unsigned int xfrm_policy_count[XFRM_POLICY_MAX*2];
EXPORT_SYMBOL(xfrm_policy_count);
@@ -164,7 +164,7 @@
read_lock(&xp->lock);
- if (xp->dead)
+ if (xp->walk.dead)
goto out;
dir = xfrm_policy_id2dir(xp->index);
@@ -236,7 +236,7 @@
policy = kzalloc(sizeof(struct xfrm_policy), gfp);
if (policy) {
- INIT_LIST_HEAD(&policy->bytype);
+ INIT_LIST_HEAD(&policy->walk.all);
INIT_HLIST_NODE(&policy->bydst);
INIT_HLIST_NODE(&policy->byidx);
rwlock_init(&policy->lock);
@@ -252,17 +252,13 @@
void xfrm_policy_destroy(struct xfrm_policy *policy)
{
- BUG_ON(!policy->dead);
+ BUG_ON(!policy->walk.dead);
BUG_ON(policy->bundles);
if (del_timer(&policy->timer))
BUG();
- write_lock_bh(&xfrm_policy_lock);
- list_del(&policy->bytype);
- write_unlock_bh(&xfrm_policy_lock);
-
security_xfrm_policy_free(policy->security);
kfree(policy);
}
@@ -310,8 +306,8 @@
int dead;
write_lock_bh(&policy->lock);
- dead = policy->dead;
- policy->dead = 1;
+ dead = policy->walk.dead;
+ policy->walk.dead = 1;
write_unlock_bh(&policy->lock);
if (unlikely(dead)) {
@@ -609,6 +605,7 @@
if (delpol) {
hlist_del(&delpol->bydst);
hlist_del(&delpol->byidx);
+ list_del(&delpol->walk.all);
xfrm_policy_count[dir]--;
}
policy->index = delpol ? delpol->index : xfrm_gen_index(policy->type, dir);
@@ -617,7 +614,7 @@
policy->curlft.use_time = 0;
if (!mod_timer(&policy->timer, jiffies + HZ))
xfrm_pol_hold(policy);
- list_add_tail(&policy->bytype, &xfrm_policy_bytype[policy->type]);
+ list_add(&policy->walk.all, &xfrm_policy_all);
write_unlock_bh(&xfrm_policy_lock);
if (delpol)
@@ -684,6 +681,7 @@
}
hlist_del(&pol->bydst);
hlist_del(&pol->byidx);
+ list_del(&pol->walk.all);
xfrm_policy_count[dir]--;
}
ret = pol;
@@ -727,6 +725,7 @@
}
hlist_del(&pol->bydst);
hlist_del(&pol->byidx);
+ list_del(&pol->walk.all);
xfrm_policy_count[dir]--;
}
ret = pol;
@@ -840,6 +839,7 @@
continue;
hlist_del(&pol->bydst);
hlist_del(&pol->byidx);
+ list_del(&pol->walk.all);
write_unlock_bh(&xfrm_policy_lock);
xfrm_audit_policy_delete(pol, 1,
@@ -867,60 +867,68 @@
int (*func)(struct xfrm_policy *, int, int, void*),
void *data)
{
- struct xfrm_policy *old, *pol, *last = NULL;
+ struct xfrm_policy *pol;
+ struct xfrm_policy_walk_entry *x;
int error = 0;
if (walk->type >= XFRM_POLICY_TYPE_MAX &&
walk->type != XFRM_POLICY_TYPE_ANY)
return -EINVAL;
- if (walk->policy == NULL && walk->count != 0)
+ if (list_empty(&walk->walk.all) && walk->seq != 0)
return 0;
- old = pol = walk->policy;
- walk->policy = NULL;
- read_lock_bh(&xfrm_policy_lock);
-
- for (; walk->cur_type < XFRM_POLICY_TYPE_MAX; walk->cur_type++) {
- if (walk->type != walk->cur_type &&
- walk->type != XFRM_POLICY_TYPE_ANY)
+ write_lock_bh(&xfrm_policy_lock);
+ if (list_empty(&walk->walk.all))
+ x = list_first_entry(&xfrm_policy_all, struct xfrm_policy_walk_entry, all);
+ else
+ x = list_entry(&walk->walk.all, struct xfrm_policy_walk_entry, all);
+ list_for_each_entry_from(x, &xfrm_policy_all, all) {
+ if (x->dead)
continue;
-
- if (pol == NULL) {
- pol = list_first_entry(&xfrm_policy_bytype[walk->cur_type],
- struct xfrm_policy, bytype);
+ pol = container_of(x, struct xfrm_policy, walk);
+ if (walk->type != XFRM_POLICY_TYPE_ANY &&
+ walk->type != pol->type)
+ continue;
+ error = func(pol, xfrm_policy_id2dir(pol->index),
+ walk->seq, data);
+ if (error) {
+ list_move_tail(&walk->walk.all, &x->all);
+ goto out;
}
- list_for_each_entry_from(pol, &xfrm_policy_bytype[walk->cur_type], bytype) {
- if (pol->dead)
- continue;
- if (last) {
- error = func(last, xfrm_policy_id2dir(last->index),
- walk->count, data);
- if (error) {
- xfrm_pol_hold(last);
- walk->policy = last;
- goto out;
- }
- }
- last = pol;
- walk->count++;
- }
- pol = NULL;
+ walk->seq++;
}
- if (walk->count == 0) {
+ if (walk->seq == 0) {
error = -ENOENT;
goto out;
}
- if (last)
- error = func(last, xfrm_policy_id2dir(last->index), 0, data);
+ list_del_init(&walk->walk.all);
out:
- read_unlock_bh(&xfrm_policy_lock);
- if (old != NULL)
- xfrm_pol_put(old);
+ write_unlock_bh(&xfrm_policy_lock);
return error;
}
EXPORT_SYMBOL(xfrm_policy_walk);
+void xfrm_policy_walk_init(struct xfrm_policy_walk *walk, u8 type)
+{
+ INIT_LIST_HEAD(&walk->walk.all);
+ walk->walk.dead = 1;
+ walk->type = type;
+ walk->seq = 0;
+}
+EXPORT_SYMBOL(xfrm_policy_walk_init);
+
+void xfrm_policy_walk_done(struct xfrm_policy_walk *walk)
+{
+ if (list_empty(&walk->walk.all))
+ return;
+
+ write_lock_bh(&xfrm_policy_lock);
+ list_del(&walk->walk.all);
+ write_unlock_bh(&xfrm_policy_lock);
+}
+EXPORT_SYMBOL(xfrm_policy_walk_done);
+
/*
* Find policy to apply to this flow.
*
@@ -1077,7 +1085,7 @@
struct hlist_head *chain = policy_hash_bysel(&pol->selector,
pol->family, dir);
- list_add_tail(&pol->bytype, &xfrm_policy_bytype[pol->type]);
+ list_add(&pol->walk.all, &xfrm_policy_all);
hlist_add_head(&pol->bydst, chain);
hlist_add_head(&pol->byidx, xfrm_policy_byidx+idx_hash(pol->index));
xfrm_policy_count[dir]++;
@@ -1095,6 +1103,7 @@
hlist_del(&pol->bydst);
hlist_del(&pol->byidx);
+ list_del(&pol->walk.all);
xfrm_policy_count[dir]--;
return pol;
@@ -1720,7 +1729,7 @@
for (pi = 0; pi < npols; pi++) {
read_lock_bh(&pols[pi]->lock);
- pol_dead |= pols[pi]->dead;
+ pol_dead |= pols[pi]->walk.dead;
read_unlock_bh(&pols[pi]->lock);
}
@@ -2415,9 +2424,7 @@
panic("XFRM: failed to allocate bydst hash\n");
}
- for (dir = 0; dir < XFRM_POLICY_TYPE_MAX; dir++)
- INIT_LIST_HEAD(&xfrm_policy_bytype[dir]);
-
+ INIT_LIST_HEAD(&xfrm_policy_all);
INIT_WORK(&xfrm_policy_gc_work, xfrm_policy_gc_task);
register_netdevice_notifier(&xfrm_dev_notifier);
}
@@ -2601,7 +2608,7 @@
int i, j, n = 0;
write_lock_bh(&pol->lock);
- if (unlikely(pol->dead)) {
+ if (unlikely(pol->walk.dead)) {
/* target policy has been deleted */
write_unlock_bh(&pol->lock);
return -ENOENT;
diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c
index 053970e..747fd8c 100644
--- a/net/xfrm/xfrm_state.c
+++ b/net/xfrm/xfrm_state.c
@@ -59,14 +59,6 @@
static unsigned int xfrm_state_num;
static unsigned int xfrm_state_genid;
-/* Counter indicating ongoing walk, protected by xfrm_state_lock. */
-static unsigned long xfrm_state_walk_ongoing;
-/* Counter indicating walk completion, protected by xfrm_cfg_mutex. */
-static unsigned long xfrm_state_walk_completed;
-
-/* List of outstanding state walks used to set the completed counter. */
-static LIST_HEAD(xfrm_state_walks);
-
static struct xfrm_state_afinfo *xfrm_state_get_afinfo(unsigned int family);
static void xfrm_state_put_afinfo(struct xfrm_state_afinfo *afinfo);
@@ -199,8 +191,7 @@
static struct xfrm_state_afinfo *xfrm_state_afinfo[NPROTO];
static struct work_struct xfrm_state_gc_work;
-static LIST_HEAD(xfrm_state_gc_leftovers);
-static LIST_HEAD(xfrm_state_gc_list);
+static HLIST_HEAD(xfrm_state_gc_list);
static DEFINE_SPINLOCK(xfrm_state_gc_lock);
int __xfrm_state_delete(struct xfrm_state *x);
@@ -412,23 +403,16 @@
static void xfrm_state_gc_task(struct work_struct *data)
{
- struct xfrm_state *x, *tmp;
- unsigned long completed;
+ struct xfrm_state *x;
+ struct hlist_node *entry, *tmp;
+ struct hlist_head gc_list;
- mutex_lock(&xfrm_cfg_mutex);
spin_lock_bh(&xfrm_state_gc_lock);
- list_splice_tail_init(&xfrm_state_gc_list, &xfrm_state_gc_leftovers);
+ hlist_move_list(&xfrm_state_gc_list, &gc_list);
spin_unlock_bh(&xfrm_state_gc_lock);
- completed = xfrm_state_walk_completed;
- mutex_unlock(&xfrm_cfg_mutex);
-
- list_for_each_entry_safe(x, tmp, &xfrm_state_gc_leftovers, gclist) {
- if ((long)(x->lastused - completed) > 0)
- break;
- list_del(&x->gclist);
+ hlist_for_each_entry_safe(x, entry, tmp, &gc_list, gclist)
xfrm_state_gc_destroy(x);
- }
wake_up(&km_waitq);
}
@@ -529,7 +513,7 @@
if (x) {
atomic_set(&x->refcnt, 1);
atomic_set(&x->tunnel_users, 0);
- INIT_LIST_HEAD(&x->all);
+ INIT_LIST_HEAD(&x->km.all);
INIT_HLIST_NODE(&x->bydst);
INIT_HLIST_NODE(&x->bysrc);
INIT_HLIST_NODE(&x->byspi);
@@ -556,7 +540,7 @@
WARN_ON(x->km.state != XFRM_STATE_DEAD);
spin_lock_bh(&xfrm_state_gc_lock);
- list_add_tail(&x->gclist, &xfrm_state_gc_list);
+ hlist_add_head(&x->gclist, &xfrm_state_gc_list);
spin_unlock_bh(&xfrm_state_gc_lock);
schedule_work(&xfrm_state_gc_work);
}
@@ -569,8 +553,7 @@
if (x->km.state != XFRM_STATE_DEAD) {
x->km.state = XFRM_STATE_DEAD;
spin_lock(&xfrm_state_lock);
- x->lastused = xfrm_state_walk_ongoing;
- list_del_rcu(&x->all);
+ list_del(&x->km.all);
hlist_del(&x->bydst);
hlist_del(&x->bysrc);
if (x->id.spi)
@@ -871,7 +854,7 @@
if (km_query(x, tmpl, pol) == 0) {
x->km.state = XFRM_STATE_ACQ;
- list_add_tail(&x->all, &xfrm_state_all);
+ list_add(&x->km.all, &xfrm_state_all);
hlist_add_head(&x->bydst, xfrm_state_bydst+h);
h = xfrm_src_hash(daddr, saddr, family);
hlist_add_head(&x->bysrc, xfrm_state_bysrc+h);
@@ -940,7 +923,7 @@
x->genid = ++xfrm_state_genid;
- list_add_tail(&x->all, &xfrm_state_all);
+ list_add(&x->km.all, &xfrm_state_all);
h = xfrm_dst_hash(&x->id.daddr, &x->props.saddr,
x->props.reqid, x->props.family);
@@ -1069,7 +1052,7 @@
xfrm_state_hold(x);
x->timer.expires = jiffies + sysctl_xfrm_acq_expires*HZ;
add_timer(&x->timer);
- list_add_tail(&x->all, &xfrm_state_all);
+ list_add(&x->km.all, &xfrm_state_all);
hlist_add_head(&x->bydst, xfrm_state_bydst+h);
h = xfrm_src_hash(daddr, saddr, family);
hlist_add_head(&x->bysrc, xfrm_state_bysrc+h);
@@ -1566,79 +1549,59 @@
int (*func)(struct xfrm_state *, int, void*),
void *data)
{
- struct xfrm_state *old, *x, *last = NULL;
+ struct xfrm_state *state;
+ struct xfrm_state_walk *x;
int err = 0;
- if (walk->state == NULL && walk->count != 0)
+ if (walk->seq != 0 && list_empty(&walk->all))
return 0;
- old = x = walk->state;
- walk->state = NULL;
spin_lock_bh(&xfrm_state_lock);
- if (x == NULL)
- x = list_first_entry(&xfrm_state_all, struct xfrm_state, all);
+ if (list_empty(&walk->all))
+ x = list_first_entry(&xfrm_state_all, struct xfrm_state_walk, all);
+ else
+ x = list_entry(&walk->all, struct xfrm_state_walk, all);
list_for_each_entry_from(x, &xfrm_state_all, all) {
- if (x->km.state == XFRM_STATE_DEAD)
+ if (x->state == XFRM_STATE_DEAD)
continue;
- if (!xfrm_id_proto_match(x->id.proto, walk->proto))
+ state = container_of(x, struct xfrm_state, km);
+ if (!xfrm_id_proto_match(state->id.proto, walk->proto))
continue;
- if (last) {
- err = func(last, walk->count, data);
- if (err) {
- xfrm_state_hold(last);
- walk->state = last;
- goto out;
- }
+ err = func(state, walk->seq, data);
+ if (err) {
+ list_move_tail(&walk->all, &x->all);
+ goto out;
}
- last = x;
- walk->count++;
+ walk->seq++;
}
- if (walk->count == 0) {
+ if (walk->seq == 0) {
err = -ENOENT;
goto out;
}
- if (last)
- err = func(last, 0, data);
+ list_del_init(&walk->all);
out:
spin_unlock_bh(&xfrm_state_lock);
- if (old != NULL)
- xfrm_state_put(old);
return err;
}
EXPORT_SYMBOL(xfrm_state_walk);
void xfrm_state_walk_init(struct xfrm_state_walk *walk, u8 proto)
{
+ INIT_LIST_HEAD(&walk->all);
walk->proto = proto;
- walk->state = NULL;
- walk->count = 0;
- list_add_tail(&walk->list, &xfrm_state_walks);
- walk->genid = ++xfrm_state_walk_ongoing;
+ walk->state = XFRM_STATE_DEAD;
+ walk->seq = 0;
}
EXPORT_SYMBOL(xfrm_state_walk_init);
void xfrm_state_walk_done(struct xfrm_state_walk *walk)
{
- struct list_head *prev;
-
- if (walk->state != NULL) {
- xfrm_state_put(walk->state);
- walk->state = NULL;
- }
-
- prev = walk->list.prev;
- list_del(&walk->list);
-
- if (prev != &xfrm_state_walks) {
- list_entry(prev, struct xfrm_state_walk, list)->genid =
- walk->genid;
+ if (list_empty(&walk->all))
return;
- }
- xfrm_state_walk_completed = walk->genid;
-
- if (!list_empty(&xfrm_state_gc_leftovers))
- schedule_work(&xfrm_state_gc_work);
+ spin_lock_bh(&xfrm_state_lock);
+ list_del(&walk->all);
+ spin_lock_bh(&xfrm_state_lock);
}
EXPORT_SYMBOL(xfrm_state_walk_done);
diff --git a/net/xfrm/xfrm_user.c b/net/xfrm/xfrm_user.c
index 04c41504..76f75df 100644
--- a/net/xfrm/xfrm_user.c
+++ b/net/xfrm/xfrm_user.c
@@ -1102,7 +1102,7 @@
return xp;
error:
*errp = err;
- xp->dead = 1;
+ xp->walk.dead = 1;
xfrm_policy_destroy(xp);
return NULL;
}
@@ -1595,7 +1595,7 @@
return -ENOENT;
read_lock(&xp->lock);
- if (xp->dead) {
+ if (xp->walk.dead) {
read_unlock(&xp->lock);
goto out;
}
