blob: 7554c56b2e63c8a7c1a53b9ce0b21b126d54b079 [file] [log] [blame]
Florian Westphal625c5562017-12-09 21:01:08 +01001/*
2 * count the number of connections matching an arbitrary key.
3 *
4 * (C) 2017 Red Hat GmbH
5 * Author: Florian Westphal <fw@strlen.de>
6 *
7 * split from xt_connlimit.c:
8 * (c) 2000 Gerd Knorr <kraxel@bytesex.org>
9 * Nov 2002: Martin Bene <martin.bene@icomedias.com>:
10 * only ignore TIME_WAIT or gone connections
11 * (C) CC Computer Consultants GmbH, 2007
12 */
13#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
14#include <linux/in.h>
15#include <linux/in6.h>
16#include <linux/ip.h>
17#include <linux/ipv6.h>
18#include <linux/jhash.h>
19#include <linux/slab.h>
20#include <linux/list.h>
21#include <linux/rbtree.h>
22#include <linux/module.h>
23#include <linux/random.h>
24#include <linux/skbuff.h>
25#include <linux/spinlock.h>
26#include <linux/netfilter/nf_conntrack_tcp.h>
27#include <linux/netfilter/x_tables.h>
28#include <net/netfilter/nf_conntrack.h>
29#include <net/netfilter/nf_conntrack_count.h>
30#include <net/netfilter/nf_conntrack_core.h>
31#include <net/netfilter/nf_conntrack_tuple.h>
32#include <net/netfilter/nf_conntrack_zones.h>
33
34#define CONNCOUNT_SLOTS 256U
35
Florian Westphal625c5562017-12-09 21:01:08 +010036#define CONNCOUNT_GC_MAX_NODES 8
37#define MAX_KEYLEN 5
38
39/* we will save the tuples of all connections we care about */
40struct nf_conncount_tuple {
Yi-Hung Weicb2b36f2018-07-02 17:33:40 -070041 struct list_head node;
Florian Westphal625c5562017-12-09 21:01:08 +010042 struct nf_conntrack_tuple tuple;
Yi-Hung Wei21ba8842018-06-12 10:51:34 -070043 struct nf_conntrack_zone zone;
Florian Westphalb36e4522018-06-20 23:32:26 +020044 int cpu;
45 u32 jiffies32;
Florian Westphal625c5562017-12-09 21:01:08 +010046};
47
48struct nf_conncount_rb {
49 struct rb_node node;
Yi-Hung Weicb2b36f2018-07-02 17:33:40 -070050 struct nf_conncount_list list;
Florian Westphal625c5562017-12-09 21:01:08 +010051 u32 key[MAX_KEYLEN];
Yi-Hung Wei5c789e12018-07-02 17:33:44 -070052 struct rcu_head rcu_head;
Florian Westphal625c5562017-12-09 21:01:08 +010053};
54
Shawn Bohrerc78e7812018-12-28 01:24:42 +010055static spinlock_t nf_conncount_locks[CONNCOUNT_SLOTS] __cacheline_aligned_in_smp;
Florian Westphal625c5562017-12-09 21:01:08 +010056
57struct nf_conncount_data {
58 unsigned int keylen;
59 struct rb_root root[CONNCOUNT_SLOTS];
Yi-Hung Wei5c789e12018-07-02 17:33:44 -070060 struct net *net;
61 struct work_struct gc_work;
62 unsigned long pending_trees[BITS_TO_LONGS(CONNCOUNT_SLOTS)];
63 unsigned int gc_tree;
Florian Westphal625c5562017-12-09 21:01:08 +010064};
65
66static u_int32_t conncount_rnd __read_mostly;
67static struct kmem_cache *conncount_rb_cachep __read_mostly;
68static struct kmem_cache *conncount_conn_cachep __read_mostly;
69
70static inline bool already_closed(const struct nf_conn *conn)
71{
72 if (nf_ct_protonum(conn) == IPPROTO_TCP)
73 return conn->proto.tcp.state == TCP_CONNTRACK_TIME_WAIT ||
74 conn->proto.tcp.state == TCP_CONNTRACK_CLOSE;
75 else
Gustavo A. R. Silvad384e652018-01-18 17:25:12 -060076 return false;
Florian Westphal625c5562017-12-09 21:01:08 +010077}
78
79static int key_diff(const u32 *a, const u32 *b, unsigned int klen)
80{
81 return memcmp(a, b, klen * sizeof(u32));
82}
83
Pablo Neira Ayusoc80f10b2018-12-28 01:24:48 +010084static void conn_free(struct nf_conncount_list *list,
Yi-Hung Weicb2b36f2018-07-02 17:33:40 -070085 struct nf_conncount_tuple *conn)
86{
Pablo Neira Ayuso2f971a82018-12-28 01:24:47 +010087 lockdep_assert_held(&list->list_lock);
Yi-Hung Weicb2b36f2018-07-02 17:33:40 -070088
89 list->count--;
Pablo Neira Ayuso2f971a82018-12-28 01:24:47 +010090 list_del(&conn->node);
Yi-Hung Wei5c789e12018-07-02 17:33:44 -070091
Pablo Neira Ayuso2f971a82018-12-28 01:24:47 +010092 kmem_cache_free(conncount_conn_cachep, conn);
Yi-Hung Weicb2b36f2018-07-02 17:33:40 -070093}
94
Florian Westphalb36e4522018-06-20 23:32:26 +020095static const struct nf_conntrack_tuple_hash *
Yi-Hung Weicb2b36f2018-07-02 17:33:40 -070096find_or_evict(struct net *net, struct nf_conncount_list *list,
Pablo Neira Ayusoc80f10b2018-12-28 01:24:48 +010097 struct nf_conncount_tuple *conn)
Florian Westphalb36e4522018-06-20 23:32:26 +020098{
99 const struct nf_conntrack_tuple_hash *found;
100 unsigned long a, b;
101 int cpu = raw_smp_processor_id();
Florian Westphal4cd273b2018-12-28 01:24:43 +0100102 u32 age;
Florian Westphalb36e4522018-06-20 23:32:26 +0200103
104 found = nf_conntrack_find_get(net, &conn->zone, &conn->tuple);
105 if (found)
106 return found;
107 b = conn->jiffies32;
108 a = (u32)jiffies;
109
110 /* conn might have been added just before by another cpu and
111 * might still be unconfirmed. In this case, nf_conntrack_find()
112 * returns no result. Thus only evict if this cpu added the
113 * stale entry or if the entry is older than two jiffies.
114 */
115 age = a - b;
116 if (conn->cpu == cpu || age >= 2) {
Pablo Neira Ayusoc80f10b2018-12-28 01:24:48 +0100117 conn_free(list, conn);
Florian Westphalb36e4522018-06-20 23:32:26 +0200118 return ERR_PTR(-ENOENT);
119 }
120
121 return ERR_PTR(-EAGAIN);
122}
123
Florian Westphaldf4a9022018-12-28 01:24:46 +0100124static int __nf_conncount_add(struct net *net,
125 struct nf_conncount_list *list,
126 const struct nf_conntrack_tuple *tuple,
127 const struct nf_conntrack_zone *zone)
Florian Westphal625c5562017-12-09 21:01:08 +0100128{
129 const struct nf_conntrack_tuple_hash *found;
Yi-Hung Weicb2b36f2018-07-02 17:33:40 -0700130 struct nf_conncount_tuple *conn, *conn_n;
Florian Westphal625c5562017-12-09 21:01:08 +0100131 struct nf_conn *found_ct;
Yi-Hung Wei976afca2018-07-02 17:33:41 -0700132 unsigned int collect = 0;
Florian Westphal625c5562017-12-09 21:01:08 +0100133
Florian Westphal625c5562017-12-09 21:01:08 +0100134 /* check the saved connections */
Yi-Hung Weicb2b36f2018-07-02 17:33:40 -0700135 list_for_each_entry_safe(conn, conn_n, &list->head, node) {
Yi-Hung Wei976afca2018-07-02 17:33:41 -0700136 if (collect > CONNCOUNT_GC_MAX_NODES)
137 break;
138
Pablo Neira Ayusoc80f10b2018-12-28 01:24:48 +0100139 found = find_or_evict(net, list, conn);
Florian Westphalb36e4522018-06-20 23:32:26 +0200140 if (IS_ERR(found)) {
141 /* Not found, but might be about to be confirmed */
142 if (PTR_ERR(found) == -EAGAIN) {
Florian Westphalb36e4522018-06-20 23:32:26 +0200143 if (nf_ct_tuple_equal(&conn->tuple, tuple) &&
144 nf_ct_zone_id(&conn->zone, conn->zone.dir) ==
145 nf_ct_zone_id(zone, zone->dir))
Florian Westphaldf4a9022018-12-28 01:24:46 +0100146 return 0; /* already exists */
147 } else {
Yi-Hung Wei976afca2018-07-02 17:33:41 -0700148 collect++;
Florian Westphaldf4a9022018-12-28 01:24:46 +0100149 }
Florian Westphal625c5562017-12-09 21:01:08 +0100150 continue;
151 }
152
153 found_ct = nf_ct_tuplehash_to_ctrack(found);
154
Florian Westphaldf4a9022018-12-28 01:24:46 +0100155 if (nf_ct_tuple_equal(&conn->tuple, tuple) &&
Yi-Hung Wei21ba8842018-06-12 10:51:34 -0700156 nf_ct_zone_equal(found_ct, zone, zone->dir)) {
Florian Westphal625c5562017-12-09 21:01:08 +0100157 /*
Florian Westphal625c5562017-12-09 21:01:08 +0100158 * We should not see tuples twice unless someone hooks
159 * this into a table without "-p tcp --syn".
Yi-Hung Wei976afca2018-07-02 17:33:41 -0700160 *
161 * Attempt to avoid a re-add in this case.
Florian Westphal625c5562017-12-09 21:01:08 +0100162 */
Florian Westphaldf4a9022018-12-28 01:24:46 +0100163 nf_ct_put(found_ct);
164 return 0;
Florian Westphal625c5562017-12-09 21:01:08 +0100165 } else if (already_closed(found_ct)) {
166 /*
167 * we do not care about connections which are
168 * closed already -> ditch it
169 */
170 nf_ct_put(found_ct);
Yi-Hung Weicb2b36f2018-07-02 17:33:40 -0700171 conn_free(list, conn);
Yi-Hung Wei976afca2018-07-02 17:33:41 -0700172 collect++;
Florian Westphal625c5562017-12-09 21:01:08 +0100173 continue;
174 }
175
176 nf_ct_put(found_ct);
Florian Westphal625c5562017-12-09 21:01:08 +0100177 }
Florian Westphaldf4a9022018-12-28 01:24:46 +0100178
179 if (WARN_ON_ONCE(list->count > INT_MAX))
180 return -EOVERFLOW;
181
182 conn = kmem_cache_alloc(conncount_conn_cachep, GFP_ATOMIC);
183 if (conn == NULL)
184 return -ENOMEM;
185
186 conn->tuple = *tuple;
187 conn->zone = *zone;
188 conn->cpu = raw_smp_processor_id();
189 conn->jiffies32 = (u32)jiffies;
190 list_add_tail(&conn->node, &list->head);
191 list->count++;
192 return 0;
Florian Westphal625c5562017-12-09 21:01:08 +0100193}
Florian Westphaldf4a9022018-12-28 01:24:46 +0100194
195int nf_conncount_add(struct net *net,
196 struct nf_conncount_list *list,
197 const struct nf_conntrack_tuple *tuple,
198 const struct nf_conntrack_zone *zone)
199{
200 int ret;
201
202 /* check the saved connections */
203 spin_lock_bh(&list->list_lock);
204 ret = __nf_conncount_add(net, list, tuple, zone);
205 spin_unlock_bh(&list->list_lock);
206
207 return ret;
208}
209EXPORT_SYMBOL_GPL(nf_conncount_add);
Florian Westphal625c5562017-12-09 21:01:08 +0100210
Yi-Hung Weicb2b36f2018-07-02 17:33:40 -0700211void nf_conncount_list_init(struct nf_conncount_list *list)
212{
Yi-Hung Wei5c789e12018-07-02 17:33:44 -0700213 spin_lock_init(&list->list_lock);
Yi-Hung Weicb2b36f2018-07-02 17:33:40 -0700214 INIT_LIST_HEAD(&list->head);
Taehee Yoo3c5cdb12018-11-05 03:44:39 +0900215 list->count = 0;
Yi-Hung Weicb2b36f2018-07-02 17:33:40 -0700216}
217EXPORT_SYMBOL_GPL(nf_conncount_list_init);
218
Pablo Neira Ayuso2f971a82018-12-28 01:24:47 +0100219/* Return true if the list is empty. Must be called with BH disabled. */
Yi-Hung Wei5c789e12018-07-02 17:33:44 -0700220bool nf_conncount_gc_list(struct net *net,
Yi-Hung Wei976afca2018-07-02 17:33:41 -0700221 struct nf_conncount_list *list)
Yi-Hung Wei2a406e82018-07-02 17:33:39 -0700222{
223 const struct nf_conntrack_tuple_hash *found;
Yi-Hung Weicb2b36f2018-07-02 17:33:40 -0700224 struct nf_conncount_tuple *conn, *conn_n;
Yi-Hung Wei2a406e82018-07-02 17:33:39 -0700225 struct nf_conn *found_ct;
226 unsigned int collected = 0;
Taehee Yoo3c5cdb12018-11-05 03:44:39 +0900227 bool ret = false;
Yi-Hung Wei2a406e82018-07-02 17:33:39 -0700228
Pablo Neira Ayuso2f971a82018-12-28 01:24:47 +0100229 /* don't bother if other cpu is already doing GC */
230 if (!spin_trylock(&list->list_lock))
231 return false;
232
Yi-Hung Weicb2b36f2018-07-02 17:33:40 -0700233 list_for_each_entry_safe(conn, conn_n, &list->head, node) {
Pablo Neira Ayusoc80f10b2018-12-28 01:24:48 +0100234 found = find_or_evict(net, list, conn);
Yi-Hung Wei2a406e82018-07-02 17:33:39 -0700235 if (IS_ERR(found)) {
Pablo Neira Ayusoc80f10b2018-12-28 01:24:48 +0100236 if (PTR_ERR(found) == -ENOENT)
Yi-Hung Wei2a406e82018-07-02 17:33:39 -0700237 collected++;
238 continue;
239 }
240
241 found_ct = nf_ct_tuplehash_to_ctrack(found);
242 if (already_closed(found_ct)) {
243 /*
244 * we do not care about connections which are
245 * closed already -> ditch it
246 */
247 nf_ct_put(found_ct);
Pablo Neira Ayusoc80f10b2018-12-28 01:24:48 +0100248 conn_free(list, conn);
Yi-Hung Wei2a406e82018-07-02 17:33:39 -0700249 collected++;
250 continue;
251 }
252
253 nf_ct_put(found_ct);
254 if (collected > CONNCOUNT_GC_MAX_NODES)
Pablo Neira Ayuso2f971a82018-12-28 01:24:47 +0100255 break;
Yi-Hung Wei2a406e82018-07-02 17:33:39 -0700256 }
Taehee Yoo3c5cdb12018-11-05 03:44:39 +0900257
Pablo Neira Ayusoc80f10b2018-12-28 01:24:48 +0100258 if (!list->count)
Taehee Yoo3c5cdb12018-11-05 03:44:39 +0900259 ret = true;
Pablo Neira Ayuso2f971a82018-12-28 01:24:47 +0100260 spin_unlock(&list->list_lock);
Taehee Yoo3c5cdb12018-11-05 03:44:39 +0900261
262 return ret;
Yi-Hung Wei2a406e82018-07-02 17:33:39 -0700263}
Yi-Hung Wei976afca2018-07-02 17:33:41 -0700264EXPORT_SYMBOL_GPL(nf_conncount_gc_list);
Yi-Hung Wei2a406e82018-07-02 17:33:39 -0700265
Yi-Hung Wei5c789e12018-07-02 17:33:44 -0700266static void __tree_nodes_free(struct rcu_head *h)
267{
268 struct nf_conncount_rb *rbconn;
269
270 rbconn = container_of(h, struct nf_conncount_rb, rcu_head);
271 kmem_cache_free(conncount_rb_cachep, rbconn);
272}
273
Pablo Neira Ayusoc80f10b2018-12-28 01:24:48 +0100274/* caller must hold tree nf_conncount_locks[] lock */
Florian Westphal625c5562017-12-09 21:01:08 +0100275static void tree_nodes_free(struct rb_root *root,
276 struct nf_conncount_rb *gc_nodes[],
277 unsigned int gc_count)
278{
279 struct nf_conncount_rb *rbconn;
280
281 while (gc_count) {
282 rbconn = gc_nodes[--gc_count];
Yi-Hung Wei5c789e12018-07-02 17:33:44 -0700283 spin_lock(&rbconn->list.list_lock);
Pablo Neira Ayusoc80f10b2018-12-28 01:24:48 +0100284 if (!rbconn->list.count) {
285 rb_erase(&rbconn->node, root);
286 call_rcu(&rbconn->rcu_head, __tree_nodes_free);
287 }
Yi-Hung Wei5c789e12018-07-02 17:33:44 -0700288 spin_unlock(&rbconn->list.list_lock);
Florian Westphal625c5562017-12-09 21:01:08 +0100289 }
290}
291
Yi-Hung Wei5c789e12018-07-02 17:33:44 -0700292static void schedule_gc_worker(struct nf_conncount_data *data, int tree)
293{
294 set_bit(tree, data->pending_trees);
295 schedule_work(&data->gc_work);
296}
297
Florian Westphal625c5562017-12-09 21:01:08 +0100298static unsigned int
Yi-Hung Wei5c789e12018-07-02 17:33:44 -0700299insert_tree(struct net *net,
300 struct nf_conncount_data *data,
301 struct rb_root *root,
Yi-Hung Wei34848d52018-07-02 17:33:43 -0700302 unsigned int hash,
303 const u32 *key,
Yi-Hung Wei34848d52018-07-02 17:33:43 -0700304 const struct nf_conntrack_tuple *tuple,
305 const struct nf_conntrack_zone *zone)
306{
Yi-Hung Wei5c789e12018-07-02 17:33:44 -0700307 struct nf_conncount_rb *gc_nodes[CONNCOUNT_GC_MAX_NODES];
Yi-Hung Wei34848d52018-07-02 17:33:43 -0700308 struct rb_node **rbnode, *parent;
309 struct nf_conncount_rb *rbconn;
310 struct nf_conncount_tuple *conn;
Yi-Hung Wei5c789e12018-07-02 17:33:44 -0700311 unsigned int count = 0, gc_count = 0;
Pablo Neira Ayusoc80f10b2018-12-28 01:24:48 +0100312 u8 keylen = data->keylen;
Florian Westphale8cfb372018-12-28 01:24:45 +0100313 bool do_gc = true;
Yi-Hung Wei34848d52018-07-02 17:33:43 -0700314
Shawn Bohrerc78e7812018-12-28 01:24:42 +0100315 spin_lock_bh(&nf_conncount_locks[hash]);
Florian Westphale8cfb372018-12-28 01:24:45 +0100316restart:
Yi-Hung Wei34848d52018-07-02 17:33:43 -0700317 parent = NULL;
318 rbnode = &(root->rb_node);
319 while (*rbnode) {
320 int diff;
321 rbconn = rb_entry(*rbnode, struct nf_conncount_rb, node);
322
323 parent = *rbnode;
324 diff = key_diff(key, rbconn->key, keylen);
325 if (diff < 0) {
326 rbnode = &((*rbnode)->rb_left);
327 } else if (diff > 0) {
328 rbnode = &((*rbnode)->rb_right);
329 } else {
Florian Westphaldf4a9022018-12-28 01:24:46 +0100330 int ret;
331
332 ret = nf_conncount_add(net, &rbconn->list, tuple, zone);
333 if (ret)
Yi-Hung Wei34848d52018-07-02 17:33:43 -0700334 count = 0; /* hotdrop */
Florian Westphaldf4a9022018-12-28 01:24:46 +0100335 else
Yi-Hung Wei5c789e12018-07-02 17:33:44 -0700336 count = rbconn->list.count;
Florian Westphaldf4a9022018-12-28 01:24:46 +0100337 tree_nodes_free(root, gc_nodes, gc_count);
338 goto out_unlock;
Yi-Hung Wei34848d52018-07-02 17:33:43 -0700339 }
Yi-Hung Wei5c789e12018-07-02 17:33:44 -0700340
341 if (gc_count >= ARRAY_SIZE(gc_nodes))
342 continue;
343
Florian Westphale8cfb372018-12-28 01:24:45 +0100344 if (do_gc && nf_conncount_gc_list(net, &rbconn->list))
Yi-Hung Wei5c789e12018-07-02 17:33:44 -0700345 gc_nodes[gc_count++] = rbconn;
Yi-Hung Wei34848d52018-07-02 17:33:43 -0700346 }
347
Yi-Hung Wei5c789e12018-07-02 17:33:44 -0700348 if (gc_count) {
349 tree_nodes_free(root, gc_nodes, gc_count);
Florian Westphale8cfb372018-12-28 01:24:45 +0100350 schedule_gc_worker(data, hash);
351 gc_count = 0;
352 do_gc = false;
353 goto restart;
Yi-Hung Wei5c789e12018-07-02 17:33:44 -0700354 }
355
Yi-Hung Wei34848d52018-07-02 17:33:43 -0700356 /* expected case: match, insert new node */
357 rbconn = kmem_cache_alloc(conncount_rb_cachep, GFP_ATOMIC);
358 if (rbconn == NULL)
359 goto out_unlock;
360
361 conn = kmem_cache_alloc(conncount_conn_cachep, GFP_ATOMIC);
362 if (conn == NULL) {
363 kmem_cache_free(conncount_rb_cachep, rbconn);
364 goto out_unlock;
365 }
366
367 conn->tuple = *tuple;
368 conn->zone = *zone;
369 memcpy(rbconn->key, key, sizeof(u32) * keylen);
370
371 nf_conncount_list_init(&rbconn->list);
372 list_add(&conn->node, &rbconn->list.head);
373 count = 1;
Taehee Yoo3c5cdb12018-11-05 03:44:39 +0900374 rbconn->list.count = count;
Yi-Hung Wei34848d52018-07-02 17:33:43 -0700375
Taehee Yood4e7df12018-12-08 11:03:01 +0900376 rb_link_node_rcu(&rbconn->node, parent, rbnode);
Yi-Hung Wei34848d52018-07-02 17:33:43 -0700377 rb_insert_color(&rbconn->node, root);
378out_unlock:
Shawn Bohrerc78e7812018-12-28 01:24:42 +0100379 spin_unlock_bh(&nf_conncount_locks[hash]);
Yi-Hung Wei34848d52018-07-02 17:33:43 -0700380 return count;
381}
382
383static unsigned int
Yi-Hung Wei2ba39112018-07-02 17:33:42 -0700384count_tree(struct net *net,
385 struct nf_conncount_data *data,
386 const u32 *key,
Florian Westphal625c5562017-12-09 21:01:08 +0100387 const struct nf_conntrack_tuple *tuple,
388 const struct nf_conntrack_zone *zone)
389{
Yi-Hung Wei2ba39112018-07-02 17:33:42 -0700390 struct rb_root *root;
Yi-Hung Wei5c789e12018-07-02 17:33:44 -0700391 struct rb_node *parent;
Florian Westphal625c5562017-12-09 21:01:08 +0100392 struct nf_conncount_rb *rbconn;
Yi-Hung Wei5c789e12018-07-02 17:33:44 -0700393 unsigned int hash;
Yi-Hung Wei2ba39112018-07-02 17:33:42 -0700394 u8 keylen = data->keylen;
Florian Westphal625c5562017-12-09 21:01:08 +0100395
Yi-Hung Wei2ba39112018-07-02 17:33:42 -0700396 hash = jhash2(key, data->keylen, conncount_rnd) % CONNCOUNT_SLOTS;
397 root = &data->root[hash];
398
Yi-Hung Wei5c789e12018-07-02 17:33:44 -0700399 parent = rcu_dereference_raw(root->rb_node);
400 while (parent) {
Florian Westphal625c5562017-12-09 21:01:08 +0100401 int diff;
Florian Westphal625c5562017-12-09 21:01:08 +0100402
Yi-Hung Wei5c789e12018-07-02 17:33:44 -0700403 rbconn = rb_entry(parent, struct nf_conncount_rb, node);
Florian Westphal625c5562017-12-09 21:01:08 +0100404
Florian Westphal625c5562017-12-09 21:01:08 +0100405 diff = key_diff(key, rbconn->key, keylen);
406 if (diff < 0) {
Yi-Hung Wei5c789e12018-07-02 17:33:44 -0700407 parent = rcu_dereference_raw(parent->rb_left);
Florian Westphal625c5562017-12-09 21:01:08 +0100408 } else if (diff > 0) {
Yi-Hung Wei5c789e12018-07-02 17:33:44 -0700409 parent = rcu_dereference_raw(parent->rb_right);
Florian Westphal625c5562017-12-09 21:01:08 +0100410 } else {
Florian Westphaldf4a9022018-12-28 01:24:46 +0100411 int ret;
Florian Westphal625c5562017-12-09 21:01:08 +0100412
Florian Westphaldf4a9022018-12-28 01:24:46 +0100413 if (!tuple) {
414 nf_conncount_gc_list(net, &rbconn->list);
Yi-Hung Wei5c789e12018-07-02 17:33:44 -0700415 return rbconn->list.count;
Florian Westphaldf4a9022018-12-28 01:24:46 +0100416 }
Florian Westphal625c5562017-12-09 21:01:08 +0100417
Florian Westphaldf4a9022018-12-28 01:24:46 +0100418 spin_lock_bh(&rbconn->list.list_lock);
419 /* Node might be about to be free'd.
420 * We need to defer to insert_tree() in this case.
421 */
422 if (rbconn->list.count == 0) {
423 spin_unlock_bh(&rbconn->list.list_lock);
Yi-Hung Wei5c789e12018-07-02 17:33:44 -0700424 break;
425 }
Florian Westphaldf4a9022018-12-28 01:24:46 +0100426
427 /* same source network -> be counted! */
428 ret = __nf_conncount_add(net, &rbconn->list, tuple, zone);
429 spin_unlock_bh(&rbconn->list.list_lock);
430 if (ret)
431 return 0; /* hotdrop */
432 else
433 return rbconn->list.count;
Florian Westphal625c5562017-12-09 21:01:08 +0100434 }
Yi-Hung Wei5c789e12018-07-02 17:33:44 -0700435 }
Florian Westphal625c5562017-12-09 21:01:08 +0100436
Yi-Hung Wei5c789e12018-07-02 17:33:44 -0700437 if (!tuple)
438 return 0;
Florian Westphal625c5562017-12-09 21:01:08 +0100439
Pablo Neira Ayusoc80f10b2018-12-28 01:24:48 +0100440 return insert_tree(net, data, root, hash, key, tuple, zone);
Yi-Hung Wei5c789e12018-07-02 17:33:44 -0700441}
442
443static void tree_gc_worker(struct work_struct *work)
444{
445 struct nf_conncount_data *data = container_of(work, struct nf_conncount_data, gc_work);
446 struct nf_conncount_rb *gc_nodes[CONNCOUNT_GC_MAX_NODES], *rbconn;
447 struct rb_root *root;
448 struct rb_node *node;
449 unsigned int tree, next_tree, gc_count = 0;
450
Shawn Bohrerc78e7812018-12-28 01:24:42 +0100451 tree = data->gc_tree % CONNCOUNT_SLOTS;
Yi-Hung Wei5c789e12018-07-02 17:33:44 -0700452 root = &data->root[tree];
453
Pablo Neira Ayuso2f971a82018-12-28 01:24:47 +0100454 local_bh_disable();
Yi-Hung Wei5c789e12018-07-02 17:33:44 -0700455 rcu_read_lock();
456 for (node = rb_first(root); node != NULL; node = rb_next(node)) {
457 rbconn = rb_entry(node, struct nf_conncount_rb, node);
458 if (nf_conncount_gc_list(data->net, &rbconn->list))
Florian Westphalf7fcc982018-12-28 01:24:44 +0100459 gc_count++;
Florian Westphal625c5562017-12-09 21:01:08 +0100460 }
Yi-Hung Wei5c789e12018-07-02 17:33:44 -0700461 rcu_read_unlock();
Pablo Neira Ayuso2f971a82018-12-28 01:24:47 +0100462 local_bh_enable();
463
464 cond_resched();
Yi-Hung Wei5c789e12018-07-02 17:33:44 -0700465
466 spin_lock_bh(&nf_conncount_locks[tree]);
Florian Westphalf7fcc982018-12-28 01:24:44 +0100467 if (gc_count < ARRAY_SIZE(gc_nodes))
468 goto next; /* do not bother */
Florian Westphal625c5562017-12-09 21:01:08 +0100469
Florian Westphalf7fcc982018-12-28 01:24:44 +0100470 gc_count = 0;
471 node = rb_first(root);
472 while (node != NULL) {
473 rbconn = rb_entry(node, struct nf_conncount_rb, node);
474 node = rb_next(node);
475
476 if (rbconn->list.count > 0)
477 continue;
478
479 gc_nodes[gc_count++] = rbconn;
480 if (gc_count >= ARRAY_SIZE(gc_nodes)) {
481 tree_nodes_free(root, gc_nodes, gc_count);
482 gc_count = 0;
483 }
Florian Westphal625c5562017-12-09 21:01:08 +0100484 }
485
Florian Westphalf7fcc982018-12-28 01:24:44 +0100486 tree_nodes_free(root, gc_nodes, gc_count);
487next:
Yi-Hung Wei5c789e12018-07-02 17:33:44 -0700488 clear_bit(tree, data->pending_trees);
Florian Westphal625c5562017-12-09 21:01:08 +0100489
Yi-Hung Wei5c789e12018-07-02 17:33:44 -0700490 next_tree = (tree + 1) % CONNCOUNT_SLOTS;
Florian Westphala0072322018-12-28 01:24:49 +0100491 next_tree = find_next_bit(data->pending_trees, CONNCOUNT_SLOTS, next_tree);
Florian Westphal625c5562017-12-09 21:01:08 +0100492
Yi-Hung Wei5c789e12018-07-02 17:33:44 -0700493 if (next_tree < CONNCOUNT_SLOTS) {
494 data->gc_tree = next_tree;
495 schedule_work(work);
496 }
497
498 spin_unlock_bh(&nf_conncount_locks[tree]);
Florian Westphal625c5562017-12-09 21:01:08 +0100499}
500
Yi-Hung Wei35d8deb2018-03-04 15:29:52 -0800501/* Count and return number of conntrack entries in 'net' with particular 'key'.
502 * If 'tuple' is not null, insert it into the accounting data structure.
Yi-Hung Wei5c789e12018-07-02 17:33:44 -0700503 * Call with RCU read lock.
Yi-Hung Wei35d8deb2018-03-04 15:29:52 -0800504 */
Florian Westphal625c5562017-12-09 21:01:08 +0100505unsigned int nf_conncount_count(struct net *net,
506 struct nf_conncount_data *data,
507 const u32 *key,
Florian Westphal625c5562017-12-09 21:01:08 +0100508 const struct nf_conntrack_tuple *tuple,
509 const struct nf_conntrack_zone *zone)
510{
Yi-Hung Wei2ba39112018-07-02 17:33:42 -0700511 return count_tree(net, data, key, tuple, zone);
Florian Westphal625c5562017-12-09 21:01:08 +0100512}
513EXPORT_SYMBOL_GPL(nf_conncount_count);
514
515struct nf_conncount_data *nf_conncount_init(struct net *net, unsigned int family,
516 unsigned int keylen)
517{
518 struct nf_conncount_data *data;
519 int ret, i;
520
521 if (keylen % sizeof(u32) ||
522 keylen / sizeof(u32) > MAX_KEYLEN ||
523 keylen == 0)
524 return ERR_PTR(-EINVAL);
525
526 net_get_random_once(&conncount_rnd, sizeof(conncount_rnd));
527
528 data = kmalloc(sizeof(*data), GFP_KERNEL);
529 if (!data)
530 return ERR_PTR(-ENOMEM);
531
532 ret = nf_ct_netns_get(net, family);
533 if (ret < 0) {
534 kfree(data);
535 return ERR_PTR(ret);
536 }
537
538 for (i = 0; i < ARRAY_SIZE(data->root); ++i)
539 data->root[i] = RB_ROOT;
540
541 data->keylen = keylen / sizeof(u32);
Yi-Hung Wei5c789e12018-07-02 17:33:44 -0700542 data->net = net;
543 INIT_WORK(&data->gc_work, tree_gc_worker);
Florian Westphal625c5562017-12-09 21:01:08 +0100544
545 return data;
546}
547EXPORT_SYMBOL_GPL(nf_conncount_init);
548
Yi-Hung Weicb2b36f2018-07-02 17:33:40 -0700549void nf_conncount_cache_free(struct nf_conncount_list *list)
Florian Westphal625c5562017-12-09 21:01:08 +0100550{
Yi-Hung Weicb2b36f2018-07-02 17:33:40 -0700551 struct nf_conncount_tuple *conn, *conn_n;
Pablo Neira Ayuso5e5cbc72018-06-02 23:38:47 +0200552
Yi-Hung Weicb2b36f2018-07-02 17:33:40 -0700553 list_for_each_entry_safe(conn, conn_n, &list->head, node)
Pablo Neira Ayuso5e5cbc72018-06-02 23:38:47 +0200554 kmem_cache_free(conncount_conn_cachep, conn);
555}
556EXPORT_SYMBOL_GPL(nf_conncount_cache_free);
557
558static void destroy_tree(struct rb_root *r)
559{
560 struct nf_conncount_rb *rbconn;
Florian Westphal625c5562017-12-09 21:01:08 +0100561 struct rb_node *node;
562
563 while ((node = rb_first(r)) != NULL) {
564 rbconn = rb_entry(node, struct nf_conncount_rb, node);
565
566 rb_erase(node, r);
567
Yi-Hung Weicb2b36f2018-07-02 17:33:40 -0700568 nf_conncount_cache_free(&rbconn->list);
Florian Westphal625c5562017-12-09 21:01:08 +0100569
570 kmem_cache_free(conncount_rb_cachep, rbconn);
571 }
572}
573
574void nf_conncount_destroy(struct net *net, unsigned int family,
575 struct nf_conncount_data *data)
576{
577 unsigned int i;
578
Yi-Hung Wei5c789e12018-07-02 17:33:44 -0700579 cancel_work_sync(&data->gc_work);
Florian Westphal625c5562017-12-09 21:01:08 +0100580 nf_ct_netns_put(net, family);
581
582 for (i = 0; i < ARRAY_SIZE(data->root); ++i)
583 destroy_tree(&data->root[i]);
584
585 kfree(data);
586}
587EXPORT_SYMBOL_GPL(nf_conncount_destroy);
588
589static int __init nf_conncount_modinit(void)
590{
591 int i;
592
Shawn Bohrerc78e7812018-12-28 01:24:42 +0100593 for (i = 0; i < CONNCOUNT_SLOTS; ++i)
Florian Westphal625c5562017-12-09 21:01:08 +0100594 spin_lock_init(&nf_conncount_locks[i]);
595
596 conncount_conn_cachep = kmem_cache_create("nf_conncount_tuple",
597 sizeof(struct nf_conncount_tuple),
598 0, 0, NULL);
599 if (!conncount_conn_cachep)
600 return -ENOMEM;
601
602 conncount_rb_cachep = kmem_cache_create("nf_conncount_rb",
603 sizeof(struct nf_conncount_rb),
604 0, 0, NULL);
605 if (!conncount_rb_cachep) {
606 kmem_cache_destroy(conncount_conn_cachep);
607 return -ENOMEM;
608 }
609
610 return 0;
611}
612
613static void __exit nf_conncount_modexit(void)
614{
615 kmem_cache_destroy(conncount_conn_cachep);
616 kmem_cache_destroy(conncount_rb_cachep);
617}
618
619module_init(nf_conncount_modinit);
620module_exit(nf_conncount_modexit);
621MODULE_AUTHOR("Jan Engelhardt <jengelh@medozas.de>");
622MODULE_AUTHOR("Florian Westphal <fw@strlen.de>");
623MODULE_DESCRIPTION("netfilter: count number of connections matching a key");
624MODULE_LICENSE("GPL");