blob: 8ce2a050797049a1b20715a88e44a8542f637966 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * net/sched/cls_api.c Packet classifier API.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
10 *
11 * Changes:
12 *
13 * Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support
14 *
15 */
16
Linus Torvalds1da177e2005-04-16 15:20:36 -070017#include <linux/module.h>
18#include <linux/types.h>
19#include <linux/kernel.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070020#include <linux/string.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070021#include <linux/errno.h>
Jiri Pirko33a48922017-02-09 14:38:57 +010022#include <linux/err.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070023#include <linux/skbuff.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024#include <linux/init.h>
25#include <linux/kmod.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090026#include <linux/slab.h>
Jiri Pirko48617382018-01-17 11:46:46 +010027#include <linux/idr.h>
John Hurley7f76fa32018-11-09 21:21:26 -080028#include <linux/rhashtable.h>
Denis V. Lunevb8542722007-12-01 00:21:31 +110029#include <net/net_namespace.h>
30#include <net/sock.h>
Arnaldo Carvalho de Melodc5fc572007-03-25 23:06:12 -070031#include <net/netlink.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070032#include <net/pkt_sched.h>
33#include <net/pkt_cls.h>
34
Davide Carattie3314732018-10-10 22:00:58 +020035extern const struct nla_policy rtm_tca_policy[TCA_MAX + 1];
36
Linus Torvalds1da177e2005-04-16 15:20:36 -070037/* The list of all installed classifier types */
WANG Cong36272872013-12-15 20:15:11 -080038static LIST_HEAD(tcf_proto_base);
Linus Torvalds1da177e2005-04-16 15:20:36 -070039
40/* Protects list of registered TC modules. It is pure SMP lock. */
41static DEFINE_RWLOCK(cls_mod_lock);
42
43/* Find classifier type by string name */
44
Jiri Pirkof34e8bf2018-07-23 09:23:04 +020045static const struct tcf_proto_ops *__tcf_proto_lookup_ops(const char *kind)
Linus Torvalds1da177e2005-04-16 15:20:36 -070046{
Eric Dumazetdcd76082013-12-20 10:04:18 -080047 const struct tcf_proto_ops *t, *res = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -070048
49 if (kind) {
50 read_lock(&cls_mod_lock);
WANG Cong36272872013-12-15 20:15:11 -080051 list_for_each_entry(t, &tcf_proto_base, head) {
Jiri Pirko33a48922017-02-09 14:38:57 +010052 if (strcmp(kind, t->kind) == 0) {
Eric Dumazetdcd76082013-12-20 10:04:18 -080053 if (try_module_get(t->owner))
54 res = t;
Linus Torvalds1da177e2005-04-16 15:20:36 -070055 break;
56 }
57 }
58 read_unlock(&cls_mod_lock);
59 }
Eric Dumazetdcd76082013-12-20 10:04:18 -080060 return res;
Linus Torvalds1da177e2005-04-16 15:20:36 -070061}
62
Jiri Pirkof34e8bf2018-07-23 09:23:04 +020063static const struct tcf_proto_ops *
64tcf_proto_lookup_ops(const char *kind, struct netlink_ext_ack *extack)
65{
66 const struct tcf_proto_ops *ops;
67
68 ops = __tcf_proto_lookup_ops(kind);
69 if (ops)
70 return ops;
71#ifdef CONFIG_MODULES
72 rtnl_unlock();
73 request_module("cls_%s", kind);
74 rtnl_lock();
75 ops = __tcf_proto_lookup_ops(kind);
76 /* We dropped the RTNL semaphore in order to perform
77 * the module load. So, even if we succeeded in loading
78 * the module we have to replay the request. We indicate
79 * this using -EAGAIN.
80 */
81 if (ops) {
82 module_put(ops->owner);
83 return ERR_PTR(-EAGAIN);
84 }
85#endif
86 NL_SET_ERR_MSG(extack, "TC classifier not found");
87 return ERR_PTR(-ENOENT);
88}
89
Linus Torvalds1da177e2005-04-16 15:20:36 -070090/* Register(unregister) new classifier type */
91
92int register_tcf_proto_ops(struct tcf_proto_ops *ops)
93{
WANG Cong36272872013-12-15 20:15:11 -080094 struct tcf_proto_ops *t;
Linus Torvalds1da177e2005-04-16 15:20:36 -070095 int rc = -EEXIST;
96
97 write_lock(&cls_mod_lock);
WANG Cong36272872013-12-15 20:15:11 -080098 list_for_each_entry(t, &tcf_proto_base, head)
Linus Torvalds1da177e2005-04-16 15:20:36 -070099 if (!strcmp(ops->kind, t->kind))
100 goto out;
101
WANG Cong36272872013-12-15 20:15:11 -0800102 list_add_tail(&ops->head, &tcf_proto_base);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103 rc = 0;
104out:
105 write_unlock(&cls_mod_lock);
106 return rc;
107}
Stephen Hemmingeraa767bf2008-01-21 02:26:41 -0800108EXPORT_SYMBOL(register_tcf_proto_ops);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109
Cong Wang7aa00452017-10-26 18:24:28 -0700110static struct workqueue_struct *tc_filter_wq;
111
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112int unregister_tcf_proto_ops(struct tcf_proto_ops *ops)
113{
WANG Cong36272872013-12-15 20:15:11 -0800114 struct tcf_proto_ops *t;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115 int rc = -ENOENT;
116
Daniel Borkmannc78e1742015-05-20 17:13:33 +0200117 /* Wait for outstanding call_rcu()s, if any, from a
118 * tcf_proto_ops's destroy() handler.
119 */
120 rcu_barrier();
Cong Wang7aa00452017-10-26 18:24:28 -0700121 flush_workqueue(tc_filter_wq);
Daniel Borkmannc78e1742015-05-20 17:13:33 +0200122
Linus Torvalds1da177e2005-04-16 15:20:36 -0700123 write_lock(&cls_mod_lock);
Eric Dumazetdcd76082013-12-20 10:04:18 -0800124 list_for_each_entry(t, &tcf_proto_base, head) {
125 if (t == ops) {
126 list_del(&t->head);
127 rc = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700128 break;
Eric Dumazetdcd76082013-12-20 10:04:18 -0800129 }
130 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700131 write_unlock(&cls_mod_lock);
132 return rc;
133}
Stephen Hemmingeraa767bf2008-01-21 02:26:41 -0800134EXPORT_SYMBOL(unregister_tcf_proto_ops);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700135
Cong Wangaaa908f2018-05-23 15:26:53 -0700136bool tcf_queue_work(struct rcu_work *rwork, work_func_t func)
Cong Wang7aa00452017-10-26 18:24:28 -0700137{
Cong Wangaaa908f2018-05-23 15:26:53 -0700138 INIT_RCU_WORK(rwork, func);
139 return queue_rcu_work(tc_filter_wq, rwork);
Cong Wang7aa00452017-10-26 18:24:28 -0700140}
141EXPORT_SYMBOL(tcf_queue_work);
142
Linus Torvalds1da177e2005-04-16 15:20:36 -0700143/* Select new prio value from the range, managed by kernel. */
144
Stephen Hemmingeraa767bf2008-01-21 02:26:41 -0800145static inline u32 tcf_auto_prio(struct tcf_proto *tp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146{
Stephen Hemmingeraa767bf2008-01-21 02:26:41 -0800147 u32 first = TC_H_MAKE(0xC0000000U, 0U);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700148
149 if (tp)
Eric Dumazetcc7ec452011-01-19 19:26:56 +0000150 first = tp->prio - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700151
Jiri Pirko79619732017-05-17 11:07:58 +0200152 return TC_H_MAJ(first);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700153}
154
Jiri Pirko33a48922017-02-09 14:38:57 +0100155static struct tcf_proto *tcf_proto_create(const char *kind, u32 protocol,
Alexander Aringc35a4ac2018-01-18 11:20:50 -0500156 u32 prio, struct tcf_chain *chain,
157 struct netlink_ext_ack *extack)
Jiri Pirko33a48922017-02-09 14:38:57 +0100158{
159 struct tcf_proto *tp;
160 int err;
161
162 tp = kzalloc(sizeof(*tp), GFP_KERNEL);
163 if (!tp)
164 return ERR_PTR(-ENOBUFS);
165
Jiri Pirkof34e8bf2018-07-23 09:23:04 +0200166 tp->ops = tcf_proto_lookup_ops(kind, extack);
167 if (IS_ERR(tp->ops)) {
168 err = PTR_ERR(tp->ops);
Jiri Pirkod68d75f2018-05-11 17:45:32 +0200169 goto errout;
Jiri Pirko33a48922017-02-09 14:38:57 +0100170 }
171 tp->classify = tp->ops->classify;
172 tp->protocol = protocol;
173 tp->prio = prio;
Jiri Pirko5bc17012017-05-17 11:08:01 +0200174 tp->chain = chain;
Jiri Pirko33a48922017-02-09 14:38:57 +0100175
176 err = tp->ops->init(tp);
177 if (err) {
178 module_put(tp->ops->owner);
179 goto errout;
180 }
181 return tp;
182
183errout:
184 kfree(tp);
185 return ERR_PTR(err);
186}
187
Jakub Kicinski715df5e2018-01-24 12:54:13 -0800188static void tcf_proto_destroy(struct tcf_proto *tp,
189 struct netlink_ext_ack *extack)
Jiri Pirkocf1facd2017-02-09 14:38:56 +0100190{
Jakub Kicinski715df5e2018-01-24 12:54:13 -0800191 tp->ops->destroy(tp, extack);
WANG Cong763dbf62017-04-19 14:21:21 -0700192 module_put(tp->ops->owner);
193 kfree_rcu(tp, rcu);
Jiri Pirkocf1facd2017-02-09 14:38:56 +0100194}
195
Jiri Pirkoa9b19442018-01-17 11:46:45 +0100196struct tcf_filter_chain_list_item {
197 struct list_head list;
198 tcf_chain_head_change_t *chain_head_change;
199 void *chain_head_change_priv;
200};
201
Jiri Pirko5bc17012017-05-17 11:08:01 +0200202static struct tcf_chain *tcf_chain_create(struct tcf_block *block,
203 u32 chain_index)
Jiri Pirko2190d1d2017-05-17 11:07:59 +0200204{
Jiri Pirko5bc17012017-05-17 11:08:01 +0200205 struct tcf_chain *chain;
206
207 chain = kzalloc(sizeof(*chain), GFP_KERNEL);
208 if (!chain)
209 return NULL;
210 list_add_tail(&chain->list, &block->chain_list);
211 chain->block = block;
212 chain->index = chain_index;
Cong Wange2ef7542017-09-11 16:33:31 -0700213 chain->refcnt = 1;
Jiri Pirkof71e0ca42018-07-23 09:23:05 +0200214 if (!chain->index)
215 block->chain0.chain = chain;
Jiri Pirko5bc17012017-05-17 11:08:01 +0200216 return chain;
Jiri Pirko2190d1d2017-05-17 11:07:59 +0200217}
218
Jiri Pirkoa9b19442018-01-17 11:46:45 +0100219static void tcf_chain_head_change_item(struct tcf_filter_chain_list_item *item,
220 struct tcf_proto *tp_head)
221{
222 if (item->chain_head_change)
223 item->chain_head_change(tp_head, item->chain_head_change_priv);
224}
Jiri Pirkof71e0ca42018-07-23 09:23:05 +0200225
226static void tcf_chain0_head_change(struct tcf_chain *chain,
227 struct tcf_proto *tp_head)
Jiri Pirkoc7eb7d72017-11-03 11:46:24 +0100228{
Jiri Pirkoa9b19442018-01-17 11:46:45 +0100229 struct tcf_filter_chain_list_item *item;
Jiri Pirkof71e0ca42018-07-23 09:23:05 +0200230 struct tcf_block *block = chain->block;
Jiri Pirkoa9b19442018-01-17 11:46:45 +0100231
Jiri Pirkof71e0ca42018-07-23 09:23:05 +0200232 if (chain->index)
233 return;
234 list_for_each_entry(item, &block->chain0.filter_chain_list, list)
Jiri Pirkoa9b19442018-01-17 11:46:45 +0100235 tcf_chain_head_change_item(item, tp_head);
Jiri Pirkoc7eb7d72017-11-03 11:46:24 +0100236}
237
Jiri Pirkof93e1cd2017-05-20 15:01:32 +0200238static void tcf_chain_destroy(struct tcf_chain *chain)
239{
Cong Wangefbf7892017-12-04 10:48:18 -0800240 struct tcf_block *block = chain->block;
241
Cong Wange2ef7542017-09-11 16:33:31 -0700242 list_del(&chain->list);
Jiri Pirkof71e0ca42018-07-23 09:23:05 +0200243 if (!chain->index)
244 block->chain0.chain = NULL;
Cong Wange2ef7542017-09-11 16:33:31 -0700245 kfree(chain);
Vlad Buslovcfebd7e2018-09-24 19:22:54 +0300246 if (list_empty(&block->chain_list) && !refcount_read(&block->refcnt))
Vlad Buslov0607e432018-09-24 19:22:57 +0300247 kfree_rcu(block, rcu);
Cong Wange2ef7542017-09-11 16:33:31 -0700248}
Jiri Pirko744a4cf2017-08-22 22:46:49 +0200249
Cong Wange2ef7542017-09-11 16:33:31 -0700250static void tcf_chain_hold(struct tcf_chain *chain)
251{
252 ++chain->refcnt;
Jiri Pirko2190d1d2017-05-17 11:07:59 +0200253}
254
Jiri Pirko3d32f4c2018-08-01 12:36:55 +0200255static bool tcf_chain_held_by_acts_only(struct tcf_chain *chain)
Jiri Pirko1f3ed382018-07-27 09:45:05 +0200256{
257 /* In case all the references are action references, this
Jiri Pirko3d32f4c2018-08-01 12:36:55 +0200258 * chain should not be shown to the user.
Jiri Pirko1f3ed382018-07-27 09:45:05 +0200259 */
260 return chain->refcnt == chain->action_refcnt;
261}
262
Jiri Pirko32a4f5e2018-07-23 09:23:06 +0200263static struct tcf_chain *tcf_chain_lookup(struct tcf_block *block,
264 u32 chain_index)
Jiri Pirko5bc17012017-05-17 11:08:01 +0200265{
266 struct tcf_chain *chain;
267
268 list_for_each_entry(chain, &block->chain_list, list) {
Jiri Pirko32a4f5e2018-07-23 09:23:06 +0200269 if (chain->index == chain_index)
Cong Wange2ef7542017-09-11 16:33:31 -0700270 return chain;
Jiri Pirko32a4f5e2018-07-23 09:23:06 +0200271 }
272 return NULL;
273}
274
275static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb,
276 u32 seq, u16 flags, int event, bool unicast);
277
Jiri Pirko53681402018-08-01 12:36:56 +0200278static struct tcf_chain *__tcf_chain_get(struct tcf_block *block,
279 u32 chain_index, bool create,
280 bool by_act)
Jiri Pirko32a4f5e2018-07-23 09:23:06 +0200281{
282 struct tcf_chain *chain = tcf_chain_lookup(block, chain_index);
283
284 if (chain) {
285 tcf_chain_hold(chain);
Jiri Pirko53681402018-08-01 12:36:56 +0200286 } else {
287 if (!create)
288 return NULL;
289 chain = tcf_chain_create(block, chain_index);
290 if (!chain)
291 return NULL;
Jiri Pirko5bc17012017-05-17 11:08:01 +0200292 }
Jiri Pirko80532382017-09-06 13:14:19 +0200293
Jiri Pirko53681402018-08-01 12:36:56 +0200294 if (by_act)
295 ++chain->action_refcnt;
296
297 /* Send notification only in case we got the first
298 * non-action reference. Until then, the chain acts only as
299 * a placeholder for actions pointing to it and user ought
300 * not know about them.
301 */
302 if (chain->refcnt - chain->action_refcnt == 1 && !by_act)
303 tc_chain_notify(chain, NULL, 0, NLM_F_CREATE | NLM_F_EXCL,
304 RTM_NEWCHAIN, false);
305
Jiri Pirko32a4f5e2018-07-23 09:23:06 +0200306 return chain;
Jiri Pirko5bc17012017-05-17 11:08:01 +0200307}
Jiri Pirko53681402018-08-01 12:36:56 +0200308
Jiri Pirko290b1c82018-08-01 12:36:57 +0200309static struct tcf_chain *tcf_chain_get(struct tcf_block *block, u32 chain_index,
310 bool create)
Jiri Pirko53681402018-08-01 12:36:56 +0200311{
312 return __tcf_chain_get(block, chain_index, create, false);
313}
Jiri Pirko5bc17012017-05-17 11:08:01 +0200314
Jiri Pirko1f3ed382018-07-27 09:45:05 +0200315struct tcf_chain *tcf_chain_get_by_act(struct tcf_block *block, u32 chain_index)
316{
Jiri Pirko53681402018-08-01 12:36:56 +0200317 return __tcf_chain_get(block, chain_index, true, true);
Jiri Pirko1f3ed382018-07-27 09:45:05 +0200318}
319EXPORT_SYMBOL(tcf_chain_get_by_act);
320
Jiri Pirko9f407f12018-07-23 09:23:07 +0200321static void tc_chain_tmplt_del(struct tcf_chain *chain);
322
Jiri Pirko53681402018-08-01 12:36:56 +0200323static void __tcf_chain_put(struct tcf_chain *chain, bool by_act)
Jiri Pirko5bc17012017-05-17 11:08:01 +0200324{
Jiri Pirko53681402018-08-01 12:36:56 +0200325 if (by_act)
326 chain->action_refcnt--;
327 chain->refcnt--;
328
329 /* The last dropped non-action reference will trigger notification. */
330 if (chain->refcnt - chain->action_refcnt == 0 && !by_act)
Jiri Pirko32a4f5e2018-07-23 09:23:06 +0200331 tc_chain_notify(chain, NULL, 0, 0, RTM_DELCHAIN, false);
Jiri Pirko53681402018-08-01 12:36:56 +0200332
333 if (chain->refcnt == 0) {
Jiri Pirko9f407f12018-07-23 09:23:07 +0200334 tc_chain_tmplt_del(chain);
Jiri Pirko5bc17012017-05-17 11:08:01 +0200335 tcf_chain_destroy(chain);
Jiri Pirko32a4f5e2018-07-23 09:23:06 +0200336 }
Jiri Pirko5bc17012017-05-17 11:08:01 +0200337}
Jiri Pirko53681402018-08-01 12:36:56 +0200338
Jiri Pirko290b1c82018-08-01 12:36:57 +0200339static void tcf_chain_put(struct tcf_chain *chain)
Jiri Pirko53681402018-08-01 12:36:56 +0200340{
341 __tcf_chain_put(chain, false);
342}
Jiri Pirko5bc17012017-05-17 11:08:01 +0200343
Jiri Pirko1f3ed382018-07-27 09:45:05 +0200344void tcf_chain_put_by_act(struct tcf_chain *chain)
345{
Jiri Pirko53681402018-08-01 12:36:56 +0200346 __tcf_chain_put(chain, true);
Jiri Pirko1f3ed382018-07-27 09:45:05 +0200347}
348EXPORT_SYMBOL(tcf_chain_put_by_act);
349
Jiri Pirko32a4f5e2018-07-23 09:23:06 +0200350static void tcf_chain_put_explicitly_created(struct tcf_chain *chain)
351{
352 if (chain->explicitly_created)
353 tcf_chain_put(chain);
354}
355
Jiri Pirko290b1c82018-08-01 12:36:57 +0200356static void tcf_chain_flush(struct tcf_chain *chain)
357{
358 struct tcf_proto *tp = rtnl_dereference(chain->filter_chain);
359
360 tcf_chain0_head_change(chain, NULL);
361 while (tp) {
362 RCU_INIT_POINTER(chain->filter_chain, tp->next);
363 tcf_proto_destroy(tp, NULL);
364 tp = rtnl_dereference(chain->filter_chain);
365 tcf_chain_put(chain);
366 }
367}
368
John Hurley7f76fa32018-11-09 21:21:26 -0800369static struct tcf_block *tc_dev_ingress_block(struct net_device *dev)
370{
371 const struct Qdisc_class_ops *cops;
372 struct Qdisc *qdisc;
373
374 if (!dev_ingress_queue(dev))
375 return NULL;
376
377 qdisc = dev_ingress_queue(dev)->qdisc_sleeping;
378 if (!qdisc)
379 return NULL;
380
381 cops = qdisc->ops->cl_ops;
382 if (!cops)
383 return NULL;
384
385 if (!cops->tcf_block)
386 return NULL;
387
388 return cops->tcf_block(qdisc, TC_H_MIN_INGRESS, NULL);
389}
390
391static struct rhashtable indr_setup_block_ht;
392
393struct tc_indr_block_dev {
394 struct rhash_head ht_node;
395 struct net_device *dev;
396 unsigned int refcnt;
397 struct list_head cb_list;
398 struct tcf_block *block;
399};
400
401struct tc_indr_block_cb {
402 struct list_head list;
403 void *cb_priv;
404 tc_indr_block_bind_cb_t *cb;
405 void *cb_ident;
406};
407
408static const struct rhashtable_params tc_indr_setup_block_ht_params = {
409 .key_offset = offsetof(struct tc_indr_block_dev, dev),
410 .head_offset = offsetof(struct tc_indr_block_dev, ht_node),
411 .key_len = sizeof(struct net_device *),
412};
413
414static struct tc_indr_block_dev *
415tc_indr_block_dev_lookup(struct net_device *dev)
416{
417 return rhashtable_lookup_fast(&indr_setup_block_ht, &dev,
418 tc_indr_setup_block_ht_params);
419}
420
421static struct tc_indr_block_dev *tc_indr_block_dev_get(struct net_device *dev)
422{
423 struct tc_indr_block_dev *indr_dev;
424
425 indr_dev = tc_indr_block_dev_lookup(dev);
426 if (indr_dev)
427 goto inc_ref;
428
429 indr_dev = kzalloc(sizeof(*indr_dev), GFP_KERNEL);
430 if (!indr_dev)
431 return NULL;
432
433 INIT_LIST_HEAD(&indr_dev->cb_list);
434 indr_dev->dev = dev;
435 indr_dev->block = tc_dev_ingress_block(dev);
436 if (rhashtable_insert_fast(&indr_setup_block_ht, &indr_dev->ht_node,
437 tc_indr_setup_block_ht_params)) {
438 kfree(indr_dev);
439 return NULL;
440 }
441
442inc_ref:
443 indr_dev->refcnt++;
444 return indr_dev;
445}
446
447static void tc_indr_block_dev_put(struct tc_indr_block_dev *indr_dev)
448{
449 if (--indr_dev->refcnt)
450 return;
451
452 rhashtable_remove_fast(&indr_setup_block_ht, &indr_dev->ht_node,
453 tc_indr_setup_block_ht_params);
454 kfree(indr_dev);
455}
456
457static struct tc_indr_block_cb *
458tc_indr_block_cb_lookup(struct tc_indr_block_dev *indr_dev,
459 tc_indr_block_bind_cb_t *cb, void *cb_ident)
460{
461 struct tc_indr_block_cb *indr_block_cb;
462
463 list_for_each_entry(indr_block_cb, &indr_dev->cb_list, list)
464 if (indr_block_cb->cb == cb &&
465 indr_block_cb->cb_ident == cb_ident)
466 return indr_block_cb;
467 return NULL;
468}
469
470static struct tc_indr_block_cb *
471tc_indr_block_cb_add(struct tc_indr_block_dev *indr_dev, void *cb_priv,
472 tc_indr_block_bind_cb_t *cb, void *cb_ident)
473{
474 struct tc_indr_block_cb *indr_block_cb;
475
476 indr_block_cb = tc_indr_block_cb_lookup(indr_dev, cb, cb_ident);
477 if (indr_block_cb)
478 return ERR_PTR(-EEXIST);
479
480 indr_block_cb = kzalloc(sizeof(*indr_block_cb), GFP_KERNEL);
481 if (!indr_block_cb)
482 return ERR_PTR(-ENOMEM);
483
484 indr_block_cb->cb_priv = cb_priv;
485 indr_block_cb->cb = cb;
486 indr_block_cb->cb_ident = cb_ident;
487 list_add(&indr_block_cb->list, &indr_dev->cb_list);
488
489 return indr_block_cb;
490}
491
492static void tc_indr_block_cb_del(struct tc_indr_block_cb *indr_block_cb)
493{
494 list_del(&indr_block_cb->list);
495 kfree(indr_block_cb);
496}
497
498static void tc_indr_block_ing_cmd(struct tc_indr_block_dev *indr_dev,
499 struct tc_indr_block_cb *indr_block_cb,
500 enum tc_block_command command)
501{
502 struct tc_block_offload bo = {
503 .command = command,
504 .binder_type = TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS,
505 .block = indr_dev->block,
506 };
507
508 if (!indr_dev->block)
509 return;
510
511 indr_block_cb->cb(indr_dev->dev, indr_block_cb->cb_priv, TC_SETUP_BLOCK,
512 &bo);
513}
514
515int __tc_indr_block_cb_register(struct net_device *dev, void *cb_priv,
516 tc_indr_block_bind_cb_t *cb, void *cb_ident)
517{
518 struct tc_indr_block_cb *indr_block_cb;
519 struct tc_indr_block_dev *indr_dev;
520 int err;
521
522 indr_dev = tc_indr_block_dev_get(dev);
523 if (!indr_dev)
524 return -ENOMEM;
525
526 indr_block_cb = tc_indr_block_cb_add(indr_dev, cb_priv, cb, cb_ident);
527 err = PTR_ERR_OR_ZERO(indr_block_cb);
528 if (err)
529 goto err_dev_put;
530
531 tc_indr_block_ing_cmd(indr_dev, indr_block_cb, TC_BLOCK_BIND);
532 return 0;
533
534err_dev_put:
535 tc_indr_block_dev_put(indr_dev);
536 return err;
537}
538EXPORT_SYMBOL_GPL(__tc_indr_block_cb_register);
539
540int tc_indr_block_cb_register(struct net_device *dev, void *cb_priv,
541 tc_indr_block_bind_cb_t *cb, void *cb_ident)
542{
543 int err;
544
545 rtnl_lock();
546 err = __tc_indr_block_cb_register(dev, cb_priv, cb, cb_ident);
547 rtnl_unlock();
548
549 return err;
550}
551EXPORT_SYMBOL_GPL(tc_indr_block_cb_register);
552
553void __tc_indr_block_cb_unregister(struct net_device *dev,
554 tc_indr_block_bind_cb_t *cb, void *cb_ident)
555{
556 struct tc_indr_block_cb *indr_block_cb;
557 struct tc_indr_block_dev *indr_dev;
558
559 indr_dev = tc_indr_block_dev_lookup(dev);
560 if (!indr_dev)
561 return;
562
563 indr_block_cb = tc_indr_block_cb_lookup(indr_dev, cb, cb_ident);
564 if (!indr_block_cb)
565 return;
566
567 /* Send unbind message if required to free any block cbs. */
568 tc_indr_block_ing_cmd(indr_dev, indr_block_cb, TC_BLOCK_UNBIND);
569 tc_indr_block_cb_del(indr_block_cb);
570 tc_indr_block_dev_put(indr_dev);
571}
572EXPORT_SYMBOL_GPL(__tc_indr_block_cb_unregister);
573
574void tc_indr_block_cb_unregister(struct net_device *dev,
575 tc_indr_block_bind_cb_t *cb, void *cb_ident)
576{
577 rtnl_lock();
578 __tc_indr_block_cb_unregister(dev, cb, cb_ident);
579 rtnl_unlock();
580}
581EXPORT_SYMBOL_GPL(tc_indr_block_cb_unregister);
582
583static void tc_indr_block_call(struct tcf_block *block, struct net_device *dev,
584 struct tcf_block_ext_info *ei,
585 enum tc_block_command command,
586 struct netlink_ext_ack *extack)
587{
588 struct tc_indr_block_cb *indr_block_cb;
589 struct tc_indr_block_dev *indr_dev;
590 struct tc_block_offload bo = {
591 .command = command,
592 .binder_type = ei->binder_type,
593 .block = block,
594 .extack = extack,
595 };
596
597 indr_dev = tc_indr_block_dev_lookup(dev);
598 if (!indr_dev)
599 return;
600
601 indr_dev->block = command == TC_BLOCK_BIND ? block : NULL;
602
603 list_for_each_entry(indr_block_cb, &indr_dev->cb_list, list)
604 indr_block_cb->cb(dev, indr_block_cb->cb_priv, TC_SETUP_BLOCK,
605 &bo);
606}
607
Jiri Pirkocaa72602018-01-17 11:46:50 +0100608static bool tcf_block_offload_in_use(struct tcf_block *block)
Jiri Pirko8c4083b2017-10-19 15:50:29 +0200609{
Jiri Pirkocaa72602018-01-17 11:46:50 +0100610 return block->offloadcnt;
611}
612
613static int tcf_block_offload_cmd(struct tcf_block *block,
614 struct net_device *dev,
615 struct tcf_block_ext_info *ei,
John Hurley60513bd2018-06-25 14:30:04 -0700616 enum tc_block_command command,
617 struct netlink_ext_ack *extack)
Jiri Pirkocaa72602018-01-17 11:46:50 +0100618{
Jiri Pirko8c4083b2017-10-19 15:50:29 +0200619 struct tc_block_offload bo = {};
620
Jiri Pirko8c4083b2017-10-19 15:50:29 +0200621 bo.command = command;
622 bo.binder_type = ei->binder_type;
623 bo.block = block;
John Hurley60513bd2018-06-25 14:30:04 -0700624 bo.extack = extack;
Jiri Pirkocaa72602018-01-17 11:46:50 +0100625 return dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo);
Jiri Pirko8c4083b2017-10-19 15:50:29 +0200626}
627
Jiri Pirkocaa72602018-01-17 11:46:50 +0100628static int tcf_block_offload_bind(struct tcf_block *block, struct Qdisc *q,
John Hurley60513bd2018-06-25 14:30:04 -0700629 struct tcf_block_ext_info *ei,
630 struct netlink_ext_ack *extack)
Jiri Pirko8c4083b2017-10-19 15:50:29 +0200631{
Jiri Pirkocaa72602018-01-17 11:46:50 +0100632 struct net_device *dev = q->dev_queue->dev;
633 int err;
634
635 if (!dev->netdev_ops->ndo_setup_tc)
636 goto no_offload_dev_inc;
637
638 /* If tc offload feature is disabled and the block we try to bind
639 * to already has some offloaded filters, forbid to bind.
640 */
John Hurley60513bd2018-06-25 14:30:04 -0700641 if (!tc_can_offload(dev) && tcf_block_offload_in_use(block)) {
642 NL_SET_ERR_MSG(extack, "Bind to offloaded block failed as dev has offload disabled");
Jiri Pirkocaa72602018-01-17 11:46:50 +0100643 return -EOPNOTSUPP;
John Hurley60513bd2018-06-25 14:30:04 -0700644 }
Jiri Pirkocaa72602018-01-17 11:46:50 +0100645
John Hurley60513bd2018-06-25 14:30:04 -0700646 err = tcf_block_offload_cmd(block, dev, ei, TC_BLOCK_BIND, extack);
Jiri Pirkocaa72602018-01-17 11:46:50 +0100647 if (err == -EOPNOTSUPP)
648 goto no_offload_dev_inc;
John Hurley7f76fa32018-11-09 21:21:26 -0800649 if (err)
650 return err;
651
652 tc_indr_block_call(block, dev, ei, TC_BLOCK_BIND, extack);
653 return 0;
Jiri Pirkocaa72602018-01-17 11:46:50 +0100654
655no_offload_dev_inc:
656 if (tcf_block_offload_in_use(block))
657 return -EOPNOTSUPP;
658 block->nooffloaddevcnt++;
John Hurley7f76fa32018-11-09 21:21:26 -0800659 tc_indr_block_call(block, dev, ei, TC_BLOCK_BIND, extack);
Jiri Pirkocaa72602018-01-17 11:46:50 +0100660 return 0;
Jiri Pirko8c4083b2017-10-19 15:50:29 +0200661}
662
663static void tcf_block_offload_unbind(struct tcf_block *block, struct Qdisc *q,
664 struct tcf_block_ext_info *ei)
665{
Jiri Pirkocaa72602018-01-17 11:46:50 +0100666 struct net_device *dev = q->dev_queue->dev;
667 int err;
668
John Hurley7f76fa32018-11-09 21:21:26 -0800669 tc_indr_block_call(block, dev, ei, TC_BLOCK_UNBIND, NULL);
670
Jiri Pirkocaa72602018-01-17 11:46:50 +0100671 if (!dev->netdev_ops->ndo_setup_tc)
672 goto no_offload_dev_dec;
John Hurley60513bd2018-06-25 14:30:04 -0700673 err = tcf_block_offload_cmd(block, dev, ei, TC_BLOCK_UNBIND, NULL);
Jiri Pirkocaa72602018-01-17 11:46:50 +0100674 if (err == -EOPNOTSUPP)
675 goto no_offload_dev_dec;
676 return;
677
678no_offload_dev_dec:
679 WARN_ON(block->nooffloaddevcnt-- == 0);
Jiri Pirko8c4083b2017-10-19 15:50:29 +0200680}
681
Jiri Pirkoa9b19442018-01-17 11:46:45 +0100682static int
Jiri Pirkof71e0ca42018-07-23 09:23:05 +0200683tcf_chain0_head_change_cb_add(struct tcf_block *block,
684 struct tcf_block_ext_info *ei,
685 struct netlink_ext_ack *extack)
Jiri Pirkoa9b19442018-01-17 11:46:45 +0100686{
Jiri Pirkof71e0ca42018-07-23 09:23:05 +0200687 struct tcf_chain *chain0 = block->chain0.chain;
Jiri Pirkoa9b19442018-01-17 11:46:45 +0100688 struct tcf_filter_chain_list_item *item;
689
690 item = kmalloc(sizeof(*item), GFP_KERNEL);
691 if (!item) {
692 NL_SET_ERR_MSG(extack, "Memory allocation for head change callback item failed");
693 return -ENOMEM;
694 }
695 item->chain_head_change = ei->chain_head_change;
696 item->chain_head_change_priv = ei->chain_head_change_priv;
Jiri Pirkof71e0ca42018-07-23 09:23:05 +0200697 if (chain0 && chain0->filter_chain)
698 tcf_chain_head_change_item(item, chain0->filter_chain);
699 list_add(&item->list, &block->chain0.filter_chain_list);
Jiri Pirkoa9b19442018-01-17 11:46:45 +0100700 return 0;
701}
702
703static void
Jiri Pirkof71e0ca42018-07-23 09:23:05 +0200704tcf_chain0_head_change_cb_del(struct tcf_block *block,
705 struct tcf_block_ext_info *ei)
Jiri Pirkoa9b19442018-01-17 11:46:45 +0100706{
Jiri Pirkof71e0ca42018-07-23 09:23:05 +0200707 struct tcf_chain *chain0 = block->chain0.chain;
Jiri Pirkoa9b19442018-01-17 11:46:45 +0100708 struct tcf_filter_chain_list_item *item;
709
Jiri Pirkof71e0ca42018-07-23 09:23:05 +0200710 list_for_each_entry(item, &block->chain0.filter_chain_list, list) {
Jiri Pirkoa9b19442018-01-17 11:46:45 +0100711 if ((!ei->chain_head_change && !ei->chain_head_change_priv) ||
712 (item->chain_head_change == ei->chain_head_change &&
713 item->chain_head_change_priv == ei->chain_head_change_priv)) {
Jiri Pirkof71e0ca42018-07-23 09:23:05 +0200714 if (chain0)
715 tcf_chain_head_change_item(item, NULL);
Jiri Pirkoa9b19442018-01-17 11:46:45 +0100716 list_del(&item->list);
717 kfree(item);
718 return;
719 }
720 }
721 WARN_ON(1);
722}
723
Jiri Pirko48617382018-01-17 11:46:46 +0100724struct tcf_net {
Vlad Buslovab281622018-09-24 19:22:56 +0300725 spinlock_t idr_lock; /* Protects idr */
Jiri Pirko48617382018-01-17 11:46:46 +0100726 struct idr idr;
727};
728
729static unsigned int tcf_net_id;
730
731static int tcf_block_insert(struct tcf_block *block, struct net *net,
Jiri Pirkobb047dd2018-02-13 12:00:16 +0100732 struct netlink_ext_ack *extack)
Jiri Pirkoa9b19442018-01-17 11:46:45 +0100733{
Jiri Pirko48617382018-01-17 11:46:46 +0100734 struct tcf_net *tn = net_generic(net, tcf_net_id);
Vlad Buslovab281622018-09-24 19:22:56 +0300735 int err;
Jiri Pirko48617382018-01-17 11:46:46 +0100736
Vlad Buslovab281622018-09-24 19:22:56 +0300737 idr_preload(GFP_KERNEL);
738 spin_lock(&tn->idr_lock);
739 err = idr_alloc_u32(&tn->idr, block, &block->index, block->index,
740 GFP_NOWAIT);
741 spin_unlock(&tn->idr_lock);
742 idr_preload_end();
743
744 return err;
Jiri Pirkoa9b19442018-01-17 11:46:45 +0100745}
746
Jiri Pirko48617382018-01-17 11:46:46 +0100747static void tcf_block_remove(struct tcf_block *block, struct net *net)
Jiri Pirko6529eab2017-05-17 11:07:55 +0200748{
Jiri Pirko48617382018-01-17 11:46:46 +0100749 struct tcf_net *tn = net_generic(net, tcf_net_id);
750
Vlad Buslovab281622018-09-24 19:22:56 +0300751 spin_lock(&tn->idr_lock);
Matthew Wilcox9c160942017-11-28 09:48:43 -0500752 idr_remove(&tn->idr, block->index);
Vlad Buslovab281622018-09-24 19:22:56 +0300753 spin_unlock(&tn->idr_lock);
Jiri Pirko48617382018-01-17 11:46:46 +0100754}
755
756static struct tcf_block *tcf_block_create(struct net *net, struct Qdisc *q,
Jiri Pirkobb047dd2018-02-13 12:00:16 +0100757 u32 block_index,
Jiri Pirko48617382018-01-17 11:46:46 +0100758 struct netlink_ext_ack *extack)
759{
760 struct tcf_block *block;
Jiri Pirko6529eab2017-05-17 11:07:55 +0200761
Jiri Pirko48617382018-01-17 11:46:46 +0100762 block = kzalloc(sizeof(*block), GFP_KERNEL);
Alexander Aring8d1a77f2017-12-20 12:35:19 -0500763 if (!block) {
764 NL_SET_ERR_MSG(extack, "Memory allocation for block failed");
Jiri Pirko48617382018-01-17 11:46:46 +0100765 return ERR_PTR(-ENOMEM);
Alexander Aring8d1a77f2017-12-20 12:35:19 -0500766 }
Jiri Pirko5bc17012017-05-17 11:08:01 +0200767 INIT_LIST_HEAD(&block->chain_list);
Jiri Pirkoacb67442017-10-19 15:50:31 +0200768 INIT_LIST_HEAD(&block->cb_list);
Jiri Pirkof36fe1c2018-01-17 11:46:48 +0100769 INIT_LIST_HEAD(&block->owner_list);
Jiri Pirkof71e0ca42018-07-23 09:23:05 +0200770 INIT_LIST_HEAD(&block->chain0.filter_chain_list);
Jiri Pirkoacb67442017-10-19 15:50:31 +0200771
Vlad Buslovcfebd7e2018-09-24 19:22:54 +0300772 refcount_set(&block->refcnt, 1);
Jiri Pirko48617382018-01-17 11:46:46 +0100773 block->net = net;
Jiri Pirkobb047dd2018-02-13 12:00:16 +0100774 block->index = block_index;
775
776 /* Don't store q pointer for blocks which are shared */
777 if (!tcf_block_shared(block))
778 block->q = q;
Jiri Pirko48617382018-01-17 11:46:46 +0100779 return block;
Jiri Pirko48617382018-01-17 11:46:46 +0100780}
781
782static struct tcf_block *tcf_block_lookup(struct net *net, u32 block_index)
783{
784 struct tcf_net *tn = net_generic(net, tcf_net_id);
785
Matthew Wilcox322d8842017-11-28 10:01:24 -0500786 return idr_find(&tn->idr, block_index);
Jiri Pirko48617382018-01-17 11:46:46 +0100787}
788
Vlad Buslov0607e432018-09-24 19:22:57 +0300789static struct tcf_block *tcf_block_refcnt_get(struct net *net, u32 block_index)
790{
791 struct tcf_block *block;
792
793 rcu_read_lock();
794 block = tcf_block_lookup(net, block_index);
795 if (block && !refcount_inc_not_zero(&block->refcnt))
796 block = NULL;
797 rcu_read_unlock();
798
799 return block;
800}
801
Vlad Buslovf0023432018-09-24 19:22:55 +0300802static void tcf_block_flush_all_chains(struct tcf_block *block)
803{
804 struct tcf_chain *chain;
805
806 /* Hold a refcnt for all chains, so that they don't disappear
807 * while we are iterating.
808 */
809 list_for_each_entry(chain, &block->chain_list, list)
810 tcf_chain_hold(chain);
811
812 list_for_each_entry(chain, &block->chain_list, list)
813 tcf_chain_flush(chain);
814}
815
816static void tcf_block_put_all_chains(struct tcf_block *block)
817{
818 struct tcf_chain *chain, *tmp;
819
820 /* At this point, all the chains should have refcnt >= 1. */
821 list_for_each_entry_safe(chain, tmp, &block->chain_list, list) {
822 tcf_chain_put_explicitly_created(chain);
823 tcf_chain_put(chain);
824 }
825}
826
Vlad Buslov0607e432018-09-24 19:22:57 +0300827static void __tcf_block_put(struct tcf_block *block, struct Qdisc *q,
828 struct tcf_block_ext_info *ei)
829{
830 if (refcount_dec_and_test(&block->refcnt)) {
831 /* Flushing/putting all chains will cause the block to be
832 * deallocated when last chain is freed. However, if chain_list
833 * is empty, block has to be manually deallocated. After block
834 * reference counter reached 0, it is no longer possible to
835 * increment it or add new chains to block.
836 */
837 bool free_block = list_empty(&block->chain_list);
838
839 if (tcf_block_shared(block))
840 tcf_block_remove(block, block->net);
841 if (!free_block)
842 tcf_block_flush_all_chains(block);
843
844 if (q)
845 tcf_block_offload_unbind(block, q, ei);
846
847 if (free_block)
848 kfree_rcu(block, rcu);
849 else
850 tcf_block_put_all_chains(block);
851 } else if (q) {
852 tcf_block_offload_unbind(block, q, ei);
853 }
854}
855
856static void tcf_block_refcnt_put(struct tcf_block *block)
857{
858 __tcf_block_put(block, NULL, NULL);
859}
860
Vlad Buslovc431f892018-05-31 09:52:53 +0300861/* Find tcf block.
862 * Set q, parent, cl when appropriate.
863 */
864
865static struct tcf_block *tcf_block_find(struct net *net, struct Qdisc **q,
866 u32 *parent, unsigned long *cl,
867 int ifindex, u32 block_index,
868 struct netlink_ext_ack *extack)
869{
870 struct tcf_block *block;
Vlad Buslove368fdb2018-09-24 19:22:53 +0300871 int err = 0;
Vlad Buslovc431f892018-05-31 09:52:53 +0300872
873 if (ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
Vlad Buslov787ce6d2018-09-24 19:22:58 +0300874 block = tcf_block_refcnt_get(net, block_index);
Vlad Buslovc431f892018-05-31 09:52:53 +0300875 if (!block) {
876 NL_SET_ERR_MSG(extack, "Block of given index was not found");
877 return ERR_PTR(-EINVAL);
878 }
879 } else {
880 const struct Qdisc_class_ops *cops;
881 struct net_device *dev;
882
Vlad Buslove368fdb2018-09-24 19:22:53 +0300883 rcu_read_lock();
884
Vlad Buslovc431f892018-05-31 09:52:53 +0300885 /* Find link */
Vlad Buslove368fdb2018-09-24 19:22:53 +0300886 dev = dev_get_by_index_rcu(net, ifindex);
887 if (!dev) {
888 rcu_read_unlock();
Vlad Buslovc431f892018-05-31 09:52:53 +0300889 return ERR_PTR(-ENODEV);
Vlad Buslove368fdb2018-09-24 19:22:53 +0300890 }
Vlad Buslovc431f892018-05-31 09:52:53 +0300891
892 /* Find qdisc */
893 if (!*parent) {
894 *q = dev->qdisc;
895 *parent = (*q)->handle;
896 } else {
Vlad Buslove368fdb2018-09-24 19:22:53 +0300897 *q = qdisc_lookup_rcu(dev, TC_H_MAJ(*parent));
Vlad Buslovc431f892018-05-31 09:52:53 +0300898 if (!*q) {
899 NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists");
Vlad Buslove368fdb2018-09-24 19:22:53 +0300900 err = -EINVAL;
901 goto errout_rcu;
Vlad Buslovc431f892018-05-31 09:52:53 +0300902 }
903 }
904
Vlad Buslove368fdb2018-09-24 19:22:53 +0300905 *q = qdisc_refcount_inc_nz(*q);
906 if (!*q) {
907 NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists");
908 err = -EINVAL;
909 goto errout_rcu;
910 }
911
Vlad Buslovc431f892018-05-31 09:52:53 +0300912 /* Is it classful? */
913 cops = (*q)->ops->cl_ops;
914 if (!cops) {
915 NL_SET_ERR_MSG(extack, "Qdisc not classful");
Vlad Buslove368fdb2018-09-24 19:22:53 +0300916 err = -EINVAL;
917 goto errout_rcu;
Vlad Buslovc431f892018-05-31 09:52:53 +0300918 }
919
920 if (!cops->tcf_block) {
921 NL_SET_ERR_MSG(extack, "Class doesn't support blocks");
Vlad Buslove368fdb2018-09-24 19:22:53 +0300922 err = -EOPNOTSUPP;
923 goto errout_rcu;
Vlad Buslovc431f892018-05-31 09:52:53 +0300924 }
925
Vlad Buslove368fdb2018-09-24 19:22:53 +0300926 /* At this point we know that qdisc is not noop_qdisc,
927 * which means that qdisc holds a reference to net_device
928 * and we hold a reference to qdisc, so it is safe to release
929 * rcu read lock.
930 */
931 rcu_read_unlock();
932
Vlad Buslovc431f892018-05-31 09:52:53 +0300933 /* Do we search for filter, attached to class? */
934 if (TC_H_MIN(*parent)) {
935 *cl = cops->find(*q, *parent);
936 if (*cl == 0) {
937 NL_SET_ERR_MSG(extack, "Specified class doesn't exist");
Vlad Buslove368fdb2018-09-24 19:22:53 +0300938 err = -ENOENT;
939 goto errout_qdisc;
Vlad Buslovc431f892018-05-31 09:52:53 +0300940 }
941 }
942
943 /* And the last stroke */
944 block = cops->tcf_block(*q, *cl, extack);
Vlad Buslove368fdb2018-09-24 19:22:53 +0300945 if (!block) {
946 err = -EINVAL;
947 goto errout_qdisc;
948 }
Vlad Buslovc431f892018-05-31 09:52:53 +0300949 if (tcf_block_shared(block)) {
950 NL_SET_ERR_MSG(extack, "This filter block is shared. Please use the block index to manipulate the filters");
Vlad Buslove368fdb2018-09-24 19:22:53 +0300951 err = -EOPNOTSUPP;
952 goto errout_qdisc;
Vlad Buslovc431f892018-05-31 09:52:53 +0300953 }
Vlad Buslov787ce6d2018-09-24 19:22:58 +0300954
955 /* Always take reference to block in order to support execution
956 * of rules update path of cls API without rtnl lock. Caller
957 * must release block when it is finished using it. 'if' block
958 * of this conditional obtain reference to block by calling
959 * tcf_block_refcnt_get().
960 */
961 refcount_inc(&block->refcnt);
Vlad Buslovc431f892018-05-31 09:52:53 +0300962 }
963
964 return block;
Vlad Buslove368fdb2018-09-24 19:22:53 +0300965
966errout_rcu:
967 rcu_read_unlock();
968errout_qdisc:
Cong Wang460b3602018-09-27 13:42:19 -0700969 if (*q) {
Vlad Buslove368fdb2018-09-24 19:22:53 +0300970 qdisc_put(*q);
Cong Wang460b3602018-09-27 13:42:19 -0700971 *q = NULL;
972 }
Vlad Buslove368fdb2018-09-24 19:22:53 +0300973 return ERR_PTR(err);
974}
975
976static void tcf_block_release(struct Qdisc *q, struct tcf_block *block)
977{
Vlad Buslov787ce6d2018-09-24 19:22:58 +0300978 if (!IS_ERR_OR_NULL(block))
979 tcf_block_refcnt_put(block);
980
Vlad Buslove368fdb2018-09-24 19:22:53 +0300981 if (q)
982 qdisc_put(q);
Vlad Buslovc431f892018-05-31 09:52:53 +0300983}
984
Jiri Pirkof36fe1c2018-01-17 11:46:48 +0100985struct tcf_block_owner_item {
986 struct list_head list;
987 struct Qdisc *q;
988 enum tcf_block_binder_type binder_type;
989};
990
991static void
992tcf_block_owner_netif_keep_dst(struct tcf_block *block,
993 struct Qdisc *q,
994 enum tcf_block_binder_type binder_type)
995{
996 if (block->keep_dst &&
997 binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS &&
998 binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_EGRESS)
999 netif_keep_dst(qdisc_dev(q));
1000}
1001
1002void tcf_block_netif_keep_dst(struct tcf_block *block)
1003{
1004 struct tcf_block_owner_item *item;
1005
1006 block->keep_dst = true;
1007 list_for_each_entry(item, &block->owner_list, list)
1008 tcf_block_owner_netif_keep_dst(block, item->q,
1009 item->binder_type);
1010}
1011EXPORT_SYMBOL(tcf_block_netif_keep_dst);
1012
1013static int tcf_block_owner_add(struct tcf_block *block,
1014 struct Qdisc *q,
1015 enum tcf_block_binder_type binder_type)
1016{
1017 struct tcf_block_owner_item *item;
1018
1019 item = kmalloc(sizeof(*item), GFP_KERNEL);
1020 if (!item)
1021 return -ENOMEM;
1022 item->q = q;
1023 item->binder_type = binder_type;
1024 list_add(&item->list, &block->owner_list);
1025 return 0;
1026}
1027
1028static void tcf_block_owner_del(struct tcf_block *block,
1029 struct Qdisc *q,
1030 enum tcf_block_binder_type binder_type)
1031{
1032 struct tcf_block_owner_item *item;
1033
1034 list_for_each_entry(item, &block->owner_list, list) {
1035 if (item->q == q && item->binder_type == binder_type) {
1036 list_del(&item->list);
1037 kfree(item);
1038 return;
1039 }
1040 }
1041 WARN_ON(1);
1042}
1043
Jiri Pirko48617382018-01-17 11:46:46 +01001044int tcf_block_get_ext(struct tcf_block **p_block, struct Qdisc *q,
1045 struct tcf_block_ext_info *ei,
1046 struct netlink_ext_ack *extack)
1047{
1048 struct net *net = qdisc_net(q);
1049 struct tcf_block *block = NULL;
Jiri Pirko48617382018-01-17 11:46:46 +01001050 int err;
1051
Vlad Buslov787ce6d2018-09-24 19:22:58 +03001052 if (ei->block_index)
Jiri Pirko48617382018-01-17 11:46:46 +01001053 /* block_index not 0 means the shared block is requested */
Vlad Buslov787ce6d2018-09-24 19:22:58 +03001054 block = tcf_block_refcnt_get(net, ei->block_index);
Jiri Pirko48617382018-01-17 11:46:46 +01001055
1056 if (!block) {
Jiri Pirkobb047dd2018-02-13 12:00:16 +01001057 block = tcf_block_create(net, q, ei->block_index, extack);
Jiri Pirko48617382018-01-17 11:46:46 +01001058 if (IS_ERR(block))
1059 return PTR_ERR(block);
Jiri Pirkobb047dd2018-02-13 12:00:16 +01001060 if (tcf_block_shared(block)) {
1061 err = tcf_block_insert(block, net, extack);
Jiri Pirko48617382018-01-17 11:46:46 +01001062 if (err)
1063 goto err_block_insert;
1064 }
1065 }
1066
Jiri Pirkof36fe1c2018-01-17 11:46:48 +01001067 err = tcf_block_owner_add(block, q, ei->binder_type);
1068 if (err)
1069 goto err_block_owner_add;
1070
1071 tcf_block_owner_netif_keep_dst(block, q, ei->binder_type);
1072
Jiri Pirkof71e0ca42018-07-23 09:23:05 +02001073 err = tcf_chain0_head_change_cb_add(block, ei, extack);
Jiri Pirkoa9b19442018-01-17 11:46:45 +01001074 if (err)
Jiri Pirkof71e0ca42018-07-23 09:23:05 +02001075 goto err_chain0_head_change_cb_add;
Jiri Pirkocaa72602018-01-17 11:46:50 +01001076
John Hurley60513bd2018-06-25 14:30:04 -07001077 err = tcf_block_offload_bind(block, q, ei, extack);
Jiri Pirkocaa72602018-01-17 11:46:50 +01001078 if (err)
1079 goto err_block_offload_bind;
1080
Jiri Pirko6529eab2017-05-17 11:07:55 +02001081 *p_block = block;
1082 return 0;
Jiri Pirko2190d1d2017-05-17 11:07:59 +02001083
Jiri Pirkocaa72602018-01-17 11:46:50 +01001084err_block_offload_bind:
Jiri Pirkof71e0ca42018-07-23 09:23:05 +02001085 tcf_chain0_head_change_cb_del(block, ei);
1086err_chain0_head_change_cb_add:
Jiri Pirkof36fe1c2018-01-17 11:46:48 +01001087 tcf_block_owner_del(block, q, ei->binder_type);
1088err_block_owner_add:
Jiri Pirko48617382018-01-17 11:46:46 +01001089err_block_insert:
Vlad Buslov787ce6d2018-09-24 19:22:58 +03001090 tcf_block_refcnt_put(block);
Jiri Pirko2190d1d2017-05-17 11:07:59 +02001091 return err;
Jiri Pirko6529eab2017-05-17 11:07:55 +02001092}
Jiri Pirko8c4083b2017-10-19 15:50:29 +02001093EXPORT_SYMBOL(tcf_block_get_ext);
1094
Jiri Pirkoc7eb7d72017-11-03 11:46:24 +01001095static void tcf_chain_head_change_dflt(struct tcf_proto *tp_head, void *priv)
1096{
1097 struct tcf_proto __rcu **p_filter_chain = priv;
1098
1099 rcu_assign_pointer(*p_filter_chain, tp_head);
1100}
1101
Jiri Pirko8c4083b2017-10-19 15:50:29 +02001102int tcf_block_get(struct tcf_block **p_block,
Alexander Aring8d1a77f2017-12-20 12:35:19 -05001103 struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q,
1104 struct netlink_ext_ack *extack)
Jiri Pirko8c4083b2017-10-19 15:50:29 +02001105{
Jiri Pirkoc7eb7d72017-11-03 11:46:24 +01001106 struct tcf_block_ext_info ei = {
1107 .chain_head_change = tcf_chain_head_change_dflt,
1108 .chain_head_change_priv = p_filter_chain,
1109 };
Jiri Pirko8c4083b2017-10-19 15:50:29 +02001110
Jiri Pirkoc7eb7d72017-11-03 11:46:24 +01001111 WARN_ON(!p_filter_chain);
Alexander Aring8d1a77f2017-12-20 12:35:19 -05001112 return tcf_block_get_ext(p_block, q, &ei, extack);
Jiri Pirko8c4083b2017-10-19 15:50:29 +02001113}
Jiri Pirko6529eab2017-05-17 11:07:55 +02001114EXPORT_SYMBOL(tcf_block_get);
1115
Cong Wang7aa00452017-10-26 18:24:28 -07001116/* XXX: Standalone actions are not allowed to jump to any chain, and bound
Roman Kapla60b3f52017-11-24 12:27:58 +01001117 * actions should be all removed after flushing.
Cong Wang7aa00452017-10-26 18:24:28 -07001118 */
Jiri Pirkoc7eb7d72017-11-03 11:46:24 +01001119void tcf_block_put_ext(struct tcf_block *block, struct Qdisc *q,
David S. Millere1ea2f92017-10-30 14:10:01 +09001120 struct tcf_block_ext_info *ei)
Cong Wang7aa00452017-10-26 18:24:28 -07001121{
David S. Millerc30abd52017-12-16 22:11:55 -05001122 if (!block)
1123 return;
Jiri Pirkof71e0ca42018-07-23 09:23:05 +02001124 tcf_chain0_head_change_cb_del(block, ei);
Jiri Pirkof36fe1c2018-01-17 11:46:48 +01001125 tcf_block_owner_del(block, q, ei->binder_type);
Roman Kapla60b3f52017-11-24 12:27:58 +01001126
Vlad Buslov0607e432018-09-24 19:22:57 +03001127 __tcf_block_put(block, q, ei);
Jiri Pirko6529eab2017-05-17 11:07:55 +02001128}
Jiri Pirko8c4083b2017-10-19 15:50:29 +02001129EXPORT_SYMBOL(tcf_block_put_ext);
1130
1131void tcf_block_put(struct tcf_block *block)
1132{
1133 struct tcf_block_ext_info ei = {0, };
1134
Jiri Pirko4853f122017-12-21 13:13:59 +01001135 if (!block)
1136 return;
Jiri Pirkoc7eb7d72017-11-03 11:46:24 +01001137 tcf_block_put_ext(block, block->q, &ei);
Jiri Pirko8c4083b2017-10-19 15:50:29 +02001138}
David S. Millere1ea2f92017-10-30 14:10:01 +09001139
Jiri Pirko6529eab2017-05-17 11:07:55 +02001140EXPORT_SYMBOL(tcf_block_put);
Jiri Pirkocf1facd2017-02-09 14:38:56 +01001141
Jiri Pirkoacb67442017-10-19 15:50:31 +02001142struct tcf_block_cb {
1143 struct list_head list;
1144 tc_setup_cb_t *cb;
1145 void *cb_ident;
1146 void *cb_priv;
1147 unsigned int refcnt;
1148};
1149
1150void *tcf_block_cb_priv(struct tcf_block_cb *block_cb)
1151{
1152 return block_cb->cb_priv;
1153}
1154EXPORT_SYMBOL(tcf_block_cb_priv);
1155
1156struct tcf_block_cb *tcf_block_cb_lookup(struct tcf_block *block,
1157 tc_setup_cb_t *cb, void *cb_ident)
1158{ struct tcf_block_cb *block_cb;
1159
1160 list_for_each_entry(block_cb, &block->cb_list, list)
1161 if (block_cb->cb == cb && block_cb->cb_ident == cb_ident)
1162 return block_cb;
1163 return NULL;
1164}
1165EXPORT_SYMBOL(tcf_block_cb_lookup);
1166
1167void tcf_block_cb_incref(struct tcf_block_cb *block_cb)
1168{
1169 block_cb->refcnt++;
1170}
1171EXPORT_SYMBOL(tcf_block_cb_incref);
1172
1173unsigned int tcf_block_cb_decref(struct tcf_block_cb *block_cb)
1174{
1175 return --block_cb->refcnt;
1176}
1177EXPORT_SYMBOL(tcf_block_cb_decref);
1178
John Hurley32636742018-06-25 14:30:10 -07001179static int
1180tcf_block_playback_offloads(struct tcf_block *block, tc_setup_cb_t *cb,
1181 void *cb_priv, bool add, bool offload_in_use,
1182 struct netlink_ext_ack *extack)
1183{
1184 struct tcf_chain *chain;
1185 struct tcf_proto *tp;
1186 int err;
1187
1188 list_for_each_entry(chain, &block->chain_list, list) {
1189 for (tp = rtnl_dereference(chain->filter_chain); tp;
1190 tp = rtnl_dereference(tp->next)) {
1191 if (tp->ops->reoffload) {
1192 err = tp->ops->reoffload(tp, add, cb, cb_priv,
1193 extack);
1194 if (err && add)
1195 goto err_playback_remove;
1196 } else if (add && offload_in_use) {
1197 err = -EOPNOTSUPP;
1198 NL_SET_ERR_MSG(extack, "Filter HW offload failed - classifier without re-offloading support");
1199 goto err_playback_remove;
1200 }
1201 }
1202 }
1203
1204 return 0;
1205
1206err_playback_remove:
1207 tcf_block_playback_offloads(block, cb, cb_priv, false, offload_in_use,
1208 extack);
1209 return err;
1210}
1211
Jiri Pirkoacb67442017-10-19 15:50:31 +02001212struct tcf_block_cb *__tcf_block_cb_register(struct tcf_block *block,
1213 tc_setup_cb_t *cb, void *cb_ident,
John Hurley60513bd2018-06-25 14:30:04 -07001214 void *cb_priv,
1215 struct netlink_ext_ack *extack)
Jiri Pirkoacb67442017-10-19 15:50:31 +02001216{
1217 struct tcf_block_cb *block_cb;
John Hurley32636742018-06-25 14:30:10 -07001218 int err;
Jiri Pirkoacb67442017-10-19 15:50:31 +02001219
John Hurley32636742018-06-25 14:30:10 -07001220 /* Replay any already present rules */
1221 err = tcf_block_playback_offloads(block, cb, cb_priv, true,
1222 tcf_block_offload_in_use(block),
1223 extack);
1224 if (err)
1225 return ERR_PTR(err);
Jiri Pirkocaa72602018-01-17 11:46:50 +01001226
Jiri Pirkoacb67442017-10-19 15:50:31 +02001227 block_cb = kzalloc(sizeof(*block_cb), GFP_KERNEL);
1228 if (!block_cb)
Jiri Pirkocaa72602018-01-17 11:46:50 +01001229 return ERR_PTR(-ENOMEM);
Jiri Pirkoacb67442017-10-19 15:50:31 +02001230 block_cb->cb = cb;
1231 block_cb->cb_ident = cb_ident;
1232 block_cb->cb_priv = cb_priv;
1233 list_add(&block_cb->list, &block->cb_list);
1234 return block_cb;
1235}
1236EXPORT_SYMBOL(__tcf_block_cb_register);
1237
1238int tcf_block_cb_register(struct tcf_block *block,
1239 tc_setup_cb_t *cb, void *cb_ident,
John Hurley60513bd2018-06-25 14:30:04 -07001240 void *cb_priv, struct netlink_ext_ack *extack)
Jiri Pirkoacb67442017-10-19 15:50:31 +02001241{
1242 struct tcf_block_cb *block_cb;
1243
John Hurley60513bd2018-06-25 14:30:04 -07001244 block_cb = __tcf_block_cb_register(block, cb, cb_ident, cb_priv,
1245 extack);
Gustavo A. R. Silvabaa2d2b2018-07-18 23:14:17 -05001246 return PTR_ERR_OR_ZERO(block_cb);
Jiri Pirkoacb67442017-10-19 15:50:31 +02001247}
1248EXPORT_SYMBOL(tcf_block_cb_register);
1249
John Hurley32636742018-06-25 14:30:10 -07001250void __tcf_block_cb_unregister(struct tcf_block *block,
1251 struct tcf_block_cb *block_cb)
Jiri Pirkoacb67442017-10-19 15:50:31 +02001252{
John Hurley32636742018-06-25 14:30:10 -07001253 tcf_block_playback_offloads(block, block_cb->cb, block_cb->cb_priv,
1254 false, tcf_block_offload_in_use(block),
1255 NULL);
Jiri Pirkoacb67442017-10-19 15:50:31 +02001256 list_del(&block_cb->list);
1257 kfree(block_cb);
1258}
1259EXPORT_SYMBOL(__tcf_block_cb_unregister);
1260
1261void tcf_block_cb_unregister(struct tcf_block *block,
1262 tc_setup_cb_t *cb, void *cb_ident)
1263{
1264 struct tcf_block_cb *block_cb;
1265
1266 block_cb = tcf_block_cb_lookup(block, cb, cb_ident);
1267 if (!block_cb)
1268 return;
John Hurley32636742018-06-25 14:30:10 -07001269 __tcf_block_cb_unregister(block, block_cb);
Jiri Pirkoacb67442017-10-19 15:50:31 +02001270}
1271EXPORT_SYMBOL(tcf_block_cb_unregister);
1272
Jiri Pirko87d83092017-05-17 11:07:54 +02001273/* Main classifier routine: scans classifier chain attached
1274 * to this qdisc, (optionally) tests for protocol and asks
1275 * specific classifiers.
1276 */
1277int tcf_classify(struct sk_buff *skb, const struct tcf_proto *tp,
1278 struct tcf_result *res, bool compat_mode)
1279{
1280 __be16 protocol = tc_skb_protocol(skb);
1281#ifdef CONFIG_NET_CLS_ACT
1282 const int max_reclassify_loop = 4;
Jiri Pirkoee538dc2017-05-23 09:11:59 +02001283 const struct tcf_proto *orig_tp = tp;
1284 const struct tcf_proto *first_tp;
Jiri Pirko87d83092017-05-17 11:07:54 +02001285 int limit = 0;
1286
1287reclassify:
1288#endif
1289 for (; tp; tp = rcu_dereference_bh(tp->next)) {
1290 int err;
1291
1292 if (tp->protocol != protocol &&
1293 tp->protocol != htons(ETH_P_ALL))
1294 continue;
1295
1296 err = tp->classify(skb, tp, res);
1297#ifdef CONFIG_NET_CLS_ACT
Jiri Pirkodb505142017-05-17 11:08:03 +02001298 if (unlikely(err == TC_ACT_RECLASSIFY && !compat_mode)) {
Jiri Pirkoee538dc2017-05-23 09:11:59 +02001299 first_tp = orig_tp;
Jiri Pirko87d83092017-05-17 11:07:54 +02001300 goto reset;
Jiri Pirkodb505142017-05-17 11:08:03 +02001301 } else if (unlikely(TC_ACT_EXT_CMP(err, TC_ACT_GOTO_CHAIN))) {
Jiri Pirkoee538dc2017-05-23 09:11:59 +02001302 first_tp = res->goto_tp;
Jiri Pirkodb505142017-05-17 11:08:03 +02001303 goto reset;
1304 }
Jiri Pirko87d83092017-05-17 11:07:54 +02001305#endif
1306 if (err >= 0)
1307 return err;
1308 }
1309
1310 return TC_ACT_UNSPEC; /* signal: continue lookup */
1311#ifdef CONFIG_NET_CLS_ACT
1312reset:
1313 if (unlikely(limit++ >= max_reclassify_loop)) {
Jiri Pirko9d3aaff2018-01-17 11:46:47 +01001314 net_notice_ratelimited("%u: reclassify loop, rule prio %u, protocol %02x\n",
1315 tp->chain->block->index,
1316 tp->prio & 0xffff,
Jiri Pirko87d83092017-05-17 11:07:54 +02001317 ntohs(tp->protocol));
1318 return TC_ACT_SHOT;
1319 }
1320
Jiri Pirkoee538dc2017-05-23 09:11:59 +02001321 tp = first_tp;
Jiri Pirko87d83092017-05-17 11:07:54 +02001322 protocol = tc_skb_protocol(skb);
1323 goto reclassify;
1324#endif
1325}
1326EXPORT_SYMBOL(tcf_classify);
1327
Jiri Pirko2190d1d2017-05-17 11:07:59 +02001328struct tcf_chain_info {
1329 struct tcf_proto __rcu **pprev;
1330 struct tcf_proto __rcu *next;
1331};
1332
1333static struct tcf_proto *tcf_chain_tp_prev(struct tcf_chain_info *chain_info)
1334{
1335 return rtnl_dereference(*chain_info->pprev);
1336}
1337
1338static void tcf_chain_tp_insert(struct tcf_chain *chain,
1339 struct tcf_chain_info *chain_info,
1340 struct tcf_proto *tp)
1341{
Jiri Pirkoc7eb7d72017-11-03 11:46:24 +01001342 if (*chain_info->pprev == chain->filter_chain)
Jiri Pirkof71e0ca42018-07-23 09:23:05 +02001343 tcf_chain0_head_change(chain, tp);
Jiri Pirko2190d1d2017-05-17 11:07:59 +02001344 RCU_INIT_POINTER(tp->next, tcf_chain_tp_prev(chain_info));
1345 rcu_assign_pointer(*chain_info->pprev, tp);
Cong Wange2ef7542017-09-11 16:33:31 -07001346 tcf_chain_hold(chain);
Jiri Pirko2190d1d2017-05-17 11:07:59 +02001347}
1348
1349static void tcf_chain_tp_remove(struct tcf_chain *chain,
1350 struct tcf_chain_info *chain_info,
1351 struct tcf_proto *tp)
1352{
1353 struct tcf_proto *next = rtnl_dereference(chain_info->next);
1354
Jiri Pirkoc7eb7d72017-11-03 11:46:24 +01001355 if (tp == chain->filter_chain)
Jiri Pirkof71e0ca42018-07-23 09:23:05 +02001356 tcf_chain0_head_change(chain, next);
Jiri Pirko2190d1d2017-05-17 11:07:59 +02001357 RCU_INIT_POINTER(*chain_info->pprev, next);
Cong Wange2ef7542017-09-11 16:33:31 -07001358 tcf_chain_put(chain);
Jiri Pirko2190d1d2017-05-17 11:07:59 +02001359}
1360
1361static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain,
1362 struct tcf_chain_info *chain_info,
1363 u32 protocol, u32 prio,
1364 bool prio_allocate)
1365{
1366 struct tcf_proto **pprev;
1367 struct tcf_proto *tp;
1368
1369 /* Check the chain for existence of proto-tcf with this priority */
1370 for (pprev = &chain->filter_chain;
1371 (tp = rtnl_dereference(*pprev)); pprev = &tp->next) {
1372 if (tp->prio >= prio) {
1373 if (tp->prio == prio) {
1374 if (prio_allocate ||
1375 (tp->protocol != protocol && protocol))
1376 return ERR_PTR(-EINVAL);
1377 } else {
1378 tp = NULL;
1379 }
1380 break;
1381 }
1382 }
1383 chain_info->pprev = pprev;
1384 chain_info->next = tp ? tp->next : NULL;
1385 return tp;
1386}
1387
WANG Cong71203712017-08-07 15:26:50 -07001388static int tcf_fill_node(struct net *net, struct sk_buff *skb,
Jiri Pirko7960d1d2018-01-17 11:46:51 +01001389 struct tcf_proto *tp, struct tcf_block *block,
1390 struct Qdisc *q, u32 parent, void *fh,
1391 u32 portid, u32 seq, u16 flags, int event)
WANG Cong71203712017-08-07 15:26:50 -07001392{
1393 struct tcmsg *tcm;
1394 struct nlmsghdr *nlh;
1395 unsigned char *b = skb_tail_pointer(skb);
1396
1397 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
1398 if (!nlh)
1399 goto out_nlmsg_trim;
1400 tcm = nlmsg_data(nlh);
1401 tcm->tcm_family = AF_UNSPEC;
1402 tcm->tcm__pad1 = 0;
1403 tcm->tcm__pad2 = 0;
Jiri Pirko7960d1d2018-01-17 11:46:51 +01001404 if (q) {
1405 tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
1406 tcm->tcm_parent = parent;
1407 } else {
1408 tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK;
1409 tcm->tcm_block_index = block->index;
1410 }
WANG Cong71203712017-08-07 15:26:50 -07001411 tcm->tcm_info = TC_H_MAKE(tp->prio, tp->protocol);
1412 if (nla_put_string(skb, TCA_KIND, tp->ops->kind))
1413 goto nla_put_failure;
1414 if (nla_put_u32(skb, TCA_CHAIN, tp->chain->index))
1415 goto nla_put_failure;
1416 if (!fh) {
1417 tcm->tcm_handle = 0;
1418 } else {
1419 if (tp->ops->dump && tp->ops->dump(net, tp, fh, skb, tcm) < 0)
1420 goto nla_put_failure;
1421 }
1422 nlh->nlmsg_len = skb_tail_pointer(skb) - b;
1423 return skb->len;
1424
1425out_nlmsg_trim:
1426nla_put_failure:
1427 nlmsg_trim(skb, b);
1428 return -1;
1429}
1430
1431static int tfilter_notify(struct net *net, struct sk_buff *oskb,
1432 struct nlmsghdr *n, struct tcf_proto *tp,
Jiri Pirko7960d1d2018-01-17 11:46:51 +01001433 struct tcf_block *block, struct Qdisc *q,
1434 u32 parent, void *fh, int event, bool unicast)
WANG Cong71203712017-08-07 15:26:50 -07001435{
1436 struct sk_buff *skb;
1437 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
1438
1439 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1440 if (!skb)
1441 return -ENOBUFS;
1442
Jiri Pirko7960d1d2018-01-17 11:46:51 +01001443 if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid,
1444 n->nlmsg_seq, n->nlmsg_flags, event) <= 0) {
WANG Cong71203712017-08-07 15:26:50 -07001445 kfree_skb(skb);
1446 return -EINVAL;
1447 }
1448
1449 if (unicast)
1450 return netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
1451
1452 return rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1453 n->nlmsg_flags & NLM_F_ECHO);
1454}
1455
1456static int tfilter_del_notify(struct net *net, struct sk_buff *oskb,
1457 struct nlmsghdr *n, struct tcf_proto *tp,
Jiri Pirko7960d1d2018-01-17 11:46:51 +01001458 struct tcf_block *block, struct Qdisc *q,
Alexander Aringc35a4ac2018-01-18 11:20:50 -05001459 u32 parent, void *fh, bool unicast, bool *last,
1460 struct netlink_ext_ack *extack)
WANG Cong71203712017-08-07 15:26:50 -07001461{
1462 struct sk_buff *skb;
1463 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
1464 int err;
1465
1466 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1467 if (!skb)
1468 return -ENOBUFS;
1469
Jiri Pirko7960d1d2018-01-17 11:46:51 +01001470 if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid,
1471 n->nlmsg_seq, n->nlmsg_flags, RTM_DELTFILTER) <= 0) {
Alexander Aringc35a4ac2018-01-18 11:20:50 -05001472 NL_SET_ERR_MSG(extack, "Failed to build del event notification");
WANG Cong71203712017-08-07 15:26:50 -07001473 kfree_skb(skb);
1474 return -EINVAL;
1475 }
1476
Alexander Aring571acf22018-01-18 11:20:53 -05001477 err = tp->ops->delete(tp, fh, last, extack);
WANG Cong71203712017-08-07 15:26:50 -07001478 if (err) {
1479 kfree_skb(skb);
1480 return err;
1481 }
1482
1483 if (unicast)
1484 return netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
1485
Alexander Aringc35a4ac2018-01-18 11:20:50 -05001486 err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1487 n->nlmsg_flags & NLM_F_ECHO);
1488 if (err < 0)
1489 NL_SET_ERR_MSG(extack, "Failed to send filter delete notification");
1490 return err;
WANG Cong71203712017-08-07 15:26:50 -07001491}
1492
1493static void tfilter_notify_chain(struct net *net, struct sk_buff *oskb,
Jiri Pirko7960d1d2018-01-17 11:46:51 +01001494 struct tcf_block *block, struct Qdisc *q,
1495 u32 parent, struct nlmsghdr *n,
WANG Cong71203712017-08-07 15:26:50 -07001496 struct tcf_chain *chain, int event)
1497{
1498 struct tcf_proto *tp;
1499
1500 for (tp = rtnl_dereference(chain->filter_chain);
1501 tp; tp = rtnl_dereference(tp->next))
Jiri Pirko7960d1d2018-01-17 11:46:51 +01001502 tfilter_notify(net, oskb, n, tp, block,
YueHaibing53189182018-07-17 20:58:14 +08001503 q, parent, NULL, event, false);
WANG Cong71203712017-08-07 15:26:50 -07001504}
1505
Vlad Buslovc431f892018-05-31 09:52:53 +03001506static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
David Ahernc21ef3e2017-04-16 09:48:24 -07001507 struct netlink_ext_ack *extack)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001508{
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09001509 struct net *net = sock_net(skb->sk);
Patrick McHardyadd93b62008-01-22 22:11:33 -08001510 struct nlattr *tca[TCA_MAX + 1];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001511 struct tcmsg *t;
1512 u32 protocol;
1513 u32 prio;
Jiri Pirko9d36d9e2017-05-17 11:07:57 +02001514 bool prio_allocate;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001515 u32 parent;
Jiri Pirko5bc17012017-05-17 11:08:01 +02001516 u32 chain_index;
Jiri Pirko7960d1d2018-01-17 11:46:51 +01001517 struct Qdisc *q = NULL;
Jiri Pirko2190d1d2017-05-17 11:07:59 +02001518 struct tcf_chain_info chain_info;
Jiri Pirko5bc17012017-05-17 11:08:01 +02001519 struct tcf_chain *chain = NULL;
Jiri Pirko6529eab2017-05-17 11:07:55 +02001520 struct tcf_block *block;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001521 struct tcf_proto *tp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001522 unsigned long cl;
WANG Cong8113c092017-08-04 21:31:43 -07001523 void *fh;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001524 int err;
Daniel Borkmann628185c2016-12-21 18:04:11 +01001525 int tp_created;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001526
Vlad Buslovc431f892018-05-31 09:52:53 +03001527 if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
Eric W. Biedermandfc47ef2012-11-16 03:03:00 +00001528 return -EPERM;
Hong zhi guode179c82013-03-25 17:36:33 +00001529
Linus Torvalds1da177e2005-04-16 15:20:36 -07001530replay:
Daniel Borkmann628185c2016-12-21 18:04:11 +01001531 tp_created = 0;
1532
Davide Carattie3314732018-10-10 22:00:58 +02001533 err = nlmsg_parse(n, sizeof(*t), tca, TCA_MAX, rtm_tca_policy, extack);
Hong zhi guode179c82013-03-25 17:36:33 +00001534 if (err < 0)
1535 return err;
1536
David S. Miller942b8162012-06-26 21:48:50 -07001537 t = nlmsg_data(n);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001538 protocol = TC_H_MIN(t->tcm_info);
1539 prio = TC_H_MAJ(t->tcm_info);
Jiri Pirko9d36d9e2017-05-17 11:07:57 +02001540 prio_allocate = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001541 parent = t->tcm_parent;
1542 cl = 0;
1543
1544 if (prio == 0) {
Vlad Buslovc431f892018-05-31 09:52:53 +03001545 /* If no priority is provided by the user,
1546 * we allocate one.
1547 */
1548 if (n->nlmsg_flags & NLM_F_CREATE) {
1549 prio = TC_H_MAKE(0x80000000U, 0U);
1550 prio_allocate = true;
1551 } else {
Alexander Aringc35a4ac2018-01-18 11:20:50 -05001552 NL_SET_ERR_MSG(extack, "Invalid filter command with priority of zero");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001553 return -ENOENT;
Daniel Borkmannea7f8272016-06-10 23:10:22 +02001554 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001555 }
1556
1557 /* Find head of filter chain. */
1558
Vlad Buslovc431f892018-05-31 09:52:53 +03001559 block = tcf_block_find(net, &q, &parent, &cl,
1560 t->tcm_ifindex, t->tcm_block_index, extack);
1561 if (IS_ERR(block)) {
1562 err = PTR_ERR(block);
1563 goto errout;
Jiri Pirko6bb16e72017-02-09 14:38:58 +01001564 }
Jiri Pirko5bc17012017-05-17 11:08:01 +02001565
1566 chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
1567 if (chain_index > TC_ACT_EXT_VAL_MASK) {
Alexander Aringc35a4ac2018-01-18 11:20:50 -05001568 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
Jiri Pirko5bc17012017-05-17 11:08:01 +02001569 err = -EINVAL;
1570 goto errout;
1571 }
Vlad Buslovc431f892018-05-31 09:52:53 +03001572 chain = tcf_chain_get(block, chain_index, true);
Jiri Pirko5bc17012017-05-17 11:08:01 +02001573 if (!chain) {
Jiri Pirkod5ed72a2018-08-27 20:58:43 +02001574 NL_SET_ERR_MSG(extack, "Cannot create specified filter chain");
Vlad Buslovc431f892018-05-31 09:52:53 +03001575 err = -ENOMEM;
Daniel Borkmannea7f8272016-06-10 23:10:22 +02001576 goto errout;
1577 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001578
Jiri Pirko2190d1d2017-05-17 11:07:59 +02001579 tp = tcf_chain_tp_find(chain, &chain_info, protocol,
1580 prio, prio_allocate);
1581 if (IS_ERR(tp)) {
Alexander Aringc35a4ac2018-01-18 11:20:50 -05001582 NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
Jiri Pirko2190d1d2017-05-17 11:07:59 +02001583 err = PTR_ERR(tp);
1584 goto errout;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001585 }
1586
1587 if (tp == NULL) {
1588 /* Proto-tcf does not exist, create new one */
1589
Jiri Pirko6bb16e72017-02-09 14:38:58 +01001590 if (tca[TCA_KIND] == NULL || !protocol) {
Alexander Aringc35a4ac2018-01-18 11:20:50 -05001591 NL_SET_ERR_MSG(extack, "Filter kind and protocol must be specified");
Jiri Pirko6bb16e72017-02-09 14:38:58 +01001592 err = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001593 goto errout;
Jiri Pirko6bb16e72017-02-09 14:38:58 +01001594 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001595
Vlad Buslovc431f892018-05-31 09:52:53 +03001596 if (!(n->nlmsg_flags & NLM_F_CREATE)) {
Alexander Aringc35a4ac2018-01-18 11:20:50 -05001597 NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter");
Jiri Pirko6bb16e72017-02-09 14:38:58 +01001598 err = -ENOENT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001599 goto errout;
Jiri Pirko6bb16e72017-02-09 14:38:58 +01001600 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001601
Jiri Pirko9d36d9e2017-05-17 11:07:57 +02001602 if (prio_allocate)
Jiri Pirko2190d1d2017-05-17 11:07:59 +02001603 prio = tcf_auto_prio(tcf_chain_tp_prev(&chain_info));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001604
Jiri Pirko33a48922017-02-09 14:38:57 +01001605 tp = tcf_proto_create(nla_data(tca[TCA_KIND]),
Alexander Aringc35a4ac2018-01-18 11:20:50 -05001606 protocol, prio, chain, extack);
Jiri Pirko33a48922017-02-09 14:38:57 +01001607 if (IS_ERR(tp)) {
1608 err = PTR_ERR(tp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001609 goto errout;
1610 }
Minoru Usui12186be2009-06-02 02:17:34 -07001611 tp_created = 1;
Jiri Pirko6bb16e72017-02-09 14:38:58 +01001612 } else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
Alexander Aringc35a4ac2018-01-18 11:20:50 -05001613 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
Jiri Pirko6bb16e72017-02-09 14:38:58 +01001614 err = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001615 goto errout;
Jiri Pirko6bb16e72017-02-09 14:38:58 +01001616 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001617
1618 fh = tp->ops->get(tp, t->tcm_handle);
1619
WANG Cong8113c092017-08-04 21:31:43 -07001620 if (!fh) {
Vlad Buslovc431f892018-05-31 09:52:53 +03001621 if (!(n->nlmsg_flags & NLM_F_CREATE)) {
Alexander Aringc35a4ac2018-01-18 11:20:50 -05001622 NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter");
Jiri Pirko6bb16e72017-02-09 14:38:58 +01001623 err = -ENOENT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001624 goto errout;
Jiri Pirko6bb16e72017-02-09 14:38:58 +01001625 }
Vlad Buslovc431f892018-05-31 09:52:53 +03001626 } else if (n->nlmsg_flags & NLM_F_EXCL) {
1627 NL_SET_ERR_MSG(extack, "Filter already exists");
1628 err = -EEXIST;
1629 goto errout;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001630 }
1631
Jiri Pirko9f407f12018-07-23 09:23:07 +02001632 if (chain->tmplt_ops && chain->tmplt_ops != tp->ops) {
1633 NL_SET_ERR_MSG(extack, "Chain template is set to a different filter kind");
1634 err = -EINVAL;
1635 goto errout;
1636 }
1637
Cong Wang2f7ef2f2014-04-25 13:54:06 -07001638 err = tp->ops->change(net, skb, tp, cl, t->tcm_handle, tca, &fh,
Alexander Aring7306db32018-01-18 11:20:51 -05001639 n->nlmsg_flags & NLM_F_CREATE ? TCA_ACT_NOREPLACE : TCA_ACT_REPLACE,
1640 extack);
Minoru Usui12186be2009-06-02 02:17:34 -07001641 if (err == 0) {
Jiri Pirko2190d1d2017-05-17 11:07:59 +02001642 if (tp_created)
1643 tcf_chain_tp_insert(chain, &chain_info, tp);
Jiri Pirko7960d1d2018-01-17 11:46:51 +01001644 tfilter_notify(net, skb, n, tp, block, q, parent, fh,
Jiri Pirkoa10fa202017-10-13 14:01:05 +02001645 RTM_NEWTFILTER, false);
Minoru Usui12186be2009-06-02 02:17:34 -07001646 } else {
1647 if (tp_created)
Jakub Kicinski715df5e2018-01-24 12:54:13 -08001648 tcf_proto_destroy(tp, NULL);
Minoru Usui12186be2009-06-02 02:17:34 -07001649 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001650
1651errout:
Jiri Pirko5bc17012017-05-17 11:08:01 +02001652 if (chain)
1653 tcf_chain_put(chain);
Vlad Buslove368fdb2018-09-24 19:22:53 +03001654 tcf_block_release(q, block);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001655 if (err == -EAGAIN)
1656 /* Replay the request. */
1657 goto replay;
1658 return err;
1659}
1660
Vlad Buslovc431f892018-05-31 09:52:53 +03001661static int tc_del_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
1662 struct netlink_ext_ack *extack)
1663{
1664 struct net *net = sock_net(skb->sk);
1665 struct nlattr *tca[TCA_MAX + 1];
1666 struct tcmsg *t;
1667 u32 protocol;
1668 u32 prio;
1669 u32 parent;
1670 u32 chain_index;
1671 struct Qdisc *q = NULL;
1672 struct tcf_chain_info chain_info;
1673 struct tcf_chain *chain = NULL;
1674 struct tcf_block *block;
1675 struct tcf_proto *tp = NULL;
1676 unsigned long cl = 0;
1677 void *fh = NULL;
1678 int err;
1679
1680 if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
1681 return -EPERM;
1682
Davide Carattie3314732018-10-10 22:00:58 +02001683 err = nlmsg_parse(n, sizeof(*t), tca, TCA_MAX, rtm_tca_policy, extack);
Vlad Buslovc431f892018-05-31 09:52:53 +03001684 if (err < 0)
1685 return err;
1686
1687 t = nlmsg_data(n);
1688 protocol = TC_H_MIN(t->tcm_info);
1689 prio = TC_H_MAJ(t->tcm_info);
1690 parent = t->tcm_parent;
1691
1692 if (prio == 0 && (protocol || t->tcm_handle || tca[TCA_KIND])) {
1693 NL_SET_ERR_MSG(extack, "Cannot flush filters with protocol, handle or kind set");
1694 return -ENOENT;
1695 }
1696
1697 /* Find head of filter chain. */
1698
1699 block = tcf_block_find(net, &q, &parent, &cl,
1700 t->tcm_ifindex, t->tcm_block_index, extack);
1701 if (IS_ERR(block)) {
1702 err = PTR_ERR(block);
1703 goto errout;
1704 }
1705
1706 chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
1707 if (chain_index > TC_ACT_EXT_VAL_MASK) {
1708 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
1709 err = -EINVAL;
1710 goto errout;
1711 }
1712 chain = tcf_chain_get(block, chain_index, false);
1713 if (!chain) {
Jiri Pirko5ca8a252018-08-03 11:08:47 +02001714 /* User requested flush on non-existent chain. Nothing to do,
1715 * so just return success.
1716 */
1717 if (prio == 0) {
1718 err = 0;
1719 goto errout;
1720 }
Vlad Buslovc431f892018-05-31 09:52:53 +03001721 NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
Jiri Pirkob7b42472018-08-27 20:58:44 +02001722 err = -ENOENT;
Vlad Buslovc431f892018-05-31 09:52:53 +03001723 goto errout;
1724 }
1725
1726 if (prio == 0) {
1727 tfilter_notify_chain(net, skb, block, q, parent, n,
1728 chain, RTM_DELTFILTER);
1729 tcf_chain_flush(chain);
1730 err = 0;
1731 goto errout;
1732 }
1733
1734 tp = tcf_chain_tp_find(chain, &chain_info, protocol,
1735 prio, false);
1736 if (!tp || IS_ERR(tp)) {
1737 NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
Vlad Buslov0e399032018-06-04 18:32:23 +03001738 err = tp ? PTR_ERR(tp) : -ENOENT;
Vlad Buslovc431f892018-05-31 09:52:53 +03001739 goto errout;
1740 } else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
1741 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
1742 err = -EINVAL;
1743 goto errout;
1744 }
1745
1746 fh = tp->ops->get(tp, t->tcm_handle);
1747
1748 if (!fh) {
1749 if (t->tcm_handle == 0) {
1750 tcf_chain_tp_remove(chain, &chain_info, tp);
1751 tfilter_notify(net, skb, n, tp, block, q, parent, fh,
1752 RTM_DELTFILTER, false);
1753 tcf_proto_destroy(tp, extack);
1754 err = 0;
1755 } else {
1756 NL_SET_ERR_MSG(extack, "Specified filter handle not found");
1757 err = -ENOENT;
1758 }
1759 } else {
1760 bool last;
1761
1762 err = tfilter_del_notify(net, skb, n, tp, block,
1763 q, parent, fh, false, &last,
1764 extack);
1765 if (err)
1766 goto errout;
1767 if (last) {
1768 tcf_chain_tp_remove(chain, &chain_info, tp);
1769 tcf_proto_destroy(tp, extack);
1770 }
1771 }
1772
1773errout:
1774 if (chain)
1775 tcf_chain_put(chain);
Vlad Buslove368fdb2018-09-24 19:22:53 +03001776 tcf_block_release(q, block);
Vlad Buslovc431f892018-05-31 09:52:53 +03001777 return err;
1778}
1779
1780static int tc_get_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
1781 struct netlink_ext_ack *extack)
1782{
1783 struct net *net = sock_net(skb->sk);
1784 struct nlattr *tca[TCA_MAX + 1];
1785 struct tcmsg *t;
1786 u32 protocol;
1787 u32 prio;
1788 u32 parent;
1789 u32 chain_index;
1790 struct Qdisc *q = NULL;
1791 struct tcf_chain_info chain_info;
1792 struct tcf_chain *chain = NULL;
1793 struct tcf_block *block;
1794 struct tcf_proto *tp = NULL;
1795 unsigned long cl = 0;
1796 void *fh = NULL;
1797 int err;
1798
Davide Carattie3314732018-10-10 22:00:58 +02001799 err = nlmsg_parse(n, sizeof(*t), tca, TCA_MAX, rtm_tca_policy, extack);
Vlad Buslovc431f892018-05-31 09:52:53 +03001800 if (err < 0)
1801 return err;
1802
1803 t = nlmsg_data(n);
1804 protocol = TC_H_MIN(t->tcm_info);
1805 prio = TC_H_MAJ(t->tcm_info);
1806 parent = t->tcm_parent;
1807
1808 if (prio == 0) {
1809 NL_SET_ERR_MSG(extack, "Invalid filter command with priority of zero");
1810 return -ENOENT;
1811 }
1812
1813 /* Find head of filter chain. */
1814
1815 block = tcf_block_find(net, &q, &parent, &cl,
1816 t->tcm_ifindex, t->tcm_block_index, extack);
1817 if (IS_ERR(block)) {
1818 err = PTR_ERR(block);
1819 goto errout;
1820 }
1821
1822 chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
1823 if (chain_index > TC_ACT_EXT_VAL_MASK) {
1824 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
1825 err = -EINVAL;
1826 goto errout;
1827 }
1828 chain = tcf_chain_get(block, chain_index, false);
1829 if (!chain) {
1830 NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
1831 err = -EINVAL;
1832 goto errout;
1833 }
1834
1835 tp = tcf_chain_tp_find(chain, &chain_info, protocol,
1836 prio, false);
1837 if (!tp || IS_ERR(tp)) {
1838 NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
Vlad Buslov0e399032018-06-04 18:32:23 +03001839 err = tp ? PTR_ERR(tp) : -ENOENT;
Vlad Buslovc431f892018-05-31 09:52:53 +03001840 goto errout;
1841 } else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
1842 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
1843 err = -EINVAL;
1844 goto errout;
1845 }
1846
1847 fh = tp->ops->get(tp, t->tcm_handle);
1848
1849 if (!fh) {
1850 NL_SET_ERR_MSG(extack, "Specified filter handle not found");
1851 err = -ENOENT;
1852 } else {
1853 err = tfilter_notify(net, skb, n, tp, block, q, parent,
1854 fh, RTM_NEWTFILTER, true);
1855 if (err < 0)
1856 NL_SET_ERR_MSG(extack, "Failed to send filter notify message");
1857 }
1858
1859errout:
1860 if (chain)
1861 tcf_chain_put(chain);
Vlad Buslove368fdb2018-09-24 19:22:53 +03001862 tcf_block_release(q, block);
Vlad Buslovc431f892018-05-31 09:52:53 +03001863 return err;
1864}
1865
Stephen Hemmingeraa767bf2008-01-21 02:26:41 -08001866struct tcf_dump_args {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001867 struct tcf_walker w;
1868 struct sk_buff *skb;
1869 struct netlink_callback *cb;
Jiri Pirko7960d1d2018-01-17 11:46:51 +01001870 struct tcf_block *block;
Jiri Pirkoa10fa202017-10-13 14:01:05 +02001871 struct Qdisc *q;
1872 u32 parent;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001873};
1874
WANG Cong8113c092017-08-04 21:31:43 -07001875static int tcf_node_dump(struct tcf_proto *tp, void *n, struct tcf_walker *arg)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001876{
Stephen Hemmingeraa767bf2008-01-21 02:26:41 -08001877 struct tcf_dump_args *a = (void *)arg;
WANG Cong832d1d52014-01-09 16:14:01 -08001878 struct net *net = sock_net(a->skb->sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001879
Jiri Pirko7960d1d2018-01-17 11:46:51 +01001880 return tcf_fill_node(net, a->skb, tp, a->block, a->q, a->parent,
Jiri Pirkoa10fa202017-10-13 14:01:05 +02001881 n, NETLINK_CB(a->cb->skb).portid,
Jamal Hadi Salim5a7a5552016-09-18 08:45:33 -04001882 a->cb->nlh->nlmsg_seq, NLM_F_MULTI,
1883 RTM_NEWTFILTER);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001884}
1885
Jiri Pirkoa10fa202017-10-13 14:01:05 +02001886static bool tcf_chain_dump(struct tcf_chain *chain, struct Qdisc *q, u32 parent,
1887 struct sk_buff *skb, struct netlink_callback *cb,
Jiri Pirkoacb31fa2017-05-17 11:08:00 +02001888 long index_start, long *p_index)
1889{
1890 struct net *net = sock_net(skb->sk);
Jiri Pirko7960d1d2018-01-17 11:46:51 +01001891 struct tcf_block *block = chain->block;
Jiri Pirkoacb31fa2017-05-17 11:08:00 +02001892 struct tcmsg *tcm = nlmsg_data(cb->nlh);
1893 struct tcf_dump_args arg;
1894 struct tcf_proto *tp;
1895
1896 for (tp = rtnl_dereference(chain->filter_chain);
1897 tp; tp = rtnl_dereference(tp->next), (*p_index)++) {
1898 if (*p_index < index_start)
1899 continue;
1900 if (TC_H_MAJ(tcm->tcm_info) &&
1901 TC_H_MAJ(tcm->tcm_info) != tp->prio)
1902 continue;
1903 if (TC_H_MIN(tcm->tcm_info) &&
1904 TC_H_MIN(tcm->tcm_info) != tp->protocol)
1905 continue;
1906 if (*p_index > index_start)
1907 memset(&cb->args[1], 0,
1908 sizeof(cb->args) - sizeof(cb->args[0]));
1909 if (cb->args[1] == 0) {
YueHaibing53189182018-07-17 20:58:14 +08001910 if (tcf_fill_node(net, skb, tp, block, q, parent, NULL,
Jiri Pirkoacb31fa2017-05-17 11:08:00 +02001911 NETLINK_CB(cb->skb).portid,
1912 cb->nlh->nlmsg_seq, NLM_F_MULTI,
1913 RTM_NEWTFILTER) <= 0)
Jiri Pirko5bc17012017-05-17 11:08:01 +02001914 return false;
Jiri Pirkoacb31fa2017-05-17 11:08:00 +02001915
1916 cb->args[1] = 1;
1917 }
1918 if (!tp->ops->walk)
1919 continue;
1920 arg.w.fn = tcf_node_dump;
1921 arg.skb = skb;
1922 arg.cb = cb;
Jiri Pirko7960d1d2018-01-17 11:46:51 +01001923 arg.block = block;
Jiri Pirkoa10fa202017-10-13 14:01:05 +02001924 arg.q = q;
1925 arg.parent = parent;
Jiri Pirkoacb31fa2017-05-17 11:08:00 +02001926 arg.w.stop = 0;
1927 arg.w.skip = cb->args[1] - 1;
1928 arg.w.count = 0;
Vlad Buslov01683a12018-07-09 13:29:11 +03001929 arg.w.cookie = cb->args[2];
Jiri Pirkoacb31fa2017-05-17 11:08:00 +02001930 tp->ops->walk(tp, &arg.w);
Vlad Buslov01683a12018-07-09 13:29:11 +03001931 cb->args[2] = arg.w.cookie;
Jiri Pirkoacb31fa2017-05-17 11:08:00 +02001932 cb->args[1] = arg.w.count + 1;
1933 if (arg.w.stop)
Jiri Pirko5bc17012017-05-17 11:08:01 +02001934 return false;
Jiri Pirkoacb31fa2017-05-17 11:08:00 +02001935 }
Jiri Pirko5bc17012017-05-17 11:08:01 +02001936 return true;
Jiri Pirkoacb31fa2017-05-17 11:08:00 +02001937}
1938
Eric Dumazetbd27a872009-11-05 20:57:26 -08001939/* called with RTNL */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001940static int tc_dump_tfilter(struct sk_buff *skb, struct netlink_callback *cb)
1941{
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09001942 struct net *net = sock_net(skb->sk);
Jiri Pirko5bc17012017-05-17 11:08:01 +02001943 struct nlattr *tca[TCA_MAX + 1];
Jiri Pirko7960d1d2018-01-17 11:46:51 +01001944 struct Qdisc *q = NULL;
Jiri Pirko6529eab2017-05-17 11:07:55 +02001945 struct tcf_block *block;
Jiri Pirko2190d1d2017-05-17 11:07:59 +02001946 struct tcf_chain *chain;
David S. Miller942b8162012-06-26 21:48:50 -07001947 struct tcmsg *tcm = nlmsg_data(cb->nlh);
Jiri Pirkoacb31fa2017-05-17 11:08:00 +02001948 long index_start;
1949 long index;
Jiri Pirkoa10fa202017-10-13 14:01:05 +02001950 u32 parent;
Jiri Pirko5bc17012017-05-17 11:08:01 +02001951 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001952
Hong zhi guo573ce262013-03-27 06:47:04 +00001953 if (nlmsg_len(cb->nlh) < sizeof(*tcm))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001954 return skb->len;
Jiri Pirko5bc17012017-05-17 11:08:01 +02001955
David Aherndac9c972018-10-07 20:16:24 -07001956 err = nlmsg_parse(cb->nlh, sizeof(*tcm), tca, TCA_MAX, NULL,
1957 cb->extack);
Jiri Pirko5bc17012017-05-17 11:08:01 +02001958 if (err)
1959 return err;
1960
Jiri Pirko7960d1d2018-01-17 11:46:51 +01001961 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
Vlad Buslov787ce6d2018-09-24 19:22:58 +03001962 block = tcf_block_refcnt_get(net, tcm->tcm_block_index);
Jiri Pirko7960d1d2018-01-17 11:46:51 +01001963 if (!block)
WANG Cong143976c2017-08-24 16:51:29 -07001964 goto out;
Jiri Pirkod680b352018-01-18 16:14:49 +01001965 /* If we work with block index, q is NULL and parent value
1966 * will never be used in the following code. The check
1967 * in tcf_fill_node prevents it. However, compiler does not
1968 * see that far, so set parent to zero to silence the warning
1969 * about parent being uninitialized.
1970 */
1971 parent = 0;
Jiri Pirko7960d1d2018-01-17 11:46:51 +01001972 } else {
1973 const struct Qdisc_class_ops *cops;
1974 struct net_device *dev;
1975 unsigned long cl = 0;
1976
1977 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
1978 if (!dev)
1979 return skb->len;
1980
1981 parent = tcm->tcm_parent;
1982 if (!parent) {
1983 q = dev->qdisc;
1984 parent = q->handle;
1985 } else {
1986 q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent));
1987 }
1988 if (!q)
1989 goto out;
1990 cops = q->ops->cl_ops;
1991 if (!cops)
1992 goto out;
1993 if (!cops->tcf_block)
1994 goto out;
1995 if (TC_H_MIN(tcm->tcm_parent)) {
1996 cl = cops->find(q, tcm->tcm_parent);
1997 if (cl == 0)
1998 goto out;
1999 }
2000 block = cops->tcf_block(q, cl, NULL);
2001 if (!block)
2002 goto out;
2003 if (tcf_block_shared(block))
2004 q = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002005 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002006
Jiri Pirkoacb31fa2017-05-17 11:08:00 +02002007 index_start = cb->args[0];
2008 index = 0;
Jiri Pirko5bc17012017-05-17 11:08:01 +02002009
2010 list_for_each_entry(chain, &block->chain_list, list) {
2011 if (tca[TCA_CHAIN] &&
2012 nla_get_u32(tca[TCA_CHAIN]) != chain->index)
2013 continue;
Jiri Pirkoa10fa202017-10-13 14:01:05 +02002014 if (!tcf_chain_dump(chain, q, parent, skb, cb,
Roman Kapl5ae437a2018-02-19 21:32:51 +01002015 index_start, &index)) {
2016 err = -EMSGSIZE;
Jiri Pirko5bc17012017-05-17 11:08:01 +02002017 break;
Roman Kapl5ae437a2018-02-19 21:32:51 +01002018 }
Jiri Pirko5bc17012017-05-17 11:08:01 +02002019 }
2020
Vlad Buslov787ce6d2018-09-24 19:22:58 +03002021 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK)
2022 tcf_block_refcnt_put(block);
Jiri Pirkoacb31fa2017-05-17 11:08:00 +02002023 cb->args[0] = index;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002024
Linus Torvalds1da177e2005-04-16 15:20:36 -07002025out:
Roman Kapl5ae437a2018-02-19 21:32:51 +01002026 /* If we did no progress, the error (EMSGSIZE) is real */
2027 if (skb->len == 0 && err)
2028 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002029 return skb->len;
2030}
2031
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002032static int tc_chain_fill_node(struct tcf_chain *chain, struct net *net,
2033 struct sk_buff *skb, struct tcf_block *block,
2034 u32 portid, u32 seq, u16 flags, int event)
2035{
2036 unsigned char *b = skb_tail_pointer(skb);
Jiri Pirko9f407f12018-07-23 09:23:07 +02002037 const struct tcf_proto_ops *ops;
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002038 struct nlmsghdr *nlh;
2039 struct tcmsg *tcm;
Jiri Pirko9f407f12018-07-23 09:23:07 +02002040 void *priv;
2041
2042 ops = chain->tmplt_ops;
2043 priv = chain->tmplt_priv;
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002044
2045 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
2046 if (!nlh)
2047 goto out_nlmsg_trim;
2048 tcm = nlmsg_data(nlh);
2049 tcm->tcm_family = AF_UNSPEC;
2050 tcm->tcm__pad1 = 0;
2051 tcm->tcm__pad2 = 0;
2052 tcm->tcm_handle = 0;
2053 if (block->q) {
2054 tcm->tcm_ifindex = qdisc_dev(block->q)->ifindex;
2055 tcm->tcm_parent = block->q->handle;
2056 } else {
2057 tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK;
2058 tcm->tcm_block_index = block->index;
2059 }
2060
2061 if (nla_put_u32(skb, TCA_CHAIN, chain->index))
2062 goto nla_put_failure;
2063
Jiri Pirko9f407f12018-07-23 09:23:07 +02002064 if (ops) {
2065 if (nla_put_string(skb, TCA_KIND, ops->kind))
2066 goto nla_put_failure;
2067 if (ops->tmplt_dump(skb, net, priv) < 0)
2068 goto nla_put_failure;
2069 }
2070
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002071 nlh->nlmsg_len = skb_tail_pointer(skb) - b;
2072 return skb->len;
2073
2074out_nlmsg_trim:
2075nla_put_failure:
2076 nlmsg_trim(skb, b);
2077 return -EMSGSIZE;
2078}
2079
2080static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb,
2081 u32 seq, u16 flags, int event, bool unicast)
2082{
2083 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
2084 struct tcf_block *block = chain->block;
2085 struct net *net = block->net;
2086 struct sk_buff *skb;
2087
2088 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2089 if (!skb)
2090 return -ENOBUFS;
2091
2092 if (tc_chain_fill_node(chain, net, skb, block, portid,
2093 seq, flags, event) <= 0) {
2094 kfree_skb(skb);
2095 return -EINVAL;
2096 }
2097
2098 if (unicast)
2099 return netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
2100
2101 return rtnetlink_send(skb, net, portid, RTNLGRP_TC, flags & NLM_F_ECHO);
2102}
2103
Jiri Pirko9f407f12018-07-23 09:23:07 +02002104static int tc_chain_tmplt_add(struct tcf_chain *chain, struct net *net,
2105 struct nlattr **tca,
2106 struct netlink_ext_ack *extack)
2107{
2108 const struct tcf_proto_ops *ops;
2109 void *tmplt_priv;
2110
2111 /* If kind is not set, user did not specify template. */
2112 if (!tca[TCA_KIND])
2113 return 0;
2114
2115 ops = tcf_proto_lookup_ops(nla_data(tca[TCA_KIND]), extack);
2116 if (IS_ERR(ops))
2117 return PTR_ERR(ops);
2118 if (!ops->tmplt_create || !ops->tmplt_destroy || !ops->tmplt_dump) {
2119 NL_SET_ERR_MSG(extack, "Chain templates are not supported with specified classifier");
2120 return -EOPNOTSUPP;
2121 }
2122
2123 tmplt_priv = ops->tmplt_create(net, chain, tca, extack);
2124 if (IS_ERR(tmplt_priv)) {
2125 module_put(ops->owner);
2126 return PTR_ERR(tmplt_priv);
2127 }
2128 chain->tmplt_ops = ops;
2129 chain->tmplt_priv = tmplt_priv;
2130 return 0;
2131}
2132
2133static void tc_chain_tmplt_del(struct tcf_chain *chain)
2134{
2135 const struct tcf_proto_ops *ops = chain->tmplt_ops;
2136
2137 /* If template ops are set, no work to do for us. */
2138 if (!ops)
2139 return;
2140
2141 ops->tmplt_destroy(chain->tmplt_priv);
2142 module_put(ops->owner);
2143}
2144
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002145/* Add/delete/get a chain */
2146
2147static int tc_ctl_chain(struct sk_buff *skb, struct nlmsghdr *n,
2148 struct netlink_ext_ack *extack)
2149{
2150 struct net *net = sock_net(skb->sk);
2151 struct nlattr *tca[TCA_MAX + 1];
2152 struct tcmsg *t;
2153 u32 parent;
2154 u32 chain_index;
2155 struct Qdisc *q = NULL;
2156 struct tcf_chain *chain = NULL;
2157 struct tcf_block *block;
2158 unsigned long cl;
2159 int err;
2160
2161 if (n->nlmsg_type != RTM_GETCHAIN &&
2162 !netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
2163 return -EPERM;
2164
2165replay:
Davide Carattie3314732018-10-10 22:00:58 +02002166 err = nlmsg_parse(n, sizeof(*t), tca, TCA_MAX, rtm_tca_policy, extack);
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002167 if (err < 0)
2168 return err;
2169
2170 t = nlmsg_data(n);
2171 parent = t->tcm_parent;
2172 cl = 0;
2173
2174 block = tcf_block_find(net, &q, &parent, &cl,
2175 t->tcm_ifindex, t->tcm_block_index, extack);
2176 if (IS_ERR(block))
2177 return PTR_ERR(block);
2178
2179 chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
2180 if (chain_index > TC_ACT_EXT_VAL_MASK) {
2181 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
Vlad Buslove368fdb2018-09-24 19:22:53 +03002182 err = -EINVAL;
2183 goto errout_block;
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002184 }
2185 chain = tcf_chain_lookup(block, chain_index);
2186 if (n->nlmsg_type == RTM_NEWCHAIN) {
2187 if (chain) {
Jiri Pirko3d32f4c2018-08-01 12:36:55 +02002188 if (tcf_chain_held_by_acts_only(chain)) {
Jiri Pirko1f3ed382018-07-27 09:45:05 +02002189 /* The chain exists only because there is
Jiri Pirko3d32f4c2018-08-01 12:36:55 +02002190 * some action referencing it.
Jiri Pirko1f3ed382018-07-27 09:45:05 +02002191 */
2192 tcf_chain_hold(chain);
2193 } else {
2194 NL_SET_ERR_MSG(extack, "Filter chain already exists");
Vlad Buslove368fdb2018-09-24 19:22:53 +03002195 err = -EEXIST;
2196 goto errout_block;
Jiri Pirko1f3ed382018-07-27 09:45:05 +02002197 }
2198 } else {
2199 if (!(n->nlmsg_flags & NLM_F_CREATE)) {
2200 NL_SET_ERR_MSG(extack, "Need both RTM_NEWCHAIN and NLM_F_CREATE to create a new chain");
Vlad Buslove368fdb2018-09-24 19:22:53 +03002201 err = -ENOENT;
2202 goto errout_block;
Jiri Pirko1f3ed382018-07-27 09:45:05 +02002203 }
2204 chain = tcf_chain_create(block, chain_index);
2205 if (!chain) {
2206 NL_SET_ERR_MSG(extack, "Failed to create filter chain");
Vlad Buslove368fdb2018-09-24 19:22:53 +03002207 err = -ENOMEM;
2208 goto errout_block;
Jiri Pirko1f3ed382018-07-27 09:45:05 +02002209 }
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002210 }
2211 } else {
Jiri Pirko3d32f4c2018-08-01 12:36:55 +02002212 if (!chain || tcf_chain_held_by_acts_only(chain)) {
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002213 NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
Vlad Buslove368fdb2018-09-24 19:22:53 +03002214 err = -EINVAL;
2215 goto errout_block;
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002216 }
2217 tcf_chain_hold(chain);
2218 }
2219
2220 switch (n->nlmsg_type) {
2221 case RTM_NEWCHAIN:
Jiri Pirko9f407f12018-07-23 09:23:07 +02002222 err = tc_chain_tmplt_add(chain, net, tca, extack);
2223 if (err)
2224 goto errout;
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002225 /* In case the chain was successfully added, take a reference
2226 * to the chain. This ensures that an empty chain
2227 * does not disappear at the end of this function.
2228 */
2229 tcf_chain_hold(chain);
2230 chain->explicitly_created = true;
2231 tc_chain_notify(chain, NULL, 0, NLM_F_CREATE | NLM_F_EXCL,
2232 RTM_NEWCHAIN, false);
2233 break;
2234 case RTM_DELCHAIN:
Cong Wangf5b9bac2018-09-11 14:22:23 -07002235 tfilter_notify_chain(net, skb, block, q, parent, n,
2236 chain, RTM_DELTFILTER);
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002237 /* Flush the chain first as the user requested chain removal. */
2238 tcf_chain_flush(chain);
2239 /* In case the chain was successfully deleted, put a reference
2240 * to the chain previously taken during addition.
2241 */
2242 tcf_chain_put_explicitly_created(chain);
Jiri Pirkoc921d7d2018-07-26 18:27:58 +02002243 chain->explicitly_created = false;
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002244 break;
2245 case RTM_GETCHAIN:
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002246 err = tc_chain_notify(chain, skb, n->nlmsg_seq,
2247 n->nlmsg_seq, n->nlmsg_type, true);
2248 if (err < 0)
2249 NL_SET_ERR_MSG(extack, "Failed to send chain notify message");
2250 break;
2251 default:
2252 err = -EOPNOTSUPP;
2253 NL_SET_ERR_MSG(extack, "Unsupported message type");
2254 goto errout;
2255 }
2256
2257errout:
2258 tcf_chain_put(chain);
Vlad Buslove368fdb2018-09-24 19:22:53 +03002259errout_block:
2260 tcf_block_release(q, block);
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002261 if (err == -EAGAIN)
2262 /* Replay the request. */
2263 goto replay;
2264 return err;
2265}
2266
2267/* called with RTNL */
2268static int tc_dump_chain(struct sk_buff *skb, struct netlink_callback *cb)
2269{
2270 struct net *net = sock_net(skb->sk);
2271 struct nlattr *tca[TCA_MAX + 1];
2272 struct Qdisc *q = NULL;
2273 struct tcf_block *block;
2274 struct tcf_chain *chain;
2275 struct tcmsg *tcm = nlmsg_data(cb->nlh);
2276 long index_start;
2277 long index;
2278 u32 parent;
2279 int err;
2280
2281 if (nlmsg_len(cb->nlh) < sizeof(*tcm))
2282 return skb->len;
2283
Davide Carattie3314732018-10-10 22:00:58 +02002284 err = nlmsg_parse(cb->nlh, sizeof(*tcm), tca, TCA_MAX, rtm_tca_policy,
David Aherndac9c972018-10-07 20:16:24 -07002285 cb->extack);
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002286 if (err)
2287 return err;
2288
2289 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
Vlad Buslov787ce6d2018-09-24 19:22:58 +03002290 block = tcf_block_refcnt_get(net, tcm->tcm_block_index);
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002291 if (!block)
2292 goto out;
2293 /* If we work with block index, q is NULL and parent value
2294 * will never be used in the following code. The check
2295 * in tcf_fill_node prevents it. However, compiler does not
2296 * see that far, so set parent to zero to silence the warning
2297 * about parent being uninitialized.
2298 */
2299 parent = 0;
2300 } else {
2301 const struct Qdisc_class_ops *cops;
2302 struct net_device *dev;
2303 unsigned long cl = 0;
2304
2305 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
2306 if (!dev)
2307 return skb->len;
2308
2309 parent = tcm->tcm_parent;
2310 if (!parent) {
2311 q = dev->qdisc;
2312 parent = q->handle;
2313 } else {
2314 q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent));
2315 }
2316 if (!q)
2317 goto out;
2318 cops = q->ops->cl_ops;
2319 if (!cops)
2320 goto out;
2321 if (!cops->tcf_block)
2322 goto out;
2323 if (TC_H_MIN(tcm->tcm_parent)) {
2324 cl = cops->find(q, tcm->tcm_parent);
2325 if (cl == 0)
2326 goto out;
2327 }
2328 block = cops->tcf_block(q, cl, NULL);
2329 if (!block)
2330 goto out;
2331 if (tcf_block_shared(block))
2332 q = NULL;
2333 }
2334
2335 index_start = cb->args[0];
2336 index = 0;
2337
2338 list_for_each_entry(chain, &block->chain_list, list) {
2339 if ((tca[TCA_CHAIN] &&
2340 nla_get_u32(tca[TCA_CHAIN]) != chain->index))
2341 continue;
2342 if (index < index_start) {
2343 index++;
2344 continue;
2345 }
Jiri Pirko3d32f4c2018-08-01 12:36:55 +02002346 if (tcf_chain_held_by_acts_only(chain))
Jiri Pirko1f3ed382018-07-27 09:45:05 +02002347 continue;
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002348 err = tc_chain_fill_node(chain, net, skb, block,
2349 NETLINK_CB(cb->skb).portid,
2350 cb->nlh->nlmsg_seq, NLM_F_MULTI,
2351 RTM_NEWCHAIN);
2352 if (err <= 0)
2353 break;
2354 index++;
2355 }
2356
Vlad Buslov787ce6d2018-09-24 19:22:58 +03002357 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK)
2358 tcf_block_refcnt_put(block);
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002359 cb->args[0] = index;
2360
2361out:
2362 /* If we did no progress, the error (EMSGSIZE) is real */
2363 if (skb->len == 0 && err)
2364 return err;
2365 return skb->len;
2366}
2367
WANG Cong18d02642014-09-25 10:26:37 -07002368void tcf_exts_destroy(struct tcf_exts *exts)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002369{
2370#ifdef CONFIG_NET_CLS_ACT
Vlad Buslov90b73b72018-07-05 17:24:33 +03002371 tcf_action_destroy(exts->actions, TCA_ACT_UNBIND);
WANG Cong22dc13c2016-08-13 22:35:00 -07002372 kfree(exts->actions);
2373 exts->nr_actions = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002374#endif
2375}
Stephen Hemmingeraa767bf2008-01-21 02:26:41 -08002376EXPORT_SYMBOL(tcf_exts_destroy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002377
Benjamin LaHaisec1b52732013-01-14 05:15:39 +00002378int tcf_exts_validate(struct net *net, struct tcf_proto *tp, struct nlattr **tb,
Alexander Aring50a56192018-01-18 11:20:52 -05002379 struct nlattr *rate_tlv, struct tcf_exts *exts, bool ovr,
2380 struct netlink_ext_ack *extack)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002381{
Linus Torvalds1da177e2005-04-16 15:20:36 -07002382#ifdef CONFIG_NET_CLS_ACT
2383 {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002384 struct tc_action *act;
Roman Mashakd04e6992018-03-08 16:59:17 -05002385 size_t attr_size = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002386
WANG Cong5da57f42013-12-15 20:15:07 -08002387 if (exts->police && tb[exts->police]) {
Jiri Pirko9fb9f252017-05-17 11:08:02 +02002388 act = tcf_action_init_1(net, tp, tb[exts->police],
2389 rate_tlv, "police", ovr,
Vlad Buslov789871b2018-07-05 17:24:25 +03002390 TCA_ACT_BIND, true, extack);
Patrick McHardyab27cfb2008-01-23 20:33:13 -08002391 if (IS_ERR(act))
2392 return PTR_ERR(act);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002393
WANG Cong33be6272013-12-15 20:15:05 -08002394 act->type = exts->type = TCA_OLD_COMPAT;
WANG Cong22dc13c2016-08-13 22:35:00 -07002395 exts->actions[0] = act;
2396 exts->nr_actions = 1;
WANG Cong5da57f42013-12-15 20:15:07 -08002397 } else if (exts->action && tb[exts->action]) {
Vlad Buslov90b73b72018-07-05 17:24:33 +03002398 int err;
WANG Cong22dc13c2016-08-13 22:35:00 -07002399
Jiri Pirko9fb9f252017-05-17 11:08:02 +02002400 err = tcf_action_init(net, tp, tb[exts->action],
2401 rate_tlv, NULL, ovr, TCA_ACT_BIND,
Vlad Buslov90b73b72018-07-05 17:24:33 +03002402 exts->actions, &attr_size, true,
Vlad Buslov789871b2018-07-05 17:24:25 +03002403 extack);
Vlad Buslov90b73b72018-07-05 17:24:33 +03002404 if (err < 0)
WANG Cong33be6272013-12-15 20:15:05 -08002405 return err;
Vlad Buslov90b73b72018-07-05 17:24:33 +03002406 exts->nr_actions = err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002407 }
Cong Wange4b95c42017-11-06 13:47:19 -08002408 exts->net = net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002409 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002410#else
WANG Cong5da57f42013-12-15 20:15:07 -08002411 if ((exts->action && tb[exts->action]) ||
Alexander Aring50a56192018-01-18 11:20:52 -05002412 (exts->police && tb[exts->police])) {
2413 NL_SET_ERR_MSG(extack, "Classifier actions are not supported per compile options (CONFIG_NET_CLS_ACT)");
Linus Torvalds1da177e2005-04-16 15:20:36 -07002414 return -EOPNOTSUPP;
Alexander Aring50a56192018-01-18 11:20:52 -05002415 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002416#endif
2417
2418 return 0;
2419}
Stephen Hemmingeraa767bf2008-01-21 02:26:41 -08002420EXPORT_SYMBOL(tcf_exts_validate);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002421
Jiri Pirko9b0d4442017-08-04 14:29:15 +02002422void tcf_exts_change(struct tcf_exts *dst, struct tcf_exts *src)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002423{
2424#ifdef CONFIG_NET_CLS_ACT
WANG Cong22dc13c2016-08-13 22:35:00 -07002425 struct tcf_exts old = *dst;
2426
Jiri Pirko9b0d4442017-08-04 14:29:15 +02002427 *dst = *src;
WANG Cong22dc13c2016-08-13 22:35:00 -07002428 tcf_exts_destroy(&old);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002429#endif
2430}
Stephen Hemmingeraa767bf2008-01-21 02:26:41 -08002431EXPORT_SYMBOL(tcf_exts_change);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002432
WANG Cong22dc13c2016-08-13 22:35:00 -07002433#ifdef CONFIG_NET_CLS_ACT
2434static struct tc_action *tcf_exts_first_act(struct tcf_exts *exts)
2435{
2436 if (exts->nr_actions == 0)
2437 return NULL;
2438 else
2439 return exts->actions[0];
2440}
2441#endif
WANG Cong33be6272013-12-15 20:15:05 -08002442
WANG Cong5da57f42013-12-15 20:15:07 -08002443int tcf_exts_dump(struct sk_buff *skb, struct tcf_exts *exts)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002444{
2445#ifdef CONFIG_NET_CLS_ACT
Cong Wang9cc63db2014-07-16 14:25:30 -07002446 struct nlattr *nest;
2447
Jiri Pirko978dfd82017-08-04 14:29:03 +02002448 if (exts->action && tcf_exts_has_actions(exts)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002449 /*
2450 * again for backward compatible mode - we want
2451 * to work with both old and new modes of entering
2452 * tc data even if iproute2 was newer - jhs
2453 */
WANG Cong33be6272013-12-15 20:15:05 -08002454 if (exts->type != TCA_OLD_COMPAT) {
WANG Cong5da57f42013-12-15 20:15:07 -08002455 nest = nla_nest_start(skb, exts->action);
Patrick McHardy4b3550ef2008-01-23 20:34:11 -08002456 if (nest == NULL)
2457 goto nla_put_failure;
WANG Cong22dc13c2016-08-13 22:35:00 -07002458
Vlad Buslov90b73b72018-07-05 17:24:33 +03002459 if (tcf_action_dump(skb, exts->actions, 0, 0) < 0)
Patrick McHardyadd93b62008-01-22 22:11:33 -08002460 goto nla_put_failure;
Patrick McHardy4b3550ef2008-01-23 20:34:11 -08002461 nla_nest_end(skb, nest);
WANG Cong5da57f42013-12-15 20:15:07 -08002462 } else if (exts->police) {
WANG Cong33be6272013-12-15 20:15:05 -08002463 struct tc_action *act = tcf_exts_first_act(exts);
WANG Cong5da57f42013-12-15 20:15:07 -08002464 nest = nla_nest_start(skb, exts->police);
Jamal Hadi Salim63acd682013-12-23 08:02:12 -05002465 if (nest == NULL || !act)
Patrick McHardy4b3550ef2008-01-23 20:34:11 -08002466 goto nla_put_failure;
WANG Cong33be6272013-12-15 20:15:05 -08002467 if (tcf_action_dump_old(skb, act, 0, 0) < 0)
Patrick McHardyadd93b62008-01-22 22:11:33 -08002468 goto nla_put_failure;
Patrick McHardy4b3550ef2008-01-23 20:34:11 -08002469 nla_nest_end(skb, nest);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002470 }
2471 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002472 return 0;
Cong Wang9cc63db2014-07-16 14:25:30 -07002473
2474nla_put_failure:
2475 nla_nest_cancel(skb, nest);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002476 return -1;
Cong Wang9cc63db2014-07-16 14:25:30 -07002477#else
2478 return 0;
2479#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002480}
Stephen Hemmingeraa767bf2008-01-21 02:26:41 -08002481EXPORT_SYMBOL(tcf_exts_dump);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002482
Stephen Hemmingeraa767bf2008-01-21 02:26:41 -08002483
WANG Cong5da57f42013-12-15 20:15:07 -08002484int tcf_exts_dump_stats(struct sk_buff *skb, struct tcf_exts *exts)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002485{
2486#ifdef CONFIG_NET_CLS_ACT
WANG Cong33be6272013-12-15 20:15:05 -08002487 struct tc_action *a = tcf_exts_first_act(exts);
Ignacy Gawędzkib057df22015-02-03 19:05:18 +01002488 if (a != NULL && tcf_action_copy_stats(skb, a, 1) < 0)
WANG Cong33be6272013-12-15 20:15:05 -08002489 return -1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002490#endif
2491 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002492}
Stephen Hemmingeraa767bf2008-01-21 02:26:41 -08002493EXPORT_SYMBOL(tcf_exts_dump_stats);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002494
Cong Wangaeb3fec2018-12-11 11:15:46 -08002495int tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type,
2496 void *type_data, bool err_stop)
Jiri Pirko717503b2017-10-11 09:41:09 +02002497{
Cong Wangaeb3fec2018-12-11 11:15:46 -08002498 struct tcf_block_cb *block_cb;
2499 int ok_count = 0;
2500 int err;
2501
2502 /* Make sure all netdevs sharing this block are offload-capable. */
2503 if (block->nooffloaddevcnt && err_stop)
2504 return -EOPNOTSUPP;
2505
2506 list_for_each_entry(block_cb, &block->cb_list, list) {
2507 err = block_cb->cb(type, type_data, block_cb->cb_priv);
2508 if (err) {
2509 if (err_stop)
2510 return err;
2511 } else {
2512 ok_count++;
2513 }
2514 }
2515 return ok_count;
Jiri Pirko717503b2017-10-11 09:41:09 +02002516}
2517EXPORT_SYMBOL(tc_setup_cb_call);
Jiri Pirkob3f55bd2017-10-11 09:41:08 +02002518
Jiri Pirko48617382018-01-17 11:46:46 +01002519static __net_init int tcf_net_init(struct net *net)
2520{
2521 struct tcf_net *tn = net_generic(net, tcf_net_id);
2522
Vlad Buslovab281622018-09-24 19:22:56 +03002523 spin_lock_init(&tn->idr_lock);
Jiri Pirko48617382018-01-17 11:46:46 +01002524 idr_init(&tn->idr);
2525 return 0;
2526}
2527
2528static void __net_exit tcf_net_exit(struct net *net)
2529{
2530 struct tcf_net *tn = net_generic(net, tcf_net_id);
2531
2532 idr_destroy(&tn->idr);
2533}
2534
2535static struct pernet_operations tcf_net_ops = {
2536 .init = tcf_net_init,
2537 .exit = tcf_net_exit,
2538 .id = &tcf_net_id,
2539 .size = sizeof(struct tcf_net),
2540};
2541
Linus Torvalds1da177e2005-04-16 15:20:36 -07002542static int __init tc_filter_init(void)
2543{
Jiri Pirko48617382018-01-17 11:46:46 +01002544 int err;
2545
Cong Wang7aa00452017-10-26 18:24:28 -07002546 tc_filter_wq = alloc_ordered_workqueue("tc_filter_workqueue", 0);
2547 if (!tc_filter_wq)
2548 return -ENOMEM;
2549
Jiri Pirko48617382018-01-17 11:46:46 +01002550 err = register_pernet_subsys(&tcf_net_ops);
2551 if (err)
2552 goto err_register_pernet_subsys;
2553
John Hurley7f76fa32018-11-09 21:21:26 -08002554 err = rhashtable_init(&indr_setup_block_ht,
2555 &tc_indr_setup_block_ht_params);
2556 if (err)
2557 goto err_rhash_setup_block_ht;
2558
Vlad Buslovc431f892018-05-31 09:52:53 +03002559 rtnl_register(PF_UNSPEC, RTM_NEWTFILTER, tc_new_tfilter, NULL, 0);
2560 rtnl_register(PF_UNSPEC, RTM_DELTFILTER, tc_del_tfilter, NULL, 0);
2561 rtnl_register(PF_UNSPEC, RTM_GETTFILTER, tc_get_tfilter,
Florian Westphalb97bac62017-08-09 20:41:48 +02002562 tc_dump_tfilter, 0);
Jiri Pirko32a4f5e2018-07-23 09:23:06 +02002563 rtnl_register(PF_UNSPEC, RTM_NEWCHAIN, tc_ctl_chain, NULL, 0);
2564 rtnl_register(PF_UNSPEC, RTM_DELCHAIN, tc_ctl_chain, NULL, 0);
2565 rtnl_register(PF_UNSPEC, RTM_GETCHAIN, tc_ctl_chain,
2566 tc_dump_chain, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002567
Linus Torvalds1da177e2005-04-16 15:20:36 -07002568 return 0;
Jiri Pirko48617382018-01-17 11:46:46 +01002569
John Hurley7f76fa32018-11-09 21:21:26 -08002570err_rhash_setup_block_ht:
2571 unregister_pernet_subsys(&tcf_net_ops);
Jiri Pirko48617382018-01-17 11:46:46 +01002572err_register_pernet_subsys:
2573 destroy_workqueue(tc_filter_wq);
2574 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002575}
2576
2577subsys_initcall(tc_filter_init);