blob: f409406254ddf2e204676bb8bdfb95d0cb3a0e71 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
tcharding722c9a02017-02-09 17:56:04 +11002 * NET3 Protocol independent device support routines.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Derived from the non IP parts of dev.c 1.0.19
tcharding722c9a02017-02-09 17:56:04 +110010 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Mark Evans, <evansmp@uhura.aston.ac.uk>
13 *
14 * Additional Authors:
15 * Florian la Roche <rzsfl@rz.uni-sb.de>
16 * Alan Cox <gw4pts@gw4pts.ampr.org>
17 * David Hinds <dahinds@users.sourceforge.net>
18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19 * Adam Sulmicki <adam@cfar.umd.edu>
20 * Pekka Riikonen <priikone@poesidon.pspt.fi>
21 *
22 * Changes:
23 * D.J. Barrow : Fixed bug where dev->refcnt gets set
tcharding722c9a02017-02-09 17:56:04 +110024 * to 2 if register_netdev gets called
25 * before net_dev_init & also removed a
26 * few lines of code in the process.
Linus Torvalds1da177e2005-04-16 15:20:36 -070027 * Alan Cox : device private ioctl copies fields back.
28 * Alan Cox : Transmit queue code does relevant
29 * stunts to keep the queue safe.
30 * Alan Cox : Fixed double lock.
31 * Alan Cox : Fixed promisc NULL pointer trap
32 * ???????? : Support the full private ioctl range
33 * Alan Cox : Moved ioctl permission check into
34 * drivers
35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
36 * Alan Cox : 100 backlog just doesn't cut it when
37 * you start doing multicast video 8)
38 * Alan Cox : Rewrote net_bh and list manager.
tcharding722c9a02017-02-09 17:56:04 +110039 * Alan Cox : Fix ETH_P_ALL echoback lengths.
Linus Torvalds1da177e2005-04-16 15:20:36 -070040 * Alan Cox : Took out transmit every packet pass
41 * Saved a few bytes in the ioctl handler
42 * Alan Cox : Network driver sets packet type before
43 * calling netif_rx. Saves a function
44 * call a packet.
45 * Alan Cox : Hashed net_bh()
46 * Richard Kooijman: Timestamp fixes.
47 * Alan Cox : Wrong field in SIOCGIFDSTADDR
48 * Alan Cox : Device lock protection.
tcharding722c9a02017-02-09 17:56:04 +110049 * Alan Cox : Fixed nasty side effect of device close
Linus Torvalds1da177e2005-04-16 15:20:36 -070050 * changes.
51 * Rudi Cilibrasi : Pass the right thing to
52 * set_mac_address()
53 * Dave Miller : 32bit quantity for the device lock to
54 * make it work out on a Sparc.
55 * Bjorn Ekwall : Added KERNELD hack.
56 * Alan Cox : Cleaned up the backlog initialise.
57 * Craig Metz : SIOCGIFCONF fix if space for under
58 * 1 device.
59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
60 * is no device open function.
61 * Andi Kleen : Fix error reporting for SIOCGIFCONF
62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
63 * Cyrus Durgin : Cleaned for KMOD
64 * Adam Sulmicki : Bug Fix : Network Device Unload
65 * A network device unload needs to purge
66 * the backlog queue.
67 * Paul Rusty Russell : SIOCSIFNAME
68 * Pekka Riikonen : Netdev boot-time settings code
69 * Andrew Morton : Make unregister_netdevice wait
tcharding722c9a02017-02-09 17:56:04 +110070 * indefinitely on dev->refcnt
71 * J Hadi Salim : - Backlog queue sampling
Linus Torvalds1da177e2005-04-16 15:20:36 -070072 * - netif_rx() feedback
73 */
74
Linus Torvalds7c0f6ba2016-12-24 11:46:01 -080075#include <linux/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070076#include <linux/bitops.h>
Randy Dunlap4fc268d2006-01-11 12:17:47 -080077#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070078#include <linux/cpu.h>
79#include <linux/types.h>
80#include <linux/kernel.h>
stephen hemminger08e98972009-11-10 07:20:34 +000081#include <linux/hash.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090082#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070083#include <linux/sched.h>
Vlastimil Babkaf1083042017-05-08 15:59:53 -070084#include <linux/sched/mm.h>
Arjan van de Ven4a3e2f72006-03-20 22:33:17 -080085#include <linux/mutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070086#include <linux/string.h>
87#include <linux/mm.h>
88#include <linux/socket.h>
89#include <linux/sockios.h>
90#include <linux/errno.h>
91#include <linux/interrupt.h>
92#include <linux/if_ether.h>
93#include <linux/netdevice.h>
94#include <linux/etherdevice.h>
Ben Hutchings0187bdf2008-06-19 16:15:47 -070095#include <linux/ethtool.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070096#include <linux/skbuff.h>
Brenden Blancoa7862b42016-07-19 12:16:48 -070097#include <linux/bpf.h>
David S. Millerb5cdae32017-04-18 15:36:58 -040098#include <linux/bpf_trace.h>
Eric W. Biederman457c4cb2007-09-12 12:01:34 +020099#include <net/net_namespace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100#include <net/sock.h>
Eric Dumazet02d62e82015-11-18 06:30:52 -0800101#include <net/busy_poll.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700102#include <linux/rtnetlink.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103#include <linux/stat.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700104#include <net/dst.h>
Pravin B Shelarfc4099f2015-10-22 18:17:16 -0700105#include <net/dst_metadata.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106#include <net/pkt_sched.h>
Jiri Pirko87d83092017-05-17 11:07:54 +0200107#include <net/pkt_cls.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108#include <net/checksum.h>
Arnd Bergmann44540962009-11-26 06:07:08 +0000109#include <net/xfrm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110#include <linux/highmem.h>
111#include <linux/init.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113#include <linux/netpoll.h>
114#include <linux/rcupdate.h>
115#include <linux/delay.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116#include <net/iw_handler.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700117#include <asm/current.h>
Steve Grubb5bdb9882005-12-03 08:39:35 -0500118#include <linux/audit.h>
Chris Leechdb217332006-06-17 21:24:58 -0700119#include <linux/dmaengine.h>
Herbert Xuf6a78bf2006-06-22 02:57:17 -0700120#include <linux/err.h>
David S. Millerc7fa9d12006-08-15 16:34:13 -0700121#include <linux/ctype.h>
Jarek Poplawski723e98b72007-05-15 22:46:18 -0700122#include <linux/if_arp.h>
Ben Hutchings6de329e2008-06-16 17:02:28 -0700123#include <linux/if_vlan.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700124#include <linux/ip.h>
Alexander Duyckad55dca2008-09-20 22:05:50 -0700125#include <net/ip.h>
Simon Horman25cd9ba2014-10-06 05:05:13 -0700126#include <net/mpls.h>
David S. Miller8f0f2222008-07-15 03:47:03 -0700127#include <linux/ipv6.h>
128#include <linux/in.h>
David S. Millerb6b2fed2008-07-21 09:48:06 -0700129#include <linux/jhash.h>
130#include <linux/random.h>
David S. Miller9cbc1cb2009-06-15 03:02:23 -0700131#include <trace/events/napi.h>
Koki Sanagicf66ba52010-08-23 18:45:02 +0900132#include <trace/events/net.h>
Koki Sanagi07dc22e2010-08-23 18:46:12 +0900133#include <trace/events/skb.h>
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +0000134#include <linux/pci.h>
Stephen Rothwellcaeda9b2010-09-16 21:39:16 -0700135#include <linux/inetdevice.h>
Ben Hutchingsc4454772011-01-19 11:03:53 +0000136#include <linux/cpu_rmap.h>
Ingo Molnarc5905af2012-02-24 08:31:31 +0100137#include <linux/static_key.h>
Eliezer Tamiraf12fa62013-06-10 11:39:41 +0300138#include <linux/hashtable.h>
Eric Dumazet60877a32013-06-20 01:15:51 -0700139#include <linux/vmalloc.h>
Michal Kubeček529d0482013-11-15 06:18:50 +0100140#include <linux/if_macvlan.h>
Willem de Bruijne7fd2882014-08-04 22:11:48 -0400141#include <linux/errqueue.h>
Eric Dumazet3b47d302014-11-06 21:09:44 -0800142#include <linux/hrtimer.h>
Pablo Neirae687ad62015-05-13 18:19:38 +0200143#include <linux/netfilter_ingress.h>
Hariprasad Shenai40e4e712016-06-08 18:09:08 +0530144#include <linux/crash_dump.h>
Davide Carattib72b5bf2017-05-18 15:44:38 +0200145#include <linux/sctp.h>
Sabrina Dubrocaae847f42017-07-21 12:49:31 +0200146#include <net/udp_tunnel.h>
Nicolas Dichtel6621dd22017-10-03 13:53:23 +0200147#include <linux/net_namespace.h>
Paolo Abeniaaa5d902018-12-14 11:51:58 +0100148#include <linux/indirect_call_wrapper.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700149
Pavel Emelyanov342709e2007-10-23 21:14:45 -0700150#include "net-sysfs.h"
151
Herbert Xud565b0a2008-12-15 23:38:52 -0800152#define MAX_GRO_SKBS 8
153
Herbert Xu5d38a072009-01-04 16:13:40 -0800154/* This should be increased if a protocol with a bigger head is added. */
155#define GRO_MAX_HEAD (MAX_HEADER + 128)
156
Linus Torvalds1da177e2005-04-16 15:20:36 -0700157static DEFINE_SPINLOCK(ptype_lock);
Vlad Yasevich62532da9d2012-11-15 08:49:10 +0000158static DEFINE_SPINLOCK(offload_lock);
Cong Wang900ff8c2013-02-18 19:20:33 +0000159struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
160struct list_head ptype_all __read_mostly; /* Taps */
Vlad Yasevich62532da9d2012-11-15 08:49:10 +0000161static struct list_head offload_base __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700162
Ben Hutchingsae78dbf2014-01-10 22:17:24 +0000163static int netif_rx_internal(struct sk_buff *skb);
Loic Prylli54951192014-07-01 21:39:43 -0700164static int call_netdevice_notifiers_info(unsigned long val,
Loic Prylli54951192014-07-01 21:39:43 -0700165 struct netdev_notifier_info *info);
Petr Machata26372602018-12-06 17:05:45 +0000166static int call_netdevice_notifiers_extack(unsigned long val,
167 struct net_device *dev,
168 struct netlink_ext_ack *extack);
Miroslav Lichvar90b602f2017-05-19 17:52:37 +0200169static struct napi_struct *napi_by_id(unsigned int napi_id);
Ben Hutchingsae78dbf2014-01-10 22:17:24 +0000170
Linus Torvalds1da177e2005-04-16 15:20:36 -0700171/*
Pavel Emelianov7562f872007-05-03 15:13:45 -0700172 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173 * semaphore.
174 *
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800175 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
Linus Torvalds1da177e2005-04-16 15:20:36 -0700176 *
177 * Writers must hold the rtnl semaphore while they loop through the
Pavel Emelianov7562f872007-05-03 15:13:45 -0700178 * dev_base_head list, and hold dev_base_lock for writing when they do the
Linus Torvalds1da177e2005-04-16 15:20:36 -0700179 * actual updates. This allows pure readers to access the list even
180 * while a writer is preparing to update it.
181 *
182 * To put it another way, dev_base_lock is held for writing only to
183 * protect against pure readers; the rtnl semaphore provides the
184 * protection against other writers.
185 *
186 * See, for example usages, register_netdevice() and
187 * unregister_netdevice(), which must be called with the rtnl
188 * semaphore held.
189 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190DEFINE_RWLOCK(dev_base_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700191EXPORT_SYMBOL(dev_base_lock);
192
Florian Westphal6c557002017-10-02 23:50:05 +0200193static DEFINE_MUTEX(ifalias_mutex);
194
Eliezer Tamiraf12fa62013-06-10 11:39:41 +0300195/* protects napi_hash addition/deletion and napi_gen_id */
196static DEFINE_SPINLOCK(napi_hash_lock);
197
Eric Dumazet52bd2d62015-11-18 06:30:50 -0800198static unsigned int napi_gen_id = NR_CPUS;
Eric Dumazet6180d9d2015-11-18 06:31:01 -0800199static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
Eliezer Tamiraf12fa62013-06-10 11:39:41 +0300200
Thomas Gleixner18afa4b2013-07-23 16:13:17 +0200201static seqcount_t devnet_rename_seq;
Brian Haleyc91f6df2012-11-26 05:21:08 +0000202
Thomas Graf4e985ad2011-06-21 03:11:20 +0000203static inline void dev_base_seq_inc(struct net *net)
204{
tcharding643aa9c2017-02-09 17:56:05 +1100205 while (++net->dev_base_seq == 0)
206 ;
Thomas Graf4e985ad2011-06-21 03:11:20 +0000207}
208
Eric W. Biederman881d9662007-09-17 11:56:21 -0700209static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700210{
Linus Torvalds8387ff22016-06-10 07:51:30 -0700211 unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ));
Eric Dumazet95c96172012-04-15 05:58:06 +0000212
stephen hemminger08e98972009-11-10 07:20:34 +0000213 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700214}
215
Eric W. Biederman881d9662007-09-17 11:56:21 -0700216static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217{
Eric Dumazet7c28bd02009-10-24 06:13:17 -0700218 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700219}
220
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000221static inline void rps_lock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000222{
223#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000224 spin_lock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000225#endif
226}
227
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000228static inline void rps_unlock(struct softnet_data *sd)
Changli Gao152102c2010-03-30 20:16:22 +0000229{
230#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +0000231 spin_unlock(&sd->input_pkt_queue.lock);
Changli Gao152102c2010-03-30 20:16:22 +0000232#endif
233}
234
Eric W. Biedermance286d32007-09-12 13:53:49 +0200235/* Device list insertion */
dingtianhong53759be2013-04-17 22:17:50 +0000236static void list_netdevice(struct net_device *dev)
Eric W. Biedermance286d32007-09-12 13:53:49 +0200237{
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +0900238 struct net *net = dev_net(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200239
240 ASSERT_RTNL();
241
242 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800243 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
Eric Dumazet72c95282009-10-30 07:11:27 +0000244 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000245 hlist_add_head_rcu(&dev->index_hlist,
246 dev_index_hash(net, dev->ifindex));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200247 write_unlock_bh(&dev_base_lock);
Thomas Graf4e985ad2011-06-21 03:11:20 +0000248
249 dev_base_seq_inc(net);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200250}
251
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000252/* Device list removal
253 * caller must respect a RCU grace period before freeing/reusing dev
254 */
Eric W. Biedermance286d32007-09-12 13:53:49 +0200255static void unlist_netdevice(struct net_device *dev)
256{
257 ASSERT_RTNL();
258
259 /* Unlink dev from the device chain */
260 write_lock_bh(&dev_base_lock);
Eric Dumazetc6d14c82009-11-04 05:43:23 -0800261 list_del_rcu(&dev->dev_list);
Eric Dumazet72c95282009-10-30 07:11:27 +0000262 hlist_del_rcu(&dev->name_hlist);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000263 hlist_del_rcu(&dev->index_hlist);
Eric W. Biedermance286d32007-09-12 13:53:49 +0200264 write_unlock_bh(&dev_base_lock);
Thomas Graf4e985ad2011-06-21 03:11:20 +0000265
266 dev_base_seq_inc(dev_net(dev));
Eric W. Biedermance286d32007-09-12 13:53:49 +0200267}
268
Linus Torvalds1da177e2005-04-16 15:20:36 -0700269/*
270 * Our notifier list
271 */
272
Alan Sternf07d5b92006-05-09 15:23:03 -0700273static RAW_NOTIFIER_HEAD(netdev_chain);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700274
275/*
276 * Device drivers call our routines to queue packets here. We empty the
277 * queue in the local softnet handler.
278 */
Stephen Hemmingerbea33482007-10-03 16:41:36 -0700279
Eric Dumazet9958da02010-04-17 04:17:02 +0000280DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700281EXPORT_PER_CPU_SYMBOL(softnet_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700282
David S. Millercf508b12008-07-22 14:16:42 -0700283#ifdef CONFIG_LOCKDEP
Jarek Poplawski723e98b72007-05-15 22:46:18 -0700284/*
David S. Millerc773e842008-07-08 23:13:53 -0700285 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
Jarek Poplawski723e98b72007-05-15 22:46:18 -0700286 * according to dev->type
287 */
tcharding643aa9c2017-02-09 17:56:05 +1100288static const unsigned short netdev_lock_type[] = {
289 ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
Jarek Poplawski723e98b72007-05-15 22:46:18 -0700290 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
291 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
292 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
293 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
294 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
295 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
296 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
297 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
298 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
299 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
300 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
Paul Gortmaker211ed862012-05-10 17:14:35 -0400301 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
302 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
303 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
Jarek Poplawski723e98b72007-05-15 22:46:18 -0700304
tcharding643aa9c2017-02-09 17:56:05 +1100305static const char *const netdev_lock_name[] = {
306 "_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
307 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
308 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
309 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
310 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
311 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
312 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
313 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
314 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
315 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
316 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
317 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
318 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
319 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
320 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
Jarek Poplawski723e98b72007-05-15 22:46:18 -0700321
322static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
David S. Millercf508b12008-07-22 14:16:42 -0700323static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
Jarek Poplawski723e98b72007-05-15 22:46:18 -0700324
325static inline unsigned short netdev_lock_pos(unsigned short dev_type)
326{
327 int i;
328
329 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
330 if (netdev_lock_type[i] == dev_type)
331 return i;
332 /* the last key is used by default */
333 return ARRAY_SIZE(netdev_lock_type) - 1;
334}
335
David S. Millercf508b12008-07-22 14:16:42 -0700336static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
337 unsigned short dev_type)
Jarek Poplawski723e98b72007-05-15 22:46:18 -0700338{
339 int i;
340
341 i = netdev_lock_pos(dev_type);
342 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
343 netdev_lock_name[i]);
344}
David S. Millercf508b12008-07-22 14:16:42 -0700345
346static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
347{
348 int i;
349
350 i = netdev_lock_pos(dev->type);
351 lockdep_set_class_and_name(&dev->addr_list_lock,
352 &netdev_addr_lock_key[i],
353 netdev_lock_name[i]);
354}
Jarek Poplawski723e98b72007-05-15 22:46:18 -0700355#else
David S. Millercf508b12008-07-22 14:16:42 -0700356static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
357 unsigned short dev_type)
358{
359}
360static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
Jarek Poplawski723e98b72007-05-15 22:46:18 -0700361{
362}
363#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700364
365/*******************************************************************************
tchardingeb13da12017-02-09 17:56:06 +1100366 *
367 * Protocol management and registration routines
368 *
369 *******************************************************************************/
Linus Torvalds1da177e2005-04-16 15:20:36 -0700370
Linus Torvalds1da177e2005-04-16 15:20:36 -0700371
372/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700373 * Add a protocol ID to the list. Now that the input handler is
374 * smarter we can dispense with all the messy stuff that used to be
375 * here.
376 *
377 * BEWARE!!! Protocol handlers, mangling input packets,
378 * MUST BE last in hash buckets and checking protocol handlers
379 * MUST start from promiscuous ptype_all chain in net_bh.
380 * It is true now, do not change it.
381 * Explanation follows: if protocol handler, mangling packet, will
382 * be the first on list, it is not able to sense, that packet
383 * is cloned and should be copied-on-write, so that it will
384 * change it and subsequent readers will get broken packet.
385 * --ANK (980803)
386 */
387
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000388static inline struct list_head *ptype_head(const struct packet_type *pt)
389{
390 if (pt->type == htons(ETH_P_ALL))
Salam Noureddine7866a622015-01-27 11:35:48 -0800391 return pt->dev ? &pt->dev->ptype_all : &ptype_all;
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000392 else
Salam Noureddine7866a622015-01-27 11:35:48 -0800393 return pt->dev ? &pt->dev->ptype_specific :
394 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000395}
396
Linus Torvalds1da177e2005-04-16 15:20:36 -0700397/**
398 * dev_add_pack - add packet handler
399 * @pt: packet type declaration
400 *
401 * Add a protocol handler to the networking stack. The passed &packet_type
402 * is linked into kernel lists and may not be freed until it has been
403 * removed from the kernel lists.
404 *
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900405 * This call does not sleep therefore it can not
Linus Torvalds1da177e2005-04-16 15:20:36 -0700406 * guarantee all CPU's that are in middle of receiving packets
407 * will see the new packet type (until the next received packet).
408 */
409
410void dev_add_pack(struct packet_type *pt)
411{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000412 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700413
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000414 spin_lock(&ptype_lock);
415 list_add_rcu(&pt->list, head);
416 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700417}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700418EXPORT_SYMBOL(dev_add_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700419
Linus Torvalds1da177e2005-04-16 15:20:36 -0700420/**
421 * __dev_remove_pack - remove packet handler
422 * @pt: packet type declaration
423 *
424 * Remove a protocol handler that was previously added to the kernel
425 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
426 * from the kernel lists and can be freed or reused once this function
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900427 * returns.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700428 *
429 * The packet type might still be in use by receivers
430 * and must not be freed until after all the CPU's have gone
431 * through a quiescent state.
432 */
433void __dev_remove_pack(struct packet_type *pt)
434{
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000435 struct list_head *head = ptype_head(pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700436 struct packet_type *pt1;
437
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000438 spin_lock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700439
440 list_for_each_entry(pt1, head, list) {
441 if (pt == pt1) {
442 list_del_rcu(&pt->list);
443 goto out;
444 }
445 }
446
Joe Perches7b6cd1c2012-02-01 10:54:43 +0000447 pr_warn("dev_remove_pack: %p not found\n", pt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700448out:
Eric Dumazetc07b68e2010-09-02 03:53:46 +0000449 spin_unlock(&ptype_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700450}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700451EXPORT_SYMBOL(__dev_remove_pack);
452
Linus Torvalds1da177e2005-04-16 15:20:36 -0700453/**
454 * dev_remove_pack - remove packet handler
455 * @pt: packet type declaration
456 *
457 * Remove a protocol handler that was previously added to the kernel
458 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
459 * from the kernel lists and can be freed or reused once this function
460 * returns.
461 *
462 * This call sleeps to guarantee that no CPU is looking at the packet
463 * type after return.
464 */
465void dev_remove_pack(struct packet_type *pt)
466{
467 __dev_remove_pack(pt);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900468
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469 synchronize_net();
470}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700471EXPORT_SYMBOL(dev_remove_pack);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700472
Vlad Yasevich62532da9d2012-11-15 08:49:10 +0000473
474/**
475 * dev_add_offload - register offload handlers
476 * @po: protocol offload declaration
477 *
478 * Add protocol offload handlers to the networking stack. The passed
479 * &proto_offload is linked into kernel lists and may not be freed until
480 * it has been removed from the kernel lists.
481 *
482 * This call does not sleep therefore it can not
483 * guarantee all CPU's that are in middle of receiving packets
484 * will see the new offload handlers (until the next received packet).
485 */
486void dev_add_offload(struct packet_offload *po)
487{
David S. Millerbdef7de2015-06-01 14:56:09 -0700488 struct packet_offload *elem;
Vlad Yasevich62532da9d2012-11-15 08:49:10 +0000489
490 spin_lock(&offload_lock);
David S. Millerbdef7de2015-06-01 14:56:09 -0700491 list_for_each_entry(elem, &offload_base, list) {
492 if (po->priority < elem->priority)
493 break;
494 }
495 list_add_rcu(&po->list, elem->list.prev);
Vlad Yasevich62532da9d2012-11-15 08:49:10 +0000496 spin_unlock(&offload_lock);
497}
498EXPORT_SYMBOL(dev_add_offload);
499
500/**
501 * __dev_remove_offload - remove offload handler
502 * @po: packet offload declaration
503 *
504 * Remove a protocol offload handler that was previously added to the
505 * kernel offload handlers by dev_add_offload(). The passed &offload_type
506 * is removed from the kernel lists and can be freed or reused once this
507 * function returns.
508 *
509 * The packet type might still be in use by receivers
510 * and must not be freed until after all the CPU's have gone
511 * through a quiescent state.
512 */
stephen hemminger1d143d92013-12-29 14:01:29 -0800513static void __dev_remove_offload(struct packet_offload *po)
Vlad Yasevich62532da9d2012-11-15 08:49:10 +0000514{
515 struct list_head *head = &offload_base;
516 struct packet_offload *po1;
517
Eric Dumazetc53aa502012-11-16 08:08:23 +0000518 spin_lock(&offload_lock);
Vlad Yasevich62532da9d2012-11-15 08:49:10 +0000519
520 list_for_each_entry(po1, head, list) {
521 if (po == po1) {
522 list_del_rcu(&po->list);
523 goto out;
524 }
525 }
526
527 pr_warn("dev_remove_offload: %p not found\n", po);
528out:
Eric Dumazetc53aa502012-11-16 08:08:23 +0000529 spin_unlock(&offload_lock);
Vlad Yasevich62532da9d2012-11-15 08:49:10 +0000530}
Vlad Yasevich62532da9d2012-11-15 08:49:10 +0000531
532/**
533 * dev_remove_offload - remove packet offload handler
534 * @po: packet offload declaration
535 *
536 * Remove a packet offload handler that was previously added to the kernel
537 * offload handlers by dev_add_offload(). The passed &offload_type is
538 * removed from the kernel lists and can be freed or reused once this
539 * function returns.
540 *
541 * This call sleeps to guarantee that no CPU is looking at the packet
542 * type after return.
543 */
544void dev_remove_offload(struct packet_offload *po)
545{
546 __dev_remove_offload(po);
547
548 synchronize_net();
549}
550EXPORT_SYMBOL(dev_remove_offload);
551
Linus Torvalds1da177e2005-04-16 15:20:36 -0700552/******************************************************************************
tchardingeb13da12017-02-09 17:56:06 +1100553 *
554 * Device Boot-time Settings Routines
555 *
556 ******************************************************************************/
Linus Torvalds1da177e2005-04-16 15:20:36 -0700557
558/* Boot time configuration table */
559static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
560
561/**
562 * netdev_boot_setup_add - add new setup entry
563 * @name: name of the device
564 * @map: configured settings for the device
565 *
566 * Adds new setup entry to the dev_boot_setup list. The function
567 * returns 0 on error and 1 on success. This is a generic routine to
568 * all netdevices.
569 */
570static int netdev_boot_setup_add(char *name, struct ifmap *map)
571{
572 struct netdev_boot_setup *s;
573 int i;
574
575 s = dev_boot_setup;
576 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
577 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
578 memset(s[i].name, 0, sizeof(s[i].name));
Wang Chen93b3cff92008-07-01 19:57:19 -0700579 strlcpy(s[i].name, name, IFNAMSIZ);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700580 memcpy(&s[i].map, map, sizeof(s[i].map));
581 break;
582 }
583 }
584
585 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
586}
587
588/**
tcharding722c9a02017-02-09 17:56:04 +1100589 * netdev_boot_setup_check - check boot time settings
590 * @dev: the netdevice
Linus Torvalds1da177e2005-04-16 15:20:36 -0700591 *
tcharding722c9a02017-02-09 17:56:04 +1100592 * Check boot time settings for the device.
593 * The found settings are set for the device to be used
594 * later in the device probing.
595 * Returns 0 if no settings found, 1 if they are.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700596 */
597int netdev_boot_setup_check(struct net_device *dev)
598{
599 struct netdev_boot_setup *s = dev_boot_setup;
600 int i;
601
602 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
603 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
Wang Chen93b3cff92008-07-01 19:57:19 -0700604 !strcmp(dev->name, s[i].name)) {
tcharding722c9a02017-02-09 17:56:04 +1100605 dev->irq = s[i].map.irq;
606 dev->base_addr = s[i].map.base_addr;
607 dev->mem_start = s[i].map.mem_start;
608 dev->mem_end = s[i].map.mem_end;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700609 return 1;
610 }
611 }
612 return 0;
613}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700614EXPORT_SYMBOL(netdev_boot_setup_check);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700615
616
617/**
tcharding722c9a02017-02-09 17:56:04 +1100618 * netdev_boot_base - get address from boot time settings
619 * @prefix: prefix for network device
620 * @unit: id for network device
Linus Torvalds1da177e2005-04-16 15:20:36 -0700621 *
tcharding722c9a02017-02-09 17:56:04 +1100622 * Check boot time settings for the base address of device.
623 * The found settings are set for the device to be used
624 * later in the device probing.
625 * Returns 0 if no settings found.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700626 */
627unsigned long netdev_boot_base(const char *prefix, int unit)
628{
629 const struct netdev_boot_setup *s = dev_boot_setup;
630 char name[IFNAMSIZ];
631 int i;
632
633 sprintf(name, "%s%d", prefix, unit);
634
635 /*
636 * If device already registered then return base of 1
637 * to indicate not to probe for this interface
638 */
Eric W. Biederman881d9662007-09-17 11:56:21 -0700639 if (__dev_get_by_name(&init_net, name))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700640 return 1;
641
642 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
643 if (!strcmp(name, s[i].name))
644 return s[i].map.base_addr;
645 return 0;
646}
647
648/*
649 * Saves at boot time configured settings for any netdevice.
650 */
651int __init netdev_boot_setup(char *str)
652{
653 int ints[5];
654 struct ifmap map;
655
656 str = get_options(str, ARRAY_SIZE(ints), ints);
657 if (!str || !*str)
658 return 0;
659
660 /* Save settings */
661 memset(&map, 0, sizeof(map));
662 if (ints[0] > 0)
663 map.irq = ints[1];
664 if (ints[0] > 1)
665 map.base_addr = ints[2];
666 if (ints[0] > 2)
667 map.mem_start = ints[3];
668 if (ints[0] > 3)
669 map.mem_end = ints[4];
670
671 /* Add new entry to the list */
672 return netdev_boot_setup_add(str, &map);
673}
674
675__setup("netdev=", netdev_boot_setup);
676
677/*******************************************************************************
tchardingeb13da12017-02-09 17:56:06 +1100678 *
679 * Device Interface Subroutines
680 *
681 *******************************************************************************/
Linus Torvalds1da177e2005-04-16 15:20:36 -0700682
683/**
Nicolas Dichtela54acb32015-04-02 17:07:00 +0200684 * dev_get_iflink - get 'iflink' value of a interface
685 * @dev: targeted interface
686 *
687 * Indicates the ifindex the interface is linked to.
688 * Physical interfaces have the same 'ifindex' and 'iflink' values.
689 */
690
691int dev_get_iflink(const struct net_device *dev)
692{
693 if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
694 return dev->netdev_ops->ndo_get_iflink(dev);
695
Nicolas Dichtel7a66bbc2015-04-02 17:07:09 +0200696 return dev->ifindex;
Nicolas Dichtela54acb32015-04-02 17:07:00 +0200697}
698EXPORT_SYMBOL(dev_get_iflink);
699
700/**
Pravin B Shelarfc4099f2015-10-22 18:17:16 -0700701 * dev_fill_metadata_dst - Retrieve tunnel egress information.
702 * @dev: targeted interface
703 * @skb: The packet.
704 *
705 * For better visibility of tunnel traffic OVS needs to retrieve
706 * egress tunnel information for a packet. Following API allows
707 * user to get this info.
708 */
709int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
710{
711 struct ip_tunnel_info *info;
712
713 if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst)
714 return -EINVAL;
715
716 info = skb_tunnel_info_unclone(skb);
717 if (!info)
718 return -ENOMEM;
719 if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
720 return -EINVAL;
721
722 return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
723}
724EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
725
726/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700727 * __dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700728 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700729 * @name: name to find
730 *
731 * Find an interface by name. Must be called under RTNL semaphore
732 * or @dev_base_lock. If the name is found a pointer to the device
733 * is returned. If the name is not found then %NULL is returned. The
734 * reference counters are not incremented so the caller must be
735 * careful with locks.
736 */
737
Eric W. Biederman881d9662007-09-17 11:56:21 -0700738struct net_device *__dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700739{
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700740 struct net_device *dev;
741 struct hlist_head *head = dev_name_hash(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700742
Sasha Levinb67bfe02013-02-27 17:06:00 -0800743 hlist_for_each_entry(dev, head, name_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700744 if (!strncmp(dev->name, name, IFNAMSIZ))
745 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700746
Linus Torvalds1da177e2005-04-16 15:20:36 -0700747 return NULL;
748}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700749EXPORT_SYMBOL(__dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700750
751/**
tcharding722c9a02017-02-09 17:56:04 +1100752 * dev_get_by_name_rcu - find a device by its name
753 * @net: the applicable net namespace
754 * @name: name to find
Eric Dumazet72c95282009-10-30 07:11:27 +0000755 *
tcharding722c9a02017-02-09 17:56:04 +1100756 * Find an interface by name.
757 * If the name is found a pointer to the device is returned.
758 * If the name is not found then %NULL is returned.
759 * The reference counters are not incremented so the caller must be
760 * careful with locks. The caller must hold RCU lock.
Eric Dumazet72c95282009-10-30 07:11:27 +0000761 */
762
763struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
764{
Eric Dumazet72c95282009-10-30 07:11:27 +0000765 struct net_device *dev;
766 struct hlist_head *head = dev_name_hash(net, name);
767
Sasha Levinb67bfe02013-02-27 17:06:00 -0800768 hlist_for_each_entry_rcu(dev, head, name_hlist)
Eric Dumazet72c95282009-10-30 07:11:27 +0000769 if (!strncmp(dev->name, name, IFNAMSIZ))
770 return dev;
771
772 return NULL;
773}
774EXPORT_SYMBOL(dev_get_by_name_rcu);
775
776/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700777 * dev_get_by_name - find a device by its name
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700778 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700779 * @name: name to find
780 *
781 * Find an interface by name. This can be called from any
782 * context and does its own locking. The returned handle has
783 * the usage count incremented and the caller must use dev_put() to
784 * release it when it is no longer needed. %NULL is returned if no
785 * matching device is found.
786 */
787
Eric W. Biederman881d9662007-09-17 11:56:21 -0700788struct net_device *dev_get_by_name(struct net *net, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700789{
790 struct net_device *dev;
791
Eric Dumazet72c95282009-10-30 07:11:27 +0000792 rcu_read_lock();
793 dev = dev_get_by_name_rcu(net, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700794 if (dev)
795 dev_hold(dev);
Eric Dumazet72c95282009-10-30 07:11:27 +0000796 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700797 return dev;
798}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700799EXPORT_SYMBOL(dev_get_by_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700800
801/**
802 * __dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700803 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700804 * @ifindex: index of device
805 *
806 * Search for an interface by index. Returns %NULL if the device
807 * is not found or a pointer to the device. The device has not
808 * had its reference counter increased so the caller must be careful
809 * about locking. The caller must hold either the RTNL semaphore
810 * or @dev_base_lock.
811 */
812
Eric W. Biederman881d9662007-09-17 11:56:21 -0700813struct net_device *__dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700814{
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700815 struct net_device *dev;
816 struct hlist_head *head = dev_index_hash(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700817
Sasha Levinb67bfe02013-02-27 17:06:00 -0800818 hlist_for_each_entry(dev, head, index_hlist)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700819 if (dev->ifindex == ifindex)
820 return dev;
Eric Dumazet0bd8d532009-10-30 01:40:11 -0700821
Linus Torvalds1da177e2005-04-16 15:20:36 -0700822 return NULL;
823}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700824EXPORT_SYMBOL(__dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700825
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000826/**
827 * dev_get_by_index_rcu - find a device by its ifindex
828 * @net: the applicable net namespace
829 * @ifindex: index of device
830 *
831 * Search for an interface by index. Returns %NULL if the device
832 * is not found or a pointer to the device. The device has not
833 * had its reference counter increased so the caller must be careful
834 * about locking. The caller must hold RCU lock.
835 */
836
837struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
838{
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000839 struct net_device *dev;
840 struct hlist_head *head = dev_index_hash(net, ifindex);
841
Sasha Levinb67bfe02013-02-27 17:06:00 -0800842 hlist_for_each_entry_rcu(dev, head, index_hlist)
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000843 if (dev->ifindex == ifindex)
844 return dev;
845
846 return NULL;
847}
848EXPORT_SYMBOL(dev_get_by_index_rcu);
849
Linus Torvalds1da177e2005-04-16 15:20:36 -0700850
851/**
852 * dev_get_by_index - find a device by its ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700853 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700854 * @ifindex: index of device
855 *
856 * Search for an interface by index. Returns NULL if the device
857 * is not found or a pointer to the device. The device returned has
858 * had a reference added and the pointer is safe until the user calls
859 * dev_put to indicate they have finished with it.
860 */
861
Eric W. Biederman881d9662007-09-17 11:56:21 -0700862struct net_device *dev_get_by_index(struct net *net, int ifindex)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700863{
864 struct net_device *dev;
865
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000866 rcu_read_lock();
867 dev = dev_get_by_index_rcu(net, ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700868 if (dev)
869 dev_hold(dev);
Eric Dumazetfb699dfd2009-10-19 19:18:49 +0000870 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700871 return dev;
872}
Eric Dumazetd1b19df2009-09-03 01:29:39 -0700873EXPORT_SYMBOL(dev_get_by_index);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700874
875/**
Miroslav Lichvar90b602f2017-05-19 17:52:37 +0200876 * dev_get_by_napi_id - find a device by napi_id
877 * @napi_id: ID of the NAPI struct
878 *
879 * Search for an interface by NAPI ID. Returns %NULL if the device
880 * is not found or a pointer to the device. The device has not had
881 * its reference counter increased so the caller must be careful
882 * about locking. The caller must hold RCU lock.
883 */
884
885struct net_device *dev_get_by_napi_id(unsigned int napi_id)
886{
887 struct napi_struct *napi;
888
889 WARN_ON_ONCE(!rcu_read_lock_held());
890
891 if (napi_id < MIN_NAPI_ID)
892 return NULL;
893
894 napi = napi_by_id(napi_id);
895
896 return napi ? napi->dev : NULL;
897}
898EXPORT_SYMBOL(dev_get_by_napi_id);
899
900/**
Nicolas Schichan5dbe7c12013-06-26 17:23:42 +0200901 * netdev_get_name - get a netdevice name, knowing its ifindex.
902 * @net: network namespace
903 * @name: a pointer to the buffer where the name will be stored.
904 * @ifindex: the ifindex of the interface to get the name from.
905 *
906 * The use of raw_seqcount_begin() and cond_resched() before
907 * retrying is required as we want to give the writers a chance
908 * to complete when CONFIG_PREEMPT is not set.
909 */
910int netdev_get_name(struct net *net, char *name, int ifindex)
911{
912 struct net_device *dev;
913 unsigned int seq;
914
915retry:
916 seq = raw_seqcount_begin(&devnet_rename_seq);
917 rcu_read_lock();
918 dev = dev_get_by_index_rcu(net, ifindex);
919 if (!dev) {
920 rcu_read_unlock();
921 return -ENODEV;
922 }
923
924 strcpy(name, dev->name);
925 rcu_read_unlock();
926 if (read_seqcount_retry(&devnet_rename_seq, seq)) {
927 cond_resched();
928 goto retry;
929 }
930
931 return 0;
932}
933
934/**
Eric Dumazet941666c2010-12-05 01:23:53 +0000935 * dev_getbyhwaddr_rcu - find a device by its hardware address
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700936 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700937 * @type: media type of device
938 * @ha: hardware address
939 *
940 * Search for an interface by MAC address. Returns NULL if the device
Eric Dumazetc5066532011-01-24 13:16:16 -0800941 * is not found or a pointer to the device.
942 * The caller must hold RCU or RTNL.
Eric Dumazet941666c2010-12-05 01:23:53 +0000943 * The returned device has not had its ref count increased
Linus Torvalds1da177e2005-04-16 15:20:36 -0700944 * and the caller must therefore be careful about locking
945 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700946 */
947
Eric Dumazet941666c2010-12-05 01:23:53 +0000948struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
949 const char *ha)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700950{
951 struct net_device *dev;
952
Eric Dumazet941666c2010-12-05 01:23:53 +0000953 for_each_netdev_rcu(net, dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700954 if (dev->type == type &&
955 !memcmp(dev->dev_addr, ha, dev->addr_len))
Pavel Emelianov7562f872007-05-03 15:13:45 -0700956 return dev;
957
958 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700959}
Eric Dumazet941666c2010-12-05 01:23:53 +0000960EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
Jochen Friedrichcf309e32005-09-22 04:44:55 -0300961
Eric W. Biederman881d9662007-09-17 11:56:21 -0700962struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700963{
964 struct net_device *dev;
965
966 ASSERT_RTNL();
Eric W. Biederman881d9662007-09-17 11:56:21 -0700967 for_each_netdev(net, dev)
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700968 if (dev->type == type)
Pavel Emelianov7562f872007-05-03 15:13:45 -0700969 return dev;
970
971 return NULL;
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700972}
Patrick McHardy4e9cac22007-05-03 03:28:13 -0700973EXPORT_SYMBOL(__dev_getfirstbyhwtype);
974
Eric W. Biederman881d9662007-09-17 11:56:21 -0700975struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700976{
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000977 struct net_device *dev, *ret = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700978
Eric Dumazet99fe3c32010-03-18 11:27:25 +0000979 rcu_read_lock();
980 for_each_netdev_rcu(net, dev)
981 if (dev->type == type) {
982 dev_hold(dev);
983 ret = dev;
984 break;
985 }
986 rcu_read_unlock();
987 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700988}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700989EXPORT_SYMBOL(dev_getfirstbyhwtype);
990
991/**
WANG Cong6c555492014-09-11 15:35:09 -0700992 * __dev_get_by_flags - find any device with given flags
Randy Dunlapc4ea43c2007-10-12 21:17:49 -0700993 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -0700994 * @if_flags: IFF_* values
995 * @mask: bitmask of bits in if_flags to check
996 *
997 * Search for any interface with the given flags. Returns NULL if a device
Eric Dumazetbb69ae02010-06-07 11:42:13 +0000998 * is not found or a pointer to the device. Must be called inside
WANG Cong6c555492014-09-11 15:35:09 -0700999 * rtnl_lock(), and result refcount is unchanged.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001000 */
1001
WANG Cong6c555492014-09-11 15:35:09 -07001002struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
1003 unsigned short mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001004{
Pavel Emelianov7562f872007-05-03 15:13:45 -07001005 struct net_device *dev, *ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001006
WANG Cong6c555492014-09-11 15:35:09 -07001007 ASSERT_RTNL();
1008
Pavel Emelianov7562f872007-05-03 15:13:45 -07001009 ret = NULL;
WANG Cong6c555492014-09-11 15:35:09 -07001010 for_each_netdev(net, dev) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001011 if (((dev->flags ^ if_flags) & mask) == 0) {
Pavel Emelianov7562f872007-05-03 15:13:45 -07001012 ret = dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001013 break;
1014 }
1015 }
Pavel Emelianov7562f872007-05-03 15:13:45 -07001016 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001017}
WANG Cong6c555492014-09-11 15:35:09 -07001018EXPORT_SYMBOL(__dev_get_by_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001019
1020/**
1021 * dev_valid_name - check if name is okay for network device
1022 * @name: name string
1023 *
1024 * Network device names need to be valid file names to
David S. Millerc7fa9d12006-08-15 16:34:13 -07001025 * to allow sysfs to work. We also disallow any kind of
1026 * whitespace.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001027 */
David S. Miller95f050b2012-03-06 16:12:15 -05001028bool dev_valid_name(const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001029{
David S. Millerc7fa9d12006-08-15 16:34:13 -07001030 if (*name == '\0')
David S. Miller95f050b2012-03-06 16:12:15 -05001031 return false;
Eric Dumazeta9d48202018-04-05 06:39:26 -07001032 if (strnlen(name, IFNAMSIZ) == IFNAMSIZ)
David S. Miller95f050b2012-03-06 16:12:15 -05001033 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -07001034 if (!strcmp(name, ".") || !strcmp(name, ".."))
David S. Miller95f050b2012-03-06 16:12:15 -05001035 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -07001036
1037 while (*name) {
Matthew Thodea4176a92015-02-17 18:31:57 -06001038 if (*name == '/' || *name == ':' || isspace(*name))
David S. Miller95f050b2012-03-06 16:12:15 -05001039 return false;
David S. Millerc7fa9d12006-08-15 16:34:13 -07001040 name++;
1041 }
David S. Miller95f050b2012-03-06 16:12:15 -05001042 return true;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001043}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001044EXPORT_SYMBOL(dev_valid_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001045
1046/**
Eric W. Biedermanb267b1792007-09-12 13:48:45 +02001047 * __dev_alloc_name - allocate a name for a device
1048 * @net: network namespace to allocate the device name in
Linus Torvalds1da177e2005-04-16 15:20:36 -07001049 * @name: name format string
Eric W. Biedermanb267b1792007-09-12 13:48:45 +02001050 * @buf: scratch buffer and result name string
Linus Torvalds1da177e2005-04-16 15:20:36 -07001051 *
1052 * Passed a format string - eg "lt%d" it will try and find a suitable
Stephen Hemminger3041a062006-05-26 13:25:24 -07001053 * id. It scans list of devices to build up a free map, then chooses
1054 * the first empty slot. The caller must hold the dev_base or rtnl lock
1055 * while allocating the name and adding the device in order to avoid
1056 * duplicates.
1057 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1058 * Returns the number of the unit assigned or a negative errno code.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001059 */
1060
Eric W. Biedermanb267b1792007-09-12 13:48:45 +02001061static int __dev_alloc_name(struct net *net, const char *name, char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001062{
1063 int i = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001064 const char *p;
1065 const int max_netdevices = 8*PAGE_SIZE;
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -07001066 unsigned long *inuse;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001067 struct net_device *d;
1068
Rasmus Villemoes93809102017-11-13 00:15:08 +01001069 if (!dev_valid_name(name))
1070 return -EINVAL;
1071
Rasmus Villemoes51f299d2017-11-13 00:15:04 +01001072 p = strchr(name, '%');
Linus Torvalds1da177e2005-04-16 15:20:36 -07001073 if (p) {
1074 /*
1075 * Verify the string as this thing may have come from
1076 * the user. There must be either one "%d" and no other "%"
1077 * characters.
1078 */
1079 if (p[1] != 'd' || strchr(p + 2, '%'))
1080 return -EINVAL;
1081
1082 /* Use one page as a bit array of possible slots */
Stephen Hemmingercfcabdc2007-10-09 01:59:42 -07001083 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001084 if (!inuse)
1085 return -ENOMEM;
1086
Eric W. Biederman881d9662007-09-17 11:56:21 -07001087 for_each_netdev(net, d) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001088 if (!sscanf(d->name, name, &i))
1089 continue;
1090 if (i < 0 || i >= max_netdevices)
1091 continue;
1092
1093 /* avoid cases where sscanf is not exact inverse of printf */
Eric W. Biedermanb267b1792007-09-12 13:48:45 +02001094 snprintf(buf, IFNAMSIZ, name, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001095 if (!strncmp(buf, d->name, IFNAMSIZ))
1096 set_bit(i, inuse);
1097 }
1098
1099 i = find_first_zero_bit(inuse, max_netdevices);
1100 free_page((unsigned long) inuse);
1101 }
1102
Rasmus Villemoes6224abd2017-11-13 00:15:07 +01001103 snprintf(buf, IFNAMSIZ, name, i);
Eric W. Biedermanb267b1792007-09-12 13:48:45 +02001104 if (!__dev_get_by_name(net, buf))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001105 return i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001106
1107 /* It is possible to run out of possible slots
1108 * when the name is long and there isn't enough space left
1109 * for the digits, or if all bits are used.
1110 */
Johannes Berg029b6d12017-12-02 08:41:55 +01001111 return -ENFILE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001112}
1113
Rasmus Villemoes2c88b852017-11-13 00:15:05 +01001114static int dev_alloc_name_ns(struct net *net,
1115 struct net_device *dev,
1116 const char *name)
1117{
1118 char buf[IFNAMSIZ];
1119 int ret;
1120
Rasmus Villemoesc46d7642017-11-13 00:15:06 +01001121 BUG_ON(!net);
Rasmus Villemoes2c88b852017-11-13 00:15:05 +01001122 ret = __dev_alloc_name(net, name, buf);
1123 if (ret >= 0)
1124 strlcpy(dev->name, buf, IFNAMSIZ);
1125 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001126}
1127
Eric W. Biedermanb267b1792007-09-12 13:48:45 +02001128/**
1129 * dev_alloc_name - allocate a name for a device
1130 * @dev: device
1131 * @name: name format string
1132 *
1133 * Passed a format string - eg "lt%d" it will try and find a suitable
1134 * id. It scans list of devices to build up a free map, then chooses
1135 * the first empty slot. The caller must hold the dev_base or rtnl lock
1136 * while allocating the name and adding the device in order to avoid
1137 * duplicates.
1138 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1139 * Returns the number of the unit assigned or a negative errno code.
1140 */
1141
1142int dev_alloc_name(struct net_device *dev, const char *name)
1143{
Rasmus Villemoesc46d7642017-11-13 00:15:06 +01001144 return dev_alloc_name_ns(dev_net(dev), dev, name);
Eric W. Biedermanb267b1792007-09-12 13:48:45 +02001145}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001146EXPORT_SYMBOL(dev_alloc_name);
Eric W. Biedermanb267b1792007-09-12 13:48:45 +02001147
Cong Wang0ad646c2017-10-13 11:58:53 -07001148int dev_get_valid_name(struct net *net, struct net_device *dev,
1149 const char *name)
Gao feng828de4f2012-09-13 20:58:27 +00001150{
David S. Miller55a5ec92018-01-02 11:45:07 -05001151 BUG_ON(!net);
1152
1153 if (!dev_valid_name(name))
1154 return -EINVAL;
1155
1156 if (strchr(name, '%'))
1157 return dev_alloc_name_ns(net, dev, name);
1158 else if (__dev_get_by_name(net, name))
1159 return -EEXIST;
1160 else if (dev->name != name)
1161 strlcpy(dev->name, name, IFNAMSIZ);
1162
1163 return 0;
Octavian Purdilad9031022009-11-18 02:36:59 +00001164}
Cong Wang0ad646c2017-10-13 11:58:53 -07001165EXPORT_SYMBOL(dev_get_valid_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001166
1167/**
1168 * dev_change_name - change name of a device
1169 * @dev: device
1170 * @newname: name (or format string) must be at least IFNAMSIZ
1171 *
1172 * Change name of a device, can pass format strings "eth%d".
1173 * for wildcarding.
1174 */
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07001175int dev_change_name(struct net_device *dev, const char *newname)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001176{
Tom Gundersen238fa362014-07-14 16:37:23 +02001177 unsigned char old_assign_type;
Herbert Xufcc5a032007-07-30 17:03:38 -07001178 char oldname[IFNAMSIZ];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001179 int err = 0;
Herbert Xufcc5a032007-07-30 17:03:38 -07001180 int ret;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001181 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001182
1183 ASSERT_RTNL();
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001184 BUG_ON(!dev_net(dev));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001185
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09001186 net = dev_net(dev);
Si-Wei Liu8065a772019-04-08 19:45:27 -04001187
1188 /* Some auto-enslaved devices e.g. failover slaves are
1189 * special, as userspace might rename the device after
1190 * the interface had been brought up and running since
1191 * the point kernel initiated auto-enslavement. Allow
1192 * live name change even when these slave devices are
1193 * up and running.
1194 *
1195 * Typically, users of these auto-enslaving devices
1196 * don't actually care about slave name change, as
1197 * they are supposed to operate on master interface
1198 * directly.
1199 */
1200 if (dev->flags & IFF_UP &&
1201 likely(!(dev->priv_flags & IFF_LIVE_RENAME_OK)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001202 return -EBUSY;
1203
Eric Dumazet30e6c9fa2012-12-20 17:25:08 +00001204 write_seqcount_begin(&devnet_rename_seq);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001205
1206 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
Eric Dumazet30e6c9fa2012-12-20 17:25:08 +00001207 write_seqcount_end(&devnet_rename_seq);
Stephen Hemmingerc8d90dc2007-10-26 03:53:42 -07001208 return 0;
Brian Haleyc91f6df2012-11-26 05:21:08 +00001209 }
Stephen Hemmingerc8d90dc2007-10-26 03:53:42 -07001210
Herbert Xufcc5a032007-07-30 17:03:38 -07001211 memcpy(oldname, dev->name, IFNAMSIZ);
1212
Gao feng828de4f2012-09-13 20:58:27 +00001213 err = dev_get_valid_name(net, dev, newname);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001214 if (err < 0) {
Eric Dumazet30e6c9fa2012-12-20 17:25:08 +00001215 write_seqcount_end(&devnet_rename_seq);
Octavian Purdilad9031022009-11-18 02:36:59 +00001216 return err;
Brian Haleyc91f6df2012-11-26 05:21:08 +00001217 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001218
Veaceslav Falico6fe82a32014-07-17 20:33:32 +02001219 if (oldname[0] && !strchr(oldname, '%'))
1220 netdev_info(dev, "renamed from %s\n", oldname);
1221
Tom Gundersen238fa362014-07-14 16:37:23 +02001222 old_assign_type = dev->name_assign_type;
1223 dev->name_assign_type = NET_NAME_RENAMED;
1224
Herbert Xufcc5a032007-07-30 17:03:38 -07001225rollback:
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07001226 ret = device_rename(&dev->dev, dev->name);
1227 if (ret) {
1228 memcpy(dev->name, oldname, IFNAMSIZ);
Tom Gundersen238fa362014-07-14 16:37:23 +02001229 dev->name_assign_type = old_assign_type;
Eric Dumazet30e6c9fa2012-12-20 17:25:08 +00001230 write_seqcount_end(&devnet_rename_seq);
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07001231 return ret;
Stephen Hemmingerdcc99772008-05-14 22:33:38 -07001232 }
Herbert Xu7f988eab52007-07-30 16:35:46 -07001233
Eric Dumazet30e6c9fa2012-12-20 17:25:08 +00001234 write_seqcount_end(&devnet_rename_seq);
Brian Haleyc91f6df2012-11-26 05:21:08 +00001235
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01001236 netdev_adjacent_rename_links(dev, oldname);
1237
Herbert Xu7f988eab52007-07-30 16:35:46 -07001238 write_lock_bh(&dev_base_lock);
Eric Dumazet372b2312011-05-17 13:56:59 -04001239 hlist_del_rcu(&dev->name_hlist);
Eric Dumazet72c95282009-10-30 07:11:27 +00001240 write_unlock_bh(&dev_base_lock);
1241
1242 synchronize_rcu();
1243
1244 write_lock_bh(&dev_base_lock);
1245 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
Herbert Xu7f988eab52007-07-30 16:35:46 -07001246 write_unlock_bh(&dev_base_lock);
1247
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001248 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001249 ret = notifier_to_errno(ret);
1250
1251 if (ret) {
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001252 /* err >= 0 after dev_alloc_name() or stores the first errno */
1253 if (err >= 0) {
Herbert Xufcc5a032007-07-30 17:03:38 -07001254 err = ret;
Eric Dumazet30e6c9fa2012-12-20 17:25:08 +00001255 write_seqcount_begin(&devnet_rename_seq);
Herbert Xufcc5a032007-07-30 17:03:38 -07001256 memcpy(dev->name, oldname, IFNAMSIZ);
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01001257 memcpy(oldname, newname, IFNAMSIZ);
Tom Gundersen238fa362014-07-14 16:37:23 +02001258 dev->name_assign_type = old_assign_type;
1259 old_assign_type = NET_NAME_RENAMED;
Herbert Xufcc5a032007-07-30 17:03:38 -07001260 goto rollback;
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001261 } else {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00001262 pr_err("%s: name change rollback failed: %d\n",
Eric Dumazet91e9c07b2009-11-15 23:30:24 +00001263 dev->name, ret);
Herbert Xufcc5a032007-07-30 17:03:38 -07001264 }
1265 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001266
1267 return err;
1268}
1269
1270/**
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001271 * dev_set_alias - change ifalias of a device
1272 * @dev: device
1273 * @alias: name up to IFALIASZ
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07001274 * @len: limit of bytes to copy from info
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001275 *
1276 * Set ifalias for a device,
1277 */
1278int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1279{
Florian Westphal6c557002017-10-02 23:50:05 +02001280 struct dev_ifalias *new_alias = NULL;
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001281
1282 if (len >= IFALIASZ)
1283 return -EINVAL;
1284
Florian Westphal6c557002017-10-02 23:50:05 +02001285 if (len) {
1286 new_alias = kmalloc(sizeof(*new_alias) + len + 1, GFP_KERNEL);
1287 if (!new_alias)
1288 return -ENOMEM;
1289
1290 memcpy(new_alias->ifalias, alias, len);
1291 new_alias->ifalias[len] = 0;
Oliver Hartkopp96ca4a2c2008-09-23 21:23:19 -07001292 }
1293
Florian Westphal6c557002017-10-02 23:50:05 +02001294 mutex_lock(&ifalias_mutex);
1295 rcu_swap_protected(dev->ifalias, new_alias,
1296 mutex_is_locked(&ifalias_mutex));
1297 mutex_unlock(&ifalias_mutex);
1298
1299 if (new_alias)
1300 kfree_rcu(new_alias, rcuhead);
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001301
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001302 return len;
1303}
Stephen Hemminger0fe554a2018-04-17 14:25:30 -07001304EXPORT_SYMBOL(dev_set_alias);
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001305
Florian Westphal6c557002017-10-02 23:50:05 +02001306/**
1307 * dev_get_alias - get ifalias of a device
1308 * @dev: device
Florian Westphal20e88322017-10-04 13:56:50 +02001309 * @name: buffer to store name of ifalias
Florian Westphal6c557002017-10-02 23:50:05 +02001310 * @len: size of buffer
1311 *
1312 * get ifalias for a device. Caller must make sure dev cannot go
1313 * away, e.g. rcu read lock or own a reference count to device.
1314 */
1315int dev_get_alias(const struct net_device *dev, char *name, size_t len)
1316{
1317 const struct dev_ifalias *alias;
1318 int ret = 0;
1319
1320 rcu_read_lock();
1321 alias = rcu_dereference(dev->ifalias);
1322 if (alias)
1323 ret = snprintf(name, len, "%s", alias->ifalias);
1324 rcu_read_unlock();
1325
1326 return ret;
1327}
Stephen Hemminger0b815a12008-09-22 21:28:11 -07001328
1329/**
Stephen Hemminger3041a062006-05-26 13:25:24 -07001330 * netdev_features_change - device changes features
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001331 * @dev: device to cause notification
1332 *
1333 * Called to indicate a device has changed features.
1334 */
1335void netdev_features_change(struct net_device *dev)
1336{
Pavel Emelyanov056925a2007-09-16 15:42:43 -07001337 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
Stephen Hemmingerd8a33ac2005-05-29 14:13:47 -07001338}
1339EXPORT_SYMBOL(netdev_features_change);
1340
1341/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07001342 * netdev_state_change - device changes state
1343 * @dev: device to cause notification
1344 *
1345 * Called to indicate a device has changed state. This function calls
1346 * the notifier chains for netdev_chain and sends a NEWLINK message
1347 * to the routing socket.
1348 */
1349void netdev_state_change(struct net_device *dev)
1350{
1351 if (dev->flags & IFF_UP) {
David Ahern51d0c0472017-10-04 17:48:45 -07001352 struct netdev_notifier_change_info change_info = {
1353 .info.dev = dev,
1354 };
Loic Prylli54951192014-07-01 21:39:43 -07001355
David Ahern51d0c0472017-10-04 17:48:45 -07001356 call_netdevice_notifiers_info(NETDEV_CHANGE,
Loic Prylli54951192014-07-01 21:39:43 -07001357 &change_info.info);
Alexei Starovoitov7f294052013-10-23 16:02:42 -07001358 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001359 }
1360}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001361EXPORT_SYMBOL(netdev_state_change);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001362
Amerigo Wangee89bab2012-08-09 22:14:56 +00001363/**
tcharding722c9a02017-02-09 17:56:04 +11001364 * netdev_notify_peers - notify network peers about existence of @dev
1365 * @dev: network device
Amerigo Wangee89bab2012-08-09 22:14:56 +00001366 *
1367 * Generate traffic such that interested network peers are aware of
1368 * @dev, such as by generating a gratuitous ARP. This may be used when
1369 * a device wants to inform the rest of the network about some sort of
1370 * reconfiguration such as a failover event or virtual machine
1371 * migration.
1372 */
1373void netdev_notify_peers(struct net_device *dev)
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001374{
Amerigo Wangee89bab2012-08-09 22:14:56 +00001375 rtnl_lock();
1376 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
Vlad Yasevich37c343b2017-03-14 08:58:08 -04001377 call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev);
Amerigo Wangee89bab2012-08-09 22:14:56 +00001378 rtnl_unlock();
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001379}
Amerigo Wangee89bab2012-08-09 22:14:56 +00001380EXPORT_SYMBOL(netdev_notify_peers);
Or Gerlitzc1da4ac2008-06-13 18:12:00 -07001381
Petr Machata40c900a2018-12-06 17:05:47 +00001382static int __dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001383{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001384 const struct net_device_ops *ops = dev->netdev_ops;
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001385 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001386
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001387 ASSERT_RTNL();
1388
Linus Torvalds1da177e2005-04-16 15:20:36 -07001389 if (!netif_device_present(dev))
1390 return -ENODEV;
1391
Neil Hormanca99ca12013-02-05 08:05:43 +00001392 /* Block netpoll from trying to do any rx path servicing.
1393 * If we don't do this there is a chance ndo_poll_controller
1394 * or ndo_poll may be running while we open the device
1395 */
Eric W. Biederman66b55522014-03-27 15:39:03 -07001396 netpoll_poll_disable(dev);
Neil Hormanca99ca12013-02-05 08:05:43 +00001397
Petr Machata40c900a2018-12-06 17:05:47 +00001398 ret = call_netdevice_notifiers_extack(NETDEV_PRE_UP, dev, extack);
Johannes Berg3b8bcfd2009-05-30 01:39:53 +02001399 ret = notifier_to_errno(ret);
1400 if (ret)
1401 return ret;
1402
Linus Torvalds1da177e2005-04-16 15:20:36 -07001403 set_bit(__LINK_STATE_START, &dev->state);
Jeff Garzikbada3392007-10-23 20:19:37 -07001404
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001405 if (ops->ndo_validate_addr)
1406 ret = ops->ndo_validate_addr(dev);
Jeff Garzikbada3392007-10-23 20:19:37 -07001407
Stephen Hemmingerd3147742008-11-19 21:32:24 -08001408 if (!ret && ops->ndo_open)
1409 ret = ops->ndo_open(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001410
Eric W. Biederman66b55522014-03-27 15:39:03 -07001411 netpoll_poll_enable(dev);
Neil Hormanca99ca12013-02-05 08:05:43 +00001412
Jeff Garzikbada3392007-10-23 20:19:37 -07001413 if (ret)
1414 clear_bit(__LINK_STATE_START, &dev->state);
1415 else {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001416 dev->flags |= IFF_UP;
Patrick McHardy4417da62007-06-27 01:28:10 -07001417 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001418 dev_activate(dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04001419 add_device_randomness(dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001420 }
Jeff Garzikbada3392007-10-23 20:19:37 -07001421
Linus Torvalds1da177e2005-04-16 15:20:36 -07001422 return ret;
1423}
Patrick McHardybd380812010-02-26 06:34:53 +00001424
1425/**
1426 * dev_open - prepare an interface for use.
Petr Machata00f54e62018-12-06 17:05:36 +00001427 * @dev: device to open
1428 * @extack: netlink extended ack
Patrick McHardybd380812010-02-26 06:34:53 +00001429 *
1430 * Takes a device from down to up state. The device's private open
1431 * function is invoked and then the multicast lists are loaded. Finally
1432 * the device is moved into the up state and a %NETDEV_UP message is
1433 * sent to the netdev notifier chain.
1434 *
1435 * Calling this function on an active interface is a nop. On a failure
1436 * a negative errno code is returned.
1437 */
Petr Machata00f54e62018-12-06 17:05:36 +00001438int dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
Patrick McHardybd380812010-02-26 06:34:53 +00001439{
1440 int ret;
1441
Patrick McHardybd380812010-02-26 06:34:53 +00001442 if (dev->flags & IFF_UP)
1443 return 0;
1444
Petr Machata40c900a2018-12-06 17:05:47 +00001445 ret = __dev_open(dev, extack);
Patrick McHardybd380812010-02-26 06:34:53 +00001446 if (ret < 0)
1447 return ret;
1448
Alexei Starovoitov7f294052013-10-23 16:02:42 -07001449 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
Patrick McHardybd380812010-02-26 06:34:53 +00001450 call_netdevice_notifiers(NETDEV_UP, dev);
1451
1452 return ret;
1453}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001454EXPORT_SYMBOL(dev_open);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001455
stephen hemminger7051b882017-07-18 15:59:27 -07001456static void __dev_close_many(struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001457{
Octavian Purdila44345722010-12-13 12:44:07 +00001458 struct net_device *dev;
Patrick McHardybd380812010-02-26 06:34:53 +00001459
Ben Hutchingse46b66b2008-05-08 02:53:17 -07001460 ASSERT_RTNL();
David S. Miller9d5010d2007-09-12 14:33:25 +02001461 might_sleep();
1462
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001463 list_for_each_entry(dev, head, close_list) {
Eric W. Biederman3f4df202014-03-27 15:38:17 -07001464 /* Temporarily disable netpoll until the interface is down */
Eric W. Biederman66b55522014-03-27 15:39:03 -07001465 netpoll_poll_disable(dev);
Eric W. Biederman3f4df202014-03-27 15:38:17 -07001466
Octavian Purdila44345722010-12-13 12:44:07 +00001467 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001468
Octavian Purdila44345722010-12-13 12:44:07 +00001469 clear_bit(__LINK_STATE_START, &dev->state);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001470
Octavian Purdila44345722010-12-13 12:44:07 +00001471 /* Synchronize to scheduled poll. We cannot touch poll list, it
1472 * can be even on different cpu. So just clear netif_running().
1473 *
1474 * dev->stop() will invoke napi_disable() on all of it's
1475 * napi_struct instances on this device.
1476 */
Peter Zijlstra4e857c52014-03-17 18:06:10 +01001477 smp_mb__after_atomic(); /* Commit netif_running(). */
Octavian Purdila44345722010-12-13 12:44:07 +00001478 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001479
Octavian Purdila44345722010-12-13 12:44:07 +00001480 dev_deactivate_many(head);
1481
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001482 list_for_each_entry(dev, head, close_list) {
Octavian Purdila44345722010-12-13 12:44:07 +00001483 const struct net_device_ops *ops = dev->netdev_ops;
1484
1485 /*
1486 * Call the device specific close. This cannot fail.
1487 * Only if device is UP
1488 *
1489 * We allow it to be called even after a DETACH hot-plug
1490 * event.
1491 */
1492 if (ops->ndo_stop)
1493 ops->ndo_stop(dev);
1494
Octavian Purdila44345722010-12-13 12:44:07 +00001495 dev->flags &= ~IFF_UP;
Eric W. Biederman66b55522014-03-27 15:39:03 -07001496 netpoll_poll_enable(dev);
Octavian Purdila44345722010-12-13 12:44:07 +00001497 }
Octavian Purdila44345722010-12-13 12:44:07 +00001498}
1499
stephen hemminger7051b882017-07-18 15:59:27 -07001500static void __dev_close(struct net_device *dev)
Octavian Purdila44345722010-12-13 12:44:07 +00001501{
1502 LIST_HEAD(single);
1503
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001504 list_add(&dev->close_list, &single);
stephen hemminger7051b882017-07-18 15:59:27 -07001505 __dev_close_many(&single);
Linus Torvaldsf87e6f42011-02-17 22:54:38 +00001506 list_del(&single);
Octavian Purdila44345722010-12-13 12:44:07 +00001507}
1508
stephen hemminger7051b882017-07-18 15:59:27 -07001509void dev_close_many(struct list_head *head, bool unlink)
Octavian Purdila44345722010-12-13 12:44:07 +00001510{
1511 struct net_device *dev, *tmp;
Octavian Purdila44345722010-12-13 12:44:07 +00001512
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001513 /* Remove the devices that don't need to be closed */
1514 list_for_each_entry_safe(dev, tmp, head, close_list)
Octavian Purdila44345722010-12-13 12:44:07 +00001515 if (!(dev->flags & IFF_UP))
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001516 list_del_init(&dev->close_list);
Octavian Purdila44345722010-12-13 12:44:07 +00001517
1518 __dev_close_many(head);
Matti Linnanvuorid8b2a4d2008-02-12 23:10:11 -08001519
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001520 list_for_each_entry_safe(dev, tmp, head, close_list) {
Alexei Starovoitov7f294052013-10-23 16:02:42 -07001521 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
Octavian Purdila44345722010-12-13 12:44:07 +00001522 call_netdevice_notifiers(NETDEV_DOWN, dev);
David S. Miller99c4a262015-03-18 22:52:33 -04001523 if (unlink)
1524 list_del_init(&dev->close_list);
Octavian Purdila44345722010-12-13 12:44:07 +00001525 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001526}
David S. Miller99c4a262015-03-18 22:52:33 -04001527EXPORT_SYMBOL(dev_close_many);
Patrick McHardybd380812010-02-26 06:34:53 +00001528
1529/**
1530 * dev_close - shutdown an interface.
1531 * @dev: device to shutdown
1532 *
1533 * This function moves an active device into down state. A
1534 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1535 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1536 * chain.
1537 */
stephen hemminger7051b882017-07-18 15:59:27 -07001538void dev_close(struct net_device *dev)
Patrick McHardybd380812010-02-26 06:34:53 +00001539{
Eric Dumazete14a5992011-05-10 12:26:06 -07001540 if (dev->flags & IFF_UP) {
1541 LIST_HEAD(single);
Patrick McHardybd380812010-02-26 06:34:53 +00001542
Eric W. Biederman5cde2822013-10-05 19:26:05 -07001543 list_add(&dev->close_list, &single);
David S. Miller99c4a262015-03-18 22:52:33 -04001544 dev_close_many(&single, true);
Eric Dumazete14a5992011-05-10 12:26:06 -07001545 list_del(&single);
1546 }
Patrick McHardybd380812010-02-26 06:34:53 +00001547}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001548EXPORT_SYMBOL(dev_close);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001549
1550
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001551/**
1552 * dev_disable_lro - disable Large Receive Offload on a device
1553 * @dev: device
1554 *
1555 * Disable Large Receive Offload (LRO) on a net device. Must be
1556 * called under RTNL. This is needed if received packets may be
1557 * forwarded to another interface.
1558 */
1559void dev_disable_lro(struct net_device *dev)
1560{
Michal Kubečekfbe168b2014-11-13 07:54:50 +01001561 struct net_device *lower_dev;
1562 struct list_head *iter;
Michal Kubeček529d0482013-11-15 06:18:50 +01001563
Michał Mirosławbc5787c62011-11-15 15:29:55 +00001564 dev->wanted_features &= ~NETIF_F_LRO;
1565 netdev_update_features(dev);
Michał Mirosław27660512011-03-18 16:56:34 +00001566
Michał Mirosław22d59692011-04-21 12:42:15 +00001567 if (unlikely(dev->features & NETIF_F_LRO))
1568 netdev_WARN(dev, "failed to disable LRO!\n");
Michal Kubečekfbe168b2014-11-13 07:54:50 +01001569
1570 netdev_for_each_lower_dev(dev, lower_dev, iter)
1571 dev_disable_lro(lower_dev);
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001572}
1573EXPORT_SYMBOL(dev_disable_lro);
1574
Michael Chan56f5aa72017-12-16 03:09:41 -05001575/**
1576 * dev_disable_gro_hw - disable HW Generic Receive Offload on a device
1577 * @dev: device
1578 *
1579 * Disable HW Generic Receive Offload (GRO_HW) on a net device. Must be
1580 * called under RTNL. This is needed if Generic XDP is installed on
1581 * the device.
1582 */
1583static void dev_disable_gro_hw(struct net_device *dev)
1584{
1585 dev->wanted_features &= ~NETIF_F_GRO_HW;
1586 netdev_update_features(dev);
1587
1588 if (unlikely(dev->features & NETIF_F_GRO_HW))
1589 netdev_WARN(dev, "failed to disable GRO_HW!\n");
1590}
1591
Kirill Tkhaiede27622018-03-23 19:47:19 +03001592const char *netdev_cmd_to_name(enum netdev_cmd cmd)
1593{
1594#define N(val) \
1595 case NETDEV_##val: \
1596 return "NETDEV_" __stringify(val);
1597 switch (cmd) {
1598 N(UP) N(DOWN) N(REBOOT) N(CHANGE) N(REGISTER) N(UNREGISTER)
1599 N(CHANGEMTU) N(CHANGEADDR) N(GOING_DOWN) N(CHANGENAME) N(FEAT_CHANGE)
1600 N(BONDING_FAILOVER) N(PRE_UP) N(PRE_TYPE_CHANGE) N(POST_TYPE_CHANGE)
1601 N(POST_INIT) N(RELEASE) N(NOTIFY_PEERS) N(JOIN) N(CHANGEUPPER)
1602 N(RESEND_IGMP) N(PRECHANGEMTU) N(CHANGEINFODATA) N(BONDING_INFO)
1603 N(PRECHANGEUPPER) N(CHANGELOWERSTATE) N(UDP_TUNNEL_PUSH_INFO)
1604 N(UDP_TUNNEL_DROP_INFO) N(CHANGE_TX_QUEUE_LEN)
Gal Pressman9daae9b2018-03-28 17:46:54 +03001605 N(CVLAN_FILTER_PUSH_INFO) N(CVLAN_FILTER_DROP_INFO)
1606 N(SVLAN_FILTER_PUSH_INFO) N(SVLAN_FILTER_DROP_INFO)
Petr Machata15704152018-12-13 11:54:33 +00001607 N(PRE_CHANGEADDR)
Kirill Tkhai3f5ecd82018-04-26 15:18:38 +03001608 }
Kirill Tkhaiede27622018-03-23 19:47:19 +03001609#undef N
1610 return "UNKNOWN_NETDEV_EVENT";
1611}
1612EXPORT_SYMBOL_GPL(netdev_cmd_to_name);
1613
Jiri Pirko351638e2013-05-28 01:30:21 +00001614static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1615 struct net_device *dev)
1616{
David Ahern51d0c0472017-10-04 17:48:45 -07001617 struct netdev_notifier_info info = {
1618 .dev = dev,
1619 };
Jiri Pirko351638e2013-05-28 01:30:21 +00001620
Jiri Pirko351638e2013-05-28 01:30:21 +00001621 return nb->notifier_call(nb, val, &info);
1622}
Ben Hutchings0187bdf2008-06-19 16:15:47 -07001623
Eric W. Biederman881d9662007-09-17 11:56:21 -07001624static int dev_boot_phase = 1;
1625
Linus Torvalds1da177e2005-04-16 15:20:36 -07001626/**
tcharding722c9a02017-02-09 17:56:04 +11001627 * register_netdevice_notifier - register a network notifier block
1628 * @nb: notifier
Linus Torvalds1da177e2005-04-16 15:20:36 -07001629 *
tcharding722c9a02017-02-09 17:56:04 +11001630 * Register a notifier to be called when network device events occur.
1631 * The notifier passed is linked into the kernel structures and must
1632 * not be reused until it has been unregistered. A negative errno code
1633 * is returned on a failure.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001634 *
tcharding722c9a02017-02-09 17:56:04 +11001635 * When registered all registration and up events are replayed
1636 * to the new notifier to allow device to have a race free
1637 * view of the network device list.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001638 */
1639
1640int register_netdevice_notifier(struct notifier_block *nb)
1641{
1642 struct net_device *dev;
Herbert Xufcc5a032007-07-30 17:03:38 -07001643 struct net_device *last;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001644 struct net *net;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001645 int err;
1646
Kirill Tkhai328fbe72018-03-29 17:03:45 +03001647 /* Close race with setup_net() and cleanup_net() */
1648 down_write(&pernet_ops_rwsem);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001649 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001650 err = raw_notifier_chain_register(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001651 if (err)
1652 goto unlock;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001653 if (dev_boot_phase)
1654 goto unlock;
1655 for_each_net(net) {
1656 for_each_netdev(net, dev) {
Jiri Pirko351638e2013-05-28 01:30:21 +00001657 err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
Eric W. Biederman881d9662007-09-17 11:56:21 -07001658 err = notifier_to_errno(err);
1659 if (err)
1660 goto rollback;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001661
Eric W. Biederman881d9662007-09-17 11:56:21 -07001662 if (!(dev->flags & IFF_UP))
1663 continue;
Herbert Xufcc5a032007-07-30 17:03:38 -07001664
Jiri Pirko351638e2013-05-28 01:30:21 +00001665 call_netdevice_notifier(nb, NETDEV_UP, dev);
Eric W. Biederman881d9662007-09-17 11:56:21 -07001666 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001667 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001668
1669unlock:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001670 rtnl_unlock();
Kirill Tkhai328fbe72018-03-29 17:03:45 +03001671 up_write(&pernet_ops_rwsem);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001672 return err;
Herbert Xufcc5a032007-07-30 17:03:38 -07001673
1674rollback:
1675 last = dev;
Eric W. Biederman881d9662007-09-17 11:56:21 -07001676 for_each_net(net) {
1677 for_each_netdev(net, dev) {
1678 if (dev == last)
RongQing.Li8f891482011-11-30 23:43:07 -05001679 goto outroll;
Herbert Xufcc5a032007-07-30 17:03:38 -07001680
Eric W. Biederman881d9662007-09-17 11:56:21 -07001681 if (dev->flags & IFF_UP) {
Jiri Pirko351638e2013-05-28 01:30:21 +00001682 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1683 dev);
1684 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
Eric W. Biederman881d9662007-09-17 11:56:21 -07001685 }
Jiri Pirko351638e2013-05-28 01:30:21 +00001686 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07001687 }
Herbert Xufcc5a032007-07-30 17:03:38 -07001688 }
Pavel Emelyanovc67625a2007-11-14 15:53:16 -08001689
RongQing.Li8f891482011-11-30 23:43:07 -05001690outroll:
Pavel Emelyanovc67625a2007-11-14 15:53:16 -08001691 raw_notifier_chain_unregister(&netdev_chain, nb);
Herbert Xufcc5a032007-07-30 17:03:38 -07001692 goto unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001693}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001694EXPORT_SYMBOL(register_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001695
1696/**
tcharding722c9a02017-02-09 17:56:04 +11001697 * unregister_netdevice_notifier - unregister a network notifier block
1698 * @nb: notifier
Linus Torvalds1da177e2005-04-16 15:20:36 -07001699 *
tcharding722c9a02017-02-09 17:56:04 +11001700 * Unregister a notifier previously registered by
1701 * register_netdevice_notifier(). The notifier is unlinked into the
1702 * kernel structures and may then be reused. A negative errno code
1703 * is returned on a failure.
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001704 *
tcharding722c9a02017-02-09 17:56:04 +11001705 * After unregistering unregister and down device events are synthesized
1706 * for all devices on the device list to the removed notifier to remove
1707 * the need for special case cleanup code.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001708 */
1709
1710int unregister_netdevice_notifier(struct notifier_block *nb)
1711{
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001712 struct net_device *dev;
1713 struct net *net;
Herbert Xu9f514952006-03-25 01:24:25 -08001714 int err;
1715
Kirill Tkhai328fbe72018-03-29 17:03:45 +03001716 /* Close race with setup_net() and cleanup_net() */
1717 down_write(&pernet_ops_rwsem);
Herbert Xu9f514952006-03-25 01:24:25 -08001718 rtnl_lock();
Alan Sternf07d5b92006-05-09 15:23:03 -07001719 err = raw_notifier_chain_unregister(&netdev_chain, nb);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001720 if (err)
1721 goto unlock;
1722
1723 for_each_net(net) {
1724 for_each_netdev(net, dev) {
1725 if (dev->flags & IFF_UP) {
Jiri Pirko351638e2013-05-28 01:30:21 +00001726 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1727 dev);
1728 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001729 }
Jiri Pirko351638e2013-05-28 01:30:21 +00001730 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
Eric W. Biederman7d3d43d2012-04-06 15:33:35 +00001731 }
1732 }
1733unlock:
Herbert Xu9f514952006-03-25 01:24:25 -08001734 rtnl_unlock();
Kirill Tkhai328fbe72018-03-29 17:03:45 +03001735 up_write(&pernet_ops_rwsem);
Herbert Xu9f514952006-03-25 01:24:25 -08001736 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001737}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001738EXPORT_SYMBOL(unregister_netdevice_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001739
1740/**
Jiri Pirko351638e2013-05-28 01:30:21 +00001741 * call_netdevice_notifiers_info - call all network notifier blocks
1742 * @val: value passed unmodified to notifier function
Jiri Pirko351638e2013-05-28 01:30:21 +00001743 * @info: notifier information data
1744 *
1745 * Call all network notifier blocks. Parameters and return value
1746 * are as for raw_notifier_call_chain().
1747 */
1748
stephen hemminger1d143d92013-12-29 14:01:29 -08001749static int call_netdevice_notifiers_info(unsigned long val,
stephen hemminger1d143d92013-12-29 14:01:29 -08001750 struct netdev_notifier_info *info)
Jiri Pirko351638e2013-05-28 01:30:21 +00001751{
1752 ASSERT_RTNL();
Jiri Pirko351638e2013-05-28 01:30:21 +00001753 return raw_notifier_call_chain(&netdev_chain, val, info);
1754}
Jiri Pirko351638e2013-05-28 01:30:21 +00001755
Petr Machata26372602018-12-06 17:05:45 +00001756static int call_netdevice_notifiers_extack(unsigned long val,
1757 struct net_device *dev,
1758 struct netlink_ext_ack *extack)
1759{
1760 struct netdev_notifier_info info = {
1761 .dev = dev,
1762 .extack = extack,
1763 };
1764
1765 return call_netdevice_notifiers_info(val, &info);
1766}
1767
Jiri Pirko351638e2013-05-28 01:30:21 +00001768/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07001769 * call_netdevice_notifiers - call all network notifier blocks
1770 * @val: value passed unmodified to notifier function
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07001771 * @dev: net_device pointer passed unmodified to notifier function
Linus Torvalds1da177e2005-04-16 15:20:36 -07001772 *
1773 * Call all network notifier blocks. Parameters and return value
Alan Sternf07d5b92006-05-09 15:23:03 -07001774 * are as for raw_notifier_call_chain().
Linus Torvalds1da177e2005-04-16 15:20:36 -07001775 */
1776
Eric W. Biedermanad7379d2007-09-16 15:33:32 -07001777int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001778{
Petr Machata26372602018-12-06 17:05:45 +00001779 return call_netdevice_notifiers_extack(val, dev, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001780}
stephen hemmingeredf947f2011-03-24 13:24:01 +00001781EXPORT_SYMBOL(call_netdevice_notifiers);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001782
Sabrina Dubrocaaf7d6cc2018-10-09 17:48:14 +02001783/**
1784 * call_netdevice_notifiers_mtu - call all network notifier blocks
1785 * @val: value passed unmodified to notifier function
1786 * @dev: net_device pointer passed unmodified to notifier function
1787 * @arg: additional u32 argument passed to the notifier function
1788 *
1789 * Call all network notifier blocks. Parameters and return value
1790 * are as for raw_notifier_call_chain().
1791 */
1792static int call_netdevice_notifiers_mtu(unsigned long val,
1793 struct net_device *dev, u32 arg)
1794{
1795 struct netdev_notifier_info_ext info = {
1796 .info.dev = dev,
1797 .ext.mtu = arg,
1798 };
1799
1800 BUILD_BUG_ON(offsetof(struct netdev_notifier_info_ext, info) != 0);
1801
1802 return call_netdevice_notifiers_info(val, &info.info);
1803}
1804
Pablo Neira1cf519002015-05-13 18:19:37 +02001805#ifdef CONFIG_NET_INGRESS
Davidlohr Buesoaabf6772018-05-08 09:07:00 -07001806static DEFINE_STATIC_KEY_FALSE(ingress_needed_key);
Daniel Borkmann45771392015-04-10 23:07:54 +02001807
1808void net_inc_ingress_queue(void)
1809{
Davidlohr Buesoaabf6772018-05-08 09:07:00 -07001810 static_branch_inc(&ingress_needed_key);
Daniel Borkmann45771392015-04-10 23:07:54 +02001811}
1812EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
1813
1814void net_dec_ingress_queue(void)
1815{
Davidlohr Buesoaabf6772018-05-08 09:07:00 -07001816 static_branch_dec(&ingress_needed_key);
Daniel Borkmann45771392015-04-10 23:07:54 +02001817}
1818EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
1819#endif
1820
Daniel Borkmann1f211a12016-01-07 22:29:47 +01001821#ifdef CONFIG_NET_EGRESS
Davidlohr Buesoaabf6772018-05-08 09:07:00 -07001822static DEFINE_STATIC_KEY_FALSE(egress_needed_key);
Daniel Borkmann1f211a12016-01-07 22:29:47 +01001823
1824void net_inc_egress_queue(void)
1825{
Davidlohr Buesoaabf6772018-05-08 09:07:00 -07001826 static_branch_inc(&egress_needed_key);
Daniel Borkmann1f211a12016-01-07 22:29:47 +01001827}
1828EXPORT_SYMBOL_GPL(net_inc_egress_queue);
1829
1830void net_dec_egress_queue(void)
1831{
Davidlohr Buesoaabf6772018-05-08 09:07:00 -07001832 static_branch_dec(&egress_needed_key);
Daniel Borkmann1f211a12016-01-07 22:29:47 +01001833}
1834EXPORT_SYMBOL_GPL(net_dec_egress_queue);
1835#endif
1836
Davidlohr Bueso39e83922018-05-08 09:07:01 -07001837static DEFINE_STATIC_KEY_FALSE(netstamp_needed_key);
Masahiro Yamadae9666d12018-12-31 00:14:15 +09001838#ifdef CONFIG_JUMP_LABEL
Eric Dumazetb90e5792011-11-28 11:16:50 +00001839static atomic_t netstamp_needed_deferred;
Eric Dumazet13baa002017-03-01 14:28:39 -08001840static atomic_t netstamp_wanted;
Eric Dumazet5fa8bbd2017-02-02 10:31:35 -08001841static void netstamp_clear(struct work_struct *work)
1842{
1843 int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
Eric Dumazet13baa002017-03-01 14:28:39 -08001844 int wanted;
Eric Dumazet5fa8bbd2017-02-02 10:31:35 -08001845
Eric Dumazet13baa002017-03-01 14:28:39 -08001846 wanted = atomic_add_return(deferred, &netstamp_wanted);
1847 if (wanted > 0)
Davidlohr Bueso39e83922018-05-08 09:07:01 -07001848 static_branch_enable(&netstamp_needed_key);
Eric Dumazet13baa002017-03-01 14:28:39 -08001849 else
Davidlohr Bueso39e83922018-05-08 09:07:01 -07001850 static_branch_disable(&netstamp_needed_key);
Eric Dumazet5fa8bbd2017-02-02 10:31:35 -08001851}
1852static DECLARE_WORK(netstamp_work, netstamp_clear);
Eric Dumazetb90e5792011-11-28 11:16:50 +00001853#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001854
1855void net_enable_timestamp(void)
1856{
Masahiro Yamadae9666d12018-12-31 00:14:15 +09001857#ifdef CONFIG_JUMP_LABEL
Eric Dumazet13baa002017-03-01 14:28:39 -08001858 int wanted;
1859
1860 while (1) {
1861 wanted = atomic_read(&netstamp_wanted);
1862 if (wanted <= 0)
1863 break;
1864 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted + 1) == wanted)
1865 return;
1866 }
1867 atomic_inc(&netstamp_needed_deferred);
1868 schedule_work(&netstamp_work);
1869#else
Davidlohr Bueso39e83922018-05-08 09:07:01 -07001870 static_branch_inc(&netstamp_needed_key);
Eric Dumazet13baa002017-03-01 14:28:39 -08001871#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001872}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001873EXPORT_SYMBOL(net_enable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001874
1875void net_disable_timestamp(void)
1876{
Masahiro Yamadae9666d12018-12-31 00:14:15 +09001877#ifdef CONFIG_JUMP_LABEL
Eric Dumazet13baa002017-03-01 14:28:39 -08001878 int wanted;
1879
1880 while (1) {
1881 wanted = atomic_read(&netstamp_wanted);
1882 if (wanted <= 1)
1883 break;
1884 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted - 1) == wanted)
1885 return;
1886 }
1887 atomic_dec(&netstamp_needed_deferred);
Eric Dumazet5fa8bbd2017-02-02 10:31:35 -08001888 schedule_work(&netstamp_work);
1889#else
Davidlohr Bueso39e83922018-05-08 09:07:01 -07001890 static_branch_dec(&netstamp_needed_key);
Eric Dumazet5fa8bbd2017-02-02 10:31:35 -08001891#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001892}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07001893EXPORT_SYMBOL(net_disable_timestamp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001894
Eric Dumazet3b098e22010-05-15 23:57:10 -07001895static inline void net_timestamp_set(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001896{
Thomas Gleixner2456e852016-12-25 11:38:40 +01001897 skb->tstamp = 0;
Davidlohr Bueso39e83922018-05-08 09:07:01 -07001898 if (static_branch_unlikely(&netstamp_needed_key))
Patrick McHardya61bbcf2005-08-14 17:24:31 -07001899 __net_timestamp(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001900}
1901
Davidlohr Bueso39e83922018-05-08 09:07:01 -07001902#define net_timestamp_check(COND, SKB) \
1903 if (static_branch_unlikely(&netstamp_needed_key)) { \
1904 if ((COND) && !(SKB)->tstamp) \
1905 __net_timestamp(SKB); \
1906 } \
Eric Dumazet3b098e22010-05-15 23:57:10 -07001907
Nikolay Aleksandrovf4b05d22016-04-28 17:59:28 +02001908bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb)
Daniel Lezcano79b569f2011-03-30 02:42:17 -07001909{
1910 unsigned int len;
1911
1912 if (!(dev->flags & IFF_UP))
1913 return false;
1914
1915 len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1916 if (skb->len <= len)
1917 return true;
1918
1919 /* if TSO is enabled, we don't care about the length as the packet
1920 * could be forwarded without being segmented before
1921 */
1922 if (skb_is_gso(skb))
1923 return true;
1924
1925 return false;
1926}
Vlad Yasevich1ee481f2014-03-27 17:32:29 -04001927EXPORT_SYMBOL_GPL(is_skb_forwardable);
Daniel Lezcano79b569f2011-03-30 02:42:17 -07001928
Herbert Xua0265d22014-04-17 13:45:03 +08001929int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1930{
Martin KaFai Lau4e3264d2016-11-09 15:36:33 -08001931 int ret = ____dev_forward_skb(dev, skb);
1932
1933 if (likely(!ret)) {
1934 skb->protocol = eth_type_trans(skb, dev);
1935 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
Herbert Xua0265d22014-04-17 13:45:03 +08001936 }
1937
Martin KaFai Lau4e3264d2016-11-09 15:36:33 -08001938 return ret;
Herbert Xua0265d22014-04-17 13:45:03 +08001939}
1940EXPORT_SYMBOL_GPL(__dev_forward_skb);
1941
Arnd Bergmann44540962009-11-26 06:07:08 +00001942/**
1943 * dev_forward_skb - loopback an skb to another netif
1944 *
1945 * @dev: destination network device
1946 * @skb: buffer to forward
1947 *
1948 * return values:
1949 * NET_RX_SUCCESS (no congestion)
Eric Dumazet6ec82562010-05-06 00:53:53 -07001950 * NET_RX_DROP (packet was dropped, but freed)
Arnd Bergmann44540962009-11-26 06:07:08 +00001951 *
1952 * dev_forward_skb can be used for injecting an skb from the
1953 * start_xmit function of one device into the receive queue
1954 * of another device.
1955 *
1956 * The receiving device may be in another namespace, so
1957 * we have to clear all information in the skb that could
1958 * impact namespace isolation.
1959 */
1960int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1961{
Herbert Xua0265d22014-04-17 13:45:03 +08001962 return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
Arnd Bergmann44540962009-11-26 06:07:08 +00001963}
1964EXPORT_SYMBOL_GPL(dev_forward_skb);
1965
Changli Gao71d9dec2010-12-15 19:57:25 +00001966static inline int deliver_skb(struct sk_buff *skb,
1967 struct packet_type *pt_prev,
1968 struct net_device *orig_dev)
1969{
Willem de Bruijn1f8b9772017-08-03 16:29:41 -04001970 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
Michael S. Tsirkin1080e512012-07-20 09:23:17 +00001971 return -ENOMEM;
Reshetova, Elena63354792017-06-30 13:07:58 +03001972 refcount_inc(&skb->users);
Changli Gao71d9dec2010-12-15 19:57:25 +00001973 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1974}
1975
Salam Noureddine7866a622015-01-27 11:35:48 -08001976static inline void deliver_ptype_list_skb(struct sk_buff *skb,
1977 struct packet_type **pt,
Jiri Pirkofbcb2172015-03-30 16:56:01 +02001978 struct net_device *orig_dev,
1979 __be16 type,
Salam Noureddine7866a622015-01-27 11:35:48 -08001980 struct list_head *ptype_list)
1981{
1982 struct packet_type *ptype, *pt_prev = *pt;
1983
1984 list_for_each_entry_rcu(ptype, ptype_list, list) {
1985 if (ptype->type != type)
1986 continue;
1987 if (pt_prev)
Jiri Pirkofbcb2172015-03-30 16:56:01 +02001988 deliver_skb(skb, pt_prev, orig_dev);
Salam Noureddine7866a622015-01-27 11:35:48 -08001989 pt_prev = ptype;
1990 }
1991 *pt = pt_prev;
1992}
1993
Eric Leblondc0de08d2012-08-16 22:02:58 +00001994static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
1995{
Eric Leblonda3d744e2012-11-06 02:10:10 +00001996 if (!ptype->af_packet_priv || !skb->sk)
Eric Leblondc0de08d2012-08-16 22:02:58 +00001997 return false;
1998
1999 if (ptype->id_match)
2000 return ptype->id_match(ptype, skb->sk);
2001 else if ((struct sock *)ptype->af_packet_priv == skb->sk)
2002 return true;
2003
2004 return false;
2005}
2006
Maciej W. Rozycki9f9a7422018-10-09 23:57:49 +01002007/**
2008 * dev_nit_active - return true if any network interface taps are in use
2009 *
2010 * @dev: network device to check for the presence of taps
2011 */
2012bool dev_nit_active(struct net_device *dev)
2013{
2014 return !list_empty(&ptype_all) || !list_empty(&dev->ptype_all);
2015}
2016EXPORT_SYMBOL_GPL(dev_nit_active);
2017
Linus Torvalds1da177e2005-04-16 15:20:36 -07002018/*
2019 * Support routine. Sends outgoing frames to any network
2020 * taps currently in use.
2021 */
2022
David Ahern74b20582016-05-10 11:19:50 -07002023void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002024{
2025 struct packet_type *ptype;
Changli Gao71d9dec2010-12-15 19:57:25 +00002026 struct sk_buff *skb2 = NULL;
2027 struct packet_type *pt_prev = NULL;
Salam Noureddine7866a622015-01-27 11:35:48 -08002028 struct list_head *ptype_list = &ptype_all;
Patrick McHardya61bbcf2005-08-14 17:24:31 -07002029
Linus Torvalds1da177e2005-04-16 15:20:36 -07002030 rcu_read_lock();
Salam Noureddine7866a622015-01-27 11:35:48 -08002031again:
2032 list_for_each_entry_rcu(ptype, ptype_list, list) {
Vincent Whitchurchfa788d92018-09-03 16:23:36 +02002033 if (ptype->ignore_outgoing)
2034 continue;
2035
Linus Torvalds1da177e2005-04-16 15:20:36 -07002036 /* Never send packets back to the socket
2037 * they originated from - MvS (miquels@drinkel.ow.org)
2038 */
Salam Noureddine7866a622015-01-27 11:35:48 -08002039 if (skb_loop_sk(ptype, skb))
2040 continue;
Changli Gao71d9dec2010-12-15 19:57:25 +00002041
Salam Noureddine7866a622015-01-27 11:35:48 -08002042 if (pt_prev) {
2043 deliver_skb(skb2, pt_prev, skb->dev);
Changli Gao71d9dec2010-12-15 19:57:25 +00002044 pt_prev = ptype;
Salam Noureddine7866a622015-01-27 11:35:48 -08002045 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002046 }
Salam Noureddine7866a622015-01-27 11:35:48 -08002047
2048 /* need to clone skb, done only once */
2049 skb2 = skb_clone(skb, GFP_ATOMIC);
2050 if (!skb2)
2051 goto out_unlock;
2052
2053 net_timestamp_set(skb2);
2054
2055 /* skb->nh should be correctly
2056 * set by sender, so that the second statement is
2057 * just protection against buggy protocols.
2058 */
2059 skb_reset_mac_header(skb2);
2060
2061 if (skb_network_header(skb2) < skb2->data ||
2062 skb_network_header(skb2) > skb_tail_pointer(skb2)) {
2063 net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
2064 ntohs(skb2->protocol),
2065 dev->name);
2066 skb_reset_network_header(skb2);
2067 }
2068
2069 skb2->transport_header = skb2->network_header;
2070 skb2->pkt_type = PACKET_OUTGOING;
2071 pt_prev = ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002072 }
Salam Noureddine7866a622015-01-27 11:35:48 -08002073
2074 if (ptype_list == &ptype_all) {
2075 ptype_list = &dev->ptype_all;
2076 goto again;
2077 }
2078out_unlock:
Willem de Bruijn581fe0e2017-09-22 19:42:37 -04002079 if (pt_prev) {
2080 if (!skb_orphan_frags_rx(skb2, GFP_ATOMIC))
2081 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
2082 else
2083 kfree_skb(skb2);
2084 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002085 rcu_read_unlock();
2086}
David Ahern74b20582016-05-10 11:19:50 -07002087EXPORT_SYMBOL_GPL(dev_queue_xmit_nit);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002088
Ben Hutchings2c530402012-07-10 10:55:09 +00002089/**
2090 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
John Fastabend4f57c082011-01-17 08:06:04 +00002091 * @dev: Network device
2092 * @txq: number of queues available
2093 *
2094 * If real_num_tx_queues is changed the tc mappings may no longer be
2095 * valid. To resolve this verify the tc mapping remains valid and if
2096 * not NULL the mapping. With no priorities mapping to this
2097 * offset/count pair it will no longer be used. In the worst case TC0
2098 * is invalid nothing can be done so disable priority mappings. If is
2099 * expected that drivers will fix this mapping if they can before
2100 * calling netif_set_real_num_tx_queues.
2101 */
Eric Dumazetbb134d222011-01-20 19:18:08 +00002102static void netif_setup_tc(struct net_device *dev, unsigned int txq)
John Fastabend4f57c082011-01-17 08:06:04 +00002103{
2104 int i;
2105 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2106
2107 /* If TC0 is invalidated disable TC mapping */
2108 if (tc->offset + tc->count > txq) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00002109 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
John Fastabend4f57c082011-01-17 08:06:04 +00002110 dev->num_tc = 0;
2111 return;
2112 }
2113
2114 /* Invalidated prio to tc mappings set to TC0 */
2115 for (i = 1; i < TC_BITMASK + 1; i++) {
2116 int q = netdev_get_prio_tc_map(dev, i);
2117
2118 tc = &dev->tc_to_txq[q];
2119 if (tc->offset + tc->count > txq) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00002120 pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
2121 i, q);
John Fastabend4f57c082011-01-17 08:06:04 +00002122 netdev_set_prio_tc_map(dev, i, 0);
2123 }
2124 }
2125}
2126
Alexander Duyck8d059b02016-10-28 11:43:49 -04002127int netdev_txq_to_tc(struct net_device *dev, unsigned int txq)
2128{
2129 if (dev->num_tc) {
2130 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2131 int i;
2132
Alexander Duyckffcfe252018-07-09 12:19:38 -04002133 /* walk through the TCs and see if it falls into any of them */
Alexander Duyck8d059b02016-10-28 11:43:49 -04002134 for (i = 0; i < TC_MAX_QUEUE; i++, tc++) {
2135 if ((txq - tc->offset) < tc->count)
2136 return i;
2137 }
2138
Alexander Duyckffcfe252018-07-09 12:19:38 -04002139 /* didn't find it, just return -1 to indicate no match */
Alexander Duyck8d059b02016-10-28 11:43:49 -04002140 return -1;
2141 }
2142
2143 return 0;
2144}
Henrik Austad8a5f2162017-10-17 12:10:10 +02002145EXPORT_SYMBOL(netdev_txq_to_tc);
Alexander Duyck8d059b02016-10-28 11:43:49 -04002146
Alexander Duyck537c00d2013-01-10 08:57:02 +00002147#ifdef CONFIG_XPS
Amritha Nambiar04157462018-06-29 21:26:46 -07002148struct static_key xps_needed __read_mostly;
2149EXPORT_SYMBOL(xps_needed);
2150struct static_key xps_rxqs_needed __read_mostly;
2151EXPORT_SYMBOL(xps_rxqs_needed);
Alexander Duyck537c00d2013-01-10 08:57:02 +00002152static DEFINE_MUTEX(xps_map_mutex);
2153#define xmap_dereference(P) \
2154 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
2155
Alexander Duyck6234f872016-10-28 11:46:49 -04002156static bool remove_xps_queue(struct xps_dev_maps *dev_maps,
2157 int tci, u16 index)
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00002158{
2159 struct xps_map *map = NULL;
2160 int pos;
2161
2162 if (dev_maps)
Amritha Nambiar80d19662018-06-29 21:26:41 -07002163 map = xmap_dereference(dev_maps->attr_map[tci]);
Alexander Duyck6234f872016-10-28 11:46:49 -04002164 if (!map)
2165 return false;
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00002166
Alexander Duyck6234f872016-10-28 11:46:49 -04002167 for (pos = map->len; pos--;) {
2168 if (map->queues[pos] != index)
2169 continue;
2170
2171 if (map->len > 1) {
2172 map->queues[pos] = map->queues[--map->len];
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00002173 break;
2174 }
Alexander Duyck6234f872016-10-28 11:46:49 -04002175
Amritha Nambiar80d19662018-06-29 21:26:41 -07002176 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
Alexander Duyck6234f872016-10-28 11:46:49 -04002177 kfree_rcu(map, rcu);
2178 return false;
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00002179 }
2180
Alexander Duyck6234f872016-10-28 11:46:49 -04002181 return true;
Alexander Duyck10cdc3f2013-01-10 08:57:17 +00002182}
2183
Alexander Duyck6234f872016-10-28 11:46:49 -04002184static bool remove_xps_queue_cpu(struct net_device *dev,
2185 struct xps_dev_maps *dev_maps,
2186 int cpu, u16 offset, u16 count)
2187{
Alexander Duyck184c4492016-10-28 11:50:13 -04002188 int num_tc = dev->num_tc ? : 1;
2189 bool active = false;
2190 int tci;
Alexander Duyck6234f872016-10-28 11:46:49 -04002191
Alexander Duyck184c4492016-10-28 11:50:13 -04002192 for (tci = cpu * num_tc; num_tc--; tci++) {
2193 int i, j;
2194
2195 for (i = count, j = offset; i--; j++) {
Amritha Nambiar6358d492018-05-17 14:50:44 -07002196 if (!remove_xps_queue(dev_maps, tci, j))
Alexander Duyck184c4492016-10-28 11:50:13 -04002197 break;
2198 }
2199
2200 active |= i < 0;
Alexander Duyck6234f872016-10-28 11:46:49 -04002201 }
2202
Alexander Duyck184c4492016-10-28 11:50:13 -04002203 return active;
Alexander Duyck6234f872016-10-28 11:46:49 -04002204}
2205
Sabrina Dubroca867d0ad2018-11-29 14:14:49 +01002206static void reset_xps_maps(struct net_device *dev,
2207 struct xps_dev_maps *dev_maps,
2208 bool is_rxqs_map)
2209{
2210 if (is_rxqs_map) {
2211 static_key_slow_dec_cpuslocked(&xps_rxqs_needed);
2212 RCU_INIT_POINTER(dev->xps_rxqs_map, NULL);
2213 } else {
2214 RCU_INIT_POINTER(dev->xps_cpus_map, NULL);
2215 }
2216 static_key_slow_dec_cpuslocked(&xps_needed);
2217 kfree_rcu(dev_maps, rcu);
2218}
2219
Amritha Nambiar80d19662018-06-29 21:26:41 -07002220static void clean_xps_maps(struct net_device *dev, const unsigned long *mask,
2221 struct xps_dev_maps *dev_maps, unsigned int nr_ids,
2222 u16 offset, u16 count, bool is_rxqs_map)
2223{
2224 bool active = false;
2225 int i, j;
2226
2227 for (j = -1; j = netif_attrmask_next(j, mask, nr_ids),
2228 j < nr_ids;)
2229 active |= remove_xps_queue_cpu(dev, dev_maps, j, offset,
2230 count);
Sabrina Dubroca867d0ad2018-11-29 14:14:49 +01002231 if (!active)
2232 reset_xps_maps(dev, dev_maps, is_rxqs_map);
Amritha Nambiar80d19662018-06-29 21:26:41 -07002233
Sabrina Dubrocaf28c0202018-11-29 14:14:48 +01002234 if (!is_rxqs_map) {
2235 for (i = offset + (count - 1); count--; i--) {
2236 netdev_queue_numa_node_write(
2237 netdev_get_tx_queue(dev, i),
2238 NUMA_NO_NODE);
Amritha Nambiar80d19662018-06-29 21:26:41 -07002239 }
Amritha Nambiar80d19662018-06-29 21:26:41 -07002240 }
2241}
2242
Alexander Duyck6234f872016-10-28 11:46:49 -04002243static void netif_reset_xps_queues(struct net_device *dev, u16 offset,
2244 u16 count)
Alexander Duyck537c00d2013-01-10 08:57:02 +00002245{
Amritha Nambiar80d19662018-06-29 21:26:41 -07002246 const unsigned long *possible_mask = NULL;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002247 struct xps_dev_maps *dev_maps;
Amritha Nambiar80d19662018-06-29 21:26:41 -07002248 unsigned int nr_ids;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002249
Amritha Nambiar04157462018-06-29 21:26:46 -07002250 if (!static_key_false(&xps_needed))
2251 return;
2252
Andrei Vagin4d99f662018-08-08 20:07:35 -07002253 cpus_read_lock();
Alexander Duyck537c00d2013-01-10 08:57:02 +00002254 mutex_lock(&xps_map_mutex);
Alexander Duyck537c00d2013-01-10 08:57:02 +00002255
Amritha Nambiar04157462018-06-29 21:26:46 -07002256 if (static_key_false(&xps_rxqs_needed)) {
2257 dev_maps = xmap_dereference(dev->xps_rxqs_map);
2258 if (dev_maps) {
2259 nr_ids = dev->num_rx_queues;
2260 clean_xps_maps(dev, possible_mask, dev_maps, nr_ids,
2261 offset, count, true);
2262 }
Amritha Nambiar80d19662018-06-29 21:26:41 -07002263 }
2264
2265 dev_maps = xmap_dereference(dev->xps_cpus_map);
Alexander Duyck537c00d2013-01-10 08:57:02 +00002266 if (!dev_maps)
2267 goto out_no_maps;
2268
Amritha Nambiar80d19662018-06-29 21:26:41 -07002269 if (num_possible_cpus() > 1)
2270 possible_mask = cpumask_bits(cpu_possible_mask);
2271 nr_ids = nr_cpu_ids;
2272 clean_xps_maps(dev, possible_mask, dev_maps, nr_ids, offset, count,
2273 false);
Alexander Duyck024e9672013-01-10 08:57:46 +00002274
Alexander Duyck537c00d2013-01-10 08:57:02 +00002275out_no_maps:
2276 mutex_unlock(&xps_map_mutex);
Andrei Vagin4d99f662018-08-08 20:07:35 -07002277 cpus_read_unlock();
Alexander Duyck537c00d2013-01-10 08:57:02 +00002278}
2279
Alexander Duyck6234f872016-10-28 11:46:49 -04002280static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
2281{
2282 netif_reset_xps_queues(dev, index, dev->num_tx_queues - index);
2283}
2284
Amritha Nambiar80d19662018-06-29 21:26:41 -07002285static struct xps_map *expand_xps_map(struct xps_map *map, int attr_index,
2286 u16 index, bool is_rxqs_map)
Alexander Duyck01c5f862013-01-10 08:57:35 +00002287{
2288 struct xps_map *new_map;
2289 int alloc_len = XPS_MIN_MAP_ALLOC;
2290 int i, pos;
2291
2292 for (pos = 0; map && pos < map->len; pos++) {
2293 if (map->queues[pos] != index)
2294 continue;
2295 return map;
2296 }
2297
Amritha Nambiar80d19662018-06-29 21:26:41 -07002298 /* Need to add tx-queue to this CPU's/rx-queue's existing map */
Alexander Duyck01c5f862013-01-10 08:57:35 +00002299 if (map) {
2300 if (pos < map->alloc_len)
2301 return map;
2302
2303 alloc_len = map->alloc_len * 2;
2304 }
2305
Amritha Nambiar80d19662018-06-29 21:26:41 -07002306 /* Need to allocate new map to store tx-queue on this CPU's/rx-queue's
2307 * map
2308 */
2309 if (is_rxqs_map)
2310 new_map = kzalloc(XPS_MAP_SIZE(alloc_len), GFP_KERNEL);
2311 else
2312 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2313 cpu_to_node(attr_index));
Alexander Duyck01c5f862013-01-10 08:57:35 +00002314 if (!new_map)
2315 return NULL;
2316
2317 for (i = 0; i < pos; i++)
2318 new_map->queues[i] = map->queues[i];
2319 new_map->alloc_len = alloc_len;
2320 new_map->len = pos;
2321
2322 return new_map;
2323}
2324
Andrei Vagin4d99f662018-08-08 20:07:35 -07002325/* Must be called under cpus_read_lock */
Amritha Nambiar80d19662018-06-29 21:26:41 -07002326int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
2327 u16 index, bool is_rxqs_map)
Alexander Duyck537c00d2013-01-10 08:57:02 +00002328{
Amritha Nambiar80d19662018-06-29 21:26:41 -07002329 const unsigned long *online_mask = NULL, *possible_mask = NULL;
Alexander Duyck01c5f862013-01-10 08:57:35 +00002330 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
Amritha Nambiar80d19662018-06-29 21:26:41 -07002331 int i, j, tci, numa_node_id = -2;
Alexander Duyck184c4492016-10-28 11:50:13 -04002332 int maps_sz, num_tc = 1, tc = 0;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002333 struct xps_map *map, *new_map;
Alexander Duyck01c5f862013-01-10 08:57:35 +00002334 bool active = false;
Amritha Nambiar80d19662018-06-29 21:26:41 -07002335 unsigned int nr_ids;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002336
Alexander Duyck184c4492016-10-28 11:50:13 -04002337 if (dev->num_tc) {
Alexander Duyckffcfe252018-07-09 12:19:38 -04002338 /* Do not allow XPS on subordinate device directly */
Alexander Duyck184c4492016-10-28 11:50:13 -04002339 num_tc = dev->num_tc;
Alexander Duyckffcfe252018-07-09 12:19:38 -04002340 if (num_tc < 0)
2341 return -EINVAL;
2342
2343 /* If queue belongs to subordinate dev use its map */
2344 dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev;
2345
Alexander Duyck184c4492016-10-28 11:50:13 -04002346 tc = netdev_txq_to_tc(dev, index);
2347 if (tc < 0)
2348 return -EINVAL;
2349 }
2350
Amritha Nambiar80d19662018-06-29 21:26:41 -07002351 mutex_lock(&xps_map_mutex);
2352 if (is_rxqs_map) {
2353 maps_sz = XPS_RXQ_DEV_MAPS_SIZE(num_tc, dev->num_rx_queues);
2354 dev_maps = xmap_dereference(dev->xps_rxqs_map);
2355 nr_ids = dev->num_rx_queues;
2356 } else {
2357 maps_sz = XPS_CPU_DEV_MAPS_SIZE(num_tc);
2358 if (num_possible_cpus() > 1) {
2359 online_mask = cpumask_bits(cpu_online_mask);
2360 possible_mask = cpumask_bits(cpu_possible_mask);
2361 }
2362 dev_maps = xmap_dereference(dev->xps_cpus_map);
2363 nr_ids = nr_cpu_ids;
2364 }
2365
Alexander Duyck184c4492016-10-28 11:50:13 -04002366 if (maps_sz < L1_CACHE_BYTES)
2367 maps_sz = L1_CACHE_BYTES;
2368
Alexander Duyck01c5f862013-01-10 08:57:35 +00002369 /* allocate memory for queue storage */
Amritha Nambiar80d19662018-06-29 21:26:41 -07002370 for (j = -1; j = netif_attrmask_next_and(j, online_mask, mask, nr_ids),
2371 j < nr_ids;) {
Alexander Duyck01c5f862013-01-10 08:57:35 +00002372 if (!new_dev_maps)
2373 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
Alexander Duyck2bb60cb2013-02-22 06:38:44 +00002374 if (!new_dev_maps) {
2375 mutex_unlock(&xps_map_mutex);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002376 return -ENOMEM;
Alexander Duyck2bb60cb2013-02-22 06:38:44 +00002377 }
Alexander Duyck01c5f862013-01-10 08:57:35 +00002378
Amritha Nambiar80d19662018-06-29 21:26:41 -07002379 tci = j * num_tc + tc;
2380 map = dev_maps ? xmap_dereference(dev_maps->attr_map[tci]) :
Alexander Duyck01c5f862013-01-10 08:57:35 +00002381 NULL;
2382
Amritha Nambiar80d19662018-06-29 21:26:41 -07002383 map = expand_xps_map(map, j, index, is_rxqs_map);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002384 if (!map)
2385 goto error;
2386
Amritha Nambiar80d19662018-06-29 21:26:41 -07002387 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002388 }
2389
2390 if (!new_dev_maps)
2391 goto out_no_new_maps;
2392
Sabrina Dubroca867d0ad2018-11-29 14:14:49 +01002393 if (!dev_maps) {
2394 /* Increment static keys at most once per type */
2395 static_key_slow_inc_cpuslocked(&xps_needed);
2396 if (is_rxqs_map)
2397 static_key_slow_inc_cpuslocked(&xps_rxqs_needed);
2398 }
Amritha Nambiar04157462018-06-29 21:26:46 -07002399
Amritha Nambiar80d19662018-06-29 21:26:41 -07002400 for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids),
2401 j < nr_ids;) {
Alexander Duyck184c4492016-10-28 11:50:13 -04002402 /* copy maps belonging to foreign traffic classes */
Amritha Nambiar80d19662018-06-29 21:26:41 -07002403 for (i = tc, tci = j * num_tc; dev_maps && i--; tci++) {
Alexander Duyck184c4492016-10-28 11:50:13 -04002404 /* fill in the new device map from the old device map */
Amritha Nambiar80d19662018-06-29 21:26:41 -07002405 map = xmap_dereference(dev_maps->attr_map[tci]);
2406 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
Alexander Duyck184c4492016-10-28 11:50:13 -04002407 }
2408
2409 /* We need to explicitly update tci as prevous loop
2410 * could break out early if dev_maps is NULL.
2411 */
Amritha Nambiar80d19662018-06-29 21:26:41 -07002412 tci = j * num_tc + tc;
Alexander Duyck184c4492016-10-28 11:50:13 -04002413
Amritha Nambiar80d19662018-06-29 21:26:41 -07002414 if (netif_attr_test_mask(j, mask, nr_ids) &&
2415 netif_attr_test_online(j, online_mask, nr_ids)) {
2416 /* add tx-queue to CPU/rx-queue maps */
Alexander Duyck01c5f862013-01-10 08:57:35 +00002417 int pos = 0;
2418
Amritha Nambiar80d19662018-06-29 21:26:41 -07002419 map = xmap_dereference(new_dev_maps->attr_map[tci]);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002420 while ((pos < map->len) && (map->queues[pos] != index))
2421 pos++;
2422
2423 if (pos == map->len)
2424 map->queues[map->len++] = index;
Alexander Duyck537c00d2013-01-10 08:57:02 +00002425#ifdef CONFIG_NUMA
Amritha Nambiar80d19662018-06-29 21:26:41 -07002426 if (!is_rxqs_map) {
2427 if (numa_node_id == -2)
2428 numa_node_id = cpu_to_node(j);
2429 else if (numa_node_id != cpu_to_node(j))
2430 numa_node_id = -1;
2431 }
Alexander Duyck537c00d2013-01-10 08:57:02 +00002432#endif
Alexander Duyck01c5f862013-01-10 08:57:35 +00002433 } else if (dev_maps) {
2434 /* fill in the new device map from the old device map */
Amritha Nambiar80d19662018-06-29 21:26:41 -07002435 map = xmap_dereference(dev_maps->attr_map[tci]);
2436 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
Alexander Duyck537c00d2013-01-10 08:57:02 +00002437 }
Alexander Duyck01c5f862013-01-10 08:57:35 +00002438
Alexander Duyck184c4492016-10-28 11:50:13 -04002439 /* copy maps belonging to foreign traffic classes */
2440 for (i = num_tc - tc, tci++; dev_maps && --i; tci++) {
2441 /* fill in the new device map from the old device map */
Amritha Nambiar80d19662018-06-29 21:26:41 -07002442 map = xmap_dereference(dev_maps->attr_map[tci]);
2443 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
Alexander Duyck184c4492016-10-28 11:50:13 -04002444 }
Alexander Duyck537c00d2013-01-10 08:57:02 +00002445 }
2446
Amritha Nambiar80d19662018-06-29 21:26:41 -07002447 if (is_rxqs_map)
2448 rcu_assign_pointer(dev->xps_rxqs_map, new_dev_maps);
2449 else
2450 rcu_assign_pointer(dev->xps_cpus_map, new_dev_maps);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002451
Alexander Duyck537c00d2013-01-10 08:57:02 +00002452 /* Cleanup old maps */
Alexander Duyck184c4492016-10-28 11:50:13 -04002453 if (!dev_maps)
2454 goto out_no_old_maps;
2455
Amritha Nambiar80d19662018-06-29 21:26:41 -07002456 for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids),
2457 j < nr_ids;) {
2458 for (i = num_tc, tci = j * num_tc; i--; tci++) {
2459 new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
2460 map = xmap_dereference(dev_maps->attr_map[tci]);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002461 if (map && map != new_map)
2462 kfree_rcu(map, rcu);
2463 }
Alexander Duyck01c5f862013-01-10 08:57:35 +00002464 }
Alexander Duyck537c00d2013-01-10 08:57:02 +00002465
Alexander Duyck184c4492016-10-28 11:50:13 -04002466 kfree_rcu(dev_maps, rcu);
2467
2468out_no_old_maps:
Alexander Duyck01c5f862013-01-10 08:57:35 +00002469 dev_maps = new_dev_maps;
2470 active = true;
2471
2472out_no_new_maps:
Amritha Nambiar80d19662018-06-29 21:26:41 -07002473 if (!is_rxqs_map) {
2474 /* update Tx queue numa node */
2475 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2476 (numa_node_id >= 0) ?
2477 numa_node_id : NUMA_NO_NODE);
2478 }
Alexander Duyck537c00d2013-01-10 08:57:02 +00002479
Alexander Duyck01c5f862013-01-10 08:57:35 +00002480 if (!dev_maps)
2481 goto out_no_maps;
2482
Amritha Nambiar80d19662018-06-29 21:26:41 -07002483 /* removes tx-queue from unused CPUs/rx-queues */
2484 for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids),
2485 j < nr_ids;) {
2486 for (i = tc, tci = j * num_tc; i--; tci++)
Alexander Duyck184c4492016-10-28 11:50:13 -04002487 active |= remove_xps_queue(dev_maps, tci, index);
Amritha Nambiar80d19662018-06-29 21:26:41 -07002488 if (!netif_attr_test_mask(j, mask, nr_ids) ||
2489 !netif_attr_test_online(j, online_mask, nr_ids))
Alexander Duyck184c4492016-10-28 11:50:13 -04002490 active |= remove_xps_queue(dev_maps, tci, index);
2491 for (i = num_tc - tc, tci++; --i; tci++)
2492 active |= remove_xps_queue(dev_maps, tci, index);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002493 }
2494
2495 /* free map if not active */
Sabrina Dubroca867d0ad2018-11-29 14:14:49 +01002496 if (!active)
2497 reset_xps_maps(dev, dev_maps, is_rxqs_map);
Alexander Duyck01c5f862013-01-10 08:57:35 +00002498
2499out_no_maps:
Alexander Duyck537c00d2013-01-10 08:57:02 +00002500 mutex_unlock(&xps_map_mutex);
2501
2502 return 0;
2503error:
Alexander Duyck01c5f862013-01-10 08:57:35 +00002504 /* remove any maps that we added */
Amritha Nambiar80d19662018-06-29 21:26:41 -07002505 for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids),
2506 j < nr_ids;) {
2507 for (i = num_tc, tci = j * num_tc; i--; tci++) {
2508 new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
Alexander Duyck184c4492016-10-28 11:50:13 -04002509 map = dev_maps ?
Amritha Nambiar80d19662018-06-29 21:26:41 -07002510 xmap_dereference(dev_maps->attr_map[tci]) :
Alexander Duyck184c4492016-10-28 11:50:13 -04002511 NULL;
2512 if (new_map && new_map != map)
2513 kfree(new_map);
2514 }
Alexander Duyck01c5f862013-01-10 08:57:35 +00002515 }
2516
Alexander Duyck537c00d2013-01-10 08:57:02 +00002517 mutex_unlock(&xps_map_mutex);
2518
Alexander Duyck537c00d2013-01-10 08:57:02 +00002519 kfree(new_dev_maps);
2520 return -ENOMEM;
2521}
Andrei Vagin4d99f662018-08-08 20:07:35 -07002522EXPORT_SYMBOL_GPL(__netif_set_xps_queue);
Amritha Nambiar80d19662018-06-29 21:26:41 -07002523
2524int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2525 u16 index)
2526{
Andrei Vagin4d99f662018-08-08 20:07:35 -07002527 int ret;
2528
2529 cpus_read_lock();
2530 ret = __netif_set_xps_queue(dev, cpumask_bits(mask), index, false);
2531 cpus_read_unlock();
2532
2533 return ret;
Amritha Nambiar80d19662018-06-29 21:26:41 -07002534}
Alexander Duyck537c00d2013-01-10 08:57:02 +00002535EXPORT_SYMBOL(netif_set_xps_queue);
2536
2537#endif
Alexander Duyckffcfe252018-07-09 12:19:38 -04002538static void netdev_unbind_all_sb_channels(struct net_device *dev)
2539{
2540 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
2541
2542 /* Unbind any subordinate channels */
2543 while (txq-- != &dev->_tx[0]) {
2544 if (txq->sb_dev)
2545 netdev_unbind_sb_channel(dev, txq->sb_dev);
2546 }
2547}
2548
Alexander Duyck9cf1f6a2016-10-28 11:43:20 -04002549void netdev_reset_tc(struct net_device *dev)
2550{
Alexander Duyck6234f872016-10-28 11:46:49 -04002551#ifdef CONFIG_XPS
2552 netif_reset_xps_queues_gt(dev, 0);
2553#endif
Alexander Duyckffcfe252018-07-09 12:19:38 -04002554 netdev_unbind_all_sb_channels(dev);
2555
2556 /* Reset TC configuration of device */
Alexander Duyck9cf1f6a2016-10-28 11:43:20 -04002557 dev->num_tc = 0;
2558 memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq));
2559 memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map));
2560}
2561EXPORT_SYMBOL(netdev_reset_tc);
2562
2563int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
2564{
2565 if (tc >= dev->num_tc)
2566 return -EINVAL;
2567
Alexander Duyck6234f872016-10-28 11:46:49 -04002568#ifdef CONFIG_XPS
2569 netif_reset_xps_queues(dev, offset, count);
2570#endif
Alexander Duyck9cf1f6a2016-10-28 11:43:20 -04002571 dev->tc_to_txq[tc].count = count;
2572 dev->tc_to_txq[tc].offset = offset;
2573 return 0;
2574}
2575EXPORT_SYMBOL(netdev_set_tc_queue);
2576
2577int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
2578{
2579 if (num_tc > TC_MAX_QUEUE)
2580 return -EINVAL;
2581
Alexander Duyck6234f872016-10-28 11:46:49 -04002582#ifdef CONFIG_XPS
2583 netif_reset_xps_queues_gt(dev, 0);
2584#endif
Alexander Duyckffcfe252018-07-09 12:19:38 -04002585 netdev_unbind_all_sb_channels(dev);
2586
Alexander Duyck9cf1f6a2016-10-28 11:43:20 -04002587 dev->num_tc = num_tc;
2588 return 0;
2589}
2590EXPORT_SYMBOL(netdev_set_num_tc);
2591
Alexander Duyckffcfe252018-07-09 12:19:38 -04002592void netdev_unbind_sb_channel(struct net_device *dev,
2593 struct net_device *sb_dev)
2594{
2595 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
2596
2597#ifdef CONFIG_XPS
2598 netif_reset_xps_queues_gt(sb_dev, 0);
2599#endif
2600 memset(sb_dev->tc_to_txq, 0, sizeof(sb_dev->tc_to_txq));
2601 memset(sb_dev->prio_tc_map, 0, sizeof(sb_dev->prio_tc_map));
2602
2603 while (txq-- != &dev->_tx[0]) {
2604 if (txq->sb_dev == sb_dev)
2605 txq->sb_dev = NULL;
2606 }
2607}
2608EXPORT_SYMBOL(netdev_unbind_sb_channel);
2609
2610int netdev_bind_sb_channel_queue(struct net_device *dev,
2611 struct net_device *sb_dev,
2612 u8 tc, u16 count, u16 offset)
2613{
2614 /* Make certain the sb_dev and dev are already configured */
2615 if (sb_dev->num_tc >= 0 || tc >= dev->num_tc)
2616 return -EINVAL;
2617
2618 /* We cannot hand out queues we don't have */
2619 if ((offset + count) > dev->real_num_tx_queues)
2620 return -EINVAL;
2621
2622 /* Record the mapping */
2623 sb_dev->tc_to_txq[tc].count = count;
2624 sb_dev->tc_to_txq[tc].offset = offset;
2625
2626 /* Provide a way for Tx queue to find the tc_to_txq map or
2627 * XPS map for itself.
2628 */
2629 while (count--)
2630 netdev_get_tx_queue(dev, count + offset)->sb_dev = sb_dev;
2631
2632 return 0;
2633}
2634EXPORT_SYMBOL(netdev_bind_sb_channel_queue);
2635
2636int netdev_set_sb_channel(struct net_device *dev, u16 channel)
2637{
2638 /* Do not use a multiqueue device to represent a subordinate channel */
2639 if (netif_is_multiqueue(dev))
2640 return -ENODEV;
2641
2642 /* We allow channels 1 - 32767 to be used for subordinate channels.
2643 * Channel 0 is meant to be "native" mode and used only to represent
2644 * the main root device. We allow writing 0 to reset the device back
2645 * to normal mode after being used as a subordinate channel.
2646 */
2647 if (channel > S16_MAX)
2648 return -EINVAL;
2649
2650 dev->num_tc = -channel;
2651
2652 return 0;
2653}
2654EXPORT_SYMBOL(netdev_set_sb_channel);
2655
John Fastabendf0796d52010-07-01 13:21:57 +00002656/*
2657 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
Gal Pressman3a053b12018-02-28 15:59:15 +02002658 * greater than real_num_tx_queues stale skbs on the qdisc must be flushed.
John Fastabendf0796d52010-07-01 13:21:57 +00002659 */
Tom Herberte6484932010-10-18 18:04:39 +00002660int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
John Fastabendf0796d52010-07-01 13:21:57 +00002661{
Jakub Kicinskiac5b7012018-02-12 21:35:31 -08002662 bool disabling;
Tom Herbert1d24eb42010-11-21 13:17:27 +00002663 int rc;
2664
Jakub Kicinskiac5b7012018-02-12 21:35:31 -08002665 disabling = txq < dev->real_num_tx_queues;
2666
Tom Herberte6484932010-10-18 18:04:39 +00002667 if (txq < 1 || txq > dev->num_tx_queues)
2668 return -EINVAL;
John Fastabendf0796d52010-07-01 13:21:57 +00002669
Ben Hutchings5c565802011-02-15 19:39:21 +00002670 if (dev->reg_state == NETREG_REGISTERED ||
2671 dev->reg_state == NETREG_UNREGISTERING) {
Tom Herberte6484932010-10-18 18:04:39 +00002672 ASSERT_RTNL();
2673
Tom Herbert1d24eb42010-11-21 13:17:27 +00002674 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2675 txq);
Tom Herbertbf264142010-11-26 08:36:09 +00002676 if (rc)
2677 return rc;
2678
John Fastabend4f57c082011-01-17 08:06:04 +00002679 if (dev->num_tc)
2680 netif_setup_tc(dev, txq);
2681
Jakub Kicinskiac5b7012018-02-12 21:35:31 -08002682 dev->real_num_tx_queues = txq;
2683
2684 if (disabling) {
2685 synchronize_net();
Tom Herberte6484932010-10-18 18:04:39 +00002686 qdisc_reset_all_tx_gt(dev, txq);
Alexander Duyck024e9672013-01-10 08:57:46 +00002687#ifdef CONFIG_XPS
2688 netif_reset_xps_queues_gt(dev, txq);
2689#endif
2690 }
Jakub Kicinskiac5b7012018-02-12 21:35:31 -08002691 } else {
2692 dev->real_num_tx_queues = txq;
John Fastabendf0796d52010-07-01 13:21:57 +00002693 }
Tom Herberte6484932010-10-18 18:04:39 +00002694
Tom Herberte6484932010-10-18 18:04:39 +00002695 return 0;
John Fastabendf0796d52010-07-01 13:21:57 +00002696}
2697EXPORT_SYMBOL(netif_set_real_num_tx_queues);
Denis Vlasenko56079432006-03-29 15:57:29 -08002698
Michael Daltona953be52014-01-16 22:23:28 -08002699#ifdef CONFIG_SYSFS
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002700/**
2701 * netif_set_real_num_rx_queues - set actual number of RX queues used
2702 * @dev: Network device
2703 * @rxq: Actual number of RX queues
2704 *
2705 * This must be called either with the rtnl_lock held or before
2706 * registration of the net device. Returns 0 on success, or a
Ben Hutchings4e7f7952010-10-08 10:33:39 -07002707 * negative error code. If called before registration, it always
2708 * succeeds.
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002709 */
2710int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2711{
2712 int rc;
2713
Tom Herbertbd25fa72010-10-18 18:00:16 +00002714 if (rxq < 1 || rxq > dev->num_rx_queues)
2715 return -EINVAL;
2716
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002717 if (dev->reg_state == NETREG_REGISTERED) {
2718 ASSERT_RTNL();
2719
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002720 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2721 rxq);
2722 if (rc)
2723 return rc;
Ben Hutchings62fe0b42010-09-27 08:24:33 +00002724 }
2725
2726 dev->real_num_rx_queues = rxq;
2727 return 0;
2728}
2729EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2730#endif
2731
Ben Hutchings2c530402012-07-10 10:55:09 +00002732/**
2733 * netif_get_num_default_rss_queues - default number of RSS queues
Yuval Mintz16917b82012-07-01 03:18:50 +00002734 *
2735 * This routine should set an upper limit on the number of RSS queues
2736 * used by default by multiqueue devices.
2737 */
Ben Hutchingsa55b1382012-07-10 10:54:38 +00002738int netif_get_num_default_rss_queues(void)
Yuval Mintz16917b82012-07-01 03:18:50 +00002739{
Hariprasad Shenai40e4e712016-06-08 18:09:08 +05302740 return is_kdump_kernel() ?
2741 1 : min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
Yuval Mintz16917b82012-07-01 03:18:50 +00002742}
2743EXPORT_SYMBOL(netif_get_num_default_rss_queues);
2744
Eric Dumazet3bcb8462016-06-04 20:02:28 -07002745static void __netif_reschedule(struct Qdisc *q)
Jarek Poplawskidef82a12008-08-17 21:54:43 -07002746{
2747 struct softnet_data *sd;
2748 unsigned long flags;
2749
2750 local_irq_save(flags);
Christoph Lameter903ceff2014-08-17 12:30:35 -05002751 sd = this_cpu_ptr(&softnet_data);
Changli Gaoa9cbd582010-04-26 23:06:24 +00002752 q->next_sched = NULL;
2753 *sd->output_queue_tailp = q;
2754 sd->output_queue_tailp = &q->next_sched;
Jarek Poplawskidef82a12008-08-17 21:54:43 -07002755 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2756 local_irq_restore(flags);
2757}
2758
David S. Miller37437bb2008-07-16 02:15:04 -07002759void __netif_schedule(struct Qdisc *q)
Denis Vlasenko56079432006-03-29 15:57:29 -08002760{
Jarek Poplawskidef82a12008-08-17 21:54:43 -07002761 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
2762 __netif_reschedule(q);
Denis Vlasenko56079432006-03-29 15:57:29 -08002763}
2764EXPORT_SYMBOL(__netif_schedule);
2765
Eric Dumazete6247022013-12-05 04:45:08 -08002766struct dev_kfree_skb_cb {
2767 enum skb_free_reason reason;
2768};
2769
2770static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
Denis Vlasenko56079432006-03-29 15:57:29 -08002771{
Eric Dumazete6247022013-12-05 04:45:08 -08002772 return (struct dev_kfree_skb_cb *)skb->cb;
Denis Vlasenko56079432006-03-29 15:57:29 -08002773}
Denis Vlasenko56079432006-03-29 15:57:29 -08002774
John Fastabend46e5da40a2014-09-12 20:04:52 -07002775void netif_schedule_queue(struct netdev_queue *txq)
2776{
2777 rcu_read_lock();
2778 if (!(txq->state & QUEUE_STATE_ANY_XOFF)) {
2779 struct Qdisc *q = rcu_dereference(txq->qdisc);
2780
2781 __netif_schedule(q);
2782 }
2783 rcu_read_unlock();
2784}
2785EXPORT_SYMBOL(netif_schedule_queue);
2786
John Fastabend46e5da40a2014-09-12 20:04:52 -07002787void netif_tx_wake_queue(struct netdev_queue *dev_queue)
2788{
2789 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
2790 struct Qdisc *q;
2791
2792 rcu_read_lock();
2793 q = rcu_dereference(dev_queue->qdisc);
2794 __netif_schedule(q);
2795 rcu_read_unlock();
2796 }
2797}
2798EXPORT_SYMBOL(netif_tx_wake_queue);
2799
Eric Dumazete6247022013-12-05 04:45:08 -08002800void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
2801{
2802 unsigned long flags;
2803
Myungho Jung98998862017-04-25 11:58:15 -07002804 if (unlikely(!skb))
2805 return;
2806
Reshetova, Elena63354792017-06-30 13:07:58 +03002807 if (likely(refcount_read(&skb->users) == 1)) {
Eric Dumazete6247022013-12-05 04:45:08 -08002808 smp_rmb();
Reshetova, Elena63354792017-06-30 13:07:58 +03002809 refcount_set(&skb->users, 0);
2810 } else if (likely(!refcount_dec_and_test(&skb->users))) {
Eric Dumazete6247022013-12-05 04:45:08 -08002811 return;
2812 }
2813 get_kfree_skb_cb(skb)->reason = reason;
2814 local_irq_save(flags);
2815 skb->next = __this_cpu_read(softnet_data.completion_queue);
2816 __this_cpu_write(softnet_data.completion_queue, skb);
2817 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2818 local_irq_restore(flags);
2819}
2820EXPORT_SYMBOL(__dev_kfree_skb_irq);
2821
2822void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
Denis Vlasenko56079432006-03-29 15:57:29 -08002823{
2824 if (in_irq() || irqs_disabled())
Eric Dumazete6247022013-12-05 04:45:08 -08002825 __dev_kfree_skb_irq(skb, reason);
Denis Vlasenko56079432006-03-29 15:57:29 -08002826 else
2827 dev_kfree_skb(skb);
2828}
Eric Dumazete6247022013-12-05 04:45:08 -08002829EXPORT_SYMBOL(__dev_kfree_skb_any);
Denis Vlasenko56079432006-03-29 15:57:29 -08002830
2831
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002832/**
2833 * netif_device_detach - mark device as removed
2834 * @dev: network device
2835 *
2836 * Mark device as removed from system and therefore no longer available.
2837 */
Denis Vlasenko56079432006-03-29 15:57:29 -08002838void netif_device_detach(struct net_device *dev)
2839{
2840 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
2841 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00002842 netif_tx_stop_all_queues(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08002843 }
2844}
2845EXPORT_SYMBOL(netif_device_detach);
2846
Stephen Hemmingerbea33482007-10-03 16:41:36 -07002847/**
2848 * netif_device_attach - mark device as attached
2849 * @dev: network device
2850 *
2851 * Mark device as attached from system and restart if needed.
2852 */
Denis Vlasenko56079432006-03-29 15:57:29 -08002853void netif_device_attach(struct net_device *dev)
2854{
2855 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
2856 netif_running(dev)) {
Alexander Duyckd5431032009-04-08 13:15:22 +00002857 netif_tx_wake_all_queues(dev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002858 __netdev_watchdog_up(dev);
Denis Vlasenko56079432006-03-29 15:57:29 -08002859 }
2860}
2861EXPORT_SYMBOL(netif_device_attach);
2862
Jiri Pirko5605c762015-05-12 14:56:12 +02002863/*
2864 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2865 * to be used as a distribution range.
2866 */
Alexander Duyckeadec8772018-07-09 12:19:48 -04002867static u16 skb_tx_hash(const struct net_device *dev,
2868 const struct net_device *sb_dev,
2869 struct sk_buff *skb)
Jiri Pirko5605c762015-05-12 14:56:12 +02002870{
2871 u32 hash;
2872 u16 qoffset = 0;
Alexander Duyck1b837d42018-04-27 14:06:53 -04002873 u16 qcount = dev->real_num_tx_queues;
Jiri Pirko5605c762015-05-12 14:56:12 +02002874
Alexander Duyckeadec8772018-07-09 12:19:48 -04002875 if (dev->num_tc) {
2876 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
2877
2878 qoffset = sb_dev->tc_to_txq[tc].offset;
2879 qcount = sb_dev->tc_to_txq[tc].count;
2880 }
2881
Jiri Pirko5605c762015-05-12 14:56:12 +02002882 if (skb_rx_queue_recorded(skb)) {
2883 hash = skb_get_rx_queue(skb);
Alexander Duyck1b837d42018-04-27 14:06:53 -04002884 while (unlikely(hash >= qcount))
2885 hash -= qcount;
Alexander Duyckeadec8772018-07-09 12:19:48 -04002886 return hash + qoffset;
Jiri Pirko5605c762015-05-12 14:56:12 +02002887 }
2888
2889 return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
2890}
Jiri Pirko5605c762015-05-12 14:56:12 +02002891
Ben Hutchings36c92472012-01-17 07:57:56 +00002892static void skb_warn_bad_offload(const struct sk_buff *skb)
2893{
Wei Tang84d15ae2016-06-16 21:17:49 +08002894 static const netdev_features_t null_features;
Ben Hutchings36c92472012-01-17 07:57:56 +00002895 struct net_device *dev = skb->dev;
Bjørn Mork88ad4172015-11-16 19:16:40 +01002896 const char *name = "";
Ben Hutchings36c92472012-01-17 07:57:56 +00002897
Ben Greearc846ad92013-04-19 10:45:52 +00002898 if (!net_ratelimit())
2899 return;
2900
Bjørn Mork88ad4172015-11-16 19:16:40 +01002901 if (dev) {
2902 if (dev->dev.parent)
2903 name = dev_driver_string(dev->dev.parent);
2904 else
2905 name = netdev_name(dev);
2906 }
Ben Hutchings36c92472012-01-17 07:57:56 +00002907 WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
2908 "gso_type=%d ip_summed=%d\n",
Bjørn Mork88ad4172015-11-16 19:16:40 +01002909 name, dev ? &dev->features : &null_features,
Michał Mirosław65e9d2f2012-01-17 10:00:40 +00002910 skb->sk ? &skb->sk->sk_route_caps : &null_features,
Ben Hutchings36c92472012-01-17 07:57:56 +00002911 skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
2912 skb_shinfo(skb)->gso_type, skb->ip_summed);
2913}
2914
Linus Torvalds1da177e2005-04-16 15:20:36 -07002915/*
2916 * Invalidate hardware checksum when packet is to be mangled, and
2917 * complete checksum manually on outgoing path.
2918 */
Patrick McHardy84fa7932006-08-29 16:44:56 -07002919int skb_checksum_help(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002920{
Al Virod3bc23e2006-11-14 21:24:49 -08002921 __wsum csum;
Herbert Xu663ead32007-04-09 11:59:07 -07002922 int ret = 0, offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002923
Patrick McHardy84fa7932006-08-29 16:44:56 -07002924 if (skb->ip_summed == CHECKSUM_COMPLETE)
Herbert Xua430a432006-07-08 13:34:56 -07002925 goto out_set_summed;
2926
2927 if (unlikely(skb_shinfo(skb)->gso_size)) {
Ben Hutchings36c92472012-01-17 07:57:56 +00002928 skb_warn_bad_offload(skb);
2929 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002930 }
2931
Eric Dumazetcef401d2013-01-25 20:34:37 +00002932 /* Before computing a checksum, we should make sure no frag could
2933 * be modified by an external entity : checksum could be wrong.
2934 */
2935 if (skb_has_shared_frag(skb)) {
2936 ret = __skb_linearize(skb);
2937 if (ret)
2938 goto out;
2939 }
2940
Michał Mirosław55508d62010-12-14 15:24:08 +00002941 offset = skb_checksum_start_offset(skb);
Herbert Xua0308472007-10-15 01:47:15 -07002942 BUG_ON(offset >= skb_headlen(skb));
2943 csum = skb_checksum(skb, offset, skb->len - offset, 0);
2944
2945 offset += skb->csum_offset;
2946 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
2947
2948 if (skb_cloned(skb) &&
2949 !skb_clone_writable(skb, offset + sizeof(__sum16))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002950 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2951 if (ret)
2952 goto out;
2953 }
2954
Eric Dumazet4f2e4ad2016-10-29 11:02:36 -07002955 *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
Herbert Xua430a432006-07-08 13:34:56 -07002956out_set_summed:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002957 skb->ip_summed = CHECKSUM_NONE;
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09002958out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002959 return ret;
2960}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07002961EXPORT_SYMBOL(skb_checksum_help);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002962
Davide Carattib72b5bf2017-05-18 15:44:38 +02002963int skb_crc32c_csum_help(struct sk_buff *skb)
2964{
2965 __le32 crc32c_csum;
2966 int ret = 0, offset, start;
2967
2968 if (skb->ip_summed != CHECKSUM_PARTIAL)
2969 goto out;
2970
2971 if (unlikely(skb_is_gso(skb)))
2972 goto out;
2973
2974 /* Before computing a checksum, we should make sure no frag could
2975 * be modified by an external entity : checksum could be wrong.
2976 */
2977 if (unlikely(skb_has_shared_frag(skb))) {
2978 ret = __skb_linearize(skb);
2979 if (ret)
2980 goto out;
2981 }
2982 start = skb_checksum_start_offset(skb);
2983 offset = start + offsetof(struct sctphdr, checksum);
2984 if (WARN_ON_ONCE(offset >= skb_headlen(skb))) {
2985 ret = -EINVAL;
2986 goto out;
2987 }
2988 if (skb_cloned(skb) &&
2989 !skb_clone_writable(skb, offset + sizeof(__le32))) {
2990 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2991 if (ret)
2992 goto out;
2993 }
2994 crc32c_csum = cpu_to_le32(~__skb_checksum(skb, start,
2995 skb->len - start, ~(__u32)0,
2996 crc32c_csum_stub));
2997 *(__le32 *)(skb->data + offset) = crc32c_csum;
2998 skb->ip_summed = CHECKSUM_NONE;
Davide Carattidba00302017-05-18 15:44:40 +02002999 skb->csum_not_inet = 0;
Davide Carattib72b5bf2017-05-18 15:44:38 +02003000out:
3001 return ret;
3002}
3003
Vlad Yasevich53d64712014-03-27 17:26:18 -04003004__be16 skb_network_protocol(struct sk_buff *skb, int *depth)
Pravin B Shelarec5f0612013-03-07 09:28:01 +00003005{
3006 __be16 type = skb->protocol;
3007
Pravin B Shelar19acc322013-05-07 20:41:07 +00003008 /* Tunnel gso handlers can set protocol to ethernet. */
3009 if (type == htons(ETH_P_TEB)) {
3010 struct ethhdr *eth;
3011
3012 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
3013 return 0;
3014
Eric Dumazet1dfe82e2018-03-26 08:08:07 -07003015 eth = (struct ethhdr *)skb->data;
Pravin B Shelar19acc322013-05-07 20:41:07 +00003016 type = eth->h_proto;
3017 }
3018
Toshiaki Makitad4bcef32015-01-29 20:37:07 +09003019 return __vlan_get_protocol(skb, type, depth);
Pravin B Shelarec5f0612013-03-07 09:28:01 +00003020}
3021
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00003022/**
3023 * skb_mac_gso_segment - mac layer segmentation handler.
3024 * @skb: buffer to segment
3025 * @features: features for the output path (see dev->features)
3026 */
3027struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
3028 netdev_features_t features)
3029{
3030 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
3031 struct packet_offload *ptype;
Vlad Yasevich53d64712014-03-27 17:26:18 -04003032 int vlan_depth = skb->mac_len;
3033 __be16 type = skb_network_protocol(skb, &vlan_depth);
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00003034
Pravin B Shelarec5f0612013-03-07 09:28:01 +00003035 if (unlikely(!type))
3036 return ERR_PTR(-EINVAL);
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00003037
Vlad Yasevich53d64712014-03-27 17:26:18 -04003038 __skb_pull(skb, vlan_depth);
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00003039
3040 rcu_read_lock();
3041 list_for_each_entry_rcu(ptype, &offload_base, list) {
3042 if (ptype->type == type && ptype->callbacks.gso_segment) {
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00003043 segs = ptype->callbacks.gso_segment(skb, features);
3044 break;
3045 }
3046 }
3047 rcu_read_unlock();
3048
3049 __skb_push(skb, skb->data - skb_mac_header(skb));
3050
3051 return segs;
3052}
3053EXPORT_SYMBOL(skb_mac_gso_segment);
3054
3055
Cong Wang12b00042013-02-05 16:36:38 +00003056/* openvswitch calls this on rx path, so we need a different check.
3057 */
3058static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
3059{
3060 if (tx_path)
Willem de Bruijn0c19f8462017-11-21 10:22:25 -05003061 return skb->ip_summed != CHECKSUM_PARTIAL &&
3062 skb->ip_summed != CHECKSUM_UNNECESSARY;
Eric Dumazet6e7bc472017-02-03 14:29:42 -08003063
3064 return skb->ip_summed == CHECKSUM_NONE;
Cong Wang12b00042013-02-05 16:36:38 +00003065}
3066
Herbert Xuf6a78bf2006-06-22 02:57:17 -07003067/**
Cong Wang12b00042013-02-05 16:36:38 +00003068 * __skb_gso_segment - Perform segmentation on skb.
Herbert Xuf6a78bf2006-06-22 02:57:17 -07003069 * @skb: buffer to segment
Herbert Xu576a30e2006-06-27 13:22:38 -07003070 * @features: features for the output path (see dev->features)
Cong Wang12b00042013-02-05 16:36:38 +00003071 * @tx_path: whether it is called in TX path
Herbert Xuf6a78bf2006-06-22 02:57:17 -07003072 *
3073 * This function segments the given skb and returns a list of segments.
Herbert Xu576a30e2006-06-27 13:22:38 -07003074 *
3075 * It may return NULL if the skb requires no segmentation. This is
3076 * only possible when GSO is used for verifying header integrity.
Konstantin Khlebnikov9207f9d2016-01-08 15:21:46 +03003077 *
3078 * Segmentation preserves SKB_SGO_CB_OFFSET bytes of previous skb cb.
Herbert Xuf6a78bf2006-06-22 02:57:17 -07003079 */
Cong Wang12b00042013-02-05 16:36:38 +00003080struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
3081 netdev_features_t features, bool tx_path)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07003082{
Eric Dumazetb2504a52017-01-31 10:20:32 -08003083 struct sk_buff *segs;
3084
Cong Wang12b00042013-02-05 16:36:38 +00003085 if (unlikely(skb_needs_check(skb, tx_path))) {
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00003086 int err;
3087
Eric Dumazetb2504a52017-01-31 10:20:32 -08003088 /* We're going to init ->check field in TCP or UDP header */
françois romieua40e0a62014-07-15 23:55:35 +02003089 err = skb_cow_head(skb, 0);
3090 if (err < 0)
Herbert Xua430a432006-07-08 13:34:56 -07003091 return ERR_PTR(err);
3092 }
3093
Alexander Duyck802ab552016-04-10 21:45:03 -04003094 /* Only report GSO partial support if it will enable us to
3095 * support segmentation on this frame without needing additional
3096 * work.
3097 */
3098 if (features & NETIF_F_GSO_PARTIAL) {
3099 netdev_features_t partial_features = NETIF_F_GSO_ROBUST;
3100 struct net_device *dev = skb->dev;
3101
3102 partial_features |= dev->features & dev->gso_partial_features;
3103 if (!skb_gso_ok(skb, features | partial_features))
3104 features &= ~NETIF_F_GSO_PARTIAL;
3105 }
3106
Konstantin Khlebnikov9207f9d2016-01-08 15:21:46 +03003107 BUILD_BUG_ON(SKB_SGO_CB_OFFSET +
3108 sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb));
3109
Pravin B Shelar68c33162013-02-14 14:02:41 +00003110 SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
Eric Dumazet3347c962013-10-19 11:42:56 -07003111 SKB_GSO_CB(skb)->encap_level = 0;
3112
Pravin B Shelar05e8ef42013-02-14 09:44:55 +00003113 skb_reset_mac_header(skb);
3114 skb_reset_mac_len(skb);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07003115
Eric Dumazetb2504a52017-01-31 10:20:32 -08003116 segs = skb_mac_gso_segment(skb, features);
3117
Willem de Bruijn8d74e9f2017-12-12 11:39:04 -05003118 if (unlikely(skb_needs_check(skb, tx_path) && !IS_ERR(segs)))
Eric Dumazetb2504a52017-01-31 10:20:32 -08003119 skb_warn_bad_offload(skb);
3120
3121 return segs;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07003122}
Cong Wang12b00042013-02-05 16:36:38 +00003123EXPORT_SYMBOL(__skb_gso_segment);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07003124
Herbert Xufb286bb2005-11-10 13:01:24 -08003125/* Take action when hardware reception checksum errors are detected. */
3126#ifdef CONFIG_BUG
Cong Wang7fe50ac2018-11-12 14:47:18 -08003127void netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb)
Herbert Xufb286bb2005-11-10 13:01:24 -08003128{
3129 if (net_ratelimit()) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00003130 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
Cong Wang7fe50ac2018-11-12 14:47:18 -08003131 if (dev)
3132 pr_err("dev features: %pNF\n", &dev->features);
3133 pr_err("skb len=%u data_len=%u pkt_type=%u gso_size=%u gso_type=%u nr_frags=%u ip_summed=%u csum=%x csum_complete_sw=%d csum_valid=%d csum_level=%u\n",
3134 skb->len, skb->data_len, skb->pkt_type,
3135 skb_shinfo(skb)->gso_size, skb_shinfo(skb)->gso_type,
3136 skb_shinfo(skb)->nr_frags, skb->ip_summed, skb->csum,
3137 skb->csum_complete_sw, skb->csum_valid, skb->csum_level);
Herbert Xufb286bb2005-11-10 13:01:24 -08003138 dump_stack();
3139 }
3140}
3141EXPORT_SYMBOL(netdev_rx_csum_fault);
3142#endif
3143
Christoph Hellwigab74cfe2018-04-03 20:31:35 +02003144/* XXX: check that highmem exists at all on the given machine. */
Florian Westphalc1e756b2014-05-05 15:00:44 +02003145static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003146{
Herbert Xu3d3a8532006-06-27 13:33:10 -07003147#ifdef CONFIG_HIGHMEM
Linus Torvalds1da177e2005-04-16 15:20:36 -07003148 int i;
tchardingf4563a72017-02-09 17:56:07 +11003149
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00003150 if (!(dev->features & NETIF_F_HIGHDMA)) {
Ian Campbellea2ab692011-08-22 23:44:58 +00003151 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3152 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
tchardingf4563a72017-02-09 17:56:07 +11003153
Ian Campbellea2ab692011-08-22 23:44:58 +00003154 if (PageHighMem(skb_frag_page(frag)))
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00003155 return 1;
Ian Campbellea2ab692011-08-22 23:44:58 +00003156 }
FUJITA Tomonori5acbbd42010-03-30 22:35:50 +00003157 }
Herbert Xu3d3a8532006-06-27 13:33:10 -07003158#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07003159 return 0;
3160}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003161
Simon Horman3b392dd2014-06-04 08:53:17 +09003162/* If MPLS offload request, verify we are testing hardware MPLS features
3163 * instead of standard features for the netdev.
3164 */
Pravin B Shelard0edc7b2014-12-23 16:20:11 -08003165#if IS_ENABLED(CONFIG_NET_MPLS_GSO)
Simon Horman3b392dd2014-06-04 08:53:17 +09003166static netdev_features_t net_mpls_features(struct sk_buff *skb,
3167 netdev_features_t features,
3168 __be16 type)
3169{
Simon Horman25cd9ba2014-10-06 05:05:13 -07003170 if (eth_p_mpls(type))
Simon Horman3b392dd2014-06-04 08:53:17 +09003171 features &= skb->dev->mpls_features;
3172
3173 return features;
3174}
3175#else
3176static netdev_features_t net_mpls_features(struct sk_buff *skb,
3177 netdev_features_t features,
3178 __be16 type)
3179{
3180 return features;
3181}
3182#endif
3183
Michał Mirosławc8f44af2011-11-15 15:29:55 +00003184static netdev_features_t harmonize_features(struct sk_buff *skb,
Florian Westphalc1e756b2014-05-05 15:00:44 +02003185 netdev_features_t features)
Jesse Grossf01a5232011-01-09 06:23:31 +00003186{
Vlad Yasevich53d64712014-03-27 17:26:18 -04003187 int tmp;
Simon Horman3b392dd2014-06-04 08:53:17 +09003188 __be16 type;
3189
3190 type = skb_network_protocol(skb, &tmp);
3191 features = net_mpls_features(skb, features, type);
Vlad Yasevich53d64712014-03-27 17:26:18 -04003192
Ed Cashinc0d680e2012-09-19 15:49:00 +00003193 if (skb->ip_summed != CHECKSUM_NONE &&
Simon Horman3b392dd2014-06-04 08:53:17 +09003194 !can_checksum_protocol(features, type)) {
Alexander Duyck996e8022016-05-02 09:25:10 -07003195 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
Jesse Grossf01a5232011-01-09 06:23:31 +00003196 }
Eric Dumazet7be2c822017-01-18 12:12:17 -08003197 if (illegal_highdma(skb->dev, skb))
3198 features &= ~NETIF_F_SG;
Jesse Grossf01a5232011-01-09 06:23:31 +00003199
3200 return features;
3201}
3202
Toshiaki Makitae38f3022015-03-27 14:31:13 +09003203netdev_features_t passthru_features_check(struct sk_buff *skb,
3204 struct net_device *dev,
3205 netdev_features_t features)
3206{
3207 return features;
3208}
3209EXPORT_SYMBOL(passthru_features_check);
3210
Toshiaki Makita7ce23672018-04-17 18:46:14 +09003211static netdev_features_t dflt_features_check(struct sk_buff *skb,
Toshiaki Makita8cb65d02015-03-27 14:31:12 +09003212 struct net_device *dev,
3213 netdev_features_t features)
3214{
3215 return vlan_features_check(skb, features);
3216}
3217
Alexander Duyckcbc53e02016-04-10 21:44:51 -04003218static netdev_features_t gso_features_check(const struct sk_buff *skb,
3219 struct net_device *dev,
3220 netdev_features_t features)
3221{
3222 u16 gso_segs = skb_shinfo(skb)->gso_segs;
3223
3224 if (gso_segs > dev->gso_max_segs)
3225 return features & ~NETIF_F_GSO_MASK;
3226
Alexander Duyck802ab552016-04-10 21:45:03 -04003227 /* Support for GSO partial features requires software
3228 * intervention before we can actually process the packets
3229 * so we need to strip support for any partial features now
3230 * and we can pull them back in after we have partially
3231 * segmented the frame.
3232 */
3233 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL))
3234 features &= ~dev->gso_partial_features;
3235
3236 /* Make sure to clear the IPv4 ID mangling feature if the
3237 * IPv4 header has the potential to be fragmented.
Alexander Duyckcbc53e02016-04-10 21:44:51 -04003238 */
3239 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
3240 struct iphdr *iph = skb->encapsulation ?
3241 inner_ip_hdr(skb) : ip_hdr(skb);
3242
3243 if (!(iph->frag_off & htons(IP_DF)))
3244 features &= ~NETIF_F_TSO_MANGLEID;
3245 }
3246
3247 return features;
3248}
3249
Florian Westphalc1e756b2014-05-05 15:00:44 +02003250netdev_features_t netif_skb_features(struct sk_buff *skb)
Jesse Gross58e998c2010-10-29 12:14:55 +00003251{
Jesse Gross5f352272014-12-23 22:37:26 -08003252 struct net_device *dev = skb->dev;
Eric Dumazetfcbeb972014-10-05 10:11:27 -07003253 netdev_features_t features = dev->features;
Jesse Gross58e998c2010-10-29 12:14:55 +00003254
Alexander Duyckcbc53e02016-04-10 21:44:51 -04003255 if (skb_is_gso(skb))
3256 features = gso_features_check(skb, dev, features);
Ben Hutchings30b678d2012-07-30 15:57:00 +00003257
Jesse Gross5f352272014-12-23 22:37:26 -08003258 /* If encapsulation offload request, verify we are testing
3259 * hardware encapsulation features instead of standard
3260 * features for the netdev
3261 */
3262 if (skb->encapsulation)
3263 features &= dev->hw_enc_features;
3264
Toshiaki Makitaf5a7fb82015-03-27 14:31:11 +09003265 if (skb_vlan_tagged(skb))
3266 features = netdev_intersect_features(features,
3267 dev->vlan_features |
3268 NETIF_F_HW_VLAN_CTAG_TX |
3269 NETIF_F_HW_VLAN_STAG_TX);
Jesse Gross58e998c2010-10-29 12:14:55 +00003270
Jesse Gross5f352272014-12-23 22:37:26 -08003271 if (dev->netdev_ops->ndo_features_check)
3272 features &= dev->netdev_ops->ndo_features_check(skb, dev,
3273 features);
Toshiaki Makita8cb65d02015-03-27 14:31:12 +09003274 else
3275 features &= dflt_features_check(skb, dev, features);
Jesse Gross5f352272014-12-23 22:37:26 -08003276
Florian Westphalc1e756b2014-05-05 15:00:44 +02003277 return harmonize_features(skb, features);
Jesse Gross58e998c2010-10-29 12:14:55 +00003278}
Florian Westphalc1e756b2014-05-05 15:00:44 +02003279EXPORT_SYMBOL(netif_skb_features);
Jesse Gross58e998c2010-10-29 12:14:55 +00003280
David S. Miller2ea25512014-08-29 21:10:01 -07003281static int xmit_one(struct sk_buff *skb, struct net_device *dev,
David S. Miller95f6b3d2014-08-29 21:57:30 -07003282 struct netdev_queue *txq, bool more)
Herbert Xuf6a78bf2006-06-22 02:57:17 -07003283{
David S. Miller2ea25512014-08-29 21:10:01 -07003284 unsigned int len;
3285 int rc;
Stephen Hemminger00829822008-11-20 20:14:53 -08003286
Maciej W. Rozycki9f9a7422018-10-09 23:57:49 +01003287 if (dev_nit_active(dev))
David S. Miller2ea25512014-08-29 21:10:01 -07003288 dev_queue_xmit_nit(skb, dev);
Jesse Grossfc741212011-01-09 06:23:32 +00003289
David S. Miller2ea25512014-08-29 21:10:01 -07003290 len = skb->len;
3291 trace_net_dev_start_xmit(skb, dev);
David S. Miller95f6b3d2014-08-29 21:57:30 -07003292 rc = netdev_start_xmit(skb, dev, txq, more);
David S. Miller2ea25512014-08-29 21:10:01 -07003293 trace_net_dev_xmit(skb, rc, dev, len);
Eric Dumazetadf30902009-06-02 05:19:30 +00003294
Patrick McHardy572a9d72009-11-10 06:14:14 +00003295 return rc;
Herbert Xuf6a78bf2006-06-22 02:57:17 -07003296}
David S. Miller2ea25512014-08-29 21:10:01 -07003297
David S. Miller8dcda222014-09-01 15:06:40 -07003298struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
3299 struct netdev_queue *txq, int *ret)
David S. Miller7f2e8702014-08-29 21:19:14 -07003300{
3301 struct sk_buff *skb = first;
3302 int rc = NETDEV_TX_OK;
3303
3304 while (skb) {
3305 struct sk_buff *next = skb->next;
3306
David S. Millera8305bf2018-07-29 20:42:53 -07003307 skb_mark_not_on_list(skb);
David S. Miller95f6b3d2014-08-29 21:57:30 -07003308 rc = xmit_one(skb, dev, txq, next != NULL);
David S. Miller7f2e8702014-08-29 21:19:14 -07003309 if (unlikely(!dev_xmit_complete(rc))) {
3310 skb->next = next;
3311 goto out;
3312 }
3313
3314 skb = next;
Eric Dumazetfe60faa2018-10-31 08:39:13 -07003315 if (netif_tx_queue_stopped(txq) && skb) {
David S. Miller7f2e8702014-08-29 21:19:14 -07003316 rc = NETDEV_TX_BUSY;
3317 break;
3318 }
3319 }
3320
3321out:
3322 *ret = rc;
3323 return skb;
3324}
3325
Eric Dumazet1ff0dc92014-10-06 11:26:27 -07003326static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
3327 netdev_features_t features)
David S. Millereae3f882014-08-30 15:17:13 -07003328{
Jiri Pirkodf8a39de2015-01-13 17:13:44 +01003329 if (skb_vlan_tag_present(skb) &&
Jiri Pirko59682502014-11-19 14:04:59 +01003330 !vlan_hw_offload_capable(features, skb->vlan_proto))
3331 skb = __vlan_hwaccel_push_inside(skb);
David S. Millereae3f882014-08-30 15:17:13 -07003332 return skb;
3333}
3334
Davide Caratti43c26a12017-05-18 15:44:41 +02003335int skb_csum_hwoffload_help(struct sk_buff *skb,
3336 const netdev_features_t features)
3337{
3338 if (unlikely(skb->csum_not_inet))
3339 return !!(features & NETIF_F_SCTP_CRC) ? 0 :
3340 skb_crc32c_csum_help(skb);
3341
3342 return !!(features & NETIF_F_CSUM_MASK) ? 0 : skb_checksum_help(skb);
3343}
3344EXPORT_SYMBOL(skb_csum_hwoffload_help);
3345
Steffen Klassertf53c7232017-12-20 10:41:36 +01003346static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev, bool *again)
David S. Millereae3f882014-08-30 15:17:13 -07003347{
3348 netdev_features_t features;
3349
David S. Millereae3f882014-08-30 15:17:13 -07003350 features = netif_skb_features(skb);
3351 skb = validate_xmit_vlan(skb, features);
3352 if (unlikely(!skb))
3353 goto out_null;
3354
Ilya Lesokhinebf4e802018-04-30 10:16:12 +03003355 skb = sk_validate_xmit_skb(skb, dev);
3356 if (unlikely(!skb))
3357 goto out_null;
3358
Johannes Berg8b86a612015-04-17 15:45:04 +02003359 if (netif_needs_gso(skb, features)) {
David S. Millerce937182014-08-30 19:22:20 -07003360 struct sk_buff *segs;
3361
3362 segs = skb_gso_segment(skb, features);
Jason Wangcecda692014-09-19 16:04:38 +08003363 if (IS_ERR(segs)) {
Jason Wangaf6dabc2014-12-19 11:09:13 +08003364 goto out_kfree_skb;
Jason Wangcecda692014-09-19 16:04:38 +08003365 } else if (segs) {
3366 consume_skb(skb);
3367 skb = segs;
3368 }
David S. Millereae3f882014-08-30 15:17:13 -07003369 } else {
3370 if (skb_needs_linearize(skb, features) &&
3371 __skb_linearize(skb))
3372 goto out_kfree_skb;
3373
3374 /* If packet is not checksummed and device does not
3375 * support checksumming for this protocol, complete
3376 * checksumming here.
3377 */
3378 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3379 if (skb->encapsulation)
3380 skb_set_inner_transport_header(skb,
3381 skb_checksum_start_offset(skb));
3382 else
3383 skb_set_transport_header(skb,
3384 skb_checksum_start_offset(skb));
Davide Caratti43c26a12017-05-18 15:44:41 +02003385 if (skb_csum_hwoffload_help(skb, features))
David S. Millereae3f882014-08-30 15:17:13 -07003386 goto out_kfree_skb;
3387 }
3388 }
3389
Steffen Klassertf53c7232017-12-20 10:41:36 +01003390 skb = validate_xmit_xfrm(skb, features, again);
Steffen Klassert3dca3f32017-12-20 10:41:31 +01003391
David S. Millereae3f882014-08-30 15:17:13 -07003392 return skb;
3393
3394out_kfree_skb:
3395 kfree_skb(skb);
3396out_null:
Eric Dumazetd21fd632016-04-12 21:50:07 -07003397 atomic_long_inc(&dev->tx_dropped);
David S. Millereae3f882014-08-30 15:17:13 -07003398 return NULL;
3399}
3400
Steffen Klassertf53c7232017-12-20 10:41:36 +01003401struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again)
Eric Dumazet55a93b32014-10-03 15:31:07 -07003402{
3403 struct sk_buff *next, *head = NULL, *tail;
3404
Eric Dumazetbec3cfd2014-10-03 20:59:19 -07003405 for (; skb != NULL; skb = next) {
Eric Dumazet55a93b32014-10-03 15:31:07 -07003406 next = skb->next;
David S. Millera8305bf2018-07-29 20:42:53 -07003407 skb_mark_not_on_list(skb);
Eric Dumazet55a93b32014-10-03 15:31:07 -07003408
Eric Dumazetbec3cfd2014-10-03 20:59:19 -07003409 /* in case skb wont be segmented, point to itself */
3410 skb->prev = skb;
3411
Steffen Klassertf53c7232017-12-20 10:41:36 +01003412 skb = validate_xmit_skb(skb, dev, again);
Eric Dumazetbec3cfd2014-10-03 20:59:19 -07003413 if (!skb)
3414 continue;
3415
3416 if (!head)
3417 head = skb;
3418 else
3419 tail->next = skb;
3420 /* If skb was segmented, skb->prev points to
3421 * the last segment. If not, it still contains skb.
3422 */
3423 tail = skb->prev;
Eric Dumazet55a93b32014-10-03 15:31:07 -07003424 }
3425 return head;
3426}
Willem de Bruijn104ba782016-10-26 11:23:07 -04003427EXPORT_SYMBOL_GPL(validate_xmit_skb_list);
Herbert Xuf6a78bf2006-06-22 02:57:17 -07003428
Eric Dumazet1def9232013-01-10 12:36:42 +00003429static void qdisc_pkt_len_init(struct sk_buff *skb)
3430{
3431 const struct skb_shared_info *shinfo = skb_shinfo(skb);
3432
3433 qdisc_skb_cb(skb)->pkt_len = skb->len;
3434
3435 /* To get more precise estimation of bytes sent on wire,
3436 * we add to pkt_len the headers size of all segments
3437 */
Maxim Mikityanskiya0dce872019-02-22 12:55:22 +00003438 if (shinfo->gso_size && skb_transport_header_was_set(skb)) {
Eric Dumazet757b8b12013-01-15 21:14:21 -08003439 unsigned int hdr_len;
Jason Wang15e5a032013-03-25 20:19:59 +00003440 u16 gso_segs = shinfo->gso_segs;
Eric Dumazet1def9232013-01-10 12:36:42 +00003441
Eric Dumazet757b8b12013-01-15 21:14:21 -08003442 /* mac layer + network layer */
3443 hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
3444
3445 /* + transport layer */
Eric Dumazet7c68d1a2018-01-18 19:59:19 -08003446 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) {
3447 const struct tcphdr *th;
3448 struct tcphdr _tcphdr;
3449
3450 th = skb_header_pointer(skb, skb_transport_offset(skb),
3451 sizeof(_tcphdr), &_tcphdr);
3452 if (likely(th))
3453 hdr_len += __tcp_hdrlen(th);
3454 } else {
3455 struct udphdr _udphdr;
3456
3457 if (skb_header_pointer(skb, skb_transport_offset(skb),
3458 sizeof(_udphdr), &_udphdr))
3459 hdr_len += sizeof(struct udphdr);
3460 }
Jason Wang15e5a032013-03-25 20:19:59 +00003461
3462 if (shinfo->gso_type & SKB_GSO_DODGY)
3463 gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
3464 shinfo->gso_size);
3465
3466 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
Eric Dumazet1def9232013-01-10 12:36:42 +00003467 }
3468}
3469
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003470static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
3471 struct net_device *dev,
3472 struct netdev_queue *txq)
3473{
3474 spinlock_t *root_lock = qdisc_lock(q);
Eric Dumazet520ac302016-06-21 23:16:49 -07003475 struct sk_buff *to_free = NULL;
Eric Dumazeta2da5702011-01-20 03:48:19 +00003476 bool contended;
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003477 int rc;
3478
Eric Dumazeta2da5702011-01-20 03:48:19 +00003479 qdisc_calculate_pkt_len(skb, q);
John Fastabend6b3ba912017-12-07 09:54:25 -08003480
3481 if (q->flags & TCQ_F_NOLOCK) {
3482 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
3483 __qdisc_drop(skb, &to_free);
3484 rc = NET_XMIT_DROP;
3485 } else {
3486 rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
Paolo Abeni32f7b442018-05-15 10:50:31 +02003487 qdisc_run(q);
John Fastabend6b3ba912017-12-07 09:54:25 -08003488 }
3489
3490 if (unlikely(to_free))
3491 kfree_skb_list(to_free);
3492 return rc;
3493 }
3494
Eric Dumazet79640a42010-06-02 05:09:29 -07003495 /*
3496 * Heuristic to force contended enqueues to serialize on a
3497 * separate lock before trying to get qdisc main lock.
Eric Dumazetf9eb8ae2016-06-06 09:37:15 -07003498 * This permits qdisc->running owner to get the lock more
Ying Xue9bf2b8c2014-06-26 15:56:31 +08003499 * often and dequeue packets faster.
Eric Dumazet79640a42010-06-02 05:09:29 -07003500 */
Eric Dumazeta2da5702011-01-20 03:48:19 +00003501 contended = qdisc_is_running(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07003502 if (unlikely(contended))
3503 spin_lock(&q->busylock);
3504
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003505 spin_lock(root_lock);
3506 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
Eric Dumazet520ac302016-06-21 23:16:49 -07003507 __qdisc_drop(skb, &to_free);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003508 rc = NET_XMIT_DROP;
3509 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
Eric Dumazetbc135b22010-06-02 03:23:51 -07003510 qdisc_run_begin(q)) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003511 /*
3512 * This is a work-conserving queue; there are no old skbs
3513 * waiting to be sent out; and the qdisc is not running -
3514 * xmit the skb directly.
3515 */
Eric Dumazetbfe0d022011-01-09 08:30:54 +00003516
Eric Dumazetbfe0d022011-01-09 08:30:54 +00003517 qdisc_bstats_update(q, skb);
3518
Eric Dumazet55a93b32014-10-03 15:31:07 -07003519 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
Eric Dumazet79640a42010-06-02 05:09:29 -07003520 if (unlikely(contended)) {
3521 spin_unlock(&q->busylock);
3522 contended = false;
3523 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003524 __qdisc_run(q);
John Fastabend6c148182017-12-07 09:54:06 -08003525 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003526
John Fastabend6c148182017-12-07 09:54:06 -08003527 qdisc_run_end(q);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003528 rc = NET_XMIT_SUCCESS;
3529 } else {
Eric Dumazet520ac302016-06-21 23:16:49 -07003530 rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
Eric Dumazet79640a42010-06-02 05:09:29 -07003531 if (qdisc_run_begin(q)) {
3532 if (unlikely(contended)) {
3533 spin_unlock(&q->busylock);
3534 contended = false;
3535 }
3536 __qdisc_run(q);
John Fastabend6c148182017-12-07 09:54:06 -08003537 qdisc_run_end(q);
Eric Dumazet79640a42010-06-02 05:09:29 -07003538 }
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003539 }
3540 spin_unlock(root_lock);
Eric Dumazet520ac302016-06-21 23:16:49 -07003541 if (unlikely(to_free))
3542 kfree_skb_list(to_free);
Eric Dumazet79640a42010-06-02 05:09:29 -07003543 if (unlikely(contended))
3544 spin_unlock(&q->busylock);
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003545 return rc;
3546}
3547
Daniel Borkmann86f85152013-12-29 17:27:11 +01003548#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
Neil Horman5bc14212011-11-22 05:10:51 +00003549static void skb_update_prio(struct sk_buff *skb)
3550{
Eric Dumazet4dcb31d2018-03-14 09:04:16 -07003551 const struct netprio_map *map;
3552 const struct sock *sk;
3553 unsigned int prioidx;
Neil Horman5bc14212011-11-22 05:10:51 +00003554
Eric Dumazet4dcb31d2018-03-14 09:04:16 -07003555 if (skb->priority)
3556 return;
3557 map = rcu_dereference_bh(skb->dev->priomap);
3558 if (!map)
3559 return;
3560 sk = skb_to_full_sk(skb);
3561 if (!sk)
3562 return;
Eric Dumazet91c68ce2012-07-08 21:45:10 +00003563
Eric Dumazet4dcb31d2018-03-14 09:04:16 -07003564 prioidx = sock_cgroup_prioidx(&sk->sk_cgrp_data);
3565
3566 if (prioidx < map->priomap_len)
3567 skb->priority = map->priomap[prioidx];
Neil Horman5bc14212011-11-22 05:10:51 +00003568}
3569#else
3570#define skb_update_prio(skb)
3571#endif
3572
hannes@stressinduktion.orgf60e5992015-04-01 17:07:44 +02003573DEFINE_PER_CPU(int, xmit_recursion);
3574EXPORT_SYMBOL(xmit_recursion);
3575
Dave Jonesd29f7492008-07-22 14:09:06 -07003576/**
Michel Machado95603e22012-06-12 10:16:35 +00003577 * dev_loopback_xmit - loop back @skb
Eric W. Biederman0c4b51f2015-09-15 20:04:18 -05003578 * @net: network namespace this loopback is happening in
3579 * @sk: sk needed to be a netfilter okfn
Michel Machado95603e22012-06-12 10:16:35 +00003580 * @skb: buffer to transmit
3581 */
Eric W. Biederman0c4b51f2015-09-15 20:04:18 -05003582int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
Michel Machado95603e22012-06-12 10:16:35 +00003583{
3584 skb_reset_mac_header(skb);
3585 __skb_pull(skb, skb_network_offset(skb));
3586 skb->pkt_type = PACKET_LOOPBACK;
3587 skb->ip_summed = CHECKSUM_UNNECESSARY;
3588 WARN_ON(!skb_dst(skb));
3589 skb_dst_force(skb);
3590 netif_rx_ni(skb);
3591 return 0;
3592}
3593EXPORT_SYMBOL(dev_loopback_xmit);
3594
Daniel Borkmann1f211a12016-01-07 22:29:47 +01003595#ifdef CONFIG_NET_EGRESS
3596static struct sk_buff *
3597sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
3598{
Jiri Pirko46209402017-11-03 11:46:25 +01003599 struct mini_Qdisc *miniq = rcu_dereference_bh(dev->miniq_egress);
Daniel Borkmann1f211a12016-01-07 22:29:47 +01003600 struct tcf_result cl_res;
3601
Jiri Pirko46209402017-11-03 11:46:25 +01003602 if (!miniq)
Daniel Borkmann1f211a12016-01-07 22:29:47 +01003603 return skb;
3604
Willem de Bruijn8dc07fd2017-01-07 17:06:37 -05003605 /* qdisc_skb_cb(skb)->pkt_len was already set by the caller. */
Jiri Pirko46209402017-11-03 11:46:25 +01003606 mini_qdisc_bstats_cpu_update(miniq, skb);
Daniel Borkmann1f211a12016-01-07 22:29:47 +01003607
Jiri Pirko46209402017-11-03 11:46:25 +01003608 switch (tcf_classify(skb, miniq->filter_list, &cl_res, false)) {
Daniel Borkmann1f211a12016-01-07 22:29:47 +01003609 case TC_ACT_OK:
3610 case TC_ACT_RECLASSIFY:
3611 skb->tc_index = TC_H_MIN(cl_res.classid);
3612 break;
3613 case TC_ACT_SHOT:
Jiri Pirko46209402017-11-03 11:46:25 +01003614 mini_qdisc_qstats_cpu_drop(miniq);
Daniel Borkmann1f211a12016-01-07 22:29:47 +01003615 *ret = NET_XMIT_DROP;
Daniel Borkmann7e2c3ae2016-05-15 23:28:29 +02003616 kfree_skb(skb);
3617 return NULL;
Daniel Borkmann1f211a12016-01-07 22:29:47 +01003618 case TC_ACT_STOLEN:
3619 case TC_ACT_QUEUED:
Jiri Pirkoe25ea212017-06-06 14:12:02 +02003620 case TC_ACT_TRAP:
Daniel Borkmann1f211a12016-01-07 22:29:47 +01003621 *ret = NET_XMIT_SUCCESS;
Daniel Borkmann7e2c3ae2016-05-15 23:28:29 +02003622 consume_skb(skb);
Daniel Borkmann1f211a12016-01-07 22:29:47 +01003623 return NULL;
3624 case TC_ACT_REDIRECT:
3625 /* No need to push/pop skb's mac_header here on egress! */
3626 skb_do_redirect(skb);
3627 *ret = NET_XMIT_SUCCESS;
3628 return NULL;
3629 default:
3630 break;
3631 }
3632
3633 return skb;
3634}
3635#endif /* CONFIG_NET_EGRESS */
3636
Amritha Nambiarfc9bab22018-06-29 21:27:02 -07003637#ifdef CONFIG_XPS
3638static int __get_xps_queue_idx(struct net_device *dev, struct sk_buff *skb,
3639 struct xps_dev_maps *dev_maps, unsigned int tci)
3640{
3641 struct xps_map *map;
3642 int queue_index = -1;
3643
3644 if (dev->num_tc) {
3645 tci *= dev->num_tc;
3646 tci += netdev_get_prio_tc_map(dev, skb->priority);
3647 }
3648
3649 map = rcu_dereference(dev_maps->attr_map[tci]);
3650 if (map) {
3651 if (map->len == 1)
3652 queue_index = map->queues[0];
3653 else
3654 queue_index = map->queues[reciprocal_scale(
3655 skb_get_hash(skb), map->len)];
3656 if (unlikely(queue_index >= dev->real_num_tx_queues))
3657 queue_index = -1;
3658 }
3659 return queue_index;
3660}
3661#endif
3662
Alexander Duyckeadec8772018-07-09 12:19:48 -04003663static int get_xps_queue(struct net_device *dev, struct net_device *sb_dev,
3664 struct sk_buff *skb)
Jiri Pirko638b2a62015-05-12 14:56:13 +02003665{
3666#ifdef CONFIG_XPS
3667 struct xps_dev_maps *dev_maps;
Amritha Nambiarfc9bab22018-06-29 21:27:02 -07003668 struct sock *sk = skb->sk;
Jiri Pirko638b2a62015-05-12 14:56:13 +02003669 int queue_index = -1;
3670
Amritha Nambiar04157462018-06-29 21:26:46 -07003671 if (!static_key_false(&xps_needed))
3672 return -1;
3673
Jiri Pirko638b2a62015-05-12 14:56:13 +02003674 rcu_read_lock();
Amritha Nambiarfc9bab22018-06-29 21:27:02 -07003675 if (!static_key_false(&xps_rxqs_needed))
3676 goto get_cpus_map;
3677
Alexander Duyckeadec8772018-07-09 12:19:48 -04003678 dev_maps = rcu_dereference(sb_dev->xps_rxqs_map);
Jiri Pirko638b2a62015-05-12 14:56:13 +02003679 if (dev_maps) {
Amritha Nambiarfc9bab22018-06-29 21:27:02 -07003680 int tci = sk_rx_queue_get(sk);
Alexander Duyck184c4492016-10-28 11:50:13 -04003681
Amritha Nambiarfc9bab22018-06-29 21:27:02 -07003682 if (tci >= 0 && tci < dev->num_rx_queues)
3683 queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
3684 tci);
3685 }
Alexander Duyck184c4492016-10-28 11:50:13 -04003686
Amritha Nambiarfc9bab22018-06-29 21:27:02 -07003687get_cpus_map:
3688 if (queue_index < 0) {
Alexander Duyckeadec8772018-07-09 12:19:48 -04003689 dev_maps = rcu_dereference(sb_dev->xps_cpus_map);
Amritha Nambiarfc9bab22018-06-29 21:27:02 -07003690 if (dev_maps) {
3691 unsigned int tci = skb->sender_cpu - 1;
3692
3693 queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
3694 tci);
Jiri Pirko638b2a62015-05-12 14:56:13 +02003695 }
3696 }
3697 rcu_read_unlock();
3698
3699 return queue_index;
3700#else
3701 return -1;
3702#endif
3703}
3704
Alexander Duycka4ea8a32018-07-09 12:19:54 -04003705u16 dev_pick_tx_zero(struct net_device *dev, struct sk_buff *skb,
Alexander Duyck4f49dec2018-07-09 12:19:59 -04003706 struct net_device *sb_dev,
3707 select_queue_fallback_t fallback)
Alexander Duycka4ea8a32018-07-09 12:19:54 -04003708{
3709 return 0;
3710}
3711EXPORT_SYMBOL(dev_pick_tx_zero);
3712
3713u16 dev_pick_tx_cpu_id(struct net_device *dev, struct sk_buff *skb,
Alexander Duyck4f49dec2018-07-09 12:19:59 -04003714 struct net_device *sb_dev,
3715 select_queue_fallback_t fallback)
Alexander Duycka4ea8a32018-07-09 12:19:54 -04003716{
3717 return (u16)raw_smp_processor_id() % dev->real_num_tx_queues;
3718}
3719EXPORT_SYMBOL(dev_pick_tx_cpu_id);
3720
Alexander Duyck8ec56fc2018-07-09 12:20:04 -04003721static u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb,
3722 struct net_device *sb_dev)
Jiri Pirko638b2a62015-05-12 14:56:13 +02003723{
3724 struct sock *sk = skb->sk;
3725 int queue_index = sk_tx_queue_get(sk);
3726
Alexander Duyckeadec8772018-07-09 12:19:48 -04003727 sb_dev = sb_dev ? : dev;
3728
Jiri Pirko638b2a62015-05-12 14:56:13 +02003729 if (queue_index < 0 || skb->ooo_okay ||
3730 queue_index >= dev->real_num_tx_queues) {
Alexander Duyckeadec8772018-07-09 12:19:48 -04003731 int new_index = get_xps_queue(dev, sb_dev, skb);
tchardingf4563a72017-02-09 17:56:07 +11003732
Jiri Pirko638b2a62015-05-12 14:56:13 +02003733 if (new_index < 0)
Alexander Duyckeadec8772018-07-09 12:19:48 -04003734 new_index = skb_tx_hash(dev, sb_dev, skb);
Jiri Pirko638b2a62015-05-12 14:56:13 +02003735
3736 if (queue_index != new_index && sk &&
Eric Dumazet004a5d02015-10-04 21:08:10 -07003737 sk_fullsock(sk) &&
Jiri Pirko638b2a62015-05-12 14:56:13 +02003738 rcu_access_pointer(sk->sk_dst_cache))
3739 sk_tx_queue_set(sk, new_index);
3740
3741 queue_index = new_index;
3742 }
3743
3744 return queue_index;
3745}
3746
3747struct netdev_queue *netdev_pick_tx(struct net_device *dev,
3748 struct sk_buff *skb,
Alexander Duyckeadec8772018-07-09 12:19:48 -04003749 struct net_device *sb_dev)
Jiri Pirko638b2a62015-05-12 14:56:13 +02003750{
3751 int queue_index = 0;
3752
3753#ifdef CONFIG_XPS
Eric Dumazet52bd2d62015-11-18 06:30:50 -08003754 u32 sender_cpu = skb->sender_cpu - 1;
3755
3756 if (sender_cpu >= (u32)NR_CPUS)
Jiri Pirko638b2a62015-05-12 14:56:13 +02003757 skb->sender_cpu = raw_smp_processor_id() + 1;
3758#endif
3759
3760 if (dev->real_num_tx_queues != 1) {
3761 const struct net_device_ops *ops = dev->netdev_ops;
tchardingf4563a72017-02-09 17:56:07 +11003762
Jiri Pirko638b2a62015-05-12 14:56:13 +02003763 if (ops->ndo_select_queue)
Alexander Duyckeadec8772018-07-09 12:19:48 -04003764 queue_index = ops->ndo_select_queue(dev, skb, sb_dev,
Jiri Pirko638b2a62015-05-12 14:56:13 +02003765 __netdev_pick_tx);
3766 else
Alexander Duyck8ec56fc2018-07-09 12:20:04 -04003767 queue_index = __netdev_pick_tx(dev, skb, sb_dev);
Jiri Pirko638b2a62015-05-12 14:56:13 +02003768
Alexander Duyckd5845272017-11-22 10:57:41 -08003769 queue_index = netdev_cap_txqueue(dev, queue_index);
Jiri Pirko638b2a62015-05-12 14:56:13 +02003770 }
3771
3772 skb_set_queue_mapping(skb, queue_index);
3773 return netdev_get_tx_queue(dev, queue_index);
3774}
3775
Michel Machado95603e22012-06-12 10:16:35 +00003776/**
Jason Wang9d08dd32014-01-20 11:25:13 +08003777 * __dev_queue_xmit - transmit a buffer
Dave Jonesd29f7492008-07-22 14:09:06 -07003778 * @skb: buffer to transmit
Alexander Duyckeadec8772018-07-09 12:19:48 -04003779 * @sb_dev: suboordinate device used for L2 forwarding offload
Dave Jonesd29f7492008-07-22 14:09:06 -07003780 *
3781 * Queue a buffer for transmission to a network device. The caller must
3782 * have set the device and priority and built the buffer before calling
3783 * this function. The function can be called from an interrupt.
3784 *
3785 * A negative errno code is returned on a failure. A success does not
3786 * guarantee the frame will be transmitted as it may be dropped due
3787 * to congestion or traffic shaping.
3788 *
3789 * -----------------------------------------------------------------------------------
3790 * I notice this method can also return errors from the queue disciplines,
3791 * including NET_XMIT_DROP, which is a positive value. So, errors can also
3792 * be positive.
3793 *
3794 * Regardless of the return value, the skb is consumed, so it is currently
3795 * difficult to retry a send to this method. (You can bump the ref count
3796 * before sending to hold a reference for retry if you are careful.)
3797 *
3798 * When calling this method, interrupts MUST be enabled. This is because
3799 * the BH enable code must have IRQs enabled so that it will not deadlock.
3800 * --BLG
3801 */
Alexander Duyckeadec8772018-07-09 12:19:48 -04003802static int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003803{
3804 struct net_device *dev = skb->dev;
David S. Millerdc2b4842008-07-08 17:18:23 -07003805 struct netdev_queue *txq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003806 struct Qdisc *q;
3807 int rc = -ENOMEM;
Steffen Klassertf53c7232017-12-20 10:41:36 +01003808 bool again = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003809
Eric Dumazet6d1ccff2013-02-05 20:22:20 +00003810 skb_reset_mac_header(skb);
3811
Willem de Bruijne7fd2882014-08-04 22:11:48 -04003812 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
3813 __skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED);
3814
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003815 /* Disable soft irqs for various locks below. Also
3816 * stops preemption for RCU.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003817 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09003818 rcu_read_lock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003819
Neil Horman5bc14212011-11-22 05:10:51 +00003820 skb_update_prio(skb);
3821
Daniel Borkmann1f211a12016-01-07 22:29:47 +01003822 qdisc_pkt_len_init(skb);
3823#ifdef CONFIG_NET_CLS_ACT
Willem de Bruijn8dc07fd2017-01-07 17:06:37 -05003824 skb->tc_at_ingress = 0;
Daniel Borkmann1f211a12016-01-07 22:29:47 +01003825# ifdef CONFIG_NET_EGRESS
Davidlohr Buesoaabf6772018-05-08 09:07:00 -07003826 if (static_branch_unlikely(&egress_needed_key)) {
Daniel Borkmann1f211a12016-01-07 22:29:47 +01003827 skb = sch_handle_egress(skb, &rc, dev);
3828 if (!skb)
3829 goto out;
3830 }
3831# endif
3832#endif
Eric Dumazet02875872014-10-05 18:38:35 -07003833 /* If device/qdisc don't need skb->dst, release it right now while
3834 * its hot in this cpu cache.
3835 */
3836 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
3837 skb_dst_drop(skb);
3838 else
3839 skb_dst_force(skb);
3840
Alexander Duyckeadec8772018-07-09 12:19:48 -04003841 txq = netdev_pick_tx(dev, skb, sb_dev);
Paul E. McKenneya898def2010-02-22 17:04:49 -08003842 q = rcu_dereference_bh(txq->qdisc);
David S. Miller37437bb2008-07-16 02:15:04 -07003843
Koki Sanagicf66ba52010-08-23 18:45:02 +09003844 trace_net_dev_queue(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003845 if (q->enqueue) {
Krishna Kumarbbd8a0d2009-08-06 01:44:21 +00003846 rc = __dev_xmit_skb(skb, q, dev, txq);
David S. Miller37437bb2008-07-16 02:15:04 -07003847 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003848 }
3849
3850 /* The device has no queue. Common case for software devices:
tchardingeb13da12017-02-09 17:56:06 +11003851 * loopback, all the sorts of tunnels...
Linus Torvalds1da177e2005-04-16 15:20:36 -07003852
tchardingeb13da12017-02-09 17:56:06 +11003853 * Really, it is unlikely that netif_tx_lock protection is necessary
3854 * here. (f.e. loopback and IP tunnels are clean ignoring statistics
3855 * counters.)
3856 * However, it is possible, that they rely on protection
3857 * made by us here.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003858
tchardingeb13da12017-02-09 17:56:06 +11003859 * Check this and shot the lock. It is not prone from deadlocks.
3860 *Either shot noqueue qdisc, it is even simpler 8)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003861 */
3862 if (dev->flags & IFF_UP) {
3863 int cpu = smp_processor_id(); /* ok because BHs are off */
3864
David S. Millerc773e842008-07-08 23:13:53 -07003865 if (txq->xmit_lock_owner != cpu) {
Daniel Borkmanna70b5062016-06-10 21:19:06 +02003866 if (unlikely(__this_cpu_read(xmit_recursion) >
3867 XMIT_RECURSION_LIMIT))
Eric Dumazet745e20f2010-09-29 13:23:09 -07003868 goto recursion_alert;
3869
Steffen Klassertf53c7232017-12-20 10:41:36 +01003870 skb = validate_xmit_skb(skb, dev, &again);
Jesper Dangaard Brouer1f595332014-09-03 17:56:09 +02003871 if (!skb)
Eric Dumazetd21fd632016-04-12 21:50:07 -07003872 goto out;
Jesper Dangaard Brouer1f595332014-09-03 17:56:09 +02003873
David S. Millerc773e842008-07-08 23:13:53 -07003874 HARD_TX_LOCK(dev, txq, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003875
Tom Herbert734664982011-11-28 16:32:44 +00003876 if (!netif_xmit_stopped(txq)) {
Eric Dumazet745e20f2010-09-29 13:23:09 -07003877 __this_cpu_inc(xmit_recursion);
David S. Millerce937182014-08-30 19:22:20 -07003878 skb = dev_hard_start_xmit(skb, dev, txq, &rc);
Eric Dumazet745e20f2010-09-29 13:23:09 -07003879 __this_cpu_dec(xmit_recursion);
Patrick McHardy572a9d72009-11-10 06:14:14 +00003880 if (dev_xmit_complete(rc)) {
David S. Millerc773e842008-07-08 23:13:53 -07003881 HARD_TX_UNLOCK(dev, txq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003882 goto out;
3883 }
3884 }
David S. Millerc773e842008-07-08 23:13:53 -07003885 HARD_TX_UNLOCK(dev, txq);
Joe Perchese87cc472012-05-13 21:56:26 +00003886 net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
3887 dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003888 } else {
3889 /* Recursion is detected! It is possible,
Eric Dumazet745e20f2010-09-29 13:23:09 -07003890 * unfortunately
3891 */
3892recursion_alert:
Joe Perchese87cc472012-05-13 21:56:26 +00003893 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
3894 dev->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003895 }
3896 }
3897
3898 rc = -ENETDOWN;
Herbert Xud4828d82006-06-22 02:28:18 -07003899 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003900
Eric Dumazet015f0682014-03-27 08:45:56 -07003901 atomic_long_inc(&dev->tx_dropped);
Jesper Dangaard Brouer1f595332014-09-03 17:56:09 +02003902 kfree_skb_list(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003903 return rc;
3904out:
Herbert Xud4828d82006-06-22 02:28:18 -07003905 rcu_read_unlock_bh();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003906 return rc;
3907}
Jason Wangf663dd92014-01-10 16:18:26 +08003908
Eric W. Biederman2b4aa3c2015-09-15 20:04:07 -05003909int dev_queue_xmit(struct sk_buff *skb)
Jason Wangf663dd92014-01-10 16:18:26 +08003910{
3911 return __dev_queue_xmit(skb, NULL);
3912}
Eric W. Biederman2b4aa3c2015-09-15 20:04:07 -05003913EXPORT_SYMBOL(dev_queue_xmit);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003914
Alexander Duyckeadec8772018-07-09 12:19:48 -04003915int dev_queue_xmit_accel(struct sk_buff *skb, struct net_device *sb_dev)
Jason Wangf663dd92014-01-10 16:18:26 +08003916{
Alexander Duyckeadec8772018-07-09 12:19:48 -04003917 return __dev_queue_xmit(skb, sb_dev);
Jason Wangf663dd92014-01-10 16:18:26 +08003918}
3919EXPORT_SYMBOL(dev_queue_xmit_accel);
3920
Magnus Karlsson865b03f2018-05-02 13:01:33 +02003921int dev_direct_xmit(struct sk_buff *skb, u16 queue_id)
3922{
3923 struct net_device *dev = skb->dev;
3924 struct sk_buff *orig_skb = skb;
3925 struct netdev_queue *txq;
3926 int ret = NETDEV_TX_BUSY;
3927 bool again = false;
3928
3929 if (unlikely(!netif_running(dev) ||
3930 !netif_carrier_ok(dev)))
3931 goto drop;
3932
3933 skb = validate_xmit_skb_list(skb, dev, &again);
3934 if (skb != orig_skb)
3935 goto drop;
3936
3937 skb_set_queue_mapping(skb, queue_id);
3938 txq = skb_get_tx_queue(dev, skb);
3939
3940 local_bh_disable();
3941
3942 HARD_TX_LOCK(dev, txq, smp_processor_id());
3943 if (!netif_xmit_frozen_or_drv_stopped(txq))
3944 ret = netdev_start_xmit(skb, dev, txq, false);
3945 HARD_TX_UNLOCK(dev, txq);
3946
3947 local_bh_enable();
3948
3949 if (!dev_xmit_complete(ret))
3950 kfree_skb(skb);
3951
3952 return ret;
3953drop:
3954 atomic_long_inc(&dev->tx_dropped);
3955 kfree_skb_list(skb);
3956 return NET_XMIT_DROP;
3957}
3958EXPORT_SYMBOL(dev_direct_xmit);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003959
tchardingeb13da12017-02-09 17:56:06 +11003960/*************************************************************************
3961 * Receiver routines
3962 *************************************************************************/
Linus Torvalds1da177e2005-04-16 15:20:36 -07003963
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07003964int netdev_max_backlog __read_mostly = 1000;
Eric Dumazetc9e6bc62012-09-27 19:29:05 +00003965EXPORT_SYMBOL(netdev_max_backlog);
3966
Eric Dumazet3b098e22010-05-15 23:57:10 -07003967int netdev_tstamp_prequeue __read_mostly = 1;
Stephen Hemminger6b2bedc2007-03-12 14:33:50 -07003968int netdev_budget __read_mostly = 300;
Matthew Whitehead7acf8a12017-04-19 12:37:10 -04003969unsigned int __read_mostly netdev_budget_usecs = 2000;
Matthias Tafelmeier3d48b532016-12-29 21:37:21 +01003970int weight_p __read_mostly = 64; /* old backlog weight */
3971int dev_weight_rx_bias __read_mostly = 1; /* bias for backlog weight */
3972int dev_weight_tx_bias __read_mostly = 1; /* bias for output_queue quota */
3973int dev_rx_weight __read_mostly = 64;
3974int dev_tx_weight __read_mostly = 64;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003975
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07003976/* Called with irq disabled */
3977static inline void ____napi_schedule(struct softnet_data *sd,
3978 struct napi_struct *napi)
3979{
3980 list_add_tail(&napi->poll_list, &sd->poll_list);
3981 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3982}
3983
Eric Dumazetdf334542010-03-24 19:13:54 +00003984#ifdef CONFIG_RPS
Tom Herbertfec5e652010-04-16 16:01:27 -07003985
3986/* One global table that all flow-based protocols share. */
Eric Dumazet6e3f7fa2010-10-25 03:02:02 +00003987struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
Tom Herbertfec5e652010-04-16 16:01:27 -07003988EXPORT_SYMBOL(rps_sock_flow_table);
Eric Dumazet567e4b72015-02-06 12:59:01 -08003989u32 rps_cpu_mask __read_mostly;
3990EXPORT_SYMBOL(rps_cpu_mask);
Tom Herbertfec5e652010-04-16 16:01:27 -07003991
Ingo Molnarc5905af2012-02-24 08:31:31 +01003992struct static_key rps_needed __read_mostly;
Jason Wang3df97ba2016-04-25 23:13:42 -04003993EXPORT_SYMBOL(rps_needed);
Eric Dumazet13bfff22016-12-07 08:29:10 -08003994struct static_key rfs_needed __read_mostly;
3995EXPORT_SYMBOL(rfs_needed);
Eric Dumazetadc93002011-11-17 03:13:26 +00003996
Ben Hutchingsc4454772011-01-19 11:03:53 +00003997static struct rps_dev_flow *
3998set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3999 struct rps_dev_flow *rflow, u16 next_cpu)
4000{
Eric Dumazeta31196b2015-04-25 09:35:24 -07004001 if (next_cpu < nr_cpu_ids) {
Ben Hutchingsc4454772011-01-19 11:03:53 +00004002#ifdef CONFIG_RFS_ACCEL
4003 struct netdev_rx_queue *rxqueue;
4004 struct rps_dev_flow_table *flow_table;
4005 struct rps_dev_flow *old_rflow;
4006 u32 flow_id;
4007 u16 rxq_index;
4008 int rc;
4009
4010 /* Should we steer this flow to a different hardware queue? */
Ben Hutchings69a19ee2011-02-15 20:32:04 +00004011 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
4012 !(dev->features & NETIF_F_NTUPLE))
Ben Hutchingsc4454772011-01-19 11:03:53 +00004013 goto out;
4014 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
4015 if (rxq_index == skb_get_rx_queue(skb))
4016 goto out;
4017
4018 rxqueue = dev->_rx + rxq_index;
4019 flow_table = rcu_dereference(rxqueue->rps_flow_table);
4020 if (!flow_table)
4021 goto out;
Tom Herbert61b905d2014-03-24 15:34:47 -07004022 flow_id = skb_get_hash(skb) & flow_table->mask;
Ben Hutchingsc4454772011-01-19 11:03:53 +00004023 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
4024 rxq_index, flow_id);
4025 if (rc < 0)
4026 goto out;
4027 old_rflow = rflow;
4028 rflow = &flow_table->flows[flow_id];
Ben Hutchingsc4454772011-01-19 11:03:53 +00004029 rflow->filter = rc;
4030 if (old_rflow->filter == rflow->filter)
4031 old_rflow->filter = RPS_NO_FILTER;
4032 out:
4033#endif
4034 rflow->last_qtail =
Ben Hutchings09994d12011-10-03 04:42:46 +00004035 per_cpu(softnet_data, next_cpu).input_queue_head;
Ben Hutchingsc4454772011-01-19 11:03:53 +00004036 }
4037
Ben Hutchings09994d12011-10-03 04:42:46 +00004038 rflow->cpu = next_cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00004039 return rflow;
4040}
4041
Tom Herbert0a9627f2010-03-16 08:03:29 +00004042/*
4043 * get_rps_cpu is called from netif_receive_skb and returns the target
4044 * CPU from the RPS map of the receiving queue for a given skb.
Eric Dumazetb0e28f12010-04-15 00:14:07 -07004045 * rcu_read_lock must be held on entry.
Tom Herbert0a9627f2010-03-16 08:03:29 +00004046 */
Tom Herbertfec5e652010-04-16 16:01:27 -07004047static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
4048 struct rps_dev_flow **rflowp)
Tom Herbert0a9627f2010-03-16 08:03:29 +00004049{
Eric Dumazet567e4b72015-02-06 12:59:01 -08004050 const struct rps_sock_flow_table *sock_flow_table;
4051 struct netdev_rx_queue *rxqueue = dev->_rx;
Tom Herbertfec5e652010-04-16 16:01:27 -07004052 struct rps_dev_flow_table *flow_table;
Eric Dumazet567e4b72015-02-06 12:59:01 -08004053 struct rps_map *map;
Tom Herbert0a9627f2010-03-16 08:03:29 +00004054 int cpu = -1;
Eric Dumazet567e4b72015-02-06 12:59:01 -08004055 u32 tcpu;
Tom Herbert61b905d2014-03-24 15:34:47 -07004056 u32 hash;
Tom Herbert0a9627f2010-03-16 08:03:29 +00004057
Tom Herbert0a9627f2010-03-16 08:03:29 +00004058 if (skb_rx_queue_recorded(skb)) {
4059 u16 index = skb_get_rx_queue(skb);
Eric Dumazet567e4b72015-02-06 12:59:01 -08004060
Ben Hutchings62fe0b42010-09-27 08:24:33 +00004061 if (unlikely(index >= dev->real_num_rx_queues)) {
4062 WARN_ONCE(dev->real_num_rx_queues > 1,
4063 "%s received packet on queue %u, but number "
4064 "of RX queues is %u\n",
4065 dev->name, index, dev->real_num_rx_queues);
Tom Herbert0a9627f2010-03-16 08:03:29 +00004066 goto done;
4067 }
Eric Dumazet567e4b72015-02-06 12:59:01 -08004068 rxqueue += index;
Tom Herbert0a9627f2010-03-16 08:03:29 +00004069 }
4070
Eric Dumazet567e4b72015-02-06 12:59:01 -08004071 /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
4072
4073 flow_table = rcu_dereference(rxqueue->rps_flow_table);
4074 map = rcu_dereference(rxqueue->rps_map);
4075 if (!flow_table && !map)
4076 goto done;
4077
Changli Gao2d47b452010-08-17 19:00:56 +00004078 skb_reset_network_header(skb);
Tom Herbert61b905d2014-03-24 15:34:47 -07004079 hash = skb_get_hash(skb);
4080 if (!hash)
Tom Herbert0a9627f2010-03-16 08:03:29 +00004081 goto done;
Tom Herbert0a9627f2010-03-16 08:03:29 +00004082
Tom Herbertfec5e652010-04-16 16:01:27 -07004083 sock_flow_table = rcu_dereference(rps_sock_flow_table);
4084 if (flow_table && sock_flow_table) {
Tom Herbertfec5e652010-04-16 16:01:27 -07004085 struct rps_dev_flow *rflow;
Eric Dumazet567e4b72015-02-06 12:59:01 -08004086 u32 next_cpu;
4087 u32 ident;
Tom Herbertfec5e652010-04-16 16:01:27 -07004088
Eric Dumazet567e4b72015-02-06 12:59:01 -08004089 /* First check into global flow table if there is a match */
4090 ident = sock_flow_table->ents[hash & sock_flow_table->mask];
4091 if ((ident ^ hash) & ~rps_cpu_mask)
4092 goto try_rps;
4093
4094 next_cpu = ident & rps_cpu_mask;
4095
4096 /* OK, now we know there is a match,
4097 * we can look at the local (per receive queue) flow table
4098 */
Tom Herbert61b905d2014-03-24 15:34:47 -07004099 rflow = &flow_table->flows[hash & flow_table->mask];
Tom Herbertfec5e652010-04-16 16:01:27 -07004100 tcpu = rflow->cpu;
4101
Tom Herbertfec5e652010-04-16 16:01:27 -07004102 /*
4103 * If the desired CPU (where last recvmsg was done) is
4104 * different from current CPU (one in the rx-queue flow
4105 * table entry), switch if one of the following holds:
Eric Dumazeta31196b2015-04-25 09:35:24 -07004106 * - Current CPU is unset (>= nr_cpu_ids).
Tom Herbertfec5e652010-04-16 16:01:27 -07004107 * - Current CPU is offline.
4108 * - The current CPU's queue tail has advanced beyond the
4109 * last packet that was enqueued using this table entry.
4110 * This guarantees that all previous packets for the flow
4111 * have been dequeued, thus preserving in order delivery.
4112 */
4113 if (unlikely(tcpu != next_cpu) &&
Eric Dumazeta31196b2015-04-25 09:35:24 -07004114 (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
Tom Herbertfec5e652010-04-16 16:01:27 -07004115 ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
Tom Herbertbaefa312012-11-16 09:04:15 +00004116 rflow->last_qtail)) >= 0)) {
4117 tcpu = next_cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00004118 rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
Tom Herbertbaefa312012-11-16 09:04:15 +00004119 }
Ben Hutchingsc4454772011-01-19 11:03:53 +00004120
Eric Dumazeta31196b2015-04-25 09:35:24 -07004121 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
Tom Herbertfec5e652010-04-16 16:01:27 -07004122 *rflowp = rflow;
4123 cpu = tcpu;
4124 goto done;
4125 }
4126 }
4127
Eric Dumazet567e4b72015-02-06 12:59:01 -08004128try_rps:
4129
Tom Herbert0a9627f2010-03-16 08:03:29 +00004130 if (map) {
Daniel Borkmann8fc54f62014-08-23 20:58:54 +02004131 tcpu = map->cpus[reciprocal_scale(hash, map->len)];
Tom Herbert0a9627f2010-03-16 08:03:29 +00004132 if (cpu_online(tcpu)) {
4133 cpu = tcpu;
4134 goto done;
4135 }
4136 }
4137
4138done:
Tom Herbert0a9627f2010-03-16 08:03:29 +00004139 return cpu;
4140}
4141
Ben Hutchingsc4454772011-01-19 11:03:53 +00004142#ifdef CONFIG_RFS_ACCEL
4143
4144/**
4145 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
4146 * @dev: Device on which the filter was set
4147 * @rxq_index: RX queue index
4148 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
4149 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
4150 *
4151 * Drivers that implement ndo_rx_flow_steer() should periodically call
4152 * this function for each installed filter and remove the filters for
4153 * which it returns %true.
4154 */
4155bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
4156 u32 flow_id, u16 filter_id)
4157{
4158 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
4159 struct rps_dev_flow_table *flow_table;
4160 struct rps_dev_flow *rflow;
4161 bool expire = true;
Eric Dumazeta31196b2015-04-25 09:35:24 -07004162 unsigned int cpu;
Ben Hutchingsc4454772011-01-19 11:03:53 +00004163
4164 rcu_read_lock();
4165 flow_table = rcu_dereference(rxqueue->rps_flow_table);
4166 if (flow_table && flow_id <= flow_table->mask) {
4167 rflow = &flow_table->flows[flow_id];
Mark Rutland6aa7de02017-10-23 14:07:29 -07004168 cpu = READ_ONCE(rflow->cpu);
Eric Dumazeta31196b2015-04-25 09:35:24 -07004169 if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
Ben Hutchingsc4454772011-01-19 11:03:53 +00004170 ((int)(per_cpu(softnet_data, cpu).input_queue_head -
4171 rflow->last_qtail) <
4172 (int)(10 * flow_table->mask)))
4173 expire = false;
4174 }
4175 rcu_read_unlock();
4176 return expire;
4177}
4178EXPORT_SYMBOL(rps_may_expire_flow);
4179
4180#endif /* CONFIG_RFS_ACCEL */
4181
Tom Herbert0a9627f2010-03-16 08:03:29 +00004182/* Called from hardirq (IPI) context */
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004183static void rps_trigger_softirq(void *data)
Tom Herbert0a9627f2010-03-16 08:03:29 +00004184{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004185 struct softnet_data *sd = data;
4186
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07004187 ____napi_schedule(sd, &sd->backlog);
Changli Gaodee42872010-05-02 05:42:16 +00004188 sd->received_rps++;
Tom Herbert0a9627f2010-03-16 08:03:29 +00004189}
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004190
Tom Herbertfec5e652010-04-16 16:01:27 -07004191#endif /* CONFIG_RPS */
Tom Herbert0a9627f2010-03-16 08:03:29 +00004192
4193/*
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004194 * Check if this softnet_data structure is another cpu one
4195 * If yes, queue it to our IPI list and return 1
4196 * If no, return 0
4197 */
4198static int rps_ipi_queued(struct softnet_data *sd)
4199{
4200#ifdef CONFIG_RPS
Christoph Lameter903ceff2014-08-17 12:30:35 -05004201 struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004202
4203 if (sd != mysd) {
4204 sd->rps_ipi_next = mysd->rps_ipi_list;
4205 mysd->rps_ipi_list = sd;
4206
4207 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4208 return 1;
4209 }
4210#endif /* CONFIG_RPS */
4211 return 0;
4212}
4213
Willem de Bruijn99bbc702013-05-20 04:02:32 +00004214#ifdef CONFIG_NET_FLOW_LIMIT
4215int netdev_flow_limit_table_len __read_mostly = (1 << 12);
4216#endif
4217
4218static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
4219{
4220#ifdef CONFIG_NET_FLOW_LIMIT
4221 struct sd_flow_limit *fl;
4222 struct softnet_data *sd;
4223 unsigned int old_flow, new_flow;
4224
4225 if (qlen < (netdev_max_backlog >> 1))
4226 return false;
4227
Christoph Lameter903ceff2014-08-17 12:30:35 -05004228 sd = this_cpu_ptr(&softnet_data);
Willem de Bruijn99bbc702013-05-20 04:02:32 +00004229
4230 rcu_read_lock();
4231 fl = rcu_dereference(sd->flow_limit);
4232 if (fl) {
Tom Herbert3958afa1b2013-12-15 22:12:06 -08004233 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
Willem de Bruijn99bbc702013-05-20 04:02:32 +00004234 old_flow = fl->history[fl->history_head];
4235 fl->history[fl->history_head] = new_flow;
4236
4237 fl->history_head++;
4238 fl->history_head &= FLOW_LIMIT_HISTORY - 1;
4239
4240 if (likely(fl->buckets[old_flow]))
4241 fl->buckets[old_flow]--;
4242
4243 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
4244 fl->count++;
4245 rcu_read_unlock();
4246 return true;
4247 }
4248 }
4249 rcu_read_unlock();
4250#endif
4251 return false;
4252}
4253
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004254/*
Tom Herbert0a9627f2010-03-16 08:03:29 +00004255 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
4256 * queue (may be a remote CPU queue).
4257 */
Tom Herbertfec5e652010-04-16 16:01:27 -07004258static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
4259 unsigned int *qtail)
Tom Herbert0a9627f2010-03-16 08:03:29 +00004260{
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004261 struct softnet_data *sd;
Tom Herbert0a9627f2010-03-16 08:03:29 +00004262 unsigned long flags;
Willem de Bruijn99bbc702013-05-20 04:02:32 +00004263 unsigned int qlen;
Tom Herbert0a9627f2010-03-16 08:03:29 +00004264
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004265 sd = &per_cpu(softnet_data, cpu);
Tom Herbert0a9627f2010-03-16 08:03:29 +00004266
4267 local_irq_save(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00004268
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004269 rps_lock(sd);
Julian Anastasove9e4dd32015-07-09 09:59:09 +03004270 if (!netif_running(skb->dev))
4271 goto drop;
Willem de Bruijn99bbc702013-05-20 04:02:32 +00004272 qlen = skb_queue_len(&sd->input_pkt_queue);
4273 if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
Li RongQinge008f3f2014-12-08 09:42:55 +08004274 if (qlen) {
Tom Herbert0a9627f2010-03-16 08:03:29 +00004275enqueue:
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004276 __skb_queue_tail(&sd->input_pkt_queue, skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00004277 input_queue_tail_incr_save(sd, qtail);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004278 rps_unlock(sd);
Changli Gao152102c2010-03-30 20:16:22 +00004279 local_irq_restore(flags);
Tom Herbert0a9627f2010-03-16 08:03:29 +00004280 return NET_RX_SUCCESS;
4281 }
4282
Eric Dumazetebda37c22010-05-06 23:51:21 +00004283 /* Schedule NAPI for backlog device
4284 * We can use non atomic operation since we own the queue lock
4285 */
4286 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004287 if (!rps_ipi_queued(sd))
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07004288 ____napi_schedule(sd, &sd->backlog);
Tom Herbert0a9627f2010-03-16 08:03:29 +00004289 }
4290 goto enqueue;
4291 }
4292
Julian Anastasove9e4dd32015-07-09 09:59:09 +03004293drop:
Changli Gaodee42872010-05-02 05:42:16 +00004294 sd->dropped++;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00004295 rps_unlock(sd);
Tom Herbert0a9627f2010-03-16 08:03:29 +00004296
Tom Herbert0a9627f2010-03-16 08:03:29 +00004297 local_irq_restore(flags);
4298
Eric Dumazetcaf586e2010-09-30 21:06:55 +00004299 atomic_long_inc(&skb->dev->rx_dropped);
Tom Herbert0a9627f2010-03-16 08:03:29 +00004300 kfree_skb(skb);
4301 return NET_RX_DROP;
4302}
Linus Torvalds1da177e2005-04-16 15:20:36 -07004303
Jesper Dangaard Brouere817f852018-01-03 11:26:09 +01004304static struct netdev_rx_queue *netif_get_rxqueue(struct sk_buff *skb)
4305{
4306 struct net_device *dev = skb->dev;
4307 struct netdev_rx_queue *rxqueue;
4308
4309 rxqueue = dev->_rx;
4310
4311 if (skb_rx_queue_recorded(skb)) {
4312 u16 index = skb_get_rx_queue(skb);
4313
4314 if (unlikely(index >= dev->real_num_rx_queues)) {
4315 WARN_ONCE(dev->real_num_rx_queues > 1,
4316 "%s received packet on queue %u, but number "
4317 "of RX queues is %u\n",
4318 dev->name, index, dev->real_num_rx_queues);
4319
4320 return rxqueue; /* Return first rxqueue */
4321 }
4322 rxqueue += index;
4323 }
4324 return rxqueue;
4325}
4326
John Fastabendd4455162017-07-17 09:26:45 -07004327static u32 netif_receive_generic_xdp(struct sk_buff *skb,
Björn Töpel02671e22018-05-02 13:01:30 +02004328 struct xdp_buff *xdp,
John Fastabendd4455162017-07-17 09:26:45 -07004329 struct bpf_prog *xdp_prog)
4330{
Jesper Dangaard Brouere817f852018-01-03 11:26:09 +01004331 struct netdev_rx_queue *rxqueue;
Nikita V. Shirokov198d83b2018-04-17 21:42:14 -07004332 void *orig_data, *orig_data_end;
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02004333 u32 metalen, act = XDP_DROP;
Jesper Dangaard Brouer29724952018-10-09 12:04:43 +02004334 __be16 orig_eth_type;
4335 struct ethhdr *eth;
4336 bool orig_bcast;
John Fastabendd4455162017-07-17 09:26:45 -07004337 int hlen, off;
4338 u32 mac_len;
4339
4340 /* Reinjected packets coming from act_mirred or similar should
4341 * not get XDP generic processing.
4342 */
Paolo Abenicd11b1642018-07-30 14:30:44 +02004343 if (skb_cloned(skb) || skb_is_tc_redirected(skb))
John Fastabendd4455162017-07-17 09:26:45 -07004344 return XDP_PASS;
4345
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02004346 /* XDP packets must be linear and must have sufficient headroom
4347 * of XDP_PACKET_HEADROOM bytes. This is the guarantee that also
4348 * native XDP provides, thus we need to do it here as well.
4349 */
4350 if (skb_is_nonlinear(skb) ||
4351 skb_headroom(skb) < XDP_PACKET_HEADROOM) {
4352 int hroom = XDP_PACKET_HEADROOM - skb_headroom(skb);
4353 int troom = skb->tail + skb->data_len - skb->end;
4354
4355 /* In case we have to go down the path and also linearize,
4356 * then lets do the pskb_expand_head() work just once here.
4357 */
4358 if (pskb_expand_head(skb,
4359 hroom > 0 ? ALIGN(hroom, NET_SKB_PAD) : 0,
4360 troom > 0 ? troom + 128 : 0, GFP_ATOMIC))
4361 goto do_drop;
Song Liu2d17d8d2017-12-14 17:17:56 -08004362 if (skb_linearize(skb))
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02004363 goto do_drop;
4364 }
John Fastabendd4455162017-07-17 09:26:45 -07004365
4366 /* The XDP program wants to see the packet starting at the MAC
4367 * header.
4368 */
4369 mac_len = skb->data - skb_mac_header(skb);
4370 hlen = skb_headlen(skb) + mac_len;
Björn Töpel02671e22018-05-02 13:01:30 +02004371 xdp->data = skb->data - mac_len;
4372 xdp->data_meta = xdp->data;
4373 xdp->data_end = xdp->data + hlen;
4374 xdp->data_hard_start = skb->data - skb_headroom(skb);
4375 orig_data_end = xdp->data_end;
4376 orig_data = xdp->data;
Jesper Dangaard Brouer29724952018-10-09 12:04:43 +02004377 eth = (struct ethhdr *)xdp->data;
4378 orig_bcast = is_multicast_ether_addr_64bits(eth->h_dest);
4379 orig_eth_type = eth->h_proto;
John Fastabendd4455162017-07-17 09:26:45 -07004380
Jesper Dangaard Brouere817f852018-01-03 11:26:09 +01004381 rxqueue = netif_get_rxqueue(skb);
Björn Töpel02671e22018-05-02 13:01:30 +02004382 xdp->rxq = &rxqueue->xdp_rxq;
Jesper Dangaard Brouere817f852018-01-03 11:26:09 +01004383
Björn Töpel02671e22018-05-02 13:01:30 +02004384 act = bpf_prog_run_xdp(xdp_prog, xdp);
John Fastabendd4455162017-07-17 09:26:45 -07004385
Björn Töpel02671e22018-05-02 13:01:30 +02004386 off = xdp->data - orig_data;
John Fastabendd4455162017-07-17 09:26:45 -07004387 if (off > 0)
4388 __skb_pull(skb, off);
4389 else if (off < 0)
4390 __skb_push(skb, -off);
Edward Cree92dd5452017-09-19 18:45:56 +01004391 skb->mac_header += off;
John Fastabendd4455162017-07-17 09:26:45 -07004392
Nikita V. Shirokov198d83b2018-04-17 21:42:14 -07004393 /* check if bpf_xdp_adjust_tail was used. it can only "shrink"
4394 * pckt.
4395 */
Björn Töpel02671e22018-05-02 13:01:30 +02004396 off = orig_data_end - xdp->data_end;
Nikita V. Shirokovf7613122018-04-25 07:15:03 -07004397 if (off != 0) {
Björn Töpel02671e22018-05-02 13:01:30 +02004398 skb_set_tail_pointer(skb, xdp->data_end - xdp->data);
Nikita V. Shirokovf7613122018-04-25 07:15:03 -07004399 skb->len -= off;
Björn Töpel02671e22018-05-02 13:01:30 +02004400
Nikita V. Shirokovf7613122018-04-25 07:15:03 -07004401 }
Nikita V. Shirokov198d83b2018-04-17 21:42:14 -07004402
Jesper Dangaard Brouer29724952018-10-09 12:04:43 +02004403 /* check if XDP changed eth hdr such SKB needs update */
4404 eth = (struct ethhdr *)xdp->data;
4405 if ((orig_eth_type != eth->h_proto) ||
4406 (orig_bcast != is_multicast_ether_addr_64bits(eth->h_dest))) {
4407 __skb_push(skb, ETH_HLEN);
4408 skb->protocol = eth_type_trans(skb, skb->dev);
4409 }
4410
John Fastabendd4455162017-07-17 09:26:45 -07004411 switch (act) {
John Fastabend6103aa92017-07-17 09:27:50 -07004412 case XDP_REDIRECT:
John Fastabendd4455162017-07-17 09:26:45 -07004413 case XDP_TX:
4414 __skb_push(skb, mac_len);
John Fastabendd4455162017-07-17 09:26:45 -07004415 break;
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02004416 case XDP_PASS:
Björn Töpel02671e22018-05-02 13:01:30 +02004417 metalen = xdp->data - xdp->data_meta;
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02004418 if (metalen)
4419 skb_metadata_set(skb, metalen);
4420 break;
John Fastabendd4455162017-07-17 09:26:45 -07004421 default:
4422 bpf_warn_invalid_xdp_action(act);
4423 /* fall through */
4424 case XDP_ABORTED:
4425 trace_xdp_exception(skb->dev, xdp_prog, act);
4426 /* fall through */
4427 case XDP_DROP:
4428 do_drop:
4429 kfree_skb(skb);
4430 break;
4431 }
4432
4433 return act;
4434}
4435
4436/* When doing generic XDP we have to bypass the qdisc layer and the
4437 * network taps in order to match in-driver-XDP behavior.
4438 */
Jason Wang7c497472017-08-11 19:41:17 +08004439void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog)
John Fastabendd4455162017-07-17 09:26:45 -07004440{
4441 struct net_device *dev = skb->dev;
4442 struct netdev_queue *txq;
4443 bool free_skb = true;
4444 int cpu, rc;
4445
4446 txq = netdev_pick_tx(dev, skb, NULL);
4447 cpu = smp_processor_id();
4448 HARD_TX_LOCK(dev, txq, cpu);
4449 if (!netif_xmit_stopped(txq)) {
4450 rc = netdev_start_xmit(skb, dev, txq, 0);
4451 if (dev_xmit_complete(rc))
4452 free_skb = false;
4453 }
4454 HARD_TX_UNLOCK(dev, txq);
4455 if (free_skb) {
4456 trace_xdp_exception(dev, xdp_prog, XDP_TX);
4457 kfree_skb(skb);
4458 }
4459}
Jason Wang7c497472017-08-11 19:41:17 +08004460EXPORT_SYMBOL_GPL(generic_xdp_tx);
John Fastabendd4455162017-07-17 09:26:45 -07004461
Davidlohr Bueso02786472018-05-08 09:07:02 -07004462static DEFINE_STATIC_KEY_FALSE(generic_xdp_needed_key);
John Fastabendd4455162017-07-17 09:26:45 -07004463
Jason Wang7c497472017-08-11 19:41:17 +08004464int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb)
John Fastabendd4455162017-07-17 09:26:45 -07004465{
John Fastabendd4455162017-07-17 09:26:45 -07004466 if (xdp_prog) {
Björn Töpel02671e22018-05-02 13:01:30 +02004467 struct xdp_buff xdp;
4468 u32 act;
John Fastabend6103aa92017-07-17 09:27:50 -07004469 int err;
John Fastabendd4455162017-07-17 09:26:45 -07004470
Björn Töpel02671e22018-05-02 13:01:30 +02004471 act = netif_receive_generic_xdp(skb, &xdp, xdp_prog);
John Fastabendd4455162017-07-17 09:26:45 -07004472 if (act != XDP_PASS) {
John Fastabend6103aa92017-07-17 09:27:50 -07004473 switch (act) {
4474 case XDP_REDIRECT:
Jesper Dangaard Brouer2facaad2017-08-24 12:33:08 +02004475 err = xdp_do_generic_redirect(skb->dev, skb,
Björn Töpel02671e22018-05-02 13:01:30 +02004476 &xdp, xdp_prog);
John Fastabend6103aa92017-07-17 09:27:50 -07004477 if (err)
4478 goto out_redir;
Björn Töpel02671e22018-05-02 13:01:30 +02004479 break;
John Fastabend6103aa92017-07-17 09:27:50 -07004480 case XDP_TX:
John Fastabendd4455162017-07-17 09:26:45 -07004481 generic_xdp_tx(skb, xdp_prog);
John Fastabend6103aa92017-07-17 09:27:50 -07004482 break;
4483 }
John Fastabendd4455162017-07-17 09:26:45 -07004484 return XDP_DROP;
4485 }
4486 }
4487 return XDP_PASS;
John Fastabend6103aa92017-07-17 09:27:50 -07004488out_redir:
John Fastabend6103aa92017-07-17 09:27:50 -07004489 kfree_skb(skb);
4490 return XDP_DROP;
John Fastabendd4455162017-07-17 09:26:45 -07004491}
Jason Wang7c497472017-08-11 19:41:17 +08004492EXPORT_SYMBOL_GPL(do_xdp_generic);
John Fastabendd4455162017-07-17 09:26:45 -07004493
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004494static int netif_rx_internal(struct sk_buff *skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004495{
Eric Dumazetb0e28f12010-04-15 00:14:07 -07004496 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004497
Eric Dumazet588f0332011-11-15 04:12:55 +00004498 net_timestamp_check(netdev_tstamp_prequeue, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004499
Koki Sanagicf66ba52010-08-23 18:45:02 +09004500 trace_netif_rx(skb);
John Fastabendd4455162017-07-17 09:26:45 -07004501
Davidlohr Bueso02786472018-05-08 09:07:02 -07004502 if (static_branch_unlikely(&generic_xdp_needed_key)) {
John Fastabendbbbe2112017-09-08 14:00:30 -07004503 int ret;
4504
4505 preempt_disable();
4506 rcu_read_lock();
4507 ret = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), skb);
4508 rcu_read_unlock();
4509 preempt_enable();
John Fastabendd4455162017-07-17 09:26:45 -07004510
John Fastabend6103aa92017-07-17 09:27:50 -07004511 /* Consider XDP consuming the packet a success from
4512 * the netdev point of view we do not want to count
4513 * this as an error.
4514 */
John Fastabendd4455162017-07-17 09:26:45 -07004515 if (ret != XDP_PASS)
John Fastabend6103aa92017-07-17 09:27:50 -07004516 return NET_RX_SUCCESS;
John Fastabendd4455162017-07-17 09:26:45 -07004517 }
4518
Eric Dumazetdf334542010-03-24 19:13:54 +00004519#ifdef CONFIG_RPS
Ingo Molnarc5905af2012-02-24 08:31:31 +01004520 if (static_key_false(&rps_needed)) {
Tom Herbertfec5e652010-04-16 16:01:27 -07004521 struct rps_dev_flow voidflow, *rflow = &voidflow;
Eric Dumazetb0e28f12010-04-15 00:14:07 -07004522 int cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004523
Changli Gaocece1942010-08-07 20:35:43 -07004524 preempt_disable();
Eric Dumazetb0e28f12010-04-15 00:14:07 -07004525 rcu_read_lock();
Tom Herbertfec5e652010-04-16 16:01:27 -07004526
4527 cpu = get_rps_cpu(skb->dev, skb, &rflow);
Eric Dumazetb0e28f12010-04-15 00:14:07 -07004528 if (cpu < 0)
4529 cpu = smp_processor_id();
Tom Herbertfec5e652010-04-16 16:01:27 -07004530
4531 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
4532
Eric Dumazetb0e28f12010-04-15 00:14:07 -07004533 rcu_read_unlock();
Changli Gaocece1942010-08-07 20:35:43 -07004534 preempt_enable();
Eric Dumazetadc93002011-11-17 03:13:26 +00004535 } else
4536#endif
Tom Herbertfec5e652010-04-16 16:01:27 -07004537 {
4538 unsigned int qtail;
tchardingf4563a72017-02-09 17:56:07 +11004539
Tom Herbertfec5e652010-04-16 16:01:27 -07004540 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
4541 put_cpu();
4542 }
Eric Dumazetb0e28f12010-04-15 00:14:07 -07004543 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004544}
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004545
4546/**
4547 * netif_rx - post buffer to the network code
4548 * @skb: buffer to post
4549 *
4550 * This function receives a packet from a device driver and queues it for
4551 * the upper (protocol) levels to process. It always succeeds. The buffer
4552 * may be dropped during processing for congestion control or by the
4553 * protocol layers.
4554 *
4555 * return values:
4556 * NET_RX_SUCCESS (no congestion)
4557 * NET_RX_DROP (packet was dropped)
4558 *
4559 */
4560
4561int netif_rx(struct sk_buff *skb)
4562{
Geneviève Bastienb0e3f1b2018-11-27 12:52:39 -05004563 int ret;
4564
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004565 trace_netif_rx_entry(skb);
4566
Geneviève Bastienb0e3f1b2018-11-27 12:52:39 -05004567 ret = netif_rx_internal(skb);
4568 trace_netif_rx_exit(ret);
4569
4570 return ret;
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004571}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07004572EXPORT_SYMBOL(netif_rx);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004573
4574int netif_rx_ni(struct sk_buff *skb)
4575{
4576 int err;
4577
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004578 trace_netif_rx_ni_entry(skb);
4579
Linus Torvalds1da177e2005-04-16 15:20:36 -07004580 preempt_disable();
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00004581 err = netif_rx_internal(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004582 if (local_softirq_pending())
4583 do_softirq();
4584 preempt_enable();
Geneviève Bastienb0e3f1b2018-11-27 12:52:39 -05004585 trace_netif_rx_ni_exit(err);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004586
4587 return err;
4588}
Linus Torvalds1da177e2005-04-16 15:20:36 -07004589EXPORT_SYMBOL(netif_rx_ni);
4590
Emese Revfy0766f782016-06-20 20:42:34 +02004591static __latent_entropy void net_tx_action(struct softirq_action *h)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004592{
Christoph Lameter903ceff2014-08-17 12:30:35 -05004593 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004594
4595 if (sd->completion_queue) {
4596 struct sk_buff *clist;
4597
4598 local_irq_disable();
4599 clist = sd->completion_queue;
4600 sd->completion_queue = NULL;
4601 local_irq_enable();
4602
4603 while (clist) {
4604 struct sk_buff *skb = clist;
tchardingf4563a72017-02-09 17:56:07 +11004605
Linus Torvalds1da177e2005-04-16 15:20:36 -07004606 clist = clist->next;
4607
Reshetova, Elena63354792017-06-30 13:07:58 +03004608 WARN_ON(refcount_read(&skb->users));
Eric Dumazete6247022013-12-05 04:45:08 -08004609 if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
4610 trace_consume_skb(skb);
4611 else
4612 trace_kfree_skb(skb, net_tx_action);
Jesper Dangaard Brouer15fad712016-02-08 13:15:04 +01004613
4614 if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
4615 __kfree_skb(skb);
4616 else
4617 __kfree_skb_defer(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004618 }
Jesper Dangaard Brouer15fad712016-02-08 13:15:04 +01004619
4620 __kfree_skb_flush();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004621 }
4622
4623 if (sd->output_queue) {
David S. Miller37437bb2008-07-16 02:15:04 -07004624 struct Qdisc *head;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004625
4626 local_irq_disable();
4627 head = sd->output_queue;
4628 sd->output_queue = NULL;
Changli Gaoa9cbd582010-04-26 23:06:24 +00004629 sd->output_queue_tailp = &sd->output_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004630 local_irq_enable();
4631
4632 while (head) {
David S. Miller37437bb2008-07-16 02:15:04 -07004633 struct Qdisc *q = head;
John Fastabend6b3ba912017-12-07 09:54:25 -08004634 spinlock_t *root_lock = NULL;
David S. Miller37437bb2008-07-16 02:15:04 -07004635
Linus Torvalds1da177e2005-04-16 15:20:36 -07004636 head = head->next_sched;
4637
John Fastabend6b3ba912017-12-07 09:54:25 -08004638 if (!(q->flags & TCQ_F_NOLOCK)) {
4639 root_lock = qdisc_lock(q);
4640 spin_lock(root_lock);
4641 }
Eric Dumazet3bcb8462016-06-04 20:02:28 -07004642 /* We need to make sure head->next_sched is read
4643 * before clearing __QDISC_STATE_SCHED
4644 */
4645 smp_mb__before_atomic();
4646 clear_bit(__QDISC_STATE_SCHED, &q->state);
4647 qdisc_run(q);
John Fastabend6b3ba912017-12-07 09:54:25 -08004648 if (root_lock)
4649 spin_unlock(root_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004650 }
4651 }
Steffen Klassertf53c7232017-12-20 10:41:36 +01004652
4653 xfrm_dev_backlog(sd);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004654}
4655
Javier Martinez Canillas181402a2016-09-09 08:43:15 -04004656#if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE)
Michał Mirosławda678292009-06-05 05:35:28 +00004657/* This hook is defined here for ATM LANE */
4658int (*br_fdb_test_addr_hook)(struct net_device *dev,
4659 unsigned char *addr) __read_mostly;
Stephen Hemminger4fb019a2009-09-11 11:50:08 -07004660EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
Michał Mirosławda678292009-06-05 05:35:28 +00004661#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07004662
Daniel Borkmann1f211a12016-01-07 22:29:47 +01004663static inline struct sk_buff *
4664sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
4665 struct net_device *orig_dev)
Herbert Xuf697c3e82007-10-14 00:38:47 -07004666{
Daniel Borkmanne7582ba2015-05-19 22:33:25 +02004667#ifdef CONFIG_NET_CLS_ACT
Jiri Pirko46209402017-11-03 11:46:25 +01004668 struct mini_Qdisc *miniq = rcu_dereference_bh(skb->dev->miniq_ingress);
Daniel Borkmannd2788d32015-05-09 22:51:32 +02004669 struct tcf_result cl_res;
Eric Dumazet24824a02010-10-02 06:11:55 +00004670
Daniel Borkmannc9e99fd2015-05-09 22:51:31 +02004671 /* If there's at least one ingress present somewhere (so
4672 * we get here via enabled static key), remaining devices
4673 * that are not configured with an ingress qdisc will bail
Daniel Borkmannd2788d32015-05-09 22:51:32 +02004674 * out here.
Daniel Borkmannc9e99fd2015-05-09 22:51:31 +02004675 */
Jiri Pirko46209402017-11-03 11:46:25 +01004676 if (!miniq)
Daniel Borkmann45771392015-04-10 23:07:54 +02004677 return skb;
Jiri Pirko46209402017-11-03 11:46:25 +01004678
Herbert Xuf697c3e82007-10-14 00:38:47 -07004679 if (*pt_prev) {
4680 *ret = deliver_skb(skb, *pt_prev, orig_dev);
4681 *pt_prev = NULL;
Herbert Xuf697c3e82007-10-14 00:38:47 -07004682 }
4683
Florian Westphal33654952015-05-14 00:36:28 +02004684 qdisc_skb_cb(skb)->pkt_len = skb->len;
Willem de Bruijn8dc07fd2017-01-07 17:06:37 -05004685 skb->tc_at_ingress = 1;
Jiri Pirko46209402017-11-03 11:46:25 +01004686 mini_qdisc_bstats_cpu_update(miniq, skb);
Daniel Borkmannc9e99fd2015-05-09 22:51:31 +02004687
Jiri Pirko46209402017-11-03 11:46:25 +01004688 switch (tcf_classify(skb, miniq->filter_list, &cl_res, false)) {
Daniel Borkmannd2788d32015-05-09 22:51:32 +02004689 case TC_ACT_OK:
4690 case TC_ACT_RECLASSIFY:
4691 skb->tc_index = TC_H_MIN(cl_res.classid);
4692 break;
4693 case TC_ACT_SHOT:
Jiri Pirko46209402017-11-03 11:46:25 +01004694 mini_qdisc_qstats_cpu_drop(miniq);
Eric Dumazet8a3a4c62016-05-06 15:55:50 -07004695 kfree_skb(skb);
4696 return NULL;
Daniel Borkmannd2788d32015-05-09 22:51:32 +02004697 case TC_ACT_STOLEN:
4698 case TC_ACT_QUEUED:
Jiri Pirkoe25ea212017-06-06 14:12:02 +02004699 case TC_ACT_TRAP:
Eric Dumazet8a3a4c62016-05-06 15:55:50 -07004700 consume_skb(skb);
Daniel Borkmannd2788d32015-05-09 22:51:32 +02004701 return NULL;
Alexei Starovoitov27b29f62015-09-15 23:05:43 -07004702 case TC_ACT_REDIRECT:
4703 /* skb_mac_header check was done by cls/act_bpf, so
4704 * we can safely push the L2 header back before
4705 * redirecting to another netdev
4706 */
4707 __skb_push(skb, skb->mac_len);
4708 skb_do_redirect(skb);
4709 return NULL;
Paolo Abenicd11b1642018-07-30 14:30:44 +02004710 case TC_ACT_REINSERT:
4711 /* this does not scrub the packet, and updates stats on error */
4712 skb_tc_reinsert(skb, &cl_res);
4713 return NULL;
Daniel Borkmannd2788d32015-05-09 22:51:32 +02004714 default:
4715 break;
Herbert Xuf697c3e82007-10-14 00:38:47 -07004716 }
Daniel Borkmanne7582ba2015-05-19 22:33:25 +02004717#endif /* CONFIG_NET_CLS_ACT */
Herbert Xuf697c3e82007-10-14 00:38:47 -07004718 return skb;
4719}
Linus Torvalds1da177e2005-04-16 15:20:36 -07004720
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004721/**
Mahesh Bandewar24b27fc2016-09-01 22:18:34 -07004722 * netdev_is_rx_handler_busy - check if receive handler is registered
4723 * @dev: device to check
4724 *
4725 * Check if a receive handler is already registered for a given device.
4726 * Return true if there one.
4727 *
4728 * The caller must hold the rtnl_mutex.
4729 */
4730bool netdev_is_rx_handler_busy(struct net_device *dev)
4731{
4732 ASSERT_RTNL();
4733 return dev && rtnl_dereference(dev->rx_handler);
4734}
4735EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy);
4736
4737/**
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004738 * netdev_rx_handler_register - register receive handler
4739 * @dev: device to register a handler for
4740 * @rx_handler: receive handler to register
Jiri Pirko93e2c322010-06-10 03:34:59 +00004741 * @rx_handler_data: data pointer that is used by rx handler
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004742 *
Masanari Iidae2278672014-02-18 22:54:36 +09004743 * Register a receive handler for a device. This handler will then be
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004744 * called from __netif_receive_skb. A negative errno code is returned
4745 * on a failure.
4746 *
4747 * The caller must hold the rtnl_mutex.
Jiri Pirko8a4eb572011-03-12 03:14:39 +00004748 *
4749 * For a general description of rx_handler, see enum rx_handler_result.
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004750 */
4751int netdev_rx_handler_register(struct net_device *dev,
Jiri Pirko93e2c322010-06-10 03:34:59 +00004752 rx_handler_func_t *rx_handler,
4753 void *rx_handler_data)
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004754{
Mahesh Bandewar1b7cd002017-01-18 15:02:49 -08004755 if (netdev_is_rx_handler_busy(dev))
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004756 return -EBUSY;
4757
Paolo Abenif54262502018-03-09 10:39:24 +01004758 if (dev->priv_flags & IFF_NO_RX_HANDLER)
4759 return -EINVAL;
4760
Eric Dumazet00cfec32013-03-29 03:01:22 +00004761 /* Note: rx_handler_data must be set before rx_handler */
Jiri Pirko93e2c322010-06-10 03:34:59 +00004762 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004763 rcu_assign_pointer(dev->rx_handler, rx_handler);
4764
4765 return 0;
4766}
4767EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
4768
4769/**
4770 * netdev_rx_handler_unregister - unregister receive handler
4771 * @dev: device to unregister a handler from
4772 *
Kusanagi Kouichi166ec362013-03-18 02:59:52 +00004773 * Unregister a receive handler from a device.
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004774 *
4775 * The caller must hold the rtnl_mutex.
4776 */
4777void netdev_rx_handler_unregister(struct net_device *dev)
4778{
4779
4780 ASSERT_RTNL();
Stephen Hemmingera9b3cd72011-08-01 16:19:00 +00004781 RCU_INIT_POINTER(dev->rx_handler, NULL);
Eric Dumazet00cfec32013-03-29 03:01:22 +00004782 /* a reader seeing a non NULL rx_handler in a rcu_read_lock()
4783 * section has a guarantee to see a non NULL rx_handler_data
4784 * as well.
4785 */
4786 synchronize_net();
Stephen Hemmingera9b3cd72011-08-01 16:19:00 +00004787 RCU_INIT_POINTER(dev->rx_handler_data, NULL);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004788}
4789EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
4790
Mel Gormanb4b9e352012-07-31 16:44:26 -07004791/*
4792 * Limit the use of PFMEMALLOC reserves to those protocols that implement
4793 * the special handling of PFMEMALLOC skbs.
4794 */
4795static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
4796{
4797 switch (skb->protocol) {
Joe Perches2b8837a2014-03-12 10:04:17 -07004798 case htons(ETH_P_ARP):
4799 case htons(ETH_P_IP):
4800 case htons(ETH_P_IPV6):
4801 case htons(ETH_P_8021Q):
4802 case htons(ETH_P_8021AD):
Mel Gormanb4b9e352012-07-31 16:44:26 -07004803 return true;
4804 default:
4805 return false;
4806 }
4807}
4808
Pablo Neirae687ad62015-05-13 18:19:38 +02004809static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
4810 int *ret, struct net_device *orig_dev)
4811{
Daniel Borkmanne7582ba2015-05-19 22:33:25 +02004812#ifdef CONFIG_NETFILTER_INGRESS
Pablo Neirae687ad62015-05-13 18:19:38 +02004813 if (nf_hook_ingress_active(skb)) {
Aaron Conole2c1e2702016-09-21 11:35:03 -04004814 int ingress_retval;
4815
Pablo Neirae687ad62015-05-13 18:19:38 +02004816 if (*pt_prev) {
4817 *ret = deliver_skb(skb, *pt_prev, orig_dev);
4818 *pt_prev = NULL;
4819 }
4820
Aaron Conole2c1e2702016-09-21 11:35:03 -04004821 rcu_read_lock();
4822 ingress_retval = nf_hook_ingress(skb);
4823 rcu_read_unlock();
4824 return ingress_retval;
Pablo Neirae687ad62015-05-13 18:19:38 +02004825 }
Daniel Borkmanne7582ba2015-05-19 22:33:25 +02004826#endif /* CONFIG_NETFILTER_INGRESS */
Pablo Neirae687ad62015-05-13 18:19:38 +02004827 return 0;
4828}
Pablo Neirae687ad62015-05-13 18:19:38 +02004829
Edward Cree88eb1942018-07-02 16:13:56 +01004830static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc,
4831 struct packet_type **ppt_prev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004832{
4833 struct packet_type *ptype, *pt_prev;
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004834 rx_handler_func_t *rx_handler;
David S. Millerf2ccd8fa2005-08-09 19:34:12 -07004835 struct net_device *orig_dev;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00004836 bool deliver_exact = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004837 int ret = NET_RX_DROP;
Al Viro252e33462006-11-14 20:48:11 -08004838 __be16 type;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004839
Eric Dumazet588f0332011-11-15 04:12:55 +00004840 net_timestamp_check(!netdev_tstamp_prequeue, skb);
Eric Dumazet81bbb3d2009-09-30 16:42:42 -07004841
Koki Sanagicf66ba52010-08-23 18:45:02 +09004842 trace_netif_receive_skb(skb);
Patrick McHardy9b22ea52008-11-04 14:49:57 -08004843
Joe Eykholtcc9bd5c2008-07-02 18:22:00 -07004844 orig_dev = skb->dev;
Jiri Pirko1765a572011-02-12 06:48:36 +00004845
Arnaldo Carvalho de Meloc1d2bbe2007-04-10 20:45:18 -07004846 skb_reset_network_header(skb);
Eric Dumazetfda55ec2013-01-07 09:28:21 +00004847 if (!skb_transport_header_was_set(skb))
4848 skb_reset_transport_header(skb);
Jiri Pirko0b5c9db2011-06-10 06:56:58 +00004849 skb_reset_mac_len(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004850
4851 pt_prev = NULL;
4852
David S. Miller63d8ea72011-02-28 10:48:59 -08004853another_round:
David S. Millerb6858172012-07-23 16:27:54 -07004854 skb->skb_iif = skb->dev->ifindex;
David S. Miller63d8ea72011-02-28 10:48:59 -08004855
4856 __this_cpu_inc(softnet_data.processed);
4857
Patrick McHardy8ad227f2013-04-19 02:04:31 +00004858 if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
4859 skb->protocol == cpu_to_be16(ETH_P_8021AD)) {
Vlad Yasevich0d5501c2014-08-08 14:42:13 -04004860 skb = skb_vlan_untag(skb);
Jiri Pirkobcc6d472011-04-07 19:48:33 +00004861 if (unlikely(!skb))
Julian Anastasov2c17d272015-07-09 09:59:10 +03004862 goto out;
Jiri Pirkobcc6d472011-04-07 19:48:33 +00004863 }
4864
Willem de Bruijne7246e12017-01-07 17:06:35 -05004865 if (skb_skip_tc_classify(skb))
4866 goto skip_classify;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004867
David S. Miller9754e292013-02-14 15:57:38 -05004868 if (pfmemalloc)
Mel Gormanb4b9e352012-07-31 16:44:26 -07004869 goto skip_taps;
4870
Linus Torvalds1da177e2005-04-16 15:20:36 -07004871 list_for_each_entry_rcu(ptype, &ptype_all, list) {
Salam Noureddine7866a622015-01-27 11:35:48 -08004872 if (pt_prev)
4873 ret = deliver_skb(skb, pt_prev, orig_dev);
4874 pt_prev = ptype;
4875 }
4876
4877 list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
4878 if (pt_prev)
4879 ret = deliver_skb(skb, pt_prev, orig_dev);
4880 pt_prev = ptype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004881 }
4882
Mel Gormanb4b9e352012-07-31 16:44:26 -07004883skip_taps:
Pablo Neira1cf519002015-05-13 18:19:37 +02004884#ifdef CONFIG_NET_INGRESS
Davidlohr Buesoaabf6772018-05-08 09:07:00 -07004885 if (static_branch_unlikely(&ingress_needed_key)) {
Daniel Borkmann1f211a12016-01-07 22:29:47 +01004886 skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev);
Daniel Borkmann45771392015-04-10 23:07:54 +02004887 if (!skb)
Julian Anastasov2c17d272015-07-09 09:59:10 +03004888 goto out;
Pablo Neirae687ad62015-05-13 18:19:38 +02004889
4890 if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
Julian Anastasov2c17d272015-07-09 09:59:10 +03004891 goto out;
Daniel Borkmann45771392015-04-10 23:07:54 +02004892 }
Pablo Neira1cf519002015-05-13 18:19:37 +02004893#endif
Willem de Bruijna5135bc2017-01-07 17:06:36 -05004894 skb_reset_tc(skb);
Willem de Bruijne7246e12017-01-07 17:06:35 -05004895skip_classify:
David S. Miller9754e292013-02-14 15:57:38 -05004896 if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
Mel Gormanb4b9e352012-07-31 16:44:26 -07004897 goto drop;
4898
Jiri Pirkodf8a39de2015-01-13 17:13:44 +01004899 if (skb_vlan_tag_present(skb)) {
John Fastabend24257172011-10-10 09:16:41 +00004900 if (pt_prev) {
4901 ret = deliver_skb(skb, pt_prev, orig_dev);
4902 pt_prev = NULL;
4903 }
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00004904 if (vlan_do_receive(&skb))
John Fastabend24257172011-10-10 09:16:41 +00004905 goto another_round;
4906 else if (unlikely(!skb))
Julian Anastasov2c17d272015-07-09 09:59:10 +03004907 goto out;
John Fastabend24257172011-10-10 09:16:41 +00004908 }
4909
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00004910 rx_handler = rcu_dereference(skb->dev->rx_handler);
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004911 if (rx_handler) {
4912 if (pt_prev) {
4913 ret = deliver_skb(skb, pt_prev, orig_dev);
4914 pt_prev = NULL;
4915 }
Jiri Pirko8a4eb572011-03-12 03:14:39 +00004916 switch (rx_handler(&skb)) {
4917 case RX_HANDLER_CONSUMED:
Cristian Bercaru3bc1b1a2013-03-08 07:03:38 +00004918 ret = NET_RX_SUCCESS;
Julian Anastasov2c17d272015-07-09 09:59:10 +03004919 goto out;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00004920 case RX_HANDLER_ANOTHER:
David S. Miller63d8ea72011-02-28 10:48:59 -08004921 goto another_round;
Jiri Pirko8a4eb572011-03-12 03:14:39 +00004922 case RX_HANDLER_EXACT:
4923 deliver_exact = true;
4924 case RX_HANDLER_PASS:
4925 break;
4926 default:
4927 BUG();
4928 }
Jiri Pirkoab95bfe2010-06-01 21:52:08 +00004929 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004930
Jiri Pirkodf8a39de2015-01-13 17:13:44 +01004931 if (unlikely(skb_vlan_tag_present(skb))) {
4932 if (skb_vlan_tag_get_id(skb))
Eric Dumazetd4b812d2013-07-18 07:19:26 -07004933 skb->pkt_type = PACKET_OTHERHOST;
4934 /* Note: we might in the future use prio bits
4935 * and set skb->priority like in vlan_do_receive()
4936 * For the time being, just ignore Priority Code Point
4937 */
Michał Mirosławb18175242018-11-09 00:18:02 +01004938 __vlan_hwaccel_clear_tag(skb);
Eric Dumazetd4b812d2013-07-18 07:19:26 -07004939 }
Florian Zumbiehl48cc32d32012-10-07 15:51:58 +00004940
Linus Torvalds1da177e2005-04-16 15:20:36 -07004941 type = skb->protocol;
Salam Noureddine7866a622015-01-27 11:35:48 -08004942
4943 /* deliver only exact match when indicated */
4944 if (likely(!deliver_exact)) {
4945 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4946 &ptype_base[ntohs(type) &
4947 PTYPE_HASH_MASK]);
4948 }
4949
4950 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4951 &orig_dev->ptype_specific);
4952
4953 if (unlikely(skb->dev != orig_dev)) {
4954 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4955 &skb->dev->ptype_specific);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004956 }
4957
4958 if (pt_prev) {
Willem de Bruijn1f8b9772017-08-03 16:29:41 -04004959 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
Michael S. Tsirkin0e698bf2012-09-15 22:44:16 +00004960 goto drop;
Edward Cree88eb1942018-07-02 16:13:56 +01004961 *ppt_prev = pt_prev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004962 } else {
Mel Gormanb4b9e352012-07-31 16:44:26 -07004963drop:
Jarod Wilson6e7333d2016-02-01 18:51:05 -05004964 if (!deliver_exact)
4965 atomic_long_inc(&skb->dev->rx_dropped);
4966 else
4967 atomic_long_inc(&skb->dev->rx_nohandler);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004968 kfree_skb(skb);
4969 /* Jamal, now you will not able to escape explaining
4970 * me how you were going to use this. :-)
4971 */
4972 ret = NET_RX_DROP;
4973 }
4974
Julian Anastasov2c17d272015-07-09 09:59:10 +03004975out:
David S. Miller9754e292013-02-14 15:57:38 -05004976 return ret;
4977}
4978
Edward Cree88eb1942018-07-02 16:13:56 +01004979static int __netif_receive_skb_one_core(struct sk_buff *skb, bool pfmemalloc)
4980{
4981 struct net_device *orig_dev = skb->dev;
4982 struct packet_type *pt_prev = NULL;
4983 int ret;
4984
4985 ret = __netif_receive_skb_core(skb, pfmemalloc, &pt_prev);
4986 if (pt_prev)
4987 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
4988 return ret;
4989}
4990
Jesper Dangaard Brouer1c601d82017-10-16 12:19:39 +02004991/**
4992 * netif_receive_skb_core - special purpose version of netif_receive_skb
4993 * @skb: buffer to process
4994 *
4995 * More direct receive version of netif_receive_skb(). It should
4996 * only be used by callers that have a need to skip RPS and Generic XDP.
4997 * Caller must also take care of handling if (page_is_)pfmemalloc.
4998 *
4999 * This function may only be called from softirq context and interrupts
5000 * should be enabled.
5001 *
5002 * Return values (usually ignored):
5003 * NET_RX_SUCCESS: no congestion
5004 * NET_RX_DROP: packet was dropped
5005 */
5006int netif_receive_skb_core(struct sk_buff *skb)
5007{
5008 int ret;
5009
5010 rcu_read_lock();
Edward Cree88eb1942018-07-02 16:13:56 +01005011 ret = __netif_receive_skb_one_core(skb, false);
Jesper Dangaard Brouer1c601d82017-10-16 12:19:39 +02005012 rcu_read_unlock();
5013
5014 return ret;
5015}
5016EXPORT_SYMBOL(netif_receive_skb_core);
5017
Edward Cree88eb1942018-07-02 16:13:56 +01005018static inline void __netif_receive_skb_list_ptype(struct list_head *head,
5019 struct packet_type *pt_prev,
5020 struct net_device *orig_dev)
Edward Cree4ce00172018-07-02 16:13:40 +01005021{
5022 struct sk_buff *skb, *next;
5023
Edward Cree88eb1942018-07-02 16:13:56 +01005024 if (!pt_prev)
5025 return;
5026 if (list_empty(head))
5027 return;
Edward Cree17266ee2018-07-02 16:14:12 +01005028 if (pt_prev->list_func != NULL)
5029 pt_prev->list_func(head, pt_prev, orig_dev);
5030 else
Alexander Lobakin9a5a90d2019-03-28 18:23:04 +03005031 list_for_each_entry_safe(skb, next, head, list) {
5032 skb_list_del_init(skb);
Edward Cree17266ee2018-07-02 16:14:12 +01005033 pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
Alexander Lobakin9a5a90d2019-03-28 18:23:04 +03005034 }
Edward Cree88eb1942018-07-02 16:13:56 +01005035}
5036
5037static void __netif_receive_skb_list_core(struct list_head *head, bool pfmemalloc)
5038{
5039 /* Fast-path assumptions:
5040 * - There is no RX handler.
5041 * - Only one packet_type matches.
5042 * If either of these fails, we will end up doing some per-packet
5043 * processing in-line, then handling the 'last ptype' for the whole
5044 * sublist. This can't cause out-of-order delivery to any single ptype,
5045 * because the 'last ptype' must be constant across the sublist, and all
5046 * other ptypes are handled per-packet.
5047 */
5048 /* Current (common) ptype of sublist */
5049 struct packet_type *pt_curr = NULL;
5050 /* Current (common) orig_dev of sublist */
5051 struct net_device *od_curr = NULL;
5052 struct list_head sublist;
5053 struct sk_buff *skb, *next;
5054
Edward Cree9af86f932018-07-09 18:10:19 +01005055 INIT_LIST_HEAD(&sublist);
Edward Cree88eb1942018-07-02 16:13:56 +01005056 list_for_each_entry_safe(skb, next, head, list) {
5057 struct net_device *orig_dev = skb->dev;
5058 struct packet_type *pt_prev = NULL;
5059
Edward Cree22f6bbb2018-12-04 17:37:57 +00005060 skb_list_del_init(skb);
Edward Cree88eb1942018-07-02 16:13:56 +01005061 __netif_receive_skb_core(skb, pfmemalloc, &pt_prev);
Edward Cree9af86f932018-07-09 18:10:19 +01005062 if (!pt_prev)
5063 continue;
Edward Cree88eb1942018-07-02 16:13:56 +01005064 if (pt_curr != pt_prev || od_curr != orig_dev) {
5065 /* dispatch old sublist */
Edward Cree88eb1942018-07-02 16:13:56 +01005066 __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
5067 /* start new sublist */
Edward Cree9af86f932018-07-09 18:10:19 +01005068 INIT_LIST_HEAD(&sublist);
Edward Cree88eb1942018-07-02 16:13:56 +01005069 pt_curr = pt_prev;
5070 od_curr = orig_dev;
5071 }
Edward Cree9af86f932018-07-09 18:10:19 +01005072 list_add_tail(&skb->list, &sublist);
Edward Cree88eb1942018-07-02 16:13:56 +01005073 }
5074
5075 /* dispatch final sublist */
Edward Cree9af86f932018-07-09 18:10:19 +01005076 __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
Edward Cree4ce00172018-07-02 16:13:40 +01005077}
5078
David S. Miller9754e292013-02-14 15:57:38 -05005079static int __netif_receive_skb(struct sk_buff *skb)
5080{
5081 int ret;
5082
5083 if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
Vlastimil Babkaf1083042017-05-08 15:59:53 -07005084 unsigned int noreclaim_flag;
David S. Miller9754e292013-02-14 15:57:38 -05005085
5086 /*
5087 * PFMEMALLOC skbs are special, they should
5088 * - be delivered to SOCK_MEMALLOC sockets only
5089 * - stay away from userspace
5090 * - have bounded memory usage
5091 *
5092 * Use PF_MEMALLOC as this saves us from propagating the allocation
5093 * context down to all allocation sites.
5094 */
Vlastimil Babkaf1083042017-05-08 15:59:53 -07005095 noreclaim_flag = memalloc_noreclaim_save();
Edward Cree88eb1942018-07-02 16:13:56 +01005096 ret = __netif_receive_skb_one_core(skb, true);
Vlastimil Babkaf1083042017-05-08 15:59:53 -07005097 memalloc_noreclaim_restore(noreclaim_flag);
David S. Miller9754e292013-02-14 15:57:38 -05005098 } else
Edward Cree88eb1942018-07-02 16:13:56 +01005099 ret = __netif_receive_skb_one_core(skb, false);
David S. Miller9754e292013-02-14 15:57:38 -05005100
Linus Torvalds1da177e2005-04-16 15:20:36 -07005101 return ret;
5102}
Tom Herbert0a9627f2010-03-16 08:03:29 +00005103
Edward Cree4ce00172018-07-02 16:13:40 +01005104static void __netif_receive_skb_list(struct list_head *head)
5105{
5106 unsigned long noreclaim_flag = 0;
5107 struct sk_buff *skb, *next;
5108 bool pfmemalloc = false; /* Is current sublist PF_MEMALLOC? */
5109
5110 list_for_each_entry_safe(skb, next, head, list) {
5111 if ((sk_memalloc_socks() && skb_pfmemalloc(skb)) != pfmemalloc) {
5112 struct list_head sublist;
5113
5114 /* Handle the previous sublist */
5115 list_cut_before(&sublist, head, &skb->list);
Edward Creeb9f463d2018-07-02 16:14:44 +01005116 if (!list_empty(&sublist))
5117 __netif_receive_skb_list_core(&sublist, pfmemalloc);
Edward Cree4ce00172018-07-02 16:13:40 +01005118 pfmemalloc = !pfmemalloc;
5119 /* See comments in __netif_receive_skb */
5120 if (pfmemalloc)
5121 noreclaim_flag = memalloc_noreclaim_save();
5122 else
5123 memalloc_noreclaim_restore(noreclaim_flag);
5124 }
5125 }
5126 /* Handle the remaining sublist */
Edward Creeb9f463d2018-07-02 16:14:44 +01005127 if (!list_empty(head))
5128 __netif_receive_skb_list_core(head, pfmemalloc);
Edward Cree4ce00172018-07-02 16:13:40 +01005129 /* Restore pflags */
5130 if (pfmemalloc)
5131 memalloc_noreclaim_restore(noreclaim_flag);
5132}
5133
Jakub Kicinskif4e63522017-11-03 13:56:16 -07005134static int generic_xdp_install(struct net_device *dev, struct netdev_bpf *xdp)
David S. Millerb5cdae32017-04-18 15:36:58 -04005135{
Martin KaFai Lau58038692017-06-15 17:29:09 -07005136 struct bpf_prog *old = rtnl_dereference(dev->xdp_prog);
David S. Millerb5cdae32017-04-18 15:36:58 -04005137 struct bpf_prog *new = xdp->prog;
5138 int ret = 0;
5139
5140 switch (xdp->command) {
Martin KaFai Lau58038692017-06-15 17:29:09 -07005141 case XDP_SETUP_PROG:
David S. Millerb5cdae32017-04-18 15:36:58 -04005142 rcu_assign_pointer(dev->xdp_prog, new);
5143 if (old)
5144 bpf_prog_put(old);
5145
5146 if (old && !new) {
Davidlohr Bueso02786472018-05-08 09:07:02 -07005147 static_branch_dec(&generic_xdp_needed_key);
David S. Millerb5cdae32017-04-18 15:36:58 -04005148 } else if (new && !old) {
Davidlohr Bueso02786472018-05-08 09:07:02 -07005149 static_branch_inc(&generic_xdp_needed_key);
David S. Millerb5cdae32017-04-18 15:36:58 -04005150 dev_disable_lro(dev);
Michael Chan56f5aa72017-12-16 03:09:41 -05005151 dev_disable_gro_hw(dev);
David S. Millerb5cdae32017-04-18 15:36:58 -04005152 }
5153 break;
David S. Millerb5cdae32017-04-18 15:36:58 -04005154
5155 case XDP_QUERY_PROG:
Martin KaFai Lau58038692017-06-15 17:29:09 -07005156 xdp->prog_id = old ? old->aux->id : 0;
David S. Millerb5cdae32017-04-18 15:36:58 -04005157 break;
5158
5159 default:
5160 ret = -EINVAL;
5161 break;
5162 }
5163
5164 return ret;
5165}
5166
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00005167static int netif_receive_skb_internal(struct sk_buff *skb)
Tom Herbert0a9627f2010-03-16 08:03:29 +00005168{
Julian Anastasov2c17d272015-07-09 09:59:10 +03005169 int ret;
5170
Eric Dumazet588f0332011-11-15 04:12:55 +00005171 net_timestamp_check(netdev_tstamp_prequeue, skb);
Eric Dumazet3b098e22010-05-15 23:57:10 -07005172
Richard Cochranc1f19b52010-07-17 08:49:36 +00005173 if (skb_defer_rx_timestamp(skb))
5174 return NET_RX_SUCCESS;
5175
Davidlohr Bueso02786472018-05-08 09:07:02 -07005176 if (static_branch_unlikely(&generic_xdp_needed_key)) {
John Fastabendbbbe2112017-09-08 14:00:30 -07005177 int ret;
David S. Millerb5cdae32017-04-18 15:36:58 -04005178
John Fastabendbbbe2112017-09-08 14:00:30 -07005179 preempt_disable();
5180 rcu_read_lock();
5181 ret = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), skb);
5182 rcu_read_unlock();
5183 preempt_enable();
5184
5185 if (ret != XDP_PASS)
John Fastabendd4455162017-07-17 09:26:45 -07005186 return NET_RX_DROP;
David S. Millerb5cdae32017-04-18 15:36:58 -04005187 }
5188
John Fastabendbbbe2112017-09-08 14:00:30 -07005189 rcu_read_lock();
Eric Dumazetdf334542010-03-24 19:13:54 +00005190#ifdef CONFIG_RPS
Ingo Molnarc5905af2012-02-24 08:31:31 +01005191 if (static_key_false(&rps_needed)) {
Eric Dumazet3b098e22010-05-15 23:57:10 -07005192 struct rps_dev_flow voidflow, *rflow = &voidflow;
Julian Anastasov2c17d272015-07-09 09:59:10 +03005193 int cpu = get_rps_cpu(skb->dev, skb, &rflow);
Tom Herbertfec5e652010-04-16 16:01:27 -07005194
Eric Dumazet3b098e22010-05-15 23:57:10 -07005195 if (cpu >= 0) {
5196 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
5197 rcu_read_unlock();
Eric Dumazetadc93002011-11-17 03:13:26 +00005198 return ret;
Eric Dumazet3b098e22010-05-15 23:57:10 -07005199 }
Tom Herbertfec5e652010-04-16 16:01:27 -07005200 }
Tom Herbert1e94d722010-03-18 17:45:44 -07005201#endif
Julian Anastasov2c17d272015-07-09 09:59:10 +03005202 ret = __netif_receive_skb(skb);
5203 rcu_read_unlock();
5204 return ret;
Tom Herbert0a9627f2010-03-16 08:03:29 +00005205}
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00005206
Edward Cree7da517a2018-07-02 16:13:24 +01005207static void netif_receive_skb_list_internal(struct list_head *head)
5208{
5209 struct bpf_prog *xdp_prog = NULL;
5210 struct sk_buff *skb, *next;
Edward Cree8c057ef2018-07-09 18:09:54 +01005211 struct list_head sublist;
Edward Cree7da517a2018-07-02 16:13:24 +01005212
Edward Cree8c057ef2018-07-09 18:09:54 +01005213 INIT_LIST_HEAD(&sublist);
Edward Cree7da517a2018-07-02 16:13:24 +01005214 list_for_each_entry_safe(skb, next, head, list) {
5215 net_timestamp_check(netdev_tstamp_prequeue, skb);
Edward Cree22f6bbb2018-12-04 17:37:57 +00005216 skb_list_del_init(skb);
Edward Cree8c057ef2018-07-09 18:09:54 +01005217 if (!skb_defer_rx_timestamp(skb))
5218 list_add_tail(&skb->list, &sublist);
Edward Cree7da517a2018-07-02 16:13:24 +01005219 }
Edward Cree8c057ef2018-07-09 18:09:54 +01005220 list_splice_init(&sublist, head);
Edward Cree7da517a2018-07-02 16:13:24 +01005221
5222 if (static_branch_unlikely(&generic_xdp_needed_key)) {
5223 preempt_disable();
5224 rcu_read_lock();
5225 list_for_each_entry_safe(skb, next, head, list) {
5226 xdp_prog = rcu_dereference(skb->dev->xdp_prog);
Edward Cree22f6bbb2018-12-04 17:37:57 +00005227 skb_list_del_init(skb);
Edward Cree8c057ef2018-07-09 18:09:54 +01005228 if (do_xdp_generic(xdp_prog, skb) == XDP_PASS)
5229 list_add_tail(&skb->list, &sublist);
Edward Cree7da517a2018-07-02 16:13:24 +01005230 }
5231 rcu_read_unlock();
5232 preempt_enable();
Edward Cree8c057ef2018-07-09 18:09:54 +01005233 /* Put passed packets back on main list */
5234 list_splice_init(&sublist, head);
Edward Cree7da517a2018-07-02 16:13:24 +01005235 }
5236
5237 rcu_read_lock();
5238#ifdef CONFIG_RPS
5239 if (static_key_false(&rps_needed)) {
5240 list_for_each_entry_safe(skb, next, head, list) {
5241 struct rps_dev_flow voidflow, *rflow = &voidflow;
5242 int cpu = get_rps_cpu(skb->dev, skb, &rflow);
5243
5244 if (cpu >= 0) {
Edward Cree8c057ef2018-07-09 18:09:54 +01005245 /* Will be handled, remove from list */
Edward Cree22f6bbb2018-12-04 17:37:57 +00005246 skb_list_del_init(skb);
Edward Cree8c057ef2018-07-09 18:09:54 +01005247 enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
Edward Cree7da517a2018-07-02 16:13:24 +01005248 }
5249 }
5250 }
5251#endif
5252 __netif_receive_skb_list(head);
5253 rcu_read_unlock();
5254}
5255
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00005256/**
5257 * netif_receive_skb - process receive buffer from network
5258 * @skb: buffer to process
5259 *
5260 * netif_receive_skb() is the main receive data processing function.
5261 * It always succeeds. The buffer may be dropped during processing
5262 * for congestion control or by the protocol layers.
5263 *
5264 * This function may only be called from softirq context and interrupts
5265 * should be enabled.
5266 *
5267 * Return values (usually ignored):
5268 * NET_RX_SUCCESS: no congestion
5269 * NET_RX_DROP: packet was dropped
5270 */
Eric W. Biederman04eb4482015-09-15 20:04:15 -05005271int netif_receive_skb(struct sk_buff *skb)
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00005272{
Geneviève Bastienb0e3f1b2018-11-27 12:52:39 -05005273 int ret;
5274
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00005275 trace_netif_receive_skb_entry(skb);
5276
Geneviève Bastienb0e3f1b2018-11-27 12:52:39 -05005277 ret = netif_receive_skb_internal(skb);
5278 trace_netif_receive_skb_exit(ret);
5279
5280 return ret;
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00005281}
Eric W. Biederman04eb4482015-09-15 20:04:15 -05005282EXPORT_SYMBOL(netif_receive_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005283
Edward Creef6ad8c12018-07-02 16:12:45 +01005284/**
5285 * netif_receive_skb_list - process many receive buffers from network
5286 * @head: list of skbs to process.
5287 *
Edward Cree7da517a2018-07-02 16:13:24 +01005288 * Since return value of netif_receive_skb() is normally ignored, and
5289 * wouldn't be meaningful for a list, this function returns void.
Edward Creef6ad8c12018-07-02 16:12:45 +01005290 *
5291 * This function may only be called from softirq context and interrupts
5292 * should be enabled.
5293 */
5294void netif_receive_skb_list(struct list_head *head)
5295{
Edward Cree7da517a2018-07-02 16:13:24 +01005296 struct sk_buff *skb;
Edward Creef6ad8c12018-07-02 16:12:45 +01005297
Edward Creeb9f463d2018-07-02 16:14:44 +01005298 if (list_empty(head))
5299 return;
Geneviève Bastienb0e3f1b2018-11-27 12:52:39 -05005300 if (trace_netif_receive_skb_list_entry_enabled()) {
5301 list_for_each_entry(skb, head, list)
5302 trace_netif_receive_skb_list_entry(skb);
5303 }
Edward Cree7da517a2018-07-02 16:13:24 +01005304 netif_receive_skb_list_internal(head);
Geneviève Bastienb0e3f1b2018-11-27 12:52:39 -05005305 trace_netif_receive_skb_list_exit(0);
Edward Creef6ad8c12018-07-02 16:12:45 +01005306}
5307EXPORT_SYMBOL(netif_receive_skb_list);
5308
Eric Dumazet41852492016-08-26 12:50:39 -07005309DEFINE_PER_CPU(struct work_struct, flush_works);
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005310
5311/* Network device is going away, flush any packets still pending */
5312static void flush_backlog(struct work_struct *work)
5313{
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005314 struct sk_buff *skb, *tmp;
5315 struct softnet_data *sd;
5316
5317 local_bh_disable();
5318 sd = this_cpu_ptr(&softnet_data);
5319
5320 local_irq_disable();
Eric Dumazete36fa2f2010-04-19 21:17:14 +00005321 rps_lock(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07005322 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
Eric Dumazet41852492016-08-26 12:50:39 -07005323 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
Eric Dumazete36fa2f2010-04-19 21:17:14 +00005324 __skb_unlink(skb, &sd->input_pkt_queue);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07005325 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00005326 input_queue_head_incr(sd);
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07005327 }
Changli Gao6e7676c2010-04-27 15:07:33 -07005328 }
Eric Dumazete36fa2f2010-04-19 21:17:14 +00005329 rps_unlock(sd);
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005330 local_irq_enable();
Changli Gao6e7676c2010-04-27 15:07:33 -07005331
5332 skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
Eric Dumazet41852492016-08-26 12:50:39 -07005333 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
Changli Gao6e7676c2010-04-27 15:07:33 -07005334 __skb_unlink(skb, &sd->process_queue);
5335 kfree_skb(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00005336 input_queue_head_incr(sd);
Changli Gao6e7676c2010-04-27 15:07:33 -07005337 }
5338 }
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005339 local_bh_enable();
5340}
5341
Eric Dumazet41852492016-08-26 12:50:39 -07005342static void flush_all_backlogs(void)
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005343{
5344 unsigned int cpu;
5345
5346 get_online_cpus();
5347
Eric Dumazet41852492016-08-26 12:50:39 -07005348 for_each_online_cpu(cpu)
5349 queue_work_on(cpu, system_highpri_wq,
5350 per_cpu_ptr(&flush_works, cpu));
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005351
5352 for_each_online_cpu(cpu)
Eric Dumazet41852492016-08-26 12:50:39 -07005353 flush_work(per_cpu_ptr(&flush_works, cpu));
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005354
5355 put_online_cpus();
Stephen Hemminger6e583ce2008-08-03 21:29:57 -07005356}
5357
Paolo Abeniaaa5d902018-12-14 11:51:58 +01005358INDIRECT_CALLABLE_DECLARE(int inet_gro_complete(struct sk_buff *, int));
5359INDIRECT_CALLABLE_DECLARE(int ipv6_gro_complete(struct sk_buff *, int));
Herbert Xud565b0a2008-12-15 23:38:52 -08005360static int napi_gro_complete(struct sk_buff *skb)
5361{
Vlad Yasevich22061d82012-11-15 08:49:11 +00005362 struct packet_offload *ptype;
Herbert Xud565b0a2008-12-15 23:38:52 -08005363 __be16 type = skb->protocol;
Vlad Yasevich22061d82012-11-15 08:49:11 +00005364 struct list_head *head = &offload_base;
Herbert Xud565b0a2008-12-15 23:38:52 -08005365 int err = -ENOENT;
5366
Eric Dumazetc3c7c252012-12-06 13:54:59 +00005367 BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
5368
Herbert Xufc59f9a2009-04-14 15:11:06 -07005369 if (NAPI_GRO_CB(skb)->count == 1) {
5370 skb_shinfo(skb)->gso_size = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08005371 goto out;
Herbert Xufc59f9a2009-04-14 15:11:06 -07005372 }
Herbert Xud565b0a2008-12-15 23:38:52 -08005373
5374 rcu_read_lock();
5375 list_for_each_entry_rcu(ptype, head, list) {
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00005376 if (ptype->type != type || !ptype->callbacks.gro_complete)
Herbert Xud565b0a2008-12-15 23:38:52 -08005377 continue;
5378
Paolo Abeniaaa5d902018-12-14 11:51:58 +01005379 err = INDIRECT_CALL_INET(ptype->callbacks.gro_complete,
5380 ipv6_gro_complete, inet_gro_complete,
5381 skb, 0);
Herbert Xud565b0a2008-12-15 23:38:52 -08005382 break;
5383 }
5384 rcu_read_unlock();
5385
5386 if (err) {
5387 WARN_ON(&ptype->list == head);
5388 kfree_skb(skb);
5389 return NET_RX_SUCCESS;
5390 }
5391
5392out:
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00005393 return netif_receive_skb_internal(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08005394}
5395
Li RongQing6312fe72018-07-05 14:34:32 +08005396static void __napi_gro_flush_chain(struct napi_struct *napi, u32 index,
David Miller07d78362018-06-24 14:14:02 +09005397 bool flush_old)
Herbert Xud565b0a2008-12-15 23:38:52 -08005398{
Li RongQing6312fe72018-07-05 14:34:32 +08005399 struct list_head *head = &napi->gro_hash[index].list;
David Millerd4546c22018-06-24 14:13:49 +09005400 struct sk_buff *skb, *p;
Herbert Xud565b0a2008-12-15 23:38:52 -08005401
David Miller07d78362018-06-24 14:14:02 +09005402 list_for_each_entry_safe_reverse(skb, p, head, list) {
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00005403 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
5404 return;
David S. Miller992cba72018-07-31 15:27:56 -07005405 skb_list_del_init(skb);
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00005406 napi_gro_complete(skb);
Li RongQing6312fe72018-07-05 14:34:32 +08005407 napi->gro_hash[index].count--;
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00005408 }
Li RongQingd9f37d02018-07-13 14:41:36 +08005409
5410 if (!napi->gro_hash[index].count)
5411 __clear_bit(index, &napi->gro_bitmask);
Herbert Xud565b0a2008-12-15 23:38:52 -08005412}
David Miller07d78362018-06-24 14:14:02 +09005413
Li RongQing6312fe72018-07-05 14:34:32 +08005414/* napi->gro_hash[].list contains packets ordered by age.
David Miller07d78362018-06-24 14:14:02 +09005415 * youngest packets at the head of it.
5416 * Complete skbs in reverse order to reduce latencies.
5417 */
5418void napi_gro_flush(struct napi_struct *napi, bool flush_old)
5419{
Eric Dumazet42519ed2018-11-21 11:39:28 -08005420 unsigned long bitmask = napi->gro_bitmask;
5421 unsigned int i, base = ~0U;
David Miller07d78362018-06-24 14:14:02 +09005422
Eric Dumazet42519ed2018-11-21 11:39:28 -08005423 while ((i = ffs(bitmask)) != 0) {
5424 bitmask >>= i;
5425 base += i;
5426 __napi_gro_flush_chain(napi, base, flush_old);
Li RongQingd9f37d02018-07-13 14:41:36 +08005427 }
David Miller07d78362018-06-24 14:14:02 +09005428}
Eric Dumazet86cac582010-08-31 18:25:32 +00005429EXPORT_SYMBOL(napi_gro_flush);
Herbert Xud565b0a2008-12-15 23:38:52 -08005430
David Miller07d78362018-06-24 14:14:02 +09005431static struct list_head *gro_list_prepare(struct napi_struct *napi,
5432 struct sk_buff *skb)
Eric Dumazet89c5fa32012-12-10 13:28:16 +00005433{
Eric Dumazet89c5fa32012-12-10 13:28:16 +00005434 unsigned int maclen = skb->dev->hard_header_len;
Tom Herbert0b4cec82014-01-15 08:58:06 -08005435 u32 hash = skb_get_hash_raw(skb);
David Miller07d78362018-06-24 14:14:02 +09005436 struct list_head *head;
David Millerd4546c22018-06-24 14:13:49 +09005437 struct sk_buff *p;
Eric Dumazet89c5fa32012-12-10 13:28:16 +00005438
Li RongQing6312fe72018-07-05 14:34:32 +08005439 head = &napi->gro_hash[hash & (GRO_HASH_BUCKETS - 1)].list;
David Miller07d78362018-06-24 14:14:02 +09005440 list_for_each_entry(p, head, list) {
Eric Dumazet89c5fa32012-12-10 13:28:16 +00005441 unsigned long diffs;
5442
Tom Herbert0b4cec82014-01-15 08:58:06 -08005443 NAPI_GRO_CB(p)->flush = 0;
5444
5445 if (hash != skb_get_hash_raw(p)) {
5446 NAPI_GRO_CB(p)->same_flow = 0;
5447 continue;
5448 }
5449
Eric Dumazet89c5fa32012-12-10 13:28:16 +00005450 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
Michał Mirosławb18175242018-11-09 00:18:02 +01005451 diffs |= skb_vlan_tag_present(p) ^ skb_vlan_tag_present(skb);
5452 if (skb_vlan_tag_present(p))
5453 diffs |= p->vlan_tci ^ skb->vlan_tci;
Jesse Grossce87fc62016-01-20 17:59:49 -08005454 diffs |= skb_metadata_dst_cmp(p, skb);
Daniel Borkmannde8f3a82017-09-25 02:25:51 +02005455 diffs |= skb_metadata_differs(p, skb);
Eric Dumazet89c5fa32012-12-10 13:28:16 +00005456 if (maclen == ETH_HLEN)
5457 diffs |= compare_ether_header(skb_mac_header(p),
Eric Dumazeta50e2332014-03-29 21:28:21 -07005458 skb_mac_header(skb));
Eric Dumazet89c5fa32012-12-10 13:28:16 +00005459 else if (!diffs)
5460 diffs = memcmp(skb_mac_header(p),
Eric Dumazeta50e2332014-03-29 21:28:21 -07005461 skb_mac_header(skb),
Eric Dumazet89c5fa32012-12-10 13:28:16 +00005462 maclen);
5463 NAPI_GRO_CB(p)->same_flow = !diffs;
Eric Dumazet89c5fa32012-12-10 13:28:16 +00005464 }
David Miller07d78362018-06-24 14:14:02 +09005465
5466 return head;
Eric Dumazet89c5fa32012-12-10 13:28:16 +00005467}
5468
Jerry Chu299603e82013-12-11 20:53:45 -08005469static void skb_gro_reset_offset(struct sk_buff *skb)
5470{
5471 const struct skb_shared_info *pinfo = skb_shinfo(skb);
5472 const skb_frag_t *frag0 = &pinfo->frags[0];
5473
5474 NAPI_GRO_CB(skb)->data_offset = 0;
5475 NAPI_GRO_CB(skb)->frag0 = NULL;
5476 NAPI_GRO_CB(skb)->frag0_len = 0;
5477
5478 if (skb_mac_header(skb) == skb_tail_pointer(skb) &&
5479 pinfo->nr_frags &&
5480 !PageHighMem(skb_frag_page(frag0))) {
5481 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
Eric Dumazet7cfd5fd2017-01-10 19:52:43 -08005482 NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int,
5483 skb_frag_size(frag0),
5484 skb->end - skb->tail);
Herbert Xud565b0a2008-12-15 23:38:52 -08005485 }
5486}
5487
Eric Dumazeta50e2332014-03-29 21:28:21 -07005488static void gro_pull_from_frag0(struct sk_buff *skb, int grow)
5489{
5490 struct skb_shared_info *pinfo = skb_shinfo(skb);
5491
5492 BUG_ON(skb->end - skb->tail < grow);
5493
5494 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
5495
5496 skb->data_len -= grow;
5497 skb->tail += grow;
5498
5499 pinfo->frags[0].page_offset += grow;
5500 skb_frag_size_sub(&pinfo->frags[0], grow);
5501
5502 if (unlikely(!skb_frag_size(&pinfo->frags[0]))) {
5503 skb_frag_unref(skb, 0);
5504 memmove(pinfo->frags, pinfo->frags + 1,
5505 --pinfo->nr_frags * sizeof(pinfo->frags[0]));
5506 }
5507}
5508
Li RongQing6312fe72018-07-05 14:34:32 +08005509static void gro_flush_oldest(struct list_head *head)
David Miller07d78362018-06-24 14:14:02 +09005510{
Li RongQing6312fe72018-07-05 14:34:32 +08005511 struct sk_buff *oldest;
David Miller07d78362018-06-24 14:14:02 +09005512
Li RongQing6312fe72018-07-05 14:34:32 +08005513 oldest = list_last_entry(head, struct sk_buff, list);
David Miller07d78362018-06-24 14:14:02 +09005514
Li RongQing6312fe72018-07-05 14:34:32 +08005515 /* We are called with head length >= MAX_GRO_SKBS, so this is
David Miller07d78362018-06-24 14:14:02 +09005516 * impossible.
5517 */
5518 if (WARN_ON_ONCE(!oldest))
5519 return;
5520
Li RongQingd9f37d02018-07-13 14:41:36 +08005521 /* Do not adjust napi->gro_hash[].count, caller is adding a new
5522 * SKB to the chain.
David Miller07d78362018-06-24 14:14:02 +09005523 */
David S. Millerece23712018-10-28 10:35:12 -07005524 skb_list_del_init(oldest);
David Miller07d78362018-06-24 14:14:02 +09005525 napi_gro_complete(oldest);
5526}
5527
Paolo Abeniaaa5d902018-12-14 11:51:58 +01005528INDIRECT_CALLABLE_DECLARE(struct sk_buff *inet_gro_receive(struct list_head *,
5529 struct sk_buff *));
5530INDIRECT_CALLABLE_DECLARE(struct sk_buff *ipv6_gro_receive(struct list_head *,
5531 struct sk_buff *));
Rami Rosenbb728822012-11-28 21:55:25 +00005532static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xud565b0a2008-12-15 23:38:52 -08005533{
Li RongQing6312fe72018-07-05 14:34:32 +08005534 u32 hash = skb_get_hash_raw(skb) & (GRO_HASH_BUCKETS - 1);
David Millerd4546c22018-06-24 14:13:49 +09005535 struct list_head *head = &offload_base;
Vlad Yasevich22061d82012-11-15 08:49:11 +00005536 struct packet_offload *ptype;
Herbert Xud565b0a2008-12-15 23:38:52 -08005537 __be16 type = skb->protocol;
David Miller07d78362018-06-24 14:14:02 +09005538 struct list_head *gro_head;
David Millerd4546c22018-06-24 14:13:49 +09005539 struct sk_buff *pp = NULL;
Ben Hutchings5b252f02009-10-29 07:17:09 +00005540 enum gro_result ret;
David Millerd4546c22018-06-24 14:13:49 +09005541 int same_flow;
Eric Dumazeta50e2332014-03-29 21:28:21 -07005542 int grow;
Herbert Xud565b0a2008-12-15 23:38:52 -08005543
David S. Millerb5cdae32017-04-18 15:36:58 -04005544 if (netif_elide_gro(skb->dev))
Herbert Xud565b0a2008-12-15 23:38:52 -08005545 goto normal;
5546
David Miller07d78362018-06-24 14:14:02 +09005547 gro_head = gro_list_prepare(napi, skb);
Eric Dumazet89c5fa32012-12-10 13:28:16 +00005548
Herbert Xud565b0a2008-12-15 23:38:52 -08005549 rcu_read_lock();
5550 list_for_each_entry_rcu(ptype, head, list) {
Vlad Yasevichf191a1d2012-11-15 08:49:23 +00005551 if (ptype->type != type || !ptype->callbacks.gro_receive)
Herbert Xud565b0a2008-12-15 23:38:52 -08005552 continue;
5553
Herbert Xu86911732009-01-29 14:19:50 +00005554 skb_set_network_header(skb, skb_gro_offset(skb));
Eric Dumazetefd94502013-02-14 17:31:48 +00005555 skb_reset_mac_len(skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08005556 NAPI_GRO_CB(skb)->same_flow = 0;
Eric Dumazetd61d0722016-11-07 11:12:27 -08005557 NAPI_GRO_CB(skb)->flush = skb_is_gso(skb) || skb_has_frag_list(skb);
Herbert Xu5d38a072009-01-04 16:13:40 -08005558 NAPI_GRO_CB(skb)->free = 0;
Jesse Grossfac8e0f2016-03-19 09:32:01 -07005559 NAPI_GRO_CB(skb)->encap_mark = 0;
Sabrina Dubrocafcd91dd2016-10-20 15:58:02 +02005560 NAPI_GRO_CB(skb)->recursion_counter = 0;
Alexander Duycka0ca1532016-04-05 09:13:39 -07005561 NAPI_GRO_CB(skb)->is_fou = 0;
Alexander Duyck15305452016-04-10 21:44:57 -04005562 NAPI_GRO_CB(skb)->is_atomic = 1;
Tom Herbert15e23962015-02-10 16:30:31 -08005563 NAPI_GRO_CB(skb)->gro_remcsum_start = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08005564
Tom Herbert662880f2014-08-27 21:26:56 -07005565 /* Setup for GRO checksum validation */
5566 switch (skb->ip_summed) {
5567 case CHECKSUM_COMPLETE:
5568 NAPI_GRO_CB(skb)->csum = skb->csum;
5569 NAPI_GRO_CB(skb)->csum_valid = 1;
5570 NAPI_GRO_CB(skb)->csum_cnt = 0;
5571 break;
5572 case CHECKSUM_UNNECESSARY:
5573 NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1;
5574 NAPI_GRO_CB(skb)->csum_valid = 0;
5575 break;
5576 default:
5577 NAPI_GRO_CB(skb)->csum_cnt = 0;
5578 NAPI_GRO_CB(skb)->csum_valid = 0;
5579 }
Herbert Xud565b0a2008-12-15 23:38:52 -08005580
Paolo Abeniaaa5d902018-12-14 11:51:58 +01005581 pp = INDIRECT_CALL_INET(ptype->callbacks.gro_receive,
5582 ipv6_gro_receive, inet_gro_receive,
5583 gro_head, skb);
Herbert Xud565b0a2008-12-15 23:38:52 -08005584 break;
5585 }
5586 rcu_read_unlock();
5587
5588 if (&ptype->list == head)
5589 goto normal;
5590
Steffen Klassert25393d32017-02-15 09:39:44 +01005591 if (IS_ERR(pp) && PTR_ERR(pp) == -EINPROGRESS) {
5592 ret = GRO_CONSUMED;
5593 goto ok;
5594 }
5595
Herbert Xu0da2afd52008-12-26 14:57:42 -08005596 same_flow = NAPI_GRO_CB(skb)->same_flow;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00005597 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
Herbert Xu0da2afd52008-12-26 14:57:42 -08005598
Herbert Xud565b0a2008-12-15 23:38:52 -08005599 if (pp) {
David S. Miller992cba72018-07-31 15:27:56 -07005600 skb_list_del_init(pp);
David Millerd4546c22018-06-24 14:13:49 +09005601 napi_gro_complete(pp);
Li RongQing6312fe72018-07-05 14:34:32 +08005602 napi->gro_hash[hash].count--;
Herbert Xud565b0a2008-12-15 23:38:52 -08005603 }
5604
Herbert Xu0da2afd52008-12-26 14:57:42 -08005605 if (same_flow)
Herbert Xud565b0a2008-12-15 23:38:52 -08005606 goto ok;
5607
Eric Dumazet600adc12014-01-09 14:12:19 -08005608 if (NAPI_GRO_CB(skb)->flush)
Herbert Xud565b0a2008-12-15 23:38:52 -08005609 goto normal;
Herbert Xud565b0a2008-12-15 23:38:52 -08005610
Li RongQing6312fe72018-07-05 14:34:32 +08005611 if (unlikely(napi->gro_hash[hash].count >= MAX_GRO_SKBS)) {
5612 gro_flush_oldest(gro_head);
Eric Dumazet600adc12014-01-09 14:12:19 -08005613 } else {
Li RongQing6312fe72018-07-05 14:34:32 +08005614 napi->gro_hash[hash].count++;
Eric Dumazet600adc12014-01-09 14:12:19 -08005615 }
Herbert Xud565b0a2008-12-15 23:38:52 -08005616 NAPI_GRO_CB(skb)->count = 1;
Eric Dumazet2e71a6f2012-10-06 08:08:49 +00005617 NAPI_GRO_CB(skb)->age = jiffies;
Eric Dumazet29e98242014-05-16 11:34:37 -07005618 NAPI_GRO_CB(skb)->last = skb;
Herbert Xu86911732009-01-29 14:19:50 +00005619 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
David Miller07d78362018-06-24 14:14:02 +09005620 list_add(&skb->list, gro_head);
Herbert Xu5d0d9be2009-01-29 14:19:48 +00005621 ret = GRO_HELD;
Herbert Xud565b0a2008-12-15 23:38:52 -08005622
Herbert Xuad0f99042009-02-01 01:24:55 -08005623pull:
Eric Dumazeta50e2332014-03-29 21:28:21 -07005624 grow = skb_gro_offset(skb) - skb_headlen(skb);
5625 if (grow > 0)
5626 gro_pull_from_frag0(skb, grow);
Herbert Xud565b0a2008-12-15 23:38:52 -08005627ok:
Li RongQingd9f37d02018-07-13 14:41:36 +08005628 if (napi->gro_hash[hash].count) {
5629 if (!test_bit(hash, &napi->gro_bitmask))
5630 __set_bit(hash, &napi->gro_bitmask);
5631 } else if (test_bit(hash, &napi->gro_bitmask)) {
5632 __clear_bit(hash, &napi->gro_bitmask);
5633 }
5634
Herbert Xu5d0d9be2009-01-29 14:19:48 +00005635 return ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08005636
5637normal:
Herbert Xuad0f99042009-02-01 01:24:55 -08005638 ret = GRO_NORMAL;
5639 goto pull;
Herbert Xu5d38a072009-01-04 16:13:40 -08005640}
Herbert Xu96e93ea2009-01-06 10:49:34 -08005641
Jerry Chubf5a7552014-01-07 10:23:19 -08005642struct packet_offload *gro_find_receive_by_type(__be16 type)
5643{
5644 struct list_head *offload_head = &offload_base;
5645 struct packet_offload *ptype;
5646
5647 list_for_each_entry_rcu(ptype, offload_head, list) {
5648 if (ptype->type != type || !ptype->callbacks.gro_receive)
5649 continue;
5650 return ptype;
5651 }
5652 return NULL;
5653}
Or Gerlitze27a2f82014-01-20 13:59:20 +02005654EXPORT_SYMBOL(gro_find_receive_by_type);
Jerry Chubf5a7552014-01-07 10:23:19 -08005655
5656struct packet_offload *gro_find_complete_by_type(__be16 type)
5657{
5658 struct list_head *offload_head = &offload_base;
5659 struct packet_offload *ptype;
5660
5661 list_for_each_entry_rcu(ptype, offload_head, list) {
5662 if (ptype->type != type || !ptype->callbacks.gro_complete)
5663 continue;
5664 return ptype;
5665 }
5666 return NULL;
5667}
Or Gerlitze27a2f82014-01-20 13:59:20 +02005668EXPORT_SYMBOL(gro_find_complete_by_type);
Herbert Xu96e93ea2009-01-06 10:49:34 -08005669
Michal Kubečeke44699d2017-06-29 11:13:36 +02005670static void napi_skb_free_stolen_head(struct sk_buff *skb)
5671{
5672 skb_dst_drop(skb);
5673 secpath_reset(skb);
5674 kmem_cache_free(skbuff_head_cache, skb);
5675}
5676
Rami Rosenbb728822012-11-28 21:55:25 +00005677static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
Herbert Xu5d38a072009-01-04 16:13:40 -08005678{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00005679 switch (ret) {
5680 case GRO_NORMAL:
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00005681 if (netif_receive_skb_internal(skb))
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07005682 ret = GRO_DROP;
5683 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08005684
Herbert Xu5d0d9be2009-01-29 14:19:48 +00005685 case GRO_DROP:
Herbert Xu5d38a072009-01-04 16:13:40 -08005686 kfree_skb(skb);
5687 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00005688
Eric Dumazetdaa86542012-04-19 07:07:40 +00005689 case GRO_MERGED_FREE:
Michal Kubečeke44699d2017-06-29 11:13:36 +02005690 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
5691 napi_skb_free_stolen_head(skb);
5692 else
Eric Dumazetd7e88832012-04-30 08:10:34 +00005693 __kfree_skb(skb);
Eric Dumazetdaa86542012-04-19 07:07:40 +00005694 break;
5695
Ben Hutchings5b252f02009-10-29 07:17:09 +00005696 case GRO_HELD:
5697 case GRO_MERGED:
Steffen Klassert25393d32017-02-15 09:39:44 +01005698 case GRO_CONSUMED:
Ben Hutchings5b252f02009-10-29 07:17:09 +00005699 break;
Herbert Xu5d38a072009-01-04 16:13:40 -08005700 }
5701
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07005702 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00005703}
Herbert Xu5d0d9be2009-01-29 14:19:48 +00005704
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07005705gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00005706{
Geneviève Bastienb0e3f1b2018-11-27 12:52:39 -05005707 gro_result_t ret;
5708
Eric Dumazet93f93a42015-11-18 06:30:59 -08005709 skb_mark_napi_id(skb, napi);
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00005710 trace_napi_gro_receive_entry(skb);
Herbert Xu86911732009-01-29 14:19:50 +00005711
Eric Dumazeta50e2332014-03-29 21:28:21 -07005712 skb_gro_reset_offset(skb);
5713
Geneviève Bastienb0e3f1b2018-11-27 12:52:39 -05005714 ret = napi_skb_finish(dev_gro_receive(napi, skb), skb);
5715 trace_napi_gro_receive_exit(ret);
5716
5717 return ret;
Herbert Xud565b0a2008-12-15 23:38:52 -08005718}
5719EXPORT_SYMBOL(napi_gro_receive);
5720
stephen hemmingerd0c2b0d2010-10-19 07:12:10 +00005721static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
Herbert Xu96e93ea2009-01-06 10:49:34 -08005722{
Eric Dumazet93a35f52014-10-23 06:30:30 -07005723 if (unlikely(skb->pfmemalloc)) {
5724 consume_skb(skb);
5725 return;
5726 }
Herbert Xu96e93ea2009-01-06 10:49:34 -08005727 __skb_pull(skb, skb_headlen(skb));
Eric Dumazet2a2a4592012-03-21 06:58:03 +00005728 /* restore the reserve we had after netdev_alloc_skb_ip_align() */
5729 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
Michał Mirosławb18175242018-11-09 00:18:02 +01005730 __vlan_hwaccel_clear_tag(skb);
Herbert Xu66c46d72011-01-29 20:44:54 -08005731 skb->dev = napi->dev;
Andy Gospodarek6d152e22011-02-02 14:53:25 -08005732 skb->skb_iif = 0;
Eric Dumazet33d9a2c2018-11-17 21:57:02 -08005733
5734 /* eth_type_trans() assumes pkt_type is PACKET_HOST */
5735 skb->pkt_type = PACKET_HOST;
5736
Jerry Chuc3caf112014-07-14 15:54:46 -07005737 skb->encapsulation = 0;
5738 skb_shinfo(skb)->gso_type = 0;
Eric Dumazete33d0ba2014-04-03 09:28:10 -07005739 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
Steffen Klassertf991bb92017-01-30 06:45:38 +01005740 secpath_reset(skb);
Herbert Xu96e93ea2009-01-06 10:49:34 -08005741
5742 napi->skb = skb;
5743}
Herbert Xu96e93ea2009-01-06 10:49:34 -08005744
Herbert Xu76620aa2009-04-16 02:02:07 -07005745struct sk_buff *napi_get_frags(struct napi_struct *napi)
Herbert Xu5d38a072009-01-04 16:13:40 -08005746{
Herbert Xu5d38a072009-01-04 16:13:40 -08005747 struct sk_buff *skb = napi->skb;
Herbert Xu5d38a072009-01-04 16:13:40 -08005748
5749 if (!skb) {
Alexander Duyckfd11a832014-12-09 19:40:49 -08005750 skb = napi_alloc_skb(napi, GRO_MAX_HEAD);
Eric Dumazete2f9dc32015-11-19 12:11:23 -08005751 if (skb) {
5752 napi->skb = skb;
5753 skb_mark_napi_id(skb, napi);
5754 }
Herbert Xu5d38a072009-01-04 16:13:40 -08005755 }
Herbert Xu96e93ea2009-01-06 10:49:34 -08005756 return skb;
5757}
Herbert Xu76620aa2009-04-16 02:02:07 -07005758EXPORT_SYMBOL(napi_get_frags);
Herbert Xu96e93ea2009-01-06 10:49:34 -08005759
Eric Dumazeta50e2332014-03-29 21:28:21 -07005760static gro_result_t napi_frags_finish(struct napi_struct *napi,
5761 struct sk_buff *skb,
5762 gro_result_t ret)
Herbert Xu5d0d9be2009-01-29 14:19:48 +00005763{
Herbert Xu5d0d9be2009-01-29 14:19:48 +00005764 switch (ret) {
5765 case GRO_NORMAL:
Eric Dumazeta50e2332014-03-29 21:28:21 -07005766 case GRO_HELD:
5767 __skb_push(skb, ETH_HLEN);
5768 skb->protocol = eth_type_trans(skb, skb->dev);
5769 if (ret == GRO_NORMAL && netif_receive_skb_internal(skb))
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07005770 ret = GRO_DROP;
Herbert Xu86911732009-01-29 14:19:50 +00005771 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00005772
5773 case GRO_DROP:
Herbert Xu5d0d9be2009-01-29 14:19:48 +00005774 napi_reuse_skb(napi, skb);
5775 break;
Ben Hutchings5b252f02009-10-29 07:17:09 +00005776
Michal Kubečeke44699d2017-06-29 11:13:36 +02005777 case GRO_MERGED_FREE:
5778 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
5779 napi_skb_free_stolen_head(skb);
5780 else
5781 napi_reuse_skb(napi, skb);
5782 break;
5783
Ben Hutchings5b252f02009-10-29 07:17:09 +00005784 case GRO_MERGED:
Steffen Klassert25393d32017-02-15 09:39:44 +01005785 case GRO_CONSUMED:
Ben Hutchings5b252f02009-10-29 07:17:09 +00005786 break;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00005787 }
5788
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07005789 return ret;
Herbert Xu5d0d9be2009-01-29 14:19:48 +00005790}
Herbert Xu5d0d9be2009-01-29 14:19:48 +00005791
Eric Dumazeta50e2332014-03-29 21:28:21 -07005792/* Upper GRO stack assumes network header starts at gro_offset=0
5793 * Drivers could call both napi_gro_frags() and napi_gro_receive()
5794 * We copy ethernet header into skb->data to have a common layout.
5795 */
Eric Dumazet4adb9c42012-05-18 20:49:06 +00005796static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
Herbert Xu96e93ea2009-01-06 10:49:34 -08005797{
Herbert Xu76620aa2009-04-16 02:02:07 -07005798 struct sk_buff *skb = napi->skb;
Eric Dumazeta50e2332014-03-29 21:28:21 -07005799 const struct ethhdr *eth;
5800 unsigned int hlen = sizeof(*eth);
Herbert Xu76620aa2009-04-16 02:02:07 -07005801
5802 napi->skb = NULL;
5803
Eric Dumazeta50e2332014-03-29 21:28:21 -07005804 skb_reset_mac_header(skb);
5805 skb_gro_reset_offset(skb);
5806
5807 eth = skb_gro_header_fast(skb, 0);
5808 if (unlikely(skb_gro_header_hard(skb, hlen))) {
5809 eth = skb_gro_header_slow(skb, hlen, 0);
5810 if (unlikely(!eth)) {
Aaron Conole4da46ce2016-04-02 15:26:43 -04005811 net_warn_ratelimited("%s: dropping impossible skb from %s\n",
5812 __func__, napi->dev->name);
Eric Dumazeta50e2332014-03-29 21:28:21 -07005813 napi_reuse_skb(napi, skb);
5814 return NULL;
5815 }
5816 } else {
5817 gro_pull_from_frag0(skb, hlen);
5818 NAPI_GRO_CB(skb)->frag0 += hlen;
5819 NAPI_GRO_CB(skb)->frag0_len -= hlen;
Herbert Xu76620aa2009-04-16 02:02:07 -07005820 }
Eric Dumazeta50e2332014-03-29 21:28:21 -07005821 __skb_pull(skb, hlen);
5822
5823 /*
5824 * This works because the only protocols we care about don't require
5825 * special handling.
5826 * We'll fix it up properly in napi_frags_finish()
5827 */
5828 skb->protocol = eth->h_proto;
Herbert Xu76620aa2009-04-16 02:02:07 -07005829
Herbert Xu76620aa2009-04-16 02:02:07 -07005830 return skb;
5831}
Herbert Xu76620aa2009-04-16 02:02:07 -07005832
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07005833gro_result_t napi_gro_frags(struct napi_struct *napi)
Herbert Xu76620aa2009-04-16 02:02:07 -07005834{
Geneviève Bastienb0e3f1b2018-11-27 12:52:39 -05005835 gro_result_t ret;
Herbert Xu76620aa2009-04-16 02:02:07 -07005836 struct sk_buff *skb = napi_frags_skb(napi);
Herbert Xu96e93ea2009-01-06 10:49:34 -08005837
5838 if (!skb)
Ben Hutchingsc7c4b3b2009-10-29 21:36:53 -07005839 return GRO_DROP;
Herbert Xu96e93ea2009-01-06 10:49:34 -08005840
Ben Hutchingsae78dbf2014-01-10 22:17:24 +00005841 trace_napi_gro_frags_entry(skb);
5842
Geneviève Bastienb0e3f1b2018-11-27 12:52:39 -05005843 ret = napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
5844 trace_napi_gro_frags_exit(ret);
5845
5846 return ret;
Herbert Xu5d38a072009-01-04 16:13:40 -08005847}
5848EXPORT_SYMBOL(napi_gro_frags);
5849
Tom Herbert573e8fc2014-08-22 13:33:47 -07005850/* Compute the checksum from gro_offset and return the folded value
5851 * after adding in any pseudo checksum.
5852 */
5853__sum16 __skb_gro_checksum_complete(struct sk_buff *skb)
5854{
5855 __wsum wsum;
5856 __sum16 sum;
5857
5858 wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0);
5859
5860 /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */
5861 sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum));
Cong Wang14641932018-11-26 09:31:26 -08005862 /* See comments in __skb_checksum_complete(). */
Tom Herbert573e8fc2014-08-22 13:33:47 -07005863 if (likely(!sum)) {
5864 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
5865 !skb->csum_complete_sw)
Cong Wang7fe50ac2018-11-12 14:47:18 -08005866 netdev_rx_csum_fault(skb->dev, skb);
Tom Herbert573e8fc2014-08-22 13:33:47 -07005867 }
5868
5869 NAPI_GRO_CB(skb)->csum = wsum;
5870 NAPI_GRO_CB(skb)->csum_valid = 1;
5871
5872 return sum;
5873}
5874EXPORT_SYMBOL(__skb_gro_checksum_complete);
5875
ashwanth@codeaurora.org773fc8f2017-06-09 14:24:58 +05305876static void net_rps_send_ipi(struct softnet_data *remsd)
5877{
5878#ifdef CONFIG_RPS
5879 while (remsd) {
5880 struct softnet_data *next = remsd->rps_ipi_next;
5881
5882 if (cpu_online(remsd->cpu))
5883 smp_call_function_single_async(remsd->cpu, &remsd->csd);
5884 remsd = next;
5885 }
5886#endif
5887}
5888
Eric Dumazete326bed2010-04-22 00:22:45 -07005889/*
Zhi Yong Wu855abcf2014-01-01 04:34:50 +08005890 * net_rps_action_and_irq_enable sends any pending IPI's for rps.
Eric Dumazete326bed2010-04-22 00:22:45 -07005891 * Note: called with local irq disabled, but exits with local irq enabled.
5892 */
5893static void net_rps_action_and_irq_enable(struct softnet_data *sd)
5894{
5895#ifdef CONFIG_RPS
5896 struct softnet_data *remsd = sd->rps_ipi_list;
5897
5898 if (remsd) {
5899 sd->rps_ipi_list = NULL;
5900
5901 local_irq_enable();
5902
5903 /* Send pending IPI's to kick RPS processing on remote cpus. */
ashwanth@codeaurora.org773fc8f2017-06-09 14:24:58 +05305904 net_rps_send_ipi(remsd);
Eric Dumazete326bed2010-04-22 00:22:45 -07005905 } else
5906#endif
5907 local_irq_enable();
5908}
5909
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08005910static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
5911{
5912#ifdef CONFIG_RPS
5913 return sd->rps_ipi_list != NULL;
5914#else
5915 return false;
5916#endif
5917}
5918
Stephen Hemmingerbea33482007-10-03 16:41:36 -07005919static int process_backlog(struct napi_struct *napi, int quota)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005920{
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07005921 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005922 bool again = true;
5923 int work = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005924
Eric Dumazete326bed2010-04-22 00:22:45 -07005925 /* Check if we have pending ipi, its better to send them now,
5926 * not waiting net_rx_action() end.
5927 */
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08005928 if (sd_has_rps_ipi_waiting(sd)) {
Eric Dumazete326bed2010-04-22 00:22:45 -07005929 local_irq_disable();
5930 net_rps_action_and_irq_enable(sd);
5931 }
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08005932
Matthias Tafelmeier3d48b532016-12-29 21:37:21 +01005933 napi->weight = dev_rx_weight;
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005934 while (again) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07005935 struct sk_buff *skb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005936
Changli Gao6e7676c2010-04-27 15:07:33 -07005937 while ((skb = __skb_dequeue(&sd->process_queue))) {
Julian Anastasov2c17d272015-07-09 09:59:10 +03005938 rcu_read_lock();
Changli Gao6e7676c2010-04-27 15:07:33 -07005939 __netif_receive_skb(skb);
Julian Anastasov2c17d272015-07-09 09:59:10 +03005940 rcu_read_unlock();
Tom Herbert76cc8b12010-05-20 18:37:59 +00005941 input_queue_head_incr(sd);
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005942 if (++work >= quota)
Tom Herbert76cc8b12010-05-20 18:37:59 +00005943 return work;
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005944
Stephen Hemmingerbea33482007-10-03 16:41:36 -07005945 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005946
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005947 local_irq_disable();
Changli Gao6e7676c2010-04-27 15:07:33 -07005948 rps_lock(sd);
Tom Herbert11ef7a82014-06-30 09:50:40 -07005949 if (skb_queue_empty(&sd->input_pkt_queue)) {
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07005950 /*
5951 * Inline a custom version of __napi_complete().
5952 * only current cpu owns and manipulates this napi,
Tom Herbert11ef7a82014-06-30 09:50:40 -07005953 * and NAPI_STATE_SCHED is the only possible flag set
5954 * on backlog.
5955 * We can use a plain write instead of clear_bit(),
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07005956 * and we dont need an smp_mb() memory barrier.
5957 */
Eric Dumazeteecfd7c2010-05-06 22:07:48 -07005958 napi->state = 0;
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005959 again = false;
5960 } else {
5961 skb_queue_splice_tail_init(&sd->input_pkt_queue,
5962 &sd->process_queue);
Changli Gao6e7676c2010-04-27 15:07:33 -07005963 }
5964 rps_unlock(sd);
Paolo Abeni145dd5f2016-08-25 15:58:44 +02005965 local_irq_enable();
Changli Gao6e7676c2010-04-27 15:07:33 -07005966 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005967
Stephen Hemmingerbea33482007-10-03 16:41:36 -07005968 return work;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005969}
5970
Stephen Hemmingerbea33482007-10-03 16:41:36 -07005971/**
5972 * __napi_schedule - schedule for receive
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07005973 * @n: entry to schedule
Stephen Hemmingerbea33482007-10-03 16:41:36 -07005974 *
Eric Dumazetbc9ad162014-10-28 18:05:13 -07005975 * The entry's receive function will be scheduled to run.
5976 * Consider using __napi_schedule_irqoff() if hard irqs are masked.
Stephen Hemmingerbea33482007-10-03 16:41:36 -07005977 */
Harvey Harrisonb5606c22008-02-13 15:03:16 -08005978void __napi_schedule(struct napi_struct *n)
Stephen Hemmingerbea33482007-10-03 16:41:36 -07005979{
5980 unsigned long flags;
5981
5982 local_irq_save(flags);
Christoph Lameter903ceff2014-08-17 12:30:35 -05005983 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
Stephen Hemmingerbea33482007-10-03 16:41:36 -07005984 local_irq_restore(flags);
5985}
5986EXPORT_SYMBOL(__napi_schedule);
5987
Eric Dumazetbc9ad162014-10-28 18:05:13 -07005988/**
Eric Dumazet39e6c822017-02-28 10:34:50 -08005989 * napi_schedule_prep - check if napi can be scheduled
5990 * @n: napi context
5991 *
5992 * Test if NAPI routine is already running, and if not mark
5993 * it as running. This is used as a condition variable
5994 * insure only one NAPI poll instance runs. We also make
5995 * sure there is no pending NAPI disable.
5996 */
5997bool napi_schedule_prep(struct napi_struct *n)
5998{
5999 unsigned long val, new;
6000
6001 do {
6002 val = READ_ONCE(n->state);
6003 if (unlikely(val & NAPIF_STATE_DISABLE))
6004 return false;
6005 new = val | NAPIF_STATE_SCHED;
6006
6007 /* Sets STATE_MISSED bit if STATE_SCHED was already set
6008 * This was suggested by Alexander Duyck, as compiler
6009 * emits better code than :
6010 * if (val & NAPIF_STATE_SCHED)
6011 * new |= NAPIF_STATE_MISSED;
6012 */
6013 new |= (val & NAPIF_STATE_SCHED) / NAPIF_STATE_SCHED *
6014 NAPIF_STATE_MISSED;
6015 } while (cmpxchg(&n->state, val, new) != val);
6016
6017 return !(val & NAPIF_STATE_SCHED);
6018}
6019EXPORT_SYMBOL(napi_schedule_prep);
6020
6021/**
Eric Dumazetbc9ad162014-10-28 18:05:13 -07006022 * __napi_schedule_irqoff - schedule for receive
6023 * @n: entry to schedule
6024 *
6025 * Variant of __napi_schedule() assuming hard irqs are masked
6026 */
6027void __napi_schedule_irqoff(struct napi_struct *n)
6028{
6029 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
6030}
6031EXPORT_SYMBOL(__napi_schedule_irqoff);
6032
Eric Dumazet364b6052016-11-15 10:15:13 -08006033bool napi_complete_done(struct napi_struct *n, int work_done)
Herbert Xud565b0a2008-12-15 23:38:52 -08006034{
Eric Dumazet39e6c822017-02-28 10:34:50 -08006035 unsigned long flags, val, new;
Herbert Xud565b0a2008-12-15 23:38:52 -08006036
6037 /*
Eric Dumazet217f6972016-11-15 10:15:11 -08006038 * 1) Don't let napi dequeue from the cpu poll list
6039 * just in case its running on a different cpu.
6040 * 2) If we are busy polling, do nothing here, we have
6041 * the guarantee we will be called later.
Herbert Xud565b0a2008-12-15 23:38:52 -08006042 */
Eric Dumazet217f6972016-11-15 10:15:11 -08006043 if (unlikely(n->state & (NAPIF_STATE_NPSVC |
6044 NAPIF_STATE_IN_BUSY_POLL)))
Eric Dumazet364b6052016-11-15 10:15:13 -08006045 return false;
Herbert Xud565b0a2008-12-15 23:38:52 -08006046
Li RongQingd9f37d02018-07-13 14:41:36 +08006047 if (n->gro_bitmask) {
Eric Dumazet3b47d302014-11-06 21:09:44 -08006048 unsigned long timeout = 0;
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08006049
Eric Dumazet3b47d302014-11-06 21:09:44 -08006050 if (work_done)
6051 timeout = n->dev->gro_flush_timeout;
6052
Paolo Abeni605108a2018-11-21 18:21:35 +01006053 /* When the NAPI instance uses a timeout and keeps postponing
6054 * it, we need to bound somehow the time packets are kept in
6055 * the GRO layer
6056 */
6057 napi_gro_flush(n, !!timeout);
Eric Dumazet3b47d302014-11-06 21:09:44 -08006058 if (timeout)
6059 hrtimer_start(&n->timer, ns_to_ktime(timeout),
6060 HRTIMER_MODE_REL_PINNED);
Eric Dumazet3b47d302014-11-06 21:09:44 -08006061 }
Eric Dumazet02c16022017-02-04 15:25:02 -08006062 if (unlikely(!list_empty(&n->poll_list))) {
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08006063 /* If n->poll_list is not empty, we need to mask irqs */
6064 local_irq_save(flags);
Eric Dumazet02c16022017-02-04 15:25:02 -08006065 list_del_init(&n->poll_list);
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08006066 local_irq_restore(flags);
6067 }
Eric Dumazet39e6c822017-02-28 10:34:50 -08006068
6069 do {
6070 val = READ_ONCE(n->state);
6071
6072 WARN_ON_ONCE(!(val & NAPIF_STATE_SCHED));
6073
6074 new = val & ~(NAPIF_STATE_MISSED | NAPIF_STATE_SCHED);
6075
6076 /* If STATE_MISSED was set, leave STATE_SCHED set,
6077 * because we will call napi->poll() one more time.
6078 * This C code was suggested by Alexander Duyck to help gcc.
6079 */
6080 new |= (val & NAPIF_STATE_MISSED) / NAPIF_STATE_MISSED *
6081 NAPIF_STATE_SCHED;
6082 } while (cmpxchg(&n->state, val, new) != val);
6083
6084 if (unlikely(val & NAPIF_STATE_MISSED)) {
6085 __napi_schedule(n);
6086 return false;
6087 }
6088
Eric Dumazet364b6052016-11-15 10:15:13 -08006089 return true;
Herbert Xud565b0a2008-12-15 23:38:52 -08006090}
Eric Dumazet3b47d302014-11-06 21:09:44 -08006091EXPORT_SYMBOL(napi_complete_done);
Herbert Xud565b0a2008-12-15 23:38:52 -08006092
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03006093/* must be called under rcu_read_lock(), as we dont take a reference */
Eric Dumazet02d62e82015-11-18 06:30:52 -08006094static struct napi_struct *napi_by_id(unsigned int napi_id)
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03006095{
6096 unsigned int hash = napi_id % HASH_SIZE(napi_hash);
6097 struct napi_struct *napi;
6098
6099 hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
6100 if (napi->napi_id == napi_id)
6101 return napi;
6102
6103 return NULL;
6104}
Eric Dumazet02d62e82015-11-18 06:30:52 -08006105
6106#if defined(CONFIG_NET_RX_BUSY_POLL)
Eric Dumazet217f6972016-11-15 10:15:11 -08006107
Eric Dumazetce6aea92015-11-18 06:30:54 -08006108#define BUSY_POLL_BUDGET 8
Eric Dumazet217f6972016-11-15 10:15:11 -08006109
6110static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock)
6111{
6112 int rc;
6113
Eric Dumazet39e6c822017-02-28 10:34:50 -08006114 /* Busy polling means there is a high chance device driver hard irq
6115 * could not grab NAPI_STATE_SCHED, and that NAPI_STATE_MISSED was
6116 * set in napi_schedule_prep().
6117 * Since we are about to call napi->poll() once more, we can safely
6118 * clear NAPI_STATE_MISSED.
6119 *
6120 * Note: x86 could use a single "lock and ..." instruction
6121 * to perform these two clear_bit()
6122 */
6123 clear_bit(NAPI_STATE_MISSED, &napi->state);
Eric Dumazet217f6972016-11-15 10:15:11 -08006124 clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state);
6125
6126 local_bh_disable();
6127
6128 /* All we really want here is to re-enable device interrupts.
6129 * Ideally, a new ndo_busy_poll_stop() could avoid another round.
6130 */
6131 rc = napi->poll(napi, BUSY_POLL_BUDGET);
Jesper Dangaard Brouer1e223912017-08-25 15:04:32 +02006132 trace_napi_poll(napi, rc, BUSY_POLL_BUDGET);
Eric Dumazet217f6972016-11-15 10:15:11 -08006133 netpoll_poll_unlock(have_poll_lock);
6134 if (rc == BUSY_POLL_BUDGET)
6135 __napi_schedule(napi);
6136 local_bh_enable();
Eric Dumazet217f6972016-11-15 10:15:11 -08006137}
6138
Sridhar Samudrala7db6b042017-03-24 10:08:24 -07006139void napi_busy_loop(unsigned int napi_id,
6140 bool (*loop_end)(void *, unsigned long),
6141 void *loop_end_arg)
Eric Dumazet02d62e82015-11-18 06:30:52 -08006142{
Sridhar Samudrala7db6b042017-03-24 10:08:24 -07006143 unsigned long start_time = loop_end ? busy_loop_current_time() : 0;
Eric Dumazet217f6972016-11-15 10:15:11 -08006144 int (*napi_poll)(struct napi_struct *napi, int budget);
Eric Dumazet217f6972016-11-15 10:15:11 -08006145 void *have_poll_lock = NULL;
Eric Dumazet02d62e82015-11-18 06:30:52 -08006146 struct napi_struct *napi;
Eric Dumazet217f6972016-11-15 10:15:11 -08006147
6148restart:
Eric Dumazet217f6972016-11-15 10:15:11 -08006149 napi_poll = NULL;
Eric Dumazet02d62e82015-11-18 06:30:52 -08006150
Eric Dumazet2a028ec2015-11-18 06:30:53 -08006151 rcu_read_lock();
Eric Dumazet02d62e82015-11-18 06:30:52 -08006152
Alexander Duyck545cd5e2017-03-24 10:07:53 -07006153 napi = napi_by_id(napi_id);
Eric Dumazet02d62e82015-11-18 06:30:52 -08006154 if (!napi)
6155 goto out;
6156
Eric Dumazet217f6972016-11-15 10:15:11 -08006157 preempt_disable();
6158 for (;;) {
Alexander Duyck2b5cd0d2017-03-24 10:08:12 -07006159 int work = 0;
6160
Eric Dumazet2a028ec2015-11-18 06:30:53 -08006161 local_bh_disable();
Eric Dumazet217f6972016-11-15 10:15:11 -08006162 if (!napi_poll) {
6163 unsigned long val = READ_ONCE(napi->state);
6164
6165 /* If multiple threads are competing for this napi,
6166 * we avoid dirtying napi->state as much as we can.
6167 */
6168 if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED |
6169 NAPIF_STATE_IN_BUSY_POLL))
6170 goto count;
6171 if (cmpxchg(&napi->state, val,
6172 val | NAPIF_STATE_IN_BUSY_POLL |
6173 NAPIF_STATE_SCHED) != val)
6174 goto count;
6175 have_poll_lock = netpoll_poll_lock(napi);
6176 napi_poll = napi->poll;
6177 }
Alexander Duyck2b5cd0d2017-03-24 10:08:12 -07006178 work = napi_poll(napi, BUSY_POLL_BUDGET);
6179 trace_napi_poll(napi, work, BUSY_POLL_BUDGET);
Eric Dumazet217f6972016-11-15 10:15:11 -08006180count:
Alexander Duyck2b5cd0d2017-03-24 10:08:12 -07006181 if (work > 0)
Sridhar Samudrala7db6b042017-03-24 10:08:24 -07006182 __NET_ADD_STATS(dev_net(napi->dev),
Alexander Duyck2b5cd0d2017-03-24 10:08:12 -07006183 LINUX_MIB_BUSYPOLLRXPACKETS, work);
Eric Dumazet2a028ec2015-11-18 06:30:53 -08006184 local_bh_enable();
Eric Dumazet02d62e82015-11-18 06:30:52 -08006185
Sridhar Samudrala7db6b042017-03-24 10:08:24 -07006186 if (!loop_end || loop_end(loop_end_arg, start_time))
Eric Dumazet217f6972016-11-15 10:15:11 -08006187 break;
Eric Dumazet02d62e82015-11-18 06:30:52 -08006188
Eric Dumazet217f6972016-11-15 10:15:11 -08006189 if (unlikely(need_resched())) {
6190 if (napi_poll)
6191 busy_poll_stop(napi, have_poll_lock);
6192 preempt_enable();
6193 rcu_read_unlock();
6194 cond_resched();
Sridhar Samudrala7db6b042017-03-24 10:08:24 -07006195 if (loop_end(loop_end_arg, start_time))
Alexander Duyck2b5cd0d2017-03-24 10:08:12 -07006196 return;
Eric Dumazet217f6972016-11-15 10:15:11 -08006197 goto restart;
6198 }
Linus Torvalds6cdf89b2016-12-12 10:48:02 -08006199 cpu_relax();
Eric Dumazet217f6972016-11-15 10:15:11 -08006200 }
6201 if (napi_poll)
6202 busy_poll_stop(napi, have_poll_lock);
6203 preempt_enable();
Eric Dumazet02d62e82015-11-18 06:30:52 -08006204out:
Eric Dumazet2a028ec2015-11-18 06:30:53 -08006205 rcu_read_unlock();
Eric Dumazet02d62e82015-11-18 06:30:52 -08006206}
Sridhar Samudrala7db6b042017-03-24 10:08:24 -07006207EXPORT_SYMBOL(napi_busy_loop);
Eric Dumazet02d62e82015-11-18 06:30:52 -08006208
6209#endif /* CONFIG_NET_RX_BUSY_POLL */
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03006210
Eric Dumazet149d6ad2016-11-08 11:07:28 -08006211static void napi_hash_add(struct napi_struct *napi)
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03006212{
Eric Dumazetd64b5e82015-11-18 06:31:00 -08006213 if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state) ||
6214 test_and_set_bit(NAPI_STATE_HASHED, &napi->state))
Eric Dumazet52bd2d62015-11-18 06:30:50 -08006215 return;
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03006216
Eric Dumazet52bd2d62015-11-18 06:30:50 -08006217 spin_lock(&napi_hash_lock);
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03006218
Alexander Duyck545cd5e2017-03-24 10:07:53 -07006219 /* 0..NR_CPUS range is reserved for sender_cpu use */
Eric Dumazet52bd2d62015-11-18 06:30:50 -08006220 do {
Alexander Duyck545cd5e2017-03-24 10:07:53 -07006221 if (unlikely(++napi_gen_id < MIN_NAPI_ID))
6222 napi_gen_id = MIN_NAPI_ID;
Eric Dumazet52bd2d62015-11-18 06:30:50 -08006223 } while (napi_by_id(napi_gen_id));
6224 napi->napi_id = napi_gen_id;
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03006225
Eric Dumazet52bd2d62015-11-18 06:30:50 -08006226 hlist_add_head_rcu(&napi->napi_hash_node,
6227 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03006228
Eric Dumazet52bd2d62015-11-18 06:30:50 -08006229 spin_unlock(&napi_hash_lock);
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03006230}
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03006231
6232/* Warning : caller is responsible to make sure rcu grace period
6233 * is respected before freeing memory containing @napi
6234 */
Eric Dumazet34cbe272015-11-18 06:31:02 -08006235bool napi_hash_del(struct napi_struct *napi)
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03006236{
Eric Dumazet34cbe272015-11-18 06:31:02 -08006237 bool rcu_sync_needed = false;
6238
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03006239 spin_lock(&napi_hash_lock);
6240
Eric Dumazet34cbe272015-11-18 06:31:02 -08006241 if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state)) {
6242 rcu_sync_needed = true;
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03006243 hlist_del_rcu(&napi->napi_hash_node);
Eric Dumazet34cbe272015-11-18 06:31:02 -08006244 }
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03006245 spin_unlock(&napi_hash_lock);
Eric Dumazet34cbe272015-11-18 06:31:02 -08006246 return rcu_sync_needed;
Eliezer Tamiraf12fa62013-06-10 11:39:41 +03006247}
6248EXPORT_SYMBOL_GPL(napi_hash_del);
6249
Eric Dumazet3b47d302014-11-06 21:09:44 -08006250static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
6251{
6252 struct napi_struct *napi;
6253
6254 napi = container_of(timer, struct napi_struct, timer);
Eric Dumazet39e6c822017-02-28 10:34:50 -08006255
6256 /* Note : we use a relaxed variant of napi_schedule_prep() not setting
6257 * NAPI_STATE_MISSED, since we do not react to a device IRQ.
6258 */
Li RongQingd9f37d02018-07-13 14:41:36 +08006259 if (napi->gro_bitmask && !napi_disable_pending(napi) &&
Eric Dumazet39e6c822017-02-28 10:34:50 -08006260 !test_and_set_bit(NAPI_STATE_SCHED, &napi->state))
6261 __napi_schedule_irqoff(napi);
Eric Dumazet3b47d302014-11-06 21:09:44 -08006262
6263 return HRTIMER_NORESTART;
6264}
6265
David S. Miller7c4ec742018-07-20 23:37:55 -07006266static void init_gro_hash(struct napi_struct *napi)
Herbert Xud565b0a2008-12-15 23:38:52 -08006267{
David Miller07d78362018-06-24 14:14:02 +09006268 int i;
6269
Li RongQing6312fe72018-07-05 14:34:32 +08006270 for (i = 0; i < GRO_HASH_BUCKETS; i++) {
6271 INIT_LIST_HEAD(&napi->gro_hash[i].list);
6272 napi->gro_hash[i].count = 0;
6273 }
David S. Miller7c4ec742018-07-20 23:37:55 -07006274 napi->gro_bitmask = 0;
6275}
6276
6277void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
6278 int (*poll)(struct napi_struct *, int), int weight)
6279{
6280 INIT_LIST_HEAD(&napi->poll_list);
6281 hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
6282 napi->timer.function = napi_watchdog;
6283 init_gro_hash(napi);
Herbert Xu5d38a072009-01-04 16:13:40 -08006284 napi->skb = NULL;
Herbert Xud565b0a2008-12-15 23:38:52 -08006285 napi->poll = poll;
Eric Dumazet82dc3c63c2013-03-05 15:57:22 +00006286 if (weight > NAPI_POLL_WEIGHT)
Qian Caibf29e9e2018-12-01 21:11:19 -05006287 netdev_err_once(dev, "%s() called with weight %d\n", __func__,
6288 weight);
Herbert Xud565b0a2008-12-15 23:38:52 -08006289 napi->weight = weight;
6290 list_add(&napi->dev_list, &dev->napi_list);
Herbert Xud565b0a2008-12-15 23:38:52 -08006291 napi->dev = dev;
Herbert Xu5d38a072009-01-04 16:13:40 -08006292#ifdef CONFIG_NETPOLL
Herbert Xud565b0a2008-12-15 23:38:52 -08006293 napi->poll_owner = -1;
6294#endif
6295 set_bit(NAPI_STATE_SCHED, &napi->state);
Eric Dumazet93d05d42015-11-18 06:31:03 -08006296 napi_hash_add(napi);
Herbert Xud565b0a2008-12-15 23:38:52 -08006297}
6298EXPORT_SYMBOL(netif_napi_add);
6299
Eric Dumazet3b47d302014-11-06 21:09:44 -08006300void napi_disable(struct napi_struct *n)
6301{
6302 might_sleep();
6303 set_bit(NAPI_STATE_DISABLE, &n->state);
6304
6305 while (test_and_set_bit(NAPI_STATE_SCHED, &n->state))
6306 msleep(1);
Neil Horman2d8bff1262015-09-23 14:57:58 -04006307 while (test_and_set_bit(NAPI_STATE_NPSVC, &n->state))
6308 msleep(1);
Eric Dumazet3b47d302014-11-06 21:09:44 -08006309
6310 hrtimer_cancel(&n->timer);
6311
6312 clear_bit(NAPI_STATE_DISABLE, &n->state);
6313}
6314EXPORT_SYMBOL(napi_disable);
6315
David Miller07d78362018-06-24 14:14:02 +09006316static void flush_gro_hash(struct napi_struct *napi)
David Millerd4546c22018-06-24 14:13:49 +09006317{
David Miller07d78362018-06-24 14:14:02 +09006318 int i;
David Millerd4546c22018-06-24 14:13:49 +09006319
David Miller07d78362018-06-24 14:14:02 +09006320 for (i = 0; i < GRO_HASH_BUCKETS; i++) {
6321 struct sk_buff *skb, *n;
6322
Li RongQing6312fe72018-07-05 14:34:32 +08006323 list_for_each_entry_safe(skb, n, &napi->gro_hash[i].list, list)
David Miller07d78362018-06-24 14:14:02 +09006324 kfree_skb(skb);
Li RongQing6312fe72018-07-05 14:34:32 +08006325 napi->gro_hash[i].count = 0;
David Miller07d78362018-06-24 14:14:02 +09006326 }
David Millerd4546c22018-06-24 14:13:49 +09006327}
6328
Eric Dumazet93d05d42015-11-18 06:31:03 -08006329/* Must be called in process context */
Herbert Xud565b0a2008-12-15 23:38:52 -08006330void netif_napi_del(struct napi_struct *napi)
6331{
Eric Dumazet93d05d42015-11-18 06:31:03 -08006332 might_sleep();
6333 if (napi_hash_del(napi))
6334 synchronize_net();
Peter P Waskiewicz Jrd7b06632008-12-26 01:35:35 -08006335 list_del_init(&napi->dev_list);
Herbert Xu76620aa2009-04-16 02:02:07 -07006336 napi_free_frags(napi);
Herbert Xud565b0a2008-12-15 23:38:52 -08006337
David Miller07d78362018-06-24 14:14:02 +09006338 flush_gro_hash(napi);
Li RongQingd9f37d02018-07-13 14:41:36 +08006339 napi->gro_bitmask = 0;
Herbert Xud565b0a2008-12-15 23:38:52 -08006340}
6341EXPORT_SYMBOL(netif_napi_del);
6342
Herbert Xu726ce702014-12-21 07:16:21 +11006343static int napi_poll(struct napi_struct *n, struct list_head *repoll)
6344{
6345 void *have;
6346 int work, weight;
6347
6348 list_del_init(&n->poll_list);
6349
6350 have = netpoll_poll_lock(n);
6351
6352 weight = n->weight;
6353
6354 /* This NAPI_STATE_SCHED test is for avoiding a race
6355 * with netpoll's poll_napi(). Only the entity which
6356 * obtains the lock and sees NAPI_STATE_SCHED set will
6357 * actually make the ->poll() call. Therefore we avoid
6358 * accidentally calling ->poll() when NAPI is not scheduled.
6359 */
6360 work = 0;
6361 if (test_bit(NAPI_STATE_SCHED, &n->state)) {
6362 work = n->poll(n, weight);
Jesper Dangaard Brouer1db19db2016-07-07 18:01:32 +02006363 trace_napi_poll(n, work, weight);
Herbert Xu726ce702014-12-21 07:16:21 +11006364 }
6365
6366 WARN_ON_ONCE(work > weight);
6367
6368 if (likely(work < weight))
6369 goto out_unlock;
6370
6371 /* Drivers must not modify the NAPI state if they
6372 * consume the entire weight. In such cases this code
6373 * still "owns" the NAPI instance and therefore can
6374 * move the instance around on the list at-will.
6375 */
6376 if (unlikely(napi_disable_pending(n))) {
6377 napi_complete(n);
6378 goto out_unlock;
6379 }
6380
Li RongQingd9f37d02018-07-13 14:41:36 +08006381 if (n->gro_bitmask) {
Herbert Xu726ce702014-12-21 07:16:21 +11006382 /* flush too old packets
6383 * If HZ < 1000, flush all packets.
6384 */
6385 napi_gro_flush(n, HZ >= 1000);
6386 }
6387
Herbert Xu001ce542014-12-21 07:16:22 +11006388 /* Some drivers may have called napi_schedule
6389 * prior to exhausting their budget.
6390 */
6391 if (unlikely(!list_empty(&n->poll_list))) {
6392 pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
6393 n->dev ? n->dev->name : "backlog");
6394 goto out_unlock;
6395 }
6396
Herbert Xu726ce702014-12-21 07:16:21 +11006397 list_add_tail(&n->poll_list, repoll);
6398
6399out_unlock:
6400 netpoll_poll_unlock(have);
6401
6402 return work;
6403}
6404
Emese Revfy0766f782016-06-20 20:42:34 +02006405static __latent_entropy void net_rx_action(struct softirq_action *h)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006406{
Christoph Lameter903ceff2014-08-17 12:30:35 -05006407 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
Matthew Whitehead7acf8a12017-04-19 12:37:10 -04006408 unsigned long time_limit = jiffies +
6409 usecs_to_jiffies(netdev_budget_usecs);
Stephen Hemminger51b0bde2005-06-23 20:14:40 -07006410 int budget = netdev_budget;
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08006411 LIST_HEAD(list);
6412 LIST_HEAD(repoll);
Matt Mackall53fb95d2005-08-11 19:27:43 -07006413
Linus Torvalds1da177e2005-04-16 15:20:36 -07006414 local_irq_disable();
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08006415 list_splice_init(&sd->poll_list, &list);
6416 local_irq_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07006417
Herbert Xuceb8d5b2014-12-21 07:16:25 +11006418 for (;;) {
Stephen Hemmingerbea33482007-10-03 16:41:36 -07006419 struct napi_struct *n;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006420
Herbert Xuceb8d5b2014-12-21 07:16:25 +11006421 if (list_empty(&list)) {
6422 if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll))
Eric Dumazetf52dffe2016-11-23 08:44:56 -08006423 goto out;
Herbert Xuceb8d5b2014-12-21 07:16:25 +11006424 break;
6425 }
6426
Herbert Xu6bd373e2014-12-21 07:16:24 +11006427 n = list_first_entry(&list, struct napi_struct, poll_list);
6428 budget -= napi_poll(n, &repoll);
6429
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08006430 /* If softirq window is exhausted then punt.
Stephen Hemminger24f8b232008-11-03 17:14:38 -08006431 * Allow this to run for 2 jiffies since which will allow
6432 * an average latency of 1.5/HZ.
Stephen Hemmingerbea33482007-10-03 16:41:36 -07006433 */
Herbert Xuceb8d5b2014-12-21 07:16:25 +11006434 if (unlikely(budget <= 0 ||
6435 time_after_eq(jiffies, time_limit))) {
6436 sd->time_squeeze++;
6437 break;
6438 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006439 }
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08006440
Eric Dumazetd75b1ad2014-11-02 06:19:33 -08006441 local_irq_disable();
6442
6443 list_splice_tail_init(&sd->poll_list, &list);
6444 list_splice_tail(&repoll, &list);
6445 list_splice(&list, &sd->poll_list);
6446 if (!list_empty(&sd->poll_list))
6447 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
6448
Eric Dumazete326bed2010-04-22 00:22:45 -07006449 net_rps_action_and_irq_enable(sd);
Eric Dumazetf52dffe2016-11-23 08:44:56 -08006450out:
6451 __kfree_skb_flush();
Linus Torvalds1da177e2005-04-16 15:20:36 -07006452}
6453
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02006454struct netdev_adjacent {
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006455 struct net_device *dev;
Veaceslav Falico5d261912013-08-28 23:25:05 +02006456
6457 /* upper master flag, there can only be one master device per list */
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006458 bool master;
Veaceslav Falico5d261912013-08-28 23:25:05 +02006459
Veaceslav Falico5d261912013-08-28 23:25:05 +02006460 /* counter for the number of times this device was added to us */
6461 u16 ref_nr;
6462
Veaceslav Falico402dae92013-09-25 09:20:09 +02006463 /* private field for the users */
6464 void *private;
6465
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006466 struct list_head list;
6467 struct rcu_head rcu;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006468};
6469
Michal Kubeček6ea29da2015-09-24 10:59:05 +02006470static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev,
Veaceslav Falico2f268f12013-09-25 09:20:07 +02006471 struct list_head *adj_list)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006472{
Veaceslav Falico5d261912013-08-28 23:25:05 +02006473 struct netdev_adjacent *adj;
Veaceslav Falico5d261912013-08-28 23:25:05 +02006474
Veaceslav Falico2f268f12013-09-25 09:20:07 +02006475 list_for_each_entry(adj, adj_list, list) {
Veaceslav Falico5d261912013-08-28 23:25:05 +02006476 if (adj->dev == adj_dev)
6477 return adj;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006478 }
6479 return NULL;
6480}
6481
David Ahernf1170fd2016-10-17 19:15:51 -07006482static int __netdev_has_upper_dev(struct net_device *upper_dev, void *data)
6483{
6484 struct net_device *dev = data;
6485
6486 return upper_dev == dev;
6487}
6488
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006489/**
6490 * netdev_has_upper_dev - Check if device is linked to an upper device
6491 * @dev: device
6492 * @upper_dev: upper device to check
6493 *
6494 * Find out if a device is linked to specified upper device and return true
6495 * in case it is. Note that this checks only immediate upper device,
6496 * not through a complete stack of devices. The caller must hold the RTNL lock.
6497 */
6498bool netdev_has_upper_dev(struct net_device *dev,
6499 struct net_device *upper_dev)
6500{
6501 ASSERT_RTNL();
6502
David Ahernf1170fd2016-10-17 19:15:51 -07006503 return netdev_walk_all_upper_dev_rcu(dev, __netdev_has_upper_dev,
6504 upper_dev);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006505}
6506EXPORT_SYMBOL(netdev_has_upper_dev);
6507
6508/**
David Ahern1a3f0602016-10-17 19:15:44 -07006509 * netdev_has_upper_dev_all - Check if device is linked to an upper device
6510 * @dev: device
6511 * @upper_dev: upper device to check
6512 *
6513 * Find out if a device is linked to specified upper device and return true
6514 * in case it is. Note that this checks the entire upper device chain.
6515 * The caller must hold rcu lock.
6516 */
6517
David Ahern1a3f0602016-10-17 19:15:44 -07006518bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
6519 struct net_device *upper_dev)
6520{
6521 return !!netdev_walk_all_upper_dev_rcu(dev, __netdev_has_upper_dev,
6522 upper_dev);
6523}
6524EXPORT_SYMBOL(netdev_has_upper_dev_all_rcu);
6525
6526/**
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006527 * netdev_has_any_upper_dev - Check if device is linked to some device
6528 * @dev: device
6529 *
6530 * Find out if a device is linked to an upper device and return true in case
6531 * it is. The caller must hold the RTNL lock.
6532 */
Ido Schimmel25cc72a2017-09-01 10:52:31 +02006533bool netdev_has_any_upper_dev(struct net_device *dev)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006534{
6535 ASSERT_RTNL();
6536
David Ahernf1170fd2016-10-17 19:15:51 -07006537 return !list_empty(&dev->adj_list.upper);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006538}
Ido Schimmel25cc72a2017-09-01 10:52:31 +02006539EXPORT_SYMBOL(netdev_has_any_upper_dev);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006540
6541/**
6542 * netdev_master_upper_dev_get - Get master upper device
6543 * @dev: device
6544 *
6545 * Find a master upper device and return pointer to it or NULL in case
6546 * it's not there. The caller must hold the RTNL lock.
6547 */
6548struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
6549{
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02006550 struct netdev_adjacent *upper;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006551
6552 ASSERT_RTNL();
6553
Veaceslav Falico2f268f12013-09-25 09:20:07 +02006554 if (list_empty(&dev->adj_list.upper))
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006555 return NULL;
6556
Veaceslav Falico2f268f12013-09-25 09:20:07 +02006557 upper = list_first_entry(&dev->adj_list.upper,
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02006558 struct netdev_adjacent, list);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006559 if (likely(upper->master))
6560 return upper->dev;
6561 return NULL;
6562}
6563EXPORT_SYMBOL(netdev_master_upper_dev_get);
6564
David Ahern0f524a82016-10-17 19:15:52 -07006565/**
6566 * netdev_has_any_lower_dev - Check if device is linked to some device
6567 * @dev: device
6568 *
6569 * Find out if a device is linked to a lower device and return true in case
6570 * it is. The caller must hold the RTNL lock.
6571 */
6572static bool netdev_has_any_lower_dev(struct net_device *dev)
6573{
6574 ASSERT_RTNL();
6575
6576 return !list_empty(&dev->adj_list.lower);
6577}
6578
Veaceslav Falicob6ccba42013-09-25 09:20:23 +02006579void *netdev_adjacent_get_private(struct list_head *adj_list)
6580{
6581 struct netdev_adjacent *adj;
6582
6583 adj = list_entry(adj_list, struct netdev_adjacent, list);
6584
6585 return adj->private;
6586}
6587EXPORT_SYMBOL(netdev_adjacent_get_private);
6588
Veaceslav Falico31088a12013-09-25 09:20:12 +02006589/**
Vlad Yasevich44a40852014-05-16 17:20:38 -04006590 * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
6591 * @dev: device
6592 * @iter: list_head ** of the current position
6593 *
6594 * Gets the next device from the dev's upper list, starting from iter
6595 * position. The caller must hold RCU read lock.
6596 */
6597struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
6598 struct list_head **iter)
6599{
6600 struct netdev_adjacent *upper;
6601
6602 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
6603
6604 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
6605
6606 if (&upper->list == &dev->adj_list.upper)
6607 return NULL;
6608
6609 *iter = &upper->list;
6610
6611 return upper->dev;
6612}
6613EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
6614
David Ahern1a3f0602016-10-17 19:15:44 -07006615static struct net_device *netdev_next_upper_dev_rcu(struct net_device *dev,
6616 struct list_head **iter)
6617{
6618 struct netdev_adjacent *upper;
6619
6620 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
6621
6622 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
6623
6624 if (&upper->list == &dev->adj_list.upper)
6625 return NULL;
6626
6627 *iter = &upper->list;
6628
6629 return upper->dev;
6630}
6631
6632int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
6633 int (*fn)(struct net_device *dev,
6634 void *data),
6635 void *data)
6636{
6637 struct net_device *udev;
6638 struct list_head *iter;
6639 int ret;
6640
6641 for (iter = &dev->adj_list.upper,
6642 udev = netdev_next_upper_dev_rcu(dev, &iter);
6643 udev;
6644 udev = netdev_next_upper_dev_rcu(dev, &iter)) {
6645 /* first is the upper device itself */
6646 ret = fn(udev, data);
6647 if (ret)
6648 return ret;
6649
6650 /* then look at all of its upper devices */
6651 ret = netdev_walk_all_upper_dev_rcu(udev, fn, data);
6652 if (ret)
6653 return ret;
6654 }
6655
6656 return 0;
6657}
6658EXPORT_SYMBOL_GPL(netdev_walk_all_upper_dev_rcu);
6659
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006660/**
Veaceslav Falico31088a12013-09-25 09:20:12 +02006661 * netdev_lower_get_next_private - Get the next ->private from the
6662 * lower neighbour list
6663 * @dev: device
6664 * @iter: list_head ** of the current position
6665 *
6666 * Gets the next netdev_adjacent->private from the dev's lower neighbour
6667 * list, starting from iter position. The caller must hold either hold the
6668 * RTNL lock or its own locking that guarantees that the neighbour lower
subashab@codeaurora.orgb4691392015-07-24 03:03:29 +00006669 * list will remain unchanged.
Veaceslav Falico31088a12013-09-25 09:20:12 +02006670 */
6671void *netdev_lower_get_next_private(struct net_device *dev,
6672 struct list_head **iter)
6673{
6674 struct netdev_adjacent *lower;
6675
6676 lower = list_entry(*iter, struct netdev_adjacent, list);
6677
6678 if (&lower->list == &dev->adj_list.lower)
6679 return NULL;
6680
Veaceslav Falico6859e7d2014-04-07 11:25:12 +02006681 *iter = lower->list.next;
Veaceslav Falico31088a12013-09-25 09:20:12 +02006682
6683 return lower->private;
6684}
6685EXPORT_SYMBOL(netdev_lower_get_next_private);
6686
6687/**
6688 * netdev_lower_get_next_private_rcu - Get the next ->private from the
6689 * lower neighbour list, RCU
6690 * variant
6691 * @dev: device
6692 * @iter: list_head ** of the current position
6693 *
6694 * Gets the next netdev_adjacent->private from the dev's lower neighbour
6695 * list, starting from iter position. The caller must hold RCU read lock.
6696 */
6697void *netdev_lower_get_next_private_rcu(struct net_device *dev,
6698 struct list_head **iter)
6699{
6700 struct netdev_adjacent *lower;
6701
6702 WARN_ON_ONCE(!rcu_read_lock_held());
6703
6704 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
6705
6706 if (&lower->list == &dev->adj_list.lower)
6707 return NULL;
6708
Veaceslav Falico6859e7d2014-04-07 11:25:12 +02006709 *iter = &lower->list;
Veaceslav Falico31088a12013-09-25 09:20:12 +02006710
6711 return lower->private;
6712}
6713EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
6714
6715/**
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04006716 * netdev_lower_get_next - Get the next device from the lower neighbour
6717 * list
6718 * @dev: device
6719 * @iter: list_head ** of the current position
6720 *
6721 * Gets the next netdev_adjacent from the dev's lower neighbour
6722 * list, starting from iter position. The caller must hold RTNL lock or
6723 * its own locking that guarantees that the neighbour lower
subashab@codeaurora.orgb4691392015-07-24 03:03:29 +00006724 * list will remain unchanged.
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04006725 */
6726void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter)
6727{
6728 struct netdev_adjacent *lower;
6729
Nikolay Aleksandrovcfdd28b2016-02-17 18:00:31 +01006730 lower = list_entry(*iter, struct netdev_adjacent, list);
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04006731
6732 if (&lower->list == &dev->adj_list.lower)
6733 return NULL;
6734
Nikolay Aleksandrovcfdd28b2016-02-17 18:00:31 +01006735 *iter = lower->list.next;
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04006736
6737 return lower->dev;
6738}
6739EXPORT_SYMBOL(netdev_lower_get_next);
6740
David Ahern1a3f0602016-10-17 19:15:44 -07006741static struct net_device *netdev_next_lower_dev(struct net_device *dev,
6742 struct list_head **iter)
6743{
6744 struct netdev_adjacent *lower;
6745
David Ahern46b5ab12016-10-26 13:21:33 -07006746 lower = list_entry((*iter)->next, struct netdev_adjacent, list);
David Ahern1a3f0602016-10-17 19:15:44 -07006747
6748 if (&lower->list == &dev->adj_list.lower)
6749 return NULL;
6750
David Ahern46b5ab12016-10-26 13:21:33 -07006751 *iter = &lower->list;
David Ahern1a3f0602016-10-17 19:15:44 -07006752
6753 return lower->dev;
6754}
6755
6756int netdev_walk_all_lower_dev(struct net_device *dev,
6757 int (*fn)(struct net_device *dev,
6758 void *data),
6759 void *data)
6760{
6761 struct net_device *ldev;
6762 struct list_head *iter;
6763 int ret;
6764
6765 for (iter = &dev->adj_list.lower,
6766 ldev = netdev_next_lower_dev(dev, &iter);
6767 ldev;
6768 ldev = netdev_next_lower_dev(dev, &iter)) {
6769 /* first is the lower device itself */
6770 ret = fn(ldev, data);
6771 if (ret)
6772 return ret;
6773
6774 /* then look at all of its lower devices */
6775 ret = netdev_walk_all_lower_dev(ldev, fn, data);
6776 if (ret)
6777 return ret;
6778 }
6779
6780 return 0;
6781}
6782EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev);
6783
David Ahern1a3f0602016-10-17 19:15:44 -07006784static struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev,
6785 struct list_head **iter)
6786{
6787 struct netdev_adjacent *lower;
6788
6789 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
6790 if (&lower->list == &dev->adj_list.lower)
6791 return NULL;
6792
6793 *iter = &lower->list;
6794
6795 return lower->dev;
6796}
6797
6798int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
6799 int (*fn)(struct net_device *dev,
6800 void *data),
6801 void *data)
6802{
6803 struct net_device *ldev;
6804 struct list_head *iter;
6805 int ret;
6806
6807 for (iter = &dev->adj_list.lower,
6808 ldev = netdev_next_lower_dev_rcu(dev, &iter);
6809 ldev;
6810 ldev = netdev_next_lower_dev_rcu(dev, &iter)) {
6811 /* first is the lower device itself */
6812 ret = fn(ldev, data);
6813 if (ret)
6814 return ret;
6815
6816 /* then look at all of its lower devices */
6817 ret = netdev_walk_all_lower_dev_rcu(ldev, fn, data);
6818 if (ret)
6819 return ret;
6820 }
6821
6822 return 0;
6823}
6824EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev_rcu);
6825
Jiri Pirko7ce856a2016-07-04 08:23:12 +02006826/**
dingtianhonge001bfa2013-12-13 10:19:55 +08006827 * netdev_lower_get_first_private_rcu - Get the first ->private from the
6828 * lower neighbour list, RCU
6829 * variant
6830 * @dev: device
6831 *
6832 * Gets the first netdev_adjacent->private from the dev's lower neighbour
6833 * list. The caller must hold RCU read lock.
6834 */
6835void *netdev_lower_get_first_private_rcu(struct net_device *dev)
6836{
6837 struct netdev_adjacent *lower;
6838
6839 lower = list_first_or_null_rcu(&dev->adj_list.lower,
6840 struct netdev_adjacent, list);
6841 if (lower)
6842 return lower->private;
6843 return NULL;
6844}
6845EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
6846
6847/**
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006848 * netdev_master_upper_dev_get_rcu - Get master upper device
6849 * @dev: device
6850 *
6851 * Find a master upper device and return pointer to it or NULL in case
6852 * it's not there. The caller must hold the RCU read lock.
6853 */
6854struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
6855{
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02006856 struct netdev_adjacent *upper;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006857
Veaceslav Falico2f268f12013-09-25 09:20:07 +02006858 upper = list_first_or_null_rcu(&dev->adj_list.upper,
Veaceslav Falicoaa9d8562013-08-28 23:25:04 +02006859 struct netdev_adjacent, list);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00006860 if (upper && likely(upper->master))
6861 return upper->dev;
6862 return NULL;
6863}
6864EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
6865
Rashika Kheria0a59f3a2014-02-09 20:26:25 +05306866static int netdev_adjacent_sysfs_add(struct net_device *dev,
Veaceslav Falico3ee32702014-01-14 21:58:50 +01006867 struct net_device *adj_dev,
6868 struct list_head *dev_list)
6869{
6870 char linkname[IFNAMSIZ+7];
tchardingf4563a72017-02-09 17:56:07 +11006871
Veaceslav Falico3ee32702014-01-14 21:58:50 +01006872 sprintf(linkname, dev_list == &dev->adj_list.upper ?
6873 "upper_%s" : "lower_%s", adj_dev->name);
6874 return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
6875 linkname);
6876}
Rashika Kheria0a59f3a2014-02-09 20:26:25 +05306877static void netdev_adjacent_sysfs_del(struct net_device *dev,
Veaceslav Falico3ee32702014-01-14 21:58:50 +01006878 char *name,
6879 struct list_head *dev_list)
6880{
6881 char linkname[IFNAMSIZ+7];
tchardingf4563a72017-02-09 17:56:07 +11006882
Veaceslav Falico3ee32702014-01-14 21:58:50 +01006883 sprintf(linkname, dev_list == &dev->adj_list.upper ?
6884 "upper_%s" : "lower_%s", name);
6885 sysfs_remove_link(&(dev->dev.kobj), linkname);
6886}
6887
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04006888static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev,
6889 struct net_device *adj_dev,
6890 struct list_head *dev_list)
6891{
6892 return (dev_list == &dev->adj_list.upper ||
6893 dev_list == &dev->adj_list.lower) &&
6894 net_eq(dev_net(dev), dev_net(adj_dev));
6895}
Veaceslav Falico3ee32702014-01-14 21:58:50 +01006896
Veaceslav Falico5d261912013-08-28 23:25:05 +02006897static int __netdev_adjacent_dev_insert(struct net_device *dev,
6898 struct net_device *adj_dev,
Veaceslav Falico7863c052013-09-25 09:20:06 +02006899 struct list_head *dev_list,
Veaceslav Falico402dae92013-09-25 09:20:09 +02006900 void *private, bool master)
Veaceslav Falico5d261912013-08-28 23:25:05 +02006901{
6902 struct netdev_adjacent *adj;
Veaceslav Falico842d67a2013-09-25 09:20:31 +02006903 int ret;
Veaceslav Falico5d261912013-08-28 23:25:05 +02006904
Michal Kubeček6ea29da2015-09-24 10:59:05 +02006905 adj = __netdev_find_adj(adj_dev, dev_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02006906
6907 if (adj) {
David Ahern790510d2016-10-17 19:15:43 -07006908 adj->ref_nr += 1;
David Ahern67b62f92016-10-17 19:15:53 -07006909 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d\n",
6910 dev->name, adj_dev->name, adj->ref_nr);
6911
Veaceslav Falico5d261912013-08-28 23:25:05 +02006912 return 0;
6913 }
6914
6915 adj = kmalloc(sizeof(*adj), GFP_KERNEL);
6916 if (!adj)
6917 return -ENOMEM;
6918
6919 adj->dev = adj_dev;
6920 adj->master = master;
David Ahern790510d2016-10-17 19:15:43 -07006921 adj->ref_nr = 1;
Veaceslav Falico402dae92013-09-25 09:20:09 +02006922 adj->private = private;
Veaceslav Falico5d261912013-08-28 23:25:05 +02006923 dev_hold(adj_dev);
Veaceslav Falico2f268f12013-09-25 09:20:07 +02006924
David Ahern67b62f92016-10-17 19:15:53 -07006925 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n",
6926 dev->name, adj_dev->name, adj->ref_nr, adj_dev->name);
Veaceslav Falico5d261912013-08-28 23:25:05 +02006927
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04006928 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) {
Veaceslav Falico3ee32702014-01-14 21:58:50 +01006929 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
Veaceslav Falico5831d662013-09-25 09:20:32 +02006930 if (ret)
6931 goto free_adj;
6932 }
6933
Veaceslav Falico7863c052013-09-25 09:20:06 +02006934 /* Ensure that master link is always the first item in list. */
Veaceslav Falico842d67a2013-09-25 09:20:31 +02006935 if (master) {
6936 ret = sysfs_create_link(&(dev->dev.kobj),
6937 &(adj_dev->dev.kobj), "master");
6938 if (ret)
Veaceslav Falico5831d662013-09-25 09:20:32 +02006939 goto remove_symlinks;
Veaceslav Falico842d67a2013-09-25 09:20:31 +02006940
Veaceslav Falico7863c052013-09-25 09:20:06 +02006941 list_add_rcu(&adj->list, dev_list);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02006942 } else {
Veaceslav Falico7863c052013-09-25 09:20:06 +02006943 list_add_tail_rcu(&adj->list, dev_list);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02006944 }
Veaceslav Falico5d261912013-08-28 23:25:05 +02006945
6946 return 0;
Veaceslav Falico842d67a2013-09-25 09:20:31 +02006947
Veaceslav Falico5831d662013-09-25 09:20:32 +02006948remove_symlinks:
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04006949 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
Veaceslav Falico3ee32702014-01-14 21:58:50 +01006950 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02006951free_adj:
6952 kfree(adj);
Nikolay Aleksandrov974daef2013-10-23 15:28:56 +02006953 dev_put(adj_dev);
Veaceslav Falico842d67a2013-09-25 09:20:31 +02006954
6955 return ret;
Veaceslav Falico5d261912013-08-28 23:25:05 +02006956}
6957
stephen hemminger1d143d92013-12-29 14:01:29 -08006958static void __netdev_adjacent_dev_remove(struct net_device *dev,
6959 struct net_device *adj_dev,
Andrew Collins93409032016-10-03 13:43:02 -06006960 u16 ref_nr,
stephen hemminger1d143d92013-12-29 14:01:29 -08006961 struct list_head *dev_list)
Veaceslav Falico5d261912013-08-28 23:25:05 +02006962{
6963 struct netdev_adjacent *adj;
6964
David Ahern67b62f92016-10-17 19:15:53 -07006965 pr_debug("Remove adjacency: dev %s adj_dev %s ref_nr %d\n",
6966 dev->name, adj_dev->name, ref_nr);
6967
Michal Kubeček6ea29da2015-09-24 10:59:05 +02006968 adj = __netdev_find_adj(adj_dev, dev_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02006969
Veaceslav Falico2f268f12013-09-25 09:20:07 +02006970 if (!adj) {
David Ahern67b62f92016-10-17 19:15:53 -07006971 pr_err("Adjacency does not exist for device %s from %s\n",
Veaceslav Falico2f268f12013-09-25 09:20:07 +02006972 dev->name, adj_dev->name);
David Ahern67b62f92016-10-17 19:15:53 -07006973 WARN_ON(1);
6974 return;
Veaceslav Falico2f268f12013-09-25 09:20:07 +02006975 }
Veaceslav Falico5d261912013-08-28 23:25:05 +02006976
Andrew Collins93409032016-10-03 13:43:02 -06006977 if (adj->ref_nr > ref_nr) {
David Ahern67b62f92016-10-17 19:15:53 -07006978 pr_debug("adjacency: %s to %s ref_nr - %d = %d\n",
6979 dev->name, adj_dev->name, ref_nr,
6980 adj->ref_nr - ref_nr);
Andrew Collins93409032016-10-03 13:43:02 -06006981 adj->ref_nr -= ref_nr;
Veaceslav Falico5d261912013-08-28 23:25:05 +02006982 return;
6983 }
6984
Veaceslav Falico842d67a2013-09-25 09:20:31 +02006985 if (adj->master)
6986 sysfs_remove_link(&(dev->dev.kobj), "master");
6987
Alexander Y. Fomichev7ce64c72014-09-15 14:22:35 +04006988 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
Veaceslav Falico3ee32702014-01-14 21:58:50 +01006989 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
Veaceslav Falico5831d662013-09-25 09:20:32 +02006990
Veaceslav Falico5d261912013-08-28 23:25:05 +02006991 list_del_rcu(&adj->list);
David Ahern67b62f92016-10-17 19:15:53 -07006992 pr_debug("adjacency: dev_put for %s, because link removed from %s to %s\n",
Veaceslav Falico2f268f12013-09-25 09:20:07 +02006993 adj_dev->name, dev->name, adj_dev->name);
Veaceslav Falico5d261912013-08-28 23:25:05 +02006994 dev_put(adj_dev);
6995 kfree_rcu(adj, rcu);
6996}
6997
stephen hemminger1d143d92013-12-29 14:01:29 -08006998static int __netdev_adjacent_dev_link_lists(struct net_device *dev,
6999 struct net_device *upper_dev,
7000 struct list_head *up_list,
7001 struct list_head *down_list,
7002 void *private, bool master)
Veaceslav Falico5d261912013-08-28 23:25:05 +02007003{
7004 int ret;
7005
David Ahern790510d2016-10-17 19:15:43 -07007006 ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list,
Andrew Collins93409032016-10-03 13:43:02 -06007007 private, master);
Veaceslav Falico5d261912013-08-28 23:25:05 +02007008 if (ret)
7009 return ret;
7010
David Ahern790510d2016-10-17 19:15:43 -07007011 ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list,
Andrew Collins93409032016-10-03 13:43:02 -06007012 private, false);
Veaceslav Falico5d261912013-08-28 23:25:05 +02007013 if (ret) {
David Ahern790510d2016-10-17 19:15:43 -07007014 __netdev_adjacent_dev_remove(dev, upper_dev, 1, up_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02007015 return ret;
7016 }
7017
7018 return 0;
7019}
7020
stephen hemminger1d143d92013-12-29 14:01:29 -08007021static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev,
7022 struct net_device *upper_dev,
Andrew Collins93409032016-10-03 13:43:02 -06007023 u16 ref_nr,
stephen hemminger1d143d92013-12-29 14:01:29 -08007024 struct list_head *up_list,
7025 struct list_head *down_list)
Veaceslav Falico5d261912013-08-28 23:25:05 +02007026{
Andrew Collins93409032016-10-03 13:43:02 -06007027 __netdev_adjacent_dev_remove(dev, upper_dev, ref_nr, up_list);
7028 __netdev_adjacent_dev_remove(upper_dev, dev, ref_nr, down_list);
Veaceslav Falico5d261912013-08-28 23:25:05 +02007029}
7030
stephen hemminger1d143d92013-12-29 14:01:29 -08007031static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
7032 struct net_device *upper_dev,
7033 void *private, bool master)
Veaceslav Falico2f268f12013-09-25 09:20:07 +02007034{
David Ahernf1170fd2016-10-17 19:15:51 -07007035 return __netdev_adjacent_dev_link_lists(dev, upper_dev,
7036 &dev->adj_list.upper,
7037 &upper_dev->adj_list.lower,
7038 private, master);
Veaceslav Falico2f268f12013-09-25 09:20:07 +02007039}
7040
stephen hemminger1d143d92013-12-29 14:01:29 -08007041static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev,
7042 struct net_device *upper_dev)
Veaceslav Falico2f268f12013-09-25 09:20:07 +02007043{
Andrew Collins93409032016-10-03 13:43:02 -06007044 __netdev_adjacent_dev_unlink_lists(dev, upper_dev, 1,
Veaceslav Falico2f268f12013-09-25 09:20:07 +02007045 &dev->adj_list.upper,
7046 &upper_dev->adj_list.lower);
7047}
Veaceslav Falico5d261912013-08-28 23:25:05 +02007048
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007049static int __netdev_upper_dev_link(struct net_device *dev,
Veaceslav Falico402dae92013-09-25 09:20:09 +02007050 struct net_device *upper_dev, bool master,
David Ahern42ab19e2017-10-04 17:48:47 -07007051 void *upper_priv, void *upper_info,
7052 struct netlink_ext_ack *extack)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007053{
David Ahern51d0c0472017-10-04 17:48:45 -07007054 struct netdev_notifier_changeupper_info changeupper_info = {
7055 .info = {
7056 .dev = dev,
David Ahern42ab19e2017-10-04 17:48:47 -07007057 .extack = extack,
David Ahern51d0c0472017-10-04 17:48:45 -07007058 },
7059 .upper_dev = upper_dev,
7060 .master = master,
7061 .linking = true,
7062 .upper_info = upper_info,
7063 };
Mike Manning50d629e2018-02-26 23:49:30 +00007064 struct net_device *master_dev;
Veaceslav Falico5d261912013-08-28 23:25:05 +02007065 int ret = 0;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007066
7067 ASSERT_RTNL();
7068
7069 if (dev == upper_dev)
7070 return -EBUSY;
7071
7072 /* To prevent loops, check if dev is not upper device to upper_dev. */
David Ahernf1170fd2016-10-17 19:15:51 -07007073 if (netdev_has_upper_dev(upper_dev, dev))
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007074 return -EBUSY;
7075
Mike Manning50d629e2018-02-26 23:49:30 +00007076 if (!master) {
7077 if (netdev_has_upper_dev(dev, upper_dev))
7078 return -EEXIST;
7079 } else {
7080 master_dev = netdev_master_upper_dev_get(dev);
7081 if (master_dev)
7082 return master_dev == upper_dev ? -EEXIST : -EBUSY;
7083 }
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007084
David Ahern51d0c0472017-10-04 17:48:45 -07007085 ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER,
Jiri Pirko573c7ba2015-10-16 14:01:22 +02007086 &changeupper_info.info);
7087 ret = notifier_to_errno(ret);
7088 if (ret)
7089 return ret;
7090
Jiri Pirko6dffb042015-12-03 12:12:10 +01007091 ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv,
Veaceslav Falico402dae92013-09-25 09:20:09 +02007092 master);
Veaceslav Falico5d261912013-08-28 23:25:05 +02007093 if (ret)
7094 return ret;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007095
David Ahern51d0c0472017-10-04 17:48:45 -07007096 ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER,
Ido Schimmelb03804e2015-12-03 12:12:03 +01007097 &changeupper_info.info);
7098 ret = notifier_to_errno(ret);
7099 if (ret)
David Ahernf1170fd2016-10-17 19:15:51 -07007100 goto rollback;
Ido Schimmelb03804e2015-12-03 12:12:03 +01007101
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007102 return 0;
Veaceslav Falico5d261912013-08-28 23:25:05 +02007103
David Ahernf1170fd2016-10-17 19:15:51 -07007104rollback:
Veaceslav Falico2f268f12013-09-25 09:20:07 +02007105 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
Veaceslav Falico5d261912013-08-28 23:25:05 +02007106
7107 return ret;
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007108}
7109
7110/**
7111 * netdev_upper_dev_link - Add a link to the upper device
7112 * @dev: device
7113 * @upper_dev: new upper device
Florian Fainelli7a006d52018-01-22 19:14:28 -08007114 * @extack: netlink extended ack
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007115 *
7116 * Adds a link to device which is upper to this one. The caller must hold
7117 * the RTNL lock. On a failure a negative errno code is returned.
7118 * On success the reference counts are adjusted and the function
7119 * returns zero.
7120 */
7121int netdev_upper_dev_link(struct net_device *dev,
David Ahern42ab19e2017-10-04 17:48:47 -07007122 struct net_device *upper_dev,
7123 struct netlink_ext_ack *extack)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007124{
David Ahern42ab19e2017-10-04 17:48:47 -07007125 return __netdev_upper_dev_link(dev, upper_dev, false,
7126 NULL, NULL, extack);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007127}
7128EXPORT_SYMBOL(netdev_upper_dev_link);
7129
7130/**
7131 * netdev_master_upper_dev_link - Add a master link to the upper device
7132 * @dev: device
7133 * @upper_dev: new upper device
Jiri Pirko6dffb042015-12-03 12:12:10 +01007134 * @upper_priv: upper device private
Jiri Pirko29bf24a2015-12-03 12:12:11 +01007135 * @upper_info: upper info to be passed down via notifier
Florian Fainelli7a006d52018-01-22 19:14:28 -08007136 * @extack: netlink extended ack
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007137 *
7138 * Adds a link to device which is upper to this one. In this case, only
7139 * one master upper device can be linked, although other non-master devices
7140 * might be linked as well. The caller must hold the RTNL lock.
7141 * On a failure a negative errno code is returned. On success the reference
7142 * counts are adjusted and the function returns zero.
7143 */
7144int netdev_master_upper_dev_link(struct net_device *dev,
Jiri Pirko6dffb042015-12-03 12:12:10 +01007145 struct net_device *upper_dev,
David Ahern42ab19e2017-10-04 17:48:47 -07007146 void *upper_priv, void *upper_info,
7147 struct netlink_ext_ack *extack)
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007148{
Jiri Pirko29bf24a2015-12-03 12:12:11 +01007149 return __netdev_upper_dev_link(dev, upper_dev, true,
David Ahern42ab19e2017-10-04 17:48:47 -07007150 upper_priv, upper_info, extack);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007151}
7152EXPORT_SYMBOL(netdev_master_upper_dev_link);
7153
7154/**
7155 * netdev_upper_dev_unlink - Removes a link to upper device
7156 * @dev: device
7157 * @upper_dev: new upper device
7158 *
7159 * Removes a link to device which is upper to this one. The caller must hold
7160 * the RTNL lock.
7161 */
7162void netdev_upper_dev_unlink(struct net_device *dev,
7163 struct net_device *upper_dev)
7164{
David Ahern51d0c0472017-10-04 17:48:45 -07007165 struct netdev_notifier_changeupper_info changeupper_info = {
7166 .info = {
7167 .dev = dev,
7168 },
7169 .upper_dev = upper_dev,
7170 .linking = false,
7171 };
tchardingf4563a72017-02-09 17:56:07 +11007172
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007173 ASSERT_RTNL();
7174
Jiri Pirko0e4ead92015-08-27 09:31:18 +02007175 changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev;
Jiri Pirko0e4ead92015-08-27 09:31:18 +02007176
David Ahern51d0c0472017-10-04 17:48:45 -07007177 call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER,
Jiri Pirko573c7ba2015-10-16 14:01:22 +02007178 &changeupper_info.info);
7179
Veaceslav Falico2f268f12013-09-25 09:20:07 +02007180 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
Veaceslav Falico5d261912013-08-28 23:25:05 +02007181
David Ahern51d0c0472017-10-04 17:48:45 -07007182 call_netdevice_notifiers_info(NETDEV_CHANGEUPPER,
Jiri Pirko0e4ead92015-08-27 09:31:18 +02007183 &changeupper_info.info);
Jiri Pirko9ff162a2013-01-03 22:48:49 +00007184}
7185EXPORT_SYMBOL(netdev_upper_dev_unlink);
7186
Moni Shoua61bd3852015-02-03 16:48:29 +02007187/**
7188 * netdev_bonding_info_change - Dispatch event about slave change
7189 * @dev: device
Masanari Iida4a26e4532015-02-14 22:26:34 +09007190 * @bonding_info: info to dispatch
Moni Shoua61bd3852015-02-03 16:48:29 +02007191 *
7192 * Send NETDEV_BONDING_INFO to netdev notifiers with info.
7193 * The caller must hold the RTNL lock.
7194 */
7195void netdev_bonding_info_change(struct net_device *dev,
7196 struct netdev_bonding_info *bonding_info)
7197{
David Ahern51d0c0472017-10-04 17:48:45 -07007198 struct netdev_notifier_bonding_info info = {
7199 .info.dev = dev,
7200 };
Moni Shoua61bd3852015-02-03 16:48:29 +02007201
7202 memcpy(&info.bonding_info, bonding_info,
7203 sizeof(struct netdev_bonding_info));
David Ahern51d0c0472017-10-04 17:48:45 -07007204 call_netdevice_notifiers_info(NETDEV_BONDING_INFO,
Moni Shoua61bd3852015-02-03 16:48:29 +02007205 &info.info);
7206}
7207EXPORT_SYMBOL(netdev_bonding_info_change);
7208
Eric Dumazet2ce1ee12015-02-04 13:37:44 -08007209static void netdev_adjacent_add_links(struct net_device *dev)
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04007210{
7211 struct netdev_adjacent *iter;
7212
7213 struct net *net = dev_net(dev);
7214
7215 list_for_each_entry(iter, &dev->adj_list.upper, list) {
Wei Tangbe4da0e2016-06-16 21:30:12 +08007216 if (!net_eq(net, dev_net(iter->dev)))
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04007217 continue;
7218 netdev_adjacent_sysfs_add(iter->dev, dev,
7219 &iter->dev->adj_list.lower);
7220 netdev_adjacent_sysfs_add(dev, iter->dev,
7221 &dev->adj_list.upper);
7222 }
7223
7224 list_for_each_entry(iter, &dev->adj_list.lower, list) {
Wei Tangbe4da0e2016-06-16 21:30:12 +08007225 if (!net_eq(net, dev_net(iter->dev)))
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04007226 continue;
7227 netdev_adjacent_sysfs_add(iter->dev, dev,
7228 &iter->dev->adj_list.upper);
7229 netdev_adjacent_sysfs_add(dev, iter->dev,
7230 &dev->adj_list.lower);
7231 }
7232}
7233
Eric Dumazet2ce1ee12015-02-04 13:37:44 -08007234static void netdev_adjacent_del_links(struct net_device *dev)
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04007235{
7236 struct netdev_adjacent *iter;
7237
7238 struct net *net = dev_net(dev);
7239
7240 list_for_each_entry(iter, &dev->adj_list.upper, list) {
Wei Tangbe4da0e2016-06-16 21:30:12 +08007241 if (!net_eq(net, dev_net(iter->dev)))
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04007242 continue;
7243 netdev_adjacent_sysfs_del(iter->dev, dev->name,
7244 &iter->dev->adj_list.lower);
7245 netdev_adjacent_sysfs_del(dev, iter->dev->name,
7246 &dev->adj_list.upper);
7247 }
7248
7249 list_for_each_entry(iter, &dev->adj_list.lower, list) {
Wei Tangbe4da0e2016-06-16 21:30:12 +08007250 if (!net_eq(net, dev_net(iter->dev)))
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04007251 continue;
7252 netdev_adjacent_sysfs_del(iter->dev, dev->name,
7253 &iter->dev->adj_list.upper);
7254 netdev_adjacent_sysfs_del(dev, iter->dev->name,
7255 &dev->adj_list.lower);
7256 }
7257}
7258
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01007259void netdev_adjacent_rename_links(struct net_device *dev, char *oldname)
Veaceslav Falico402dae92013-09-25 09:20:09 +02007260{
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01007261 struct netdev_adjacent *iter;
Veaceslav Falico402dae92013-09-25 09:20:09 +02007262
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04007263 struct net *net = dev_net(dev);
7264
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01007265 list_for_each_entry(iter, &dev->adj_list.upper, list) {
Wei Tangbe4da0e2016-06-16 21:30:12 +08007266 if (!net_eq(net, dev_net(iter->dev)))
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04007267 continue;
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01007268 netdev_adjacent_sysfs_del(iter->dev, oldname,
7269 &iter->dev->adj_list.lower);
7270 netdev_adjacent_sysfs_add(iter->dev, dev,
7271 &iter->dev->adj_list.lower);
7272 }
Veaceslav Falico402dae92013-09-25 09:20:09 +02007273
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01007274 list_for_each_entry(iter, &dev->adj_list.lower, list) {
Wei Tangbe4da0e2016-06-16 21:30:12 +08007275 if (!net_eq(net, dev_net(iter->dev)))
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04007276 continue;
Veaceslav Falico5bb025f2014-01-14 21:58:51 +01007277 netdev_adjacent_sysfs_del(iter->dev, oldname,
7278 &iter->dev->adj_list.upper);
7279 netdev_adjacent_sysfs_add(iter->dev, dev,
7280 &iter->dev->adj_list.upper);
7281 }
Veaceslav Falico402dae92013-09-25 09:20:09 +02007282}
Veaceslav Falico402dae92013-09-25 09:20:09 +02007283
7284void *netdev_lower_dev_get_private(struct net_device *dev,
7285 struct net_device *lower_dev)
7286{
7287 struct netdev_adjacent *lower;
7288
7289 if (!lower_dev)
7290 return NULL;
Michal Kubeček6ea29da2015-09-24 10:59:05 +02007291 lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower);
Veaceslav Falico402dae92013-09-25 09:20:09 +02007292 if (!lower)
7293 return NULL;
7294
7295 return lower->private;
7296}
7297EXPORT_SYMBOL(netdev_lower_dev_get_private);
7298
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04007299
Sabrina Dubroca952fcfd2016-08-12 16:10:33 +02007300int dev_get_nest_level(struct net_device *dev)
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04007301{
7302 struct net_device *lower = NULL;
7303 struct list_head *iter;
7304 int max_nest = -1;
7305 int nest;
7306
7307 ASSERT_RTNL();
7308
7309 netdev_for_each_lower_dev(dev, lower, iter) {
Sabrina Dubroca952fcfd2016-08-12 16:10:33 +02007310 nest = dev_get_nest_level(lower);
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04007311 if (max_nest < nest)
7312 max_nest = nest;
7313 }
7314
Sabrina Dubroca952fcfd2016-08-12 16:10:33 +02007315 return max_nest + 1;
Vlad Yasevich4085ebe2014-05-16 17:04:53 -04007316}
7317EXPORT_SYMBOL(dev_get_nest_level);
7318
Jiri Pirko04d48262015-12-03 12:12:15 +01007319/**
7320 * netdev_lower_change - Dispatch event about lower device state change
7321 * @lower_dev: device
7322 * @lower_state_info: state to dispatch
7323 *
7324 * Send NETDEV_CHANGELOWERSTATE to netdev notifiers with info.
7325 * The caller must hold the RTNL lock.
7326 */
7327void netdev_lower_state_changed(struct net_device *lower_dev,
7328 void *lower_state_info)
7329{
David Ahern51d0c0472017-10-04 17:48:45 -07007330 struct netdev_notifier_changelowerstate_info changelowerstate_info = {
7331 .info.dev = lower_dev,
7332 };
Jiri Pirko04d48262015-12-03 12:12:15 +01007333
7334 ASSERT_RTNL();
7335 changelowerstate_info.lower_state_info = lower_state_info;
David Ahern51d0c0472017-10-04 17:48:45 -07007336 call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE,
Jiri Pirko04d48262015-12-03 12:12:15 +01007337 &changelowerstate_info.info);
7338}
7339EXPORT_SYMBOL(netdev_lower_state_changed);
7340
Patrick McHardyb6c40d62008-10-07 15:26:48 -07007341static void dev_change_rx_flags(struct net_device *dev, int flags)
7342{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08007343 const struct net_device_ops *ops = dev->netdev_ops;
7344
Vlad Yasevichd2615bf2013-11-19 20:47:15 -05007345 if (ops->ndo_change_rx_flags)
Stephen Hemmingerd3147742008-11-19 21:32:24 -08007346 ops->ndo_change_rx_flags(dev, flags);
Patrick McHardyb6c40d62008-10-07 15:26:48 -07007347}
7348
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02007349static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
Patrick McHardy4417da62007-06-27 01:28:10 -07007350{
Eric Dumazetb536db92011-11-30 21:42:26 +00007351 unsigned int old_flags = dev->flags;
Eric W. Biedermand04a48b2012-05-23 17:01:57 -06007352 kuid_t uid;
7353 kgid_t gid;
Patrick McHardy4417da62007-06-27 01:28:10 -07007354
Patrick McHardy24023452007-07-14 18:51:31 -07007355 ASSERT_RTNL();
7356
Wang Chendad9b332008-06-18 01:48:28 -07007357 dev->flags |= IFF_PROMISC;
7358 dev->promiscuity += inc;
7359 if (dev->promiscuity == 0) {
7360 /*
7361 * Avoid overflow.
7362 * If inc causes overflow, untouch promisc and return error.
7363 */
7364 if (inc < 0)
7365 dev->flags &= ~IFF_PROMISC;
7366 else {
7367 dev->promiscuity -= inc;
Joe Perches7b6cd1c2012-02-01 10:54:43 +00007368 pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n",
7369 dev->name);
Wang Chendad9b332008-06-18 01:48:28 -07007370 return -EOVERFLOW;
7371 }
7372 }
Patrick McHardy4417da62007-06-27 01:28:10 -07007373 if (dev->flags != old_flags) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00007374 pr_info("device %s %s promiscuous mode\n",
7375 dev->name,
7376 dev->flags & IFF_PROMISC ? "entered" : "left");
David Howells8192b0c2008-11-14 10:39:10 +11007377 if (audit_enabled) {
7378 current_uid_gid(&uid, &gid);
Richard Guy Briggscdfb6b32018-05-12 21:58:20 -04007379 audit_log(audit_context(), GFP_ATOMIC,
7380 AUDIT_ANOM_PROMISCUOUS,
7381 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
7382 dev->name, (dev->flags & IFF_PROMISC),
7383 (old_flags & IFF_PROMISC),
7384 from_kuid(&init_user_ns, audit_get_loginuid(current)),
7385 from_kuid(&init_user_ns, uid),
7386 from_kgid(&init_user_ns, gid),
7387 audit_get_sessionid(current));
David Howells8192b0c2008-11-14 10:39:10 +11007388 }
Patrick McHardy24023452007-07-14 18:51:31 -07007389
Patrick McHardyb6c40d62008-10-07 15:26:48 -07007390 dev_change_rx_flags(dev, IFF_PROMISC);
Patrick McHardy4417da62007-06-27 01:28:10 -07007391 }
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02007392 if (notify)
7393 __dev_notify_flags(dev, old_flags, IFF_PROMISC);
Wang Chendad9b332008-06-18 01:48:28 -07007394 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07007395}
7396
Linus Torvalds1da177e2005-04-16 15:20:36 -07007397/**
7398 * dev_set_promiscuity - update promiscuity count on a device
7399 * @dev: device
7400 * @inc: modifier
7401 *
Stephen Hemminger3041a062006-05-26 13:25:24 -07007402 * Add or remove promiscuity from a device. While the count in the device
Linus Torvalds1da177e2005-04-16 15:20:36 -07007403 * remains above zero the interface remains promiscuous. Once it hits zero
7404 * the device reverts back to normal filtering operation. A negative inc
7405 * value is used to drop promiscuity on the device.
Wang Chendad9b332008-06-18 01:48:28 -07007406 * Return 0 if successful or a negative errno code on error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07007407 */
Wang Chendad9b332008-06-18 01:48:28 -07007408int dev_set_promiscuity(struct net_device *dev, int inc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007409{
Eric Dumazetb536db92011-11-30 21:42:26 +00007410 unsigned int old_flags = dev->flags;
Wang Chendad9b332008-06-18 01:48:28 -07007411 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007412
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02007413 err = __dev_set_promiscuity(dev, inc, true);
Patrick McHardy4b5a6982008-07-06 15:49:08 -07007414 if (err < 0)
Wang Chendad9b332008-06-18 01:48:28 -07007415 return err;
Patrick McHardy4417da62007-06-27 01:28:10 -07007416 if (dev->flags != old_flags)
7417 dev_set_rx_mode(dev);
Wang Chendad9b332008-06-18 01:48:28 -07007418 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007419}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07007420EXPORT_SYMBOL(dev_set_promiscuity);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007421
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02007422static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007423{
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02007424 unsigned int old_flags = dev->flags, old_gflags = dev->gflags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007425
Patrick McHardy24023452007-07-14 18:51:31 -07007426 ASSERT_RTNL();
7427
Linus Torvalds1da177e2005-04-16 15:20:36 -07007428 dev->flags |= IFF_ALLMULTI;
Wang Chendad9b332008-06-18 01:48:28 -07007429 dev->allmulti += inc;
7430 if (dev->allmulti == 0) {
7431 /*
7432 * Avoid overflow.
7433 * If inc causes overflow, untouch allmulti and return error.
7434 */
7435 if (inc < 0)
7436 dev->flags &= ~IFF_ALLMULTI;
7437 else {
7438 dev->allmulti -= inc;
Joe Perches7b6cd1c2012-02-01 10:54:43 +00007439 pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n",
7440 dev->name);
Wang Chendad9b332008-06-18 01:48:28 -07007441 return -EOVERFLOW;
7442 }
7443 }
Patrick McHardy24023452007-07-14 18:51:31 -07007444 if (dev->flags ^ old_flags) {
Patrick McHardyb6c40d62008-10-07 15:26:48 -07007445 dev_change_rx_flags(dev, IFF_ALLMULTI);
Patrick McHardy4417da62007-06-27 01:28:10 -07007446 dev_set_rx_mode(dev);
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02007447 if (notify)
7448 __dev_notify_flags(dev, old_flags,
7449 dev->gflags ^ old_gflags);
Patrick McHardy24023452007-07-14 18:51:31 -07007450 }
Wang Chendad9b332008-06-18 01:48:28 -07007451 return 0;
Patrick McHardy4417da62007-06-27 01:28:10 -07007452}
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02007453
7454/**
7455 * dev_set_allmulti - update allmulti count on a device
7456 * @dev: device
7457 * @inc: modifier
7458 *
7459 * Add or remove reception of all multicast frames to a device. While the
7460 * count in the device remains above zero the interface remains listening
7461 * to all interfaces. Once it hits zero the device reverts back to normal
7462 * filtering operation. A negative @inc value is used to drop the counter
7463 * when releasing a resource needing all multicasts.
7464 * Return 0 if successful or a negative errno code on error.
7465 */
7466
7467int dev_set_allmulti(struct net_device *dev, int inc)
7468{
7469 return __dev_set_allmulti(dev, inc, true);
7470}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07007471EXPORT_SYMBOL(dev_set_allmulti);
Patrick McHardy4417da62007-06-27 01:28:10 -07007472
7473/*
7474 * Upload unicast and multicast address lists to device and
7475 * configure RX filtering. When the device doesn't support unicast
Joe Perches53ccaae2007-12-20 14:02:06 -08007476 * filtering it is put in promiscuous mode while unicast addresses
Patrick McHardy4417da62007-06-27 01:28:10 -07007477 * are present.
7478 */
7479void __dev_set_rx_mode(struct net_device *dev)
7480{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08007481 const struct net_device_ops *ops = dev->netdev_ops;
7482
Patrick McHardy4417da62007-06-27 01:28:10 -07007483 /* dev_open will call this function so the list will stay sane. */
7484 if (!(dev->flags&IFF_UP))
7485 return;
7486
7487 if (!netif_device_present(dev))
YOSHIFUJI Hideaki40b77c92007-07-19 10:43:23 +09007488 return;
Patrick McHardy4417da62007-06-27 01:28:10 -07007489
Jiri Pirko01789342011-08-16 06:29:00 +00007490 if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
Patrick McHardy4417da62007-06-27 01:28:10 -07007491 /* Unicast addresses changes may only happen under the rtnl,
7492 * therefore calling __dev_set_promiscuity here is safe.
7493 */
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08007494 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02007495 __dev_set_promiscuity(dev, 1, false);
Joe Perches2d348d12011-07-25 16:17:35 -07007496 dev->uc_promisc = true;
Jiri Pirko32e7bfc2010-01-25 13:36:10 -08007497 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02007498 __dev_set_promiscuity(dev, -1, false);
Joe Perches2d348d12011-07-25 16:17:35 -07007499 dev->uc_promisc = false;
Patrick McHardy4417da62007-06-27 01:28:10 -07007500 }
Patrick McHardy4417da62007-06-27 01:28:10 -07007501 }
Jiri Pirko01789342011-08-16 06:29:00 +00007502
7503 if (ops->ndo_set_rx_mode)
7504 ops->ndo_set_rx_mode(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07007505}
7506
7507void dev_set_rx_mode(struct net_device *dev)
7508{
David S. Millerb9e40852008-07-15 00:15:08 -07007509 netif_addr_lock_bh(dev);
Patrick McHardy4417da62007-06-27 01:28:10 -07007510 __dev_set_rx_mode(dev);
David S. Millerb9e40852008-07-15 00:15:08 -07007511 netif_addr_unlock_bh(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007512}
7513
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07007514/**
7515 * dev_get_flags - get flags reported to userspace
7516 * @dev: device
7517 *
7518 * Get the combination of flag bits exported through APIs to userspace.
7519 */
Eric Dumazet95c96172012-04-15 05:58:06 +00007520unsigned int dev_get_flags(const struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007521{
Eric Dumazet95c96172012-04-15 05:58:06 +00007522 unsigned int flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007523
7524 flags = (dev->flags & ~(IFF_PROMISC |
7525 IFF_ALLMULTI |
Stefan Rompfb00055a2006-03-20 17:09:11 -08007526 IFF_RUNNING |
7527 IFF_LOWER_UP |
7528 IFF_DORMANT)) |
Linus Torvalds1da177e2005-04-16 15:20:36 -07007529 (dev->gflags & (IFF_PROMISC |
7530 IFF_ALLMULTI));
7531
Stefan Rompfb00055a2006-03-20 17:09:11 -08007532 if (netif_running(dev)) {
7533 if (netif_oper_up(dev))
7534 flags |= IFF_RUNNING;
7535 if (netif_carrier_ok(dev))
7536 flags |= IFF_LOWER_UP;
7537 if (netif_dormant(dev))
7538 flags |= IFF_DORMANT;
7539 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07007540
7541 return flags;
7542}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07007543EXPORT_SYMBOL(dev_get_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007544
Petr Machata6d040322018-12-06 17:05:43 +00007545int __dev_change_flags(struct net_device *dev, unsigned int flags,
7546 struct netlink_ext_ack *extack)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007547{
Eric Dumazetb536db92011-11-30 21:42:26 +00007548 unsigned int old_flags = dev->flags;
Patrick McHardybd380812010-02-26 06:34:53 +00007549 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007550
Patrick McHardy24023452007-07-14 18:51:31 -07007551 ASSERT_RTNL();
7552
Linus Torvalds1da177e2005-04-16 15:20:36 -07007553 /*
7554 * Set the flags on our device.
7555 */
7556
7557 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
7558 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
7559 IFF_AUTOMEDIA)) |
7560 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
7561 IFF_ALLMULTI));
7562
7563 /*
7564 * Load in the correct multicast list now the flags have changed.
7565 */
7566
Patrick McHardyb6c40d62008-10-07 15:26:48 -07007567 if ((old_flags ^ flags) & IFF_MULTICAST)
7568 dev_change_rx_flags(dev, IFF_MULTICAST);
Patrick McHardy24023452007-07-14 18:51:31 -07007569
Patrick McHardy4417da62007-06-27 01:28:10 -07007570 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007571
7572 /*
7573 * Have we downed the interface. We handle IFF_UP ourselves
7574 * according to user attempts to set it, rather than blindly
7575 * setting it.
7576 */
7577
7578 ret = 0;
stephen hemminger7051b882017-07-18 15:59:27 -07007579 if ((old_flags ^ flags) & IFF_UP) {
7580 if (old_flags & IFF_UP)
7581 __dev_close(dev);
7582 else
Petr Machata40c900a2018-12-06 17:05:47 +00007583 ret = __dev_open(dev, extack);
stephen hemminger7051b882017-07-18 15:59:27 -07007584 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07007585
Linus Torvalds1da177e2005-04-16 15:20:36 -07007586 if ((flags ^ dev->gflags) & IFF_PROMISC) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07007587 int inc = (flags & IFF_PROMISC) ? 1 : -1;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02007588 unsigned int old_flags = dev->flags;
Eric Dumazetd1b19df2009-09-03 01:29:39 -07007589
Linus Torvalds1da177e2005-04-16 15:20:36 -07007590 dev->gflags ^= IFF_PROMISC;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02007591
7592 if (__dev_set_promiscuity(dev, inc, false) >= 0)
7593 if (dev->flags != old_flags)
7594 dev_set_rx_mode(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007595 }
7596
7597 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
tchardingeb13da12017-02-09 17:56:06 +11007598 * is important. Some (broken) drivers set IFF_PROMISC, when
7599 * IFF_ALLMULTI is requested not asking us and not reporting.
Linus Torvalds1da177e2005-04-16 15:20:36 -07007600 */
7601 if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
Eric Dumazetd1b19df2009-09-03 01:29:39 -07007602 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
7603
Linus Torvalds1da177e2005-04-16 15:20:36 -07007604 dev->gflags ^= IFF_ALLMULTI;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02007605 __dev_set_allmulti(dev, inc, false);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007606 }
7607
Patrick McHardybd380812010-02-26 06:34:53 +00007608 return ret;
7609}
7610
Nicolas Dichtela528c212013-09-25 12:02:44 +02007611void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
7612 unsigned int gchanges)
Patrick McHardybd380812010-02-26 06:34:53 +00007613{
7614 unsigned int changes = dev->flags ^ old_flags;
7615
Nicolas Dichtela528c212013-09-25 12:02:44 +02007616 if (gchanges)
Alexei Starovoitov7f294052013-10-23 16:02:42 -07007617 rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC);
Nicolas Dichtela528c212013-09-25 12:02:44 +02007618
Patrick McHardybd380812010-02-26 06:34:53 +00007619 if (changes & IFF_UP) {
7620 if (dev->flags & IFF_UP)
7621 call_netdevice_notifiers(NETDEV_UP, dev);
7622 else
7623 call_netdevice_notifiers(NETDEV_DOWN, dev);
7624 }
7625
7626 if (dev->flags & IFF_UP &&
Jiri Pirkobe9efd32013-05-28 01:30:22 +00007627 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) {
David Ahern51d0c0472017-10-04 17:48:45 -07007628 struct netdev_notifier_change_info change_info = {
7629 .info = {
7630 .dev = dev,
7631 },
7632 .flags_changed = changes,
7633 };
Jiri Pirkobe9efd32013-05-28 01:30:22 +00007634
David Ahern51d0c0472017-10-04 17:48:45 -07007635 call_netdevice_notifiers_info(NETDEV_CHANGE, &change_info.info);
Jiri Pirkobe9efd32013-05-28 01:30:22 +00007636 }
Patrick McHardybd380812010-02-26 06:34:53 +00007637}
7638
7639/**
7640 * dev_change_flags - change device settings
7641 * @dev: device
7642 * @flags: device state flags
Petr Machata567c5e12018-12-06 17:05:42 +00007643 * @extack: netlink extended ack
Patrick McHardybd380812010-02-26 06:34:53 +00007644 *
7645 * Change settings on device based state flags. The flags are
7646 * in the userspace exported format.
7647 */
Petr Machata567c5e12018-12-06 17:05:42 +00007648int dev_change_flags(struct net_device *dev, unsigned int flags,
7649 struct netlink_ext_ack *extack)
Patrick McHardybd380812010-02-26 06:34:53 +00007650{
Eric Dumazetb536db92011-11-30 21:42:26 +00007651 int ret;
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02007652 unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags;
Patrick McHardybd380812010-02-26 06:34:53 +00007653
Petr Machata6d040322018-12-06 17:05:43 +00007654 ret = __dev_change_flags(dev, flags, extack);
Patrick McHardybd380812010-02-26 06:34:53 +00007655 if (ret < 0)
7656 return ret;
7657
Nicolas Dichtel991fb3f2013-09-25 12:02:45 +02007658 changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags);
Nicolas Dichtela528c212013-09-25 12:02:44 +02007659 __dev_notify_flags(dev, old_flags, changes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007660 return ret;
7661}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07007662EXPORT_SYMBOL(dev_change_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007663
WANG Congf51048c2017-07-06 15:01:57 -07007664int __dev_set_mtu(struct net_device *dev, int new_mtu)
Veaceslav Falico2315dc92014-01-10 16:56:25 +01007665{
7666 const struct net_device_ops *ops = dev->netdev_ops;
7667
7668 if (ops->ndo_change_mtu)
7669 return ops->ndo_change_mtu(dev, new_mtu);
7670
7671 dev->mtu = new_mtu;
7672 return 0;
7673}
WANG Congf51048c2017-07-06 15:01:57 -07007674EXPORT_SYMBOL(__dev_set_mtu);
Veaceslav Falico2315dc92014-01-10 16:56:25 +01007675
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07007676/**
Stephen Hemminger7a4c53b2018-07-27 13:43:23 -07007677 * dev_set_mtu_ext - Change maximum transfer unit
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07007678 * @dev: device
7679 * @new_mtu: new transfer unit
Stephen Hemminger7a4c53b2018-07-27 13:43:23 -07007680 * @extack: netlink extended ack
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07007681 *
7682 * Change the maximum transfer size of the network device.
7683 */
Stephen Hemminger7a4c53b2018-07-27 13:43:23 -07007684int dev_set_mtu_ext(struct net_device *dev, int new_mtu,
7685 struct netlink_ext_ack *extack)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007686{
Veaceslav Falico2315dc92014-01-10 16:56:25 +01007687 int err, orig_mtu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007688
7689 if (new_mtu == dev->mtu)
7690 return 0;
7691
Jarod Wilson61e84622016-10-07 22:04:33 -04007692 /* MTU must be positive, and in range */
7693 if (new_mtu < 0 || new_mtu < dev->min_mtu) {
Stephen Hemminger7a4c53b2018-07-27 13:43:23 -07007694 NL_SET_ERR_MSG(extack, "mtu less than device minimum");
Linus Torvalds1da177e2005-04-16 15:20:36 -07007695 return -EINVAL;
Jarod Wilson61e84622016-10-07 22:04:33 -04007696 }
7697
7698 if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) {
Stephen Hemminger7a4c53b2018-07-27 13:43:23 -07007699 NL_SET_ERR_MSG(extack, "mtu greater than device maximum");
Jarod Wilson61e84622016-10-07 22:04:33 -04007700 return -EINVAL;
7701 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07007702
7703 if (!netif_device_present(dev))
7704 return -ENODEV;
7705
Veaceslav Falico1d486bf2014-01-16 00:02:18 +01007706 err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev);
7707 err = notifier_to_errno(err);
7708 if (err)
7709 return err;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08007710
Veaceslav Falico2315dc92014-01-10 16:56:25 +01007711 orig_mtu = dev->mtu;
7712 err = __dev_set_mtu(dev, new_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007713
Veaceslav Falico2315dc92014-01-10 16:56:25 +01007714 if (!err) {
Sabrina Dubrocaaf7d6cc2018-10-09 17:48:14 +02007715 err = call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev,
7716 orig_mtu);
Veaceslav Falico2315dc92014-01-10 16:56:25 +01007717 err = notifier_to_errno(err);
7718 if (err) {
7719 /* setting mtu back and notifying everyone again,
7720 * so that they have a chance to revert changes.
7721 */
7722 __dev_set_mtu(dev, orig_mtu);
Sabrina Dubrocaaf7d6cc2018-10-09 17:48:14 +02007723 call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev,
7724 new_mtu);
Veaceslav Falico2315dc92014-01-10 16:56:25 +01007725 }
7726 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07007727 return err;
7728}
Stephen Hemminger7a4c53b2018-07-27 13:43:23 -07007729
7730int dev_set_mtu(struct net_device *dev, int new_mtu)
7731{
7732 struct netlink_ext_ack extack;
7733 int err;
7734
Li RongQinga6bcfc82018-08-03 15:45:21 +08007735 memset(&extack, 0, sizeof(extack));
Stephen Hemminger7a4c53b2018-07-27 13:43:23 -07007736 err = dev_set_mtu_ext(dev, new_mtu, &extack);
Li RongQinga6bcfc82018-08-03 15:45:21 +08007737 if (err && extack._msg)
Stephen Hemminger7a4c53b2018-07-27 13:43:23 -07007738 net_err_ratelimited("%s: %s\n", dev->name, extack._msg);
7739 return err;
7740}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07007741EXPORT_SYMBOL(dev_set_mtu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007742
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07007743/**
Cong Wang6a643dd2018-01-25 18:26:22 -08007744 * dev_change_tx_queue_len - Change TX queue length of a netdevice
7745 * @dev: device
7746 * @new_len: new tx queue length
7747 */
7748int dev_change_tx_queue_len(struct net_device *dev, unsigned long new_len)
7749{
7750 unsigned int orig_len = dev->tx_queue_len;
7751 int res;
7752
7753 if (new_len != (unsigned int)new_len)
7754 return -ERANGE;
7755
7756 if (new_len != orig_len) {
7757 dev->tx_queue_len = new_len;
7758 res = call_netdevice_notifiers(NETDEV_CHANGE_TX_QUEUE_LEN, dev);
7759 res = notifier_to_errno(res);
Tariq Toukan7effaf02018-07-24 14:12:20 +03007760 if (res)
7761 goto err_rollback;
7762 res = dev_qdisc_change_tx_queue_len(dev);
7763 if (res)
7764 goto err_rollback;
Cong Wang6a643dd2018-01-25 18:26:22 -08007765 }
7766
7767 return 0;
Tariq Toukan7effaf02018-07-24 14:12:20 +03007768
7769err_rollback:
7770 netdev_err(dev, "refused to change device tx_queue_len\n");
7771 dev->tx_queue_len = orig_len;
7772 return res;
Cong Wang6a643dd2018-01-25 18:26:22 -08007773}
7774
7775/**
Vlad Dogarucbda10f2011-01-13 23:38:30 +00007776 * dev_set_group - Change group this device belongs to
7777 * @dev: device
7778 * @new_group: group this device should belong to
7779 */
7780void dev_set_group(struct net_device *dev, int new_group)
7781{
7782 dev->group = new_group;
7783}
7784EXPORT_SYMBOL(dev_set_group);
7785
7786/**
Petr Machatad59cdf92018-12-13 11:54:35 +00007787 * dev_pre_changeaddr_notify - Call NETDEV_PRE_CHANGEADDR.
7788 * @dev: device
7789 * @addr: new address
7790 * @extack: netlink extended ack
7791 */
7792int dev_pre_changeaddr_notify(struct net_device *dev, const char *addr,
7793 struct netlink_ext_ack *extack)
7794{
7795 struct netdev_notifier_pre_changeaddr_info info = {
7796 .info.dev = dev,
7797 .info.extack = extack,
7798 .dev_addr = addr,
7799 };
7800 int rc;
7801
7802 rc = call_netdevice_notifiers_info(NETDEV_PRE_CHANGEADDR, &info.info);
7803 return notifier_to_errno(rc);
7804}
7805EXPORT_SYMBOL(dev_pre_changeaddr_notify);
7806
7807/**
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07007808 * dev_set_mac_address - Change Media Access Control Address
7809 * @dev: device
7810 * @sa: new address
Petr Machata3a37a962018-12-13 11:54:30 +00007811 * @extack: netlink extended ack
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07007812 *
7813 * Change the hardware (MAC) address of the device
7814 */
Petr Machata3a37a962018-12-13 11:54:30 +00007815int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa,
7816 struct netlink_ext_ack *extack)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007817{
Stephen Hemmingerd3147742008-11-19 21:32:24 -08007818 const struct net_device_ops *ops = dev->netdev_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007819 int err;
7820
Stephen Hemmingerd3147742008-11-19 21:32:24 -08007821 if (!ops->ndo_set_mac_address)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007822 return -EOPNOTSUPP;
7823 if (sa->sa_family != dev->type)
7824 return -EINVAL;
7825 if (!netif_device_present(dev))
7826 return -ENODEV;
Petr Machatad59cdf92018-12-13 11:54:35 +00007827 err = dev_pre_changeaddr_notify(dev, sa->sa_data, extack);
7828 if (err)
7829 return err;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08007830 err = ops->ndo_set_mac_address(dev, sa);
Jiri Pirkof6521512013-01-01 03:30:14 +00007831 if (err)
7832 return err;
Jiri Pirkofbdeca22013-01-01 03:30:16 +00007833 dev->addr_assign_type = NET_ADDR_SET;
Jiri Pirkof6521512013-01-01 03:30:14 +00007834 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04007835 add_device_randomness(dev->dev_addr, dev->addr_len);
Jiri Pirkof6521512013-01-01 03:30:14 +00007836 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007837}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07007838EXPORT_SYMBOL(dev_set_mac_address);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007839
Jiri Pirko4bf84c32012-12-27 23:49:37 +00007840/**
7841 * dev_change_carrier - Change device carrier
7842 * @dev: device
Randy Dunlap691b3b72013-03-04 12:32:43 +00007843 * @new_carrier: new value
Jiri Pirko4bf84c32012-12-27 23:49:37 +00007844 *
7845 * Change device carrier
7846 */
7847int dev_change_carrier(struct net_device *dev, bool new_carrier)
7848{
7849 const struct net_device_ops *ops = dev->netdev_ops;
7850
7851 if (!ops->ndo_change_carrier)
7852 return -EOPNOTSUPP;
7853 if (!netif_device_present(dev))
7854 return -ENODEV;
7855 return ops->ndo_change_carrier(dev, new_carrier);
7856}
7857EXPORT_SYMBOL(dev_change_carrier);
7858
Linus Torvalds1da177e2005-04-16 15:20:36 -07007859/**
Jiri Pirko66b52b02013-07-29 18:16:49 +02007860 * dev_get_phys_port_id - Get device physical port ID
7861 * @dev: device
7862 * @ppid: port ID
7863 *
7864 * Get device physical port ID
7865 */
7866int dev_get_phys_port_id(struct net_device *dev,
Jiri Pirko02637fc2014-11-28 14:34:16 +01007867 struct netdev_phys_item_id *ppid)
Jiri Pirko66b52b02013-07-29 18:16:49 +02007868{
7869 const struct net_device_ops *ops = dev->netdev_ops;
7870
7871 if (!ops->ndo_get_phys_port_id)
7872 return -EOPNOTSUPP;
7873 return ops->ndo_get_phys_port_id(dev, ppid);
7874}
7875EXPORT_SYMBOL(dev_get_phys_port_id);
7876
7877/**
David Aherndb24a902015-03-17 20:23:15 -06007878 * dev_get_phys_port_name - Get device physical port name
7879 * @dev: device
7880 * @name: port name
Luis de Bethencourted49e652016-03-21 16:31:14 +00007881 * @len: limit of bytes to copy to name
David Aherndb24a902015-03-17 20:23:15 -06007882 *
7883 * Get device physical port name
7884 */
7885int dev_get_phys_port_name(struct net_device *dev,
7886 char *name, size_t len)
7887{
7888 const struct net_device_ops *ops = dev->netdev_ops;
7889
7890 if (!ops->ndo_get_phys_port_name)
7891 return -EOPNOTSUPP;
7892 return ops->ndo_get_phys_port_name(dev, name, len);
7893}
7894EXPORT_SYMBOL(dev_get_phys_port_name);
7895
7896/**
Florian Fainellid6abc5962019-02-06 09:45:35 -08007897 * dev_get_port_parent_id - Get the device's port parent identifier
7898 * @dev: network device
7899 * @ppid: pointer to a storage for the port's parent identifier
7900 * @recurse: allow/disallow recursion to lower devices
7901 *
7902 * Get the devices's port parent identifier
7903 */
7904int dev_get_port_parent_id(struct net_device *dev,
7905 struct netdev_phys_item_id *ppid,
7906 bool recurse)
7907{
7908 const struct net_device_ops *ops = dev->netdev_ops;
7909 struct netdev_phys_item_id first = { };
7910 struct net_device *lower_dev;
7911 struct list_head *iter;
7912 int err = -EOPNOTSUPP;
7913
7914 if (ops->ndo_get_port_parent_id)
7915 return ops->ndo_get_port_parent_id(dev, ppid);
7916
7917 if (!recurse)
7918 return err;
7919
7920 netdev_for_each_lower_dev(dev, lower_dev, iter) {
7921 err = dev_get_port_parent_id(lower_dev, ppid, recurse);
7922 if (err)
7923 break;
7924 if (!first.id_len)
7925 first = *ppid;
7926 else if (memcmp(&first, ppid, sizeof(*ppid)))
7927 return -ENODATA;
7928 }
7929
7930 return err;
7931}
7932EXPORT_SYMBOL(dev_get_port_parent_id);
7933
7934/**
7935 * netdev_port_same_parent_id - Indicate if two network devices have
7936 * the same port parent identifier
7937 * @a: first network device
7938 * @b: second network device
7939 */
7940bool netdev_port_same_parent_id(struct net_device *a, struct net_device *b)
7941{
7942 struct netdev_phys_item_id a_id = { };
7943 struct netdev_phys_item_id b_id = { };
7944
7945 if (dev_get_port_parent_id(a, &a_id, true) ||
7946 dev_get_port_parent_id(b, &b_id, true))
7947 return false;
7948
7949 return netdev_phys_item_id_same(&a_id, &b_id);
7950}
7951EXPORT_SYMBOL(netdev_port_same_parent_id);
7952
7953/**
Anuradha Karuppiahd746d702015-07-14 13:43:19 -07007954 * dev_change_proto_down - update protocol port state information
7955 * @dev: device
7956 * @proto_down: new value
7957 *
7958 * This info can be used by switch drivers to set the phys state of the
7959 * port.
7960 */
7961int dev_change_proto_down(struct net_device *dev, bool proto_down)
7962{
7963 const struct net_device_ops *ops = dev->netdev_ops;
7964
7965 if (!ops->ndo_change_proto_down)
7966 return -EOPNOTSUPP;
7967 if (!netif_device_present(dev))
7968 return -ENODEV;
7969 return ops->ndo_change_proto_down(dev, proto_down);
7970}
7971EXPORT_SYMBOL(dev_change_proto_down);
7972
Andy Roulinb5899672019-02-22 18:06:36 +00007973/**
7974 * dev_change_proto_down_generic - generic implementation for
7975 * ndo_change_proto_down that sets carrier according to
7976 * proto_down.
7977 *
7978 * @dev: device
7979 * @proto_down: new value
7980 */
7981int dev_change_proto_down_generic(struct net_device *dev, bool proto_down)
7982{
7983 if (proto_down)
7984 netif_carrier_off(dev);
7985 else
7986 netif_carrier_on(dev);
7987 dev->proto_down = proto_down;
7988 return 0;
7989}
7990EXPORT_SYMBOL(dev_change_proto_down_generic);
7991
Jakub Kicinskia25717d2018-07-11 20:36:41 -07007992u32 __dev_xdp_query(struct net_device *dev, bpf_op_t bpf_op,
7993 enum bpf_netdev_command cmd)
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02007994{
Jakub Kicinskif4e63522017-11-03 13:56:16 -07007995 struct netdev_bpf xdp;
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02007996
Jakub Kicinskia25717d2018-07-11 20:36:41 -07007997 if (!bpf_op)
7998 return 0;
Martin KaFai Lau58038692017-06-15 17:29:09 -07007999
Jakub Kicinskia25717d2018-07-11 20:36:41 -07008000 memset(&xdp, 0, sizeof(xdp));
8001 xdp.command = cmd;
8002
8003 /* Query must always succeed. */
8004 WARN_ON(bpf_op(dev, &xdp) < 0 && cmd == XDP_QUERY_PROG);
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02008005
Jakub Kicinski6b867582018-07-11 20:36:39 -07008006 return xdp.prog_id;
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02008007}
8008
Jakub Kicinskif4e63522017-11-03 13:56:16 -07008009static int dev_xdp_install(struct net_device *dev, bpf_op_t bpf_op,
Jakub Kicinski32d60272017-06-21 18:25:03 -07008010 struct netlink_ext_ack *extack, u32 flags,
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02008011 struct bpf_prog *prog)
8012{
Jakub Kicinskif4e63522017-11-03 13:56:16 -07008013 struct netdev_bpf xdp;
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02008014
8015 memset(&xdp, 0, sizeof(xdp));
Jakub Kicinskiee5d0322017-06-21 18:25:04 -07008016 if (flags & XDP_FLAGS_HW_MODE)
8017 xdp.command = XDP_SETUP_PROG_HW;
8018 else
8019 xdp.command = XDP_SETUP_PROG;
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02008020 xdp.extack = extack;
Jakub Kicinski32d60272017-06-21 18:25:03 -07008021 xdp.flags = flags;
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02008022 xdp.prog = prog;
8023
Jakub Kicinskif4e63522017-11-03 13:56:16 -07008024 return bpf_op(dev, &xdp);
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02008025}
8026
Jakub Kicinskibd0b2e72017-12-01 15:08:57 -08008027static void dev_xdp_uninstall(struct net_device *dev)
8028{
8029 struct netdev_bpf xdp;
8030 bpf_op_t ndo_bpf;
8031
8032 /* Remove generic XDP */
8033 WARN_ON(dev_xdp_install(dev, generic_xdp_install, NULL, 0, NULL));
8034
8035 /* Remove from the driver */
8036 ndo_bpf = dev->netdev_ops->ndo_bpf;
8037 if (!ndo_bpf)
8038 return;
8039
Jakub Kicinskia25717d2018-07-11 20:36:41 -07008040 memset(&xdp, 0, sizeof(xdp));
8041 xdp.command = XDP_QUERY_PROG;
8042 WARN_ON(ndo_bpf(dev, &xdp));
8043 if (xdp.prog_id)
8044 WARN_ON(dev_xdp_install(dev, ndo_bpf, NULL, xdp.prog_flags,
8045 NULL));
Jakub Kicinskibd0b2e72017-12-01 15:08:57 -08008046
Jakub Kicinskia25717d2018-07-11 20:36:41 -07008047 /* Remove HW offload */
8048 memset(&xdp, 0, sizeof(xdp));
8049 xdp.command = XDP_QUERY_PROG_HW;
8050 if (!ndo_bpf(dev, &xdp) && xdp.prog_id)
8051 WARN_ON(dev_xdp_install(dev, ndo_bpf, NULL, xdp.prog_flags,
8052 NULL));
Jakub Kicinskibd0b2e72017-12-01 15:08:57 -08008053}
8054
Anuradha Karuppiahd746d702015-07-14 13:43:19 -07008055/**
Brenden Blancoa7862b42016-07-19 12:16:48 -07008056 * dev_change_xdp_fd - set or clear a bpf program for a device rx path
8057 * @dev: device
Jakub Kicinskib5d60982017-05-01 15:53:43 -07008058 * @extack: netlink extended ack
Brenden Blancoa7862b42016-07-19 12:16:48 -07008059 * @fd: new program fd or negative value to clear
Daniel Borkmann85de8572016-11-28 23:16:54 +01008060 * @flags: xdp-related flags
Brenden Blancoa7862b42016-07-19 12:16:48 -07008061 *
8062 * Set or clear a bpf program for a device
8063 */
Jakub Kicinskiddf9f972017-04-30 21:46:46 -07008064int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
8065 int fd, u32 flags)
Brenden Blancoa7862b42016-07-19 12:16:48 -07008066{
8067 const struct net_device_ops *ops = dev->netdev_ops;
Jakub Kicinskia25717d2018-07-11 20:36:41 -07008068 enum bpf_netdev_command query;
Brenden Blancoa7862b42016-07-19 12:16:48 -07008069 struct bpf_prog *prog = NULL;
Jakub Kicinskif4e63522017-11-03 13:56:16 -07008070 bpf_op_t bpf_op, bpf_chk;
Jakub Kicinski9ee963d2019-02-05 20:03:21 -08008071 bool offload;
Brenden Blancoa7862b42016-07-19 12:16:48 -07008072 int err;
8073
Daniel Borkmann85de8572016-11-28 23:16:54 +01008074 ASSERT_RTNL();
8075
Jakub Kicinski9ee963d2019-02-05 20:03:21 -08008076 offload = flags & XDP_FLAGS_HW_MODE;
8077 query = offload ? XDP_QUERY_PROG_HW : XDP_QUERY_PROG;
Jakub Kicinskia25717d2018-07-11 20:36:41 -07008078
Jakub Kicinskif4e63522017-11-03 13:56:16 -07008079 bpf_op = bpf_chk = ops->ndo_bpf;
Maciej Fijalkowski01dde202019-02-01 22:42:27 +01008080 if (!bpf_op && (flags & (XDP_FLAGS_DRV_MODE | XDP_FLAGS_HW_MODE))) {
8081 NL_SET_ERR_MSG(extack, "underlying driver does not support XDP in native mode");
Daniel Borkmann0489df92017-05-12 01:04:45 +02008082 return -EOPNOTSUPP;
Maciej Fijalkowski01dde202019-02-01 22:42:27 +01008083 }
Jakub Kicinskif4e63522017-11-03 13:56:16 -07008084 if (!bpf_op || (flags & XDP_FLAGS_SKB_MODE))
8085 bpf_op = generic_xdp_install;
8086 if (bpf_op == bpf_chk)
8087 bpf_chk = generic_xdp_install;
David S. Millerb5cdae32017-04-18 15:36:58 -04008088
Brenden Blancoa7862b42016-07-19 12:16:48 -07008089 if (fd >= 0) {
Jakub Kicinski9ee963d2019-02-05 20:03:21 -08008090 if (!offload && __dev_xdp_query(dev, bpf_chk, XDP_QUERY_PROG)) {
Maciej Fijalkowski01dde202019-02-01 22:42:27 +01008091 NL_SET_ERR_MSG(extack, "native and generic XDP can't be active at the same time");
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02008092 return -EEXIST;
Maciej Fijalkowski01dde202019-02-01 22:42:27 +01008093 }
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02008094 if ((flags & XDP_FLAGS_UPDATE_IF_NOEXIST) &&
Maciej Fijalkowski01dde202019-02-01 22:42:27 +01008095 __dev_xdp_query(dev, bpf_op, query)) {
8096 NL_SET_ERR_MSG(extack, "XDP program already attached");
Daniel Borkmannd67b9cd2017-05-12 01:04:46 +02008097 return -EBUSY;
Maciej Fijalkowski01dde202019-02-01 22:42:27 +01008098 }
Daniel Borkmann85de8572016-11-28 23:16:54 +01008099
Jakub Kicinski288b3de2017-11-20 15:21:54 -08008100 prog = bpf_prog_get_type_dev(fd, BPF_PROG_TYPE_XDP,
8101 bpf_op == ops->ndo_bpf);
Brenden Blancoa7862b42016-07-19 12:16:48 -07008102 if (IS_ERR(prog))
8103 return PTR_ERR(prog);
Jakub Kicinski441a3302017-11-20 15:21:55 -08008104
Jakub Kicinski9ee963d2019-02-05 20:03:21 -08008105 if (!offload && bpf_prog_is_dev_bound(prog->aux)) {
Jakub Kicinski441a3302017-11-20 15:21:55 -08008106 NL_SET_ERR_MSG(extack, "using device-bound program without HW_MODE flag is not supported");
8107 bpf_prog_put(prog);
8108 return -EINVAL;
8109 }
Brenden Blancoa7862b42016-07-19 12:16:48 -07008110 }
8111
Jakub Kicinskif4e63522017-11-03 13:56:16 -07008112 err = dev_xdp_install(dev, bpf_op, extack, flags, prog);
Brenden Blancoa7862b42016-07-19 12:16:48 -07008113 if (err < 0 && prog)
8114 bpf_prog_put(prog);
8115
8116 return err;
8117}
Brenden Blancoa7862b42016-07-19 12:16:48 -07008118
8119/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07008120 * dev_new_index - allocate an ifindex
Randy Dunlapc4ea43c2007-10-12 21:17:49 -07008121 * @net: the applicable net namespace
Linus Torvalds1da177e2005-04-16 15:20:36 -07008122 *
8123 * Returns a suitable unique value for a new device interface
8124 * number. The caller must hold the rtnl semaphore or the
8125 * dev_base_lock to be sure it remains unique.
8126 */
Eric W. Biederman881d9662007-09-17 11:56:21 -07008127static int dev_new_index(struct net *net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07008128{
Pavel Emelyanovaa79e662012-08-08 21:53:19 +00008129 int ifindex = net->ifindex;
tchardingf4563a72017-02-09 17:56:07 +11008130
Linus Torvalds1da177e2005-04-16 15:20:36 -07008131 for (;;) {
8132 if (++ifindex <= 0)
8133 ifindex = 1;
Eric W. Biederman881d9662007-09-17 11:56:21 -07008134 if (!__dev_get_by_index(net, ifindex))
Pavel Emelyanovaa79e662012-08-08 21:53:19 +00008135 return net->ifindex = ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008136 }
8137}
8138
Linus Torvalds1da177e2005-04-16 15:20:36 -07008139/* Delayed registration/unregisteration */
Denis Cheng3b5b34f2007-12-07 00:49:17 -08008140static LIST_HEAD(net_todo_list);
Cong Wang200b9162014-05-12 15:11:20 -07008141DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008142
Stephen Hemminger6f05f622007-03-08 20:46:03 -08008143static void net_set_todo(struct net_device *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07008144{
Linus Torvalds1da177e2005-04-16 15:20:36 -07008145 list_add_tail(&dev->todo_list, &net_todo_list);
Eric W. Biederman50624c92013-09-23 21:19:49 -07008146 dev_net(dev)->dev_unreg_count++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008147}
8148
Eric Dumazet9b5e3832009-10-27 07:04:19 +00008149static void rollback_registered_many(struct list_head *head)
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07008150{
Krishna Kumare93737b2009-12-08 22:26:02 +00008151 struct net_device *dev, *tmp;
Eric W. Biederman5cde2822013-10-05 19:26:05 -07008152 LIST_HEAD(close_head);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00008153
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07008154 BUG_ON(dev_boot_phase);
8155 ASSERT_RTNL();
8156
Krishna Kumare93737b2009-12-08 22:26:02 +00008157 list_for_each_entry_safe(dev, tmp, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00008158 /* Some devices call without registering
Krishna Kumare93737b2009-12-08 22:26:02 +00008159 * for initialization unwind. Remove those
8160 * devices and proceed with the remaining.
Eric Dumazet9b5e3832009-10-27 07:04:19 +00008161 */
8162 if (dev->reg_state == NETREG_UNINITIALIZED) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00008163 pr_debug("unregister_netdevice: device %s/%p never was registered\n",
8164 dev->name, dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07008165
Eric Dumazet9b5e3832009-10-27 07:04:19 +00008166 WARN_ON(1);
Krishna Kumare93737b2009-12-08 22:26:02 +00008167 list_del(&dev->unreg_list);
8168 continue;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00008169 }
Eric Dumazet449f4542011-05-19 12:24:16 +00008170 dev->dismantle = true;
Eric Dumazet9b5e3832009-10-27 07:04:19 +00008171 BUG_ON(dev->reg_state != NETREG_REGISTERED);
Octavian Purdila44345722010-12-13 12:44:07 +00008172 }
Eric Dumazet9b5e3832009-10-27 07:04:19 +00008173
Octavian Purdila44345722010-12-13 12:44:07 +00008174 /* If device is running, close it first. */
Eric W. Biederman5cde2822013-10-05 19:26:05 -07008175 list_for_each_entry(dev, head, unreg_list)
8176 list_add_tail(&dev->close_list, &close_head);
David S. Miller99c4a262015-03-18 22:52:33 -04008177 dev_close_many(&close_head, true);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00008178
Octavian Purdila44345722010-12-13 12:44:07 +00008179 list_for_each_entry(dev, head, unreg_list) {
Eric Dumazet9b5e3832009-10-27 07:04:19 +00008180 /* And unlink it from device chain. */
8181 unlist_netdevice(dev);
8182
8183 dev->reg_state = NETREG_UNREGISTERING;
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07008184 }
Eric Dumazet41852492016-08-26 12:50:39 -07008185 flush_all_backlogs();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07008186
Eric Dumazet9b5e3832009-10-27 07:04:19 +00008187 synchronize_net();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07008188
Eric Dumazet9b5e3832009-10-27 07:04:19 +00008189 list_for_each_entry(dev, head, unreg_list) {
Mahesh Bandewar395eea62014-12-03 13:46:24 -08008190 struct sk_buff *skb = NULL;
8191
Eric Dumazet9b5e3832009-10-27 07:04:19 +00008192 /* Shutdown queueing discipline. */
8193 dev_shutdown(dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07008194
Jakub Kicinskibd0b2e72017-12-01 15:08:57 -08008195 dev_xdp_uninstall(dev);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07008196
Eric Dumazet9b5e3832009-10-27 07:04:19 +00008197 /* Notify protocols, that we are about to destroy
tchardingeb13da12017-02-09 17:56:06 +11008198 * this device. They should clean all the things.
8199 */
Eric Dumazet9b5e3832009-10-27 07:04:19 +00008200 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
8201
Mahesh Bandewar395eea62014-12-03 13:46:24 -08008202 if (!dev->rtnl_link_ops ||
8203 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
Vlad Yasevich3d3ea5a2017-05-27 10:14:34 -04008204 skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U, 0,
Nicolas Dichtel38e01b32018-01-25 15:01:39 +01008205 GFP_KERNEL, NULL, 0);
Mahesh Bandewar395eea62014-12-03 13:46:24 -08008206
Eric Dumazet9b5e3832009-10-27 07:04:19 +00008207 /*
8208 * Flush the unicast and multicast chains
8209 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00008210 dev_uc_flush(dev);
Jiri Pirko22bedad32010-04-01 21:22:57 +00008211 dev_mc_flush(dev);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00008212
8213 if (dev->netdev_ops->ndo_uninit)
8214 dev->netdev_ops->ndo_uninit(dev);
8215
Mahesh Bandewar395eea62014-12-03 13:46:24 -08008216 if (skb)
8217 rtmsg_ifinfo_send(skb, dev, GFP_KERNEL);
Roopa Prabhu56bfa7e2014-05-01 11:40:30 -07008218
Jiri Pirko9ff162a2013-01-03 22:48:49 +00008219 /* Notifier chain MUST detach us all upper devices. */
8220 WARN_ON(netdev_has_any_upper_dev(dev));
David Ahern0f524a82016-10-17 19:15:52 -07008221 WARN_ON(netdev_has_any_lower_dev(dev));
Eric Dumazet9b5e3832009-10-27 07:04:19 +00008222
8223 /* Remove entries from kobject tree */
8224 netdev_unregister_kobject(dev);
Alexander Duyck024e9672013-01-10 08:57:46 +00008225#ifdef CONFIG_XPS
8226 /* Remove XPS queueing entries */
8227 netif_reset_xps_queues_gt(dev, 0);
8228#endif
Eric Dumazet9b5e3832009-10-27 07:04:19 +00008229 }
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07008230
Eric W. Biederman850a5452011-10-13 22:25:23 +00008231 synchronize_net();
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07008232
Eric W. Biedermana5ee1552009-11-29 15:45:58 +00008233 list_for_each_entry(dev, head, unreg_list)
Eric Dumazet9b5e3832009-10-27 07:04:19 +00008234 dev_put(dev);
8235}
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07008236
Eric Dumazet9b5e3832009-10-27 07:04:19 +00008237static void rollback_registered(struct net_device *dev)
8238{
8239 LIST_HEAD(single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07008240
Eric Dumazet9b5e3832009-10-27 07:04:19 +00008241 list_add(&dev->unreg_list, &single);
8242 rollback_registered_many(&single);
Eric Dumazetceaaec982011-02-17 22:59:19 +00008243 list_del(&single);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07008244}
8245
Jarod Wilsonfd867d52015-11-02 21:55:59 -05008246static netdev_features_t netdev_sync_upper_features(struct net_device *lower,
8247 struct net_device *upper, netdev_features_t features)
8248{
8249 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
8250 netdev_features_t feature;
Jarod Wilson5ba3f7d2015-11-03 10:15:59 -05008251 int feature_bit;
Jarod Wilsonfd867d52015-11-02 21:55:59 -05008252
Hauke Mehrtens3b89ea92019-02-15 17:58:54 +01008253 for_each_netdev_feature(upper_disables, feature_bit) {
Jarod Wilson5ba3f7d2015-11-03 10:15:59 -05008254 feature = __NETIF_F_BIT(feature_bit);
Jarod Wilsonfd867d52015-11-02 21:55:59 -05008255 if (!(upper->wanted_features & feature)
8256 && (features & feature)) {
8257 netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n",
8258 &feature, upper->name);
8259 features &= ~feature;
8260 }
8261 }
8262
8263 return features;
8264}
8265
8266static void netdev_sync_lower_features(struct net_device *upper,
8267 struct net_device *lower, netdev_features_t features)
8268{
8269 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
8270 netdev_features_t feature;
Jarod Wilson5ba3f7d2015-11-03 10:15:59 -05008271 int feature_bit;
Jarod Wilsonfd867d52015-11-02 21:55:59 -05008272
Hauke Mehrtens3b89ea92019-02-15 17:58:54 +01008273 for_each_netdev_feature(upper_disables, feature_bit) {
Jarod Wilson5ba3f7d2015-11-03 10:15:59 -05008274 feature = __NETIF_F_BIT(feature_bit);
Jarod Wilsonfd867d52015-11-02 21:55:59 -05008275 if (!(features & feature) && (lower->features & feature)) {
8276 netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n",
8277 &feature, lower->name);
8278 lower->wanted_features &= ~feature;
8279 netdev_update_features(lower);
8280
8281 if (unlikely(lower->features & feature))
8282 netdev_WARN(upper, "failed to disable %pNF on %s!\n",
8283 &feature, lower->name);
8284 }
8285 }
8286}
8287
Michał Mirosławc8f44af2011-11-15 15:29:55 +00008288static netdev_features_t netdev_fix_features(struct net_device *dev,
8289 netdev_features_t features)
Herbert Xub63365a2008-10-23 01:11:29 -07008290{
Michał Mirosław57422dc2011-01-22 12:14:12 +00008291 /* Fix illegal checksum combinations */
8292 if ((features & NETIF_F_HW_CSUM) &&
8293 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04008294 netdev_warn(dev, "mixed HW and IP checksum settings.\n");
Michał Mirosław57422dc2011-01-22 12:14:12 +00008295 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
8296 }
8297
Herbert Xub63365a2008-10-23 01:11:29 -07008298 /* TSO requires that SG is present as well. */
Ben Hutchingsea2d3682011-04-12 14:38:37 +00008299 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04008300 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
Ben Hutchingsea2d3682011-04-12 14:38:37 +00008301 features &= ~NETIF_F_ALL_TSO;
Herbert Xub63365a2008-10-23 01:11:29 -07008302 }
8303
Pravin B Shelarec5f0612013-03-07 09:28:01 +00008304 if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
8305 !(features & NETIF_F_IP_CSUM)) {
8306 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
8307 features &= ~NETIF_F_TSO;
8308 features &= ~NETIF_F_TSO_ECN;
8309 }
8310
8311 if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
8312 !(features & NETIF_F_IPV6_CSUM)) {
8313 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
8314 features &= ~NETIF_F_TSO6;
8315 }
8316
Alexander Duyckb1dc4972016-05-02 09:38:24 -07008317 /* TSO with IPv4 ID mangling requires IPv4 TSO be enabled */
8318 if ((features & NETIF_F_TSO_MANGLEID) && !(features & NETIF_F_TSO))
8319 features &= ~NETIF_F_TSO_MANGLEID;
8320
Ben Hutchings31d8b9e2011-04-12 14:47:15 +00008321 /* TSO ECN requires that TSO is present as well. */
8322 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
8323 features &= ~NETIF_F_TSO_ECN;
8324
Michał Mirosław212b5732011-02-15 16:59:16 +00008325 /* Software GSO depends on SG. */
8326 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
Michał Mirosław6f404e42011-05-16 15:14:21 -04008327 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
Michał Mirosław212b5732011-02-15 16:59:16 +00008328 features &= ~NETIF_F_GSO;
8329 }
8330
Alexander Duyck802ab552016-04-10 21:45:03 -04008331 /* GSO partial features require GSO partial be set */
8332 if ((features & dev->gso_partial_features) &&
8333 !(features & NETIF_F_GSO_PARTIAL)) {
8334 netdev_dbg(dev,
8335 "Dropping partially supported GSO features since no GSO partial.\n");
8336 features &= ~dev->gso_partial_features;
8337 }
8338
Michael Chanfb1f5f72017-12-16 03:09:40 -05008339 if (!(features & NETIF_F_RXCSUM)) {
8340 /* NETIF_F_GRO_HW implies doing RXCSUM since every packet
8341 * successfully merged by hardware must also have the
8342 * checksum verified by hardware. If the user does not
8343 * want to enable RXCSUM, logically, we should disable GRO_HW.
8344 */
8345 if (features & NETIF_F_GRO_HW) {
8346 netdev_dbg(dev, "Dropping NETIF_F_GRO_HW since no RXCSUM feature.\n");
8347 features &= ~NETIF_F_GRO_HW;
8348 }
8349 }
8350
Gal Pressmande8d5ab2018-03-12 11:48:49 +02008351 /* LRO/HW-GRO features cannot be combined with RX-FCS */
8352 if (features & NETIF_F_RXFCS) {
8353 if (features & NETIF_F_LRO) {
8354 netdev_dbg(dev, "Dropping LRO feature since RX-FCS is requested.\n");
8355 features &= ~NETIF_F_LRO;
8356 }
8357
8358 if (features & NETIF_F_GRO_HW) {
8359 netdev_dbg(dev, "Dropping HW-GRO feature since RX-FCS is requested.\n");
8360 features &= ~NETIF_F_GRO_HW;
8361 }
Gal Pressmane6c6a922018-03-04 14:12:04 +02008362 }
8363
Herbert Xub63365a2008-10-23 01:11:29 -07008364 return features;
8365}
Herbert Xub63365a2008-10-23 01:11:29 -07008366
Michał Mirosław6cb6a272011-04-02 22:48:47 -07008367int __netdev_update_features(struct net_device *dev)
Michał Mirosław5455c692011-02-15 16:59:17 +00008368{
Jarod Wilsonfd867d52015-11-02 21:55:59 -05008369 struct net_device *upper, *lower;
Michał Mirosławc8f44af2011-11-15 15:29:55 +00008370 netdev_features_t features;
Jarod Wilsonfd867d52015-11-02 21:55:59 -05008371 struct list_head *iter;
Jarod Wilsone7868a82015-11-03 23:09:32 -05008372 int err = -1;
Michał Mirosław5455c692011-02-15 16:59:17 +00008373
Michał Mirosław87267482011-04-12 09:56:38 +00008374 ASSERT_RTNL();
8375
Michał Mirosław5455c692011-02-15 16:59:17 +00008376 features = netdev_get_wanted_features(dev);
8377
8378 if (dev->netdev_ops->ndo_fix_features)
8379 features = dev->netdev_ops->ndo_fix_features(dev, features);
8380
8381 /* driver might be less strict about feature dependencies */
8382 features = netdev_fix_features(dev, features);
8383
Jarod Wilsonfd867d52015-11-02 21:55:59 -05008384 /* some features can't be enabled if they're off an an upper device */
8385 netdev_for_each_upper_dev_rcu(dev, upper, iter)
8386 features = netdev_sync_upper_features(dev, upper, features);
8387
Michał Mirosław5455c692011-02-15 16:59:17 +00008388 if (dev->features == features)
Jarod Wilsone7868a82015-11-03 23:09:32 -05008389 goto sync_lower;
Michał Mirosław5455c692011-02-15 16:59:17 +00008390
Michał Mirosławc8f44af2011-11-15 15:29:55 +00008391 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
8392 &dev->features, &features);
Michał Mirosław5455c692011-02-15 16:59:17 +00008393
8394 if (dev->netdev_ops->ndo_set_features)
8395 err = dev->netdev_ops->ndo_set_features(dev, features);
Nikolay Aleksandrov5f8dc332015-11-13 14:54:01 +01008396 else
8397 err = 0;
Michał Mirosław5455c692011-02-15 16:59:17 +00008398
Michał Mirosław6cb6a272011-04-02 22:48:47 -07008399 if (unlikely(err < 0)) {
Michał Mirosław5455c692011-02-15 16:59:17 +00008400 netdev_err(dev,
Michał Mirosławc8f44af2011-11-15 15:29:55 +00008401 "set_features() failed (%d); wanted %pNF, left %pNF\n",
8402 err, &features, &dev->features);
Nikolay Aleksandrov17b85d22015-11-17 15:49:06 +01008403 /* return non-0 since some features might have changed and
8404 * it's better to fire a spurious notification than miss it
8405 */
8406 return -1;
Michał Mirosław6cb6a272011-04-02 22:48:47 -07008407 }
8408
Jarod Wilsone7868a82015-11-03 23:09:32 -05008409sync_lower:
Jarod Wilsonfd867d52015-11-02 21:55:59 -05008410 /* some features must be disabled on lower devices when disabled
8411 * on an upper device (think: bonding master or bridge)
8412 */
8413 netdev_for_each_lower_dev(dev, lower, iter)
8414 netdev_sync_lower_features(dev, lower, features);
8415
Sabrina Dubrocaae847f42017-07-21 12:49:31 +02008416 if (!err) {
8417 netdev_features_t diff = features ^ dev->features;
8418
8419 if (diff & NETIF_F_RX_UDP_TUNNEL_PORT) {
8420 /* udp_tunnel_{get,drop}_rx_info both need
8421 * NETIF_F_RX_UDP_TUNNEL_PORT enabled on the
8422 * device, or they won't do anything.
8423 * Thus we need to update dev->features
8424 * *before* calling udp_tunnel_get_rx_info,
8425 * but *after* calling udp_tunnel_drop_rx_info.
8426 */
8427 if (features & NETIF_F_RX_UDP_TUNNEL_PORT) {
8428 dev->features = features;
8429 udp_tunnel_get_rx_info(dev);
8430 } else {
8431 udp_tunnel_drop_rx_info(dev);
8432 }
8433 }
8434
Gal Pressman9daae9b2018-03-28 17:46:54 +03008435 if (diff & NETIF_F_HW_VLAN_CTAG_FILTER) {
8436 if (features & NETIF_F_HW_VLAN_CTAG_FILTER) {
8437 dev->features = features;
8438 err |= vlan_get_rx_ctag_filter_info(dev);
8439 } else {
8440 vlan_drop_rx_ctag_filter_info(dev);
8441 }
8442 }
8443
8444 if (diff & NETIF_F_HW_VLAN_STAG_FILTER) {
8445 if (features & NETIF_F_HW_VLAN_STAG_FILTER) {
8446 dev->features = features;
8447 err |= vlan_get_rx_stag_filter_info(dev);
8448 } else {
8449 vlan_drop_rx_stag_filter_info(dev);
8450 }
8451 }
8452
Michał Mirosław6cb6a272011-04-02 22:48:47 -07008453 dev->features = features;
Sabrina Dubrocaae847f42017-07-21 12:49:31 +02008454 }
Michał Mirosław6cb6a272011-04-02 22:48:47 -07008455
Jarod Wilsone7868a82015-11-03 23:09:32 -05008456 return err < 0 ? 0 : 1;
Michał Mirosław6cb6a272011-04-02 22:48:47 -07008457}
8458
Michał Mirosławafe12cc2011-05-07 03:22:17 +00008459/**
8460 * netdev_update_features - recalculate device features
8461 * @dev: the device to check
8462 *
8463 * Recalculate dev->features set and send notifications if it
8464 * has changed. Should be called after driver or hardware dependent
8465 * conditions might have changed that influence the features.
8466 */
Michał Mirosław6cb6a272011-04-02 22:48:47 -07008467void netdev_update_features(struct net_device *dev)
8468{
8469 if (__netdev_update_features(dev))
8470 netdev_features_change(dev);
Michał Mirosław5455c692011-02-15 16:59:17 +00008471}
8472EXPORT_SYMBOL(netdev_update_features);
8473
Linus Torvalds1da177e2005-04-16 15:20:36 -07008474/**
Michał Mirosławafe12cc2011-05-07 03:22:17 +00008475 * netdev_change_features - recalculate device features
8476 * @dev: the device to check
8477 *
8478 * Recalculate dev->features set and send notifications even
8479 * if they have not changed. Should be called instead of
8480 * netdev_update_features() if also dev->vlan_features might
8481 * have changed to allow the changes to be propagated to stacked
8482 * VLAN devices.
8483 */
8484void netdev_change_features(struct net_device *dev)
8485{
8486 __netdev_update_features(dev);
8487 netdev_features_change(dev);
8488}
8489EXPORT_SYMBOL(netdev_change_features);
8490
8491/**
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08008492 * netif_stacked_transfer_operstate - transfer operstate
8493 * @rootdev: the root or lower level device to transfer state from
8494 * @dev: the device to transfer operstate to
8495 *
8496 * Transfer operational state from root to device. This is normally
8497 * called when a stacking relationship exists between the root
8498 * device and the device(a leaf device).
8499 */
8500void netif_stacked_transfer_operstate(const struct net_device *rootdev,
8501 struct net_device *dev)
8502{
8503 if (rootdev->operstate == IF_OPER_DORMANT)
8504 netif_dormant_on(dev);
8505 else
8506 netif_dormant_off(dev);
8507
Zhang Shengju0575c862017-04-26 17:49:38 +08008508 if (netif_carrier_ok(rootdev))
8509 netif_carrier_on(dev);
8510 else
8511 netif_carrier_off(dev);
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08008512}
8513EXPORT_SYMBOL(netif_stacked_transfer_operstate);
8514
Eric Dumazet1b4bf462010-09-23 17:26:35 +00008515static int netif_alloc_rx_queues(struct net_device *dev)
8516{
Eric Dumazet1b4bf462010-09-23 17:26:35 +00008517 unsigned int i, count = dev->num_rx_queues;
Tom Herbertbd25fa72010-10-18 18:00:16 +00008518 struct netdev_rx_queue *rx;
Pankaj Gupta10595902015-01-12 11:41:28 +05308519 size_t sz = count * sizeof(*rx);
Jesper Dangaard Brouere817f852018-01-03 11:26:09 +01008520 int err = 0;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00008521
Tom Herbertbd25fa72010-10-18 18:00:16 +00008522 BUG_ON(count < 1);
Eric Dumazet1b4bf462010-09-23 17:26:35 +00008523
Michal Hockodcda9b02017-07-12 14:36:45 -07008524 rx = kvzalloc(sz, GFP_KERNEL | __GFP_RETRY_MAYFAIL);
Michal Hockoda6bc572017-05-08 15:57:31 -07008525 if (!rx)
8526 return -ENOMEM;
8527
Tom Herbertbd25fa72010-10-18 18:00:16 +00008528 dev->_rx = rx;
8529
Jesper Dangaard Brouere817f852018-01-03 11:26:09 +01008530 for (i = 0; i < count; i++) {
Tom Herbertfe822242010-11-09 10:47:38 +00008531 rx[i].dev = dev;
Jesper Dangaard Brouere817f852018-01-03 11:26:09 +01008532
8533 /* XDP RX-queue setup */
8534 err = xdp_rxq_info_reg(&rx[i].xdp_rxq, dev, i);
8535 if (err < 0)
8536 goto err_rxq_info;
8537 }
Eric Dumazet1b4bf462010-09-23 17:26:35 +00008538 return 0;
Jesper Dangaard Brouere817f852018-01-03 11:26:09 +01008539
8540err_rxq_info:
8541 /* Rollback successful reg's and free other resources */
8542 while (i--)
8543 xdp_rxq_info_unreg(&rx[i].xdp_rxq);
Jakub Kicinski141b52a2018-01-10 01:20:01 -08008544 kvfree(dev->_rx);
Jesper Dangaard Brouere817f852018-01-03 11:26:09 +01008545 dev->_rx = NULL;
8546 return err;
Eric Dumazet1b4bf462010-09-23 17:26:35 +00008547}
Jesper Dangaard Brouere817f852018-01-03 11:26:09 +01008548
8549static void netif_free_rx_queues(struct net_device *dev)
8550{
8551 unsigned int i, count = dev->num_rx_queues;
Jesper Dangaard Brouere817f852018-01-03 11:26:09 +01008552
8553 /* netif_alloc_rx_queues alloc failed, resources have been unreg'ed */
8554 if (!dev->_rx)
8555 return;
8556
Jesper Dangaard Brouere817f852018-01-03 11:26:09 +01008557 for (i = 0; i < count; i++)
Jakub Kicinski82aaff22018-01-10 01:20:02 -08008558 xdp_rxq_info_unreg(&dev->_rx[i].xdp_rxq);
8559
8560 kvfree(dev->_rx);
Jesper Dangaard Brouere817f852018-01-03 11:26:09 +01008561}
Eric Dumazet1b4bf462010-09-23 17:26:35 +00008562
Changli Gaoaa942102010-12-04 02:31:41 +00008563static void netdev_init_one_queue(struct net_device *dev,
8564 struct netdev_queue *queue, void *_unused)
8565{
8566 /* Initialize queue lock */
8567 spin_lock_init(&queue->_xmit_lock);
8568 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
8569 queue->xmit_lock_owner = -1;
Changli Gaob236da62010-12-14 03:09:15 +00008570 netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
Changli Gaoaa942102010-12-04 02:31:41 +00008571 queue->dev = dev;
Tom Herbert114cf582011-11-28 16:33:09 +00008572#ifdef CONFIG_BQL
8573 dql_init(&queue->dql, HZ);
8574#endif
Changli Gaoaa942102010-12-04 02:31:41 +00008575}
8576
Eric Dumazet60877a32013-06-20 01:15:51 -07008577static void netif_free_tx_queues(struct net_device *dev)
8578{
WANG Cong4cb28972014-06-02 15:55:22 -07008579 kvfree(dev->_tx);
Eric Dumazet60877a32013-06-20 01:15:51 -07008580}
8581
Tom Herberte6484932010-10-18 18:04:39 +00008582static int netif_alloc_netdev_queues(struct net_device *dev)
8583{
8584 unsigned int count = dev->num_tx_queues;
8585 struct netdev_queue *tx;
Eric Dumazet60877a32013-06-20 01:15:51 -07008586 size_t sz = count * sizeof(*tx);
Tom Herberte6484932010-10-18 18:04:39 +00008587
Eric Dumazetd3397272015-07-06 17:13:26 +02008588 if (count < 1 || count > 0xffff)
8589 return -EINVAL;
Tom Herberte6484932010-10-18 18:04:39 +00008590
Michal Hockodcda9b02017-07-12 14:36:45 -07008591 tx = kvzalloc(sz, GFP_KERNEL | __GFP_RETRY_MAYFAIL);
Michal Hockoda6bc572017-05-08 15:57:31 -07008592 if (!tx)
8593 return -ENOMEM;
8594
Tom Herberte6484932010-10-18 18:04:39 +00008595 dev->_tx = tx;
Tom Herbert1d24eb42010-11-21 13:17:27 +00008596
Tom Herberte6484932010-10-18 18:04:39 +00008597 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
8598 spin_lock_init(&dev->tx_global_lock);
Changli Gaoaa942102010-12-04 02:31:41 +00008599
8600 return 0;
Tom Herberte6484932010-10-18 18:04:39 +00008601}
8602
Denys Vlasenkoa2029242015-05-11 21:17:53 +02008603void netif_tx_stop_all_queues(struct net_device *dev)
8604{
8605 unsigned int i;
8606
8607 for (i = 0; i < dev->num_tx_queues; i++) {
8608 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
tchardingf4563a72017-02-09 17:56:07 +11008609
Denys Vlasenkoa2029242015-05-11 21:17:53 +02008610 netif_tx_stop_queue(txq);
8611 }
8612}
8613EXPORT_SYMBOL(netif_tx_stop_all_queues);
8614
Patrick Mullaneyfc4a7482009-12-03 15:59:22 -08008615/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07008616 * register_netdevice - register a network device
8617 * @dev: device to register
8618 *
8619 * Take a completed network device structure and add it to the kernel
8620 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
8621 * chain. 0 is returned on success. A negative errno code is returned
8622 * on a failure to set up the device, or if the name is a duplicate.
8623 *
8624 * Callers must hold the rtnl semaphore. You may want
8625 * register_netdev() instead of this.
8626 *
8627 * BUGS:
8628 * The locking appears insufficient to guarantee two parallel registers
8629 * will not get the same name.
8630 */
8631
8632int register_netdevice(struct net_device *dev)
8633{
Linus Torvalds1da177e2005-04-16 15:20:36 -07008634 int ret;
Stephen Hemmingerd3147742008-11-19 21:32:24 -08008635 struct net *net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008636
Florian Fainellie283de32018-04-30 14:20:05 -07008637 BUILD_BUG_ON(sizeof(netdev_features_t) * BITS_PER_BYTE <
8638 NETDEV_FEATURE_COUNT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008639 BUG_ON(dev_boot_phase);
8640 ASSERT_RTNL();
8641
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07008642 might_sleep();
8643
Linus Torvalds1da177e2005-04-16 15:20:36 -07008644 /* When net_device's are persistent, this will be fatal. */
8645 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
Stephen Hemmingerd3147742008-11-19 21:32:24 -08008646 BUG_ON(!net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008647
David S. Millerf1f28aa2008-07-15 00:08:33 -07008648 spin_lock_init(&dev->addr_list_lock);
David S. Millercf508b12008-07-22 14:16:42 -07008649 netdev_set_addr_lockdep_class(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008650
Gao feng828de4f2012-09-13 20:58:27 +00008651 ret = dev_get_valid_name(net, dev, dev->name);
Peter Pan(潘卫平)0696c3a2011-05-12 15:46:56 +00008652 if (ret < 0)
8653 goto out;
8654
Linus Torvalds1da177e2005-04-16 15:20:36 -07008655 /* Init, if this function is available */
Stephen Hemmingerd3147742008-11-19 21:32:24 -08008656 if (dev->netdev_ops->ndo_init) {
8657 ret = dev->netdev_ops->ndo_init(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008658 if (ret) {
8659 if (ret > 0)
8660 ret = -EIO;
Adrian Bunk90833aa2006-11-13 16:02:22 -08008661 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008662 }
8663 }
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09008664
Patrick McHardyf6469682013-04-19 02:04:27 +00008665 if (((dev->hw_features | dev->features) &
8666 NETIF_F_HW_VLAN_CTAG_FILTER) &&
Michał Mirosławd2ed2732013-01-29 15:14:16 +00008667 (!dev->netdev_ops->ndo_vlan_rx_add_vid ||
8668 !dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
8669 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
8670 ret = -EINVAL;
8671 goto err_uninit;
8672 }
8673
Pavel Emelyanov9c7dafb2012-08-08 21:52:46 +00008674 ret = -EBUSY;
8675 if (!dev->ifindex)
8676 dev->ifindex = dev_new_index(net);
8677 else if (__dev_get_by_index(net, dev->ifindex))
8678 goto err_uninit;
8679
Michał Mirosław5455c692011-02-15 16:59:17 +00008680 /* Transfer changeable features to wanted_features and enable
8681 * software offloads (GSO and GRO).
8682 */
8683 dev->hw_features |= NETIF_F_SOFT_FEATURES;
Michał Mirosław14d12322011-02-22 16:52:28 +00008684 dev->features |= NETIF_F_SOFT_FEATURES;
Sabrina Dubrocad764a122017-07-21 12:49:28 +02008685
8686 if (dev->netdev_ops->ndo_udp_tunnel_add) {
8687 dev->features |= NETIF_F_RX_UDP_TUNNEL_PORT;
8688 dev->hw_features |= NETIF_F_RX_UDP_TUNNEL_PORT;
8689 }
8690
Michał Mirosław14d12322011-02-22 16:52:28 +00008691 dev->wanted_features = dev->features & dev->hw_features;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008692
Alexander Duyckcbc53e02016-04-10 21:44:51 -04008693 if (!(dev->flags & IFF_LOOPBACK))
Michał Mirosław34324dc2011-11-15 15:29:55 +00008694 dev->hw_features |= NETIF_F_NOCACHE_COPY;
Alexander Duyckcbc53e02016-04-10 21:44:51 -04008695
Alexander Duyck7f348a62016-04-20 16:51:00 -04008696 /* If IPv4 TCP segmentation offload is supported we should also
8697 * allow the device to enable segmenting the frame with the option
8698 * of ignoring a static IP ID value. This doesn't enable the
8699 * feature itself but allows the user to enable it later.
8700 */
Alexander Duyckcbc53e02016-04-10 21:44:51 -04008701 if (dev->hw_features & NETIF_F_TSO)
8702 dev->hw_features |= NETIF_F_TSO_MANGLEID;
Alexander Duyck7f348a62016-04-20 16:51:00 -04008703 if (dev->vlan_features & NETIF_F_TSO)
8704 dev->vlan_features |= NETIF_F_TSO_MANGLEID;
8705 if (dev->mpls_features & NETIF_F_TSO)
8706 dev->mpls_features |= NETIF_F_TSO_MANGLEID;
8707 if (dev->hw_enc_features & NETIF_F_TSO)
8708 dev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
Tom Herbertc6e1a0d2011-04-04 22:30:30 -07008709
Michał Mirosław1180e7d2011-07-14 14:41:11 -07008710 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
Brandon Philips16c3ea72010-09-15 09:24:24 +00008711 */
Michał Mirosław1180e7d2011-07-14 14:41:11 -07008712 dev->vlan_features |= NETIF_F_HIGHDMA;
Brandon Philips16c3ea72010-09-15 09:24:24 +00008713
Pravin B Shelaree579672013-03-07 09:28:08 +00008714 /* Make NETIF_F_SG inheritable to tunnel devices.
8715 */
Alexander Duyck802ab552016-04-10 21:45:03 -04008716 dev->hw_enc_features |= NETIF_F_SG | NETIF_F_GSO_PARTIAL;
Pravin B Shelaree579672013-03-07 09:28:08 +00008717
Simon Horman0d89d202013-05-23 21:02:52 +00008718 /* Make NETIF_F_SG inheritable to MPLS.
8719 */
8720 dev->mpls_features |= NETIF_F_SG;
8721
Johannes Berg7ffbe3f2009-10-02 05:15:27 +00008722 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
8723 ret = notifier_to_errno(ret);
8724 if (ret)
8725 goto err_uninit;
8726
Eric W. Biederman8b41d182007-09-26 22:02:53 -07008727 ret = netdev_register_kobject(dev);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07008728 if (ret)
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07008729 goto err_uninit;
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07008730 dev->reg_state = NETREG_REGISTERED;
8731
Michał Mirosław6cb6a272011-04-02 22:48:47 -07008732 __netdev_update_features(dev);
Michał Mirosław8e9b59b2011-02-22 16:52:28 +00008733
Linus Torvalds1da177e2005-04-16 15:20:36 -07008734 /*
8735 * Default initial state at registry is that the
8736 * device is present.
8737 */
8738
8739 set_bit(__LINK_STATE_PRESENT, &dev->state);
8740
Ben Hutchings8f4cccb2012-08-20 22:16:51 +01008741 linkwatch_init_dev(dev);
8742
Linus Torvalds1da177e2005-04-16 15:20:36 -07008743 dev_init_scheduler(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008744 dev_hold(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02008745 list_netdevice(dev);
Theodore Ts'o7bf23572012-07-04 21:23:25 -04008746 add_device_randomness(dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008747
Jiri Pirko948b3372013-01-08 01:38:25 +00008748 /* If the device has permanent device address, driver should
8749 * set dev_addr and also addr_assign_type should be set to
8750 * NET_ADDR_PERM (default value).
8751 */
8752 if (dev->addr_assign_type == NET_ADDR_PERM)
8753 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
8754
Linus Torvalds1da177e2005-04-16 15:20:36 -07008755 /* Notify protocols, that a new device appeared. */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07008756 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
Herbert Xufcc5a032007-07-30 17:03:38 -07008757 ret = notifier_to_errno(ret);
Daniel Lezcano93ee31f2007-10-30 15:38:18 -07008758 if (ret) {
8759 rollback_registered(dev);
8760 dev->reg_state = NETREG_UNREGISTERED;
8761 }
Eric W. Biedermand90a9092009-12-12 22:11:15 +00008762 /*
8763 * Prevent userspace races by waiting until the network
8764 * device is fully setup before sending notifications.
8765 */
Patrick McHardya2835762010-02-26 06:34:51 +00008766 if (!dev->rtnl_link_ops ||
8767 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
Alexei Starovoitov7f294052013-10-23 16:02:42 -07008768 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008769
8770out:
8771 return ret;
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07008772
8773err_uninit:
Stephen Hemmingerd3147742008-11-19 21:32:24 -08008774 if (dev->netdev_ops->ndo_uninit)
8775 dev->netdev_ops->ndo_uninit(dev);
David S. Millercf124db2017-05-08 12:52:56 -04008776 if (dev->priv_destructor)
8777 dev->priv_destructor(dev);
Herbert Xu7ce1b0e2007-07-30 16:29:40 -07008778 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008779}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07008780EXPORT_SYMBOL(register_netdevice);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008781
8782/**
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08008783 * init_dummy_netdev - init a dummy network device for NAPI
8784 * @dev: device to init
8785 *
8786 * This takes a network device structure and initialize the minimum
8787 * amount of fields so it can be used to schedule NAPI polls without
8788 * registering a full blown interface. This is to be used by drivers
8789 * that need to tie several hardware interfaces to a single NAPI
8790 * poll scheduler due to HW limitations.
8791 */
8792int init_dummy_netdev(struct net_device *dev)
8793{
8794 /* Clear everything. Note we don't initialize spinlocks
8795 * are they aren't supposed to be taken by any of the
8796 * NAPI code and this dummy netdev is supposed to be
8797 * only ever used for NAPI polls
8798 */
8799 memset(dev, 0, sizeof(struct net_device));
8800
8801 /* make sure we BUG if trying to hit standard
8802 * register/unregister code path
8803 */
8804 dev->reg_state = NETREG_DUMMY;
8805
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08008806 /* NAPI wants this */
8807 INIT_LIST_HEAD(&dev->napi_list);
8808
8809 /* a dummy interface is started by default */
8810 set_bit(__LINK_STATE_PRESENT, &dev->state);
8811 set_bit(__LINK_STATE_START, &dev->state);
8812
Josh Elsasser35edfdc2019-01-26 14:38:33 -08008813 /* napi_busy_loop stats accounting wants this */
8814 dev_net_set(dev, &init_net);
8815
Eric Dumazet29b44332010-10-11 10:22:12 +00008816 /* Note : We dont allocate pcpu_refcnt for dummy devices,
8817 * because users of this 'device' dont need to change
8818 * its refcount.
8819 */
8820
Benjamin Herrenschmidt937f1ba2009-01-14 21:05:05 -08008821 return 0;
8822}
8823EXPORT_SYMBOL_GPL(init_dummy_netdev);
8824
8825
8826/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07008827 * register_netdev - register a network device
8828 * @dev: device to register
8829 *
8830 * Take a completed network device structure and add it to the kernel
8831 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
8832 * chain. 0 is returned on success. A negative errno code is returned
8833 * on a failure to set up the device, or if the name is a duplicate.
8834 *
Borislav Petkov38b4da382007-04-20 22:14:10 -07008835 * This is a wrapper around register_netdevice that takes the rtnl semaphore
Linus Torvalds1da177e2005-04-16 15:20:36 -07008836 * and expands the device name if you passed a format string to
8837 * alloc_netdev.
8838 */
8839int register_netdev(struct net_device *dev)
8840{
8841 int err;
8842
Kirill Tkhaib0f3deb2018-03-14 22:17:28 +03008843 if (rtnl_lock_killable())
8844 return -EINTR;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008845 err = register_netdevice(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008846 rtnl_unlock();
8847 return err;
8848}
8849EXPORT_SYMBOL(register_netdev);
8850
Eric Dumazet29b44332010-10-11 10:22:12 +00008851int netdev_refcnt_read(const struct net_device *dev)
8852{
8853 int i, refcnt = 0;
8854
8855 for_each_possible_cpu(i)
8856 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
8857 return refcnt;
8858}
8859EXPORT_SYMBOL(netdev_refcnt_read);
8860
Ben Hutchings2c530402012-07-10 10:55:09 +00008861/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07008862 * netdev_wait_allrefs - wait until all references are gone.
Randy Dunlap3de7a372012-08-18 14:36:44 +00008863 * @dev: target net_device
Linus Torvalds1da177e2005-04-16 15:20:36 -07008864 *
8865 * This is called when unregistering network devices.
8866 *
8867 * Any protocol or device that holds a reference should register
8868 * for netdevice notification, and cleanup and put back the
8869 * reference if they receive an UNREGISTER event.
8870 * We can get stuck here if buggy protocols don't correctly
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09008871 * call dev_put.
Linus Torvalds1da177e2005-04-16 15:20:36 -07008872 */
8873static void netdev_wait_allrefs(struct net_device *dev)
8874{
8875 unsigned long rebroadcast_time, warning_time;
Eric Dumazet29b44332010-10-11 10:22:12 +00008876 int refcnt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008877
Eric Dumazete014deb2009-11-17 05:59:21 +00008878 linkwatch_forget_dev(dev);
8879
Linus Torvalds1da177e2005-04-16 15:20:36 -07008880 rebroadcast_time = warning_time = jiffies;
Eric Dumazet29b44332010-10-11 10:22:12 +00008881 refcnt = netdev_refcnt_read(dev);
8882
8883 while (refcnt != 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07008884 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
Stephen Hemminger6756ae42006-03-20 22:23:58 -08008885 rtnl_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07008886
8887 /* Rebroadcast unregister notification */
Pavel Emelyanov056925a2007-09-16 15:42:43 -07008888 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008889
Eric Dumazet748e2d92012-08-22 21:50:59 +00008890 __rtnl_unlock();
Eric Dumazet0115e8e2012-08-22 17:19:46 +00008891 rcu_barrier();
Eric Dumazet748e2d92012-08-22 21:50:59 +00008892 rtnl_lock();
8893
Linus Torvalds1da177e2005-04-16 15:20:36 -07008894 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
8895 &dev->state)) {
8896 /* We must not have linkwatch events
8897 * pending on unregister. If this
8898 * happens, we simply run the queue
8899 * unscheduled, resulting in a noop
8900 * for this device.
8901 */
8902 linkwatch_run_queue();
8903 }
8904
Stephen Hemminger6756ae42006-03-20 22:23:58 -08008905 __rtnl_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07008906
8907 rebroadcast_time = jiffies;
8908 }
8909
8910 msleep(250);
8911
Eric Dumazet29b44332010-10-11 10:22:12 +00008912 refcnt = netdev_refcnt_read(dev);
8913
Linus Torvalds1da177e2005-04-16 15:20:36 -07008914 if (time_after(jiffies, warning_time + 10 * HZ)) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00008915 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
8916 dev->name, refcnt);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008917 warning_time = jiffies;
8918 }
8919 }
8920}
8921
8922/* The sequence is:
8923 *
8924 * rtnl_lock();
8925 * ...
8926 * register_netdevice(x1);
8927 * register_netdevice(x2);
8928 * ...
8929 * unregister_netdevice(y1);
8930 * unregister_netdevice(y2);
8931 * ...
8932 * rtnl_unlock();
8933 * free_netdev(y1);
8934 * free_netdev(y2);
8935 *
Herbert Xu58ec3b42008-10-07 15:50:03 -07008936 * We are invoked by rtnl_unlock().
Linus Torvalds1da177e2005-04-16 15:20:36 -07008937 * This allows us to deal with problems:
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07008938 * 1) We can delete sysfs objects which invoke hotplug
Linus Torvalds1da177e2005-04-16 15:20:36 -07008939 * without deadlocking with linkwatch via keventd.
8940 * 2) Since we run with the RTNL semaphore not held, we can sleep
8941 * safely in order to wait for the netdev refcnt to drop to zero.
Herbert Xu58ec3b42008-10-07 15:50:03 -07008942 *
8943 * We must not return until all unregister events added during
8944 * the interval the lock was held have been completed.
Linus Torvalds1da177e2005-04-16 15:20:36 -07008945 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07008946void netdev_run_todo(void)
8947{
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07008948 struct list_head list;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008949
Linus Torvalds1da177e2005-04-16 15:20:36 -07008950 /* Snapshot list, allow later requests */
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07008951 list_replace_init(&net_todo_list, &list);
Herbert Xu58ec3b42008-10-07 15:50:03 -07008952
8953 __rtnl_unlock();
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07008954
Eric Dumazet0115e8e2012-08-22 17:19:46 +00008955
8956 /* Wait for rcu callbacks to finish before next phase */
Eric W. Biederman850a5452011-10-13 22:25:23 +00008957 if (!list_empty(&list))
8958 rcu_barrier();
8959
Linus Torvalds1da177e2005-04-16 15:20:36 -07008960 while (!list_empty(&list)) {
8961 struct net_device *dev
stephen hemmingere5e26d72010-02-24 14:01:38 +00008962 = list_first_entry(&list, struct net_device, todo_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008963 list_del(&dev->todo_list);
8964
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07008965 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00008966 pr_err("network todo '%s' but state %d\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07008967 dev->name, dev->reg_state);
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07008968 dump_stack();
8969 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008970 }
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07008971
Stephen Hemmingerb17a7c12006-05-10 13:21:17 -07008972 dev->reg_state = NETREG_UNREGISTERED;
8973
8974 netdev_wait_allrefs(dev);
8975
8976 /* paranoia */
Eric Dumazet29b44332010-10-11 10:22:12 +00008977 BUG_ON(netdev_refcnt_read(dev));
Salam Noureddine7866a622015-01-27 11:35:48 -08008978 BUG_ON(!list_empty(&dev->ptype_all));
8979 BUG_ON(!list_empty(&dev->ptype_specific));
Eric Dumazet33d480c2011-08-11 19:30:52 +00008980 WARN_ON(rcu_access_pointer(dev->ip_ptr));
8981 WARN_ON(rcu_access_pointer(dev->ip6_ptr));
David Ahern330c7272018-02-13 08:52:00 -08008982#if IS_ENABLED(CONFIG_DECNET)
Ilpo Järvinen547b7922008-07-25 21:43:18 -07008983 WARN_ON(dev->dn_ptr);
David Ahern330c7272018-02-13 08:52:00 -08008984#endif
David S. Millercf124db2017-05-08 12:52:56 -04008985 if (dev->priv_destructor)
8986 dev->priv_destructor(dev);
8987 if (dev->needs_free_netdev)
8988 free_netdev(dev);
Stephen Hemminger9093bbb2007-05-19 15:39:25 -07008989
Eric W. Biederman50624c92013-09-23 21:19:49 -07008990 /* Report a network device has been unregistered */
8991 rtnl_lock();
8992 dev_net(dev)->dev_unreg_count--;
8993 __rtnl_unlock();
8994 wake_up(&netdev_unregistering_wq);
8995
Stephen Hemminger9093bbb2007-05-19 15:39:25 -07008996 /* Free network device */
8997 kobject_put(&dev->dev.kobj);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008998 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07008999}
9000
Jarod Wilson92566452016-02-01 18:51:04 -05009001/* Convert net_device_stats to rtnl_link_stats64. rtnl_link_stats64 has
9002 * all the same fields in the same order as net_device_stats, with only
9003 * the type differing, but rtnl_link_stats64 may have additional fields
9004 * at the end for newer counters.
Ben Hutchings3cfde792010-07-09 09:11:52 +00009005 */
Eric Dumazet77a1abf2012-03-05 04:50:09 +00009006void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
9007 const struct net_device_stats *netdev_stats)
Ben Hutchings3cfde792010-07-09 09:11:52 +00009008{
9009#if BITS_PER_LONG == 64
Jarod Wilson92566452016-02-01 18:51:04 -05009010 BUILD_BUG_ON(sizeof(*stats64) < sizeof(*netdev_stats));
Alban Browaeys9af99592017-07-03 03:20:13 +02009011 memcpy(stats64, netdev_stats, sizeof(*netdev_stats));
Jarod Wilson92566452016-02-01 18:51:04 -05009012 /* zero out counters that only exist in rtnl_link_stats64 */
9013 memset((char *)stats64 + sizeof(*netdev_stats), 0,
9014 sizeof(*stats64) - sizeof(*netdev_stats));
Ben Hutchings3cfde792010-07-09 09:11:52 +00009015#else
Jarod Wilson92566452016-02-01 18:51:04 -05009016 size_t i, n = sizeof(*netdev_stats) / sizeof(unsigned long);
Ben Hutchings3cfde792010-07-09 09:11:52 +00009017 const unsigned long *src = (const unsigned long *)netdev_stats;
9018 u64 *dst = (u64 *)stats64;
9019
Jarod Wilson92566452016-02-01 18:51:04 -05009020 BUILD_BUG_ON(n > sizeof(*stats64) / sizeof(u64));
Ben Hutchings3cfde792010-07-09 09:11:52 +00009021 for (i = 0; i < n; i++)
9022 dst[i] = src[i];
Jarod Wilson92566452016-02-01 18:51:04 -05009023 /* zero out counters that only exist in rtnl_link_stats64 */
9024 memset((char *)stats64 + n * sizeof(u64), 0,
9025 sizeof(*stats64) - n * sizeof(u64));
Ben Hutchings3cfde792010-07-09 09:11:52 +00009026#endif
9027}
Eric Dumazet77a1abf2012-03-05 04:50:09 +00009028EXPORT_SYMBOL(netdev_stats_to_stats64);
Ben Hutchings3cfde792010-07-09 09:11:52 +00009029
Eric Dumazetd83345a2009-11-16 03:36:51 +00009030/**
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08009031 * dev_get_stats - get network device statistics
9032 * @dev: device to get statistics from
Eric Dumazet28172732010-07-07 14:58:56 -07009033 * @storage: place to store stats
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08009034 *
Ben Hutchingsd7753512010-07-09 09:12:41 +00009035 * Get network statistics from device. Return @storage.
9036 * The device driver may provide its own method by setting
9037 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
9038 * otherwise the internal statistics structure is used.
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08009039 */
Ben Hutchingsd7753512010-07-09 09:12:41 +00009040struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
9041 struct rtnl_link_stats64 *storage)
Eric Dumazet7004bf22009-05-18 00:34:33 +00009042{
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08009043 const struct net_device_ops *ops = dev->netdev_ops;
9044
Eric Dumazet28172732010-07-07 14:58:56 -07009045 if (ops->ndo_get_stats64) {
9046 memset(storage, 0, sizeof(*storage));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00009047 ops->ndo_get_stats64(dev, storage);
9048 } else if (ops->ndo_get_stats) {
Ben Hutchings3cfde792010-07-09 09:11:52 +00009049 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
Eric Dumazetcaf586e2010-09-30 21:06:55 +00009050 } else {
9051 netdev_stats_to_stats64(storage, &dev->stats);
Eric Dumazet28172732010-07-07 14:58:56 -07009052 }
Eric Dumazet6f64ec72017-06-27 07:02:20 -07009053 storage->rx_dropped += (unsigned long)atomic_long_read(&dev->rx_dropped);
9054 storage->tx_dropped += (unsigned long)atomic_long_read(&dev->tx_dropped);
9055 storage->rx_nohandler += (unsigned long)atomic_long_read(&dev->rx_nohandler);
Eric Dumazet28172732010-07-07 14:58:56 -07009056 return storage;
Rusty Russellc45d2862007-03-28 14:29:08 -07009057}
Stephen Hemmingereeda3fd2008-11-19 21:40:23 -08009058EXPORT_SYMBOL(dev_get_stats);
Rusty Russellc45d2862007-03-28 14:29:08 -07009059
Eric Dumazet24824a02010-10-02 06:11:55 +00009060struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
David S. Millerdc2b4842008-07-08 17:18:23 -07009061{
Eric Dumazet24824a02010-10-02 06:11:55 +00009062 struct netdev_queue *queue = dev_ingress_queue(dev);
David S. Millerdc2b4842008-07-08 17:18:23 -07009063
Eric Dumazet24824a02010-10-02 06:11:55 +00009064#ifdef CONFIG_NET_CLS_ACT
9065 if (queue)
9066 return queue;
9067 queue = kzalloc(sizeof(*queue), GFP_KERNEL);
9068 if (!queue)
9069 return NULL;
9070 netdev_init_one_queue(dev, queue, NULL);
Eric Dumazet2ce1ee12015-02-04 13:37:44 -08009071 RCU_INIT_POINTER(queue->qdisc, &noop_qdisc);
Eric Dumazet24824a02010-10-02 06:11:55 +00009072 queue->qdisc_sleeping = &noop_qdisc;
9073 rcu_assign_pointer(dev->ingress_queue, queue);
9074#endif
9075 return queue;
David S. Millerbb949fb2008-07-08 16:55:56 -07009076}
9077
Eric Dumazet2c60db02012-09-16 09:17:26 +00009078static const struct ethtool_ops default_ethtool_ops;
9079
Stanislaw Gruszkad07d7502013-01-10 23:19:10 +00009080void netdev_set_default_ethtool_ops(struct net_device *dev,
9081 const struct ethtool_ops *ops)
9082{
9083 if (dev->ethtool_ops == &default_ethtool_ops)
9084 dev->ethtool_ops = ops;
9085}
9086EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
9087
Eric Dumazet74d332c2013-10-30 13:10:44 -07009088void netdev_freemem(struct net_device *dev)
9089{
9090 char *addr = (char *)dev - dev->padded;
9091
WANG Cong4cb28972014-06-02 15:55:22 -07009092 kvfree(addr);
Eric Dumazet74d332c2013-10-30 13:10:44 -07009093}
9094
Linus Torvalds1da177e2005-04-16 15:20:36 -07009095/**
tcharding722c9a02017-02-09 17:56:04 +11009096 * alloc_netdev_mqs - allocate network device
9097 * @sizeof_priv: size of private data to allocate space for
9098 * @name: device name format string
9099 * @name_assign_type: origin of device name
9100 * @setup: callback to initialize device
9101 * @txqs: the number of TX subqueues to allocate
9102 * @rxqs: the number of RX subqueues to allocate
Linus Torvalds1da177e2005-04-16 15:20:36 -07009103 *
tcharding722c9a02017-02-09 17:56:04 +11009104 * Allocates a struct net_device with private data area for driver use
9105 * and performs basic initialization. Also allocates subqueue structs
9106 * for each queue on the device.
Linus Torvalds1da177e2005-04-16 15:20:36 -07009107 */
Tom Herbert36909ea2011-01-09 19:36:31 +00009108struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
Tom Gundersenc835a672014-07-14 16:37:24 +02009109 unsigned char name_assign_type,
Tom Herbert36909ea2011-01-09 19:36:31 +00009110 void (*setup)(struct net_device *),
9111 unsigned int txqs, unsigned int rxqs)
Linus Torvalds1da177e2005-04-16 15:20:36 -07009112{
Linus Torvalds1da177e2005-04-16 15:20:36 -07009113 struct net_device *dev;
Alexey Dobriyan52a59bd2017-09-21 23:33:29 +03009114 unsigned int alloc_size;
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00009115 struct net_device *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07009116
Stephen Hemmingerb6fe17d2006-08-29 17:06:13 -07009117 BUG_ON(strlen(name) >= sizeof(dev->name));
9118
Tom Herbert36909ea2011-01-09 19:36:31 +00009119 if (txqs < 1) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00009120 pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
Tom Herbert55513fb2010-10-18 17:55:58 +00009121 return NULL;
9122 }
9123
Tom Herbert36909ea2011-01-09 19:36:31 +00009124 if (rxqs < 1) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00009125 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
Tom Herbert36909ea2011-01-09 19:36:31 +00009126 return NULL;
9127 }
Tom Herbert36909ea2011-01-09 19:36:31 +00009128
David S. Millerfd2ea0a2008-07-17 01:56:23 -07009129 alloc_size = sizeof(struct net_device);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07009130 if (sizeof_priv) {
9131 /* ensure 32-byte alignment of private area */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00009132 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
Alexey Dobriyand1643d22008-04-18 15:43:32 -07009133 alloc_size += sizeof_priv;
9134 }
9135 /* ensure 32-byte alignment of whole construct */
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00009136 alloc_size += NETDEV_ALIGN - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07009137
Michal Hockodcda9b02017-07-12 14:36:45 -07009138 p = kvzalloc(alloc_size, GFP_KERNEL | __GFP_RETRY_MAYFAIL);
Joe Perches62b59422013-02-04 16:48:16 +00009139 if (!p)
Linus Torvalds1da177e2005-04-16 15:20:36 -07009140 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07009141
Eric Dumazet1ce8e7b2009-05-27 04:42:37 +00009142 dev = PTR_ALIGN(p, NETDEV_ALIGN);
Linus Torvalds1da177e2005-04-16 15:20:36 -07009143 dev->padded = (char *)dev - (char *)p;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00009144
Eric Dumazet29b44332010-10-11 10:22:12 +00009145 dev->pcpu_refcnt = alloc_percpu(int);
9146 if (!dev->pcpu_refcnt)
Eric Dumazet74d332c2013-10-30 13:10:44 -07009147 goto free_dev;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00009148
Linus Torvalds1da177e2005-04-16 15:20:36 -07009149 if (dev_addr_init(dev))
Eric Dumazet29b44332010-10-11 10:22:12 +00009150 goto free_pcpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07009151
Jiri Pirko22bedad32010-04-01 21:22:57 +00009152 dev_mc_init(dev);
Jiri Pirkoa748ee22010-04-01 21:22:09 +00009153 dev_uc_init(dev);
Jiri Pirkoccffad252009-05-22 23:22:17 +00009154
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09009155 dev_net_set(dev, &init_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07009156
Peter P Waskiewicz Jr82cc1a72008-03-21 03:43:19 -07009157 dev->gso_max_size = GSO_MAX_SIZE;
Ben Hutchings30b678d2012-07-30 15:57:00 +00009158 dev->gso_max_segs = GSO_MAX_SEGS;
Linus Torvalds1da177e2005-04-16 15:20:36 -07009159
Herbert Xud565b0a2008-12-15 23:38:52 -08009160 INIT_LIST_HEAD(&dev->napi_list);
Eric W. Biederman9fdce092009-10-30 14:51:13 +00009161 INIT_LIST_HEAD(&dev->unreg_list);
Eric W. Biederman5cde2822013-10-05 19:26:05 -07009162 INIT_LIST_HEAD(&dev->close_list);
Eric Dumazete014deb2009-11-17 05:59:21 +00009163 INIT_LIST_HEAD(&dev->link_watch_list);
Veaceslav Falico2f268f12013-09-25 09:20:07 +02009164 INIT_LIST_HEAD(&dev->adj_list.upper);
9165 INIT_LIST_HEAD(&dev->adj_list.lower);
Salam Noureddine7866a622015-01-27 11:35:48 -08009166 INIT_LIST_HEAD(&dev->ptype_all);
9167 INIT_LIST_HEAD(&dev->ptype_specific);
Jiri Kosina59cc1f62016-08-10 11:05:15 +02009168#ifdef CONFIG_NET_SCHED
9169 hash_init(dev->qdisc_hash);
9170#endif
Eric Dumazet02875872014-10-05 18:38:35 -07009171 dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07009172 setup(dev);
David S. Miller8d3bdbd2011-02-08 15:02:50 -08009173
Phil Suttera8131042016-02-17 15:37:43 +01009174 if (!dev->tx_queue_len) {
Phil Sutterf84bb1e2015-08-27 21:21:36 +02009175 dev->priv_flags |= IFF_NO_QUEUE;
Jesper Dangaard Brouer11597082016-11-03 14:56:06 +01009176 dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN;
Phil Suttera8131042016-02-17 15:37:43 +01009177 }
Phil Sutter906470c2015-08-18 10:30:48 +02009178
David S. Miller8d3bdbd2011-02-08 15:02:50 -08009179 dev->num_tx_queues = txqs;
9180 dev->real_num_tx_queues = txqs;
9181 if (netif_alloc_netdev_queues(dev))
9182 goto free_all;
9183
David S. Miller8d3bdbd2011-02-08 15:02:50 -08009184 dev->num_rx_queues = rxqs;
9185 dev->real_num_rx_queues = rxqs;
9186 if (netif_alloc_rx_queues(dev))
9187 goto free_all;
David S. Miller8d3bdbd2011-02-08 15:02:50 -08009188
Linus Torvalds1da177e2005-04-16 15:20:36 -07009189 strcpy(dev->name, name);
Tom Gundersenc835a672014-07-14 16:37:24 +02009190 dev->name_assign_type = name_assign_type;
Vlad Dogarucbda10f2011-01-13 23:38:30 +00009191 dev->group = INIT_NETDEV_GROUP;
Eric Dumazet2c60db02012-09-16 09:17:26 +00009192 if (!dev->ethtool_ops)
9193 dev->ethtool_ops = &default_ethtool_ops;
Pablo Neirae687ad62015-05-13 18:19:38 +02009194
9195 nf_hook_ingress_init(dev);
9196
Linus Torvalds1da177e2005-04-16 15:20:36 -07009197 return dev;
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00009198
David S. Miller8d3bdbd2011-02-08 15:02:50 -08009199free_all:
9200 free_netdev(dev);
9201 return NULL;
9202
Eric Dumazet29b44332010-10-11 10:22:12 +00009203free_pcpu:
9204 free_percpu(dev->pcpu_refcnt);
Eric Dumazet74d332c2013-10-30 13:10:44 -07009205free_dev:
9206 netdev_freemem(dev);
Jiri Pirkoab9c73c2009-05-08 13:30:17 +00009207 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07009208}
Tom Herbert36909ea2011-01-09 19:36:31 +00009209EXPORT_SYMBOL(alloc_netdev_mqs);
Linus Torvalds1da177e2005-04-16 15:20:36 -07009210
9211/**
tcharding722c9a02017-02-09 17:56:04 +11009212 * free_netdev - free network device
9213 * @dev: device
Linus Torvalds1da177e2005-04-16 15:20:36 -07009214 *
tcharding722c9a02017-02-09 17:56:04 +11009215 * This function does the last stage of destroying an allocated device
9216 * interface. The reference to the device object is released. If this
9217 * is the last reference then it will be freed.Must be called in process
9218 * context.
Linus Torvalds1da177e2005-04-16 15:20:36 -07009219 */
9220void free_netdev(struct net_device *dev)
9221{
Herbert Xud565b0a2008-12-15 23:38:52 -08009222 struct napi_struct *p, *n;
9223
Eric Dumazet93d05d42015-11-18 06:31:03 -08009224 might_sleep();
Eric Dumazet60877a32013-06-20 01:15:51 -07009225 netif_free_tx_queues(dev);
Jesper Dangaard Brouere817f852018-01-03 11:26:09 +01009226 netif_free_rx_queues(dev);
David S. Millere8a04642008-07-17 00:34:19 -07009227
Eric Dumazet33d480c2011-08-11 19:30:52 +00009228 kfree(rcu_dereference_protected(dev->ingress_queue, 1));
Eric Dumazet24824a02010-10-02 06:11:55 +00009229
Jiri Pirkof001fde2009-05-05 02:48:28 +00009230 /* Flush device addresses */
9231 dev_addr_flush(dev);
9232
Herbert Xud565b0a2008-12-15 23:38:52 -08009233 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
9234 netif_napi_del(p);
9235
Eric Dumazet29b44332010-10-11 10:22:12 +00009236 free_percpu(dev->pcpu_refcnt);
9237 dev->pcpu_refcnt = NULL;
9238
Stephen Hemminger3041a062006-05-26 13:25:24 -07009239 /* Compatibility with error handling in drivers */
Linus Torvalds1da177e2005-04-16 15:20:36 -07009240 if (dev->reg_state == NETREG_UNINITIALIZED) {
Eric Dumazet74d332c2013-10-30 13:10:44 -07009241 netdev_freemem(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07009242 return;
9243 }
9244
9245 BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
9246 dev->reg_state = NETREG_RELEASED;
9247
Greg Kroah-Hartman43cb76d2002-04-09 12:14:34 -07009248 /* will free via device release */
9249 put_device(&dev->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07009250}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07009251EXPORT_SYMBOL(free_netdev);
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09009252
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07009253/**
9254 * synchronize_net - Synchronize with packet receive processing
9255 *
9256 * Wait for packets currently being received to be done.
9257 * Does not block later packets from starting.
9258 */
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +09009259void synchronize_net(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07009260{
9261 might_sleep();
Eric Dumazetbe3fc412011-05-23 23:07:32 +00009262 if (rtnl_is_locked())
9263 synchronize_rcu_expedited();
9264 else
9265 synchronize_rcu();
Linus Torvalds1da177e2005-04-16 15:20:36 -07009266}
Eric Dumazetd1b19df2009-09-03 01:29:39 -07009267EXPORT_SYMBOL(synchronize_net);
Linus Torvalds1da177e2005-04-16 15:20:36 -07009268
9269/**
Eric Dumazet44a08732009-10-27 07:03:04 +00009270 * unregister_netdevice_queue - remove device from the kernel
Linus Torvalds1da177e2005-04-16 15:20:36 -07009271 * @dev: device
Eric Dumazet44a08732009-10-27 07:03:04 +00009272 * @head: list
Jaswinder Singh Rajput6ebfbc02009-11-22 20:43:13 -08009273 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07009274 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08009275 * from the kernel tables.
Eric Dumazet44a08732009-10-27 07:03:04 +00009276 * If head not NULL, device is queued to be unregistered later.
Linus Torvalds1da177e2005-04-16 15:20:36 -07009277 *
9278 * Callers must hold the rtnl semaphore. You may want
9279 * unregister_netdev() instead of this.
9280 */
9281
Eric Dumazet44a08732009-10-27 07:03:04 +00009282void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07009283{
Herbert Xua6620712007-12-12 19:21:56 -08009284 ASSERT_RTNL();
9285
Eric Dumazet44a08732009-10-27 07:03:04 +00009286 if (head) {
Eric W. Biederman9fdce092009-10-30 14:51:13 +00009287 list_move_tail(&dev->unreg_list, head);
Eric Dumazet44a08732009-10-27 07:03:04 +00009288 } else {
9289 rollback_registered(dev);
9290 /* Finish processing unregister after unlock */
9291 net_set_todo(dev);
9292 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07009293}
Eric Dumazet44a08732009-10-27 07:03:04 +00009294EXPORT_SYMBOL(unregister_netdevice_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07009295
9296/**
Eric Dumazet9b5e3832009-10-27 07:04:19 +00009297 * unregister_netdevice_many - unregister many devices
9298 * @head: list of devices
Eric Dumazet87757a92014-06-06 06:44:03 -07009299 *
9300 * Note: As most callers use a stack allocated list_head,
9301 * we force a list_del() to make sure stack wont be corrupted later.
Eric Dumazet9b5e3832009-10-27 07:04:19 +00009302 */
9303void unregister_netdevice_many(struct list_head *head)
9304{
9305 struct net_device *dev;
9306
9307 if (!list_empty(head)) {
9308 rollback_registered_many(head);
9309 list_for_each_entry(dev, head, unreg_list)
9310 net_set_todo(dev);
Eric Dumazet87757a92014-06-06 06:44:03 -07009311 list_del(head);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00009312 }
9313}
Eric Dumazet63c80992009-10-27 07:06:49 +00009314EXPORT_SYMBOL(unregister_netdevice_many);
Eric Dumazet9b5e3832009-10-27 07:04:19 +00009315
9316/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07009317 * unregister_netdev - remove device from the kernel
9318 * @dev: device
9319 *
9320 * This function shuts down a device interface and removes it
Wang Chend59b54b2007-12-11 02:28:03 -08009321 * from the kernel tables.
Linus Torvalds1da177e2005-04-16 15:20:36 -07009322 *
9323 * This is just a wrapper for unregister_netdevice that takes
9324 * the rtnl semaphore. In general you want to use this and not
9325 * unregister_netdevice.
9326 */
9327void unregister_netdev(struct net_device *dev)
9328{
9329 rtnl_lock();
9330 unregister_netdevice(dev);
9331 rtnl_unlock();
9332}
Linus Torvalds1da177e2005-04-16 15:20:36 -07009333EXPORT_SYMBOL(unregister_netdev);
9334
Eric W. Biedermance286d32007-09-12 13:53:49 +02009335/**
9336 * dev_change_net_namespace - move device to different nethost namespace
9337 * @dev: device
9338 * @net: network namespace
9339 * @pat: If not NULL name pattern to try if the current device name
9340 * is already taken in the destination network namespace.
9341 *
9342 * This function shuts down a device interface and moves it
9343 * to a new network namespace. On success 0 is returned, on
9344 * a failure a netagive errno code is returned.
9345 *
9346 * Callers must hold the rtnl semaphore.
9347 */
9348
9349int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
9350{
Nicolas Dichtel38e01b32018-01-25 15:01:39 +01009351 int err, new_nsid, new_ifindex;
Eric W. Biedermance286d32007-09-12 13:53:49 +02009352
9353 ASSERT_RTNL();
9354
9355 /* Don't allow namespace local devices to be moved. */
9356 err = -EINVAL;
9357 if (dev->features & NETIF_F_NETNS_LOCAL)
9358 goto out;
9359
9360 /* Ensure the device has been registrered */
Eric W. Biedermance286d32007-09-12 13:53:49 +02009361 if (dev->reg_state != NETREG_REGISTERED)
9362 goto out;
9363
9364 /* Get out if there is nothing todo */
9365 err = 0;
YOSHIFUJI Hideaki878628f2008-03-26 03:57:35 +09009366 if (net_eq(dev_net(dev), net))
Eric W. Biedermance286d32007-09-12 13:53:49 +02009367 goto out;
9368
9369 /* Pick the destination device name, and ensure
9370 * we can use it in the destination network namespace.
9371 */
9372 err = -EEXIST;
Octavian Purdilad9031022009-11-18 02:36:59 +00009373 if (__dev_get_by_name(net, dev->name)) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02009374 /* We get here if we can't use the current device name */
9375 if (!pat)
9376 goto out;
Li RongQing7892bd02018-06-19 17:23:17 +08009377 err = dev_get_valid_name(net, dev, pat);
9378 if (err < 0)
Eric W. Biedermance286d32007-09-12 13:53:49 +02009379 goto out;
9380 }
9381
9382 /*
9383 * And now a mini version of register_netdevice unregister_netdevice.
9384 */
9385
9386 /* If device is running close it first. */
Pavel Emelyanov9b772652007-10-10 02:49:09 -07009387 dev_close(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02009388
9389 /* And unlink it from device chain */
Eric W. Biedermance286d32007-09-12 13:53:49 +02009390 unlist_netdevice(dev);
9391
9392 synchronize_net();
9393
9394 /* Shutdown queueing discipline. */
9395 dev_shutdown(dev);
9396
9397 /* Notify protocols, that we are about to destroy
tchardingeb13da12017-02-09 17:56:06 +11009398 * this device. They should clean all the things.
9399 *
9400 * Note that dev->reg_state stays at NETREG_REGISTERED.
9401 * This is wanted because this way 8021q and macvlan know
9402 * the device is just moving and can keep their slaves up.
9403 */
Eric W. Biedermance286d32007-09-12 13:53:49 +02009404 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
Gao feng6549dd42012-08-23 15:36:55 +00009405 rcu_barrier();
Nicolas Dichtel38e01b32018-01-25 15:01:39 +01009406
Nicolas Dichtelc36ac8e2018-01-25 15:01:38 +01009407 new_nsid = peernet2id_alloc(dev_net(dev), net);
Nicolas Dichtel38e01b32018-01-25 15:01:39 +01009408 /* If there is an ifindex conflict assign a new one */
9409 if (__dev_get_by_index(net, dev->ifindex))
9410 new_ifindex = dev_new_index(net);
9411 else
9412 new_ifindex = dev->ifindex;
9413
9414 rtmsg_ifinfo_newnet(RTM_DELLINK, dev, ~0U, GFP_KERNEL, &new_nsid,
9415 new_ifindex);
Eric W. Biedermance286d32007-09-12 13:53:49 +02009416
9417 /*
9418 * Flush the unicast and multicast chains
9419 */
Jiri Pirkoa748ee22010-04-01 21:22:09 +00009420 dev_uc_flush(dev);
Jiri Pirko22bedad32010-04-01 21:22:57 +00009421 dev_mc_flush(dev);
Eric W. Biedermance286d32007-09-12 13:53:49 +02009422
Serge Hallyn4e66ae22012-12-03 16:17:12 +00009423 /* Send a netdev-removed uevent to the old namespace */
9424 kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04009425 netdev_adjacent_del_links(dev);
Serge Hallyn4e66ae22012-12-03 16:17:12 +00009426
Eric W. Biedermance286d32007-09-12 13:53:49 +02009427 /* Actually switch the network namespace */
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09009428 dev_net_set(dev, net);
Nicolas Dichtel38e01b32018-01-25 15:01:39 +01009429 dev->ifindex = new_ifindex;
Eric W. Biedermance286d32007-09-12 13:53:49 +02009430
Serge Hallyn4e66ae22012-12-03 16:17:12 +00009431 /* Send a netdev-add uevent to the new namespace */
9432 kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
Alexander Y. Fomichev4c754312014-08-25 16:26:45 +04009433 netdev_adjacent_add_links(dev);
Serge Hallyn4e66ae22012-12-03 16:17:12 +00009434
Eric W. Biederman8b41d182007-09-26 22:02:53 -07009435 /* Fixup kobjects */
Eric W. Biedermana1b3f592010-05-04 17:36:49 -07009436 err = device_rename(&dev->dev, dev->name);
Eric W. Biederman8b41d182007-09-26 22:02:53 -07009437 WARN_ON(err);
Eric W. Biedermance286d32007-09-12 13:53:49 +02009438
9439 /* Add the device back in the hashes */
9440 list_netdevice(dev);
9441
9442 /* Notify protocols, that a new device appeared. */
9443 call_netdevice_notifiers(NETDEV_REGISTER, dev);
9444
Eric W. Biedermand90a9092009-12-12 22:11:15 +00009445 /*
9446 * Prevent userspace races by waiting until the network
9447 * device is fully setup before sending notifications.
9448 */
Alexei Starovoitov7f294052013-10-23 16:02:42 -07009449 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
Eric W. Biedermand90a9092009-12-12 22:11:15 +00009450
Eric W. Biedermance286d32007-09-12 13:53:49 +02009451 synchronize_net();
9452 err = 0;
9453out:
9454 return err;
9455}
Johannes Berg463d0182009-07-14 00:33:35 +02009456EXPORT_SYMBOL_GPL(dev_change_net_namespace);
Eric W. Biedermance286d32007-09-12 13:53:49 +02009457
Sebastian Andrzej Siewiorf0bf90d2016-11-03 15:50:04 +01009458static int dev_cpu_dead(unsigned int oldcpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07009459{
9460 struct sk_buff **list_skb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07009461 struct sk_buff *skb;
Sebastian Andrzej Siewiorf0bf90d2016-11-03 15:50:04 +01009462 unsigned int cpu;
Ashwanth Goli97d8b6e2017-06-13 16:54:55 +05309463 struct softnet_data *sd, *oldsd, *remsd = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07009464
Linus Torvalds1da177e2005-04-16 15:20:36 -07009465 local_irq_disable();
9466 cpu = smp_processor_id();
9467 sd = &per_cpu(softnet_data, cpu);
9468 oldsd = &per_cpu(softnet_data, oldcpu);
9469
9470 /* Find end of our completion_queue. */
9471 list_skb = &sd->completion_queue;
9472 while (*list_skb)
9473 list_skb = &(*list_skb)->next;
9474 /* Append completion queue from offline CPU. */
9475 *list_skb = oldsd->completion_queue;
9476 oldsd->completion_queue = NULL;
9477
Linus Torvalds1da177e2005-04-16 15:20:36 -07009478 /* Append output queue from offline CPU. */
Changli Gaoa9cbd582010-04-26 23:06:24 +00009479 if (oldsd->output_queue) {
9480 *sd->output_queue_tailp = oldsd->output_queue;
9481 sd->output_queue_tailp = oldsd->output_queue_tailp;
9482 oldsd->output_queue = NULL;
9483 oldsd->output_queue_tailp = &oldsd->output_queue;
9484 }
Eric Dumazetac64da02015-01-15 17:04:22 -08009485 /* Append NAPI poll list from offline CPU, with one exception :
9486 * process_backlog() must be called by cpu owning percpu backlog.
9487 * We properly handle process_queue & input_pkt_queue later.
9488 */
9489 while (!list_empty(&oldsd->poll_list)) {
9490 struct napi_struct *napi = list_first_entry(&oldsd->poll_list,
9491 struct napi_struct,
9492 poll_list);
9493
9494 list_del_init(&napi->poll_list);
9495 if (napi->poll == process_backlog)
9496 napi->state = 0;
9497 else
9498 ____napi_schedule(sd, napi);
Heiko Carstens264524d2011-06-06 20:50:03 +00009499 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07009500
9501 raise_softirq_irqoff(NET_TX_SOFTIRQ);
9502 local_irq_enable();
9503
ashwanth@codeaurora.org773fc8f2017-06-09 14:24:58 +05309504#ifdef CONFIG_RPS
9505 remsd = oldsd->rps_ipi_list;
9506 oldsd->rps_ipi_list = NULL;
9507#endif
9508 /* send out pending IPI's on offline CPU */
9509 net_rps_send_ipi(remsd);
9510
Linus Torvalds1da177e2005-04-16 15:20:36 -07009511 /* Process offline CPU's input_pkt_queue */
Tom Herbert76cc8b12010-05-20 18:37:59 +00009512 while ((skb = __skb_dequeue(&oldsd->process_queue))) {
Eric Dumazet91e83132015-02-05 14:58:14 -08009513 netif_rx_ni(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00009514 input_queue_head_incr(oldsd);
9515 }
Eric Dumazetac64da02015-01-15 17:04:22 -08009516 while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
Eric Dumazet91e83132015-02-05 14:58:14 -08009517 netif_rx_ni(skb);
Tom Herbert76cc8b12010-05-20 18:37:59 +00009518 input_queue_head_incr(oldsd);
Tom Herbertfec5e652010-04-16 16:01:27 -07009519 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07009520
Sebastian Andrzej Siewiorf0bf90d2016-11-03 15:50:04 +01009521 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07009522}
Linus Torvalds1da177e2005-04-16 15:20:36 -07009523
Herbert Xu7f353bf2007-08-10 15:47:58 -07009524/**
Herbert Xub63365a2008-10-23 01:11:29 -07009525 * netdev_increment_features - increment feature set by one
9526 * @all: current feature set
9527 * @one: new feature set
9528 * @mask: mask feature set
Herbert Xu7f353bf2007-08-10 15:47:58 -07009529 *
9530 * Computes a new feature set after adding a device with feature set
Herbert Xub63365a2008-10-23 01:11:29 -07009531 * @one to the master device with current feature set @all. Will not
9532 * enable anything that is off in @mask. Returns the new feature set.
Herbert Xu7f353bf2007-08-10 15:47:58 -07009533 */
Michał Mirosławc8f44af2011-11-15 15:29:55 +00009534netdev_features_t netdev_increment_features(netdev_features_t all,
9535 netdev_features_t one, netdev_features_t mask)
Herbert Xu7f353bf2007-08-10 15:47:58 -07009536{
Tom Herbertc8cd0982015-12-14 11:19:44 -08009537 if (mask & NETIF_F_HW_CSUM)
Tom Herberta1882222015-12-14 11:19:43 -08009538 mask |= NETIF_F_CSUM_MASK;
Michał Mirosław1742f182011-04-22 06:31:16 +00009539 mask |= NETIF_F_VLAN_CHALLENGED;
9540
Tom Herberta1882222015-12-14 11:19:43 -08009541 all |= one & (NETIF_F_ONE_FOR_ALL | NETIF_F_CSUM_MASK) & mask;
Michał Mirosław1742f182011-04-22 06:31:16 +00009542 all &= one | ~NETIF_F_ALL_FOR_ALL;
9543
Michał Mirosław1742f182011-04-22 06:31:16 +00009544 /* If one device supports hw checksumming, set for all. */
Tom Herbertc8cd0982015-12-14 11:19:44 -08009545 if (all & NETIF_F_HW_CSUM)
9546 all &= ~(NETIF_F_CSUM_MASK & ~NETIF_F_HW_CSUM);
Herbert Xu7f353bf2007-08-10 15:47:58 -07009547
9548 return all;
9549}
Herbert Xub63365a2008-10-23 01:11:29 -07009550EXPORT_SYMBOL(netdev_increment_features);
Herbert Xu7f353bf2007-08-10 15:47:58 -07009551
Baruch Siach430f03c2013-06-02 20:43:55 +00009552static struct hlist_head * __net_init netdev_create_hash(void)
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07009553{
9554 int i;
9555 struct hlist_head *hash;
9556
Kees Cook6da2ec52018-06-12 13:55:00 -07009557 hash = kmalloc_array(NETDEV_HASHENTRIES, sizeof(*hash), GFP_KERNEL);
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07009558 if (hash != NULL)
9559 for (i = 0; i < NETDEV_HASHENTRIES; i++)
9560 INIT_HLIST_HEAD(&hash[i]);
9561
9562 return hash;
9563}
9564
Eric W. Biederman881d9662007-09-17 11:56:21 -07009565/* Initialize per network namespace state */
Pavel Emelyanov46650792007-10-08 20:38:39 -07009566static int __net_init netdev_init(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07009567{
Li RongQingd9f37d02018-07-13 14:41:36 +08009568 BUILD_BUG_ON(GRO_HASH_BUCKETS >
David S. Millerccdb5172018-07-16 17:02:04 -07009569 8 * FIELD_SIZEOF(struct napi_struct, gro_bitmask));
Li RongQingd9f37d02018-07-13 14:41:36 +08009570
Rustad, Mark D734b6542012-07-18 09:06:07 +00009571 if (net != &init_net)
9572 INIT_LIST_HEAD(&net->dev_base_head);
Eric W. Biederman881d9662007-09-17 11:56:21 -07009573
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07009574 net->dev_name_head = netdev_create_hash();
9575 if (net->dev_name_head == NULL)
9576 goto err_name;
Eric W. Biederman881d9662007-09-17 11:56:21 -07009577
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07009578 net->dev_index_head = netdev_create_hash();
9579 if (net->dev_index_head == NULL)
9580 goto err_idx;
Eric W. Biederman881d9662007-09-17 11:56:21 -07009581
9582 return 0;
Pavel Emelyanov30d97d32007-09-16 15:40:33 -07009583
9584err_idx:
9585 kfree(net->dev_name_head);
9586err_name:
9587 return -ENOMEM;
Eric W. Biederman881d9662007-09-17 11:56:21 -07009588}
9589
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07009590/**
9591 * netdev_drivername - network driver for the device
9592 * @dev: network device
Stephen Hemmingerf0db2752008-09-30 02:23:58 -07009593 *
9594 * Determine network driver for device.
9595 */
David S. Miller3019de12011-06-06 16:41:33 -07009596const char *netdev_drivername(const struct net_device *dev)
Arjan van de Ven6579e572008-07-21 13:31:48 -07009597{
Stephen Hemmingercf04a4c72008-09-30 02:22:14 -07009598 const struct device_driver *driver;
9599 const struct device *parent;
David S. Miller3019de12011-06-06 16:41:33 -07009600 const char *empty = "";
Arjan van de Ven6579e572008-07-21 13:31:48 -07009601
9602 parent = dev->dev.parent;
Arjan van de Ven6579e572008-07-21 13:31:48 -07009603 if (!parent)
David S. Miller3019de12011-06-06 16:41:33 -07009604 return empty;
Arjan van de Ven6579e572008-07-21 13:31:48 -07009605
9606 driver = parent->driver;
9607 if (driver && driver->name)
David S. Miller3019de12011-06-06 16:41:33 -07009608 return driver->name;
9609 return empty;
Arjan van de Ven6579e572008-07-21 13:31:48 -07009610}
9611
Joe Perches6ea754e2014-09-22 11:10:50 -07009612static void __netdev_printk(const char *level, const struct net_device *dev,
9613 struct va_format *vaf)
Joe Perches256df2f2010-06-27 01:02:35 +00009614{
Joe Perchesb004ff42012-09-12 20:12:19 -07009615 if (dev && dev->dev.parent) {
Joe Perches6ea754e2014-09-22 11:10:50 -07009616 dev_printk_emit(level[1] - '0',
9617 dev->dev.parent,
9618 "%s %s %s%s: %pV",
9619 dev_driver_string(dev->dev.parent),
9620 dev_name(dev->dev.parent),
9621 netdev_name(dev), netdev_reg_state(dev),
9622 vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07009623 } else if (dev) {
Joe Perches6ea754e2014-09-22 11:10:50 -07009624 printk("%s%s%s: %pV",
9625 level, netdev_name(dev), netdev_reg_state(dev), vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07009626 } else {
Joe Perches6ea754e2014-09-22 11:10:50 -07009627 printk("%s(NULL net_device): %pV", level, vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07009628 }
Joe Perches256df2f2010-06-27 01:02:35 +00009629}
9630
Joe Perches6ea754e2014-09-22 11:10:50 -07009631void netdev_printk(const char *level, const struct net_device *dev,
9632 const char *format, ...)
Joe Perches256df2f2010-06-27 01:02:35 +00009633{
9634 struct va_format vaf;
9635 va_list args;
Joe Perches256df2f2010-06-27 01:02:35 +00009636
9637 va_start(args, format);
9638
9639 vaf.fmt = format;
9640 vaf.va = &args;
9641
Joe Perches6ea754e2014-09-22 11:10:50 -07009642 __netdev_printk(level, dev, &vaf);
Joe Perchesb004ff42012-09-12 20:12:19 -07009643
Joe Perches256df2f2010-06-27 01:02:35 +00009644 va_end(args);
Joe Perches256df2f2010-06-27 01:02:35 +00009645}
9646EXPORT_SYMBOL(netdev_printk);
9647
9648#define define_netdev_printk_level(func, level) \
Joe Perches6ea754e2014-09-22 11:10:50 -07009649void func(const struct net_device *dev, const char *fmt, ...) \
Joe Perches256df2f2010-06-27 01:02:35 +00009650{ \
Joe Perches256df2f2010-06-27 01:02:35 +00009651 struct va_format vaf; \
9652 va_list args; \
9653 \
9654 va_start(args, fmt); \
9655 \
9656 vaf.fmt = fmt; \
9657 vaf.va = &args; \
9658 \
Joe Perches6ea754e2014-09-22 11:10:50 -07009659 __netdev_printk(level, dev, &vaf); \
Joe Perchesb004ff42012-09-12 20:12:19 -07009660 \
Joe Perches256df2f2010-06-27 01:02:35 +00009661 va_end(args); \
Joe Perches256df2f2010-06-27 01:02:35 +00009662} \
9663EXPORT_SYMBOL(func);
9664
9665define_netdev_printk_level(netdev_emerg, KERN_EMERG);
9666define_netdev_printk_level(netdev_alert, KERN_ALERT);
9667define_netdev_printk_level(netdev_crit, KERN_CRIT);
9668define_netdev_printk_level(netdev_err, KERN_ERR);
9669define_netdev_printk_level(netdev_warn, KERN_WARNING);
9670define_netdev_printk_level(netdev_notice, KERN_NOTICE);
9671define_netdev_printk_level(netdev_info, KERN_INFO);
9672
Pavel Emelyanov46650792007-10-08 20:38:39 -07009673static void __net_exit netdev_exit(struct net *net)
Eric W. Biederman881d9662007-09-17 11:56:21 -07009674{
9675 kfree(net->dev_name_head);
9676 kfree(net->dev_index_head);
Vasily Averinee21b18b2017-11-12 22:28:46 +03009677 if (net != &init_net)
9678 WARN_ON_ONCE(!list_empty(&net->dev_base_head));
Eric W. Biederman881d9662007-09-17 11:56:21 -07009679}
9680
Denis V. Lunev022cbae2007-11-13 03:23:50 -08009681static struct pernet_operations __net_initdata netdev_net_ops = {
Eric W. Biederman881d9662007-09-17 11:56:21 -07009682 .init = netdev_init,
9683 .exit = netdev_exit,
9684};
9685
Pavel Emelyanov46650792007-10-08 20:38:39 -07009686static void __net_exit default_device_exit(struct net *net)
Eric W. Biedermance286d32007-09-12 13:53:49 +02009687{
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00009688 struct net_device *dev, *aux;
Eric W. Biedermance286d32007-09-12 13:53:49 +02009689 /*
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00009690 * Push all migratable network devices back to the
Eric W. Biedermance286d32007-09-12 13:53:49 +02009691 * initial network namespace
9692 */
9693 rtnl_lock();
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00009694 for_each_netdev_safe(net, dev, aux) {
Eric W. Biedermance286d32007-09-12 13:53:49 +02009695 int err;
Pavel Emelyanovaca51392008-05-08 01:24:25 -07009696 char fb_name[IFNAMSIZ];
Eric W. Biedermance286d32007-09-12 13:53:49 +02009697
9698 /* Ignore unmoveable devices (i.e. loopback) */
9699 if (dev->features & NETIF_F_NETNS_LOCAL)
9700 continue;
9701
Eric W. Biedermane008b5f2009-11-29 22:25:30 +00009702 /* Leave virtual devices for the generic cleanup */
9703 if (dev->rtnl_link_ops)
9704 continue;
Eric W. Biedermand0c082c2008-11-05 15:59:38 -08009705
Lucas De Marchi25985ed2011-03-30 22:57:33 -03009706 /* Push remaining network devices to init_net */
Pavel Emelyanovaca51392008-05-08 01:24:25 -07009707 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
9708 err = dev_change_net_namespace(dev, &init_net, fb_name);
Eric W. Biedermance286d32007-09-12 13:53:49 +02009709 if (err) {
Joe Perches7b6cd1c2012-02-01 10:54:43 +00009710 pr_emerg("%s: failed to move %s to init_net: %d\n",
9711 __func__, dev->name, err);
Pavel Emelyanovaca51392008-05-08 01:24:25 -07009712 BUG();
Eric W. Biedermance286d32007-09-12 13:53:49 +02009713 }
9714 }
9715 rtnl_unlock();
9716}
9717
Eric W. Biederman50624c92013-09-23 21:19:49 -07009718static void __net_exit rtnl_lock_unregistering(struct list_head *net_list)
9719{
9720 /* Return with the rtnl_lock held when there are no network
9721 * devices unregistering in any network namespace in net_list.
9722 */
9723 struct net *net;
9724 bool unregistering;
Peter Zijlstraff960a72014-10-29 17:04:56 +01009725 DEFINE_WAIT_FUNC(wait, woken_wake_function);
Eric W. Biederman50624c92013-09-23 21:19:49 -07009726
Peter Zijlstraff960a72014-10-29 17:04:56 +01009727 add_wait_queue(&netdev_unregistering_wq, &wait);
Eric W. Biederman50624c92013-09-23 21:19:49 -07009728 for (;;) {
Eric W. Biederman50624c92013-09-23 21:19:49 -07009729 unregistering = false;
9730 rtnl_lock();
9731 list_for_each_entry(net, net_list, exit_list) {
9732 if (net->dev_unreg_count > 0) {
9733 unregistering = true;
9734 break;
9735 }
9736 }
9737 if (!unregistering)
9738 break;
9739 __rtnl_unlock();
Peter Zijlstraff960a72014-10-29 17:04:56 +01009740
9741 wait_woken(&wait, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
Eric W. Biederman50624c92013-09-23 21:19:49 -07009742 }
Peter Zijlstraff960a72014-10-29 17:04:56 +01009743 remove_wait_queue(&netdev_unregistering_wq, &wait);
Eric W. Biederman50624c92013-09-23 21:19:49 -07009744}
9745
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00009746static void __net_exit default_device_exit_batch(struct list_head *net_list)
9747{
9748 /* At exit all network devices most be removed from a network
Uwe Kleine-Königb5950762010-11-01 15:38:34 -04009749 * namespace. Do this in the reverse order of registration.
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00009750 * Do this across as many network namespaces as possible to
9751 * improve batching efficiency.
9752 */
9753 struct net_device *dev;
9754 struct net *net;
9755 LIST_HEAD(dev_kill_list);
9756
Eric W. Biederman50624c92013-09-23 21:19:49 -07009757 /* To prevent network device cleanup code from dereferencing
9758 * loopback devices or network devices that have been freed
9759 * wait here for all pending unregistrations to complete,
9760 * before unregistring the loopback device and allowing the
9761 * network namespace be freed.
9762 *
9763 * The netdev todo list containing all network devices
9764 * unregistrations that happen in default_device_exit_batch
9765 * will run in the rtnl_unlock() at the end of
9766 * default_device_exit_batch.
9767 */
9768 rtnl_lock_unregistering(net_list);
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00009769 list_for_each_entry(net, net_list, exit_list) {
9770 for_each_netdev_reverse(net, dev) {
Jiri Pirkob0ab2fa2014-06-26 09:58:25 +02009771 if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink)
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00009772 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
9773 else
9774 unregister_netdevice_queue(dev, &dev_kill_list);
9775 }
9776 }
9777 unregister_netdevice_many(&dev_kill_list);
9778 rtnl_unlock();
9779}
9780
Denis V. Lunev022cbae2007-11-13 03:23:50 -08009781static struct pernet_operations __net_initdata default_device_ops = {
Eric W. Biedermance286d32007-09-12 13:53:49 +02009782 .exit = default_device_exit,
Eric W. Biederman04dc7f6b2009-12-03 02:29:04 +00009783 .exit_batch = default_device_exit_batch,
Eric W. Biedermance286d32007-09-12 13:53:49 +02009784};
9785
Linus Torvalds1da177e2005-04-16 15:20:36 -07009786/*
9787 * Initialize the DEV module. At boot time this walks the device list and
9788 * unhooks any devices that fail to initialise (normally hardware not
9789 * present) and leaves us with a valid list of present and active devices.
9790 *
9791 */
9792
9793/*
9794 * This is called single threaded during boot, so no need
9795 * to take the rtnl semaphore.
9796 */
9797static int __init net_dev_init(void)
9798{
9799 int i, rc = -ENOMEM;
9800
9801 BUG_ON(!dev_boot_phase);
9802
Linus Torvalds1da177e2005-04-16 15:20:36 -07009803 if (dev_proc_init())
9804 goto out;
9805
Eric W. Biederman8b41d182007-09-26 22:02:53 -07009806 if (netdev_kobject_init())
Linus Torvalds1da177e2005-04-16 15:20:36 -07009807 goto out;
9808
9809 INIT_LIST_HEAD(&ptype_all);
Pavel Emelyanov82d8a8672007-11-26 20:12:58 +08009810 for (i = 0; i < PTYPE_HASH_SIZE; i++)
Linus Torvalds1da177e2005-04-16 15:20:36 -07009811 INIT_LIST_HEAD(&ptype_base[i]);
9812
Vlad Yasevich62532da9d2012-11-15 08:49:10 +00009813 INIT_LIST_HEAD(&offload_base);
9814
Eric W. Biederman881d9662007-09-17 11:56:21 -07009815 if (register_pernet_subsys(&netdev_net_ops))
9816 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07009817
9818 /*
9819 * Initialise the packet receive queues.
9820 */
9821
KAMEZAWA Hiroyuki6f912042006-04-10 22:52:50 -07009822 for_each_possible_cpu(i) {
Eric Dumazet41852492016-08-26 12:50:39 -07009823 struct work_struct *flush = per_cpu_ptr(&flush_works, i);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00009824 struct softnet_data *sd = &per_cpu(softnet_data, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07009825
Eric Dumazet41852492016-08-26 12:50:39 -07009826 INIT_WORK(flush, flush_backlog);
9827
Eric Dumazete36fa2f2010-04-19 21:17:14 +00009828 skb_queue_head_init(&sd->input_pkt_queue);
Changli Gao6e7676c2010-04-27 15:07:33 -07009829 skb_queue_head_init(&sd->process_queue);
Steffen Klassertf53c7232017-12-20 10:41:36 +01009830#ifdef CONFIG_XFRM_OFFLOAD
9831 skb_queue_head_init(&sd->xfrm_backlog);
9832#endif
Eric Dumazete36fa2f2010-04-19 21:17:14 +00009833 INIT_LIST_HEAD(&sd->poll_list);
Changli Gaoa9cbd582010-04-26 23:06:24 +00009834 sd->output_queue_tailp = &sd->output_queue;
Eric Dumazetdf334542010-03-24 19:13:54 +00009835#ifdef CONFIG_RPS
Eric Dumazete36fa2f2010-04-19 21:17:14 +00009836 sd->csd.func = rps_trigger_softirq;
9837 sd->csd.info = sd;
Eric Dumazete36fa2f2010-04-19 21:17:14 +00009838 sd->cpu = i;
Tom Herbert1e94d722010-03-18 17:45:44 -07009839#endif
Tom Herbert0a9627f2010-03-16 08:03:29 +00009840
David S. Miller7c4ec742018-07-20 23:37:55 -07009841 init_gro_hash(&sd->backlog);
Eric Dumazete36fa2f2010-04-19 21:17:14 +00009842 sd->backlog.poll = process_backlog;
9843 sd->backlog.weight = weight_p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07009844 }
9845
Linus Torvalds1da177e2005-04-16 15:20:36 -07009846 dev_boot_phase = 0;
9847
Eric W. Biederman505d4f72008-11-07 22:54:20 -08009848 /* The loopback device is special if any other network devices
9849 * is present in a network namespace the loopback device must
9850 * be present. Since we now dynamically allocate and free the
9851 * loopback device ensure this invariant is maintained by
9852 * keeping the loopback device as the first device on the
9853 * list of network devices. Ensuring the loopback devices
9854 * is the first device that appears and the last network device
9855 * that disappears.
9856 */
9857 if (register_pernet_device(&loopback_net_ops))
9858 goto out;
9859
9860 if (register_pernet_device(&default_device_ops))
9861 goto out;
9862
Carlos R. Mafra962cf362008-05-15 11:15:37 -03009863 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
9864 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
Linus Torvalds1da177e2005-04-16 15:20:36 -07009865
Sebastian Andrzej Siewiorf0bf90d2016-11-03 15:50:04 +01009866 rc = cpuhp_setup_state_nocalls(CPUHP_NET_DEV_DEAD, "net/dev:dead",
9867 NULL, dev_cpu_dead);
9868 WARN_ON(rc < 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07009869 rc = 0;
9870out:
9871 return rc;
9872}
9873
9874subsys_initcall(net_dev_init);