blob: 6ce8dd19f33c3aafef0b9dab648a44fdd3478866 [file] [log] [blame]
Thomas Gleixner2874c5f2019-05-27 08:55:01 +02001// SPDX-License-Identifier: GPL-2.0-or-later
Linus Torvalds1da177e2005-04-16 15:20:36 -07002/*
3 * INET An implementation of the TCP/IP protocol suite for the LINUX
4 * operating system. INET is implemented using the BSD Socket
5 * interface as the means of communication with the user level.
6 *
7 * PACKET - implements raw packet sockets.
8 *
Jesper Juhl02c30a82005-05-05 16:16:16 -07009 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -070010 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
11 * Alan Cox, <gw4pts@gw4pts.ampr.org>
12 *
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +090013 * Fixes:
Linus Torvalds1da177e2005-04-16 15:20:36 -070014 * Alan Cox : verify_area() now used correctly
15 * Alan Cox : new skbuff lists, look ma no backlogs!
16 * Alan Cox : tidied skbuff lists.
17 * Alan Cox : Now uses generic datagram routines I
18 * added. Also fixed the peek/read crash
19 * from all old Linux datagram code.
20 * Alan Cox : Uses the improved datagram code.
21 * Alan Cox : Added NULL's for socket options.
22 * Alan Cox : Re-commented the code.
23 * Alan Cox : Use new kernel side addressing
24 * Rob Janssen : Correct MTU usage.
25 * Dave Platt : Counter leaks caused by incorrect
26 * interrupt locking and some slightly
27 * dubious gcc output. Can you read
28 * compiler: it said _VOLATILE_
29 * Richard Kooijman : Timestamp fixes.
30 * Alan Cox : New buffers. Use sk->mac.raw.
31 * Alan Cox : sendmsg/recvmsg support.
32 * Alan Cox : Protocol setting support
33 * Alexey Kuznetsov : Untied from IPv4 stack.
34 * Cyrus Durgin : Fixed kerneld for kmod.
35 * Michal Ostrowski : Module initialization cleanup.
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +090036 * Ulises Alonso : Frame number limit removal and
Linus Torvalds1da177e2005-04-16 15:20:36 -070037 * packet_set_ring memory leak.
Eric W. Biederman0fb375f2005-09-21 00:11:37 -070038 * Eric Biederman : Allow for > 8 byte hardware addresses.
39 * The convention is that longer addresses
40 * will simply extend the hardware address
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +090041 * byte arrays at the end of sockaddr_ll
Eric W. Biederman0fb375f2005-09-21 00:11:37 -070042 * and packet_mreq.
Johann Baudy69e3c752009-05-18 22:11:22 -070043 * Johann Baudy : Added TX RING.
chetan lokef6fb8f12011-08-19 10:18:16 +000044 * Chetan Loke : Implemented TPACKET_V3 block abstraction
45 * layer.
46 * Copyright (C) 2011, <lokec@ccs.neu.edu>
Linus Torvalds1da177e2005-04-16 15:20:36 -070047 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +090048
Baruch Siachdc41c4a2021-09-09 20:49:47 +030049#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
50
Jakub Kicinskicc698372020-11-20 14:50:52 -080051#include <linux/ethtool.h>
Jakub Kicinskib6459412021-12-28 16:49:13 -080052#include <linux/filter.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070053#include <linux/types.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070054#include <linux/mm.h>
Randy Dunlap4fc268d2006-01-11 12:17:47 -080055#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070056#include <linux/fcntl.h>
57#include <linux/socket.h>
58#include <linux/in.h>
59#include <linux/inet.h>
60#include <linux/netdevice.h>
61#include <linux/if_packet.h>
62#include <linux/wireless.h>
Herbert Xuffbc6112007-02-04 23:33:10 -080063#include <linux/kernel.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070064#include <linux/kmod.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090065#include <linux/slab.h>
Neil Horman0e3125c2010-11-16 10:26:47 -080066#include <linux/vmalloc.h>
Eric W. Biederman457c4cb2007-09-12 12:01:34 +020067#include <net/net_namespace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070068#include <net/ip.h>
69#include <net/protocol.h>
70#include <linux/skbuff.h>
71#include <net/sock.h>
72#include <linux/errno.h>
73#include <linux/timer.h>
Linus Torvalds7c0f6ba2016-12-24 11:46:01 -080074#include <linux/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070075#include <asm/ioctls.h>
76#include <asm/page.h>
Al Viroa1f8e7f72006-10-19 16:08:53 -040077#include <asm/cacheflush.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070078#include <asm/io.h>
79#include <linux/proc_fs.h>
80#include <linux/seq_file.h>
81#include <linux/poll.h>
82#include <linux/module.h>
83#include <linux/init.h>
Herbert Xu905db442009-01-30 14:12:06 -080084#include <linux/mutex.h>
Eric Dumazet05423b22009-10-26 18:40:35 -070085#include <linux/if_vlan.h>
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -080086#include <linux/virtio_net.h>
Richard Cochraned85b562010-04-07 22:41:28 +000087#include <linux/errqueue.h>
Scott McMillan614f60f2010-06-02 05:53:56 -070088#include <linux/net_tstamp.h>
Daniel Borkmannb0138402014-01-15 16:25:36 +010089#include <linux/percpu.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070090#ifdef CONFIG_INET
91#include <net/inet_common.h>
92#endif
Willem de Bruijn47dceb82015-08-14 22:31:34 -040093#include <linux/bpf.h>
Willem de Bruijn719c44d2016-06-07 12:06:34 -040094#include <net/compat.h>
Pablo Neira Ayuso0d7308c2021-10-08 22:06:04 +020095#include <linux/netfilter_netdev.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070096
Pavel Emelyanov2787b042012-08-13 05:49:39 +000097#include "internal.h"
98
Linus Torvalds1da177e2005-04-16 15:20:36 -070099/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100 Assumptions:
Eyal Birgerd5496992020-11-21 08:28:17 +0200101 - If the device has no dev->header_ops->create, there is no LL header
102 visible above the device. In this case, its hard_header_len should be 0.
Xie Heb4c58812020-09-14 00:41:54 -0700103 The device may prepend its own header internally. In this case, its
104 needed_headroom should be set to the space needed for it to add its
105 internal header.
106 For example, a WiFi driver pretending to be an Ethernet driver should
107 set its hard_header_len to be the Ethernet header length, and set its
108 needed_headroom to be (the real WiFi header length - the fake Ethernet
109 header length).
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110 - packet socket receives packets with pulled ll header,
111 so that SOCK_RAW should push it back.
112
113On receive:
114-----------
115
Eyal Birgerd5496992020-11-21 08:28:17 +0200116Incoming, dev_has_header(dev) == true
Arnaldo Carvalho de Melob0e380b12007-04-10 21:21:55 -0700117 mac_header -> ll header
118 data -> data
Linus Torvalds1da177e2005-04-16 15:20:36 -0700119
Eyal Birgerd5496992020-11-21 08:28:17 +0200120Outgoing, dev_has_header(dev) == true
Arnaldo Carvalho de Melob0e380b12007-04-10 21:21:55 -0700121 mac_header -> ll header
122 data -> ll header
Linus Torvalds1da177e2005-04-16 15:20:36 -0700123
Eyal Birgerd5496992020-11-21 08:28:17 +0200124Incoming, dev_has_header(dev) == false
Xie Heb79a80b2020-09-16 05:23:08 -0700125 mac_header -> data
126 However drivers often make it point to the ll header.
127 This is incorrect because the ll header should be invisible to us.
Arnaldo Carvalho de Melob0e380b12007-04-10 21:21:55 -0700128 data -> data
Linus Torvalds1da177e2005-04-16 15:20:36 -0700129
Eyal Birgerd5496992020-11-21 08:28:17 +0200130Outgoing, dev_has_header(dev) == false
Xie Heb79a80b2020-09-16 05:23:08 -0700131 mac_header -> data. ll header is invisible to us.
Arnaldo Carvalho de Melob0e380b12007-04-10 21:21:55 -0700132 data -> data
Linus Torvalds1da177e2005-04-16 15:20:36 -0700133
134Resume
Eyal Birgerd5496992020-11-21 08:28:17 +0200135 If dev_has_header(dev) == false we are unable to restore the ll header,
Xie Heb79a80b2020-09-16 05:23:08 -0700136 because it is invisible to us.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700137
138
139On transmit:
140------------
141
Xie He21c859742021-02-05 14:41:24 -0800142dev_has_header(dev) == true
Arnaldo Carvalho de Melob0e380b12007-04-10 21:21:55 -0700143 mac_header -> ll header
144 data -> ll header
Linus Torvalds1da177e2005-04-16 15:20:36 -0700145
Xie He21c859742021-02-05 14:41:24 -0800146dev_has_header(dev) == false (ll header is invisible to us)
Arnaldo Carvalho de Melob0e380b12007-04-10 21:21:55 -0700147 mac_header -> data
148 data -> data
Linus Torvalds1da177e2005-04-16 15:20:36 -0700149
Xie He09599722020-09-18 06:56:16 -0700150 We should set network_header on output to the correct position,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700151 packet classifier depends on it.
152 */
153
Linus Torvalds1da177e2005-04-16 15:20:36 -0700154/* Private packet socket structures. */
155
Eric W. Biederman0fb375f2005-09-21 00:11:37 -0700156/* identical to struct packet_mreq except it has
157 * a longer address field.
158 */
Eric Dumazet40d4e3d2009-07-21 21:57:59 +0000159struct packet_mreq_max {
Eric W. Biederman0fb375f2005-09-21 00:11:37 -0700160 int mr_ifindex;
161 unsigned short mr_type;
162 unsigned short mr_alen;
163 unsigned char mr_address[MAX_ADDR_LEN];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700164};
David S. Millera2efcfa2007-05-29 13:12:50 -0700165
Daniel Borkmann184f4892013-04-16 01:57:46 +0000166union tpacket_uhdr {
167 struct tpacket_hdr *h1;
168 struct tpacket2_hdr *h2;
169 struct tpacket3_hdr *h3;
170 void *raw;
171};
172
chetan lokef6fb8f12011-08-19 10:18:16 +0000173static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
Johann Baudy69e3c752009-05-18 22:11:22 -0700174 int closing, int tx_ring);
175
chetan lokef6fb8f12011-08-19 10:18:16 +0000176#define V3_ALIGNMENT (8)
177
chetan lokebc59ba32011-08-25 10:43:30 +0000178#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
chetan lokef6fb8f12011-08-19 10:18:16 +0000179
180#define BLK_PLUS_PRIV(sz_of_priv) \
181 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
182
chetan lokef6fb8f12011-08-19 10:18:16 +0000183#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
184#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
185#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
186#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
187#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
188#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
chetan lokef6fb8f12011-08-19 10:18:16 +0000189
Johann Baudy69e3c752009-05-18 22:11:22 -0700190struct packet_sock;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +0000191static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
192 struct packet_type *pt, struct net_device *orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700193
chetan lokef6fb8f12011-08-19 10:18:16 +0000194static void *packet_previous_frame(struct packet_sock *po,
195 struct packet_ring_buffer *rb,
196 int status);
197static void packet_increment_head(struct packet_ring_buffer *buff);
Rosen, Rami878cd3b2017-05-24 18:34:11 +0300198static int prb_curr_blk_in_use(struct tpacket_block_desc *);
chetan lokebc59ba32011-08-25 10:43:30 +0000199static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
chetan lokef6fb8f12011-08-19 10:18:16 +0000200 struct packet_sock *);
chetan lokebc59ba32011-08-25 10:43:30 +0000201static void prb_retire_current_block(struct tpacket_kbdq_core *,
chetan lokef6fb8f12011-08-19 10:18:16 +0000202 struct packet_sock *, unsigned int status);
chetan lokebc59ba32011-08-25 10:43:30 +0000203static int prb_queue_frozen(struct tpacket_kbdq_core *);
204static void prb_open_block(struct tpacket_kbdq_core *,
205 struct tpacket_block_desc *);
Kees Cook17bfd8c2017-10-24 01:46:26 -0700206static void prb_retire_rx_blk_timer_expired(struct timer_list *);
chetan lokebc59ba32011-08-25 10:43:30 +0000207static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
chetan lokebc59ba32011-08-25 10:43:30 +0000208static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
209static void prb_clear_rxhash(struct tpacket_kbdq_core *,
210 struct tpacket3_hdr *);
211static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
212 struct tpacket3_hdr *);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700213static void packet_flush_mclist(struct sock *sk);
Magnus Karlsson865b03f2018-05-02 13:01:33 +0200214static u16 packet_pick_tx_queue(struct sk_buff *skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700215
Herbert Xuffbc6112007-02-04 23:33:10 -0800216struct packet_skb_cb {
Herbert Xuffbc6112007-02-04 23:33:10 -0800217 union {
218 struct sockaddr_pkt pkt;
Eyal Birger2472d762015-03-01 14:58:28 +0200219 union {
220 /* Trick: alias skb original length with
221 * ll.sll_family and ll.protocol in order
222 * to save room.
223 */
224 unsigned int origlen;
225 struct sockaddr_ll ll;
226 };
Herbert Xuffbc6112007-02-04 23:33:10 -0800227 } sa;
228};
229
David Woodhoused3869ef2015-09-23 19:45:08 +0100230#define vio_le() virtio_legacy_is_little_endian()
231
Herbert Xuffbc6112007-02-04 23:33:10 -0800232#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
Herbert Xu8dc41942007-02-04 23:31:32 -0800233
chetan lokebc59ba32011-08-25 10:43:30 +0000234#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
chetan lokef6fb8f12011-08-19 10:18:16 +0000235#define GET_PBLOCK_DESC(x, bid) \
chetan lokebc59ba32011-08-25 10:43:30 +0000236 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
chetan lokef6fb8f12011-08-19 10:18:16 +0000237#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
chetan lokebc59ba32011-08-25 10:43:30 +0000238 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
chetan lokef6fb8f12011-08-19 10:18:16 +0000239#define GET_NEXT_PRB_BLK_NUM(x) \
240 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
241 ((x)->kactive_blk_num+1) : 0)
242
David S. Millerdc99f602011-07-05 01:45:05 -0700243static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
244static void __fanout_link(struct sock *sk, struct packet_sock *po);
245
Pablo Neira Ayuso0d7308c2021-10-08 22:06:04 +0200246#ifdef CONFIG_NETFILTER_EGRESS
247static noinline struct sk_buff *nf_hook_direct_egress(struct sk_buff *skb)
248{
249 struct sk_buff *next, *head = NULL, *tail;
250 int rc;
251
252 rcu_read_lock();
253 for (; skb != NULL; skb = next) {
254 next = skb->next;
255 skb_mark_not_on_list(skb);
256
257 if (!nf_hook_egress(skb, &rc, skb->dev))
258 continue;
259
260 if (!head)
261 head = skb;
262 else
263 tail->next = skb;
264
265 tail = skb;
266 }
267 rcu_read_unlock();
268
269 return head;
270}
271#endif
272
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100273static int packet_direct_xmit(struct sk_buff *skb)
274{
Pablo Neira Ayuso0d7308c2021-10-08 22:06:04 +0200275#ifdef CONFIG_NETFILTER_EGRESS
276 if (nf_hook_egress_active()) {
277 skb = nf_hook_direct_egress(skb);
278 if (!skb)
279 return NET_XMIT_DROP;
280 }
281#endif
Magnus Karlsson865b03f2018-05-02 13:01:33 +0200282 return dev_direct_xmit(skb, packet_pick_tx_queue(skb));
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100283}
284
Daniel Borkmann66e56cd2013-12-06 11:36:15 +0100285static struct net_device *packet_cached_dev_get(struct packet_sock *po)
286{
287 struct net_device *dev;
288
289 rcu_read_lock();
290 dev = rcu_dereference(po->cached_dev);
Yajun Deng1160dfa2021-08-05 19:55:27 +0800291 dev_hold(dev);
Daniel Borkmann66e56cd2013-12-06 11:36:15 +0100292 rcu_read_unlock();
293
294 return dev;
295}
296
297static void packet_cached_dev_assign(struct packet_sock *po,
298 struct net_device *dev)
299{
300 rcu_assign_pointer(po->cached_dev, dev);
301}
302
303static void packet_cached_dev_reset(struct packet_sock *po)
304{
305 RCU_INIT_POINTER(po->cached_dev, NULL);
306}
307
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100308static bool packet_use_direct_xmit(const struct packet_sock *po)
309{
310 return po->xmit == packet_direct_xmit;
311}
312
Magnus Karlsson865b03f2018-05-02 13:01:33 +0200313static u16 packet_pick_tx_queue(struct sk_buff *skb)
Daniel Borkmann0fd5d572014-02-16 15:55:22 +0100314{
Magnus Karlsson865b03f2018-05-02 13:01:33 +0200315 struct net_device *dev = skb->dev;
Daniel Borkmann0fd5d572014-02-16 15:55:22 +0100316 const struct net_device_ops *ops = dev->netdev_ops;
Paolo Abenib71b5832019-03-20 11:02:05 +0100317 int cpu = raw_smp_processor_id();
Daniel Borkmann0fd5d572014-02-16 15:55:22 +0100318 u16 queue_index;
319
Paolo Abenib71b5832019-03-20 11:02:05 +0100320#ifdef CONFIG_XPS
321 skb->sender_cpu = cpu + 1;
322#endif
323 skb_record_rx_queue(skb, cpu % dev->real_num_tx_queues);
Daniel Borkmann0fd5d572014-02-16 15:55:22 +0100324 if (ops->ndo_select_queue) {
Paolo Abenia350ecc2019-03-20 11:02:06 +0100325 queue_index = ops->ndo_select_queue(dev, skb, NULL);
Daniel Borkmann0fd5d572014-02-16 15:55:22 +0100326 queue_index = netdev_cap_txqueue(dev, queue_index);
327 } else {
Paolo Abenib71b5832019-03-20 11:02:05 +0100328 queue_index = netdev_pick_tx(dev, skb, NULL);
Daniel Borkmann0fd5d572014-02-16 15:55:22 +0100329 }
330
Magnus Karlsson865b03f2018-05-02 13:01:33 +0200331 return queue_index;
Daniel Borkmann0fd5d572014-02-16 15:55:22 +0100332}
333
Willem de Bruijna6361f02018-04-23 17:37:03 -0400334/* __register_prot_hook must be invoked through register_prot_hook
David S. Millerce06b032011-07-04 01:44:29 -0700335 * or from a context in which asynchronous accesses to the packet
336 * socket is not possible (packet_create()).
337 */
Willem de Bruijna6361f02018-04-23 17:37:03 -0400338static void __register_prot_hook(struct sock *sk)
David S. Millerce06b032011-07-04 01:44:29 -0700339{
340 struct packet_sock *po = pkt_sk(sk);
Daniel Borkmanne40526c2013-11-21 16:50:58 +0100341
David S. Millerce06b032011-07-04 01:44:29 -0700342 if (!po->running) {
Daniel Borkmann66e56cd2013-12-06 11:36:15 +0100343 if (po->fanout)
David S. Millerdc99f602011-07-05 01:45:05 -0700344 __fanout_link(sk, po);
Daniel Borkmann66e56cd2013-12-06 11:36:15 +0100345 else
David S. Millerdc99f602011-07-05 01:45:05 -0700346 dev_add_pack(&po->prot_hook);
Daniel Borkmanne40526c2013-11-21 16:50:58 +0100347
David S. Millerce06b032011-07-04 01:44:29 -0700348 sock_hold(sk);
349 po->running = 1;
350 }
351}
352
Willem de Bruijna6361f02018-04-23 17:37:03 -0400353static void register_prot_hook(struct sock *sk)
354{
355 lockdep_assert_held_once(&pkt_sk(sk)->bind_lock);
356 __register_prot_hook(sk);
357}
358
359/* If the sync parameter is true, we will temporarily drop
David S. Millerce06b032011-07-04 01:44:29 -0700360 * the po->bind_lock and do a synchronize_net to make sure no
361 * asynchronous packet processing paths still refer to the elements
362 * of po->prot_hook. If the sync parameter is false, it is the
363 * callers responsibility to take care of this.
364 */
365static void __unregister_prot_hook(struct sock *sk, bool sync)
366{
367 struct packet_sock *po = pkt_sk(sk);
368
Willem de Bruijna6361f02018-04-23 17:37:03 -0400369 lockdep_assert_held_once(&po->bind_lock);
370
David S. Millerce06b032011-07-04 01:44:29 -0700371 po->running = 0;
Daniel Borkmann66e56cd2013-12-06 11:36:15 +0100372
373 if (po->fanout)
David S. Millerdc99f602011-07-05 01:45:05 -0700374 __fanout_unlink(sk, po);
Daniel Borkmann66e56cd2013-12-06 11:36:15 +0100375 else
David S. Millerdc99f602011-07-05 01:45:05 -0700376 __dev_remove_pack(&po->prot_hook);
Daniel Borkmanne40526c2013-11-21 16:50:58 +0100377
David S. Millerce06b032011-07-04 01:44:29 -0700378 __sock_put(sk);
379
380 if (sync) {
381 spin_unlock(&po->bind_lock);
382 synchronize_net();
383 spin_lock(&po->bind_lock);
384 }
385}
386
387static void unregister_prot_hook(struct sock *sk, bool sync)
388{
389 struct packet_sock *po = pkt_sk(sk);
390
391 if (po->running)
392 __unregister_prot_hook(sk, sync);
393}
394
Michael S. Tsirkin6e580402014-11-24 13:32:16 +0200395static inline struct page * __pure pgv_to_page(void *addr)
Changli Gao0af55bb2010-12-01 02:52:20 +0000396{
397 if (is_vmalloc_addr(addr))
398 return vmalloc_to_page(addr);
399 return virt_to_page(addr);
400}
401
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700402static void __packet_set_status(struct packet_sock *po, void *frame, int status)
403{
Daniel Borkmann184f4892013-04-16 01:57:46 +0000404 union tpacket_uhdr h;
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700405
406 h.raw = frame;
407 switch (po->tp_version) {
408 case TPACKET_V1:
409 h.h1->tp_status = status;
Changli Gao0af55bb2010-12-01 02:52:20 +0000410 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700411 break;
412 case TPACKET_V2:
413 h.h2->tp_status = status;
Changli Gao0af55bb2010-12-01 02:52:20 +0000414 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700415 break;
chetan lokef6fb8f12011-08-19 10:18:16 +0000416 case TPACKET_V3:
Sowmini Varadhan7f953ab2017-01-03 06:31:47 -0800417 h.h3->tp_status = status;
418 flush_dcache_page(pgv_to_page(&h.h3->tp_status));
419 break;
Johann Baudy69e3c752009-05-18 22:11:22 -0700420 default:
chetan lokef6fb8f12011-08-19 10:18:16 +0000421 WARN(1, "TPACKET version not supported.\n");
Johann Baudy69e3c752009-05-18 22:11:22 -0700422 BUG();
423 }
424
425 smp_wmb();
426}
427
Eric Dumazet96f657e2019-06-12 09:52:26 -0700428static int __packet_get_status(const struct packet_sock *po, void *frame)
Johann Baudy69e3c752009-05-18 22:11:22 -0700429{
Daniel Borkmann184f4892013-04-16 01:57:46 +0000430 union tpacket_uhdr h;
Johann Baudy69e3c752009-05-18 22:11:22 -0700431
432 smp_rmb();
433
434 h.raw = frame;
435 switch (po->tp_version) {
436 case TPACKET_V1:
Changli Gao0af55bb2010-12-01 02:52:20 +0000437 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
Johann Baudy69e3c752009-05-18 22:11:22 -0700438 return h.h1->tp_status;
439 case TPACKET_V2:
Changli Gao0af55bb2010-12-01 02:52:20 +0000440 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
Johann Baudy69e3c752009-05-18 22:11:22 -0700441 return h.h2->tp_status;
chetan lokef6fb8f12011-08-19 10:18:16 +0000442 case TPACKET_V3:
Sowmini Varadhan7f953ab2017-01-03 06:31:47 -0800443 flush_dcache_page(pgv_to_page(&h.h3->tp_status));
444 return h.h3->tp_status;
Johann Baudy69e3c752009-05-18 22:11:22 -0700445 default:
chetan lokef6fb8f12011-08-19 10:18:16 +0000446 WARN(1, "TPACKET version not supported.\n");
Johann Baudy69e3c752009-05-18 22:11:22 -0700447 BUG();
448 return 0;
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700449 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700450}
Johann Baudy69e3c752009-05-18 22:11:22 -0700451
Arnd Bergmannd413fcb2017-11-27 10:09:24 +0100452static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec64 *ts,
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000453 unsigned int flags)
Daniel Borkmann7a513842013-04-23 00:39:29 +0000454{
455 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
456
Willem de Bruijn68a360e2014-07-25 18:01:31 -0400457 if (shhwtstamps &&
458 (flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
Arnd Bergmannd413fcb2017-11-27 10:09:24 +0100459 ktime_to_timespec64_cond(shhwtstamps->hwtstamp, ts))
Willem de Bruijn68a360e2014-07-25 18:01:31 -0400460 return TP_STATUS_TS_RAW_HARDWARE;
Daniel Borkmann7a513842013-04-23 00:39:29 +0000461
Richard Sanger171c3b12021-05-12 13:31:22 +1200462 if ((flags & SOF_TIMESTAMPING_SOFTWARE) &&
Martin KaFai Lau27942a12022-03-02 11:55:38 -0800463 ktime_to_timespec64_cond(skb_tstamp(skb), ts))
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000464 return TP_STATUS_TS_SOFTWARE;
Daniel Borkmann7a513842013-04-23 00:39:29 +0000465
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000466 return 0;
Daniel Borkmann7a513842013-04-23 00:39:29 +0000467}
468
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000469static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
470 struct sk_buff *skb)
Willem de Bruijn2e313962013-04-23 00:39:28 +0000471{
472 union tpacket_uhdr h;
Arnd Bergmannd413fcb2017-11-27 10:09:24 +0100473 struct timespec64 ts;
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000474 __u32 ts_status;
Willem de Bruijn2e313962013-04-23 00:39:28 +0000475
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000476 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
477 return 0;
Willem de Bruijn2e313962013-04-23 00:39:28 +0000478
479 h.raw = frame;
Arnd Bergmannd413fcb2017-11-27 10:09:24 +0100480 /*
481 * versions 1 through 3 overflow the timestamps in y2106, since they
482 * all store the seconds in a 32-bit unsigned integer.
483 * If we create a version 4, that should have a 64-bit timestamp,
484 * either 64-bit seconds + 32-bit nanoseconds, or just 64-bit
485 * nanoseconds.
486 */
Willem de Bruijn2e313962013-04-23 00:39:28 +0000487 switch (po->tp_version) {
488 case TPACKET_V1:
489 h.h1->tp_sec = ts.tv_sec;
490 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
491 break;
492 case TPACKET_V2:
493 h.h2->tp_sec = ts.tv_sec;
494 h.h2->tp_nsec = ts.tv_nsec;
495 break;
496 case TPACKET_V3:
Daniel Borkmann57ea8842017-01-05 02:34:28 +0100497 h.h3->tp_sec = ts.tv_sec;
498 h.h3->tp_nsec = ts.tv_nsec;
499 break;
Willem de Bruijn2e313962013-04-23 00:39:28 +0000500 default:
501 WARN(1, "TPACKET version not supported.\n");
502 BUG();
503 }
504
505 /* one flush is safe, as both fields always lie on the same cacheline */
506 flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
507 smp_wmb();
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000508
509 return ts_status;
Willem de Bruijn2e313962013-04-23 00:39:28 +0000510}
511
Eric Dumazetd4b5bd92019-06-12 09:52:27 -0700512static void *packet_lookup_frame(const struct packet_sock *po,
513 const struct packet_ring_buffer *rb,
514 unsigned int position,
515 int status)
Johann Baudy69e3c752009-05-18 22:11:22 -0700516{
517 unsigned int pg_vec_pos, frame_offset;
Daniel Borkmann184f4892013-04-16 01:57:46 +0000518 union tpacket_uhdr h;
Johann Baudy69e3c752009-05-18 22:11:22 -0700519
520 pg_vec_pos = position / rb->frames_per_block;
521 frame_offset = position % rb->frames_per_block;
522
Neil Horman0e3125c2010-11-16 10:26:47 -0800523 h.raw = rb->pg_vec[pg_vec_pos].buffer +
524 (frame_offset * rb->frame_size);
Johann Baudy69e3c752009-05-18 22:11:22 -0700525
526 if (status != __packet_get_status(po, h.raw))
527 return NULL;
528
529 return h.raw;
530}
531
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000532static void *packet_current_frame(struct packet_sock *po,
Johann Baudy69e3c752009-05-18 22:11:22 -0700533 struct packet_ring_buffer *rb,
534 int status)
535{
536 return packet_lookup_frame(po, rb, rb->head, status);
537}
538
chetan lokebc59ba32011-08-25 10:43:30 +0000539static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
chetan lokef6fb8f12011-08-19 10:18:16 +0000540{
541 del_timer_sync(&pkc->retire_blk_timer);
542}
543
544static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
chetan lokef6fb8f12011-08-19 10:18:16 +0000545 struct sk_buff_head *rb_queue)
546{
chetan lokebc59ba32011-08-25 10:43:30 +0000547 struct tpacket_kbdq_core *pkc;
chetan lokef6fb8f12011-08-19 10:18:16 +0000548
Tobias Klauser73d0fcf2015-07-28 14:21:26 +0200549 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
chetan lokef6fb8f12011-08-19 10:18:16 +0000550
Veaceslav Falicoec6f8092013-11-29 09:53:23 +0100551 spin_lock_bh(&rb_queue->lock);
chetan lokef6fb8f12011-08-19 10:18:16 +0000552 pkc->delete_blk_timer = 1;
Veaceslav Falicoec6f8092013-11-29 09:53:23 +0100553 spin_unlock_bh(&rb_queue->lock);
chetan lokef6fb8f12011-08-19 10:18:16 +0000554
555 prb_del_retire_blk_timer(pkc);
556}
557
Maninder Singhe8e85cc2015-06-22 12:39:16 +0530558static void prb_setup_retire_blk_timer(struct packet_sock *po)
chetan lokef6fb8f12011-08-19 10:18:16 +0000559{
chetan lokebc59ba32011-08-25 10:43:30 +0000560 struct tpacket_kbdq_core *pkc;
chetan lokef6fb8f12011-08-19 10:18:16 +0000561
Maninder Singhe8e85cc2015-06-22 12:39:16 +0530562 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
Kees Cook17bfd8c2017-10-24 01:46:26 -0700563 timer_setup(&pkc->retire_blk_timer, prb_retire_rx_blk_timer_expired,
564 0);
565 pkc->retire_blk_timer.expires = jiffies;
chetan lokef6fb8f12011-08-19 10:18:16 +0000566}
567
568static int prb_calc_retire_blk_tmo(struct packet_sock *po,
569 int blk_size_in_bytes)
570{
571 struct net_device *dev;
Mao Wenan0914d2b2019-12-23 18:42:57 +0800572 unsigned int mbits, div;
David Decotigny7cad1ba2016-02-24 10:58:10 -0800573 struct ethtool_link_ksettings ecmd;
Jiri Pirko4bc71cb2011-09-03 03:34:30 +0000574 int err;
chetan lokef6fb8f12011-08-19 10:18:16 +0000575
Jiri Pirko4bc71cb2011-09-03 03:34:30 +0000576 rtnl_lock();
577 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
578 if (unlikely(!dev)) {
579 rtnl_unlock();
chetan lokef6fb8f12011-08-19 10:18:16 +0000580 return DEFAULT_PRB_RETIRE_TOV;
Jiri Pirko4bc71cb2011-09-03 03:34:30 +0000581 }
David Decotigny7cad1ba2016-02-24 10:58:10 -0800582 err = __ethtool_get_link_ksettings(dev, &ecmd);
Jiri Pirko4bc71cb2011-09-03 03:34:30 +0000583 rtnl_unlock();
Mao Wenan0914d2b2019-12-23 18:42:57 +0800584 if (err)
Mao Wenanb43d1f92019-12-09 21:31:25 +0800585 return DEFAULT_PRB_RETIRE_TOV;
chetan lokef6fb8f12011-08-19 10:18:16 +0000586
Mao Wenan0914d2b2019-12-23 18:42:57 +0800587 /* If the link speed is so slow you don't really
588 * need to worry about perf anyways
589 */
590 if (ecmd.base.speed < SPEED_1000 ||
591 ecmd.base.speed == SPEED_UNKNOWN)
592 return DEFAULT_PRB_RETIRE_TOV;
593
594 div = ecmd.base.speed / 1000;
chetan lokef6fb8f12011-08-19 10:18:16 +0000595 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
596
597 if (div)
598 mbits /= div;
599
chetan lokef6fb8f12011-08-19 10:18:16 +0000600 if (div)
Mao Wenan0914d2b2019-12-23 18:42:57 +0800601 return mbits + 1;
602 return mbits;
chetan lokef6fb8f12011-08-19 10:18:16 +0000603}
604
chetan lokebc59ba32011-08-25 10:43:30 +0000605static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
chetan lokef6fb8f12011-08-19 10:18:16 +0000606 union tpacket_req_u *req_u)
607{
608 p1->feature_req_word = req_u->req3.tp_feature_req_word;
609}
610
611static void init_prb_bdqc(struct packet_sock *po,
612 struct packet_ring_buffer *rb,
613 struct pgv *pg_vec,
Maninder Singhe8e85cc2015-06-22 12:39:16 +0530614 union tpacket_req_u *req_u)
chetan lokef6fb8f12011-08-19 10:18:16 +0000615{
Duan Jiong22781a52013-12-06 13:29:36 +0800616 struct tpacket_kbdq_core *p1 = GET_PBDQC_FROM_RB(rb);
chetan lokebc59ba32011-08-25 10:43:30 +0000617 struct tpacket_block_desc *pbd;
chetan lokef6fb8f12011-08-19 10:18:16 +0000618
619 memset(p1, 0x0, sizeof(*p1));
620
621 p1->knxt_seq_num = 1;
622 p1->pkbdq = pg_vec;
chetan lokebc59ba32011-08-25 10:43:30 +0000623 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
Joe Perchese3192692012-06-03 17:41:40 +0000624 p1->pkblk_start = pg_vec[0].buffer;
chetan lokef6fb8f12011-08-19 10:18:16 +0000625 p1->kblk_size = req_u->req3.tp_block_size;
626 p1->knum_blocks = req_u->req3.tp_block_nr;
627 p1->hdrlen = po->tp_hdrlen;
628 p1->version = po->tp_version;
629 p1->last_kactive_blk_num = 0;
Daniel Borkmannee80fbf2013-04-19 06:12:29 +0000630 po->stats.stats3.tp_freeze_q_cnt = 0;
chetan lokef6fb8f12011-08-19 10:18:16 +0000631 if (req_u->req3.tp_retire_blk_tov)
632 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
633 else
634 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
635 req_u->req3.tp_block_size);
636 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
637 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
John Ogness632ca502020-07-07 17:28:04 +0206638 rwlock_init(&p1->blk_fill_in_prog_lock);
chetan lokef6fb8f12011-08-19 10:18:16 +0000639
Eric Dumazetdc808112014-08-15 09:16:04 -0700640 p1->max_frame_len = p1->kblk_size - BLK_PLUS_PRIV(p1->blk_sizeof_priv);
chetan lokef6fb8f12011-08-19 10:18:16 +0000641 prb_init_ft_ops(p1, req_u);
Maninder Singhe8e85cc2015-06-22 12:39:16 +0530642 prb_setup_retire_blk_timer(po);
chetan lokef6fb8f12011-08-19 10:18:16 +0000643 prb_open_block(p1, pbd);
644}
645
646/* Do NOT update the last_blk_num first.
647 * Assumes sk_buff_head lock is held.
648 */
chetan lokebc59ba32011-08-25 10:43:30 +0000649static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
chetan lokef6fb8f12011-08-19 10:18:16 +0000650{
651 mod_timer(&pkc->retire_blk_timer,
652 jiffies + pkc->tov_in_jiffies);
653 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
654}
655
656/*
657 * Timer logic:
658 * 1) We refresh the timer only when we open a block.
659 * By doing this we don't waste cycles refreshing the timer
660 * on packet-by-packet basis.
661 *
662 * With a 1MB block-size, on a 1Gbps line, it will take
663 * i) ~8 ms to fill a block + ii) memcpy etc.
664 * In this cut we are not accounting for the memcpy time.
665 *
666 * So, if the user sets the 'tmo' to 10ms then the timer
667 * will never fire while the block is still getting filled
668 * (which is what we want). However, the user could choose
669 * to close a block early and that's fine.
670 *
671 * But when the timer does fire, we check whether or not to refresh it.
672 * Since the tmo granularity is in msecs, it is not too expensive
673 * to refresh the timer, lets say every '8' msecs.
674 * Either the user can set the 'tmo' or we can derive it based on
675 * a) line-speed and b) block-size.
676 * prb_calc_retire_blk_tmo() calculates the tmo.
677 *
678 */
Kees Cook17bfd8c2017-10-24 01:46:26 -0700679static void prb_retire_rx_blk_timer_expired(struct timer_list *t)
chetan lokef6fb8f12011-08-19 10:18:16 +0000680{
Kees Cook17bfd8c2017-10-24 01:46:26 -0700681 struct packet_sock *po =
682 from_timer(po, t, rx_ring.prb_bdqc.retire_blk_timer);
Duan Jiong22781a52013-12-06 13:29:36 +0800683 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
chetan lokef6fb8f12011-08-19 10:18:16 +0000684 unsigned int frozen;
chetan lokebc59ba32011-08-25 10:43:30 +0000685 struct tpacket_block_desc *pbd;
chetan lokef6fb8f12011-08-19 10:18:16 +0000686
687 spin_lock(&po->sk.sk_receive_queue.lock);
688
689 frozen = prb_queue_frozen(pkc);
690 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
691
692 if (unlikely(pkc->delete_blk_timer))
693 goto out;
694
695 /* We only need to plug the race when the block is partially filled.
696 * tpacket_rcv:
697 * lock(); increment BLOCK_NUM_PKTS; unlock()
698 * copy_bits() is in progress ...
699 * timer fires on other cpu:
700 * we can't retire the current block because copy_bits
701 * is in progress.
702 *
703 */
704 if (BLOCK_NUM_PKTS(pbd)) {
John Ogness632ca502020-07-07 17:28:04 +0206705 /* Waiting for skb_copy_bits to finish... */
706 write_lock(&pkc->blk_fill_in_prog_lock);
707 write_unlock(&pkc->blk_fill_in_prog_lock);
chetan lokef6fb8f12011-08-19 10:18:16 +0000708 }
709
710 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
711 if (!frozen) {
Alexander Drozdov41a50d62015-02-24 08:18:28 +0300712 if (!BLOCK_NUM_PKTS(pbd)) {
713 /* An empty block. Just refresh the timer. */
714 goto refresh_timer;
715 }
chetan lokef6fb8f12011-08-19 10:18:16 +0000716 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
717 if (!prb_dispatch_next_block(pkc, po))
718 goto refresh_timer;
719 else
720 goto out;
721 } else {
722 /* Case 1. Queue was frozen because user-space was
723 * lagging behind.
724 */
Rosen, Rami878cd3b2017-05-24 18:34:11 +0300725 if (prb_curr_blk_in_use(pbd)) {
chetan lokef6fb8f12011-08-19 10:18:16 +0000726 /*
727 * Ok, user-space is still behind.
728 * So just refresh the timer.
729 */
730 goto refresh_timer;
731 } else {
732 /* Case 2. queue was frozen,user-space caught up,
733 * now the link went idle && the timer fired.
734 * We don't have a block to close.So we open this
735 * block and restart the timer.
736 * opening a block thaws the queue,restarts timer
737 * Thawing/timer-refresh is a side effect.
738 */
739 prb_open_block(pkc, pbd);
740 goto out;
741 }
742 }
743 }
744
745refresh_timer:
746 _prb_refresh_rx_retire_blk_timer(pkc);
747
748out:
749 spin_unlock(&po->sk.sk_receive_queue.lock);
750}
751
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000752static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
chetan lokebc59ba32011-08-25 10:43:30 +0000753 struct tpacket_block_desc *pbd1, __u32 status)
chetan lokef6fb8f12011-08-19 10:18:16 +0000754{
755 /* Flush everything minus the block header */
756
757#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
758 u8 *start, *end;
759
760 start = (u8 *)pbd1;
761
762 /* Skip the block header(we know header WILL fit in 4K) */
763 start += PAGE_SIZE;
764
765 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
766 for (; start < end; start += PAGE_SIZE)
767 flush_dcache_page(pgv_to_page(start));
768
769 smp_wmb();
770#endif
771
772 /* Now update the block status. */
773
774 BLOCK_STATUS(pbd1) = status;
775
776 /* Flush the block header */
777
778#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
779 start = (u8 *)pbd1;
780 flush_dcache_page(pgv_to_page(start));
781
782 smp_wmb();
783#endif
784}
785
786/*
787 * Side effect:
788 *
789 * 1) flush the block
790 * 2) Increment active_blk_num
791 *
792 * Note:We DONT refresh the timer on purpose.
793 * Because almost always the next block will be opened.
794 */
chetan lokebc59ba32011-08-25 10:43:30 +0000795static void prb_close_block(struct tpacket_kbdq_core *pkc1,
796 struct tpacket_block_desc *pbd1,
chetan lokef6fb8f12011-08-19 10:18:16 +0000797 struct packet_sock *po, unsigned int stat)
798{
799 __u32 status = TP_STATUS_USER | stat;
800
801 struct tpacket3_hdr *last_pkt;
chetan lokebc59ba32011-08-25 10:43:30 +0000802 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
Dan Collinsda413ee2014-12-19 16:49:25 +1300803 struct sock *sk = &po->sk;
chetan lokef6fb8f12011-08-19 10:18:16 +0000804
Eric Dumazet8e8e2952019-06-12 09:52:30 -0700805 if (atomic_read(&po->tp_drops))
chetan lokef6fb8f12011-08-19 10:18:16 +0000806 status |= TP_STATUS_LOSING;
807
808 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
809 last_pkt->tp_next_offset = 0;
810
811 /* Get the ts of the last pkt */
812 if (BLOCK_NUM_PKTS(pbd1)) {
813 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
814 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
815 } else {
Alexander Drozdov41a50d62015-02-24 08:18:28 +0300816 /* Ok, we tmo'd - so get the current time.
817 *
818 * It shouldn't really happen as we don't close empty
819 * blocks. See prb_retire_rx_blk_timer_expired().
820 */
Arnd Bergmannd413fcb2017-11-27 10:09:24 +0100821 struct timespec64 ts;
822 ktime_get_real_ts64(&ts);
chetan lokef6fb8f12011-08-19 10:18:16 +0000823 h1->ts_last_pkt.ts_sec = ts.tv_sec;
824 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
825 }
826
827 smp_wmb();
828
829 /* Flush the block */
830 prb_flush_block(pkc1, pbd1, status);
831
Dan Collinsda413ee2014-12-19 16:49:25 +1300832 sk->sk_data_ready(sk);
833
chetan lokef6fb8f12011-08-19 10:18:16 +0000834 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
835}
836
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000837static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
chetan lokef6fb8f12011-08-19 10:18:16 +0000838{
839 pkc->reset_pending_on_curr_blk = 0;
840}
841
842/*
843 * Side effect of opening a block:
844 *
845 * 1) prb_queue is thawed.
846 * 2) retire_blk_timer is refreshed.
847 *
848 */
chetan lokebc59ba32011-08-25 10:43:30 +0000849static void prb_open_block(struct tpacket_kbdq_core *pkc1,
850 struct tpacket_block_desc *pbd1)
chetan lokef6fb8f12011-08-19 10:18:16 +0000851{
Arnd Bergmannd413fcb2017-11-27 10:09:24 +0100852 struct timespec64 ts;
chetan lokebc59ba32011-08-25 10:43:30 +0000853 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
chetan lokef6fb8f12011-08-19 10:18:16 +0000854
855 smp_rmb();
856
Daniel Borkmann8da30562013-05-03 02:57:00 +0000857 /* We could have just memset this but we will lose the
858 * flexibility of making the priv area sticky
859 */
chetan lokef6fb8f12011-08-19 10:18:16 +0000860
Daniel Borkmann8da30562013-05-03 02:57:00 +0000861 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
862 BLOCK_NUM_PKTS(pbd1) = 0;
863 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
chetan lokef6fb8f12011-08-19 10:18:16 +0000864
Arnd Bergmannd413fcb2017-11-27 10:09:24 +0100865 ktime_get_real_ts64(&ts);
chetan lokef6fb8f12011-08-19 10:18:16 +0000866
Daniel Borkmann8da30562013-05-03 02:57:00 +0000867 h1->ts_first_pkt.ts_sec = ts.tv_sec;
868 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
chetan lokef6fb8f12011-08-19 10:18:16 +0000869
Daniel Borkmann8da30562013-05-03 02:57:00 +0000870 pkc1->pkblk_start = (char *)pbd1;
871 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
872
873 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
874 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
875
876 pbd1->version = pkc1->version;
877 pkc1->prev = pkc1->nxt_offset;
878 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
879
880 prb_thaw_queue(pkc1);
881 _prb_refresh_rx_retire_blk_timer(pkc1);
882
883 smp_wmb();
chetan lokef6fb8f12011-08-19 10:18:16 +0000884}
885
886/*
887 * Queue freeze logic:
888 * 1) Assume tp_block_nr = 8 blocks.
889 * 2) At time 't0', user opens Rx ring.
890 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
891 * 4) user-space is either sleeping or processing block '0'.
892 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
893 * it will close block-7,loop around and try to fill block '0'.
894 * call-flow:
895 * __packet_lookup_frame_in_block
896 * prb_retire_current_block()
897 * prb_dispatch_next_block()
898 * |->(BLOCK_STATUS == USER) evaluates to true
899 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
900 * 6) Now there are two cases:
901 * 6.1) Link goes idle right after the queue is frozen.
902 * But remember, the last open_block() refreshed the timer.
903 * When this timer expires,it will refresh itself so that we can
904 * re-open block-0 in near future.
905 * 6.2) Link is busy and keeps on receiving packets. This is a simple
906 * case and __packet_lookup_frame_in_block will check if block-0
907 * is free and can now be re-used.
908 */
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000909static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f12011-08-19 10:18:16 +0000910 struct packet_sock *po)
911{
912 pkc->reset_pending_on_curr_blk = 1;
Daniel Borkmannee80fbf2013-04-19 06:12:29 +0000913 po->stats.stats3.tp_freeze_q_cnt++;
chetan lokef6fb8f12011-08-19 10:18:16 +0000914}
915
916#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
917
918/*
919 * If the next block is free then we will dispatch it
920 * and return a good offset.
921 * Else, we will freeze the queue.
922 * So, caller must check the return value.
923 */
chetan lokebc59ba32011-08-25 10:43:30 +0000924static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f12011-08-19 10:18:16 +0000925 struct packet_sock *po)
926{
chetan lokebc59ba32011-08-25 10:43:30 +0000927 struct tpacket_block_desc *pbd;
chetan lokef6fb8f12011-08-19 10:18:16 +0000928
929 smp_rmb();
930
931 /* 1. Get current block num */
932 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
933
934 /* 2. If this block is currently in_use then freeze the queue */
935 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
936 prb_freeze_queue(pkc, po);
937 return NULL;
938 }
939
940 /*
941 * 3.
942 * open this block and return the offset where the first packet
943 * needs to get stored.
944 */
945 prb_open_block(pkc, pbd);
946 return (void *)pkc->nxt_offset;
947}
948
chetan lokebc59ba32011-08-25 10:43:30 +0000949static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f12011-08-19 10:18:16 +0000950 struct packet_sock *po, unsigned int status)
951{
chetan lokebc59ba32011-08-25 10:43:30 +0000952 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
chetan lokef6fb8f12011-08-19 10:18:16 +0000953
954 /* retire/close the current block */
955 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
956 /*
957 * Plug the case where copy_bits() is in progress on
958 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
959 * have space to copy the pkt in the current block and
960 * called prb_retire_current_block()
961 *
962 * We don't need to worry about the TMO case because
963 * the timer-handler already handled this case.
964 */
965 if (!(status & TP_STATUS_BLK_TMO)) {
John Ogness632ca502020-07-07 17:28:04 +0206966 /* Waiting for skb_copy_bits to finish... */
967 write_lock(&pkc->blk_fill_in_prog_lock);
968 write_unlock(&pkc->blk_fill_in_prog_lock);
chetan lokef6fb8f12011-08-19 10:18:16 +0000969 }
970 prb_close_block(pkc, pbd, po, status);
971 return;
972 }
chetan lokef6fb8f12011-08-19 10:18:16 +0000973}
974
Rosen, Rami878cd3b2017-05-24 18:34:11 +0300975static int prb_curr_blk_in_use(struct tpacket_block_desc *pbd)
chetan lokef6fb8f12011-08-19 10:18:16 +0000976{
977 return TP_STATUS_USER & BLOCK_STATUS(pbd);
978}
979
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000980static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
chetan lokef6fb8f12011-08-19 10:18:16 +0000981{
982 return pkc->reset_pending_on_curr_blk;
983}
984
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000985static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
John Ogness88fd1cb2020-08-13 21:45:25 +0206986 __releases(&pkc->blk_fill_in_prog_lock)
chetan lokef6fb8f12011-08-19 10:18:16 +0000987{
chetan lokebc59ba32011-08-25 10:43:30 +0000988 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
John Ogness632ca502020-07-07 17:28:04 +0206989
990 read_unlock(&pkc->blk_fill_in_prog_lock);
chetan lokef6fb8f12011-08-19 10:18:16 +0000991}
992
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000993static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f12011-08-19 10:18:16 +0000994 struct tpacket3_hdr *ppd)
995{
Tom Herbert3958afa1b2013-12-15 22:12:06 -0800996 ppd->hv1.tp_rxhash = skb_get_hash(pkc->skb);
chetan lokef6fb8f12011-08-19 10:18:16 +0000997}
998
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000999static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f12011-08-19 10:18:16 +00001000 struct tpacket3_hdr *ppd)
1001{
1002 ppd->hv1.tp_rxhash = 0;
1003}
1004
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001005static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f12011-08-19 10:18:16 +00001006 struct tpacket3_hdr *ppd)
1007{
Jiri Pirkodf8a39de2015-01-13 17:13:44 +01001008 if (skb_vlan_tag_present(pkc->skb)) {
1009 ppd->hv1.tp_vlan_tci = skb_vlan_tag_get(pkc->skb);
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +09001010 ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->vlan_proto);
1011 ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
chetan lokef6fb8f12011-08-19 10:18:16 +00001012 } else {
danborkmann@iogearbox.net9e670302012-08-20 03:34:03 +00001013 ppd->hv1.tp_vlan_tci = 0;
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +09001014 ppd->hv1.tp_vlan_tpid = 0;
danborkmann@iogearbox.net9e670302012-08-20 03:34:03 +00001015 ppd->tp_status = TP_STATUS_AVAILABLE;
chetan lokef6fb8f12011-08-19 10:18:16 +00001016 }
1017}
1018
chetan lokebc59ba32011-08-25 10:43:30 +00001019static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f12011-08-19 10:18:16 +00001020 struct tpacket3_hdr *ppd)
1021{
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +09001022 ppd->hv1.tp_padding = 0;
chetan lokef6fb8f12011-08-19 10:18:16 +00001023 prb_fill_vlan_info(pkc, ppd);
1024
1025 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
1026 prb_fill_rxhash(pkc, ppd);
1027 else
1028 prb_clear_rxhash(pkc, ppd);
1029}
1030
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001031static void prb_fill_curr_block(char *curr,
chetan lokebc59ba32011-08-25 10:43:30 +00001032 struct tpacket_kbdq_core *pkc,
1033 struct tpacket_block_desc *pbd,
chetan lokef6fb8f12011-08-19 10:18:16 +00001034 unsigned int len)
John Ogness88fd1cb2020-08-13 21:45:25 +02061035 __acquires(&pkc->blk_fill_in_prog_lock)
chetan lokef6fb8f12011-08-19 10:18:16 +00001036{
1037 struct tpacket3_hdr *ppd;
1038
1039 ppd = (struct tpacket3_hdr *)curr;
1040 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
1041 pkc->prev = curr;
1042 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
1043 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
1044 BLOCK_NUM_PKTS(pbd) += 1;
John Ogness632ca502020-07-07 17:28:04 +02061045 read_lock(&pkc->blk_fill_in_prog_lock);
chetan lokef6fb8f12011-08-19 10:18:16 +00001046 prb_run_all_ft_ops(pkc, ppd);
1047}
1048
1049/* Assumes caller has the sk->rx_queue.lock */
1050static void *__packet_lookup_frame_in_block(struct packet_sock *po,
1051 struct sk_buff *skb,
chetan lokef6fb8f12011-08-19 10:18:16 +00001052 unsigned int len
1053 )
1054{
chetan lokebc59ba32011-08-25 10:43:30 +00001055 struct tpacket_kbdq_core *pkc;
1056 struct tpacket_block_desc *pbd;
chetan lokef6fb8f12011-08-19 10:18:16 +00001057 char *curr, *end;
1058
Joe Perchese3192692012-06-03 17:41:40 +00001059 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
chetan lokef6fb8f12011-08-19 10:18:16 +00001060 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1061
1062 /* Queue is frozen when user space is lagging behind */
1063 if (prb_queue_frozen(pkc)) {
1064 /*
1065 * Check if that last block which caused the queue to freeze,
1066 * is still in_use by user-space.
1067 */
Rosen, Rami878cd3b2017-05-24 18:34:11 +03001068 if (prb_curr_blk_in_use(pbd)) {
chetan lokef6fb8f12011-08-19 10:18:16 +00001069 /* Can't record this packet */
1070 return NULL;
1071 } else {
1072 /*
1073 * Ok, the block was released by user-space.
1074 * Now let's open that block.
1075 * opening a block also thaws the queue.
1076 * Thawing is a side effect.
1077 */
1078 prb_open_block(pkc, pbd);
1079 }
1080 }
1081
1082 smp_mb();
1083 curr = pkc->nxt_offset;
1084 pkc->skb = skb;
Joe Perchese3192692012-06-03 17:41:40 +00001085 end = (char *)pbd + pkc->kblk_size;
chetan lokef6fb8f12011-08-19 10:18:16 +00001086
1087 /* first try the current block */
1088 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1089 prb_fill_curr_block(curr, pkc, pbd, len);
1090 return (void *)curr;
1091 }
1092
1093 /* Ok, close the current block */
1094 prb_retire_current_block(pkc, po, 0);
1095
1096 /* Now, try to dispatch the next block */
1097 curr = (char *)prb_dispatch_next_block(pkc, po);
1098 if (curr) {
1099 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1100 prb_fill_curr_block(curr, pkc, pbd, len);
1101 return (void *)curr;
1102 }
1103
1104 /*
1105 * No free blocks are available.user_space hasn't caught up yet.
1106 * Queue was just frozen and now this packet will get dropped.
1107 */
1108 return NULL;
1109}
1110
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001111static void *packet_current_rx_frame(struct packet_sock *po,
chetan lokef6fb8f12011-08-19 10:18:16 +00001112 struct sk_buff *skb,
1113 int status, unsigned int len)
1114{
1115 char *curr = NULL;
1116 switch (po->tp_version) {
1117 case TPACKET_V1:
1118 case TPACKET_V2:
1119 curr = packet_lookup_frame(po, &po->rx_ring,
1120 po->rx_ring.head, status);
1121 return curr;
1122 case TPACKET_V3:
Mao Wenan46088052019-06-11 09:32:13 +08001123 return __packet_lookup_frame_in_block(po, skb, len);
chetan lokef6fb8f12011-08-19 10:18:16 +00001124 default:
1125 WARN(1, "TPACKET version not supported\n");
1126 BUG();
Ying Xue99aa3472012-08-06 16:27:10 +00001127 return NULL;
chetan lokef6fb8f12011-08-19 10:18:16 +00001128 }
1129}
1130
Eric Dumazetdcf70ce2019-06-12 09:52:28 -07001131static void *prb_lookup_block(const struct packet_sock *po,
1132 const struct packet_ring_buffer *rb,
1133 unsigned int idx,
1134 int status)
chetan lokef6fb8f12011-08-19 10:18:16 +00001135{
chetan lokebc59ba32011-08-25 10:43:30 +00001136 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001137 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
chetan lokef6fb8f12011-08-19 10:18:16 +00001138
1139 if (status != BLOCK_STATUS(pbd))
1140 return NULL;
1141 return pbd;
1142}
1143
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001144static int prb_previous_blk_num(struct packet_ring_buffer *rb)
chetan lokef6fb8f12011-08-19 10:18:16 +00001145{
1146 unsigned int prev;
1147 if (rb->prb_bdqc.kactive_blk_num)
1148 prev = rb->prb_bdqc.kactive_blk_num-1;
1149 else
1150 prev = rb->prb_bdqc.knum_blocks-1;
1151 return prev;
1152}
1153
1154/* Assumes caller has held the rx_queue.lock */
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001155static void *__prb_previous_block(struct packet_sock *po,
chetan lokef6fb8f12011-08-19 10:18:16 +00001156 struct packet_ring_buffer *rb,
1157 int status)
1158{
1159 unsigned int previous = prb_previous_blk_num(rb);
1160 return prb_lookup_block(po, rb, previous, status);
1161}
1162
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001163static void *packet_previous_rx_frame(struct packet_sock *po,
chetan lokef6fb8f12011-08-19 10:18:16 +00001164 struct packet_ring_buffer *rb,
1165 int status)
1166{
1167 if (po->tp_version <= TPACKET_V2)
1168 return packet_previous_frame(po, rb, status);
1169
1170 return __prb_previous_block(po, rb, status);
1171}
1172
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001173static void packet_increment_rx_head(struct packet_sock *po,
chetan lokef6fb8f12011-08-19 10:18:16 +00001174 struct packet_ring_buffer *rb)
1175{
1176 switch (po->tp_version) {
1177 case TPACKET_V1:
1178 case TPACKET_V2:
1179 return packet_increment_head(rb);
1180 case TPACKET_V3:
1181 default:
1182 WARN(1, "TPACKET version not supported.\n");
1183 BUG();
1184 return;
1185 }
1186}
1187
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001188static void *packet_previous_frame(struct packet_sock *po,
Johann Baudy69e3c752009-05-18 22:11:22 -07001189 struct packet_ring_buffer *rb,
1190 int status)
1191{
1192 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1193 return packet_lookup_frame(po, rb, previous, status);
1194}
1195
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001196static void packet_increment_head(struct packet_ring_buffer *buff)
Johann Baudy69e3c752009-05-18 22:11:22 -07001197{
1198 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1199}
1200
Daniel Borkmannb0138402014-01-15 16:25:36 +01001201static void packet_inc_pending(struct packet_ring_buffer *rb)
1202{
1203 this_cpu_inc(*rb->pending_refcnt);
1204}
1205
1206static void packet_dec_pending(struct packet_ring_buffer *rb)
1207{
1208 this_cpu_dec(*rb->pending_refcnt);
1209}
1210
1211static unsigned int packet_read_pending(const struct packet_ring_buffer *rb)
1212{
1213 unsigned int refcnt = 0;
1214 int cpu;
1215
1216 /* We don't use pending refcount in rx_ring. */
1217 if (rb->pending_refcnt == NULL)
1218 return 0;
1219
1220 for_each_possible_cpu(cpu)
1221 refcnt += *per_cpu_ptr(rb->pending_refcnt, cpu);
1222
1223 return refcnt;
1224}
1225
1226static int packet_alloc_pending(struct packet_sock *po)
1227{
1228 po->rx_ring.pending_refcnt = NULL;
1229
1230 po->tx_ring.pending_refcnt = alloc_percpu(unsigned int);
1231 if (unlikely(po->tx_ring.pending_refcnt == NULL))
1232 return -ENOBUFS;
1233
1234 return 0;
1235}
1236
1237static void packet_free_pending(struct packet_sock *po)
1238{
1239 free_percpu(po->tx_ring.pending_refcnt);
1240}
1241
Willem de Bruijn99547292015-05-12 11:56:47 -04001242#define ROOM_POW_OFF 2
1243#define ROOM_NONE 0x0
1244#define ROOM_LOW 0x1
1245#define ROOM_NORMAL 0x2
1246
Eric Dumazetd4b5bd92019-06-12 09:52:27 -07001247static bool __tpacket_has_room(const struct packet_sock *po, int pow_off)
Willem de Bruijn99547292015-05-12 11:56:47 -04001248{
1249 int idx, len;
1250
Eric Dumazetd4b5bd92019-06-12 09:52:27 -07001251 len = READ_ONCE(po->rx_ring.frame_max) + 1;
1252 idx = READ_ONCE(po->rx_ring.head);
Willem de Bruijn99547292015-05-12 11:56:47 -04001253 if (pow_off)
1254 idx += len >> pow_off;
1255 if (idx >= len)
1256 idx -= len;
1257 return packet_lookup_frame(po, &po->rx_ring, idx, TP_STATUS_KERNEL);
1258}
1259
Eric Dumazetdcf70ce2019-06-12 09:52:28 -07001260static bool __tpacket_v3_has_room(const struct packet_sock *po, int pow_off)
Willem de Bruijn99547292015-05-12 11:56:47 -04001261{
1262 int idx, len;
1263
Eric Dumazetdcf70ce2019-06-12 09:52:28 -07001264 len = READ_ONCE(po->rx_ring.prb_bdqc.knum_blocks);
1265 idx = READ_ONCE(po->rx_ring.prb_bdqc.kactive_blk_num);
Willem de Bruijn99547292015-05-12 11:56:47 -04001266 if (pow_off)
1267 idx += len >> pow_off;
1268 if (idx >= len)
1269 idx -= len;
1270 return prb_lookup_block(po, &po->rx_ring, idx, TP_STATUS_KERNEL);
1271}
1272
Eric Dumazet0338a142019-06-12 09:52:29 -07001273static int __packet_rcv_has_room(const struct packet_sock *po,
1274 const struct sk_buff *skb)
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001275{
Eric Dumazet0338a142019-06-12 09:52:29 -07001276 const struct sock *sk = &po->sk;
Willem de Bruijn99547292015-05-12 11:56:47 -04001277 int ret = ROOM_NONE;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001278
Willem de Bruijn99547292015-05-12 11:56:47 -04001279 if (po->prot_hook.func != tpacket_rcv) {
Eric Dumazet0338a142019-06-12 09:52:29 -07001280 int rcvbuf = READ_ONCE(sk->sk_rcvbuf);
1281 int avail = rcvbuf - atomic_read(&sk->sk_rmem_alloc)
1282 - (skb ? skb->truesize : 0);
1283
1284 if (avail > (rcvbuf >> ROOM_POW_OFF))
Willem de Bruijn99547292015-05-12 11:56:47 -04001285 return ROOM_NORMAL;
1286 else if (avail > 0)
1287 return ROOM_LOW;
1288 else
1289 return ROOM_NONE;
1290 }
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001291
Willem de Bruijn99547292015-05-12 11:56:47 -04001292 if (po->tp_version == TPACKET_V3) {
1293 if (__tpacket_v3_has_room(po, ROOM_POW_OFF))
1294 ret = ROOM_NORMAL;
1295 else if (__tpacket_v3_has_room(po, 0))
1296 ret = ROOM_LOW;
1297 } else {
1298 if (__tpacket_has_room(po, ROOM_POW_OFF))
1299 ret = ROOM_NORMAL;
1300 else if (__tpacket_has_room(po, 0))
1301 ret = ROOM_LOW;
1302 }
Willem de Bruijn2ccdbaa2015-05-12 11:56:48 -04001303
1304 return ret;
1305}
1306
1307static int packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1308{
Eric Dumazet3a2bb842019-06-12 09:52:32 -07001309 int pressure, ret;
Willem de Bruijn2ccdbaa2015-05-12 11:56:48 -04001310
Willem de Bruijn54d7c012015-05-14 15:25:02 -04001311 ret = __packet_rcv_has_room(po, skb);
Eric Dumazet3a2bb842019-06-12 09:52:32 -07001312 pressure = ret != ROOM_NORMAL;
1313
1314 if (READ_ONCE(po->pressure) != pressure)
1315 WRITE_ONCE(po->pressure, pressure);
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001316
Willem de Bruijn99547292015-05-12 11:56:47 -04001317 return ret;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001318}
1319
Eric Dumazet9bb6cd62019-06-12 09:52:33 -07001320static void packet_rcv_try_clear_pressure(struct packet_sock *po)
1321{
1322 if (READ_ONCE(po->pressure) &&
1323 __packet_rcv_has_room(po, NULL) == ROOM_NORMAL)
1324 WRITE_ONCE(po->pressure, 0);
1325}
1326
Linus Torvalds1da177e2005-04-16 15:20:36 -07001327static void packet_sock_destruct(struct sock *sk)
1328{
Richard Cochraned85b562010-04-07 22:41:28 +00001329 skb_queue_purge(&sk->sk_error_queue);
1330
Ilpo Järvinen547b7922008-07-25 21:43:18 -07001331 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
Reshetova, Elena14afee42017-06-30 13:08:00 +03001332 WARN_ON(refcount_read(&sk->sk_wmem_alloc));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001333
1334 if (!sock_flag(sk, SOCK_DEAD)) {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001335 pr_err("Attempt to release alive packet socket: %p\n", sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001336 return;
1337 }
1338
Pavel Emelyanov17ab56a2007-11-10 21:38:48 -08001339 sk_refcnt_debug_dec(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001340}
1341
Willem de Bruijn3b3a5b02015-05-12 11:56:49 -04001342static bool fanout_flow_is_huge(struct packet_sock *po, struct sk_buff *skb)
1343{
Eric Dumazetb756ad92019-11-08 05:07:46 -08001344 u32 *history = po->rollover->history;
1345 u32 victim, rxhash;
Willem de Bruijn3b3a5b02015-05-12 11:56:49 -04001346 int i, count = 0;
1347
1348 rxhash = skb_get_hash(skb);
1349 for (i = 0; i < ROLLOVER_HLEN; i++)
Eric Dumazetb756ad92019-11-08 05:07:46 -08001350 if (READ_ONCE(history[i]) == rxhash)
Willem de Bruijn3b3a5b02015-05-12 11:56:49 -04001351 count++;
1352
Jason A. Donenfeld81895a62022-10-05 16:43:38 +02001353 victim = prandom_u32_max(ROLLOVER_HLEN);
Eric Dumazetb756ad92019-11-08 05:07:46 -08001354
1355 /* Avoid dirtying the cache line if possible */
1356 if (READ_ONCE(history[victim]) != rxhash)
1357 WRITE_ONCE(history[victim], rxhash);
1358
Willem de Bruijn3b3a5b02015-05-12 11:56:49 -04001359 return count > (ROLLOVER_HLEN >> 1);
1360}
1361
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001362static unsigned int fanout_demux_hash(struct packet_fanout *f,
1363 struct sk_buff *skb,
1364 unsigned int num)
David S. Millerdc99f602011-07-05 01:45:05 -07001365{
David S. Millereb70db82016-07-01 16:07:50 -04001366 return reciprocal_scale(__skb_get_hash_symmetric(skb), num);
David S. Millerdc99f602011-07-05 01:45:05 -07001367}
1368
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001369static unsigned int fanout_demux_lb(struct packet_fanout *f,
1370 struct sk_buff *skb,
1371 unsigned int num)
David S. Millerdc99f602011-07-05 01:45:05 -07001372{
Willem de Bruijn468479e2015-06-17 15:59:34 -04001373 unsigned int val = atomic_inc_return(&f->rr_cur);
David S. Millerdc99f602011-07-05 01:45:05 -07001374
Willem de Bruijn468479e2015-06-17 15:59:34 -04001375 return val % num;
David S. Millerdc99f602011-07-05 01:45:05 -07001376}
1377
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001378static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1379 struct sk_buff *skb,
1380 unsigned int num)
David S. Miller95ec3eb2011-07-06 01:56:38 -07001381{
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001382 return smp_processor_id() % num;
1383}
David S. Miller95ec3eb2011-07-06 01:56:38 -07001384
Daniel Borkmann5df0ddf2013-08-28 22:13:09 +02001385static unsigned int fanout_demux_rnd(struct packet_fanout *f,
1386 struct sk_buff *skb,
1387 unsigned int num)
1388{
Daniel Borkmannf337db62014-01-22 02:29:39 +01001389 return prandom_u32_max(num);
Daniel Borkmann5df0ddf2013-08-28 22:13:09 +02001390}
1391
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001392static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1393 struct sk_buff *skb,
Willem de Bruijnad377ca2015-05-12 11:56:45 -04001394 unsigned int idx, bool try_self,
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001395 unsigned int num)
1396{
Willem de Bruijn4633c9e2015-05-17 19:44:02 -04001397 struct packet_sock *po, *po_next, *po_skip = NULL;
Willem de Bruijna9b63912015-05-12 11:56:50 -04001398 unsigned int i, j, room = ROOM_NONE;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001399
Eric Dumazet94f633e2021-04-14 12:36:44 -07001400 po = pkt_sk(rcu_dereference(f->arr[idx]));
Willem de Bruijn3b3a5b02015-05-12 11:56:49 -04001401
1402 if (try_self) {
1403 room = packet_rcv_has_room(po, skb);
1404 if (room == ROOM_NORMAL ||
1405 (room == ROOM_LOW && !fanout_flow_is_huge(po, skb)))
1406 return idx;
Willem de Bruijn4633c9e2015-05-17 19:44:02 -04001407 po_skip = po;
Willem de Bruijn3b3a5b02015-05-12 11:56:49 -04001408 }
Willem de Bruijnad377ca2015-05-12 11:56:45 -04001409
Willem de Bruijn0648ab72015-05-12 11:56:46 -04001410 i = j = min_t(int, po->rollover->sock, num - 1);
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001411 do {
Eric Dumazet94f633e2021-04-14 12:36:44 -07001412 po_next = pkt_sk(rcu_dereference(f->arr[i]));
Eric Dumazet3a2bb842019-06-12 09:52:32 -07001413 if (po_next != po_skip && !READ_ONCE(po_next->pressure) &&
Willem de Bruijn2ccdbaa2015-05-12 11:56:48 -04001414 packet_rcv_has_room(po_next, skb) == ROOM_NORMAL) {
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001415 if (i != j)
Willem de Bruijn0648ab72015-05-12 11:56:46 -04001416 po->rollover->sock = i;
Willem de Bruijna9b63912015-05-12 11:56:50 -04001417 atomic_long_inc(&po->rollover->num);
1418 if (room == ROOM_LOW)
1419 atomic_long_inc(&po->rollover->num_huge);
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001420 return i;
1421 }
Willem de Bruijnad377ca2015-05-12 11:56:45 -04001422
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001423 if (++i == num)
1424 i = 0;
1425 } while (i != j);
1426
Willem de Bruijna9b63912015-05-12 11:56:50 -04001427 atomic_long_inc(&po->rollover->num_failed);
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001428 return idx;
1429}
1430
Neil Horman2d360972014-01-22 16:01:44 -05001431static unsigned int fanout_demux_qm(struct packet_fanout *f,
1432 struct sk_buff *skb,
1433 unsigned int num)
1434{
1435 return skb_get_queue_mapping(skb) % num;
1436}
1437
Willem de Bruijn47dceb82015-08-14 22:31:34 -04001438static unsigned int fanout_demux_bpf(struct packet_fanout *f,
1439 struct sk_buff *skb,
1440 unsigned int num)
1441{
1442 struct bpf_prog *prog;
1443 unsigned int ret = 0;
1444
1445 rcu_read_lock();
1446 prog = rcu_dereference(f->bpf_prog);
1447 if (prog)
Alexei Starovoitovff936a02015-10-07 10:55:41 -07001448 ret = bpf_prog_run_clear_cb(prog, skb) % num;
Willem de Bruijn47dceb82015-08-14 22:31:34 -04001449 rcu_read_unlock();
1450
1451 return ret;
1452}
1453
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001454static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1455{
1456 return f->flags & (flag >> 8);
David S. Miller95ec3eb2011-07-06 01:56:38 -07001457}
1458
David S. Miller95ec3eb2011-07-06 01:56:38 -07001459static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1460 struct packet_type *pt, struct net_device *orig_dev)
David S. Millerdc99f602011-07-05 01:45:05 -07001461{
1462 struct packet_fanout *f = pt->af_packet_priv;
Eric Dumazetf98f4512015-06-16 07:59:11 -07001463 unsigned int num = READ_ONCE(f->num_members);
Eric W. Biederman19bcf9f2015-10-09 13:44:54 -05001464 struct net *net = read_pnet(&f->net);
David S. Millerdc99f602011-07-05 01:45:05 -07001465 struct packet_sock *po;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001466 unsigned int idx;
David S. Millerdc99f602011-07-05 01:45:05 -07001467
Eric W. Biederman19bcf9f2015-10-09 13:44:54 -05001468 if (!net_eq(dev_net(dev), net) || !num) {
David S. Millerdc99f602011-07-05 01:45:05 -07001469 kfree_skb(skb);
1470 return 0;
1471 }
1472
Alexander Drozdov3f34b242015-02-20 08:24:27 +03001473 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
Eric W. Biederman19bcf9f2015-10-09 13:44:54 -05001474 skb = ip_check_defrag(net, skb, IP_DEFRAG_AF_PACKET);
Alexander Drozdov3f34b242015-02-20 08:24:27 +03001475 if (!skb)
1476 return 0;
1477 }
David S. Miller95ec3eb2011-07-06 01:56:38 -07001478 switch (f->type) {
1479 case PACKET_FANOUT_HASH:
1480 default:
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001481 idx = fanout_demux_hash(f, skb, num);
David S. Miller95ec3eb2011-07-06 01:56:38 -07001482 break;
1483 case PACKET_FANOUT_LB:
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001484 idx = fanout_demux_lb(f, skb, num);
David S. Miller95ec3eb2011-07-06 01:56:38 -07001485 break;
1486 case PACKET_FANOUT_CPU:
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001487 idx = fanout_demux_cpu(f, skb, num);
1488 break;
Daniel Borkmann5df0ddf2013-08-28 22:13:09 +02001489 case PACKET_FANOUT_RND:
1490 idx = fanout_demux_rnd(f, skb, num);
1491 break;
Neil Horman2d360972014-01-22 16:01:44 -05001492 case PACKET_FANOUT_QM:
1493 idx = fanout_demux_qm(f, skb, num);
1494 break;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001495 case PACKET_FANOUT_ROLLOVER:
Willem de Bruijnad377ca2015-05-12 11:56:45 -04001496 idx = fanout_demux_rollover(f, skb, 0, false, num);
David S. Miller95ec3eb2011-07-06 01:56:38 -07001497 break;
Willem de Bruijn47dceb82015-08-14 22:31:34 -04001498 case PACKET_FANOUT_CBPF:
Willem de Bruijnf2e52092015-08-14 22:31:35 -04001499 case PACKET_FANOUT_EBPF:
Willem de Bruijn47dceb82015-08-14 22:31:34 -04001500 idx = fanout_demux_bpf(f, skb, num);
1501 break;
David S. Miller7736d332011-07-05 01:43:20 -07001502 }
1503
Willem de Bruijnad377ca2015-05-12 11:56:45 -04001504 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER))
1505 idx = fanout_demux_rollover(f, skb, idx, true, num);
David S. Millerdc99f602011-07-05 01:45:05 -07001506
Eric Dumazet94f633e2021-04-14 12:36:44 -07001507 po = pkt_sk(rcu_dereference(f->arr[idx]));
David S. Millerdc99f602011-07-05 01:45:05 -07001508 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1509}
1510
Pavel Emelyanovfff33212012-08-16 05:36:48 +00001511DEFINE_MUTEX(fanout_mutex);
1512EXPORT_SYMBOL_GPL(fanout_mutex);
David S. Millerdc99f602011-07-05 01:45:05 -07001513static LIST_HEAD(fanout_list);
Mike Maloney4a69a862017-04-21 10:56:11 -04001514static u16 fanout_next_id;
David S. Millerdc99f602011-07-05 01:45:05 -07001515
1516static void __fanout_link(struct sock *sk, struct packet_sock *po)
1517{
1518 struct packet_fanout *f = po->fanout;
1519
1520 spin_lock(&f->lock);
Eric Dumazet94f633e2021-04-14 12:36:44 -07001521 rcu_assign_pointer(f->arr[f->num_members], sk);
David S. Millerdc99f602011-07-05 01:45:05 -07001522 smp_wmb();
1523 f->num_members++;
Anoob Soman2bd624b2017-02-15 20:25:39 +00001524 if (f->num_members == 1)
1525 dev_add_pack(&f->prot_hook);
David S. Millerdc99f602011-07-05 01:45:05 -07001526 spin_unlock(&f->lock);
1527}
1528
1529static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1530{
1531 struct packet_fanout *f = po->fanout;
1532 int i;
1533
1534 spin_lock(&f->lock);
1535 for (i = 0; i < f->num_members; i++) {
Eric Dumazet94f633e2021-04-14 12:36:44 -07001536 if (rcu_dereference_protected(f->arr[i],
1537 lockdep_is_held(&f->lock)) == sk)
David S. Millerdc99f602011-07-05 01:45:05 -07001538 break;
1539 }
1540 BUG_ON(i >= f->num_members);
Eric Dumazet94f633e2021-04-14 12:36:44 -07001541 rcu_assign_pointer(f->arr[i],
1542 rcu_dereference_protected(f->arr[f->num_members - 1],
1543 lockdep_is_held(&f->lock)));
David S. Millerdc99f602011-07-05 01:45:05 -07001544 f->num_members--;
Anoob Soman2bd624b2017-02-15 20:25:39 +00001545 if (f->num_members == 0)
1546 __dev_remove_pack(&f->prot_hook);
David S. Millerdc99f602011-07-05 01:45:05 -07001547 spin_unlock(&f->lock);
1548}
1549
Weilong Chend4dd8ae2013-12-23 11:31:38 +08001550static bool match_fanout_group(struct packet_type *ptype, struct sock *sk)
Eric Leblondc0de08d2012-08-16 22:02:58 +00001551{
Eric Dumazet161642e2015-10-09 11:29:32 -07001552 if (sk->sk_family != PF_PACKET)
1553 return false;
Eric Leblondc0de08d2012-08-16 22:02:58 +00001554
Eric Dumazet161642e2015-10-09 11:29:32 -07001555 return ptype->af_packet_priv == pkt_sk(sk)->fanout;
Eric Leblondc0de08d2012-08-16 22:02:58 +00001556}
1557
Willem de Bruijn47dceb82015-08-14 22:31:34 -04001558static void fanout_init_data(struct packet_fanout *f)
1559{
1560 switch (f->type) {
1561 case PACKET_FANOUT_LB:
1562 atomic_set(&f->rr_cur, 0);
1563 break;
1564 case PACKET_FANOUT_CBPF:
Willem de Bruijnf2e52092015-08-14 22:31:35 -04001565 case PACKET_FANOUT_EBPF:
Willem de Bruijn47dceb82015-08-14 22:31:34 -04001566 RCU_INIT_POINTER(f->bpf_prog, NULL);
1567 break;
1568 }
1569}
1570
1571static void __fanout_set_data_bpf(struct packet_fanout *f, struct bpf_prog *new)
1572{
1573 struct bpf_prog *old;
1574
1575 spin_lock(&f->lock);
1576 old = rcu_dereference_protected(f->bpf_prog, lockdep_is_held(&f->lock));
1577 rcu_assign_pointer(f->bpf_prog, new);
1578 spin_unlock(&f->lock);
1579
1580 if (old) {
1581 synchronize_net();
1582 bpf_prog_destroy(old);
1583 }
1584}
1585
Christoph Hellwigb1ea9ff2020-07-23 08:08:47 +02001586static int fanout_set_data_cbpf(struct packet_sock *po, sockptr_t data,
Willem de Bruijn47dceb82015-08-14 22:31:34 -04001587 unsigned int len)
1588{
1589 struct bpf_prog *new;
1590 struct sock_fprog fprog;
1591 int ret;
1592
1593 if (sock_flag(&po->sk, SOCK_FILTER_LOCKED))
1594 return -EPERM;
Christoph Hellwig4d295e52020-07-17 08:23:13 +02001595
1596 ret = copy_bpf_fprog_from_user(&fprog, data, len);
1597 if (ret)
1598 return ret;
Willem de Bruijn47dceb82015-08-14 22:31:34 -04001599
Daniel Borkmannbab18992015-10-02 15:17:33 +02001600 ret = bpf_prog_create_from_user(&new, &fprog, NULL, false);
Willem de Bruijn47dceb82015-08-14 22:31:34 -04001601 if (ret)
1602 return ret;
1603
1604 __fanout_set_data_bpf(po->fanout, new);
1605 return 0;
1606}
1607
Christoph Hellwiga7b75c52020-07-23 08:09:07 +02001608static int fanout_set_data_ebpf(struct packet_sock *po, sockptr_t data,
Willem de Bruijnf2e52092015-08-14 22:31:35 -04001609 unsigned int len)
1610{
1611 struct bpf_prog *new;
1612 u32 fd;
1613
1614 if (sock_flag(&po->sk, SOCK_FILTER_LOCKED))
1615 return -EPERM;
1616 if (len != sizeof(fd))
1617 return -EINVAL;
Christoph Hellwiga7b75c52020-07-23 08:09:07 +02001618 if (copy_from_sockptr(&fd, data, len))
Willem de Bruijnf2e52092015-08-14 22:31:35 -04001619 return -EFAULT;
1620
Daniel Borkmann113214b2016-06-30 17:24:44 +02001621 new = bpf_prog_get_type(fd, BPF_PROG_TYPE_SOCKET_FILTER);
Willem de Bruijnf2e52092015-08-14 22:31:35 -04001622 if (IS_ERR(new))
1623 return PTR_ERR(new);
Willem de Bruijnf2e52092015-08-14 22:31:35 -04001624
1625 __fanout_set_data_bpf(po->fanout, new);
1626 return 0;
1627}
1628
Christoph Hellwiga7b75c52020-07-23 08:09:07 +02001629static int fanout_set_data(struct packet_sock *po, sockptr_t data,
Willem de Bruijn47dceb82015-08-14 22:31:34 -04001630 unsigned int len)
1631{
1632 switch (po->fanout->type) {
1633 case PACKET_FANOUT_CBPF:
Christoph Hellwiga7b75c52020-07-23 08:09:07 +02001634 return fanout_set_data_cbpf(po, data, len);
Willem de Bruijnf2e52092015-08-14 22:31:35 -04001635 case PACKET_FANOUT_EBPF:
1636 return fanout_set_data_ebpf(po, data, len);
Willem de Bruijn47dceb82015-08-14 22:31:34 -04001637 default:
1638 return -EINVAL;
zhong jiang07d53ae2018-08-04 19:41:41 +08001639 }
Willem de Bruijn47dceb82015-08-14 22:31:34 -04001640}
1641
1642static void fanout_release_data(struct packet_fanout *f)
1643{
1644 switch (f->type) {
1645 case PACKET_FANOUT_CBPF:
Willem de Bruijnf2e52092015-08-14 22:31:35 -04001646 case PACKET_FANOUT_EBPF:
Willem de Bruijn47dceb82015-08-14 22:31:34 -04001647 __fanout_set_data_bpf(f, NULL);
zhong jiang07d53ae2018-08-04 19:41:41 +08001648 }
Willem de Bruijn47dceb82015-08-14 22:31:34 -04001649}
1650
Mike Maloney4a69a862017-04-21 10:56:11 -04001651static bool __fanout_id_is_free(struct sock *sk, u16 candidate_id)
1652{
1653 struct packet_fanout *f;
1654
1655 list_for_each_entry(f, &fanout_list, list) {
1656 if (f->id == candidate_id &&
1657 read_pnet(&f->net) == sock_net(sk)) {
1658 return false;
1659 }
1660 }
1661 return true;
1662}
1663
1664static bool fanout_find_new_id(struct sock *sk, u16 *new_id)
1665{
1666 u16 id = fanout_next_id;
1667
1668 do {
1669 if (__fanout_id_is_free(sk, id)) {
1670 *new_id = id;
1671 fanout_next_id = id + 1;
1672 return true;
1673 }
1674
1675 id++;
1676 } while (id != fanout_next_id);
1677
1678 return false;
1679}
1680
Tanner Love9c661b02020-11-06 13:07:40 -05001681static int fanout_add(struct sock *sk, struct fanout_args *args)
David S. Millerdc99f602011-07-05 01:45:05 -07001682{
Eric Dumazetd199fab2017-02-14 09:03:51 -08001683 struct packet_rollover *rollover = NULL;
David S. Millerdc99f602011-07-05 01:45:05 -07001684 struct packet_sock *po = pkt_sk(sk);
Tanner Love9c661b02020-11-06 13:07:40 -05001685 u16 type_flags = args->type_flags;
David S. Millerdc99f602011-07-05 01:45:05 -07001686 struct packet_fanout *f, *match;
David S. Miller7736d332011-07-05 01:43:20 -07001687 u8 type = type_flags & 0xff;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001688 u8 flags = type_flags >> 8;
Tanner Love9c661b02020-11-06 13:07:40 -05001689 u16 id = args->id;
David S. Millerdc99f602011-07-05 01:45:05 -07001690 int err;
1691
1692 switch (type) {
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001693 case PACKET_FANOUT_ROLLOVER:
1694 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1695 return -EINVAL;
Gustavo A. R. Silva5af5a022020-11-20 12:38:20 -06001696 break;
David S. Millerdc99f602011-07-05 01:45:05 -07001697 case PACKET_FANOUT_HASH:
1698 case PACKET_FANOUT_LB:
David S. Miller95ec3eb2011-07-06 01:56:38 -07001699 case PACKET_FANOUT_CPU:
Daniel Borkmann5df0ddf2013-08-28 22:13:09 +02001700 case PACKET_FANOUT_RND:
Neil Horman2d360972014-01-22 16:01:44 -05001701 case PACKET_FANOUT_QM:
Willem de Bruijn47dceb82015-08-14 22:31:34 -04001702 case PACKET_FANOUT_CBPF:
Willem de Bruijnf2e52092015-08-14 22:31:35 -04001703 case PACKET_FANOUT_EBPF:
David S. Millerdc99f602011-07-05 01:45:05 -07001704 break;
1705 default:
1706 return -EINVAL;
1707 }
1708
Eric Dumazetd199fab2017-02-14 09:03:51 -08001709 mutex_lock(&fanout_mutex);
David S. Millerdc99f602011-07-05 01:45:05 -07001710
Eric Dumazetd199fab2017-02-14 09:03:51 -08001711 err = -EALREADY;
David S. Millerdc99f602011-07-05 01:45:05 -07001712 if (po->fanout)
Eric Dumazetd199fab2017-02-14 09:03:51 -08001713 goto out;
David S. Millerdc99f602011-07-05 01:45:05 -07001714
Willem de Bruijn4633c9e2015-05-17 19:44:02 -04001715 if (type == PACKET_FANOUT_ROLLOVER ||
1716 (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)) {
Eric Dumazetd199fab2017-02-14 09:03:51 -08001717 err = -ENOMEM;
1718 rollover = kzalloc(sizeof(*rollover), GFP_KERNEL);
1719 if (!rollover)
1720 goto out;
1721 atomic_long_set(&rollover->num, 0);
1722 atomic_long_set(&rollover->num_huge, 0);
1723 atomic_long_set(&rollover->num_failed, 0);
Willem de Bruijn0648ab72015-05-12 11:56:46 -04001724 }
1725
Mike Maloney4a69a862017-04-21 10:56:11 -04001726 if (type_flags & PACKET_FANOUT_FLAG_UNIQUEID) {
1727 if (id != 0) {
1728 err = -EINVAL;
1729 goto out;
1730 }
1731 if (!fanout_find_new_id(sk, &id)) {
1732 err = -ENOMEM;
1733 goto out;
1734 }
1735 /* ephemeral flag for the first socket in the group: drop it */
1736 flags &= ~(PACKET_FANOUT_FLAG_UNIQUEID >> 8);
1737 }
1738
David S. Millerdc99f602011-07-05 01:45:05 -07001739 match = NULL;
1740 list_for_each_entry(f, &fanout_list, list) {
1741 if (f->id == id &&
1742 read_pnet(&f->net) == sock_net(sk)) {
1743 match = f;
1744 break;
1745 }
1746 }
Eric Dumazetafe62c62011-07-07 06:41:29 -07001747 err = -EINVAL;
Tanner Love9c661b02020-11-06 13:07:40 -05001748 if (match) {
1749 if (match->flags != flags)
1750 goto out;
1751 if (args->max_num_members &&
1752 args->max_num_members != match->max_num_members)
1753 goto out;
1754 } else {
1755 if (args->max_num_members > PACKET_FANOUT_MAX)
1756 goto out;
1757 if (!args->max_num_members)
1758 /* legacy PACKET_FANOUT_MAX */
1759 args->max_num_members = 256;
Eric Dumazetafe62c62011-07-07 06:41:29 -07001760 err = -ENOMEM;
Tanner Love9c661b02020-11-06 13:07:40 -05001761 match = kvzalloc(struct_size(match, arr, args->max_num_members),
1762 GFP_KERNEL);
Eric Dumazetafe62c62011-07-07 06:41:29 -07001763 if (!match)
1764 goto out;
1765 write_pnet(&match->net, sock_net(sk));
1766 match->id = id;
1767 match->type = type;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001768 match->flags = flags;
Eric Dumazetafe62c62011-07-07 06:41:29 -07001769 INIT_LIST_HEAD(&match->list);
1770 spin_lock_init(&match->lock);
Reshetova, Elenafb5c2c12017-06-30 13:08:10 +03001771 refcount_set(&match->sk_ref, 0);
Willem de Bruijn47dceb82015-08-14 22:31:34 -04001772 fanout_init_data(match);
Eric Dumazetafe62c62011-07-07 06:41:29 -07001773 match->prot_hook.type = po->prot_hook.type;
1774 match->prot_hook.dev = po->prot_hook.dev;
1775 match->prot_hook.func = packet_rcv_fanout;
1776 match->prot_hook.af_packet_priv = match;
Congyu Liu47934e062022-01-18 14:20:13 -05001777 match->prot_hook.af_packet_net = read_pnet(&match->net);
Eric Leblondc0de08d2012-08-16 22:02:58 +00001778 match->prot_hook.id_match = match_fanout_group;
Tanner Love9c661b02020-11-06 13:07:40 -05001779 match->max_num_members = args->max_num_members;
Eric Dumazetafe62c62011-07-07 06:41:29 -07001780 list_add(&match->list, &fanout_list);
1781 }
1782 err = -EINVAL;
Willem de Bruijn008ba2a2017-09-14 17:14:41 -04001783
1784 spin_lock(&po->bind_lock);
1785 if (po->running &&
1786 match->type == type &&
Eric Dumazetafe62c62011-07-07 06:41:29 -07001787 match->prot_hook.type == po->prot_hook.type &&
1788 match->prot_hook.dev == po->prot_hook.dev) {
1789 err = -ENOSPC;
Tanner Love9c661b02020-11-06 13:07:40 -05001790 if (refcount_read(&match->sk_ref) < match->max_num_members) {
Eric Dumazetafe62c62011-07-07 06:41:29 -07001791 __dev_remove_pack(&po->prot_hook);
Eric Dumazete42e70a2022-01-31 18:23:58 -08001792
1793 /* Paired with packet_setsockopt(PACKET_FANOUT_DATA) */
1794 WRITE_ONCE(po->fanout, match);
1795
Mike Maloney57f015f2017-11-28 10:44:29 -05001796 po->rollover = rollover;
1797 rollover = NULL;
Reshetova, Elenafb5c2c12017-06-30 13:08:10 +03001798 refcount_set(&match->sk_ref, refcount_read(&match->sk_ref) + 1);
Eric Dumazetafe62c62011-07-07 06:41:29 -07001799 __fanout_link(sk, po);
1800 err = 0;
David S. Millerdc99f602011-07-05 01:45:05 -07001801 }
1802 }
Willem de Bruijn008ba2a2017-09-14 17:14:41 -04001803 spin_unlock(&po->bind_lock);
1804
1805 if (err && !refcount_read(&match->sk_ref)) {
1806 list_del(&match->list);
Tanner Love9c661b02020-11-06 13:07:40 -05001807 kvfree(match);
Willem de Bruijn008ba2a2017-09-14 17:14:41 -04001808 }
1809
Eric Dumazetafe62c62011-07-07 06:41:29 -07001810out:
Mike Maloney57f015f2017-11-28 10:44:29 -05001811 kfree(rollover);
Eric Dumazetd199fab2017-02-14 09:03:51 -08001812 mutex_unlock(&fanout_mutex);
David S. Millerdc99f602011-07-05 01:45:05 -07001813 return err;
1814}
1815
Anoob Soman2bd624b2017-02-15 20:25:39 +00001816/* If pkt_sk(sk)->fanout->sk_ref is zero, this function removes
1817 * pkt_sk(sk)->fanout from fanout_list and returns pkt_sk(sk)->fanout.
1818 * It is the responsibility of the caller to call fanout_release_data() and
1819 * free the returned packet_fanout (after synchronize_net())
1820 */
1821static struct packet_fanout *fanout_release(struct sock *sk)
David S. Millerdc99f602011-07-05 01:45:05 -07001822{
1823 struct packet_sock *po = pkt_sk(sk);
1824 struct packet_fanout *f;
1825
Pavel Emelyanovfff33212012-08-16 05:36:48 +00001826 mutex_lock(&fanout_mutex);
Eric Dumazetd199fab2017-02-14 09:03:51 -08001827 f = po->fanout;
1828 if (f) {
1829 po->fanout = NULL;
David S. Millerdc99f602011-07-05 01:45:05 -07001830
Reshetova, Elenafb5c2c12017-06-30 13:08:10 +03001831 if (refcount_dec_and_test(&f->sk_ref))
Eric Dumazetd199fab2017-02-14 09:03:51 -08001832 list_del(&f->list);
Anoob Soman2bd624b2017-02-15 20:25:39 +00001833 else
1834 f = NULL;
David S. Millerdc99f602011-07-05 01:45:05 -07001835 }
1836 mutex_unlock(&fanout_mutex);
Anoob Soman2bd624b2017-02-15 20:25:39 +00001837
1838 return f;
David S. Millerdc99f602011-07-05 01:45:05 -07001839}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001840
Daniel Borkmann3c70c132015-11-11 23:25:42 +01001841static bool packet_extra_vlan_len_allowed(const struct net_device *dev,
1842 struct sk_buff *skb)
1843{
1844 /* Earlier code assumed this would be a VLAN pkt, double-check
1845 * this now that we have the actual packet in hand. We can only
1846 * do this check on Ethernet devices.
1847 */
1848 if (unlikely(dev->type != ARPHRD_ETHER))
1849 return false;
1850
1851 skb_reset_mac_header(skb);
1852 return likely(eth_hdr(skb)->h_proto == htons(ETH_P_8021Q));
1853}
1854
Eric Dumazet90ddc4f2005-12-22 12:49:22 -08001855static const struct proto_ops packet_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001856
Eric Dumazet90ddc4f2005-12-22 12:49:22 -08001857static const struct proto_ops packet_ops_spkt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001858
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001859static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1860 struct packet_type *pt, struct net_device *orig_dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001861{
1862 struct sock *sk;
1863 struct sockaddr_pkt *spkt;
1864
1865 /*
1866 * When we registered the protocol we saved the socket in the data
1867 * field for just this event.
1868 */
1869
1870 sk = pt->af_packet_priv;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001871
Linus Torvalds1da177e2005-04-16 15:20:36 -07001872 /*
1873 * Yank back the headers [hope the device set this
1874 * right or kerboom...]
1875 *
1876 * Incoming packets have ll header pulled,
1877 * push it back.
1878 *
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07001879 * For outgoing ones skb->data == skb_mac_header(skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001880 * so that this procedure is noop.
1881 */
1882
1883 if (skb->pkt_type == PACKET_LOOPBACK)
1884 goto out;
1885
Octavian Purdila09ad9bc2009-11-25 15:14:13 -08001886 if (!net_eq(dev_net(dev), sock_net(sk)))
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08001887 goto out;
1888
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001889 skb = skb_share_check(skb, GFP_ATOMIC);
1890 if (skb == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001891 goto oom;
1892
1893 /* drop any routing info */
Eric Dumazetadf30902009-06-02 05:19:30 +00001894 skb_dst_drop(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001895
Phil Oester84531c22005-07-12 11:57:52 -07001896 /* drop conntrack reference */
Florian Westphal895b5c92019-09-29 20:54:03 +02001897 nf_reset_ct(skb);
Phil Oester84531c22005-07-12 11:57:52 -07001898
Herbert Xuffbc6112007-02-04 23:33:10 -08001899 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001900
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07001901 skb_push(skb, skb->data - skb_mac_header(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001902
1903 /*
1904 * The SOCK_PACKET socket receives _all_ frames.
1905 */
1906
1907 spkt->spkt_family = dev->type;
Wolfram Sang8fc9d512022-08-18 23:02:27 +02001908 strscpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001909 spkt->spkt_protocol = skb->protocol;
1910
1911 /*
1912 * Charge the memory to the socket. This is done specifically
1913 * to prevent sockets using all the memory up.
1914 */
1915
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001916 if (sock_queue_rcv_skb(sk, skb) == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001917 return 0;
1918
1919out:
1920 kfree_skb(skb);
1921oom:
1922 return 0;
1923}
1924
Maxim Mikityanskiy75c65772019-02-21 12:40:01 +00001925static void packet_parse_headers(struct sk_buff *skb, struct socket *sock)
1926{
Hangbin Liudfed9132022-04-25 09:45:02 +08001927 int depth;
1928
Yoshiki Komachi18bed892019-03-18 14:39:52 +09001929 if ((!skb->protocol || skb->protocol == htons(ETH_P_ALL)) &&
1930 sock->type == SOCK_RAW) {
Maxim Mikityanskiy75c65772019-02-21 12:40:01 +00001931 skb_reset_mac_header(skb);
1932 skb->protocol = dev_parse_header_protocol(skb);
1933 }
1934
Hangbin Liudfed9132022-04-25 09:45:02 +08001935 /* Move network header to the right position for VLAN tagged packets */
1936 if (likely(skb->dev->type == ARPHRD_ETHER) &&
1937 eth_type_vlan(skb->protocol) &&
Eric Dumazete9d3f802022-06-02 09:18:59 -07001938 __vlan_get_protocol(skb, skb->protocol, &depth) != 0) {
1939 if (pskb_may_pull(skb, depth))
1940 skb_set_network_header(skb, depth);
1941 }
Hangbin Liudfed9132022-04-25 09:45:02 +08001942
Maxim Mikityanskiy75c65772019-02-21 12:40:01 +00001943 skb_probe_transport_header(skb);
1944}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001945
1946/*
1947 * Output a raw packet to a device layer. This bypasses all the other
1948 * protocol layers and you must therefore supply it with a complete frame
1949 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001950
Ying Xue1b784142015-03-02 15:37:48 +08001951static int packet_sendmsg_spkt(struct socket *sock, struct msghdr *msg,
1952 size_t len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001953{
1954 struct sock *sk = sock->sk;
Steffen Hurrle342dfc32014-01-17 22:53:15 +01001955 DECLARE_SOCKADDR(struct sockaddr_pkt *, saddr, msg->msg_name);
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001956 struct sk_buff *skb = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001957 struct net_device *dev;
Soheil Hassas Yeganehc14ac942016-04-02 23:08:12 -04001958 struct sockcm_cookie sockc;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001959 __be16 proto = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001960 int err;
Ben Greear3bdc0eb2012-02-11 15:39:30 +00001961 int extra_len = 0;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001962
Linus Torvalds1da177e2005-04-16 15:20:36 -07001963 /*
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001964 * Get and verify the address.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001965 */
1966
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001967 if (saddr) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001968 if (msg->msg_namelen < sizeof(struct sockaddr))
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001969 return -EINVAL;
1970 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1971 proto = saddr->spkt_protocol;
1972 } else
1973 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001974
1975 /*
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001976 * Find the device first to size check it
Linus Torvalds1da177e2005-04-16 15:20:36 -07001977 */
1978
danborkmann@iogearbox.netde74e922012-06-10 08:59:28 +00001979 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001980retry:
Eric Dumazet654d1f82009-11-02 10:43:32 +01001981 rcu_read_lock();
1982 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001983 err = -ENODEV;
1984 if (dev == NULL)
1985 goto out_unlock;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001986
David S. Millerd5e76b02007-01-25 19:30:36 -08001987 err = -ENETDOWN;
1988 if (!(dev->flags & IFF_UP))
1989 goto out_unlock;
1990
Linus Torvalds1da177e2005-04-16 15:20:36 -07001991 /*
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001992 * You may not queue a frame bigger than the mtu. This is the lowest level
1993 * raw protocol and you must do your own fragmentation at this level.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001994 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001995
Ben Greear3bdc0eb2012-02-11 15:39:30 +00001996 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1997 if (!netif_supports_nofcs(dev)) {
1998 err = -EPROTONOSUPPORT;
1999 goto out_unlock;
2000 }
2001 extra_len = 4; /* We're doing our own CRC */
2002 }
2003
Linus Torvalds1da177e2005-04-16 15:20:36 -07002004 err = -EMSGSIZE;
Ben Greear3bdc0eb2012-02-11 15:39:30 +00002005 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002006 goto out_unlock;
2007
Eric Dumazet1a35ca82009-12-15 05:47:03 +00002008 if (!skb) {
2009 size_t reserved = LL_RESERVED_SPACE(dev);
Herbert Xu4ce40912011-11-18 02:20:05 +00002010 int tlen = dev->needed_tailroom;
Eric Dumazet1a35ca82009-12-15 05:47:03 +00002011 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002012
Eric Dumazet1a35ca82009-12-15 05:47:03 +00002013 rcu_read_unlock();
Herbert Xu4ce40912011-11-18 02:20:05 +00002014 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
Eric Dumazet1a35ca82009-12-15 05:47:03 +00002015 if (skb == NULL)
2016 return -ENOBUFS;
2017 /* FIXME: Save some space for broken drivers that write a hard
2018 * header at transmission time by themselves. PPP is the notable
2019 * one here. This should really be fixed at the driver level.
2020 */
2021 skb_reserve(skb, reserved);
2022 skb_reset_network_header(skb);
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002023
Eric Dumazet1a35ca82009-12-15 05:47:03 +00002024 /* Try to align data part correctly */
2025 if (hhlen) {
2026 skb->data -= hhlen;
2027 skb->tail -= hhlen;
2028 if (len < hhlen)
2029 skb_reset_network_header(skb);
2030 }
Al Viro6ce8e9c2014-04-06 21:25:44 -04002031 err = memcpy_from_msg(skb_put(skb, len), msg, len);
Eric Dumazet1a35ca82009-12-15 05:47:03 +00002032 if (err)
2033 goto out_free;
2034 goto retry;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002035 }
2036
Willem de Bruijn9ed988c2016-03-09 21:58:34 -05002037 if (!dev_validate_header(dev, skb->data, len)) {
2038 err = -EINVAL;
2039 goto out_unlock;
2040 }
Daniel Borkmann3c70c132015-11-11 23:25:42 +01002041 if (len > (dev->mtu + dev->hard_header_len + extra_len) &&
2042 !packet_extra_vlan_len_allowed(dev, skb)) {
2043 err = -EMSGSIZE;
2044 goto out_unlock;
Ben Greear57f89bf2011-02-11 09:35:18 +00002045 }
Eric Dumazet1a35ca82009-12-15 05:47:03 +00002046
Willem de Bruijn657a0662018-07-06 10:12:56 -04002047 sockcm_init(&sockc, sk);
Soheil Hassas Yeganehc14ac942016-04-02 23:08:12 -04002048 if (msg->msg_controllen) {
2049 err = sock_cmsg_send(sk, msg, &sockc);
Soheil Hassas Yeganehf8e77182016-07-20 18:01:18 -04002050 if (unlikely(err))
Soheil Hassas Yeganehc14ac942016-04-02 23:08:12 -04002051 goto out_unlock;
Soheil Hassas Yeganehc14ac942016-04-02 23:08:12 -04002052 }
2053
Linus Torvalds1da177e2005-04-16 15:20:36 -07002054 skb->protocol = proto;
2055 skb->dev = dev;
2056 skb->priority = sk->sk_priority;
Eric Dumazet2d37a182009-10-01 19:14:46 +00002057 skb->mark = sk->sk_mark;
Richard Cochran3d0ba8c2018-07-03 15:42:51 -07002058 skb->tstamp = sockc.transmit_time;
Daniel Borkmannbf84a0102013-04-14 08:08:13 +00002059
Willem de Bruijn8f932f72018-12-17 12:24:00 -05002060 skb_setup_tx_timestamp(skb, sockc.tsflags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002061
Ben Greear3bdc0eb2012-02-11 15:39:30 +00002062 if (unlikely(extra_len == 4))
2063 skb->no_fcs = 1;
2064
Maxim Mikityanskiy75c65772019-02-21 12:40:01 +00002065 packet_parse_headers(skb, sock);
Jason Wangc1aad272013-03-25 20:19:57 +00002066
Linus Torvalds1da177e2005-04-16 15:20:36 -07002067 dev_queue_xmit(skb);
Eric Dumazet654d1f82009-11-02 10:43:32 +01002068 rcu_read_unlock();
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002069 return len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002070
Linus Torvalds1da177e2005-04-16 15:20:36 -07002071out_unlock:
Eric Dumazet654d1f82009-11-02 10:43:32 +01002072 rcu_read_unlock();
Eric Dumazet1a35ca82009-12-15 05:47:03 +00002073out_free:
2074 kfree_skb(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002075 return err;
2076}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002077
Alexei Starovoitovff936a02015-10-07 10:55:41 -07002078static unsigned int run_filter(struct sk_buff *skb,
2079 const struct sock *sk,
2080 unsigned int res)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002081{
2082 struct sk_filter *filter;
2083
Eric Dumazet80f8f102011-01-18 07:46:52 +00002084 rcu_read_lock();
2085 filter = rcu_dereference(sk->sk_filter);
David S. Millerdbcb5852007-01-24 15:21:02 -08002086 if (filter != NULL)
Alexei Starovoitovff936a02015-10-07 10:55:41 -07002087 res = bpf_prog_run_clear_cb(filter->prog, skb);
Eric Dumazet80f8f102011-01-18 07:46:52 +00002088 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002089
David S. Millerdbcb5852007-01-24 15:21:02 -08002090 return res;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002091}
2092
Willem de Bruijn16cc1402016-02-03 18:02:14 -05002093static int packet_rcv_vnet(struct msghdr *msg, const struct sk_buff *skb,
2094 size_t *len)
2095{
2096 struct virtio_net_hdr vnet_hdr;
2097
2098 if (*len < sizeof(vnet_hdr))
2099 return -EINVAL;
2100 *len -= sizeof(vnet_hdr);
2101
Willem de Bruijnfd3a8862018-06-06 11:23:01 -04002102 if (virtio_net_hdr_from_skb(skb, &vnet_hdr, vio_le(), true, 0))
Willem de Bruijn16cc1402016-02-03 18:02:14 -05002103 return -EINVAL;
2104
2105 return memcpy_to_msg(msg, (void *)&vnet_hdr, sizeof(vnet_hdr));
2106}
2107
Linus Torvalds1da177e2005-04-16 15:20:36 -07002108/*
Eric Dumazet62ab0812010-12-06 20:50:09 +00002109 * This function makes lazy skb cloning in hope that most of packets
2110 * are discarded by BPF.
2111 *
2112 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
2113 * and skb->cb are mangled. It works because (and until) packets
2114 * falling here are owned by current CPU. Output packets are cloned
2115 * by dev_queue_xmit_nit(), input packets are processed by net_bh
Wang Hai0e4161d2021-03-24 14:19:31 +08002116 * sequentially, so that if we return skb to original state on exit,
Eric Dumazet62ab0812010-12-06 20:50:09 +00002117 * we will not harm anyone.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002118 */
2119
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002120static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
2121 struct packet_type *pt, struct net_device *orig_dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002122{
2123 struct sock *sk;
2124 struct sockaddr_ll *sll;
2125 struct packet_sock *po;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002126 u8 *skb_head = skb->data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002127 int skb_len = skb->len;
David S. Millerdbcb5852007-01-24 15:21:02 -08002128 unsigned int snaplen, res;
Weongyo Jeongda378452016-04-14 14:10:04 -07002129 bool is_drop_n_account = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002130
2131 if (skb->pkt_type == PACKET_LOOPBACK)
2132 goto drop;
2133
2134 sk = pt->af_packet_priv;
2135 po = pkt_sk(sk);
2136
Octavian Purdila09ad9bc2009-11-25 15:14:13 -08002137 if (!net_eq(dev_net(dev), sock_net(sk)))
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08002138 goto drop;
2139
Linus Torvalds1da177e2005-04-16 15:20:36 -07002140 skb->dev = dev;
2141
Eyal Birgerd5496992020-11-21 08:28:17 +02002142 if (dev_has_header(dev)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002143 /* The device has an explicit notion of ll header,
Eric Dumazet62ab0812010-12-06 20:50:09 +00002144 * exported to higher levels.
2145 *
2146 * Otherwise, the device hides details of its frame
2147 * structure, so that corresponding packet head is
2148 * never delivered to user.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002149 */
2150 if (sk->sk_type != SOCK_DGRAM)
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07002151 skb_push(skb, skb->data - skb_mac_header(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002152 else if (skb->pkt_type == PACKET_OUTGOING) {
2153 /* Special case: outgoing packets have ll header at head */
Arnaldo Carvalho de Melobbe735e2007-03-10 22:16:10 -03002154 skb_pull(skb, skb_network_offset(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002155 }
2156 }
2157
2158 snaplen = skb->len;
2159
David S. Millerdbcb5852007-01-24 15:21:02 -08002160 res = run_filter(skb, sk, snaplen);
2161 if (!res)
Dmitry Mishinfda9ef52006-08-31 15:28:39 -07002162 goto drop_n_restore;
David S. Millerdbcb5852007-01-24 15:21:02 -08002163 if (snaplen > res)
2164 snaplen = res;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002165
Eric Dumazet0fd7bac2011-12-21 07:11:44 +00002166 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002167 goto drop_n_acct;
2168
2169 if (skb_shared(skb)) {
2170 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
2171 if (nskb == NULL)
2172 goto drop_n_acct;
2173
2174 if (skb_head != skb->data) {
2175 skb->data = skb_head;
2176 skb->len = skb_len;
2177 }
Eric Dumazetabc4e4f2012-04-19 02:24:42 +00002178 consume_skb(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002179 skb = nskb;
2180 }
2181
Eyal Birgerb4772ef2015-03-01 14:58:29 +02002182 sock_skb_cb_check_size(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8);
Herbert Xuffbc6112007-02-04 23:33:10 -08002183
2184 sll = &PACKET_SKB_CB(skb)->sa.ll;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002185 sll->sll_hatype = dev->type;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002186 sll->sll_pkttype = skb->pkt_type;
Peter P Waskiewicz Jr8032b462007-11-10 22:03:25 -08002187 if (unlikely(po->origdev))
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07002188 sll->sll_ifindex = orig_dev->ifindex;
2189 else
2190 sll->sll_ifindex = dev->ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002191
Stephen Hemmingerb95cce32007-09-26 22:13:38 -07002192 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002193
Eyal Birger2472d762015-03-01 14:58:28 +02002194 /* sll->sll_family and sll->sll_protocol are set in packet_recvmsg().
2195 * Use their space for storing the original skb length.
2196 */
2197 PACKET_SKB_CB(skb)->sa.origlen = skb->len;
Herbert Xu8dc41942007-02-04 23:31:32 -08002198
Linus Torvalds1da177e2005-04-16 15:20:36 -07002199 if (pskb_trim(skb, snaplen))
2200 goto drop_n_acct;
2201
2202 skb_set_owner_r(skb, sk);
2203 skb->dev = NULL;
Eric Dumazetadf30902009-06-02 05:19:30 +00002204 skb_dst_drop(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002205
Phil Oester84531c22005-07-12 11:57:52 -07002206 /* drop conntrack reference */
Florian Westphal895b5c92019-09-29 20:54:03 +02002207 nf_reset_ct(skb);
Phil Oester84531c22005-07-12 11:57:52 -07002208
Linus Torvalds1da177e2005-04-16 15:20:36 -07002209 spin_lock(&sk->sk_receive_queue.lock);
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00002210 po->stats.stats1.tp_packets++;
Eyal Birger3bc3b962015-03-01 14:58:30 +02002211 sock_skb_set_dropcount(sk, skb);
Martin KaFai Lau27942a12022-03-02 11:55:38 -08002212 skb_clear_delivery_time(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002213 __skb_queue_tail(&sk->sk_receive_queue, skb);
2214 spin_unlock(&sk->sk_receive_queue.lock);
David S. Miller676d2362014-04-11 16:15:36 -04002215 sk->sk_data_ready(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002216 return 0;
2217
2218drop_n_acct:
Weongyo Jeongda378452016-04-14 14:10:04 -07002219 is_drop_n_account = true;
Eric Dumazet8e8e2952019-06-12 09:52:30 -07002220 atomic_inc(&po->tp_drops);
Willem de Bruijn7091fbd2011-09-30 10:38:28 +00002221 atomic_inc(&sk->sk_drops);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002222
2223drop_n_restore:
2224 if (skb_head != skb->data && skb_shared(skb)) {
2225 skb->data = skb_head;
2226 skb->len = skb_len;
2227 }
2228drop:
Weongyo Jeongda378452016-04-14 14:10:04 -07002229 if (!is_drop_n_account)
2230 consume_skb(skb);
2231 else
2232 kfree_skb(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002233 return 0;
2234}
2235
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002236static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
2237 struct packet_type *pt, struct net_device *orig_dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002238{
2239 struct sock *sk;
2240 struct packet_sock *po;
2241 struct sockaddr_ll *sll;
Daniel Borkmann184f4892013-04-16 01:57:46 +00002242 union tpacket_uhdr h;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002243 u8 *skb_head = skb->data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002244 int skb_len = skb->len;
David S. Millerdbcb5852007-01-24 15:21:02 -08002245 unsigned int snaplen, res;
chetan lokef6fb8f12011-08-19 10:18:16 +00002246 unsigned long status = TP_STATUS_USER;
Or Cohenacf69c92020-09-03 21:05:28 -07002247 unsigned short macoff, hdrlen;
2248 unsigned int netoff;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002249 struct sk_buff *copy_skb = NULL;
Arnd Bergmannd413fcb2017-11-27 10:09:24 +01002250 struct timespec64 ts;
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +00002251 __u32 ts_status;
Weongyo Jeongda378452016-04-14 14:10:04 -07002252 bool is_drop_n_account = false;
Willem de Bruijn61fad682020-03-13 12:18:09 -04002253 unsigned int slot_id = 0;
Benjamin Poirieredbd58b2017-08-28 14:29:41 -04002254 bool do_vnet = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002255
Atzm Watanabe51846352013-12-17 22:53:32 +09002256 /* struct tpacket{2,3}_hdr is aligned to a multiple of TPACKET_ALIGNMENT.
2257 * We may add members to them until current aligned size without forcing
2258 * userspace to call getsockopt(..., PACKET_HDRLEN, ...).
2259 */
2260 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h2)) != 32);
2261 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h3)) != 48);
2262
Linus Torvalds1da177e2005-04-16 15:20:36 -07002263 if (skb->pkt_type == PACKET_LOOPBACK)
2264 goto drop;
2265
2266 sk = pt->af_packet_priv;
2267 po = pkt_sk(sk);
2268
Octavian Purdila09ad9bc2009-11-25 15:14:13 -08002269 if (!net_eq(dev_net(dev), sock_net(sk)))
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08002270 goto drop;
2271
Eyal Birgerd5496992020-11-21 08:28:17 +02002272 if (dev_has_header(dev)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002273 if (sk->sk_type != SOCK_DGRAM)
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07002274 skb_push(skb, skb->data - skb_mac_header(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002275 else if (skb->pkt_type == PACKET_OUTGOING) {
2276 /* Special case: outgoing packets have ll header at head */
Arnaldo Carvalho de Melobbe735e2007-03-10 22:16:10 -03002277 skb_pull(skb, skb_network_offset(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002278 }
2279 }
2280
2281 snaplen = skb->len;
2282
David S. Millerdbcb5852007-01-24 15:21:02 -08002283 res = run_filter(skb, sk, snaplen);
2284 if (!res)
Dmitry Mishinfda9ef52006-08-31 15:28:39 -07002285 goto drop_n_restore;
Alexander Drozdov68c2e5d2015-03-23 09:11:12 +03002286
Eric Dumazet2c51c622019-06-12 09:52:31 -07002287 /* If we are flooded, just give up */
2288 if (__packet_rcv_has_room(po, skb) == ROOM_NONE) {
2289 atomic_inc(&po->tp_drops);
2290 goto drop_n_restore;
2291 }
2292
Alexander Drozdov68c2e5d2015-03-23 09:11:12 +03002293 if (skb->ip_summed == CHECKSUM_PARTIAL)
2294 status |= TP_STATUS_CSUMNOTREADY;
Alexander Drozdov682f0482015-03-23 09:11:13 +03002295 else if (skb->pkt_type != PACKET_OUTGOING &&
2296 (skb->ip_summed == CHECKSUM_COMPLETE ||
2297 skb_csum_unnecessary(skb)))
2298 status |= TP_STATUS_CSUM_VALID;
Alexander Drozdov68c2e5d2015-03-23 09:11:12 +03002299
David S. Millerdbcb5852007-01-24 15:21:02 -08002300 if (snaplen > res)
2301 snaplen = res;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002302
2303 if (sk->sk_type == SOCK_DGRAM) {
Patrick McHardy8913336a2008-07-18 18:05:19 -07002304 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
2305 po->tp_reserve;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002306 } else {
Eric Dumazet95c96172012-04-15 05:58:06 +00002307 unsigned int maclen = skb_network_offset(skb);
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002308 netoff = TPACKET_ALIGN(po->tp_hdrlen +
Patrick McHardy8913336a2008-07-18 18:05:19 -07002309 (maclen < 16 ? 16 : maclen)) +
Willem de Bruijn58d19b12016-02-03 18:02:15 -05002310 po->tp_reserve;
Benjamin Poirieredbd58b2017-08-28 14:29:41 -04002311 if (po->has_vnet_hdr) {
Willem de Bruijn58d19b12016-02-03 18:02:15 -05002312 netoff += sizeof(struct virtio_net_hdr);
Benjamin Poirieredbd58b2017-08-28 14:29:41 -04002313 do_vnet = true;
2314 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002315 macoff = netoff - maclen;
2316 }
Or Cohenacf69c92020-09-03 21:05:28 -07002317 if (netoff > USHRT_MAX) {
2318 atomic_inc(&po->tp_drops);
2319 goto drop_n_restore;
2320 }
chetan lokef6fb8f12011-08-19 10:18:16 +00002321 if (po->tp_version <= TPACKET_V2) {
2322 if (macoff + snaplen > po->rx_ring.frame_size) {
2323 if (po->copy_thresh &&
Eric Dumazet0fd7bac2011-12-21 07:11:44 +00002324 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
chetan lokef6fb8f12011-08-19 10:18:16 +00002325 if (skb_shared(skb)) {
2326 copy_skb = skb_clone(skb, GFP_ATOMIC);
2327 } else {
2328 copy_skb = skb_get(skb);
2329 skb_head = skb->data;
2330 }
Eric Dumazetc7005252022-03-12 15:29:58 -08002331 if (copy_skb) {
2332 memset(&PACKET_SKB_CB(copy_skb)->sa.ll, 0,
2333 sizeof(PACKET_SKB_CB(copy_skb)->sa.ll));
chetan lokef6fb8f12011-08-19 10:18:16 +00002334 skb_set_owner_r(copy_skb, sk);
Eric Dumazetc7005252022-03-12 15:29:58 -08002335 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002336 }
chetan lokef6fb8f12011-08-19 10:18:16 +00002337 snaplen = po->rx_ring.frame_size - macoff;
Benjamin Poirieredbd58b2017-08-28 14:29:41 -04002338 if ((int)snaplen < 0) {
chetan lokef6fb8f12011-08-19 10:18:16 +00002339 snaplen = 0;
Benjamin Poirieredbd58b2017-08-28 14:29:41 -04002340 do_vnet = false;
2341 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002342 }
Eric Dumazetdc808112014-08-15 09:16:04 -07002343 } else if (unlikely(macoff + snaplen >
2344 GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len)) {
2345 u32 nval;
2346
2347 nval = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len - macoff;
2348 pr_err_once("tpacket_rcv: packet too big, clamped from %u to %u. macoff=%u\n",
2349 snaplen, nval, macoff);
2350 snaplen = nval;
2351 if (unlikely((int)snaplen < 0)) {
2352 snaplen = 0;
2353 macoff = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len;
Benjamin Poirieredbd58b2017-08-28 14:29:41 -04002354 do_vnet = false;
Eric Dumazetdc808112014-08-15 09:16:04 -07002355 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002356 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002357 spin_lock(&sk->sk_receive_queue.lock);
chetan lokef6fb8f12011-08-19 10:18:16 +00002358 h.raw = packet_current_rx_frame(po, skb,
2359 TP_STATUS_KERNEL, (macoff+snaplen));
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002360 if (!h.raw)
Willem de Bruijn58d19b12016-02-03 18:02:15 -05002361 goto drop_n_account;
Willem de Bruijn46e4c422020-03-09 11:34:35 -04002362
Willem de Bruijn61fad682020-03-13 12:18:09 -04002363 if (po->tp_version <= TPACKET_V2) {
2364 slot_id = po->rx_ring.head;
2365 if (test_bit(slot_id, po->rx_ring.rx_owner_map))
2366 goto drop_n_account;
2367 __set_bit(slot_id, po->rx_ring.rx_owner_map);
2368 }
2369
Willem de Bruijn46e4c422020-03-09 11:34:35 -04002370 if (do_vnet &&
2371 virtio_net_hdr_from_skb(skb, h.raw + macoff -
2372 sizeof(struct virtio_net_hdr),
John Ogness88fd1cb2020-08-13 21:45:25 +02062373 vio_le(), true, 0)) {
2374 if (po->tp_version == TPACKET_V3)
2375 prb_clear_blk_fill_status(&po->rx_ring);
Willem de Bruijn46e4c422020-03-09 11:34:35 -04002376 goto drop_n_account;
John Ogness88fd1cb2020-08-13 21:45:25 +02062377 }
Willem de Bruijn46e4c422020-03-09 11:34:35 -04002378
chetan lokef6fb8f12011-08-19 10:18:16 +00002379 if (po->tp_version <= TPACKET_V2) {
2380 packet_increment_rx_head(po, &po->rx_ring);
2381 /*
2382 * LOSING will be reported till you read the stats,
2383 * because it's COR - Clear On Read.
2384 * Anyways, moving it for V1/V2 only as V3 doesn't need this
2385 * at packet level.
2386 */
Eric Dumazet8e8e2952019-06-12 09:52:30 -07002387 if (atomic_read(&po->tp_drops))
chetan lokef6fb8f12011-08-19 10:18:16 +00002388 status |= TP_STATUS_LOSING;
2389 }
Eric Dumazet945d0152018-06-21 14:16:02 -07002390
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00002391 po->stats.stats1.tp_packets++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002392 if (copy_skb) {
2393 status |= TP_STATUS_COPY;
Martin KaFai Lau27942a12022-03-02 11:55:38 -08002394 skb_clear_delivery_time(copy_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002395 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
2396 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002397 spin_unlock(&sk->sk_receive_queue.lock);
2398
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002399 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +00002400
Richard Sanger171c3b12021-05-12 13:31:22 +12002401 /* Always timestamp; prefer an existing software timestamp taken
2402 * closer to the time of capture.
2403 */
2404 ts_status = tpacket_get_timestamp(skb, &ts,
2405 po->tp_tstamp | SOF_TIMESTAMPING_SOFTWARE);
2406 if (!ts_status)
Arnd Bergmannd413fcb2017-11-27 10:09:24 +01002407 ktime_get_real_ts64(&ts);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002408
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +00002409 status |= ts_status;
2410
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002411 switch (po->tp_version) {
2412 case TPACKET_V1:
2413 h.h1->tp_len = skb->len;
2414 h.h1->tp_snaplen = snaplen;
2415 h.h1->tp_mac = macoff;
2416 h.h1->tp_net = netoff;
Daniel Borkmann4b457bd2013-04-16 01:29:11 +00002417 h.h1->tp_sec = ts.tv_sec;
2418 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002419 hdrlen = sizeof(*h.h1);
2420 break;
2421 case TPACKET_V2:
2422 h.h2->tp_len = skb->len;
2423 h.h2->tp_snaplen = snaplen;
2424 h.h2->tp_mac = macoff;
2425 h.h2->tp_net = netoff;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002426 h.h2->tp_sec = ts.tv_sec;
2427 h.h2->tp_nsec = ts.tv_nsec;
Jiri Pirkodf8a39de2015-01-13 17:13:44 +01002428 if (skb_vlan_tag_present(skb)) {
2429 h.h2->tp_vlan_tci = skb_vlan_tag_get(skb);
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +09002430 h.h2->tp_vlan_tpid = ntohs(skb->vlan_proto);
2431 status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
Ben Greeara3bcc232011-06-01 06:49:10 +00002432 } else {
2433 h.h2->tp_vlan_tci = 0;
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +09002434 h.h2->tp_vlan_tpid = 0;
Ben Greeara3bcc232011-06-01 06:49:10 +00002435 }
Atzm Watanabee4d26f42013-12-17 22:53:36 +09002436 memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding));
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002437 hdrlen = sizeof(*h.h2);
2438 break;
chetan lokef6fb8f12011-08-19 10:18:16 +00002439 case TPACKET_V3:
2440 /* tp_nxt_offset,vlan are already populated above.
2441 * So DONT clear those fields here
2442 */
2443 h.h3->tp_status |= status;
2444 h.h3->tp_len = skb->len;
2445 h.h3->tp_snaplen = snaplen;
2446 h.h3->tp_mac = macoff;
2447 h.h3->tp_net = netoff;
chetan lokef6fb8f12011-08-19 10:18:16 +00002448 h.h3->tp_sec = ts.tv_sec;
2449 h.h3->tp_nsec = ts.tv_nsec;
Atzm Watanabee4d26f42013-12-17 22:53:36 +09002450 memset(h.h3->tp_padding, 0, sizeof(h.h3->tp_padding));
chetan lokef6fb8f12011-08-19 10:18:16 +00002451 hdrlen = sizeof(*h.h3);
2452 break;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002453 default:
2454 BUG();
2455 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002456
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002457 sll = h.raw + TPACKET_ALIGN(hdrlen);
Stephen Hemmingerb95cce32007-09-26 22:13:38 -07002458 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002459 sll->sll_family = AF_PACKET;
2460 sll->sll_hatype = dev->type;
2461 sll->sll_protocol = skb->protocol;
2462 sll->sll_pkttype = skb->pkt_type;
Peter P Waskiewicz Jr8032b462007-11-10 22:03:25 -08002463 if (unlikely(po->origdev))
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07002464 sll->sll_ifindex = orig_dev->ifindex;
2465 else
2466 sll->sll_ifindex = dev->ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002467
Ralf Baechlee16aa202006-12-07 00:11:33 -08002468 smp_mb();
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002469
Changli Gaof6dafa92010-12-07 04:26:16 +00002470#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002471 if (po->tp_version <= TPACKET_V2) {
Changli Gao0af55bb2010-12-01 02:52:20 +00002472 u8 *start, *end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002473
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002474 end = (u8 *) PAGE_ALIGN((unsigned long) h.raw +
2475 macoff + snaplen);
2476
2477 for (start = h.raw; start < end; start += PAGE_SIZE)
2478 flush_dcache_page(pgv_to_page(start));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002479 }
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002480 smp_wmb();
Changli Gaof6dafa92010-12-07 04:26:16 +00002481#endif
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002482
Dan Collinsda413ee2014-12-19 16:49:25 +13002483 if (po->tp_version <= TPACKET_V2) {
Willem de Bruijn61fad682020-03-13 12:18:09 -04002484 spin_lock(&sk->sk_receive_queue.lock);
chetan lokef6fb8f12011-08-19 10:18:16 +00002485 __packet_set_status(po, h.raw, status);
Willem de Bruijn61fad682020-03-13 12:18:09 -04002486 __clear_bit(slot_id, po->rx_ring.rx_owner_map);
2487 spin_unlock(&sk->sk_receive_queue.lock);
Dan Collinsda413ee2014-12-19 16:49:25 +13002488 sk->sk_data_ready(sk);
John Ogness88fd1cb2020-08-13 21:45:25 +02062489 } else if (po->tp_version == TPACKET_V3) {
chetan lokef6fb8f12011-08-19 10:18:16 +00002490 prb_clear_blk_fill_status(&po->rx_ring);
Dan Collinsda413ee2014-12-19 16:49:25 +13002491 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002492
2493drop_n_restore:
2494 if (skb_head != skb->data && skb_shared(skb)) {
2495 skb->data = skb_head;
2496 skb->len = skb_len;
2497 }
2498drop:
Weongyo Jeongda378452016-04-14 14:10:04 -07002499 if (!is_drop_n_account)
2500 consume_skb(skb);
2501 else
2502 kfree_skb(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002503 return 0;
2504
Willem de Bruijn58d19b12016-02-03 18:02:15 -05002505drop_n_account:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002506 spin_unlock(&sk->sk_receive_queue.lock);
Eric Dumazet8e8e2952019-06-12 09:52:30 -07002507 atomic_inc(&po->tp_drops);
2508 is_drop_n_account = true;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002509
David S. Miller676d2362014-04-11 16:15:36 -04002510 sk->sk_data_ready(sk);
Wei Yongjunacb5d752009-02-25 00:36:42 +00002511 kfree_skb(copy_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002512 goto drop_n_restore;
2513}
2514
Johann Baudy69e3c752009-05-18 22:11:22 -07002515static void tpacket_destruct_skb(struct sk_buff *skb)
2516{
2517 struct packet_sock *po = pkt_sk(skb->sk);
Johann Baudy69e3c752009-05-18 22:11:22 -07002518
Johann Baudy69e3c752009-05-18 22:11:22 -07002519 if (likely(po->tx_ring.pg_vec)) {
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002520 void *ph;
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +00002521 __u32 ts;
2522
Willem de Bruijn5cd8d462018-11-20 13:00:18 -05002523 ph = skb_zcopy_get_nouarg(skb);
Daniel Borkmannb0138402014-01-15 16:25:36 +01002524 packet_dec_pending(&po->tx_ring);
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +00002525
2526 ts = __packet_set_timestamp(po, ph, skb);
2527 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
Neil Horman89ed5b52019-06-25 17:57:49 -04002528
2529 if (!packet_read_pending(&po->tx_ring))
2530 complete(&po->skb_completion);
Johann Baudy69e3c752009-05-18 22:11:22 -07002531 }
2532
2533 sock_wfree(skb);
2534}
2535
Willem de Bruijn16cc1402016-02-03 18:02:14 -05002536static int __packet_snd_vnet_parse(struct virtio_net_hdr *vnet_hdr, size_t len)
2537{
Willem de Bruijn16cc1402016-02-03 18:02:14 -05002538 if ((vnet_hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2539 (__virtio16_to_cpu(vio_le(), vnet_hdr->csum_start) +
2540 __virtio16_to_cpu(vio_le(), vnet_hdr->csum_offset) + 2 >
2541 __virtio16_to_cpu(vio_le(), vnet_hdr->hdr_len)))
2542 vnet_hdr->hdr_len = __cpu_to_virtio16(vio_le(),
2543 __virtio16_to_cpu(vio_le(), vnet_hdr->csum_start) +
2544 __virtio16_to_cpu(vio_le(), vnet_hdr->csum_offset) + 2);
2545
2546 if (__virtio16_to_cpu(vio_le(), vnet_hdr->hdr_len) > len)
2547 return -EINVAL;
2548
Willem de Bruijn16cc1402016-02-03 18:02:14 -05002549 return 0;
2550}
2551
2552static int packet_snd_vnet_parse(struct msghdr *msg, size_t *len,
2553 struct virtio_net_hdr *vnet_hdr)
2554{
Willem de Bruijn16cc1402016-02-03 18:02:14 -05002555 if (*len < sizeof(*vnet_hdr))
2556 return -EINVAL;
2557 *len -= sizeof(*vnet_hdr);
2558
Al Virocbbd26b2016-11-01 22:09:04 -04002559 if (!copy_from_iter_full(vnet_hdr, sizeof(*vnet_hdr), &msg->msg_iter))
Willem de Bruijn16cc1402016-02-03 18:02:14 -05002560 return -EFAULT;
2561
2562 return __packet_snd_vnet_parse(vnet_hdr, *len);
2563}
2564
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002565static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
Willem de Bruijn8d39b4a2016-02-03 18:02:16 -05002566 void *frame, struct net_device *dev, void *data, int tp_len,
Soheil Hassas Yeganehc14ac942016-04-02 23:08:12 -04002567 __be16 proto, unsigned char *addr, int hlen, int copylen,
2568 const struct sockcm_cookie *sockc)
Johann Baudy69e3c752009-05-18 22:11:22 -07002569{
Daniel Borkmann184f4892013-04-16 01:57:46 +00002570 union tpacket_uhdr ph;
Willem de Bruijn8d39b4a2016-02-03 18:02:16 -05002571 int to_write, offset, len, nr_frags, len_max;
Johann Baudy69e3c752009-05-18 22:11:22 -07002572 struct socket *sock = po->sk.sk_socket;
2573 struct page *page;
Johann Baudy69e3c752009-05-18 22:11:22 -07002574 int err;
2575
2576 ph.raw = frame;
2577
2578 skb->protocol = proto;
2579 skb->dev = dev;
2580 skb->priority = po->sk.sk_priority;
Eric Dumazet2d37a182009-10-01 19:14:46 +00002581 skb->mark = po->sk.sk_mark;
Richard Cochran3d0ba8c2018-07-03 15:42:51 -07002582 skb->tstamp = sockc->transmit_time;
Willem de Bruijn8f932f72018-12-17 12:24:00 -05002583 skb_setup_tx_timestamp(skb, sockc->tsflags);
Willem de Bruijn5cd8d462018-11-20 13:00:18 -05002584 skb_zcopy_set_nouarg(skb, ph.raw);
Johann Baudy69e3c752009-05-18 22:11:22 -07002585
Herbert Xuae641942011-11-18 02:20:04 +00002586 skb_reserve(skb, hlen);
Johann Baudy69e3c752009-05-18 22:11:22 -07002587 skb_reset_network_header(skb);
Jason Wangc1aad272013-03-25 20:19:57 +00002588
Johann Baudy69e3c752009-05-18 22:11:22 -07002589 to_write = tp_len;
2590
2591 if (sock->type == SOCK_DGRAM) {
2592 err = dev_hard_header(skb, dev, ntohs(proto), addr,
2593 NULL, tp_len);
2594 if (unlikely(err < 0))
2595 return -EINVAL;
Willem de Bruijn1d036d22016-02-03 18:02:17 -05002596 } else if (copylen) {
Willem de Bruijn9ed988c2016-03-09 21:58:34 -05002597 int hdrlen = min_t(int, copylen, tp_len);
2598
Johann Baudy69e3c752009-05-18 22:11:22 -07002599 skb_push(skb, dev->hard_header_len);
Willem de Bruijn1d036d22016-02-03 18:02:17 -05002600 skb_put(skb, copylen - dev->hard_header_len);
Willem de Bruijn9ed988c2016-03-09 21:58:34 -05002601 err = skb_store_bits(skb, 0, data, hdrlen);
Johann Baudy69e3c752009-05-18 22:11:22 -07002602 if (unlikely(err))
2603 return err;
Willem de Bruijn9ed988c2016-03-09 21:58:34 -05002604 if (!dev_validate_header(dev, skb->data, hdrlen))
2605 return -EINVAL;
Johann Baudy69e3c752009-05-18 22:11:22 -07002606
Willem de Bruijn9ed988c2016-03-09 21:58:34 -05002607 data += hdrlen;
2608 to_write -= hdrlen;
Johann Baudy69e3c752009-05-18 22:11:22 -07002609 }
2610
Johann Baudy69e3c752009-05-18 22:11:22 -07002611 offset = offset_in_page(data);
2612 len_max = PAGE_SIZE - offset;
2613 len = ((to_write > len_max) ? len_max : to_write);
2614
2615 skb->data_len = to_write;
2616 skb->len += to_write;
2617 skb->truesize += to_write;
Reshetova, Elena14afee42017-06-30 13:08:00 +03002618 refcount_add(to_write, &po->sk.sk_wmem_alloc);
Johann Baudy69e3c752009-05-18 22:11:22 -07002619
2620 while (likely(to_write)) {
2621 nr_frags = skb_shinfo(skb)->nr_frags;
2622
2623 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002624 pr_err("Packet exceed the number of skb frags(%lu)\n",
2625 MAX_SKB_FRAGS);
Johann Baudy69e3c752009-05-18 22:11:22 -07002626 return -EFAULT;
2627 }
2628
Changli Gao0af55bb2010-12-01 02:52:20 +00002629 page = pgv_to_page(data);
2630 data += len;
Johann Baudy69e3c752009-05-18 22:11:22 -07002631 flush_dcache_page(page);
2632 get_page(page);
Changli Gao0af55bb2010-12-01 02:52:20 +00002633 skb_fill_page_desc(skb, nr_frags, page, offset, len);
Johann Baudy69e3c752009-05-18 22:11:22 -07002634 to_write -= len;
2635 offset = 0;
2636 len_max = PAGE_SIZE;
2637 len = ((to_write > len_max) ? len_max : to_write);
2638 }
2639
Maxim Mikityanskiy75c65772019-02-21 12:40:01 +00002640 packet_parse_headers(skb, sock);
Daniel Borkmannefdfa2f2015-11-11 23:25:40 +01002641
Johann Baudy69e3c752009-05-18 22:11:22 -07002642 return tp_len;
2643}
2644
Willem de Bruijn8d39b4a2016-02-03 18:02:16 -05002645static int tpacket_parse_header(struct packet_sock *po, void *frame,
2646 int size_max, void **data)
2647{
2648 union tpacket_uhdr ph;
2649 int tp_len, off;
2650
2651 ph.raw = frame;
2652
2653 switch (po->tp_version) {
Sowmini Varadhan7f953ab2017-01-03 06:31:47 -08002654 case TPACKET_V3:
2655 if (ph.h3->tp_next_offset != 0) {
2656 pr_warn_once("variable sized slot not supported");
2657 return -EINVAL;
2658 }
2659 tp_len = ph.h3->tp_len;
2660 break;
Willem de Bruijn8d39b4a2016-02-03 18:02:16 -05002661 case TPACKET_V2:
2662 tp_len = ph.h2->tp_len;
2663 break;
2664 default:
2665 tp_len = ph.h1->tp_len;
2666 break;
2667 }
2668 if (unlikely(tp_len > size_max)) {
2669 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
2670 return -EMSGSIZE;
2671 }
2672
2673 if (unlikely(po->tp_tx_has_off)) {
2674 int off_min, off_max;
2675
2676 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2677 off_max = po->tx_ring.frame_size - tp_len;
2678 if (po->sk.sk_type == SOCK_DGRAM) {
2679 switch (po->tp_version) {
Sowmini Varadhan7f953ab2017-01-03 06:31:47 -08002680 case TPACKET_V3:
2681 off = ph.h3->tp_net;
2682 break;
Willem de Bruijn8d39b4a2016-02-03 18:02:16 -05002683 case TPACKET_V2:
2684 off = ph.h2->tp_net;
2685 break;
2686 default:
2687 off = ph.h1->tp_net;
2688 break;
2689 }
2690 } else {
2691 switch (po->tp_version) {
Sowmini Varadhan7f953ab2017-01-03 06:31:47 -08002692 case TPACKET_V3:
2693 off = ph.h3->tp_mac;
2694 break;
Willem de Bruijn8d39b4a2016-02-03 18:02:16 -05002695 case TPACKET_V2:
2696 off = ph.h2->tp_mac;
2697 break;
2698 default:
2699 off = ph.h1->tp_mac;
2700 break;
2701 }
2702 }
2703 if (unlikely((off < off_min) || (off_max < off)))
2704 return -EINVAL;
2705 } else {
2706 off = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2707 }
2708
2709 *data = frame + off;
2710 return tp_len;
2711}
2712
Johann Baudy69e3c752009-05-18 22:11:22 -07002713static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2714{
Neil Horman89ed5b52019-06-25 17:57:49 -04002715 struct sk_buff *skb = NULL;
Johann Baudy69e3c752009-05-18 22:11:22 -07002716 struct net_device *dev;
Willem de Bruijn1d036d22016-02-03 18:02:17 -05002717 struct virtio_net_hdr *vnet_hdr = NULL;
Soheil Hassas Yeganehc14ac942016-04-02 23:08:12 -04002718 struct sockcm_cookie sockc;
Johann Baudy69e3c752009-05-18 22:11:22 -07002719 __be16 proto;
David S. Miller09effa62013-08-07 17:11:00 -07002720 int err, reserve = 0;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002721 void *ph;
Steffen Hurrle342dfc32014-01-17 22:53:15 +01002722 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
Daniel Borkmann87a2fd22014-01-15 16:25:35 +01002723 bool need_wait = !(msg->msg_flags & MSG_DONTWAIT);
Willem de Bruijn486efdc2019-04-29 11:53:18 -04002724 unsigned char *addr = NULL;
Johann Baudy69e3c752009-05-18 22:11:22 -07002725 int tp_len, size_max;
Willem de Bruijn8d39b4a2016-02-03 18:02:16 -05002726 void *data;
Johann Baudy69e3c752009-05-18 22:11:22 -07002727 int len_sum = 0;
danborkmann@iogearbox.net9e670302012-08-20 03:34:03 +00002728 int status = TP_STATUS_AVAILABLE;
Willem de Bruijn1d036d22016-02-03 18:02:17 -05002729 int hlen, tlen, copylen = 0;
Neil Horman89ed5b52019-06-25 17:57:49 -04002730 long timeo = 0;
Johann Baudy69e3c752009-05-18 22:11:22 -07002731
Johann Baudy69e3c752009-05-18 22:11:22 -07002732 mutex_lock(&po->pg_vec_lock);
2733
Eric Dumazet32d31822019-08-14 02:11:57 -07002734 /* packet_sendmsg() check on tx_ring.pg_vec was lockless,
2735 * we need to confirm it under protection of pg_vec_lock.
2736 */
2737 if (unlikely(!po->tx_ring.pg_vec)) {
2738 err = -EBUSY;
2739 goto out;
2740 }
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01002741 if (likely(saddr == NULL)) {
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002742 dev = packet_cached_dev_get(po);
Eric Dumazetc7d2ef52021-06-16 06:42:01 -07002743 proto = READ_ONCE(po->num);
Johann Baudy69e3c752009-05-18 22:11:22 -07002744 } else {
2745 err = -EINVAL;
2746 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2747 goto out;
2748 if (msg->msg_namelen < (saddr->sll_halen
2749 + offsetof(struct sockaddr_ll,
2750 sll_addr)))
2751 goto out;
Johann Baudy69e3c752009-05-18 22:11:22 -07002752 proto = saddr->sll_protocol;
Ben Greear827d9782011-06-01 07:18:53 +00002753 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
Willem de Bruijn486efdc2019-04-29 11:53:18 -04002754 if (po->sk.sk_socket->type == SOCK_DGRAM) {
2755 if (dev && msg->msg_namelen < dev->addr_len +
2756 offsetof(struct sockaddr_ll, sll_addr))
2757 goto out_put;
2758 addr = saddr->sll_addr;
2759 }
Johann Baudy69e3c752009-05-18 22:11:22 -07002760 }
2761
Johann Baudy69e3c752009-05-18 22:11:22 -07002762 err = -ENXIO;
2763 if (unlikely(dev == NULL))
2764 goto out;
Johann Baudy69e3c752009-05-18 22:11:22 -07002765 err = -ENETDOWN;
2766 if (unlikely(!(dev->flags & IFF_UP)))
2767 goto out_put;
2768
Willem de Bruijn657a0662018-07-06 10:12:56 -04002769 sockcm_init(&sockc, &po->sk);
Douglas Caetano dos Santosd19b1832017-05-12 15:19:15 -03002770 if (msg->msg_controllen) {
2771 err = sock_cmsg_send(&po->sk, msg, &sockc);
2772 if (unlikely(err))
2773 goto out_put;
2774 }
2775
Daniel Borkmann5cfb4c82015-11-11 23:25:44 +01002776 if (po->sk.sk_socket->type == SOCK_RAW)
2777 reserve = dev->hard_header_len;
Johann Baudy69e3c752009-05-18 22:11:22 -07002778 size_max = po->tx_ring.frame_size
Gabor Gombasb5dd8842009-10-29 03:19:11 -07002779 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
Johann Baudy69e3c752009-05-18 22:11:22 -07002780
Willem de Bruijn1d036d22016-02-03 18:02:17 -05002781 if ((size_max > dev->mtu + reserve + VLAN_HLEN) && !po->has_vnet_hdr)
Daniel Borkmann5cfb4c82015-11-11 23:25:44 +01002782 size_max = dev->mtu + reserve + VLAN_HLEN;
David S. Miller09effa62013-08-07 17:11:00 -07002783
Neil Horman89ed5b52019-06-25 17:57:49 -04002784 reinit_completion(&po->skb_completion);
2785
Johann Baudy69e3c752009-05-18 22:11:22 -07002786 do {
2787 ph = packet_current_frame(po, &po->tx_ring,
Daniel Borkmann87a2fd22014-01-15 16:25:35 +01002788 TP_STATUS_SEND_REQUEST);
Johann Baudy69e3c752009-05-18 22:11:22 -07002789 if (unlikely(ph == NULL)) {
Neil Horman89ed5b52019-06-25 17:57:49 -04002790 if (need_wait && skb) {
2791 timeo = sock_sndtimeo(&po->sk, msg->msg_flags & MSG_DONTWAIT);
2792 timeo = wait_for_completion_interruptible_timeout(&po->skb_completion, timeo);
2793 if (timeo <= 0) {
2794 err = !timeo ? -ETIMEDOUT : -ERESTARTSYS;
2795 goto out_put;
2796 }
2797 }
2798 /* check for additional frames */
Johann Baudy69e3c752009-05-18 22:11:22 -07002799 continue;
2800 }
2801
Willem de Bruijn8d39b4a2016-02-03 18:02:16 -05002802 skb = NULL;
2803 tp_len = tpacket_parse_header(po, ph, size_max, &data);
2804 if (tp_len < 0)
2805 goto tpacket_error;
2806
Johann Baudy69e3c752009-05-18 22:11:22 -07002807 status = TP_STATUS_SEND_REQUEST;
Herbert Xuae641942011-11-18 02:20:04 +00002808 hlen = LL_RESERVED_SPACE(dev);
2809 tlen = dev->needed_tailroom;
Willem de Bruijn1d036d22016-02-03 18:02:17 -05002810 if (po->has_vnet_hdr) {
2811 vnet_hdr = data;
2812 data += sizeof(*vnet_hdr);
2813 tp_len -= sizeof(*vnet_hdr);
2814 if (tp_len < 0 ||
2815 __packet_snd_vnet_parse(vnet_hdr, tp_len)) {
2816 tp_len = -EINVAL;
2817 goto tpacket_error;
2818 }
2819 copylen = __virtio16_to_cpu(vio_le(),
2820 vnet_hdr->hdr_len);
2821 }
Willem de Bruijn9ed988c2016-03-09 21:58:34 -05002822 copylen = max_t(int, copylen, dev->hard_header_len);
Johann Baudy69e3c752009-05-18 22:11:22 -07002823 skb = sock_alloc_send_skb(&po->sk,
Willem de Bruijn1d036d22016-02-03 18:02:17 -05002824 hlen + tlen + sizeof(struct sockaddr_ll) +
2825 (copylen - dev->hard_header_len),
Kretschmer, Mathiasfbf33a22015-05-08 15:44:37 +02002826 !need_wait, &err);
Johann Baudy69e3c752009-05-18 22:11:22 -07002827
Kretschmer, Mathiasfbf33a22015-05-08 15:44:37 +02002828 if (unlikely(skb == NULL)) {
2829 /* we assume the socket was initially writeable ... */
2830 if (likely(len_sum > 0))
2831 err = len_sum;
Johann Baudy69e3c752009-05-18 22:11:22 -07002832 goto out_status;
Kretschmer, Mathiasfbf33a22015-05-08 15:44:37 +02002833 }
Willem de Bruijn8d39b4a2016-02-03 18:02:16 -05002834 tp_len = tpacket_fill_skb(po, skb, ph, dev, data, tp_len, proto,
Soheil Hassas Yeganehc14ac942016-04-02 23:08:12 -04002835 addr, hlen, copylen, &sockc);
Alexander Drozdovdbd46ab2015-07-28 13:57:01 +03002836 if (likely(tp_len >= 0) &&
Daniel Borkmann5cfb4c82015-11-11 23:25:44 +01002837 tp_len > dev->mtu + reserve &&
Willem de Bruijn1d036d22016-02-03 18:02:17 -05002838 !po->has_vnet_hdr &&
Daniel Borkmann3c70c132015-11-11 23:25:42 +01002839 !packet_extra_vlan_len_allowed(dev, skb))
2840 tp_len = -EMSGSIZE;
Johann Baudy69e3c752009-05-18 22:11:22 -07002841
2842 if (unlikely(tp_len < 0)) {
Willem de Bruijn8d39b4a2016-02-03 18:02:16 -05002843tpacket_error:
Johann Baudy69e3c752009-05-18 22:11:22 -07002844 if (po->tp_loss) {
2845 __packet_set_status(po, ph,
2846 TP_STATUS_AVAILABLE);
2847 packet_increment_head(&po->tx_ring);
2848 kfree_skb(skb);
2849 continue;
2850 } else {
2851 status = TP_STATUS_WRONG_FORMAT;
2852 err = tp_len;
2853 goto out_status;
2854 }
2855 }
2856
Jianfeng Tan9d2f67e2018-09-29 15:41:27 +00002857 if (po->has_vnet_hdr) {
2858 if (virtio_net_hdr_to_skb(skb, vnet_hdr, vio_le())) {
2859 tp_len = -EINVAL;
2860 goto tpacket_error;
2861 }
2862 virtio_net_hdr_set_proto(skb, vnet_hdr);
Willem de Bruijn1d036d22016-02-03 18:02:17 -05002863 }
2864
Johann Baudy69e3c752009-05-18 22:11:22 -07002865 skb->destructor = tpacket_destruct_skb;
2866 __packet_set_status(po, ph, TP_STATUS_SENDING);
Daniel Borkmannb0138402014-01-15 16:25:36 +01002867 packet_inc_pending(&po->tx_ring);
Johann Baudy69e3c752009-05-18 22:11:22 -07002868
2869 status = TP_STATUS_SEND_REQUEST;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002870 err = po->xmit(skb);
Hangbin Liu29e8e652022-04-14 16:49:25 +08002871 if (unlikely(err != 0)) {
2872 if (err > 0)
2873 err = net_xmit_errno(err);
Jarek Poplawskieb70df12010-01-10 22:04:19 +00002874 if (err && __packet_get_status(po, ph) ==
2875 TP_STATUS_AVAILABLE) {
2876 /* skb was destructed already */
2877 skb = NULL;
2878 goto out_status;
2879 }
2880 /*
2881 * skb was dropped but not destructed yet;
2882 * let's treat it like congestion or err < 0
2883 */
2884 err = 0;
2885 }
Johann Baudy69e3c752009-05-18 22:11:22 -07002886 packet_increment_head(&po->tx_ring);
2887 len_sum += tp_len;
Daniel Borkmannb0138402014-01-15 16:25:36 +01002888 } while (likely((ph != NULL) ||
2889 /* Note: packet_read_pending() might be slow if we have
2890 * to call it as it's per_cpu variable, but in fast-path
2891 * we already short-circuit the loop with the first
2892 * condition, and luckily don't have to go that path
2893 * anyway.
2894 */
2895 (need_wait && packet_read_pending(&po->tx_ring))));
Johann Baudy69e3c752009-05-18 22:11:22 -07002896
2897 err = len_sum;
2898 goto out_put;
2899
Johann Baudy69e3c752009-05-18 22:11:22 -07002900out_status:
2901 __packet_set_status(po, ph, status);
2902 kfree_skb(skb);
2903out_put:
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002904 dev_put(dev);
Johann Baudy69e3c752009-05-18 22:11:22 -07002905out:
2906 mutex_unlock(&po->pg_vec_lock);
2907 return err;
2908}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002909
Olof Johanssoneea49cc92011-11-02 11:00:49 +00002910static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2911 size_t reserve, size_t len,
2912 size_t linear, int noblock,
2913 int *err)
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002914{
2915 struct sk_buff *skb;
2916
2917 /* Under a page? Don't bother with paged skb. */
2918 if (prepad + len < PAGE_SIZE || !linear)
2919 linear = len;
2920
2921 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
Eric Dumazet28d64272013-08-08 14:38:47 -07002922 err, 0);
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002923 if (!skb)
2924 return NULL;
2925
2926 skb_reserve(skb, reserve);
2927 skb_put(skb, linear);
2928 skb->data_len = len - linear;
2929 skb->len += len - linear;
2930
2931 return skb;
2932}
2933
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002934static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002935{
2936 struct sock *sk = sock->sk;
Steffen Hurrle342dfc32014-01-17 22:53:15 +01002937 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002938 struct sk_buff *skb;
2939 struct net_device *dev;
Al Viro0e11c912006-11-08 00:26:29 -08002940 __be16 proto;
Willem de Bruijn486efdc2019-04-29 11:53:18 -04002941 unsigned char *addr = NULL;
Ben Greear827d9782011-06-01 07:18:53 +00002942 int err, reserve = 0;
Edward Jeec7d39e32015-10-08 14:56:49 -07002943 struct sockcm_cookie sockc;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002944 struct virtio_net_hdr vnet_hdr = { 0 };
2945 int offset = 0;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002946 struct packet_sock *po = pkt_sk(sk);
Willem de Bruijnda7c9562017-09-26 12:20:17 -04002947 bool has_vnet_hdr = false;
Willem de Bruijn57031eb2017-02-07 15:57:21 -05002948 int hlen, tlen, linear;
Ben Greear3bdc0eb2012-02-11 15:39:30 +00002949 int extra_len = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002950
2951 /*
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002952 * Get and verify the address.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002953 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002954
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01002955 if (likely(saddr == NULL)) {
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002956 dev = packet_cached_dev_get(po);
Eric Dumazetc7d2ef52021-06-16 06:42:01 -07002957 proto = READ_ONCE(po->num);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002958 } else {
2959 err = -EINVAL;
2960 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2961 goto out;
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07002962 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2963 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002964 proto = saddr->sll_protocol;
Ben Greear827d9782011-06-01 07:18:53 +00002965 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
Willem de Bruijn486efdc2019-04-29 11:53:18 -04002966 if (sock->type == SOCK_DGRAM) {
2967 if (dev && msg->msg_namelen < dev->addr_len +
2968 offsetof(struct sockaddr_ll, sll_addr))
2969 goto out_unlock;
2970 addr = saddr->sll_addr;
2971 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002972 }
2973
Linus Torvalds1da177e2005-04-16 15:20:36 -07002974 err = -ENXIO;
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002975 if (unlikely(dev == NULL))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002976 goto out_unlock;
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002977 err = -ENETDOWN;
2978 if (unlikely(!(dev->flags & IFF_UP)))
2979 goto out_unlock;
2980
Willem de Bruijn657a0662018-07-06 10:12:56 -04002981 sockcm_init(&sockc, sk);
Edward Jeec7d39e32015-10-08 14:56:49 -07002982 sockc.mark = sk->sk_mark;
2983 if (msg->msg_controllen) {
2984 err = sock_cmsg_send(sk, msg, &sockc);
2985 if (unlikely(err))
2986 goto out_unlock;
2987 }
2988
Linus Torvalds1da177e2005-04-16 15:20:36 -07002989 if (sock->type == SOCK_RAW)
2990 reserve = dev->hard_header_len;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002991 if (po->has_vnet_hdr) {
Willem de Bruijn16cc1402016-02-03 18:02:14 -05002992 err = packet_snd_vnet_parse(msg, &len, &vnet_hdr);
2993 if (err)
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002994 goto out_unlock;
Willem de Bruijnda7c9562017-09-26 12:20:17 -04002995 has_vnet_hdr = true;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002996 }
2997
Ben Greear3bdc0eb2012-02-11 15:39:30 +00002998 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2999 if (!netif_supports_nofcs(dev)) {
3000 err = -EPROTONOSUPPORT;
3001 goto out_unlock;
3002 }
3003 extra_len = 4; /* We're doing our own CRC */
3004 }
3005
Linus Torvalds1da177e2005-04-16 15:20:36 -07003006 err = -EMSGSIZE;
Willem de Bruijn16cc1402016-02-03 18:02:14 -05003007 if (!vnet_hdr.gso_type &&
3008 (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003009 goto out_unlock;
3010
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003011 err = -ENOBUFS;
Herbert Xuae641942011-11-18 02:20:04 +00003012 hlen = LL_RESERVED_SPACE(dev);
3013 tlen = dev->needed_tailroom;
Willem de Bruijn57031eb2017-02-07 15:57:21 -05003014 linear = __virtio16_to_cpu(vio_le(), vnet_hdr.hdr_len);
3015 linear = max(linear, min_t(int, len, dev->hard_header_len));
3016 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, linear,
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003017 msg->msg_flags & MSG_DONTWAIT, &err);
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003018 if (skb == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003019 goto out_unlock;
3020
Willem de Bruijnb84bbaf2018-05-11 13:24:25 -04003021 skb_reset_network_header(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003022
Stephen Hemminger0c4e8582007-10-09 01:36:32 -07003023 err = -EINVAL;
Willem de Bruijn9c707762014-11-19 13:10:16 -05003024 if (sock->type == SOCK_DGRAM) {
3025 offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len);
Christoph Jaeger46d2cfb2015-01-11 13:01:16 -05003026 if (unlikely(offset < 0))
Willem de Bruijn9c707762014-11-19 13:10:16 -05003027 goto out_free;
Willem de Bruijnb84bbaf2018-05-11 13:24:25 -04003028 } else if (reserve) {
Willem de Bruijn9aad13b2018-05-24 18:10:30 -04003029 skb_reserve(skb, -reserve);
Nicolas Dichtel88a81212019-01-17 11:27:22 +01003030 if (len < reserve + sizeof(struct ipv6hdr) &&
3031 dev->min_header_len != dev->hard_header_len)
Willem de Bruijn993675a2018-07-11 12:00:45 -04003032 skb_reset_network_header(skb);
Willem de Bruijn9c707762014-11-19 13:10:16 -05003033 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003034
3035 /* Returns -EFAULT on error */
Al Viroc0371da2014-11-24 10:42:55 -05003036 err = skb_copy_datagram_from_iter(skb, offset, &msg->msg_iter, len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003037 if (err)
3038 goto out_free;
Daniel Borkmannbf84a0102013-04-14 08:08:13 +00003039
Zhengchao Shaodc633702022-07-27 17:33:12 +08003040 if ((sock->type == SOCK_RAW &&
3041 !dev_validate_header(dev, skb->data, len)) || !skb->len) {
Willem de Bruijn9ed988c2016-03-09 21:58:34 -05003042 err = -EINVAL;
3043 goto out_free;
3044 }
3045
Willem de Bruijn8f932f72018-12-17 12:24:00 -05003046 skb_setup_tx_timestamp(skb, sockc.tsflags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003047
Willem de Bruijn16cc1402016-02-03 18:02:14 -05003048 if (!vnet_hdr.gso_type && (len > dev->mtu + reserve + extra_len) &&
Daniel Borkmann3c70c132015-11-11 23:25:42 +01003049 !packet_extra_vlan_len_allowed(dev, skb)) {
3050 err = -EMSGSIZE;
3051 goto out_free;
Ben Greear57f89bf2011-02-11 09:35:18 +00003052 }
3053
David S. Miller09effa62013-08-07 17:11:00 -07003054 skb->protocol = proto;
3055 skb->dev = dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003056 skb->priority = sk->sk_priority;
Edward Jeec7d39e32015-10-08 14:56:49 -07003057 skb->mark = sockc.mark;
Richard Cochran3d0ba8c2018-07-03 15:42:51 -07003058 skb->tstamp = sockc.transmit_time;
Daniel Borkmann0fd5d572014-02-16 15:55:22 +01003059
Hangbin Liudfed9132022-04-25 09:45:02 +08003060 if (unlikely(extra_len == 4))
3061 skb->no_fcs = 1;
3062
3063 packet_parse_headers(skb, sock);
3064
Willem de Bruijnda7c9562017-09-26 12:20:17 -04003065 if (has_vnet_hdr) {
Jarno Rajahalmedb60eb5f2016-11-18 15:40:41 -08003066 err = virtio_net_hdr_to_skb(skb, &vnet_hdr, vio_le());
Willem de Bruijn16cc1402016-02-03 18:02:14 -05003067 if (err)
3068 goto out_free;
3069 len += sizeof(vnet_hdr);
Jianfeng Tan9d2f67e2018-09-29 15:41:27 +00003070 virtio_net_hdr_set_proto(skb, &vnet_hdr);
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003071 }
3072
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01003073 err = po->xmit(skb);
Hangbin Liu29e8e652022-04-14 16:49:25 +08003074 if (unlikely(err != 0)) {
3075 if (err > 0)
3076 err = net_xmit_errno(err);
3077 if (err)
3078 goto out_unlock;
3079 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003080
Daniel Borkmanne40526c2013-11-21 16:50:58 +01003081 dev_put(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003082
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003083 return len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003084
3085out_free:
3086 kfree_skb(skb);
3087out_unlock:
Yajun Deng1160dfa2021-08-05 19:55:27 +08003088 dev_put(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003089out:
3090 return err;
3091}
3092
Ying Xue1b784142015-03-02 15:37:48 +08003093static int packet_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
Johann Baudy69e3c752009-05-18 22:11:22 -07003094{
Johann Baudy69e3c752009-05-18 22:11:22 -07003095 struct sock *sk = sock->sk;
3096 struct packet_sock *po = pkt_sk(sk);
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01003097
Eric Dumazetd1b5bee2021-06-10 09:00:12 -07003098 /* Reading tx_ring.pg_vec without holding pg_vec_lock is racy.
3099 * tpacket_snd() will redo the check safely.
3100 */
3101 if (data_race(po->tx_ring.pg_vec))
Johann Baudy69e3c752009-05-18 22:11:22 -07003102 return tpacket_snd(po, msg);
Eric Dumazetd1b5bee2021-06-10 09:00:12 -07003103
3104 return packet_snd(sock, msg, len);
Johann Baudy69e3c752009-05-18 22:11:22 -07003105}
3106
Linus Torvalds1da177e2005-04-16 15:20:36 -07003107/*
3108 * Close a PACKET socket. This is fairly simple. We immediately go
3109 * to 'closed' state and remove our protocol entry in the device list.
3110 */
3111
3112static int packet_release(struct socket *sock)
3113{
3114 struct sock *sk = sock->sk;
3115 struct packet_sock *po;
Anoob Soman2bd624b2017-02-15 20:25:39 +00003116 struct packet_fanout *f;
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08003117 struct net *net;
chetan lokef6fb8f12011-08-19 10:18:16 +00003118 union tpacket_req_u req_u;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003119
3120 if (!sk)
3121 return 0;
3122
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09003123 net = sock_net(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003124 po = pkt_sk(sk);
3125
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00003126 mutex_lock(&net->packet.sklist_lock);
stephen hemminger808f5112010-02-22 07:57:18 +00003127 sk_del_node_init_rcu(sk);
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00003128 mutex_unlock(&net->packet.sklist_lock);
3129
Eric Dumazet920de802008-11-24 00:09:29 -08003130 sock_prot_inuse_add(net, sk->sk_prot, -1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003131
stephen hemminger808f5112010-02-22 07:57:18 +00003132 spin_lock(&po->bind_lock);
David S. Millerce06b032011-07-04 01:44:29 -07003133 unregister_prot_hook(sk, false);
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01003134 packet_cached_dev_reset(po);
3135
Ben Greear160ff182011-06-01 07:18:52 +00003136 if (po->prot_hook.dev) {
Jakub Kicinskid62607c2022-06-07 21:39:55 -07003137 netdev_put(po->prot_hook.dev, &po->prot_hook.dev_tracker);
Ben Greear160ff182011-06-01 07:18:52 +00003138 po->prot_hook.dev = NULL;
3139 }
stephen hemminger808f5112010-02-22 07:57:18 +00003140 spin_unlock(&po->bind_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003141
Linus Torvalds1da177e2005-04-16 15:20:36 -07003142 packet_flush_mclist(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003143
Eric Dumazet5171b372018-04-15 17:52:04 -07003144 lock_sock(sk);
Phil Sutter9665d5d2013-02-01 07:21:41 +00003145 if (po->rx_ring.pg_vec) {
3146 memset(&req_u, 0, sizeof(req_u));
chetan lokef6fb8f12011-08-19 10:18:16 +00003147 packet_set_ring(sk, &req_u, 1, 0);
Phil Sutter9665d5d2013-02-01 07:21:41 +00003148 }
Johann Baudy69e3c752009-05-18 22:11:22 -07003149
Phil Sutter9665d5d2013-02-01 07:21:41 +00003150 if (po->tx_ring.pg_vec) {
3151 memset(&req_u, 0, sizeof(req_u));
chetan lokef6fb8f12011-08-19 10:18:16 +00003152 packet_set_ring(sk, &req_u, 1, 1);
Phil Sutter9665d5d2013-02-01 07:21:41 +00003153 }
Eric Dumazet5171b372018-04-15 17:52:04 -07003154 release_sock(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003155
Anoob Soman2bd624b2017-02-15 20:25:39 +00003156 f = fanout_release(sk);
David S. Millerdc99f602011-07-05 01:45:05 -07003157
stephen hemminger808f5112010-02-22 07:57:18 +00003158 synchronize_net();
Anoob Soman2bd624b2017-02-15 20:25:39 +00003159
Willem de Bruijnafa09252019-05-31 12:37:23 -04003160 kfree(po->rollover);
Anoob Soman2bd624b2017-02-15 20:25:39 +00003161 if (f) {
3162 fanout_release_data(f);
Tanner Love9c661b02020-11-06 13:07:40 -05003163 kvfree(f);
Anoob Soman2bd624b2017-02-15 20:25:39 +00003164 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003165 /*
3166 * Now the socket is dead. No more input will appear.
3167 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003168 sock_orphan(sk);
3169 sock->sk = NULL;
3170
3171 /* Purge queues */
3172
3173 skb_queue_purge(&sk->sk_receive_queue);
Daniel Borkmannb0138402014-01-15 16:25:36 +01003174 packet_free_pending(po);
Pavel Emelyanov17ab56a2007-11-10 21:38:48 -08003175 sk_refcnt_debug_release(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003176
3177 sock_put(sk);
3178 return 0;
3179}
3180
3181/*
3182 * Attach a packet hook.
3183 */
3184
Francesco Ruggeri30f7ea12015-11-05 08:16:14 -08003185static int packet_do_bind(struct sock *sk, const char *name, int ifindex,
3186 __be16 proto)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003187{
3188 struct packet_sock *po = pkt_sk(sk);
Francesco Ruggeri30f7ea12015-11-05 08:16:14 -08003189 struct net_device *dev = NULL;
Francesco Ruggeri30f7ea12015-11-05 08:16:14 -08003190 bool unlisted = false;
Eric Dumazetbf440772022-01-07 10:39:53 -08003191 bool need_rehook;
3192 int ret = 0;
David S. Millerdc99f602011-07-05 01:45:05 -07003193
Linus Torvalds1da177e2005-04-16 15:20:36 -07003194 lock_sock(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003195 spin_lock(&po->bind_lock);
Francesco Ruggeri30f7ea12015-11-05 08:16:14 -08003196 rcu_read_lock();
3197
Willem de Bruijn49716132017-09-26 12:19:37 -04003198 if (po->fanout) {
3199 ret = -EINVAL;
3200 goto out_unlock;
3201 }
3202
Francesco Ruggeri30f7ea12015-11-05 08:16:14 -08003203 if (name) {
3204 dev = dev_get_by_name_rcu(sock_net(sk), name);
3205 if (!dev) {
3206 ret = -ENODEV;
3207 goto out_unlock;
3208 }
3209 } else if (ifindex) {
3210 dev = dev_get_by_index_rcu(sock_net(sk), ifindex);
3211 if (!dev) {
3212 ret = -ENODEV;
3213 goto out_unlock;
3214 }
3215 }
3216
Eric Dumazetbf440772022-01-07 10:39:53 -08003217 need_rehook = po->prot_hook.type != proto || po->prot_hook.dev != dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003218
Daniel Borkmann902fefb2014-01-15 16:25:34 +01003219 if (need_rehook) {
Eric Dumazetbf440772022-01-07 10:39:53 -08003220 dev_hold(dev);
Francesco Ruggeri30f7ea12015-11-05 08:16:14 -08003221 if (po->running) {
3222 rcu_read_unlock();
Eric Dumazet15fe0762017-11-28 08:03:30 -08003223 /* prevents packet_notifier() from calling
3224 * register_prot_hook()
3225 */
Eric Dumazetc7d2ef52021-06-16 06:42:01 -07003226 WRITE_ONCE(po->num, 0);
Francesco Ruggeri30f7ea12015-11-05 08:16:14 -08003227 __unregister_prot_hook(sk, true);
3228 rcu_read_lock();
Francesco Ruggeri30f7ea12015-11-05 08:16:14 -08003229 if (dev)
3230 unlisted = !dev_get_by_index_rcu(sock_net(sk),
3231 dev->ifindex);
3232 }
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01003233
Eric Dumazet15fe0762017-11-28 08:03:30 -08003234 BUG_ON(po->running);
Eric Dumazetc7d2ef52021-06-16 06:42:01 -07003235 WRITE_ONCE(po->num, proto);
Daniel Borkmann902fefb2014-01-15 16:25:34 +01003236 po->prot_hook.type = proto;
Daniel Borkmann902fefb2014-01-15 16:25:34 +01003237
Jakub Kicinskid62607c2022-06-07 21:39:55 -07003238 netdev_put(po->prot_hook.dev, &po->prot_hook.dev_tracker);
Eric Dumazetf1d9268e2021-12-14 07:09:33 -08003239
Francesco Ruggeri30f7ea12015-11-05 08:16:14 -08003240 if (unlikely(unlisted)) {
Francesco Ruggeri30f7ea12015-11-05 08:16:14 -08003241 po->prot_hook.dev = NULL;
Eric Dumazete032f7c2021-06-16 06:42:02 -07003242 WRITE_ONCE(po->ifindex, -1);
Francesco Ruggeri30f7ea12015-11-05 08:16:14 -08003243 packet_cached_dev_reset(po);
3244 } else {
Jakub Kicinskid62607c2022-06-07 21:39:55 -07003245 netdev_hold(dev, &po->prot_hook.dev_tracker,
3246 GFP_ATOMIC);
Francesco Ruggeri30f7ea12015-11-05 08:16:14 -08003247 po->prot_hook.dev = dev;
Eric Dumazete032f7c2021-06-16 06:42:02 -07003248 WRITE_ONCE(po->ifindex, dev ? dev->ifindex : 0);
Francesco Ruggeri30f7ea12015-11-05 08:16:14 -08003249 packet_cached_dev_assign(po, dev);
3250 }
Eric Dumazetbf440772022-01-07 10:39:53 -08003251 dev_put(dev);
Daniel Borkmann902fefb2014-01-15 16:25:34 +01003252 }
3253
3254 if (proto == 0 || !need_rehook)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003255 goto out_unlock;
3256
Francesco Ruggeri30f7ea12015-11-05 08:16:14 -08003257 if (!unlisted && (!dev || (dev->flags & IFF_UP))) {
David S. Millerce06b032011-07-04 01:44:29 -07003258 register_prot_hook(sk);
Urs Thuermannbe85d4a2007-11-12 21:05:20 -08003259 } else {
3260 sk->sk_err = ENETDOWN;
3261 if (!sock_flag(sk, SOCK_DEAD))
Alexander Aringe3ae2362021-06-27 18:48:21 -04003262 sk_error_report(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003263 }
3264
3265out_unlock:
Francesco Ruggeri30f7ea12015-11-05 08:16:14 -08003266 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003267 spin_unlock(&po->bind_lock);
3268 release_sock(sk);
Francesco Ruggeri30f7ea12015-11-05 08:16:14 -08003269 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003270}
3271
3272/*
3273 * Bind a packet socket to a device
3274 */
3275
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003276static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
3277 int addr_len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003278{
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003279 struct sock *sk = sock->sk;
Alexander Potapenko540e2892017-03-01 12:57:20 +01003280 char name[sizeof(uaddr->sa_data) + 1];
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003281
Linus Torvalds1da177e2005-04-16 15:20:36 -07003282 /*
3283 * Check legality
3284 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003285
Kris Katterjohn8ae55f02006-01-23 16:28:02 -08003286 if (addr_len != sizeof(struct sockaddr))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003287 return -EINVAL;
Alexander Potapenko540e2892017-03-01 12:57:20 +01003288 /* uaddr->sa_data comes from the userspace, it's not guaranteed to be
3289 * zero-terminated.
3290 */
3291 memcpy(name, uaddr->sa_data, sizeof(uaddr->sa_data));
3292 name[sizeof(uaddr->sa_data)] = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003293
Francesco Ruggeri30f7ea12015-11-05 08:16:14 -08003294 return packet_do_bind(sk, name, 0, pkt_sk(sk)->num);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003295}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003296
3297static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
3298{
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003299 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
3300 struct sock *sk = sock->sk;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003301
3302 /*
3303 * Check legality
3304 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003305
Linus Torvalds1da177e2005-04-16 15:20:36 -07003306 if (addr_len < sizeof(struct sockaddr_ll))
3307 return -EINVAL;
3308 if (sll->sll_family != AF_PACKET)
3309 return -EINVAL;
3310
Francesco Ruggeri30f7ea12015-11-05 08:16:14 -08003311 return packet_do_bind(sk, NULL, sll->sll_ifindex,
3312 sll->sll_protocol ? : pkt_sk(sk)->num);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003313}
3314
3315static struct proto packet_proto = {
3316 .name = "PACKET",
3317 .owner = THIS_MODULE,
3318 .obj_size = sizeof(struct packet_sock),
3319};
3320
3321/*
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003322 * Create a packet of type SOCK_PACKET.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003323 */
3324
Eric Paris3f378b682009-11-05 22:18:14 -08003325static int packet_create(struct net *net, struct socket *sock, int protocol,
3326 int kern)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003327{
3328 struct sock *sk;
3329 struct packet_sock *po;
Al Viro0e11c912006-11-08 00:26:29 -08003330 __be16 proto = (__force __be16)protocol; /* weird, but documented */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003331 int err;
3332
Eric W. Biedermandf008c92012-11-16 03:03:07 +00003333 if (!ns_capable(net->user_ns, CAP_NET_RAW))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003334 return -EPERM;
David S. Millerbe020972007-05-29 13:16:31 -07003335 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
3336 sock->type != SOCK_PACKET)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003337 return -ESOCKTNOSUPPORT;
3338
3339 sock->state = SS_UNCONNECTED;
3340
3341 err = -ENOBUFS;
Eric W. Biederman11aa9c22015-05-08 21:09:13 -05003342 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto, kern);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003343 if (sk == NULL)
3344 goto out;
3345
3346 sock->ops = &packet_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003347 if (sock->type == SOCK_PACKET)
3348 sock->ops = &packet_ops_spkt;
David S. Millerbe020972007-05-29 13:16:31 -07003349
Linus Torvalds1da177e2005-04-16 15:20:36 -07003350 sock_init_data(sock, sk);
3351
3352 po = pkt_sk(sk);
Neil Horman89ed5b52019-06-25 17:57:49 -04003353 init_completion(&po->skb_completion);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003354 sk->sk_family = PF_PACKET;
Al Viro0e11c912006-11-08 00:26:29 -08003355 po->num = proto;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01003356 po->xmit = dev_queue_xmit;
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01003357
Daniel Borkmannb0138402014-01-15 16:25:36 +01003358 err = packet_alloc_pending(po);
3359 if (err)
3360 goto out2;
3361
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01003362 packet_cached_dev_reset(po);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003363
3364 sk->sk_destruct = packet_sock_destruct;
Pavel Emelyanov17ab56a2007-11-10 21:38:48 -08003365 sk_refcnt_debug_inc(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003366
3367 /*
3368 * Attach a protocol block
3369 */
3370
3371 spin_lock_init(&po->bind_lock);
Herbert Xu905db442009-01-30 14:12:06 -08003372 mutex_init(&po->pg_vec_lock);
Willem de Bruijn0648ab72015-05-12 11:56:46 -04003373 po->rollover = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003374 po->prot_hook.func = packet_rcv;
David S. Millerbe020972007-05-29 13:16:31 -07003375
Linus Torvalds1da177e2005-04-16 15:20:36 -07003376 if (sock->type == SOCK_PACKET)
3377 po->prot_hook.func = packet_rcv_spkt;
David S. Millerbe020972007-05-29 13:16:31 -07003378
Linus Torvalds1da177e2005-04-16 15:20:36 -07003379 po->prot_hook.af_packet_priv = sk;
Congyu Liu47934e062022-01-18 14:20:13 -05003380 po->prot_hook.af_packet_net = sock_net(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003381
Al Viro0e11c912006-11-08 00:26:29 -08003382 if (proto) {
3383 po->prot_hook.type = proto;
Willem de Bruijna6361f02018-04-23 17:37:03 -04003384 __register_prot_hook(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003385 }
3386
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00003387 mutex_lock(&net->packet.sklist_lock);
Maxime Chevalliera4dc6a42019-03-16 14:41:30 +01003388 sk_add_node_tail_rcu(sk, &net->packet.sklist);
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00003389 mutex_unlock(&net->packet.sklist_lock);
3390
Eric Dumazet36804532008-11-19 14:25:35 -08003391 sock_prot_inuse_add(net, &packet_proto, 1);
stephen hemminger808f5112010-02-22 07:57:18 +00003392
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003393 return 0;
Daniel Borkmannb0138402014-01-15 16:25:36 +01003394out2:
3395 sk_free(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003396out:
3397 return err;
3398}
3399
3400/*
3401 * Pull a packet from our receive queue and hand it to the user.
3402 * If necessary we block.
3403 */
3404
Ying Xue1b784142015-03-02 15:37:48 +08003405static int packet_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
3406 int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003407{
3408 struct sock *sk = sock->sk;
3409 struct sk_buff *skb;
3410 int copied, err;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003411 int vnet_hdr_len = 0;
Eyal Birger2472d762015-03-01 14:58:28 +02003412 unsigned int origlen = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003413
3414 err = -EINVAL;
Richard Cochraned85b562010-04-07 22:41:28 +00003415 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003416 goto out;
3417
3418#if 0
3419 /* What error should we return now? EUNATTACH? */
3420 if (pkt_sk(sk)->ifindex < 0)
3421 return -ENODEV;
3422#endif
3423
Richard Cochraned85b562010-04-07 22:41:28 +00003424 if (flags & MSG_ERRQUEUE) {
Richard Cochrancb820f82013-07-19 19:40:09 +02003425 err = sock_recv_errqueue(sk, msg, len,
3426 SOL_PACKET, PACKET_TX_TIMESTAMP);
Richard Cochraned85b562010-04-07 22:41:28 +00003427 goto out;
3428 }
3429
Linus Torvalds1da177e2005-04-16 15:20:36 -07003430 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -07003431 * Call the generic datagram receiver. This handles all sorts
3432 * of horrible races and re-entrancy so we can forget about it
3433 * in the protocol layers.
3434 *
3435 * Now it will return ENETDOWN, if device have just gone down,
3436 * but then it will block.
3437 */
3438
Oliver Hartkoppf4b41f02022-04-04 18:30:22 +02003439 skb = skb_recv_datagram(sk, flags, &err);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003440
3441 /*
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003442 * An error occurred so return it. Because skb_recv_datagram()
Linus Torvalds1da177e2005-04-16 15:20:36 -07003443 * handles the blocking we don't see and worry about blocking
3444 * retries.
3445 */
3446
Kris Katterjohn8ae55f02006-01-23 16:28:02 -08003447 if (skb == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003448 goto out;
3449
Eric Dumazet9bb6cd62019-06-12 09:52:33 -07003450 packet_rcv_try_clear_pressure(pkt_sk(sk));
Willem de Bruijn2ccdbaa2015-05-12 11:56:48 -04003451
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003452 if (pkt_sk(sk)->has_vnet_hdr) {
Willem de Bruijn16cc1402016-02-03 18:02:14 -05003453 err = packet_rcv_vnet(msg, skb, &len);
3454 if (err)
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003455 goto out_free;
Willem de Bruijn16cc1402016-02-03 18:02:14 -05003456 vnet_hdr_len = sizeof(struct virtio_net_hdr);
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003457 }
3458
Hannes Frederic Sowaf3d33422013-11-21 03:14:22 +01003459 /* You lose any data beyond the buffer you gave. If it worries
3460 * a user program they can ask the device for its MTU
3461 * anyway.
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003462 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003463 copied = skb->len;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003464 if (copied > len) {
3465 copied = len;
3466 msg->msg_flags |= MSG_TRUNC;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003467 }
3468
David S. Miller51f3d022014-11-05 16:46:40 -05003469 err = skb_copy_datagram_msg(skb, 0, msg, copied);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003470 if (err)
3471 goto out_free;
3472
Eyal Birger2472d762015-03-01 14:58:28 +02003473 if (sock->type != SOCK_PACKET) {
3474 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
3475
3476 /* Original length was stored in sockaddr_ll fields */
3477 origlen = PACKET_SKB_CB(skb)->sa.origlen;
3478 sll->sll_family = AF_PACKET;
3479 sll->sll_protocol = skb->protocol;
3480 }
3481
Erin MacNeil6fd1d512022-04-27 16:02:37 -04003482 sock_recv_cmsgs(msg, sk, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003483
Hannes Frederic Sowaf3d33422013-11-21 03:14:22 +01003484 if (msg->msg_name) {
Eric Dumazetc7005252022-03-12 15:29:58 -08003485 const size_t max_len = min(sizeof(skb->cb),
3486 sizeof(struct sockaddr_storage));
Willem de Bruijnb2cf86e2019-04-29 11:46:55 -04003487 int copy_len;
3488
Hannes Frederic Sowaf3d33422013-11-21 03:14:22 +01003489 /* If the address length field is there to be filled
3490 * in, we fill it in now.
3491 */
3492 if (sock->type == SOCK_PACKET) {
Steffen Hurrle342dfc32014-01-17 22:53:15 +01003493 __sockaddr_check_size(sizeof(struct sockaddr_pkt));
Hannes Frederic Sowaf3d33422013-11-21 03:14:22 +01003494 msg->msg_namelen = sizeof(struct sockaddr_pkt);
Willem de Bruijnb2cf86e2019-04-29 11:46:55 -04003495 copy_len = msg->msg_namelen;
Hannes Frederic Sowaf3d33422013-11-21 03:14:22 +01003496 } else {
3497 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
Eyal Birger2472d762015-03-01 14:58:28 +02003498
Hannes Frederic Sowaf3d33422013-11-21 03:14:22 +01003499 msg->msg_namelen = sll->sll_halen +
3500 offsetof(struct sockaddr_ll, sll_addr);
Willem de Bruijnb2cf86e2019-04-29 11:46:55 -04003501 copy_len = msg->msg_namelen;
3502 if (msg->msg_namelen < sizeof(struct sockaddr_ll)) {
3503 memset(msg->msg_name +
3504 offsetof(struct sockaddr_ll, sll_addr),
3505 0, sizeof(sll->sll_addr));
3506 msg->msg_namelen = sizeof(struct sockaddr_ll);
3507 }
Hannes Frederic Sowaf3d33422013-11-21 03:14:22 +01003508 }
Eric Dumazetc7005252022-03-12 15:29:58 -08003509 if (WARN_ON_ONCE(copy_len > max_len)) {
3510 copy_len = max_len;
3511 msg->msg_namelen = copy_len;
3512 }
Willem de Bruijnb2cf86e2019-04-29 11:46:55 -04003513 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa, copy_len);
Hannes Frederic Sowaf3d33422013-11-21 03:14:22 +01003514 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003515
Herbert Xu8dc41942007-02-04 23:31:32 -08003516 if (pkt_sk(sk)->auxdata) {
Herbert Xuffbc6112007-02-04 23:33:10 -08003517 struct tpacket_auxdata aux;
3518
3519 aux.tp_status = TP_STATUS_USER;
3520 if (skb->ip_summed == CHECKSUM_PARTIAL)
3521 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
Alexander Drozdov682f0482015-03-23 09:11:13 +03003522 else if (skb->pkt_type != PACKET_OUTGOING &&
3523 (skb->ip_summed == CHECKSUM_COMPLETE ||
3524 skb_csum_unnecessary(skb)))
3525 aux.tp_status |= TP_STATUS_CSUM_VALID;
3526
Eyal Birger2472d762015-03-01 14:58:28 +02003527 aux.tp_len = origlen;
Herbert Xuffbc6112007-02-04 23:33:10 -08003528 aux.tp_snaplen = skb->len;
3529 aux.tp_mac = 0;
Arnaldo Carvalho de Melobbe735e2007-03-10 22:16:10 -03003530 aux.tp_net = skb_network_offset(skb);
Jiri Pirkodf8a39de2015-01-13 17:13:44 +01003531 if (skb_vlan_tag_present(skb)) {
3532 aux.tp_vlan_tci = skb_vlan_tag_get(skb);
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +09003533 aux.tp_vlan_tpid = ntohs(skb->vlan_proto);
3534 aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
Ben Greeara3bcc232011-06-01 06:49:10 +00003535 } else {
3536 aux.tp_vlan_tci = 0;
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +09003537 aux.tp_vlan_tpid = 0;
Ben Greeara3bcc232011-06-01 06:49:10 +00003538 }
Herbert Xuffbc6112007-02-04 23:33:10 -08003539 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
Herbert Xu8dc41942007-02-04 23:31:32 -08003540 }
3541
Linus Torvalds1da177e2005-04-16 15:20:36 -07003542 /*
3543 * Free or return the buffer as appropriate. Again this
3544 * hides all the races and re-entrancy issues from us.
3545 */
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003546 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003547
3548out_free:
3549 skb_free_datagram(sk, skb);
3550out:
3551 return err;
3552}
3553
Linus Torvalds1da177e2005-04-16 15:20:36 -07003554static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
Denys Vlasenko9b2c45d2018-02-12 20:00:20 +01003555 int peer)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003556{
3557 struct net_device *dev;
3558 struct sock *sk = sock->sk;
3559
3560 if (peer)
3561 return -EOPNOTSUPP;
3562
3563 uaddr->sa_family = AF_PACKET;
Daniel Borkmann2dc85bf2013-06-12 16:02:27 +02003564 memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
Eric Dumazet654d1f82009-11-02 10:43:32 +01003565 rcu_read_lock();
Eric Dumazete032f7c2021-06-16 06:42:02 -07003566 dev = dev_get_by_index_rcu(sock_net(sk), READ_ONCE(pkt_sk(sk)->ifindex));
Eric Dumazet654d1f82009-11-02 10:43:32 +01003567 if (dev)
Wolfram Sang8fc9d512022-08-18 23:02:27 +02003568 strscpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
Eric Dumazet654d1f82009-11-02 10:43:32 +01003569 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003570
Denys Vlasenko9b2c45d2018-02-12 20:00:20 +01003571 return sizeof(*uaddr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003572}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003573
3574static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
Denys Vlasenko9b2c45d2018-02-12 20:00:20 +01003575 int peer)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003576{
3577 struct net_device *dev;
3578 struct sock *sk = sock->sk;
3579 struct packet_sock *po = pkt_sk(sk);
Cyrill Gorcunov13cfa972009-11-08 05:51:19 +00003580 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
Eric Dumazete032f7c2021-06-16 06:42:02 -07003581 int ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003582
3583 if (peer)
3584 return -EOPNOTSUPP;
3585
Eric Dumazete032f7c2021-06-16 06:42:02 -07003586 ifindex = READ_ONCE(po->ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003587 sll->sll_family = AF_PACKET;
Eric Dumazete032f7c2021-06-16 06:42:02 -07003588 sll->sll_ifindex = ifindex;
Eric Dumazetc7d2ef52021-06-16 06:42:01 -07003589 sll->sll_protocol = READ_ONCE(po->num);
Vasiliy Kulikov67286642010-11-10 12:09:10 -08003590 sll->sll_pkttype = 0;
Eric Dumazet654d1f82009-11-02 10:43:32 +01003591 rcu_read_lock();
Eric Dumazete032f7c2021-06-16 06:42:02 -07003592 dev = dev_get_by_index_rcu(sock_net(sk), ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003593 if (dev) {
3594 sll->sll_hatype = dev->type;
3595 sll->sll_halen = dev->addr_len;
3596 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003597 } else {
3598 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
3599 sll->sll_halen = 0;
3600 }
Eric Dumazet654d1f82009-11-02 10:43:32 +01003601 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003602
Denys Vlasenko9b2c45d2018-02-12 20:00:20 +01003603 return offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003604}
3605
Wang Chen2aeb0b82008-07-14 20:49:46 -07003606static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
3607 int what)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003608{
3609 switch (i->type) {
3610 case PACKET_MR_MULTICAST:
Jiri Pirko11625632010-03-02 20:40:01 +00003611 if (i->alen != dev->addr_len)
3612 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003613 if (what > 0)
Jiri Pirko22bedad32010-04-01 21:22:57 +00003614 return dev_mc_add(dev, i->addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003615 else
Jiri Pirko22bedad32010-04-01 21:22:57 +00003616 return dev_mc_del(dev, i->addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003617 break;
3618 case PACKET_MR_PROMISC:
Wang Chen2aeb0b82008-07-14 20:49:46 -07003619 return dev_set_promiscuity(dev, what);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003620 case PACKET_MR_ALLMULTI:
Wang Chen2aeb0b82008-07-14 20:49:46 -07003621 return dev_set_allmulti(dev, what);
Eric W. Biedermand95ed922009-05-19 18:27:17 +00003622 case PACKET_MR_UNICAST:
Jiri Pirko11625632010-03-02 20:40:01 +00003623 if (i->alen != dev->addr_len)
3624 return -EINVAL;
Eric W. Biedermand95ed922009-05-19 18:27:17 +00003625 if (what > 0)
Jiri Pirkoa748ee22010-04-01 21:22:09 +00003626 return dev_uc_add(dev, i->addr);
Eric W. Biedermand95ed922009-05-19 18:27:17 +00003627 else
Jiri Pirkoa748ee22010-04-01 21:22:09 +00003628 return dev_uc_del(dev, i->addr);
Eric W. Biedermand95ed922009-05-19 18:27:17 +00003629 break;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003630 default:
3631 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003632 }
Wang Chen2aeb0b82008-07-14 20:49:46 -07003633 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003634}
3635
Francesco Ruggeri82f17092015-03-09 11:51:04 -07003636static void packet_dev_mclist_delete(struct net_device *dev,
3637 struct packet_mclist **mlp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003638{
Francesco Ruggeri82f17092015-03-09 11:51:04 -07003639 struct packet_mclist *ml;
3640
3641 while ((ml = *mlp) != NULL) {
3642 if (ml->ifindex == dev->ifindex) {
3643 packet_dev_mc(dev, ml, -1);
3644 *mlp = ml->next;
3645 kfree(ml);
3646 } else
3647 mlp = &ml->next;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003648 }
3649}
3650
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003651static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003652{
3653 struct packet_sock *po = pkt_sk(sk);
3654 struct packet_mclist *ml, *i;
3655 struct net_device *dev;
3656 int err;
3657
3658 rtnl_lock();
3659
3660 err = -ENODEV;
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09003661 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003662 if (!dev)
3663 goto done;
3664
3665 err = -EINVAL;
Jiri Pirko11625632010-03-02 20:40:01 +00003666 if (mreq->mr_alen > dev->addr_len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003667 goto done;
3668
3669 err = -ENOBUFS;
Kris Katterjohn8b3a7002006-01-11 15:56:43 -08003670 i = kmalloc(sizeof(*i), GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003671 if (i == NULL)
3672 goto done;
3673
3674 err = 0;
3675 for (ml = po->mclist; ml; ml = ml->next) {
3676 if (ml->ifindex == mreq->mr_ifindex &&
3677 ml->type == mreq->mr_type &&
3678 ml->alen == mreq->mr_alen &&
3679 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3680 ml->count++;
3681 /* Free the new element ... */
3682 kfree(i);
3683 goto done;
3684 }
3685 }
3686
3687 i->type = mreq->mr_type;
3688 i->ifindex = mreq->mr_ifindex;
3689 i->alen = mreq->mr_alen;
3690 memcpy(i->addr, mreq->mr_address, i->alen);
Mathias Krause309cf372016-04-10 12:52:28 +02003691 memset(i->addr + i->alen, 0, sizeof(i->addr) - i->alen);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003692 i->count = 1;
3693 i->next = po->mclist;
3694 po->mclist = i;
Wang Chen2aeb0b82008-07-14 20:49:46 -07003695 err = packet_dev_mc(dev, i, 1);
3696 if (err) {
3697 po->mclist = i->next;
3698 kfree(i);
3699 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003700
3701done:
3702 rtnl_unlock();
3703 return err;
3704}
3705
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003706static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003707{
3708 struct packet_mclist *ml, **mlp;
3709
3710 rtnl_lock();
3711
3712 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
3713 if (ml->ifindex == mreq->mr_ifindex &&
3714 ml->type == mreq->mr_type &&
3715 ml->alen == mreq->mr_alen &&
3716 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3717 if (--ml->count == 0) {
3718 struct net_device *dev;
3719 *mlp = ml->next;
Eric Dumazetad959e72009-10-16 06:38:46 +00003720 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3721 if (dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003722 packet_dev_mc(dev, ml, -1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003723 kfree(ml);
3724 }
Francesco Ruggeri82f17092015-03-09 11:51:04 -07003725 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003726 }
3727 }
3728 rtnl_unlock();
Francesco Ruggeri82f17092015-03-09 11:51:04 -07003729 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003730}
3731
3732static void packet_flush_mclist(struct sock *sk)
3733{
3734 struct packet_sock *po = pkt_sk(sk);
3735 struct packet_mclist *ml;
3736
3737 if (!po->mclist)
3738 return;
3739
3740 rtnl_lock();
3741 while ((ml = po->mclist) != NULL) {
3742 struct net_device *dev;
3743
3744 po->mclist = ml->next;
Eric Dumazetad959e72009-10-16 06:38:46 +00003745 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3746 if (dev != NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003747 packet_dev_mc(dev, ml, -1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003748 kfree(ml);
3749 }
3750 rtnl_unlock();
3751}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003752
3753static int
Christoph Hellwiga7b75c52020-07-23 08:09:07 +02003754packet_setsockopt(struct socket *sock, int level, int optname, sockptr_t optval,
3755 unsigned int optlen)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003756{
3757 struct sock *sk = sock->sk;
Herbert Xu8dc41942007-02-04 23:31:32 -08003758 struct packet_sock *po = pkt_sk(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003759 int ret;
3760
3761 if (level != SOL_PACKET)
3762 return -ENOPROTOOPT;
3763
Johann Baudy69e3c752009-05-18 22:11:22 -07003764 switch (optname) {
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003765 case PACKET_ADD_MEMBERSHIP:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003766 case PACKET_DROP_MEMBERSHIP:
3767 {
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003768 struct packet_mreq_max mreq;
3769 int len = optlen;
3770 memset(&mreq, 0, sizeof(mreq));
3771 if (len < sizeof(struct packet_mreq))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003772 return -EINVAL;
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003773 if (len > sizeof(mreq))
3774 len = sizeof(mreq);
Christoph Hellwiga7b75c52020-07-23 08:09:07 +02003775 if (copy_from_sockptr(&mreq, optval, len))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003776 return -EFAULT;
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003777 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3778 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003779 if (optname == PACKET_ADD_MEMBERSHIP)
3780 ret = packet_mc_add(sk, &mreq);
3781 else
3782 ret = packet_mc_drop(sk, &mreq);
3783 return ret;
3784 }
David S. Millera2efcfa2007-05-29 13:12:50 -07003785
Linus Torvalds1da177e2005-04-16 15:20:36 -07003786 case PACKET_RX_RING:
Johann Baudy69e3c752009-05-18 22:11:22 -07003787 case PACKET_TX_RING:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003788 {
chetan lokef6fb8f12011-08-19 10:18:16 +00003789 union tpacket_req_u req_u;
3790 int len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003791
Eric Dumazet5171b372018-04-15 17:52:04 -07003792 lock_sock(sk);
chetan lokef6fb8f12011-08-19 10:18:16 +00003793 switch (po->tp_version) {
3794 case TPACKET_V1:
3795 case TPACKET_V2:
3796 len = sizeof(req_u.req);
3797 break;
3798 case TPACKET_V3:
3799 default:
3800 len = sizeof(req_u.req3);
3801 break;
3802 }
Eric Dumazet5171b372018-04-15 17:52:04 -07003803 if (optlen < len) {
3804 ret = -EINVAL;
3805 } else {
Christoph Hellwiga7b75c52020-07-23 08:09:07 +02003806 if (copy_from_sockptr(&req_u.req, optval, len))
Eric Dumazet5171b372018-04-15 17:52:04 -07003807 ret = -EFAULT;
3808 else
3809 ret = packet_set_ring(sk, &req_u, 0,
3810 optname == PACKET_TX_RING);
3811 }
3812 release_sock(sk);
3813 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003814 }
3815 case PACKET_COPY_THRESH:
3816 {
3817 int val;
3818
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003819 if (optlen != sizeof(val))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003820 return -EINVAL;
Christoph Hellwiga7b75c52020-07-23 08:09:07 +02003821 if (copy_from_sockptr(&val, optval, sizeof(val)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003822 return -EFAULT;
3823
3824 pkt_sk(sk)->copy_thresh = val;
3825 return 0;
3826 }
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003827 case PACKET_VERSION:
3828 {
3829 int val;
3830
3831 if (optlen != sizeof(val))
3832 return -EINVAL;
Christoph Hellwiga7b75c52020-07-23 08:09:07 +02003833 if (copy_from_sockptr(&val, optval, sizeof(val)))
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003834 return -EFAULT;
3835 switch (val) {
3836 case TPACKET_V1:
3837 case TPACKET_V2:
chetan lokef6fb8f12011-08-19 10:18:16 +00003838 case TPACKET_V3:
Philip Pettersson84ac7262016-11-30 14:55:36 -08003839 break;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003840 default:
3841 return -EINVAL;
3842 }
Philip Pettersson84ac7262016-11-30 14:55:36 -08003843 lock_sock(sk);
3844 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
3845 ret = -EBUSY;
3846 } else {
3847 po->tp_version = val;
3848 ret = 0;
3849 }
3850 release_sock(sk);
3851 return ret;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003852 }
Patrick McHardy8913336a2008-07-18 18:05:19 -07003853 case PACKET_RESERVE:
3854 {
3855 unsigned int val;
3856
3857 if (optlen != sizeof(val))
3858 return -EINVAL;
Christoph Hellwiga7b75c52020-07-23 08:09:07 +02003859 if (copy_from_sockptr(&val, optval, sizeof(val)))
Patrick McHardy8913336a2008-07-18 18:05:19 -07003860 return -EFAULT;
Andrey Konovalovbcc53642017-03-29 16:11:22 +02003861 if (val > INT_MAX)
3862 return -EINVAL;
Willem de Bruijnc27927e2017-08-10 12:41:58 -04003863 lock_sock(sk);
3864 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
3865 ret = -EBUSY;
3866 } else {
3867 po->tp_reserve = val;
3868 ret = 0;
3869 }
3870 release_sock(sk);
3871 return ret;
Patrick McHardy8913336a2008-07-18 18:05:19 -07003872 }
Johann Baudy69e3c752009-05-18 22:11:22 -07003873 case PACKET_LOSS:
3874 {
3875 unsigned int val;
3876
3877 if (optlen != sizeof(val))
3878 return -EINVAL;
Christoph Hellwiga7b75c52020-07-23 08:09:07 +02003879 if (copy_from_sockptr(&val, optval, sizeof(val)))
Johann Baudy69e3c752009-05-18 22:11:22 -07003880 return -EFAULT;
Willem de Bruijna6361f02018-04-23 17:37:03 -04003881
3882 lock_sock(sk);
3883 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
3884 ret = -EBUSY;
3885 } else {
3886 po->tp_loss = !!val;
3887 ret = 0;
3888 }
3889 release_sock(sk);
3890 return ret;
Johann Baudy69e3c752009-05-18 22:11:22 -07003891 }
Herbert Xu8dc41942007-02-04 23:31:32 -08003892 case PACKET_AUXDATA:
3893 {
3894 int val;
3895
3896 if (optlen < sizeof(val))
3897 return -EINVAL;
Christoph Hellwiga7b75c52020-07-23 08:09:07 +02003898 if (copy_from_sockptr(&val, optval, sizeof(val)))
Herbert Xu8dc41942007-02-04 23:31:32 -08003899 return -EFAULT;
3900
Willem de Bruijna6361f02018-04-23 17:37:03 -04003901 lock_sock(sk);
Herbert Xu8dc41942007-02-04 23:31:32 -08003902 po->auxdata = !!val;
Willem de Bruijna6361f02018-04-23 17:37:03 -04003903 release_sock(sk);
Herbert Xu8dc41942007-02-04 23:31:32 -08003904 return 0;
3905 }
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07003906 case PACKET_ORIGDEV:
3907 {
3908 int val;
3909
3910 if (optlen < sizeof(val))
3911 return -EINVAL;
Christoph Hellwiga7b75c52020-07-23 08:09:07 +02003912 if (copy_from_sockptr(&val, optval, sizeof(val)))
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07003913 return -EFAULT;
3914
Willem de Bruijna6361f02018-04-23 17:37:03 -04003915 lock_sock(sk);
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07003916 po->origdev = !!val;
Willem de Bruijna6361f02018-04-23 17:37:03 -04003917 release_sock(sk);
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07003918 return 0;
3919 }
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003920 case PACKET_VNET_HDR:
3921 {
3922 int val;
3923
3924 if (sock->type != SOCK_RAW)
3925 return -EINVAL;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003926 if (optlen < sizeof(val))
3927 return -EINVAL;
Christoph Hellwiga7b75c52020-07-23 08:09:07 +02003928 if (copy_from_sockptr(&val, optval, sizeof(val)))
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003929 return -EFAULT;
3930
Willem de Bruijna6361f02018-04-23 17:37:03 -04003931 lock_sock(sk);
3932 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
3933 ret = -EBUSY;
3934 } else {
3935 po->has_vnet_hdr = !!val;
3936 ret = 0;
3937 }
3938 release_sock(sk);
3939 return ret;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003940 }
Scott McMillan614f60f2010-06-02 05:53:56 -07003941 case PACKET_TIMESTAMP:
3942 {
3943 int val;
3944
3945 if (optlen != sizeof(val))
3946 return -EINVAL;
Christoph Hellwiga7b75c52020-07-23 08:09:07 +02003947 if (copy_from_sockptr(&val, optval, sizeof(val)))
Scott McMillan614f60f2010-06-02 05:53:56 -07003948 return -EFAULT;
3949
3950 po->tp_tstamp = val;
3951 return 0;
3952 }
David S. Millerdc99f602011-07-05 01:45:05 -07003953 case PACKET_FANOUT:
3954 {
Tanner Love9c661b02020-11-06 13:07:40 -05003955 struct fanout_args args = { 0 };
David S. Millerdc99f602011-07-05 01:45:05 -07003956
Tanner Love9c661b02020-11-06 13:07:40 -05003957 if (optlen != sizeof(int) && optlen != sizeof(args))
David S. Millerdc99f602011-07-05 01:45:05 -07003958 return -EINVAL;
Tanner Love9c661b02020-11-06 13:07:40 -05003959 if (copy_from_sockptr(&args, optval, optlen))
David S. Millerdc99f602011-07-05 01:45:05 -07003960 return -EFAULT;
3961
Tanner Love9c661b02020-11-06 13:07:40 -05003962 return fanout_add(sk, &args);
David S. Millerdc99f602011-07-05 01:45:05 -07003963 }
Willem de Bruijn47dceb82015-08-14 22:31:34 -04003964 case PACKET_FANOUT_DATA:
3965 {
Eric Dumazete42e70a2022-01-31 18:23:58 -08003966 /* Paired with the WRITE_ONCE() in fanout_add() */
3967 if (!READ_ONCE(po->fanout))
Willem de Bruijn47dceb82015-08-14 22:31:34 -04003968 return -EINVAL;
3969
3970 return fanout_set_data(po, optval, optlen);
3971 }
Vincent Whitchurchfa788d92018-09-03 16:23:36 +02003972 case PACKET_IGNORE_OUTGOING:
3973 {
3974 int val;
3975
3976 if (optlen != sizeof(val))
3977 return -EINVAL;
Christoph Hellwiga7b75c52020-07-23 08:09:07 +02003978 if (copy_from_sockptr(&val, optval, sizeof(val)))
Vincent Whitchurchfa788d92018-09-03 16:23:36 +02003979 return -EFAULT;
3980 if (val < 0 || val > 1)
3981 return -EINVAL;
3982
3983 po->prot_hook.ignore_outgoing = !!val;
3984 return 0;
3985 }
Paul Chavent5920cd3a2012-11-06 23:10:47 +00003986 case PACKET_TX_HAS_OFF:
3987 {
3988 unsigned int val;
3989
3990 if (optlen != sizeof(val))
3991 return -EINVAL;
Christoph Hellwiga7b75c52020-07-23 08:09:07 +02003992 if (copy_from_sockptr(&val, optval, sizeof(val)))
Paul Chavent5920cd3a2012-11-06 23:10:47 +00003993 return -EFAULT;
Willem de Bruijna6361f02018-04-23 17:37:03 -04003994
3995 lock_sock(sk);
Jiapeng Chong25c55b382021-05-17 18:15:25 +08003996 if (!po->rx_ring.pg_vec && !po->tx_ring.pg_vec)
Willem de Bruijna6361f02018-04-23 17:37:03 -04003997 po->tp_tx_has_off = !!val;
Jiapeng Chong25c55b382021-05-17 18:15:25 +08003998
Willem de Bruijna6361f02018-04-23 17:37:03 -04003999 release_sock(sk);
Paul Chavent5920cd3a2012-11-06 23:10:47 +00004000 return 0;
4001 }
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01004002 case PACKET_QDISC_BYPASS:
4003 {
4004 int val;
4005
4006 if (optlen != sizeof(val))
4007 return -EINVAL;
Christoph Hellwiga7b75c52020-07-23 08:09:07 +02004008 if (copy_from_sockptr(&val, optval, sizeof(val)))
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01004009 return -EFAULT;
4010
4011 po->xmit = val ? packet_direct_xmit : dev_queue_xmit;
4012 return 0;
4013 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004014 default:
4015 return -ENOPROTOOPT;
4016 }
4017}
4018
4019static int packet_getsockopt(struct socket *sock, int level, int optname,
4020 char __user *optval, int __user *optlen)
4021{
4022 int len;
Eric Dumazetc06fff62012-04-19 21:56:11 +00004023 int val, lv = sizeof(val);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004024 struct sock *sk = sock->sk;
4025 struct packet_sock *po = pkt_sk(sk);
Eric Dumazetc06fff62012-04-19 21:56:11 +00004026 void *data = &val;
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00004027 union tpacket_stats_u st;
Willem de Bruijna9b63912015-05-12 11:56:50 -04004028 struct tpacket_rollover_stats rstats;
Eric Dumazet8e8e2952019-06-12 09:52:30 -07004029 int drops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004030
4031 if (level != SOL_PACKET)
4032 return -ENOPROTOOPT;
4033
Kris Katterjohn8ae55f02006-01-23 16:28:02 -08004034 if (get_user(len, optlen))
4035 return -EFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004036
4037 if (len < 0)
4038 return -EINVAL;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09004039
Johann Baudy69e3c752009-05-18 22:11:22 -07004040 switch (optname) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004041 case PACKET_STATISTICS:
Linus Torvalds1da177e2005-04-16 15:20:36 -07004042 spin_lock_bh(&sk->sk_receive_queue.lock);
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00004043 memcpy(&st, &po->stats, sizeof(st));
4044 memset(&po->stats, 0, sizeof(po->stats));
4045 spin_unlock_bh(&sk->sk_receive_queue.lock);
Eric Dumazet8e8e2952019-06-12 09:52:30 -07004046 drops = atomic_xchg(&po->tp_drops, 0);
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00004047
chetan lokef6fb8f12011-08-19 10:18:16 +00004048 if (po->tp_version == TPACKET_V3) {
Eric Dumazetc06fff62012-04-19 21:56:11 +00004049 lv = sizeof(struct tpacket_stats_v3);
Eric Dumazet8e8e2952019-06-12 09:52:30 -07004050 st.stats3.tp_drops = drops;
4051 st.stats3.tp_packets += drops;
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00004052 data = &st.stats3;
chetan lokef6fb8f12011-08-19 10:18:16 +00004053 } else {
Eric Dumazetc06fff62012-04-19 21:56:11 +00004054 lv = sizeof(struct tpacket_stats);
Eric Dumazet8e8e2952019-06-12 09:52:30 -07004055 st.stats1.tp_drops = drops;
4056 st.stats1.tp_packets += drops;
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00004057 data = &st.stats1;
chetan lokef6fb8f12011-08-19 10:18:16 +00004058 }
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00004059
Linus Torvalds1da177e2005-04-16 15:20:36 -07004060 break;
Herbert Xu8dc41942007-02-04 23:31:32 -08004061 case PACKET_AUXDATA:
Herbert Xu8dc41942007-02-04 23:31:32 -08004062 val = po->auxdata;
Herbert Xu8dc41942007-02-04 23:31:32 -08004063 break;
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07004064 case PACKET_ORIGDEV:
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07004065 val = po->origdev;
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07004066 break;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08004067 case PACKET_VNET_HDR:
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08004068 val = po->has_vnet_hdr;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08004069 break;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07004070 case PACKET_VERSION:
Patrick McHardybbd6ef82008-07-14 22:50:15 -07004071 val = po->tp_version;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07004072 break;
4073 case PACKET_HDRLEN:
4074 if (len > sizeof(int))
4075 len = sizeof(int);
Alexander Potapenkofd2c83b2017-04-25 18:51:46 +02004076 if (len < sizeof(int))
4077 return -EINVAL;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07004078 if (copy_from_user(&val, optval, len))
4079 return -EFAULT;
4080 switch (val) {
4081 case TPACKET_V1:
4082 val = sizeof(struct tpacket_hdr);
4083 break;
4084 case TPACKET_V2:
4085 val = sizeof(struct tpacket2_hdr);
4086 break;
chetan lokef6fb8f12011-08-19 10:18:16 +00004087 case TPACKET_V3:
4088 val = sizeof(struct tpacket3_hdr);
4089 break;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07004090 default:
4091 return -EINVAL;
4092 }
Patrick McHardybbd6ef82008-07-14 22:50:15 -07004093 break;
Patrick McHardy8913336a2008-07-18 18:05:19 -07004094 case PACKET_RESERVE:
Patrick McHardy8913336a2008-07-18 18:05:19 -07004095 val = po->tp_reserve;
Patrick McHardy8913336a2008-07-18 18:05:19 -07004096 break;
Johann Baudy69e3c752009-05-18 22:11:22 -07004097 case PACKET_LOSS:
Johann Baudy69e3c752009-05-18 22:11:22 -07004098 val = po->tp_loss;
Johann Baudy69e3c752009-05-18 22:11:22 -07004099 break;
Scott McMillan614f60f2010-06-02 05:53:56 -07004100 case PACKET_TIMESTAMP:
Scott McMillan614f60f2010-06-02 05:53:56 -07004101 val = po->tp_tstamp;
Scott McMillan614f60f2010-06-02 05:53:56 -07004102 break;
David S. Millerdc99f602011-07-05 01:45:05 -07004103 case PACKET_FANOUT:
David S. Millerdc99f602011-07-05 01:45:05 -07004104 val = (po->fanout ?
4105 ((u32)po->fanout->id |
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00004106 ((u32)po->fanout->type << 16) |
4107 ((u32)po->fanout->flags << 24)) :
David S. Millerdc99f602011-07-05 01:45:05 -07004108 0);
David S. Millerdc99f602011-07-05 01:45:05 -07004109 break;
Vincent Whitchurchfa788d92018-09-03 16:23:36 +02004110 case PACKET_IGNORE_OUTGOING:
4111 val = po->prot_hook.ignore_outgoing;
4112 break;
Willem de Bruijna9b63912015-05-12 11:56:50 -04004113 case PACKET_ROLLOVER_STATS:
Mike Maloney57f015f2017-11-28 10:44:29 -05004114 if (!po->rollover)
Willem de Bruijna9b63912015-05-12 11:56:50 -04004115 return -EINVAL;
Mike Maloney57f015f2017-11-28 10:44:29 -05004116 rstats.tp_all = atomic_long_read(&po->rollover->num);
4117 rstats.tp_huge = atomic_long_read(&po->rollover->num_huge);
4118 rstats.tp_failed = atomic_long_read(&po->rollover->num_failed);
4119 data = &rstats;
4120 lv = sizeof(rstats);
Willem de Bruijna9b63912015-05-12 11:56:50 -04004121 break;
Paul Chavent5920cd3a2012-11-06 23:10:47 +00004122 case PACKET_TX_HAS_OFF:
4123 val = po->tp_tx_has_off;
4124 break;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01004125 case PACKET_QDISC_BYPASS:
4126 val = packet_use_direct_xmit(po);
4127 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004128 default:
4129 return -ENOPROTOOPT;
4130 }
4131
Eric Dumazetc06fff62012-04-19 21:56:11 +00004132 if (len > lv)
4133 len = lv;
Kris Katterjohn8ae55f02006-01-23 16:28:02 -08004134 if (put_user(len, optlen))
4135 return -EFAULT;
Herbert Xu8dc41942007-02-04 23:31:32 -08004136 if (copy_to_user(optval, data, len))
4137 return -EFAULT;
Kris Katterjohn8ae55f02006-01-23 16:28:02 -08004138 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004139}
4140
Jiri Pirko351638e2013-05-28 01:30:21 +00004141static int packet_notifier(struct notifier_block *this,
4142 unsigned long msg, void *ptr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004143{
4144 struct sock *sk;
Jiri Pirko351638e2013-05-28 01:30:21 +00004145 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09004146 struct net *net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004147
stephen hemminger808f5112010-02-22 07:57:18 +00004148 rcu_read_lock();
Sasha Levinb67bfe02013-02-27 17:06:00 -08004149 sk_for_each_rcu(sk, &net->packet.sklist) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004150 struct packet_sock *po = pkt_sk(sk);
4151
4152 switch (msg) {
4153 case NETDEV_UNREGISTER:
Linus Torvalds1da177e2005-04-16 15:20:36 -07004154 if (po->mclist)
Francesco Ruggeri82f17092015-03-09 11:51:04 -07004155 packet_dev_mclist_delete(dev, &po->mclist);
Gustavo A. R. Silvadf561f662020-08-23 17:36:59 -05004156 fallthrough;
David S. Millera2efcfa2007-05-29 13:12:50 -07004157
Linus Torvalds1da177e2005-04-16 15:20:36 -07004158 case NETDEV_DOWN:
4159 if (dev->ifindex == po->ifindex) {
4160 spin_lock(&po->bind_lock);
4161 if (po->running) {
David S. Millerce06b032011-07-04 01:44:29 -07004162 __unregister_prot_hook(sk, false);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004163 sk->sk_err = ENETDOWN;
4164 if (!sock_flag(sk, SOCK_DEAD))
Alexander Aringe3ae2362021-06-27 18:48:21 -04004165 sk_error_report(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004166 }
4167 if (msg == NETDEV_UNREGISTER) {
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01004168 packet_cached_dev_reset(po);
Eric Dumazete032f7c2021-06-16 06:42:02 -07004169 WRITE_ONCE(po->ifindex, -1);
Jakub Kicinskid62607c2022-06-07 21:39:55 -07004170 netdev_put(po->prot_hook.dev,
4171 &po->prot_hook.dev_tracker);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004172 po->prot_hook.dev = NULL;
4173 }
4174 spin_unlock(&po->bind_lock);
4175 }
4176 break;
4177 case NETDEV_UP:
stephen hemminger808f5112010-02-22 07:57:18 +00004178 if (dev->ifindex == po->ifindex) {
4179 spin_lock(&po->bind_lock);
David S. Millerce06b032011-07-04 01:44:29 -07004180 if (po->num)
4181 register_prot_hook(sk);
stephen hemminger808f5112010-02-22 07:57:18 +00004182 spin_unlock(&po->bind_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004183 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004184 break;
4185 }
4186 }
stephen hemminger808f5112010-02-22 07:57:18 +00004187 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004188 return NOTIFY_DONE;
4189}
4190
4191
4192static int packet_ioctl(struct socket *sock, unsigned int cmd,
4193 unsigned long arg)
4194{
4195 struct sock *sk = sock->sk;
4196
Johann Baudy69e3c752009-05-18 22:11:22 -07004197 switch (cmd) {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00004198 case SIOCOUTQ:
4199 {
4200 int amount = sk_wmem_alloc_get(sk);
Eric Dumazet31e6d362009-06-17 19:05:41 -07004201
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00004202 return put_user(amount, (int __user *)arg);
4203 }
4204 case SIOCINQ:
4205 {
4206 struct sk_buff *skb;
4207 int amount = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004208
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00004209 spin_lock_bh(&sk->sk_receive_queue.lock);
4210 skb = skb_peek(&sk->sk_receive_queue);
4211 if (skb)
4212 amount = skb->len;
4213 spin_unlock_bh(&sk->sk_receive_queue.lock);
4214 return put_user(amount, (int __user *)arg);
4215 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004216#ifdef CONFIG_INET
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00004217 case SIOCADDRT:
4218 case SIOCDELRT:
4219 case SIOCDARP:
4220 case SIOCGARP:
4221 case SIOCSARP:
4222 case SIOCGIFADDR:
4223 case SIOCSIFADDR:
4224 case SIOCGIFBRDADDR:
4225 case SIOCSIFBRDADDR:
4226 case SIOCGIFNETMASK:
4227 case SIOCSIFNETMASK:
4228 case SIOCGIFDSTADDR:
4229 case SIOCSIFDSTADDR:
4230 case SIOCSIFFLAGS:
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00004231 return inet_dgram_ops.ioctl(sock, cmd, arg);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004232#endif
4233
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00004234 default:
4235 return -ENOIOCTLCMD;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004236 }
4237 return 0;
4238}
4239
Linus Torvaldsa11e1d42018-06-28 09:43:44 -07004240static __poll_t packet_poll(struct file *file, struct socket *sock,
4241 poll_table *wait)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004242{
4243 struct sock *sk = sock->sk;
4244 struct packet_sock *po = pkt_sk(sk);
Linus Torvaldsa11e1d42018-06-28 09:43:44 -07004245 __poll_t mask = datagram_poll(file, sock, wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004246
4247 spin_lock_bh(&sk->sk_receive_queue.lock);
Johann Baudy69e3c752009-05-18 22:11:22 -07004248 if (po->rx_ring.pg_vec) {
chetan lokef6fb8f12011-08-19 10:18:16 +00004249 if (!packet_previous_rx_frame(po, &po->rx_ring,
4250 TP_STATUS_KERNEL))
Linus Torvaldsa9a08842018-02-11 14:34:03 -08004251 mask |= EPOLLIN | EPOLLRDNORM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004252 }
Eric Dumazet9bb6cd62019-06-12 09:52:33 -07004253 packet_rcv_try_clear_pressure(po);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004254 spin_unlock_bh(&sk->sk_receive_queue.lock);
Johann Baudy69e3c752009-05-18 22:11:22 -07004255 spin_lock_bh(&sk->sk_write_queue.lock);
4256 if (po->tx_ring.pg_vec) {
4257 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
Linus Torvaldsa9a08842018-02-11 14:34:03 -08004258 mask |= EPOLLOUT | EPOLLWRNORM;
Johann Baudy69e3c752009-05-18 22:11:22 -07004259 }
4260 spin_unlock_bh(&sk->sk_write_queue.lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004261 return mask;
4262}
4263
4264
4265/* Dirty? Well, I still did not learn better way to account
4266 * for user mmaps.
4267 */
4268
4269static void packet_mm_open(struct vm_area_struct *vma)
4270{
4271 struct file *file = vma->vm_file;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00004272 struct socket *sock = file->private_data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004273 struct sock *sk = sock->sk;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09004274
Linus Torvalds1da177e2005-04-16 15:20:36 -07004275 if (sk)
4276 atomic_inc(&pkt_sk(sk)->mapped);
4277}
4278
4279static void packet_mm_close(struct vm_area_struct *vma)
4280{
4281 struct file *file = vma->vm_file;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00004282 struct socket *sock = file->private_data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004283 struct sock *sk = sock->sk;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09004284
Linus Torvalds1da177e2005-04-16 15:20:36 -07004285 if (sk)
4286 atomic_dec(&pkt_sk(sk)->mapped);
4287}
4288
Alexey Dobriyanf0f37e2f2009-09-27 22:29:37 +04004289static const struct vm_operations_struct packet_mmap_ops = {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00004290 .open = packet_mm_open,
4291 .close = packet_mm_close,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004292};
4293
Eric Dumazet3a7ad062018-08-29 11:50:12 -07004294static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
4295 unsigned int len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004296{
4297 int i;
4298
David S. Miller4ebf0ae2005-12-06 16:38:35 -08004299 for (i = 0; i < len; i++) {
Neil Horman0e3125c2010-11-16 10:26:47 -08004300 if (likely(pg_vec[i].buffer)) {
Eric Dumazet3a7ad062018-08-29 11:50:12 -07004301 if (is_vmalloc_addr(pg_vec[i].buffer))
4302 vfree(pg_vec[i].buffer);
4303 else
4304 free_pages((unsigned long)pg_vec[i].buffer,
4305 order);
Neil Horman0e3125c2010-11-16 10:26:47 -08004306 pg_vec[i].buffer = NULL;
4307 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004308 }
4309 kfree(pg_vec);
4310}
4311
Eric Dumazet3a7ad062018-08-29 11:50:12 -07004312static char *alloc_one_pg_vec_page(unsigned long order)
David S. Miller4ebf0ae2005-12-06 16:38:35 -08004313{
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01004314 char *buffer;
Eric Dumazet3a7ad062018-08-29 11:50:12 -07004315 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
4316 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
Eric Dumazet719bfea2009-04-15 03:39:52 -07004317
Eric Dumazet3a7ad062018-08-29 11:50:12 -07004318 buffer = (char *) __get_free_pages(gfp_flags, order);
Neil Horman0e3125c2010-11-16 10:26:47 -08004319 if (buffer)
4320 return buffer;
4321
Eric Dumazet3a7ad062018-08-29 11:50:12 -07004322 /* __get_free_pages failed, fall back to vmalloc */
4323 buffer = vzalloc(array_size((1 << order), PAGE_SIZE));
4324 if (buffer)
4325 return buffer;
Neil Horman0e3125c2010-11-16 10:26:47 -08004326
Eric Dumazet3a7ad062018-08-29 11:50:12 -07004327 /* vmalloc failed, lets dig into swap here */
4328 gfp_flags &= ~__GFP_NORETRY;
4329 buffer = (char *) __get_free_pages(gfp_flags, order);
4330 if (buffer)
4331 return buffer;
4332
4333 /* complete and utter failure */
4334 return NULL;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08004335}
4336
Eric Dumazet3a7ad062018-08-29 11:50:12 -07004337static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
David S. Miller4ebf0ae2005-12-06 16:38:35 -08004338{
4339 unsigned int block_nr = req->tp_block_nr;
Neil Horman0e3125c2010-11-16 10:26:47 -08004340 struct pgv *pg_vec;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08004341 int i;
4342
Christoph Paasch398f0132019-03-18 23:14:52 -07004343 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL | __GFP_NOWARN);
David S. Miller4ebf0ae2005-12-06 16:38:35 -08004344 if (unlikely(!pg_vec))
4345 goto out;
4346
4347 for (i = 0; i < block_nr; i++) {
Eric Dumazet3a7ad062018-08-29 11:50:12 -07004348 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
Neil Horman0e3125c2010-11-16 10:26:47 -08004349 if (unlikely(!pg_vec[i].buffer))
David S. Miller4ebf0ae2005-12-06 16:38:35 -08004350 goto out_free_pgvec;
4351 }
4352
4353out:
4354 return pg_vec;
4355
4356out_free_pgvec:
Eric Dumazet3a7ad062018-08-29 11:50:12 -07004357 free_pg_vec(pg_vec, order, block_nr);
David S. Miller4ebf0ae2005-12-06 16:38:35 -08004358 pg_vec = NULL;
4359 goto out;
4360}
Linus Torvalds1da177e2005-04-16 15:20:36 -07004361
chetan lokef6fb8f12011-08-19 10:18:16 +00004362static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
Johann Baudy69e3c752009-05-18 22:11:22 -07004363 int closing, int tx_ring)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004364{
Neil Horman0e3125c2010-11-16 10:26:47 -08004365 struct pgv *pg_vec = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004366 struct packet_sock *po = pkt_sk(sk);
Willem de Bruijn61fad682020-03-13 12:18:09 -04004367 unsigned long *rx_owner_map = NULL;
Eric Dumazet3a7ad062018-08-29 11:50:12 -07004368 int was_running, order = 0;
Johann Baudy69e3c752009-05-18 22:11:22 -07004369 struct packet_ring_buffer *rb;
4370 struct sk_buff_head *rb_queue;
Al Viro0e11c912006-11-08 00:26:29 -08004371 __be16 num;
Colin Ian King2a6d6c32020-07-01 16:04:33 +01004372 int err;
chetan lokef6fb8f12011-08-19 10:18:16 +00004373 /* Added to avoid minimal code churn */
4374 struct tpacket_req *req = &req_u->req;
4375
Johann Baudy69e3c752009-05-18 22:11:22 -07004376 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
4377 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
4378
4379 err = -EBUSY;
4380 if (!closing) {
4381 if (atomic_read(&po->mapped))
4382 goto out;
Daniel Borkmannb0138402014-01-15 16:25:36 +01004383 if (packet_read_pending(rb))
Johann Baudy69e3c752009-05-18 22:11:22 -07004384 goto out;
4385 }
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09004386
Linus Torvalds1da177e2005-04-16 15:20:36 -07004387 if (req->tp_block_nr) {
Willem de Bruijn4576cd42018-08-06 10:38:34 -04004388 unsigned int min_frame_size;
4389
Linus Torvalds1da177e2005-04-16 15:20:36 -07004390 /* Sanity tests and some calculations */
Johann Baudy69e3c752009-05-18 22:11:22 -07004391 err = -EBUSY;
4392 if (unlikely(rb->pg_vec))
4393 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004394
Patrick McHardybbd6ef82008-07-14 22:50:15 -07004395 switch (po->tp_version) {
4396 case TPACKET_V1:
4397 po->tp_hdrlen = TPACKET_HDRLEN;
4398 break;
4399 case TPACKET_V2:
4400 po->tp_hdrlen = TPACKET2_HDRLEN;
4401 break;
chetan lokef6fb8f12011-08-19 10:18:16 +00004402 case TPACKET_V3:
4403 po->tp_hdrlen = TPACKET3_HDRLEN;
4404 break;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07004405 }
4406
Johann Baudy69e3c752009-05-18 22:11:22 -07004407 err = -EINVAL;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08004408 if (unlikely((int)req->tp_block_size <= 0))
Johann Baudy69e3c752009-05-18 22:11:22 -07004409 goto out;
Tobias Klauser90836b62015-11-17 10:40:21 +01004410 if (unlikely(!PAGE_ALIGNED(req->tp_block_size)))
Johann Baudy69e3c752009-05-18 22:11:22 -07004411 goto out;
Willem de Bruijn4576cd42018-08-06 10:38:34 -04004412 min_frame_size = po->tp_hdrlen + po->tp_reserve;
Eric Dumazetdc808112014-08-15 09:16:04 -07004413 if (po->tp_version >= TPACKET_V3 &&
Willem de Bruijn4576cd42018-08-06 10:38:34 -04004414 req->tp_block_size <
4415 BLK_PLUS_PRIV((u64)req_u->req3.tp_sizeof_priv) + min_frame_size)
Eric Dumazetdc808112014-08-15 09:16:04 -07004416 goto out;
Willem de Bruijn4576cd42018-08-06 10:38:34 -04004417 if (unlikely(req->tp_frame_size < min_frame_size))
Johann Baudy69e3c752009-05-18 22:11:22 -07004418 goto out;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08004419 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
Johann Baudy69e3c752009-05-18 22:11:22 -07004420 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004421
Tobias Klauser4194b492015-11-17 10:38:36 +01004422 rb->frames_per_block = req->tp_block_size / req->tp_frame_size;
4423 if (unlikely(rb->frames_per_block == 0))
Johann Baudy69e3c752009-05-18 22:11:22 -07004424 goto out;
Kal Conleyfc628142019-02-10 09:57:11 +01004425 if (unlikely(rb->frames_per_block > UINT_MAX / req->tp_block_nr))
Andrey Konovalov8f8d28e2017-03-29 16:11:21 +02004426 goto out;
Johann Baudy69e3c752009-05-18 22:11:22 -07004427 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
4428 req->tp_frame_nr))
4429 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004430
4431 err = -ENOMEM;
Eric Dumazet3a7ad062018-08-29 11:50:12 -07004432 order = get_order(req->tp_block_size);
4433 pg_vec = alloc_pg_vec(req, order);
David S. Miller4ebf0ae2005-12-06 16:38:35 -08004434 if (unlikely(!pg_vec))
Linus Torvalds1da177e2005-04-16 15:20:36 -07004435 goto out;
chetan lokef6fb8f12011-08-19 10:18:16 +00004436 switch (po->tp_version) {
4437 case TPACKET_V3:
Sowmini Varadhan7f953ab2017-01-03 06:31:47 -08004438 /* Block transmit is not supported yet */
4439 if (!tx_ring) {
Maninder Singhe8e85cc2015-06-22 12:39:16 +05304440 init_prb_bdqc(po, rb, pg_vec, req_u);
Sowmini Varadhan7f953ab2017-01-03 06:31:47 -08004441 } else {
4442 struct tpacket_req3 *req3 = &req_u->req3;
4443
4444 if (req3->tp_retire_blk_tov ||
4445 req3->tp_sizeof_priv ||
4446 req3->tp_feature_req_word) {
4447 err = -EINVAL;
Eric Dumazet55655e32019-06-24 02:38:20 -07004448 goto out_free_pg_vec;
Sowmini Varadhan7f953ab2017-01-03 06:31:47 -08004449 }
4450 }
Dan Carpenterd7cf0c32014-02-18 15:20:51 +03004451 break;
chetan lokef6fb8f12011-08-19 10:18:16 +00004452 default:
Willem de Bruijn61fad682020-03-13 12:18:09 -04004453 if (!tx_ring) {
4454 rx_owner_map = bitmap_alloc(req->tp_frame_nr,
4455 GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO);
4456 if (!rx_owner_map)
4457 goto out_free_pg_vec;
4458 }
chetan lokef6fb8f12011-08-19 10:18:16 +00004459 break;
4460 }
Johann Baudy69e3c752009-05-18 22:11:22 -07004461 }
4462 /* Done */
4463 else {
4464 err = -EINVAL;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08004465 if (unlikely(req->tp_frame_nr))
Johann Baudy69e3c752009-05-18 22:11:22 -07004466 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004467 }
4468
Linus Torvalds1da177e2005-04-16 15:20:36 -07004469
4470 /* Detach socket from network */
4471 spin_lock(&po->bind_lock);
4472 was_running = po->running;
4473 num = po->num;
4474 if (was_running) {
Eric Dumazetc7d2ef52021-06-16 06:42:01 -07004475 WRITE_ONCE(po->num, 0);
David S. Millerce06b032011-07-04 01:44:29 -07004476 __unregister_prot_hook(sk, false);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004477 }
4478 spin_unlock(&po->bind_lock);
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09004479
Linus Torvalds1da177e2005-04-16 15:20:36 -07004480 synchronize_net();
4481
4482 err = -EBUSY;
Herbert Xu905db442009-01-30 14:12:06 -08004483 mutex_lock(&po->pg_vec_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004484 if (closing || atomic_read(&po->mapped) == 0) {
4485 err = 0;
Johann Baudy69e3c752009-05-18 22:11:22 -07004486 spin_lock_bh(&rb_queue->lock);
Changli Gaoc053fd92010-12-10 16:02:20 -08004487 swap(rb->pg_vec, pg_vec);
Willem de Bruijn61fad682020-03-13 12:18:09 -04004488 if (po->tp_version <= TPACKET_V2)
4489 swap(rb->rx_owner_map, rx_owner_map);
Johann Baudy69e3c752009-05-18 22:11:22 -07004490 rb->frame_max = (req->tp_frame_nr - 1);
4491 rb->head = 0;
4492 rb->frame_size = req->tp_frame_size;
4493 spin_unlock_bh(&rb_queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004494
Eric Dumazet3a7ad062018-08-29 11:50:12 -07004495 swap(rb->pg_vec_order, order);
Changli Gaoc053fd92010-12-10 16:02:20 -08004496 swap(rb->pg_vec_len, req->tp_block_nr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004497
Johann Baudy69e3c752009-05-18 22:11:22 -07004498 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
4499 po->prot_hook.func = (po->rx_ring.pg_vec) ?
4500 tpacket_rcv : packet_rcv;
4501 skb_queue_purge(rb_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004502 if (atomic_read(&po->mapped))
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00004503 pr_err("packet_mmap: vma is busy: %d\n",
4504 atomic_read(&po->mapped));
Linus Torvalds1da177e2005-04-16 15:20:36 -07004505 }
Herbert Xu905db442009-01-30 14:12:06 -08004506 mutex_unlock(&po->pg_vec_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004507
4508 spin_lock(&po->bind_lock);
David S. Millerce06b032011-07-04 01:44:29 -07004509 if (was_running) {
Eric Dumazetc7d2ef52021-06-16 06:42:01 -07004510 WRITE_ONCE(po->num, num);
David S. Millerce06b032011-07-04 01:44:29 -07004511 register_prot_hook(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004512 }
4513 spin_unlock(&po->bind_lock);
WANG Congc800aaf2017-07-24 10:07:32 -07004514 if (pg_vec && (po->tp_version > TPACKET_V2)) {
chetan lokef6fb8f12011-08-19 10:18:16 +00004515 /* Because we don't support block-based V3 on tx-ring */
4516 if (!tx_ring)
Tobias Klauser73d0fcf2015-07-28 14:21:26 +02004517 prb_shutdown_retire_blk_timer(po, rb_queue);
chetan lokef6fb8f12011-08-19 10:18:16 +00004518 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004519
Eric Dumazet55655e32019-06-24 02:38:20 -07004520out_free_pg_vec:
Willem de Bruijnec6af092021-12-15 09:39:37 -05004521 if (pg_vec) {
4522 bitmap_free(rx_owner_map);
Eric Dumazet3a7ad062018-08-29 11:50:12 -07004523 free_pg_vec(pg_vec, order, req->tp_block_nr);
Willem de Bruijnec6af092021-12-15 09:39:37 -05004524 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004525out:
4526 return err;
4527}
4528
Johann Baudy69e3c752009-05-18 22:11:22 -07004529static int packet_mmap(struct file *file, struct socket *sock,
4530 struct vm_area_struct *vma)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004531{
4532 struct sock *sk = sock->sk;
4533 struct packet_sock *po = pkt_sk(sk);
Johann Baudy69e3c752009-05-18 22:11:22 -07004534 unsigned long size, expected_size;
4535 struct packet_ring_buffer *rb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004536 unsigned long start;
4537 int err = -EINVAL;
4538 int i;
4539
4540 if (vma->vm_pgoff)
4541 return -EINVAL;
4542
Herbert Xu905db442009-01-30 14:12:06 -08004543 mutex_lock(&po->pg_vec_lock);
Johann Baudy69e3c752009-05-18 22:11:22 -07004544
4545 expected_size = 0;
4546 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
4547 if (rb->pg_vec) {
4548 expected_size += rb->pg_vec_len
4549 * rb->pg_vec_pages
4550 * PAGE_SIZE;
4551 }
4552 }
4553
4554 if (expected_size == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004555 goto out;
Johann Baudy69e3c752009-05-18 22:11:22 -07004556
4557 size = vma->vm_end - vma->vm_start;
4558 if (size != expected_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004559 goto out;
4560
Linus Torvalds1da177e2005-04-16 15:20:36 -07004561 start = vma->vm_start;
Johann Baudy69e3c752009-05-18 22:11:22 -07004562 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
4563 if (rb->pg_vec == NULL)
4564 continue;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08004565
Johann Baudy69e3c752009-05-18 22:11:22 -07004566 for (i = 0; i < rb->pg_vec_len; i++) {
Neil Horman0e3125c2010-11-16 10:26:47 -08004567 struct page *page;
4568 void *kaddr = rb->pg_vec[i].buffer;
Johann Baudy69e3c752009-05-18 22:11:22 -07004569 int pg_num;
4570
Changli Gaoc56b4d92010-12-01 02:52:57 +00004571 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
4572 page = pgv_to_page(kaddr);
Johann Baudy69e3c752009-05-18 22:11:22 -07004573 err = vm_insert_page(vma, start, page);
4574 if (unlikely(err))
4575 goto out;
4576 start += PAGE_SIZE;
Neil Horman0e3125c2010-11-16 10:26:47 -08004577 kaddr += PAGE_SIZE;
Johann Baudy69e3c752009-05-18 22:11:22 -07004578 }
David S. Miller4ebf0ae2005-12-06 16:38:35 -08004579 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004580 }
Johann Baudy69e3c752009-05-18 22:11:22 -07004581
David S. Miller4ebf0ae2005-12-06 16:38:35 -08004582 atomic_inc(&po->mapped);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004583 vma->vm_ops = &packet_mmap_ops;
4584 err = 0;
4585
4586out:
Herbert Xu905db442009-01-30 14:12:06 -08004587 mutex_unlock(&po->pg_vec_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004588 return err;
4589}
Linus Torvalds1da177e2005-04-16 15:20:36 -07004590
Eric Dumazet90ddc4f2005-12-22 12:49:22 -08004591static const struct proto_ops packet_ops_spkt = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004592 .family = PF_PACKET,
4593 .owner = THIS_MODULE,
4594 .release = packet_release,
4595 .bind = packet_bind_spkt,
4596 .connect = sock_no_connect,
4597 .socketpair = sock_no_socketpair,
4598 .accept = sock_no_accept,
4599 .getname = packet_getname_spkt,
Linus Torvaldsa11e1d42018-06-28 09:43:44 -07004600 .poll = datagram_poll,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004601 .ioctl = packet_ioctl,
Arnd Bergmannc7cbdbf2019-04-17 22:51:48 +02004602 .gettstamp = sock_gettstamp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004603 .listen = sock_no_listen,
4604 .shutdown = sock_no_shutdown,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004605 .sendmsg = packet_sendmsg_spkt,
4606 .recvmsg = packet_recvmsg,
4607 .mmap = sock_no_mmap,
4608 .sendpage = sock_no_sendpage,
4609};
Linus Torvalds1da177e2005-04-16 15:20:36 -07004610
Eric Dumazet90ddc4f2005-12-22 12:49:22 -08004611static const struct proto_ops packet_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004612 .family = PF_PACKET,
4613 .owner = THIS_MODULE,
4614 .release = packet_release,
4615 .bind = packet_bind,
4616 .connect = sock_no_connect,
4617 .socketpair = sock_no_socketpair,
4618 .accept = sock_no_accept,
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09004619 .getname = packet_getname,
Linus Torvaldsa11e1d42018-06-28 09:43:44 -07004620 .poll = packet_poll,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004621 .ioctl = packet_ioctl,
Arnd Bergmannc7cbdbf2019-04-17 22:51:48 +02004622 .gettstamp = sock_gettstamp,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004623 .listen = sock_no_listen,
4624 .shutdown = sock_no_shutdown,
4625 .setsockopt = packet_setsockopt,
4626 .getsockopt = packet_getsockopt,
4627 .sendmsg = packet_sendmsg,
4628 .recvmsg = packet_recvmsg,
4629 .mmap = packet_mmap,
4630 .sendpage = sock_no_sendpage,
4631};
4632
Stephen Hemmingerec1b4cf2009-10-05 05:58:39 +00004633static const struct net_proto_family packet_family_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004634 .family = PF_PACKET,
4635 .create = packet_create,
4636 .owner = THIS_MODULE,
4637};
4638
4639static struct notifier_block packet_netdev_notifier = {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00004640 .notifier_call = packet_notifier,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004641};
4642
4643#ifdef CONFIG_PROC_FS
Linus Torvalds1da177e2005-04-16 15:20:36 -07004644
4645static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
stephen hemminger808f5112010-02-22 07:57:18 +00004646 __acquires(RCU)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004647{
Denis V. Luneve372c412007-11-19 22:31:54 -08004648 struct net *net = seq_file_net(seq);
stephen hemminger808f5112010-02-22 07:57:18 +00004649
4650 rcu_read_lock();
4651 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004652}
4653
4654static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4655{
Herbert Xu1bf40952007-12-16 14:04:02 -08004656 struct net *net = seq_file_net(seq);
stephen hemminger808f5112010-02-22 07:57:18 +00004657 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004658}
4659
4660static void packet_seq_stop(struct seq_file *seq, void *v)
stephen hemminger808f5112010-02-22 07:57:18 +00004661 __releases(RCU)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004662{
stephen hemminger808f5112010-02-22 07:57:18 +00004663 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004664}
4665
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09004666static int packet_seq_show(struct seq_file *seq, void *v)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004667{
4668 if (v == SEQ_START_TOKEN)
Baruch Siachabdcd062020-12-16 09:28:04 +02004669 seq_printf(seq,
4670 "%*sRefCnt Type Proto Iface R Rmem User Inode\n",
4671 IS_ENABLED(CONFIG_64BIT) ? -17 : -9, "sk");
Linus Torvalds1da177e2005-04-16 15:20:36 -07004672 else {
Li Zefanb7ceabd2010-02-08 23:19:29 +00004673 struct sock *s = sk_entry(v);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004674 const struct packet_sock *po = pkt_sk(s);
4675
4676 seq_printf(seq,
Dan Rosenberg71338aa2011-05-23 12:17:35 +00004677 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07004678 s,
Reshetova, Elena41c6d652017-06-30 13:08:01 +03004679 refcount_read(&s->sk_refcnt),
Linus Torvalds1da177e2005-04-16 15:20:36 -07004680 s->sk_type,
Eric Dumazetc7d2ef52021-06-16 06:42:01 -07004681 ntohs(READ_ONCE(po->num)),
Eric Dumazete032f7c2021-06-16 06:42:02 -07004682 READ_ONCE(po->ifindex),
Linus Torvalds1da177e2005-04-16 15:20:36 -07004683 po->running,
4684 atomic_read(&s->sk_rmem_alloc),
Eric W. Biedermana7cb5a42012-05-24 01:10:10 -06004685 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00004686 sock_i_ino(s));
Linus Torvalds1da177e2005-04-16 15:20:36 -07004687 }
4688
4689 return 0;
4690}
4691
Philippe De Muyter56b3d972007-07-10 23:07:31 -07004692static const struct seq_operations packet_seq_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004693 .start = packet_seq_start,
4694 .next = packet_seq_next,
4695 .stop = packet_seq_stop,
4696 .show = packet_seq_show,
4697};
Linus Torvalds1da177e2005-04-16 15:20:36 -07004698#endif
4699
Alexey Dobriyan2c8c1e72010-01-17 03:35:32 +00004700static int __net_init packet_net_init(struct net *net)
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004701{
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00004702 mutex_init(&net->packet.sklist_lock);
Denis V. Lunev2aaef4e2007-12-11 04:19:54 -08004703 INIT_HLIST_HEAD(&net->packet.sklist);
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004704
Yonatan Linika268e0f2020-12-14 22:25:50 +02004705#ifdef CONFIG_PROC_FS
Christoph Hellwigc3506372018-04-10 19:42:55 +02004706 if (!proc_create_net("packet", 0, net->proc_net, &packet_seq_ops,
4707 sizeof(struct seq_net_private)))
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004708 return -ENOMEM;
Yonatan Linika268e0f2020-12-14 22:25:50 +02004709#endif /* CONFIG_PROC_FS */
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004710
4711 return 0;
4712}
4713
Alexey Dobriyan2c8c1e72010-01-17 03:35:32 +00004714static void __net_exit packet_net_exit(struct net *net)
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004715{
Gao fengece31ff2013-02-18 01:34:56 +00004716 remove_proc_entry("packet", net->proc_net);
Vasily Averin669f8f12017-11-12 22:27:49 +03004717 WARN_ON_ONCE(!hlist_empty(&net->packet.sklist));
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004718}
4719
4720static struct pernet_operations packet_net_ops = {
4721 .init = packet_net_init,
4722 .exit = packet_net_exit,
4723};
4724
4725
Linus Torvalds1da177e2005-04-16 15:20:36 -07004726static void __exit packet_exit(void)
4727{
Linus Torvalds1da177e2005-04-16 15:20:36 -07004728 sock_unregister(PF_PACKET);
4729 proto_unregister(&packet_proto);
Ziyang Xuan63b7c2e2022-09-15 09:08:35 +08004730 unregister_netdevice_notifier(&packet_netdev_notifier);
4731 unregister_pernet_subsys(&packet_net_ops);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004732}
4733
4734static int __init packet_init(void)
4735{
YueHaibing36096f22019-05-09 22:52:20 +08004736 int rc;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004737
YueHaibing36096f22019-05-09 22:52:20 +08004738 rc = register_pernet_subsys(&packet_net_ops);
4739 if (rc)
Ziyang Xuan63b7c2e2022-09-15 09:08:35 +08004740 goto out;
YueHaibing36096f22019-05-09 22:52:20 +08004741 rc = register_netdevice_notifier(&packet_netdev_notifier);
4742 if (rc)
4743 goto out_pernet;
Ziyang Xuan63b7c2e2022-09-15 09:08:35 +08004744 rc = proto_register(&packet_proto, 0);
4745 if (rc)
4746 goto out_notifier;
4747 rc = sock_register(&packet_family_ops);
4748 if (rc)
4749 goto out_proto;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004750
YueHaibing36096f22019-05-09 22:52:20 +08004751 return 0;
4752
YueHaibing36096f22019-05-09 22:52:20 +08004753out_proto:
4754 proto_unregister(&packet_proto);
Ziyang Xuan63b7c2e2022-09-15 09:08:35 +08004755out_notifier:
4756 unregister_netdevice_notifier(&packet_netdev_notifier);
4757out_pernet:
4758 unregister_pernet_subsys(&packet_net_ops);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004759out:
4760 return rc;
4761}
4762
4763module_init(packet_init);
4764module_exit(packet_exit);
4765MODULE_LICENSE("GPL");
4766MODULE_ALIAS_NETPROTO(PF_PACKET);