blob: 586229a14ad3984f154753d45ac357613c5a442a [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * PACKET - implements raw packet sockets.
7 *
Jesper Juhl02c30a82005-05-05 16:16:16 -07008 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -07009 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 *
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +090012 * Fixes:
Linus Torvalds1da177e2005-04-16 15:20:36 -070013 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +090035 * Ulises Alonso : Frame number limit removal and
Linus Torvalds1da177e2005-04-16 15:20:36 -070036 * packet_set_ring memory leak.
Eric W. Biederman0fb375f2005-09-21 00:11:37 -070037 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +090040 * byte arrays at the end of sockaddr_ll
Eric W. Biederman0fb375f2005-09-21 00:11:37 -070041 * and packet_mreq.
Johann Baudy69e3c752009-05-18 22:11:22 -070042 * Johann Baudy : Added TX RING.
chetan lokef6fb8f12011-08-19 10:18:16 +000043 * Chetan Loke : Implemented TPACKET_V3 block abstraction
44 * layer.
45 * Copyright (C) 2011, <lokec@ccs.neu.edu>
46 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070047 *
48 * This program is free software; you can redistribute it and/or
49 * modify it under the terms of the GNU General Public License
50 * as published by the Free Software Foundation; either version
51 * 2 of the License, or (at your option) any later version.
52 *
53 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +090054
Linus Torvalds1da177e2005-04-16 15:20:36 -070055#include <linux/types.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070056#include <linux/mm.h>
Randy Dunlap4fc268d2006-01-11 12:17:47 -080057#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070058#include <linux/fcntl.h>
59#include <linux/socket.h>
60#include <linux/in.h>
61#include <linux/inet.h>
62#include <linux/netdevice.h>
63#include <linux/if_packet.h>
64#include <linux/wireless.h>
Herbert Xuffbc6112007-02-04 23:33:10 -080065#include <linux/kernel.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070066#include <linux/kmod.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090067#include <linux/slab.h>
Neil Horman0e3125c2010-11-16 10:26:47 -080068#include <linux/vmalloc.h>
Eric W. Biederman457c4cb2007-09-12 12:01:34 +020069#include <net/net_namespace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070070#include <net/ip.h>
71#include <net/protocol.h>
72#include <linux/skbuff.h>
73#include <net/sock.h>
74#include <linux/errno.h>
75#include <linux/timer.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070076#include <asm/uaccess.h>
77#include <asm/ioctls.h>
78#include <asm/page.h>
Al Viroa1f8e7f72006-10-19 16:08:53 -040079#include <asm/cacheflush.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070080#include <asm/io.h>
81#include <linux/proc_fs.h>
82#include <linux/seq_file.h>
83#include <linux/poll.h>
84#include <linux/module.h>
85#include <linux/init.h>
Herbert Xu905db442009-01-30 14:12:06 -080086#include <linux/mutex.h>
Eric Dumazet05423b22009-10-26 18:40:35 -070087#include <linux/if_vlan.h>
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -080088#include <linux/virtio_net.h>
Richard Cochraned85b562010-04-07 22:41:28 +000089#include <linux/errqueue.h>
Scott McMillan614f60f2010-06-02 05:53:56 -070090#include <linux/net_tstamp.h>
Daniel Borkmannb0138402014-01-15 16:25:36 +010091#include <linux/percpu.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070092#ifdef CONFIG_INET
93#include <net/inet_common.h>
94#endif
95
Pavel Emelyanov2787b042012-08-13 05:49:39 +000096#include "internal.h"
97
Linus Torvalds1da177e2005-04-16 15:20:36 -070098/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070099 Assumptions:
100 - if device has no dev->hard_header routine, it adds and removes ll header
101 inside itself. In this case ll header is invisible outside of device,
102 but higher levels still should reserve dev->hard_header_len.
103 Some devices are enough clever to reallocate skb, when header
104 will not fit to reserved space (tunnel), another ones are silly
105 (PPP).
106 - packet socket receives packets with pulled ll header,
107 so that SOCK_RAW should push it back.
108
109On receive:
110-----------
111
112Incoming, dev->hard_header!=NULL
Arnaldo Carvalho de Melob0e380b12007-04-10 21:21:55 -0700113 mac_header -> ll header
114 data -> data
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115
116Outgoing, dev->hard_header!=NULL
Arnaldo Carvalho de Melob0e380b12007-04-10 21:21:55 -0700117 mac_header -> ll header
118 data -> ll header
Linus Torvalds1da177e2005-04-16 15:20:36 -0700119
120Incoming, dev->hard_header==NULL
Arnaldo Carvalho de Melob0e380b12007-04-10 21:21:55 -0700121 mac_header -> UNKNOWN position. It is very likely, that it points to ll
122 header. PPP makes it, that is wrong, because introduce
YOSHIFUJI Hideakidb0c58f2007-07-19 10:44:35 +0900123 assymetry between rx and tx paths.
Arnaldo Carvalho de Melob0e380b12007-04-10 21:21:55 -0700124 data -> data
Linus Torvalds1da177e2005-04-16 15:20:36 -0700125
126Outgoing, dev->hard_header==NULL
Arnaldo Carvalho de Melob0e380b12007-04-10 21:21:55 -0700127 mac_header -> data. ll header is still not built!
128 data -> data
Linus Torvalds1da177e2005-04-16 15:20:36 -0700129
130Resume
131 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
132
133
134On transmit:
135------------
136
137dev->hard_header != NULL
Arnaldo Carvalho de Melob0e380b12007-04-10 21:21:55 -0700138 mac_header -> ll header
139 data -> ll header
Linus Torvalds1da177e2005-04-16 15:20:36 -0700140
141dev->hard_header == NULL (ll header is added by device, we cannot control it)
Arnaldo Carvalho de Melob0e380b12007-04-10 21:21:55 -0700142 mac_header -> data
143 data -> data
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144
145 We should set nh.raw on output to correct posistion,
146 packet classifier depends on it.
147 */
148
Linus Torvalds1da177e2005-04-16 15:20:36 -0700149/* Private packet socket structures. */
150
Eric W. Biederman0fb375f2005-09-21 00:11:37 -0700151/* identical to struct packet_mreq except it has
152 * a longer address field.
153 */
Eric Dumazet40d4e3d2009-07-21 21:57:59 +0000154struct packet_mreq_max {
Eric W. Biederman0fb375f2005-09-21 00:11:37 -0700155 int mr_ifindex;
156 unsigned short mr_type;
157 unsigned short mr_alen;
158 unsigned char mr_address[MAX_ADDR_LEN];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700159};
David S. Millera2efcfa2007-05-29 13:12:50 -0700160
Daniel Borkmann184f4892013-04-16 01:57:46 +0000161union tpacket_uhdr {
162 struct tpacket_hdr *h1;
163 struct tpacket2_hdr *h2;
164 struct tpacket3_hdr *h3;
165 void *raw;
166};
167
chetan lokef6fb8f12011-08-19 10:18:16 +0000168static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
Johann Baudy69e3c752009-05-18 22:11:22 -0700169 int closing, int tx_ring);
170
chetan lokef6fb8f12011-08-19 10:18:16 +0000171#define V3_ALIGNMENT (8)
172
chetan lokebc59ba32011-08-25 10:43:30 +0000173#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
chetan lokef6fb8f12011-08-19 10:18:16 +0000174
175#define BLK_PLUS_PRIV(sz_of_priv) \
176 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
177
chetan lokef6fb8f12011-08-19 10:18:16 +0000178#define PGV_FROM_VMALLOC 1
Johann Baudy69e3c752009-05-18 22:11:22 -0700179
chetan lokef6fb8f12011-08-19 10:18:16 +0000180#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
181#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
182#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
183#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
184#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
185#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
186#define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
187
Johann Baudy69e3c752009-05-18 22:11:22 -0700188struct packet_sock;
189static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
Willem de Bruijn77f65eb2013-03-19 10:18:11 +0000190static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
191 struct packet_type *pt, struct net_device *orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700192
chetan lokef6fb8f12011-08-19 10:18:16 +0000193static void *packet_previous_frame(struct packet_sock *po,
194 struct packet_ring_buffer *rb,
195 int status);
196static void packet_increment_head(struct packet_ring_buffer *buff);
chetan lokebc59ba32011-08-25 10:43:30 +0000197static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
198 struct tpacket_block_desc *);
199static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
chetan lokef6fb8f12011-08-19 10:18:16 +0000200 struct packet_sock *);
chetan lokebc59ba32011-08-25 10:43:30 +0000201static void prb_retire_current_block(struct tpacket_kbdq_core *,
chetan lokef6fb8f12011-08-19 10:18:16 +0000202 struct packet_sock *, unsigned int status);
chetan lokebc59ba32011-08-25 10:43:30 +0000203static int prb_queue_frozen(struct tpacket_kbdq_core *);
204static void prb_open_block(struct tpacket_kbdq_core *,
205 struct tpacket_block_desc *);
chetan lokef6fb8f12011-08-19 10:18:16 +0000206static void prb_retire_rx_blk_timer_expired(unsigned long);
chetan lokebc59ba32011-08-25 10:43:30 +0000207static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
208static void prb_init_blk_timer(struct packet_sock *,
209 struct tpacket_kbdq_core *,
210 void (*func) (unsigned long));
211static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
212static void prb_clear_rxhash(struct tpacket_kbdq_core *,
213 struct tpacket3_hdr *);
214static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
215 struct tpacket3_hdr *);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700216static void packet_flush_mclist(struct sock *sk);
217
Herbert Xuffbc6112007-02-04 23:33:10 -0800218struct packet_skb_cb {
219 unsigned int origlen;
220 union {
221 struct sockaddr_pkt pkt;
222 struct sockaddr_ll ll;
223 } sa;
224};
225
226#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
Herbert Xu8dc41942007-02-04 23:31:32 -0800227
chetan lokebc59ba32011-08-25 10:43:30 +0000228#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
chetan lokef6fb8f12011-08-19 10:18:16 +0000229#define GET_PBLOCK_DESC(x, bid) \
chetan lokebc59ba32011-08-25 10:43:30 +0000230 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
chetan lokef6fb8f12011-08-19 10:18:16 +0000231#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
chetan lokebc59ba32011-08-25 10:43:30 +0000232 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
chetan lokef6fb8f12011-08-19 10:18:16 +0000233#define GET_NEXT_PRB_BLK_NUM(x) \
234 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
235 ((x)->kactive_blk_num+1) : 0)
236
David S. Millerdc99f602011-07-05 01:45:05 -0700237static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
238static void __fanout_link(struct sock *sk, struct packet_sock *po);
239
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100240static int packet_direct_xmit(struct sk_buff *skb)
241{
242 struct net_device *dev = skb->dev;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100243 netdev_features_t features;
244 struct netdev_queue *txq;
Daniel Borkmann43279502014-03-27 16:38:30 +0100245 int ret = NETDEV_TX_BUSY;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100246
247 if (unlikely(!netif_running(dev) ||
Daniel Borkmann43279502014-03-27 16:38:30 +0100248 !netif_carrier_ok(dev)))
249 goto drop;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100250
251 features = netif_skb_features(skb);
252 if (skb_needs_linearize(skb, features) &&
Daniel Borkmann43279502014-03-27 16:38:30 +0100253 __skb_linearize(skb))
254 goto drop;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100255
Daniel Borkmann10c51b56232014-08-27 11:11:27 +0200256 txq = skb_get_tx_queue(dev, skb);
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100257
Daniel Borkmann43279502014-03-27 16:38:30 +0100258 local_bh_disable();
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100259
Daniel Borkmann43279502014-03-27 16:38:30 +0100260 HARD_TX_LOCK(dev, txq, smp_processor_id());
David S. Miller10b3ad82014-08-29 21:07:24 -0700261 if (!netif_xmit_frozen_or_drv_stopped(txq))
David S. Millerfa2dbdc22014-08-29 21:55:22 -0700262 ret = netdev_start_xmit(skb, dev, txq, false);
Daniel Borkmann43279502014-03-27 16:38:30 +0100263 HARD_TX_UNLOCK(dev, txq);
264
265 local_bh_enable();
266
267 if (!dev_xmit_complete(ret))
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100268 kfree_skb(skb);
Daniel Borkmann43279502014-03-27 16:38:30 +0100269
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100270 return ret;
Daniel Borkmann43279502014-03-27 16:38:30 +0100271drop:
Daniel Borkmann0f97ede2014-04-02 20:52:56 +0200272 atomic_long_inc(&dev->tx_dropped);
Daniel Borkmann43279502014-03-27 16:38:30 +0100273 kfree_skb(skb);
274 return NET_XMIT_DROP;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100275}
276
Daniel Borkmann66e56cd2013-12-06 11:36:15 +0100277static struct net_device *packet_cached_dev_get(struct packet_sock *po)
278{
279 struct net_device *dev;
280
281 rcu_read_lock();
282 dev = rcu_dereference(po->cached_dev);
283 if (likely(dev))
284 dev_hold(dev);
285 rcu_read_unlock();
286
287 return dev;
288}
289
290static void packet_cached_dev_assign(struct packet_sock *po,
291 struct net_device *dev)
292{
293 rcu_assign_pointer(po->cached_dev, dev);
294}
295
296static void packet_cached_dev_reset(struct packet_sock *po)
297{
298 RCU_INIT_POINTER(po->cached_dev, NULL);
299}
300
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100301static bool packet_use_direct_xmit(const struct packet_sock *po)
302{
303 return po->xmit == packet_direct_xmit;
304}
305
Daniel Borkmann0fd5d572014-02-16 15:55:22 +0100306static u16 __packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100307{
Li Zhong1cbac012013-12-12 22:39:55 +0100308 return (u16) raw_smp_processor_id() % dev->real_num_tx_queues;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100309}
310
Daniel Borkmann0fd5d572014-02-16 15:55:22 +0100311static void packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
312{
313 const struct net_device_ops *ops = dev->netdev_ops;
314 u16 queue_index;
315
316 if (ops->ndo_select_queue) {
317 queue_index = ops->ndo_select_queue(dev, skb, NULL,
318 __packet_pick_tx_queue);
319 queue_index = netdev_cap_txqueue(dev, queue_index);
320 } else {
321 queue_index = __packet_pick_tx_queue(dev, skb);
322 }
323
324 skb_set_queue_mapping(skb, queue_index);
325}
326
David S. Millerce06b032011-07-04 01:44:29 -0700327/* register_prot_hook must be invoked with the po->bind_lock held,
328 * or from a context in which asynchronous accesses to the packet
329 * socket is not possible (packet_create()).
330 */
331static void register_prot_hook(struct sock *sk)
332{
333 struct packet_sock *po = pkt_sk(sk);
Daniel Borkmanne40526c2013-11-21 16:50:58 +0100334
David S. Millerce06b032011-07-04 01:44:29 -0700335 if (!po->running) {
Daniel Borkmann66e56cd2013-12-06 11:36:15 +0100336 if (po->fanout)
David S. Millerdc99f602011-07-05 01:45:05 -0700337 __fanout_link(sk, po);
Daniel Borkmann66e56cd2013-12-06 11:36:15 +0100338 else
David S. Millerdc99f602011-07-05 01:45:05 -0700339 dev_add_pack(&po->prot_hook);
Daniel Borkmanne40526c2013-11-21 16:50:58 +0100340
David S. Millerce06b032011-07-04 01:44:29 -0700341 sock_hold(sk);
342 po->running = 1;
343 }
344}
345
346/* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
347 * held. If the sync parameter is true, we will temporarily drop
348 * the po->bind_lock and do a synchronize_net to make sure no
349 * asynchronous packet processing paths still refer to the elements
350 * of po->prot_hook. If the sync parameter is false, it is the
351 * callers responsibility to take care of this.
352 */
353static void __unregister_prot_hook(struct sock *sk, bool sync)
354{
355 struct packet_sock *po = pkt_sk(sk);
356
357 po->running = 0;
Daniel Borkmann66e56cd2013-12-06 11:36:15 +0100358
359 if (po->fanout)
David S. Millerdc99f602011-07-05 01:45:05 -0700360 __fanout_unlink(sk, po);
Daniel Borkmann66e56cd2013-12-06 11:36:15 +0100361 else
David S. Millerdc99f602011-07-05 01:45:05 -0700362 __dev_remove_pack(&po->prot_hook);
Daniel Borkmanne40526c2013-11-21 16:50:58 +0100363
David S. Millerce06b032011-07-04 01:44:29 -0700364 __sock_put(sk);
365
366 if (sync) {
367 spin_unlock(&po->bind_lock);
368 synchronize_net();
369 spin_lock(&po->bind_lock);
370 }
371}
372
373static void unregister_prot_hook(struct sock *sk, bool sync)
374{
375 struct packet_sock *po = pkt_sk(sk);
376
377 if (po->running)
378 __unregister_prot_hook(sk, sync);
379}
380
Michael S. Tsirkin6e580402014-11-24 13:32:16 +0200381static inline struct page * __pure pgv_to_page(void *addr)
Changli Gao0af55bb2010-12-01 02:52:20 +0000382{
383 if (is_vmalloc_addr(addr))
384 return vmalloc_to_page(addr);
385 return virt_to_page(addr);
386}
387
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700388static void __packet_set_status(struct packet_sock *po, void *frame, int status)
389{
Daniel Borkmann184f4892013-04-16 01:57:46 +0000390 union tpacket_uhdr h;
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700391
392 h.raw = frame;
393 switch (po->tp_version) {
394 case TPACKET_V1:
395 h.h1->tp_status = status;
Changli Gao0af55bb2010-12-01 02:52:20 +0000396 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700397 break;
398 case TPACKET_V2:
399 h.h2->tp_status = status;
Changli Gao0af55bb2010-12-01 02:52:20 +0000400 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700401 break;
chetan lokef6fb8f12011-08-19 10:18:16 +0000402 case TPACKET_V3:
Johann Baudy69e3c752009-05-18 22:11:22 -0700403 default:
chetan lokef6fb8f12011-08-19 10:18:16 +0000404 WARN(1, "TPACKET version not supported.\n");
Johann Baudy69e3c752009-05-18 22:11:22 -0700405 BUG();
406 }
407
408 smp_wmb();
409}
410
411static int __packet_get_status(struct packet_sock *po, void *frame)
412{
Daniel Borkmann184f4892013-04-16 01:57:46 +0000413 union tpacket_uhdr h;
Johann Baudy69e3c752009-05-18 22:11:22 -0700414
415 smp_rmb();
416
417 h.raw = frame;
418 switch (po->tp_version) {
419 case TPACKET_V1:
Changli Gao0af55bb2010-12-01 02:52:20 +0000420 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
Johann Baudy69e3c752009-05-18 22:11:22 -0700421 return h.h1->tp_status;
422 case TPACKET_V2:
Changli Gao0af55bb2010-12-01 02:52:20 +0000423 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
Johann Baudy69e3c752009-05-18 22:11:22 -0700424 return h.h2->tp_status;
chetan lokef6fb8f12011-08-19 10:18:16 +0000425 case TPACKET_V3:
Johann Baudy69e3c752009-05-18 22:11:22 -0700426 default:
chetan lokef6fb8f12011-08-19 10:18:16 +0000427 WARN(1, "TPACKET version not supported.\n");
Johann Baudy69e3c752009-05-18 22:11:22 -0700428 BUG();
429 return 0;
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700430 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700431}
Johann Baudy69e3c752009-05-18 22:11:22 -0700432
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000433static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
434 unsigned int flags)
Daniel Borkmann7a513842013-04-23 00:39:29 +0000435{
436 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
437
Willem de Bruijn68a360e2014-07-25 18:01:31 -0400438 if (shhwtstamps &&
439 (flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
440 ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
441 return TP_STATUS_TS_RAW_HARDWARE;
Daniel Borkmann7a513842013-04-23 00:39:29 +0000442
443 if (ktime_to_timespec_cond(skb->tstamp, ts))
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000444 return TP_STATUS_TS_SOFTWARE;
Daniel Borkmann7a513842013-04-23 00:39:29 +0000445
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000446 return 0;
Daniel Borkmann7a513842013-04-23 00:39:29 +0000447}
448
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000449static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
450 struct sk_buff *skb)
Willem de Bruijn2e313962013-04-23 00:39:28 +0000451{
452 union tpacket_uhdr h;
453 struct timespec ts;
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000454 __u32 ts_status;
Willem de Bruijn2e313962013-04-23 00:39:28 +0000455
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000456 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
457 return 0;
Willem de Bruijn2e313962013-04-23 00:39:28 +0000458
459 h.raw = frame;
460 switch (po->tp_version) {
461 case TPACKET_V1:
462 h.h1->tp_sec = ts.tv_sec;
463 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
464 break;
465 case TPACKET_V2:
466 h.h2->tp_sec = ts.tv_sec;
467 h.h2->tp_nsec = ts.tv_nsec;
468 break;
469 case TPACKET_V3:
470 default:
471 WARN(1, "TPACKET version not supported.\n");
472 BUG();
473 }
474
475 /* one flush is safe, as both fields always lie on the same cacheline */
476 flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
477 smp_wmb();
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000478
479 return ts_status;
Willem de Bruijn2e313962013-04-23 00:39:28 +0000480}
481
Johann Baudy69e3c752009-05-18 22:11:22 -0700482static void *packet_lookup_frame(struct packet_sock *po,
483 struct packet_ring_buffer *rb,
484 unsigned int position,
485 int status)
486{
487 unsigned int pg_vec_pos, frame_offset;
Daniel Borkmann184f4892013-04-16 01:57:46 +0000488 union tpacket_uhdr h;
Johann Baudy69e3c752009-05-18 22:11:22 -0700489
490 pg_vec_pos = position / rb->frames_per_block;
491 frame_offset = position % rb->frames_per_block;
492
Neil Horman0e3125c2010-11-16 10:26:47 -0800493 h.raw = rb->pg_vec[pg_vec_pos].buffer +
494 (frame_offset * rb->frame_size);
Johann Baudy69e3c752009-05-18 22:11:22 -0700495
496 if (status != __packet_get_status(po, h.raw))
497 return NULL;
498
499 return h.raw;
500}
501
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000502static void *packet_current_frame(struct packet_sock *po,
Johann Baudy69e3c752009-05-18 22:11:22 -0700503 struct packet_ring_buffer *rb,
504 int status)
505{
506 return packet_lookup_frame(po, rb, rb->head, status);
507}
508
chetan lokebc59ba32011-08-25 10:43:30 +0000509static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
chetan lokef6fb8f12011-08-19 10:18:16 +0000510{
511 del_timer_sync(&pkc->retire_blk_timer);
512}
513
514static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
515 int tx_ring,
516 struct sk_buff_head *rb_queue)
517{
chetan lokebc59ba32011-08-25 10:43:30 +0000518 struct tpacket_kbdq_core *pkc;
chetan lokef6fb8f12011-08-19 10:18:16 +0000519
Duan Jiong22781a52013-12-06 13:29:36 +0800520 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
521 GET_PBDQC_FROM_RB(&po->rx_ring);
chetan lokef6fb8f12011-08-19 10:18:16 +0000522
Veaceslav Falicoec6f8092013-11-29 09:53:23 +0100523 spin_lock_bh(&rb_queue->lock);
chetan lokef6fb8f12011-08-19 10:18:16 +0000524 pkc->delete_blk_timer = 1;
Veaceslav Falicoec6f8092013-11-29 09:53:23 +0100525 spin_unlock_bh(&rb_queue->lock);
chetan lokef6fb8f12011-08-19 10:18:16 +0000526
527 prb_del_retire_blk_timer(pkc);
528}
529
530static void prb_init_blk_timer(struct packet_sock *po,
chetan lokebc59ba32011-08-25 10:43:30 +0000531 struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f12011-08-19 10:18:16 +0000532 void (*func) (unsigned long))
533{
534 init_timer(&pkc->retire_blk_timer);
535 pkc->retire_blk_timer.data = (long)po;
536 pkc->retire_blk_timer.function = func;
537 pkc->retire_blk_timer.expires = jiffies;
538}
539
540static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring)
541{
chetan lokebc59ba32011-08-25 10:43:30 +0000542 struct tpacket_kbdq_core *pkc;
chetan lokef6fb8f12011-08-19 10:18:16 +0000543
544 if (tx_ring)
545 BUG();
546
Duan Jiong22781a52013-12-06 13:29:36 +0800547 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
548 GET_PBDQC_FROM_RB(&po->rx_ring);
chetan lokef6fb8f12011-08-19 10:18:16 +0000549 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
550}
551
552static int prb_calc_retire_blk_tmo(struct packet_sock *po,
553 int blk_size_in_bytes)
554{
555 struct net_device *dev;
556 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
Jiri Pirko4bc71cb2011-09-03 03:34:30 +0000557 struct ethtool_cmd ecmd;
558 int err;
parav.pandit@emulex.come440cf22012-06-27 03:56:12 +0000559 u32 speed;
chetan lokef6fb8f12011-08-19 10:18:16 +0000560
Jiri Pirko4bc71cb2011-09-03 03:34:30 +0000561 rtnl_lock();
562 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
563 if (unlikely(!dev)) {
564 rtnl_unlock();
chetan lokef6fb8f12011-08-19 10:18:16 +0000565 return DEFAULT_PRB_RETIRE_TOV;
Jiri Pirko4bc71cb2011-09-03 03:34:30 +0000566 }
567 err = __ethtool_get_settings(dev, &ecmd);
parav.pandit@emulex.come440cf22012-06-27 03:56:12 +0000568 speed = ethtool_cmd_speed(&ecmd);
Jiri Pirko4bc71cb2011-09-03 03:34:30 +0000569 rtnl_unlock();
570 if (!err) {
Jiri Pirko4bc71cb2011-09-03 03:34:30 +0000571 /*
572 * If the link speed is so slow you don't really
573 * need to worry about perf anyways
574 */
parav.pandit@emulex.come440cf22012-06-27 03:56:12 +0000575 if (speed < SPEED_1000 || speed == SPEED_UNKNOWN) {
Jiri Pirko4bc71cb2011-09-03 03:34:30 +0000576 return DEFAULT_PRB_RETIRE_TOV;
parav.pandit@emulex.come440cf22012-06-27 03:56:12 +0000577 } else {
578 msec = 1;
579 div = speed / 1000;
chetan lokef6fb8f12011-08-19 10:18:16 +0000580 }
581 }
582
583 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
584
585 if (div)
586 mbits /= div;
587
588 tmo = mbits * msec;
589
590 if (div)
591 return tmo+1;
592 return tmo;
593}
594
chetan lokebc59ba32011-08-25 10:43:30 +0000595static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
chetan lokef6fb8f12011-08-19 10:18:16 +0000596 union tpacket_req_u *req_u)
597{
598 p1->feature_req_word = req_u->req3.tp_feature_req_word;
599}
600
601static void init_prb_bdqc(struct packet_sock *po,
602 struct packet_ring_buffer *rb,
603 struct pgv *pg_vec,
604 union tpacket_req_u *req_u, int tx_ring)
605{
Duan Jiong22781a52013-12-06 13:29:36 +0800606 struct tpacket_kbdq_core *p1 = GET_PBDQC_FROM_RB(rb);
chetan lokebc59ba32011-08-25 10:43:30 +0000607 struct tpacket_block_desc *pbd;
chetan lokef6fb8f12011-08-19 10:18:16 +0000608
609 memset(p1, 0x0, sizeof(*p1));
610
611 p1->knxt_seq_num = 1;
612 p1->pkbdq = pg_vec;
chetan lokebc59ba32011-08-25 10:43:30 +0000613 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
Joe Perchese3192692012-06-03 17:41:40 +0000614 p1->pkblk_start = pg_vec[0].buffer;
chetan lokef6fb8f12011-08-19 10:18:16 +0000615 p1->kblk_size = req_u->req3.tp_block_size;
616 p1->knum_blocks = req_u->req3.tp_block_nr;
617 p1->hdrlen = po->tp_hdrlen;
618 p1->version = po->tp_version;
619 p1->last_kactive_blk_num = 0;
Daniel Borkmannee80fbf2013-04-19 06:12:29 +0000620 po->stats.stats3.tp_freeze_q_cnt = 0;
chetan lokef6fb8f12011-08-19 10:18:16 +0000621 if (req_u->req3.tp_retire_blk_tov)
622 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
623 else
624 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
625 req_u->req3.tp_block_size);
626 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
627 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
628
Eric Dumazetdc808112014-08-15 09:16:04 -0700629 p1->max_frame_len = p1->kblk_size - BLK_PLUS_PRIV(p1->blk_sizeof_priv);
chetan lokef6fb8f12011-08-19 10:18:16 +0000630 prb_init_ft_ops(p1, req_u);
631 prb_setup_retire_blk_timer(po, tx_ring);
632 prb_open_block(p1, pbd);
633}
634
635/* Do NOT update the last_blk_num first.
636 * Assumes sk_buff_head lock is held.
637 */
chetan lokebc59ba32011-08-25 10:43:30 +0000638static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
chetan lokef6fb8f12011-08-19 10:18:16 +0000639{
640 mod_timer(&pkc->retire_blk_timer,
641 jiffies + pkc->tov_in_jiffies);
642 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
643}
644
645/*
646 * Timer logic:
647 * 1) We refresh the timer only when we open a block.
648 * By doing this we don't waste cycles refreshing the timer
649 * on packet-by-packet basis.
650 *
651 * With a 1MB block-size, on a 1Gbps line, it will take
652 * i) ~8 ms to fill a block + ii) memcpy etc.
653 * In this cut we are not accounting for the memcpy time.
654 *
655 * So, if the user sets the 'tmo' to 10ms then the timer
656 * will never fire while the block is still getting filled
657 * (which is what we want). However, the user could choose
658 * to close a block early and that's fine.
659 *
660 * But when the timer does fire, we check whether or not to refresh it.
661 * Since the tmo granularity is in msecs, it is not too expensive
662 * to refresh the timer, lets say every '8' msecs.
663 * Either the user can set the 'tmo' or we can derive it based on
664 * a) line-speed and b) block-size.
665 * prb_calc_retire_blk_tmo() calculates the tmo.
666 *
667 */
668static void prb_retire_rx_blk_timer_expired(unsigned long data)
669{
670 struct packet_sock *po = (struct packet_sock *)data;
Duan Jiong22781a52013-12-06 13:29:36 +0800671 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
chetan lokef6fb8f12011-08-19 10:18:16 +0000672 unsigned int frozen;
chetan lokebc59ba32011-08-25 10:43:30 +0000673 struct tpacket_block_desc *pbd;
chetan lokef6fb8f12011-08-19 10:18:16 +0000674
675 spin_lock(&po->sk.sk_receive_queue.lock);
676
677 frozen = prb_queue_frozen(pkc);
678 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
679
680 if (unlikely(pkc->delete_blk_timer))
681 goto out;
682
683 /* We only need to plug the race when the block is partially filled.
684 * tpacket_rcv:
685 * lock(); increment BLOCK_NUM_PKTS; unlock()
686 * copy_bits() is in progress ...
687 * timer fires on other cpu:
688 * we can't retire the current block because copy_bits
689 * is in progress.
690 *
691 */
692 if (BLOCK_NUM_PKTS(pbd)) {
693 while (atomic_read(&pkc->blk_fill_in_prog)) {
694 /* Waiting for skb_copy_bits to finish... */
695 cpu_relax();
696 }
697 }
698
699 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
700 if (!frozen) {
701 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
702 if (!prb_dispatch_next_block(pkc, po))
703 goto refresh_timer;
704 else
705 goto out;
706 } else {
707 /* Case 1. Queue was frozen because user-space was
708 * lagging behind.
709 */
710 if (prb_curr_blk_in_use(pkc, pbd)) {
711 /*
712 * Ok, user-space is still behind.
713 * So just refresh the timer.
714 */
715 goto refresh_timer;
716 } else {
717 /* Case 2. queue was frozen,user-space caught up,
718 * now the link went idle && the timer fired.
719 * We don't have a block to close.So we open this
720 * block and restart the timer.
721 * opening a block thaws the queue,restarts timer
722 * Thawing/timer-refresh is a side effect.
723 */
724 prb_open_block(pkc, pbd);
725 goto out;
726 }
727 }
728 }
729
730refresh_timer:
731 _prb_refresh_rx_retire_blk_timer(pkc);
732
733out:
734 spin_unlock(&po->sk.sk_receive_queue.lock);
735}
736
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000737static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
chetan lokebc59ba32011-08-25 10:43:30 +0000738 struct tpacket_block_desc *pbd1, __u32 status)
chetan lokef6fb8f12011-08-19 10:18:16 +0000739{
740 /* Flush everything minus the block header */
741
742#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
743 u8 *start, *end;
744
745 start = (u8 *)pbd1;
746
747 /* Skip the block header(we know header WILL fit in 4K) */
748 start += PAGE_SIZE;
749
750 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
751 for (; start < end; start += PAGE_SIZE)
752 flush_dcache_page(pgv_to_page(start));
753
754 smp_wmb();
755#endif
756
757 /* Now update the block status. */
758
759 BLOCK_STATUS(pbd1) = status;
760
761 /* Flush the block header */
762
763#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
764 start = (u8 *)pbd1;
765 flush_dcache_page(pgv_to_page(start));
766
767 smp_wmb();
768#endif
769}
770
771/*
772 * Side effect:
773 *
774 * 1) flush the block
775 * 2) Increment active_blk_num
776 *
777 * Note:We DONT refresh the timer on purpose.
778 * Because almost always the next block will be opened.
779 */
chetan lokebc59ba32011-08-25 10:43:30 +0000780static void prb_close_block(struct tpacket_kbdq_core *pkc1,
781 struct tpacket_block_desc *pbd1,
chetan lokef6fb8f12011-08-19 10:18:16 +0000782 struct packet_sock *po, unsigned int stat)
783{
784 __u32 status = TP_STATUS_USER | stat;
785
786 struct tpacket3_hdr *last_pkt;
chetan lokebc59ba32011-08-25 10:43:30 +0000787 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
chetan lokef6fb8f12011-08-19 10:18:16 +0000788
Daniel Borkmannee80fbf2013-04-19 06:12:29 +0000789 if (po->stats.stats3.tp_drops)
chetan lokef6fb8f12011-08-19 10:18:16 +0000790 status |= TP_STATUS_LOSING;
791
792 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
793 last_pkt->tp_next_offset = 0;
794
795 /* Get the ts of the last pkt */
796 if (BLOCK_NUM_PKTS(pbd1)) {
797 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
798 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
799 } else {
800 /* Ok, we tmo'd - so get the current time */
801 struct timespec ts;
802 getnstimeofday(&ts);
803 h1->ts_last_pkt.ts_sec = ts.tv_sec;
804 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
805 }
806
807 smp_wmb();
808
809 /* Flush the block */
810 prb_flush_block(pkc1, pbd1, status);
811
812 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
813}
814
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000815static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
chetan lokef6fb8f12011-08-19 10:18:16 +0000816{
817 pkc->reset_pending_on_curr_blk = 0;
818}
819
820/*
821 * Side effect of opening a block:
822 *
823 * 1) prb_queue is thawed.
824 * 2) retire_blk_timer is refreshed.
825 *
826 */
chetan lokebc59ba32011-08-25 10:43:30 +0000827static void prb_open_block(struct tpacket_kbdq_core *pkc1,
828 struct tpacket_block_desc *pbd1)
chetan lokef6fb8f12011-08-19 10:18:16 +0000829{
830 struct timespec ts;
chetan lokebc59ba32011-08-25 10:43:30 +0000831 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
chetan lokef6fb8f12011-08-19 10:18:16 +0000832
833 smp_rmb();
834
Daniel Borkmann8da30562013-05-03 02:57:00 +0000835 /* We could have just memset this but we will lose the
836 * flexibility of making the priv area sticky
837 */
chetan lokef6fb8f12011-08-19 10:18:16 +0000838
Daniel Borkmann8da30562013-05-03 02:57:00 +0000839 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
840 BLOCK_NUM_PKTS(pbd1) = 0;
841 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
chetan lokef6fb8f12011-08-19 10:18:16 +0000842
Daniel Borkmann8da30562013-05-03 02:57:00 +0000843 getnstimeofday(&ts);
chetan lokef6fb8f12011-08-19 10:18:16 +0000844
Daniel Borkmann8da30562013-05-03 02:57:00 +0000845 h1->ts_first_pkt.ts_sec = ts.tv_sec;
846 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
chetan lokef6fb8f12011-08-19 10:18:16 +0000847
Daniel Borkmann8da30562013-05-03 02:57:00 +0000848 pkc1->pkblk_start = (char *)pbd1;
849 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
850
851 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
852 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
853
854 pbd1->version = pkc1->version;
855 pkc1->prev = pkc1->nxt_offset;
856 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
857
858 prb_thaw_queue(pkc1);
859 _prb_refresh_rx_retire_blk_timer(pkc1);
860
861 smp_wmb();
chetan lokef6fb8f12011-08-19 10:18:16 +0000862}
863
864/*
865 * Queue freeze logic:
866 * 1) Assume tp_block_nr = 8 blocks.
867 * 2) At time 't0', user opens Rx ring.
868 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
869 * 4) user-space is either sleeping or processing block '0'.
870 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
871 * it will close block-7,loop around and try to fill block '0'.
872 * call-flow:
873 * __packet_lookup_frame_in_block
874 * prb_retire_current_block()
875 * prb_dispatch_next_block()
876 * |->(BLOCK_STATUS == USER) evaluates to true
877 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
878 * 6) Now there are two cases:
879 * 6.1) Link goes idle right after the queue is frozen.
880 * But remember, the last open_block() refreshed the timer.
881 * When this timer expires,it will refresh itself so that we can
882 * re-open block-0 in near future.
883 * 6.2) Link is busy and keeps on receiving packets. This is a simple
884 * case and __packet_lookup_frame_in_block will check if block-0
885 * is free and can now be re-used.
886 */
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000887static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f12011-08-19 10:18:16 +0000888 struct packet_sock *po)
889{
890 pkc->reset_pending_on_curr_blk = 1;
Daniel Borkmannee80fbf2013-04-19 06:12:29 +0000891 po->stats.stats3.tp_freeze_q_cnt++;
chetan lokef6fb8f12011-08-19 10:18:16 +0000892}
893
894#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
895
896/*
897 * If the next block is free then we will dispatch it
898 * and return a good offset.
899 * Else, we will freeze the queue.
900 * So, caller must check the return value.
901 */
chetan lokebc59ba32011-08-25 10:43:30 +0000902static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f12011-08-19 10:18:16 +0000903 struct packet_sock *po)
904{
chetan lokebc59ba32011-08-25 10:43:30 +0000905 struct tpacket_block_desc *pbd;
chetan lokef6fb8f12011-08-19 10:18:16 +0000906
907 smp_rmb();
908
909 /* 1. Get current block num */
910 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
911
912 /* 2. If this block is currently in_use then freeze the queue */
913 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
914 prb_freeze_queue(pkc, po);
915 return NULL;
916 }
917
918 /*
919 * 3.
920 * open this block and return the offset where the first packet
921 * needs to get stored.
922 */
923 prb_open_block(pkc, pbd);
924 return (void *)pkc->nxt_offset;
925}
926
chetan lokebc59ba32011-08-25 10:43:30 +0000927static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f12011-08-19 10:18:16 +0000928 struct packet_sock *po, unsigned int status)
929{
chetan lokebc59ba32011-08-25 10:43:30 +0000930 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
chetan lokef6fb8f12011-08-19 10:18:16 +0000931
932 /* retire/close the current block */
933 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
934 /*
935 * Plug the case where copy_bits() is in progress on
936 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
937 * have space to copy the pkt in the current block and
938 * called prb_retire_current_block()
939 *
940 * We don't need to worry about the TMO case because
941 * the timer-handler already handled this case.
942 */
943 if (!(status & TP_STATUS_BLK_TMO)) {
944 while (atomic_read(&pkc->blk_fill_in_prog)) {
945 /* Waiting for skb_copy_bits to finish... */
946 cpu_relax();
947 }
948 }
949 prb_close_block(pkc, pbd, po, status);
950 return;
951 }
chetan lokef6fb8f12011-08-19 10:18:16 +0000952}
953
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000954static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
chetan lokebc59ba32011-08-25 10:43:30 +0000955 struct tpacket_block_desc *pbd)
chetan lokef6fb8f12011-08-19 10:18:16 +0000956{
957 return TP_STATUS_USER & BLOCK_STATUS(pbd);
958}
959
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000960static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
chetan lokef6fb8f12011-08-19 10:18:16 +0000961{
962 return pkc->reset_pending_on_curr_blk;
963}
964
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000965static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
chetan lokef6fb8f12011-08-19 10:18:16 +0000966{
chetan lokebc59ba32011-08-25 10:43:30 +0000967 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
chetan lokef6fb8f12011-08-19 10:18:16 +0000968 atomic_dec(&pkc->blk_fill_in_prog);
969}
970
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000971static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f12011-08-19 10:18:16 +0000972 struct tpacket3_hdr *ppd)
973{
Tom Herbert3958afa1b2013-12-15 22:12:06 -0800974 ppd->hv1.tp_rxhash = skb_get_hash(pkc->skb);
chetan lokef6fb8f12011-08-19 10:18:16 +0000975}
976
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000977static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f12011-08-19 10:18:16 +0000978 struct tpacket3_hdr *ppd)
979{
980 ppd->hv1.tp_rxhash = 0;
981}
982
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000983static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f12011-08-19 10:18:16 +0000984 struct tpacket3_hdr *ppd)
985{
986 if (vlan_tx_tag_present(pkc->skb)) {
987 ppd->hv1.tp_vlan_tci = vlan_tx_tag_get(pkc->skb);
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +0900988 ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->vlan_proto);
989 ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
chetan lokef6fb8f12011-08-19 10:18:16 +0000990 } else {
danborkmann@iogearbox.net9e670302012-08-20 03:34:03 +0000991 ppd->hv1.tp_vlan_tci = 0;
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +0900992 ppd->hv1.tp_vlan_tpid = 0;
danborkmann@iogearbox.net9e670302012-08-20 03:34:03 +0000993 ppd->tp_status = TP_STATUS_AVAILABLE;
chetan lokef6fb8f12011-08-19 10:18:16 +0000994 }
995}
996
chetan lokebc59ba32011-08-25 10:43:30 +0000997static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f12011-08-19 10:18:16 +0000998 struct tpacket3_hdr *ppd)
999{
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +09001000 ppd->hv1.tp_padding = 0;
chetan lokef6fb8f12011-08-19 10:18:16 +00001001 prb_fill_vlan_info(pkc, ppd);
1002
1003 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
1004 prb_fill_rxhash(pkc, ppd);
1005 else
1006 prb_clear_rxhash(pkc, ppd);
1007}
1008
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001009static void prb_fill_curr_block(char *curr,
chetan lokebc59ba32011-08-25 10:43:30 +00001010 struct tpacket_kbdq_core *pkc,
1011 struct tpacket_block_desc *pbd,
chetan lokef6fb8f12011-08-19 10:18:16 +00001012 unsigned int len)
1013{
1014 struct tpacket3_hdr *ppd;
1015
1016 ppd = (struct tpacket3_hdr *)curr;
1017 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
1018 pkc->prev = curr;
1019 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
1020 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
1021 BLOCK_NUM_PKTS(pbd) += 1;
1022 atomic_inc(&pkc->blk_fill_in_prog);
1023 prb_run_all_ft_ops(pkc, ppd);
1024}
1025
1026/* Assumes caller has the sk->rx_queue.lock */
1027static void *__packet_lookup_frame_in_block(struct packet_sock *po,
1028 struct sk_buff *skb,
1029 int status,
1030 unsigned int len
1031 )
1032{
chetan lokebc59ba32011-08-25 10:43:30 +00001033 struct tpacket_kbdq_core *pkc;
1034 struct tpacket_block_desc *pbd;
chetan lokef6fb8f12011-08-19 10:18:16 +00001035 char *curr, *end;
1036
Joe Perchese3192692012-06-03 17:41:40 +00001037 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
chetan lokef6fb8f12011-08-19 10:18:16 +00001038 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1039
1040 /* Queue is frozen when user space is lagging behind */
1041 if (prb_queue_frozen(pkc)) {
1042 /*
1043 * Check if that last block which caused the queue to freeze,
1044 * is still in_use by user-space.
1045 */
1046 if (prb_curr_blk_in_use(pkc, pbd)) {
1047 /* Can't record this packet */
1048 return NULL;
1049 } else {
1050 /*
1051 * Ok, the block was released by user-space.
1052 * Now let's open that block.
1053 * opening a block also thaws the queue.
1054 * Thawing is a side effect.
1055 */
1056 prb_open_block(pkc, pbd);
1057 }
1058 }
1059
1060 smp_mb();
1061 curr = pkc->nxt_offset;
1062 pkc->skb = skb;
Joe Perchese3192692012-06-03 17:41:40 +00001063 end = (char *)pbd + pkc->kblk_size;
chetan lokef6fb8f12011-08-19 10:18:16 +00001064
1065 /* first try the current block */
1066 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1067 prb_fill_curr_block(curr, pkc, pbd, len);
1068 return (void *)curr;
1069 }
1070
1071 /* Ok, close the current block */
1072 prb_retire_current_block(pkc, po, 0);
1073
1074 /* Now, try to dispatch the next block */
1075 curr = (char *)prb_dispatch_next_block(pkc, po);
1076 if (curr) {
1077 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1078 prb_fill_curr_block(curr, pkc, pbd, len);
1079 return (void *)curr;
1080 }
1081
1082 /*
1083 * No free blocks are available.user_space hasn't caught up yet.
1084 * Queue was just frozen and now this packet will get dropped.
1085 */
1086 return NULL;
1087}
1088
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001089static void *packet_current_rx_frame(struct packet_sock *po,
chetan lokef6fb8f12011-08-19 10:18:16 +00001090 struct sk_buff *skb,
1091 int status, unsigned int len)
1092{
1093 char *curr = NULL;
1094 switch (po->tp_version) {
1095 case TPACKET_V1:
1096 case TPACKET_V2:
1097 curr = packet_lookup_frame(po, &po->rx_ring,
1098 po->rx_ring.head, status);
1099 return curr;
1100 case TPACKET_V3:
1101 return __packet_lookup_frame_in_block(po, skb, status, len);
1102 default:
1103 WARN(1, "TPACKET version not supported\n");
1104 BUG();
Ying Xue99aa3472012-08-06 16:27:10 +00001105 return NULL;
chetan lokef6fb8f12011-08-19 10:18:16 +00001106 }
1107}
1108
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001109static void *prb_lookup_block(struct packet_sock *po,
chetan lokef6fb8f12011-08-19 10:18:16 +00001110 struct packet_ring_buffer *rb,
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001111 unsigned int idx,
chetan lokef6fb8f12011-08-19 10:18:16 +00001112 int status)
1113{
chetan lokebc59ba32011-08-25 10:43:30 +00001114 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001115 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
chetan lokef6fb8f12011-08-19 10:18:16 +00001116
1117 if (status != BLOCK_STATUS(pbd))
1118 return NULL;
1119 return pbd;
1120}
1121
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001122static int prb_previous_blk_num(struct packet_ring_buffer *rb)
chetan lokef6fb8f12011-08-19 10:18:16 +00001123{
1124 unsigned int prev;
1125 if (rb->prb_bdqc.kactive_blk_num)
1126 prev = rb->prb_bdqc.kactive_blk_num-1;
1127 else
1128 prev = rb->prb_bdqc.knum_blocks-1;
1129 return prev;
1130}
1131
1132/* Assumes caller has held the rx_queue.lock */
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001133static void *__prb_previous_block(struct packet_sock *po,
chetan lokef6fb8f12011-08-19 10:18:16 +00001134 struct packet_ring_buffer *rb,
1135 int status)
1136{
1137 unsigned int previous = prb_previous_blk_num(rb);
1138 return prb_lookup_block(po, rb, previous, status);
1139}
1140
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001141static void *packet_previous_rx_frame(struct packet_sock *po,
chetan lokef6fb8f12011-08-19 10:18:16 +00001142 struct packet_ring_buffer *rb,
1143 int status)
1144{
1145 if (po->tp_version <= TPACKET_V2)
1146 return packet_previous_frame(po, rb, status);
1147
1148 return __prb_previous_block(po, rb, status);
1149}
1150
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001151static void packet_increment_rx_head(struct packet_sock *po,
chetan lokef6fb8f12011-08-19 10:18:16 +00001152 struct packet_ring_buffer *rb)
1153{
1154 switch (po->tp_version) {
1155 case TPACKET_V1:
1156 case TPACKET_V2:
1157 return packet_increment_head(rb);
1158 case TPACKET_V3:
1159 default:
1160 WARN(1, "TPACKET version not supported.\n");
1161 BUG();
1162 return;
1163 }
1164}
1165
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001166static void *packet_previous_frame(struct packet_sock *po,
Johann Baudy69e3c752009-05-18 22:11:22 -07001167 struct packet_ring_buffer *rb,
1168 int status)
1169{
1170 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1171 return packet_lookup_frame(po, rb, previous, status);
1172}
1173
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001174static void packet_increment_head(struct packet_ring_buffer *buff)
Johann Baudy69e3c752009-05-18 22:11:22 -07001175{
1176 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1177}
1178
Daniel Borkmannb0138402014-01-15 16:25:36 +01001179static void packet_inc_pending(struct packet_ring_buffer *rb)
1180{
1181 this_cpu_inc(*rb->pending_refcnt);
1182}
1183
1184static void packet_dec_pending(struct packet_ring_buffer *rb)
1185{
1186 this_cpu_dec(*rb->pending_refcnt);
1187}
1188
1189static unsigned int packet_read_pending(const struct packet_ring_buffer *rb)
1190{
1191 unsigned int refcnt = 0;
1192 int cpu;
1193
1194 /* We don't use pending refcount in rx_ring. */
1195 if (rb->pending_refcnt == NULL)
1196 return 0;
1197
1198 for_each_possible_cpu(cpu)
1199 refcnt += *per_cpu_ptr(rb->pending_refcnt, cpu);
1200
1201 return refcnt;
1202}
1203
1204static int packet_alloc_pending(struct packet_sock *po)
1205{
1206 po->rx_ring.pending_refcnt = NULL;
1207
1208 po->tx_ring.pending_refcnt = alloc_percpu(unsigned int);
1209 if (unlikely(po->tx_ring.pending_refcnt == NULL))
1210 return -ENOBUFS;
1211
1212 return 0;
1213}
1214
1215static void packet_free_pending(struct packet_sock *po)
1216{
1217 free_percpu(po->tx_ring.pending_refcnt);
1218}
1219
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001220static bool packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1221{
1222 struct sock *sk = &po->sk;
1223 bool has_room;
1224
1225 if (po->prot_hook.func != tpacket_rcv)
1226 return (atomic_read(&sk->sk_rmem_alloc) + skb->truesize)
1227 <= sk->sk_rcvbuf;
1228
1229 spin_lock(&sk->sk_receive_queue.lock);
1230 if (po->tp_version == TPACKET_V3)
1231 has_room = prb_lookup_block(po, &po->rx_ring,
1232 po->rx_ring.prb_bdqc.kactive_blk_num,
1233 TP_STATUS_KERNEL);
1234 else
1235 has_room = packet_lookup_frame(po, &po->rx_ring,
1236 po->rx_ring.head,
1237 TP_STATUS_KERNEL);
1238 spin_unlock(&sk->sk_receive_queue.lock);
1239
1240 return has_room;
1241}
1242
Linus Torvalds1da177e2005-04-16 15:20:36 -07001243static void packet_sock_destruct(struct sock *sk)
1244{
Richard Cochraned85b562010-04-07 22:41:28 +00001245 skb_queue_purge(&sk->sk_error_queue);
1246
Ilpo Järvinen547b7922008-07-25 21:43:18 -07001247 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1248 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001249
1250 if (!sock_flag(sk, SOCK_DEAD)) {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001251 pr_err("Attempt to release alive packet socket: %p\n", sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001252 return;
1253 }
1254
Pavel Emelyanov17ab56a2007-11-10 21:38:48 -08001255 sk_refcnt_debug_dec(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001256}
1257
David S. Millerdc99f602011-07-05 01:45:05 -07001258static int fanout_rr_next(struct packet_fanout *f, unsigned int num)
1259{
1260 int x = atomic_read(&f->rr_cur) + 1;
1261
1262 if (x >= num)
1263 x = 0;
1264
1265 return x;
1266}
1267
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001268static unsigned int fanout_demux_hash(struct packet_fanout *f,
1269 struct sk_buff *skb,
1270 unsigned int num)
David S. Millerdc99f602011-07-05 01:45:05 -07001271{
Tom Herbert61b905d2014-03-24 15:34:47 -07001272 return reciprocal_scale(skb_get_hash(skb), num);
David S. Millerdc99f602011-07-05 01:45:05 -07001273}
1274
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001275static unsigned int fanout_demux_lb(struct packet_fanout *f,
1276 struct sk_buff *skb,
1277 unsigned int num)
David S. Millerdc99f602011-07-05 01:45:05 -07001278{
1279 int cur, old;
1280
1281 cur = atomic_read(&f->rr_cur);
1282 while ((old = atomic_cmpxchg(&f->rr_cur, cur,
1283 fanout_rr_next(f, num))) != cur)
1284 cur = old;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001285 return cur;
David S. Millerdc99f602011-07-05 01:45:05 -07001286}
1287
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001288static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1289 struct sk_buff *skb,
1290 unsigned int num)
David S. Miller95ec3eb2011-07-06 01:56:38 -07001291{
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001292 return smp_processor_id() % num;
1293}
David S. Miller95ec3eb2011-07-06 01:56:38 -07001294
Daniel Borkmann5df0ddf2013-08-28 22:13:09 +02001295static unsigned int fanout_demux_rnd(struct packet_fanout *f,
1296 struct sk_buff *skb,
1297 unsigned int num)
1298{
Daniel Borkmannf337db62014-01-22 02:29:39 +01001299 return prandom_u32_max(num);
Daniel Borkmann5df0ddf2013-08-28 22:13:09 +02001300}
1301
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001302static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1303 struct sk_buff *skb,
1304 unsigned int idx, unsigned int skip,
1305 unsigned int num)
1306{
1307 unsigned int i, j;
1308
1309 i = j = min_t(int, f->next[idx], num - 1);
1310 do {
1311 if (i != skip && packet_rcv_has_room(pkt_sk(f->arr[i]), skb)) {
1312 if (i != j)
1313 f->next[idx] = i;
1314 return i;
1315 }
1316 if (++i == num)
1317 i = 0;
1318 } while (i != j);
1319
1320 return idx;
1321}
1322
Neil Horman2d360972014-01-22 16:01:44 -05001323static unsigned int fanout_demux_qm(struct packet_fanout *f,
1324 struct sk_buff *skb,
1325 unsigned int num)
1326{
1327 return skb_get_queue_mapping(skb) % num;
1328}
1329
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001330static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1331{
1332 return f->flags & (flag >> 8);
David S. Miller95ec3eb2011-07-06 01:56:38 -07001333}
1334
David S. Miller95ec3eb2011-07-06 01:56:38 -07001335static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1336 struct packet_type *pt, struct net_device *orig_dev)
David S. Millerdc99f602011-07-05 01:45:05 -07001337{
1338 struct packet_fanout *f = pt->af_packet_priv;
1339 unsigned int num = f->num_members;
1340 struct packet_sock *po;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001341 unsigned int idx;
David S. Millerdc99f602011-07-05 01:45:05 -07001342
1343 if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
1344 !num) {
1345 kfree_skb(skb);
1346 return 0;
1347 }
1348
David S. Miller95ec3eb2011-07-06 01:56:38 -07001349 switch (f->type) {
1350 case PACKET_FANOUT_HASH:
1351 default:
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001352 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
Eric Dumazetbc416d92011-10-06 10:28:31 +00001353 skb = ip_check_defrag(skb, IP_DEFRAG_AF_PACKET);
David S. Miller95ec3eb2011-07-06 01:56:38 -07001354 if (!skb)
1355 return 0;
1356 }
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001357 idx = fanout_demux_hash(f, skb, num);
David S. Miller95ec3eb2011-07-06 01:56:38 -07001358 break;
1359 case PACKET_FANOUT_LB:
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001360 idx = fanout_demux_lb(f, skb, num);
David S. Miller95ec3eb2011-07-06 01:56:38 -07001361 break;
1362 case PACKET_FANOUT_CPU:
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001363 idx = fanout_demux_cpu(f, skb, num);
1364 break;
Daniel Borkmann5df0ddf2013-08-28 22:13:09 +02001365 case PACKET_FANOUT_RND:
1366 idx = fanout_demux_rnd(f, skb, num);
1367 break;
Neil Horman2d360972014-01-22 16:01:44 -05001368 case PACKET_FANOUT_QM:
1369 idx = fanout_demux_qm(f, skb, num);
1370 break;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001371 case PACKET_FANOUT_ROLLOVER:
1372 idx = fanout_demux_rollover(f, skb, 0, (unsigned int) -1, num);
David S. Miller95ec3eb2011-07-06 01:56:38 -07001373 break;
David S. Miller7736d332011-07-05 01:43:20 -07001374 }
1375
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001376 po = pkt_sk(f->arr[idx]);
1377 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER) &&
1378 unlikely(!packet_rcv_has_room(po, skb))) {
1379 idx = fanout_demux_rollover(f, skb, idx, idx, num);
1380 po = pkt_sk(f->arr[idx]);
1381 }
David S. Millerdc99f602011-07-05 01:45:05 -07001382
1383 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1384}
1385
Pavel Emelyanovfff33212012-08-16 05:36:48 +00001386DEFINE_MUTEX(fanout_mutex);
1387EXPORT_SYMBOL_GPL(fanout_mutex);
David S. Millerdc99f602011-07-05 01:45:05 -07001388static LIST_HEAD(fanout_list);
1389
1390static void __fanout_link(struct sock *sk, struct packet_sock *po)
1391{
1392 struct packet_fanout *f = po->fanout;
1393
1394 spin_lock(&f->lock);
1395 f->arr[f->num_members] = sk;
1396 smp_wmb();
1397 f->num_members++;
1398 spin_unlock(&f->lock);
1399}
1400
1401static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1402{
1403 struct packet_fanout *f = po->fanout;
1404 int i;
1405
1406 spin_lock(&f->lock);
1407 for (i = 0; i < f->num_members; i++) {
1408 if (f->arr[i] == sk)
1409 break;
1410 }
1411 BUG_ON(i >= f->num_members);
1412 f->arr[i] = f->arr[f->num_members - 1];
1413 f->num_members--;
1414 spin_unlock(&f->lock);
1415}
1416
Weilong Chend4dd8ae2013-12-23 11:31:38 +08001417static bool match_fanout_group(struct packet_type *ptype, struct sock *sk)
Eric Leblondc0de08d2012-08-16 22:02:58 +00001418{
Weilong Chend4dd8ae2013-12-23 11:31:38 +08001419 if (ptype->af_packet_priv == (void *)((struct packet_sock *)sk)->fanout)
Eric Leblondc0de08d2012-08-16 22:02:58 +00001420 return true;
1421
1422 return false;
1423}
1424
David S. Miller7736d332011-07-05 01:43:20 -07001425static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
David S. Millerdc99f602011-07-05 01:45:05 -07001426{
1427 struct packet_sock *po = pkt_sk(sk);
1428 struct packet_fanout *f, *match;
David S. Miller7736d332011-07-05 01:43:20 -07001429 u8 type = type_flags & 0xff;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001430 u8 flags = type_flags >> 8;
David S. Millerdc99f602011-07-05 01:45:05 -07001431 int err;
1432
1433 switch (type) {
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001434 case PACKET_FANOUT_ROLLOVER:
1435 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1436 return -EINVAL;
David S. Millerdc99f602011-07-05 01:45:05 -07001437 case PACKET_FANOUT_HASH:
1438 case PACKET_FANOUT_LB:
David S. Miller95ec3eb2011-07-06 01:56:38 -07001439 case PACKET_FANOUT_CPU:
Daniel Borkmann5df0ddf2013-08-28 22:13:09 +02001440 case PACKET_FANOUT_RND:
Neil Horman2d360972014-01-22 16:01:44 -05001441 case PACKET_FANOUT_QM:
David S. Millerdc99f602011-07-05 01:45:05 -07001442 break;
1443 default:
1444 return -EINVAL;
1445 }
1446
1447 if (!po->running)
1448 return -EINVAL;
1449
1450 if (po->fanout)
1451 return -EALREADY;
1452
1453 mutex_lock(&fanout_mutex);
1454 match = NULL;
1455 list_for_each_entry(f, &fanout_list, list) {
1456 if (f->id == id &&
1457 read_pnet(&f->net) == sock_net(sk)) {
1458 match = f;
1459 break;
1460 }
1461 }
Eric Dumazetafe62c62011-07-07 06:41:29 -07001462 err = -EINVAL;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001463 if (match && match->flags != flags)
Eric Dumazetafe62c62011-07-07 06:41:29 -07001464 goto out;
David S. Millerdc99f602011-07-05 01:45:05 -07001465 if (!match) {
Eric Dumazetafe62c62011-07-07 06:41:29 -07001466 err = -ENOMEM;
David S. Millerdc99f602011-07-05 01:45:05 -07001467 match = kzalloc(sizeof(*match), GFP_KERNEL);
Eric Dumazetafe62c62011-07-07 06:41:29 -07001468 if (!match)
1469 goto out;
1470 write_pnet(&match->net, sock_net(sk));
1471 match->id = id;
1472 match->type = type;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001473 match->flags = flags;
Eric Dumazetafe62c62011-07-07 06:41:29 -07001474 atomic_set(&match->rr_cur, 0);
1475 INIT_LIST_HEAD(&match->list);
1476 spin_lock_init(&match->lock);
1477 atomic_set(&match->sk_ref, 0);
1478 match->prot_hook.type = po->prot_hook.type;
1479 match->prot_hook.dev = po->prot_hook.dev;
1480 match->prot_hook.func = packet_rcv_fanout;
1481 match->prot_hook.af_packet_priv = match;
Eric Leblondc0de08d2012-08-16 22:02:58 +00001482 match->prot_hook.id_match = match_fanout_group;
Eric Dumazetafe62c62011-07-07 06:41:29 -07001483 dev_add_pack(&match->prot_hook);
1484 list_add(&match->list, &fanout_list);
1485 }
1486 err = -EINVAL;
1487 if (match->type == type &&
1488 match->prot_hook.type == po->prot_hook.type &&
1489 match->prot_hook.dev == po->prot_hook.dev) {
1490 err = -ENOSPC;
1491 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1492 __dev_remove_pack(&po->prot_hook);
1493 po->fanout = match;
1494 atomic_inc(&match->sk_ref);
1495 __fanout_link(sk, po);
1496 err = 0;
David S. Millerdc99f602011-07-05 01:45:05 -07001497 }
1498 }
Eric Dumazetafe62c62011-07-07 06:41:29 -07001499out:
David S. Millerdc99f602011-07-05 01:45:05 -07001500 mutex_unlock(&fanout_mutex);
1501 return err;
1502}
1503
1504static void fanout_release(struct sock *sk)
1505{
1506 struct packet_sock *po = pkt_sk(sk);
1507 struct packet_fanout *f;
1508
1509 f = po->fanout;
1510 if (!f)
1511 return;
1512
Pavel Emelyanovfff33212012-08-16 05:36:48 +00001513 mutex_lock(&fanout_mutex);
David S. Millerdc99f602011-07-05 01:45:05 -07001514 po->fanout = NULL;
1515
David S. Millerdc99f602011-07-05 01:45:05 -07001516 if (atomic_dec_and_test(&f->sk_ref)) {
1517 list_del(&f->list);
1518 dev_remove_pack(&f->prot_hook);
1519 kfree(f);
1520 }
1521 mutex_unlock(&fanout_mutex);
1522}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001523
Eric Dumazet90ddc4f2005-12-22 12:49:22 -08001524static const struct proto_ops packet_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001525
Eric Dumazet90ddc4f2005-12-22 12:49:22 -08001526static const struct proto_ops packet_ops_spkt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001527
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001528static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1529 struct packet_type *pt, struct net_device *orig_dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001530{
1531 struct sock *sk;
1532 struct sockaddr_pkt *spkt;
1533
1534 /*
1535 * When we registered the protocol we saved the socket in the data
1536 * field for just this event.
1537 */
1538
1539 sk = pt->af_packet_priv;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001540
Linus Torvalds1da177e2005-04-16 15:20:36 -07001541 /*
1542 * Yank back the headers [hope the device set this
1543 * right or kerboom...]
1544 *
1545 * Incoming packets have ll header pulled,
1546 * push it back.
1547 *
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07001548 * For outgoing ones skb->data == skb_mac_header(skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001549 * so that this procedure is noop.
1550 */
1551
1552 if (skb->pkt_type == PACKET_LOOPBACK)
1553 goto out;
1554
Octavian Purdila09ad9bc2009-11-25 15:14:13 -08001555 if (!net_eq(dev_net(dev), sock_net(sk)))
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08001556 goto out;
1557
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001558 skb = skb_share_check(skb, GFP_ATOMIC);
1559 if (skb == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001560 goto oom;
1561
1562 /* drop any routing info */
Eric Dumazetadf30902009-06-02 05:19:30 +00001563 skb_dst_drop(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001564
Phil Oester84531c22005-07-12 11:57:52 -07001565 /* drop conntrack reference */
1566 nf_reset(skb);
1567
Herbert Xuffbc6112007-02-04 23:33:10 -08001568 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001569
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07001570 skb_push(skb, skb->data - skb_mac_header(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001571
1572 /*
1573 * The SOCK_PACKET socket receives _all_ frames.
1574 */
1575
1576 spkt->spkt_family = dev->type;
1577 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1578 spkt->spkt_protocol = skb->protocol;
1579
1580 /*
1581 * Charge the memory to the socket. This is done specifically
1582 * to prevent sockets using all the memory up.
1583 */
1584
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001585 if (sock_queue_rcv_skb(sk, skb) == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001586 return 0;
1587
1588out:
1589 kfree_skb(skb);
1590oom:
1591 return 0;
1592}
1593
1594
1595/*
1596 * Output a raw packet to a device layer. This bypasses all the other
1597 * protocol layers and you must therefore supply it with a complete frame
1598 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001599
Linus Torvalds1da177e2005-04-16 15:20:36 -07001600static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
1601 struct msghdr *msg, size_t len)
1602{
1603 struct sock *sk = sock->sk;
Steffen Hurrle342dfc32014-01-17 22:53:15 +01001604 DECLARE_SOCKADDR(struct sockaddr_pkt *, saddr, msg->msg_name);
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001605 struct sk_buff *skb = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001606 struct net_device *dev;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001607 __be16 proto = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001608 int err;
Ben Greear3bdc0eb2012-02-11 15:39:30 +00001609 int extra_len = 0;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001610
Linus Torvalds1da177e2005-04-16 15:20:36 -07001611 /*
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001612 * Get and verify the address.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001613 */
1614
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001615 if (saddr) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001616 if (msg->msg_namelen < sizeof(struct sockaddr))
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001617 return -EINVAL;
1618 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1619 proto = saddr->spkt_protocol;
1620 } else
1621 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001622
1623 /*
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001624 * Find the device first to size check it
Linus Torvalds1da177e2005-04-16 15:20:36 -07001625 */
1626
danborkmann@iogearbox.netde74e922012-06-10 08:59:28 +00001627 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001628retry:
Eric Dumazet654d1f82009-11-02 10:43:32 +01001629 rcu_read_lock();
1630 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001631 err = -ENODEV;
1632 if (dev == NULL)
1633 goto out_unlock;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001634
David S. Millerd5e76b02007-01-25 19:30:36 -08001635 err = -ENETDOWN;
1636 if (!(dev->flags & IFF_UP))
1637 goto out_unlock;
1638
Linus Torvalds1da177e2005-04-16 15:20:36 -07001639 /*
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001640 * You may not queue a frame bigger than the mtu. This is the lowest level
1641 * raw protocol and you must do your own fragmentation at this level.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001642 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001643
Ben Greear3bdc0eb2012-02-11 15:39:30 +00001644 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1645 if (!netif_supports_nofcs(dev)) {
1646 err = -EPROTONOSUPPORT;
1647 goto out_unlock;
1648 }
1649 extra_len = 4; /* We're doing our own CRC */
1650 }
1651
Linus Torvalds1da177e2005-04-16 15:20:36 -07001652 err = -EMSGSIZE;
Ben Greear3bdc0eb2012-02-11 15:39:30 +00001653 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001654 goto out_unlock;
1655
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001656 if (!skb) {
1657 size_t reserved = LL_RESERVED_SPACE(dev);
Herbert Xu4ce40912011-11-18 02:20:05 +00001658 int tlen = dev->needed_tailroom;
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001659 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001660
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001661 rcu_read_unlock();
Herbert Xu4ce40912011-11-18 02:20:05 +00001662 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001663 if (skb == NULL)
1664 return -ENOBUFS;
1665 /* FIXME: Save some space for broken drivers that write a hard
1666 * header at transmission time by themselves. PPP is the notable
1667 * one here. This should really be fixed at the driver level.
1668 */
1669 skb_reserve(skb, reserved);
1670 skb_reset_network_header(skb);
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001671
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001672 /* Try to align data part correctly */
1673 if (hhlen) {
1674 skb->data -= hhlen;
1675 skb->tail -= hhlen;
1676 if (len < hhlen)
1677 skb_reset_network_header(skb);
1678 }
1679 err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
1680 if (err)
1681 goto out_free;
1682 goto retry;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001683 }
1684
Ben Greear3bdc0eb2012-02-11 15:39:30 +00001685 if (len > (dev->mtu + dev->hard_header_len + extra_len)) {
Ben Greear57f89bf2011-02-11 09:35:18 +00001686 /* Earlier code assumed this would be a VLAN pkt,
1687 * double-check this now that we have the actual
1688 * packet in hand.
1689 */
1690 struct ethhdr *ehdr;
1691 skb_reset_mac_header(skb);
1692 ehdr = eth_hdr(skb);
1693 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
1694 err = -EMSGSIZE;
1695 goto out_unlock;
1696 }
1697 }
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001698
Linus Torvalds1da177e2005-04-16 15:20:36 -07001699 skb->protocol = proto;
1700 skb->dev = dev;
1701 skb->priority = sk->sk_priority;
Eric Dumazet2d37a182009-10-01 19:14:46 +00001702 skb->mark = sk->sk_mark;
Daniel Borkmannbf84a0102013-04-14 08:08:13 +00001703
1704 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001705
Ben Greear3bdc0eb2012-02-11 15:39:30 +00001706 if (unlikely(extra_len == 4))
1707 skb->no_fcs = 1;
1708
Jason Wang40893fd2013-03-26 23:11:22 +00001709 skb_probe_transport_header(skb, 0);
Jason Wangc1aad272013-03-25 20:19:57 +00001710
Linus Torvalds1da177e2005-04-16 15:20:36 -07001711 dev_queue_xmit(skb);
Eric Dumazet654d1f82009-11-02 10:43:32 +01001712 rcu_read_unlock();
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001713 return len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001714
Linus Torvalds1da177e2005-04-16 15:20:36 -07001715out_unlock:
Eric Dumazet654d1f82009-11-02 10:43:32 +01001716 rcu_read_unlock();
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001717out_free:
1718 kfree_skb(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001719 return err;
1720}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001721
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001722static unsigned int run_filter(const struct sk_buff *skb,
Eric Dumazet62ab0812010-12-06 20:50:09 +00001723 const struct sock *sk,
David S. Millerdbcb5852007-01-24 15:21:02 -08001724 unsigned int res)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001725{
1726 struct sk_filter *filter;
1727
Eric Dumazet80f8f102011-01-18 07:46:52 +00001728 rcu_read_lock();
1729 filter = rcu_dereference(sk->sk_filter);
David S. Millerdbcb5852007-01-24 15:21:02 -08001730 if (filter != NULL)
Eric Dumazet0a148422011-04-20 09:27:32 +00001731 res = SK_RUN_FILTER(filter, skb);
Eric Dumazet80f8f102011-01-18 07:46:52 +00001732 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001733
David S. Millerdbcb5852007-01-24 15:21:02 -08001734 return res;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001735}
1736
1737/*
Eric Dumazet62ab0812010-12-06 20:50:09 +00001738 * This function makes lazy skb cloning in hope that most of packets
1739 * are discarded by BPF.
1740 *
1741 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
1742 * and skb->cb are mangled. It works because (and until) packets
1743 * falling here are owned by current CPU. Output packets are cloned
1744 * by dev_queue_xmit_nit(), input packets are processed by net_bh
1745 * sequencially, so that if we return skb to original state on exit,
1746 * we will not harm anyone.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001747 */
1748
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001749static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
1750 struct packet_type *pt, struct net_device *orig_dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001751{
1752 struct sock *sk;
1753 struct sockaddr_ll *sll;
1754 struct packet_sock *po;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001755 u8 *skb_head = skb->data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001756 int skb_len = skb->len;
David S. Millerdbcb5852007-01-24 15:21:02 -08001757 unsigned int snaplen, res;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001758
1759 if (skb->pkt_type == PACKET_LOOPBACK)
1760 goto drop;
1761
1762 sk = pt->af_packet_priv;
1763 po = pkt_sk(sk);
1764
Octavian Purdila09ad9bc2009-11-25 15:14:13 -08001765 if (!net_eq(dev_net(dev), sock_net(sk)))
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08001766 goto drop;
1767
Linus Torvalds1da177e2005-04-16 15:20:36 -07001768 skb->dev = dev;
1769
Stephen Hemminger3b04ddd2007-10-09 01:40:57 -07001770 if (dev->header_ops) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001771 /* The device has an explicit notion of ll header,
Eric Dumazet62ab0812010-12-06 20:50:09 +00001772 * exported to higher levels.
1773 *
1774 * Otherwise, the device hides details of its frame
1775 * structure, so that corresponding packet head is
1776 * never delivered to user.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001777 */
1778 if (sk->sk_type != SOCK_DGRAM)
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07001779 skb_push(skb, skb->data - skb_mac_header(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001780 else if (skb->pkt_type == PACKET_OUTGOING) {
1781 /* Special case: outgoing packets have ll header at head */
Arnaldo Carvalho de Melobbe735e2007-03-10 22:16:10 -03001782 skb_pull(skb, skb_network_offset(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001783 }
1784 }
1785
1786 snaplen = skb->len;
1787
David S. Millerdbcb5852007-01-24 15:21:02 -08001788 res = run_filter(skb, sk, snaplen);
1789 if (!res)
Dmitry Mishinfda9ef52006-08-31 15:28:39 -07001790 goto drop_n_restore;
David S. Millerdbcb5852007-01-24 15:21:02 -08001791 if (snaplen > res)
1792 snaplen = res;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001793
Eric Dumazet0fd7bac2011-12-21 07:11:44 +00001794 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001795 goto drop_n_acct;
1796
1797 if (skb_shared(skb)) {
1798 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
1799 if (nskb == NULL)
1800 goto drop_n_acct;
1801
1802 if (skb_head != skb->data) {
1803 skb->data = skb_head;
1804 skb->len = skb_len;
1805 }
Eric Dumazetabc4e4f2012-04-19 02:24:42 +00001806 consume_skb(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001807 skb = nskb;
1808 }
1809
Herbert Xuffbc6112007-02-04 23:33:10 -08001810 BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8 >
1811 sizeof(skb->cb));
1812
1813 sll = &PACKET_SKB_CB(skb)->sa.ll;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001814 sll->sll_family = AF_PACKET;
1815 sll->sll_hatype = dev->type;
1816 sll->sll_protocol = skb->protocol;
1817 sll->sll_pkttype = skb->pkt_type;
Peter P Waskiewicz Jr8032b462007-11-10 22:03:25 -08001818 if (unlikely(po->origdev))
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07001819 sll->sll_ifindex = orig_dev->ifindex;
1820 else
1821 sll->sll_ifindex = dev->ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001822
Stephen Hemmingerb95cce32007-09-26 22:13:38 -07001823 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001824
Herbert Xuffbc6112007-02-04 23:33:10 -08001825 PACKET_SKB_CB(skb)->origlen = skb->len;
Herbert Xu8dc41942007-02-04 23:31:32 -08001826
Linus Torvalds1da177e2005-04-16 15:20:36 -07001827 if (pskb_trim(skb, snaplen))
1828 goto drop_n_acct;
1829
1830 skb_set_owner_r(skb, sk);
1831 skb->dev = NULL;
Eric Dumazetadf30902009-06-02 05:19:30 +00001832 skb_dst_drop(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001833
Phil Oester84531c22005-07-12 11:57:52 -07001834 /* drop conntrack reference */
1835 nf_reset(skb);
1836
Linus Torvalds1da177e2005-04-16 15:20:36 -07001837 spin_lock(&sk->sk_receive_queue.lock);
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00001838 po->stats.stats1.tp_packets++;
Neil Horman3b885782009-10-12 13:26:31 -07001839 skb->dropcount = atomic_read(&sk->sk_drops);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001840 __skb_queue_tail(&sk->sk_receive_queue, skb);
1841 spin_unlock(&sk->sk_receive_queue.lock);
David S. Miller676d2362014-04-11 16:15:36 -04001842 sk->sk_data_ready(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001843 return 0;
1844
1845drop_n_acct:
Willem de Bruijn7091fbd2011-09-30 10:38:28 +00001846 spin_lock(&sk->sk_receive_queue.lock);
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00001847 po->stats.stats1.tp_drops++;
Willem de Bruijn7091fbd2011-09-30 10:38:28 +00001848 atomic_inc(&sk->sk_drops);
1849 spin_unlock(&sk->sk_receive_queue.lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001850
1851drop_n_restore:
1852 if (skb_head != skb->data && skb_shared(skb)) {
1853 skb->data = skb_head;
1854 skb->len = skb_len;
1855 }
1856drop:
Neil Hormanead2ceb2009-03-11 09:49:55 +00001857 consume_skb(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001858 return 0;
1859}
1860
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001861static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
1862 struct packet_type *pt, struct net_device *orig_dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001863{
1864 struct sock *sk;
1865 struct packet_sock *po;
1866 struct sockaddr_ll *sll;
Daniel Borkmann184f4892013-04-16 01:57:46 +00001867 union tpacket_uhdr h;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001868 u8 *skb_head = skb->data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001869 int skb_len = skb->len;
David S. Millerdbcb5852007-01-24 15:21:02 -08001870 unsigned int snaplen, res;
chetan lokef6fb8f12011-08-19 10:18:16 +00001871 unsigned long status = TP_STATUS_USER;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07001872 unsigned short macoff, netoff, hdrlen;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001873 struct sk_buff *copy_skb = NULL;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07001874 struct timespec ts;
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +00001875 __u32 ts_status;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001876
Atzm Watanabe51846352013-12-17 22:53:32 +09001877 /* struct tpacket{2,3}_hdr is aligned to a multiple of TPACKET_ALIGNMENT.
1878 * We may add members to them until current aligned size without forcing
1879 * userspace to call getsockopt(..., PACKET_HDRLEN, ...).
1880 */
1881 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h2)) != 32);
1882 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h3)) != 48);
1883
Linus Torvalds1da177e2005-04-16 15:20:36 -07001884 if (skb->pkt_type == PACKET_LOOPBACK)
1885 goto drop;
1886
1887 sk = pt->af_packet_priv;
1888 po = pkt_sk(sk);
1889
Octavian Purdila09ad9bc2009-11-25 15:14:13 -08001890 if (!net_eq(dev_net(dev), sock_net(sk)))
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08001891 goto drop;
1892
Stephen Hemminger3b04ddd2007-10-09 01:40:57 -07001893 if (dev->header_ops) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001894 if (sk->sk_type != SOCK_DGRAM)
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07001895 skb_push(skb, skb->data - skb_mac_header(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001896 else if (skb->pkt_type == PACKET_OUTGOING) {
1897 /* Special case: outgoing packets have ll header at head */
Arnaldo Carvalho de Melobbe735e2007-03-10 22:16:10 -03001898 skb_pull(skb, skb_network_offset(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001899 }
1900 }
1901
Herbert Xu8dc41942007-02-04 23:31:32 -08001902 if (skb->ip_summed == CHECKSUM_PARTIAL)
1903 status |= TP_STATUS_CSUMNOTREADY;
1904
Linus Torvalds1da177e2005-04-16 15:20:36 -07001905 snaplen = skb->len;
1906
David S. Millerdbcb5852007-01-24 15:21:02 -08001907 res = run_filter(skb, sk, snaplen);
1908 if (!res)
Dmitry Mishinfda9ef52006-08-31 15:28:39 -07001909 goto drop_n_restore;
David S. Millerdbcb5852007-01-24 15:21:02 -08001910 if (snaplen > res)
1911 snaplen = res;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001912
1913 if (sk->sk_type == SOCK_DGRAM) {
Patrick McHardy8913336a2008-07-18 18:05:19 -07001914 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
1915 po->tp_reserve;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001916 } else {
Eric Dumazet95c96172012-04-15 05:58:06 +00001917 unsigned int maclen = skb_network_offset(skb);
Patrick McHardybbd6ef82008-07-14 22:50:15 -07001918 netoff = TPACKET_ALIGN(po->tp_hdrlen +
Patrick McHardy8913336a2008-07-18 18:05:19 -07001919 (maclen < 16 ? 16 : maclen)) +
1920 po->tp_reserve;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001921 macoff = netoff - maclen;
1922 }
chetan lokef6fb8f12011-08-19 10:18:16 +00001923 if (po->tp_version <= TPACKET_V2) {
1924 if (macoff + snaplen > po->rx_ring.frame_size) {
1925 if (po->copy_thresh &&
Eric Dumazet0fd7bac2011-12-21 07:11:44 +00001926 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
chetan lokef6fb8f12011-08-19 10:18:16 +00001927 if (skb_shared(skb)) {
1928 copy_skb = skb_clone(skb, GFP_ATOMIC);
1929 } else {
1930 copy_skb = skb_get(skb);
1931 skb_head = skb->data;
1932 }
1933 if (copy_skb)
1934 skb_set_owner_r(copy_skb, sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001935 }
chetan lokef6fb8f12011-08-19 10:18:16 +00001936 snaplen = po->rx_ring.frame_size - macoff;
1937 if ((int)snaplen < 0)
1938 snaplen = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001939 }
Eric Dumazetdc808112014-08-15 09:16:04 -07001940 } else if (unlikely(macoff + snaplen >
1941 GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len)) {
1942 u32 nval;
1943
1944 nval = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len - macoff;
1945 pr_err_once("tpacket_rcv: packet too big, clamped from %u to %u. macoff=%u\n",
1946 snaplen, nval, macoff);
1947 snaplen = nval;
1948 if (unlikely((int)snaplen < 0)) {
1949 snaplen = 0;
1950 macoff = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len;
1951 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001952 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001953 spin_lock(&sk->sk_receive_queue.lock);
chetan lokef6fb8f12011-08-19 10:18:16 +00001954 h.raw = packet_current_rx_frame(po, skb,
1955 TP_STATUS_KERNEL, (macoff+snaplen));
Patrick McHardybbd6ef82008-07-14 22:50:15 -07001956 if (!h.raw)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001957 goto ring_is_full;
chetan lokef6fb8f12011-08-19 10:18:16 +00001958 if (po->tp_version <= TPACKET_V2) {
1959 packet_increment_rx_head(po, &po->rx_ring);
1960 /*
1961 * LOSING will be reported till you read the stats,
1962 * because it's COR - Clear On Read.
1963 * Anyways, moving it for V1/V2 only as V3 doesn't need this
1964 * at packet level.
1965 */
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00001966 if (po->stats.stats1.tp_drops)
chetan lokef6fb8f12011-08-19 10:18:16 +00001967 status |= TP_STATUS_LOSING;
1968 }
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00001969 po->stats.stats1.tp_packets++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001970 if (copy_skb) {
1971 status |= TP_STATUS_COPY;
1972 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
1973 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001974 spin_unlock(&sk->sk_receive_queue.lock);
1975
Patrick McHardybbd6ef82008-07-14 22:50:15 -07001976 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +00001977
1978 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
Daniel Borkmann7a513842013-04-23 00:39:29 +00001979 getnstimeofday(&ts);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001980
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +00001981 status |= ts_status;
1982
Patrick McHardybbd6ef82008-07-14 22:50:15 -07001983 switch (po->tp_version) {
1984 case TPACKET_V1:
1985 h.h1->tp_len = skb->len;
1986 h.h1->tp_snaplen = snaplen;
1987 h.h1->tp_mac = macoff;
1988 h.h1->tp_net = netoff;
Daniel Borkmann4b457bd2013-04-16 01:29:11 +00001989 h.h1->tp_sec = ts.tv_sec;
1990 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07001991 hdrlen = sizeof(*h.h1);
1992 break;
1993 case TPACKET_V2:
1994 h.h2->tp_len = skb->len;
1995 h.h2->tp_snaplen = snaplen;
1996 h.h2->tp_mac = macoff;
1997 h.h2->tp_net = netoff;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07001998 h.h2->tp_sec = ts.tv_sec;
1999 h.h2->tp_nsec = ts.tv_nsec;
Ben Greeara3bcc232011-06-01 06:49:10 +00002000 if (vlan_tx_tag_present(skb)) {
2001 h.h2->tp_vlan_tci = vlan_tx_tag_get(skb);
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +09002002 h.h2->tp_vlan_tpid = ntohs(skb->vlan_proto);
2003 status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
Ben Greeara3bcc232011-06-01 06:49:10 +00002004 } else {
2005 h.h2->tp_vlan_tci = 0;
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +09002006 h.h2->tp_vlan_tpid = 0;
Ben Greeara3bcc232011-06-01 06:49:10 +00002007 }
Atzm Watanabee4d26f42013-12-17 22:53:36 +09002008 memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding));
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002009 hdrlen = sizeof(*h.h2);
2010 break;
chetan lokef6fb8f12011-08-19 10:18:16 +00002011 case TPACKET_V3:
2012 /* tp_nxt_offset,vlan are already populated above.
2013 * So DONT clear those fields here
2014 */
2015 h.h3->tp_status |= status;
2016 h.h3->tp_len = skb->len;
2017 h.h3->tp_snaplen = snaplen;
2018 h.h3->tp_mac = macoff;
2019 h.h3->tp_net = netoff;
chetan lokef6fb8f12011-08-19 10:18:16 +00002020 h.h3->tp_sec = ts.tv_sec;
2021 h.h3->tp_nsec = ts.tv_nsec;
Atzm Watanabee4d26f42013-12-17 22:53:36 +09002022 memset(h.h3->tp_padding, 0, sizeof(h.h3->tp_padding));
chetan lokef6fb8f12011-08-19 10:18:16 +00002023 hdrlen = sizeof(*h.h3);
2024 break;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002025 default:
2026 BUG();
2027 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002028
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002029 sll = h.raw + TPACKET_ALIGN(hdrlen);
Stephen Hemmingerb95cce32007-09-26 22:13:38 -07002030 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002031 sll->sll_family = AF_PACKET;
2032 sll->sll_hatype = dev->type;
2033 sll->sll_protocol = skb->protocol;
2034 sll->sll_pkttype = skb->pkt_type;
Peter P Waskiewicz Jr8032b462007-11-10 22:03:25 -08002035 if (unlikely(po->origdev))
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07002036 sll->sll_ifindex = orig_dev->ifindex;
2037 else
2038 sll->sll_ifindex = dev->ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002039
Ralf Baechlee16aa202006-12-07 00:11:33 -08002040 smp_mb();
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002041
Changli Gaof6dafa92010-12-07 04:26:16 +00002042#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002043 if (po->tp_version <= TPACKET_V2) {
Changli Gao0af55bb2010-12-01 02:52:20 +00002044 u8 *start, *end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002045
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002046 end = (u8 *) PAGE_ALIGN((unsigned long) h.raw +
2047 macoff + snaplen);
2048
2049 for (start = h.raw; start < end; start += PAGE_SIZE)
2050 flush_dcache_page(pgv_to_page(start));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002051 }
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002052 smp_wmb();
Changli Gaof6dafa92010-12-07 04:26:16 +00002053#endif
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002054
chetan lokef6fb8f12011-08-19 10:18:16 +00002055 if (po->tp_version <= TPACKET_V2)
2056 __packet_set_status(po, h.raw, status);
2057 else
2058 prb_clear_blk_fill_status(&po->rx_ring);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002059
David S. Miller676d2362014-04-11 16:15:36 -04002060 sk->sk_data_ready(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002061
2062drop_n_restore:
2063 if (skb_head != skb->data && skb_shared(skb)) {
2064 skb->data = skb_head;
2065 skb->len = skb_len;
2066 }
2067drop:
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002068 kfree_skb(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002069 return 0;
2070
2071ring_is_full:
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00002072 po->stats.stats1.tp_drops++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002073 spin_unlock(&sk->sk_receive_queue.lock);
2074
David S. Miller676d2362014-04-11 16:15:36 -04002075 sk->sk_data_ready(sk);
Wei Yongjunacb5d752009-02-25 00:36:42 +00002076 kfree_skb(copy_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002077 goto drop_n_restore;
2078}
2079
Johann Baudy69e3c752009-05-18 22:11:22 -07002080static void tpacket_destruct_skb(struct sk_buff *skb)
2081{
2082 struct packet_sock *po = pkt_sk(skb->sk);
Johann Baudy69e3c752009-05-18 22:11:22 -07002083
Johann Baudy69e3c752009-05-18 22:11:22 -07002084 if (likely(po->tx_ring.pg_vec)) {
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002085 void *ph;
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +00002086 __u32 ts;
2087
Johann Baudy69e3c752009-05-18 22:11:22 -07002088 ph = skb_shinfo(skb)->destructor_arg;
Daniel Borkmannb0138402014-01-15 16:25:36 +01002089 packet_dec_pending(&po->tx_ring);
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +00002090
2091 ts = __packet_set_timestamp(po, ph, skb);
2092 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
Johann Baudy69e3c752009-05-18 22:11:22 -07002093 }
2094
2095 sock_wfree(skb);
2096}
2097
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002098static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
2099 void *frame, struct net_device *dev, int size_max,
Herbert Xuae641942011-11-18 02:20:04 +00002100 __be16 proto, unsigned char *addr, int hlen)
Johann Baudy69e3c752009-05-18 22:11:22 -07002101{
Daniel Borkmann184f4892013-04-16 01:57:46 +00002102 union tpacket_uhdr ph;
David S. Miller09effa62013-08-07 17:11:00 -07002103 int to_write, offset, len, tp_len, nr_frags, len_max;
Johann Baudy69e3c752009-05-18 22:11:22 -07002104 struct socket *sock = po->sk.sk_socket;
2105 struct page *page;
2106 void *data;
2107 int err;
2108
2109 ph.raw = frame;
2110
2111 skb->protocol = proto;
2112 skb->dev = dev;
2113 skb->priority = po->sk.sk_priority;
Eric Dumazet2d37a182009-10-01 19:14:46 +00002114 skb->mark = po->sk.sk_mark;
Willem de Bruijn2e313962013-04-23 00:39:28 +00002115 sock_tx_timestamp(&po->sk, &skb_shinfo(skb)->tx_flags);
Johann Baudy69e3c752009-05-18 22:11:22 -07002116 skb_shinfo(skb)->destructor_arg = ph.raw;
2117
2118 switch (po->tp_version) {
2119 case TPACKET_V2:
2120 tp_len = ph.h2->tp_len;
2121 break;
2122 default:
2123 tp_len = ph.h1->tp_len;
2124 break;
2125 }
David S. Miller09effa62013-08-07 17:11:00 -07002126 if (unlikely(tp_len > size_max)) {
2127 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
2128 return -EMSGSIZE;
2129 }
Johann Baudy69e3c752009-05-18 22:11:22 -07002130
Herbert Xuae641942011-11-18 02:20:04 +00002131 skb_reserve(skb, hlen);
Johann Baudy69e3c752009-05-18 22:11:22 -07002132 skb_reset_network_header(skb);
Jason Wangc1aad272013-03-25 20:19:57 +00002133
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002134 if (!packet_use_direct_xmit(po))
2135 skb_probe_transport_header(skb, 0);
2136 if (unlikely(po->tp_tx_has_off)) {
Paul Chavent5920cd3a2012-11-06 23:10:47 +00002137 int off_min, off_max, off;
2138 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2139 off_max = po->tx_ring.frame_size - tp_len;
2140 if (sock->type == SOCK_DGRAM) {
2141 switch (po->tp_version) {
2142 case TPACKET_V2:
2143 off = ph.h2->tp_net;
2144 break;
2145 default:
2146 off = ph.h1->tp_net;
2147 break;
2148 }
2149 } else {
2150 switch (po->tp_version) {
2151 case TPACKET_V2:
2152 off = ph.h2->tp_mac;
2153 break;
2154 default:
2155 off = ph.h1->tp_mac;
2156 break;
2157 }
2158 }
2159 if (unlikely((off < off_min) || (off_max < off)))
2160 return -EINVAL;
2161 data = ph.raw + off;
2162 } else {
2163 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
2164 }
Johann Baudy69e3c752009-05-18 22:11:22 -07002165 to_write = tp_len;
2166
2167 if (sock->type == SOCK_DGRAM) {
2168 err = dev_hard_header(skb, dev, ntohs(proto), addr,
2169 NULL, tp_len);
2170 if (unlikely(err < 0))
2171 return -EINVAL;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002172 } else if (dev->hard_header_len) {
Johann Baudy69e3c752009-05-18 22:11:22 -07002173 /* net device doesn't like empty head */
2174 if (unlikely(tp_len <= dev->hard_header_len)) {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002175 pr_err("packet size is too short (%d < %d)\n",
2176 tp_len, dev->hard_header_len);
Johann Baudy69e3c752009-05-18 22:11:22 -07002177 return -EINVAL;
2178 }
2179
2180 skb_push(skb, dev->hard_header_len);
2181 err = skb_store_bits(skb, 0, data,
2182 dev->hard_header_len);
2183 if (unlikely(err))
2184 return err;
2185
2186 data += dev->hard_header_len;
2187 to_write -= dev->hard_header_len;
2188 }
2189
Johann Baudy69e3c752009-05-18 22:11:22 -07002190 offset = offset_in_page(data);
2191 len_max = PAGE_SIZE - offset;
2192 len = ((to_write > len_max) ? len_max : to_write);
2193
2194 skb->data_len = to_write;
2195 skb->len += to_write;
2196 skb->truesize += to_write;
2197 atomic_add(to_write, &po->sk.sk_wmem_alloc);
2198
2199 while (likely(to_write)) {
2200 nr_frags = skb_shinfo(skb)->nr_frags;
2201
2202 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002203 pr_err("Packet exceed the number of skb frags(%lu)\n",
2204 MAX_SKB_FRAGS);
Johann Baudy69e3c752009-05-18 22:11:22 -07002205 return -EFAULT;
2206 }
2207
Changli Gao0af55bb2010-12-01 02:52:20 +00002208 page = pgv_to_page(data);
2209 data += len;
Johann Baudy69e3c752009-05-18 22:11:22 -07002210 flush_dcache_page(page);
2211 get_page(page);
Changli Gao0af55bb2010-12-01 02:52:20 +00002212 skb_fill_page_desc(skb, nr_frags, page, offset, len);
Johann Baudy69e3c752009-05-18 22:11:22 -07002213 to_write -= len;
2214 offset = 0;
2215 len_max = PAGE_SIZE;
2216 len = ((to_write > len_max) ? len_max : to_write);
2217 }
2218
2219 return tp_len;
2220}
2221
2222static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2223{
Johann Baudy69e3c752009-05-18 22:11:22 -07002224 struct sk_buff *skb;
2225 struct net_device *dev;
2226 __be16 proto;
David S. Miller09effa62013-08-07 17:11:00 -07002227 int err, reserve = 0;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002228 void *ph;
Steffen Hurrle342dfc32014-01-17 22:53:15 +01002229 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
Daniel Borkmann87a2fd22014-01-15 16:25:35 +01002230 bool need_wait = !(msg->msg_flags & MSG_DONTWAIT);
Johann Baudy69e3c752009-05-18 22:11:22 -07002231 int tp_len, size_max;
2232 unsigned char *addr;
2233 int len_sum = 0;
danborkmann@iogearbox.net9e670302012-08-20 03:34:03 +00002234 int status = TP_STATUS_AVAILABLE;
Herbert Xuae641942011-11-18 02:20:04 +00002235 int hlen, tlen;
Johann Baudy69e3c752009-05-18 22:11:22 -07002236
Johann Baudy69e3c752009-05-18 22:11:22 -07002237 mutex_lock(&po->pg_vec_lock);
2238
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01002239 if (likely(saddr == NULL)) {
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002240 dev = packet_cached_dev_get(po);
Johann Baudy69e3c752009-05-18 22:11:22 -07002241 proto = po->num;
2242 addr = NULL;
2243 } else {
2244 err = -EINVAL;
2245 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2246 goto out;
2247 if (msg->msg_namelen < (saddr->sll_halen
2248 + offsetof(struct sockaddr_ll,
2249 sll_addr)))
2250 goto out;
Johann Baudy69e3c752009-05-18 22:11:22 -07002251 proto = saddr->sll_protocol;
2252 addr = saddr->sll_addr;
Ben Greear827d9782011-06-01 07:18:53 +00002253 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
Johann Baudy69e3c752009-05-18 22:11:22 -07002254 }
2255
Johann Baudy69e3c752009-05-18 22:11:22 -07002256 err = -ENXIO;
2257 if (unlikely(dev == NULL))
2258 goto out;
Johann Baudy69e3c752009-05-18 22:11:22 -07002259 err = -ENETDOWN;
2260 if (unlikely(!(dev->flags & IFF_UP)))
2261 goto out_put;
2262
Daniel Borkmann52f14542014-02-28 02:22:06 +01002263 reserve = dev->hard_header_len + VLAN_HLEN;
Johann Baudy69e3c752009-05-18 22:11:22 -07002264 size_max = po->tx_ring.frame_size
Gabor Gombasb5dd8842009-10-29 03:19:11 -07002265 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
Johann Baudy69e3c752009-05-18 22:11:22 -07002266
David S. Miller09effa62013-08-07 17:11:00 -07002267 if (size_max > dev->mtu + reserve)
2268 size_max = dev->mtu + reserve;
2269
Johann Baudy69e3c752009-05-18 22:11:22 -07002270 do {
2271 ph = packet_current_frame(po, &po->tx_ring,
Daniel Borkmann87a2fd22014-01-15 16:25:35 +01002272 TP_STATUS_SEND_REQUEST);
Johann Baudy69e3c752009-05-18 22:11:22 -07002273 if (unlikely(ph == NULL)) {
Daniel Borkmann87a2fd22014-01-15 16:25:35 +01002274 if (need_wait && need_resched())
2275 schedule();
Johann Baudy69e3c752009-05-18 22:11:22 -07002276 continue;
2277 }
2278
2279 status = TP_STATUS_SEND_REQUEST;
Herbert Xuae641942011-11-18 02:20:04 +00002280 hlen = LL_RESERVED_SPACE(dev);
2281 tlen = dev->needed_tailroom;
Johann Baudy69e3c752009-05-18 22:11:22 -07002282 skb = sock_alloc_send_skb(&po->sk,
Herbert Xuae641942011-11-18 02:20:04 +00002283 hlen + tlen + sizeof(struct sockaddr_ll),
Johann Baudy69e3c752009-05-18 22:11:22 -07002284 0, &err);
2285
2286 if (unlikely(skb == NULL))
2287 goto out_status;
2288
2289 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
Daniel Borkmann52f14542014-02-28 02:22:06 +01002290 addr, hlen);
2291 if (tp_len > dev->mtu + dev->hard_header_len) {
2292 struct ethhdr *ehdr;
2293 /* Earlier code assumed this would be a VLAN pkt,
2294 * double-check this now that we have the actual
2295 * packet in hand.
2296 */
Johann Baudy69e3c752009-05-18 22:11:22 -07002297
Daniel Borkmann52f14542014-02-28 02:22:06 +01002298 skb_reset_mac_header(skb);
2299 ehdr = eth_hdr(skb);
2300 if (ehdr->h_proto != htons(ETH_P_8021Q))
2301 tp_len = -EMSGSIZE;
2302 }
Johann Baudy69e3c752009-05-18 22:11:22 -07002303 if (unlikely(tp_len < 0)) {
2304 if (po->tp_loss) {
2305 __packet_set_status(po, ph,
2306 TP_STATUS_AVAILABLE);
2307 packet_increment_head(&po->tx_ring);
2308 kfree_skb(skb);
2309 continue;
2310 } else {
2311 status = TP_STATUS_WRONG_FORMAT;
2312 err = tp_len;
2313 goto out_status;
2314 }
2315 }
2316
Daniel Borkmann0fd5d572014-02-16 15:55:22 +01002317 packet_pick_tx_queue(dev, skb);
2318
Johann Baudy69e3c752009-05-18 22:11:22 -07002319 skb->destructor = tpacket_destruct_skb;
2320 __packet_set_status(po, ph, TP_STATUS_SENDING);
Daniel Borkmannb0138402014-01-15 16:25:36 +01002321 packet_inc_pending(&po->tx_ring);
Johann Baudy69e3c752009-05-18 22:11:22 -07002322
2323 status = TP_STATUS_SEND_REQUEST;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002324 err = po->xmit(skb);
Jarek Poplawskieb70df12010-01-10 22:04:19 +00002325 if (unlikely(err > 0)) {
2326 err = net_xmit_errno(err);
2327 if (err && __packet_get_status(po, ph) ==
2328 TP_STATUS_AVAILABLE) {
2329 /* skb was destructed already */
2330 skb = NULL;
2331 goto out_status;
2332 }
2333 /*
2334 * skb was dropped but not destructed yet;
2335 * let's treat it like congestion or err < 0
2336 */
2337 err = 0;
2338 }
Johann Baudy69e3c752009-05-18 22:11:22 -07002339 packet_increment_head(&po->tx_ring);
2340 len_sum += tp_len;
Daniel Borkmannb0138402014-01-15 16:25:36 +01002341 } while (likely((ph != NULL) ||
2342 /* Note: packet_read_pending() might be slow if we have
2343 * to call it as it's per_cpu variable, but in fast-path
2344 * we already short-circuit the loop with the first
2345 * condition, and luckily don't have to go that path
2346 * anyway.
2347 */
2348 (need_wait && packet_read_pending(&po->tx_ring))));
Johann Baudy69e3c752009-05-18 22:11:22 -07002349
2350 err = len_sum;
2351 goto out_put;
2352
Johann Baudy69e3c752009-05-18 22:11:22 -07002353out_status:
2354 __packet_set_status(po, ph, status);
2355 kfree_skb(skb);
2356out_put:
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002357 dev_put(dev);
Johann Baudy69e3c752009-05-18 22:11:22 -07002358out:
2359 mutex_unlock(&po->pg_vec_lock);
2360 return err;
2361}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002362
Olof Johanssoneea49cc92011-11-02 11:00:49 +00002363static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2364 size_t reserve, size_t len,
2365 size_t linear, int noblock,
2366 int *err)
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002367{
2368 struct sk_buff *skb;
2369
2370 /* Under a page? Don't bother with paged skb. */
2371 if (prepad + len < PAGE_SIZE || !linear)
2372 linear = len;
2373
2374 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
Eric Dumazet28d64272013-08-08 14:38:47 -07002375 err, 0);
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002376 if (!skb)
2377 return NULL;
2378
2379 skb_reserve(skb, reserve);
2380 skb_put(skb, linear);
2381 skb->data_len = len - linear;
2382 skb->len += len - linear;
2383
2384 return skb;
2385}
2386
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002387static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002388{
2389 struct sock *sk = sock->sk;
Steffen Hurrle342dfc32014-01-17 22:53:15 +01002390 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002391 struct sk_buff *skb;
2392 struct net_device *dev;
Al Viro0e11c912006-11-08 00:26:29 -08002393 __be16 proto;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002394 unsigned char *addr;
Ben Greear827d9782011-06-01 07:18:53 +00002395 int err, reserve = 0;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002396 struct virtio_net_hdr vnet_hdr = { 0 };
2397 int offset = 0;
2398 int vnet_hdr_len;
2399 struct packet_sock *po = pkt_sk(sk);
2400 unsigned short gso_type = 0;
Herbert Xuae641942011-11-18 02:20:04 +00002401 int hlen, tlen;
Ben Greear3bdc0eb2012-02-11 15:39:30 +00002402 int extra_len = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002403
2404 /*
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002405 * Get and verify the address.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002406 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002407
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01002408 if (likely(saddr == NULL)) {
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002409 dev = packet_cached_dev_get(po);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002410 proto = po->num;
2411 addr = NULL;
2412 } else {
2413 err = -EINVAL;
2414 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2415 goto out;
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07002416 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2417 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002418 proto = saddr->sll_protocol;
2419 addr = saddr->sll_addr;
Ben Greear827d9782011-06-01 07:18:53 +00002420 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002421 }
2422
Linus Torvalds1da177e2005-04-16 15:20:36 -07002423 err = -ENXIO;
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002424 if (unlikely(dev == NULL))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002425 goto out_unlock;
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002426 err = -ENETDOWN;
2427 if (unlikely(!(dev->flags & IFF_UP)))
2428 goto out_unlock;
2429
Linus Torvalds1da177e2005-04-16 15:20:36 -07002430 if (sock->type == SOCK_RAW)
2431 reserve = dev->hard_header_len;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002432 if (po->has_vnet_hdr) {
2433 vnet_hdr_len = sizeof(vnet_hdr);
2434
2435 err = -EINVAL;
2436 if (len < vnet_hdr_len)
2437 goto out_unlock;
2438
2439 len -= vnet_hdr_len;
2440
2441 err = memcpy_fromiovec((void *)&vnet_hdr, msg->msg_iov,
2442 vnet_hdr_len);
2443 if (err < 0)
2444 goto out_unlock;
2445
2446 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
Michael S. Tsirkindc9e5152014-11-23 18:16:19 +02002447 (__virtio16_to_cpu(false, vnet_hdr.csum_start) +
2448 __virtio16_to_cpu(false, vnet_hdr.csum_offset) + 2 >
2449 __virtio16_to_cpu(false, vnet_hdr.hdr_len)))
2450 vnet_hdr.hdr_len = __cpu_to_virtio16(false,
2451 __virtio16_to_cpu(false, vnet_hdr.csum_start) +
2452 __virtio16_to_cpu(false, vnet_hdr.csum_offset) + 2);
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002453
2454 err = -EINVAL;
Michael S. Tsirkindc9e5152014-11-23 18:16:19 +02002455 if (__virtio16_to_cpu(false, vnet_hdr.hdr_len) > len)
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002456 goto out_unlock;
2457
2458 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2459 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2460 case VIRTIO_NET_HDR_GSO_TCPV4:
2461 gso_type = SKB_GSO_TCPV4;
2462 break;
2463 case VIRTIO_NET_HDR_GSO_TCPV6:
2464 gso_type = SKB_GSO_TCPV6;
2465 break;
2466 case VIRTIO_NET_HDR_GSO_UDP:
2467 gso_type = SKB_GSO_UDP;
2468 break;
2469 default:
2470 goto out_unlock;
2471 }
2472
2473 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
2474 gso_type |= SKB_GSO_TCP_ECN;
2475
2476 if (vnet_hdr.gso_size == 0)
2477 goto out_unlock;
2478
2479 }
2480 }
2481
Ben Greear3bdc0eb2012-02-11 15:39:30 +00002482 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2483 if (!netif_supports_nofcs(dev)) {
2484 err = -EPROTONOSUPPORT;
2485 goto out_unlock;
2486 }
2487 extra_len = 4; /* We're doing our own CRC */
2488 }
2489
Linus Torvalds1da177e2005-04-16 15:20:36 -07002490 err = -EMSGSIZE;
Ben Greear3bdc0eb2012-02-11 15:39:30 +00002491 if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002492 goto out_unlock;
2493
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002494 err = -ENOBUFS;
Herbert Xuae641942011-11-18 02:20:04 +00002495 hlen = LL_RESERVED_SPACE(dev);
2496 tlen = dev->needed_tailroom;
Michael S. Tsirkindc9e5152014-11-23 18:16:19 +02002497 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len,
2498 __virtio16_to_cpu(false, vnet_hdr.hdr_len),
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002499 msg->msg_flags & MSG_DONTWAIT, &err);
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002500 if (skb == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002501 goto out_unlock;
2502
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002503 skb_set_network_header(skb, reserve);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002504
Stephen Hemminger0c4e8582007-10-09 01:36:32 -07002505 err = -EINVAL;
2506 if (sock->type == SOCK_DGRAM &&
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002507 (offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len)) < 0)
Stephen Hemminger0c4e8582007-10-09 01:36:32 -07002508 goto out_free;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002509
2510 /* Returns -EFAULT on error */
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002511 err = skb_copy_datagram_from_iovec(skb, offset, msg->msg_iov, 0, len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002512 if (err)
2513 goto out_free;
Daniel Borkmannbf84a0102013-04-14 08:08:13 +00002514
2515 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002516
Ben Greear3bdc0eb2012-02-11 15:39:30 +00002517 if (!gso_type && (len > dev->mtu + reserve + extra_len)) {
David S. Miller09effa62013-08-07 17:11:00 -07002518 /* Earlier code assumed this would be a VLAN pkt,
2519 * double-check this now that we have the actual
2520 * packet in hand.
2521 */
2522 struct ethhdr *ehdr;
2523 skb_reset_mac_header(skb);
2524 ehdr = eth_hdr(skb);
2525 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
2526 err = -EMSGSIZE;
2527 goto out_free;
2528 }
Ben Greear57f89bf2011-02-11 09:35:18 +00002529 }
2530
David S. Miller09effa62013-08-07 17:11:00 -07002531 skb->protocol = proto;
2532 skb->dev = dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002533 skb->priority = sk->sk_priority;
Eric Dumazet2d37a182009-10-01 19:14:46 +00002534 skb->mark = sk->sk_mark;
Daniel Borkmann0fd5d572014-02-16 15:55:22 +01002535
2536 packet_pick_tx_queue(dev, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002537
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002538 if (po->has_vnet_hdr) {
2539 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
Michael S. Tsirkindc9e5152014-11-23 18:16:19 +02002540 u16 s = __virtio16_to_cpu(false, vnet_hdr.csum_start);
2541 u16 o = __virtio16_to_cpu(false, vnet_hdr.csum_offset);
2542 if (!skb_partial_csum_set(skb, s, o)) {
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002543 err = -EINVAL;
2544 goto out_free;
2545 }
2546 }
2547
Michael S. Tsirkindc9e5152014-11-23 18:16:19 +02002548 skb_shinfo(skb)->gso_size =
2549 __virtio16_to_cpu(false, vnet_hdr.gso_size);
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002550 skb_shinfo(skb)->gso_type = gso_type;
2551
2552 /* Header must be checked, and gso_segs computed. */
2553 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2554 skb_shinfo(skb)->gso_segs = 0;
2555
2556 len += vnet_hdr_len;
2557 }
2558
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002559 if (!packet_use_direct_xmit(po))
2560 skb_probe_transport_header(skb, reserve);
Ben Greear3bdc0eb2012-02-11 15:39:30 +00002561 if (unlikely(extra_len == 4))
2562 skb->no_fcs = 1;
2563
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002564 err = po->xmit(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002565 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2566 goto out_unlock;
2567
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002568 dev_put(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002569
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002570 return len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002571
2572out_free:
2573 kfree_skb(skb);
2574out_unlock:
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002575 if (dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002576 dev_put(dev);
2577out:
2578 return err;
2579}
2580
Johann Baudy69e3c752009-05-18 22:11:22 -07002581static int packet_sendmsg(struct kiocb *iocb, struct socket *sock,
2582 struct msghdr *msg, size_t len)
2583{
Johann Baudy69e3c752009-05-18 22:11:22 -07002584 struct sock *sk = sock->sk;
2585 struct packet_sock *po = pkt_sk(sk);
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002586
Johann Baudy69e3c752009-05-18 22:11:22 -07002587 if (po->tx_ring.pg_vec)
2588 return tpacket_snd(po, msg);
2589 else
Johann Baudy69e3c752009-05-18 22:11:22 -07002590 return packet_snd(sock, msg, len);
2591}
2592
Linus Torvalds1da177e2005-04-16 15:20:36 -07002593/*
2594 * Close a PACKET socket. This is fairly simple. We immediately go
2595 * to 'closed' state and remove our protocol entry in the device list.
2596 */
2597
2598static int packet_release(struct socket *sock)
2599{
2600 struct sock *sk = sock->sk;
2601 struct packet_sock *po;
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08002602 struct net *net;
chetan lokef6fb8f12011-08-19 10:18:16 +00002603 union tpacket_req_u req_u;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002604
2605 if (!sk)
2606 return 0;
2607
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09002608 net = sock_net(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002609 po = pkt_sk(sk);
2610
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00002611 mutex_lock(&net->packet.sklist_lock);
stephen hemminger808f5112010-02-22 07:57:18 +00002612 sk_del_node_init_rcu(sk);
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00002613 mutex_unlock(&net->packet.sklist_lock);
2614
2615 preempt_disable();
Eric Dumazet920de802008-11-24 00:09:29 -08002616 sock_prot_inuse_add(net, sk->sk_prot, -1);
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00002617 preempt_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002618
stephen hemminger808f5112010-02-22 07:57:18 +00002619 spin_lock(&po->bind_lock);
David S. Millerce06b032011-07-04 01:44:29 -07002620 unregister_prot_hook(sk, false);
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01002621 packet_cached_dev_reset(po);
2622
Ben Greear160ff182011-06-01 07:18:52 +00002623 if (po->prot_hook.dev) {
2624 dev_put(po->prot_hook.dev);
2625 po->prot_hook.dev = NULL;
2626 }
stephen hemminger808f5112010-02-22 07:57:18 +00002627 spin_unlock(&po->bind_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002628
Linus Torvalds1da177e2005-04-16 15:20:36 -07002629 packet_flush_mclist(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002630
Phil Sutter9665d5d2013-02-01 07:21:41 +00002631 if (po->rx_ring.pg_vec) {
2632 memset(&req_u, 0, sizeof(req_u));
chetan lokef6fb8f12011-08-19 10:18:16 +00002633 packet_set_ring(sk, &req_u, 1, 0);
Phil Sutter9665d5d2013-02-01 07:21:41 +00002634 }
Johann Baudy69e3c752009-05-18 22:11:22 -07002635
Phil Sutter9665d5d2013-02-01 07:21:41 +00002636 if (po->tx_ring.pg_vec) {
2637 memset(&req_u, 0, sizeof(req_u));
chetan lokef6fb8f12011-08-19 10:18:16 +00002638 packet_set_ring(sk, &req_u, 1, 1);
Phil Sutter9665d5d2013-02-01 07:21:41 +00002639 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002640
David S. Millerdc99f602011-07-05 01:45:05 -07002641 fanout_release(sk);
2642
stephen hemminger808f5112010-02-22 07:57:18 +00002643 synchronize_net();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002644 /*
2645 * Now the socket is dead. No more input will appear.
2646 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002647 sock_orphan(sk);
2648 sock->sk = NULL;
2649
2650 /* Purge queues */
2651
2652 skb_queue_purge(&sk->sk_receive_queue);
Daniel Borkmannb0138402014-01-15 16:25:36 +01002653 packet_free_pending(po);
Pavel Emelyanov17ab56a2007-11-10 21:38:48 -08002654 sk_refcnt_debug_release(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002655
2656 sock_put(sk);
2657 return 0;
2658}
2659
2660/*
2661 * Attach a packet hook.
2662 */
2663
Daniel Borkmann902fefb2014-01-15 16:25:34 +01002664static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 proto)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002665{
2666 struct packet_sock *po = pkt_sk(sk);
Daniel Borkmann902fefb2014-01-15 16:25:34 +01002667 const struct net_device *dev_curr;
2668 __be16 proto_curr;
2669 bool need_rehook;
David S. Millerdc99f602011-07-05 01:45:05 -07002670
Wei Yongjunaef950b2011-12-27 22:32:41 -05002671 if (po->fanout) {
2672 if (dev)
2673 dev_put(dev);
2674
David S. Millerdc99f602011-07-05 01:45:05 -07002675 return -EINVAL;
Wei Yongjunaef950b2011-12-27 22:32:41 -05002676 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002677
2678 lock_sock(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002679 spin_lock(&po->bind_lock);
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01002680
Daniel Borkmann902fefb2014-01-15 16:25:34 +01002681 proto_curr = po->prot_hook.type;
2682 dev_curr = po->prot_hook.dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002683
Daniel Borkmann902fefb2014-01-15 16:25:34 +01002684 need_rehook = proto_curr != proto || dev_curr != dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002685
Daniel Borkmann902fefb2014-01-15 16:25:34 +01002686 if (need_rehook) {
2687 unregister_prot_hook(sk, true);
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01002688
Daniel Borkmann902fefb2014-01-15 16:25:34 +01002689 po->num = proto;
2690 po->prot_hook.type = proto;
2691
2692 if (po->prot_hook.dev)
2693 dev_put(po->prot_hook.dev);
2694
2695 po->prot_hook.dev = dev;
2696
2697 po->ifindex = dev ? dev->ifindex : 0;
2698 packet_cached_dev_assign(po, dev);
2699 }
2700
2701 if (proto == 0 || !need_rehook)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002702 goto out_unlock;
2703
Urs Thuermannbe85d4a2007-11-12 21:05:20 -08002704 if (!dev || (dev->flags & IFF_UP)) {
David S. Millerce06b032011-07-04 01:44:29 -07002705 register_prot_hook(sk);
Urs Thuermannbe85d4a2007-11-12 21:05:20 -08002706 } else {
2707 sk->sk_err = ENETDOWN;
2708 if (!sock_flag(sk, SOCK_DEAD))
2709 sk->sk_error_report(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002710 }
2711
2712out_unlock:
2713 spin_unlock(&po->bind_lock);
2714 release_sock(sk);
2715 return 0;
2716}
2717
2718/*
2719 * Bind a packet socket to a device
2720 */
2721
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002722static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
2723 int addr_len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002724{
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002725 struct sock *sk = sock->sk;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002726 char name[15];
2727 struct net_device *dev;
2728 int err = -ENODEV;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002729
Linus Torvalds1da177e2005-04-16 15:20:36 -07002730 /*
2731 * Check legality
2732 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002733
Kris Katterjohn8ae55f02006-01-23 16:28:02 -08002734 if (addr_len != sizeof(struct sockaddr))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002735 return -EINVAL;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002736 strlcpy(name, uaddr->sa_data, sizeof(name));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002737
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09002738 dev = dev_get_by_name(sock_net(sk), name);
Ben Greear160ff182011-06-01 07:18:52 +00002739 if (dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002740 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002741 return err;
2742}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002743
2744static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
2745{
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002746 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
2747 struct sock *sk = sock->sk;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002748 struct net_device *dev = NULL;
2749 int err;
2750
2751
2752 /*
2753 * Check legality
2754 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002755
Linus Torvalds1da177e2005-04-16 15:20:36 -07002756 if (addr_len < sizeof(struct sockaddr_ll))
2757 return -EINVAL;
2758 if (sll->sll_family != AF_PACKET)
2759 return -EINVAL;
2760
2761 if (sll->sll_ifindex) {
2762 err = -ENODEV;
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09002763 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002764 if (dev == NULL)
2765 goto out;
2766 }
2767 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002768
2769out:
2770 return err;
2771}
2772
2773static struct proto packet_proto = {
2774 .name = "PACKET",
2775 .owner = THIS_MODULE,
2776 .obj_size = sizeof(struct packet_sock),
2777};
2778
2779/*
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002780 * Create a packet of type SOCK_PACKET.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002781 */
2782
Eric Paris3f378b682009-11-05 22:18:14 -08002783static int packet_create(struct net *net, struct socket *sock, int protocol,
2784 int kern)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002785{
2786 struct sock *sk;
2787 struct packet_sock *po;
Al Viro0e11c912006-11-08 00:26:29 -08002788 __be16 proto = (__force __be16)protocol; /* weird, but documented */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002789 int err;
2790
Eric W. Biedermandf008c92012-11-16 03:03:07 +00002791 if (!ns_capable(net->user_ns, CAP_NET_RAW))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002792 return -EPERM;
David S. Millerbe020972007-05-29 13:16:31 -07002793 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
2794 sock->type != SOCK_PACKET)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002795 return -ESOCKTNOSUPPORT;
2796
2797 sock->state = SS_UNCONNECTED;
2798
2799 err = -ENOBUFS;
Pavel Emelyanov6257ff22007-11-01 00:39:31 -07002800 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002801 if (sk == NULL)
2802 goto out;
2803
2804 sock->ops = &packet_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002805 if (sock->type == SOCK_PACKET)
2806 sock->ops = &packet_ops_spkt;
David S. Millerbe020972007-05-29 13:16:31 -07002807
Linus Torvalds1da177e2005-04-16 15:20:36 -07002808 sock_init_data(sock, sk);
2809
2810 po = pkt_sk(sk);
2811 sk->sk_family = PF_PACKET;
Al Viro0e11c912006-11-08 00:26:29 -08002812 po->num = proto;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002813 po->xmit = dev_queue_xmit;
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01002814
Daniel Borkmannb0138402014-01-15 16:25:36 +01002815 err = packet_alloc_pending(po);
2816 if (err)
2817 goto out2;
2818
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01002819 packet_cached_dev_reset(po);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002820
2821 sk->sk_destruct = packet_sock_destruct;
Pavel Emelyanov17ab56a2007-11-10 21:38:48 -08002822 sk_refcnt_debug_inc(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002823
2824 /*
2825 * Attach a protocol block
2826 */
2827
2828 spin_lock_init(&po->bind_lock);
Herbert Xu905db442009-01-30 14:12:06 -08002829 mutex_init(&po->pg_vec_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002830 po->prot_hook.func = packet_rcv;
David S. Millerbe020972007-05-29 13:16:31 -07002831
Linus Torvalds1da177e2005-04-16 15:20:36 -07002832 if (sock->type == SOCK_PACKET)
2833 po->prot_hook.func = packet_rcv_spkt;
David S. Millerbe020972007-05-29 13:16:31 -07002834
Linus Torvalds1da177e2005-04-16 15:20:36 -07002835 po->prot_hook.af_packet_priv = sk;
2836
Al Viro0e11c912006-11-08 00:26:29 -08002837 if (proto) {
2838 po->prot_hook.type = proto;
David S. Millerce06b032011-07-04 01:44:29 -07002839 register_prot_hook(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002840 }
2841
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00002842 mutex_lock(&net->packet.sklist_lock);
stephen hemminger808f5112010-02-22 07:57:18 +00002843 sk_add_node_rcu(sk, &net->packet.sklist);
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00002844 mutex_unlock(&net->packet.sklist_lock);
2845
2846 preempt_disable();
Eric Dumazet36804532008-11-19 14:25:35 -08002847 sock_prot_inuse_add(net, &packet_proto, 1);
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00002848 preempt_enable();
stephen hemminger808f5112010-02-22 07:57:18 +00002849
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002850 return 0;
Daniel Borkmannb0138402014-01-15 16:25:36 +01002851out2:
2852 sk_free(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002853out:
2854 return err;
2855}
2856
2857/*
2858 * Pull a packet from our receive queue and hand it to the user.
2859 * If necessary we block.
2860 */
2861
2862static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
2863 struct msghdr *msg, size_t len, int flags)
2864{
2865 struct sock *sk = sock->sk;
2866 struct sk_buff *skb;
2867 int copied, err;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002868 int vnet_hdr_len = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002869
2870 err = -EINVAL;
Richard Cochraned85b562010-04-07 22:41:28 +00002871 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002872 goto out;
2873
2874#if 0
2875 /* What error should we return now? EUNATTACH? */
2876 if (pkt_sk(sk)->ifindex < 0)
2877 return -ENODEV;
2878#endif
2879
Richard Cochraned85b562010-04-07 22:41:28 +00002880 if (flags & MSG_ERRQUEUE) {
Richard Cochrancb820f82013-07-19 19:40:09 +02002881 err = sock_recv_errqueue(sk, msg, len,
2882 SOL_PACKET, PACKET_TX_TIMESTAMP);
Richard Cochraned85b562010-04-07 22:41:28 +00002883 goto out;
2884 }
2885
Linus Torvalds1da177e2005-04-16 15:20:36 -07002886 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002887 * Call the generic datagram receiver. This handles all sorts
2888 * of horrible races and re-entrancy so we can forget about it
2889 * in the protocol layers.
2890 *
2891 * Now it will return ENETDOWN, if device have just gone down,
2892 * but then it will block.
2893 */
2894
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002895 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002896
2897 /*
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002898 * An error occurred so return it. Because skb_recv_datagram()
Linus Torvalds1da177e2005-04-16 15:20:36 -07002899 * handles the blocking we don't see and worry about blocking
2900 * retries.
2901 */
2902
Kris Katterjohn8ae55f02006-01-23 16:28:02 -08002903 if (skb == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002904 goto out;
2905
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002906 if (pkt_sk(sk)->has_vnet_hdr) {
2907 struct virtio_net_hdr vnet_hdr = { 0 };
2908
2909 err = -EINVAL;
2910 vnet_hdr_len = sizeof(vnet_hdr);
Mariusz Kozlowski1f18b712010-11-08 11:58:45 +00002911 if (len < vnet_hdr_len)
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002912 goto out_free;
2913
Mariusz Kozlowski1f18b712010-11-08 11:58:45 +00002914 len -= vnet_hdr_len;
2915
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002916 if (skb_is_gso(skb)) {
2917 struct skb_shared_info *sinfo = skb_shinfo(skb);
2918
2919 /* This is a hint as to how much should be linear. */
Michael S. Tsirkindc9e5152014-11-23 18:16:19 +02002920 vnet_hdr.hdr_len =
2921 __cpu_to_virtio16(false, skb_headlen(skb));
2922 vnet_hdr.gso_size =
2923 __cpu_to_virtio16(false, sinfo->gso_size);
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002924 if (sinfo->gso_type & SKB_GSO_TCPV4)
2925 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
2926 else if (sinfo->gso_type & SKB_GSO_TCPV6)
2927 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
2928 else if (sinfo->gso_type & SKB_GSO_UDP)
2929 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
2930 else if (sinfo->gso_type & SKB_GSO_FCOE)
2931 goto out_free;
2932 else
2933 BUG();
2934 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
2935 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
2936 } else
2937 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
2938
2939 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2940 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
Michael S. Tsirkindc9e5152014-11-23 18:16:19 +02002941 vnet_hdr.csum_start = __cpu_to_virtio16(false,
2942 skb_checksum_start_offset(skb));
2943 vnet_hdr.csum_offset = __cpu_to_virtio16(false,
2944 skb->csum_offset);
Jason Wang10a8d942011-06-10 00:56:17 +00002945 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
2946 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002947 } /* else everything is zero */
2948
2949 err = memcpy_toiovec(msg->msg_iov, (void *)&vnet_hdr,
2950 vnet_hdr_len);
2951 if (err < 0)
2952 goto out_free;
2953 }
2954
Hannes Frederic Sowaf3d33422013-11-21 03:14:22 +01002955 /* You lose any data beyond the buffer you gave. If it worries
2956 * a user program they can ask the device for its MTU
2957 * anyway.
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07002958 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002959 copied = skb->len;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002960 if (copied > len) {
2961 copied = len;
2962 msg->msg_flags |= MSG_TRUNC;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002963 }
2964
2965 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2966 if (err)
2967 goto out_free;
2968
Neil Horman3b885782009-10-12 13:26:31 -07002969 sock_recv_ts_and_drops(msg, sk, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002970
Hannes Frederic Sowaf3d33422013-11-21 03:14:22 +01002971 if (msg->msg_name) {
2972 /* If the address length field is there to be filled
2973 * in, we fill it in now.
2974 */
2975 if (sock->type == SOCK_PACKET) {
Steffen Hurrle342dfc32014-01-17 22:53:15 +01002976 __sockaddr_check_size(sizeof(struct sockaddr_pkt));
Hannes Frederic Sowaf3d33422013-11-21 03:14:22 +01002977 msg->msg_namelen = sizeof(struct sockaddr_pkt);
2978 } else {
2979 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
2980 msg->msg_namelen = sll->sll_halen +
2981 offsetof(struct sockaddr_ll, sll_addr);
2982 }
Herbert Xuffbc6112007-02-04 23:33:10 -08002983 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
2984 msg->msg_namelen);
Hannes Frederic Sowaf3d33422013-11-21 03:14:22 +01002985 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002986
Herbert Xu8dc41942007-02-04 23:31:32 -08002987 if (pkt_sk(sk)->auxdata) {
Herbert Xuffbc6112007-02-04 23:33:10 -08002988 struct tpacket_auxdata aux;
2989
2990 aux.tp_status = TP_STATUS_USER;
2991 if (skb->ip_summed == CHECKSUM_PARTIAL)
2992 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
2993 aux.tp_len = PACKET_SKB_CB(skb)->origlen;
2994 aux.tp_snaplen = skb->len;
2995 aux.tp_mac = 0;
Arnaldo Carvalho de Melobbe735e2007-03-10 22:16:10 -03002996 aux.tp_net = skb_network_offset(skb);
Ben Greeara3bcc232011-06-01 06:49:10 +00002997 if (vlan_tx_tag_present(skb)) {
2998 aux.tp_vlan_tci = vlan_tx_tag_get(skb);
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +09002999 aux.tp_vlan_tpid = ntohs(skb->vlan_proto);
3000 aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
Ben Greeara3bcc232011-06-01 06:49:10 +00003001 } else {
3002 aux.tp_vlan_tci = 0;
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +09003003 aux.tp_vlan_tpid = 0;
Ben Greeara3bcc232011-06-01 06:49:10 +00003004 }
Herbert Xuffbc6112007-02-04 23:33:10 -08003005 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
Herbert Xu8dc41942007-02-04 23:31:32 -08003006 }
3007
Linus Torvalds1da177e2005-04-16 15:20:36 -07003008 /*
3009 * Free or return the buffer as appropriate. Again this
3010 * hides all the races and re-entrancy issues from us.
3011 */
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003012 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003013
3014out_free:
3015 skb_free_datagram(sk, skb);
3016out:
3017 return err;
3018}
3019
Linus Torvalds1da177e2005-04-16 15:20:36 -07003020static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
3021 int *uaddr_len, int peer)
3022{
3023 struct net_device *dev;
3024 struct sock *sk = sock->sk;
3025
3026 if (peer)
3027 return -EOPNOTSUPP;
3028
3029 uaddr->sa_family = AF_PACKET;
Daniel Borkmann2dc85bf2013-06-12 16:02:27 +02003030 memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
Eric Dumazet654d1f82009-11-02 10:43:32 +01003031 rcu_read_lock();
3032 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
3033 if (dev)
Daniel Borkmann2dc85bf2013-06-12 16:02:27 +02003034 strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
Eric Dumazet654d1f82009-11-02 10:43:32 +01003035 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003036 *uaddr_len = sizeof(*uaddr);
3037
3038 return 0;
3039}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003040
3041static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
3042 int *uaddr_len, int peer)
3043{
3044 struct net_device *dev;
3045 struct sock *sk = sock->sk;
3046 struct packet_sock *po = pkt_sk(sk);
Cyrill Gorcunov13cfa972009-11-08 05:51:19 +00003047 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003048
3049 if (peer)
3050 return -EOPNOTSUPP;
3051
3052 sll->sll_family = AF_PACKET;
3053 sll->sll_ifindex = po->ifindex;
3054 sll->sll_protocol = po->num;
Vasiliy Kulikov67286642010-11-10 12:09:10 -08003055 sll->sll_pkttype = 0;
Eric Dumazet654d1f82009-11-02 10:43:32 +01003056 rcu_read_lock();
3057 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003058 if (dev) {
3059 sll->sll_hatype = dev->type;
3060 sll->sll_halen = dev->addr_len;
3061 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003062 } else {
3063 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
3064 sll->sll_halen = 0;
3065 }
Eric Dumazet654d1f82009-11-02 10:43:32 +01003066 rcu_read_unlock();
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003067 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003068
3069 return 0;
3070}
3071
Wang Chen2aeb0b82008-07-14 20:49:46 -07003072static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
3073 int what)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003074{
3075 switch (i->type) {
3076 case PACKET_MR_MULTICAST:
Jiri Pirko11625632010-03-02 20:40:01 +00003077 if (i->alen != dev->addr_len)
3078 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003079 if (what > 0)
Jiri Pirko22bedad32010-04-01 21:22:57 +00003080 return dev_mc_add(dev, i->addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003081 else
Jiri Pirko22bedad32010-04-01 21:22:57 +00003082 return dev_mc_del(dev, i->addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003083 break;
3084 case PACKET_MR_PROMISC:
Wang Chen2aeb0b82008-07-14 20:49:46 -07003085 return dev_set_promiscuity(dev, what);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003086 case PACKET_MR_ALLMULTI:
Wang Chen2aeb0b82008-07-14 20:49:46 -07003087 return dev_set_allmulti(dev, what);
Eric W. Biedermand95ed922009-05-19 18:27:17 +00003088 case PACKET_MR_UNICAST:
Jiri Pirko11625632010-03-02 20:40:01 +00003089 if (i->alen != dev->addr_len)
3090 return -EINVAL;
Eric W. Biedermand95ed922009-05-19 18:27:17 +00003091 if (what > 0)
Jiri Pirkoa748ee22010-04-01 21:22:09 +00003092 return dev_uc_add(dev, i->addr);
Eric W. Biedermand95ed922009-05-19 18:27:17 +00003093 else
Jiri Pirkoa748ee22010-04-01 21:22:09 +00003094 return dev_uc_del(dev, i->addr);
Eric W. Biedermand95ed922009-05-19 18:27:17 +00003095 break;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003096 default:
3097 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003098 }
Wang Chen2aeb0b82008-07-14 20:49:46 -07003099 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003100}
3101
3102static void packet_dev_mclist(struct net_device *dev, struct packet_mclist *i, int what)
3103{
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003104 for ( ; i; i = i->next) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003105 if (i->ifindex == dev->ifindex)
3106 packet_dev_mc(dev, i, what);
3107 }
3108}
3109
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003110static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003111{
3112 struct packet_sock *po = pkt_sk(sk);
3113 struct packet_mclist *ml, *i;
3114 struct net_device *dev;
3115 int err;
3116
3117 rtnl_lock();
3118
3119 err = -ENODEV;
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09003120 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003121 if (!dev)
3122 goto done;
3123
3124 err = -EINVAL;
Jiri Pirko11625632010-03-02 20:40:01 +00003125 if (mreq->mr_alen > dev->addr_len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003126 goto done;
3127
3128 err = -ENOBUFS;
Kris Katterjohn8b3a7002006-01-11 15:56:43 -08003129 i = kmalloc(sizeof(*i), GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003130 if (i == NULL)
3131 goto done;
3132
3133 err = 0;
3134 for (ml = po->mclist; ml; ml = ml->next) {
3135 if (ml->ifindex == mreq->mr_ifindex &&
3136 ml->type == mreq->mr_type &&
3137 ml->alen == mreq->mr_alen &&
3138 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3139 ml->count++;
3140 /* Free the new element ... */
3141 kfree(i);
3142 goto done;
3143 }
3144 }
3145
3146 i->type = mreq->mr_type;
3147 i->ifindex = mreq->mr_ifindex;
3148 i->alen = mreq->mr_alen;
3149 memcpy(i->addr, mreq->mr_address, i->alen);
3150 i->count = 1;
3151 i->next = po->mclist;
3152 po->mclist = i;
Wang Chen2aeb0b82008-07-14 20:49:46 -07003153 err = packet_dev_mc(dev, i, 1);
3154 if (err) {
3155 po->mclist = i->next;
3156 kfree(i);
3157 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003158
3159done:
3160 rtnl_unlock();
3161 return err;
3162}
3163
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003164static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003165{
3166 struct packet_mclist *ml, **mlp;
3167
3168 rtnl_lock();
3169
3170 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
3171 if (ml->ifindex == mreq->mr_ifindex &&
3172 ml->type == mreq->mr_type &&
3173 ml->alen == mreq->mr_alen &&
3174 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3175 if (--ml->count == 0) {
3176 struct net_device *dev;
3177 *mlp = ml->next;
Eric Dumazetad959e72009-10-16 06:38:46 +00003178 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3179 if (dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003180 packet_dev_mc(dev, ml, -1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003181 kfree(ml);
3182 }
3183 rtnl_unlock();
3184 return 0;
3185 }
3186 }
3187 rtnl_unlock();
3188 return -EADDRNOTAVAIL;
3189}
3190
3191static void packet_flush_mclist(struct sock *sk)
3192{
3193 struct packet_sock *po = pkt_sk(sk);
3194 struct packet_mclist *ml;
3195
3196 if (!po->mclist)
3197 return;
3198
3199 rtnl_lock();
3200 while ((ml = po->mclist) != NULL) {
3201 struct net_device *dev;
3202
3203 po->mclist = ml->next;
Eric Dumazetad959e72009-10-16 06:38:46 +00003204 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3205 if (dev != NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003206 packet_dev_mc(dev, ml, -1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003207 kfree(ml);
3208 }
3209 rtnl_unlock();
3210}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003211
3212static int
David S. Millerb7058842009-09-30 16:12:20 -07003213packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003214{
3215 struct sock *sk = sock->sk;
Herbert Xu8dc41942007-02-04 23:31:32 -08003216 struct packet_sock *po = pkt_sk(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003217 int ret;
3218
3219 if (level != SOL_PACKET)
3220 return -ENOPROTOOPT;
3221
Johann Baudy69e3c752009-05-18 22:11:22 -07003222 switch (optname) {
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003223 case PACKET_ADD_MEMBERSHIP:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003224 case PACKET_DROP_MEMBERSHIP:
3225 {
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003226 struct packet_mreq_max mreq;
3227 int len = optlen;
3228 memset(&mreq, 0, sizeof(mreq));
3229 if (len < sizeof(struct packet_mreq))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003230 return -EINVAL;
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003231 if (len > sizeof(mreq))
3232 len = sizeof(mreq);
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003233 if (copy_from_user(&mreq, optval, len))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003234 return -EFAULT;
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003235 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3236 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003237 if (optname == PACKET_ADD_MEMBERSHIP)
3238 ret = packet_mc_add(sk, &mreq);
3239 else
3240 ret = packet_mc_drop(sk, &mreq);
3241 return ret;
3242 }
David S. Millera2efcfa2007-05-29 13:12:50 -07003243
Linus Torvalds1da177e2005-04-16 15:20:36 -07003244 case PACKET_RX_RING:
Johann Baudy69e3c752009-05-18 22:11:22 -07003245 case PACKET_TX_RING:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003246 {
chetan lokef6fb8f12011-08-19 10:18:16 +00003247 union tpacket_req_u req_u;
3248 int len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003249
chetan lokef6fb8f12011-08-19 10:18:16 +00003250 switch (po->tp_version) {
3251 case TPACKET_V1:
3252 case TPACKET_V2:
3253 len = sizeof(req_u.req);
3254 break;
3255 case TPACKET_V3:
3256 default:
3257 len = sizeof(req_u.req3);
3258 break;
3259 }
3260 if (optlen < len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003261 return -EINVAL;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003262 if (pkt_sk(sk)->has_vnet_hdr)
3263 return -EINVAL;
chetan lokef6fb8f12011-08-19 10:18:16 +00003264 if (copy_from_user(&req_u.req, optval, len))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003265 return -EFAULT;
chetan lokef6fb8f12011-08-19 10:18:16 +00003266 return packet_set_ring(sk, &req_u, 0,
3267 optname == PACKET_TX_RING);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003268 }
3269 case PACKET_COPY_THRESH:
3270 {
3271 int val;
3272
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003273 if (optlen != sizeof(val))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003274 return -EINVAL;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003275 if (copy_from_user(&val, optval, sizeof(val)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003276 return -EFAULT;
3277
3278 pkt_sk(sk)->copy_thresh = val;
3279 return 0;
3280 }
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003281 case PACKET_VERSION:
3282 {
3283 int val;
3284
3285 if (optlen != sizeof(val))
3286 return -EINVAL;
Johann Baudy69e3c752009-05-18 22:11:22 -07003287 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003288 return -EBUSY;
3289 if (copy_from_user(&val, optval, sizeof(val)))
3290 return -EFAULT;
3291 switch (val) {
3292 case TPACKET_V1:
3293 case TPACKET_V2:
chetan lokef6fb8f12011-08-19 10:18:16 +00003294 case TPACKET_V3:
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003295 po->tp_version = val;
3296 return 0;
3297 default:
3298 return -EINVAL;
3299 }
3300 }
Patrick McHardy8913336a2008-07-18 18:05:19 -07003301 case PACKET_RESERVE:
3302 {
3303 unsigned int val;
3304
3305 if (optlen != sizeof(val))
3306 return -EINVAL;
Johann Baudy69e3c752009-05-18 22:11:22 -07003307 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
Patrick McHardy8913336a2008-07-18 18:05:19 -07003308 return -EBUSY;
3309 if (copy_from_user(&val, optval, sizeof(val)))
3310 return -EFAULT;
3311 po->tp_reserve = val;
3312 return 0;
3313 }
Johann Baudy69e3c752009-05-18 22:11:22 -07003314 case PACKET_LOSS:
3315 {
3316 unsigned int val;
3317
3318 if (optlen != sizeof(val))
3319 return -EINVAL;
3320 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3321 return -EBUSY;
3322 if (copy_from_user(&val, optval, sizeof(val)))
3323 return -EFAULT;
3324 po->tp_loss = !!val;
3325 return 0;
3326 }
Herbert Xu8dc41942007-02-04 23:31:32 -08003327 case PACKET_AUXDATA:
3328 {
3329 int val;
3330
3331 if (optlen < sizeof(val))
3332 return -EINVAL;
3333 if (copy_from_user(&val, optval, sizeof(val)))
3334 return -EFAULT;
3335
3336 po->auxdata = !!val;
3337 return 0;
3338 }
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07003339 case PACKET_ORIGDEV:
3340 {
3341 int val;
3342
3343 if (optlen < sizeof(val))
3344 return -EINVAL;
3345 if (copy_from_user(&val, optval, sizeof(val)))
3346 return -EFAULT;
3347
3348 po->origdev = !!val;
3349 return 0;
3350 }
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003351 case PACKET_VNET_HDR:
3352 {
3353 int val;
3354
3355 if (sock->type != SOCK_RAW)
3356 return -EINVAL;
3357 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3358 return -EBUSY;
3359 if (optlen < sizeof(val))
3360 return -EINVAL;
3361 if (copy_from_user(&val, optval, sizeof(val)))
3362 return -EFAULT;
3363
3364 po->has_vnet_hdr = !!val;
3365 return 0;
3366 }
Scott McMillan614f60f2010-06-02 05:53:56 -07003367 case PACKET_TIMESTAMP:
3368 {
3369 int val;
3370
3371 if (optlen != sizeof(val))
3372 return -EINVAL;
3373 if (copy_from_user(&val, optval, sizeof(val)))
3374 return -EFAULT;
3375
3376 po->tp_tstamp = val;
3377 return 0;
3378 }
David S. Millerdc99f602011-07-05 01:45:05 -07003379 case PACKET_FANOUT:
3380 {
3381 int val;
3382
3383 if (optlen != sizeof(val))
3384 return -EINVAL;
3385 if (copy_from_user(&val, optval, sizeof(val)))
3386 return -EFAULT;
3387
3388 return fanout_add(sk, val & 0xffff, val >> 16);
3389 }
Paul Chavent5920cd3a2012-11-06 23:10:47 +00003390 case PACKET_TX_HAS_OFF:
3391 {
3392 unsigned int val;
3393
3394 if (optlen != sizeof(val))
3395 return -EINVAL;
3396 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3397 return -EBUSY;
3398 if (copy_from_user(&val, optval, sizeof(val)))
3399 return -EFAULT;
3400 po->tp_tx_has_off = !!val;
3401 return 0;
3402 }
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01003403 case PACKET_QDISC_BYPASS:
3404 {
3405 int val;
3406
3407 if (optlen != sizeof(val))
3408 return -EINVAL;
3409 if (copy_from_user(&val, optval, sizeof(val)))
3410 return -EFAULT;
3411
3412 po->xmit = val ? packet_direct_xmit : dev_queue_xmit;
3413 return 0;
3414 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003415 default:
3416 return -ENOPROTOOPT;
3417 }
3418}
3419
3420static int packet_getsockopt(struct socket *sock, int level, int optname,
3421 char __user *optval, int __user *optlen)
3422{
3423 int len;
Eric Dumazetc06fff62012-04-19 21:56:11 +00003424 int val, lv = sizeof(val);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003425 struct sock *sk = sock->sk;
3426 struct packet_sock *po = pkt_sk(sk);
Eric Dumazetc06fff62012-04-19 21:56:11 +00003427 void *data = &val;
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00003428 union tpacket_stats_u st;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003429
3430 if (level != SOL_PACKET)
3431 return -ENOPROTOOPT;
3432
Kris Katterjohn8ae55f02006-01-23 16:28:02 -08003433 if (get_user(len, optlen))
3434 return -EFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003435
3436 if (len < 0)
3437 return -EINVAL;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003438
Johann Baudy69e3c752009-05-18 22:11:22 -07003439 switch (optname) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003440 case PACKET_STATISTICS:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003441 spin_lock_bh(&sk->sk_receive_queue.lock);
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00003442 memcpy(&st, &po->stats, sizeof(st));
3443 memset(&po->stats, 0, sizeof(po->stats));
3444 spin_unlock_bh(&sk->sk_receive_queue.lock);
3445
chetan lokef6fb8f12011-08-19 10:18:16 +00003446 if (po->tp_version == TPACKET_V3) {
Eric Dumazetc06fff62012-04-19 21:56:11 +00003447 lv = sizeof(struct tpacket_stats_v3);
Willem de Bruijn8bcdeaf2013-08-19 16:40:22 -04003448 st.stats3.tp_packets += st.stats3.tp_drops;
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00003449 data = &st.stats3;
chetan lokef6fb8f12011-08-19 10:18:16 +00003450 } else {
Eric Dumazetc06fff62012-04-19 21:56:11 +00003451 lv = sizeof(struct tpacket_stats);
Willem de Bruijn8bcdeaf2013-08-19 16:40:22 -04003452 st.stats1.tp_packets += st.stats1.tp_drops;
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00003453 data = &st.stats1;
chetan lokef6fb8f12011-08-19 10:18:16 +00003454 }
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00003455
Linus Torvalds1da177e2005-04-16 15:20:36 -07003456 break;
Herbert Xu8dc41942007-02-04 23:31:32 -08003457 case PACKET_AUXDATA:
Herbert Xu8dc41942007-02-04 23:31:32 -08003458 val = po->auxdata;
Herbert Xu8dc41942007-02-04 23:31:32 -08003459 break;
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07003460 case PACKET_ORIGDEV:
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07003461 val = po->origdev;
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07003462 break;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003463 case PACKET_VNET_HDR:
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003464 val = po->has_vnet_hdr;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003465 break;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003466 case PACKET_VERSION:
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003467 val = po->tp_version;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003468 break;
3469 case PACKET_HDRLEN:
3470 if (len > sizeof(int))
3471 len = sizeof(int);
3472 if (copy_from_user(&val, optval, len))
3473 return -EFAULT;
3474 switch (val) {
3475 case TPACKET_V1:
3476 val = sizeof(struct tpacket_hdr);
3477 break;
3478 case TPACKET_V2:
3479 val = sizeof(struct tpacket2_hdr);
3480 break;
chetan lokef6fb8f12011-08-19 10:18:16 +00003481 case TPACKET_V3:
3482 val = sizeof(struct tpacket3_hdr);
3483 break;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003484 default:
3485 return -EINVAL;
3486 }
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003487 break;
Patrick McHardy8913336a2008-07-18 18:05:19 -07003488 case PACKET_RESERVE:
Patrick McHardy8913336a2008-07-18 18:05:19 -07003489 val = po->tp_reserve;
Patrick McHardy8913336a2008-07-18 18:05:19 -07003490 break;
Johann Baudy69e3c752009-05-18 22:11:22 -07003491 case PACKET_LOSS:
Johann Baudy69e3c752009-05-18 22:11:22 -07003492 val = po->tp_loss;
Johann Baudy69e3c752009-05-18 22:11:22 -07003493 break;
Scott McMillan614f60f2010-06-02 05:53:56 -07003494 case PACKET_TIMESTAMP:
Scott McMillan614f60f2010-06-02 05:53:56 -07003495 val = po->tp_tstamp;
Scott McMillan614f60f2010-06-02 05:53:56 -07003496 break;
David S. Millerdc99f602011-07-05 01:45:05 -07003497 case PACKET_FANOUT:
David S. Millerdc99f602011-07-05 01:45:05 -07003498 val = (po->fanout ?
3499 ((u32)po->fanout->id |
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00003500 ((u32)po->fanout->type << 16) |
3501 ((u32)po->fanout->flags << 24)) :
David S. Millerdc99f602011-07-05 01:45:05 -07003502 0);
David S. Millerdc99f602011-07-05 01:45:05 -07003503 break;
Paul Chavent5920cd3a2012-11-06 23:10:47 +00003504 case PACKET_TX_HAS_OFF:
3505 val = po->tp_tx_has_off;
3506 break;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01003507 case PACKET_QDISC_BYPASS:
3508 val = packet_use_direct_xmit(po);
3509 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003510 default:
3511 return -ENOPROTOOPT;
3512 }
3513
Eric Dumazetc06fff62012-04-19 21:56:11 +00003514 if (len > lv)
3515 len = lv;
Kris Katterjohn8ae55f02006-01-23 16:28:02 -08003516 if (put_user(len, optlen))
3517 return -EFAULT;
Herbert Xu8dc41942007-02-04 23:31:32 -08003518 if (copy_to_user(optval, data, len))
3519 return -EFAULT;
Kris Katterjohn8ae55f02006-01-23 16:28:02 -08003520 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003521}
3522
3523
Jiri Pirko351638e2013-05-28 01:30:21 +00003524static int packet_notifier(struct notifier_block *this,
3525 unsigned long msg, void *ptr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003526{
3527 struct sock *sk;
Jiri Pirko351638e2013-05-28 01:30:21 +00003528 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09003529 struct net *net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003530
stephen hemminger808f5112010-02-22 07:57:18 +00003531 rcu_read_lock();
Sasha Levinb67bfe02013-02-27 17:06:00 -08003532 sk_for_each_rcu(sk, &net->packet.sklist) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003533 struct packet_sock *po = pkt_sk(sk);
3534
3535 switch (msg) {
3536 case NETDEV_UNREGISTER:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003537 if (po->mclist)
3538 packet_dev_mclist(dev, po->mclist, -1);
David S. Millera2efcfa2007-05-29 13:12:50 -07003539 /* fallthrough */
3540
Linus Torvalds1da177e2005-04-16 15:20:36 -07003541 case NETDEV_DOWN:
3542 if (dev->ifindex == po->ifindex) {
3543 spin_lock(&po->bind_lock);
3544 if (po->running) {
David S. Millerce06b032011-07-04 01:44:29 -07003545 __unregister_prot_hook(sk, false);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003546 sk->sk_err = ENETDOWN;
3547 if (!sock_flag(sk, SOCK_DEAD))
3548 sk->sk_error_report(sk);
3549 }
3550 if (msg == NETDEV_UNREGISTER) {
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01003551 packet_cached_dev_reset(po);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003552 po->ifindex = -1;
Ben Greear160ff182011-06-01 07:18:52 +00003553 if (po->prot_hook.dev)
3554 dev_put(po->prot_hook.dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003555 po->prot_hook.dev = NULL;
3556 }
3557 spin_unlock(&po->bind_lock);
3558 }
3559 break;
3560 case NETDEV_UP:
stephen hemminger808f5112010-02-22 07:57:18 +00003561 if (dev->ifindex == po->ifindex) {
3562 spin_lock(&po->bind_lock);
David S. Millerce06b032011-07-04 01:44:29 -07003563 if (po->num)
3564 register_prot_hook(sk);
stephen hemminger808f5112010-02-22 07:57:18 +00003565 spin_unlock(&po->bind_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003566 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003567 break;
3568 }
3569 }
stephen hemminger808f5112010-02-22 07:57:18 +00003570 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003571 return NOTIFY_DONE;
3572}
3573
3574
3575static int packet_ioctl(struct socket *sock, unsigned int cmd,
3576 unsigned long arg)
3577{
3578 struct sock *sk = sock->sk;
3579
Johann Baudy69e3c752009-05-18 22:11:22 -07003580 switch (cmd) {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003581 case SIOCOUTQ:
3582 {
3583 int amount = sk_wmem_alloc_get(sk);
Eric Dumazet31e6d362009-06-17 19:05:41 -07003584
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003585 return put_user(amount, (int __user *)arg);
3586 }
3587 case SIOCINQ:
3588 {
3589 struct sk_buff *skb;
3590 int amount = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003591
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003592 spin_lock_bh(&sk->sk_receive_queue.lock);
3593 skb = skb_peek(&sk->sk_receive_queue);
3594 if (skb)
3595 amount = skb->len;
3596 spin_unlock_bh(&sk->sk_receive_queue.lock);
3597 return put_user(amount, (int __user *)arg);
3598 }
3599 case SIOCGSTAMP:
3600 return sock_get_timestamp(sk, (struct timeval __user *)arg);
3601 case SIOCGSTAMPNS:
3602 return sock_get_timestampns(sk, (struct timespec __user *)arg);
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003603
Linus Torvalds1da177e2005-04-16 15:20:36 -07003604#ifdef CONFIG_INET
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003605 case SIOCADDRT:
3606 case SIOCDELRT:
3607 case SIOCDARP:
3608 case SIOCGARP:
3609 case SIOCSARP:
3610 case SIOCGIFADDR:
3611 case SIOCSIFADDR:
3612 case SIOCGIFBRDADDR:
3613 case SIOCSIFBRDADDR:
3614 case SIOCGIFNETMASK:
3615 case SIOCSIFNETMASK:
3616 case SIOCGIFDSTADDR:
3617 case SIOCSIFDSTADDR:
3618 case SIOCSIFFLAGS:
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003619 return inet_dgram_ops.ioctl(sock, cmd, arg);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003620#endif
3621
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003622 default:
3623 return -ENOIOCTLCMD;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003624 }
3625 return 0;
3626}
3627
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003628static unsigned int packet_poll(struct file *file, struct socket *sock,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003629 poll_table *wait)
3630{
3631 struct sock *sk = sock->sk;
3632 struct packet_sock *po = pkt_sk(sk);
3633 unsigned int mask = datagram_poll(file, sock, wait);
3634
3635 spin_lock_bh(&sk->sk_receive_queue.lock);
Johann Baudy69e3c752009-05-18 22:11:22 -07003636 if (po->rx_ring.pg_vec) {
chetan lokef6fb8f12011-08-19 10:18:16 +00003637 if (!packet_previous_rx_frame(po, &po->rx_ring,
3638 TP_STATUS_KERNEL))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003639 mask |= POLLIN | POLLRDNORM;
3640 }
3641 spin_unlock_bh(&sk->sk_receive_queue.lock);
Johann Baudy69e3c752009-05-18 22:11:22 -07003642 spin_lock_bh(&sk->sk_write_queue.lock);
3643 if (po->tx_ring.pg_vec) {
3644 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
3645 mask |= POLLOUT | POLLWRNORM;
3646 }
3647 spin_unlock_bh(&sk->sk_write_queue.lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003648 return mask;
3649}
3650
3651
3652/* Dirty? Well, I still did not learn better way to account
3653 * for user mmaps.
3654 */
3655
3656static void packet_mm_open(struct vm_area_struct *vma)
3657{
3658 struct file *file = vma->vm_file;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003659 struct socket *sock = file->private_data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003660 struct sock *sk = sock->sk;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003661
Linus Torvalds1da177e2005-04-16 15:20:36 -07003662 if (sk)
3663 atomic_inc(&pkt_sk(sk)->mapped);
3664}
3665
3666static void packet_mm_close(struct vm_area_struct *vma)
3667{
3668 struct file *file = vma->vm_file;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003669 struct socket *sock = file->private_data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003670 struct sock *sk = sock->sk;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003671
Linus Torvalds1da177e2005-04-16 15:20:36 -07003672 if (sk)
3673 atomic_dec(&pkt_sk(sk)->mapped);
3674}
3675
Alexey Dobriyanf0f37e2f2009-09-27 22:29:37 +04003676static const struct vm_operations_struct packet_mmap_ops = {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003677 .open = packet_mm_open,
3678 .close = packet_mm_close,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003679};
3680
Neil Horman0e3125c2010-11-16 10:26:47 -08003681static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
3682 unsigned int len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003683{
3684 int i;
3685
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003686 for (i = 0; i < len; i++) {
Neil Horman0e3125c2010-11-16 10:26:47 -08003687 if (likely(pg_vec[i].buffer)) {
Changli Gaoc56b4d92010-12-01 02:52:57 +00003688 if (is_vmalloc_addr(pg_vec[i].buffer))
Neil Horman0e3125c2010-11-16 10:26:47 -08003689 vfree(pg_vec[i].buffer);
3690 else
3691 free_pages((unsigned long)pg_vec[i].buffer,
3692 order);
3693 pg_vec[i].buffer = NULL;
3694 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003695 }
3696 kfree(pg_vec);
3697}
3698
Olof Johanssoneea49cc92011-11-02 11:00:49 +00003699static char *alloc_one_pg_vec_page(unsigned long order)
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003700{
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01003701 char *buffer;
Neil Horman0e3125c2010-11-16 10:26:47 -08003702 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
3703 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
Eric Dumazet719bfea2009-04-15 03:39:52 -07003704
Neil Horman0e3125c2010-11-16 10:26:47 -08003705 buffer = (char *) __get_free_pages(gfp_flags, order);
Neil Horman0e3125c2010-11-16 10:26:47 -08003706 if (buffer)
3707 return buffer;
3708
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01003709 /* __get_free_pages failed, fall back to vmalloc */
Eric Dumazetbbce5a52010-11-20 07:31:54 +00003710 buffer = vzalloc((1 << order) * PAGE_SIZE);
Neil Horman0e3125c2010-11-16 10:26:47 -08003711 if (buffer)
3712 return buffer;
3713
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01003714 /* vmalloc failed, lets dig into swap here */
Neil Horman0e3125c2010-11-16 10:26:47 -08003715 gfp_flags &= ~__GFP_NORETRY;
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01003716 buffer = (char *) __get_free_pages(gfp_flags, order);
Neil Horman0e3125c2010-11-16 10:26:47 -08003717 if (buffer)
3718 return buffer;
3719
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01003720 /* complete and utter failure */
Neil Horman0e3125c2010-11-16 10:26:47 -08003721 return NULL;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003722}
3723
Neil Horman0e3125c2010-11-16 10:26:47 -08003724static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003725{
3726 unsigned int block_nr = req->tp_block_nr;
Neil Horman0e3125c2010-11-16 10:26:47 -08003727 struct pgv *pg_vec;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003728 int i;
3729
Neil Horman0e3125c2010-11-16 10:26:47 -08003730 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003731 if (unlikely(!pg_vec))
3732 goto out;
3733
3734 for (i = 0; i < block_nr; i++) {
Changli Gaoc56b4d92010-12-01 02:52:57 +00003735 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
Neil Horman0e3125c2010-11-16 10:26:47 -08003736 if (unlikely(!pg_vec[i].buffer))
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003737 goto out_free_pgvec;
3738 }
3739
3740out:
3741 return pg_vec;
3742
3743out_free_pgvec:
3744 free_pg_vec(pg_vec, order, block_nr);
3745 pg_vec = NULL;
3746 goto out;
3747}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003748
chetan lokef6fb8f12011-08-19 10:18:16 +00003749static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
Johann Baudy69e3c752009-05-18 22:11:22 -07003750 int closing, int tx_ring)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003751{
Neil Horman0e3125c2010-11-16 10:26:47 -08003752 struct pgv *pg_vec = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003753 struct packet_sock *po = pkt_sk(sk);
Al Viro0e11c912006-11-08 00:26:29 -08003754 int was_running, order = 0;
Johann Baudy69e3c752009-05-18 22:11:22 -07003755 struct packet_ring_buffer *rb;
3756 struct sk_buff_head *rb_queue;
Al Viro0e11c912006-11-08 00:26:29 -08003757 __be16 num;
chetan lokef6fb8f12011-08-19 10:18:16 +00003758 int err = -EINVAL;
3759 /* Added to avoid minimal code churn */
3760 struct tpacket_req *req = &req_u->req;
3761
3762 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */
3763 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
3764 WARN(1, "Tx-ring is not supported.\n");
3765 goto out;
3766 }
Johann Baudy69e3c752009-05-18 22:11:22 -07003767
3768 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
3769 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
3770
3771 err = -EBUSY;
3772 if (!closing) {
3773 if (atomic_read(&po->mapped))
3774 goto out;
Daniel Borkmannb0138402014-01-15 16:25:36 +01003775 if (packet_read_pending(rb))
Johann Baudy69e3c752009-05-18 22:11:22 -07003776 goto out;
3777 }
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003778
Linus Torvalds1da177e2005-04-16 15:20:36 -07003779 if (req->tp_block_nr) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003780 /* Sanity tests and some calculations */
Johann Baudy69e3c752009-05-18 22:11:22 -07003781 err = -EBUSY;
3782 if (unlikely(rb->pg_vec))
3783 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003784
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003785 switch (po->tp_version) {
3786 case TPACKET_V1:
3787 po->tp_hdrlen = TPACKET_HDRLEN;
3788 break;
3789 case TPACKET_V2:
3790 po->tp_hdrlen = TPACKET2_HDRLEN;
3791 break;
chetan lokef6fb8f12011-08-19 10:18:16 +00003792 case TPACKET_V3:
3793 po->tp_hdrlen = TPACKET3_HDRLEN;
3794 break;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003795 }
3796
Johann Baudy69e3c752009-05-18 22:11:22 -07003797 err = -EINVAL;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003798 if (unlikely((int)req->tp_block_size <= 0))
Johann Baudy69e3c752009-05-18 22:11:22 -07003799 goto out;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003800 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
Johann Baudy69e3c752009-05-18 22:11:22 -07003801 goto out;
Eric Dumazetdc808112014-08-15 09:16:04 -07003802 if (po->tp_version >= TPACKET_V3 &&
3803 (int)(req->tp_block_size -
3804 BLK_PLUS_PRIV(req_u->req3.tp_sizeof_priv)) <= 0)
3805 goto out;
Patrick McHardy8913336a2008-07-18 18:05:19 -07003806 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
Johann Baudy69e3c752009-05-18 22:11:22 -07003807 po->tp_reserve))
3808 goto out;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003809 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
Johann Baudy69e3c752009-05-18 22:11:22 -07003810 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003811
Johann Baudy69e3c752009-05-18 22:11:22 -07003812 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
3813 if (unlikely(rb->frames_per_block <= 0))
3814 goto out;
3815 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
3816 req->tp_frame_nr))
3817 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003818
3819 err = -ENOMEM;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003820 order = get_order(req->tp_block_size);
3821 pg_vec = alloc_pg_vec(req, order);
3822 if (unlikely(!pg_vec))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003823 goto out;
chetan lokef6fb8f12011-08-19 10:18:16 +00003824 switch (po->tp_version) {
3825 case TPACKET_V3:
3826 /* Transmit path is not supported. We checked
3827 * it above but just being paranoid
3828 */
3829 if (!tx_ring)
3830 init_prb_bdqc(po, rb, pg_vec, req_u, tx_ring);
Dan Carpenterd7cf0c32014-02-18 15:20:51 +03003831 break;
chetan lokef6fb8f12011-08-19 10:18:16 +00003832 default:
3833 break;
3834 }
Johann Baudy69e3c752009-05-18 22:11:22 -07003835 }
3836 /* Done */
3837 else {
3838 err = -EINVAL;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003839 if (unlikely(req->tp_frame_nr))
Johann Baudy69e3c752009-05-18 22:11:22 -07003840 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003841 }
3842
3843 lock_sock(sk);
3844
3845 /* Detach socket from network */
3846 spin_lock(&po->bind_lock);
3847 was_running = po->running;
3848 num = po->num;
3849 if (was_running) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003850 po->num = 0;
David S. Millerce06b032011-07-04 01:44:29 -07003851 __unregister_prot_hook(sk, false);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003852 }
3853 spin_unlock(&po->bind_lock);
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003854
Linus Torvalds1da177e2005-04-16 15:20:36 -07003855 synchronize_net();
3856
3857 err = -EBUSY;
Herbert Xu905db442009-01-30 14:12:06 -08003858 mutex_lock(&po->pg_vec_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003859 if (closing || atomic_read(&po->mapped) == 0) {
3860 err = 0;
Johann Baudy69e3c752009-05-18 22:11:22 -07003861 spin_lock_bh(&rb_queue->lock);
Changli Gaoc053fd92010-12-10 16:02:20 -08003862 swap(rb->pg_vec, pg_vec);
Johann Baudy69e3c752009-05-18 22:11:22 -07003863 rb->frame_max = (req->tp_frame_nr - 1);
3864 rb->head = 0;
3865 rb->frame_size = req->tp_frame_size;
3866 spin_unlock_bh(&rb_queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003867
Changli Gaoc053fd92010-12-10 16:02:20 -08003868 swap(rb->pg_vec_order, order);
3869 swap(rb->pg_vec_len, req->tp_block_nr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003870
Johann Baudy69e3c752009-05-18 22:11:22 -07003871 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
3872 po->prot_hook.func = (po->rx_ring.pg_vec) ?
3873 tpacket_rcv : packet_rcv;
3874 skb_queue_purge(rb_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003875 if (atomic_read(&po->mapped))
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003876 pr_err("packet_mmap: vma is busy: %d\n",
3877 atomic_read(&po->mapped));
Linus Torvalds1da177e2005-04-16 15:20:36 -07003878 }
Herbert Xu905db442009-01-30 14:12:06 -08003879 mutex_unlock(&po->pg_vec_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003880
3881 spin_lock(&po->bind_lock);
David S. Millerce06b032011-07-04 01:44:29 -07003882 if (was_running) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003883 po->num = num;
David S. Millerce06b032011-07-04 01:44:29 -07003884 register_prot_hook(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003885 }
3886 spin_unlock(&po->bind_lock);
chetan lokef6fb8f12011-08-19 10:18:16 +00003887 if (closing && (po->tp_version > TPACKET_V2)) {
3888 /* Because we don't support block-based V3 on tx-ring */
3889 if (!tx_ring)
3890 prb_shutdown_retire_blk_timer(po, tx_ring, rb_queue);
3891 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003892 release_sock(sk);
3893
Linus Torvalds1da177e2005-04-16 15:20:36 -07003894 if (pg_vec)
3895 free_pg_vec(pg_vec, order, req->tp_block_nr);
3896out:
3897 return err;
3898}
3899
Johann Baudy69e3c752009-05-18 22:11:22 -07003900static int packet_mmap(struct file *file, struct socket *sock,
3901 struct vm_area_struct *vma)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003902{
3903 struct sock *sk = sock->sk;
3904 struct packet_sock *po = pkt_sk(sk);
Johann Baudy69e3c752009-05-18 22:11:22 -07003905 unsigned long size, expected_size;
3906 struct packet_ring_buffer *rb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003907 unsigned long start;
3908 int err = -EINVAL;
3909 int i;
3910
3911 if (vma->vm_pgoff)
3912 return -EINVAL;
3913
Herbert Xu905db442009-01-30 14:12:06 -08003914 mutex_lock(&po->pg_vec_lock);
Johann Baudy69e3c752009-05-18 22:11:22 -07003915
3916 expected_size = 0;
3917 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3918 if (rb->pg_vec) {
3919 expected_size += rb->pg_vec_len
3920 * rb->pg_vec_pages
3921 * PAGE_SIZE;
3922 }
3923 }
3924
3925 if (expected_size == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003926 goto out;
Johann Baudy69e3c752009-05-18 22:11:22 -07003927
3928 size = vma->vm_end - vma->vm_start;
3929 if (size != expected_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003930 goto out;
3931
Linus Torvalds1da177e2005-04-16 15:20:36 -07003932 start = vma->vm_start;
Johann Baudy69e3c752009-05-18 22:11:22 -07003933 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3934 if (rb->pg_vec == NULL)
3935 continue;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003936
Johann Baudy69e3c752009-05-18 22:11:22 -07003937 for (i = 0; i < rb->pg_vec_len; i++) {
Neil Horman0e3125c2010-11-16 10:26:47 -08003938 struct page *page;
3939 void *kaddr = rb->pg_vec[i].buffer;
Johann Baudy69e3c752009-05-18 22:11:22 -07003940 int pg_num;
3941
Changli Gaoc56b4d92010-12-01 02:52:57 +00003942 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
3943 page = pgv_to_page(kaddr);
Johann Baudy69e3c752009-05-18 22:11:22 -07003944 err = vm_insert_page(vma, start, page);
3945 if (unlikely(err))
3946 goto out;
3947 start += PAGE_SIZE;
Neil Horman0e3125c2010-11-16 10:26:47 -08003948 kaddr += PAGE_SIZE;
Johann Baudy69e3c752009-05-18 22:11:22 -07003949 }
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003950 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003951 }
Johann Baudy69e3c752009-05-18 22:11:22 -07003952
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003953 atomic_inc(&po->mapped);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003954 vma->vm_ops = &packet_mmap_ops;
3955 err = 0;
3956
3957out:
Herbert Xu905db442009-01-30 14:12:06 -08003958 mutex_unlock(&po->pg_vec_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003959 return err;
3960}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003961
Eric Dumazet90ddc4f2005-12-22 12:49:22 -08003962static const struct proto_ops packet_ops_spkt = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003963 .family = PF_PACKET,
3964 .owner = THIS_MODULE,
3965 .release = packet_release,
3966 .bind = packet_bind_spkt,
3967 .connect = sock_no_connect,
3968 .socketpair = sock_no_socketpair,
3969 .accept = sock_no_accept,
3970 .getname = packet_getname_spkt,
3971 .poll = datagram_poll,
3972 .ioctl = packet_ioctl,
3973 .listen = sock_no_listen,
3974 .shutdown = sock_no_shutdown,
3975 .setsockopt = sock_no_setsockopt,
3976 .getsockopt = sock_no_getsockopt,
3977 .sendmsg = packet_sendmsg_spkt,
3978 .recvmsg = packet_recvmsg,
3979 .mmap = sock_no_mmap,
3980 .sendpage = sock_no_sendpage,
3981};
Linus Torvalds1da177e2005-04-16 15:20:36 -07003982
Eric Dumazet90ddc4f2005-12-22 12:49:22 -08003983static const struct proto_ops packet_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003984 .family = PF_PACKET,
3985 .owner = THIS_MODULE,
3986 .release = packet_release,
3987 .bind = packet_bind,
3988 .connect = sock_no_connect,
3989 .socketpair = sock_no_socketpair,
3990 .accept = sock_no_accept,
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003991 .getname = packet_getname,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003992 .poll = packet_poll,
3993 .ioctl = packet_ioctl,
3994 .listen = sock_no_listen,
3995 .shutdown = sock_no_shutdown,
3996 .setsockopt = packet_setsockopt,
3997 .getsockopt = packet_getsockopt,
3998 .sendmsg = packet_sendmsg,
3999 .recvmsg = packet_recvmsg,
4000 .mmap = packet_mmap,
4001 .sendpage = sock_no_sendpage,
4002};
4003
Stephen Hemmingerec1b4cf2009-10-05 05:58:39 +00004004static const struct net_proto_family packet_family_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004005 .family = PF_PACKET,
4006 .create = packet_create,
4007 .owner = THIS_MODULE,
4008};
4009
4010static struct notifier_block packet_netdev_notifier = {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00004011 .notifier_call = packet_notifier,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004012};
4013
4014#ifdef CONFIG_PROC_FS
Linus Torvalds1da177e2005-04-16 15:20:36 -07004015
4016static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
stephen hemminger808f5112010-02-22 07:57:18 +00004017 __acquires(RCU)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004018{
Denis V. Luneve372c412007-11-19 22:31:54 -08004019 struct net *net = seq_file_net(seq);
stephen hemminger808f5112010-02-22 07:57:18 +00004020
4021 rcu_read_lock();
4022 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004023}
4024
4025static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4026{
Herbert Xu1bf40952007-12-16 14:04:02 -08004027 struct net *net = seq_file_net(seq);
stephen hemminger808f5112010-02-22 07:57:18 +00004028 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004029}
4030
4031static void packet_seq_stop(struct seq_file *seq, void *v)
stephen hemminger808f5112010-02-22 07:57:18 +00004032 __releases(RCU)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004033{
stephen hemminger808f5112010-02-22 07:57:18 +00004034 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004035}
4036
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09004037static int packet_seq_show(struct seq_file *seq, void *v)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004038{
4039 if (v == SEQ_START_TOKEN)
4040 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
4041 else {
Li Zefanb7ceabd2010-02-08 23:19:29 +00004042 struct sock *s = sk_entry(v);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004043 const struct packet_sock *po = pkt_sk(s);
4044
4045 seq_printf(seq,
Dan Rosenberg71338aa2011-05-23 12:17:35 +00004046 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07004047 s,
4048 atomic_read(&s->sk_refcnt),
4049 s->sk_type,
4050 ntohs(po->num),
4051 po->ifindex,
4052 po->running,
4053 atomic_read(&s->sk_rmem_alloc),
Eric W. Biedermana7cb5a42012-05-24 01:10:10 -06004054 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00004055 sock_i_ino(s));
Linus Torvalds1da177e2005-04-16 15:20:36 -07004056 }
4057
4058 return 0;
4059}
4060
Philippe De Muyter56b3d972007-07-10 23:07:31 -07004061static const struct seq_operations packet_seq_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004062 .start = packet_seq_start,
4063 .next = packet_seq_next,
4064 .stop = packet_seq_stop,
4065 .show = packet_seq_show,
4066};
4067
4068static int packet_seq_open(struct inode *inode, struct file *file)
4069{
Denis V. Luneve372c412007-11-19 22:31:54 -08004070 return seq_open_net(inode, file, &packet_seq_ops,
4071 sizeof(struct seq_net_private));
Linus Torvalds1da177e2005-04-16 15:20:36 -07004072}
4073
Arjan van de Venda7071d2007-02-12 00:55:36 -08004074static const struct file_operations packet_seq_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004075 .owner = THIS_MODULE,
4076 .open = packet_seq_open,
4077 .read = seq_read,
4078 .llseek = seq_lseek,
Denis V. Luneve372c412007-11-19 22:31:54 -08004079 .release = seq_release_net,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004080};
4081
4082#endif
4083
Alexey Dobriyan2c8c1e72010-01-17 03:35:32 +00004084static int __net_init packet_net_init(struct net *net)
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004085{
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00004086 mutex_init(&net->packet.sklist_lock);
Denis V. Lunev2aaef4e2007-12-11 04:19:54 -08004087 INIT_HLIST_HEAD(&net->packet.sklist);
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004088
Gao fengd4beaa62013-02-18 01:34:54 +00004089 if (!proc_create("packet", 0, net->proc_net, &packet_seq_fops))
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004090 return -ENOMEM;
4091
4092 return 0;
4093}
4094
Alexey Dobriyan2c8c1e72010-01-17 03:35:32 +00004095static void __net_exit packet_net_exit(struct net *net)
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004096{
Gao fengece31ff2013-02-18 01:34:56 +00004097 remove_proc_entry("packet", net->proc_net);
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004098}
4099
4100static struct pernet_operations packet_net_ops = {
4101 .init = packet_net_init,
4102 .exit = packet_net_exit,
4103};
4104
4105
Linus Torvalds1da177e2005-04-16 15:20:36 -07004106static void __exit packet_exit(void)
4107{
Linus Torvalds1da177e2005-04-16 15:20:36 -07004108 unregister_netdevice_notifier(&packet_netdev_notifier);
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004109 unregister_pernet_subsys(&packet_net_ops);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004110 sock_unregister(PF_PACKET);
4111 proto_unregister(&packet_proto);
4112}
4113
4114static int __init packet_init(void)
4115{
4116 int rc = proto_register(&packet_proto, 0);
4117
4118 if (rc != 0)
4119 goto out;
4120
4121 sock_register(&packet_family_ops);
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004122 register_pernet_subsys(&packet_net_ops);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004123 register_netdevice_notifier(&packet_netdev_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004124out:
4125 return rc;
4126}
4127
4128module_init(packet_init);
4129module_exit(packet_exit);
4130MODULE_LICENSE("GPL");
4131MODULE_ALIAS_NETPROTO(PF_PACKET);