blob: 6a2bb37506c567c1f03de74bf51eb8aa357e318e [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * PACKET - implements raw packet sockets.
7 *
Jesper Juhl02c30a82005-05-05 16:16:16 -07008 * Authors: Ross Biro
Linus Torvalds1da177e2005-04-16 15:20:36 -07009 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 *
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +090012 * Fixes:
Linus Torvalds1da177e2005-04-16 15:20:36 -070013 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +090035 * Ulises Alonso : Frame number limit removal and
Linus Torvalds1da177e2005-04-16 15:20:36 -070036 * packet_set_ring memory leak.
Eric W. Biederman0fb375f2005-09-21 00:11:37 -070037 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +090040 * byte arrays at the end of sockaddr_ll
Eric W. Biederman0fb375f2005-09-21 00:11:37 -070041 * and packet_mreq.
Johann Baudy69e3c752009-05-18 22:11:22 -070042 * Johann Baudy : Added TX RING.
chetan lokef6fb8f12011-08-19 10:18:16 +000043 * Chetan Loke : Implemented TPACKET_V3 block abstraction
44 * layer.
45 * Copyright (C) 2011, <lokec@ccs.neu.edu>
46 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070047 *
48 * This program is free software; you can redistribute it and/or
49 * modify it under the terms of the GNU General Public License
50 * as published by the Free Software Foundation; either version
51 * 2 of the License, or (at your option) any later version.
52 *
53 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +090054
Linus Torvalds1da177e2005-04-16 15:20:36 -070055#include <linux/types.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070056#include <linux/mm.h>
Randy Dunlap4fc268d2006-01-11 12:17:47 -080057#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070058#include <linux/fcntl.h>
59#include <linux/socket.h>
60#include <linux/in.h>
61#include <linux/inet.h>
62#include <linux/netdevice.h>
63#include <linux/if_packet.h>
64#include <linux/wireless.h>
Herbert Xuffbc6112007-02-04 23:33:10 -080065#include <linux/kernel.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070066#include <linux/kmod.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090067#include <linux/slab.h>
Neil Horman0e3125c2010-11-16 10:26:47 -080068#include <linux/vmalloc.h>
Eric W. Biederman457c4cb2007-09-12 12:01:34 +020069#include <net/net_namespace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070070#include <net/ip.h>
71#include <net/protocol.h>
72#include <linux/skbuff.h>
73#include <net/sock.h>
74#include <linux/errno.h>
75#include <linux/timer.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070076#include <asm/uaccess.h>
77#include <asm/ioctls.h>
78#include <asm/page.h>
Al Viroa1f8e7f72006-10-19 16:08:53 -040079#include <asm/cacheflush.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070080#include <asm/io.h>
81#include <linux/proc_fs.h>
82#include <linux/seq_file.h>
83#include <linux/poll.h>
84#include <linux/module.h>
85#include <linux/init.h>
Herbert Xu905db442009-01-30 14:12:06 -080086#include <linux/mutex.h>
Eric Dumazet05423b22009-10-26 18:40:35 -070087#include <linux/if_vlan.h>
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -080088#include <linux/virtio_net.h>
Richard Cochraned85b562010-04-07 22:41:28 +000089#include <linux/errqueue.h>
Scott McMillan614f60f2010-06-02 05:53:56 -070090#include <linux/net_tstamp.h>
Daniel Borkmannb0138402014-01-15 16:25:36 +010091#include <linux/percpu.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070092#ifdef CONFIG_INET
93#include <net/inet_common.h>
94#endif
95
Pavel Emelyanov2787b042012-08-13 05:49:39 +000096#include "internal.h"
97
Linus Torvalds1da177e2005-04-16 15:20:36 -070098/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070099 Assumptions:
100 - if device has no dev->hard_header routine, it adds and removes ll header
101 inside itself. In this case ll header is invisible outside of device,
102 but higher levels still should reserve dev->hard_header_len.
103 Some devices are enough clever to reallocate skb, when header
104 will not fit to reserved space (tunnel), another ones are silly
105 (PPP).
106 - packet socket receives packets with pulled ll header,
107 so that SOCK_RAW should push it back.
108
109On receive:
110-----------
111
112Incoming, dev->hard_header!=NULL
Arnaldo Carvalho de Melob0e380b12007-04-10 21:21:55 -0700113 mac_header -> ll header
114 data -> data
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115
116Outgoing, dev->hard_header!=NULL
Arnaldo Carvalho de Melob0e380b12007-04-10 21:21:55 -0700117 mac_header -> ll header
118 data -> ll header
Linus Torvalds1da177e2005-04-16 15:20:36 -0700119
120Incoming, dev->hard_header==NULL
Arnaldo Carvalho de Melob0e380b12007-04-10 21:21:55 -0700121 mac_header -> UNKNOWN position. It is very likely, that it points to ll
122 header. PPP makes it, that is wrong, because introduce
YOSHIFUJI Hideakidb0c58f2007-07-19 10:44:35 +0900123 assymetry between rx and tx paths.
Arnaldo Carvalho de Melob0e380b12007-04-10 21:21:55 -0700124 data -> data
Linus Torvalds1da177e2005-04-16 15:20:36 -0700125
126Outgoing, dev->hard_header==NULL
Arnaldo Carvalho de Melob0e380b12007-04-10 21:21:55 -0700127 mac_header -> data. ll header is still not built!
128 data -> data
Linus Torvalds1da177e2005-04-16 15:20:36 -0700129
130Resume
131 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
132
133
134On transmit:
135------------
136
137dev->hard_header != NULL
Arnaldo Carvalho de Melob0e380b12007-04-10 21:21:55 -0700138 mac_header -> ll header
139 data -> ll header
Linus Torvalds1da177e2005-04-16 15:20:36 -0700140
141dev->hard_header == NULL (ll header is added by device, we cannot control it)
Arnaldo Carvalho de Melob0e380b12007-04-10 21:21:55 -0700142 mac_header -> data
143 data -> data
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144
145 We should set nh.raw on output to correct posistion,
146 packet classifier depends on it.
147 */
148
Linus Torvalds1da177e2005-04-16 15:20:36 -0700149/* Private packet socket structures. */
150
Eric W. Biederman0fb375f2005-09-21 00:11:37 -0700151/* identical to struct packet_mreq except it has
152 * a longer address field.
153 */
Eric Dumazet40d4e3d2009-07-21 21:57:59 +0000154struct packet_mreq_max {
Eric W. Biederman0fb375f2005-09-21 00:11:37 -0700155 int mr_ifindex;
156 unsigned short mr_type;
157 unsigned short mr_alen;
158 unsigned char mr_address[MAX_ADDR_LEN];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700159};
David S. Millera2efcfa2007-05-29 13:12:50 -0700160
Daniel Borkmann184f4892013-04-16 01:57:46 +0000161union tpacket_uhdr {
162 struct tpacket_hdr *h1;
163 struct tpacket2_hdr *h2;
164 struct tpacket3_hdr *h3;
165 void *raw;
166};
167
chetan lokef6fb8f12011-08-19 10:18:16 +0000168static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
Johann Baudy69e3c752009-05-18 22:11:22 -0700169 int closing, int tx_ring);
170
chetan lokef6fb8f12011-08-19 10:18:16 +0000171#define V3_ALIGNMENT (8)
172
chetan lokebc59ba32011-08-25 10:43:30 +0000173#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
chetan lokef6fb8f12011-08-19 10:18:16 +0000174
175#define BLK_PLUS_PRIV(sz_of_priv) \
176 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
177
chetan lokef6fb8f12011-08-19 10:18:16 +0000178#define PGV_FROM_VMALLOC 1
Johann Baudy69e3c752009-05-18 22:11:22 -0700179
chetan lokef6fb8f12011-08-19 10:18:16 +0000180#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
181#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
182#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
183#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
184#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
185#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
186#define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
187
Johann Baudy69e3c752009-05-18 22:11:22 -0700188struct packet_sock;
189static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
Willem de Bruijn77f65eb2013-03-19 10:18:11 +0000190static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
191 struct packet_type *pt, struct net_device *orig_dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700192
chetan lokef6fb8f12011-08-19 10:18:16 +0000193static void *packet_previous_frame(struct packet_sock *po,
194 struct packet_ring_buffer *rb,
195 int status);
196static void packet_increment_head(struct packet_ring_buffer *buff);
chetan lokebc59ba32011-08-25 10:43:30 +0000197static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
198 struct tpacket_block_desc *);
199static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
chetan lokef6fb8f12011-08-19 10:18:16 +0000200 struct packet_sock *);
chetan lokebc59ba32011-08-25 10:43:30 +0000201static void prb_retire_current_block(struct tpacket_kbdq_core *,
chetan lokef6fb8f12011-08-19 10:18:16 +0000202 struct packet_sock *, unsigned int status);
chetan lokebc59ba32011-08-25 10:43:30 +0000203static int prb_queue_frozen(struct tpacket_kbdq_core *);
204static void prb_open_block(struct tpacket_kbdq_core *,
205 struct tpacket_block_desc *);
chetan lokef6fb8f12011-08-19 10:18:16 +0000206static void prb_retire_rx_blk_timer_expired(unsigned long);
chetan lokebc59ba32011-08-25 10:43:30 +0000207static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
208static void prb_init_blk_timer(struct packet_sock *,
209 struct tpacket_kbdq_core *,
210 void (*func) (unsigned long));
211static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
212static void prb_clear_rxhash(struct tpacket_kbdq_core *,
213 struct tpacket3_hdr *);
214static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
215 struct tpacket3_hdr *);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700216static void packet_flush_mclist(struct sock *sk);
217
Herbert Xuffbc6112007-02-04 23:33:10 -0800218struct packet_skb_cb {
219 unsigned int origlen;
220 union {
221 struct sockaddr_pkt pkt;
222 struct sockaddr_ll ll;
223 } sa;
224};
225
226#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
Herbert Xu8dc41942007-02-04 23:31:32 -0800227
chetan lokebc59ba32011-08-25 10:43:30 +0000228#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
chetan lokef6fb8f12011-08-19 10:18:16 +0000229#define GET_PBLOCK_DESC(x, bid) \
chetan lokebc59ba32011-08-25 10:43:30 +0000230 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
chetan lokef6fb8f12011-08-19 10:18:16 +0000231#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
chetan lokebc59ba32011-08-25 10:43:30 +0000232 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
chetan lokef6fb8f12011-08-19 10:18:16 +0000233#define GET_NEXT_PRB_BLK_NUM(x) \
234 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
235 ((x)->kactive_blk_num+1) : 0)
236
David S. Millerdc99f602011-07-05 01:45:05 -0700237static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
238static void __fanout_link(struct sock *sk, struct packet_sock *po);
239
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100240static int packet_direct_xmit(struct sk_buff *skb)
241{
242 struct net_device *dev = skb->dev;
243 const struct net_device_ops *ops = dev->netdev_ops;
244 netdev_features_t features;
245 struct netdev_queue *txq;
246 u16 queue_map;
247 int ret;
248
249 if (unlikely(!netif_running(dev) ||
250 !netif_carrier_ok(dev))) {
251 kfree_skb(skb);
252 return NET_XMIT_DROP;
253 }
254
255 features = netif_skb_features(skb);
256 if (skb_needs_linearize(skb, features) &&
257 __skb_linearize(skb)) {
258 kfree_skb(skb);
259 return NET_XMIT_DROP;
260 }
261
262 queue_map = skb_get_queue_mapping(skb);
263 txq = netdev_get_tx_queue(dev, queue_map);
264
265 __netif_tx_lock_bh(txq);
266 if (unlikely(netif_xmit_frozen_or_stopped(txq))) {
267 ret = NETDEV_TX_BUSY;
268 kfree_skb(skb);
269 goto out;
270 }
271
272 ret = ops->ndo_start_xmit(skb, dev);
273 if (likely(dev_xmit_complete(ret)))
274 txq_trans_update(txq);
275 else
276 kfree_skb(skb);
277out:
278 __netif_tx_unlock_bh(txq);
279 return ret;
280}
281
Daniel Borkmann66e56cd2013-12-06 11:36:15 +0100282static struct net_device *packet_cached_dev_get(struct packet_sock *po)
283{
284 struct net_device *dev;
285
286 rcu_read_lock();
287 dev = rcu_dereference(po->cached_dev);
288 if (likely(dev))
289 dev_hold(dev);
290 rcu_read_unlock();
291
292 return dev;
293}
294
295static void packet_cached_dev_assign(struct packet_sock *po,
296 struct net_device *dev)
297{
298 rcu_assign_pointer(po->cached_dev, dev);
299}
300
301static void packet_cached_dev_reset(struct packet_sock *po)
302{
303 RCU_INIT_POINTER(po->cached_dev, NULL);
304}
305
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100306static bool packet_use_direct_xmit(const struct packet_sock *po)
307{
308 return po->xmit == packet_direct_xmit;
309}
310
311static u16 packet_pick_tx_queue(struct net_device *dev)
312{
Li Zhong1cbac012013-12-12 22:39:55 +0100313 return (u16) raw_smp_processor_id() % dev->real_num_tx_queues;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +0100314}
315
David S. Millerce06b032011-07-04 01:44:29 -0700316/* register_prot_hook must be invoked with the po->bind_lock held,
317 * or from a context in which asynchronous accesses to the packet
318 * socket is not possible (packet_create()).
319 */
320static void register_prot_hook(struct sock *sk)
321{
322 struct packet_sock *po = pkt_sk(sk);
Daniel Borkmanne40526c2013-11-21 16:50:58 +0100323
David S. Millerce06b032011-07-04 01:44:29 -0700324 if (!po->running) {
Daniel Borkmann66e56cd2013-12-06 11:36:15 +0100325 if (po->fanout)
David S. Millerdc99f602011-07-05 01:45:05 -0700326 __fanout_link(sk, po);
Daniel Borkmann66e56cd2013-12-06 11:36:15 +0100327 else
David S. Millerdc99f602011-07-05 01:45:05 -0700328 dev_add_pack(&po->prot_hook);
Daniel Borkmanne40526c2013-11-21 16:50:58 +0100329
David S. Millerce06b032011-07-04 01:44:29 -0700330 sock_hold(sk);
331 po->running = 1;
332 }
333}
334
335/* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
336 * held. If the sync parameter is true, we will temporarily drop
337 * the po->bind_lock and do a synchronize_net to make sure no
338 * asynchronous packet processing paths still refer to the elements
339 * of po->prot_hook. If the sync parameter is false, it is the
340 * callers responsibility to take care of this.
341 */
342static void __unregister_prot_hook(struct sock *sk, bool sync)
343{
344 struct packet_sock *po = pkt_sk(sk);
345
346 po->running = 0;
Daniel Borkmann66e56cd2013-12-06 11:36:15 +0100347
348 if (po->fanout)
David S. Millerdc99f602011-07-05 01:45:05 -0700349 __fanout_unlink(sk, po);
Daniel Borkmann66e56cd2013-12-06 11:36:15 +0100350 else
David S. Millerdc99f602011-07-05 01:45:05 -0700351 __dev_remove_pack(&po->prot_hook);
Daniel Borkmanne40526c2013-11-21 16:50:58 +0100352
David S. Millerce06b032011-07-04 01:44:29 -0700353 __sock_put(sk);
354
355 if (sync) {
356 spin_unlock(&po->bind_lock);
357 synchronize_net();
358 spin_lock(&po->bind_lock);
359 }
360}
361
362static void unregister_prot_hook(struct sock *sk, bool sync)
363{
364 struct packet_sock *po = pkt_sk(sk);
365
366 if (po->running)
367 __unregister_prot_hook(sk, sync);
368}
369
Changli Gaof6dafa92010-12-07 04:26:16 +0000370static inline __pure struct page *pgv_to_page(void *addr)
Changli Gao0af55bb2010-12-01 02:52:20 +0000371{
372 if (is_vmalloc_addr(addr))
373 return vmalloc_to_page(addr);
374 return virt_to_page(addr);
375}
376
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700377static void __packet_set_status(struct packet_sock *po, void *frame, int status)
378{
Daniel Borkmann184f4892013-04-16 01:57:46 +0000379 union tpacket_uhdr h;
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700380
381 h.raw = frame;
382 switch (po->tp_version) {
383 case TPACKET_V1:
384 h.h1->tp_status = status;
Changli Gao0af55bb2010-12-01 02:52:20 +0000385 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700386 break;
387 case TPACKET_V2:
388 h.h2->tp_status = status;
Changli Gao0af55bb2010-12-01 02:52:20 +0000389 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700390 break;
chetan lokef6fb8f12011-08-19 10:18:16 +0000391 case TPACKET_V3:
Johann Baudy69e3c752009-05-18 22:11:22 -0700392 default:
chetan lokef6fb8f12011-08-19 10:18:16 +0000393 WARN(1, "TPACKET version not supported.\n");
Johann Baudy69e3c752009-05-18 22:11:22 -0700394 BUG();
395 }
396
397 smp_wmb();
398}
399
400static int __packet_get_status(struct packet_sock *po, void *frame)
401{
Daniel Borkmann184f4892013-04-16 01:57:46 +0000402 union tpacket_uhdr h;
Johann Baudy69e3c752009-05-18 22:11:22 -0700403
404 smp_rmb();
405
406 h.raw = frame;
407 switch (po->tp_version) {
408 case TPACKET_V1:
Changli Gao0af55bb2010-12-01 02:52:20 +0000409 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
Johann Baudy69e3c752009-05-18 22:11:22 -0700410 return h.h1->tp_status;
411 case TPACKET_V2:
Changli Gao0af55bb2010-12-01 02:52:20 +0000412 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
Johann Baudy69e3c752009-05-18 22:11:22 -0700413 return h.h2->tp_status;
chetan lokef6fb8f12011-08-19 10:18:16 +0000414 case TPACKET_V3:
Johann Baudy69e3c752009-05-18 22:11:22 -0700415 default:
chetan lokef6fb8f12011-08-19 10:18:16 +0000416 WARN(1, "TPACKET version not supported.\n");
Johann Baudy69e3c752009-05-18 22:11:22 -0700417 BUG();
418 return 0;
Patrick McHardybbd6ef82008-07-14 22:50:15 -0700419 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700420}
Johann Baudy69e3c752009-05-18 22:11:22 -0700421
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000422static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
423 unsigned int flags)
Daniel Borkmann7a513842013-04-23 00:39:29 +0000424{
425 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
426
427 if (shhwtstamps) {
428 if ((flags & SOF_TIMESTAMPING_SYS_HARDWARE) &&
429 ktime_to_timespec_cond(shhwtstamps->syststamp, ts))
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000430 return TP_STATUS_TS_SYS_HARDWARE;
Daniel Borkmann7a513842013-04-23 00:39:29 +0000431 if ((flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
432 ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000433 return TP_STATUS_TS_RAW_HARDWARE;
Daniel Borkmann7a513842013-04-23 00:39:29 +0000434 }
435
436 if (ktime_to_timespec_cond(skb->tstamp, ts))
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000437 return TP_STATUS_TS_SOFTWARE;
Daniel Borkmann7a513842013-04-23 00:39:29 +0000438
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000439 return 0;
Daniel Borkmann7a513842013-04-23 00:39:29 +0000440}
441
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000442static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
443 struct sk_buff *skb)
Willem de Bruijn2e313962013-04-23 00:39:28 +0000444{
445 union tpacket_uhdr h;
446 struct timespec ts;
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000447 __u32 ts_status;
Willem de Bruijn2e313962013-04-23 00:39:28 +0000448
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000449 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
450 return 0;
Willem de Bruijn2e313962013-04-23 00:39:28 +0000451
452 h.raw = frame;
453 switch (po->tp_version) {
454 case TPACKET_V1:
455 h.h1->tp_sec = ts.tv_sec;
456 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
457 break;
458 case TPACKET_V2:
459 h.h2->tp_sec = ts.tv_sec;
460 h.h2->tp_nsec = ts.tv_nsec;
461 break;
462 case TPACKET_V3:
463 default:
464 WARN(1, "TPACKET version not supported.\n");
465 BUG();
466 }
467
468 /* one flush is safe, as both fields always lie on the same cacheline */
469 flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
470 smp_wmb();
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +0000471
472 return ts_status;
Willem de Bruijn2e313962013-04-23 00:39:28 +0000473}
474
Johann Baudy69e3c752009-05-18 22:11:22 -0700475static void *packet_lookup_frame(struct packet_sock *po,
476 struct packet_ring_buffer *rb,
477 unsigned int position,
478 int status)
479{
480 unsigned int pg_vec_pos, frame_offset;
Daniel Borkmann184f4892013-04-16 01:57:46 +0000481 union tpacket_uhdr h;
Johann Baudy69e3c752009-05-18 22:11:22 -0700482
483 pg_vec_pos = position / rb->frames_per_block;
484 frame_offset = position % rb->frames_per_block;
485
Neil Horman0e3125c2010-11-16 10:26:47 -0800486 h.raw = rb->pg_vec[pg_vec_pos].buffer +
487 (frame_offset * rb->frame_size);
Johann Baudy69e3c752009-05-18 22:11:22 -0700488
489 if (status != __packet_get_status(po, h.raw))
490 return NULL;
491
492 return h.raw;
493}
494
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000495static void *packet_current_frame(struct packet_sock *po,
Johann Baudy69e3c752009-05-18 22:11:22 -0700496 struct packet_ring_buffer *rb,
497 int status)
498{
499 return packet_lookup_frame(po, rb, rb->head, status);
500}
501
chetan lokebc59ba32011-08-25 10:43:30 +0000502static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
chetan lokef6fb8f12011-08-19 10:18:16 +0000503{
504 del_timer_sync(&pkc->retire_blk_timer);
505}
506
507static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
508 int tx_ring,
509 struct sk_buff_head *rb_queue)
510{
chetan lokebc59ba32011-08-25 10:43:30 +0000511 struct tpacket_kbdq_core *pkc;
chetan lokef6fb8f12011-08-19 10:18:16 +0000512
Duan Jiong22781a52013-12-06 13:29:36 +0800513 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
514 GET_PBDQC_FROM_RB(&po->rx_ring);
chetan lokef6fb8f12011-08-19 10:18:16 +0000515
Veaceslav Falicoec6f8092013-11-29 09:53:23 +0100516 spin_lock_bh(&rb_queue->lock);
chetan lokef6fb8f12011-08-19 10:18:16 +0000517 pkc->delete_blk_timer = 1;
Veaceslav Falicoec6f8092013-11-29 09:53:23 +0100518 spin_unlock_bh(&rb_queue->lock);
chetan lokef6fb8f12011-08-19 10:18:16 +0000519
520 prb_del_retire_blk_timer(pkc);
521}
522
523static void prb_init_blk_timer(struct packet_sock *po,
chetan lokebc59ba32011-08-25 10:43:30 +0000524 struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f12011-08-19 10:18:16 +0000525 void (*func) (unsigned long))
526{
527 init_timer(&pkc->retire_blk_timer);
528 pkc->retire_blk_timer.data = (long)po;
529 pkc->retire_blk_timer.function = func;
530 pkc->retire_blk_timer.expires = jiffies;
531}
532
533static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring)
534{
chetan lokebc59ba32011-08-25 10:43:30 +0000535 struct tpacket_kbdq_core *pkc;
chetan lokef6fb8f12011-08-19 10:18:16 +0000536
537 if (tx_ring)
538 BUG();
539
Duan Jiong22781a52013-12-06 13:29:36 +0800540 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
541 GET_PBDQC_FROM_RB(&po->rx_ring);
chetan lokef6fb8f12011-08-19 10:18:16 +0000542 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
543}
544
545static int prb_calc_retire_blk_tmo(struct packet_sock *po,
546 int blk_size_in_bytes)
547{
548 struct net_device *dev;
549 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
Jiri Pirko4bc71cb2011-09-03 03:34:30 +0000550 struct ethtool_cmd ecmd;
551 int err;
parav.pandit@emulex.come440cf22012-06-27 03:56:12 +0000552 u32 speed;
chetan lokef6fb8f12011-08-19 10:18:16 +0000553
Jiri Pirko4bc71cb2011-09-03 03:34:30 +0000554 rtnl_lock();
555 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
556 if (unlikely(!dev)) {
557 rtnl_unlock();
chetan lokef6fb8f12011-08-19 10:18:16 +0000558 return DEFAULT_PRB_RETIRE_TOV;
Jiri Pirko4bc71cb2011-09-03 03:34:30 +0000559 }
560 err = __ethtool_get_settings(dev, &ecmd);
parav.pandit@emulex.come440cf22012-06-27 03:56:12 +0000561 speed = ethtool_cmd_speed(&ecmd);
Jiri Pirko4bc71cb2011-09-03 03:34:30 +0000562 rtnl_unlock();
563 if (!err) {
Jiri Pirko4bc71cb2011-09-03 03:34:30 +0000564 /*
565 * If the link speed is so slow you don't really
566 * need to worry about perf anyways
567 */
parav.pandit@emulex.come440cf22012-06-27 03:56:12 +0000568 if (speed < SPEED_1000 || speed == SPEED_UNKNOWN) {
Jiri Pirko4bc71cb2011-09-03 03:34:30 +0000569 return DEFAULT_PRB_RETIRE_TOV;
parav.pandit@emulex.come440cf22012-06-27 03:56:12 +0000570 } else {
571 msec = 1;
572 div = speed / 1000;
chetan lokef6fb8f12011-08-19 10:18:16 +0000573 }
574 }
575
576 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
577
578 if (div)
579 mbits /= div;
580
581 tmo = mbits * msec;
582
583 if (div)
584 return tmo+1;
585 return tmo;
586}
587
chetan lokebc59ba32011-08-25 10:43:30 +0000588static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
chetan lokef6fb8f12011-08-19 10:18:16 +0000589 union tpacket_req_u *req_u)
590{
591 p1->feature_req_word = req_u->req3.tp_feature_req_word;
592}
593
594static void init_prb_bdqc(struct packet_sock *po,
595 struct packet_ring_buffer *rb,
596 struct pgv *pg_vec,
597 union tpacket_req_u *req_u, int tx_ring)
598{
Duan Jiong22781a52013-12-06 13:29:36 +0800599 struct tpacket_kbdq_core *p1 = GET_PBDQC_FROM_RB(rb);
chetan lokebc59ba32011-08-25 10:43:30 +0000600 struct tpacket_block_desc *pbd;
chetan lokef6fb8f12011-08-19 10:18:16 +0000601
602 memset(p1, 0x0, sizeof(*p1));
603
604 p1->knxt_seq_num = 1;
605 p1->pkbdq = pg_vec;
chetan lokebc59ba32011-08-25 10:43:30 +0000606 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
Joe Perchese3192692012-06-03 17:41:40 +0000607 p1->pkblk_start = pg_vec[0].buffer;
chetan lokef6fb8f12011-08-19 10:18:16 +0000608 p1->kblk_size = req_u->req3.tp_block_size;
609 p1->knum_blocks = req_u->req3.tp_block_nr;
610 p1->hdrlen = po->tp_hdrlen;
611 p1->version = po->tp_version;
612 p1->last_kactive_blk_num = 0;
Daniel Borkmannee80fbf2013-04-19 06:12:29 +0000613 po->stats.stats3.tp_freeze_q_cnt = 0;
chetan lokef6fb8f12011-08-19 10:18:16 +0000614 if (req_u->req3.tp_retire_blk_tov)
615 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
616 else
617 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
618 req_u->req3.tp_block_size);
619 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
620 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
621
622 prb_init_ft_ops(p1, req_u);
623 prb_setup_retire_blk_timer(po, tx_ring);
624 prb_open_block(p1, pbd);
625}
626
627/* Do NOT update the last_blk_num first.
628 * Assumes sk_buff_head lock is held.
629 */
chetan lokebc59ba32011-08-25 10:43:30 +0000630static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
chetan lokef6fb8f12011-08-19 10:18:16 +0000631{
632 mod_timer(&pkc->retire_blk_timer,
633 jiffies + pkc->tov_in_jiffies);
634 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
635}
636
637/*
638 * Timer logic:
639 * 1) We refresh the timer only when we open a block.
640 * By doing this we don't waste cycles refreshing the timer
641 * on packet-by-packet basis.
642 *
643 * With a 1MB block-size, on a 1Gbps line, it will take
644 * i) ~8 ms to fill a block + ii) memcpy etc.
645 * In this cut we are not accounting for the memcpy time.
646 *
647 * So, if the user sets the 'tmo' to 10ms then the timer
648 * will never fire while the block is still getting filled
649 * (which is what we want). However, the user could choose
650 * to close a block early and that's fine.
651 *
652 * But when the timer does fire, we check whether or not to refresh it.
653 * Since the tmo granularity is in msecs, it is not too expensive
654 * to refresh the timer, lets say every '8' msecs.
655 * Either the user can set the 'tmo' or we can derive it based on
656 * a) line-speed and b) block-size.
657 * prb_calc_retire_blk_tmo() calculates the tmo.
658 *
659 */
660static void prb_retire_rx_blk_timer_expired(unsigned long data)
661{
662 struct packet_sock *po = (struct packet_sock *)data;
Duan Jiong22781a52013-12-06 13:29:36 +0800663 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
chetan lokef6fb8f12011-08-19 10:18:16 +0000664 unsigned int frozen;
chetan lokebc59ba32011-08-25 10:43:30 +0000665 struct tpacket_block_desc *pbd;
chetan lokef6fb8f12011-08-19 10:18:16 +0000666
667 spin_lock(&po->sk.sk_receive_queue.lock);
668
669 frozen = prb_queue_frozen(pkc);
670 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
671
672 if (unlikely(pkc->delete_blk_timer))
673 goto out;
674
675 /* We only need to plug the race when the block is partially filled.
676 * tpacket_rcv:
677 * lock(); increment BLOCK_NUM_PKTS; unlock()
678 * copy_bits() is in progress ...
679 * timer fires on other cpu:
680 * we can't retire the current block because copy_bits
681 * is in progress.
682 *
683 */
684 if (BLOCK_NUM_PKTS(pbd)) {
685 while (atomic_read(&pkc->blk_fill_in_prog)) {
686 /* Waiting for skb_copy_bits to finish... */
687 cpu_relax();
688 }
689 }
690
691 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
692 if (!frozen) {
693 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
694 if (!prb_dispatch_next_block(pkc, po))
695 goto refresh_timer;
696 else
697 goto out;
698 } else {
699 /* Case 1. Queue was frozen because user-space was
700 * lagging behind.
701 */
702 if (prb_curr_blk_in_use(pkc, pbd)) {
703 /*
704 * Ok, user-space is still behind.
705 * So just refresh the timer.
706 */
707 goto refresh_timer;
708 } else {
709 /* Case 2. queue was frozen,user-space caught up,
710 * now the link went idle && the timer fired.
711 * We don't have a block to close.So we open this
712 * block and restart the timer.
713 * opening a block thaws the queue,restarts timer
714 * Thawing/timer-refresh is a side effect.
715 */
716 prb_open_block(pkc, pbd);
717 goto out;
718 }
719 }
720 }
721
722refresh_timer:
723 _prb_refresh_rx_retire_blk_timer(pkc);
724
725out:
726 spin_unlock(&po->sk.sk_receive_queue.lock);
727}
728
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000729static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
chetan lokebc59ba32011-08-25 10:43:30 +0000730 struct tpacket_block_desc *pbd1, __u32 status)
chetan lokef6fb8f12011-08-19 10:18:16 +0000731{
732 /* Flush everything minus the block header */
733
734#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
735 u8 *start, *end;
736
737 start = (u8 *)pbd1;
738
739 /* Skip the block header(we know header WILL fit in 4K) */
740 start += PAGE_SIZE;
741
742 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
743 for (; start < end; start += PAGE_SIZE)
744 flush_dcache_page(pgv_to_page(start));
745
746 smp_wmb();
747#endif
748
749 /* Now update the block status. */
750
751 BLOCK_STATUS(pbd1) = status;
752
753 /* Flush the block header */
754
755#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
756 start = (u8 *)pbd1;
757 flush_dcache_page(pgv_to_page(start));
758
759 smp_wmb();
760#endif
761}
762
763/*
764 * Side effect:
765 *
766 * 1) flush the block
767 * 2) Increment active_blk_num
768 *
769 * Note:We DONT refresh the timer on purpose.
770 * Because almost always the next block will be opened.
771 */
chetan lokebc59ba32011-08-25 10:43:30 +0000772static void prb_close_block(struct tpacket_kbdq_core *pkc1,
773 struct tpacket_block_desc *pbd1,
chetan lokef6fb8f12011-08-19 10:18:16 +0000774 struct packet_sock *po, unsigned int stat)
775{
776 __u32 status = TP_STATUS_USER | stat;
777
778 struct tpacket3_hdr *last_pkt;
chetan lokebc59ba32011-08-25 10:43:30 +0000779 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
chetan lokef6fb8f12011-08-19 10:18:16 +0000780
Daniel Borkmannee80fbf2013-04-19 06:12:29 +0000781 if (po->stats.stats3.tp_drops)
chetan lokef6fb8f12011-08-19 10:18:16 +0000782 status |= TP_STATUS_LOSING;
783
784 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
785 last_pkt->tp_next_offset = 0;
786
787 /* Get the ts of the last pkt */
788 if (BLOCK_NUM_PKTS(pbd1)) {
789 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
790 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
791 } else {
792 /* Ok, we tmo'd - so get the current time */
793 struct timespec ts;
794 getnstimeofday(&ts);
795 h1->ts_last_pkt.ts_sec = ts.tv_sec;
796 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
797 }
798
799 smp_wmb();
800
801 /* Flush the block */
802 prb_flush_block(pkc1, pbd1, status);
803
804 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
805}
806
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000807static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
chetan lokef6fb8f12011-08-19 10:18:16 +0000808{
809 pkc->reset_pending_on_curr_blk = 0;
810}
811
812/*
813 * Side effect of opening a block:
814 *
815 * 1) prb_queue is thawed.
816 * 2) retire_blk_timer is refreshed.
817 *
818 */
chetan lokebc59ba32011-08-25 10:43:30 +0000819static void prb_open_block(struct tpacket_kbdq_core *pkc1,
820 struct tpacket_block_desc *pbd1)
chetan lokef6fb8f12011-08-19 10:18:16 +0000821{
822 struct timespec ts;
chetan lokebc59ba32011-08-25 10:43:30 +0000823 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
chetan lokef6fb8f12011-08-19 10:18:16 +0000824
825 smp_rmb();
826
Daniel Borkmann8da30562013-05-03 02:57:00 +0000827 /* We could have just memset this but we will lose the
828 * flexibility of making the priv area sticky
829 */
chetan lokef6fb8f12011-08-19 10:18:16 +0000830
Daniel Borkmann8da30562013-05-03 02:57:00 +0000831 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
832 BLOCK_NUM_PKTS(pbd1) = 0;
833 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
chetan lokef6fb8f12011-08-19 10:18:16 +0000834
Daniel Borkmann8da30562013-05-03 02:57:00 +0000835 getnstimeofday(&ts);
chetan lokef6fb8f12011-08-19 10:18:16 +0000836
Daniel Borkmann8da30562013-05-03 02:57:00 +0000837 h1->ts_first_pkt.ts_sec = ts.tv_sec;
838 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
chetan lokef6fb8f12011-08-19 10:18:16 +0000839
Daniel Borkmann8da30562013-05-03 02:57:00 +0000840 pkc1->pkblk_start = (char *)pbd1;
841 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
842
843 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
844 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
845
846 pbd1->version = pkc1->version;
847 pkc1->prev = pkc1->nxt_offset;
848 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
849
850 prb_thaw_queue(pkc1);
851 _prb_refresh_rx_retire_blk_timer(pkc1);
852
853 smp_wmb();
chetan lokef6fb8f12011-08-19 10:18:16 +0000854}
855
856/*
857 * Queue freeze logic:
858 * 1) Assume tp_block_nr = 8 blocks.
859 * 2) At time 't0', user opens Rx ring.
860 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
861 * 4) user-space is either sleeping or processing block '0'.
862 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
863 * it will close block-7,loop around and try to fill block '0'.
864 * call-flow:
865 * __packet_lookup_frame_in_block
866 * prb_retire_current_block()
867 * prb_dispatch_next_block()
868 * |->(BLOCK_STATUS == USER) evaluates to true
869 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
870 * 6) Now there are two cases:
871 * 6.1) Link goes idle right after the queue is frozen.
872 * But remember, the last open_block() refreshed the timer.
873 * When this timer expires,it will refresh itself so that we can
874 * re-open block-0 in near future.
875 * 6.2) Link is busy and keeps on receiving packets. This is a simple
876 * case and __packet_lookup_frame_in_block will check if block-0
877 * is free and can now be re-used.
878 */
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000879static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f12011-08-19 10:18:16 +0000880 struct packet_sock *po)
881{
882 pkc->reset_pending_on_curr_blk = 1;
Daniel Borkmannee80fbf2013-04-19 06:12:29 +0000883 po->stats.stats3.tp_freeze_q_cnt++;
chetan lokef6fb8f12011-08-19 10:18:16 +0000884}
885
886#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
887
888/*
889 * If the next block is free then we will dispatch it
890 * and return a good offset.
891 * Else, we will freeze the queue.
892 * So, caller must check the return value.
893 */
chetan lokebc59ba32011-08-25 10:43:30 +0000894static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f12011-08-19 10:18:16 +0000895 struct packet_sock *po)
896{
chetan lokebc59ba32011-08-25 10:43:30 +0000897 struct tpacket_block_desc *pbd;
chetan lokef6fb8f12011-08-19 10:18:16 +0000898
899 smp_rmb();
900
901 /* 1. Get current block num */
902 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
903
904 /* 2. If this block is currently in_use then freeze the queue */
905 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
906 prb_freeze_queue(pkc, po);
907 return NULL;
908 }
909
910 /*
911 * 3.
912 * open this block and return the offset where the first packet
913 * needs to get stored.
914 */
915 prb_open_block(pkc, pbd);
916 return (void *)pkc->nxt_offset;
917}
918
chetan lokebc59ba32011-08-25 10:43:30 +0000919static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f12011-08-19 10:18:16 +0000920 struct packet_sock *po, unsigned int status)
921{
chetan lokebc59ba32011-08-25 10:43:30 +0000922 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
chetan lokef6fb8f12011-08-19 10:18:16 +0000923
924 /* retire/close the current block */
925 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
926 /*
927 * Plug the case where copy_bits() is in progress on
928 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
929 * have space to copy the pkt in the current block and
930 * called prb_retire_current_block()
931 *
932 * We don't need to worry about the TMO case because
933 * the timer-handler already handled this case.
934 */
935 if (!(status & TP_STATUS_BLK_TMO)) {
936 while (atomic_read(&pkc->blk_fill_in_prog)) {
937 /* Waiting for skb_copy_bits to finish... */
938 cpu_relax();
939 }
940 }
941 prb_close_block(pkc, pbd, po, status);
942 return;
943 }
chetan lokef6fb8f12011-08-19 10:18:16 +0000944}
945
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000946static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
chetan lokebc59ba32011-08-25 10:43:30 +0000947 struct tpacket_block_desc *pbd)
chetan lokef6fb8f12011-08-19 10:18:16 +0000948{
949 return TP_STATUS_USER & BLOCK_STATUS(pbd);
950}
951
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000952static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
chetan lokef6fb8f12011-08-19 10:18:16 +0000953{
954 return pkc->reset_pending_on_curr_blk;
955}
956
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000957static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
chetan lokef6fb8f12011-08-19 10:18:16 +0000958{
chetan lokebc59ba32011-08-25 10:43:30 +0000959 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
chetan lokef6fb8f12011-08-19 10:18:16 +0000960 atomic_dec(&pkc->blk_fill_in_prog);
961}
962
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000963static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f12011-08-19 10:18:16 +0000964 struct tpacket3_hdr *ppd)
965{
Tom Herbert3958afa1b2013-12-15 22:12:06 -0800966 ppd->hv1.tp_rxhash = skb_get_hash(pkc->skb);
chetan lokef6fb8f12011-08-19 10:18:16 +0000967}
968
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000969static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f12011-08-19 10:18:16 +0000970 struct tpacket3_hdr *ppd)
971{
972 ppd->hv1.tp_rxhash = 0;
973}
974
Olof Johanssoneea49cc92011-11-02 11:00:49 +0000975static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f12011-08-19 10:18:16 +0000976 struct tpacket3_hdr *ppd)
977{
978 if (vlan_tx_tag_present(pkc->skb)) {
979 ppd->hv1.tp_vlan_tci = vlan_tx_tag_get(pkc->skb);
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +0900980 ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->vlan_proto);
981 ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
chetan lokef6fb8f12011-08-19 10:18:16 +0000982 } else {
danborkmann@iogearbox.net9e670302012-08-20 03:34:03 +0000983 ppd->hv1.tp_vlan_tci = 0;
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +0900984 ppd->hv1.tp_vlan_tpid = 0;
danborkmann@iogearbox.net9e670302012-08-20 03:34:03 +0000985 ppd->tp_status = TP_STATUS_AVAILABLE;
chetan lokef6fb8f12011-08-19 10:18:16 +0000986 }
987}
988
chetan lokebc59ba32011-08-25 10:43:30 +0000989static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
chetan lokef6fb8f12011-08-19 10:18:16 +0000990 struct tpacket3_hdr *ppd)
991{
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +0900992 ppd->hv1.tp_padding = 0;
chetan lokef6fb8f12011-08-19 10:18:16 +0000993 prb_fill_vlan_info(pkc, ppd);
994
995 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
996 prb_fill_rxhash(pkc, ppd);
997 else
998 prb_clear_rxhash(pkc, ppd);
999}
1000
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001001static void prb_fill_curr_block(char *curr,
chetan lokebc59ba32011-08-25 10:43:30 +00001002 struct tpacket_kbdq_core *pkc,
1003 struct tpacket_block_desc *pbd,
chetan lokef6fb8f12011-08-19 10:18:16 +00001004 unsigned int len)
1005{
1006 struct tpacket3_hdr *ppd;
1007
1008 ppd = (struct tpacket3_hdr *)curr;
1009 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
1010 pkc->prev = curr;
1011 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
1012 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
1013 BLOCK_NUM_PKTS(pbd) += 1;
1014 atomic_inc(&pkc->blk_fill_in_prog);
1015 prb_run_all_ft_ops(pkc, ppd);
1016}
1017
1018/* Assumes caller has the sk->rx_queue.lock */
1019static void *__packet_lookup_frame_in_block(struct packet_sock *po,
1020 struct sk_buff *skb,
1021 int status,
1022 unsigned int len
1023 )
1024{
chetan lokebc59ba32011-08-25 10:43:30 +00001025 struct tpacket_kbdq_core *pkc;
1026 struct tpacket_block_desc *pbd;
chetan lokef6fb8f12011-08-19 10:18:16 +00001027 char *curr, *end;
1028
Joe Perchese3192692012-06-03 17:41:40 +00001029 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
chetan lokef6fb8f12011-08-19 10:18:16 +00001030 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1031
1032 /* Queue is frozen when user space is lagging behind */
1033 if (prb_queue_frozen(pkc)) {
1034 /*
1035 * Check if that last block which caused the queue to freeze,
1036 * is still in_use by user-space.
1037 */
1038 if (prb_curr_blk_in_use(pkc, pbd)) {
1039 /* Can't record this packet */
1040 return NULL;
1041 } else {
1042 /*
1043 * Ok, the block was released by user-space.
1044 * Now let's open that block.
1045 * opening a block also thaws the queue.
1046 * Thawing is a side effect.
1047 */
1048 prb_open_block(pkc, pbd);
1049 }
1050 }
1051
1052 smp_mb();
1053 curr = pkc->nxt_offset;
1054 pkc->skb = skb;
Joe Perchese3192692012-06-03 17:41:40 +00001055 end = (char *)pbd + pkc->kblk_size;
chetan lokef6fb8f12011-08-19 10:18:16 +00001056
1057 /* first try the current block */
1058 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1059 prb_fill_curr_block(curr, pkc, pbd, len);
1060 return (void *)curr;
1061 }
1062
1063 /* Ok, close the current block */
1064 prb_retire_current_block(pkc, po, 0);
1065
1066 /* Now, try to dispatch the next block */
1067 curr = (char *)prb_dispatch_next_block(pkc, po);
1068 if (curr) {
1069 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1070 prb_fill_curr_block(curr, pkc, pbd, len);
1071 return (void *)curr;
1072 }
1073
1074 /*
1075 * No free blocks are available.user_space hasn't caught up yet.
1076 * Queue was just frozen and now this packet will get dropped.
1077 */
1078 return NULL;
1079}
1080
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001081static void *packet_current_rx_frame(struct packet_sock *po,
chetan lokef6fb8f12011-08-19 10:18:16 +00001082 struct sk_buff *skb,
1083 int status, unsigned int len)
1084{
1085 char *curr = NULL;
1086 switch (po->tp_version) {
1087 case TPACKET_V1:
1088 case TPACKET_V2:
1089 curr = packet_lookup_frame(po, &po->rx_ring,
1090 po->rx_ring.head, status);
1091 return curr;
1092 case TPACKET_V3:
1093 return __packet_lookup_frame_in_block(po, skb, status, len);
1094 default:
1095 WARN(1, "TPACKET version not supported\n");
1096 BUG();
Ying Xue99aa3472012-08-06 16:27:10 +00001097 return NULL;
chetan lokef6fb8f12011-08-19 10:18:16 +00001098 }
1099}
1100
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001101static void *prb_lookup_block(struct packet_sock *po,
chetan lokef6fb8f12011-08-19 10:18:16 +00001102 struct packet_ring_buffer *rb,
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001103 unsigned int idx,
chetan lokef6fb8f12011-08-19 10:18:16 +00001104 int status)
1105{
chetan lokebc59ba32011-08-25 10:43:30 +00001106 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001107 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
chetan lokef6fb8f12011-08-19 10:18:16 +00001108
1109 if (status != BLOCK_STATUS(pbd))
1110 return NULL;
1111 return pbd;
1112}
1113
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001114static int prb_previous_blk_num(struct packet_ring_buffer *rb)
chetan lokef6fb8f12011-08-19 10:18:16 +00001115{
1116 unsigned int prev;
1117 if (rb->prb_bdqc.kactive_blk_num)
1118 prev = rb->prb_bdqc.kactive_blk_num-1;
1119 else
1120 prev = rb->prb_bdqc.knum_blocks-1;
1121 return prev;
1122}
1123
1124/* Assumes caller has held the rx_queue.lock */
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001125static void *__prb_previous_block(struct packet_sock *po,
chetan lokef6fb8f12011-08-19 10:18:16 +00001126 struct packet_ring_buffer *rb,
1127 int status)
1128{
1129 unsigned int previous = prb_previous_blk_num(rb);
1130 return prb_lookup_block(po, rb, previous, status);
1131}
1132
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001133static void *packet_previous_rx_frame(struct packet_sock *po,
chetan lokef6fb8f12011-08-19 10:18:16 +00001134 struct packet_ring_buffer *rb,
1135 int status)
1136{
1137 if (po->tp_version <= TPACKET_V2)
1138 return packet_previous_frame(po, rb, status);
1139
1140 return __prb_previous_block(po, rb, status);
1141}
1142
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001143static void packet_increment_rx_head(struct packet_sock *po,
chetan lokef6fb8f12011-08-19 10:18:16 +00001144 struct packet_ring_buffer *rb)
1145{
1146 switch (po->tp_version) {
1147 case TPACKET_V1:
1148 case TPACKET_V2:
1149 return packet_increment_head(rb);
1150 case TPACKET_V3:
1151 default:
1152 WARN(1, "TPACKET version not supported.\n");
1153 BUG();
1154 return;
1155 }
1156}
1157
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001158static void *packet_previous_frame(struct packet_sock *po,
Johann Baudy69e3c752009-05-18 22:11:22 -07001159 struct packet_ring_buffer *rb,
1160 int status)
1161{
1162 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1163 return packet_lookup_frame(po, rb, previous, status);
1164}
1165
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001166static void packet_increment_head(struct packet_ring_buffer *buff)
Johann Baudy69e3c752009-05-18 22:11:22 -07001167{
1168 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1169}
1170
Daniel Borkmannb0138402014-01-15 16:25:36 +01001171static void packet_inc_pending(struct packet_ring_buffer *rb)
1172{
1173 this_cpu_inc(*rb->pending_refcnt);
1174}
1175
1176static void packet_dec_pending(struct packet_ring_buffer *rb)
1177{
1178 this_cpu_dec(*rb->pending_refcnt);
1179}
1180
1181static unsigned int packet_read_pending(const struct packet_ring_buffer *rb)
1182{
1183 unsigned int refcnt = 0;
1184 int cpu;
1185
1186 /* We don't use pending refcount in rx_ring. */
1187 if (rb->pending_refcnt == NULL)
1188 return 0;
1189
1190 for_each_possible_cpu(cpu)
1191 refcnt += *per_cpu_ptr(rb->pending_refcnt, cpu);
1192
1193 return refcnt;
1194}
1195
1196static int packet_alloc_pending(struct packet_sock *po)
1197{
1198 po->rx_ring.pending_refcnt = NULL;
1199
1200 po->tx_ring.pending_refcnt = alloc_percpu(unsigned int);
1201 if (unlikely(po->tx_ring.pending_refcnt == NULL))
1202 return -ENOBUFS;
1203
1204 return 0;
1205}
1206
1207static void packet_free_pending(struct packet_sock *po)
1208{
1209 free_percpu(po->tx_ring.pending_refcnt);
1210}
1211
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001212static bool packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1213{
1214 struct sock *sk = &po->sk;
1215 bool has_room;
1216
1217 if (po->prot_hook.func != tpacket_rcv)
1218 return (atomic_read(&sk->sk_rmem_alloc) + skb->truesize)
1219 <= sk->sk_rcvbuf;
1220
1221 spin_lock(&sk->sk_receive_queue.lock);
1222 if (po->tp_version == TPACKET_V3)
1223 has_room = prb_lookup_block(po, &po->rx_ring,
1224 po->rx_ring.prb_bdqc.kactive_blk_num,
1225 TP_STATUS_KERNEL);
1226 else
1227 has_room = packet_lookup_frame(po, &po->rx_ring,
1228 po->rx_ring.head,
1229 TP_STATUS_KERNEL);
1230 spin_unlock(&sk->sk_receive_queue.lock);
1231
1232 return has_room;
1233}
1234
Linus Torvalds1da177e2005-04-16 15:20:36 -07001235static void packet_sock_destruct(struct sock *sk)
1236{
Richard Cochraned85b562010-04-07 22:41:28 +00001237 skb_queue_purge(&sk->sk_error_queue);
1238
Ilpo Järvinen547b7922008-07-25 21:43:18 -07001239 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1240 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001241
1242 if (!sock_flag(sk, SOCK_DEAD)) {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001243 pr_err("Attempt to release alive packet socket: %p\n", sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001244 return;
1245 }
1246
Pavel Emelyanov17ab56a2007-11-10 21:38:48 -08001247 sk_refcnt_debug_dec(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001248}
1249
David S. Millerdc99f602011-07-05 01:45:05 -07001250static int fanout_rr_next(struct packet_fanout *f, unsigned int num)
1251{
1252 int x = atomic_read(&f->rr_cur) + 1;
1253
1254 if (x >= num)
1255 x = 0;
1256
1257 return x;
1258}
1259
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001260static unsigned int fanout_demux_hash(struct packet_fanout *f,
1261 struct sk_buff *skb,
1262 unsigned int num)
David S. Millerdc99f602011-07-05 01:45:05 -07001263{
Daniel Borkmann89770b02014-01-22 02:29:40 +01001264 return reciprocal_scale(skb->rxhash, num);
David S. Millerdc99f602011-07-05 01:45:05 -07001265}
1266
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001267static unsigned int fanout_demux_lb(struct packet_fanout *f,
1268 struct sk_buff *skb,
1269 unsigned int num)
David S. Millerdc99f602011-07-05 01:45:05 -07001270{
1271 int cur, old;
1272
1273 cur = atomic_read(&f->rr_cur);
1274 while ((old = atomic_cmpxchg(&f->rr_cur, cur,
1275 fanout_rr_next(f, num))) != cur)
1276 cur = old;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001277 return cur;
David S. Millerdc99f602011-07-05 01:45:05 -07001278}
1279
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001280static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1281 struct sk_buff *skb,
1282 unsigned int num)
David S. Miller95ec3eb2011-07-06 01:56:38 -07001283{
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001284 return smp_processor_id() % num;
1285}
David S. Miller95ec3eb2011-07-06 01:56:38 -07001286
Daniel Borkmann5df0ddf2013-08-28 22:13:09 +02001287static unsigned int fanout_demux_rnd(struct packet_fanout *f,
1288 struct sk_buff *skb,
1289 unsigned int num)
1290{
Daniel Borkmannf337db62014-01-22 02:29:39 +01001291 return prandom_u32_max(num);
Daniel Borkmann5df0ddf2013-08-28 22:13:09 +02001292}
1293
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001294static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1295 struct sk_buff *skb,
1296 unsigned int idx, unsigned int skip,
1297 unsigned int num)
1298{
1299 unsigned int i, j;
1300
1301 i = j = min_t(int, f->next[idx], num - 1);
1302 do {
1303 if (i != skip && packet_rcv_has_room(pkt_sk(f->arr[i]), skb)) {
1304 if (i != j)
1305 f->next[idx] = i;
1306 return i;
1307 }
1308 if (++i == num)
1309 i = 0;
1310 } while (i != j);
1311
1312 return idx;
1313}
1314
Neil Horman2d360972014-01-22 16:01:44 -05001315static unsigned int fanout_demux_qm(struct packet_fanout *f,
1316 struct sk_buff *skb,
1317 unsigned int num)
1318{
1319 return skb_get_queue_mapping(skb) % num;
1320}
1321
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001322static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1323{
1324 return f->flags & (flag >> 8);
David S. Miller95ec3eb2011-07-06 01:56:38 -07001325}
1326
David S. Miller95ec3eb2011-07-06 01:56:38 -07001327static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1328 struct packet_type *pt, struct net_device *orig_dev)
David S. Millerdc99f602011-07-05 01:45:05 -07001329{
1330 struct packet_fanout *f = pt->af_packet_priv;
1331 unsigned int num = f->num_members;
1332 struct packet_sock *po;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001333 unsigned int idx;
David S. Millerdc99f602011-07-05 01:45:05 -07001334
1335 if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
1336 !num) {
1337 kfree_skb(skb);
1338 return 0;
1339 }
1340
David S. Miller95ec3eb2011-07-06 01:56:38 -07001341 switch (f->type) {
1342 case PACKET_FANOUT_HASH:
1343 default:
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001344 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
Eric Dumazetbc416d92011-10-06 10:28:31 +00001345 skb = ip_check_defrag(skb, IP_DEFRAG_AF_PACKET);
David S. Miller95ec3eb2011-07-06 01:56:38 -07001346 if (!skb)
1347 return 0;
1348 }
Tom Herbert3958afa1b2013-12-15 22:12:06 -08001349 skb_get_hash(skb);
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001350 idx = fanout_demux_hash(f, skb, num);
David S. Miller95ec3eb2011-07-06 01:56:38 -07001351 break;
1352 case PACKET_FANOUT_LB:
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001353 idx = fanout_demux_lb(f, skb, num);
David S. Miller95ec3eb2011-07-06 01:56:38 -07001354 break;
1355 case PACKET_FANOUT_CPU:
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001356 idx = fanout_demux_cpu(f, skb, num);
1357 break;
Daniel Borkmann5df0ddf2013-08-28 22:13:09 +02001358 case PACKET_FANOUT_RND:
1359 idx = fanout_demux_rnd(f, skb, num);
1360 break;
Neil Horman2d360972014-01-22 16:01:44 -05001361 case PACKET_FANOUT_QM:
1362 idx = fanout_demux_qm(f, skb, num);
1363 break;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001364 case PACKET_FANOUT_ROLLOVER:
1365 idx = fanout_demux_rollover(f, skb, 0, (unsigned int) -1, num);
David S. Miller95ec3eb2011-07-06 01:56:38 -07001366 break;
David S. Miller7736d332011-07-05 01:43:20 -07001367 }
1368
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001369 po = pkt_sk(f->arr[idx]);
1370 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER) &&
1371 unlikely(!packet_rcv_has_room(po, skb))) {
1372 idx = fanout_demux_rollover(f, skb, idx, idx, num);
1373 po = pkt_sk(f->arr[idx]);
1374 }
David S. Millerdc99f602011-07-05 01:45:05 -07001375
1376 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1377}
1378
Pavel Emelyanovfff33212012-08-16 05:36:48 +00001379DEFINE_MUTEX(fanout_mutex);
1380EXPORT_SYMBOL_GPL(fanout_mutex);
David S. Millerdc99f602011-07-05 01:45:05 -07001381static LIST_HEAD(fanout_list);
1382
1383static void __fanout_link(struct sock *sk, struct packet_sock *po)
1384{
1385 struct packet_fanout *f = po->fanout;
1386
1387 spin_lock(&f->lock);
1388 f->arr[f->num_members] = sk;
1389 smp_wmb();
1390 f->num_members++;
1391 spin_unlock(&f->lock);
1392}
1393
1394static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1395{
1396 struct packet_fanout *f = po->fanout;
1397 int i;
1398
1399 spin_lock(&f->lock);
1400 for (i = 0; i < f->num_members; i++) {
1401 if (f->arr[i] == sk)
1402 break;
1403 }
1404 BUG_ON(i >= f->num_members);
1405 f->arr[i] = f->arr[f->num_members - 1];
1406 f->num_members--;
1407 spin_unlock(&f->lock);
1408}
1409
Weilong Chend4dd8ae2013-12-23 11:31:38 +08001410static bool match_fanout_group(struct packet_type *ptype, struct sock *sk)
Eric Leblondc0de08d2012-08-16 22:02:58 +00001411{
Weilong Chend4dd8ae2013-12-23 11:31:38 +08001412 if (ptype->af_packet_priv == (void *)((struct packet_sock *)sk)->fanout)
Eric Leblondc0de08d2012-08-16 22:02:58 +00001413 return true;
1414
1415 return false;
1416}
1417
David S. Miller7736d332011-07-05 01:43:20 -07001418static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
David S. Millerdc99f602011-07-05 01:45:05 -07001419{
1420 struct packet_sock *po = pkt_sk(sk);
1421 struct packet_fanout *f, *match;
David S. Miller7736d332011-07-05 01:43:20 -07001422 u8 type = type_flags & 0xff;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001423 u8 flags = type_flags >> 8;
David S. Millerdc99f602011-07-05 01:45:05 -07001424 int err;
1425
1426 switch (type) {
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001427 case PACKET_FANOUT_ROLLOVER:
1428 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1429 return -EINVAL;
David S. Millerdc99f602011-07-05 01:45:05 -07001430 case PACKET_FANOUT_HASH:
1431 case PACKET_FANOUT_LB:
David S. Miller95ec3eb2011-07-06 01:56:38 -07001432 case PACKET_FANOUT_CPU:
Daniel Borkmann5df0ddf2013-08-28 22:13:09 +02001433 case PACKET_FANOUT_RND:
Neil Horman2d360972014-01-22 16:01:44 -05001434 case PACKET_FANOUT_QM:
David S. Millerdc99f602011-07-05 01:45:05 -07001435 break;
1436 default:
1437 return -EINVAL;
1438 }
1439
1440 if (!po->running)
1441 return -EINVAL;
1442
1443 if (po->fanout)
1444 return -EALREADY;
1445
1446 mutex_lock(&fanout_mutex);
1447 match = NULL;
1448 list_for_each_entry(f, &fanout_list, list) {
1449 if (f->id == id &&
1450 read_pnet(&f->net) == sock_net(sk)) {
1451 match = f;
1452 break;
1453 }
1454 }
Eric Dumazetafe62c62011-07-07 06:41:29 -07001455 err = -EINVAL;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001456 if (match && match->flags != flags)
Eric Dumazetafe62c62011-07-07 06:41:29 -07001457 goto out;
David S. Millerdc99f602011-07-05 01:45:05 -07001458 if (!match) {
Eric Dumazetafe62c62011-07-07 06:41:29 -07001459 err = -ENOMEM;
David S. Millerdc99f602011-07-05 01:45:05 -07001460 match = kzalloc(sizeof(*match), GFP_KERNEL);
Eric Dumazetafe62c62011-07-07 06:41:29 -07001461 if (!match)
1462 goto out;
1463 write_pnet(&match->net, sock_net(sk));
1464 match->id = id;
1465 match->type = type;
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00001466 match->flags = flags;
Eric Dumazetafe62c62011-07-07 06:41:29 -07001467 atomic_set(&match->rr_cur, 0);
1468 INIT_LIST_HEAD(&match->list);
1469 spin_lock_init(&match->lock);
1470 atomic_set(&match->sk_ref, 0);
1471 match->prot_hook.type = po->prot_hook.type;
1472 match->prot_hook.dev = po->prot_hook.dev;
1473 match->prot_hook.func = packet_rcv_fanout;
1474 match->prot_hook.af_packet_priv = match;
Eric Leblondc0de08d2012-08-16 22:02:58 +00001475 match->prot_hook.id_match = match_fanout_group;
Eric Dumazetafe62c62011-07-07 06:41:29 -07001476 dev_add_pack(&match->prot_hook);
1477 list_add(&match->list, &fanout_list);
1478 }
1479 err = -EINVAL;
1480 if (match->type == type &&
1481 match->prot_hook.type == po->prot_hook.type &&
1482 match->prot_hook.dev == po->prot_hook.dev) {
1483 err = -ENOSPC;
1484 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1485 __dev_remove_pack(&po->prot_hook);
1486 po->fanout = match;
1487 atomic_inc(&match->sk_ref);
1488 __fanout_link(sk, po);
1489 err = 0;
David S. Millerdc99f602011-07-05 01:45:05 -07001490 }
1491 }
Eric Dumazetafe62c62011-07-07 06:41:29 -07001492out:
David S. Millerdc99f602011-07-05 01:45:05 -07001493 mutex_unlock(&fanout_mutex);
1494 return err;
1495}
1496
1497static void fanout_release(struct sock *sk)
1498{
1499 struct packet_sock *po = pkt_sk(sk);
1500 struct packet_fanout *f;
1501
1502 f = po->fanout;
1503 if (!f)
1504 return;
1505
Pavel Emelyanovfff33212012-08-16 05:36:48 +00001506 mutex_lock(&fanout_mutex);
David S. Millerdc99f602011-07-05 01:45:05 -07001507 po->fanout = NULL;
1508
David S. Millerdc99f602011-07-05 01:45:05 -07001509 if (atomic_dec_and_test(&f->sk_ref)) {
1510 list_del(&f->list);
1511 dev_remove_pack(&f->prot_hook);
1512 kfree(f);
1513 }
1514 mutex_unlock(&fanout_mutex);
1515}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001516
Eric Dumazet90ddc4f2005-12-22 12:49:22 -08001517static const struct proto_ops packet_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001518
Eric Dumazet90ddc4f2005-12-22 12:49:22 -08001519static const struct proto_ops packet_ops_spkt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001520
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001521static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1522 struct packet_type *pt, struct net_device *orig_dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001523{
1524 struct sock *sk;
1525 struct sockaddr_pkt *spkt;
1526
1527 /*
1528 * When we registered the protocol we saved the socket in the data
1529 * field for just this event.
1530 */
1531
1532 sk = pt->af_packet_priv;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001533
Linus Torvalds1da177e2005-04-16 15:20:36 -07001534 /*
1535 * Yank back the headers [hope the device set this
1536 * right or kerboom...]
1537 *
1538 * Incoming packets have ll header pulled,
1539 * push it back.
1540 *
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07001541 * For outgoing ones skb->data == skb_mac_header(skb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001542 * so that this procedure is noop.
1543 */
1544
1545 if (skb->pkt_type == PACKET_LOOPBACK)
1546 goto out;
1547
Octavian Purdila09ad9bc2009-11-25 15:14:13 -08001548 if (!net_eq(dev_net(dev), sock_net(sk)))
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08001549 goto out;
1550
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001551 skb = skb_share_check(skb, GFP_ATOMIC);
1552 if (skb == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001553 goto oom;
1554
1555 /* drop any routing info */
Eric Dumazetadf30902009-06-02 05:19:30 +00001556 skb_dst_drop(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001557
Phil Oester84531c22005-07-12 11:57:52 -07001558 /* drop conntrack reference */
1559 nf_reset(skb);
1560
Herbert Xuffbc6112007-02-04 23:33:10 -08001561 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001562
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07001563 skb_push(skb, skb->data - skb_mac_header(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001564
1565 /*
1566 * The SOCK_PACKET socket receives _all_ frames.
1567 */
1568
1569 spkt->spkt_family = dev->type;
1570 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1571 spkt->spkt_protocol = skb->protocol;
1572
1573 /*
1574 * Charge the memory to the socket. This is done specifically
1575 * to prevent sockets using all the memory up.
1576 */
1577
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001578 if (sock_queue_rcv_skb(sk, skb) == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001579 return 0;
1580
1581out:
1582 kfree_skb(skb);
1583oom:
1584 return 0;
1585}
1586
1587
1588/*
1589 * Output a raw packet to a device layer. This bypasses all the other
1590 * protocol layers and you must therefore supply it with a complete frame
1591 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001592
Linus Torvalds1da177e2005-04-16 15:20:36 -07001593static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
1594 struct msghdr *msg, size_t len)
1595{
1596 struct sock *sk = sock->sk;
Steffen Hurrle342dfc32014-01-17 22:53:15 +01001597 DECLARE_SOCKADDR(struct sockaddr_pkt *, saddr, msg->msg_name);
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001598 struct sk_buff *skb = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001599 struct net_device *dev;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001600 __be16 proto = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001601 int err;
Ben Greear3bdc0eb2012-02-11 15:39:30 +00001602 int extra_len = 0;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001603
Linus Torvalds1da177e2005-04-16 15:20:36 -07001604 /*
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001605 * Get and verify the address.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001606 */
1607
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001608 if (saddr) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001609 if (msg->msg_namelen < sizeof(struct sockaddr))
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001610 return -EINVAL;
1611 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1612 proto = saddr->spkt_protocol;
1613 } else
1614 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001615
1616 /*
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001617 * Find the device first to size check it
Linus Torvalds1da177e2005-04-16 15:20:36 -07001618 */
1619
danborkmann@iogearbox.netde74e922012-06-10 08:59:28 +00001620 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001621retry:
Eric Dumazet654d1f82009-11-02 10:43:32 +01001622 rcu_read_lock();
1623 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001624 err = -ENODEV;
1625 if (dev == NULL)
1626 goto out_unlock;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001627
David S. Millerd5e76b02007-01-25 19:30:36 -08001628 err = -ENETDOWN;
1629 if (!(dev->flags & IFF_UP))
1630 goto out_unlock;
1631
Linus Torvalds1da177e2005-04-16 15:20:36 -07001632 /*
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001633 * You may not queue a frame bigger than the mtu. This is the lowest level
1634 * raw protocol and you must do your own fragmentation at this level.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001635 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001636
Ben Greear3bdc0eb2012-02-11 15:39:30 +00001637 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1638 if (!netif_supports_nofcs(dev)) {
1639 err = -EPROTONOSUPPORT;
1640 goto out_unlock;
1641 }
1642 extra_len = 4; /* We're doing our own CRC */
1643 }
1644
Linus Torvalds1da177e2005-04-16 15:20:36 -07001645 err = -EMSGSIZE;
Ben Greear3bdc0eb2012-02-11 15:39:30 +00001646 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001647 goto out_unlock;
1648
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001649 if (!skb) {
1650 size_t reserved = LL_RESERVED_SPACE(dev);
Herbert Xu4ce40912011-11-18 02:20:05 +00001651 int tlen = dev->needed_tailroom;
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001652 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001653
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001654 rcu_read_unlock();
Herbert Xu4ce40912011-11-18 02:20:05 +00001655 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001656 if (skb == NULL)
1657 return -ENOBUFS;
1658 /* FIXME: Save some space for broken drivers that write a hard
1659 * header at transmission time by themselves. PPP is the notable
1660 * one here. This should really be fixed at the driver level.
1661 */
1662 skb_reserve(skb, reserved);
1663 skb_reset_network_header(skb);
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09001664
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001665 /* Try to align data part correctly */
1666 if (hhlen) {
1667 skb->data -= hhlen;
1668 skb->tail -= hhlen;
1669 if (len < hhlen)
1670 skb_reset_network_header(skb);
1671 }
1672 err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
1673 if (err)
1674 goto out_free;
1675 goto retry;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001676 }
1677
Ben Greear3bdc0eb2012-02-11 15:39:30 +00001678 if (len > (dev->mtu + dev->hard_header_len + extra_len)) {
Ben Greear57f89bf2011-02-11 09:35:18 +00001679 /* Earlier code assumed this would be a VLAN pkt,
1680 * double-check this now that we have the actual
1681 * packet in hand.
1682 */
1683 struct ethhdr *ehdr;
1684 skb_reset_mac_header(skb);
1685 ehdr = eth_hdr(skb);
1686 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
1687 err = -EMSGSIZE;
1688 goto out_unlock;
1689 }
1690 }
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001691
Linus Torvalds1da177e2005-04-16 15:20:36 -07001692 skb->protocol = proto;
1693 skb->dev = dev;
1694 skb->priority = sk->sk_priority;
Eric Dumazet2d37a182009-10-01 19:14:46 +00001695 skb->mark = sk->sk_mark;
Daniel Borkmannbf84a0102013-04-14 08:08:13 +00001696
1697 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001698
Ben Greear3bdc0eb2012-02-11 15:39:30 +00001699 if (unlikely(extra_len == 4))
1700 skb->no_fcs = 1;
1701
Jason Wang40893fd2013-03-26 23:11:22 +00001702 skb_probe_transport_header(skb, 0);
Jason Wangc1aad272013-03-25 20:19:57 +00001703
Linus Torvalds1da177e2005-04-16 15:20:36 -07001704 dev_queue_xmit(skb);
Eric Dumazet654d1f82009-11-02 10:43:32 +01001705 rcu_read_unlock();
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001706 return len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001707
Linus Torvalds1da177e2005-04-16 15:20:36 -07001708out_unlock:
Eric Dumazet654d1f82009-11-02 10:43:32 +01001709 rcu_read_unlock();
Eric Dumazet1a35ca82009-12-15 05:47:03 +00001710out_free:
1711 kfree_skb(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001712 return err;
1713}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001714
Olof Johanssoneea49cc92011-11-02 11:00:49 +00001715static unsigned int run_filter(const struct sk_buff *skb,
Eric Dumazet62ab0812010-12-06 20:50:09 +00001716 const struct sock *sk,
David S. Millerdbcb5852007-01-24 15:21:02 -08001717 unsigned int res)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001718{
1719 struct sk_filter *filter;
1720
Eric Dumazet80f8f102011-01-18 07:46:52 +00001721 rcu_read_lock();
1722 filter = rcu_dereference(sk->sk_filter);
David S. Millerdbcb5852007-01-24 15:21:02 -08001723 if (filter != NULL)
Eric Dumazet0a148422011-04-20 09:27:32 +00001724 res = SK_RUN_FILTER(filter, skb);
Eric Dumazet80f8f102011-01-18 07:46:52 +00001725 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001726
David S. Millerdbcb5852007-01-24 15:21:02 -08001727 return res;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001728}
1729
1730/*
Eric Dumazet62ab0812010-12-06 20:50:09 +00001731 * This function makes lazy skb cloning in hope that most of packets
1732 * are discarded by BPF.
1733 *
1734 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
1735 * and skb->cb are mangled. It works because (and until) packets
1736 * falling here are owned by current CPU. Output packets are cloned
1737 * by dev_queue_xmit_nit(), input packets are processed by net_bh
1738 * sequencially, so that if we return skb to original state on exit,
1739 * we will not harm anyone.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001740 */
1741
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001742static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
1743 struct packet_type *pt, struct net_device *orig_dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001744{
1745 struct sock *sk;
1746 struct sockaddr_ll *sll;
1747 struct packet_sock *po;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001748 u8 *skb_head = skb->data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001749 int skb_len = skb->len;
David S. Millerdbcb5852007-01-24 15:21:02 -08001750 unsigned int snaplen, res;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001751
1752 if (skb->pkt_type == PACKET_LOOPBACK)
1753 goto drop;
1754
1755 sk = pt->af_packet_priv;
1756 po = pkt_sk(sk);
1757
Octavian Purdila09ad9bc2009-11-25 15:14:13 -08001758 if (!net_eq(dev_net(dev), sock_net(sk)))
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08001759 goto drop;
1760
Linus Torvalds1da177e2005-04-16 15:20:36 -07001761 skb->dev = dev;
1762
Stephen Hemminger3b04ddd2007-10-09 01:40:57 -07001763 if (dev->header_ops) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001764 /* The device has an explicit notion of ll header,
Eric Dumazet62ab0812010-12-06 20:50:09 +00001765 * exported to higher levels.
1766 *
1767 * Otherwise, the device hides details of its frame
1768 * structure, so that corresponding packet head is
1769 * never delivered to user.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001770 */
1771 if (sk->sk_type != SOCK_DGRAM)
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07001772 skb_push(skb, skb->data - skb_mac_header(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001773 else if (skb->pkt_type == PACKET_OUTGOING) {
1774 /* Special case: outgoing packets have ll header at head */
Arnaldo Carvalho de Melobbe735e2007-03-10 22:16:10 -03001775 skb_pull(skb, skb_network_offset(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001776 }
1777 }
1778
1779 snaplen = skb->len;
1780
David S. Millerdbcb5852007-01-24 15:21:02 -08001781 res = run_filter(skb, sk, snaplen);
1782 if (!res)
Dmitry Mishinfda9ef52006-08-31 15:28:39 -07001783 goto drop_n_restore;
David S. Millerdbcb5852007-01-24 15:21:02 -08001784 if (snaplen > res)
1785 snaplen = res;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001786
Eric Dumazet0fd7bac2011-12-21 07:11:44 +00001787 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001788 goto drop_n_acct;
1789
1790 if (skb_shared(skb)) {
1791 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
1792 if (nskb == NULL)
1793 goto drop_n_acct;
1794
1795 if (skb_head != skb->data) {
1796 skb->data = skb_head;
1797 skb->len = skb_len;
1798 }
Eric Dumazetabc4e4f2012-04-19 02:24:42 +00001799 consume_skb(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001800 skb = nskb;
1801 }
1802
Herbert Xuffbc6112007-02-04 23:33:10 -08001803 BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8 >
1804 sizeof(skb->cb));
1805
1806 sll = &PACKET_SKB_CB(skb)->sa.ll;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001807 sll->sll_family = AF_PACKET;
1808 sll->sll_hatype = dev->type;
1809 sll->sll_protocol = skb->protocol;
1810 sll->sll_pkttype = skb->pkt_type;
Peter P Waskiewicz Jr8032b462007-11-10 22:03:25 -08001811 if (unlikely(po->origdev))
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07001812 sll->sll_ifindex = orig_dev->ifindex;
1813 else
1814 sll->sll_ifindex = dev->ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001815
Stephen Hemmingerb95cce32007-09-26 22:13:38 -07001816 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001817
Herbert Xuffbc6112007-02-04 23:33:10 -08001818 PACKET_SKB_CB(skb)->origlen = skb->len;
Herbert Xu8dc41942007-02-04 23:31:32 -08001819
Linus Torvalds1da177e2005-04-16 15:20:36 -07001820 if (pskb_trim(skb, snaplen))
1821 goto drop_n_acct;
1822
1823 skb_set_owner_r(skb, sk);
1824 skb->dev = NULL;
Eric Dumazetadf30902009-06-02 05:19:30 +00001825 skb_dst_drop(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001826
Phil Oester84531c22005-07-12 11:57:52 -07001827 /* drop conntrack reference */
1828 nf_reset(skb);
1829
Linus Torvalds1da177e2005-04-16 15:20:36 -07001830 spin_lock(&sk->sk_receive_queue.lock);
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00001831 po->stats.stats1.tp_packets++;
Neil Horman3b885782009-10-12 13:26:31 -07001832 skb->dropcount = atomic_read(&sk->sk_drops);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001833 __skb_queue_tail(&sk->sk_receive_queue, skb);
1834 spin_unlock(&sk->sk_receive_queue.lock);
1835 sk->sk_data_ready(sk, skb->len);
1836 return 0;
1837
1838drop_n_acct:
Willem de Bruijn7091fbd2011-09-30 10:38:28 +00001839 spin_lock(&sk->sk_receive_queue.lock);
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00001840 po->stats.stats1.tp_drops++;
Willem de Bruijn7091fbd2011-09-30 10:38:28 +00001841 atomic_inc(&sk->sk_drops);
1842 spin_unlock(&sk->sk_receive_queue.lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001843
1844drop_n_restore:
1845 if (skb_head != skb->data && skb_shared(skb)) {
1846 skb->data = skb_head;
1847 skb->len = skb_len;
1848 }
1849drop:
Neil Hormanead2ceb2009-03-11 09:49:55 +00001850 consume_skb(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001851 return 0;
1852}
1853
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001854static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
1855 struct packet_type *pt, struct net_device *orig_dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001856{
1857 struct sock *sk;
1858 struct packet_sock *po;
1859 struct sockaddr_ll *sll;
Daniel Borkmann184f4892013-04-16 01:57:46 +00001860 union tpacket_uhdr h;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00001861 u8 *skb_head = skb->data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001862 int skb_len = skb->len;
David S. Millerdbcb5852007-01-24 15:21:02 -08001863 unsigned int snaplen, res;
chetan lokef6fb8f12011-08-19 10:18:16 +00001864 unsigned long status = TP_STATUS_USER;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07001865 unsigned short macoff, netoff, hdrlen;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001866 struct sk_buff *copy_skb = NULL;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07001867 struct timespec ts;
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +00001868 __u32 ts_status;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001869
Atzm Watanabe51846352013-12-17 22:53:32 +09001870 /* struct tpacket{2,3}_hdr is aligned to a multiple of TPACKET_ALIGNMENT.
1871 * We may add members to them until current aligned size without forcing
1872 * userspace to call getsockopt(..., PACKET_HDRLEN, ...).
1873 */
1874 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h2)) != 32);
1875 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h3)) != 48);
1876
Linus Torvalds1da177e2005-04-16 15:20:36 -07001877 if (skb->pkt_type == PACKET_LOOPBACK)
1878 goto drop;
1879
1880 sk = pt->af_packet_priv;
1881 po = pkt_sk(sk);
1882
Octavian Purdila09ad9bc2009-11-25 15:14:13 -08001883 if (!net_eq(dev_net(dev), sock_net(sk)))
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08001884 goto drop;
1885
Stephen Hemminger3b04ddd2007-10-09 01:40:57 -07001886 if (dev->header_ops) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001887 if (sk->sk_type != SOCK_DGRAM)
Arnaldo Carvalho de Melo98e399f2007-03-19 15:33:04 -07001888 skb_push(skb, skb->data - skb_mac_header(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001889 else if (skb->pkt_type == PACKET_OUTGOING) {
1890 /* Special case: outgoing packets have ll header at head */
Arnaldo Carvalho de Melobbe735e2007-03-10 22:16:10 -03001891 skb_pull(skb, skb_network_offset(skb));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001892 }
1893 }
1894
Herbert Xu8dc41942007-02-04 23:31:32 -08001895 if (skb->ip_summed == CHECKSUM_PARTIAL)
1896 status |= TP_STATUS_CSUMNOTREADY;
1897
Linus Torvalds1da177e2005-04-16 15:20:36 -07001898 snaplen = skb->len;
1899
David S. Millerdbcb5852007-01-24 15:21:02 -08001900 res = run_filter(skb, sk, snaplen);
1901 if (!res)
Dmitry Mishinfda9ef52006-08-31 15:28:39 -07001902 goto drop_n_restore;
David S. Millerdbcb5852007-01-24 15:21:02 -08001903 if (snaplen > res)
1904 snaplen = res;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001905
1906 if (sk->sk_type == SOCK_DGRAM) {
Patrick McHardy8913336a2008-07-18 18:05:19 -07001907 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
1908 po->tp_reserve;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001909 } else {
Eric Dumazet95c96172012-04-15 05:58:06 +00001910 unsigned int maclen = skb_network_offset(skb);
Patrick McHardybbd6ef82008-07-14 22:50:15 -07001911 netoff = TPACKET_ALIGN(po->tp_hdrlen +
Patrick McHardy8913336a2008-07-18 18:05:19 -07001912 (maclen < 16 ? 16 : maclen)) +
1913 po->tp_reserve;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001914 macoff = netoff - maclen;
1915 }
chetan lokef6fb8f12011-08-19 10:18:16 +00001916 if (po->tp_version <= TPACKET_V2) {
1917 if (macoff + snaplen > po->rx_ring.frame_size) {
1918 if (po->copy_thresh &&
Eric Dumazet0fd7bac2011-12-21 07:11:44 +00001919 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
chetan lokef6fb8f12011-08-19 10:18:16 +00001920 if (skb_shared(skb)) {
1921 copy_skb = skb_clone(skb, GFP_ATOMIC);
1922 } else {
1923 copy_skb = skb_get(skb);
1924 skb_head = skb->data;
1925 }
1926 if (copy_skb)
1927 skb_set_owner_r(copy_skb, sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001928 }
chetan lokef6fb8f12011-08-19 10:18:16 +00001929 snaplen = po->rx_ring.frame_size - macoff;
1930 if ((int)snaplen < 0)
1931 snaplen = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001932 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001933 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001934 spin_lock(&sk->sk_receive_queue.lock);
chetan lokef6fb8f12011-08-19 10:18:16 +00001935 h.raw = packet_current_rx_frame(po, skb,
1936 TP_STATUS_KERNEL, (macoff+snaplen));
Patrick McHardybbd6ef82008-07-14 22:50:15 -07001937 if (!h.raw)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001938 goto ring_is_full;
chetan lokef6fb8f12011-08-19 10:18:16 +00001939 if (po->tp_version <= TPACKET_V2) {
1940 packet_increment_rx_head(po, &po->rx_ring);
1941 /*
1942 * LOSING will be reported till you read the stats,
1943 * because it's COR - Clear On Read.
1944 * Anyways, moving it for V1/V2 only as V3 doesn't need this
1945 * at packet level.
1946 */
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00001947 if (po->stats.stats1.tp_drops)
chetan lokef6fb8f12011-08-19 10:18:16 +00001948 status |= TP_STATUS_LOSING;
1949 }
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00001950 po->stats.stats1.tp_packets++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001951 if (copy_skb) {
1952 status |= TP_STATUS_COPY;
1953 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
1954 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001955 spin_unlock(&sk->sk_receive_queue.lock);
1956
Patrick McHardybbd6ef82008-07-14 22:50:15 -07001957 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +00001958
1959 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
Daniel Borkmann7a513842013-04-23 00:39:29 +00001960 getnstimeofday(&ts);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001961
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +00001962 status |= ts_status;
1963
Patrick McHardybbd6ef82008-07-14 22:50:15 -07001964 switch (po->tp_version) {
1965 case TPACKET_V1:
1966 h.h1->tp_len = skb->len;
1967 h.h1->tp_snaplen = snaplen;
1968 h.h1->tp_mac = macoff;
1969 h.h1->tp_net = netoff;
Daniel Borkmann4b457bd2013-04-16 01:29:11 +00001970 h.h1->tp_sec = ts.tv_sec;
1971 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07001972 hdrlen = sizeof(*h.h1);
1973 break;
1974 case TPACKET_V2:
1975 h.h2->tp_len = skb->len;
1976 h.h2->tp_snaplen = snaplen;
1977 h.h2->tp_mac = macoff;
1978 h.h2->tp_net = netoff;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07001979 h.h2->tp_sec = ts.tv_sec;
1980 h.h2->tp_nsec = ts.tv_nsec;
Ben Greeara3bcc232011-06-01 06:49:10 +00001981 if (vlan_tx_tag_present(skb)) {
1982 h.h2->tp_vlan_tci = vlan_tx_tag_get(skb);
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +09001983 h.h2->tp_vlan_tpid = ntohs(skb->vlan_proto);
1984 status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
Ben Greeara3bcc232011-06-01 06:49:10 +00001985 } else {
1986 h.h2->tp_vlan_tci = 0;
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +09001987 h.h2->tp_vlan_tpid = 0;
Ben Greeara3bcc232011-06-01 06:49:10 +00001988 }
Atzm Watanabee4d26f42013-12-17 22:53:36 +09001989 memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding));
Patrick McHardybbd6ef82008-07-14 22:50:15 -07001990 hdrlen = sizeof(*h.h2);
1991 break;
chetan lokef6fb8f12011-08-19 10:18:16 +00001992 case TPACKET_V3:
1993 /* tp_nxt_offset,vlan are already populated above.
1994 * So DONT clear those fields here
1995 */
1996 h.h3->tp_status |= status;
1997 h.h3->tp_len = skb->len;
1998 h.h3->tp_snaplen = snaplen;
1999 h.h3->tp_mac = macoff;
2000 h.h3->tp_net = netoff;
chetan lokef6fb8f12011-08-19 10:18:16 +00002001 h.h3->tp_sec = ts.tv_sec;
2002 h.h3->tp_nsec = ts.tv_nsec;
Atzm Watanabee4d26f42013-12-17 22:53:36 +09002003 memset(h.h3->tp_padding, 0, sizeof(h.h3->tp_padding));
chetan lokef6fb8f12011-08-19 10:18:16 +00002004 hdrlen = sizeof(*h.h3);
2005 break;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002006 default:
2007 BUG();
2008 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002009
Patrick McHardybbd6ef82008-07-14 22:50:15 -07002010 sll = h.raw + TPACKET_ALIGN(hdrlen);
Stephen Hemmingerb95cce32007-09-26 22:13:38 -07002011 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002012 sll->sll_family = AF_PACKET;
2013 sll->sll_hatype = dev->type;
2014 sll->sll_protocol = skb->protocol;
2015 sll->sll_pkttype = skb->pkt_type;
Peter P Waskiewicz Jr8032b462007-11-10 22:03:25 -08002016 if (unlikely(po->origdev))
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07002017 sll->sll_ifindex = orig_dev->ifindex;
2018 else
2019 sll->sll_ifindex = dev->ifindex;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002020
Ralf Baechlee16aa202006-12-07 00:11:33 -08002021 smp_mb();
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002022
Changli Gaof6dafa92010-12-07 04:26:16 +00002023#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002024 if (po->tp_version <= TPACKET_V2) {
Changli Gao0af55bb2010-12-01 02:52:20 +00002025 u8 *start, *end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002026
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002027 end = (u8 *) PAGE_ALIGN((unsigned long) h.raw +
2028 macoff + snaplen);
2029
2030 for (start = h.raw; start < end; start += PAGE_SIZE)
2031 flush_dcache_page(pgv_to_page(start));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002032 }
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002033 smp_wmb();
Changli Gaof6dafa92010-12-07 04:26:16 +00002034#endif
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002035
chetan lokef6fb8f12011-08-19 10:18:16 +00002036 if (po->tp_version <= TPACKET_V2)
2037 __packet_set_status(po, h.raw, status);
2038 else
2039 prb_clear_blk_fill_status(&po->rx_ring);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002040
2041 sk->sk_data_ready(sk, 0);
2042
2043drop_n_restore:
2044 if (skb_head != skb->data && skb_shared(skb)) {
2045 skb->data = skb_head;
2046 skb->len = skb_len;
2047 }
2048drop:
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002049 kfree_skb(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002050 return 0;
2051
2052ring_is_full:
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00002053 po->stats.stats1.tp_drops++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002054 spin_unlock(&sk->sk_receive_queue.lock);
2055
2056 sk->sk_data_ready(sk, 0);
Wei Yongjunacb5d752009-02-25 00:36:42 +00002057 kfree_skb(copy_skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002058 goto drop_n_restore;
2059}
2060
Johann Baudy69e3c752009-05-18 22:11:22 -07002061static void tpacket_destruct_skb(struct sk_buff *skb)
2062{
2063 struct packet_sock *po = pkt_sk(skb->sk);
Johann Baudy69e3c752009-05-18 22:11:22 -07002064
Johann Baudy69e3c752009-05-18 22:11:22 -07002065 if (likely(po->tx_ring.pg_vec)) {
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01002066 void *ph;
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +00002067 __u32 ts;
2068
Johann Baudy69e3c752009-05-18 22:11:22 -07002069 ph = skb_shinfo(skb)->destructor_arg;
Daniel Borkmannb0138402014-01-15 16:25:36 +01002070 packet_dec_pending(&po->tx_ring);
Daniel Borkmannb9c32fb2013-04-23 00:39:31 +00002071
2072 ts = __packet_set_timestamp(po, ph, skb);
2073 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
Johann Baudy69e3c752009-05-18 22:11:22 -07002074 }
2075
2076 sock_wfree(skb);
2077}
2078
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002079static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
2080 void *frame, struct net_device *dev, int size_max,
Herbert Xuae641942011-11-18 02:20:04 +00002081 __be16 proto, unsigned char *addr, int hlen)
Johann Baudy69e3c752009-05-18 22:11:22 -07002082{
Daniel Borkmann184f4892013-04-16 01:57:46 +00002083 union tpacket_uhdr ph;
David S. Miller09effa62013-08-07 17:11:00 -07002084 int to_write, offset, len, tp_len, nr_frags, len_max;
Johann Baudy69e3c752009-05-18 22:11:22 -07002085 struct socket *sock = po->sk.sk_socket;
2086 struct page *page;
2087 void *data;
2088 int err;
2089
2090 ph.raw = frame;
2091
2092 skb->protocol = proto;
2093 skb->dev = dev;
2094 skb->priority = po->sk.sk_priority;
Eric Dumazet2d37a182009-10-01 19:14:46 +00002095 skb->mark = po->sk.sk_mark;
Willem de Bruijn2e313962013-04-23 00:39:28 +00002096 sock_tx_timestamp(&po->sk, &skb_shinfo(skb)->tx_flags);
Johann Baudy69e3c752009-05-18 22:11:22 -07002097 skb_shinfo(skb)->destructor_arg = ph.raw;
2098
2099 switch (po->tp_version) {
2100 case TPACKET_V2:
2101 tp_len = ph.h2->tp_len;
2102 break;
2103 default:
2104 tp_len = ph.h1->tp_len;
2105 break;
2106 }
David S. Miller09effa62013-08-07 17:11:00 -07002107 if (unlikely(tp_len > size_max)) {
2108 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
2109 return -EMSGSIZE;
2110 }
Johann Baudy69e3c752009-05-18 22:11:22 -07002111
Herbert Xuae641942011-11-18 02:20:04 +00002112 skb_reserve(skb, hlen);
Johann Baudy69e3c752009-05-18 22:11:22 -07002113 skb_reset_network_header(skb);
Jason Wangc1aad272013-03-25 20:19:57 +00002114
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002115 if (!packet_use_direct_xmit(po))
2116 skb_probe_transport_header(skb, 0);
2117 if (unlikely(po->tp_tx_has_off)) {
Paul Chavent5920cd3a2012-11-06 23:10:47 +00002118 int off_min, off_max, off;
2119 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2120 off_max = po->tx_ring.frame_size - tp_len;
2121 if (sock->type == SOCK_DGRAM) {
2122 switch (po->tp_version) {
2123 case TPACKET_V2:
2124 off = ph.h2->tp_net;
2125 break;
2126 default:
2127 off = ph.h1->tp_net;
2128 break;
2129 }
2130 } else {
2131 switch (po->tp_version) {
2132 case TPACKET_V2:
2133 off = ph.h2->tp_mac;
2134 break;
2135 default:
2136 off = ph.h1->tp_mac;
2137 break;
2138 }
2139 }
2140 if (unlikely((off < off_min) || (off_max < off)))
2141 return -EINVAL;
2142 data = ph.raw + off;
2143 } else {
2144 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
2145 }
Johann Baudy69e3c752009-05-18 22:11:22 -07002146 to_write = tp_len;
2147
2148 if (sock->type == SOCK_DGRAM) {
2149 err = dev_hard_header(skb, dev, ntohs(proto), addr,
2150 NULL, tp_len);
2151 if (unlikely(err < 0))
2152 return -EINVAL;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002153 } else if (dev->hard_header_len) {
Johann Baudy69e3c752009-05-18 22:11:22 -07002154 /* net device doesn't like empty head */
2155 if (unlikely(tp_len <= dev->hard_header_len)) {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002156 pr_err("packet size is too short (%d < %d)\n",
2157 tp_len, dev->hard_header_len);
Johann Baudy69e3c752009-05-18 22:11:22 -07002158 return -EINVAL;
2159 }
2160
2161 skb_push(skb, dev->hard_header_len);
2162 err = skb_store_bits(skb, 0, data,
2163 dev->hard_header_len);
2164 if (unlikely(err))
2165 return err;
2166
2167 data += dev->hard_header_len;
2168 to_write -= dev->hard_header_len;
2169 }
2170
Johann Baudy69e3c752009-05-18 22:11:22 -07002171 offset = offset_in_page(data);
2172 len_max = PAGE_SIZE - offset;
2173 len = ((to_write > len_max) ? len_max : to_write);
2174
2175 skb->data_len = to_write;
2176 skb->len += to_write;
2177 skb->truesize += to_write;
2178 atomic_add(to_write, &po->sk.sk_wmem_alloc);
2179
2180 while (likely(to_write)) {
2181 nr_frags = skb_shinfo(skb)->nr_frags;
2182
2183 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002184 pr_err("Packet exceed the number of skb frags(%lu)\n",
2185 MAX_SKB_FRAGS);
Johann Baudy69e3c752009-05-18 22:11:22 -07002186 return -EFAULT;
2187 }
2188
Changli Gao0af55bb2010-12-01 02:52:20 +00002189 page = pgv_to_page(data);
2190 data += len;
Johann Baudy69e3c752009-05-18 22:11:22 -07002191 flush_dcache_page(page);
2192 get_page(page);
Changli Gao0af55bb2010-12-01 02:52:20 +00002193 skb_fill_page_desc(skb, nr_frags, page, offset, len);
Johann Baudy69e3c752009-05-18 22:11:22 -07002194 to_write -= len;
2195 offset = 0;
2196 len_max = PAGE_SIZE;
2197 len = ((to_write > len_max) ? len_max : to_write);
2198 }
2199
2200 return tp_len;
2201}
2202
2203static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2204{
Johann Baudy69e3c752009-05-18 22:11:22 -07002205 struct sk_buff *skb;
2206 struct net_device *dev;
2207 __be16 proto;
David S. Miller09effa62013-08-07 17:11:00 -07002208 int err, reserve = 0;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002209 void *ph;
Steffen Hurrle342dfc32014-01-17 22:53:15 +01002210 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
Daniel Borkmann87a2fd22014-01-15 16:25:35 +01002211 bool need_wait = !(msg->msg_flags & MSG_DONTWAIT);
Johann Baudy69e3c752009-05-18 22:11:22 -07002212 int tp_len, size_max;
2213 unsigned char *addr;
2214 int len_sum = 0;
danborkmann@iogearbox.net9e670302012-08-20 03:34:03 +00002215 int status = TP_STATUS_AVAILABLE;
Herbert Xuae641942011-11-18 02:20:04 +00002216 int hlen, tlen;
Johann Baudy69e3c752009-05-18 22:11:22 -07002217
Johann Baudy69e3c752009-05-18 22:11:22 -07002218 mutex_lock(&po->pg_vec_lock);
2219
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01002220 if (likely(saddr == NULL)) {
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002221 dev = packet_cached_dev_get(po);
Johann Baudy69e3c752009-05-18 22:11:22 -07002222 proto = po->num;
2223 addr = NULL;
2224 } else {
2225 err = -EINVAL;
2226 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2227 goto out;
2228 if (msg->msg_namelen < (saddr->sll_halen
2229 + offsetof(struct sockaddr_ll,
2230 sll_addr)))
2231 goto out;
Johann Baudy69e3c752009-05-18 22:11:22 -07002232 proto = saddr->sll_protocol;
2233 addr = saddr->sll_addr;
Ben Greear827d9782011-06-01 07:18:53 +00002234 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
Johann Baudy69e3c752009-05-18 22:11:22 -07002235 }
2236
Johann Baudy69e3c752009-05-18 22:11:22 -07002237 err = -ENXIO;
2238 if (unlikely(dev == NULL))
2239 goto out;
Johann Baudy69e3c752009-05-18 22:11:22 -07002240 err = -ENETDOWN;
2241 if (unlikely(!(dev->flags & IFF_UP)))
2242 goto out_put;
2243
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002244 reserve = dev->hard_header_len;
2245
Johann Baudy69e3c752009-05-18 22:11:22 -07002246 size_max = po->tx_ring.frame_size
Gabor Gombasb5dd8842009-10-29 03:19:11 -07002247 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
Johann Baudy69e3c752009-05-18 22:11:22 -07002248
David S. Miller09effa62013-08-07 17:11:00 -07002249 if (size_max > dev->mtu + reserve)
2250 size_max = dev->mtu + reserve;
2251
Johann Baudy69e3c752009-05-18 22:11:22 -07002252 do {
2253 ph = packet_current_frame(po, &po->tx_ring,
Daniel Borkmann87a2fd22014-01-15 16:25:35 +01002254 TP_STATUS_SEND_REQUEST);
Johann Baudy69e3c752009-05-18 22:11:22 -07002255 if (unlikely(ph == NULL)) {
Daniel Borkmann87a2fd22014-01-15 16:25:35 +01002256 if (need_wait && need_resched())
2257 schedule();
Johann Baudy69e3c752009-05-18 22:11:22 -07002258 continue;
2259 }
2260
2261 status = TP_STATUS_SEND_REQUEST;
Herbert Xuae641942011-11-18 02:20:04 +00002262 hlen = LL_RESERVED_SPACE(dev);
2263 tlen = dev->needed_tailroom;
Johann Baudy69e3c752009-05-18 22:11:22 -07002264 skb = sock_alloc_send_skb(&po->sk,
Herbert Xuae641942011-11-18 02:20:04 +00002265 hlen + tlen + sizeof(struct sockaddr_ll),
Johann Baudy69e3c752009-05-18 22:11:22 -07002266 0, &err);
2267
2268 if (unlikely(skb == NULL))
2269 goto out_status;
2270
2271 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
Herbert Xuae641942011-11-18 02:20:04 +00002272 addr, hlen);
Johann Baudy69e3c752009-05-18 22:11:22 -07002273
2274 if (unlikely(tp_len < 0)) {
2275 if (po->tp_loss) {
2276 __packet_set_status(po, ph,
2277 TP_STATUS_AVAILABLE);
2278 packet_increment_head(&po->tx_ring);
2279 kfree_skb(skb);
2280 continue;
2281 } else {
2282 status = TP_STATUS_WRONG_FORMAT;
2283 err = tp_len;
2284 goto out_status;
2285 }
2286 }
2287
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002288 skb_set_queue_mapping(skb, packet_pick_tx_queue(dev));
Johann Baudy69e3c752009-05-18 22:11:22 -07002289 skb->destructor = tpacket_destruct_skb;
2290 __packet_set_status(po, ph, TP_STATUS_SENDING);
Daniel Borkmannb0138402014-01-15 16:25:36 +01002291 packet_inc_pending(&po->tx_ring);
Johann Baudy69e3c752009-05-18 22:11:22 -07002292
2293 status = TP_STATUS_SEND_REQUEST;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002294 err = po->xmit(skb);
Jarek Poplawskieb70df12010-01-10 22:04:19 +00002295 if (unlikely(err > 0)) {
2296 err = net_xmit_errno(err);
2297 if (err && __packet_get_status(po, ph) ==
2298 TP_STATUS_AVAILABLE) {
2299 /* skb was destructed already */
2300 skb = NULL;
2301 goto out_status;
2302 }
2303 /*
2304 * skb was dropped but not destructed yet;
2305 * let's treat it like congestion or err < 0
2306 */
2307 err = 0;
2308 }
Johann Baudy69e3c752009-05-18 22:11:22 -07002309 packet_increment_head(&po->tx_ring);
2310 len_sum += tp_len;
Daniel Borkmannb0138402014-01-15 16:25:36 +01002311 } while (likely((ph != NULL) ||
2312 /* Note: packet_read_pending() might be slow if we have
2313 * to call it as it's per_cpu variable, but in fast-path
2314 * we already short-circuit the loop with the first
2315 * condition, and luckily don't have to go that path
2316 * anyway.
2317 */
2318 (need_wait && packet_read_pending(&po->tx_ring))));
Johann Baudy69e3c752009-05-18 22:11:22 -07002319
2320 err = len_sum;
2321 goto out_put;
2322
Johann Baudy69e3c752009-05-18 22:11:22 -07002323out_status:
2324 __packet_set_status(po, ph, status);
2325 kfree_skb(skb);
2326out_put:
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002327 dev_put(dev);
Johann Baudy69e3c752009-05-18 22:11:22 -07002328out:
2329 mutex_unlock(&po->pg_vec_lock);
2330 return err;
2331}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002332
Olof Johanssoneea49cc92011-11-02 11:00:49 +00002333static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2334 size_t reserve, size_t len,
2335 size_t linear, int noblock,
2336 int *err)
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002337{
2338 struct sk_buff *skb;
2339
2340 /* Under a page? Don't bother with paged skb. */
2341 if (prepad + len < PAGE_SIZE || !linear)
2342 linear = len;
2343
2344 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
Eric Dumazet28d64272013-08-08 14:38:47 -07002345 err, 0);
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002346 if (!skb)
2347 return NULL;
2348
2349 skb_reserve(skb, reserve);
2350 skb_put(skb, linear);
2351 skb->data_len = len - linear;
2352 skb->len += len - linear;
2353
2354 return skb;
2355}
2356
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002357static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002358{
2359 struct sock *sk = sock->sk;
Steffen Hurrle342dfc32014-01-17 22:53:15 +01002360 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002361 struct sk_buff *skb;
2362 struct net_device *dev;
Al Viro0e11c912006-11-08 00:26:29 -08002363 __be16 proto;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002364 unsigned char *addr;
Ben Greear827d9782011-06-01 07:18:53 +00002365 int err, reserve = 0;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002366 struct virtio_net_hdr vnet_hdr = { 0 };
2367 int offset = 0;
2368 int vnet_hdr_len;
2369 struct packet_sock *po = pkt_sk(sk);
2370 unsigned short gso_type = 0;
Herbert Xuae641942011-11-18 02:20:04 +00002371 int hlen, tlen;
Ben Greear3bdc0eb2012-02-11 15:39:30 +00002372 int extra_len = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002373
2374 /*
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002375 * Get and verify the address.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002376 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002377
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01002378 if (likely(saddr == NULL)) {
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002379 dev = packet_cached_dev_get(po);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002380 proto = po->num;
2381 addr = NULL;
2382 } else {
2383 err = -EINVAL;
2384 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2385 goto out;
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07002386 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2387 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002388 proto = saddr->sll_protocol;
2389 addr = saddr->sll_addr;
Ben Greear827d9782011-06-01 07:18:53 +00002390 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002391 }
2392
Linus Torvalds1da177e2005-04-16 15:20:36 -07002393 err = -ENXIO;
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002394 if (unlikely(dev == NULL))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002395 goto out_unlock;
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002396 err = -ENETDOWN;
2397 if (unlikely(!(dev->flags & IFF_UP)))
2398 goto out_unlock;
2399
Linus Torvalds1da177e2005-04-16 15:20:36 -07002400 if (sock->type == SOCK_RAW)
2401 reserve = dev->hard_header_len;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002402 if (po->has_vnet_hdr) {
2403 vnet_hdr_len = sizeof(vnet_hdr);
2404
2405 err = -EINVAL;
2406 if (len < vnet_hdr_len)
2407 goto out_unlock;
2408
2409 len -= vnet_hdr_len;
2410
2411 err = memcpy_fromiovec((void *)&vnet_hdr, msg->msg_iov,
2412 vnet_hdr_len);
2413 if (err < 0)
2414 goto out_unlock;
2415
2416 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2417 (vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 >
2418 vnet_hdr.hdr_len))
2419 vnet_hdr.hdr_len = vnet_hdr.csum_start +
2420 vnet_hdr.csum_offset + 2;
2421
2422 err = -EINVAL;
2423 if (vnet_hdr.hdr_len > len)
2424 goto out_unlock;
2425
2426 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2427 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2428 case VIRTIO_NET_HDR_GSO_TCPV4:
2429 gso_type = SKB_GSO_TCPV4;
2430 break;
2431 case VIRTIO_NET_HDR_GSO_TCPV6:
2432 gso_type = SKB_GSO_TCPV6;
2433 break;
2434 case VIRTIO_NET_HDR_GSO_UDP:
2435 gso_type = SKB_GSO_UDP;
2436 break;
2437 default:
2438 goto out_unlock;
2439 }
2440
2441 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
2442 gso_type |= SKB_GSO_TCP_ECN;
2443
2444 if (vnet_hdr.gso_size == 0)
2445 goto out_unlock;
2446
2447 }
2448 }
2449
Ben Greear3bdc0eb2012-02-11 15:39:30 +00002450 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2451 if (!netif_supports_nofcs(dev)) {
2452 err = -EPROTONOSUPPORT;
2453 goto out_unlock;
2454 }
2455 extra_len = 4; /* We're doing our own CRC */
2456 }
2457
Linus Torvalds1da177e2005-04-16 15:20:36 -07002458 err = -EMSGSIZE;
Ben Greear3bdc0eb2012-02-11 15:39:30 +00002459 if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002460 goto out_unlock;
2461
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002462 err = -ENOBUFS;
Herbert Xuae641942011-11-18 02:20:04 +00002463 hlen = LL_RESERVED_SPACE(dev);
2464 tlen = dev->needed_tailroom;
2465 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, vnet_hdr.hdr_len,
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002466 msg->msg_flags & MSG_DONTWAIT, &err);
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002467 if (skb == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002468 goto out_unlock;
2469
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002470 skb_set_network_header(skb, reserve);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002471
Stephen Hemminger0c4e8582007-10-09 01:36:32 -07002472 err = -EINVAL;
2473 if (sock->type == SOCK_DGRAM &&
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002474 (offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len)) < 0)
Stephen Hemminger0c4e8582007-10-09 01:36:32 -07002475 goto out_free;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002476
2477 /* Returns -EFAULT on error */
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002478 err = skb_copy_datagram_from_iovec(skb, offset, msg->msg_iov, 0, len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002479 if (err)
2480 goto out_free;
Daniel Borkmannbf84a0102013-04-14 08:08:13 +00002481
2482 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002483
Ben Greear3bdc0eb2012-02-11 15:39:30 +00002484 if (!gso_type && (len > dev->mtu + reserve + extra_len)) {
David S. Miller09effa62013-08-07 17:11:00 -07002485 /* Earlier code assumed this would be a VLAN pkt,
2486 * double-check this now that we have the actual
2487 * packet in hand.
2488 */
2489 struct ethhdr *ehdr;
2490 skb_reset_mac_header(skb);
2491 ehdr = eth_hdr(skb);
2492 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
2493 err = -EMSGSIZE;
2494 goto out_free;
2495 }
Ben Greear57f89bf2011-02-11 09:35:18 +00002496 }
2497
David S. Miller09effa62013-08-07 17:11:00 -07002498 skb->protocol = proto;
2499 skb->dev = dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002500 skb->priority = sk->sk_priority;
Eric Dumazet2d37a182009-10-01 19:14:46 +00002501 skb->mark = sk->sk_mark;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002502 skb_set_queue_mapping(skb, packet_pick_tx_queue(dev));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002503
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002504 if (po->has_vnet_hdr) {
2505 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
2506 if (!skb_partial_csum_set(skb, vnet_hdr.csum_start,
2507 vnet_hdr.csum_offset)) {
2508 err = -EINVAL;
2509 goto out_free;
2510 }
2511 }
2512
2513 skb_shinfo(skb)->gso_size = vnet_hdr.gso_size;
2514 skb_shinfo(skb)->gso_type = gso_type;
2515
2516 /* Header must be checked, and gso_segs computed. */
2517 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2518 skb_shinfo(skb)->gso_segs = 0;
2519
2520 len += vnet_hdr_len;
2521 }
2522
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002523 if (!packet_use_direct_xmit(po))
2524 skb_probe_transport_header(skb, reserve);
Ben Greear3bdc0eb2012-02-11 15:39:30 +00002525 if (unlikely(extra_len == 4))
2526 skb->no_fcs = 1;
2527
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002528 err = po->xmit(skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002529 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2530 goto out_unlock;
2531
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002532 dev_put(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002533
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002534 return len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002535
2536out_free:
2537 kfree_skb(skb);
2538out_unlock:
Daniel Borkmanne40526c2013-11-21 16:50:58 +01002539 if (dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002540 dev_put(dev);
2541out:
2542 return err;
2543}
2544
Johann Baudy69e3c752009-05-18 22:11:22 -07002545static int packet_sendmsg(struct kiocb *iocb, struct socket *sock,
2546 struct msghdr *msg, size_t len)
2547{
Johann Baudy69e3c752009-05-18 22:11:22 -07002548 struct sock *sk = sock->sk;
2549 struct packet_sock *po = pkt_sk(sk);
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002550
Johann Baudy69e3c752009-05-18 22:11:22 -07002551 if (po->tx_ring.pg_vec)
2552 return tpacket_snd(po, msg);
2553 else
Johann Baudy69e3c752009-05-18 22:11:22 -07002554 return packet_snd(sock, msg, len);
2555}
2556
Linus Torvalds1da177e2005-04-16 15:20:36 -07002557/*
2558 * Close a PACKET socket. This is fairly simple. We immediately go
2559 * to 'closed' state and remove our protocol entry in the device list.
2560 */
2561
2562static int packet_release(struct socket *sock)
2563{
2564 struct sock *sk = sock->sk;
2565 struct packet_sock *po;
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08002566 struct net *net;
chetan lokef6fb8f12011-08-19 10:18:16 +00002567 union tpacket_req_u req_u;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002568
2569 if (!sk)
2570 return 0;
2571
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09002572 net = sock_net(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002573 po = pkt_sk(sk);
2574
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00002575 mutex_lock(&net->packet.sklist_lock);
stephen hemminger808f5112010-02-22 07:57:18 +00002576 sk_del_node_init_rcu(sk);
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00002577 mutex_unlock(&net->packet.sklist_lock);
2578
2579 preempt_disable();
Eric Dumazet920de802008-11-24 00:09:29 -08002580 sock_prot_inuse_add(net, sk->sk_prot, -1);
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00002581 preempt_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002582
stephen hemminger808f5112010-02-22 07:57:18 +00002583 spin_lock(&po->bind_lock);
David S. Millerce06b032011-07-04 01:44:29 -07002584 unregister_prot_hook(sk, false);
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01002585 packet_cached_dev_reset(po);
2586
Ben Greear160ff182011-06-01 07:18:52 +00002587 if (po->prot_hook.dev) {
2588 dev_put(po->prot_hook.dev);
2589 po->prot_hook.dev = NULL;
2590 }
stephen hemminger808f5112010-02-22 07:57:18 +00002591 spin_unlock(&po->bind_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002592
Linus Torvalds1da177e2005-04-16 15:20:36 -07002593 packet_flush_mclist(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002594
Phil Sutter9665d5d2013-02-01 07:21:41 +00002595 if (po->rx_ring.pg_vec) {
2596 memset(&req_u, 0, sizeof(req_u));
chetan lokef6fb8f12011-08-19 10:18:16 +00002597 packet_set_ring(sk, &req_u, 1, 0);
Phil Sutter9665d5d2013-02-01 07:21:41 +00002598 }
Johann Baudy69e3c752009-05-18 22:11:22 -07002599
Phil Sutter9665d5d2013-02-01 07:21:41 +00002600 if (po->tx_ring.pg_vec) {
2601 memset(&req_u, 0, sizeof(req_u));
chetan lokef6fb8f12011-08-19 10:18:16 +00002602 packet_set_ring(sk, &req_u, 1, 1);
Phil Sutter9665d5d2013-02-01 07:21:41 +00002603 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002604
David S. Millerdc99f602011-07-05 01:45:05 -07002605 fanout_release(sk);
2606
stephen hemminger808f5112010-02-22 07:57:18 +00002607 synchronize_net();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002608 /*
2609 * Now the socket is dead. No more input will appear.
2610 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002611 sock_orphan(sk);
2612 sock->sk = NULL;
2613
2614 /* Purge queues */
2615
2616 skb_queue_purge(&sk->sk_receive_queue);
Daniel Borkmannb0138402014-01-15 16:25:36 +01002617 packet_free_pending(po);
Pavel Emelyanov17ab56a2007-11-10 21:38:48 -08002618 sk_refcnt_debug_release(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002619
2620 sock_put(sk);
2621 return 0;
2622}
2623
2624/*
2625 * Attach a packet hook.
2626 */
2627
Daniel Borkmann902fefb2014-01-15 16:25:34 +01002628static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 proto)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002629{
2630 struct packet_sock *po = pkt_sk(sk);
Daniel Borkmann902fefb2014-01-15 16:25:34 +01002631 const struct net_device *dev_curr;
2632 __be16 proto_curr;
2633 bool need_rehook;
David S. Millerdc99f602011-07-05 01:45:05 -07002634
Wei Yongjunaef950b2011-12-27 22:32:41 -05002635 if (po->fanout) {
2636 if (dev)
2637 dev_put(dev);
2638
David S. Millerdc99f602011-07-05 01:45:05 -07002639 return -EINVAL;
Wei Yongjunaef950b2011-12-27 22:32:41 -05002640 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002641
2642 lock_sock(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002643 spin_lock(&po->bind_lock);
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01002644
Daniel Borkmann902fefb2014-01-15 16:25:34 +01002645 proto_curr = po->prot_hook.type;
2646 dev_curr = po->prot_hook.dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002647
Daniel Borkmann902fefb2014-01-15 16:25:34 +01002648 need_rehook = proto_curr != proto || dev_curr != dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002649
Daniel Borkmann902fefb2014-01-15 16:25:34 +01002650 if (need_rehook) {
2651 unregister_prot_hook(sk, true);
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01002652
Daniel Borkmann902fefb2014-01-15 16:25:34 +01002653 po->num = proto;
2654 po->prot_hook.type = proto;
2655
2656 if (po->prot_hook.dev)
2657 dev_put(po->prot_hook.dev);
2658
2659 po->prot_hook.dev = dev;
2660
2661 po->ifindex = dev ? dev->ifindex : 0;
2662 packet_cached_dev_assign(po, dev);
2663 }
2664
2665 if (proto == 0 || !need_rehook)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002666 goto out_unlock;
2667
Urs Thuermannbe85d4a2007-11-12 21:05:20 -08002668 if (!dev || (dev->flags & IFF_UP)) {
David S. Millerce06b032011-07-04 01:44:29 -07002669 register_prot_hook(sk);
Urs Thuermannbe85d4a2007-11-12 21:05:20 -08002670 } else {
2671 sk->sk_err = ENETDOWN;
2672 if (!sock_flag(sk, SOCK_DEAD))
2673 sk->sk_error_report(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002674 }
2675
2676out_unlock:
2677 spin_unlock(&po->bind_lock);
2678 release_sock(sk);
2679 return 0;
2680}
2681
2682/*
2683 * Bind a packet socket to a device
2684 */
2685
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002686static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
2687 int addr_len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002688{
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002689 struct sock *sk = sock->sk;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002690 char name[15];
2691 struct net_device *dev;
2692 int err = -ENODEV;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002693
Linus Torvalds1da177e2005-04-16 15:20:36 -07002694 /*
2695 * Check legality
2696 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002697
Kris Katterjohn8ae55f02006-01-23 16:28:02 -08002698 if (addr_len != sizeof(struct sockaddr))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002699 return -EINVAL;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002700 strlcpy(name, uaddr->sa_data, sizeof(name));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002701
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09002702 dev = dev_get_by_name(sock_net(sk), name);
Ben Greear160ff182011-06-01 07:18:52 +00002703 if (dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002704 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002705 return err;
2706}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002707
2708static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
2709{
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002710 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
2711 struct sock *sk = sock->sk;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002712 struct net_device *dev = NULL;
2713 int err;
2714
2715
2716 /*
2717 * Check legality
2718 */
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002719
Linus Torvalds1da177e2005-04-16 15:20:36 -07002720 if (addr_len < sizeof(struct sockaddr_ll))
2721 return -EINVAL;
2722 if (sll->sll_family != AF_PACKET)
2723 return -EINVAL;
2724
2725 if (sll->sll_ifindex) {
2726 err = -ENODEV;
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09002727 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002728 if (dev == NULL)
2729 goto out;
2730 }
2731 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002732
2733out:
2734 return err;
2735}
2736
2737static struct proto packet_proto = {
2738 .name = "PACKET",
2739 .owner = THIS_MODULE,
2740 .obj_size = sizeof(struct packet_sock),
2741};
2742
2743/*
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002744 * Create a packet of type SOCK_PACKET.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002745 */
2746
Eric Paris3f378b682009-11-05 22:18:14 -08002747static int packet_create(struct net *net, struct socket *sock, int protocol,
2748 int kern)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002749{
2750 struct sock *sk;
2751 struct packet_sock *po;
Al Viro0e11c912006-11-08 00:26:29 -08002752 __be16 proto = (__force __be16)protocol; /* weird, but documented */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002753 int err;
2754
Eric W. Biedermandf008c92012-11-16 03:03:07 +00002755 if (!ns_capable(net->user_ns, CAP_NET_RAW))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002756 return -EPERM;
David S. Millerbe020972007-05-29 13:16:31 -07002757 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
2758 sock->type != SOCK_PACKET)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002759 return -ESOCKTNOSUPPORT;
2760
2761 sock->state = SS_UNCONNECTED;
2762
2763 err = -ENOBUFS;
Pavel Emelyanov6257ff22007-11-01 00:39:31 -07002764 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002765 if (sk == NULL)
2766 goto out;
2767
2768 sock->ops = &packet_ops;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002769 if (sock->type == SOCK_PACKET)
2770 sock->ops = &packet_ops_spkt;
David S. Millerbe020972007-05-29 13:16:31 -07002771
Linus Torvalds1da177e2005-04-16 15:20:36 -07002772 sock_init_data(sock, sk);
2773
2774 po = pkt_sk(sk);
2775 sk->sk_family = PF_PACKET;
Al Viro0e11c912006-11-08 00:26:29 -08002776 po->num = proto;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01002777 po->xmit = dev_queue_xmit;
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01002778
Daniel Borkmannb0138402014-01-15 16:25:36 +01002779 err = packet_alloc_pending(po);
2780 if (err)
2781 goto out2;
2782
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01002783 packet_cached_dev_reset(po);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002784
2785 sk->sk_destruct = packet_sock_destruct;
Pavel Emelyanov17ab56a2007-11-10 21:38:48 -08002786 sk_refcnt_debug_inc(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002787
2788 /*
2789 * Attach a protocol block
2790 */
2791
2792 spin_lock_init(&po->bind_lock);
Herbert Xu905db442009-01-30 14:12:06 -08002793 mutex_init(&po->pg_vec_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002794 po->prot_hook.func = packet_rcv;
David S. Millerbe020972007-05-29 13:16:31 -07002795
Linus Torvalds1da177e2005-04-16 15:20:36 -07002796 if (sock->type == SOCK_PACKET)
2797 po->prot_hook.func = packet_rcv_spkt;
David S. Millerbe020972007-05-29 13:16:31 -07002798
Linus Torvalds1da177e2005-04-16 15:20:36 -07002799 po->prot_hook.af_packet_priv = sk;
2800
Al Viro0e11c912006-11-08 00:26:29 -08002801 if (proto) {
2802 po->prot_hook.type = proto;
David S. Millerce06b032011-07-04 01:44:29 -07002803 register_prot_hook(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002804 }
2805
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00002806 mutex_lock(&net->packet.sklist_lock);
stephen hemminger808f5112010-02-22 07:57:18 +00002807 sk_add_node_rcu(sk, &net->packet.sklist);
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00002808 mutex_unlock(&net->packet.sklist_lock);
2809
2810 preempt_disable();
Eric Dumazet36804532008-11-19 14:25:35 -08002811 sock_prot_inuse_add(net, &packet_proto, 1);
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00002812 preempt_enable();
stephen hemminger808f5112010-02-22 07:57:18 +00002813
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002814 return 0;
Daniel Borkmannb0138402014-01-15 16:25:36 +01002815out2:
2816 sk_free(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002817out:
2818 return err;
2819}
2820
2821/*
2822 * Pull a packet from our receive queue and hand it to the user.
2823 * If necessary we block.
2824 */
2825
2826static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
2827 struct msghdr *msg, size_t len, int flags)
2828{
2829 struct sock *sk = sock->sk;
2830 struct sk_buff *skb;
2831 int copied, err;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002832 int vnet_hdr_len = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002833
2834 err = -EINVAL;
Richard Cochraned85b562010-04-07 22:41:28 +00002835 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002836 goto out;
2837
2838#if 0
2839 /* What error should we return now? EUNATTACH? */
2840 if (pkt_sk(sk)->ifindex < 0)
2841 return -ENODEV;
2842#endif
2843
Richard Cochraned85b562010-04-07 22:41:28 +00002844 if (flags & MSG_ERRQUEUE) {
Richard Cochrancb820f82013-07-19 19:40:09 +02002845 err = sock_recv_errqueue(sk, msg, len,
2846 SOL_PACKET, PACKET_TX_TIMESTAMP);
Richard Cochraned85b562010-04-07 22:41:28 +00002847 goto out;
2848 }
2849
Linus Torvalds1da177e2005-04-16 15:20:36 -07002850 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002851 * Call the generic datagram receiver. This handles all sorts
2852 * of horrible races and re-entrancy so we can forget about it
2853 * in the protocol layers.
2854 *
2855 * Now it will return ENETDOWN, if device have just gone down,
2856 * but then it will block.
2857 */
2858
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002859 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002860
2861 /*
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09002862 * An error occurred so return it. Because skb_recv_datagram()
Linus Torvalds1da177e2005-04-16 15:20:36 -07002863 * handles the blocking we don't see and worry about blocking
2864 * retries.
2865 */
2866
Kris Katterjohn8ae55f02006-01-23 16:28:02 -08002867 if (skb == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002868 goto out;
2869
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002870 if (pkt_sk(sk)->has_vnet_hdr) {
2871 struct virtio_net_hdr vnet_hdr = { 0 };
2872
2873 err = -EINVAL;
2874 vnet_hdr_len = sizeof(vnet_hdr);
Mariusz Kozlowski1f18b712010-11-08 11:58:45 +00002875 if (len < vnet_hdr_len)
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002876 goto out_free;
2877
Mariusz Kozlowski1f18b712010-11-08 11:58:45 +00002878 len -= vnet_hdr_len;
2879
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002880 if (skb_is_gso(skb)) {
2881 struct skb_shared_info *sinfo = skb_shinfo(skb);
2882
2883 /* This is a hint as to how much should be linear. */
2884 vnet_hdr.hdr_len = skb_headlen(skb);
2885 vnet_hdr.gso_size = sinfo->gso_size;
2886 if (sinfo->gso_type & SKB_GSO_TCPV4)
2887 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
2888 else if (sinfo->gso_type & SKB_GSO_TCPV6)
2889 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
2890 else if (sinfo->gso_type & SKB_GSO_UDP)
2891 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
2892 else if (sinfo->gso_type & SKB_GSO_FCOE)
2893 goto out_free;
2894 else
2895 BUG();
2896 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
2897 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
2898 } else
2899 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
2900
2901 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2902 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
Michał Mirosław55508d62010-12-14 15:24:08 +00002903 vnet_hdr.csum_start = skb_checksum_start_offset(skb);
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002904 vnet_hdr.csum_offset = skb->csum_offset;
Jason Wang10a8d942011-06-10 00:56:17 +00002905 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
2906 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002907 } /* else everything is zero */
2908
2909 err = memcpy_toiovec(msg->msg_iov, (void *)&vnet_hdr,
2910 vnet_hdr_len);
2911 if (err < 0)
2912 goto out_free;
2913 }
2914
Hannes Frederic Sowaf3d33422013-11-21 03:14:22 +01002915 /* You lose any data beyond the buffer you gave. If it worries
2916 * a user program they can ask the device for its MTU
2917 * anyway.
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07002918 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002919 copied = skb->len;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00002920 if (copied > len) {
2921 copied = len;
2922 msg->msg_flags |= MSG_TRUNC;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002923 }
2924
2925 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2926 if (err)
2927 goto out_free;
2928
Neil Horman3b885782009-10-12 13:26:31 -07002929 sock_recv_ts_and_drops(msg, sk, skb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002930
Hannes Frederic Sowaf3d33422013-11-21 03:14:22 +01002931 if (msg->msg_name) {
2932 /* If the address length field is there to be filled
2933 * in, we fill it in now.
2934 */
2935 if (sock->type == SOCK_PACKET) {
Steffen Hurrle342dfc32014-01-17 22:53:15 +01002936 __sockaddr_check_size(sizeof(struct sockaddr_pkt));
Hannes Frederic Sowaf3d33422013-11-21 03:14:22 +01002937 msg->msg_namelen = sizeof(struct sockaddr_pkt);
2938 } else {
2939 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
2940 msg->msg_namelen = sll->sll_halen +
2941 offsetof(struct sockaddr_ll, sll_addr);
2942 }
Herbert Xuffbc6112007-02-04 23:33:10 -08002943 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
2944 msg->msg_namelen);
Hannes Frederic Sowaf3d33422013-11-21 03:14:22 +01002945 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002946
Herbert Xu8dc41942007-02-04 23:31:32 -08002947 if (pkt_sk(sk)->auxdata) {
Herbert Xuffbc6112007-02-04 23:33:10 -08002948 struct tpacket_auxdata aux;
2949
2950 aux.tp_status = TP_STATUS_USER;
2951 if (skb->ip_summed == CHECKSUM_PARTIAL)
2952 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
2953 aux.tp_len = PACKET_SKB_CB(skb)->origlen;
2954 aux.tp_snaplen = skb->len;
2955 aux.tp_mac = 0;
Arnaldo Carvalho de Melobbe735e2007-03-10 22:16:10 -03002956 aux.tp_net = skb_network_offset(skb);
Ben Greeara3bcc232011-06-01 06:49:10 +00002957 if (vlan_tx_tag_present(skb)) {
2958 aux.tp_vlan_tci = vlan_tx_tag_get(skb);
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +09002959 aux.tp_vlan_tpid = ntohs(skb->vlan_proto);
2960 aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
Ben Greeara3bcc232011-06-01 06:49:10 +00002961 } else {
2962 aux.tp_vlan_tci = 0;
Atzm Watanabea0cdfcf2013-12-17 22:53:40 +09002963 aux.tp_vlan_tpid = 0;
Ben Greeara3bcc232011-06-01 06:49:10 +00002964 }
Herbert Xuffbc6112007-02-04 23:33:10 -08002965 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
Herbert Xu8dc41942007-02-04 23:31:32 -08002966 }
2967
Linus Torvalds1da177e2005-04-16 15:20:36 -07002968 /*
2969 * Free or return the buffer as appropriate. Again this
2970 * hides all the races and re-entrancy issues from us.
2971 */
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08002972 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002973
2974out_free:
2975 skb_free_datagram(sk, skb);
2976out:
2977 return err;
2978}
2979
Linus Torvalds1da177e2005-04-16 15:20:36 -07002980static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
2981 int *uaddr_len, int peer)
2982{
2983 struct net_device *dev;
2984 struct sock *sk = sock->sk;
2985
2986 if (peer)
2987 return -EOPNOTSUPP;
2988
2989 uaddr->sa_family = AF_PACKET;
Daniel Borkmann2dc85bf2013-06-12 16:02:27 +02002990 memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
Eric Dumazet654d1f82009-11-02 10:43:32 +01002991 rcu_read_lock();
2992 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
2993 if (dev)
Daniel Borkmann2dc85bf2013-06-12 16:02:27 +02002994 strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
Eric Dumazet654d1f82009-11-02 10:43:32 +01002995 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002996 *uaddr_len = sizeof(*uaddr);
2997
2998 return 0;
2999}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003000
3001static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
3002 int *uaddr_len, int peer)
3003{
3004 struct net_device *dev;
3005 struct sock *sk = sock->sk;
3006 struct packet_sock *po = pkt_sk(sk);
Cyrill Gorcunov13cfa972009-11-08 05:51:19 +00003007 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003008
3009 if (peer)
3010 return -EOPNOTSUPP;
3011
3012 sll->sll_family = AF_PACKET;
3013 sll->sll_ifindex = po->ifindex;
3014 sll->sll_protocol = po->num;
Vasiliy Kulikov67286642010-11-10 12:09:10 -08003015 sll->sll_pkttype = 0;
Eric Dumazet654d1f82009-11-02 10:43:32 +01003016 rcu_read_lock();
3017 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003018 if (dev) {
3019 sll->sll_hatype = dev->type;
3020 sll->sll_halen = dev->addr_len;
3021 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003022 } else {
3023 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
3024 sll->sll_halen = 0;
3025 }
Eric Dumazet654d1f82009-11-02 10:43:32 +01003026 rcu_read_unlock();
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003027 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003028
3029 return 0;
3030}
3031
Wang Chen2aeb0b82008-07-14 20:49:46 -07003032static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
3033 int what)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003034{
3035 switch (i->type) {
3036 case PACKET_MR_MULTICAST:
Jiri Pirko11625632010-03-02 20:40:01 +00003037 if (i->alen != dev->addr_len)
3038 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003039 if (what > 0)
Jiri Pirko22bedad32010-04-01 21:22:57 +00003040 return dev_mc_add(dev, i->addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003041 else
Jiri Pirko22bedad32010-04-01 21:22:57 +00003042 return dev_mc_del(dev, i->addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003043 break;
3044 case PACKET_MR_PROMISC:
Wang Chen2aeb0b82008-07-14 20:49:46 -07003045 return dev_set_promiscuity(dev, what);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003046 break;
3047 case PACKET_MR_ALLMULTI:
Wang Chen2aeb0b82008-07-14 20:49:46 -07003048 return dev_set_allmulti(dev, what);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003049 break;
Eric W. Biedermand95ed922009-05-19 18:27:17 +00003050 case PACKET_MR_UNICAST:
Jiri Pirko11625632010-03-02 20:40:01 +00003051 if (i->alen != dev->addr_len)
3052 return -EINVAL;
Eric W. Biedermand95ed922009-05-19 18:27:17 +00003053 if (what > 0)
Jiri Pirkoa748ee22010-04-01 21:22:09 +00003054 return dev_uc_add(dev, i->addr);
Eric W. Biedermand95ed922009-05-19 18:27:17 +00003055 else
Jiri Pirkoa748ee22010-04-01 21:22:09 +00003056 return dev_uc_del(dev, i->addr);
Eric W. Biedermand95ed922009-05-19 18:27:17 +00003057 break;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003058 default:
3059 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003060 }
Wang Chen2aeb0b82008-07-14 20:49:46 -07003061 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003062}
3063
3064static void packet_dev_mclist(struct net_device *dev, struct packet_mclist *i, int what)
3065{
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003066 for ( ; i; i = i->next) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003067 if (i->ifindex == dev->ifindex)
3068 packet_dev_mc(dev, i, what);
3069 }
3070}
3071
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003072static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003073{
3074 struct packet_sock *po = pkt_sk(sk);
3075 struct packet_mclist *ml, *i;
3076 struct net_device *dev;
3077 int err;
3078
3079 rtnl_lock();
3080
3081 err = -ENODEV;
YOSHIFUJI Hideaki3b1e0a62008-03-26 02:26:21 +09003082 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003083 if (!dev)
3084 goto done;
3085
3086 err = -EINVAL;
Jiri Pirko11625632010-03-02 20:40:01 +00003087 if (mreq->mr_alen > dev->addr_len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003088 goto done;
3089
3090 err = -ENOBUFS;
Kris Katterjohn8b3a7002006-01-11 15:56:43 -08003091 i = kmalloc(sizeof(*i), GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003092 if (i == NULL)
3093 goto done;
3094
3095 err = 0;
3096 for (ml = po->mclist; ml; ml = ml->next) {
3097 if (ml->ifindex == mreq->mr_ifindex &&
3098 ml->type == mreq->mr_type &&
3099 ml->alen == mreq->mr_alen &&
3100 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3101 ml->count++;
3102 /* Free the new element ... */
3103 kfree(i);
3104 goto done;
3105 }
3106 }
3107
3108 i->type = mreq->mr_type;
3109 i->ifindex = mreq->mr_ifindex;
3110 i->alen = mreq->mr_alen;
3111 memcpy(i->addr, mreq->mr_address, i->alen);
3112 i->count = 1;
3113 i->next = po->mclist;
3114 po->mclist = i;
Wang Chen2aeb0b82008-07-14 20:49:46 -07003115 err = packet_dev_mc(dev, i, 1);
3116 if (err) {
3117 po->mclist = i->next;
3118 kfree(i);
3119 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003120
3121done:
3122 rtnl_unlock();
3123 return err;
3124}
3125
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003126static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003127{
3128 struct packet_mclist *ml, **mlp;
3129
3130 rtnl_lock();
3131
3132 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
3133 if (ml->ifindex == mreq->mr_ifindex &&
3134 ml->type == mreq->mr_type &&
3135 ml->alen == mreq->mr_alen &&
3136 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3137 if (--ml->count == 0) {
3138 struct net_device *dev;
3139 *mlp = ml->next;
Eric Dumazetad959e72009-10-16 06:38:46 +00003140 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3141 if (dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003142 packet_dev_mc(dev, ml, -1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003143 kfree(ml);
3144 }
3145 rtnl_unlock();
3146 return 0;
3147 }
3148 }
3149 rtnl_unlock();
3150 return -EADDRNOTAVAIL;
3151}
3152
3153static void packet_flush_mclist(struct sock *sk)
3154{
3155 struct packet_sock *po = pkt_sk(sk);
3156 struct packet_mclist *ml;
3157
3158 if (!po->mclist)
3159 return;
3160
3161 rtnl_lock();
3162 while ((ml = po->mclist) != NULL) {
3163 struct net_device *dev;
3164
3165 po->mclist = ml->next;
Eric Dumazetad959e72009-10-16 06:38:46 +00003166 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3167 if (dev != NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003168 packet_dev_mc(dev, ml, -1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003169 kfree(ml);
3170 }
3171 rtnl_unlock();
3172}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003173
3174static int
David S. Millerb7058842009-09-30 16:12:20 -07003175packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003176{
3177 struct sock *sk = sock->sk;
Herbert Xu8dc41942007-02-04 23:31:32 -08003178 struct packet_sock *po = pkt_sk(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003179 int ret;
3180
3181 if (level != SOL_PACKET)
3182 return -ENOPROTOOPT;
3183
Johann Baudy69e3c752009-05-18 22:11:22 -07003184 switch (optname) {
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003185 case PACKET_ADD_MEMBERSHIP:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003186 case PACKET_DROP_MEMBERSHIP:
3187 {
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003188 struct packet_mreq_max mreq;
3189 int len = optlen;
3190 memset(&mreq, 0, sizeof(mreq));
3191 if (len < sizeof(struct packet_mreq))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003192 return -EINVAL;
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003193 if (len > sizeof(mreq))
3194 len = sizeof(mreq);
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003195 if (copy_from_user(&mreq, optval, len))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003196 return -EFAULT;
Eric W. Biederman0fb375f2005-09-21 00:11:37 -07003197 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3198 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003199 if (optname == PACKET_ADD_MEMBERSHIP)
3200 ret = packet_mc_add(sk, &mreq);
3201 else
3202 ret = packet_mc_drop(sk, &mreq);
3203 return ret;
3204 }
David S. Millera2efcfa2007-05-29 13:12:50 -07003205
Linus Torvalds1da177e2005-04-16 15:20:36 -07003206 case PACKET_RX_RING:
Johann Baudy69e3c752009-05-18 22:11:22 -07003207 case PACKET_TX_RING:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003208 {
chetan lokef6fb8f12011-08-19 10:18:16 +00003209 union tpacket_req_u req_u;
3210 int len;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003211
chetan lokef6fb8f12011-08-19 10:18:16 +00003212 switch (po->tp_version) {
3213 case TPACKET_V1:
3214 case TPACKET_V2:
3215 len = sizeof(req_u.req);
3216 break;
3217 case TPACKET_V3:
3218 default:
3219 len = sizeof(req_u.req3);
3220 break;
3221 }
3222 if (optlen < len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003223 return -EINVAL;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003224 if (pkt_sk(sk)->has_vnet_hdr)
3225 return -EINVAL;
chetan lokef6fb8f12011-08-19 10:18:16 +00003226 if (copy_from_user(&req_u.req, optval, len))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003227 return -EFAULT;
chetan lokef6fb8f12011-08-19 10:18:16 +00003228 return packet_set_ring(sk, &req_u, 0,
3229 optname == PACKET_TX_RING);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003230 }
3231 case PACKET_COPY_THRESH:
3232 {
3233 int val;
3234
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003235 if (optlen != sizeof(val))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003236 return -EINVAL;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003237 if (copy_from_user(&val, optval, sizeof(val)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003238 return -EFAULT;
3239
3240 pkt_sk(sk)->copy_thresh = val;
3241 return 0;
3242 }
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003243 case PACKET_VERSION:
3244 {
3245 int val;
3246
3247 if (optlen != sizeof(val))
3248 return -EINVAL;
Johann Baudy69e3c752009-05-18 22:11:22 -07003249 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003250 return -EBUSY;
3251 if (copy_from_user(&val, optval, sizeof(val)))
3252 return -EFAULT;
3253 switch (val) {
3254 case TPACKET_V1:
3255 case TPACKET_V2:
chetan lokef6fb8f12011-08-19 10:18:16 +00003256 case TPACKET_V3:
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003257 po->tp_version = val;
3258 return 0;
3259 default:
3260 return -EINVAL;
3261 }
3262 }
Patrick McHardy8913336a2008-07-18 18:05:19 -07003263 case PACKET_RESERVE:
3264 {
3265 unsigned int val;
3266
3267 if (optlen != sizeof(val))
3268 return -EINVAL;
Johann Baudy69e3c752009-05-18 22:11:22 -07003269 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
Patrick McHardy8913336a2008-07-18 18:05:19 -07003270 return -EBUSY;
3271 if (copy_from_user(&val, optval, sizeof(val)))
3272 return -EFAULT;
3273 po->tp_reserve = val;
3274 return 0;
3275 }
Johann Baudy69e3c752009-05-18 22:11:22 -07003276 case PACKET_LOSS:
3277 {
3278 unsigned int val;
3279
3280 if (optlen != sizeof(val))
3281 return -EINVAL;
3282 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3283 return -EBUSY;
3284 if (copy_from_user(&val, optval, sizeof(val)))
3285 return -EFAULT;
3286 po->tp_loss = !!val;
3287 return 0;
3288 }
Herbert Xu8dc41942007-02-04 23:31:32 -08003289 case PACKET_AUXDATA:
3290 {
3291 int val;
3292
3293 if (optlen < sizeof(val))
3294 return -EINVAL;
3295 if (copy_from_user(&val, optval, sizeof(val)))
3296 return -EFAULT;
3297
3298 po->auxdata = !!val;
3299 return 0;
3300 }
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07003301 case PACKET_ORIGDEV:
3302 {
3303 int val;
3304
3305 if (optlen < sizeof(val))
3306 return -EINVAL;
3307 if (copy_from_user(&val, optval, sizeof(val)))
3308 return -EFAULT;
3309
3310 po->origdev = !!val;
3311 return 0;
3312 }
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003313 case PACKET_VNET_HDR:
3314 {
3315 int val;
3316
3317 if (sock->type != SOCK_RAW)
3318 return -EINVAL;
3319 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3320 return -EBUSY;
3321 if (optlen < sizeof(val))
3322 return -EINVAL;
3323 if (copy_from_user(&val, optval, sizeof(val)))
3324 return -EFAULT;
3325
3326 po->has_vnet_hdr = !!val;
3327 return 0;
3328 }
Scott McMillan614f60f2010-06-02 05:53:56 -07003329 case PACKET_TIMESTAMP:
3330 {
3331 int val;
3332
3333 if (optlen != sizeof(val))
3334 return -EINVAL;
3335 if (copy_from_user(&val, optval, sizeof(val)))
3336 return -EFAULT;
3337
3338 po->tp_tstamp = val;
3339 return 0;
3340 }
David S. Millerdc99f602011-07-05 01:45:05 -07003341 case PACKET_FANOUT:
3342 {
3343 int val;
3344
3345 if (optlen != sizeof(val))
3346 return -EINVAL;
3347 if (copy_from_user(&val, optval, sizeof(val)))
3348 return -EFAULT;
3349
3350 return fanout_add(sk, val & 0xffff, val >> 16);
3351 }
Paul Chavent5920cd3a2012-11-06 23:10:47 +00003352 case PACKET_TX_HAS_OFF:
3353 {
3354 unsigned int val;
3355
3356 if (optlen != sizeof(val))
3357 return -EINVAL;
3358 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3359 return -EBUSY;
3360 if (copy_from_user(&val, optval, sizeof(val)))
3361 return -EFAULT;
3362 po->tp_tx_has_off = !!val;
3363 return 0;
3364 }
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01003365 case PACKET_QDISC_BYPASS:
3366 {
3367 int val;
3368
3369 if (optlen != sizeof(val))
3370 return -EINVAL;
3371 if (copy_from_user(&val, optval, sizeof(val)))
3372 return -EFAULT;
3373
3374 po->xmit = val ? packet_direct_xmit : dev_queue_xmit;
3375 return 0;
3376 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003377 default:
3378 return -ENOPROTOOPT;
3379 }
3380}
3381
3382static int packet_getsockopt(struct socket *sock, int level, int optname,
3383 char __user *optval, int __user *optlen)
3384{
3385 int len;
Eric Dumazetc06fff62012-04-19 21:56:11 +00003386 int val, lv = sizeof(val);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003387 struct sock *sk = sock->sk;
3388 struct packet_sock *po = pkt_sk(sk);
Eric Dumazetc06fff62012-04-19 21:56:11 +00003389 void *data = &val;
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00003390 union tpacket_stats_u st;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003391
3392 if (level != SOL_PACKET)
3393 return -ENOPROTOOPT;
3394
Kris Katterjohn8ae55f02006-01-23 16:28:02 -08003395 if (get_user(len, optlen))
3396 return -EFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003397
3398 if (len < 0)
3399 return -EINVAL;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003400
Johann Baudy69e3c752009-05-18 22:11:22 -07003401 switch (optname) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003402 case PACKET_STATISTICS:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003403 spin_lock_bh(&sk->sk_receive_queue.lock);
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00003404 memcpy(&st, &po->stats, sizeof(st));
3405 memset(&po->stats, 0, sizeof(po->stats));
3406 spin_unlock_bh(&sk->sk_receive_queue.lock);
3407
chetan lokef6fb8f12011-08-19 10:18:16 +00003408 if (po->tp_version == TPACKET_V3) {
Eric Dumazetc06fff62012-04-19 21:56:11 +00003409 lv = sizeof(struct tpacket_stats_v3);
Willem de Bruijn8bcdeaf2013-08-19 16:40:22 -04003410 st.stats3.tp_packets += st.stats3.tp_drops;
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00003411 data = &st.stats3;
chetan lokef6fb8f12011-08-19 10:18:16 +00003412 } else {
Eric Dumazetc06fff62012-04-19 21:56:11 +00003413 lv = sizeof(struct tpacket_stats);
Willem de Bruijn8bcdeaf2013-08-19 16:40:22 -04003414 st.stats1.tp_packets += st.stats1.tp_drops;
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00003415 data = &st.stats1;
chetan lokef6fb8f12011-08-19 10:18:16 +00003416 }
Daniel Borkmannee80fbf2013-04-19 06:12:29 +00003417
Linus Torvalds1da177e2005-04-16 15:20:36 -07003418 break;
Herbert Xu8dc41942007-02-04 23:31:32 -08003419 case PACKET_AUXDATA:
Herbert Xu8dc41942007-02-04 23:31:32 -08003420 val = po->auxdata;
Herbert Xu8dc41942007-02-04 23:31:32 -08003421 break;
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07003422 case PACKET_ORIGDEV:
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07003423 val = po->origdev;
Peter P. Waskiewicz Jr80feaac2007-04-20 16:05:39 -07003424 break;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003425 case PACKET_VNET_HDR:
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003426 val = po->has_vnet_hdr;
Sridhar Samudralabfd5f4a2010-02-04 20:24:10 -08003427 break;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003428 case PACKET_VERSION:
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003429 val = po->tp_version;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003430 break;
3431 case PACKET_HDRLEN:
3432 if (len > sizeof(int))
3433 len = sizeof(int);
3434 if (copy_from_user(&val, optval, len))
3435 return -EFAULT;
3436 switch (val) {
3437 case TPACKET_V1:
3438 val = sizeof(struct tpacket_hdr);
3439 break;
3440 case TPACKET_V2:
3441 val = sizeof(struct tpacket2_hdr);
3442 break;
chetan lokef6fb8f12011-08-19 10:18:16 +00003443 case TPACKET_V3:
3444 val = sizeof(struct tpacket3_hdr);
3445 break;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003446 default:
3447 return -EINVAL;
3448 }
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003449 break;
Patrick McHardy8913336a2008-07-18 18:05:19 -07003450 case PACKET_RESERVE:
Patrick McHardy8913336a2008-07-18 18:05:19 -07003451 val = po->tp_reserve;
Patrick McHardy8913336a2008-07-18 18:05:19 -07003452 break;
Johann Baudy69e3c752009-05-18 22:11:22 -07003453 case PACKET_LOSS:
Johann Baudy69e3c752009-05-18 22:11:22 -07003454 val = po->tp_loss;
Johann Baudy69e3c752009-05-18 22:11:22 -07003455 break;
Scott McMillan614f60f2010-06-02 05:53:56 -07003456 case PACKET_TIMESTAMP:
Scott McMillan614f60f2010-06-02 05:53:56 -07003457 val = po->tp_tstamp;
Scott McMillan614f60f2010-06-02 05:53:56 -07003458 break;
David S. Millerdc99f602011-07-05 01:45:05 -07003459 case PACKET_FANOUT:
David S. Millerdc99f602011-07-05 01:45:05 -07003460 val = (po->fanout ?
3461 ((u32)po->fanout->id |
Willem de Bruijn77f65eb2013-03-19 10:18:11 +00003462 ((u32)po->fanout->type << 16) |
3463 ((u32)po->fanout->flags << 24)) :
David S. Millerdc99f602011-07-05 01:45:05 -07003464 0);
David S. Millerdc99f602011-07-05 01:45:05 -07003465 break;
Paul Chavent5920cd3a2012-11-06 23:10:47 +00003466 case PACKET_TX_HAS_OFF:
3467 val = po->tp_tx_has_off;
3468 break;
Daniel Borkmannd346a3f2013-12-06 11:36:17 +01003469 case PACKET_QDISC_BYPASS:
3470 val = packet_use_direct_xmit(po);
3471 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003472 default:
3473 return -ENOPROTOOPT;
3474 }
3475
Eric Dumazetc06fff62012-04-19 21:56:11 +00003476 if (len > lv)
3477 len = lv;
Kris Katterjohn8ae55f02006-01-23 16:28:02 -08003478 if (put_user(len, optlen))
3479 return -EFAULT;
Herbert Xu8dc41942007-02-04 23:31:32 -08003480 if (copy_to_user(optval, data, len))
3481 return -EFAULT;
Kris Katterjohn8ae55f02006-01-23 16:28:02 -08003482 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003483}
3484
3485
Jiri Pirko351638e2013-05-28 01:30:21 +00003486static int packet_notifier(struct notifier_block *this,
3487 unsigned long msg, void *ptr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003488{
3489 struct sock *sk;
Jiri Pirko351638e2013-05-28 01:30:21 +00003490 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
YOSHIFUJI Hideakic346dca2008-03-25 21:47:49 +09003491 struct net *net = dev_net(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003492
stephen hemminger808f5112010-02-22 07:57:18 +00003493 rcu_read_lock();
Sasha Levinb67bfe02013-02-27 17:06:00 -08003494 sk_for_each_rcu(sk, &net->packet.sklist) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003495 struct packet_sock *po = pkt_sk(sk);
3496
3497 switch (msg) {
3498 case NETDEV_UNREGISTER:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003499 if (po->mclist)
3500 packet_dev_mclist(dev, po->mclist, -1);
David S. Millera2efcfa2007-05-29 13:12:50 -07003501 /* fallthrough */
3502
Linus Torvalds1da177e2005-04-16 15:20:36 -07003503 case NETDEV_DOWN:
3504 if (dev->ifindex == po->ifindex) {
3505 spin_lock(&po->bind_lock);
3506 if (po->running) {
David S. Millerce06b032011-07-04 01:44:29 -07003507 __unregister_prot_hook(sk, false);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003508 sk->sk_err = ENETDOWN;
3509 if (!sock_flag(sk, SOCK_DEAD))
3510 sk->sk_error_report(sk);
3511 }
3512 if (msg == NETDEV_UNREGISTER) {
Daniel Borkmann66e56cd2013-12-06 11:36:15 +01003513 packet_cached_dev_reset(po);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003514 po->ifindex = -1;
Ben Greear160ff182011-06-01 07:18:52 +00003515 if (po->prot_hook.dev)
3516 dev_put(po->prot_hook.dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003517 po->prot_hook.dev = NULL;
3518 }
3519 spin_unlock(&po->bind_lock);
3520 }
3521 break;
3522 case NETDEV_UP:
stephen hemminger808f5112010-02-22 07:57:18 +00003523 if (dev->ifindex == po->ifindex) {
3524 spin_lock(&po->bind_lock);
David S. Millerce06b032011-07-04 01:44:29 -07003525 if (po->num)
3526 register_prot_hook(sk);
stephen hemminger808f5112010-02-22 07:57:18 +00003527 spin_unlock(&po->bind_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003528 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003529 break;
3530 }
3531 }
stephen hemminger808f5112010-02-22 07:57:18 +00003532 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003533 return NOTIFY_DONE;
3534}
3535
3536
3537static int packet_ioctl(struct socket *sock, unsigned int cmd,
3538 unsigned long arg)
3539{
3540 struct sock *sk = sock->sk;
3541
Johann Baudy69e3c752009-05-18 22:11:22 -07003542 switch (cmd) {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003543 case SIOCOUTQ:
3544 {
3545 int amount = sk_wmem_alloc_get(sk);
Eric Dumazet31e6d362009-06-17 19:05:41 -07003546
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003547 return put_user(amount, (int __user *)arg);
3548 }
3549 case SIOCINQ:
3550 {
3551 struct sk_buff *skb;
3552 int amount = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003553
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003554 spin_lock_bh(&sk->sk_receive_queue.lock);
3555 skb = skb_peek(&sk->sk_receive_queue);
3556 if (skb)
3557 amount = skb->len;
3558 spin_unlock_bh(&sk->sk_receive_queue.lock);
3559 return put_user(amount, (int __user *)arg);
3560 }
3561 case SIOCGSTAMP:
3562 return sock_get_timestamp(sk, (struct timeval __user *)arg);
3563 case SIOCGSTAMPNS:
3564 return sock_get_timestampns(sk, (struct timespec __user *)arg);
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003565
Linus Torvalds1da177e2005-04-16 15:20:36 -07003566#ifdef CONFIG_INET
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003567 case SIOCADDRT:
3568 case SIOCDELRT:
3569 case SIOCDARP:
3570 case SIOCGARP:
3571 case SIOCSARP:
3572 case SIOCGIFADDR:
3573 case SIOCSIFADDR:
3574 case SIOCGIFBRDADDR:
3575 case SIOCSIFBRDADDR:
3576 case SIOCGIFNETMASK:
3577 case SIOCSIFNETMASK:
3578 case SIOCGIFDSTADDR:
3579 case SIOCSIFDSTADDR:
3580 case SIOCSIFFLAGS:
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003581 return inet_dgram_ops.ioctl(sock, cmd, arg);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003582#endif
3583
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003584 default:
3585 return -ENOIOCTLCMD;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003586 }
3587 return 0;
3588}
3589
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003590static unsigned int packet_poll(struct file *file, struct socket *sock,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003591 poll_table *wait)
3592{
3593 struct sock *sk = sock->sk;
3594 struct packet_sock *po = pkt_sk(sk);
3595 unsigned int mask = datagram_poll(file, sock, wait);
3596
3597 spin_lock_bh(&sk->sk_receive_queue.lock);
Johann Baudy69e3c752009-05-18 22:11:22 -07003598 if (po->rx_ring.pg_vec) {
chetan lokef6fb8f12011-08-19 10:18:16 +00003599 if (!packet_previous_rx_frame(po, &po->rx_ring,
3600 TP_STATUS_KERNEL))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003601 mask |= POLLIN | POLLRDNORM;
3602 }
3603 spin_unlock_bh(&sk->sk_receive_queue.lock);
Johann Baudy69e3c752009-05-18 22:11:22 -07003604 spin_lock_bh(&sk->sk_write_queue.lock);
3605 if (po->tx_ring.pg_vec) {
3606 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
3607 mask |= POLLOUT | POLLWRNORM;
3608 }
3609 spin_unlock_bh(&sk->sk_write_queue.lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003610 return mask;
3611}
3612
3613
3614/* Dirty? Well, I still did not learn better way to account
3615 * for user mmaps.
3616 */
3617
3618static void packet_mm_open(struct vm_area_struct *vma)
3619{
3620 struct file *file = vma->vm_file;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003621 struct socket *sock = file->private_data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003622 struct sock *sk = sock->sk;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003623
Linus Torvalds1da177e2005-04-16 15:20:36 -07003624 if (sk)
3625 atomic_inc(&pkt_sk(sk)->mapped);
3626}
3627
3628static void packet_mm_close(struct vm_area_struct *vma)
3629{
3630 struct file *file = vma->vm_file;
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003631 struct socket *sock = file->private_data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003632 struct sock *sk = sock->sk;
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003633
Linus Torvalds1da177e2005-04-16 15:20:36 -07003634 if (sk)
3635 atomic_dec(&pkt_sk(sk)->mapped);
3636}
3637
Alexey Dobriyanf0f37e2f2009-09-27 22:29:37 +04003638static const struct vm_operations_struct packet_mmap_ops = {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003639 .open = packet_mm_open,
3640 .close = packet_mm_close,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003641};
3642
Neil Horman0e3125c2010-11-16 10:26:47 -08003643static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
3644 unsigned int len)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003645{
3646 int i;
3647
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003648 for (i = 0; i < len; i++) {
Neil Horman0e3125c2010-11-16 10:26:47 -08003649 if (likely(pg_vec[i].buffer)) {
Changli Gaoc56b4d92010-12-01 02:52:57 +00003650 if (is_vmalloc_addr(pg_vec[i].buffer))
Neil Horman0e3125c2010-11-16 10:26:47 -08003651 vfree(pg_vec[i].buffer);
3652 else
3653 free_pages((unsigned long)pg_vec[i].buffer,
3654 order);
3655 pg_vec[i].buffer = NULL;
3656 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003657 }
3658 kfree(pg_vec);
3659}
3660
Olof Johanssoneea49cc92011-11-02 11:00:49 +00003661static char *alloc_one_pg_vec_page(unsigned long order)
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003662{
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01003663 char *buffer;
Neil Horman0e3125c2010-11-16 10:26:47 -08003664 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
3665 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
Eric Dumazet719bfea2009-04-15 03:39:52 -07003666
Neil Horman0e3125c2010-11-16 10:26:47 -08003667 buffer = (char *) __get_free_pages(gfp_flags, order);
Neil Horman0e3125c2010-11-16 10:26:47 -08003668 if (buffer)
3669 return buffer;
3670
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01003671 /* __get_free_pages failed, fall back to vmalloc */
Eric Dumazetbbce5a52010-11-20 07:31:54 +00003672 buffer = vzalloc((1 << order) * PAGE_SIZE);
Neil Horman0e3125c2010-11-16 10:26:47 -08003673 if (buffer)
3674 return buffer;
3675
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01003676 /* vmalloc failed, lets dig into swap here */
Neil Horman0e3125c2010-11-16 10:26:47 -08003677 gfp_flags &= ~__GFP_NORETRY;
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01003678 buffer = (char *) __get_free_pages(gfp_flags, order);
Neil Horman0e3125c2010-11-16 10:26:47 -08003679 if (buffer)
3680 return buffer;
3681
Daniel Borkmannf0d4eb22014-01-19 11:46:53 +01003682 /* complete and utter failure */
Neil Horman0e3125c2010-11-16 10:26:47 -08003683 return NULL;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003684}
3685
Neil Horman0e3125c2010-11-16 10:26:47 -08003686static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003687{
3688 unsigned int block_nr = req->tp_block_nr;
Neil Horman0e3125c2010-11-16 10:26:47 -08003689 struct pgv *pg_vec;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003690 int i;
3691
Neil Horman0e3125c2010-11-16 10:26:47 -08003692 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003693 if (unlikely(!pg_vec))
3694 goto out;
3695
3696 for (i = 0; i < block_nr; i++) {
Changli Gaoc56b4d92010-12-01 02:52:57 +00003697 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
Neil Horman0e3125c2010-11-16 10:26:47 -08003698 if (unlikely(!pg_vec[i].buffer))
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003699 goto out_free_pgvec;
3700 }
3701
3702out:
3703 return pg_vec;
3704
3705out_free_pgvec:
3706 free_pg_vec(pg_vec, order, block_nr);
3707 pg_vec = NULL;
3708 goto out;
3709}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003710
chetan lokef6fb8f12011-08-19 10:18:16 +00003711static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
Johann Baudy69e3c752009-05-18 22:11:22 -07003712 int closing, int tx_ring)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003713{
Neil Horman0e3125c2010-11-16 10:26:47 -08003714 struct pgv *pg_vec = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003715 struct packet_sock *po = pkt_sk(sk);
Al Viro0e11c912006-11-08 00:26:29 -08003716 int was_running, order = 0;
Johann Baudy69e3c752009-05-18 22:11:22 -07003717 struct packet_ring_buffer *rb;
3718 struct sk_buff_head *rb_queue;
Al Viro0e11c912006-11-08 00:26:29 -08003719 __be16 num;
chetan lokef6fb8f12011-08-19 10:18:16 +00003720 int err = -EINVAL;
3721 /* Added to avoid minimal code churn */
3722 struct tpacket_req *req = &req_u->req;
3723
3724 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */
3725 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
3726 WARN(1, "Tx-ring is not supported.\n");
3727 goto out;
3728 }
Johann Baudy69e3c752009-05-18 22:11:22 -07003729
3730 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
3731 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
3732
3733 err = -EBUSY;
3734 if (!closing) {
3735 if (atomic_read(&po->mapped))
3736 goto out;
Daniel Borkmannb0138402014-01-15 16:25:36 +01003737 if (packet_read_pending(rb))
Johann Baudy69e3c752009-05-18 22:11:22 -07003738 goto out;
3739 }
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003740
Linus Torvalds1da177e2005-04-16 15:20:36 -07003741 if (req->tp_block_nr) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003742 /* Sanity tests and some calculations */
Johann Baudy69e3c752009-05-18 22:11:22 -07003743 err = -EBUSY;
3744 if (unlikely(rb->pg_vec))
3745 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003746
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003747 switch (po->tp_version) {
3748 case TPACKET_V1:
3749 po->tp_hdrlen = TPACKET_HDRLEN;
3750 break;
3751 case TPACKET_V2:
3752 po->tp_hdrlen = TPACKET2_HDRLEN;
3753 break;
chetan lokef6fb8f12011-08-19 10:18:16 +00003754 case TPACKET_V3:
3755 po->tp_hdrlen = TPACKET3_HDRLEN;
3756 break;
Patrick McHardybbd6ef82008-07-14 22:50:15 -07003757 }
3758
Johann Baudy69e3c752009-05-18 22:11:22 -07003759 err = -EINVAL;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003760 if (unlikely((int)req->tp_block_size <= 0))
Johann Baudy69e3c752009-05-18 22:11:22 -07003761 goto out;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003762 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
Johann Baudy69e3c752009-05-18 22:11:22 -07003763 goto out;
Patrick McHardy8913336a2008-07-18 18:05:19 -07003764 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
Johann Baudy69e3c752009-05-18 22:11:22 -07003765 po->tp_reserve))
3766 goto out;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003767 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
Johann Baudy69e3c752009-05-18 22:11:22 -07003768 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003769
Johann Baudy69e3c752009-05-18 22:11:22 -07003770 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
3771 if (unlikely(rb->frames_per_block <= 0))
3772 goto out;
3773 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
3774 req->tp_frame_nr))
3775 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003776
3777 err = -ENOMEM;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003778 order = get_order(req->tp_block_size);
3779 pg_vec = alloc_pg_vec(req, order);
3780 if (unlikely(!pg_vec))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003781 goto out;
chetan lokef6fb8f12011-08-19 10:18:16 +00003782 switch (po->tp_version) {
3783 case TPACKET_V3:
3784 /* Transmit path is not supported. We checked
3785 * it above but just being paranoid
3786 */
3787 if (!tx_ring)
3788 init_prb_bdqc(po, rb, pg_vec, req_u, tx_ring);
3789 break;
3790 default:
3791 break;
3792 }
Johann Baudy69e3c752009-05-18 22:11:22 -07003793 }
3794 /* Done */
3795 else {
3796 err = -EINVAL;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003797 if (unlikely(req->tp_frame_nr))
Johann Baudy69e3c752009-05-18 22:11:22 -07003798 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003799 }
3800
3801 lock_sock(sk);
3802
3803 /* Detach socket from network */
3804 spin_lock(&po->bind_lock);
3805 was_running = po->running;
3806 num = po->num;
3807 if (was_running) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003808 po->num = 0;
David S. Millerce06b032011-07-04 01:44:29 -07003809 __unregister_prot_hook(sk, false);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003810 }
3811 spin_unlock(&po->bind_lock);
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003812
Linus Torvalds1da177e2005-04-16 15:20:36 -07003813 synchronize_net();
3814
3815 err = -EBUSY;
Herbert Xu905db442009-01-30 14:12:06 -08003816 mutex_lock(&po->pg_vec_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003817 if (closing || atomic_read(&po->mapped) == 0) {
3818 err = 0;
Johann Baudy69e3c752009-05-18 22:11:22 -07003819 spin_lock_bh(&rb_queue->lock);
Changli Gaoc053fd92010-12-10 16:02:20 -08003820 swap(rb->pg_vec, pg_vec);
Johann Baudy69e3c752009-05-18 22:11:22 -07003821 rb->frame_max = (req->tp_frame_nr - 1);
3822 rb->head = 0;
3823 rb->frame_size = req->tp_frame_size;
3824 spin_unlock_bh(&rb_queue->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003825
Changli Gaoc053fd92010-12-10 16:02:20 -08003826 swap(rb->pg_vec_order, order);
3827 swap(rb->pg_vec_len, req->tp_block_nr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003828
Johann Baudy69e3c752009-05-18 22:11:22 -07003829 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
3830 po->prot_hook.func = (po->rx_ring.pg_vec) ?
3831 tpacket_rcv : packet_rcv;
3832 skb_queue_purge(rb_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003833 if (atomic_read(&po->mapped))
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003834 pr_err("packet_mmap: vma is busy: %d\n",
3835 atomic_read(&po->mapped));
Linus Torvalds1da177e2005-04-16 15:20:36 -07003836 }
Herbert Xu905db442009-01-30 14:12:06 -08003837 mutex_unlock(&po->pg_vec_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003838
3839 spin_lock(&po->bind_lock);
David S. Millerce06b032011-07-04 01:44:29 -07003840 if (was_running) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003841 po->num = num;
David S. Millerce06b032011-07-04 01:44:29 -07003842 register_prot_hook(sk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003843 }
3844 spin_unlock(&po->bind_lock);
chetan lokef6fb8f12011-08-19 10:18:16 +00003845 if (closing && (po->tp_version > TPACKET_V2)) {
3846 /* Because we don't support block-based V3 on tx-ring */
3847 if (!tx_ring)
3848 prb_shutdown_retire_blk_timer(po, tx_ring, rb_queue);
3849 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003850 release_sock(sk);
3851
Linus Torvalds1da177e2005-04-16 15:20:36 -07003852 if (pg_vec)
3853 free_pg_vec(pg_vec, order, req->tp_block_nr);
3854out:
3855 return err;
3856}
3857
Johann Baudy69e3c752009-05-18 22:11:22 -07003858static int packet_mmap(struct file *file, struct socket *sock,
3859 struct vm_area_struct *vma)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003860{
3861 struct sock *sk = sock->sk;
3862 struct packet_sock *po = pkt_sk(sk);
Johann Baudy69e3c752009-05-18 22:11:22 -07003863 unsigned long size, expected_size;
3864 struct packet_ring_buffer *rb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003865 unsigned long start;
3866 int err = -EINVAL;
3867 int i;
3868
3869 if (vma->vm_pgoff)
3870 return -EINVAL;
3871
Herbert Xu905db442009-01-30 14:12:06 -08003872 mutex_lock(&po->pg_vec_lock);
Johann Baudy69e3c752009-05-18 22:11:22 -07003873
3874 expected_size = 0;
3875 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3876 if (rb->pg_vec) {
3877 expected_size += rb->pg_vec_len
3878 * rb->pg_vec_pages
3879 * PAGE_SIZE;
3880 }
3881 }
3882
3883 if (expected_size == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003884 goto out;
Johann Baudy69e3c752009-05-18 22:11:22 -07003885
3886 size = vma->vm_end - vma->vm_start;
3887 if (size != expected_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003888 goto out;
3889
Linus Torvalds1da177e2005-04-16 15:20:36 -07003890 start = vma->vm_start;
Johann Baudy69e3c752009-05-18 22:11:22 -07003891 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3892 if (rb->pg_vec == NULL)
3893 continue;
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003894
Johann Baudy69e3c752009-05-18 22:11:22 -07003895 for (i = 0; i < rb->pg_vec_len; i++) {
Neil Horman0e3125c2010-11-16 10:26:47 -08003896 struct page *page;
3897 void *kaddr = rb->pg_vec[i].buffer;
Johann Baudy69e3c752009-05-18 22:11:22 -07003898 int pg_num;
3899
Changli Gaoc56b4d92010-12-01 02:52:57 +00003900 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
3901 page = pgv_to_page(kaddr);
Johann Baudy69e3c752009-05-18 22:11:22 -07003902 err = vm_insert_page(vma, start, page);
3903 if (unlikely(err))
3904 goto out;
3905 start += PAGE_SIZE;
Neil Horman0e3125c2010-11-16 10:26:47 -08003906 kaddr += PAGE_SIZE;
Johann Baudy69e3c752009-05-18 22:11:22 -07003907 }
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003908 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003909 }
Johann Baudy69e3c752009-05-18 22:11:22 -07003910
David S. Miller4ebf0ae2005-12-06 16:38:35 -08003911 atomic_inc(&po->mapped);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003912 vma->vm_ops = &packet_mmap_ops;
3913 err = 0;
3914
3915out:
Herbert Xu905db442009-01-30 14:12:06 -08003916 mutex_unlock(&po->pg_vec_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003917 return err;
3918}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003919
Eric Dumazet90ddc4f2005-12-22 12:49:22 -08003920static const struct proto_ops packet_ops_spkt = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003921 .family = PF_PACKET,
3922 .owner = THIS_MODULE,
3923 .release = packet_release,
3924 .bind = packet_bind_spkt,
3925 .connect = sock_no_connect,
3926 .socketpair = sock_no_socketpair,
3927 .accept = sock_no_accept,
3928 .getname = packet_getname_spkt,
3929 .poll = datagram_poll,
3930 .ioctl = packet_ioctl,
3931 .listen = sock_no_listen,
3932 .shutdown = sock_no_shutdown,
3933 .setsockopt = sock_no_setsockopt,
3934 .getsockopt = sock_no_getsockopt,
3935 .sendmsg = packet_sendmsg_spkt,
3936 .recvmsg = packet_recvmsg,
3937 .mmap = sock_no_mmap,
3938 .sendpage = sock_no_sendpage,
3939};
Linus Torvalds1da177e2005-04-16 15:20:36 -07003940
Eric Dumazet90ddc4f2005-12-22 12:49:22 -08003941static const struct proto_ops packet_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003942 .family = PF_PACKET,
3943 .owner = THIS_MODULE,
3944 .release = packet_release,
3945 .bind = packet_bind,
3946 .connect = sock_no_connect,
3947 .socketpair = sock_no_socketpair,
3948 .accept = sock_no_accept,
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003949 .getname = packet_getname,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003950 .poll = packet_poll,
3951 .ioctl = packet_ioctl,
3952 .listen = sock_no_listen,
3953 .shutdown = sock_no_shutdown,
3954 .setsockopt = packet_setsockopt,
3955 .getsockopt = packet_getsockopt,
3956 .sendmsg = packet_sendmsg,
3957 .recvmsg = packet_recvmsg,
3958 .mmap = packet_mmap,
3959 .sendpage = sock_no_sendpage,
3960};
3961
Stephen Hemmingerec1b4cf2009-10-05 05:58:39 +00003962static const struct net_proto_family packet_family_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003963 .family = PF_PACKET,
3964 .create = packet_create,
3965 .owner = THIS_MODULE,
3966};
3967
3968static struct notifier_block packet_netdev_notifier = {
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00003969 .notifier_call = packet_notifier,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003970};
3971
3972#ifdef CONFIG_PROC_FS
Linus Torvalds1da177e2005-04-16 15:20:36 -07003973
3974static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
stephen hemminger808f5112010-02-22 07:57:18 +00003975 __acquires(RCU)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003976{
Denis V. Luneve372c412007-11-19 22:31:54 -08003977 struct net *net = seq_file_net(seq);
stephen hemminger808f5112010-02-22 07:57:18 +00003978
3979 rcu_read_lock();
3980 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003981}
3982
3983static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3984{
Herbert Xu1bf40952007-12-16 14:04:02 -08003985 struct net *net = seq_file_net(seq);
stephen hemminger808f5112010-02-22 07:57:18 +00003986 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003987}
3988
3989static void packet_seq_stop(struct seq_file *seq, void *v)
stephen hemminger808f5112010-02-22 07:57:18 +00003990 __releases(RCU)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003991{
stephen hemminger808f5112010-02-22 07:57:18 +00003992 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003993}
3994
YOSHIFUJI Hideaki1ce4f282007-02-09 23:25:10 +09003995static int packet_seq_show(struct seq_file *seq, void *v)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003996{
3997 if (v == SEQ_START_TOKEN)
3998 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
3999 else {
Li Zefanb7ceabd2010-02-08 23:19:29 +00004000 struct sock *s = sk_entry(v);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004001 const struct packet_sock *po = pkt_sk(s);
4002
4003 seq_printf(seq,
Dan Rosenberg71338aa2011-05-23 12:17:35 +00004004 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07004005 s,
4006 atomic_read(&s->sk_refcnt),
4007 s->sk_type,
4008 ntohs(po->num),
4009 po->ifindex,
4010 po->running,
4011 atomic_read(&s->sk_rmem_alloc),
Eric W. Biedermana7cb5a42012-05-24 01:10:10 -06004012 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
Eric Dumazet40d4e3d2009-07-21 21:57:59 +00004013 sock_i_ino(s));
Linus Torvalds1da177e2005-04-16 15:20:36 -07004014 }
4015
4016 return 0;
4017}
4018
Philippe De Muyter56b3d972007-07-10 23:07:31 -07004019static const struct seq_operations packet_seq_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004020 .start = packet_seq_start,
4021 .next = packet_seq_next,
4022 .stop = packet_seq_stop,
4023 .show = packet_seq_show,
4024};
4025
4026static int packet_seq_open(struct inode *inode, struct file *file)
4027{
Denis V. Luneve372c412007-11-19 22:31:54 -08004028 return seq_open_net(inode, file, &packet_seq_ops,
4029 sizeof(struct seq_net_private));
Linus Torvalds1da177e2005-04-16 15:20:36 -07004030}
4031
Arjan van de Venda7071d2007-02-12 00:55:36 -08004032static const struct file_operations packet_seq_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004033 .owner = THIS_MODULE,
4034 .open = packet_seq_open,
4035 .read = seq_read,
4036 .llseek = seq_lseek,
Denis V. Luneve372c412007-11-19 22:31:54 -08004037 .release = seq_release_net,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004038};
4039
4040#endif
4041
Alexey Dobriyan2c8c1e72010-01-17 03:35:32 +00004042static int __net_init packet_net_init(struct net *net)
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004043{
Pavel Emelyanov0fa7fa92012-08-21 01:06:47 +00004044 mutex_init(&net->packet.sklist_lock);
Denis V. Lunev2aaef4e2007-12-11 04:19:54 -08004045 INIT_HLIST_HEAD(&net->packet.sklist);
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004046
Gao fengd4beaa62013-02-18 01:34:54 +00004047 if (!proc_create("packet", 0, net->proc_net, &packet_seq_fops))
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004048 return -ENOMEM;
4049
4050 return 0;
4051}
4052
Alexey Dobriyan2c8c1e72010-01-17 03:35:32 +00004053static void __net_exit packet_net_exit(struct net *net)
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004054{
Gao fengece31ff2013-02-18 01:34:56 +00004055 remove_proc_entry("packet", net->proc_net);
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004056}
4057
4058static struct pernet_operations packet_net_ops = {
4059 .init = packet_net_init,
4060 .exit = packet_net_exit,
4061};
4062
4063
Linus Torvalds1da177e2005-04-16 15:20:36 -07004064static void __exit packet_exit(void)
4065{
Linus Torvalds1da177e2005-04-16 15:20:36 -07004066 unregister_netdevice_notifier(&packet_netdev_notifier);
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004067 unregister_pernet_subsys(&packet_net_ops);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004068 sock_unregister(PF_PACKET);
4069 proto_unregister(&packet_proto);
4070}
4071
4072static int __init packet_init(void)
4073{
4074 int rc = proto_register(&packet_proto, 0);
4075
4076 if (rc != 0)
4077 goto out;
4078
4079 sock_register(&packet_family_ops);
Denis V. Lunevd12d01d2007-11-19 22:28:35 -08004080 register_pernet_subsys(&packet_net_ops);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004081 register_netdevice_notifier(&packet_netdev_notifier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004082out:
4083 return rc;
4084}
4085
4086module_init(packet_init);
4087module_exit(packet_exit);
4088MODULE_LICENSE("GPL");
4089MODULE_ALIAS_NETPROTO(PF_PACKET);