Björn Töpel | dac09149 | 2018-05-18 14:00:21 +0200 | [diff] [blame] | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
| 2 | /* AF_XDP internal functions |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 3 | * Copyright(c) 2018 Intel Corporation. |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 4 | */ |
| 5 | |
| 6 | #ifndef _LINUX_XDP_SOCK_H |
| 7 | #define _LINUX_XDP_SOCK_H |
| 8 | |
Björn Töpel | e61e62b9 | 2018-06-04 14:05:51 +0200 | [diff] [blame] | 9 | #include <linux/workqueue.h> |
| 10 | #include <linux/if_xdp.h> |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 11 | #include <linux/mutex.h> |
Magnus Karlsson | ac98d8a | 2018-06-04 14:05:57 +0200 | [diff] [blame] | 12 | #include <linux/spinlock.h> |
Björn Töpel | e61e62b9 | 2018-06-04 14:05:51 +0200 | [diff] [blame] | 13 | #include <linux/mm.h> |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 14 | #include <net/sock.h> |
| 15 | |
Björn Töpel | b9b6b68 | 2018-05-02 13:01:25 +0200 | [diff] [blame] | 16 | struct net_device; |
| 17 | struct xsk_queue; |
Magnus Karlsson | a71506a | 2020-05-20 21:20:51 +0200 | [diff] [blame] | 18 | struct xdp_buff; |
Björn Töpel | e61e62b9 | 2018-06-04 14:05:51 +0200 | [diff] [blame] | 19 | |
Björn Töpel | e61e62b9 | 2018-06-04 14:05:51 +0200 | [diff] [blame] | 20 | struct xdp_umem { |
Magnus Karlsson | 7f7ffa4 | 2020-08-28 10:26:21 +0200 | [diff] [blame] | 21 | void *addrs; |
Magnus Karlsson | 93ee30f | 2018-08-31 13:40:02 +0200 | [diff] [blame] | 22 | u64 size; |
Björn Töpel | e61e62b9 | 2018-06-04 14:05:51 +0200 | [diff] [blame] | 23 | u32 headroom; |
Björn Töpel | 2b43470a | 2020-05-20 21:20:53 +0200 | [diff] [blame] | 24 | u32 chunk_size; |
Magnus Karlsson | 1c1efc2 | 2020-08-28 10:26:17 +0200 | [diff] [blame] | 25 | u32 chunks; |
Magnus Karlsson | 8ef4e27 | 2020-08-28 10:26:23 +0200 | [diff] [blame] | 26 | u32 npgs; |
Björn Töpel | e61e62b9 | 2018-06-04 14:05:51 +0200 | [diff] [blame] | 27 | struct user_struct *user; |
Björn Töpel | e61e62b9 | 2018-06-04 14:05:51 +0200 | [diff] [blame] | 28 | refcount_t users; |
Magnus Karlsson | 77cd0d7 | 2019-08-14 09:27:17 +0200 | [diff] [blame] | 29 | u8 flags; |
Björn Töpel | 173d3ad | 2018-06-04 14:05:55 +0200 | [diff] [blame] | 30 | bool zc; |
Magnus Karlsson | 8ef4e27 | 2020-08-28 10:26:23 +0200 | [diff] [blame] | 31 | struct page **pgs; |
| 32 | int id; |
Magnus Karlsson | 921b686 | 2020-08-28 10:26:22 +0200 | [diff] [blame] | 33 | struct list_head xsk_dma_list; |
Magnus Karlsson | 537cf4e | 2020-11-20 12:53:39 +0100 | [diff] [blame] | 34 | struct work_struct work; |
Björn Töpel | e61e62b9 | 2018-06-04 14:05:51 +0200 | [diff] [blame] | 35 | }; |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 36 | |
Björn Töpel | d817991 | 2019-11-01 12:03:46 +0100 | [diff] [blame] | 37 | struct xsk_map { |
| 38 | struct bpf_map map; |
Björn Töpel | d817991 | 2019-11-01 12:03:46 +0100 | [diff] [blame] | 39 | spinlock_t lock; /* Synchronize map updates */ |
Toke Høiland-Jørgensen | 782347b | 2021-06-24 18:05:55 +0200 | [diff] [blame] | 40 | struct xdp_sock __rcu *xsk_map[]; |
Björn Töpel | d817991 | 2019-11-01 12:03:46 +0100 | [diff] [blame] | 41 | }; |
| 42 | |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 43 | struct xdp_sock { |
| 44 | /* struct sock must be the first member of struct xdp_sock */ |
| 45 | struct sock sk; |
Magnus Karlsson | 8ef4e27 | 2020-08-28 10:26:23 +0200 | [diff] [blame] | 46 | struct xsk_queue *rx ____cacheline_aligned_in_smp; |
Björn Töpel | b9b6b68 | 2018-05-02 13:01:25 +0200 | [diff] [blame] | 47 | struct net_device *dev; |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 48 | struct xdp_umem *umem; |
Björn Töpel | fbfc504a | 2018-05-02 13:01:28 +0200 | [diff] [blame] | 49 | struct list_head flush_node; |
Magnus Karlsson | c465576 | 2020-08-28 10:26:16 +0200 | [diff] [blame] | 50 | struct xsk_buff_pool *pool; |
Magnus Karlsson | 965a9909 | 2018-05-02 13:01:26 +0200 | [diff] [blame] | 51 | u16 queue_id; |
Magnus Karlsson | ac98d8a | 2018-06-04 14:05:57 +0200 | [diff] [blame] | 52 | bool zc; |
Ilya Maximets | 455302d | 2019-06-28 11:04:07 +0300 | [diff] [blame] | 53 | enum { |
| 54 | XSK_READY = 0, |
| 55 | XSK_BOUND, |
| 56 | XSK_UNBOUND, |
| 57 | } state; |
Magnus Karlsson | 8ef4e27 | 2020-08-28 10:26:23 +0200 | [diff] [blame] | 58 | |
Jonathan Lemon | fada7fd | 2019-06-06 13:59:40 -0700 | [diff] [blame] | 59 | struct xsk_queue *tx ____cacheline_aligned_in_smp; |
Magnus Karlsson | a5aa8e5 | 2020-08-28 10:26:20 +0200 | [diff] [blame] | 60 | struct list_head tx_list; |
Ilya Maximets | bf0bdd1 | 2019-07-03 15:09:16 +0300 | [diff] [blame] | 61 | /* Protects generic receive. */ |
| 62 | spinlock_t rx_lock; |
Ciara Loftus | 8aa5a33 | 2020-07-08 07:28:33 +0000 | [diff] [blame] | 63 | |
| 64 | /* Statistics */ |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 65 | u64 rx_dropped; |
Ciara Loftus | 8aa5a33 | 2020-07-08 07:28:33 +0000 | [diff] [blame] | 66 | u64 rx_queue_full; |
| 67 | |
Björn Töpel | 0402acd | 2019-08-15 11:30:13 +0200 | [diff] [blame] | 68 | struct list_head map_list; |
| 69 | /* Protects map_list */ |
| 70 | spinlock_t map_list_lock; |
Magnus Karlsson | 8ef4e27 | 2020-08-28 10:26:23 +0200 | [diff] [blame] | 71 | /* Protects multiple processes in the control path */ |
| 72 | struct mutex mutex; |
Magnus Karlsson | 7361f9c | 2020-08-28 10:26:18 +0200 | [diff] [blame] | 73 | struct xsk_queue *fq_tmp; /* Only as tmp storage before bind */ |
| 74 | struct xsk_queue *cq_tmp; /* Only as tmp storage before bind */ |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 75 | }; |
| 76 | |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 77 | #ifdef CONFIG_XDP_SOCKETS |
Björn Töpel | 9025403 | 2018-08-28 14:44:27 +0200 | [diff] [blame] | 78 | |
Magnus Karlsson | a71506a | 2020-05-20 21:20:51 +0200 | [diff] [blame] | 79 | int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp); |
Björn Töpel | e312b9e | 2019-12-19 07:10:02 +0100 | [diff] [blame] | 80 | int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp); |
| 81 | void __xsk_map_flush(void); |
Björn Töpel | d817991 | 2019-11-01 12:03:46 +0100 | [diff] [blame] | 82 | |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 83 | #else |
Magnus Karlsson | a71506a | 2020-05-20 21:20:51 +0200 | [diff] [blame] | 84 | |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 85 | static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) |
| 86 | { |
| 87 | return -ENOTSUPP; |
| 88 | } |
| 89 | |
Magnus Karlsson | a71506a | 2020-05-20 21:20:51 +0200 | [diff] [blame] | 90 | static inline int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp) |
Maxim Mikityanskiy | d57d7642 | 2019-06-26 17:35:24 +0300 | [diff] [blame] | 91 | { |
Magnus Karlsson | a71506a | 2020-05-20 21:20:51 +0200 | [diff] [blame] | 92 | return -EOPNOTSUPP; |
Maxim Mikityanskiy | d57d7642 | 2019-06-26 17:35:24 +0300 | [diff] [blame] | 93 | } |
| 94 | |
Magnus Karlsson | a71506a | 2020-05-20 21:20:51 +0200 | [diff] [blame] | 95 | static inline void __xsk_map_flush(void) |
Björn Töpel | 9025403 | 2018-08-28 14:44:27 +0200 | [diff] [blame] | 96 | { |
| 97 | } |
| 98 | |
Björn Töpel | c497176 | 2018-05-02 13:01:27 +0200 | [diff] [blame] | 99 | #endif /* CONFIG_XDP_SOCKETS */ |
| 100 | |
Björn Töpel | c0c77d8 | 2018-05-02 13:01:23 +0200 | [diff] [blame] | 101 | #endif /* _LINUX_XDP_SOCK_H */ |