blob: fff069d2ed1bc0f025a6ac56a5d2e992143d3297 [file] [log] [blame]
Björn Töpeldac091492018-05-18 14:00:21 +02001/* SPDX-License-Identifier: GPL-2.0 */
2/* AF_XDP internal functions
Björn Töpelc0c77d82018-05-02 13:01:23 +02003 * Copyright(c) 2018 Intel Corporation.
Björn Töpelc0c77d82018-05-02 13:01:23 +02004 */
5
6#ifndef _LINUX_XDP_SOCK_H
7#define _LINUX_XDP_SOCK_H
8
Björn Töpele61e62b92018-06-04 14:05:51 +02009#include <linux/workqueue.h>
10#include <linux/if_xdp.h>
Björn Töpelc0c77d82018-05-02 13:01:23 +020011#include <linux/mutex.h>
Magnus Karlssonac98d8a2018-06-04 14:05:57 +020012#include <linux/spinlock.h>
Björn Töpele61e62b92018-06-04 14:05:51 +020013#include <linux/mm.h>
Björn Töpelc0c77d82018-05-02 13:01:23 +020014#include <net/sock.h>
15
Björn Töpelb9b6b682018-05-02 13:01:25 +020016struct net_device;
17struct xsk_queue;
Magnus Karlssona71506a2020-05-20 21:20:51 +020018struct xdp_buff;
Björn Töpele61e62b92018-06-04 14:05:51 +020019
Björn Töpele61e62b92018-06-04 14:05:51 +020020struct xdp_umem {
Magnus Karlsson7f7ffa42020-08-28 10:26:21 +020021 void *addrs;
Magnus Karlsson93ee30f2018-08-31 13:40:02 +020022 u64 size;
Björn Töpele61e62b92018-06-04 14:05:51 +020023 u32 headroom;
Björn Töpel2b43470a2020-05-20 21:20:53 +020024 u32 chunk_size;
Magnus Karlsson1c1efc22020-08-28 10:26:17 +020025 u32 chunks;
Magnus Karlsson8ef4e272020-08-28 10:26:23 +020026 u32 npgs;
Björn Töpele61e62b92018-06-04 14:05:51 +020027 struct user_struct *user;
Björn Töpele61e62b92018-06-04 14:05:51 +020028 refcount_t users;
Magnus Karlsson77cd0d72019-08-14 09:27:17 +020029 u8 flags;
Björn Töpel173d3ad2018-06-04 14:05:55 +020030 bool zc;
Magnus Karlsson8ef4e272020-08-28 10:26:23 +020031 struct page **pgs;
32 int id;
Magnus Karlsson921b6862020-08-28 10:26:22 +020033 struct list_head xsk_dma_list;
Magnus Karlsson537cf4e2020-11-20 12:53:39 +010034 struct work_struct work;
Björn Töpele61e62b92018-06-04 14:05:51 +020035};
Björn Töpelc0c77d82018-05-02 13:01:23 +020036
Björn Töpeld8179912019-11-01 12:03:46 +010037struct xsk_map {
38 struct bpf_map map;
Björn Töpeld8179912019-11-01 12:03:46 +010039 spinlock_t lock; /* Synchronize map updates */
Toke Høiland-Jørgensen782347b2021-06-24 18:05:55 +020040 struct xdp_sock __rcu *xsk_map[];
Björn Töpeld8179912019-11-01 12:03:46 +010041};
42
Björn Töpelc0c77d82018-05-02 13:01:23 +020043struct xdp_sock {
44 /* struct sock must be the first member of struct xdp_sock */
45 struct sock sk;
Magnus Karlsson8ef4e272020-08-28 10:26:23 +020046 struct xsk_queue *rx ____cacheline_aligned_in_smp;
Björn Töpelb9b6b682018-05-02 13:01:25 +020047 struct net_device *dev;
Björn Töpelc0c77d82018-05-02 13:01:23 +020048 struct xdp_umem *umem;
Björn Töpelfbfc504a2018-05-02 13:01:28 +020049 struct list_head flush_node;
Magnus Karlssonc4655762020-08-28 10:26:16 +020050 struct xsk_buff_pool *pool;
Magnus Karlsson965a99092018-05-02 13:01:26 +020051 u16 queue_id;
Magnus Karlssonac98d8a2018-06-04 14:05:57 +020052 bool zc;
Ilya Maximets455302d2019-06-28 11:04:07 +030053 enum {
54 XSK_READY = 0,
55 XSK_BOUND,
56 XSK_UNBOUND,
57 } state;
Magnus Karlsson8ef4e272020-08-28 10:26:23 +020058
Jonathan Lemonfada7fd2019-06-06 13:59:40 -070059 struct xsk_queue *tx ____cacheline_aligned_in_smp;
Magnus Karlssona5aa8e52020-08-28 10:26:20 +020060 struct list_head tx_list;
Ilya Maximetsbf0bdd12019-07-03 15:09:16 +030061 /* Protects generic receive. */
62 spinlock_t rx_lock;
Ciara Loftus8aa5a332020-07-08 07:28:33 +000063
64 /* Statistics */
Björn Töpelc4971762018-05-02 13:01:27 +020065 u64 rx_dropped;
Ciara Loftus8aa5a332020-07-08 07:28:33 +000066 u64 rx_queue_full;
67
Björn Töpel0402acd2019-08-15 11:30:13 +020068 struct list_head map_list;
69 /* Protects map_list */
70 spinlock_t map_list_lock;
Magnus Karlsson8ef4e272020-08-28 10:26:23 +020071 /* Protects multiple processes in the control path */
72 struct mutex mutex;
Magnus Karlsson7361f9c2020-08-28 10:26:18 +020073 struct xsk_queue *fq_tmp; /* Only as tmp storage before bind */
74 struct xsk_queue *cq_tmp; /* Only as tmp storage before bind */
Björn Töpelc0c77d82018-05-02 13:01:23 +020075};
76
Björn Töpelc4971762018-05-02 13:01:27 +020077#ifdef CONFIG_XDP_SOCKETS
Björn Töpel90254032018-08-28 14:44:27 +020078
Magnus Karlssona71506a2020-05-20 21:20:51 +020079int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp);
Björn Töpele312b9e2019-12-19 07:10:02 +010080int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp);
81void __xsk_map_flush(void);
Björn Töpeld8179912019-11-01 12:03:46 +010082
Björn Töpelc4971762018-05-02 13:01:27 +020083#else
Magnus Karlssona71506a2020-05-20 21:20:51 +020084
Björn Töpelc4971762018-05-02 13:01:27 +020085static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
86{
87 return -ENOTSUPP;
88}
89
Magnus Karlssona71506a2020-05-20 21:20:51 +020090static inline int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp)
Maxim Mikityanskiyd57d76422019-06-26 17:35:24 +030091{
Magnus Karlssona71506a2020-05-20 21:20:51 +020092 return -EOPNOTSUPP;
Maxim Mikityanskiyd57d76422019-06-26 17:35:24 +030093}
94
Magnus Karlssona71506a2020-05-20 21:20:51 +020095static inline void __xsk_map_flush(void)
Björn Töpel90254032018-08-28 14:44:27 +020096{
97}
98
Björn Töpelc4971762018-05-02 13:01:27 +020099#endif /* CONFIG_XDP_SOCKETS */
100
Björn Töpelc0c77d82018-05-02 13:01:23 +0200101#endif /* _LINUX_XDP_SOCK_H */