include/net/xdp_sock.h

/* SPDX-License-Identifier: GPL-2.0 */
/* AF_XDP internal functions
 * Copyright(c) 2018 Intel Corporation.
 */

#ifndef _LINUX_XDP_SOCK_H
#define _LINUX_XDP_SOCK_H

#include <linux/bpf.h>
#include <linux/workqueue.h>
#include <linux/if_xdp.h>
#include <linux/mutex.h>
#include <linux/spinlock.h>
#include <linux/mm.h>
#include <net/sock.h>

#define XDP_UMEM_SG_FLAG BIT(3)

struct net_device;
struct xsk_queue;
struct xdp_buff;

struct xdp_umem {
	void *addrs;
	u64 size;
	u32 headroom;
	u32 chunk_size;
	u32 chunks;
	u32 npgs;
	struct user_struct *user;
	refcount_t users;
	u8 flags;
	u8 tx_metadata_len;
	bool zc;
	struct page **pgs;
	int id;
	struct list_head xsk_dma_list;
	struct work_struct work;
};

struct xsk_map {
	struct bpf_map map;
	spinlock_t lock; /* Synchronize map updates */
	atomic_t count;
	struct xdp_sock __rcu *xsk_map[];
};

struct xdp_sock {
	/* struct sock must be the first member of struct xdp_sock */
	struct sock sk;
	struct xsk_queue *rx ____cacheline_aligned_in_smp;
	struct net_device *dev;
	struct xdp_umem *umem;
	struct list_head flush_node;
	struct xsk_buff_pool *pool;
	u16 queue_id;
	bool zc;
	bool sg;
	enum {
		XSK_READY = 0,
		XSK_BOUND,
		XSK_UNBOUND,
	} state;

	struct xsk_queue *tx ____cacheline_aligned_in_smp;
	struct list_head tx_list;
	/* record the number of tx descriptors sent by this xsk and
	 * when it exceeds MAX_PER_SOCKET_BUDGET, an opportunity needs
	 * to be given to other xsks for sending tx descriptors, thereby
	 * preventing other XSKs from being starved.
	 */
	u32 tx_budget_spent;

	/* Statistics */
	u64 rx_dropped;
	u64 rx_queue_full;

	/* When __xsk_generic_xmit() must return before it sees the EOP descriptor for the current
	 * packet, the partially built skb is saved here so that packet building can resume in next
	 * call of __xsk_generic_xmit().
	 */
	struct sk_buff *skb;

	struct list_head map_list;
	/* Protects map_list */
	spinlock_t map_list_lock;
	u32 max_tx_budget;
	/* Protects multiple processes in the control path */
	struct mutex mutex;
	struct xsk_queue *fq_tmp; /* Only as tmp storage before bind */
	struct xsk_queue *cq_tmp; /* Only as tmp storage before bind */
};

/*
 * AF_XDP TX metadata hooks for network devices.
 * The following hooks can be defined; unless noted otherwise, they are
 * optional and can be filled with a null pointer.
 *
 * void (*tmo_request_timestamp)(void *priv)
 *     Called when AF_XDP frame requested egress timestamp.
 *
 * u64 (*tmo_fill_timestamp)(void *priv)
 *     Called when AF_XDP frame, that had requested egress timestamp,
 *     received a completion. The hook needs to return the actual HW timestamp.
 *
 * void (*tmo_request_checksum)(u16 csum_start, u16 csum_offset, void *priv)
 *     Called when AF_XDP frame requested HW checksum offload. csum_start
 *     indicates position where checksumming should start.
 *     csum_offset indicates position where checksum should be stored.
 *
 * void (*tmo_request_launch_time)(u64 launch_time, void *priv)
 *     Called when AF_XDP frame requested launch time HW offload support.
 *     launch_time indicates the PTP time at which the device can schedule the
 *     packet for transmission.
 */
struct xsk_tx_metadata_ops {
	void	(*tmo_request_timestamp)(void *priv);
	u64	(*tmo_fill_timestamp)(void *priv);
	void	(*tmo_request_checksum)(u16 csum_start, u16 csum_offset, void *priv);
	void	(*tmo_request_launch_time)(u64 launch_time, void *priv);
};

#ifdef CONFIG_XDP_SOCKETS

int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp);
int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp);
void __xsk_map_flush(struct list_head *flush_list);
INDIRECT_CALLABLE_DECLARE(void xsk_destruct_skb(struct sk_buff *));

/**
 *  xsk_tx_metadata_to_compl - Save enough relevant metadata information
 *  to perform tx completion in the future.
 *  @meta: pointer to AF_XDP metadata area
 *  @compl: pointer to output struct xsk_tx_metadata_to_compl
 *
 *  This function should be called by the networking device when
 *  it prepares AF_XDP egress packet. The value of @compl should be stored
 *  and passed to xsk_tx_metadata_complete upon TX completion.
 */
static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta,
					    struct xsk_tx_metadata_compl *compl)
{
	if (!meta)
		return;

	if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP)
		compl->tx_timestamp = &meta->completion.tx_timestamp;
	else
		compl->tx_timestamp = NULL;
}

/**
 *  xsk_tx_metadata_request - Evaluate AF_XDP TX metadata at submission
 *  and call appropriate xsk_tx_metadata_ops operation.
 *  @meta: pointer to AF_XDP metadata area
 *  @ops: pointer to struct xsk_tx_metadata_ops
 *  @priv: pointer to driver-private aread
 *
 *  This function should be called by the networking device when
 *  it prepares AF_XDP egress packet.
 */
static inline void xsk_tx_metadata_request(const struct xsk_tx_metadata *meta,
					   const struct xsk_tx_metadata_ops *ops,
					   void *priv)
{
	if (!meta)
		return;

	if (ops->tmo_request_launch_time)
		if (meta->flags & XDP_TXMD_FLAGS_LAUNCH_TIME)
			ops->tmo_request_launch_time(meta->request.launch_time,
						     priv);

	if (ops->tmo_request_timestamp)
		if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP)
			ops->tmo_request_timestamp(priv);

	if (ops->tmo_request_checksum)
		if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM)
			ops->tmo_request_checksum(meta->request.csum_start,
						  meta->request.csum_offset, priv);
}

/**
 *  xsk_tx_metadata_complete - Evaluate AF_XDP TX metadata at completion
 *  and call appropriate xsk_tx_metadata_ops operation.
 *  @compl: pointer to completion metadata produced from xsk_tx_metadata_to_compl
 *  @ops: pointer to struct xsk_tx_metadata_ops
 *  @priv: pointer to driver-private aread
 *
 *  This function should be called by the networking device upon
 *  AF_XDP egress completion.
 */
static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl,
					    const struct xsk_tx_metadata_ops *ops,
					    void *priv)
{
	if (!compl)
		return;
	if (!compl->tx_timestamp)
		return;

	*compl->tx_timestamp = ops->tmo_fill_timestamp(priv);
}

#else

static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
{
	return -ENOTSUPP;
}

static inline int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp)
{
	return -EOPNOTSUPP;
}

static inline void __xsk_map_flush(struct list_head *flush_list)
{
}

#ifdef CONFIG_MITIGATION_RETPOLINE
static inline void xsk_destruct_skb(struct sk_buff *skb)
{
}
#endif

static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta,
					    struct xsk_tx_metadata_compl *compl)
{
}

static inline void xsk_tx_metadata_request(struct xsk_tx_metadata *meta,
					   const struct xsk_tx_metadata_ops *ops,
					   void *priv)
{
}

static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl,
					    const struct xsk_tx_metadata_ops *ops,
					    void *priv)
{
}

#endif /* CONFIG_XDP_SOCKETS */
#endif /* _LINUX_XDP_SOCK_H */