| // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB |
| /* Copyright (c) 2019 Mellanox Technologies. */ |
| |
| #include "tx.h" |
| #include "pool.h" |
| #include "en/xdp.h" |
| #include "en/params.h" |
| #include <net/xdp_sock_drv.h> |
| |
| int mlx5e_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags) |
| { |
| struct mlx5e_priv *priv = netdev_priv(dev); |
| struct mlx5e_params *params = &priv->channels.params; |
| struct mlx5e_channel *c; |
| u16 ix; |
| |
| if (unlikely(!mlx5e_xdp_is_active(priv))) |
| return -ENETDOWN; |
| |
| if (unlikely(!mlx5e_qid_get_ch_if_in_group(params, qid, MLX5E_RQ_GROUP_XSK, &ix))) |
| return -EINVAL; |
| |
| c = priv->channels.c[ix]; |
| |
| if (unlikely(!test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))) |
| return -ENXIO; |
| |
| if (!napi_if_scheduled_mark_missed(&c->napi)) { |
| /* To avoid WQE overrun, don't post a NOP if async_icosq is not |
| * active and not polled by NAPI. Return 0, because the upcoming |
| * activate will trigger the IRQ for us. |
| */ |
| if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &c->async_icosq.state))) |
| return 0; |
| |
| if (test_and_set_bit(MLX5E_SQ_STATE_PENDING_XSK_TX, &c->async_icosq.state)) |
| return 0; |
| |
| spin_lock_bh(&c->async_icosq_lock); |
| mlx5e_trigger_irq(&c->async_icosq); |
| spin_unlock_bh(&c->async_icosq_lock); |
| } |
| |
| return 0; |
| } |
| |
| /* When TX fails (because of the size of the packet), we need to get completions |
| * in order, so post a NOP to get a CQE. Since AF_XDP doesn't distinguish |
| * between successful TX and errors, handling in mlx5e_poll_xdpsq_cq is the |
| * same. |
| */ |
| static void mlx5e_xsk_tx_post_err(struct mlx5e_xdpsq *sq, |
| struct mlx5e_xdp_info *xdpi) |
| { |
| u16 pi = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->pc); |
| struct mlx5e_xdp_wqe_info *wi = &sq->db.wqe_info[pi]; |
| struct mlx5e_tx_wqe *nopwqe; |
| |
| wi->num_wqebbs = 1; |
| wi->num_pkts = 1; |
| |
| nopwqe = mlx5e_post_nop(&sq->wq, sq->sqn, &sq->pc); |
| mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo, xdpi); |
| sq->doorbell_cseg = &nopwqe->ctrl; |
| } |
| |
| bool mlx5e_xsk_tx(struct mlx5e_xdpsq *sq, unsigned int budget) |
| { |
| struct xsk_buff_pool *pool = sq->xsk_pool; |
| struct mlx5e_xmit_data xdptxd; |
| struct mlx5e_xdp_info xdpi; |
| bool work_done = true; |
| bool flush = false; |
| |
| xdpi.mode = MLX5E_XDP_XMIT_MODE_XSK; |
| |
| for (; budget; budget--) { |
| int check_result = INDIRECT_CALL_2(sq->xmit_xdp_frame_check, |
| mlx5e_xmit_xdp_frame_check_mpwqe, |
| mlx5e_xmit_xdp_frame_check, |
| sq); |
| struct xdp_desc desc; |
| bool ret; |
| |
| if (unlikely(check_result < 0)) { |
| work_done = false; |
| break; |
| } |
| |
| if (!xsk_tx_peek_desc(pool, &desc)) { |
| /* TX will get stuck until something wakes it up by |
| * triggering NAPI. Currently it's expected that the |
| * application calls sendto() if there are consumed, but |
| * not completed frames. |
| */ |
| break; |
| } |
| |
| xdptxd.dma_addr = xsk_buff_raw_get_dma(pool, desc.addr); |
| xdptxd.data = xsk_buff_raw_get_data(pool, desc.addr); |
| xdptxd.len = desc.len; |
| |
| xsk_buff_raw_dma_sync_for_device(pool, xdptxd.dma_addr, xdptxd.len); |
| |
| ret = INDIRECT_CALL_2(sq->xmit_xdp_frame, mlx5e_xmit_xdp_frame_mpwqe, |
| mlx5e_xmit_xdp_frame, sq, &xdptxd, &xdpi, check_result); |
| if (unlikely(!ret)) { |
| if (sq->mpwqe.wqe) |
| mlx5e_xdp_mpwqe_complete(sq); |
| |
| mlx5e_xsk_tx_post_err(sq, &xdpi); |
| } |
| |
| flush = true; |
| } |
| |
| if (flush) { |
| if (sq->mpwqe.wqe) |
| mlx5e_xdp_mpwqe_complete(sq); |
| mlx5e_xmit_xdp_doorbell(sq); |
| |
| xsk_tx_release(pool); |
| } |
| |
| return !(budget && work_done); |
| } |