drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_tx.c - linux - Git at Google

 // SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause

 /* Packet transmit logic for Mellanox Gigabit Ethernet driver
  *
  * Copyright (C) 2020-2021 NVIDIA CORPORATION & AFFILIATES
  */

 #include <linux/skbuff.h>

 #include "mlxbf_gige.h"
 #include "mlxbf_gige_regs.h"

 /* Transmit Initialization
  * 1) Allocates TX WQE array using coherent DMA mapping
  * 2) Allocates TX completion counter using coherent DMA mapping
  */
 int mlxbf_gige_tx_init(struct mlxbf_gige *priv)
 {
 	size_t size;

 	size = MLXBF_GIGE_TX_WQE_SZ * priv->tx_q_entries;
 	priv->tx_wqe_base = dma_alloc_coherent(priv->dev, size,
 					       &priv->tx_wqe_base_dma,
 					       GFP_KERNEL);
 	if (!priv->tx_wqe_base)
 		return -ENOMEM;

 	priv->tx_wqe_next = priv->tx_wqe_base;

 	/* Write TX WQE base address into MMIO reg */
 	writeq(priv->tx_wqe_base_dma, priv->base + MLXBF_GIGE_TX_WQ_BASE);

 	/* Allocate address for TX completion count */
 	priv->tx_cc = dma_alloc_coherent(priv->dev, MLXBF_GIGE_TX_CC_SZ,
 					 &priv->tx_cc_dma, GFP_KERNEL);
 	if (!priv->tx_cc) {
 		dma_free_coherent(priv->dev, size,
 				  priv->tx_wqe_base, priv->tx_wqe_base_dma);
 		return -ENOMEM;
 	}

 	/* Write TX CC base address into MMIO reg */
 	writeq(priv->tx_cc_dma, priv->base + MLXBF_GIGE_TX_CI_UPDATE_ADDRESS);

 	writeq(ilog2(priv->tx_q_entries),
 	       priv->base + MLXBF_GIGE_TX_WQ_SIZE_LOG2);

 	priv->prev_tx_ci = 0;
 	priv->tx_pi = 0;

 	return 0;
 }

 /* Transmit Deinitialization
  * This routine will free allocations done by mlxbf_gige_tx_init(),
  * namely the TX WQE array and the TX completion counter
  */
 void mlxbf_gige_tx_deinit(struct mlxbf_gige *priv)
 {
 	u64 *tx_wqe_addr;
 	size_t size;
 	int i;

 	tx_wqe_addr = priv->tx_wqe_base;

 	for (i = 0; i < priv->tx_q_entries; i++) {
 		if (priv->tx_skb[i]) {
 			dma_unmap_single(priv->dev, *tx_wqe_addr,
 					 priv->tx_skb[i]->len, DMA_TO_DEVICE);
 			dev_kfree_skb(priv->tx_skb[i]);
 			priv->tx_skb[i] = NULL;
 		}
 		tx_wqe_addr += 2;
 	}

 	size = MLXBF_GIGE_TX_WQE_SZ * priv->tx_q_entries;
 	dma_free_coherent(priv->dev, size,
 			  priv->tx_wqe_base, priv->tx_wqe_base_dma);

 	dma_free_coherent(priv->dev, MLXBF_GIGE_TX_CC_SZ,
 			  priv->tx_cc, priv->tx_cc_dma);

 	priv->tx_wqe_base = NULL;
 	priv->tx_wqe_base_dma = 0;
 	priv->tx_cc = NULL;
 	priv->tx_cc_dma = 0;
 	priv->tx_wqe_next = NULL;
 	writeq(0, priv->base + MLXBF_GIGE_TX_WQ_BASE);
 	writeq(0, priv->base + MLXBF_GIGE_TX_CI_UPDATE_ADDRESS);
 }

 /* Function that returns status of TX ring:
  *          0: TX ring is full, i.e. there are no
  *             available un-used entries in TX ring.
  *   non-null: TX ring is not full, i.e. there are
  *             some available entries in TX ring.
  *             The non-null value is a measure of
  *             how many TX entries are available, but
  *             it is not the exact number of available
  *             entries (see below).
  *
  * The algorithm makes the assumption that if
  * (prev_tx_ci == tx_pi) then the TX ring is empty.
  * An empty ring actually has (tx_q_entries-1)
  * entries, which allows the algorithm to differentiate
  * the case of an empty ring vs. a full ring.
  */
 static u16 mlxbf_gige_tx_buffs_avail(struct mlxbf_gige *priv)
 {
 	unsigned long flags;
 	u16 avail;

 	spin_lock_irqsave(&priv->lock, flags);

 	if (priv->prev_tx_ci == priv->tx_pi)
 		avail = priv->tx_q_entries - 1;
 	else
 		avail = ((priv->tx_q_entries + priv->prev_tx_ci - priv->tx_pi)
 			  % priv->tx_q_entries) - 1;

 	spin_unlock_irqrestore(&priv->lock, flags);

 	return avail;
 }

 bool mlxbf_gige_handle_tx_complete(struct mlxbf_gige *priv)
 {
 	struct net_device_stats *stats;
 	u16 tx_wqe_index;
 	u64 *tx_wqe_addr;
 	u64 tx_status;
 	u16 tx_ci;

 	tx_status = readq(priv->base + MLXBF_GIGE_TX_STATUS);
 	if (tx_status & MLXBF_GIGE_TX_STATUS_DATA_FIFO_FULL)
 		priv->stats.tx_fifo_full++;
 	tx_ci = readq(priv->base + MLXBF_GIGE_TX_CONSUMER_INDEX);
 	stats = &priv->netdev->stats;

 	/* Transmit completion logic needs to loop until the completion
 	 * index (in SW) equals TX consumer index (from HW).  These
 	 * parameters are unsigned 16-bit values and the wrap case needs
 	 * to be supported, that is TX consumer index wrapped from 0xFFFF
 	 * to 0 while TX completion index is still < 0xFFFF.
 	 */
 	for (; priv->prev_tx_ci != tx_ci; priv->prev_tx_ci++) {
 		tx_wqe_index = priv->prev_tx_ci % priv->tx_q_entries;
 		/* Each TX WQE is 16 bytes. The 8 MSB store the 2KB TX
 		 * buffer address and the 8 LSB contain information
 		 * about the TX WQE.
 		 */
 		tx_wqe_addr = priv->tx_wqe_base +
 			       (tx_wqe_index * MLXBF_GIGE_TX_WQE_SZ_QWORDS);

 		stats->tx_packets++;
 		stats->tx_bytes += MLXBF_GIGE_TX_WQE_PKT_LEN(tx_wqe_addr);

 		dma_unmap_single(priv->dev, *tx_wqe_addr,
 				 priv->tx_skb[tx_wqe_index]->len, DMA_TO_DEVICE);
 		dev_consume_skb_any(priv->tx_skb[tx_wqe_index]);
 		priv->tx_skb[tx_wqe_index] = NULL;

 		/* Ensure completion of updates across all cores */
 		mb();
 	}

 	/* Since the TX ring was likely just drained, check if TX queue
 	 * had previously been stopped and now that there are TX buffers
 	 * available the TX queue can be awakened.
 	 */
 	if (netif_queue_stopped(priv->netdev) &&
 	    mlxbf_gige_tx_buffs_avail(priv))
 		netif_wake_queue(priv->netdev);

 	return true;
 }

 /* Function to advance the tx_wqe_next pointer to next TX WQE */
 void mlxbf_gige_update_tx_wqe_next(struct mlxbf_gige *priv)
 {
 	/* Advance tx_wqe_next pointer */
 	priv->tx_wqe_next += MLXBF_GIGE_TX_WQE_SZ_QWORDS;

 	/* Check if 'next' pointer is beyond end of TX ring */
 	/* If so, set 'next' back to 'base' pointer of ring */
 	if (priv->tx_wqe_next == (priv->tx_wqe_base +
 				  (priv->tx_q_entries * MLXBF_GIGE_TX_WQE_SZ_QWORDS)))
 		priv->tx_wqe_next = priv->tx_wqe_base;
 }

 netdev_tx_t mlxbf_gige_start_xmit(struct sk_buff *skb,
 				  struct net_device *netdev)
 {
 	struct mlxbf_gige *priv = netdev_priv(netdev);
 	long buff_addr, start_dma_page, end_dma_page;
 	struct sk_buff *tx_skb;
 	dma_addr_t tx_buf_dma;
 	unsigned long flags;
 	u64 *tx_wqe_addr;
 	u64 word2;

 	/* If needed, linearize TX SKB as hardware DMA expects this */
 	if (skb->len > MLXBF_GIGE_DEFAULT_BUF_SZ || skb_linearize(skb)) {
 		dev_kfree_skb(skb);
 		netdev->stats.tx_dropped++;
 		return NETDEV_TX_OK;
 	}

 	buff_addr = (long)skb->data;
 	start_dma_page = buff_addr >> MLXBF_GIGE_DMA_PAGE_SHIFT;
 	end_dma_page   = (buff_addr + skb->len - 1) >> MLXBF_GIGE_DMA_PAGE_SHIFT;

 	/* Verify that payload pointer and data length of SKB to be
 	 * transmitted does not violate the hardware DMA limitation.
 	 */
 	if (start_dma_page != end_dma_page) {
 		/* DMA operation would fail as-is, alloc new aligned SKB */
 		tx_skb = mlxbf_gige_alloc_skb(priv, skb->len,
 					      &tx_buf_dma, DMA_TO_DEVICE);
 		if (!tx_skb) {
 			/* Free original skb, could not alloc new aligned SKB */
 			dev_kfree_skb(skb);
 			netdev->stats.tx_dropped++;
 			return NETDEV_TX_OK;
 		}

 		skb_put_data(tx_skb, skb->data, skb->len);

 		/* Free the original SKB */
 		dev_kfree_skb(skb);
 	} else {
 		tx_skb = skb;
 		tx_buf_dma = dma_map_single(priv->dev, skb->data,
 					    skb->len, DMA_TO_DEVICE);
 		if (dma_mapping_error(priv->dev, tx_buf_dma)) {
 			dev_kfree_skb(skb);
 			netdev->stats.tx_dropped++;
 			return NETDEV_TX_OK;
 		}
 	}

 	/* Get address of TX WQE */
 	tx_wqe_addr = priv->tx_wqe_next;

 	mlxbf_gige_update_tx_wqe_next(priv);

 	/* Put PA of buffer address into first 64-bit word of TX WQE */
 	*tx_wqe_addr = tx_buf_dma;

 	/* Set TX WQE pkt_len appropriately
 	 * NOTE: GigE silicon will automatically pad up to
 	 *       minimum packet length if needed.
 	 */
 	word2 = tx_skb->len & MLXBF_GIGE_TX_WQE_PKT_LEN_MASK;

 	/* Write entire 2nd word of TX WQE */
 	*(tx_wqe_addr + 1) = word2;

 	spin_lock_irqsave(&priv->lock, flags);
 	priv->tx_skb[priv->tx_pi % priv->tx_q_entries] = tx_skb;
 	priv->tx_pi++;
 	spin_unlock_irqrestore(&priv->lock, flags);

 	if (!netdev_xmit_more()) {
 		/* Create memory barrier before write to TX PI */
 		wmb();
 		writeq(priv->tx_pi, priv->base + MLXBF_GIGE_TX_PRODUCER_INDEX);
 	}

 	/* Check if the last TX entry was just used */
 	if (!mlxbf_gige_tx_buffs_avail(priv)) {
 		/* TX ring is full, inform stack */
 		netif_stop_queue(netdev);

 		/* Since there is no separate "TX complete" interrupt, need
 		 * to explicitly schedule NAPI poll.  This will trigger logic
 		 * which processes TX completions, and will hopefully drain
 		 * the TX ring allowing the TX queue to be awakened.
 		 */
 		napi_schedule(&priv->napi);
 	}

 	return NETDEV_TX_OK;
 }
	// SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause

	/* Packet transmit logic for Mellanox Gigabit Ethernet driver
	*
	* Copyright (C) 2020-2021 NVIDIA CORPORATION & AFFILIATES
	*/

	#include <linux/skbuff.h>

	#include "mlxbf_gige.h"
	#include "mlxbf_gige_regs.h"

	/* Transmit Initialization
	* 1) Allocates TX WQE array using coherent DMA mapping
	* 2) Allocates TX completion counter using coherent DMA mapping
	*/
	int mlxbf_gige_tx_init(struct mlxbf_gige *priv)
	{
	size_t size;

	size = MLXBF_GIGE_TX_WQE_SZ * priv->tx_q_entries;
	priv->tx_wqe_base = dma_alloc_coherent(priv->dev, size,
	&priv->tx_wqe_base_dma,
	GFP_KERNEL);
	if (!priv->tx_wqe_base)
	return -ENOMEM;

	priv->tx_wqe_next = priv->tx_wqe_base;

	/* Write TX WQE base address into MMIO reg */
	writeq(priv->tx_wqe_base_dma, priv->base + MLXBF_GIGE_TX_WQ_BASE);

	/* Allocate address for TX completion count */
	priv->tx_cc = dma_alloc_coherent(priv->dev, MLXBF_GIGE_TX_CC_SZ,
	&priv->tx_cc_dma, GFP_KERNEL);
	if (!priv->tx_cc) {
	dma_free_coherent(priv->dev, size,
	priv->tx_wqe_base, priv->tx_wqe_base_dma);
	return -ENOMEM;
	}

	/* Write TX CC base address into MMIO reg */
	writeq(priv->tx_cc_dma, priv->base + MLXBF_GIGE_TX_CI_UPDATE_ADDRESS);

	writeq(ilog2(priv->tx_q_entries),
	priv->base + MLXBF_GIGE_TX_WQ_SIZE_LOG2);

	priv->prev_tx_ci = 0;
	priv->tx_pi = 0;

	return 0;
	}

	/* Transmit Deinitialization
	* This routine will free allocations done by mlxbf_gige_tx_init(),
	* namely the TX WQE array and the TX completion counter
	*/
	void mlxbf_gige_tx_deinit(struct mlxbf_gige *priv)
	{
	u64 *tx_wqe_addr;
	size_t size;
	int i;

	tx_wqe_addr = priv->tx_wqe_base;

	for (i = 0; i < priv->tx_q_entries; i++) {
	if (priv->tx_skb[i]) {
	dma_unmap_single(priv->dev, *tx_wqe_addr,
	priv->tx_skb[i]->len, DMA_TO_DEVICE);
	dev_kfree_skb(priv->tx_skb[i]);
	priv->tx_skb[i] = NULL;
	}
	tx_wqe_addr += 2;
	}

	size = MLXBF_GIGE_TX_WQE_SZ * priv->tx_q_entries;
	dma_free_coherent(priv->dev, size,
	priv->tx_wqe_base, priv->tx_wqe_base_dma);

	dma_free_coherent(priv->dev, MLXBF_GIGE_TX_CC_SZ,
	priv->tx_cc, priv->tx_cc_dma);

	priv->tx_wqe_base = NULL;
	priv->tx_wqe_base_dma = 0;
	priv->tx_cc = NULL;
	priv->tx_cc_dma = 0;
	priv->tx_wqe_next = NULL;
	writeq(0, priv->base + MLXBF_GIGE_TX_WQ_BASE);
	writeq(0, priv->base + MLXBF_GIGE_TX_CI_UPDATE_ADDRESS);
	}

	/* Function that returns status of TX ring:
	* 0: TX ring is full, i.e. there are no
	* available un-used entries in TX ring.
	* non-null: TX ring is not full, i.e. there are
	* some available entries in TX ring.
	* The non-null value is a measure of
	* how many TX entries are available, but
	* it is not the exact number of available
	* entries (see below).
	*
	* The algorithm makes the assumption that if
	* (prev_tx_ci == tx_pi) then the TX ring is empty.
	* An empty ring actually has (tx_q_entries-1)
	* entries, which allows the algorithm to differentiate
	* the case of an empty ring vs. a full ring.
	*/
	static u16 mlxbf_gige_tx_buffs_avail(struct mlxbf_gige *priv)
	{
	unsigned long flags;
	u16 avail;

	spin_lock_irqsave(&priv->lock, flags);

	if (priv->prev_tx_ci == priv->tx_pi)
	avail = priv->tx_q_entries - 1;
	else
	avail = ((priv->tx_q_entries + priv->prev_tx_ci - priv->tx_pi)
	% priv->tx_q_entries) - 1;

	spin_unlock_irqrestore(&priv->lock, flags);

	return avail;
	}

	bool mlxbf_gige_handle_tx_complete(struct mlxbf_gige *priv)
	{
	struct net_device_stats *stats;
	u16 tx_wqe_index;
	u64 *tx_wqe_addr;
	u64 tx_status;
	u16 tx_ci;

	tx_status = readq(priv->base + MLXBF_GIGE_TX_STATUS);
	if (tx_status & MLXBF_GIGE_TX_STATUS_DATA_FIFO_FULL)
	priv->stats.tx_fifo_full++;
	tx_ci = readq(priv->base + MLXBF_GIGE_TX_CONSUMER_INDEX);
	stats = &priv->netdev->stats;

	/* Transmit completion logic needs to loop until the completion
	* index (in SW) equals TX consumer index (from HW). These
	* parameters are unsigned 16-bit values and the wrap case needs
	* to be supported, that is TX consumer index wrapped from 0xFFFF
	* to 0 while TX completion index is still < 0xFFFF.
	*/
	for (; priv->prev_tx_ci != tx_ci; priv->prev_tx_ci++) {
	tx_wqe_index = priv->prev_tx_ci % priv->tx_q_entries;
	/* Each TX WQE is 16 bytes. The 8 MSB store the 2KB TX
	* buffer address and the 8 LSB contain information
	* about the TX WQE.
	*/
	tx_wqe_addr = priv->tx_wqe_base +
	(tx_wqe_index * MLXBF_GIGE_TX_WQE_SZ_QWORDS);

	stats->tx_packets++;
	stats->tx_bytes += MLXBF_GIGE_TX_WQE_PKT_LEN(tx_wqe_addr);

	dma_unmap_single(priv->dev, *tx_wqe_addr,
	priv->tx_skb[tx_wqe_index]->len, DMA_TO_DEVICE);
	dev_consume_skb_any(priv->tx_skb[tx_wqe_index]);
	priv->tx_skb[tx_wqe_index] = NULL;

	/* Ensure completion of updates across all cores */
	mb();
	}

	/* Since the TX ring was likely just drained, check if TX queue
	* had previously been stopped and now that there are TX buffers
	* available the TX queue can be awakened.
	*/
	if (netif_queue_stopped(priv->netdev) &&
	mlxbf_gige_tx_buffs_avail(priv))
	netif_wake_queue(priv->netdev);

	return true;
	}

	/* Function to advance the tx_wqe_next pointer to next TX WQE */
	void mlxbf_gige_update_tx_wqe_next(struct mlxbf_gige *priv)
	{
	/* Advance tx_wqe_next pointer */
	priv->tx_wqe_next += MLXBF_GIGE_TX_WQE_SZ_QWORDS;

	/* Check if 'next' pointer is beyond end of TX ring */
	/* If so, set 'next' back to 'base' pointer of ring */
	if (priv->tx_wqe_next == (priv->tx_wqe_base +
	(priv->tx_q_entries * MLXBF_GIGE_TX_WQE_SZ_QWORDS)))
	priv->tx_wqe_next = priv->tx_wqe_base;
	}

	netdev_tx_t mlxbf_gige_start_xmit(struct sk_buff *skb,
	struct net_device *netdev)
	{
	struct mlxbf_gige *priv = netdev_priv(netdev);
	long buff_addr, start_dma_page, end_dma_page;
	struct sk_buff *tx_skb;
	dma_addr_t tx_buf_dma;
	unsigned long flags;
	u64 *tx_wqe_addr;
	u64 word2;

	/* If needed, linearize TX SKB as hardware DMA expects this */
	if (skb->len > MLXBF_GIGE_DEFAULT_BUF_SZ \|\| skb_linearize(skb)) {
	dev_kfree_skb(skb);
	netdev->stats.tx_dropped++;
	return NETDEV_TX_OK;
	}

	buff_addr = (long)skb->data;
	start_dma_page = buff_addr >> MLXBF_GIGE_DMA_PAGE_SHIFT;
	end_dma_page = (buff_addr + skb->len - 1) >> MLXBF_GIGE_DMA_PAGE_SHIFT;

	/* Verify that payload pointer and data length of SKB to be
	* transmitted does not violate the hardware DMA limitation.
	*/
	if (start_dma_page != end_dma_page) {
	/* DMA operation would fail as-is, alloc new aligned SKB */
	tx_skb = mlxbf_gige_alloc_skb(priv, skb->len,
	&tx_buf_dma, DMA_TO_DEVICE);
	if (!tx_skb) {
	/* Free original skb, could not alloc new aligned SKB */
	dev_kfree_skb(skb);
	netdev->stats.tx_dropped++;
	return NETDEV_TX_OK;
	}

	skb_put_data(tx_skb, skb->data, skb->len);

	/* Free the original SKB */
	dev_kfree_skb(skb);
	} else {
	tx_skb = skb;
	tx_buf_dma = dma_map_single(priv->dev, skb->data,
	skb->len, DMA_TO_DEVICE);
	if (dma_mapping_error(priv->dev, tx_buf_dma)) {
	dev_kfree_skb(skb);
	netdev->stats.tx_dropped++;
	return NETDEV_TX_OK;
	}
	}

	/* Get address of TX WQE */
	tx_wqe_addr = priv->tx_wqe_next;

	mlxbf_gige_update_tx_wqe_next(priv);

	/* Put PA of buffer address into first 64-bit word of TX WQE */
	*tx_wqe_addr = tx_buf_dma;

	/* Set TX WQE pkt_len appropriately
	* NOTE: GigE silicon will automatically pad up to
	* minimum packet length if needed.
	*/
	word2 = tx_skb->len & MLXBF_GIGE_TX_WQE_PKT_LEN_MASK;

	/* Write entire 2nd word of TX WQE */
	*(tx_wqe_addr + 1) = word2;

	spin_lock_irqsave(&priv->lock, flags);
	priv->tx_skb[priv->tx_pi % priv->tx_q_entries] = tx_skb;
	priv->tx_pi++;
	spin_unlock_irqrestore(&priv->lock, flags);

	if (!netdev_xmit_more()) {
	/* Create memory barrier before write to TX PI */
	wmb();
	writeq(priv->tx_pi, priv->base + MLXBF_GIGE_TX_PRODUCER_INDEX);
	}

	/* Check if the last TX entry was just used */
	if (!mlxbf_gige_tx_buffs_avail(priv)) {
	/* TX ring is full, inform stack */
	netif_stop_queue(netdev);

	/* Since there is no separate "TX complete" interrupt, need
	* to explicitly schedule NAPI poll. This will trigger logic
	* which processes TX completions, and will hopefully drain
	* the TX ring allowing the TX queue to be awakened.
	*/
	napi_schedule(&priv->napi);
	}

	return NETDEV_TX_OK;
	}