drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_rx.c - linux - Git at Google

 // SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause

 /* Packet receive logic for Mellanox Gigabit Ethernet driver
  *
  * Copyright (C) 2020-2021 NVIDIA CORPORATION & AFFILIATES
  */

 #include <linux/etherdevice.h>
 #include <linux/skbuff.h>

 #include "mlxbf_gige.h"
 #include "mlxbf_gige_regs.h"

 void mlxbf_gige_set_mac_rx_filter(struct mlxbf_gige *priv,
 				  unsigned int index, u64 dmac)
 {
 	void __iomem *base = priv->base;
 	u64 control;

 	/* Write destination MAC to specified MAC RX filter */
 	writeq(dmac, base + MLXBF_GIGE_RX_MAC_FILTER +
 	       (index * MLXBF_GIGE_RX_MAC_FILTER_STRIDE));

 	/* Enable MAC receive filter mask for specified index */
 	control = readq(base + MLXBF_GIGE_CONTROL);
 	control |= (MLXBF_GIGE_CONTROL_EN_SPECIFIC_MAC << index);
 	writeq(control, base + MLXBF_GIGE_CONTROL);
 }

 void mlxbf_gige_get_mac_rx_filter(struct mlxbf_gige *priv,
 				  unsigned int index, u64 *dmac)
 {
 	void __iomem *base = priv->base;

 	/* Read destination MAC from specified MAC RX filter */
 	*dmac = readq(base + MLXBF_GIGE_RX_MAC_FILTER +
 		      (index * MLXBF_GIGE_RX_MAC_FILTER_STRIDE));
 }

 void mlxbf_gige_enable_promisc(struct mlxbf_gige *priv)
 {
 	void __iomem *base = priv->base;
 	u64 control;
 	u64 end_mac;

 	/* Enable MAC_ID_RANGE match functionality */
 	control = readq(base + MLXBF_GIGE_CONTROL);
 	control |= MLXBF_GIGE_CONTROL_MAC_ID_RANGE_EN;
 	writeq(control, base + MLXBF_GIGE_CONTROL);

 	/* Set start of destination MAC range check to 0 */
 	writeq(0, base + MLXBF_GIGE_RX_MAC_FILTER_DMAC_RANGE_START);

 	/* Set end of destination MAC range check to all FFs */
 	end_mac = BCAST_MAC_ADDR;
 	writeq(end_mac, base + MLXBF_GIGE_RX_MAC_FILTER_DMAC_RANGE_END);
 }

 void mlxbf_gige_disable_promisc(struct mlxbf_gige *priv)
 {
 	void __iomem *base = priv->base;
 	u64 control;

 	/* Disable MAC_ID_RANGE match functionality */
 	control = readq(base + MLXBF_GIGE_CONTROL);
 	control &= ~MLXBF_GIGE_CONTROL_MAC_ID_RANGE_EN;
 	writeq(control, base + MLXBF_GIGE_CONTROL);

 	/* NOTE: no need to change DMAC_RANGE_START or END;
 	 * those values are ignored since MAC_ID_RANGE_EN=0
 	 */
 }

 /* Receive Initialization
  * 1) Configures RX MAC filters via MMIO registers
  * 2) Allocates RX WQE array using coherent DMA mapping
  * 3) Initializes each element of RX WQE array with a receive
  *    buffer pointer (also using coherent DMA mapping)
  * 4) Allocates RX CQE array using coherent DMA mapping
  * 5) Completes other misc receive initialization
  */
 int mlxbf_gige_rx_init(struct mlxbf_gige *priv)
 {
 	size_t wq_size, cq_size;
 	dma_addr_t *rx_wqe_ptr;
 	dma_addr_t rx_buf_dma;
 	u64 data;
 	int i, j;

 	/* Configure MAC RX filter #0 to allow RX of broadcast pkts */
 	mlxbf_gige_set_mac_rx_filter(priv, MLXBF_GIGE_BCAST_MAC_FILTER_IDX,
 				     BCAST_MAC_ADDR);

 	wq_size = MLXBF_GIGE_RX_WQE_SZ * priv->rx_q_entries;
 	priv->rx_wqe_base = dma_alloc_coherent(priv->dev, wq_size,
 					       &priv->rx_wqe_base_dma,
 					       GFP_KERNEL);
 	if (!priv->rx_wqe_base)
 		return -ENOMEM;

 	/* Initialize 'rx_wqe_ptr' to point to first RX WQE in array
 	 * Each RX WQE is simply a receive buffer pointer, so walk
 	 * the entire array, allocating a 2KB buffer for each element
 	 */
 	rx_wqe_ptr = priv->rx_wqe_base;

 	for (i = 0; i < priv->rx_q_entries; i++) {
 		priv->rx_skb[i] = mlxbf_gige_alloc_skb(priv, MLXBF_GIGE_DEFAULT_BUF_SZ,
 						       &rx_buf_dma, DMA_FROM_DEVICE);
 		if (!priv->rx_skb[i])
 			goto free_wqe_and_skb;
 		*rx_wqe_ptr++ = rx_buf_dma;
 	}

 	/* Write RX WQE base address into MMIO reg */
 	writeq(priv->rx_wqe_base_dma, priv->base + MLXBF_GIGE_RX_WQ_BASE);

 	cq_size = MLXBF_GIGE_RX_CQE_SZ * priv->rx_q_entries;
 	priv->rx_cqe_base = dma_alloc_coherent(priv->dev, cq_size,
 					       &priv->rx_cqe_base_dma,
 					       GFP_KERNEL);
 	if (!priv->rx_cqe_base)
 		goto free_wqe_and_skb;

 	for (i = 0; i < priv->rx_q_entries; i++)
 		priv->rx_cqe_base[i] |= MLXBF_GIGE_RX_CQE_VALID_MASK;

 	/* Write RX CQE base address into MMIO reg */
 	writeq(priv->rx_cqe_base_dma, priv->base + MLXBF_GIGE_RX_CQ_BASE);

 	/* Write RX_WQE_PI with current number of replenished buffers */
 	writeq(priv->rx_q_entries, priv->base + MLXBF_GIGE_RX_WQE_PI);

 	/* Enable removal of CRC during RX */
 	data = readq(priv->base + MLXBF_GIGE_RX);
 	data |= MLXBF_GIGE_RX_STRIP_CRC_EN;
 	writeq(data, priv->base + MLXBF_GIGE_RX);

 	/* Enable RX MAC filter pass and discard counters */
 	writeq(MLXBF_GIGE_RX_MAC_FILTER_COUNT_DISC_EN,
 	       priv->base + MLXBF_GIGE_RX_MAC_FILTER_COUNT_DISC);
 	writeq(MLXBF_GIGE_RX_MAC_FILTER_COUNT_PASS_EN,
 	       priv->base + MLXBF_GIGE_RX_MAC_FILTER_COUNT_PASS);

 	writeq(ilog2(priv->rx_q_entries),
 	       priv->base + MLXBF_GIGE_RX_WQE_SIZE_LOG2);

 	/* Clear MLXBF_GIGE_INT_MASK 'receive pkt' bit to
 	 * indicate readiness to receive interrupts
 	 */
 	data = readq(priv->base + MLXBF_GIGE_INT_MASK);
 	data &= ~MLXBF_GIGE_INT_MASK_RX_RECEIVE_PACKET;
 	writeq(data, priv->base + MLXBF_GIGE_INT_MASK);

 	/* Enable RX DMA to write new packets to memory */
 	data = readq(priv->base + MLXBF_GIGE_RX_DMA);
 	data |= MLXBF_GIGE_RX_DMA_EN;
 	writeq(data, priv->base + MLXBF_GIGE_RX_DMA);

 	return 0;

 free_wqe_and_skb:
 	rx_wqe_ptr = priv->rx_wqe_base;
 	for (j = 0; j < i; j++) {
 		dma_unmap_single(priv->dev, *rx_wqe_ptr,
 				 MLXBF_GIGE_DEFAULT_BUF_SZ, DMA_FROM_DEVICE);
 		dev_kfree_skb(priv->rx_skb[j]);
 		rx_wqe_ptr++;
 	}
 	dma_free_coherent(priv->dev, wq_size,
 			  priv->rx_wqe_base, priv->rx_wqe_base_dma);
 	return -ENOMEM;
 }

 /* Receive Deinitialization
  * This routine will free allocations done by mlxbf_gige_rx_init(),
  * namely the RX WQE and RX CQE arrays, as well as all RX buffers
  */
 void mlxbf_gige_rx_deinit(struct mlxbf_gige *priv)
 {
 	dma_addr_t *rx_wqe_ptr;
 	size_t size;
 	u64 data;
 	int i;

 	/* Disable RX DMA to prevent packet transfers to memory */
 	data = readq(priv->base + MLXBF_GIGE_RX_DMA);
 	data &= ~MLXBF_GIGE_RX_DMA_EN;
 	writeq(data, priv->base + MLXBF_GIGE_RX_DMA);

 	rx_wqe_ptr = priv->rx_wqe_base;

 	for (i = 0; i < priv->rx_q_entries; i++) {
 		dma_unmap_single(priv->dev, *rx_wqe_ptr, MLXBF_GIGE_DEFAULT_BUF_SZ,
 				 DMA_FROM_DEVICE);
 		dev_kfree_skb(priv->rx_skb[i]);
 		rx_wqe_ptr++;
 	}

 	size = MLXBF_GIGE_RX_WQE_SZ * priv->rx_q_entries;
 	dma_free_coherent(priv->dev, size,
 			  priv->rx_wqe_base, priv->rx_wqe_base_dma);

 	size = MLXBF_GIGE_RX_CQE_SZ * priv->rx_q_entries;
 	dma_free_coherent(priv->dev, size,
 			  priv->rx_cqe_base, priv->rx_cqe_base_dma);

 	priv->rx_wqe_base = NULL;
 	priv->rx_wqe_base_dma = 0;
 	priv->rx_cqe_base = NULL;
 	priv->rx_cqe_base_dma = 0;
 	writeq(0, priv->base + MLXBF_GIGE_RX_WQ_BASE);
 	writeq(0, priv->base + MLXBF_GIGE_RX_CQ_BASE);
 }

 static bool mlxbf_gige_rx_packet(struct mlxbf_gige *priv, int *rx_pkts)
 {
 	struct net_device *netdev = priv->netdev;
 	struct sk_buff *skb = NULL, *rx_skb;
 	u16 rx_pi_rem, rx_ci_rem;
 	dma_addr_t *rx_wqe_addr;
 	dma_addr_t rx_buf_dma;
 	u64 *rx_cqe_addr;
 	u64 datalen;
 	u64 rx_cqe;
 	u16 rx_ci;
 	u16 rx_pi;

 	/* Index into RX buffer array is rx_pi w/wrap based on RX_CQE_SIZE */
 	rx_pi = readq(priv->base + MLXBF_GIGE_RX_WQE_PI);
 	rx_pi_rem = rx_pi % priv->rx_q_entries;

 	rx_wqe_addr = priv->rx_wqe_base + rx_pi_rem;
 	rx_cqe_addr = priv->rx_cqe_base + rx_pi_rem;
 	rx_cqe = *rx_cqe_addr;

 	if ((!!(rx_cqe & MLXBF_GIGE_RX_CQE_VALID_MASK)) != priv->valid_polarity)
 		return false;

 	if ((rx_cqe & MLXBF_GIGE_RX_CQE_PKT_STATUS_MASK) == 0) {
 		/* Packet is OK, increment stats */
 		datalen = rx_cqe & MLXBF_GIGE_RX_CQE_PKT_LEN_MASK;
 		netdev->stats.rx_packets++;
 		netdev->stats.rx_bytes += datalen;

 		skb = priv->rx_skb[rx_pi_rem];

 		/* Alloc another RX SKB for this same index */
 		rx_skb = mlxbf_gige_alloc_skb(priv, MLXBF_GIGE_DEFAULT_BUF_SZ,
 					      &rx_buf_dma, DMA_FROM_DEVICE);
 		if (!rx_skb)
 			return false;
 		priv->rx_skb[rx_pi_rem] = rx_skb;
 		dma_unmap_single(priv->dev, *rx_wqe_addr,
 				 MLXBF_GIGE_DEFAULT_BUF_SZ, DMA_FROM_DEVICE);

 		skb_put(skb, datalen);

 		skb->ip_summed = CHECKSUM_NONE; /* device did not checksum packet */

 		skb->protocol = eth_type_trans(skb, netdev);

 		*rx_wqe_addr = rx_buf_dma;
 	} else if (rx_cqe & MLXBF_GIGE_RX_CQE_PKT_STATUS_MAC_ERR) {
 		priv->stats.rx_mac_errors++;
 	} else if (rx_cqe & MLXBF_GIGE_RX_CQE_PKT_STATUS_TRUNCATED) {
 		priv->stats.rx_truncate_errors++;
 	}

 	/* Read receive consumer index before replenish so that this routine
 	 * returns accurate return value even if packet is received into
 	 * just-replenished buffer prior to exiting this routine.
 	 */
 	rx_ci = readq(priv->base + MLXBF_GIGE_RX_CQE_PACKET_CI);
 	rx_ci_rem = rx_ci % priv->rx_q_entries;

 	/* Let hardware know we've replenished one buffer */
 	rx_pi++;

 	/* Ensure completion of all writes before notifying HW of replenish */
 	wmb();
 	writeq(rx_pi, priv->base + MLXBF_GIGE_RX_WQE_PI);

 	(*rx_pkts)++;

 	rx_pi_rem = rx_pi % priv->rx_q_entries;
 	if (rx_pi_rem == 0)
 		priv->valid_polarity ^= 1;

 	if (skb)
 		netif_receive_skb(skb);

 	return rx_pi_rem != rx_ci_rem;
 }

 /* Driver poll() function called by NAPI infrastructure */
 int mlxbf_gige_poll(struct napi_struct *napi, int budget)
 {
 	struct mlxbf_gige *priv;
 	bool remaining_pkts;
 	int work_done = 0;
 	u64 data;

 	priv = container_of(napi, struct mlxbf_gige, napi);

 	mlxbf_gige_handle_tx_complete(priv);

 	do {
 		remaining_pkts = mlxbf_gige_rx_packet(priv, &work_done);
 	} while (remaining_pkts && work_done < budget);

 	/* If amount of work done < budget, turn off NAPI polling
 	 * via napi_complete_done(napi, work_done) and then
 	 * re-enable interrupts.
 	 */
 	if (work_done < budget && napi_complete_done(napi, work_done)) {
 		/* Clear MLXBF_GIGE_INT_MASK 'receive pkt' bit to
 		 * indicate receive readiness
 		 */
 		data = readq(priv->base + MLXBF_GIGE_INT_MASK);
 		data &= ~MLXBF_GIGE_INT_MASK_RX_RECEIVE_PACKET;
 		writeq(data, priv->base + MLXBF_GIGE_INT_MASK);
 	}

 	return work_done;
 }
	// SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause

	/* Packet receive logic for Mellanox Gigabit Ethernet driver
	*
	* Copyright (C) 2020-2021 NVIDIA CORPORATION & AFFILIATES
	*/

	#include <linux/etherdevice.h>
	#include <linux/skbuff.h>

	#include "mlxbf_gige.h"
	#include "mlxbf_gige_regs.h"

	void mlxbf_gige_set_mac_rx_filter(struct mlxbf_gige *priv,
	unsigned int index, u64 dmac)
	{
	void __iomem *base = priv->base;
	u64 control;

	/* Write destination MAC to specified MAC RX filter */
	writeq(dmac, base + MLXBF_GIGE_RX_MAC_FILTER +
	(index * MLXBF_GIGE_RX_MAC_FILTER_STRIDE));

	/* Enable MAC receive filter mask for specified index */
	control = readq(base + MLXBF_GIGE_CONTROL);
	control \|= (MLXBF_GIGE_CONTROL_EN_SPECIFIC_MAC << index);
	writeq(control, base + MLXBF_GIGE_CONTROL);
	}

	void mlxbf_gige_get_mac_rx_filter(struct mlxbf_gige *priv,
	unsigned int index, u64 *dmac)
	{
	void __iomem *base = priv->base;

	/* Read destination MAC from specified MAC RX filter */
	*dmac = readq(base + MLXBF_GIGE_RX_MAC_FILTER +
	(index * MLXBF_GIGE_RX_MAC_FILTER_STRIDE));
	}

	void mlxbf_gige_enable_promisc(struct mlxbf_gige *priv)
	{
	void __iomem *base = priv->base;
	u64 control;
	u64 end_mac;

	/* Enable MAC_ID_RANGE match functionality */
	control = readq(base + MLXBF_GIGE_CONTROL);
	control \|= MLXBF_GIGE_CONTROL_MAC_ID_RANGE_EN;
	writeq(control, base + MLXBF_GIGE_CONTROL);

	/* Set start of destination MAC range check to 0 */
	writeq(0, base + MLXBF_GIGE_RX_MAC_FILTER_DMAC_RANGE_START);

	/* Set end of destination MAC range check to all FFs */
	end_mac = BCAST_MAC_ADDR;
	writeq(end_mac, base + MLXBF_GIGE_RX_MAC_FILTER_DMAC_RANGE_END);
	}

	void mlxbf_gige_disable_promisc(struct mlxbf_gige *priv)
	{
	void __iomem *base = priv->base;
	u64 control;

	/* Disable MAC_ID_RANGE match functionality */
	control = readq(base + MLXBF_GIGE_CONTROL);
	control &= ~MLXBF_GIGE_CONTROL_MAC_ID_RANGE_EN;
	writeq(control, base + MLXBF_GIGE_CONTROL);

	/* NOTE: no need to change DMAC_RANGE_START or END;
	* those values are ignored since MAC_ID_RANGE_EN=0
	*/
	}

	/* Receive Initialization
	* 1) Configures RX MAC filters via MMIO registers
	* 2) Allocates RX WQE array using coherent DMA mapping
	* 3) Initializes each element of RX WQE array with a receive
	* buffer pointer (also using coherent DMA mapping)
	* 4) Allocates RX CQE array using coherent DMA mapping
	* 5) Completes other misc receive initialization
	*/
	int mlxbf_gige_rx_init(struct mlxbf_gige *priv)
	{
	size_t wq_size, cq_size;
	dma_addr_t *rx_wqe_ptr;
	dma_addr_t rx_buf_dma;
	u64 data;
	int i, j;

	/* Configure MAC RX filter #0 to allow RX of broadcast pkts */
	mlxbf_gige_set_mac_rx_filter(priv, MLXBF_GIGE_BCAST_MAC_FILTER_IDX,
	BCAST_MAC_ADDR);

	wq_size = MLXBF_GIGE_RX_WQE_SZ * priv->rx_q_entries;
	priv->rx_wqe_base = dma_alloc_coherent(priv->dev, wq_size,
	&priv->rx_wqe_base_dma,
	GFP_KERNEL);
	if (!priv->rx_wqe_base)
	return -ENOMEM;

	/* Initialize 'rx_wqe_ptr' to point to first RX WQE in array
	* Each RX WQE is simply a receive buffer pointer, so walk
	* the entire array, allocating a 2KB buffer for each element
	*/
	rx_wqe_ptr = priv->rx_wqe_base;

	for (i = 0; i < priv->rx_q_entries; i++) {
	priv->rx_skb[i] = mlxbf_gige_alloc_skb(priv, MLXBF_GIGE_DEFAULT_BUF_SZ,
	&rx_buf_dma, DMA_FROM_DEVICE);
	if (!priv->rx_skb[i])
	goto free_wqe_and_skb;
	*rx_wqe_ptr++ = rx_buf_dma;
	}

	/* Write RX WQE base address into MMIO reg */
	writeq(priv->rx_wqe_base_dma, priv->base + MLXBF_GIGE_RX_WQ_BASE);

	cq_size = MLXBF_GIGE_RX_CQE_SZ * priv->rx_q_entries;
	priv->rx_cqe_base = dma_alloc_coherent(priv->dev, cq_size,
	&priv->rx_cqe_base_dma,
	GFP_KERNEL);
	if (!priv->rx_cqe_base)
	goto free_wqe_and_skb;

	for (i = 0; i < priv->rx_q_entries; i++)
	priv->rx_cqe_base[i] \|= MLXBF_GIGE_RX_CQE_VALID_MASK;

	/* Write RX CQE base address into MMIO reg */
	writeq(priv->rx_cqe_base_dma, priv->base + MLXBF_GIGE_RX_CQ_BASE);

	/* Write RX_WQE_PI with current number of replenished buffers */
	writeq(priv->rx_q_entries, priv->base + MLXBF_GIGE_RX_WQE_PI);

	/* Enable removal of CRC during RX */
	data = readq(priv->base + MLXBF_GIGE_RX);
	data \|= MLXBF_GIGE_RX_STRIP_CRC_EN;
	writeq(data, priv->base + MLXBF_GIGE_RX);

	/* Enable RX MAC filter pass and discard counters */
	writeq(MLXBF_GIGE_RX_MAC_FILTER_COUNT_DISC_EN,
	priv->base + MLXBF_GIGE_RX_MAC_FILTER_COUNT_DISC);
	writeq(MLXBF_GIGE_RX_MAC_FILTER_COUNT_PASS_EN,
	priv->base + MLXBF_GIGE_RX_MAC_FILTER_COUNT_PASS);

	writeq(ilog2(priv->rx_q_entries),
	priv->base + MLXBF_GIGE_RX_WQE_SIZE_LOG2);

	/* Clear MLXBF_GIGE_INT_MASK 'receive pkt' bit to
	* indicate readiness to receive interrupts
	*/
	data = readq(priv->base + MLXBF_GIGE_INT_MASK);
	data &= ~MLXBF_GIGE_INT_MASK_RX_RECEIVE_PACKET;
	writeq(data, priv->base + MLXBF_GIGE_INT_MASK);

	/* Enable RX DMA to write new packets to memory */
	data = readq(priv->base + MLXBF_GIGE_RX_DMA);
	data \|= MLXBF_GIGE_RX_DMA_EN;
	writeq(data, priv->base + MLXBF_GIGE_RX_DMA);

	return 0;

	free_wqe_and_skb:
	rx_wqe_ptr = priv->rx_wqe_base;
	for (j = 0; j < i; j++) {
	dma_unmap_single(priv->dev, *rx_wqe_ptr,
	MLXBF_GIGE_DEFAULT_BUF_SZ, DMA_FROM_DEVICE);
	dev_kfree_skb(priv->rx_skb[j]);
	rx_wqe_ptr++;
	}
	dma_free_coherent(priv->dev, wq_size,
	priv->rx_wqe_base, priv->rx_wqe_base_dma);
	return -ENOMEM;
	}

	/* Receive Deinitialization
	* This routine will free allocations done by mlxbf_gige_rx_init(),
	* namely the RX WQE and RX CQE arrays, as well as all RX buffers
	*/
	void mlxbf_gige_rx_deinit(struct mlxbf_gige *priv)
	{
	dma_addr_t *rx_wqe_ptr;
	size_t size;
	u64 data;
	int i;

	/* Disable RX DMA to prevent packet transfers to memory */
	data = readq(priv->base + MLXBF_GIGE_RX_DMA);
	data &= ~MLXBF_GIGE_RX_DMA_EN;
	writeq(data, priv->base + MLXBF_GIGE_RX_DMA);

	rx_wqe_ptr = priv->rx_wqe_base;

	for (i = 0; i < priv->rx_q_entries; i++) {
	dma_unmap_single(priv->dev, *rx_wqe_ptr, MLXBF_GIGE_DEFAULT_BUF_SZ,
	DMA_FROM_DEVICE);
	dev_kfree_skb(priv->rx_skb[i]);
	rx_wqe_ptr++;
	}

	size = MLXBF_GIGE_RX_WQE_SZ * priv->rx_q_entries;
	dma_free_coherent(priv->dev, size,
	priv->rx_wqe_base, priv->rx_wqe_base_dma);

	size = MLXBF_GIGE_RX_CQE_SZ * priv->rx_q_entries;
	dma_free_coherent(priv->dev, size,
	priv->rx_cqe_base, priv->rx_cqe_base_dma);

	priv->rx_wqe_base = NULL;
	priv->rx_wqe_base_dma = 0;
	priv->rx_cqe_base = NULL;
	priv->rx_cqe_base_dma = 0;
	writeq(0, priv->base + MLXBF_GIGE_RX_WQ_BASE);
	writeq(0, priv->base + MLXBF_GIGE_RX_CQ_BASE);
	}

	static bool mlxbf_gige_rx_packet(struct mlxbf_gige priv, int rx_pkts)
	{
	struct net_device *netdev = priv->netdev;
	struct sk_buff skb = NULL, rx_skb;
	u16 rx_pi_rem, rx_ci_rem;
	dma_addr_t *rx_wqe_addr;
	dma_addr_t rx_buf_dma;
	u64 *rx_cqe_addr;
	u64 datalen;
	u64 rx_cqe;
	u16 rx_ci;
	u16 rx_pi;

	/* Index into RX buffer array is rx_pi w/wrap based on RX_CQE_SIZE */
	rx_pi = readq(priv->base + MLXBF_GIGE_RX_WQE_PI);
	rx_pi_rem = rx_pi % priv->rx_q_entries;

	rx_wqe_addr = priv->rx_wqe_base + rx_pi_rem;
	rx_cqe_addr = priv->rx_cqe_base + rx_pi_rem;
	rx_cqe = *rx_cqe_addr;

	if ((!!(rx_cqe & MLXBF_GIGE_RX_CQE_VALID_MASK)) != priv->valid_polarity)
	return false;

	if ((rx_cqe & MLXBF_GIGE_RX_CQE_PKT_STATUS_MASK) == 0) {
	/* Packet is OK, increment stats */
	datalen = rx_cqe & MLXBF_GIGE_RX_CQE_PKT_LEN_MASK;
	netdev->stats.rx_packets++;
	netdev->stats.rx_bytes += datalen;

	skb = priv->rx_skb[rx_pi_rem];

	/* Alloc another RX SKB for this same index */
	rx_skb = mlxbf_gige_alloc_skb(priv, MLXBF_GIGE_DEFAULT_BUF_SZ,
	&rx_buf_dma, DMA_FROM_DEVICE);
	if (!rx_skb)
	return false;
	priv->rx_skb[rx_pi_rem] = rx_skb;
	dma_unmap_single(priv->dev, *rx_wqe_addr,
	MLXBF_GIGE_DEFAULT_BUF_SZ, DMA_FROM_DEVICE);

	skb_put(skb, datalen);

	skb->ip_summed = CHECKSUM_NONE; /* device did not checksum packet */

	skb->protocol = eth_type_trans(skb, netdev);

	*rx_wqe_addr = rx_buf_dma;
	} else if (rx_cqe & MLXBF_GIGE_RX_CQE_PKT_STATUS_MAC_ERR) {
	priv->stats.rx_mac_errors++;
	} else if (rx_cqe & MLXBF_GIGE_RX_CQE_PKT_STATUS_TRUNCATED) {
	priv->stats.rx_truncate_errors++;
	}

	/* Read receive consumer index before replenish so that this routine
	* returns accurate return value even if packet is received into
	* just-replenished buffer prior to exiting this routine.
	*/
	rx_ci = readq(priv->base + MLXBF_GIGE_RX_CQE_PACKET_CI);
	rx_ci_rem = rx_ci % priv->rx_q_entries;

	/* Let hardware know we've replenished one buffer */
	rx_pi++;

	/* Ensure completion of all writes before notifying HW of replenish */
	wmb();
	writeq(rx_pi, priv->base + MLXBF_GIGE_RX_WQE_PI);

	(*rx_pkts)++;

	rx_pi_rem = rx_pi % priv->rx_q_entries;
	if (rx_pi_rem == 0)
	priv->valid_polarity ^= 1;

	if (skb)
	netif_receive_skb(skb);

	return rx_pi_rem != rx_ci_rem;
	}

	/* Driver poll() function called by NAPI infrastructure */
	int mlxbf_gige_poll(struct napi_struct *napi, int budget)
	{
	struct mlxbf_gige *priv;
	bool remaining_pkts;
	int work_done = 0;
	u64 data;

	priv = container_of(napi, struct mlxbf_gige, napi);

	mlxbf_gige_handle_tx_complete(priv);

	do {
	remaining_pkts = mlxbf_gige_rx_packet(priv, &work_done);
	} while (remaining_pkts && work_done < budget);

	/* If amount of work done < budget, turn off NAPI polling
	* via napi_complete_done(napi, work_done) and then
	* re-enable interrupts.
	*/
	if (work_done < budget && napi_complete_done(napi, work_done)) {
	/* Clear MLXBF_GIGE_INT_MASK 'receive pkt' bit to
	* indicate receive readiness
	*/
	data = readq(priv->base + MLXBF_GIGE_INT_MASK);
	data &= ~MLXBF_GIGE_INT_MASK_RX_RECEIVE_PACKET;
	writeq(data, priv->base + MLXBF_GIGE_INT_MASK);
	}

	return work_done;
	}