drivers/net/ethernet/google/gve/gve_rx.c - linux - Git at Google

 // SPDX-License-Identifier: (GPL-2.0 OR MIT)
 /* Google virtual Ethernet (gve) driver
  *
  * Copyright (C) 2015-2021 Google, Inc.
  */

 #include "gve.h"
 #include "gve_adminq.h"
 #include "gve_utils.h"
 #include <linux/etherdevice.h>

 static void gve_rx_free_buffer(struct device *dev,
 			       struct gve_rx_slot_page_info *page_info,
 			       union gve_rx_data_slot *data_slot)
 {
 	dma_addr_t dma = (dma_addr_t)(be64_to_cpu(data_slot->addr) &
 				      GVE_DATA_SLOT_ADDR_PAGE_MASK);

 	page_ref_sub(page_info->page, page_info->pagecnt_bias - 1);
 	gve_free_page(dev, page_info->page, dma, DMA_FROM_DEVICE);
 }

 static void gve_rx_unfill_pages(struct gve_priv *priv, struct gve_rx_ring *rx)
 {
 	u32 slots = rx->mask + 1;
 	int i;

 	if (rx->data.raw_addressing) {
 		for (i = 0; i < slots; i++)
 			gve_rx_free_buffer(&priv->pdev->dev, &rx->data.page_info[i],
 					   &rx->data.data_ring[i]);
 	} else {
 		for (i = 0; i < slots; i++)
 			page_ref_sub(rx->data.page_info[i].page,
 				     rx->data.page_info[i].pagecnt_bias - 1);
 		gve_unassign_qpl(priv, rx->data.qpl->id);
 		rx->data.qpl = NULL;
 	}
 	kvfree(rx->data.page_info);
 	rx->data.page_info = NULL;
 }

 static void gve_rx_free_ring(struct gve_priv *priv, int idx)
 {
 	struct gve_rx_ring *rx = &priv->rx[idx];
 	struct device *dev = &priv->pdev->dev;
 	u32 slots = rx->mask + 1;
 	size_t bytes;

 	gve_rx_remove_from_block(priv, idx);

 	bytes = sizeof(struct gve_rx_desc) * priv->rx_desc_cnt;
 	dma_free_coherent(dev, bytes, rx->desc.desc_ring, rx->desc.bus);
 	rx->desc.desc_ring = NULL;

 	dma_free_coherent(dev, sizeof(*rx->q_resources),
 			  rx->q_resources, rx->q_resources_bus);
 	rx->q_resources = NULL;

 	gve_rx_unfill_pages(priv, rx);

 	bytes = sizeof(*rx->data.data_ring) * slots;
 	dma_free_coherent(dev, bytes, rx->data.data_ring,
 			  rx->data.data_bus);
 	rx->data.data_ring = NULL;
 	netif_dbg(priv, drv, priv->dev, "freed rx ring %d\n", idx);
 }

 static void gve_setup_rx_buffer(struct gve_rx_slot_page_info *page_info,
 			     dma_addr_t addr, struct page *page, __be64 *slot_addr)
 {
 	page_info->page = page;
 	page_info->page_offset = 0;
 	page_info->page_address = page_address(page);
 	*slot_addr = cpu_to_be64(addr);
 	/* The page already has 1 ref */
 	page_ref_add(page, INT_MAX - 1);
 	page_info->pagecnt_bias = INT_MAX;
 }

 static int gve_rx_alloc_buffer(struct gve_priv *priv, struct device *dev,
 			       struct gve_rx_slot_page_info *page_info,
 			       union gve_rx_data_slot *data_slot)
 {
 	struct page *page;
 	dma_addr_t dma;
 	int err;

 	err = gve_alloc_page(priv, dev, &page, &dma, DMA_FROM_DEVICE,
 			     GFP_ATOMIC);
 	if (err)
 		return err;

 	gve_setup_rx_buffer(page_info, dma, page, &data_slot->addr);
 	return 0;
 }

 static int gve_prefill_rx_pages(struct gve_rx_ring *rx)
 {
 	struct gve_priv *priv = rx->gve;
 	u32 slots;
 	int err;
 	int i;

 	/* Allocate one page per Rx queue slot. Each page is split into two
 	 * packet buffers, when possible we "page flip" between the two.
 	 */
 	slots = rx->mask + 1;

 	rx->data.page_info = kvzalloc(slots *
 				      sizeof(*rx->data.page_info), GFP_KERNEL);
 	if (!rx->data.page_info)
 		return -ENOMEM;

 	if (!rx->data.raw_addressing) {
 		rx->data.qpl = gve_assign_rx_qpl(priv);
 		if (!rx->data.qpl) {
 			kvfree(rx->data.page_info);
 			rx->data.page_info = NULL;
 			return -ENOMEM;
 		}
 	}
 	for (i = 0; i < slots; i++) {
 		if (!rx->data.raw_addressing) {
 			struct page *page = rx->data.qpl->pages[i];
 			dma_addr_t addr = i * PAGE_SIZE;

 			gve_setup_rx_buffer(&rx->data.page_info[i], addr, page,
 					    &rx->data.data_ring[i].qpl_offset);
 			continue;
 		}
 		err = gve_rx_alloc_buffer(priv, &priv->pdev->dev, &rx->data.page_info[i],
 					  &rx->data.data_ring[i]);
 		if (err)
 			goto alloc_err;
 	}

 	return slots;
 alloc_err:
 	while (i--)
 		gve_rx_free_buffer(&priv->pdev->dev,
 				   &rx->data.page_info[i],
 				   &rx->data.data_ring[i]);
 	return err;
 }

 static void gve_rx_ctx_clear(struct gve_rx_ctx *ctx)
 {
 	ctx->curr_frag_cnt = 0;
 	ctx->total_expected_size = 0;
 	ctx->expected_frag_cnt = 0;
 	ctx->skb_head = NULL;
 	ctx->skb_tail = NULL;
 	ctx->reuse_frags = false;
 }

 static int gve_rx_alloc_ring(struct gve_priv *priv, int idx)
 {
 	struct gve_rx_ring *rx = &priv->rx[idx];
 	struct device *hdev = &priv->pdev->dev;
 	u32 slots, npages;
 	int filled_pages;
 	size_t bytes;
 	int err;

 	netif_dbg(priv, drv, priv->dev, "allocating rx ring\n");
 	/* Make sure everything is zeroed to start with */
 	memset(rx, 0, sizeof(*rx));

 	rx->gve = priv;
 	rx->q_num = idx;

 	slots = priv->rx_data_slot_cnt;
 	rx->mask = slots - 1;
 	rx->data.raw_addressing = priv->queue_format == GVE_GQI_RDA_FORMAT;

 	/* alloc rx data ring */
 	bytes = sizeof(*rx->data.data_ring) * slots;
 	rx->data.data_ring = dma_alloc_coherent(hdev, bytes,
 						&rx->data.data_bus,
 						GFP_KERNEL);
 	if (!rx->data.data_ring)
 		return -ENOMEM;
 	filled_pages = gve_prefill_rx_pages(rx);
 	if (filled_pages < 0) {
 		err = -ENOMEM;
 		goto abort_with_slots;
 	}
 	rx->fill_cnt = filled_pages;
 	/* Ensure data ring slots (packet buffers) are visible. */
 	dma_wmb();

 	/* Alloc gve_queue_resources */
 	rx->q_resources =
 		dma_alloc_coherent(hdev,
 				   sizeof(*rx->q_resources),
 				   &rx->q_resources_bus,
 				   GFP_KERNEL);
 	if (!rx->q_resources) {
 		err = -ENOMEM;
 		goto abort_filled;
 	}
 	netif_dbg(priv, drv, priv->dev, "rx[%d]->data.data_bus=%lx\n", idx,
 		  (unsigned long)rx->data.data_bus);

 	/* alloc rx desc ring */
 	bytes = sizeof(struct gve_rx_desc) * priv->rx_desc_cnt;
 	npages = bytes / PAGE_SIZE;
 	if (npages * PAGE_SIZE != bytes) {
 		err = -EIO;
 		goto abort_with_q_resources;
 	}

 	rx->desc.desc_ring = dma_alloc_coherent(hdev, bytes, &rx->desc.bus,
 						GFP_KERNEL);
 	if (!rx->desc.desc_ring) {
 		err = -ENOMEM;
 		goto abort_with_q_resources;
 	}
 	rx->cnt = 0;
 	rx->db_threshold = priv->rx_desc_cnt / 2;
 	rx->desc.seqno = 1;

 	/* Allocating half-page buffers allows page-flipping which is faster
 	 * than copying or allocating new pages.
 	 */
 	rx->packet_buffer_size = PAGE_SIZE / 2;
 	gve_rx_ctx_clear(&rx->ctx);
 	gve_rx_add_to_block(priv, idx);

 	return 0;

 abort_with_q_resources:
 	dma_free_coherent(hdev, sizeof(*rx->q_resources),
 			  rx->q_resources, rx->q_resources_bus);
 	rx->q_resources = NULL;
 abort_filled:
 	gve_rx_unfill_pages(priv, rx);
 abort_with_slots:
 	bytes = sizeof(*rx->data.data_ring) * slots;
 	dma_free_coherent(hdev, bytes, rx->data.data_ring, rx->data.data_bus);
 	rx->data.data_ring = NULL;

 	return err;
 }

 int gve_rx_alloc_rings(struct gve_priv *priv)
 {
 	int err = 0;
 	int i;

 	for (i = 0; i < priv->rx_cfg.num_queues; i++) {
 		err = gve_rx_alloc_ring(priv, i);
 		if (err) {
 			netif_err(priv, drv, priv->dev,
 				  "Failed to alloc rx ring=%d: err=%d\n",
 				  i, err);
 			break;
 		}
 	}
 	/* Unallocate if there was an error */
 	if (err) {
 		int j;

 		for (j = 0; j < i; j++)
 			gve_rx_free_ring(priv, j);
 	}
 	return err;
 }

 void gve_rx_free_rings_gqi(struct gve_priv *priv)
 {
 	int i;

 	for (i = 0; i < priv->rx_cfg.num_queues; i++)
 		gve_rx_free_ring(priv, i);
 }

 void gve_rx_write_doorbell(struct gve_priv *priv, struct gve_rx_ring *rx)
 {
 	u32 db_idx = be32_to_cpu(rx->q_resources->db_index);

 	iowrite32be(rx->fill_cnt, &priv->db_bar2[db_idx]);
 }

 static enum pkt_hash_types gve_rss_type(__be16 pkt_flags)
 {
 	if (likely(pkt_flags & (GVE_RXF_TCP | GVE_RXF_UDP)))
 		return PKT_HASH_TYPE_L4;
 	if (pkt_flags & (GVE_RXF_IPV4 | GVE_RXF_IPV6))
 		return PKT_HASH_TYPE_L3;
 	return PKT_HASH_TYPE_L2;
 }

 static u16 gve_rx_ctx_padding(struct gve_rx_ctx *ctx)
 {
 	return (ctx->curr_frag_cnt == 0) ? GVE_RX_PAD : 0;
 }

 static struct sk_buff *gve_rx_add_frags(struct napi_struct *napi,
 					struct gve_rx_slot_page_info *page_info,
 					u16 packet_buffer_size, u16 len,
 					struct gve_rx_ctx *ctx)
 {
 	u32 offset = page_info->page_offset +  gve_rx_ctx_padding(ctx);
 	struct sk_buff *skb;

 	if (!ctx->skb_head)
 		ctx->skb_head = napi_get_frags(napi);

 	if (unlikely(!ctx->skb_head))
 		return NULL;

 	skb = ctx->skb_head;
 	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page_info->page,
 			offset, len, packet_buffer_size);

 	return skb;
 }

 static void gve_rx_flip_buff(struct gve_rx_slot_page_info *page_info, __be64 *slot_addr)
 {
 	const __be64 offset = cpu_to_be64(PAGE_SIZE / 2);

 	/* "flip" to other packet buffer on this page */
 	page_info->page_offset ^= PAGE_SIZE / 2;
 	*(slot_addr) ^= offset;
 }

 static int gve_rx_can_recycle_buffer(struct gve_rx_slot_page_info *page_info)
 {
 	int pagecount = page_count(page_info->page);

 	/* This page is not being used by any SKBs - reuse */
 	if (pagecount == page_info->pagecnt_bias)
 		return 1;
 	/* This page is still being used by an SKB - we can't reuse */
 	else if (pagecount > page_info->pagecnt_bias)
 		return 0;
 	WARN(pagecount < page_info->pagecnt_bias,
 	     "Pagecount should never be less than the bias.");
 	return -1;
 }

 static struct sk_buff *
 gve_rx_raw_addressing(struct device *dev, struct net_device *netdev,
 		      struct gve_rx_slot_page_info *page_info, u16 len,
 		      struct napi_struct *napi,
 		      union gve_rx_data_slot *data_slot,
 		      u16 packet_buffer_size, struct gve_rx_ctx *ctx)
 {
 	struct sk_buff *skb = gve_rx_add_frags(napi, page_info, packet_buffer_size, len, ctx);

 	if (!skb)
 		return NULL;

 	/* Optimistically stop the kernel from freeing the page.
 	 * We will check again in refill to determine if we need to alloc a
 	 * new page.
 	 */
 	gve_dec_pagecnt_bias(page_info);

 	return skb;
 }

 static struct sk_buff *
 gve_rx_qpl(struct device *dev, struct net_device *netdev,
 	   struct gve_rx_ring *rx, struct gve_rx_slot_page_info *page_info,
 	   u16 len, struct napi_struct *napi,
 	   union gve_rx_data_slot *data_slot)
 {
 	struct gve_rx_ctx *ctx = &rx->ctx;
 	struct sk_buff *skb;

 	/* if raw_addressing mode is not enabled gvnic can only receive into
 	 * registered segments. If the buffer can't be recycled, our only
 	 * choice is to copy the data out of it so that we can return it to the
 	 * device.
 	 */
 	if (ctx->reuse_frags) {
 		skb = gve_rx_add_frags(napi, page_info, rx->packet_buffer_size, len, ctx);
 		/* No point in recycling if we didn't get the skb */
 		if (skb) {
 			/* Make sure that the page isn't freed. */
 			gve_dec_pagecnt_bias(page_info);
 			gve_rx_flip_buff(page_info, &data_slot->qpl_offset);
 		}
 	} else {
 		const u16 padding = gve_rx_ctx_padding(ctx);

 		skb = gve_rx_copy(netdev, napi, page_info, len, padding, ctx);
 		if (skb) {
 			u64_stats_update_begin(&rx->statss);
 			rx->rx_frag_copy_cnt++;
 			u64_stats_update_end(&rx->statss);
 		}
 	}
 	return skb;
 }

 #define GVE_PKTCONT_BIT_IS_SET(x) (GVE_RXF_PKT_CONT & (x))
 static u16 gve_rx_get_fragment_size(struct gve_rx_ctx *ctx, struct gve_rx_desc *desc)
 {
 	return be16_to_cpu(desc->len) - gve_rx_ctx_padding(ctx);
 }

 static bool gve_rx_ctx_init(struct gve_rx_ctx *ctx, struct gve_rx_ring *rx)
 {
 	bool qpl_mode = !rx->data.raw_addressing, packet_size_error = false;
 	bool buffer_error = false, desc_error = false, seqno_error = false;
 	struct gve_rx_slot_page_info *page_info;
 	struct gve_priv *priv = rx->gve;
 	u32 idx = rx->cnt & rx->mask;
 	bool reuse_frags, can_flip;
 	struct gve_rx_desc *desc;
 	u16 packet_size = 0;
 	u16 n_frags = 0;
 	int recycle;

 	/** In QPL mode, we only flip buffers when all buffers containing the packet
 	 * can be flipped. RDA can_flip decisions will be made later, per frag.
 	 */
 	can_flip = qpl_mode;
 	reuse_frags = can_flip;
 	do {
 		u16 frag_size;

 		n_frags++;
 		desc = &rx->desc.desc_ring[idx];
 		desc_error = unlikely(desc->flags_seq & GVE_RXF_ERR) || desc_error;
 		if (GVE_SEQNO(desc->flags_seq) != rx->desc.seqno) {
 			seqno_error = true;
 			netdev_warn(priv->dev,
 				    "RX seqno error: want=%d, got=%d, dropping packet and scheduling reset.",
 				    rx->desc.seqno, GVE_SEQNO(desc->flags_seq));
 		}
 		frag_size = be16_to_cpu(desc->len);
 		packet_size += frag_size;
 		if (frag_size > rx->packet_buffer_size) {
 			packet_size_error = true;
 			netdev_warn(priv->dev,
 				    "RX fragment error: packet_buffer_size=%d, frag_size=%d, dropping packet.",
 				    rx->packet_buffer_size, be16_to_cpu(desc->len));
 		}
 		page_info = &rx->data.page_info[idx];
 		if (can_flip) {
 			recycle = gve_rx_can_recycle_buffer(page_info);
 			reuse_frags = reuse_frags && recycle > 0;
 			buffer_error = buffer_error || unlikely(recycle < 0);
 		}
 		idx = (idx + 1) & rx->mask;
 		rx->desc.seqno = gve_next_seqno(rx->desc.seqno);
 	} while (GVE_PKTCONT_BIT_IS_SET(desc->flags_seq));

 	prefetch(rx->desc.desc_ring + idx);

 	ctx->curr_frag_cnt = 0;
 	ctx->total_expected_size = packet_size - GVE_RX_PAD;
 	ctx->expected_frag_cnt = n_frags;
 	ctx->skb_head = NULL;
 	ctx->reuse_frags = reuse_frags;

 	if (ctx->expected_frag_cnt > 1) {
 		u64_stats_update_begin(&rx->statss);
 		rx->rx_cont_packet_cnt++;
 		u64_stats_update_end(&rx->statss);
 	}
 	if (ctx->total_expected_size > priv->rx_copybreak && !ctx->reuse_frags && qpl_mode) {
 		u64_stats_update_begin(&rx->statss);
 		rx->rx_copied_pkt++;
 		u64_stats_update_end(&rx->statss);
 	}

 	if (unlikely(buffer_error || seqno_error || packet_size_error)) {
 		gve_schedule_reset(priv);
 		return false;
 	}

 	if (unlikely(desc_error)) {
 		u64_stats_update_begin(&rx->statss);
 		rx->rx_desc_err_dropped_pkt++;
 		u64_stats_update_end(&rx->statss);
 		return false;
 	}
 	return true;
 }

 static struct sk_buff *gve_rx_skb(struct gve_priv *priv, struct gve_rx_ring *rx,
 				  struct gve_rx_slot_page_info *page_info, struct napi_struct *napi,
 				  u16 len, union gve_rx_data_slot *data_slot)
 {
 	struct net_device *netdev = priv->dev;
 	struct gve_rx_ctx *ctx = &rx->ctx;
 	struct sk_buff *skb = NULL;

 	if (len <= priv->rx_copybreak && ctx->expected_frag_cnt == 1) {
 		/* Just copy small packets */
 		skb = gve_rx_copy(netdev, napi, page_info, len, GVE_RX_PAD, ctx);
 		if (skb) {
 			u64_stats_update_begin(&rx->statss);
 			rx->rx_copied_pkt++;
 			rx->rx_frag_copy_cnt++;
 			rx->rx_copybreak_pkt++;
 			u64_stats_update_end(&rx->statss);
 		}
 	} else {
 		if (rx->data.raw_addressing) {
 			int recycle = gve_rx_can_recycle_buffer(page_info);

 			if (unlikely(recycle < 0)) {
 				gve_schedule_reset(priv);
 				return NULL;
 			}
 			page_info->can_flip = recycle;
 			if (page_info->can_flip) {
 				u64_stats_update_begin(&rx->statss);
 				rx->rx_frag_flip_cnt++;
 				u64_stats_update_end(&rx->statss);
 			}
 			skb = gve_rx_raw_addressing(&priv->pdev->dev, netdev,
 						    page_info, len, napi,
 						    data_slot,
 						    rx->packet_buffer_size, ctx);
 		} else {
 			if (ctx->reuse_frags) {
 				u64_stats_update_begin(&rx->statss);
 				rx->rx_frag_flip_cnt++;
 				u64_stats_update_end(&rx->statss);
 			}
 			skb = gve_rx_qpl(&priv->pdev->dev, netdev, rx,
 					 page_info, len, napi, data_slot);
 		}
 	}
 	return skb;
 }

 static bool gve_rx(struct gve_rx_ring *rx, netdev_features_t feat,
 		   u64 *packet_size_bytes, u32 *work_done)
 {
 	struct gve_rx_slot_page_info *page_info;
 	struct gve_rx_ctx *ctx = &rx->ctx;
 	union gve_rx_data_slot *data_slot;
 	struct gve_priv *priv = rx->gve;
 	struct gve_rx_desc *first_desc;
 	struct sk_buff *skb = NULL;
 	struct gve_rx_desc *desc;
 	struct napi_struct *napi;
 	dma_addr_t page_bus;
 	u32 work_cnt = 0;
 	void *va;
 	u32 idx;
 	u16 len;

 	idx = rx->cnt & rx->mask;
 	first_desc = &rx->desc.desc_ring[idx];
 	desc = first_desc;
 	napi = &priv->ntfy_blocks[rx->ntfy_id].napi;

 	if (unlikely(!gve_rx_ctx_init(ctx, rx)))
 		goto skb_alloc_fail;

 	while (ctx->curr_frag_cnt < ctx->expected_frag_cnt) {
 		/* Prefetch two packet buffers ahead, we will need it soon. */
 		page_info = &rx->data.page_info[(idx + 2) & rx->mask];
 		va = page_info->page_address + page_info->page_offset;

 		prefetch(page_info->page); /* Kernel page struct. */
 		prefetch(va);              /* Packet header. */
 		prefetch(va + 64);         /* Next cacheline too. */

 		len = gve_rx_get_fragment_size(ctx, desc);

 		page_info = &rx->data.page_info[idx];
 		data_slot = &rx->data.data_ring[idx];
 		page_bus = rx->data.raw_addressing ?
 			   be64_to_cpu(data_slot->addr) - page_info->page_offset :
 			   rx->data.qpl->page_buses[idx];
 		dma_sync_single_for_cpu(&priv->pdev->dev, page_bus, PAGE_SIZE, DMA_FROM_DEVICE);

 		skb = gve_rx_skb(priv, rx, page_info, napi, len, data_slot);
 		if (!skb) {
 			u64_stats_update_begin(&rx->statss);
 			rx->rx_skb_alloc_fail++;
 			u64_stats_update_end(&rx->statss);
 			goto skb_alloc_fail;
 		}

 		ctx->curr_frag_cnt++;
 		rx->cnt++;
 		idx = rx->cnt & rx->mask;
 		work_cnt++;
 		desc = &rx->desc.desc_ring[idx];
 	}

 	if (likely(feat & NETIF_F_RXCSUM)) {
 		/* NIC passes up the partial sum */
 		if (first_desc->csum)
 			skb->ip_summed = CHECKSUM_COMPLETE;
 		else
 			skb->ip_summed = CHECKSUM_NONE;
 		skb->csum = csum_unfold(first_desc->csum);
 	}

 	/* parse flags & pass relevant info up */
 	if (likely(feat & NETIF_F_RXHASH) &&
 	    gve_needs_rss(first_desc->flags_seq))
 		skb_set_hash(skb, be32_to_cpu(first_desc->rss_hash),
 			     gve_rss_type(first_desc->flags_seq));

 	*packet_size_bytes = skb->len + (skb->protocol ? ETH_HLEN : 0);
 	*work_done = work_cnt;
 	skb_record_rx_queue(skb, rx->q_num);
 	if (skb_is_nonlinear(skb))
 		napi_gro_frags(napi);
 	else
 		napi_gro_receive(napi, skb);

 	gve_rx_ctx_clear(ctx);
 	return true;

 skb_alloc_fail:
 	if (napi->skb)
 		napi_free_frags(napi);
 	*packet_size_bytes = 0;
 	*work_done = ctx->expected_frag_cnt;
 	while (ctx->curr_frag_cnt < ctx->expected_frag_cnt) {
 		rx->cnt++;
 		ctx->curr_frag_cnt++;
 	}
 	gve_rx_ctx_clear(ctx);
 	return false;
 }

 bool gve_rx_work_pending(struct gve_rx_ring *rx)
 {
 	struct gve_rx_desc *desc;
 	__be16 flags_seq;
 	u32 next_idx;

 	next_idx = rx->cnt & rx->mask;
 	desc = rx->desc.desc_ring + next_idx;

 	flags_seq = desc->flags_seq;

 	return (GVE_SEQNO(flags_seq) == rx->desc.seqno);
 }

 static bool gve_rx_refill_buffers(struct gve_priv *priv, struct gve_rx_ring *rx)
 {
 	int refill_target = rx->mask + 1;
 	u32 fill_cnt = rx->fill_cnt;

 	while (fill_cnt - rx->cnt < refill_target) {
 		struct gve_rx_slot_page_info *page_info;
 		u32 idx = fill_cnt & rx->mask;

 		page_info = &rx->data.page_info[idx];
 		if (page_info->can_flip) {
 			/* The other half of the page is free because it was
 			 * free when we processed the descriptor. Flip to it.
 			 */
 			union gve_rx_data_slot *data_slot =
 						&rx->data.data_ring[idx];

 			gve_rx_flip_buff(page_info, &data_slot->addr);
 			page_info->can_flip = 0;
 		} else {
 			/* It is possible that the networking stack has already
 			 * finished processing all outstanding packets in the buffer
 			 * and it can be reused.
 			 * Flipping is unnecessary here - if the networking stack still
 			 * owns half the page it is impossible to tell which half. Either
 			 * the whole page is free or it needs to be replaced.
 			 */
 			int recycle = gve_rx_can_recycle_buffer(page_info);

 			if (recycle < 0) {
 				if (!rx->data.raw_addressing)
 					gve_schedule_reset(priv);
 				return false;
 			}
 			if (!recycle) {
 				/* We can't reuse the buffer - alloc a new one*/
 				union gve_rx_data_slot *data_slot =
 						&rx->data.data_ring[idx];
 				struct device *dev = &priv->pdev->dev;
 				gve_rx_free_buffer(dev, page_info, data_slot);
 				page_info->page = NULL;
 				if (gve_rx_alloc_buffer(priv, dev, page_info,
 							data_slot)) {
 					u64_stats_update_begin(&rx->statss);
 					rx->rx_buf_alloc_fail++;
 					u64_stats_update_end(&rx->statss);
 					break;
 				}
 			}
 		}
 		fill_cnt++;
 	}
 	rx->fill_cnt = fill_cnt;
 	return true;
 }

 static int gve_clean_rx_done(struct gve_rx_ring *rx, int budget,
 			     netdev_features_t feat)
 {
 	u32 work_done = 0, total_packet_cnt = 0, ok_packet_cnt = 0;
 	struct gve_priv *priv = rx->gve;
 	u32 idx = rx->cnt & rx->mask;
 	struct gve_rx_desc *desc;
 	u64 bytes = 0;

 	desc = &rx->desc.desc_ring[idx];
 	while ((GVE_SEQNO(desc->flags_seq) == rx->desc.seqno) &&
 	       work_done < budget) {
 		u64 packet_size_bytes = 0;
 		u32 work_cnt = 0;
 		bool dropped;

 		netif_info(priv, rx_status, priv->dev,
 			   "[%d] idx=%d desc=%p desc->flags_seq=0x%x\n",
 			   rx->q_num, idx, desc, desc->flags_seq);
 		netif_info(priv, rx_status, priv->dev,
 			   "[%d] seqno=%d rx->desc.seqno=%d\n",
 			   rx->q_num, GVE_SEQNO(desc->flags_seq),
 			   rx->desc.seqno);

 		dropped = !gve_rx(rx, feat, &packet_size_bytes, &work_cnt);
 		if (!dropped) {
 			bytes += packet_size_bytes;
 			ok_packet_cnt++;
 		}
 		total_packet_cnt++;
 		idx = rx->cnt & rx->mask;
 		desc = &rx->desc.desc_ring[idx];
 		work_done += work_cnt;
 	}

 	if (!work_done && rx->fill_cnt - rx->cnt > rx->db_threshold)
 		return 0;

 	if (work_done) {
 		u64_stats_update_begin(&rx->statss);
 		rx->rpackets += ok_packet_cnt;
 		rx->rbytes += bytes;
 		u64_stats_update_end(&rx->statss);
 	}

 	/* restock ring slots */
 	if (!rx->data.raw_addressing) {
 		/* In QPL mode buffs are refilled as the desc are processed */
 		rx->fill_cnt += work_done;
 	} else if (rx->fill_cnt - rx->cnt <= rx->db_threshold) {
 		/* In raw addressing mode buffs are only refilled if the avail
 		 * falls below a threshold.
 		 */
 		if (!gve_rx_refill_buffers(priv, rx))
 			return 0;

 		/* If we were not able to completely refill buffers, we'll want
 		 * to schedule this queue for work again to refill buffers.
 		 */
 		if (rx->fill_cnt - rx->cnt <= rx->db_threshold) {
 			gve_rx_write_doorbell(priv, rx);
 			return budget;
 		}
 	}

 	gve_rx_write_doorbell(priv, rx);
 	return total_packet_cnt;
 }

 int gve_rx_poll(struct gve_notify_block *block, int budget)
 {
 	struct gve_rx_ring *rx = block->rx;
 	netdev_features_t feat;
 	int work_done = 0;

 	feat = block->napi.dev->features;

 	/* If budget is 0, do all the work */
 	if (budget == 0)
 		budget = INT_MAX;

 	if (budget > 0)
 		work_done = gve_clean_rx_done(rx, budget, feat);

 	return work_done;
 }
	// SPDX-License-Identifier: (GPL-2.0 OR MIT)
	/* Google virtual Ethernet (gve) driver
	*
	* Copyright (C) 2015-2021 Google, Inc.
	*/

	#include "gve.h"
	#include "gve_adminq.h"
	#include "gve_utils.h"
	#include <linux/etherdevice.h>

	static void gve_rx_free_buffer(struct device *dev,
	struct gve_rx_slot_page_info *page_info,
	union gve_rx_data_slot *data_slot)
	{
	dma_addr_t dma = (dma_addr_t)(be64_to_cpu(data_slot->addr) &
	GVE_DATA_SLOT_ADDR_PAGE_MASK);

	page_ref_sub(page_info->page, page_info->pagecnt_bias - 1);
	gve_free_page(dev, page_info->page, dma, DMA_FROM_DEVICE);
	}

	static void gve_rx_unfill_pages(struct gve_priv priv, struct gve_rx_ring rx)
	{
	u32 slots = rx->mask + 1;
	int i;

	if (rx->data.raw_addressing) {
	for (i = 0; i < slots; i++)
	gve_rx_free_buffer(&priv->pdev->dev, &rx->data.page_info[i],
	&rx->data.data_ring[i]);
	} else {
	for (i = 0; i < slots; i++)
	page_ref_sub(rx->data.page_info[i].page,
	rx->data.page_info[i].pagecnt_bias - 1);
	gve_unassign_qpl(priv, rx->data.qpl->id);
	rx->data.qpl = NULL;
	}
	kvfree(rx->data.page_info);
	rx->data.page_info = NULL;
	}

	static void gve_rx_free_ring(struct gve_priv *priv, int idx)
	{
	struct gve_rx_ring *rx = &priv->rx[idx];
	struct device *dev = &priv->pdev->dev;
	u32 slots = rx->mask + 1;
	size_t bytes;

	gve_rx_remove_from_block(priv, idx);

	bytes = sizeof(struct gve_rx_desc) * priv->rx_desc_cnt;
	dma_free_coherent(dev, bytes, rx->desc.desc_ring, rx->desc.bus);
	rx->desc.desc_ring = NULL;

	dma_free_coherent(dev, sizeof(*rx->q_resources),
	rx->q_resources, rx->q_resources_bus);
	rx->q_resources = NULL;

	gve_rx_unfill_pages(priv, rx);

	bytes = sizeof(rx->data.data_ring) slots;
	dma_free_coherent(dev, bytes, rx->data.data_ring,
	rx->data.data_bus);
	rx->data.data_ring = NULL;
	netif_dbg(priv, drv, priv->dev, "freed rx ring %d\n", idx);
	}

	static void gve_setup_rx_buffer(struct gve_rx_slot_page_info *page_info,
	dma_addr_t addr, struct page page, __be64 slot_addr)
	{
	page_info->page = page;
	page_info->page_offset = 0;
	page_info->page_address = page_address(page);
	*slot_addr = cpu_to_be64(addr);
	/* The page already has 1 ref */
	page_ref_add(page, INT_MAX - 1);
	page_info->pagecnt_bias = INT_MAX;
	}

	static int gve_rx_alloc_buffer(struct gve_priv priv, struct device dev,
	struct gve_rx_slot_page_info *page_info,
	union gve_rx_data_slot *data_slot)
	{
	struct page *page;
	dma_addr_t dma;
	int err;

	err = gve_alloc_page(priv, dev, &page, &dma, DMA_FROM_DEVICE,
	GFP_ATOMIC);
	if (err)
	return err;

	gve_setup_rx_buffer(page_info, dma, page, &data_slot->addr);
	return 0;
	}

	static int gve_prefill_rx_pages(struct gve_rx_ring *rx)
	{
	struct gve_priv *priv = rx->gve;
	u32 slots;
	int err;
	int i;

	/* Allocate one page per Rx queue slot. Each page is split into two
	* packet buffers, when possible we "page flip" between the two.
	*/
	slots = rx->mask + 1;

	rx->data.page_info = kvzalloc(slots *
	sizeof(*rx->data.page_info), GFP_KERNEL);
	if (!rx->data.page_info)
	return -ENOMEM;

	if (!rx->data.raw_addressing) {
	rx->data.qpl = gve_assign_rx_qpl(priv);
	if (!rx->data.qpl) {
	kvfree(rx->data.page_info);
	rx->data.page_info = NULL;
	return -ENOMEM;
	}
	}
	for (i = 0; i < slots; i++) {
	if (!rx->data.raw_addressing) {
	struct page *page = rx->data.qpl->pages[i];
	dma_addr_t addr = i * PAGE_SIZE;

	gve_setup_rx_buffer(&rx->data.page_info[i], addr, page,
	&rx->data.data_ring[i].qpl_offset);
	continue;
	}
	err = gve_rx_alloc_buffer(priv, &priv->pdev->dev, &rx->data.page_info[i],
	&rx->data.data_ring[i]);
	if (err)
	goto alloc_err;
	}

	return slots;
	alloc_err:
	while (i--)
	gve_rx_free_buffer(&priv->pdev->dev,
	&rx->data.page_info[i],
	&rx->data.data_ring[i]);
	return err;
	}

	static void gve_rx_ctx_clear(struct gve_rx_ctx *ctx)
	{
	ctx->curr_frag_cnt = 0;
	ctx->total_expected_size = 0;
	ctx->expected_frag_cnt = 0;
	ctx->skb_head = NULL;
	ctx->skb_tail = NULL;
	ctx->reuse_frags = false;
	}

	static int gve_rx_alloc_ring(struct gve_priv *priv, int idx)
	{
	struct gve_rx_ring *rx = &priv->rx[idx];
	struct device *hdev = &priv->pdev->dev;
	u32 slots, npages;
	int filled_pages;
	size_t bytes;
	int err;

	netif_dbg(priv, drv, priv->dev, "allocating rx ring\n");
	/* Make sure everything is zeroed to start with */
	memset(rx, 0, sizeof(*rx));

	rx->gve = priv;
	rx->q_num = idx;

	slots = priv->rx_data_slot_cnt;
	rx->mask = slots - 1;
	rx->data.raw_addressing = priv->queue_format == GVE_GQI_RDA_FORMAT;

	/* alloc rx data ring */
	bytes = sizeof(rx->data.data_ring) slots;
	rx->data.data_ring = dma_alloc_coherent(hdev, bytes,
	&rx->data.data_bus,
	GFP_KERNEL);
	if (!rx->data.data_ring)
	return -ENOMEM;
	filled_pages = gve_prefill_rx_pages(rx);
	if (filled_pages < 0) {
	err = -ENOMEM;
	goto abort_with_slots;
	}
	rx->fill_cnt = filled_pages;
	/* Ensure data ring slots (packet buffers) are visible. */
	dma_wmb();

	/* Alloc gve_queue_resources */
	rx->q_resources =
	dma_alloc_coherent(hdev,
	sizeof(*rx->q_resources),
	&rx->q_resources_bus,
	GFP_KERNEL);
	if (!rx->q_resources) {
	err = -ENOMEM;
	goto abort_filled;
	}
	netif_dbg(priv, drv, priv->dev, "rx[%d]->data.data_bus=%lx\n", idx,
	(unsigned long)rx->data.data_bus);

	/* alloc rx desc ring */
	bytes = sizeof(struct gve_rx_desc) * priv->rx_desc_cnt;
	npages = bytes / PAGE_SIZE;
	if (npages * PAGE_SIZE != bytes) {
	err = -EIO;
	goto abort_with_q_resources;
	}

	rx->desc.desc_ring = dma_alloc_coherent(hdev, bytes, &rx->desc.bus,
	GFP_KERNEL);
	if (!rx->desc.desc_ring) {
	err = -ENOMEM;
	goto abort_with_q_resources;
	}
	rx->cnt = 0;
	rx->db_threshold = priv->rx_desc_cnt / 2;
	rx->desc.seqno = 1;

	/* Allocating half-page buffers allows page-flipping which is faster
	* than copying or allocating new pages.
	*/
	rx->packet_buffer_size = PAGE_SIZE / 2;
	gve_rx_ctx_clear(&rx->ctx);
	gve_rx_add_to_block(priv, idx);

	return 0;

	abort_with_q_resources:
	dma_free_coherent(hdev, sizeof(*rx->q_resources),
	rx->q_resources, rx->q_resources_bus);
	rx->q_resources = NULL;
	abort_filled:
	gve_rx_unfill_pages(priv, rx);
	abort_with_slots:
	bytes = sizeof(rx->data.data_ring) slots;
	dma_free_coherent(hdev, bytes, rx->data.data_ring, rx->data.data_bus);
	rx->data.data_ring = NULL;

	return err;
	}

	int gve_rx_alloc_rings(struct gve_priv *priv)
	{
	int err = 0;
	int i;

	for (i = 0; i < priv->rx_cfg.num_queues; i++) {
	err = gve_rx_alloc_ring(priv, i);
	if (err) {
	netif_err(priv, drv, priv->dev,
	"Failed to alloc rx ring=%d: err=%d\n",
	i, err);
	break;
	}
	}
	/* Unallocate if there was an error */
	if (err) {
	int j;

	for (j = 0; j < i; j++)
	gve_rx_free_ring(priv, j);
	}
	return err;
	}

	void gve_rx_free_rings_gqi(struct gve_priv *priv)
	{
	int i;

	for (i = 0; i < priv->rx_cfg.num_queues; i++)
	gve_rx_free_ring(priv, i);
	}

	void gve_rx_write_doorbell(struct gve_priv priv, struct gve_rx_ring rx)
	{
	u32 db_idx = be32_to_cpu(rx->q_resources->db_index);

	iowrite32be(rx->fill_cnt, &priv->db_bar2[db_idx]);
	}

	static enum pkt_hash_types gve_rss_type(__be16 pkt_flags)
	{
	if (likely(pkt_flags & (GVE_RXF_TCP \| GVE_RXF_UDP)))
	return PKT_HASH_TYPE_L4;
	if (pkt_flags & (GVE_RXF_IPV4 \| GVE_RXF_IPV6))
	return PKT_HASH_TYPE_L3;
	return PKT_HASH_TYPE_L2;
	}

	static u16 gve_rx_ctx_padding(struct gve_rx_ctx *ctx)
	{
	return (ctx->curr_frag_cnt == 0) ? GVE_RX_PAD : 0;
	}

	static struct sk_buff gve_rx_add_frags(struct napi_struct napi,
	struct gve_rx_slot_page_info *page_info,
	u16 packet_buffer_size, u16 len,
	struct gve_rx_ctx *ctx)
	{
	u32 offset = page_info->page_offset + gve_rx_ctx_padding(ctx);
	struct sk_buff *skb;

	if (!ctx->skb_head)
	ctx->skb_head = napi_get_frags(napi);

	if (unlikely(!ctx->skb_head))
	return NULL;

	skb = ctx->skb_head;
	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page_info->page,
	offset, len, packet_buffer_size);

	return skb;
	}

	static void gve_rx_flip_buff(struct gve_rx_slot_page_info page_info, __be64 slot_addr)
	{
	const __be64 offset = cpu_to_be64(PAGE_SIZE / 2);

	/* "flip" to other packet buffer on this page */
	page_info->page_offset ^= PAGE_SIZE / 2;
	*(slot_addr) ^= offset;
	}

	static int gve_rx_can_recycle_buffer(struct gve_rx_slot_page_info *page_info)
	{
	int pagecount = page_count(page_info->page);

	/* This page is not being used by any SKBs - reuse */
	if (pagecount == page_info->pagecnt_bias)
	return 1;
	/* This page is still being used by an SKB - we can't reuse */
	else if (pagecount > page_info->pagecnt_bias)
	return 0;
	WARN(pagecount < page_info->pagecnt_bias,
	"Pagecount should never be less than the bias.");
	return -1;
	}

	static struct sk_buff *
	gve_rx_raw_addressing(struct device dev, struct net_device netdev,
	struct gve_rx_slot_page_info *page_info, u16 len,
	struct napi_struct *napi,
	union gve_rx_data_slot *data_slot,
	u16 packet_buffer_size, struct gve_rx_ctx *ctx)
	{
	struct sk_buff *skb = gve_rx_add_frags(napi, page_info, packet_buffer_size, len, ctx);

	if (!skb)
	return NULL;

	/* Optimistically stop the kernel from freeing the page.
	* We will check again in refill to determine if we need to alloc a
	* new page.
	*/
	gve_dec_pagecnt_bias(page_info);

	return skb;
	}

	static struct sk_buff *
	gve_rx_qpl(struct device dev, struct net_device netdev,
	struct gve_rx_ring rx, struct gve_rx_slot_page_info page_info,
	u16 len, struct napi_struct *napi,
	union gve_rx_data_slot *data_slot)
	{
	struct gve_rx_ctx *ctx = &rx->ctx;
	struct sk_buff *skb;

	/* if raw_addressing mode is not enabled gvnic can only receive into
	* registered segments. If the buffer can't be recycled, our only
	* choice is to copy the data out of it so that we can return it to the
	* device.
	*/
	if (ctx->reuse_frags) {
	skb = gve_rx_add_frags(napi, page_info, rx->packet_buffer_size, len, ctx);
	/* No point in recycling if we didn't get the skb */
	if (skb) {
	/* Make sure that the page isn't freed. */
	gve_dec_pagecnt_bias(page_info);
	gve_rx_flip_buff(page_info, &data_slot->qpl_offset);
	}
	} else {
	const u16 padding = gve_rx_ctx_padding(ctx);

	skb = gve_rx_copy(netdev, napi, page_info, len, padding, ctx);
	if (skb) {
	u64_stats_update_begin(&rx->statss);
	rx->rx_frag_copy_cnt++;
	u64_stats_update_end(&rx->statss);
	}
	}
	return skb;
	}

	#define GVE_PKTCONT_BIT_IS_SET(x) (GVE_RXF_PKT_CONT & (x))
	static u16 gve_rx_get_fragment_size(struct gve_rx_ctx ctx, struct gve_rx_desc desc)
	{
	return be16_to_cpu(desc->len) - gve_rx_ctx_padding(ctx);
	}

	static bool gve_rx_ctx_init(struct gve_rx_ctx ctx, struct gve_rx_ring rx)
	{
	bool qpl_mode = !rx->data.raw_addressing, packet_size_error = false;
	bool buffer_error = false, desc_error = false, seqno_error = false;
	struct gve_rx_slot_page_info *page_info;
	struct gve_priv *priv = rx->gve;
	u32 idx = rx->cnt & rx->mask;
	bool reuse_frags, can_flip;
	struct gve_rx_desc *desc;
	u16 packet_size = 0;
	u16 n_frags = 0;
	int recycle;

	/** In QPL mode, we only flip buffers when all buffers containing the packet
	* can be flipped. RDA can_flip decisions will be made later, per frag.
	*/
	can_flip = qpl_mode;
	reuse_frags = can_flip;
	do {
	u16 frag_size;

	n_frags++;
	desc = &rx->desc.desc_ring[idx];
	desc_error = unlikely(desc->flags_seq & GVE_RXF_ERR) \|\| desc_error;
	if (GVE_SEQNO(desc->flags_seq) != rx->desc.seqno) {
	seqno_error = true;
	netdev_warn(priv->dev,
	"RX seqno error: want=%d, got=%d, dropping packet and scheduling reset.",
	rx->desc.seqno, GVE_SEQNO(desc->flags_seq));
	}
	frag_size = be16_to_cpu(desc->len);
	packet_size += frag_size;
	if (frag_size > rx->packet_buffer_size) {
	packet_size_error = true;
	netdev_warn(priv->dev,
	"RX fragment error: packet_buffer_size=%d, frag_size=%d, dropping packet.",
	rx->packet_buffer_size, be16_to_cpu(desc->len));
	}
	page_info = &rx->data.page_info[idx];
	if (can_flip) {
	recycle = gve_rx_can_recycle_buffer(page_info);
	reuse_frags = reuse_frags && recycle > 0;
	buffer_error = buffer_error \|\| unlikely(recycle < 0);
	}
	idx = (idx + 1) & rx->mask;
	rx->desc.seqno = gve_next_seqno(rx->desc.seqno);
	} while (GVE_PKTCONT_BIT_IS_SET(desc->flags_seq));

	prefetch(rx->desc.desc_ring + idx);

	ctx->curr_frag_cnt = 0;
	ctx->total_expected_size = packet_size - GVE_RX_PAD;
	ctx->expected_frag_cnt = n_frags;
	ctx->skb_head = NULL;
	ctx->reuse_frags = reuse_frags;

	if (ctx->expected_frag_cnt > 1) {
	u64_stats_update_begin(&rx->statss);
	rx->rx_cont_packet_cnt++;
	u64_stats_update_end(&rx->statss);
	}
	if (ctx->total_expected_size > priv->rx_copybreak && !ctx->reuse_frags && qpl_mode) {
	u64_stats_update_begin(&rx->statss);
	rx->rx_copied_pkt++;
	u64_stats_update_end(&rx->statss);
	}

	if (unlikely(buffer_error \|\| seqno_error \|\| packet_size_error)) {
	gve_schedule_reset(priv);
	return false;
	}

	if (unlikely(desc_error)) {
	u64_stats_update_begin(&rx->statss);
	rx->rx_desc_err_dropped_pkt++;
	u64_stats_update_end(&rx->statss);
	return false;
	}
	return true;
	}

	static struct sk_buff gve_rx_skb(struct gve_priv priv, struct gve_rx_ring *rx,
	struct gve_rx_slot_page_info page_info, struct napi_struct napi,
	u16 len, union gve_rx_data_slot *data_slot)
	{
	struct net_device *netdev = priv->dev;
	struct gve_rx_ctx *ctx = &rx->ctx;
	struct sk_buff *skb = NULL;

	if (len <= priv->rx_copybreak && ctx->expected_frag_cnt == 1) {
	/* Just copy small packets */
	skb = gve_rx_copy(netdev, napi, page_info, len, GVE_RX_PAD, ctx);
	if (skb) {
	u64_stats_update_begin(&rx->statss);
	rx->rx_copied_pkt++;
	rx->rx_frag_copy_cnt++;
	rx->rx_copybreak_pkt++;
	u64_stats_update_end(&rx->statss);
	}
	} else {
	if (rx->data.raw_addressing) {
	int recycle = gve_rx_can_recycle_buffer(page_info);

	if (unlikely(recycle < 0)) {
	gve_schedule_reset(priv);
	return NULL;
	}
	page_info->can_flip = recycle;
	if (page_info->can_flip) {
	u64_stats_update_begin(&rx->statss);
	rx->rx_frag_flip_cnt++;
	u64_stats_update_end(&rx->statss);
	}
	skb = gve_rx_raw_addressing(&priv->pdev->dev, netdev,
	page_info, len, napi,
	data_slot,
	rx->packet_buffer_size, ctx);
	} else {
	if (ctx->reuse_frags) {
	u64_stats_update_begin(&rx->statss);
	rx->rx_frag_flip_cnt++;
	u64_stats_update_end(&rx->statss);
	}
	skb = gve_rx_qpl(&priv->pdev->dev, netdev, rx,
	page_info, len, napi, data_slot);
	}
	}
	return skb;
	}

	static bool gve_rx(struct gve_rx_ring *rx, netdev_features_t feat,
	u64 packet_size_bytes, u32 work_done)
	{
	struct gve_rx_slot_page_info *page_info;
	struct gve_rx_ctx *ctx = &rx->ctx;
	union gve_rx_data_slot *data_slot;
	struct gve_priv *priv = rx->gve;
	struct gve_rx_desc *first_desc;
	struct sk_buff *skb = NULL;
	struct gve_rx_desc *desc;
	struct napi_struct *napi;
	dma_addr_t page_bus;
	u32 work_cnt = 0;
	void *va;
	u32 idx;
	u16 len;

	idx = rx->cnt & rx->mask;
	first_desc = &rx->desc.desc_ring[idx];
	desc = first_desc;
	napi = &priv->ntfy_blocks[rx->ntfy_id].napi;

	if (unlikely(!gve_rx_ctx_init(ctx, rx)))
	goto skb_alloc_fail;

	while (ctx->curr_frag_cnt < ctx->expected_frag_cnt) {
	/* Prefetch two packet buffers ahead, we will need it soon. */
	page_info = &rx->data.page_info[(idx + 2) & rx->mask];
	va = page_info->page_address + page_info->page_offset;

	prefetch(page_info->page); /* Kernel page struct. */
	prefetch(va); /* Packet header. */
	prefetch(va + 64); /* Next cacheline too. */

	len = gve_rx_get_fragment_size(ctx, desc);

	page_info = &rx->data.page_info[idx];
	data_slot = &rx->data.data_ring[idx];
	page_bus = rx->data.raw_addressing ?
	be64_to_cpu(data_slot->addr) - page_info->page_offset :
	rx->data.qpl->page_buses[idx];
	dma_sync_single_for_cpu(&priv->pdev->dev, page_bus, PAGE_SIZE, DMA_FROM_DEVICE);

	skb = gve_rx_skb(priv, rx, page_info, napi, len, data_slot);
	if (!skb) {
	u64_stats_update_begin(&rx->statss);
	rx->rx_skb_alloc_fail++;
	u64_stats_update_end(&rx->statss);
	goto skb_alloc_fail;
	}

	ctx->curr_frag_cnt++;
	rx->cnt++;
	idx = rx->cnt & rx->mask;
	work_cnt++;
	desc = &rx->desc.desc_ring[idx];
	}

	if (likely(feat & NETIF_F_RXCSUM)) {
	/* NIC passes up the partial sum */
	if (first_desc->csum)
	skb->ip_summed = CHECKSUM_COMPLETE;
	else
	skb->ip_summed = CHECKSUM_NONE;
	skb->csum = csum_unfold(first_desc->csum);
	}

	/* parse flags & pass relevant info up */
	if (likely(feat & NETIF_F_RXHASH) &&
	gve_needs_rss(first_desc->flags_seq))
	skb_set_hash(skb, be32_to_cpu(first_desc->rss_hash),
	gve_rss_type(first_desc->flags_seq));

	*packet_size_bytes = skb->len + (skb->protocol ? ETH_HLEN : 0);
	*work_done = work_cnt;
	skb_record_rx_queue(skb, rx->q_num);
	if (skb_is_nonlinear(skb))
	napi_gro_frags(napi);
	else
	napi_gro_receive(napi, skb);

	gve_rx_ctx_clear(ctx);
	return true;

	skb_alloc_fail:
	if (napi->skb)
	napi_free_frags(napi);
	*packet_size_bytes = 0;
	*work_done = ctx->expected_frag_cnt;
	while (ctx->curr_frag_cnt < ctx->expected_frag_cnt) {
	rx->cnt++;
	ctx->curr_frag_cnt++;
	}
	gve_rx_ctx_clear(ctx);
	return false;
	}

	bool gve_rx_work_pending(struct gve_rx_ring *rx)
	{
	struct gve_rx_desc *desc;
	__be16 flags_seq;
	u32 next_idx;

	next_idx = rx->cnt & rx->mask;
	desc = rx->desc.desc_ring + next_idx;

	flags_seq = desc->flags_seq;

	return (GVE_SEQNO(flags_seq) == rx->desc.seqno);
	}

	static bool gve_rx_refill_buffers(struct gve_priv priv, struct gve_rx_ring rx)
	{
	int refill_target = rx->mask + 1;
	u32 fill_cnt = rx->fill_cnt;

	while (fill_cnt - rx->cnt < refill_target) {
	struct gve_rx_slot_page_info *page_info;
	u32 idx = fill_cnt & rx->mask;

	page_info = &rx->data.page_info[idx];
	if (page_info->can_flip) {
	/* The other half of the page is free because it was
	* free when we processed the descriptor. Flip to it.
	*/
	union gve_rx_data_slot *data_slot =
	&rx->data.data_ring[idx];

	gve_rx_flip_buff(page_info, &data_slot->addr);
	page_info->can_flip = 0;
	} else {
	/* It is possible that the networking stack has already
	* finished processing all outstanding packets in the buffer
	* and it can be reused.
	* Flipping is unnecessary here - if the networking stack still
	* owns half the page it is impossible to tell which half. Either
	* the whole page is free or it needs to be replaced.
	*/
	int recycle = gve_rx_can_recycle_buffer(page_info);

	if (recycle < 0) {
	if (!rx->data.raw_addressing)
	gve_schedule_reset(priv);
	return false;
	}
	if (!recycle) {
	/* We can't reuse the buffer - alloc a new one*/
	union gve_rx_data_slot *data_slot =
	&rx->data.data_ring[idx];
	struct device *dev = &priv->pdev->dev;
	gve_rx_free_buffer(dev, page_info, data_slot);
	page_info->page = NULL;
	if (gve_rx_alloc_buffer(priv, dev, page_info,
	data_slot)) {
	u64_stats_update_begin(&rx->statss);
	rx->rx_buf_alloc_fail++;
	u64_stats_update_end(&rx->statss);
	break;
	}
	}
	}
	fill_cnt++;
	}
	rx->fill_cnt = fill_cnt;
	return true;
	}

	static int gve_clean_rx_done(struct gve_rx_ring *rx, int budget,
	netdev_features_t feat)
	{
	u32 work_done = 0, total_packet_cnt = 0, ok_packet_cnt = 0;
	struct gve_priv *priv = rx->gve;
	u32 idx = rx->cnt & rx->mask;
	struct gve_rx_desc *desc;
	u64 bytes = 0;

	desc = &rx->desc.desc_ring[idx];
	while ((GVE_SEQNO(desc->flags_seq) == rx->desc.seqno) &&
	work_done < budget) {
	u64 packet_size_bytes = 0;
	u32 work_cnt = 0;
	bool dropped;

	netif_info(priv, rx_status, priv->dev,
	"[%d] idx=%d desc=%p desc->flags_seq=0x%x\n",
	rx->q_num, idx, desc, desc->flags_seq);
	netif_info(priv, rx_status, priv->dev,
	"[%d] seqno=%d rx->desc.seqno=%d\n",
	rx->q_num, GVE_SEQNO(desc->flags_seq),
	rx->desc.seqno);

	dropped = !gve_rx(rx, feat, &packet_size_bytes, &work_cnt);
	if (!dropped) {
	bytes += packet_size_bytes;
	ok_packet_cnt++;
	}
	total_packet_cnt++;
	idx = rx->cnt & rx->mask;
	desc = &rx->desc.desc_ring[idx];
	work_done += work_cnt;
	}

	if (!work_done && rx->fill_cnt - rx->cnt > rx->db_threshold)
	return 0;

	if (work_done) {
	u64_stats_update_begin(&rx->statss);
	rx->rpackets += ok_packet_cnt;
	rx->rbytes += bytes;
	u64_stats_update_end(&rx->statss);
	}

	/* restock ring slots */
	if (!rx->data.raw_addressing) {
	/* In QPL mode buffs are refilled as the desc are processed */
	rx->fill_cnt += work_done;
	} else if (rx->fill_cnt - rx->cnt <= rx->db_threshold) {
	/* In raw addressing mode buffs are only refilled if the avail
	* falls below a threshold.
	*/
	if (!gve_rx_refill_buffers(priv, rx))
	return 0;

	/* If we were not able to completely refill buffers, we'll want
	* to schedule this queue for work again to refill buffers.
	*/
	if (rx->fill_cnt - rx->cnt <= rx->db_threshold) {
	gve_rx_write_doorbell(priv, rx);
	return budget;
	}
	}

	gve_rx_write_doorbell(priv, rx);
	return total_packet_cnt;
	}

	int gve_rx_poll(struct gve_notify_block *block, int budget)
	{
	struct gve_rx_ring *rx = block->rx;
	netdev_features_t feat;
	int work_done = 0;

	feat = block->napi.dev->features;

	/* If budget is 0, do all the work */
	if (budget == 0)
	budget = INT_MAX;

	if (budget > 0)
	work_done = gve_clean_rx_done(rx, budget, feat);

	return work_done;
	}