| // SPDX-License-Identifier: GPL-2.0 |
| /* Marvell Octeon EP (EndPoint) VF Ethernet Driver |
| * |
| * Copyright (C) 2020 Marvell. |
| * |
| */ |
| |
| #include <linux/pci.h> |
| #include <linux/etherdevice.h> |
| #include <linux/vmalloc.h> |
| |
| #include "octep_vf_config.h" |
| #include "octep_vf_main.h" |
| |
| static void octep_vf_oq_reset_indices(struct octep_vf_oq *oq) |
| { |
| oq->host_read_idx = 0; |
| oq->host_refill_idx = 0; |
| oq->refill_count = 0; |
| oq->last_pkt_count = 0; |
| oq->pkts_pending = 0; |
| } |
| |
| /** |
| * octep_vf_oq_fill_ring_buffers() - fill initial receive buffers for Rx ring. |
| * |
| * @oq: Octeon Rx queue data structure. |
| * |
| * Return: 0, if successfully filled receive buffers for all descriptors. |
| * -ENOMEM, if failed to allocate a buffer or failed to map for DMA. |
| */ |
| static int octep_vf_oq_fill_ring_buffers(struct octep_vf_oq *oq) |
| { |
| struct octep_vf_oq_desc_hw *desc_ring = oq->desc_ring; |
| struct page *page; |
| u32 i; |
| |
| for (i = 0; i < oq->max_count; i++) { |
| page = dev_alloc_page(); |
| if (unlikely(!page)) { |
| dev_err(oq->dev, "Rx buffer alloc failed\n"); |
| goto rx_buf_alloc_err; |
| } |
| desc_ring[i].buffer_ptr = dma_map_page(oq->dev, page, 0, |
| PAGE_SIZE, |
| DMA_FROM_DEVICE); |
| if (dma_mapping_error(oq->dev, desc_ring[i].buffer_ptr)) { |
| dev_err(oq->dev, |
| "OQ-%d buffer alloc: DMA mapping error!\n", |
| oq->q_no); |
| goto dma_map_err; |
| } |
| oq->buff_info[i].page = page; |
| } |
| |
| return 0; |
| |
| dma_map_err: |
| put_page(page); |
| rx_buf_alloc_err: |
| while (i) { |
| i--; |
| dma_unmap_page(oq->dev, desc_ring[i].buffer_ptr, PAGE_SIZE, DMA_FROM_DEVICE); |
| put_page(oq->buff_info[i].page); |
| oq->buff_info[i].page = NULL; |
| } |
| |
| return -ENOMEM; |
| } |
| |
| /** |
| * octep_vf_oq_refill() - refill buffers for used Rx ring descriptors. |
| * |
| * @oct: Octeon device private data structure. |
| * @oq: Octeon Rx queue data structure. |
| * |
| * Return: number of descriptors successfully refilled with receive buffers. |
| */ |
| static int octep_vf_oq_refill(struct octep_vf_device *oct, struct octep_vf_oq *oq) |
| { |
| struct octep_vf_oq_desc_hw *desc_ring = oq->desc_ring; |
| struct page *page; |
| u32 refill_idx, i; |
| |
| refill_idx = oq->host_refill_idx; |
| for (i = 0; i < oq->refill_count; i++) { |
| page = dev_alloc_page(); |
| if (unlikely(!page)) { |
| dev_err(oq->dev, "refill: rx buffer alloc failed\n"); |
| oq->stats.alloc_failures++; |
| break; |
| } |
| |
| desc_ring[refill_idx].buffer_ptr = dma_map_page(oq->dev, page, 0, |
| PAGE_SIZE, DMA_FROM_DEVICE); |
| if (dma_mapping_error(oq->dev, desc_ring[refill_idx].buffer_ptr)) { |
| dev_err(oq->dev, |
| "OQ-%d buffer refill: DMA mapping error!\n", |
| oq->q_no); |
| put_page(page); |
| oq->stats.alloc_failures++; |
| break; |
| } |
| oq->buff_info[refill_idx].page = page; |
| refill_idx++; |
| if (refill_idx == oq->max_count) |
| refill_idx = 0; |
| } |
| oq->host_refill_idx = refill_idx; |
| oq->refill_count -= i; |
| |
| return i; |
| } |
| |
| /** |
| * octep_vf_setup_oq() - Setup a Rx queue. |
| * |
| * @oct: Octeon device private data structure. |
| * @q_no: Rx queue number to be setup. |
| * |
| * Allocate resources for a Rx queue. |
| */ |
| static int octep_vf_setup_oq(struct octep_vf_device *oct, int q_no) |
| { |
| struct octep_vf_oq *oq; |
| u32 desc_ring_size; |
| |
| oq = vzalloc(sizeof(*oq)); |
| if (!oq) |
| goto create_oq_fail; |
| oct->oq[q_no] = oq; |
| |
| oq->octep_vf_dev = oct; |
| oq->netdev = oct->netdev; |
| oq->dev = &oct->pdev->dev; |
| oq->q_no = q_no; |
| oq->max_count = CFG_GET_OQ_NUM_DESC(oct->conf); |
| oq->ring_size_mask = oq->max_count - 1; |
| oq->buffer_size = CFG_GET_OQ_BUF_SIZE(oct->conf); |
| oq->max_single_buffer_size = oq->buffer_size - OCTEP_VF_OQ_RESP_HW_SIZE; |
| |
| /* When the hardware/firmware supports additional capabilities, |
| * additional header is filled-in by Octeon after length field in |
| * Rx packets. this header contains additional packet information. |
| */ |
| if (oct->fw_info.rx_ol_flags) |
| oq->max_single_buffer_size -= OCTEP_VF_OQ_RESP_HW_EXT_SIZE; |
| |
| oq->refill_threshold = CFG_GET_OQ_REFILL_THRESHOLD(oct->conf); |
| |
| desc_ring_size = oq->max_count * OCTEP_VF_OQ_DESC_SIZE; |
| oq->desc_ring = dma_alloc_coherent(oq->dev, desc_ring_size, |
| &oq->desc_ring_dma, GFP_KERNEL); |
| |
| if (unlikely(!oq->desc_ring)) { |
| dev_err(oq->dev, |
| "Failed to allocate DMA memory for OQ-%d !!\n", q_no); |
| goto desc_dma_alloc_err; |
| } |
| |
| oq->buff_info = vzalloc(oq->max_count * OCTEP_VF_OQ_RECVBUF_SIZE); |
| |
| if (unlikely(!oq->buff_info)) { |
| dev_err(&oct->pdev->dev, |
| "Failed to allocate buffer info for OQ-%d\n", q_no); |
| goto buf_list_err; |
| } |
| |
| if (octep_vf_oq_fill_ring_buffers(oq)) |
| goto oq_fill_buff_err; |
| |
| octep_vf_oq_reset_indices(oq); |
| oct->hw_ops.setup_oq_regs(oct, q_no); |
| oct->num_oqs++; |
| |
| return 0; |
| |
| oq_fill_buff_err: |
| vfree(oq->buff_info); |
| oq->buff_info = NULL; |
| buf_list_err: |
| dma_free_coherent(oq->dev, desc_ring_size, |
| oq->desc_ring, oq->desc_ring_dma); |
| oq->desc_ring = NULL; |
| desc_dma_alloc_err: |
| vfree(oq); |
| oct->oq[q_no] = NULL; |
| create_oq_fail: |
| return -ENOMEM; |
| } |
| |
| /** |
| * octep_vf_oq_free_ring_buffers() - Free ring buffers. |
| * |
| * @oq: Octeon Rx queue data structure. |
| * |
| * Free receive buffers in unused Rx queue descriptors. |
| */ |
| static void octep_vf_oq_free_ring_buffers(struct octep_vf_oq *oq) |
| { |
| struct octep_vf_oq_desc_hw *desc_ring = oq->desc_ring; |
| int i; |
| |
| if (!oq->desc_ring || !oq->buff_info) |
| return; |
| |
| for (i = 0; i < oq->max_count; i++) { |
| if (oq->buff_info[i].page) { |
| dma_unmap_page(oq->dev, desc_ring[i].buffer_ptr, |
| PAGE_SIZE, DMA_FROM_DEVICE); |
| put_page(oq->buff_info[i].page); |
| oq->buff_info[i].page = NULL; |
| desc_ring[i].buffer_ptr = 0; |
| } |
| } |
| octep_vf_oq_reset_indices(oq); |
| } |
| |
| /** |
| * octep_vf_free_oq() - Free Rx queue resources. |
| * |
| * @oq: Octeon Rx queue data structure. |
| * |
| * Free all resources of a Rx queue. |
| */ |
| static int octep_vf_free_oq(struct octep_vf_oq *oq) |
| { |
| struct octep_vf_device *oct = oq->octep_vf_dev; |
| int q_no = oq->q_no; |
| |
| octep_vf_oq_free_ring_buffers(oq); |
| |
| vfree(oq->buff_info); |
| |
| if (oq->desc_ring) |
| dma_free_coherent(oq->dev, |
| oq->max_count * OCTEP_VF_OQ_DESC_SIZE, |
| oq->desc_ring, oq->desc_ring_dma); |
| |
| vfree(oq); |
| oct->oq[q_no] = NULL; |
| oct->num_oqs--; |
| return 0; |
| } |
| |
| /** |
| * octep_vf_setup_oqs() - setup resources for all Rx queues. |
| * |
| * @oct: Octeon device private data structure. |
| */ |
| int octep_vf_setup_oqs(struct octep_vf_device *oct) |
| { |
| int i, retval = 0; |
| |
| oct->num_oqs = 0; |
| for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) { |
| retval = octep_vf_setup_oq(oct, i); |
| if (retval) { |
| dev_err(&oct->pdev->dev, |
| "Failed to setup OQ(RxQ)-%d.\n", i); |
| goto oq_setup_err; |
| } |
| dev_dbg(&oct->pdev->dev, "Successfully setup OQ(RxQ)-%d.\n", i); |
| } |
| |
| return 0; |
| |
| oq_setup_err: |
| while (i) { |
| i--; |
| octep_vf_free_oq(oct->oq[i]); |
| } |
| return retval; |
| } |
| |
| /** |
| * octep_vf_oq_dbell_init() - Initialize Rx queue doorbell. |
| * |
| * @oct: Octeon device private data structure. |
| * |
| * Write number of descriptors to Rx queue doorbell register. |
| */ |
| void octep_vf_oq_dbell_init(struct octep_vf_device *oct) |
| { |
| int i; |
| |
| for (i = 0; i < oct->num_oqs; i++) |
| writel(oct->oq[i]->max_count, oct->oq[i]->pkts_credit_reg); |
| } |
| |
| /** |
| * octep_vf_free_oqs() - Free resources of all Rx queues. |
| * |
| * @oct: Octeon device private data structure. |
| */ |
| void octep_vf_free_oqs(struct octep_vf_device *oct) |
| { |
| int i; |
| |
| for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) { |
| if (!oct->oq[i]) |
| continue; |
| octep_vf_free_oq(oct->oq[i]); |
| dev_dbg(&oct->pdev->dev, |
| "Successfully freed OQ(RxQ)-%d.\n", i); |
| } |
| } |
| |
| /** |
| * octep_vf_oq_check_hw_for_pkts() - Check for new Rx packets. |
| * |
| * @oct: Octeon device private data structure. |
| * @oq: Octeon Rx queue data structure. |
| * |
| * Return: packets received after previous check. |
| */ |
| static int octep_vf_oq_check_hw_for_pkts(struct octep_vf_device *oct, |
| struct octep_vf_oq *oq) |
| { |
| u32 pkt_count, new_pkts; |
| |
| pkt_count = readl(oq->pkts_sent_reg); |
| new_pkts = pkt_count - oq->last_pkt_count; |
| |
| /* Clear the hardware packets counter register if the rx queue is |
| * being processed continuously with-in a single interrupt and |
| * reached half its max value. |
| * this counter is not cleared every time read, to save write cycles. |
| */ |
| if (unlikely(pkt_count > 0xF0000000U)) { |
| writel(pkt_count, oq->pkts_sent_reg); |
| pkt_count = readl(oq->pkts_sent_reg); |
| new_pkts += pkt_count; |
| } |
| oq->last_pkt_count = pkt_count; |
| oq->pkts_pending += new_pkts; |
| return new_pkts; |
| } |
| |
| /** |
| * __octep_vf_oq_process_rx() - Process hardware Rx queue and push to stack. |
| * |
| * @oct: Octeon device private data structure. |
| * @oq: Octeon Rx queue data structure. |
| * @pkts_to_process: number of packets to be processed. |
| * |
| * Process the new packets in Rx queue. |
| * Packets larger than single Rx buffer arrive in consecutive descriptors. |
| * But, count returned by the API only accounts full packets, not fragments. |
| * |
| * Return: number of packets processed and pushed to stack. |
| */ |
| static int __octep_vf_oq_process_rx(struct octep_vf_device *oct, |
| struct octep_vf_oq *oq, u16 pkts_to_process) |
| { |
| struct octep_vf_oq_resp_hw_ext *resp_hw_ext = NULL; |
| netdev_features_t feat = oq->netdev->features; |
| struct octep_vf_rx_buffer *buff_info; |
| struct octep_vf_oq_resp_hw *resp_hw; |
| u32 pkt, rx_bytes, desc_used; |
| u16 data_offset, rx_ol_flags; |
| struct sk_buff *skb; |
| u32 read_idx; |
| |
| read_idx = oq->host_read_idx; |
| rx_bytes = 0; |
| desc_used = 0; |
| for (pkt = 0; pkt < pkts_to_process; pkt++) { |
| buff_info = (struct octep_vf_rx_buffer *)&oq->buff_info[read_idx]; |
| dma_unmap_page(oq->dev, oq->desc_ring[read_idx].buffer_ptr, |
| PAGE_SIZE, DMA_FROM_DEVICE); |
| resp_hw = page_address(buff_info->page); |
| buff_info->page = NULL; |
| |
| /* Swap the length field that is in Big-Endian to CPU */ |
| buff_info->len = be64_to_cpu(resp_hw->length); |
| if (oct->fw_info.rx_ol_flags) { |
| /* Extended response header is immediately after |
| * response header (resp_hw) |
| */ |
| resp_hw_ext = (struct octep_vf_oq_resp_hw_ext *) |
| (resp_hw + 1); |
| buff_info->len -= OCTEP_VF_OQ_RESP_HW_EXT_SIZE; |
| /* Packet Data is immediately after |
| * extended response header. |
| */ |
| data_offset = OCTEP_VF_OQ_RESP_HW_SIZE + |
| OCTEP_VF_OQ_RESP_HW_EXT_SIZE; |
| rx_ol_flags = resp_hw_ext->rx_ol_flags; |
| } else { |
| /* Data is immediately after |
| * Hardware Rx response header. |
| */ |
| data_offset = OCTEP_VF_OQ_RESP_HW_SIZE; |
| rx_ol_flags = 0; |
| } |
| rx_bytes += buff_info->len; |
| |
| if (buff_info->len <= oq->max_single_buffer_size) { |
| skb = napi_build_skb((void *)resp_hw, PAGE_SIZE); |
| skb_reserve(skb, data_offset); |
| skb_put(skb, buff_info->len); |
| read_idx++; |
| desc_used++; |
| if (read_idx == oq->max_count) |
| read_idx = 0; |
| } else { |
| struct skb_shared_info *shinfo; |
| u16 data_len; |
| |
| skb = napi_build_skb((void *)resp_hw, PAGE_SIZE); |
| skb_reserve(skb, data_offset); |
| /* Head fragment includes response header(s); |
| * subsequent fragments contains only data. |
| */ |
| skb_put(skb, oq->max_single_buffer_size); |
| read_idx++; |
| desc_used++; |
| if (read_idx == oq->max_count) |
| read_idx = 0; |
| |
| shinfo = skb_shinfo(skb); |
| data_len = buff_info->len - oq->max_single_buffer_size; |
| while (data_len) { |
| dma_unmap_page(oq->dev, oq->desc_ring[read_idx].buffer_ptr, |
| PAGE_SIZE, DMA_FROM_DEVICE); |
| buff_info = (struct octep_vf_rx_buffer *) |
| &oq->buff_info[read_idx]; |
| if (data_len < oq->buffer_size) { |
| buff_info->len = data_len; |
| data_len = 0; |
| } else { |
| buff_info->len = oq->buffer_size; |
| data_len -= oq->buffer_size; |
| } |
| |
| skb_add_rx_frag(skb, shinfo->nr_frags, |
| buff_info->page, 0, |
| buff_info->len, |
| buff_info->len); |
| buff_info->page = NULL; |
| read_idx++; |
| desc_used++; |
| if (read_idx == oq->max_count) |
| read_idx = 0; |
| } |
| } |
| |
| skb->dev = oq->netdev; |
| skb->protocol = eth_type_trans(skb, skb->dev); |
| if (feat & NETIF_F_RXCSUM && |
| OCTEP_VF_RX_CSUM_VERIFIED(rx_ol_flags)) |
| skb->ip_summed = CHECKSUM_UNNECESSARY; |
| else |
| skb->ip_summed = CHECKSUM_NONE; |
| napi_gro_receive(oq->napi, skb); |
| } |
| |
| oq->host_read_idx = read_idx; |
| oq->refill_count += desc_used; |
| oq->stats.packets += pkt; |
| oq->stats.bytes += rx_bytes; |
| |
| return pkt; |
| } |
| |
| /** |
| * octep_vf_oq_process_rx() - Process Rx queue. |
| * |
| * @oq: Octeon Rx queue data structure. |
| * @budget: max number of packets can be processed in one invocation. |
| * |
| * Check for newly received packets and process them. |
| * Keeps checking for new packets until budget is used or no new packets seen. |
| * |
| * Return: number of packets processed. |
| */ |
| int octep_vf_oq_process_rx(struct octep_vf_oq *oq, int budget) |
| { |
| u32 pkts_available, pkts_processed, total_pkts_processed; |
| struct octep_vf_device *oct = oq->octep_vf_dev; |
| |
| pkts_available = 0; |
| pkts_processed = 0; |
| total_pkts_processed = 0; |
| while (total_pkts_processed < budget) { |
| /* update pending count only when current one exhausted */ |
| if (oq->pkts_pending == 0) |
| octep_vf_oq_check_hw_for_pkts(oct, oq); |
| pkts_available = min(budget - total_pkts_processed, |
| oq->pkts_pending); |
| if (!pkts_available) |
| break; |
| |
| pkts_processed = __octep_vf_oq_process_rx(oct, oq, |
| pkts_available); |
| oq->pkts_pending -= pkts_processed; |
| total_pkts_processed += pkts_processed; |
| } |
| |
| if (oq->refill_count >= oq->refill_threshold) { |
| u32 desc_refilled = octep_vf_oq_refill(oct, oq); |
| |
| /* flush pending writes before updating credits */ |
| smp_wmb(); |
| writel(desc_refilled, oq->pkts_credit_reg); |
| } |
| |
| return total_pkts_processed; |
| } |