| // SPDX-License-Identifier: GPL-2.0-only |
| /* Copyright (c) 2013-2016, The Linux Foundation. All rights reserved. |
| */ |
| |
| /* Qualcomm Technologies, Inc. EMAC Ethernet Controller MAC layer support |
| */ |
| |
| #include <linux/tcp.h> |
| #include <linux/ip.h> |
| #include <linux/ipv6.h> |
| #include <linux/crc32.h> |
| #include <linux/if_vlan.h> |
| #include <linux/jiffies.h> |
| #include <linux/phy.h> |
| #include <linux/of.h> |
| #include <net/ip6_checksum.h> |
| #include "emac.h" |
| #include "emac-sgmii.h" |
| |
| /* EMAC_MAC_CTRL */ |
| #define SINGLE_PAUSE_MODE 0x10000000 |
| #define DEBUG_MODE 0x08000000 |
| #define BROAD_EN 0x04000000 |
| #define MULTI_ALL 0x02000000 |
| #define RX_CHKSUM_EN 0x01000000 |
| #define HUGE 0x00800000 |
| #define SPEED(x) (((x) & 0x3) << 20) |
| #define SPEED_MASK SPEED(0x3) |
| #define SIMR 0x00080000 |
| #define TPAUSE 0x00010000 |
| #define PROM_MODE 0x00008000 |
| #define VLAN_STRIP 0x00004000 |
| #define PRLEN_BMSK 0x00003c00 |
| #define PRLEN_SHFT 10 |
| #define HUGEN 0x00000200 |
| #define FLCHK 0x00000100 |
| #define PCRCE 0x00000080 |
| #define CRCE 0x00000040 |
| #define FULLD 0x00000020 |
| #define MAC_LP_EN 0x00000010 |
| #define RXFC 0x00000008 |
| #define TXFC 0x00000004 |
| #define RXEN 0x00000002 |
| #define TXEN 0x00000001 |
| |
| /* EMAC_DESC_CTRL_3 */ |
| #define RFD_RING_SIZE_BMSK 0xfff |
| |
| /* EMAC_DESC_CTRL_4 */ |
| #define RX_BUFFER_SIZE_BMSK 0xffff |
| |
| /* EMAC_DESC_CTRL_6 */ |
| #define RRD_RING_SIZE_BMSK 0xfff |
| |
| /* EMAC_DESC_CTRL_9 */ |
| #define TPD_RING_SIZE_BMSK 0xffff |
| |
| /* EMAC_TXQ_CTRL_0 */ |
| #define NUM_TXF_BURST_PREF_BMSK 0xffff0000 |
| #define NUM_TXF_BURST_PREF_SHFT 16 |
| #define LS_8023_SP 0x80 |
| #define TXQ_MODE 0x40 |
| #define TXQ_EN 0x20 |
| #define IP_OP_SP 0x10 |
| #define NUM_TPD_BURST_PREF_BMSK 0xf |
| #define NUM_TPD_BURST_PREF_SHFT 0 |
| |
| /* EMAC_TXQ_CTRL_1 */ |
| #define JUMBO_TASK_OFFLOAD_THRESHOLD_BMSK 0x7ff |
| |
| /* EMAC_TXQ_CTRL_2 */ |
| #define TXF_HWM_BMSK 0xfff0000 |
| #define TXF_LWM_BMSK 0xfff |
| |
| /* EMAC_RXQ_CTRL_0 */ |
| #define RXQ_EN BIT(31) |
| #define CUT_THRU_EN BIT(30) |
| #define RSS_HASH_EN BIT(29) |
| #define NUM_RFD_BURST_PREF_BMSK 0x3f00000 |
| #define NUM_RFD_BURST_PREF_SHFT 20 |
| #define IDT_TABLE_SIZE_BMSK 0x1ff00 |
| #define IDT_TABLE_SIZE_SHFT 8 |
| #define SP_IPV6 0x80 |
| |
| /* EMAC_RXQ_CTRL_1 */ |
| #define JUMBO_1KAH_BMSK 0xf000 |
| #define JUMBO_1KAH_SHFT 12 |
| #define RFD_PREF_LOW_TH 0x10 |
| #define RFD_PREF_LOW_THRESHOLD_BMSK 0xfc0 |
| #define RFD_PREF_LOW_THRESHOLD_SHFT 6 |
| #define RFD_PREF_UP_TH 0x10 |
| #define RFD_PREF_UP_THRESHOLD_BMSK 0x3f |
| #define RFD_PREF_UP_THRESHOLD_SHFT 0 |
| |
| /* EMAC_RXQ_CTRL_2 */ |
| #define RXF_DOF_THRESFHOLD 0x1a0 |
| #define RXF_DOF_THRESHOLD_BMSK 0xfff0000 |
| #define RXF_DOF_THRESHOLD_SHFT 16 |
| #define RXF_UOF_THRESFHOLD 0xbe |
| #define RXF_UOF_THRESHOLD_BMSK 0xfff |
| #define RXF_UOF_THRESHOLD_SHFT 0 |
| |
| /* EMAC_RXQ_CTRL_3 */ |
| #define RXD_TIMER_BMSK 0xffff0000 |
| #define RXD_THRESHOLD_BMSK 0xfff |
| #define RXD_THRESHOLD_SHFT 0 |
| |
| /* EMAC_DMA_CTRL */ |
| #define DMAW_DLY_CNT_BMSK 0xf0000 |
| #define DMAW_DLY_CNT_SHFT 16 |
| #define DMAR_DLY_CNT_BMSK 0xf800 |
| #define DMAR_DLY_CNT_SHFT 11 |
| #define DMAR_REQ_PRI 0x400 |
| #define REGWRBLEN_BMSK 0x380 |
| #define REGWRBLEN_SHFT 7 |
| #define REGRDBLEN_BMSK 0x70 |
| #define REGRDBLEN_SHFT 4 |
| #define OUT_ORDER_MODE 0x4 |
| #define ENH_ORDER_MODE 0x2 |
| #define IN_ORDER_MODE 0x1 |
| |
| /* EMAC_MAILBOX_13 */ |
| #define RFD3_PROC_IDX_BMSK 0xfff0000 |
| #define RFD3_PROC_IDX_SHFT 16 |
| #define RFD3_PROD_IDX_BMSK 0xfff |
| #define RFD3_PROD_IDX_SHFT 0 |
| |
| /* EMAC_MAILBOX_2 */ |
| #define NTPD_CONS_IDX_BMSK 0xffff0000 |
| #define NTPD_CONS_IDX_SHFT 16 |
| |
| /* EMAC_MAILBOX_3 */ |
| #define RFD0_CONS_IDX_BMSK 0xfff |
| #define RFD0_CONS_IDX_SHFT 0 |
| |
| /* EMAC_MAILBOX_11 */ |
| #define H3TPD_PROD_IDX_BMSK 0xffff0000 |
| #define H3TPD_PROD_IDX_SHFT 16 |
| |
| /* EMAC_AXI_MAST_CTRL */ |
| #define DATA_BYTE_SWAP 0x8 |
| #define MAX_BOUND 0x2 |
| #define MAX_BTYPE 0x1 |
| |
| /* EMAC_MAILBOX_12 */ |
| #define H3TPD_CONS_IDX_BMSK 0xffff0000 |
| #define H3TPD_CONS_IDX_SHFT 16 |
| |
| /* EMAC_MAILBOX_9 */ |
| #define H2TPD_PROD_IDX_BMSK 0xffff |
| #define H2TPD_PROD_IDX_SHFT 0 |
| |
| /* EMAC_MAILBOX_10 */ |
| #define H1TPD_CONS_IDX_BMSK 0xffff0000 |
| #define H1TPD_CONS_IDX_SHFT 16 |
| #define H2TPD_CONS_IDX_BMSK 0xffff |
| #define H2TPD_CONS_IDX_SHFT 0 |
| |
| /* EMAC_ATHR_HEADER_CTRL */ |
| #define HEADER_CNT_EN 0x2 |
| #define HEADER_ENABLE 0x1 |
| |
| /* EMAC_MAILBOX_0 */ |
| #define RFD0_PROC_IDX_BMSK 0xfff0000 |
| #define RFD0_PROC_IDX_SHFT 16 |
| #define RFD0_PROD_IDX_BMSK 0xfff |
| #define RFD0_PROD_IDX_SHFT 0 |
| |
| /* EMAC_MAILBOX_5 */ |
| #define RFD1_PROC_IDX_BMSK 0xfff0000 |
| #define RFD1_PROC_IDX_SHFT 16 |
| #define RFD1_PROD_IDX_BMSK 0xfff |
| #define RFD1_PROD_IDX_SHFT 0 |
| |
| /* EMAC_MISC_CTRL */ |
| #define RX_UNCPL_INT_EN 0x1 |
| |
| /* EMAC_MAILBOX_7 */ |
| #define RFD2_CONS_IDX_BMSK 0xfff0000 |
| #define RFD2_CONS_IDX_SHFT 16 |
| #define RFD1_CONS_IDX_BMSK 0xfff |
| #define RFD1_CONS_IDX_SHFT 0 |
| |
| /* EMAC_MAILBOX_8 */ |
| #define RFD3_CONS_IDX_BMSK 0xfff |
| #define RFD3_CONS_IDX_SHFT 0 |
| |
| /* EMAC_MAILBOX_15 */ |
| #define NTPD_PROD_IDX_BMSK 0xffff |
| #define NTPD_PROD_IDX_SHFT 0 |
| |
| /* EMAC_MAILBOX_16 */ |
| #define H1TPD_PROD_IDX_BMSK 0xffff |
| #define H1TPD_PROD_IDX_SHFT 0 |
| |
| #define RXQ0_RSS_HSTYP_IPV6_TCP_EN 0x20 |
| #define RXQ0_RSS_HSTYP_IPV6_EN 0x10 |
| #define RXQ0_RSS_HSTYP_IPV4_TCP_EN 0x8 |
| #define RXQ0_RSS_HSTYP_IPV4_EN 0x4 |
| |
| /* EMAC_EMAC_WRAPPER_TX_TS_INX */ |
| #define EMAC_WRAPPER_TX_TS_EMPTY BIT(31) |
| #define EMAC_WRAPPER_TX_TS_INX_BMSK 0xffff |
| |
| struct emac_skb_cb { |
| u32 tpd_idx; |
| unsigned long jiffies; |
| }; |
| |
| #define EMAC_SKB_CB(skb) ((struct emac_skb_cb *)(skb)->cb) |
| #define EMAC_RSS_IDT_SIZE 256 |
| #define JUMBO_1KAH 0x4 |
| #define RXD_TH 0x100 |
| #define EMAC_TPD_LAST_FRAGMENT 0x80000000 |
| #define EMAC_TPD_TSTAMP_SAVE 0x80000000 |
| |
| /* EMAC Errors in emac_rrd.word[3] */ |
| #define EMAC_RRD_L4F BIT(14) |
| #define EMAC_RRD_IPF BIT(15) |
| #define EMAC_RRD_CRC BIT(21) |
| #define EMAC_RRD_FAE BIT(22) |
| #define EMAC_RRD_TRN BIT(23) |
| #define EMAC_RRD_RNT BIT(24) |
| #define EMAC_RRD_INC BIT(25) |
| #define EMAC_RRD_FOV BIT(29) |
| #define EMAC_RRD_LEN BIT(30) |
| |
| /* Error bits that will result in a received frame being discarded */ |
| #define EMAC_RRD_ERROR (EMAC_RRD_IPF | EMAC_RRD_CRC | EMAC_RRD_FAE | \ |
| EMAC_RRD_TRN | EMAC_RRD_RNT | EMAC_RRD_INC | \ |
| EMAC_RRD_FOV | EMAC_RRD_LEN) |
| #define EMAC_RRD_STATS_DW_IDX 3 |
| |
| #define EMAC_RRD(RXQ, SIZE, IDX) ((RXQ)->rrd.v_addr + (SIZE * (IDX))) |
| #define EMAC_RFD(RXQ, SIZE, IDX) ((RXQ)->rfd.v_addr + (SIZE * (IDX))) |
| #define EMAC_TPD(TXQ, SIZE, IDX) ((TXQ)->tpd.v_addr + (SIZE * (IDX))) |
| |
| #define GET_RFD_BUFFER(RXQ, IDX) (&((RXQ)->rfd.rfbuff[(IDX)])) |
| #define GET_TPD_BUFFER(RTQ, IDX) (&((RTQ)->tpd.tpbuff[(IDX)])) |
| |
| #define EMAC_TX_POLL_HWTXTSTAMP_THRESHOLD 8 |
| |
| #define ISR_RX_PKT (\ |
| RX_PKT_INT0 |\ |
| RX_PKT_INT1 |\ |
| RX_PKT_INT2 |\ |
| RX_PKT_INT3) |
| |
| void emac_mac_multicast_addr_set(struct emac_adapter *adpt, u8 *addr) |
| { |
| u32 crc32, bit, reg, mta; |
| |
| /* Calculate the CRC of the MAC address */ |
| crc32 = ether_crc(ETH_ALEN, addr); |
| |
| /* The HASH Table is an array of 2 32-bit registers. It is |
| * treated like an array of 64 bits (BitArray[hash_value]). |
| * Use the upper 6 bits of the above CRC as the hash value. |
| */ |
| reg = (crc32 >> 31) & 0x1; |
| bit = (crc32 >> 26) & 0x1F; |
| |
| mta = readl(adpt->base + EMAC_HASH_TAB_REG0 + (reg << 2)); |
| mta |= BIT(bit); |
| writel(mta, adpt->base + EMAC_HASH_TAB_REG0 + (reg << 2)); |
| } |
| |
| void emac_mac_multicast_addr_clear(struct emac_adapter *adpt) |
| { |
| writel(0, adpt->base + EMAC_HASH_TAB_REG0); |
| writel(0, adpt->base + EMAC_HASH_TAB_REG1); |
| } |
| |
| /* definitions for RSS */ |
| #define EMAC_RSS_KEY(_i, _type) \ |
| (EMAC_RSS_KEY0 + ((_i) * sizeof(_type))) |
| #define EMAC_RSS_TBL(_i, _type) \ |
| (EMAC_IDT_TABLE0 + ((_i) * sizeof(_type))) |
| |
| /* Config MAC modes */ |
| void emac_mac_mode_config(struct emac_adapter *adpt) |
| { |
| struct net_device *netdev = adpt->netdev; |
| u32 mac; |
| |
| mac = readl(adpt->base + EMAC_MAC_CTRL); |
| mac &= ~(VLAN_STRIP | PROM_MODE | MULTI_ALL | MAC_LP_EN); |
| |
| if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) |
| mac |= VLAN_STRIP; |
| |
| if (netdev->flags & IFF_PROMISC) |
| mac |= PROM_MODE; |
| |
| if (netdev->flags & IFF_ALLMULTI) |
| mac |= MULTI_ALL; |
| |
| writel(mac, adpt->base + EMAC_MAC_CTRL); |
| } |
| |
| /* Config descriptor rings */ |
| static void emac_mac_dma_rings_config(struct emac_adapter *adpt) |
| { |
| /* TPD (Transmit Packet Descriptor) */ |
| writel(upper_32_bits(adpt->tx_q.tpd.dma_addr), |
| adpt->base + EMAC_DESC_CTRL_1); |
| |
| writel(lower_32_bits(adpt->tx_q.tpd.dma_addr), |
| adpt->base + EMAC_DESC_CTRL_8); |
| |
| writel(adpt->tx_q.tpd.count & TPD_RING_SIZE_BMSK, |
| adpt->base + EMAC_DESC_CTRL_9); |
| |
| /* RFD (Receive Free Descriptor) & RRD (Receive Return Descriptor) */ |
| writel(upper_32_bits(adpt->rx_q.rfd.dma_addr), |
| adpt->base + EMAC_DESC_CTRL_0); |
| |
| writel(lower_32_bits(adpt->rx_q.rfd.dma_addr), |
| adpt->base + EMAC_DESC_CTRL_2); |
| writel(lower_32_bits(adpt->rx_q.rrd.dma_addr), |
| adpt->base + EMAC_DESC_CTRL_5); |
| |
| writel(adpt->rx_q.rfd.count & RFD_RING_SIZE_BMSK, |
| adpt->base + EMAC_DESC_CTRL_3); |
| writel(adpt->rx_q.rrd.count & RRD_RING_SIZE_BMSK, |
| adpt->base + EMAC_DESC_CTRL_6); |
| |
| writel(adpt->rxbuf_size & RX_BUFFER_SIZE_BMSK, |
| adpt->base + EMAC_DESC_CTRL_4); |
| |
| writel(0, adpt->base + EMAC_DESC_CTRL_11); |
| |
| /* Load all of the base addresses above and ensure that triggering HW to |
| * read ring pointers is flushed |
| */ |
| writel(1, adpt->base + EMAC_INTER_SRAM_PART9); |
| } |
| |
| /* Config transmit parameters */ |
| static void emac_mac_tx_config(struct emac_adapter *adpt) |
| { |
| u32 val; |
| |
| writel((EMAC_MAX_TX_OFFLOAD_THRESH >> 3) & |
| JUMBO_TASK_OFFLOAD_THRESHOLD_BMSK, adpt->base + EMAC_TXQ_CTRL_1); |
| |
| val = (adpt->tpd_burst << NUM_TPD_BURST_PREF_SHFT) & |
| NUM_TPD_BURST_PREF_BMSK; |
| |
| val |= TXQ_MODE | LS_8023_SP; |
| val |= (0x0100 << NUM_TXF_BURST_PREF_SHFT) & |
| NUM_TXF_BURST_PREF_BMSK; |
| |
| writel(val, adpt->base + EMAC_TXQ_CTRL_0); |
| emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_2, |
| (TXF_HWM_BMSK | TXF_LWM_BMSK), 0); |
| } |
| |
| /* Config receive parameters */ |
| static void emac_mac_rx_config(struct emac_adapter *adpt) |
| { |
| u32 val; |
| |
| val = (adpt->rfd_burst << NUM_RFD_BURST_PREF_SHFT) & |
| NUM_RFD_BURST_PREF_BMSK; |
| val |= (SP_IPV6 | CUT_THRU_EN); |
| |
| writel(val, adpt->base + EMAC_RXQ_CTRL_0); |
| |
| val = readl(adpt->base + EMAC_RXQ_CTRL_1); |
| val &= ~(JUMBO_1KAH_BMSK | RFD_PREF_LOW_THRESHOLD_BMSK | |
| RFD_PREF_UP_THRESHOLD_BMSK); |
| val |= (JUMBO_1KAH << JUMBO_1KAH_SHFT) | |
| (RFD_PREF_LOW_TH << RFD_PREF_LOW_THRESHOLD_SHFT) | |
| (RFD_PREF_UP_TH << RFD_PREF_UP_THRESHOLD_SHFT); |
| writel(val, adpt->base + EMAC_RXQ_CTRL_1); |
| |
| val = readl(adpt->base + EMAC_RXQ_CTRL_2); |
| val &= ~(RXF_DOF_THRESHOLD_BMSK | RXF_UOF_THRESHOLD_BMSK); |
| val |= (RXF_DOF_THRESFHOLD << RXF_DOF_THRESHOLD_SHFT) | |
| (RXF_UOF_THRESFHOLD << RXF_UOF_THRESHOLD_SHFT); |
| writel(val, adpt->base + EMAC_RXQ_CTRL_2); |
| |
| val = readl(adpt->base + EMAC_RXQ_CTRL_3); |
| val &= ~(RXD_TIMER_BMSK | RXD_THRESHOLD_BMSK); |
| val |= RXD_TH << RXD_THRESHOLD_SHFT; |
| writel(val, adpt->base + EMAC_RXQ_CTRL_3); |
| } |
| |
| /* Config dma */ |
| static void emac_mac_dma_config(struct emac_adapter *adpt) |
| { |
| u32 dma_ctrl = DMAR_REQ_PRI; |
| |
| switch (adpt->dma_order) { |
| case emac_dma_ord_in: |
| dma_ctrl |= IN_ORDER_MODE; |
| break; |
| case emac_dma_ord_enh: |
| dma_ctrl |= ENH_ORDER_MODE; |
| break; |
| case emac_dma_ord_out: |
| dma_ctrl |= OUT_ORDER_MODE; |
| break; |
| default: |
| break; |
| } |
| |
| dma_ctrl |= (((u32)adpt->dmar_block) << REGRDBLEN_SHFT) & |
| REGRDBLEN_BMSK; |
| dma_ctrl |= (((u32)adpt->dmaw_block) << REGWRBLEN_SHFT) & |
| REGWRBLEN_BMSK; |
| dma_ctrl |= (((u32)adpt->dmar_dly_cnt) << DMAR_DLY_CNT_SHFT) & |
| DMAR_DLY_CNT_BMSK; |
| dma_ctrl |= (((u32)adpt->dmaw_dly_cnt) << DMAW_DLY_CNT_SHFT) & |
| DMAW_DLY_CNT_BMSK; |
| |
| /* config DMA and ensure that configuration is flushed to HW */ |
| writel(dma_ctrl, adpt->base + EMAC_DMA_CTRL); |
| } |
| |
| /* set MAC address */ |
| static void emac_set_mac_address(struct emac_adapter *adpt, const u8 *addr) |
| { |
| u32 sta; |
| |
| /* for example: 00-A0-C6-11-22-33 |
| * 0<-->C6112233, 1<-->00A0. |
| */ |
| |
| /* low 32bit word */ |
| sta = (((u32)addr[2]) << 24) | (((u32)addr[3]) << 16) | |
| (((u32)addr[4]) << 8) | (((u32)addr[5])); |
| writel(sta, adpt->base + EMAC_MAC_STA_ADDR0); |
| |
| /* hight 32bit word */ |
| sta = (((u32)addr[0]) << 8) | (u32)addr[1]; |
| writel(sta, adpt->base + EMAC_MAC_STA_ADDR1); |
| } |
| |
| static void emac_mac_config(struct emac_adapter *adpt) |
| { |
| struct net_device *netdev = adpt->netdev; |
| unsigned int max_frame; |
| u32 val; |
| |
| emac_set_mac_address(adpt, netdev->dev_addr); |
| |
| max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; |
| adpt->rxbuf_size = netdev->mtu > EMAC_DEF_RX_BUF_SIZE ? |
| ALIGN(max_frame, 8) : EMAC_DEF_RX_BUF_SIZE; |
| |
| emac_mac_dma_rings_config(adpt); |
| |
| writel(netdev->mtu + ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN, |
| adpt->base + EMAC_MAX_FRAM_LEN_CTRL); |
| |
| emac_mac_tx_config(adpt); |
| emac_mac_rx_config(adpt); |
| emac_mac_dma_config(adpt); |
| |
| val = readl(adpt->base + EMAC_AXI_MAST_CTRL); |
| val &= ~(DATA_BYTE_SWAP | MAX_BOUND); |
| val |= MAX_BTYPE; |
| writel(val, adpt->base + EMAC_AXI_MAST_CTRL); |
| writel(0, adpt->base + EMAC_CLK_GATE_CTRL); |
| writel(RX_UNCPL_INT_EN, adpt->base + EMAC_MISC_CTRL); |
| } |
| |
| void emac_mac_reset(struct emac_adapter *adpt) |
| { |
| emac_mac_stop(adpt); |
| |
| emac_reg_update32(adpt->base + EMAC_DMA_MAS_CTRL, 0, SOFT_RST); |
| usleep_range(100, 150); /* reset may take up to 100usec */ |
| |
| /* interrupt clear-on-read */ |
| emac_reg_update32(adpt->base + EMAC_DMA_MAS_CTRL, 0, INT_RD_CLR_EN); |
| } |
| |
| static void emac_mac_start(struct emac_adapter *adpt) |
| { |
| struct phy_device *phydev = adpt->phydev; |
| u32 mac, csr1; |
| |
| /* enable tx queue */ |
| emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_0, 0, TXQ_EN); |
| |
| /* enable rx queue */ |
| emac_reg_update32(adpt->base + EMAC_RXQ_CTRL_0, 0, RXQ_EN); |
| |
| /* enable mac control */ |
| mac = readl(adpt->base + EMAC_MAC_CTRL); |
| csr1 = readl(adpt->csr + EMAC_EMAC_WRAPPER_CSR1); |
| |
| mac |= TXEN | RXEN; /* enable RX/TX */ |
| |
| /* Configure MAC flow control. If set to automatic, then match |
| * whatever the PHY does. Otherwise, enable or disable it, depending |
| * on what the user configured via ethtool. |
| */ |
| mac &= ~(RXFC | TXFC); |
| |
| if (adpt->automatic) { |
| /* If it's set to automatic, then update our local values */ |
| adpt->rx_flow_control = phydev->pause; |
| adpt->tx_flow_control = phydev->pause != phydev->asym_pause; |
| } |
| mac |= adpt->rx_flow_control ? RXFC : 0; |
| mac |= adpt->tx_flow_control ? TXFC : 0; |
| |
| /* setup link speed */ |
| mac &= ~SPEED_MASK; |
| if (phydev->speed == SPEED_1000) { |
| mac |= SPEED(2); |
| csr1 |= FREQ_MODE; |
| } else { |
| mac |= SPEED(1); |
| csr1 &= ~FREQ_MODE; |
| } |
| |
| if (phydev->duplex == DUPLEX_FULL) |
| mac |= FULLD; |
| else |
| mac &= ~FULLD; |
| |
| /* other parameters */ |
| mac |= (CRCE | PCRCE); |
| mac |= ((adpt->preamble << PRLEN_SHFT) & PRLEN_BMSK); |
| mac |= BROAD_EN; |
| mac |= FLCHK; |
| mac &= ~RX_CHKSUM_EN; |
| mac &= ~(HUGEN | VLAN_STRIP | TPAUSE | SIMR | HUGE | MULTI_ALL | |
| DEBUG_MODE | SINGLE_PAUSE_MODE); |
| |
| /* Enable single-pause-frame mode if requested. |
| * |
| * If enabled, the EMAC will send a single pause frame when the RX |
| * queue is full. This normally leads to packet loss because |
| * the pause frame disables the remote MAC only for 33ms (the quanta), |
| * and then the remote MAC continues sending packets even though |
| * the RX queue is still full. |
| * |
| * If disabled, the EMAC sends a pause frame every 31ms until the RX |
| * queue is no longer full. Normally, this is the preferred |
| * method of operation. However, when the system is hung (e.g. |
| * cores are halted), the EMAC interrupt handler is never called |
| * and so the RX queue fills up quickly and stays full. The resuling |
| * non-stop "flood" of pause frames sometimes has the effect of |
| * disabling nearby switches. In some cases, other nearby switches |
| * are also affected, shutting down the entire network. |
| * |
| * The user can enable or disable single-pause-frame mode |
| * via ethtool. |
| */ |
| mac |= adpt->single_pause_mode ? SINGLE_PAUSE_MODE : 0; |
| |
| writel_relaxed(csr1, adpt->csr + EMAC_EMAC_WRAPPER_CSR1); |
| |
| writel_relaxed(mac, adpt->base + EMAC_MAC_CTRL); |
| |
| /* enable interrupt read clear, low power sleep mode and |
| * the irq moderators |
| */ |
| |
| writel_relaxed(adpt->irq_mod, adpt->base + EMAC_IRQ_MOD_TIM_INIT); |
| writel_relaxed(INT_RD_CLR_EN | LPW_MODE | IRQ_MODERATOR_EN | |
| IRQ_MODERATOR2_EN, adpt->base + EMAC_DMA_MAS_CTRL); |
| |
| emac_mac_mode_config(adpt); |
| |
| emac_reg_update32(adpt->base + EMAC_ATHR_HEADER_CTRL, |
| (HEADER_ENABLE | HEADER_CNT_EN), 0); |
| } |
| |
| void emac_mac_stop(struct emac_adapter *adpt) |
| { |
| emac_reg_update32(adpt->base + EMAC_RXQ_CTRL_0, RXQ_EN, 0); |
| emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_0, TXQ_EN, 0); |
| emac_reg_update32(adpt->base + EMAC_MAC_CTRL, TXEN | RXEN, 0); |
| usleep_range(1000, 1050); /* stopping mac may take upto 1msec */ |
| } |
| |
| /* Free all descriptors of given transmit queue */ |
| static void emac_tx_q_descs_free(struct emac_adapter *adpt) |
| { |
| struct emac_tx_queue *tx_q = &adpt->tx_q; |
| unsigned int i; |
| size_t size; |
| |
| /* ring already cleared, nothing to do */ |
| if (!tx_q->tpd.tpbuff) |
| return; |
| |
| for (i = 0; i < tx_q->tpd.count; i++) { |
| struct emac_buffer *tpbuf = GET_TPD_BUFFER(tx_q, i); |
| |
| if (tpbuf->dma_addr) { |
| dma_unmap_single(adpt->netdev->dev.parent, |
| tpbuf->dma_addr, tpbuf->length, |
| DMA_TO_DEVICE); |
| tpbuf->dma_addr = 0; |
| } |
| if (tpbuf->skb) { |
| dev_kfree_skb_any(tpbuf->skb); |
| tpbuf->skb = NULL; |
| } |
| } |
| |
| size = sizeof(struct emac_buffer) * tx_q->tpd.count; |
| memset(tx_q->tpd.tpbuff, 0, size); |
| |
| /* clear the descriptor ring */ |
| memset(tx_q->tpd.v_addr, 0, tx_q->tpd.size); |
| |
| tx_q->tpd.consume_idx = 0; |
| tx_q->tpd.produce_idx = 0; |
| } |
| |
| /* Free all descriptors of given receive queue */ |
| static void emac_rx_q_free_descs(struct emac_adapter *adpt) |
| { |
| struct device *dev = adpt->netdev->dev.parent; |
| struct emac_rx_queue *rx_q = &adpt->rx_q; |
| unsigned int i; |
| size_t size; |
| |
| /* ring already cleared, nothing to do */ |
| if (!rx_q->rfd.rfbuff) |
| return; |
| |
| for (i = 0; i < rx_q->rfd.count; i++) { |
| struct emac_buffer *rfbuf = GET_RFD_BUFFER(rx_q, i); |
| |
| if (rfbuf->dma_addr) { |
| dma_unmap_single(dev, rfbuf->dma_addr, rfbuf->length, |
| DMA_FROM_DEVICE); |
| rfbuf->dma_addr = 0; |
| } |
| if (rfbuf->skb) { |
| dev_kfree_skb(rfbuf->skb); |
| rfbuf->skb = NULL; |
| } |
| } |
| |
| size = sizeof(struct emac_buffer) * rx_q->rfd.count; |
| memset(rx_q->rfd.rfbuff, 0, size); |
| |
| /* clear the descriptor rings */ |
| memset(rx_q->rrd.v_addr, 0, rx_q->rrd.size); |
| rx_q->rrd.produce_idx = 0; |
| rx_q->rrd.consume_idx = 0; |
| |
| memset(rx_q->rfd.v_addr, 0, rx_q->rfd.size); |
| rx_q->rfd.produce_idx = 0; |
| rx_q->rfd.consume_idx = 0; |
| } |
| |
| /* Free all buffers associated with given transmit queue */ |
| static void emac_tx_q_bufs_free(struct emac_adapter *adpt) |
| { |
| struct emac_tx_queue *tx_q = &adpt->tx_q; |
| |
| emac_tx_q_descs_free(adpt); |
| |
| kfree(tx_q->tpd.tpbuff); |
| tx_q->tpd.tpbuff = NULL; |
| tx_q->tpd.v_addr = NULL; |
| tx_q->tpd.dma_addr = 0; |
| tx_q->tpd.size = 0; |
| } |
| |
| /* Allocate TX descriptor ring for the given transmit queue */ |
| static int emac_tx_q_desc_alloc(struct emac_adapter *adpt, |
| struct emac_tx_queue *tx_q) |
| { |
| struct emac_ring_header *ring_header = &adpt->ring_header; |
| int node = dev_to_node(adpt->netdev->dev.parent); |
| size_t size; |
| |
| size = sizeof(struct emac_buffer) * tx_q->tpd.count; |
| tx_q->tpd.tpbuff = kzalloc_node(size, GFP_KERNEL, node); |
| if (!tx_q->tpd.tpbuff) |
| return -ENOMEM; |
| |
| tx_q->tpd.size = tx_q->tpd.count * (adpt->tpd_size * 4); |
| tx_q->tpd.dma_addr = ring_header->dma_addr + ring_header->used; |
| tx_q->tpd.v_addr = ring_header->v_addr + ring_header->used; |
| ring_header->used += ALIGN(tx_q->tpd.size, 8); |
| tx_q->tpd.produce_idx = 0; |
| tx_q->tpd.consume_idx = 0; |
| |
| return 0; |
| } |
| |
| /* Free all buffers associated with given transmit queue */ |
| static void emac_rx_q_bufs_free(struct emac_adapter *adpt) |
| { |
| struct emac_rx_queue *rx_q = &adpt->rx_q; |
| |
| emac_rx_q_free_descs(adpt); |
| |
| kfree(rx_q->rfd.rfbuff); |
| rx_q->rfd.rfbuff = NULL; |
| |
| rx_q->rfd.v_addr = NULL; |
| rx_q->rfd.dma_addr = 0; |
| rx_q->rfd.size = 0; |
| |
| rx_q->rrd.v_addr = NULL; |
| rx_q->rrd.dma_addr = 0; |
| rx_q->rrd.size = 0; |
| } |
| |
| /* Allocate RX descriptor rings for the given receive queue */ |
| static int emac_rx_descs_alloc(struct emac_adapter *adpt) |
| { |
| struct emac_ring_header *ring_header = &adpt->ring_header; |
| int node = dev_to_node(adpt->netdev->dev.parent); |
| struct emac_rx_queue *rx_q = &adpt->rx_q; |
| size_t size; |
| |
| size = sizeof(struct emac_buffer) * rx_q->rfd.count; |
| rx_q->rfd.rfbuff = kzalloc_node(size, GFP_KERNEL, node); |
| if (!rx_q->rfd.rfbuff) |
| return -ENOMEM; |
| |
| rx_q->rrd.size = rx_q->rrd.count * (adpt->rrd_size * 4); |
| rx_q->rfd.size = rx_q->rfd.count * (adpt->rfd_size * 4); |
| |
| rx_q->rrd.dma_addr = ring_header->dma_addr + ring_header->used; |
| rx_q->rrd.v_addr = ring_header->v_addr + ring_header->used; |
| ring_header->used += ALIGN(rx_q->rrd.size, 8); |
| |
| rx_q->rfd.dma_addr = ring_header->dma_addr + ring_header->used; |
| rx_q->rfd.v_addr = ring_header->v_addr + ring_header->used; |
| ring_header->used += ALIGN(rx_q->rfd.size, 8); |
| |
| rx_q->rrd.produce_idx = 0; |
| rx_q->rrd.consume_idx = 0; |
| |
| rx_q->rfd.produce_idx = 0; |
| rx_q->rfd.consume_idx = 0; |
| |
| return 0; |
| } |
| |
| /* Allocate all TX and RX descriptor rings */ |
| int emac_mac_rx_tx_rings_alloc_all(struct emac_adapter *adpt) |
| { |
| struct emac_ring_header *ring_header = &adpt->ring_header; |
| struct device *dev = adpt->netdev->dev.parent; |
| unsigned int num_tx_descs = adpt->tx_desc_cnt; |
| unsigned int num_rx_descs = adpt->rx_desc_cnt; |
| int ret; |
| |
| adpt->tx_q.tpd.count = adpt->tx_desc_cnt; |
| |
| adpt->rx_q.rrd.count = adpt->rx_desc_cnt; |
| adpt->rx_q.rfd.count = adpt->rx_desc_cnt; |
| |
| /* Ring DMA buffer. Each ring may need up to 8 bytes for alignment, |
| * hence the additional padding bytes are allocated. |
| */ |
| ring_header->size = num_tx_descs * (adpt->tpd_size * 4) + |
| num_rx_descs * (adpt->rfd_size * 4) + |
| num_rx_descs * (adpt->rrd_size * 4) + |
| 8 + 2 * 8; /* 8 byte per one Tx and two Rx rings */ |
| |
| ring_header->used = 0; |
| ring_header->v_addr = dma_alloc_coherent(dev, ring_header->size, |
| &ring_header->dma_addr, |
| GFP_KERNEL); |
| if (!ring_header->v_addr) |
| return -ENOMEM; |
| |
| ring_header->used = ALIGN(ring_header->dma_addr, 8) - |
| ring_header->dma_addr; |
| |
| ret = emac_tx_q_desc_alloc(adpt, &adpt->tx_q); |
| if (ret) { |
| netdev_err(adpt->netdev, "error: Tx Queue alloc failed\n"); |
| goto err_alloc_tx; |
| } |
| |
| ret = emac_rx_descs_alloc(adpt); |
| if (ret) { |
| netdev_err(adpt->netdev, "error: Rx Queue alloc failed\n"); |
| goto err_alloc_rx; |
| } |
| |
| return 0; |
| |
| err_alloc_rx: |
| emac_tx_q_bufs_free(adpt); |
| err_alloc_tx: |
| dma_free_coherent(dev, ring_header->size, |
| ring_header->v_addr, ring_header->dma_addr); |
| |
| ring_header->v_addr = NULL; |
| ring_header->dma_addr = 0; |
| ring_header->size = 0; |
| ring_header->used = 0; |
| |
| return ret; |
| } |
| |
| /* Free all TX and RX descriptor rings */ |
| void emac_mac_rx_tx_rings_free_all(struct emac_adapter *adpt) |
| { |
| struct emac_ring_header *ring_header = &adpt->ring_header; |
| struct device *dev = adpt->netdev->dev.parent; |
| |
| emac_tx_q_bufs_free(adpt); |
| emac_rx_q_bufs_free(adpt); |
| |
| dma_free_coherent(dev, ring_header->size, |
| ring_header->v_addr, ring_header->dma_addr); |
| |
| ring_header->v_addr = NULL; |
| ring_header->dma_addr = 0; |
| ring_header->size = 0; |
| ring_header->used = 0; |
| } |
| |
| /* Initialize descriptor rings */ |
| static void emac_mac_rx_tx_ring_reset_all(struct emac_adapter *adpt) |
| { |
| unsigned int i; |
| |
| adpt->tx_q.tpd.produce_idx = 0; |
| adpt->tx_q.tpd.consume_idx = 0; |
| for (i = 0; i < adpt->tx_q.tpd.count; i++) |
| adpt->tx_q.tpd.tpbuff[i].dma_addr = 0; |
| |
| adpt->rx_q.rrd.produce_idx = 0; |
| adpt->rx_q.rrd.consume_idx = 0; |
| adpt->rx_q.rfd.produce_idx = 0; |
| adpt->rx_q.rfd.consume_idx = 0; |
| for (i = 0; i < adpt->rx_q.rfd.count; i++) |
| adpt->rx_q.rfd.rfbuff[i].dma_addr = 0; |
| } |
| |
| /* Produce new receive free descriptor */ |
| static void emac_mac_rx_rfd_create(struct emac_adapter *adpt, |
| struct emac_rx_queue *rx_q, |
| dma_addr_t addr) |
| { |
| u32 *hw_rfd = EMAC_RFD(rx_q, adpt->rfd_size, rx_q->rfd.produce_idx); |
| |
| *(hw_rfd++) = lower_32_bits(addr); |
| *hw_rfd = upper_32_bits(addr); |
| |
| if (++rx_q->rfd.produce_idx == rx_q->rfd.count) |
| rx_q->rfd.produce_idx = 0; |
| } |
| |
| /* Fill up receive queue's RFD with preallocated receive buffers */ |
| static void emac_mac_rx_descs_refill(struct emac_adapter *adpt, |
| struct emac_rx_queue *rx_q) |
| { |
| struct emac_buffer *curr_rxbuf; |
| struct emac_buffer *next_rxbuf; |
| unsigned int count = 0; |
| u32 next_produce_idx; |
| |
| next_produce_idx = rx_q->rfd.produce_idx + 1; |
| if (next_produce_idx == rx_q->rfd.count) |
| next_produce_idx = 0; |
| |
| curr_rxbuf = GET_RFD_BUFFER(rx_q, rx_q->rfd.produce_idx); |
| next_rxbuf = GET_RFD_BUFFER(rx_q, next_produce_idx); |
| |
| /* this always has a blank rx_buffer*/ |
| while (!next_rxbuf->dma_addr) { |
| struct sk_buff *skb; |
| int ret; |
| |
| skb = netdev_alloc_skb_ip_align(adpt->netdev, adpt->rxbuf_size); |
| if (!skb) |
| break; |
| |
| curr_rxbuf->dma_addr = |
| dma_map_single(adpt->netdev->dev.parent, skb->data, |
| adpt->rxbuf_size, DMA_FROM_DEVICE); |
| |
| ret = dma_mapping_error(adpt->netdev->dev.parent, |
| curr_rxbuf->dma_addr); |
| if (ret) { |
| dev_kfree_skb(skb); |
| break; |
| } |
| curr_rxbuf->skb = skb; |
| curr_rxbuf->length = adpt->rxbuf_size; |
| |
| emac_mac_rx_rfd_create(adpt, rx_q, curr_rxbuf->dma_addr); |
| next_produce_idx = rx_q->rfd.produce_idx + 1; |
| if (next_produce_idx == rx_q->rfd.count) |
| next_produce_idx = 0; |
| |
| curr_rxbuf = GET_RFD_BUFFER(rx_q, rx_q->rfd.produce_idx); |
| next_rxbuf = GET_RFD_BUFFER(rx_q, next_produce_idx); |
| count++; |
| } |
| |
| if (count) { |
| u32 prod_idx = (rx_q->rfd.produce_idx << rx_q->produce_shift) & |
| rx_q->produce_mask; |
| emac_reg_update32(adpt->base + rx_q->produce_reg, |
| rx_q->produce_mask, prod_idx); |
| } |
| } |
| |
| static void emac_adjust_link(struct net_device *netdev) |
| { |
| struct emac_adapter *adpt = netdev_priv(netdev); |
| struct phy_device *phydev = netdev->phydev; |
| |
| if (phydev->link) { |
| emac_mac_start(adpt); |
| emac_sgmii_link_change(adpt, true); |
| } else { |
| emac_sgmii_link_change(adpt, false); |
| emac_mac_stop(adpt); |
| } |
| |
| phy_print_status(phydev); |
| } |
| |
| /* Bringup the interface/HW */ |
| int emac_mac_up(struct emac_adapter *adpt) |
| { |
| struct net_device *netdev = adpt->netdev; |
| int ret; |
| |
| emac_mac_rx_tx_ring_reset_all(adpt); |
| emac_mac_config(adpt); |
| emac_mac_rx_descs_refill(adpt, &adpt->rx_q); |
| |
| adpt->phydev->irq = PHY_POLL; |
| ret = phy_connect_direct(netdev, adpt->phydev, emac_adjust_link, |
| PHY_INTERFACE_MODE_SGMII); |
| if (ret) { |
| netdev_err(adpt->netdev, "could not connect phy\n"); |
| return ret; |
| } |
| |
| phy_attached_print(adpt->phydev, NULL); |
| |
| /* enable mac irq */ |
| writel((u32)~DIS_INT, adpt->base + EMAC_INT_STATUS); |
| writel(adpt->irq.mask, adpt->base + EMAC_INT_MASK); |
| |
| phy_start(adpt->phydev); |
| |
| napi_enable(&adpt->rx_q.napi); |
| netif_start_queue(netdev); |
| |
| return 0; |
| } |
| |
| /* Bring down the interface/HW */ |
| void emac_mac_down(struct emac_adapter *adpt) |
| { |
| struct net_device *netdev = adpt->netdev; |
| |
| netif_stop_queue(netdev); |
| napi_disable(&adpt->rx_q.napi); |
| |
| phy_stop(adpt->phydev); |
| |
| /* Interrupts must be disabled before the PHY is disconnected, to |
| * avoid a race condition where adjust_link is null when we get |
| * an interrupt. |
| */ |
| writel(DIS_INT, adpt->base + EMAC_INT_STATUS); |
| writel(0, adpt->base + EMAC_INT_MASK); |
| synchronize_irq(adpt->irq.irq); |
| |
| phy_disconnect(adpt->phydev); |
| |
| emac_mac_reset(adpt); |
| |
| emac_tx_q_descs_free(adpt); |
| netdev_reset_queue(adpt->netdev); |
| emac_rx_q_free_descs(adpt); |
| } |
| |
| /* Consume next received packet descriptor */ |
| static bool emac_rx_process_rrd(struct emac_adapter *adpt, |
| struct emac_rx_queue *rx_q, |
| struct emac_rrd *rrd) |
| { |
| u32 *hw_rrd = EMAC_RRD(rx_q, adpt->rrd_size, rx_q->rrd.consume_idx); |
| |
| rrd->word[3] = *(hw_rrd + 3); |
| |
| if (!RRD_UPDT(rrd)) |
| return false; |
| |
| rrd->word[4] = 0; |
| rrd->word[5] = 0; |
| |
| rrd->word[0] = *(hw_rrd++); |
| rrd->word[1] = *(hw_rrd++); |
| rrd->word[2] = *(hw_rrd++); |
| |
| if (unlikely(RRD_NOR(rrd) != 1)) { |
| netdev_err(adpt->netdev, |
| "error: multi-RFD not support yet! nor:%lu\n", |
| RRD_NOR(rrd)); |
| } |
| |
| /* mark rrd as processed */ |
| RRD_UPDT_SET(rrd, 0); |
| *hw_rrd = rrd->word[3]; |
| |
| if (++rx_q->rrd.consume_idx == rx_q->rrd.count) |
| rx_q->rrd.consume_idx = 0; |
| |
| return true; |
| } |
| |
| /* Produce new transmit descriptor */ |
| static void emac_tx_tpd_create(struct emac_adapter *adpt, |
| struct emac_tx_queue *tx_q, struct emac_tpd *tpd) |
| { |
| u32 *hw_tpd; |
| |
| tx_q->tpd.last_produce_idx = tx_q->tpd.produce_idx; |
| hw_tpd = EMAC_TPD(tx_q, adpt->tpd_size, tx_q->tpd.produce_idx); |
| |
| if (++tx_q->tpd.produce_idx == tx_q->tpd.count) |
| tx_q->tpd.produce_idx = 0; |
| |
| *(hw_tpd++) = tpd->word[0]; |
| *(hw_tpd++) = tpd->word[1]; |
| *(hw_tpd++) = tpd->word[2]; |
| *hw_tpd = tpd->word[3]; |
| } |
| |
| /* Mark the last transmit descriptor as such (for the transmit packet) */ |
| static void emac_tx_tpd_mark_last(struct emac_adapter *adpt, |
| struct emac_tx_queue *tx_q) |
| { |
| u32 *hw_tpd = |
| EMAC_TPD(tx_q, adpt->tpd_size, tx_q->tpd.last_produce_idx); |
| u32 tmp_tpd; |
| |
| tmp_tpd = *(hw_tpd + 1); |
| tmp_tpd |= EMAC_TPD_LAST_FRAGMENT; |
| *(hw_tpd + 1) = tmp_tpd; |
| } |
| |
| static void emac_rx_rfd_clean(struct emac_rx_queue *rx_q, struct emac_rrd *rrd) |
| { |
| struct emac_buffer *rfbuf = rx_q->rfd.rfbuff; |
| u32 consume_idx = RRD_SI(rrd); |
| unsigned int i; |
| |
| for (i = 0; i < RRD_NOR(rrd); i++) { |
| rfbuf[consume_idx].skb = NULL; |
| if (++consume_idx == rx_q->rfd.count) |
| consume_idx = 0; |
| } |
| |
| rx_q->rfd.consume_idx = consume_idx; |
| rx_q->rfd.process_idx = consume_idx; |
| } |
| |
| /* Push the received skb to upper layers */ |
| static void emac_receive_skb(struct emac_rx_queue *rx_q, |
| struct sk_buff *skb, |
| u16 vlan_tag, bool vlan_flag) |
| { |
| if (vlan_flag) { |
| u16 vlan; |
| |
| EMAC_TAG_TO_VLAN(vlan_tag, vlan); |
| __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan); |
| } |
| |
| napi_gro_receive(&rx_q->napi, skb); |
| } |
| |
| /* Process receive event */ |
| void emac_mac_rx_process(struct emac_adapter *adpt, struct emac_rx_queue *rx_q, |
| int *num_pkts, int max_pkts) |
| { |
| u32 proc_idx, hw_consume_idx, num_consume_pkts; |
| struct net_device *netdev = adpt->netdev; |
| struct emac_buffer *rfbuf; |
| unsigned int count = 0; |
| struct emac_rrd rrd; |
| struct sk_buff *skb; |
| u32 reg; |
| |
| reg = readl_relaxed(adpt->base + rx_q->consume_reg); |
| |
| hw_consume_idx = (reg & rx_q->consume_mask) >> rx_q->consume_shift; |
| num_consume_pkts = (hw_consume_idx >= rx_q->rrd.consume_idx) ? |
| (hw_consume_idx - rx_q->rrd.consume_idx) : |
| (hw_consume_idx + rx_q->rrd.count - rx_q->rrd.consume_idx); |
| |
| do { |
| if (!num_consume_pkts) |
| break; |
| |
| if (!emac_rx_process_rrd(adpt, rx_q, &rrd)) |
| break; |
| |
| if (likely(RRD_NOR(&rrd) == 1)) { |
| /* good receive */ |
| rfbuf = GET_RFD_BUFFER(rx_q, RRD_SI(&rrd)); |
| dma_unmap_single(adpt->netdev->dev.parent, |
| rfbuf->dma_addr, rfbuf->length, |
| DMA_FROM_DEVICE); |
| rfbuf->dma_addr = 0; |
| skb = rfbuf->skb; |
| } else { |
| netdev_err(adpt->netdev, |
| "error: multi-RFD not support yet!\n"); |
| break; |
| } |
| emac_rx_rfd_clean(rx_q, &rrd); |
| num_consume_pkts--; |
| count++; |
| |
| /* Due to a HW issue in L4 check sum detection (UDP/TCP frags |
| * with DF set are marked as error), drop packets based on the |
| * error mask rather than the summary bit (ignoring L4F errors) |
| */ |
| if (rrd.word[EMAC_RRD_STATS_DW_IDX] & EMAC_RRD_ERROR) { |
| netif_dbg(adpt, rx_status, adpt->netdev, |
| "Drop error packet[RRD: 0x%x:0x%x:0x%x:0x%x]\n", |
| rrd.word[0], rrd.word[1], |
| rrd.word[2], rrd.word[3]); |
| |
| dev_kfree_skb(skb); |
| continue; |
| } |
| |
| skb_put(skb, RRD_PKT_SIZE(&rrd) - ETH_FCS_LEN); |
| skb->dev = netdev; |
| skb->protocol = eth_type_trans(skb, skb->dev); |
| if (netdev->features & NETIF_F_RXCSUM) |
| skb->ip_summed = RRD_L4F(&rrd) ? |
| CHECKSUM_NONE : CHECKSUM_UNNECESSARY; |
| else |
| skb_checksum_none_assert(skb); |
| |
| emac_receive_skb(rx_q, skb, (u16)RRD_CVALN_TAG(&rrd), |
| (bool)RRD_CVTAG(&rrd)); |
| |
| (*num_pkts)++; |
| } while (*num_pkts < max_pkts); |
| |
| if (count) { |
| proc_idx = (rx_q->rfd.process_idx << rx_q->process_shft) & |
| rx_q->process_mask; |
| emac_reg_update32(adpt->base + rx_q->process_reg, |
| rx_q->process_mask, proc_idx); |
| emac_mac_rx_descs_refill(adpt, rx_q); |
| } |
| } |
| |
| /* get the number of free transmit descriptors */ |
| static unsigned int emac_tpd_num_free_descs(struct emac_tx_queue *tx_q) |
| { |
| u32 produce_idx = tx_q->tpd.produce_idx; |
| u32 consume_idx = tx_q->tpd.consume_idx; |
| |
| return (consume_idx > produce_idx) ? |
| (consume_idx - produce_idx - 1) : |
| (tx_q->tpd.count + consume_idx - produce_idx - 1); |
| } |
| |
| /* Process transmit event */ |
| void emac_mac_tx_process(struct emac_adapter *adpt, struct emac_tx_queue *tx_q) |
| { |
| u32 reg = readl_relaxed(adpt->base + tx_q->consume_reg); |
| u32 hw_consume_idx, pkts_compl = 0, bytes_compl = 0; |
| struct emac_buffer *tpbuf; |
| |
| hw_consume_idx = (reg & tx_q->consume_mask) >> tx_q->consume_shift; |
| |
| while (tx_q->tpd.consume_idx != hw_consume_idx) { |
| tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.consume_idx); |
| if (tpbuf->dma_addr) { |
| dma_unmap_page(adpt->netdev->dev.parent, |
| tpbuf->dma_addr, tpbuf->length, |
| DMA_TO_DEVICE); |
| tpbuf->dma_addr = 0; |
| } |
| |
| if (tpbuf->skb) { |
| pkts_compl++; |
| bytes_compl += tpbuf->skb->len; |
| dev_consume_skb_irq(tpbuf->skb); |
| tpbuf->skb = NULL; |
| } |
| |
| if (++tx_q->tpd.consume_idx == tx_q->tpd.count) |
| tx_q->tpd.consume_idx = 0; |
| } |
| |
| netdev_completed_queue(adpt->netdev, pkts_compl, bytes_compl); |
| |
| if (netif_queue_stopped(adpt->netdev)) |
| if (emac_tpd_num_free_descs(tx_q) > (MAX_SKB_FRAGS + 1)) |
| netif_wake_queue(adpt->netdev); |
| } |
| |
| /* Initialize all queue data structures */ |
| void emac_mac_rx_tx_ring_init_all(struct platform_device *pdev, |
| struct emac_adapter *adpt) |
| { |
| adpt->rx_q.netdev = adpt->netdev; |
| |
| adpt->rx_q.produce_reg = EMAC_MAILBOX_0; |
| adpt->rx_q.produce_mask = RFD0_PROD_IDX_BMSK; |
| adpt->rx_q.produce_shift = RFD0_PROD_IDX_SHFT; |
| |
| adpt->rx_q.process_reg = EMAC_MAILBOX_0; |
| adpt->rx_q.process_mask = RFD0_PROC_IDX_BMSK; |
| adpt->rx_q.process_shft = RFD0_PROC_IDX_SHFT; |
| |
| adpt->rx_q.consume_reg = EMAC_MAILBOX_3; |
| adpt->rx_q.consume_mask = RFD0_CONS_IDX_BMSK; |
| adpt->rx_q.consume_shift = RFD0_CONS_IDX_SHFT; |
| |
| adpt->rx_q.irq = &adpt->irq; |
| adpt->rx_q.intr = adpt->irq.mask & ISR_RX_PKT; |
| |
| adpt->tx_q.produce_reg = EMAC_MAILBOX_15; |
| adpt->tx_q.produce_mask = NTPD_PROD_IDX_BMSK; |
| adpt->tx_q.produce_shift = NTPD_PROD_IDX_SHFT; |
| |
| adpt->tx_q.consume_reg = EMAC_MAILBOX_2; |
| adpt->tx_q.consume_mask = NTPD_CONS_IDX_BMSK; |
| adpt->tx_q.consume_shift = NTPD_CONS_IDX_SHFT; |
| } |
| |
| /* Fill up transmit descriptors with TSO and Checksum offload information */ |
| static int emac_tso_csum(struct emac_adapter *adpt, |
| struct emac_tx_queue *tx_q, |
| struct sk_buff *skb, |
| struct emac_tpd *tpd) |
| { |
| unsigned int hdr_len; |
| int ret; |
| |
| if (skb_is_gso(skb)) { |
| if (skb_header_cloned(skb)) { |
| ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC); |
| if (unlikely(ret)) |
| return ret; |
| } |
| |
| if (skb->protocol == htons(ETH_P_IP)) { |
| u32 pkt_len = ((unsigned char *)ip_hdr(skb) - skb->data) |
| + ntohs(ip_hdr(skb)->tot_len); |
| if (skb->len > pkt_len) { |
| ret = pskb_trim(skb, pkt_len); |
| if (unlikely(ret)) |
| return ret; |
| } |
| } |
| |
| hdr_len = skb_tcp_all_headers(skb); |
| if (unlikely(skb->len == hdr_len)) { |
| /* we only need to do csum */ |
| netif_warn(adpt, tx_err, adpt->netdev, |
| "tso not needed for packet with 0 data\n"); |
| goto do_csum; |
| } |
| |
| if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) { |
| ip_hdr(skb)->check = 0; |
| tcp_hdr(skb)->check = |
| ~csum_tcpudp_magic(ip_hdr(skb)->saddr, |
| ip_hdr(skb)->daddr, |
| 0, IPPROTO_TCP, 0); |
| TPD_IPV4_SET(tpd, 1); |
| } |
| |
| if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) { |
| /* ipv6 tso need an extra tpd */ |
| struct emac_tpd extra_tpd; |
| |
| memset(tpd, 0, sizeof(*tpd)); |
| memset(&extra_tpd, 0, sizeof(extra_tpd)); |
| |
| tcp_v6_gso_csum_prep(skb); |
| |
| TPD_PKT_LEN_SET(&extra_tpd, skb->len); |
| TPD_LSO_SET(&extra_tpd, 1); |
| TPD_LSOV_SET(&extra_tpd, 1); |
| emac_tx_tpd_create(adpt, tx_q, &extra_tpd); |
| TPD_LSOV_SET(tpd, 1); |
| } |
| |
| TPD_LSO_SET(tpd, 1); |
| TPD_TCPHDR_OFFSET_SET(tpd, skb_transport_offset(skb)); |
| TPD_MSS_SET(tpd, skb_shinfo(skb)->gso_size); |
| return 0; |
| } |
| |
| do_csum: |
| if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) { |
| unsigned int css, cso; |
| |
| cso = skb_transport_offset(skb); |
| if (unlikely(cso & 0x1)) { |
| netdev_err(adpt->netdev, |
| "error: payload offset should be even\n"); |
| return -EINVAL; |
| } |
| css = cso + skb->csum_offset; |
| |
| TPD_PAYLOAD_OFFSET_SET(tpd, cso >> 1); |
| TPD_CXSUM_OFFSET_SET(tpd, css >> 1); |
| TPD_CSX_SET(tpd, 1); |
| } |
| |
| return 0; |
| } |
| |
| /* Fill up transmit descriptors */ |
| static void emac_tx_fill_tpd(struct emac_adapter *adpt, |
| struct emac_tx_queue *tx_q, struct sk_buff *skb, |
| struct emac_tpd *tpd) |
| { |
| unsigned int nr_frags = skb_shinfo(skb)->nr_frags; |
| unsigned int first = tx_q->tpd.produce_idx; |
| unsigned int len = skb_headlen(skb); |
| struct emac_buffer *tpbuf = NULL; |
| unsigned int mapped_len = 0; |
| unsigned int i; |
| int count = 0; |
| int ret; |
| |
| /* if Large Segment Offload is (in TCP Segmentation Offload struct) */ |
| if (TPD_LSO(tpd)) { |
| mapped_len = skb_tcp_all_headers(skb); |
| |
| tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx); |
| tpbuf->length = mapped_len; |
| tpbuf->dma_addr = dma_map_page(adpt->netdev->dev.parent, |
| virt_to_page(skb->data), |
| offset_in_page(skb->data), |
| tpbuf->length, |
| DMA_TO_DEVICE); |
| ret = dma_mapping_error(adpt->netdev->dev.parent, |
| tpbuf->dma_addr); |
| if (ret) |
| goto error; |
| |
| TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr)); |
| TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr)); |
| TPD_BUF_LEN_SET(tpd, tpbuf->length); |
| emac_tx_tpd_create(adpt, tx_q, tpd); |
| count++; |
| } |
| |
| if (mapped_len < len) { |
| tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx); |
| tpbuf->length = len - mapped_len; |
| tpbuf->dma_addr = dma_map_page(adpt->netdev->dev.parent, |
| virt_to_page(skb->data + |
| mapped_len), |
| offset_in_page(skb->data + |
| mapped_len), |
| tpbuf->length, DMA_TO_DEVICE); |
| ret = dma_mapping_error(adpt->netdev->dev.parent, |
| tpbuf->dma_addr); |
| if (ret) |
| goto error; |
| |
| TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr)); |
| TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr)); |
| TPD_BUF_LEN_SET(tpd, tpbuf->length); |
| emac_tx_tpd_create(adpt, tx_q, tpd); |
| count++; |
| } |
| |
| for (i = 0; i < nr_frags; i++) { |
| skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; |
| |
| tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx); |
| tpbuf->length = skb_frag_size(frag); |
| tpbuf->dma_addr = skb_frag_dma_map(adpt->netdev->dev.parent, |
| frag, 0, tpbuf->length, |
| DMA_TO_DEVICE); |
| ret = dma_mapping_error(adpt->netdev->dev.parent, |
| tpbuf->dma_addr); |
| if (ret) |
| goto error; |
| |
| TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr)); |
| TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr)); |
| TPD_BUF_LEN_SET(tpd, tpbuf->length); |
| emac_tx_tpd_create(adpt, tx_q, tpd); |
| count++; |
| } |
| |
| /* The last tpd */ |
| wmb(); |
| emac_tx_tpd_mark_last(adpt, tx_q); |
| |
| /* The last buffer info contain the skb address, |
| * so it will be freed after unmap |
| */ |
| tpbuf->skb = skb; |
| |
| return; |
| |
| error: |
| /* One of the memory mappings failed, so undo everything */ |
| tx_q->tpd.produce_idx = first; |
| |
| while (count--) { |
| tpbuf = GET_TPD_BUFFER(tx_q, first); |
| dma_unmap_page(adpt->netdev->dev.parent, tpbuf->dma_addr, |
| tpbuf->length, DMA_TO_DEVICE); |
| tpbuf->dma_addr = 0; |
| tpbuf->length = 0; |
| |
| if (++first == tx_q->tpd.count) |
| first = 0; |
| } |
| |
| dev_kfree_skb(skb); |
| } |
| |
| /* Transmit the packet using specified transmit queue */ |
| netdev_tx_t emac_mac_tx_buf_send(struct emac_adapter *adpt, |
| struct emac_tx_queue *tx_q, |
| struct sk_buff *skb) |
| { |
| struct emac_tpd tpd; |
| u32 prod_idx; |
| int len; |
| |
| memset(&tpd, 0, sizeof(tpd)); |
| |
| if (emac_tso_csum(adpt, tx_q, skb, &tpd) != 0) { |
| dev_kfree_skb_any(skb); |
| return NETDEV_TX_OK; |
| } |
| |
| if (skb_vlan_tag_present(skb)) { |
| u16 tag; |
| |
| EMAC_VLAN_TO_TAG(skb_vlan_tag_get(skb), tag); |
| TPD_CVLAN_TAG_SET(&tpd, tag); |
| TPD_INSTC_SET(&tpd, 1); |
| } |
| |
| if (skb_network_offset(skb) != ETH_HLEN) |
| TPD_TYP_SET(&tpd, 1); |
| |
| len = skb->len; |
| emac_tx_fill_tpd(adpt, tx_q, skb, &tpd); |
| |
| netdev_sent_queue(adpt->netdev, len); |
| |
| /* Make sure the are enough free descriptors to hold one |
| * maximum-sized SKB. We need one desc for each fragment, |
| * one for the checksum (emac_tso_csum), one for TSO, and |
| * one for the SKB header. |
| */ |
| if (emac_tpd_num_free_descs(tx_q) < (MAX_SKB_FRAGS + 3)) |
| netif_stop_queue(adpt->netdev); |
| |
| /* update produce idx */ |
| prod_idx = (tx_q->tpd.produce_idx << tx_q->produce_shift) & |
| tx_q->produce_mask; |
| emac_reg_update32(adpt->base + tx_q->produce_reg, |
| tx_q->produce_mask, prod_idx); |
| |
| return NETDEV_TX_OK; |
| } |
