| // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) |
| /* Copyright (C) 2015-2018 Netronome Systems, Inc. */ |
| |
| /* |
| * nfp6000_pcie.c |
| * Authors: Jakub Kicinski <jakub.kicinski@netronome.com> |
| * Jason McMullan <jason.mcmullan@netronome.com> |
| * Rolf Neugebauer <rolf.neugebauer@netronome.com> |
| * |
| * Multiplexes the NFP BARs between NFP internal resources and |
| * implements the PCIe specific interface for generic CPP bus access. |
| * |
| * The BARs are managed with refcounts and are allocated/acquired |
| * using target, token and offset/size matching. The generic CPP bus |
| * abstraction builds upon this BAR interface. |
| */ |
| |
| #include <asm/unaligned.h> |
| #include <linux/kernel.h> |
| #include <linux/module.h> |
| #include <linux/kref.h> |
| #include <linux/io.h> |
| #include <linux/delay.h> |
| #include <linux/interrupt.h> |
| #include <linux/sort.h> |
| #include <linux/sched.h> |
| #include <linux/types.h> |
| #include <linux/pci.h> |
| |
| #include "nfp_cpp.h" |
| |
| #include "nfp6000/nfp6000.h" |
| |
| #include "nfp6000_pcie.h" |
| |
| #define NFP_PCIE_BAR(_pf) (0x30000 + ((_pf) & 7) * 0xc0) |
| #define NFP_PCIE_BAR_EXPLICIT_BAR0(_x, _y) \ |
| (0x00000080 + (0x40 * ((_x) & 0x3)) + (0x10 * ((_y) & 0x3))) |
| #define NFP_PCIE_BAR_EXPLICIT_BAR0_SignalType(_x) (((_x) & 0x3) << 30) |
| #define NFP_PCIE_BAR_EXPLICIT_BAR0_SignalType_of(_x) (((_x) >> 30) & 0x3) |
| #define NFP_PCIE_BAR_EXPLICIT_BAR0_Token(_x) (((_x) & 0x3) << 28) |
| #define NFP_PCIE_BAR_EXPLICIT_BAR0_Token_of(_x) (((_x) >> 28) & 0x3) |
| #define NFP_PCIE_BAR_EXPLICIT_BAR0_Address(_x) (((_x) & 0xffffff) << 0) |
| #define NFP_PCIE_BAR_EXPLICIT_BAR0_Address_of(_x) (((_x) >> 0) & 0xffffff) |
| #define NFP_PCIE_BAR_EXPLICIT_BAR1(_x, _y) \ |
| (0x00000084 + (0x40 * ((_x) & 0x3)) + (0x10 * ((_y) & 0x3))) |
| #define NFP_PCIE_BAR_EXPLICIT_BAR1_SignalRef(_x) (((_x) & 0x7f) << 24) |
| #define NFP_PCIE_BAR_EXPLICIT_BAR1_SignalRef_of(_x) (((_x) >> 24) & 0x7f) |
| #define NFP_PCIE_BAR_EXPLICIT_BAR1_DataMaster(_x) (((_x) & 0x3ff) << 14) |
| #define NFP_PCIE_BAR_EXPLICIT_BAR1_DataMaster_of(_x) (((_x) >> 14) & 0x3ff) |
| #define NFP_PCIE_BAR_EXPLICIT_BAR1_DataRef(_x) (((_x) & 0x3fff) << 0) |
| #define NFP_PCIE_BAR_EXPLICIT_BAR1_DataRef_of(_x) (((_x) >> 0) & 0x3fff) |
| #define NFP_PCIE_BAR_EXPLICIT_BAR2(_x, _y) \ |
| (0x00000088 + (0x40 * ((_x) & 0x3)) + (0x10 * ((_y) & 0x3))) |
| #define NFP_PCIE_BAR_EXPLICIT_BAR2_Target(_x) (((_x) & 0xf) << 28) |
| #define NFP_PCIE_BAR_EXPLICIT_BAR2_Target_of(_x) (((_x) >> 28) & 0xf) |
| #define NFP_PCIE_BAR_EXPLICIT_BAR2_Action(_x) (((_x) & 0x1f) << 23) |
| #define NFP_PCIE_BAR_EXPLICIT_BAR2_Action_of(_x) (((_x) >> 23) & 0x1f) |
| #define NFP_PCIE_BAR_EXPLICIT_BAR2_Length(_x) (((_x) & 0x1f) << 18) |
| #define NFP_PCIE_BAR_EXPLICIT_BAR2_Length_of(_x) (((_x) >> 18) & 0x1f) |
| #define NFP_PCIE_BAR_EXPLICIT_BAR2_ByteMask(_x) (((_x) & 0xff) << 10) |
| #define NFP_PCIE_BAR_EXPLICIT_BAR2_ByteMask_of(_x) (((_x) >> 10) & 0xff) |
| #define NFP_PCIE_BAR_EXPLICIT_BAR2_SignalMaster(_x) (((_x) & 0x3ff) << 0) |
| #define NFP_PCIE_BAR_EXPLICIT_BAR2_SignalMaster_of(_x) (((_x) >> 0) & 0x3ff) |
| |
| #define NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress(_x) (((_x) & 0x1f) << 16) |
| #define NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress_of(_x) (((_x) >> 16) & 0x1f) |
| #define NFP_PCIE_BAR_PCIE2CPP_BaseAddress(_x) (((_x) & 0xffff) << 0) |
| #define NFP_PCIE_BAR_PCIE2CPP_BaseAddress_of(_x) (((_x) >> 0) & 0xffff) |
| #define NFP_PCIE_BAR_PCIE2CPP_LengthSelect(_x) (((_x) & 0x3) << 27) |
| #define NFP_PCIE_BAR_PCIE2CPP_LengthSelect_of(_x) (((_x) >> 27) & 0x3) |
| #define NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT 0 |
| #define NFP_PCIE_BAR_PCIE2CPP_LengthSelect_64BIT 1 |
| #define NFP_PCIE_BAR_PCIE2CPP_LengthSelect_0BYTE 3 |
| #define NFP_PCIE_BAR_PCIE2CPP_MapType(_x) (((_x) & 0x7) << 29) |
| #define NFP_PCIE_BAR_PCIE2CPP_MapType_of(_x) (((_x) >> 29) & 0x7) |
| #define NFP_PCIE_BAR_PCIE2CPP_MapType_FIXED 0 |
| #define NFP_PCIE_BAR_PCIE2CPP_MapType_BULK 1 |
| #define NFP_PCIE_BAR_PCIE2CPP_MapType_TARGET 2 |
| #define NFP_PCIE_BAR_PCIE2CPP_MapType_GENERAL 3 |
| #define NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT0 4 |
| #define NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT1 5 |
| #define NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT2 6 |
| #define NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT3 7 |
| #define NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress(_x) (((_x) & 0xf) << 23) |
| #define NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress_of(_x) (((_x) >> 23) & 0xf) |
| #define NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress(_x) (((_x) & 0x3) << 21) |
| #define NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress_of(_x) (((_x) >> 21) & 0x3) |
| #define NFP_PCIE_EM 0x020000 |
| #define NFP_PCIE_SRAM 0x000000 |
| |
| /* Minimal size of the PCIe cfg memory we depend on being mapped, |
| * queue controller and DMA controller don't have to be covered. |
| */ |
| #define NFP_PCI_MIN_MAP_SIZE 0x080000 |
| |
| #define NFP_PCIE_P2C_FIXED_SIZE(bar) (1 << (bar)->bitsize) |
| #define NFP_PCIE_P2C_BULK_SIZE(bar) (1 << (bar)->bitsize) |
| #define NFP_PCIE_P2C_GENERAL_TARGET_OFFSET(bar, x) ((x) << ((bar)->bitsize - 2)) |
| #define NFP_PCIE_P2C_GENERAL_TOKEN_OFFSET(bar, x) ((x) << ((bar)->bitsize - 4)) |
| #define NFP_PCIE_P2C_GENERAL_SIZE(bar) (1 << ((bar)->bitsize - 4)) |
| |
| #define NFP_PCIE_CFG_BAR_PCIETOCPPEXPANSIONBAR(bar, slot) \ |
| (0x400 + ((bar) * 8 + (slot)) * 4) |
| |
| #define NFP_PCIE_CPP_BAR_PCIETOCPPEXPANSIONBAR(bar, slot) \ |
| (((bar) * 8 + (slot)) * 4) |
| |
| /* The number of explicit BARs to reserve. |
| * Minimum is 0, maximum is 4 on the NFP6000. |
| * The NFP3800 can have only one per PF. |
| */ |
| #define NFP_PCIE_EXPLICIT_BARS 2 |
| |
| struct nfp6000_pcie; |
| struct nfp6000_area_priv; |
| |
| /** |
| * struct nfp_bar - describes BAR configuration and usage |
| * @nfp: backlink to owner |
| * @barcfg: cached contents of BAR config CSR |
| * @base: the BAR's base CPP offset |
| * @mask: mask for the BAR aperture (read only) |
| * @bitsize: bitsize of BAR aperture (read only) |
| * @index: index of the BAR |
| * @refcnt: number of current users |
| * @iomem: mapped IO memory |
| * @resource: iomem resource window |
| */ |
| struct nfp_bar { |
| struct nfp6000_pcie *nfp; |
| u32 barcfg; |
| u64 base; /* CPP address base */ |
| u64 mask; /* Bit mask of the bar */ |
| u32 bitsize; /* Bit size of the bar */ |
| int index; |
| atomic_t refcnt; |
| |
| void __iomem *iomem; |
| struct resource *resource; |
| }; |
| |
| #define NFP_PCI_BAR_MAX (PCI_64BIT_BAR_COUNT * 8) |
| |
| struct nfp6000_pcie { |
| struct pci_dev *pdev; |
| struct device *dev; |
| |
| /* PCI BAR management */ |
| spinlock_t bar_lock; /* Protect the PCI2CPP BAR cache */ |
| int bars; |
| struct nfp_bar bar[NFP_PCI_BAR_MAX]; |
| wait_queue_head_t bar_waiters; |
| |
| /* Reserved BAR access */ |
| struct { |
| void __iomem *csr; |
| void __iomem *em; |
| void __iomem *expl[4]; |
| } iomem; |
| |
| /* Explicit IO access */ |
| struct { |
| struct mutex mutex; /* Lock access to this explicit group */ |
| u8 master_id; |
| u8 signal_ref; |
| void __iomem *data; |
| struct { |
| void __iomem *addr; |
| int bitsize; |
| int free[4]; |
| } group[4]; |
| } expl; |
| }; |
| |
| static u32 nfp_bar_maptype(struct nfp_bar *bar) |
| { |
| return NFP_PCIE_BAR_PCIE2CPP_MapType_of(bar->barcfg); |
| } |
| |
| static resource_size_t nfp_bar_resource_len(struct nfp_bar *bar) |
| { |
| return pci_resource_len(bar->nfp->pdev, (bar->index / 8) * 2) / 8; |
| } |
| |
| static resource_size_t nfp_bar_resource_start(struct nfp_bar *bar) |
| { |
| return pci_resource_start(bar->nfp->pdev, (bar->index / 8) * 2) |
| + nfp_bar_resource_len(bar) * (bar->index & 7); |
| } |
| |
| #define TARGET_WIDTH_32 4 |
| #define TARGET_WIDTH_64 8 |
| |
| static int |
| compute_bar(const struct nfp6000_pcie *nfp, const struct nfp_bar *bar, |
| u32 *bar_config, u64 *bar_base, |
| int tgt, int act, int tok, u64 offset, size_t size, int width) |
| { |
| int bitsize; |
| u32 newcfg; |
| |
| if (tgt >= NFP_CPP_NUM_TARGETS) |
| return -EINVAL; |
| |
| switch (width) { |
| case 8: |
| newcfg = NFP_PCIE_BAR_PCIE2CPP_LengthSelect( |
| NFP_PCIE_BAR_PCIE2CPP_LengthSelect_64BIT); |
| break; |
| case 4: |
| newcfg = NFP_PCIE_BAR_PCIE2CPP_LengthSelect( |
| NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT); |
| break; |
| case 0: |
| newcfg = NFP_PCIE_BAR_PCIE2CPP_LengthSelect( |
| NFP_PCIE_BAR_PCIE2CPP_LengthSelect_0BYTE); |
| break; |
| default: |
| return -EINVAL; |
| } |
| |
| if (act != NFP_CPP_ACTION_RW && act != 0) { |
| /* Fixed CPP mapping with specific action */ |
| u64 mask = ~(NFP_PCIE_P2C_FIXED_SIZE(bar) - 1); |
| |
| newcfg |= NFP_PCIE_BAR_PCIE2CPP_MapType( |
| NFP_PCIE_BAR_PCIE2CPP_MapType_FIXED); |
| newcfg |= NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress(tgt); |
| newcfg |= NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress(act); |
| newcfg |= NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress(tok); |
| |
| if ((offset & mask) != ((offset + size - 1) & mask)) |
| return -EINVAL; |
| offset &= mask; |
| |
| bitsize = 40 - 16; |
| } else { |
| u64 mask = ~(NFP_PCIE_P2C_BULK_SIZE(bar) - 1); |
| |
| /* Bulk mapping */ |
| newcfg |= NFP_PCIE_BAR_PCIE2CPP_MapType( |
| NFP_PCIE_BAR_PCIE2CPP_MapType_BULK); |
| newcfg |= NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress(tgt); |
| newcfg |= NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress(tok); |
| |
| if ((offset & mask) != ((offset + size - 1) & mask)) |
| return -EINVAL; |
| |
| offset &= mask; |
| |
| bitsize = 40 - 21; |
| } |
| |
| if (bar->bitsize < bitsize) |
| return -EINVAL; |
| |
| newcfg |= offset >> bitsize; |
| |
| if (bar_base) |
| *bar_base = offset; |
| |
| if (bar_config) |
| *bar_config = newcfg; |
| |
| return 0; |
| } |
| |
| static int |
| nfp6000_bar_write(struct nfp6000_pcie *nfp, struct nfp_bar *bar, u32 newcfg) |
| { |
| int base, slot; |
| int xbar; |
| |
| base = bar->index >> 3; |
| slot = bar->index & 7; |
| |
| if (nfp->iomem.csr) { |
| xbar = NFP_PCIE_CPP_BAR_PCIETOCPPEXPANSIONBAR(base, slot); |
| writel(newcfg, nfp->iomem.csr + xbar); |
| /* Readback to ensure BAR is flushed */ |
| readl(nfp->iomem.csr + xbar); |
| } else { |
| xbar = NFP_PCIE_CFG_BAR_PCIETOCPPEXPANSIONBAR(base, slot); |
| pci_write_config_dword(nfp->pdev, xbar, newcfg); |
| } |
| |
| bar->barcfg = newcfg; |
| |
| return 0; |
| } |
| |
| static int |
| reconfigure_bar(struct nfp6000_pcie *nfp, struct nfp_bar *bar, |
| int tgt, int act, int tok, u64 offset, size_t size, int width) |
| { |
| u64 newbase; |
| u32 newcfg; |
| int err; |
| |
| err = compute_bar(nfp, bar, &newcfg, &newbase, |
| tgt, act, tok, offset, size, width); |
| if (err) |
| return err; |
| |
| bar->base = newbase; |
| |
| return nfp6000_bar_write(nfp, bar, newcfg); |
| } |
| |
| /* Check if BAR can be used with the given parameters. */ |
| static int matching_bar(struct nfp_bar *bar, u32 tgt, u32 act, u32 tok, |
| u64 offset, size_t size, int width) |
| { |
| int bartgt, baract, bartok; |
| int barwidth; |
| u32 maptype; |
| |
| maptype = NFP_PCIE_BAR_PCIE2CPP_MapType_of(bar->barcfg); |
| bartgt = NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress_of(bar->barcfg); |
| bartok = NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress_of(bar->barcfg); |
| baract = NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress_of(bar->barcfg); |
| |
| barwidth = NFP_PCIE_BAR_PCIE2CPP_LengthSelect_of(bar->barcfg); |
| switch (barwidth) { |
| case NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT: |
| barwidth = 4; |
| break; |
| case NFP_PCIE_BAR_PCIE2CPP_LengthSelect_64BIT: |
| barwidth = 8; |
| break; |
| case NFP_PCIE_BAR_PCIE2CPP_LengthSelect_0BYTE: |
| barwidth = 0; |
| break; |
| default: |
| barwidth = -1; |
| break; |
| } |
| |
| switch (maptype) { |
| case NFP_PCIE_BAR_PCIE2CPP_MapType_TARGET: |
| bartok = -1; |
| fallthrough; |
| case NFP_PCIE_BAR_PCIE2CPP_MapType_BULK: |
| baract = NFP_CPP_ACTION_RW; |
| if (act == 0) |
| act = NFP_CPP_ACTION_RW; |
| fallthrough; |
| case NFP_PCIE_BAR_PCIE2CPP_MapType_FIXED: |
| break; |
| default: |
| /* We don't match explicit bars through the area interface */ |
| return 0; |
| } |
| |
| /* Make sure to match up the width */ |
| if (barwidth != width) |
| return 0; |
| |
| if ((bartgt < 0 || bartgt == tgt) && |
| (bartok < 0 || bartok == tok) && |
| (baract == act) && |
| bar->base <= offset && |
| (bar->base + (1 << bar->bitsize)) >= (offset + size)) |
| return 1; |
| |
| /* No match */ |
| return 0; |
| } |
| |
| static int |
| find_matching_bar(struct nfp6000_pcie *nfp, |
| u32 tgt, u32 act, u32 tok, u64 offset, size_t size, int width) |
| { |
| int n; |
| |
| for (n = 0; n < nfp->bars; n++) { |
| struct nfp_bar *bar = &nfp->bar[n]; |
| |
| if (matching_bar(bar, tgt, act, tok, offset, size, width)) |
| return n; |
| } |
| |
| return -1; |
| } |
| |
| /* Return EAGAIN if no resource is available */ |
| static int |
| find_unused_bar_noblock(const struct nfp6000_pcie *nfp, |
| int tgt, int act, int tok, |
| u64 offset, size_t size, int width) |
| { |
| int n, busy = 0; |
| |
| for (n = 0; n < nfp->bars; n++) { |
| const struct nfp_bar *bar = &nfp->bar[n]; |
| int err; |
| |
| if (!bar->bitsize) |
| continue; |
| |
| /* Just check to see if we can make it fit... */ |
| err = compute_bar(nfp, bar, NULL, NULL, |
| tgt, act, tok, offset, size, width); |
| if (err) |
| continue; |
| |
| if (!atomic_read(&bar->refcnt)) |
| return n; |
| |
| busy++; |
| } |
| |
| if (WARN(!busy, "No suitable BAR found for request tgt:0x%x act:0x%x tok:0x%x off:0x%llx size:%zd width:%d\n", |
| tgt, act, tok, offset, size, width)) |
| return -EINVAL; |
| |
| return -EAGAIN; |
| } |
| |
| static int |
| find_unused_bar_and_lock(struct nfp6000_pcie *nfp, |
| int tgt, int act, int tok, |
| u64 offset, size_t size, int width) |
| { |
| unsigned long flags; |
| int n; |
| |
| spin_lock_irqsave(&nfp->bar_lock, flags); |
| |
| n = find_unused_bar_noblock(nfp, tgt, act, tok, offset, size, width); |
| if (n < 0) |
| spin_unlock_irqrestore(&nfp->bar_lock, flags); |
| else |
| __release(&nfp->bar_lock); |
| |
| return n; |
| } |
| |
| static void nfp_bar_get(struct nfp6000_pcie *nfp, struct nfp_bar *bar) |
| { |
| atomic_inc(&bar->refcnt); |
| } |
| |
| static void nfp_bar_put(struct nfp6000_pcie *nfp, struct nfp_bar *bar) |
| { |
| if (atomic_dec_and_test(&bar->refcnt)) |
| wake_up_interruptible(&nfp->bar_waiters); |
| } |
| |
| static int |
| nfp_wait_for_bar(struct nfp6000_pcie *nfp, int *barnum, |
| u32 tgt, u32 act, u32 tok, u64 offset, size_t size, int width) |
| { |
| return wait_event_interruptible(nfp->bar_waiters, |
| (*barnum = find_unused_bar_and_lock(nfp, tgt, act, tok, |
| offset, size, width)) |
| != -EAGAIN); |
| } |
| |
| static int |
| nfp_alloc_bar(struct nfp6000_pcie *nfp, |
| u32 tgt, u32 act, u32 tok, |
| u64 offset, size_t size, int width, int nonblocking) |
| { |
| unsigned long irqflags; |
| int barnum, retval; |
| |
| if (size > (1 << 24)) |
| return -EINVAL; |
| |
| spin_lock_irqsave(&nfp->bar_lock, irqflags); |
| barnum = find_matching_bar(nfp, tgt, act, tok, offset, size, width); |
| if (barnum >= 0) { |
| /* Found a perfect match. */ |
| nfp_bar_get(nfp, &nfp->bar[barnum]); |
| spin_unlock_irqrestore(&nfp->bar_lock, irqflags); |
| return barnum; |
| } |
| |
| barnum = find_unused_bar_noblock(nfp, tgt, act, tok, |
| offset, size, width); |
| if (barnum < 0) { |
| if (nonblocking) |
| goto err_nobar; |
| |
| /* Wait until a BAR becomes available. The |
| * find_unused_bar function will reclaim the bar_lock |
| * if a free BAR is found. |
| */ |
| spin_unlock_irqrestore(&nfp->bar_lock, irqflags); |
| retval = nfp_wait_for_bar(nfp, &barnum, tgt, act, tok, |
| offset, size, width); |
| if (retval) |
| return retval; |
| __acquire(&nfp->bar_lock); |
| } |
| |
| nfp_bar_get(nfp, &nfp->bar[barnum]); |
| retval = reconfigure_bar(nfp, &nfp->bar[barnum], |
| tgt, act, tok, offset, size, width); |
| if (retval < 0) { |
| nfp_bar_put(nfp, &nfp->bar[barnum]); |
| barnum = retval; |
| } |
| |
| err_nobar: |
| spin_unlock_irqrestore(&nfp->bar_lock, irqflags); |
| return barnum; |
| } |
| |
| static void disable_bars(struct nfp6000_pcie *nfp); |
| |
| static int bar_cmp(const void *aptr, const void *bptr) |
| { |
| const struct nfp_bar *a = aptr, *b = bptr; |
| |
| if (a->bitsize == b->bitsize) |
| return a->index - b->index; |
| else |
| return a->bitsize - b->bitsize; |
| } |
| |
| /* Map all PCI bars and fetch the actual BAR configurations from the |
| * board. We assume that the BAR with the PCIe config block is |
| * already mapped. |
| * |
| * BAR0.0: Reserved for General Mapping (for MSI-X access to PCIe SRAM) |
| * BAR0.1: Reserved for XPB access (for MSI-X access to PCIe PBA) |
| * BAR0.2: -- |
| * BAR0.3: -- |
| * BAR0.4: Reserved for Explicit 0.0-0.3 access |
| * BAR0.5: Reserved for Explicit 1.0-1.3 access |
| * BAR0.6: Reserved for Explicit 2.0-2.3 access |
| * BAR0.7: Reserved for Explicit 3.0-3.3 access |
| * |
| * BAR1.0-BAR1.7: -- |
| * BAR2.0-BAR2.7: -- |
| */ |
| static int enable_bars(struct nfp6000_pcie *nfp, u16 interface) |
| { |
| const u32 barcfg_msix_general = |
| NFP_PCIE_BAR_PCIE2CPP_MapType( |
| NFP_PCIE_BAR_PCIE2CPP_MapType_GENERAL) | |
| NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT; |
| const u32 barcfg_msix_xpb = |
| NFP_PCIE_BAR_PCIE2CPP_MapType( |
| NFP_PCIE_BAR_PCIE2CPP_MapType_BULK) | |
| NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT | |
| NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress( |
| NFP_CPP_TARGET_ISLAND_XPB); |
| const u32 barcfg_explicit[4] = { |
| NFP_PCIE_BAR_PCIE2CPP_MapType( |
| NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT0), |
| NFP_PCIE_BAR_PCIE2CPP_MapType( |
| NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT1), |
| NFP_PCIE_BAR_PCIE2CPP_MapType( |
| NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT2), |
| NFP_PCIE_BAR_PCIE2CPP_MapType( |
| NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT3), |
| }; |
| char status_msg[196] = {}; |
| int i, err, bars_free; |
| struct nfp_bar *bar; |
| int expl_groups; |
| char *msg, *end; |
| |
| msg = status_msg + |
| snprintf(status_msg, sizeof(status_msg) - 1, "RESERVED BARs: "); |
| end = status_msg + sizeof(status_msg) - 1; |
| |
| bar = &nfp->bar[0]; |
| for (i = 0; i < ARRAY_SIZE(nfp->bar); i++, bar++) { |
| struct resource *res; |
| |
| res = &nfp->pdev->resource[(i >> 3) * 2]; |
| |
| /* Skip over BARs that are not IORESOURCE_MEM */ |
| if (!(resource_type(res) & IORESOURCE_MEM)) { |
| bar--; |
| continue; |
| } |
| |
| bar->resource = res; |
| bar->barcfg = 0; |
| |
| bar->nfp = nfp; |
| bar->index = i; |
| bar->mask = nfp_bar_resource_len(bar) - 1; |
| bar->bitsize = fls(bar->mask); |
| bar->base = 0; |
| bar->iomem = NULL; |
| } |
| |
| nfp->bars = bar - &nfp->bar[0]; |
| if (nfp->bars < 8) { |
| dev_err(nfp->dev, "No usable BARs found!\n"); |
| return -EINVAL; |
| } |
| |
| bars_free = nfp->bars; |
| |
| /* Convert unit ID (0..3) to signal master/data master ID (0x40..0x70) |
| */ |
| mutex_init(&nfp->expl.mutex); |
| |
| nfp->expl.master_id = ((NFP_CPP_INTERFACE_UNIT_of(interface) & 3) + 4) |
| << 4; |
| nfp->expl.signal_ref = 0x10; |
| |
| /* Configure, and lock, BAR0.0 for General Target use (MSI-X SRAM) */ |
| bar = &nfp->bar[0]; |
| if (nfp_bar_resource_len(bar) >= NFP_PCI_MIN_MAP_SIZE) |
| bar->iomem = ioremap(nfp_bar_resource_start(bar), |
| nfp_bar_resource_len(bar)); |
| if (bar->iomem) { |
| int pf; |
| |
| msg += scnprintf(msg, end - msg, "0.0: General/MSI-X SRAM, "); |
| atomic_inc(&bar->refcnt); |
| bars_free--; |
| |
| nfp6000_bar_write(nfp, bar, barcfg_msix_general); |
| |
| nfp->expl.data = bar->iomem + NFP_PCIE_SRAM + 0x1000; |
| |
| switch (nfp->pdev->device) { |
| case PCI_DEVICE_ID_NETRONOME_NFP3800: |
| pf = nfp->pdev->devfn & 7; |
| nfp->iomem.csr = bar->iomem + NFP_PCIE_BAR(pf); |
| break; |
| case PCI_DEVICE_ID_NETRONOME_NFP4000: |
| case PCI_DEVICE_ID_NETRONOME_NFP5000: |
| case PCI_DEVICE_ID_NETRONOME_NFP6000: |
| nfp->iomem.csr = bar->iomem + NFP_PCIE_BAR(0); |
| break; |
| default: |
| dev_err(nfp->dev, "Unsupported device ID: %04hx!\n", |
| nfp->pdev->device); |
| err = -EINVAL; |
| goto err_unmap_bar0; |
| } |
| nfp->iomem.em = bar->iomem + NFP_PCIE_EM; |
| } |
| |
| switch (nfp->pdev->device) { |
| case PCI_DEVICE_ID_NETRONOME_NFP3800: |
| expl_groups = 1; |
| break; |
| case PCI_DEVICE_ID_NETRONOME_NFP4000: |
| case PCI_DEVICE_ID_NETRONOME_NFP5000: |
| case PCI_DEVICE_ID_NETRONOME_NFP6000: |
| expl_groups = 4; |
| break; |
| default: |
| dev_err(nfp->dev, "Unsupported device ID: %04hx!\n", |
| nfp->pdev->device); |
| err = -EINVAL; |
| goto err_unmap_bar0; |
| } |
| |
| /* Configure, and lock, BAR0.1 for PCIe XPB (MSI-X PBA) */ |
| bar = &nfp->bar[1]; |
| msg += scnprintf(msg, end - msg, "0.1: PCIe XPB/MSI-X PBA, "); |
| atomic_inc(&bar->refcnt); |
| bars_free--; |
| |
| nfp6000_bar_write(nfp, bar, barcfg_msix_xpb); |
| |
| /* Use BAR0.4..BAR0.7 for EXPL IO */ |
| for (i = 0; i < 4; i++) { |
| int j; |
| |
| if (i >= NFP_PCIE_EXPLICIT_BARS || i >= expl_groups) { |
| nfp->expl.group[i].bitsize = 0; |
| continue; |
| } |
| |
| bar = &nfp->bar[4 + i]; |
| bar->iomem = ioremap(nfp_bar_resource_start(bar), |
| nfp_bar_resource_len(bar)); |
| if (bar->iomem) { |
| msg += scnprintf(msg, end - msg, |
| "0.%d: Explicit%d, ", 4 + i, i); |
| atomic_inc(&bar->refcnt); |
| bars_free--; |
| |
| nfp->expl.group[i].bitsize = bar->bitsize; |
| nfp->expl.group[i].addr = bar->iomem; |
| nfp6000_bar_write(nfp, bar, barcfg_explicit[i]); |
| |
| for (j = 0; j < 4; j++) |
| nfp->expl.group[i].free[j] = true; |
| } |
| nfp->iomem.expl[i] = bar->iomem; |
| } |
| |
| /* Sort bars by bit size - use the smallest possible first. */ |
| sort(&nfp->bar[0], nfp->bars, sizeof(nfp->bar[0]), |
| bar_cmp, NULL); |
| |
| dev_info(nfp->dev, "%sfree: %d/%d\n", status_msg, bars_free, nfp->bars); |
| |
| return 0; |
| |
| err_unmap_bar0: |
| if (nfp->bar[0].iomem) |
| iounmap(nfp->bar[0].iomem); |
| return err; |
| } |
| |
| static void disable_bars(struct nfp6000_pcie *nfp) |
| { |
| struct nfp_bar *bar = &nfp->bar[0]; |
| int n; |
| |
| for (n = 0; n < nfp->bars; n++, bar++) { |
| if (bar->iomem) { |
| iounmap(bar->iomem); |
| bar->iomem = NULL; |
| } |
| } |
| } |
| |
| /* |
| * Generic CPP bus access interface. |
| */ |
| |
| struct nfp6000_area_priv { |
| atomic_t refcnt; |
| |
| struct nfp_bar *bar; |
| u32 bar_offset; |
| |
| u32 target; |
| u32 action; |
| u32 token; |
| u64 offset; |
| struct { |
| int read; |
| int write; |
| int bar; |
| } width; |
| size_t size; |
| |
| void __iomem *iomem; |
| phys_addr_t phys; |
| struct resource resource; |
| }; |
| |
| static int nfp6000_area_init(struct nfp_cpp_area *area, u32 dest, |
| unsigned long long address, unsigned long size) |
| { |
| struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area); |
| u32 target = NFP_CPP_ID_TARGET_of(dest); |
| u32 action = NFP_CPP_ID_ACTION_of(dest); |
| u32 token = NFP_CPP_ID_TOKEN_of(dest); |
| int pp; |
| |
| pp = nfp_target_pushpull(NFP_CPP_ID(target, action, token), address); |
| if (pp < 0) |
| return pp; |
| |
| priv->width.read = PUSH_WIDTH(pp); |
| priv->width.write = PULL_WIDTH(pp); |
| if (priv->width.read > 0 && |
| priv->width.write > 0 && |
| priv->width.read != priv->width.write) { |
| return -EINVAL; |
| } |
| |
| if (priv->width.read > 0) |
| priv->width.bar = priv->width.read; |
| else |
| priv->width.bar = priv->width.write; |
| |
| atomic_set(&priv->refcnt, 0); |
| priv->bar = NULL; |
| |
| priv->target = target; |
| priv->action = action; |
| priv->token = token; |
| priv->offset = address; |
| priv->size = size; |
| memset(&priv->resource, 0, sizeof(priv->resource)); |
| |
| return 0; |
| } |
| |
| static void nfp6000_area_cleanup(struct nfp_cpp_area *area) |
| { |
| } |
| |
| static void priv_area_get(struct nfp_cpp_area *area) |
| { |
| struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area); |
| |
| atomic_inc(&priv->refcnt); |
| } |
| |
| static int priv_area_put(struct nfp_cpp_area *area) |
| { |
| struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area); |
| |
| if (WARN_ON(!atomic_read(&priv->refcnt))) |
| return 0; |
| |
| return atomic_dec_and_test(&priv->refcnt); |
| } |
| |
| static int nfp6000_area_acquire(struct nfp_cpp_area *area) |
| { |
| struct nfp6000_pcie *nfp = nfp_cpp_priv(nfp_cpp_area_cpp(area)); |
| struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area); |
| int barnum, err; |
| |
| if (priv->bar) { |
| /* Already allocated. */ |
| priv_area_get(area); |
| return 0; |
| } |
| |
| barnum = nfp_alloc_bar(nfp, priv->target, priv->action, priv->token, |
| priv->offset, priv->size, priv->width.bar, 1); |
| |
| if (barnum < 0) { |
| err = barnum; |
| goto err_alloc_bar; |
| } |
| priv->bar = &nfp->bar[barnum]; |
| |
| /* Calculate offset into BAR. */ |
| if (nfp_bar_maptype(priv->bar) == |
| NFP_PCIE_BAR_PCIE2CPP_MapType_GENERAL) { |
| priv->bar_offset = priv->offset & |
| (NFP_PCIE_P2C_GENERAL_SIZE(priv->bar) - 1); |
| priv->bar_offset += NFP_PCIE_P2C_GENERAL_TARGET_OFFSET( |
| priv->bar, priv->target); |
| priv->bar_offset += NFP_PCIE_P2C_GENERAL_TOKEN_OFFSET( |
| priv->bar, priv->token); |
| } else { |
| priv->bar_offset = priv->offset & priv->bar->mask; |
| } |
| |
| /* We don't actually try to acquire the resource area using |
| * request_resource. This would prevent sharing the mapped |
| * BAR between multiple CPP areas and prevent us from |
| * effectively utilizing the limited amount of BAR resources. |
| */ |
| priv->phys = nfp_bar_resource_start(priv->bar) + priv->bar_offset; |
| priv->resource.name = nfp_cpp_area_name(area); |
| priv->resource.start = priv->phys; |
| priv->resource.end = priv->resource.start + priv->size - 1; |
| priv->resource.flags = IORESOURCE_MEM; |
| |
| /* If the bar is already mapped in, use its mapping */ |
| if (priv->bar->iomem) |
| priv->iomem = priv->bar->iomem + priv->bar_offset; |
| else |
| /* Must have been too big. Sub-allocate. */ |
| priv->iomem = ioremap(priv->phys, priv->size); |
| |
| if (IS_ERR_OR_NULL(priv->iomem)) { |
| dev_err(nfp->dev, "Can't ioremap() a %d byte region of BAR %d\n", |
| (int)priv->size, priv->bar->index); |
| err = !priv->iomem ? -ENOMEM : PTR_ERR(priv->iomem); |
| priv->iomem = NULL; |
| goto err_iomem_remap; |
| } |
| |
| priv_area_get(area); |
| return 0; |
| |
| err_iomem_remap: |
| nfp_bar_put(nfp, priv->bar); |
| priv->bar = NULL; |
| err_alloc_bar: |
| return err; |
| } |
| |
| static void nfp6000_area_release(struct nfp_cpp_area *area) |
| { |
| struct nfp6000_pcie *nfp = nfp_cpp_priv(nfp_cpp_area_cpp(area)); |
| struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area); |
| |
| if (!priv_area_put(area)) |
| return; |
| |
| if (!priv->bar->iomem) |
| iounmap(priv->iomem); |
| |
| nfp_bar_put(nfp, priv->bar); |
| |
| priv->bar = NULL; |
| priv->iomem = NULL; |
| } |
| |
| static phys_addr_t nfp6000_area_phys(struct nfp_cpp_area *area) |
| { |
| struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area); |
| |
| return priv->phys; |
| } |
| |
| static void __iomem *nfp6000_area_iomem(struct nfp_cpp_area *area) |
| { |
| struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area); |
| |
| return priv->iomem; |
| } |
| |
| static struct resource *nfp6000_area_resource(struct nfp_cpp_area *area) |
| { |
| /* Use the BAR resource as the resource for the CPP area. |
| * This enables us to share the BAR among multiple CPP areas |
| * without resource conflicts. |
| */ |
| struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area); |
| |
| return priv->bar->resource; |
| } |
| |
| static int nfp6000_area_read(struct nfp_cpp_area *area, void *kernel_vaddr, |
| unsigned long offset, unsigned int length) |
| { |
| u64 __maybe_unused *wrptr64 = kernel_vaddr; |
| const u64 __iomem __maybe_unused *rdptr64; |
| struct nfp6000_area_priv *priv; |
| u32 *wrptr32 = kernel_vaddr; |
| const u32 __iomem *rdptr32; |
| int n, width; |
| |
| priv = nfp_cpp_area_priv(area); |
| rdptr64 = priv->iomem + offset; |
| rdptr32 = priv->iomem + offset; |
| |
| if (offset + length > priv->size) |
| return -EFAULT; |
| |
| width = priv->width.read; |
| if (width <= 0) |
| return -EINVAL; |
| |
| /* MU reads via a PCIe2CPP BAR support 32bit (and other) lengths */ |
| if (priv->target == (NFP_CPP_TARGET_MU & NFP_CPP_TARGET_ID_MASK) && |
| priv->action == NFP_CPP_ACTION_RW && |
| (offset % sizeof(u64) == 4 || length % sizeof(u64) == 4)) |
| width = TARGET_WIDTH_32; |
| |
| /* Unaligned? Translate to an explicit access */ |
| if ((priv->offset + offset) & (width - 1)) |
| return nfp_cpp_explicit_read(nfp_cpp_area_cpp(area), |
| NFP_CPP_ID(priv->target, |
| priv->action, |
| priv->token), |
| priv->offset + offset, |
| kernel_vaddr, length, width); |
| |
| if (WARN_ON(!priv->bar)) |
| return -EFAULT; |
| |
| switch (width) { |
| case TARGET_WIDTH_32: |
| if (offset % sizeof(u32) != 0 || length % sizeof(u32) != 0) |
| return -EINVAL; |
| |
| for (n = 0; n < length; n += sizeof(u32)) |
| *wrptr32++ = __raw_readl(rdptr32++); |
| return n; |
| #ifdef __raw_readq |
| case TARGET_WIDTH_64: |
| if (offset % sizeof(u64) != 0 || length % sizeof(u64) != 0) |
| return -EINVAL; |
| |
| for (n = 0; n < length; n += sizeof(u64)) |
| *wrptr64++ = __raw_readq(rdptr64++); |
| return n; |
| #endif |
| default: |
| return -EINVAL; |
| } |
| } |
| |
| static int |
| nfp6000_area_write(struct nfp_cpp_area *area, |
| const void *kernel_vaddr, |
| unsigned long offset, unsigned int length) |
| { |
| const u64 __maybe_unused *rdptr64 = kernel_vaddr; |
| u64 __iomem __maybe_unused *wrptr64; |
| const u32 *rdptr32 = kernel_vaddr; |
| struct nfp6000_area_priv *priv; |
| u32 __iomem *wrptr32; |
| int n, width; |
| |
| priv = nfp_cpp_area_priv(area); |
| wrptr64 = priv->iomem + offset; |
| wrptr32 = priv->iomem + offset; |
| |
| if (offset + length > priv->size) |
| return -EFAULT; |
| |
| width = priv->width.write; |
| if (width <= 0) |
| return -EINVAL; |
| |
| /* MU writes via a PCIe2CPP BAR support 32bit (and other) lengths */ |
| if (priv->target == (NFP_CPP_TARGET_ID_MASK & NFP_CPP_TARGET_MU) && |
| priv->action == NFP_CPP_ACTION_RW && |
| (offset % sizeof(u64) == 4 || length % sizeof(u64) == 4)) |
| width = TARGET_WIDTH_32; |
| |
| /* Unaligned? Translate to an explicit access */ |
| if ((priv->offset + offset) & (width - 1)) |
| return nfp_cpp_explicit_write(nfp_cpp_area_cpp(area), |
| NFP_CPP_ID(priv->target, |
| priv->action, |
| priv->token), |
| priv->offset + offset, |
| kernel_vaddr, length, width); |
| |
| if (WARN_ON(!priv->bar)) |
| return -EFAULT; |
| |
| switch (width) { |
| case TARGET_WIDTH_32: |
| if (offset % sizeof(u32) != 0 || length % sizeof(u32) != 0) |
| return -EINVAL; |
| |
| for (n = 0; n < length; n += sizeof(u32)) { |
| __raw_writel(*rdptr32++, wrptr32++); |
| wmb(); |
| } |
| return n; |
| #ifdef __raw_writeq |
| case TARGET_WIDTH_64: |
| if (offset % sizeof(u64) != 0 || length % sizeof(u64) != 0) |
| return -EINVAL; |
| |
| for (n = 0; n < length; n += sizeof(u64)) { |
| __raw_writeq(*rdptr64++, wrptr64++); |
| wmb(); |
| } |
| return n; |
| #endif |
| default: |
| return -EINVAL; |
| } |
| } |
| |
| struct nfp6000_explicit_priv { |
| struct nfp6000_pcie *nfp; |
| struct { |
| int group; |
| int area; |
| } bar; |
| int bitsize; |
| void __iomem *data; |
| void __iomem *addr; |
| }; |
| |
| static int nfp6000_explicit_acquire(struct nfp_cpp_explicit *expl) |
| { |
| struct nfp6000_pcie *nfp = nfp_cpp_priv(nfp_cpp_explicit_cpp(expl)); |
| struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl); |
| int i, j; |
| |
| mutex_lock(&nfp->expl.mutex); |
| for (i = 0; i < ARRAY_SIZE(nfp->expl.group); i++) { |
| if (!nfp->expl.group[i].bitsize) |
| continue; |
| |
| for (j = 0; j < ARRAY_SIZE(nfp->expl.group[i].free); j++) { |
| u16 data_offset; |
| |
| if (!nfp->expl.group[i].free[j]) |
| continue; |
| |
| priv->nfp = nfp; |
| priv->bar.group = i; |
| priv->bar.area = j; |
| priv->bitsize = nfp->expl.group[i].bitsize - 2; |
| |
| data_offset = (priv->bar.group << 9) + |
| (priv->bar.area << 7); |
| priv->data = nfp->expl.data + data_offset; |
| priv->addr = nfp->expl.group[i].addr + |
| (priv->bar.area << priv->bitsize); |
| nfp->expl.group[i].free[j] = false; |
| |
| mutex_unlock(&nfp->expl.mutex); |
| return 0; |
| } |
| } |
| mutex_unlock(&nfp->expl.mutex); |
| |
| return -EAGAIN; |
| } |
| |
| static void nfp6000_explicit_release(struct nfp_cpp_explicit *expl) |
| { |
| struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl); |
| struct nfp6000_pcie *nfp = priv->nfp; |
| |
| mutex_lock(&nfp->expl.mutex); |
| nfp->expl.group[priv->bar.group].free[priv->bar.area] = true; |
| mutex_unlock(&nfp->expl.mutex); |
| } |
| |
| static int nfp6000_explicit_put(struct nfp_cpp_explicit *expl, |
| const void *buff, size_t len) |
| { |
| struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl); |
| const u32 *src = buff; |
| size_t i; |
| |
| for (i = 0; i < len; i += sizeof(u32)) |
| writel(*(src++), priv->data + i); |
| |
| return i; |
| } |
| |
| static int |
| nfp6000_explicit_do(struct nfp_cpp_explicit *expl, |
| const struct nfp_cpp_explicit_command *cmd, u64 address) |
| { |
| struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl); |
| u8 signal_master, signal_ref, data_master; |
| struct nfp6000_pcie *nfp = priv->nfp; |
| int sigmask = 0; |
| u16 data_ref; |
| u32 csr[3]; |
| |
| if (cmd->siga_mode) |
| sigmask |= 1 << cmd->siga; |
| if (cmd->sigb_mode) |
| sigmask |= 1 << cmd->sigb; |
| |
| signal_master = cmd->signal_master; |
| if (!signal_master) |
| signal_master = nfp->expl.master_id; |
| |
| signal_ref = cmd->signal_ref; |
| if (signal_master == nfp->expl.master_id) |
| signal_ref = nfp->expl.signal_ref + |
| ((priv->bar.group * 4 + priv->bar.area) << 1); |
| |
| data_master = cmd->data_master; |
| if (!data_master) |
| data_master = nfp->expl.master_id; |
| |
| data_ref = cmd->data_ref; |
| if (data_master == nfp->expl.master_id) |
| data_ref = 0x1000 + |
| (priv->bar.group << 9) + (priv->bar.area << 7); |
| |
| csr[0] = NFP_PCIE_BAR_EXPLICIT_BAR0_SignalType(sigmask) | |
| NFP_PCIE_BAR_EXPLICIT_BAR0_Token( |
| NFP_CPP_ID_TOKEN_of(cmd->cpp_id)) | |
| NFP_PCIE_BAR_EXPLICIT_BAR0_Address(address >> 16); |
| |
| csr[1] = NFP_PCIE_BAR_EXPLICIT_BAR1_SignalRef(signal_ref) | |
| NFP_PCIE_BAR_EXPLICIT_BAR1_DataMaster(data_master) | |
| NFP_PCIE_BAR_EXPLICIT_BAR1_DataRef(data_ref); |
| |
| csr[2] = NFP_PCIE_BAR_EXPLICIT_BAR2_Target( |
| NFP_CPP_ID_TARGET_of(cmd->cpp_id)) | |
| NFP_PCIE_BAR_EXPLICIT_BAR2_Action( |
| NFP_CPP_ID_ACTION_of(cmd->cpp_id)) | |
| NFP_PCIE_BAR_EXPLICIT_BAR2_Length(cmd->len) | |
| NFP_PCIE_BAR_EXPLICIT_BAR2_ByteMask(cmd->byte_mask) | |
| NFP_PCIE_BAR_EXPLICIT_BAR2_SignalMaster(signal_master); |
| |
| if (nfp->iomem.csr) { |
| writel(csr[0], nfp->iomem.csr + |
| NFP_PCIE_BAR_EXPLICIT_BAR0(priv->bar.group, |
| priv->bar.area)); |
| writel(csr[1], nfp->iomem.csr + |
| NFP_PCIE_BAR_EXPLICIT_BAR1(priv->bar.group, |
| priv->bar.area)); |
| writel(csr[2], nfp->iomem.csr + |
| NFP_PCIE_BAR_EXPLICIT_BAR2(priv->bar.group, |
| priv->bar.area)); |
| /* Readback to ensure BAR is flushed */ |
| readl(nfp->iomem.csr + |
| NFP_PCIE_BAR_EXPLICIT_BAR0(priv->bar.group, |
| priv->bar.area)); |
| readl(nfp->iomem.csr + |
| NFP_PCIE_BAR_EXPLICIT_BAR1(priv->bar.group, |
| priv->bar.area)); |
| readl(nfp->iomem.csr + |
| NFP_PCIE_BAR_EXPLICIT_BAR2(priv->bar.group, |
| priv->bar.area)); |
| } else { |
| pci_write_config_dword(nfp->pdev, 0x400 + |
| NFP_PCIE_BAR_EXPLICIT_BAR0( |
| priv->bar.group, priv->bar.area), |
| csr[0]); |
| |
| pci_write_config_dword(nfp->pdev, 0x400 + |
| NFP_PCIE_BAR_EXPLICIT_BAR1( |
| priv->bar.group, priv->bar.area), |
| csr[1]); |
| |
| pci_write_config_dword(nfp->pdev, 0x400 + |
| NFP_PCIE_BAR_EXPLICIT_BAR2( |
| priv->bar.group, priv->bar.area), |
| csr[2]); |
| } |
| |
| /* Issue the 'kickoff' transaction */ |
| readb(priv->addr + (address & ((1 << priv->bitsize) - 1))); |
| |
| return sigmask; |
| } |
| |
| static int nfp6000_explicit_get(struct nfp_cpp_explicit *expl, |
| void *buff, size_t len) |
| { |
| struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl); |
| u32 *dst = buff; |
| size_t i; |
| |
| for (i = 0; i < len; i += sizeof(u32)) |
| *(dst++) = readl(priv->data + i); |
| |
| return i; |
| } |
| |
| static int nfp6000_init(struct nfp_cpp *cpp) |
| { |
| nfp_cpp_area_cache_add(cpp, SZ_64K); |
| nfp_cpp_area_cache_add(cpp, SZ_64K); |
| nfp_cpp_area_cache_add(cpp, SZ_256K); |
| |
| return 0; |
| } |
| |
| static void nfp6000_free(struct nfp_cpp *cpp) |
| { |
| struct nfp6000_pcie *nfp = nfp_cpp_priv(cpp); |
| |
| disable_bars(nfp); |
| kfree(nfp); |
| } |
| |
| static int nfp6000_read_serial(struct device *dev, u8 *serial) |
| { |
| struct pci_dev *pdev = to_pci_dev(dev); |
| u64 dsn; |
| |
| dsn = pci_get_dsn(pdev); |
| if (!dsn) { |
| dev_err(dev, "can't find PCIe Serial Number Capability\n"); |
| return -EINVAL; |
| } |
| |
| put_unaligned_be32((u32)(dsn >> 32), serial); |
| put_unaligned_be16((u16)(dsn >> 16), serial + 4); |
| |
| return 0; |
| } |
| |
| static int nfp6000_get_interface(struct device *dev) |
| { |
| struct pci_dev *pdev = to_pci_dev(dev); |
| u64 dsn; |
| |
| dsn = pci_get_dsn(pdev); |
| if (!dsn) { |
| dev_err(dev, "can't find PCIe Serial Number Capability\n"); |
| return -EINVAL; |
| } |
| |
| return dsn & 0xffff; |
| } |
| |
| static const struct nfp_cpp_operations nfp6000_pcie_ops = { |
| .owner = THIS_MODULE, |
| |
| .init = nfp6000_init, |
| .free = nfp6000_free, |
| |
| .read_serial = nfp6000_read_serial, |
| .get_interface = nfp6000_get_interface, |
| |
| .area_priv_size = sizeof(struct nfp6000_area_priv), |
| .area_init = nfp6000_area_init, |
| .area_cleanup = nfp6000_area_cleanup, |
| .area_acquire = nfp6000_area_acquire, |
| .area_release = nfp6000_area_release, |
| .area_phys = nfp6000_area_phys, |
| .area_iomem = nfp6000_area_iomem, |
| .area_resource = nfp6000_area_resource, |
| .area_read = nfp6000_area_read, |
| .area_write = nfp6000_area_write, |
| |
| .explicit_priv_size = sizeof(struct nfp6000_explicit_priv), |
| .explicit_acquire = nfp6000_explicit_acquire, |
| .explicit_release = nfp6000_explicit_release, |
| .explicit_put = nfp6000_explicit_put, |
| .explicit_do = nfp6000_explicit_do, |
| .explicit_get = nfp6000_explicit_get, |
| }; |
| |
| /** |
| * nfp_cpp_from_nfp6000_pcie() - Build a NFP CPP bus from a NFP6000 PCI device |
| * @pdev: NFP6000 PCI device |
| * |
| * Return: NFP CPP handle |
| */ |
| struct nfp_cpp *nfp_cpp_from_nfp6000_pcie(struct pci_dev *pdev) |
| { |
| struct nfp6000_pcie *nfp; |
| u16 interface; |
| int err; |
| |
| /* Finished with card initialization. */ |
| dev_info(&pdev->dev, |
| "Netronome Flow Processor NFP4000/NFP5000/NFP6000 PCIe Card Probe\n"); |
| pcie_print_link_status(pdev); |
| |
| nfp = kzalloc(sizeof(*nfp), GFP_KERNEL); |
| if (!nfp) { |
| err = -ENOMEM; |
| goto err_ret; |
| } |
| |
| nfp->dev = &pdev->dev; |
| nfp->pdev = pdev; |
| init_waitqueue_head(&nfp->bar_waiters); |
| spin_lock_init(&nfp->bar_lock); |
| |
| interface = nfp6000_get_interface(&pdev->dev); |
| |
| if (NFP_CPP_INTERFACE_TYPE_of(interface) != |
| NFP_CPP_INTERFACE_TYPE_PCI) { |
| dev_err(&pdev->dev, |
| "Interface type %d is not the expected %d\n", |
| NFP_CPP_INTERFACE_TYPE_of(interface), |
| NFP_CPP_INTERFACE_TYPE_PCI); |
| err = -ENODEV; |
| goto err_free_nfp; |
| } |
| |
| if (NFP_CPP_INTERFACE_CHANNEL_of(interface) != |
| NFP_CPP_INTERFACE_CHANNEL_PEROPENER) { |
| dev_err(&pdev->dev, "Interface channel %d is not the expected %d\n", |
| NFP_CPP_INTERFACE_CHANNEL_of(interface), |
| NFP_CPP_INTERFACE_CHANNEL_PEROPENER); |
| err = -ENODEV; |
| goto err_free_nfp; |
| } |
| |
| err = enable_bars(nfp, interface); |
| if (err) |
| goto err_free_nfp; |
| |
| /* Probe for all the common NFP devices */ |
| return nfp_cpp_from_operations(&nfp6000_pcie_ops, &pdev->dev, nfp); |
| |
| err_free_nfp: |
| kfree(nfp); |
| err_ret: |
| dev_err(&pdev->dev, "NFP6000 PCI setup failed\n"); |
| return ERR_PTR(err); |
| } |