| // SPDX-License-Identifier: GPL-2.0 |
| #include <linux/cpumask.h> |
| #include <linux/dma-mapping.h> |
| #include <linux/dmapool.h> |
| #include <linux/delay.h> |
| #include <linux/gfp.h> |
| #include <linux/kernel.h> |
| #include <linux/module.h> |
| #include <linux/pci_regs.h> |
| #include <linux/vmalloc.h> |
| #include <linux/pci.h> |
| |
| #include "nitrox_dev.h" |
| #include "nitrox_common.h" |
| #include "nitrox_req.h" |
| #include "nitrox_csr.h" |
| |
| #define CRYPTO_CTX_SIZE 256 |
| |
| /* packet inuput ring alignments */ |
| #define PKTIN_Q_ALIGN_BYTES 16 |
| |
| static int nitrox_cmdq_init(struct nitrox_cmdq *cmdq, int align_bytes) |
| { |
| struct nitrox_device *ndev = cmdq->ndev; |
| |
| cmdq->qsize = (ndev->qlen * cmdq->instr_size) + align_bytes; |
| cmdq->unalign_base = dma_alloc_coherent(DEV(ndev), cmdq->qsize, |
| &cmdq->unalign_dma, |
| GFP_KERNEL); |
| if (!cmdq->unalign_base) |
| return -ENOMEM; |
| |
| cmdq->dma = PTR_ALIGN(cmdq->unalign_dma, align_bytes); |
| cmdq->base = cmdq->unalign_base + (cmdq->dma - cmdq->unalign_dma); |
| cmdq->write_idx = 0; |
| |
| spin_lock_init(&cmdq->cmd_qlock); |
| spin_lock_init(&cmdq->resp_qlock); |
| spin_lock_init(&cmdq->backlog_qlock); |
| |
| INIT_LIST_HEAD(&cmdq->response_head); |
| INIT_LIST_HEAD(&cmdq->backlog_head); |
| INIT_WORK(&cmdq->backlog_qflush, backlog_qflush_work); |
| |
| atomic_set(&cmdq->pending_count, 0); |
| atomic_set(&cmdq->backlog_count, 0); |
| return 0; |
| } |
| |
| static void nitrox_cmdq_reset(struct nitrox_cmdq *cmdq) |
| { |
| cmdq->write_idx = 0; |
| atomic_set(&cmdq->pending_count, 0); |
| atomic_set(&cmdq->backlog_count, 0); |
| } |
| |
| static void nitrox_cmdq_cleanup(struct nitrox_cmdq *cmdq) |
| { |
| struct nitrox_device *ndev = cmdq->ndev; |
| |
| if (!cmdq->unalign_base) |
| return; |
| |
| cancel_work_sync(&cmdq->backlog_qflush); |
| |
| dma_free_coherent(DEV(ndev), cmdq->qsize, |
| cmdq->unalign_base, cmdq->unalign_dma); |
| nitrox_cmdq_reset(cmdq); |
| |
| cmdq->dbell_csr_addr = NULL; |
| cmdq->compl_cnt_csr_addr = NULL; |
| cmdq->unalign_base = NULL; |
| cmdq->base = NULL; |
| cmdq->unalign_dma = 0; |
| cmdq->dma = 0; |
| cmdq->qsize = 0; |
| cmdq->instr_size = 0; |
| } |
| |
| static void nitrox_free_pktin_queues(struct nitrox_device *ndev) |
| { |
| int i; |
| |
| for (i = 0; i < ndev->nr_queues; i++) { |
| struct nitrox_cmdq *cmdq = &ndev->pkt_inq[i]; |
| |
| nitrox_cmdq_cleanup(cmdq); |
| } |
| kfree(ndev->pkt_inq); |
| ndev->pkt_inq = NULL; |
| } |
| |
| static int nitrox_alloc_pktin_queues(struct nitrox_device *ndev) |
| { |
| int i, err; |
| |
| ndev->pkt_inq = kcalloc_node(ndev->nr_queues, |
| sizeof(struct nitrox_cmdq), |
| GFP_KERNEL, ndev->node); |
| if (!ndev->pkt_inq) |
| return -ENOMEM; |
| |
| for (i = 0; i < ndev->nr_queues; i++) { |
| struct nitrox_cmdq *cmdq; |
| u64 offset; |
| |
| cmdq = &ndev->pkt_inq[i]; |
| cmdq->ndev = ndev; |
| cmdq->qno = i; |
| cmdq->instr_size = sizeof(struct nps_pkt_instr); |
| |
| /* packet input ring doorbell address */ |
| offset = NPS_PKT_IN_INSTR_BAOFF_DBELLX(i); |
| cmdq->dbell_csr_addr = NITROX_CSR_ADDR(ndev, offset); |
| /* packet solicit port completion count address */ |
| offset = NPS_PKT_SLC_CNTSX(i); |
| cmdq->compl_cnt_csr_addr = NITROX_CSR_ADDR(ndev, offset); |
| |
| err = nitrox_cmdq_init(cmdq, PKTIN_Q_ALIGN_BYTES); |
| if (err) |
| goto pktq_fail; |
| } |
| return 0; |
| |
| pktq_fail: |
| nitrox_free_pktin_queues(ndev); |
| return err; |
| } |
| |
| static int create_crypto_dma_pool(struct nitrox_device *ndev) |
| { |
| size_t size; |
| |
| /* Crypto context pool, 16 byte aligned */ |
| size = CRYPTO_CTX_SIZE + sizeof(struct ctx_hdr); |
| ndev->ctx_pool = dma_pool_create("nitrox-context", |
| DEV(ndev), size, 16, 0); |
| if (!ndev->ctx_pool) |
| return -ENOMEM; |
| |
| return 0; |
| } |
| |
| static void destroy_crypto_dma_pool(struct nitrox_device *ndev) |
| { |
| if (!ndev->ctx_pool) |
| return; |
| |
| dma_pool_destroy(ndev->ctx_pool); |
| ndev->ctx_pool = NULL; |
| } |
| |
| /* |
| * crypto_alloc_context - Allocate crypto context from pool |
| * @ndev: NITROX Device |
| */ |
| void *crypto_alloc_context(struct nitrox_device *ndev) |
| { |
| struct ctx_hdr *ctx; |
| struct crypto_ctx_hdr *chdr; |
| void *vaddr; |
| dma_addr_t dma; |
| |
| chdr = kmalloc(sizeof(*chdr), GFP_KERNEL); |
| if (!chdr) |
| return NULL; |
| |
| vaddr = dma_pool_zalloc(ndev->ctx_pool, GFP_KERNEL, &dma); |
| if (!vaddr) { |
| kfree(chdr); |
| return NULL; |
| } |
| |
| /* fill meta data */ |
| ctx = vaddr; |
| ctx->pool = ndev->ctx_pool; |
| ctx->dma = dma; |
| ctx->ctx_dma = dma + sizeof(struct ctx_hdr); |
| |
| chdr->pool = ndev->ctx_pool; |
| chdr->dma = dma; |
| chdr->vaddr = vaddr; |
| |
| return chdr; |
| } |
| |
| /** |
| * crypto_free_context - Free crypto context to pool |
| * @ctx: context to free |
| */ |
| void crypto_free_context(void *ctx) |
| { |
| struct crypto_ctx_hdr *ctxp; |
| |
| if (!ctx) |
| return; |
| |
| ctxp = ctx; |
| dma_pool_free(ctxp->pool, ctxp->vaddr, ctxp->dma); |
| kfree(ctxp); |
| } |
| |
| /** |
| * nitrox_common_sw_init - allocate software resources. |
| * @ndev: NITROX device |
| * |
| * Allocates crypto context pools and command queues etc. |
| * |
| * Return: 0 on success, or a negative error code on error. |
| */ |
| int nitrox_common_sw_init(struct nitrox_device *ndev) |
| { |
| int err = 0; |
| |
| /* per device crypto context pool */ |
| err = create_crypto_dma_pool(ndev); |
| if (err) |
| return err; |
| |
| err = nitrox_alloc_pktin_queues(ndev); |
| if (err) |
| destroy_crypto_dma_pool(ndev); |
| |
| return err; |
| } |
| |
| /** |
| * nitrox_common_sw_cleanup - free software resources. |
| * @ndev: NITROX device |
| */ |
| void nitrox_common_sw_cleanup(struct nitrox_device *ndev) |
| { |
| nitrox_free_pktin_queues(ndev); |
| destroy_crypto_dma_pool(ndev); |
| } |