| // SPDX-License-Identifier: GPL-2.0+ |
| #include <linux/module.h> |
| #include <linux/init.h> |
| #include <linux/mm.h> |
| #include <linux/kernel.h> /* printk() */ |
| #include <linux/slab.h> /* kmalloc() */ |
| #include <linux/fs.h> /* everything... */ |
| #include <linux/errno.h> /* error codes */ |
| #include <linux/types.h> /* size_t */ |
| #include <linux/cdev.h> |
| #include <linux/uaccess.h> /* copy_*_user */ |
| #include <linux/highmem.h> |
| #include <linux/pagemap.h> |
| #include "kpc_dma_driver.h" |
| #include "uapi.h" |
| |
| /********** Helper Functions **********/ |
| static inline |
| unsigned int count_pages(unsigned long iov_base, size_t iov_len) |
| { |
| unsigned long first = (iov_base & PAGE_MASK) >> PAGE_SHIFT; |
| unsigned long last = ((iov_base + iov_len - 1) & PAGE_MASK) >> PAGE_SHIFT; |
| |
| return last - first + 1; |
| } |
| |
| static inline |
| unsigned int count_parts_for_sge(struct scatterlist *sg) |
| { |
| return DIV_ROUND_UP(sg_dma_len(sg), 0x80000); |
| } |
| |
| /********** Transfer Helpers **********/ |
| static int kpc_dma_transfer(struct dev_private_data *priv, |
| unsigned long iov_base, size_t iov_len) |
| { |
| unsigned int i = 0; |
| int rv = 0, nr_pages = 0; |
| struct kpc_dma_device *ldev; |
| struct aio_cb_data *acd; |
| DECLARE_COMPLETION_ONSTACK(done); |
| u32 desc_needed = 0; |
| struct scatterlist *sg; |
| u32 num_descrs_avail; |
| struct kpc_dma_descriptor *desc; |
| unsigned int pcnt; |
| unsigned int p; |
| u64 card_addr; |
| u64 dma_addr; |
| u64 user_ctl; |
| |
| ldev = priv->ldev; |
| |
| acd = kzalloc(sizeof(*acd), GFP_KERNEL); |
| if (!acd) { |
| dev_err(&priv->ldev->pldev->dev, "Couldn't kmalloc space for the aio data\n"); |
| return -ENOMEM; |
| } |
| memset(acd, 0x66, sizeof(struct aio_cb_data)); |
| |
| acd->priv = priv; |
| acd->ldev = priv->ldev; |
| acd->cpl = &done; |
| acd->flags = 0; |
| acd->len = iov_len; |
| acd->page_count = count_pages(iov_base, iov_len); |
| |
| // Allocate an array of page pointers |
| acd->user_pages = kcalloc(acd->page_count, sizeof(struct page *), |
| GFP_KERNEL); |
| if (!acd->user_pages) { |
| dev_err(&priv->ldev->pldev->dev, "Couldn't kmalloc space for the page pointers\n"); |
| rv = -ENOMEM; |
| goto err_alloc_userpages; |
| } |
| |
| // Lock the user buffer pages in memory, and hold on to the page pointers (for the sglist) |
| mmap_read_lock(current->mm); /* get memory map semaphore */ |
| rv = pin_user_pages(iov_base, acd->page_count, FOLL_TOUCH | FOLL_WRITE, |
| acd->user_pages, NULL); |
| mmap_read_unlock(current->mm); /* release the semaphore */ |
| if (rv != acd->page_count) { |
| nr_pages = rv; |
| if (rv > 0) |
| rv = -EFAULT; |
| |
| dev_err(&priv->ldev->pldev->dev, "Couldn't pin_user_pages (%d)\n", rv); |
| goto unpin_pages; |
| } |
| nr_pages = acd->page_count; |
| |
| // Allocate and setup the sg_table (scatterlist entries) |
| rv = sg_alloc_table_from_pages(&acd->sgt, acd->user_pages, acd->page_count, |
| iov_base & (PAGE_SIZE - 1), iov_len, GFP_KERNEL); |
| if (rv) { |
| dev_err(&priv->ldev->pldev->dev, "Couldn't alloc sg_table (%d)\n", rv); |
| goto unpin_pages; |
| } |
| |
| // Setup the DMA mapping for all the sg entries |
| acd->mapped_entry_count = dma_map_sg(&ldev->pldev->dev, acd->sgt.sgl, acd->sgt.nents, |
| ldev->dir); |
| if (acd->mapped_entry_count <= 0) { |
| dev_err(&priv->ldev->pldev->dev, "Couldn't dma_map_sg (%d)\n", |
| acd->mapped_entry_count); |
| goto free_table; |
| } |
| |
| // Calculate how many descriptors are actually needed for this transfer. |
| for_each_sg(acd->sgt.sgl, sg, acd->mapped_entry_count, i) { |
| desc_needed += count_parts_for_sge(sg); |
| } |
| |
| lock_engine(ldev); |
| |
| // Figoure out how many descriptors are available and return an error if there aren't enough |
| num_descrs_avail = count_descriptors_available(ldev); |
| dev_dbg(&priv->ldev->pldev->dev, |
| " mapped_entry_count = %d num_descrs_needed = %d num_descrs_avail = %d\n", |
| acd->mapped_entry_count, desc_needed, num_descrs_avail); |
| |
| if (desc_needed >= ldev->desc_pool_cnt) { |
| dev_warn(&priv->ldev->pldev->dev, |
| " mapped_entry_count = %d num_descrs_needed = %d num_descrs_avail = %d TOO MANY to ever complete!\n", |
| acd->mapped_entry_count, desc_needed, num_descrs_avail); |
| rv = -EAGAIN; |
| goto err_descr_too_many; |
| } |
| if (desc_needed > num_descrs_avail) { |
| dev_warn(&priv->ldev->pldev->dev, |
| " mapped_entry_count = %d num_descrs_needed = %d num_descrs_avail = %d Too many to complete right now.\n", |
| acd->mapped_entry_count, desc_needed, num_descrs_avail); |
| rv = -EMSGSIZE; |
| goto err_descr_too_many; |
| } |
| |
| // Loop through all the sg table entries and fill out a descriptor for each one. |
| desc = ldev->desc_next; |
| card_addr = acd->priv->card_addr; |
| for_each_sg(acd->sgt.sgl, sg, acd->mapped_entry_count, i) { |
| pcnt = count_parts_for_sge(sg); |
| for (p = 0 ; p < pcnt ; p++) { |
| // Fill out the descriptor |
| BUG_ON(!desc); |
| clear_desc(desc); |
| if (p != pcnt - 1) |
| desc->DescByteCount = 0x80000; |
| else |
| desc->DescByteCount = sg_dma_len(sg) - (p * 0x80000); |
| |
| desc->DescBufferByteCount = desc->DescByteCount; |
| |
| desc->DescControlFlags |= DMA_DESC_CTL_IRQONERR; |
| if (i == 0 && p == 0) |
| desc->DescControlFlags |= DMA_DESC_CTL_SOP; |
| if (i == acd->mapped_entry_count - 1 && p == pcnt - 1) |
| desc->DescControlFlags |= DMA_DESC_CTL_EOP | DMA_DESC_CTL_IRQONDONE; |
| |
| desc->DescCardAddrLS = (card_addr & 0xFFFFFFFF); |
| desc->DescCardAddrMS = (card_addr >> 32) & 0xF; |
| card_addr += desc->DescByteCount; |
| |
| dma_addr = sg_dma_address(sg) + (p * 0x80000); |
| desc->DescSystemAddrLS = (dma_addr & 0x00000000FFFFFFFFUL) >> 0; |
| desc->DescSystemAddrMS = (dma_addr & 0xFFFFFFFF00000000UL) >> 32; |
| |
| user_ctl = acd->priv->user_ctl; |
| if (i == acd->mapped_entry_count - 1 && p == pcnt - 1) |
| user_ctl = acd->priv->user_ctl_last; |
| |
| desc->DescUserControlLS = (user_ctl & 0x00000000FFFFFFFFUL) >> 0; |
| desc->DescUserControlMS = (user_ctl & 0xFFFFFFFF00000000UL) >> 32; |
| |
| if (i == acd->mapped_entry_count - 1 && p == pcnt - 1) |
| desc->acd = acd; |
| |
| dev_dbg(&priv->ldev->pldev->dev, " Filled descriptor %p (acd = %p)\n", |
| desc, desc->acd); |
| |
| ldev->desc_next = desc->Next; |
| desc = desc->Next; |
| } |
| } |
| |
| // Send the filled descriptors off to the hardware to process! |
| SetEngineSWPtr(ldev, ldev->desc_next); |
| |
| unlock_engine(ldev); |
| |
| rv = wait_for_completion_interruptible(&done); |
| /* |
| * If the user aborted (rv == -ERESTARTSYS), we're no longer responsible |
| * for cleaning up the acd |
| */ |
| if (rv == -ERESTARTSYS) |
| acd->cpl = NULL; |
| if (rv == 0) { |
| rv = acd->len; |
| kfree(acd); |
| } |
| return rv; |
| |
| err_descr_too_many: |
| unlock_engine(ldev); |
| dma_unmap_sg(&ldev->pldev->dev, acd->sgt.sgl, acd->sgt.nents, ldev->dir); |
| free_table: |
| sg_free_table(&acd->sgt); |
| |
| unpin_pages: |
| if (nr_pages > 0) |
| unpin_user_pages(acd->user_pages, nr_pages); |
| kfree(acd->user_pages); |
| err_alloc_userpages: |
| kfree(acd); |
| dev_dbg(&priv->ldev->pldev->dev, "%s returning with error %d\n", __func__, rv); |
| return rv; |
| } |
| |
| void transfer_complete_cb(struct aio_cb_data *acd, size_t xfr_count, u32 flags) |
| { |
| unsigned int i; |
| |
| BUG_ON(!acd); |
| BUG_ON(!acd->user_pages); |
| BUG_ON(!acd->sgt.sgl); |
| BUG_ON(!acd->ldev); |
| BUG_ON(!acd->ldev->pldev); |
| |
| dma_unmap_sg(&acd->ldev->pldev->dev, acd->sgt.sgl, acd->sgt.nents, acd->ldev->dir); |
| |
| for (i = 0 ; i < acd->page_count ; i++) { |
| if (!PageReserved(acd->user_pages[i])) |
| set_page_dirty_lock(acd->user_pages[i]); |
| } |
| |
| unpin_user_pages(acd->user_pages, acd->page_count); |
| |
| sg_free_table(&acd->sgt); |
| |
| kfree(acd->user_pages); |
| |
| acd->flags = flags; |
| |
| if (acd->cpl) { |
| complete(acd->cpl); |
| } else { |
| /* |
| * There's no completion, so we're responsible for cleaning up |
| * the acd |
| */ |
| kfree(acd); |
| } |
| } |
| |
| /********** Fileops **********/ |
| static |
| int kpc_dma_open(struct inode *inode, struct file *filp) |
| { |
| struct dev_private_data *priv; |
| struct kpc_dma_device *ldev = kpc_dma_lookup_device(iminor(inode)); |
| |
| if (!ldev) |
| return -ENODEV; |
| |
| if (!atomic_dec_and_test(&ldev->open_count)) { |
| atomic_inc(&ldev->open_count); |
| return -EBUSY; /* already open */ |
| } |
| |
| priv = kzalloc(sizeof(*priv), GFP_KERNEL); |
| if (!priv) |
| return -ENOMEM; |
| |
| priv->ldev = ldev; |
| filp->private_data = priv; |
| |
| return 0; |
| } |
| |
| static |
| int kpc_dma_close(struct inode *inode, struct file *filp) |
| { |
| struct kpc_dma_descriptor *cur; |
| struct dev_private_data *priv = (struct dev_private_data *)filp->private_data; |
| struct kpc_dma_device *eng = priv->ldev; |
| |
| lock_engine(eng); |
| |
| stop_dma_engine(eng); |
| |
| cur = eng->desc_completed->Next; |
| while (cur != eng->desc_next) { |
| dev_dbg(&eng->pldev->dev, "Aborting descriptor %p (acd = %p)\n", cur, cur->acd); |
| if (cur->DescControlFlags & DMA_DESC_CTL_EOP) { |
| if (cur->acd) |
| transfer_complete_cb(cur->acd, 0, ACD_FLAG_ABORT); |
| } |
| |
| clear_desc(cur); |
| eng->desc_completed = cur; |
| |
| cur = cur->Next; |
| } |
| |
| start_dma_engine(eng); |
| |
| unlock_engine(eng); |
| |
| atomic_inc(&priv->ldev->open_count); /* release the device */ |
| kfree(priv); |
| return 0; |
| } |
| |
| static |
| ssize_t kpc_dma_read(struct file *filp, char __user *user_buf, size_t count, loff_t *ppos) |
| { |
| struct dev_private_data *priv = (struct dev_private_data *)filp->private_data; |
| |
| if (priv->ldev->dir != DMA_FROM_DEVICE) |
| return -EMEDIUMTYPE; |
| |
| return kpc_dma_transfer(priv, (unsigned long)user_buf, count); |
| } |
| |
| static |
| ssize_t kpc_dma_write(struct file *filp, const char __user *user_buf, size_t count, loff_t *ppos) |
| { |
| struct dev_private_data *priv = (struct dev_private_data *)filp->private_data; |
| |
| if (priv->ldev->dir != DMA_TO_DEVICE) |
| return -EMEDIUMTYPE; |
| |
| return kpc_dma_transfer(priv, (unsigned long)user_buf, count); |
| } |
| |
| static |
| long kpc_dma_ioctl(struct file *filp, unsigned int ioctl_num, unsigned long ioctl_param) |
| { |
| struct dev_private_data *priv = (struct dev_private_data *)filp->private_data; |
| |
| switch (ioctl_num) { |
| case KND_IOCTL_SET_CARD_ADDR: |
| priv->card_addr = ioctl_param; return priv->card_addr; |
| case KND_IOCTL_SET_USER_CTL: |
| priv->user_ctl = ioctl_param; return priv->user_ctl; |
| case KND_IOCTL_SET_USER_CTL_LAST: |
| priv->user_ctl_last = ioctl_param; return priv->user_ctl_last; |
| case KND_IOCTL_GET_USER_STS: |
| return priv->user_sts; |
| } |
| |
| return -ENOTTY; |
| } |
| |
| const struct file_operations kpc_dma_fops = { |
| .owner = THIS_MODULE, |
| .open = kpc_dma_open, |
| .release = kpc_dma_close, |
| .read = kpc_dma_read, |
| .write = kpc_dma_write, |
| .unlocked_ioctl = kpc_dma_ioctl, |
| }; |
| |