| // SPDX-License-Identifier: GPL-2.0-only |
| /* |
| * Intel MIC Platform Software Stack (MPSS) |
| * |
| * Copyright(c) 2013 Intel Corporation. |
| * |
| * Intel MIC Host driver. |
| */ |
| #include <linux/pci.h> |
| #include <linux/interrupt.h> |
| |
| #include "../common/mic_dev.h" |
| #include "mic_device.h" |
| |
| static irqreturn_t mic_thread_fn(int irq, void *dev) |
| { |
| struct mic_device *mdev = dev; |
| struct mic_intr_info *intr_info = mdev->intr_info; |
| struct mic_irq_info *irq_info = &mdev->irq_info; |
| struct mic_intr_cb *intr_cb; |
| struct pci_dev *pdev = mdev->pdev; |
| int i; |
| |
| spin_lock(&irq_info->mic_thread_lock); |
| for (i = intr_info->intr_start_idx[MIC_INTR_DB]; |
| i < intr_info->intr_len[MIC_INTR_DB]; i++) |
| if (test_and_clear_bit(i, &irq_info->mask)) { |
| list_for_each_entry(intr_cb, &irq_info->cb_list[i], |
| list) |
| if (intr_cb->thread_fn) |
| intr_cb->thread_fn(pdev->irq, |
| intr_cb->data); |
| } |
| spin_unlock(&irq_info->mic_thread_lock); |
| return IRQ_HANDLED; |
| } |
| /** |
| * mic_interrupt - Generic interrupt handler for |
| * MSI and INTx based interrupts. |
| * @irq: interrupt to handle (unused) |
| * @dev: pointer to the mic_device instance |
| */ |
| static irqreturn_t mic_interrupt(int irq, void *dev) |
| { |
| struct mic_device *mdev = dev; |
| struct mic_intr_info *intr_info = mdev->intr_info; |
| struct mic_irq_info *irq_info = &mdev->irq_info; |
| struct mic_intr_cb *intr_cb; |
| struct pci_dev *pdev = mdev->pdev; |
| u32 mask; |
| int i; |
| |
| mask = mdev->ops->ack_interrupt(mdev); |
| if (!mask) |
| return IRQ_NONE; |
| |
| spin_lock(&irq_info->mic_intr_lock); |
| for (i = intr_info->intr_start_idx[MIC_INTR_DB]; |
| i < intr_info->intr_len[MIC_INTR_DB]; i++) |
| if (mask & BIT(i)) { |
| list_for_each_entry(intr_cb, &irq_info->cb_list[i], |
| list) |
| if (intr_cb->handler) |
| intr_cb->handler(pdev->irq, |
| intr_cb->data); |
| set_bit(i, &irq_info->mask); |
| } |
| spin_unlock(&irq_info->mic_intr_lock); |
| return IRQ_WAKE_THREAD; |
| } |
| |
| /* Return the interrupt offset from the index. Index is 0 based. */ |
| static u16 mic_map_src_to_offset(struct mic_device *mdev, |
| int intr_src, enum mic_intr_type type) |
| { |
| if (type >= MIC_NUM_INTR_TYPES) |
| return MIC_NUM_OFFSETS; |
| if (intr_src >= mdev->intr_info->intr_len[type]) |
| return MIC_NUM_OFFSETS; |
| |
| return mdev->intr_info->intr_start_idx[type] + intr_src; |
| } |
| |
| /* Return next available msix_entry. */ |
| static struct msix_entry *mic_get_available_vector(struct mic_device *mdev) |
| { |
| int i; |
| struct mic_irq_info *info = &mdev->irq_info; |
| |
| for (i = 0; i < info->num_vectors; i++) |
| if (!info->mic_msi_map[i]) |
| return &info->msix_entries[i]; |
| return NULL; |
| } |
| |
| /** |
| * mic_register_intr_callback - Register a callback handler for the |
| * given source id. |
| * |
| * @mdev: pointer to the mic_device instance |
| * @idx: The source id to be registered. |
| * @handler: The function to be called when the source id receives |
| * the interrupt. |
| * @thread_fn: thread fn. corresponding to the handler |
| * @data: Private data of the requester. |
| * Return the callback structure that was registered or an |
| * appropriate error on failure. |
| */ |
| static struct mic_intr_cb *mic_register_intr_callback(struct mic_device *mdev, |
| u8 idx, irq_handler_t handler, irq_handler_t thread_fn, |
| void *data) |
| { |
| struct mic_intr_cb *intr_cb; |
| unsigned long flags; |
| int rc; |
| intr_cb = kmalloc(sizeof(*intr_cb), GFP_KERNEL); |
| |
| if (!intr_cb) |
| return ERR_PTR(-ENOMEM); |
| |
| intr_cb->handler = handler; |
| intr_cb->thread_fn = thread_fn; |
| intr_cb->data = data; |
| intr_cb->cb_id = ida_simple_get(&mdev->irq_info.cb_ida, |
| 0, 0, GFP_KERNEL); |
| if (intr_cb->cb_id < 0) { |
| rc = intr_cb->cb_id; |
| goto ida_fail; |
| } |
| |
| spin_lock(&mdev->irq_info.mic_thread_lock); |
| spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags); |
| list_add_tail(&intr_cb->list, &mdev->irq_info.cb_list[idx]); |
| spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags); |
| spin_unlock(&mdev->irq_info.mic_thread_lock); |
| |
| return intr_cb; |
| ida_fail: |
| kfree(intr_cb); |
| return ERR_PTR(rc); |
| } |
| |
| /** |
| * mic_unregister_intr_callback - Unregister the callback handler |
| * identified by its callback id. |
| * |
| * @mdev: pointer to the mic_device instance |
| * @idx: The callback structure id to be unregistered. |
| * Return the source id that was unregistered or MIC_NUM_OFFSETS if no |
| * such callback handler was found. |
| */ |
| static u8 mic_unregister_intr_callback(struct mic_device *mdev, u32 idx) |
| { |
| struct list_head *pos, *tmp; |
| struct mic_intr_cb *intr_cb; |
| unsigned long flags; |
| int i; |
| |
| spin_lock(&mdev->irq_info.mic_thread_lock); |
| spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags); |
| for (i = 0; i < MIC_NUM_OFFSETS; i++) { |
| list_for_each_safe(pos, tmp, &mdev->irq_info.cb_list[i]) { |
| intr_cb = list_entry(pos, struct mic_intr_cb, list); |
| if (intr_cb->cb_id == idx) { |
| list_del(pos); |
| ida_simple_remove(&mdev->irq_info.cb_ida, |
| intr_cb->cb_id); |
| kfree(intr_cb); |
| spin_unlock_irqrestore( |
| &mdev->irq_info.mic_intr_lock, flags); |
| spin_unlock(&mdev->irq_info.mic_thread_lock); |
| return i; |
| } |
| } |
| } |
| spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags); |
| spin_unlock(&mdev->irq_info.mic_thread_lock); |
| return MIC_NUM_OFFSETS; |
| } |
| |
| /** |
| * mic_setup_msix - Initializes MSIx interrupts. |
| * |
| * @mdev: pointer to mic_device instance |
| * @pdev: PCI device structure |
| * |
| * RETURNS: An appropriate -ERRNO error value on error, or zero for success. |
| */ |
| static int mic_setup_msix(struct mic_device *mdev, struct pci_dev *pdev) |
| { |
| int rc, i; |
| int entry_size = sizeof(*mdev->irq_info.msix_entries); |
| |
| mdev->irq_info.msix_entries = kmalloc_array(MIC_MIN_MSIX, |
| entry_size, GFP_KERNEL); |
| if (!mdev->irq_info.msix_entries) { |
| rc = -ENOMEM; |
| goto err_nomem1; |
| } |
| |
| for (i = 0; i < MIC_MIN_MSIX; i++) |
| mdev->irq_info.msix_entries[i].entry = i; |
| |
| rc = pci_enable_msix_exact(pdev, mdev->irq_info.msix_entries, |
| MIC_MIN_MSIX); |
| if (rc) { |
| dev_dbg(&pdev->dev, "Error enabling MSIx. rc = %d\n", rc); |
| goto err_enable_msix; |
| } |
| |
| mdev->irq_info.num_vectors = MIC_MIN_MSIX; |
| mdev->irq_info.mic_msi_map = kzalloc((sizeof(u32) * |
| mdev->irq_info.num_vectors), GFP_KERNEL); |
| |
| if (!mdev->irq_info.mic_msi_map) { |
| rc = -ENOMEM; |
| goto err_nomem2; |
| } |
| |
| dev_dbg(&mdev->pdev->dev, |
| "%d MSIx irqs setup\n", mdev->irq_info.num_vectors); |
| return 0; |
| err_nomem2: |
| pci_disable_msix(pdev); |
| err_enable_msix: |
| kfree(mdev->irq_info.msix_entries); |
| err_nomem1: |
| mdev->irq_info.num_vectors = 0; |
| return rc; |
| } |
| |
| /** |
| * mic_setup_callbacks - Initialize data structures needed |
| * to handle callbacks. |
| * |
| * @mdev: pointer to mic_device instance |
| */ |
| static int mic_setup_callbacks(struct mic_device *mdev) |
| { |
| int i; |
| |
| mdev->irq_info.cb_list = kmalloc_array(MIC_NUM_OFFSETS, |
| sizeof(*mdev->irq_info.cb_list), |
| GFP_KERNEL); |
| if (!mdev->irq_info.cb_list) |
| return -ENOMEM; |
| |
| for (i = 0; i < MIC_NUM_OFFSETS; i++) |
| INIT_LIST_HEAD(&mdev->irq_info.cb_list[i]); |
| ida_init(&mdev->irq_info.cb_ida); |
| spin_lock_init(&mdev->irq_info.mic_intr_lock); |
| spin_lock_init(&mdev->irq_info.mic_thread_lock); |
| return 0; |
| } |
| |
| /** |
| * mic_release_callbacks - Uninitialize data structures needed |
| * to handle callbacks. |
| * |
| * @mdev: pointer to mic_device instance |
| */ |
| static void mic_release_callbacks(struct mic_device *mdev) |
| { |
| unsigned long flags; |
| struct list_head *pos, *tmp; |
| struct mic_intr_cb *intr_cb; |
| int i; |
| |
| spin_lock(&mdev->irq_info.mic_thread_lock); |
| spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags); |
| for (i = 0; i < MIC_NUM_OFFSETS; i++) { |
| if (list_empty(&mdev->irq_info.cb_list[i])) |
| break; |
| |
| list_for_each_safe(pos, tmp, &mdev->irq_info.cb_list[i]) { |
| intr_cb = list_entry(pos, struct mic_intr_cb, list); |
| list_del(pos); |
| ida_simple_remove(&mdev->irq_info.cb_ida, |
| intr_cb->cb_id); |
| kfree(intr_cb); |
| } |
| } |
| spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags); |
| spin_unlock(&mdev->irq_info.mic_thread_lock); |
| ida_destroy(&mdev->irq_info.cb_ida); |
| kfree(mdev->irq_info.cb_list); |
| } |
| |
| /** |
| * mic_setup_msi - Initializes MSI interrupts. |
| * |
| * @mdev: pointer to mic_device instance |
| * @pdev: PCI device structure |
| * |
| * RETURNS: An appropriate -ERRNO error value on error, or zero for success. |
| */ |
| static int mic_setup_msi(struct mic_device *mdev, struct pci_dev *pdev) |
| { |
| int rc; |
| |
| rc = pci_enable_msi(pdev); |
| if (rc) { |
| dev_dbg(&pdev->dev, "Error enabling MSI. rc = %d\n", rc); |
| return rc; |
| } |
| |
| mdev->irq_info.num_vectors = 1; |
| mdev->irq_info.mic_msi_map = kzalloc((sizeof(u32) * |
| mdev->irq_info.num_vectors), GFP_KERNEL); |
| |
| if (!mdev->irq_info.mic_msi_map) { |
| rc = -ENOMEM; |
| goto err_nomem1; |
| } |
| |
| rc = mic_setup_callbacks(mdev); |
| if (rc) { |
| dev_err(&pdev->dev, "Error setting up callbacks\n"); |
| goto err_nomem2; |
| } |
| |
| rc = request_threaded_irq(pdev->irq, mic_interrupt, mic_thread_fn, |
| 0, "mic-msi", mdev); |
| if (rc) { |
| dev_err(&pdev->dev, "Error allocating MSI interrupt\n"); |
| goto err_irq_req_fail; |
| } |
| |
| dev_dbg(&pdev->dev, "%d MSI irqs setup\n", mdev->irq_info.num_vectors); |
| return 0; |
| err_irq_req_fail: |
| mic_release_callbacks(mdev); |
| err_nomem2: |
| kfree(mdev->irq_info.mic_msi_map); |
| err_nomem1: |
| pci_disable_msi(pdev); |
| mdev->irq_info.num_vectors = 0; |
| return rc; |
| } |
| |
| /** |
| * mic_setup_intx - Initializes legacy interrupts. |
| * |
| * @mdev: pointer to mic_device instance |
| * @pdev: PCI device structure |
| * |
| * RETURNS: An appropriate -ERRNO error value on error, or zero for success. |
| */ |
| static int mic_setup_intx(struct mic_device *mdev, struct pci_dev *pdev) |
| { |
| int rc; |
| |
| /* Enable intx */ |
| pci_intx(pdev, 1); |
| rc = mic_setup_callbacks(mdev); |
| if (rc) { |
| dev_err(&pdev->dev, "Error setting up callbacks\n"); |
| goto err_nomem; |
| } |
| |
| rc = request_threaded_irq(pdev->irq, mic_interrupt, mic_thread_fn, |
| IRQF_SHARED, "mic-intx", mdev); |
| if (rc) |
| goto err; |
| |
| dev_dbg(&pdev->dev, "intx irq setup\n"); |
| return 0; |
| err: |
| mic_release_callbacks(mdev); |
| err_nomem: |
| return rc; |
| } |
| |
| /** |
| * mic_next_db - Retrieve the next doorbell interrupt source id. |
| * The id is picked sequentially from the available pool of |
| * doorlbell ids. |
| * |
| * @mdev: pointer to the mic_device instance. |
| * |
| * Returns the next doorbell interrupt source. |
| */ |
| int mic_next_db(struct mic_device *mdev) |
| { |
| int next_db; |
| |
| next_db = mdev->irq_info.next_avail_src % |
| mdev->intr_info->intr_len[MIC_INTR_DB]; |
| mdev->irq_info.next_avail_src++; |
| return next_db; |
| } |
| |
| #define COOKIE_ID_SHIFT 16 |
| #define GET_ENTRY(cookie) ((cookie) & 0xFFFF) |
| #define GET_OFFSET(cookie) ((cookie) >> COOKIE_ID_SHIFT) |
| #define MK_COOKIE(x, y) ((x) | (y) << COOKIE_ID_SHIFT) |
| |
| /** |
| * mic_request_threaded_irq - request an irq. mic_mutex needs |
| * to be held before calling this function. |
| * |
| * @mdev: pointer to mic_device instance |
| * @handler: The callback function that handles the interrupt. |
| * The function needs to call ack_interrupts |
| * (mdev->ops->ack_interrupt(mdev)) when handling the interrupts. |
| * @thread_fn: thread fn required by request_threaded_irq. |
| * @name: The ASCII name of the callee requesting the irq. |
| * @data: private data that is returned back when calling the |
| * function handler. |
| * @intr_src: The source id of the requester. Its the doorbell id |
| * for Doorbell interrupts and DMA channel id for DMA interrupts. |
| * @type: The type of interrupt. Values defined in mic_intr_type |
| * |
| * returns: The cookie that is transparent to the caller. Passed |
| * back when calling mic_free_irq. An appropriate error code |
| * is returned on failure. Caller needs to use IS_ERR(return_val) |
| * to check for failure and PTR_ERR(return_val) to obtained the |
| * error code. |
| * |
| */ |
| struct mic_irq * |
| mic_request_threaded_irq(struct mic_device *mdev, |
| irq_handler_t handler, irq_handler_t thread_fn, |
| const char *name, void *data, int intr_src, |
| enum mic_intr_type type) |
| { |
| u16 offset; |
| int rc = 0; |
| struct msix_entry *msix = NULL; |
| unsigned long cookie = 0; |
| u16 entry; |
| struct mic_intr_cb *intr_cb; |
| struct pci_dev *pdev = mdev->pdev; |
| |
| offset = mic_map_src_to_offset(mdev, intr_src, type); |
| if (offset >= MIC_NUM_OFFSETS) { |
| dev_err(&mdev->pdev->dev, |
| "Error mapping index %d to a valid source id.\n", |
| intr_src); |
| rc = -EINVAL; |
| goto err; |
| } |
| |
| if (mdev->irq_info.num_vectors > 1) { |
| msix = mic_get_available_vector(mdev); |
| if (!msix) { |
| dev_err(&mdev->pdev->dev, |
| "No MSIx vectors available for use.\n"); |
| rc = -ENOSPC; |
| goto err; |
| } |
| |
| rc = request_threaded_irq(msix->vector, handler, thread_fn, |
| 0, name, data); |
| if (rc) { |
| dev_dbg(&mdev->pdev->dev, |
| "request irq failed rc = %d\n", rc); |
| goto err; |
| } |
| entry = msix->entry; |
| mdev->irq_info.mic_msi_map[entry] |= BIT(offset); |
| mdev->intr_ops->program_msi_to_src_map(mdev, |
| entry, offset, true); |
| cookie = MK_COOKIE(entry, offset); |
| dev_dbg(&mdev->pdev->dev, "irq: %d assigned for src: %d\n", |
| msix->vector, intr_src); |
| } else { |
| intr_cb = mic_register_intr_callback(mdev, offset, handler, |
| thread_fn, data); |
| if (IS_ERR(intr_cb)) { |
| dev_err(&mdev->pdev->dev, |
| "No available callback entries for use\n"); |
| rc = PTR_ERR(intr_cb); |
| goto err; |
| } |
| |
| entry = 0; |
| if (pci_dev_msi_enabled(pdev)) { |
| mdev->irq_info.mic_msi_map[entry] |= (1 << offset); |
| mdev->intr_ops->program_msi_to_src_map(mdev, |
| entry, offset, true); |
| } |
| cookie = MK_COOKIE(entry, intr_cb->cb_id); |
| dev_dbg(&mdev->pdev->dev, "callback %d registered for src: %d\n", |
| intr_cb->cb_id, intr_src); |
| } |
| return (struct mic_irq *)cookie; |
| err: |
| return ERR_PTR(rc); |
| } |
| |
| /** |
| * mic_free_irq - free irq. mic_mutex |
| * needs to be held before calling this function. |
| * |
| * @mdev: pointer to mic_device instance |
| * @cookie: cookie obtained during a successful call to mic_request_threaded_irq |
| * @data: private data specified by the calling function during the |
| * mic_request_threaded_irq |
| * |
| * returns: none. |
| */ |
| void mic_free_irq(struct mic_device *mdev, |
| struct mic_irq *cookie, void *data) |
| { |
| u32 offset; |
| u32 entry; |
| u8 src_id; |
| unsigned int irq; |
| struct pci_dev *pdev = mdev->pdev; |
| |
| entry = GET_ENTRY((unsigned long)cookie); |
| offset = GET_OFFSET((unsigned long)cookie); |
| if (mdev->irq_info.num_vectors > 1) { |
| if (entry >= mdev->irq_info.num_vectors) { |
| dev_warn(&mdev->pdev->dev, |
| "entry %d should be < num_irq %d\n", |
| entry, mdev->irq_info.num_vectors); |
| return; |
| } |
| irq = mdev->irq_info.msix_entries[entry].vector; |
| free_irq(irq, data); |
| mdev->irq_info.mic_msi_map[entry] &= ~(BIT(offset)); |
| mdev->intr_ops->program_msi_to_src_map(mdev, |
| entry, offset, false); |
| |
| dev_dbg(&mdev->pdev->dev, "irq: %d freed\n", irq); |
| } else { |
| irq = pdev->irq; |
| src_id = mic_unregister_intr_callback(mdev, offset); |
| if (src_id >= MIC_NUM_OFFSETS) { |
| dev_warn(&mdev->pdev->dev, "Error unregistering callback\n"); |
| return; |
| } |
| if (pci_dev_msi_enabled(pdev)) { |
| mdev->irq_info.mic_msi_map[entry] &= ~(BIT(src_id)); |
| mdev->intr_ops->program_msi_to_src_map(mdev, |
| entry, src_id, false); |
| } |
| dev_dbg(&mdev->pdev->dev, "callback %d unregistered for src: %d\n", |
| offset, src_id); |
| } |
| } |
| |
| /** |
| * mic_setup_interrupts - Initializes interrupts. |
| * |
| * @mdev: pointer to mic_device instance |
| * @pdev: PCI device structure |
| * |
| * RETURNS: An appropriate -ERRNO error value on error, or zero for success. |
| */ |
| int mic_setup_interrupts(struct mic_device *mdev, struct pci_dev *pdev) |
| { |
| int rc; |
| |
| rc = mic_setup_msix(mdev, pdev); |
| if (!rc) |
| goto done; |
| |
| rc = mic_setup_msi(mdev, pdev); |
| if (!rc) |
| goto done; |
| |
| rc = mic_setup_intx(mdev, pdev); |
| if (rc) { |
| dev_err(&mdev->pdev->dev, "no usable interrupts\n"); |
| return rc; |
| } |
| done: |
| mdev->intr_ops->enable_interrupts(mdev); |
| return 0; |
| } |
| |
| /** |
| * mic_free_interrupts - Frees interrupts setup by mic_setup_interrupts |
| * |
| * @mdev: pointer to mic_device instance |
| * @pdev: PCI device structure |
| * |
| * returns none. |
| */ |
| void mic_free_interrupts(struct mic_device *mdev, struct pci_dev *pdev) |
| { |
| int i; |
| |
| mdev->intr_ops->disable_interrupts(mdev); |
| if (mdev->irq_info.num_vectors > 1) { |
| for (i = 0; i < mdev->irq_info.num_vectors; i++) { |
| if (mdev->irq_info.mic_msi_map[i]) |
| dev_warn(&pdev->dev, "irq %d may still be in use.\n", |
| mdev->irq_info.msix_entries[i].vector); |
| } |
| kfree(mdev->irq_info.mic_msi_map); |
| kfree(mdev->irq_info.msix_entries); |
| pci_disable_msix(pdev); |
| } else { |
| if (pci_dev_msi_enabled(pdev)) { |
| free_irq(pdev->irq, mdev); |
| kfree(mdev->irq_info.mic_msi_map); |
| pci_disable_msi(pdev); |
| } else { |
| free_irq(pdev->irq, mdev); |
| } |
| mic_release_callbacks(mdev); |
| } |
| } |
| |
| /** |
| * mic_intr_restore - Restore MIC interrupt registers. |
| * |
| * @mdev: pointer to mic_device instance. |
| * |
| * Restore the interrupt registers to values previously |
| * stored in the SW data structures. mic_mutex needs to |
| * be held before calling this function. |
| * |
| * returns None. |
| */ |
| void mic_intr_restore(struct mic_device *mdev) |
| { |
| int entry, offset; |
| struct pci_dev *pdev = mdev->pdev; |
| |
| if (!pci_dev_msi_enabled(pdev)) |
| return; |
| |
| for (entry = 0; entry < mdev->irq_info.num_vectors; entry++) { |
| for (offset = 0; offset < MIC_NUM_OFFSETS; offset++) { |
| if (mdev->irq_info.mic_msi_map[entry] & BIT(offset)) |
| mdev->intr_ops->program_msi_to_src_map(mdev, |
| entry, offset, true); |
| } |
| } |
| } |