blob: e2278bf9f11d4904d98bd47253d4ee3488bb7517 [file] [log] [blame]
// SPDX-License-Identifier: GPL-2.0-only
/*
* Intel MIC Platform Software Stack (MPSS)
*
* Copyright(c) 2014 Intel Corporation.
*
* Intel SCIF driver.
*/
#include <linux/module.h>
#include <linux/idr.h>
#include <linux/mic_common.h>
#include "../common/mic_dev.h"
#include "../bus/scif_bus.h"
#include "scif_peer_bus.h"
#include "scif_main.h"
#include "scif_map.h"
struct scif_info scif_info = {
.mdev = {
.minor = MISC_DYNAMIC_MINOR,
.name = "scif",
.fops = &scif_fops,
}
};
struct scif_dev *scif_dev;
struct kmem_cache *unaligned_cache;
static atomic_t g_loopb_cnt;
/* Runs in the context of intr_wq */
static void scif_intr_bh_handler(struct work_struct *work)
{
struct scif_dev *scifdev =
container_of(work, struct scif_dev, intr_bh);
if (scifdev_self(scifdev))
scif_loopb_msg_handler(scifdev, scifdev->qpairs);
else
scif_nodeqp_intrhandler(scifdev, scifdev->qpairs);
}
int scif_setup_intr_wq(struct scif_dev *scifdev)
{
if (!scifdev->intr_wq) {
snprintf(scifdev->intr_wqname, sizeof(scifdev->intr_wqname),
"SCIF INTR %d", scifdev->node);
scifdev->intr_wq =
alloc_ordered_workqueue(scifdev->intr_wqname, 0);
if (!scifdev->intr_wq)
return -ENOMEM;
INIT_WORK(&scifdev->intr_bh, scif_intr_bh_handler);
}
return 0;
}
void scif_destroy_intr_wq(struct scif_dev *scifdev)
{
if (scifdev->intr_wq) {
destroy_workqueue(scifdev->intr_wq);
scifdev->intr_wq = NULL;
}
}
irqreturn_t scif_intr_handler(int irq, void *data)
{
struct scif_dev *scifdev = data;
struct scif_hw_dev *sdev = scifdev->sdev;
sdev->hw_ops->ack_interrupt(sdev, scifdev->db);
queue_work(scifdev->intr_wq, &scifdev->intr_bh);
return IRQ_HANDLED;
}
static void scif_qp_setup_handler(struct work_struct *work)
{
struct scif_dev *scifdev = container_of(work, struct scif_dev,
qp_dwork.work);
struct scif_hw_dev *sdev = scifdev->sdev;
dma_addr_t da = 0;
int err;
if (scif_is_mgmt_node()) {
struct mic_bootparam *bp = sdev->dp;
da = bp->scif_card_dma_addr;
scifdev->rdb = bp->h2c_scif_db;
} else {
struct mic_bootparam __iomem *bp = sdev->rdp;
da = readq(&bp->scif_host_dma_addr);
scifdev->rdb = ioread8(&bp->c2h_scif_db);
}
if (da) {
err = scif_qp_response(da, scifdev);
if (err)
dev_err(&scifdev->sdev->dev,
"scif_qp_response err %d\n", err);
} else {
schedule_delayed_work(&scifdev->qp_dwork,
msecs_to_jiffies(1000));
}
}
static int scif_setup_scifdev(void)
{
/* We support a maximum of 129 SCIF nodes including the mgmt node */
#define MAX_SCIF_NODES 129
int i;
u8 num_nodes = MAX_SCIF_NODES;
scif_dev = kcalloc(num_nodes, sizeof(*scif_dev), GFP_KERNEL);
if (!scif_dev)
return -ENOMEM;
for (i = 0; i < num_nodes; i++) {
struct scif_dev *scifdev = &scif_dev[i];
scifdev->node = i;
scifdev->exit = OP_IDLE;
init_waitqueue_head(&scifdev->disconn_wq);
mutex_init(&scifdev->lock);
INIT_WORK(&scifdev->peer_add_work, scif_add_peer_device);
INIT_DELAYED_WORK(&scifdev->p2p_dwork,
scif_poll_qp_state);
INIT_DELAYED_WORK(&scifdev->qp_dwork,
scif_qp_setup_handler);
INIT_LIST_HEAD(&scifdev->p2p);
RCU_INIT_POINTER(scifdev->spdev, NULL);
}
return 0;
}
static void scif_destroy_scifdev(void)
{
kfree(scif_dev);
scif_dev = NULL;
}
static int scif_probe(struct scif_hw_dev *sdev)
{
struct scif_dev *scifdev = &scif_dev[sdev->dnode];
int rc;
dev_set_drvdata(&sdev->dev, sdev);
scifdev->sdev = sdev;
if (1 == atomic_add_return(1, &g_loopb_cnt)) {
struct scif_dev *loopb_dev = &scif_dev[sdev->snode];
loopb_dev->sdev = sdev;
rc = scif_setup_loopback_qp(loopb_dev);
if (rc)
goto exit;
}
rc = scif_setup_intr_wq(scifdev);
if (rc)
goto destroy_loopb;
rc = scif_setup_qp(scifdev);
if (rc)
goto destroy_intr;
scifdev->db = sdev->hw_ops->next_db(sdev);
scifdev->cookie = sdev->hw_ops->request_irq(sdev, scif_intr_handler,
"SCIF_INTR", scifdev,
scifdev->db);
if (IS_ERR(scifdev->cookie)) {
rc = PTR_ERR(scifdev->cookie);
goto free_qp;
}
if (scif_is_mgmt_node()) {
struct mic_bootparam *bp = sdev->dp;
bp->c2h_scif_db = scifdev->db;
bp->scif_host_dma_addr = scifdev->qp_dma_addr;
} else {
struct mic_bootparam __iomem *bp = sdev->rdp;
iowrite8(scifdev->db, &bp->h2c_scif_db);
writeq(scifdev->qp_dma_addr, &bp->scif_card_dma_addr);
}
schedule_delayed_work(&scifdev->qp_dwork,
msecs_to_jiffies(1000));
return rc;
free_qp:
scif_free_qp(scifdev);
destroy_intr:
scif_destroy_intr_wq(scifdev);
destroy_loopb:
if (atomic_dec_and_test(&g_loopb_cnt))
scif_destroy_loopback_qp(&scif_dev[sdev->snode]);
exit:
return rc;
}
void scif_stop(struct scif_dev *scifdev)
{
struct scif_dev *dev;
int i;
for (i = scif_info.maxid; i >= 0; i--) {
dev = &scif_dev[i];
if (scifdev_self(dev))
continue;
scif_handle_remove_node(i);
}
}
static void scif_remove(struct scif_hw_dev *sdev)
{
struct scif_dev *scifdev = &scif_dev[sdev->dnode];
if (scif_is_mgmt_node()) {
struct mic_bootparam *bp = sdev->dp;
bp->c2h_scif_db = -1;
bp->scif_host_dma_addr = 0x0;
} else {
struct mic_bootparam __iomem *bp = sdev->rdp;
iowrite8(-1, &bp->h2c_scif_db);
writeq(0x0, &bp->scif_card_dma_addr);
}
if (scif_is_mgmt_node()) {
scif_disconnect_node(scifdev->node, true);
} else {
scif_info.card_initiated_exit = true;
scif_stop(scifdev);
}
if (atomic_dec_and_test(&g_loopb_cnt))
scif_destroy_loopback_qp(&scif_dev[sdev->snode]);
if (scifdev->cookie) {
sdev->hw_ops->free_irq(sdev, scifdev->cookie, scifdev);
scifdev->cookie = NULL;
}
scif_destroy_intr_wq(scifdev);
cancel_delayed_work(&scifdev->qp_dwork);
scif_free_qp(scifdev);
scifdev->rdb = -1;
scifdev->sdev = NULL;
}
static struct scif_hw_dev_id id_table[] = {
{ MIC_SCIF_DEV, SCIF_DEV_ANY_ID },
{ 0 },
};
static struct scif_driver scif_driver = {
.driver.name = KBUILD_MODNAME,
.driver.owner = THIS_MODULE,
.id_table = id_table,
.probe = scif_probe,
.remove = scif_remove,
};
static int _scif_init(void)
{
int rc;
mutex_init(&scif_info.eplock);
spin_lock_init(&scif_info.rmalock);
spin_lock_init(&scif_info.nb_connect_lock);
spin_lock_init(&scif_info.port_lock);
mutex_init(&scif_info.conflock);
mutex_init(&scif_info.connlock);
mutex_init(&scif_info.fencelock);
INIT_LIST_HEAD(&scif_info.uaccept);
INIT_LIST_HEAD(&scif_info.listen);
INIT_LIST_HEAD(&scif_info.zombie);
INIT_LIST_HEAD(&scif_info.connected);
INIT_LIST_HEAD(&scif_info.disconnected);
INIT_LIST_HEAD(&scif_info.rma);
INIT_LIST_HEAD(&scif_info.rma_tc);
INIT_LIST_HEAD(&scif_info.mmu_notif_cleanup);
INIT_LIST_HEAD(&scif_info.fence);
INIT_LIST_HEAD(&scif_info.nb_connect_list);
init_waitqueue_head(&scif_info.exitwq);
scif_info.rma_tc_limit = SCIF_RMA_TEMP_CACHE_LIMIT;
scif_info.en_msg_log = 0;
scif_info.p2p_enable = 1;
rc = scif_setup_scifdev();
if (rc)
goto error;
unaligned_cache = kmem_cache_create("Unaligned_DMA",
SCIF_KMEM_UNALIGNED_BUF_SIZE,
0, SLAB_HWCACHE_ALIGN, NULL);
if (!unaligned_cache) {
rc = -ENOMEM;
goto free_sdev;
}
INIT_WORK(&scif_info.misc_work, scif_misc_handler);
INIT_WORK(&scif_info.mmu_notif_work, scif_mmu_notif_handler);
INIT_WORK(&scif_info.conn_work, scif_conn_handler);
idr_init(&scif_ports);
return 0;
free_sdev:
scif_destroy_scifdev();
error:
return rc;
}
static void _scif_exit(void)
{
idr_destroy(&scif_ports);
kmem_cache_destroy(unaligned_cache);
scif_destroy_scifdev();
}
static int __init scif_init(void)
{
struct miscdevice *mdev = &scif_info.mdev;
int rc;
_scif_init();
iova_cache_get();
rc = scif_peer_bus_init();
if (rc)
goto exit;
rc = scif_register_driver(&scif_driver);
if (rc)
goto peer_bus_exit;
rc = misc_register(mdev);
if (rc)
goto unreg_scif;
scif_init_debugfs();
return 0;
unreg_scif:
scif_unregister_driver(&scif_driver);
peer_bus_exit:
scif_peer_bus_exit();
exit:
_scif_exit();
return rc;
}
static void __exit scif_exit(void)
{
scif_exit_debugfs();
misc_deregister(&scif_info.mdev);
scif_unregister_driver(&scif_driver);
scif_peer_bus_exit();
iova_cache_put();
_scif_exit();
}
module_init(scif_init);
module_exit(scif_exit);
MODULE_DEVICE_TABLE(scif, id_table);
MODULE_AUTHOR("Intel Corporation");
MODULE_DESCRIPTION("Intel(R) SCIF driver");
MODULE_LICENSE("GPL v2");