| // SPDX-License-Identifier: GPL-2.0-only |
| /* |
| * Intel MIC Platform Software Stack (MPSS) |
| * |
| * Copyright(c) 2014 Intel Corporation. |
| * |
| * Intel SCIF driver. |
| */ |
| #include <linux/scif.h> |
| #include "scif_main.h" |
| #include "scif_map.h" |
| |
| static const char * const scif_ep_states[] = { |
| "Unbound", |
| "Bound", |
| "Listening", |
| "Connected", |
| "Connecting", |
| "Mapping", |
| "Closing", |
| "Close Listening", |
| "Disconnected", |
| "Zombie"}; |
| |
| enum conn_async_state { |
| ASYNC_CONN_IDLE = 1, /* ep setup for async connect */ |
| ASYNC_CONN_INPROGRESS, /* async connect in progress */ |
| ASYNC_CONN_FLUSH_WORK /* async work flush in progress */ |
| }; |
| |
| /* |
| * File operations for anonymous inode file associated with a SCIF endpoint, |
| * used in kernel mode SCIF poll. Kernel mode SCIF poll calls portions of the |
| * poll API in the kernel and these take in a struct file *. Since a struct |
| * file is not available to kernel mode SCIF, it uses an anonymous file for |
| * this purpose. |
| */ |
| const struct file_operations scif_anon_fops = { |
| .owner = THIS_MODULE, |
| }; |
| |
| scif_epd_t scif_open(void) |
| { |
| struct scif_endpt *ep; |
| int err; |
| |
| might_sleep(); |
| ep = kzalloc(sizeof(*ep), GFP_KERNEL); |
| if (!ep) |
| goto err_ep_alloc; |
| |
| ep->qp_info.qp = kzalloc(sizeof(*ep->qp_info.qp), GFP_KERNEL); |
| if (!ep->qp_info.qp) |
| goto err_qp_alloc; |
| |
| err = scif_anon_inode_getfile(ep); |
| if (err) |
| goto err_anon_inode; |
| |
| spin_lock_init(&ep->lock); |
| mutex_init(&ep->sendlock); |
| mutex_init(&ep->recvlock); |
| |
| scif_rma_ep_init(ep); |
| ep->state = SCIFEP_UNBOUND; |
| dev_dbg(scif_info.mdev.this_device, |
| "SCIFAPI open: ep %p success\n", ep); |
| return ep; |
| |
| err_anon_inode: |
| kfree(ep->qp_info.qp); |
| err_qp_alloc: |
| kfree(ep); |
| err_ep_alloc: |
| return NULL; |
| } |
| EXPORT_SYMBOL_GPL(scif_open); |
| |
| /* |
| * scif_disconnect_ep - Disconnects the endpoint if found |
| * @epd: The end point returned from scif_open() |
| */ |
| static struct scif_endpt *scif_disconnect_ep(struct scif_endpt *ep) |
| { |
| struct scifmsg msg; |
| struct scif_endpt *fep = NULL; |
| struct scif_endpt *tmpep; |
| struct list_head *pos, *tmpq; |
| int err; |
| |
| /* |
| * Wake up any threads blocked in send()/recv() before closing |
| * out the connection. Grabbing and releasing the send/recv lock |
| * will ensure that any blocked senders/receivers have exited for |
| * Ring 0 endpoints. It is a Ring 0 bug to call send/recv after |
| * close. Ring 3 endpoints are not affected since close will not |
| * be called while there are IOCTLs executing. |
| */ |
| wake_up_interruptible(&ep->sendwq); |
| wake_up_interruptible(&ep->recvwq); |
| mutex_lock(&ep->sendlock); |
| mutex_unlock(&ep->sendlock); |
| mutex_lock(&ep->recvlock); |
| mutex_unlock(&ep->recvlock); |
| |
| /* Remove from the connected list */ |
| mutex_lock(&scif_info.connlock); |
| list_for_each_safe(pos, tmpq, &scif_info.connected) { |
| tmpep = list_entry(pos, struct scif_endpt, list); |
| if (tmpep == ep) { |
| list_del(pos); |
| fep = tmpep; |
| spin_lock(&ep->lock); |
| break; |
| } |
| } |
| |
| if (!fep) { |
| /* |
| * The other side has completed the disconnect before |
| * the end point can be removed from the list. Therefore |
| * the ep lock is not locked, traverse the disconnected |
| * list to find the endpoint and release the conn lock. |
| */ |
| list_for_each_safe(pos, tmpq, &scif_info.disconnected) { |
| tmpep = list_entry(pos, struct scif_endpt, list); |
| if (tmpep == ep) { |
| list_del(pos); |
| break; |
| } |
| } |
| mutex_unlock(&scif_info.connlock); |
| return NULL; |
| } |
| |
| init_completion(&ep->discon); |
| msg.uop = SCIF_DISCNCT; |
| msg.src = ep->port; |
| msg.dst = ep->peer; |
| msg.payload[0] = (u64)ep; |
| msg.payload[1] = ep->remote_ep; |
| |
| err = scif_nodeqp_send(ep->remote_dev, &msg); |
| spin_unlock(&ep->lock); |
| mutex_unlock(&scif_info.connlock); |
| |
| if (!err) |
| /* Wait for the remote node to respond with SCIF_DISCNT_ACK */ |
| wait_for_completion_timeout(&ep->discon, |
| SCIF_NODE_ALIVE_TIMEOUT); |
| return ep; |
| } |
| |
| int scif_close(scif_epd_t epd) |
| { |
| struct scif_endpt *ep = (struct scif_endpt *)epd; |
| struct scif_endpt *tmpep; |
| struct list_head *pos, *tmpq; |
| enum scif_epd_state oldstate; |
| bool flush_conn; |
| |
| dev_dbg(scif_info.mdev.this_device, "SCIFAPI close: ep %p %s\n", |
| ep, scif_ep_states[ep->state]); |
| might_sleep(); |
| spin_lock(&ep->lock); |
| flush_conn = (ep->conn_async_state == ASYNC_CONN_INPROGRESS); |
| spin_unlock(&ep->lock); |
| |
| if (flush_conn) |
| flush_work(&scif_info.conn_work); |
| |
| spin_lock(&ep->lock); |
| oldstate = ep->state; |
| |
| ep->state = SCIFEP_CLOSING; |
| |
| switch (oldstate) { |
| case SCIFEP_ZOMBIE: |
| dev_err(scif_info.mdev.this_device, |
| "SCIFAPI close: zombie state unexpected\n"); |
| /* fall through */ |
| case SCIFEP_DISCONNECTED: |
| spin_unlock(&ep->lock); |
| scif_unregister_all_windows(epd); |
| /* Remove from the disconnected list */ |
| mutex_lock(&scif_info.connlock); |
| list_for_each_safe(pos, tmpq, &scif_info.disconnected) { |
| tmpep = list_entry(pos, struct scif_endpt, list); |
| if (tmpep == ep) { |
| list_del(pos); |
| break; |
| } |
| } |
| mutex_unlock(&scif_info.connlock); |
| break; |
| case SCIFEP_UNBOUND: |
| case SCIFEP_BOUND: |
| case SCIFEP_CONNECTING: |
| spin_unlock(&ep->lock); |
| break; |
| case SCIFEP_MAPPING: |
| case SCIFEP_CONNECTED: |
| case SCIFEP_CLOSING: |
| { |
| spin_unlock(&ep->lock); |
| scif_unregister_all_windows(epd); |
| scif_disconnect_ep(ep); |
| break; |
| } |
| case SCIFEP_LISTENING: |
| case SCIFEP_CLLISTEN: |
| { |
| struct scif_conreq *conreq; |
| struct scifmsg msg; |
| struct scif_endpt *aep; |
| |
| spin_unlock(&ep->lock); |
| mutex_lock(&scif_info.eplock); |
| |
| /* remove from listen list */ |
| list_for_each_safe(pos, tmpq, &scif_info.listen) { |
| tmpep = list_entry(pos, struct scif_endpt, list); |
| if (tmpep == ep) |
| list_del(pos); |
| } |
| /* Remove any dangling accepts */ |
| while (ep->acceptcnt) { |
| aep = list_first_entry(&ep->li_accept, |
| struct scif_endpt, liacceptlist); |
| list_del(&aep->liacceptlist); |
| scif_put_port(aep->port.port); |
| list_for_each_safe(pos, tmpq, &scif_info.uaccept) { |
| tmpep = list_entry(pos, struct scif_endpt, |
| miacceptlist); |
| if (tmpep == aep) { |
| list_del(pos); |
| break; |
| } |
| } |
| mutex_unlock(&scif_info.eplock); |
| mutex_lock(&scif_info.connlock); |
| list_for_each_safe(pos, tmpq, &scif_info.connected) { |
| tmpep = list_entry(pos, |
| struct scif_endpt, list); |
| if (tmpep == aep) { |
| list_del(pos); |
| break; |
| } |
| } |
| list_for_each_safe(pos, tmpq, &scif_info.disconnected) { |
| tmpep = list_entry(pos, |
| struct scif_endpt, list); |
| if (tmpep == aep) { |
| list_del(pos); |
| break; |
| } |
| } |
| mutex_unlock(&scif_info.connlock); |
| scif_teardown_ep(aep); |
| mutex_lock(&scif_info.eplock); |
| scif_add_epd_to_zombie_list(aep, SCIF_EPLOCK_HELD); |
| ep->acceptcnt--; |
| } |
| |
| spin_lock(&ep->lock); |
| mutex_unlock(&scif_info.eplock); |
| |
| /* Remove and reject any pending connection requests. */ |
| while (ep->conreqcnt) { |
| conreq = list_first_entry(&ep->conlist, |
| struct scif_conreq, list); |
| list_del(&conreq->list); |
| |
| msg.uop = SCIF_CNCT_REJ; |
| msg.dst.node = conreq->msg.src.node; |
| msg.dst.port = conreq->msg.src.port; |
| msg.payload[0] = conreq->msg.payload[0]; |
| msg.payload[1] = conreq->msg.payload[1]; |
| /* |
| * No Error Handling on purpose for scif_nodeqp_send(). |
| * If the remote node is lost we still want free the |
| * connection requests on the self node. |
| */ |
| scif_nodeqp_send(&scif_dev[conreq->msg.src.node], |
| &msg); |
| ep->conreqcnt--; |
| kfree(conreq); |
| } |
| |
| spin_unlock(&ep->lock); |
| /* If a kSCIF accept is waiting wake it up */ |
| wake_up_interruptible(&ep->conwq); |
| break; |
| } |
| } |
| scif_put_port(ep->port.port); |
| scif_anon_inode_fput(ep); |
| scif_teardown_ep(ep); |
| scif_add_epd_to_zombie_list(ep, !SCIF_EPLOCK_HELD); |
| return 0; |
| } |
| EXPORT_SYMBOL_GPL(scif_close); |
| |
| /** |
| * scif_flush() - Wakes up any blocking accepts. The endpoint will no longer |
| * accept new connections. |
| * @epd: The end point returned from scif_open() |
| */ |
| int __scif_flush(scif_epd_t epd) |
| { |
| struct scif_endpt *ep = (struct scif_endpt *)epd; |
| |
| switch (ep->state) { |
| case SCIFEP_LISTENING: |
| { |
| ep->state = SCIFEP_CLLISTEN; |
| |
| /* If an accept is waiting wake it up */ |
| wake_up_interruptible(&ep->conwq); |
| break; |
| } |
| default: |
| break; |
| } |
| return 0; |
| } |
| |
| int scif_bind(scif_epd_t epd, u16 pn) |
| { |
| struct scif_endpt *ep = (struct scif_endpt *)epd; |
| int ret = 0; |
| int tmp; |
| |
| dev_dbg(scif_info.mdev.this_device, |
| "SCIFAPI bind: ep %p %s requested port number %d\n", |
| ep, scif_ep_states[ep->state], pn); |
| if (pn) { |
| /* |
| * Similar to IETF RFC 1700, SCIF ports below |
| * SCIF_ADMIN_PORT_END can only be bound by system (or root) |
| * processes or by processes executed by privileged users. |
| */ |
| if (pn < SCIF_ADMIN_PORT_END && !capable(CAP_SYS_ADMIN)) { |
| ret = -EACCES; |
| goto scif_bind_admin_exit; |
| } |
| } |
| |
| spin_lock(&ep->lock); |
| if (ep->state == SCIFEP_BOUND) { |
| ret = -EINVAL; |
| goto scif_bind_exit; |
| } else if (ep->state != SCIFEP_UNBOUND) { |
| ret = -EISCONN; |
| goto scif_bind_exit; |
| } |
| |
| if (pn) { |
| tmp = scif_rsrv_port(pn); |
| if (tmp != pn) { |
| ret = -EINVAL; |
| goto scif_bind_exit; |
| } |
| } else { |
| ret = scif_get_new_port(); |
| if (ret < 0) |
| goto scif_bind_exit; |
| pn = ret; |
| } |
| |
| ep->state = SCIFEP_BOUND; |
| ep->port.node = scif_info.nodeid; |
| ep->port.port = pn; |
| ep->conn_async_state = ASYNC_CONN_IDLE; |
| ret = pn; |
| dev_dbg(scif_info.mdev.this_device, |
| "SCIFAPI bind: bound to port number %d\n", pn); |
| scif_bind_exit: |
| spin_unlock(&ep->lock); |
| scif_bind_admin_exit: |
| return ret; |
| } |
| EXPORT_SYMBOL_GPL(scif_bind); |
| |
| int scif_listen(scif_epd_t epd, int backlog) |
| { |
| struct scif_endpt *ep = (struct scif_endpt *)epd; |
| |
| dev_dbg(scif_info.mdev.this_device, |
| "SCIFAPI listen: ep %p %s\n", ep, scif_ep_states[ep->state]); |
| spin_lock(&ep->lock); |
| switch (ep->state) { |
| case SCIFEP_ZOMBIE: |
| case SCIFEP_CLOSING: |
| case SCIFEP_CLLISTEN: |
| case SCIFEP_UNBOUND: |
| case SCIFEP_DISCONNECTED: |
| spin_unlock(&ep->lock); |
| return -EINVAL; |
| case SCIFEP_LISTENING: |
| case SCIFEP_CONNECTED: |
| case SCIFEP_CONNECTING: |
| case SCIFEP_MAPPING: |
| spin_unlock(&ep->lock); |
| return -EISCONN; |
| case SCIFEP_BOUND: |
| break; |
| } |
| |
| ep->state = SCIFEP_LISTENING; |
| ep->backlog = backlog; |
| |
| ep->conreqcnt = 0; |
| ep->acceptcnt = 0; |
| INIT_LIST_HEAD(&ep->conlist); |
| init_waitqueue_head(&ep->conwq); |
| INIT_LIST_HEAD(&ep->li_accept); |
| spin_unlock(&ep->lock); |
| |
| /* |
| * Listen status is complete so delete the qp information not needed |
| * on a listen before placing on the list of listening ep's |
| */ |
| scif_teardown_ep(ep); |
| ep->qp_info.qp = NULL; |
| |
| mutex_lock(&scif_info.eplock); |
| list_add_tail(&ep->list, &scif_info.listen); |
| mutex_unlock(&scif_info.eplock); |
| return 0; |
| } |
| EXPORT_SYMBOL_GPL(scif_listen); |
| |
| /* |
| ************************************************************************ |
| * SCIF connection flow: |
| * |
| * 1) A SCIF listening endpoint can call scif_accept(..) to wait for SCIF |
| * connections via a SCIF_CNCT_REQ message |
| * 2) A SCIF endpoint can initiate a SCIF connection by calling |
| * scif_connect(..) which calls scif_setup_qp_connect(..) which |
| * allocates the local qp for the endpoint ring buffer and then sends |
| * a SCIF_CNCT_REQ to the remote node and waits for a SCIF_CNCT_GNT or |
| * a SCIF_CNCT_REJ message |
| * 3) The peer node handles a SCIF_CNCT_REQ via scif_cnctreq_resp(..) which |
| * wakes up any threads blocked in step 1 or sends a SCIF_CNCT_REJ |
| * message otherwise |
| * 4) A thread blocked waiting for incoming connections allocates its local |
| * endpoint QP and ring buffer following which it sends a SCIF_CNCT_GNT |
| * and waits for a SCIF_CNCT_GNT(N)ACK. If the allocation fails then |
| * the node sends a SCIF_CNCT_REJ message |
| * 5) Upon receipt of a SCIF_CNCT_GNT or a SCIF_CNCT_REJ message the |
| * connecting endpoint is woken up as part of handling |
| * scif_cnctgnt_resp(..) following which it maps the remote endpoints' |
| * QP, updates its outbound QP and sends a SCIF_CNCT_GNTACK message on |
| * success or a SCIF_CNCT_GNTNACK message on failure and completes |
| * the scif_connect(..) API |
| * 6) Upon receipt of a SCIF_CNCT_GNT(N)ACK the accepting endpoint blocked |
| * in step 4 is woken up and completes the scif_accept(..) API |
| * 7) The SCIF connection is now established between the two SCIF endpoints. |
| */ |
| static int scif_conn_func(struct scif_endpt *ep) |
| { |
| int err = 0; |
| struct scifmsg msg; |
| struct device *spdev; |
| |
| err = scif_reserve_dma_chan(ep); |
| if (err) { |
| dev_err(&ep->remote_dev->sdev->dev, |
| "%s %d err %d\n", __func__, __LINE__, err); |
| ep->state = SCIFEP_BOUND; |
| goto connect_error_simple; |
| } |
| /* Initiate the first part of the endpoint QP setup */ |
| err = scif_setup_qp_connect(ep->qp_info.qp, &ep->qp_info.qp_offset, |
| SCIF_ENDPT_QP_SIZE, ep->remote_dev); |
| if (err) { |
| dev_err(&ep->remote_dev->sdev->dev, |
| "%s err %d qp_offset 0x%llx\n", |
| __func__, err, ep->qp_info.qp_offset); |
| ep->state = SCIFEP_BOUND; |
| goto connect_error_simple; |
| } |
| |
| spdev = scif_get_peer_dev(ep->remote_dev); |
| if (IS_ERR(spdev)) { |
| err = PTR_ERR(spdev); |
| goto cleanup_qp; |
| } |
| /* Format connect message and send it */ |
| msg.src = ep->port; |
| msg.dst = ep->conn_port; |
| msg.uop = SCIF_CNCT_REQ; |
| msg.payload[0] = (u64)ep; |
| msg.payload[1] = ep->qp_info.qp_offset; |
| err = _scif_nodeqp_send(ep->remote_dev, &msg); |
| if (err) |
| goto connect_error_dec; |
| scif_put_peer_dev(spdev); |
| /* |
| * Wait for the remote node to respond with SCIF_CNCT_GNT or |
| * SCIF_CNCT_REJ message. |
| */ |
| err = wait_event_timeout(ep->conwq, ep->state != SCIFEP_CONNECTING, |
| SCIF_NODE_ALIVE_TIMEOUT); |
| if (!err) { |
| dev_err(&ep->remote_dev->sdev->dev, |
| "%s %d timeout\n", __func__, __LINE__); |
| ep->state = SCIFEP_BOUND; |
| } |
| spdev = scif_get_peer_dev(ep->remote_dev); |
| if (IS_ERR(spdev)) { |
| err = PTR_ERR(spdev); |
| goto cleanup_qp; |
| } |
| if (ep->state == SCIFEP_MAPPING) { |
| err = scif_setup_qp_connect_response(ep->remote_dev, |
| ep->qp_info.qp, |
| ep->qp_info.gnt_pld); |
| /* |
| * If the resource to map the queue are not available then |
| * we need to tell the other side to terminate the accept |
| */ |
| if (err) { |
| dev_err(&ep->remote_dev->sdev->dev, |
| "%s %d err %d\n", __func__, __LINE__, err); |
| msg.uop = SCIF_CNCT_GNTNACK; |
| msg.payload[0] = ep->remote_ep; |
| _scif_nodeqp_send(ep->remote_dev, &msg); |
| ep->state = SCIFEP_BOUND; |
| goto connect_error_dec; |
| } |
| |
| msg.uop = SCIF_CNCT_GNTACK; |
| msg.payload[0] = ep->remote_ep; |
| err = _scif_nodeqp_send(ep->remote_dev, &msg); |
| if (err) { |
| ep->state = SCIFEP_BOUND; |
| goto connect_error_dec; |
| } |
| ep->state = SCIFEP_CONNECTED; |
| mutex_lock(&scif_info.connlock); |
| list_add_tail(&ep->list, &scif_info.connected); |
| mutex_unlock(&scif_info.connlock); |
| dev_dbg(&ep->remote_dev->sdev->dev, |
| "SCIFAPI connect: ep %p connected\n", ep); |
| } else if (ep->state == SCIFEP_BOUND) { |
| dev_dbg(&ep->remote_dev->sdev->dev, |
| "SCIFAPI connect: ep %p connection refused\n", ep); |
| err = -ECONNREFUSED; |
| goto connect_error_dec; |
| } |
| scif_put_peer_dev(spdev); |
| return err; |
| connect_error_dec: |
| scif_put_peer_dev(spdev); |
| cleanup_qp: |
| scif_cleanup_ep_qp(ep); |
| connect_error_simple: |
| return err; |
| } |
| |
| /* |
| * scif_conn_handler: |
| * |
| * Workqueue handler for servicing non-blocking SCIF connect |
| * |
| */ |
| void scif_conn_handler(struct work_struct *work) |
| { |
| struct scif_endpt *ep; |
| |
| do { |
| ep = NULL; |
| spin_lock(&scif_info.nb_connect_lock); |
| if (!list_empty(&scif_info.nb_connect_list)) { |
| ep = list_first_entry(&scif_info.nb_connect_list, |
| struct scif_endpt, conn_list); |
| list_del(&ep->conn_list); |
| } |
| spin_unlock(&scif_info.nb_connect_lock); |
| if (ep) { |
| ep->conn_err = scif_conn_func(ep); |
| wake_up_interruptible(&ep->conn_pend_wq); |
| } |
| } while (ep); |
| } |
| |
| int __scif_connect(scif_epd_t epd, struct scif_port_id *dst, bool non_block) |
| { |
| struct scif_endpt *ep = (struct scif_endpt *)epd; |
| int err = 0; |
| struct scif_dev *remote_dev; |
| struct device *spdev; |
| |
| dev_dbg(scif_info.mdev.this_device, "SCIFAPI connect: ep %p %s\n", ep, |
| scif_ep_states[ep->state]); |
| |
| if (!scif_dev || dst->node > scif_info.maxid) |
| return -ENODEV; |
| |
| might_sleep(); |
| |
| remote_dev = &scif_dev[dst->node]; |
| spdev = scif_get_peer_dev(remote_dev); |
| if (IS_ERR(spdev)) { |
| err = PTR_ERR(spdev); |
| return err; |
| } |
| |
| spin_lock(&ep->lock); |
| switch (ep->state) { |
| case SCIFEP_ZOMBIE: |
| case SCIFEP_CLOSING: |
| err = -EINVAL; |
| break; |
| case SCIFEP_DISCONNECTED: |
| if (ep->conn_async_state == ASYNC_CONN_INPROGRESS) |
| ep->conn_async_state = ASYNC_CONN_FLUSH_WORK; |
| else |
| err = -EINVAL; |
| break; |
| case SCIFEP_LISTENING: |
| case SCIFEP_CLLISTEN: |
| err = -EOPNOTSUPP; |
| break; |
| case SCIFEP_CONNECTING: |
| case SCIFEP_MAPPING: |
| if (ep->conn_async_state == ASYNC_CONN_INPROGRESS) |
| err = -EINPROGRESS; |
| else |
| err = -EISCONN; |
| break; |
| case SCIFEP_CONNECTED: |
| if (ep->conn_async_state == ASYNC_CONN_INPROGRESS) |
| ep->conn_async_state = ASYNC_CONN_FLUSH_WORK; |
| else |
| err = -EISCONN; |
| break; |
| case SCIFEP_UNBOUND: |
| err = scif_get_new_port(); |
| if (err < 0) |
| break; |
| ep->port.port = err; |
| ep->port.node = scif_info.nodeid; |
| ep->conn_async_state = ASYNC_CONN_IDLE; |
| /* Fall through */ |
| case SCIFEP_BOUND: |
| /* |
| * If a non-blocking connect has been already initiated |
| * (conn_async_state is either ASYNC_CONN_INPROGRESS or |
| * ASYNC_CONN_FLUSH_WORK), the end point could end up in |
| * SCIF_BOUND due an error in the connection process |
| * (e.g., connection refused) If conn_async_state is |
| * ASYNC_CONN_INPROGRESS - transition to ASYNC_CONN_FLUSH_WORK |
| * so that the error status can be collected. If the state is |
| * already ASYNC_CONN_FLUSH_WORK - then set the error to |
| * EINPROGRESS since some other thread is waiting to collect |
| * error status. |
| */ |
| if (ep->conn_async_state == ASYNC_CONN_INPROGRESS) { |
| ep->conn_async_state = ASYNC_CONN_FLUSH_WORK; |
| } else if (ep->conn_async_state == ASYNC_CONN_FLUSH_WORK) { |
| err = -EINPROGRESS; |
| } else { |
| ep->conn_port = *dst; |
| init_waitqueue_head(&ep->sendwq); |
| init_waitqueue_head(&ep->recvwq); |
| init_waitqueue_head(&ep->conwq); |
| ep->conn_async_state = 0; |
| |
| if (unlikely(non_block)) |
| ep->conn_async_state = ASYNC_CONN_INPROGRESS; |
| } |
| break; |
| } |
| |
| if (err || ep->conn_async_state == ASYNC_CONN_FLUSH_WORK) |
| goto connect_simple_unlock1; |
| |
| ep->state = SCIFEP_CONNECTING; |
| ep->remote_dev = &scif_dev[dst->node]; |
| ep->qp_info.qp->magic = SCIFEP_MAGIC; |
| if (ep->conn_async_state == ASYNC_CONN_INPROGRESS) { |
| init_waitqueue_head(&ep->conn_pend_wq); |
| spin_lock(&scif_info.nb_connect_lock); |
| list_add_tail(&ep->conn_list, &scif_info.nb_connect_list); |
| spin_unlock(&scif_info.nb_connect_lock); |
| err = -EINPROGRESS; |
| schedule_work(&scif_info.conn_work); |
| } |
| connect_simple_unlock1: |
| spin_unlock(&ep->lock); |
| scif_put_peer_dev(spdev); |
| if (err) { |
| return err; |
| } else if (ep->conn_async_state == ASYNC_CONN_FLUSH_WORK) { |
| flush_work(&scif_info.conn_work); |
| err = ep->conn_err; |
| spin_lock(&ep->lock); |
| ep->conn_async_state = ASYNC_CONN_IDLE; |
| spin_unlock(&ep->lock); |
| } else { |
| err = scif_conn_func(ep); |
| } |
| return err; |
| } |
| |
| int scif_connect(scif_epd_t epd, struct scif_port_id *dst) |
| { |
| return __scif_connect(epd, dst, false); |
| } |
| EXPORT_SYMBOL_GPL(scif_connect); |
| |
| /** |
| * scif_accept() - Accept a connection request from the remote node |
| * |
| * The function accepts a connection request from the remote node. Successful |
| * complete is indicate by a new end point being created and passed back |
| * to the caller for future reference. |
| * |
| * Upon successful complete a zero will be returned and the peer information |
| * will be filled in. |
| * |
| * If the end point is not in the listening state -EINVAL will be returned. |
| * |
| * If during the connection sequence resource allocation fails the -ENOMEM |
| * will be returned. |
| * |
| * If the function is called with the ASYNC flag set and no connection requests |
| * are pending it will return -EAGAIN. |
| * |
| * If the remote side is not sending any connection requests the caller may |
| * terminate this function with a signal. If so a -EINTR will be returned. |
| */ |
| int scif_accept(scif_epd_t epd, struct scif_port_id *peer, |
| scif_epd_t *newepd, int flags) |
| { |
| struct scif_endpt *lep = (struct scif_endpt *)epd; |
| struct scif_endpt *cep; |
| struct scif_conreq *conreq; |
| struct scifmsg msg; |
| int err; |
| struct device *spdev; |
| |
| dev_dbg(scif_info.mdev.this_device, |
| "SCIFAPI accept: ep %p %s\n", lep, scif_ep_states[lep->state]); |
| |
| if (flags & ~SCIF_ACCEPT_SYNC) |
| return -EINVAL; |
| |
| if (!peer || !newepd) |
| return -EINVAL; |
| |
| might_sleep(); |
| spin_lock(&lep->lock); |
| if (lep->state != SCIFEP_LISTENING) { |
| spin_unlock(&lep->lock); |
| return -EINVAL; |
| } |
| |
| if (!lep->conreqcnt && !(flags & SCIF_ACCEPT_SYNC)) { |
| /* No connection request present and we do not want to wait */ |
| spin_unlock(&lep->lock); |
| return -EAGAIN; |
| } |
| |
| lep->files = current->files; |
| retry_connection: |
| spin_unlock(&lep->lock); |
| /* Wait for the remote node to send us a SCIF_CNCT_REQ */ |
| err = wait_event_interruptible(lep->conwq, |
| (lep->conreqcnt || |
| (lep->state != SCIFEP_LISTENING))); |
| if (err) |
| return err; |
| |
| if (lep->state != SCIFEP_LISTENING) |
| return -EINTR; |
| |
| spin_lock(&lep->lock); |
| |
| if (!lep->conreqcnt) |
| goto retry_connection; |
| |
| /* Get the first connect request off the list */ |
| conreq = list_first_entry(&lep->conlist, struct scif_conreq, list); |
| list_del(&conreq->list); |
| lep->conreqcnt--; |
| spin_unlock(&lep->lock); |
| |
| /* Fill in the peer information */ |
| peer->node = conreq->msg.src.node; |
| peer->port = conreq->msg.src.port; |
| |
| cep = kzalloc(sizeof(*cep), GFP_KERNEL); |
| if (!cep) { |
| err = -ENOMEM; |
| goto scif_accept_error_epalloc; |
| } |
| spin_lock_init(&cep->lock); |
| mutex_init(&cep->sendlock); |
| mutex_init(&cep->recvlock); |
| cep->state = SCIFEP_CONNECTING; |
| cep->remote_dev = &scif_dev[peer->node]; |
| cep->remote_ep = conreq->msg.payload[0]; |
| |
| scif_rma_ep_init(cep); |
| |
| err = scif_reserve_dma_chan(cep); |
| if (err) { |
| dev_err(scif_info.mdev.this_device, |
| "%s %d err %d\n", __func__, __LINE__, err); |
| goto scif_accept_error_qpalloc; |
| } |
| |
| cep->qp_info.qp = kzalloc(sizeof(*cep->qp_info.qp), GFP_KERNEL); |
| if (!cep->qp_info.qp) { |
| err = -ENOMEM; |
| goto scif_accept_error_qpalloc; |
| } |
| |
| err = scif_anon_inode_getfile(cep); |
| if (err) |
| goto scif_accept_error_anon_inode; |
| |
| cep->qp_info.qp->magic = SCIFEP_MAGIC; |
| spdev = scif_get_peer_dev(cep->remote_dev); |
| if (IS_ERR(spdev)) { |
| err = PTR_ERR(spdev); |
| goto scif_accept_error_map; |
| } |
| err = scif_setup_qp_accept(cep->qp_info.qp, &cep->qp_info.qp_offset, |
| conreq->msg.payload[1], SCIF_ENDPT_QP_SIZE, |
| cep->remote_dev); |
| if (err) { |
| dev_dbg(&cep->remote_dev->sdev->dev, |
| "SCIFAPI accept: ep %p new %p scif_setup_qp_accept %d qp_offset 0x%llx\n", |
| lep, cep, err, cep->qp_info.qp_offset); |
| scif_put_peer_dev(spdev); |
| goto scif_accept_error_map; |
| } |
| |
| cep->port.node = lep->port.node; |
| cep->port.port = lep->port.port; |
| cep->peer.node = peer->node; |
| cep->peer.port = peer->port; |
| init_waitqueue_head(&cep->sendwq); |
| init_waitqueue_head(&cep->recvwq); |
| init_waitqueue_head(&cep->conwq); |
| |
| msg.uop = SCIF_CNCT_GNT; |
| msg.src = cep->port; |
| msg.payload[0] = cep->remote_ep; |
| msg.payload[1] = cep->qp_info.qp_offset; |
| msg.payload[2] = (u64)cep; |
| |
| err = _scif_nodeqp_send(cep->remote_dev, &msg); |
| scif_put_peer_dev(spdev); |
| if (err) |
| goto scif_accept_error_map; |
| retry: |
| /* Wait for the remote node to respond with SCIF_CNCT_GNT(N)ACK */ |
| err = wait_event_timeout(cep->conwq, cep->state != SCIFEP_CONNECTING, |
| SCIF_NODE_ACCEPT_TIMEOUT); |
| if (!err && scifdev_alive(cep)) |
| goto retry; |
| err = !err ? -ENODEV : 0; |
| if (err) |
| goto scif_accept_error_map; |
| kfree(conreq); |
| |
| spin_lock(&cep->lock); |
| |
| if (cep->state == SCIFEP_CLOSING) { |
| /* |
| * Remote failed to allocate resources and NAKed the grant. |
| * There is at this point nothing referencing the new end point. |
| */ |
| spin_unlock(&cep->lock); |
| scif_teardown_ep(cep); |
| kfree(cep); |
| |
| /* If call with sync flag then go back and wait. */ |
| if (flags & SCIF_ACCEPT_SYNC) { |
| spin_lock(&lep->lock); |
| goto retry_connection; |
| } |
| return -EAGAIN; |
| } |
| |
| scif_get_port(cep->port.port); |
| *newepd = (scif_epd_t)cep; |
| spin_unlock(&cep->lock); |
| return 0; |
| scif_accept_error_map: |
| scif_anon_inode_fput(cep); |
| scif_accept_error_anon_inode: |
| scif_teardown_ep(cep); |
| scif_accept_error_qpalloc: |
| kfree(cep); |
| scif_accept_error_epalloc: |
| msg.uop = SCIF_CNCT_REJ; |
| msg.dst.node = conreq->msg.src.node; |
| msg.dst.port = conreq->msg.src.port; |
| msg.payload[0] = conreq->msg.payload[0]; |
| msg.payload[1] = conreq->msg.payload[1]; |
| scif_nodeqp_send(&scif_dev[conreq->msg.src.node], &msg); |
| kfree(conreq); |
| return err; |
| } |
| EXPORT_SYMBOL_GPL(scif_accept); |
| |
| /* |
| * scif_msg_param_check: |
| * @epd: The end point returned from scif_open() |
| * @len: Length to receive |
| * @flags: blocking or non blocking |
| * |
| * Validate parameters for messaging APIs scif_send(..)/scif_recv(..). |
| */ |
| static inline int scif_msg_param_check(scif_epd_t epd, int len, int flags) |
| { |
| int ret = -EINVAL; |
| |
| if (len < 0) |
| goto err_ret; |
| if (flags && (!(flags & SCIF_RECV_BLOCK))) |
| goto err_ret; |
| ret = 0; |
| err_ret: |
| return ret; |
| } |
| |
| static int _scif_send(scif_epd_t epd, void *msg, int len, int flags) |
| { |
| struct scif_endpt *ep = (struct scif_endpt *)epd; |
| struct scifmsg notif_msg; |
| int curr_xfer_len = 0, sent_len = 0, write_count; |
| int ret = 0; |
| struct scif_qp *qp = ep->qp_info.qp; |
| |
| if (flags & SCIF_SEND_BLOCK) |
| might_sleep(); |
| |
| spin_lock(&ep->lock); |
| while (sent_len != len && SCIFEP_CONNECTED == ep->state) { |
| write_count = scif_rb_space(&qp->outbound_q); |
| if (write_count) { |
| /* Best effort to send as much data as possible */ |
| curr_xfer_len = min(len - sent_len, write_count); |
| ret = scif_rb_write(&qp->outbound_q, msg, |
| curr_xfer_len); |
| if (ret < 0) |
| break; |
| /* Success. Update write pointer */ |
| scif_rb_commit(&qp->outbound_q); |
| /* |
| * Send a notification to the peer about the |
| * produced data message. |
| */ |
| notif_msg.src = ep->port; |
| notif_msg.uop = SCIF_CLIENT_SENT; |
| notif_msg.payload[0] = ep->remote_ep; |
| ret = _scif_nodeqp_send(ep->remote_dev, ¬if_msg); |
| if (ret) |
| break; |
| sent_len += curr_xfer_len; |
| msg = msg + curr_xfer_len; |
| continue; |
| } |
| curr_xfer_len = min(len - sent_len, SCIF_ENDPT_QP_SIZE - 1); |
| /* Not enough RB space. return for the Non Blocking case */ |
| if (!(flags & SCIF_SEND_BLOCK)) |
| break; |
| |
| spin_unlock(&ep->lock); |
| /* Wait for a SCIF_CLIENT_RCVD message in the Blocking case */ |
| ret = |
| wait_event_interruptible(ep->sendwq, |
| (SCIFEP_CONNECTED != ep->state) || |
| (scif_rb_space(&qp->outbound_q) >= |
| curr_xfer_len)); |
| spin_lock(&ep->lock); |
| if (ret) |
| break; |
| } |
| if (sent_len) |
| ret = sent_len; |
| else if (!ret && SCIFEP_CONNECTED != ep->state) |
| ret = SCIFEP_DISCONNECTED == ep->state ? |
| -ECONNRESET : -ENOTCONN; |
| spin_unlock(&ep->lock); |
| return ret; |
| } |
| |
| static int _scif_recv(scif_epd_t epd, void *msg, int len, int flags) |
| { |
| int read_size; |
| struct scif_endpt *ep = (struct scif_endpt *)epd; |
| struct scifmsg notif_msg; |
| int curr_recv_len = 0, remaining_len = len, read_count; |
| int ret = 0; |
| struct scif_qp *qp = ep->qp_info.qp; |
| |
| if (flags & SCIF_RECV_BLOCK) |
| might_sleep(); |
| spin_lock(&ep->lock); |
| while (remaining_len && (SCIFEP_CONNECTED == ep->state || |
| SCIFEP_DISCONNECTED == ep->state)) { |
| read_count = scif_rb_count(&qp->inbound_q, remaining_len); |
| if (read_count) { |
| /* |
| * Best effort to recv as much data as there |
| * are bytes to read in the RB particularly |
| * important for the Non Blocking case. |
| */ |
| curr_recv_len = min(remaining_len, read_count); |
| read_size = scif_rb_get_next(&qp->inbound_q, |
| msg, curr_recv_len); |
| if (ep->state == SCIFEP_CONNECTED) { |
| /* |
| * Update the read pointer only if the endpoint |
| * is still connected else the read pointer |
| * might no longer exist since the peer has |
| * freed resources! |
| */ |
| scif_rb_update_read_ptr(&qp->inbound_q); |
| /* |
| * Send a notification to the peer about the |
| * consumed data message only if the EP is in |
| * SCIFEP_CONNECTED state. |
| */ |
| notif_msg.src = ep->port; |
| notif_msg.uop = SCIF_CLIENT_RCVD; |
| notif_msg.payload[0] = ep->remote_ep; |
| ret = _scif_nodeqp_send(ep->remote_dev, |
| ¬if_msg); |
| if (ret) |
| break; |
| } |
| remaining_len -= curr_recv_len; |
| msg = msg + curr_recv_len; |
| continue; |
| } |
| /* |
| * Bail out now if the EP is in SCIFEP_DISCONNECTED state else |
| * we will keep looping forever. |
| */ |
| if (ep->state == SCIFEP_DISCONNECTED) |
| break; |
| /* |
| * Return in the Non Blocking case if there is no data |
| * to read in this iteration. |
| */ |
| if (!(flags & SCIF_RECV_BLOCK)) |
| break; |
| curr_recv_len = min(remaining_len, SCIF_ENDPT_QP_SIZE - 1); |
| spin_unlock(&ep->lock); |
| /* |
| * Wait for a SCIF_CLIENT_SEND message in the blocking case |
| * or until other side disconnects. |
| */ |
| ret = |
| wait_event_interruptible(ep->recvwq, |
| SCIFEP_CONNECTED != ep->state || |
| scif_rb_count(&qp->inbound_q, |
| curr_recv_len) |
| >= curr_recv_len); |
| spin_lock(&ep->lock); |
| if (ret) |
| break; |
| } |
| if (len - remaining_len) |
| ret = len - remaining_len; |
| else if (!ret && ep->state != SCIFEP_CONNECTED) |
| ret = ep->state == SCIFEP_DISCONNECTED ? |
| -ECONNRESET : -ENOTCONN; |
| spin_unlock(&ep->lock); |
| return ret; |
| } |
| |
| /** |
| * scif_user_send() - Send data to connection queue |
| * @epd: The end point returned from scif_open() |
| * @msg: Address to place data |
| * @len: Length to receive |
| * @flags: blocking or non blocking |
| * |
| * This function is called from the driver IOCTL entry point |
| * only and is a wrapper for _scif_send(). |
| */ |
| int scif_user_send(scif_epd_t epd, void __user *msg, int len, int flags) |
| { |
| struct scif_endpt *ep = (struct scif_endpt *)epd; |
| int err = 0; |
| int sent_len = 0; |
| char *tmp; |
| int loop_len; |
| int chunk_len = min(len, (1 << (MAX_ORDER + PAGE_SHIFT - 1))); |
| |
| dev_dbg(scif_info.mdev.this_device, |
| "SCIFAPI send (U): ep %p %s\n", ep, scif_ep_states[ep->state]); |
| if (!len) |
| return 0; |
| |
| err = scif_msg_param_check(epd, len, flags); |
| if (err) |
| goto send_err; |
| |
| tmp = kmalloc(chunk_len, GFP_KERNEL); |
| if (!tmp) { |
| err = -ENOMEM; |
| goto send_err; |
| } |
| /* |
| * Grabbing the lock before breaking up the transfer in |
| * multiple chunks is required to ensure that messages do |
| * not get fragmented and reordered. |
| */ |
| mutex_lock(&ep->sendlock); |
| while (sent_len != len) { |
| loop_len = len - sent_len; |
| loop_len = min(chunk_len, loop_len); |
| if (copy_from_user(tmp, msg, loop_len)) { |
| err = -EFAULT; |
| goto send_free_err; |
| } |
| err = _scif_send(epd, tmp, loop_len, flags); |
| if (err < 0) |
| goto send_free_err; |
| sent_len += err; |
| msg += err; |
| if (err != loop_len) |
| goto send_free_err; |
| } |
| send_free_err: |
| mutex_unlock(&ep->sendlock); |
| kfree(tmp); |
| send_err: |
| return err < 0 ? err : sent_len; |
| } |
| |
| /** |
| * scif_user_recv() - Receive data from connection queue |
| * @epd: The end point returned from scif_open() |
| * @msg: Address to place data |
| * @len: Length to receive |
| * @flags: blocking or non blocking |
| * |
| * This function is called from the driver IOCTL entry point |
| * only and is a wrapper for _scif_recv(). |
| */ |
| int scif_user_recv(scif_epd_t epd, void __user *msg, int len, int flags) |
| { |
| struct scif_endpt *ep = (struct scif_endpt *)epd; |
| int err = 0; |
| int recv_len = 0; |
| char *tmp; |
| int loop_len; |
| int chunk_len = min(len, (1 << (MAX_ORDER + PAGE_SHIFT - 1))); |
| |
| dev_dbg(scif_info.mdev.this_device, |
| "SCIFAPI recv (U): ep %p %s\n", ep, scif_ep_states[ep->state]); |
| if (!len) |
| return 0; |
| |
| err = scif_msg_param_check(epd, len, flags); |
| if (err) |
| goto recv_err; |
| |
| tmp = kmalloc(chunk_len, GFP_KERNEL); |
| if (!tmp) { |
| err = -ENOMEM; |
| goto recv_err; |
| } |
| /* |
| * Grabbing the lock before breaking up the transfer in |
| * multiple chunks is required to ensure that messages do |
| * not get fragmented and reordered. |
| */ |
| mutex_lock(&ep->recvlock); |
| while (recv_len != len) { |
| loop_len = len - recv_len; |
| loop_len = min(chunk_len, loop_len); |
| err = _scif_recv(epd, tmp, loop_len, flags); |
| if (err < 0) |
| goto recv_free_err; |
| if (copy_to_user(msg, tmp, err)) { |
| err = -EFAULT; |
| goto recv_free_err; |
| } |
| recv_len += err; |
| msg += err; |
| if (err != loop_len) |
| goto recv_free_err; |
| } |
| recv_free_err: |
| mutex_unlock(&ep->recvlock); |
| kfree(tmp); |
| recv_err: |
| return err < 0 ? err : recv_len; |
| } |
| |
| /** |
| * scif_send() - Send data to connection queue |
| * @epd: The end point returned from scif_open() |
| * @msg: Address to place data |
| * @len: Length to receive |
| * @flags: blocking or non blocking |
| * |
| * This function is called from the kernel mode only and is |
| * a wrapper for _scif_send(). |
| */ |
| int scif_send(scif_epd_t epd, void *msg, int len, int flags) |
| { |
| struct scif_endpt *ep = (struct scif_endpt *)epd; |
| int ret; |
| |
| dev_dbg(scif_info.mdev.this_device, |
| "SCIFAPI send (K): ep %p %s\n", ep, scif_ep_states[ep->state]); |
| if (!len) |
| return 0; |
| |
| ret = scif_msg_param_check(epd, len, flags); |
| if (ret) |
| return ret; |
| if (!ep->remote_dev) |
| return -ENOTCONN; |
| /* |
| * Grab the mutex lock in the blocking case only |
| * to ensure messages do not get fragmented/reordered. |
| * The non blocking mode is protected using spin locks |
| * in _scif_send(). |
| */ |
| if (flags & SCIF_SEND_BLOCK) |
| mutex_lock(&ep->sendlock); |
| |
| ret = _scif_send(epd, msg, len, flags); |
| |
| if (flags & SCIF_SEND_BLOCK) |
| mutex_unlock(&ep->sendlock); |
| return ret; |
| } |
| EXPORT_SYMBOL_GPL(scif_send); |
| |
| /** |
| * scif_recv() - Receive data from connection queue |
| * @epd: The end point returned from scif_open() |
| * @msg: Address to place data |
| * @len: Length to receive |
| * @flags: blocking or non blocking |
| * |
| * This function is called from the kernel mode only and is |
| * a wrapper for _scif_recv(). |
| */ |
| int scif_recv(scif_epd_t epd, void *msg, int len, int flags) |
| { |
| struct scif_endpt *ep = (struct scif_endpt *)epd; |
| int ret; |
| |
| dev_dbg(scif_info.mdev.this_device, |
| "SCIFAPI recv (K): ep %p %s\n", ep, scif_ep_states[ep->state]); |
| if (!len) |
| return 0; |
| |
| ret = scif_msg_param_check(epd, len, flags); |
| if (ret) |
| return ret; |
| /* |
| * Grab the mutex lock in the blocking case only |
| * to ensure messages do not get fragmented/reordered. |
| * The non blocking mode is protected using spin locks |
| * in _scif_send(). |
| */ |
| if (flags & SCIF_RECV_BLOCK) |
| mutex_lock(&ep->recvlock); |
| |
| ret = _scif_recv(epd, msg, len, flags); |
| |
| if (flags & SCIF_RECV_BLOCK) |
| mutex_unlock(&ep->recvlock); |
| |
| return ret; |
| } |
| EXPORT_SYMBOL_GPL(scif_recv); |
| |
| static inline void _scif_poll_wait(struct file *f, wait_queue_head_t *wq, |
| poll_table *p, struct scif_endpt *ep) |
| { |
| /* |
| * Because poll_wait makes a GFP_KERNEL allocation, give up the lock |
| * and regrab it afterwards. Because the endpoint state might have |
| * changed while the lock was given up, the state must be checked |
| * again after re-acquiring the lock. The code in __scif_pollfd(..) |
| * does this. |
| */ |
| spin_unlock(&ep->lock); |
| poll_wait(f, wq, p); |
| spin_lock(&ep->lock); |
| } |
| |
| __poll_t |
| __scif_pollfd(struct file *f, poll_table *wait, struct scif_endpt *ep) |
| { |
| __poll_t mask = 0; |
| |
| dev_dbg(scif_info.mdev.this_device, |
| "SCIFAPI pollfd: ep %p %s\n", ep, scif_ep_states[ep->state]); |
| |
| spin_lock(&ep->lock); |
| |
| /* Endpoint is waiting for a non-blocking connect to complete */ |
| if (ep->conn_async_state == ASYNC_CONN_INPROGRESS) { |
| _scif_poll_wait(f, &ep->conn_pend_wq, wait, ep); |
| if (ep->conn_async_state == ASYNC_CONN_INPROGRESS) { |
| if (ep->state == SCIFEP_CONNECTED || |
| ep->state == SCIFEP_DISCONNECTED || |
| ep->conn_err) |
| mask |= EPOLLOUT; |
| goto exit; |
| } |
| } |
| |
| /* Endpoint is listening for incoming connection requests */ |
| if (ep->state == SCIFEP_LISTENING) { |
| _scif_poll_wait(f, &ep->conwq, wait, ep); |
| if (ep->state == SCIFEP_LISTENING) { |
| if (ep->conreqcnt) |
| mask |= EPOLLIN; |
| goto exit; |
| } |
| } |
| |
| /* Endpoint is connected or disconnected */ |
| if (ep->state == SCIFEP_CONNECTED || ep->state == SCIFEP_DISCONNECTED) { |
| if (poll_requested_events(wait) & EPOLLIN) |
| _scif_poll_wait(f, &ep->recvwq, wait, ep); |
| if (poll_requested_events(wait) & EPOLLOUT) |
| _scif_poll_wait(f, &ep->sendwq, wait, ep); |
| if (ep->state == SCIFEP_CONNECTED || |
| ep->state == SCIFEP_DISCONNECTED) { |
| /* Data can be read without blocking */ |
| if (scif_rb_count(&ep->qp_info.qp->inbound_q, 1)) |
| mask |= EPOLLIN; |
| /* Data can be written without blocking */ |
| if (scif_rb_space(&ep->qp_info.qp->outbound_q)) |
| mask |= EPOLLOUT; |
| /* Return EPOLLHUP if endpoint is disconnected */ |
| if (ep->state == SCIFEP_DISCONNECTED) |
| mask |= EPOLLHUP; |
| goto exit; |
| } |
| } |
| |
| /* Return EPOLLERR if the endpoint is in none of the above states */ |
| mask |= EPOLLERR; |
| exit: |
| spin_unlock(&ep->lock); |
| return mask; |
| } |
| |
| /** |
| * scif_poll() - Kernel mode SCIF poll |
| * @ufds: Array of scif_pollepd structures containing the end points |
| * and events to poll on |
| * @nfds: Size of the ufds array |
| * @timeout_msecs: Timeout in msecs, -ve implies infinite timeout |
| * |
| * The code flow in this function is based on do_poll(..) in select.c |
| * |
| * Returns the number of endpoints which have pending events or 0 in |
| * the event of a timeout. If a signal is used for wake up, -EINTR is |
| * returned. |
| */ |
| int |
| scif_poll(struct scif_pollepd *ufds, unsigned int nfds, long timeout_msecs) |
| { |
| struct poll_wqueues table; |
| poll_table *pt; |
| int i, count = 0, timed_out = timeout_msecs == 0; |
| __poll_t mask; |
| u64 timeout = timeout_msecs < 0 ? MAX_SCHEDULE_TIMEOUT |
| : msecs_to_jiffies(timeout_msecs); |
| |
| poll_initwait(&table); |
| pt = &table.pt; |
| while (1) { |
| for (i = 0; i < nfds; i++) { |
| pt->_key = ufds[i].events | EPOLLERR | EPOLLHUP; |
| mask = __scif_pollfd(ufds[i].epd->anon, |
| pt, ufds[i].epd); |
| mask &= ufds[i].events | EPOLLERR | EPOLLHUP; |
| if (mask) { |
| count++; |
| pt->_qproc = NULL; |
| } |
| ufds[i].revents = mask; |
| } |
| pt->_qproc = NULL; |
| if (!count) { |
| count = table.error; |
| if (signal_pending(current)) |
| count = -EINTR; |
| } |
| if (count || timed_out) |
| break; |
| |
| if (!schedule_timeout_interruptible(timeout)) |
| timed_out = 1; |
| } |
| poll_freewait(&table); |
| return count; |
| } |
| EXPORT_SYMBOL_GPL(scif_poll); |
| |
| int scif_get_node_ids(u16 *nodes, int len, u16 *self) |
| { |
| int online = 0; |
| int offset = 0; |
| int node; |
| |
| if (!scif_is_mgmt_node()) |
| scif_get_node_info(); |
| |
| *self = scif_info.nodeid; |
| mutex_lock(&scif_info.conflock); |
| len = min_t(int, len, scif_info.total); |
| for (node = 0; node <= scif_info.maxid; node++) { |
| if (_scifdev_alive(&scif_dev[node])) { |
| online++; |
| if (offset < len) |
| nodes[offset++] = node; |
| } |
| } |
| dev_dbg(scif_info.mdev.this_device, |
| "SCIFAPI get_node_ids total %d online %d filled in %d nodes\n", |
| scif_info.total, online, offset); |
| mutex_unlock(&scif_info.conflock); |
| |
| return online; |
| } |
| EXPORT_SYMBOL_GPL(scif_get_node_ids); |
| |
| static int scif_add_client_dev(struct device *dev, struct subsys_interface *si) |
| { |
| struct scif_client *client = |
| container_of(si, struct scif_client, si); |
| struct scif_peer_dev *spdev = |
| container_of(dev, struct scif_peer_dev, dev); |
| |
| if (client->probe) |
| client->probe(spdev); |
| return 0; |
| } |
| |
| static void scif_remove_client_dev(struct device *dev, |
| struct subsys_interface *si) |
| { |
| struct scif_client *client = |
| container_of(si, struct scif_client, si); |
| struct scif_peer_dev *spdev = |
| container_of(dev, struct scif_peer_dev, dev); |
| |
| if (client->remove) |
| client->remove(spdev); |
| } |
| |
| void scif_client_unregister(struct scif_client *client) |
| { |
| subsys_interface_unregister(&client->si); |
| } |
| EXPORT_SYMBOL_GPL(scif_client_unregister); |
| |
| int scif_client_register(struct scif_client *client) |
| { |
| struct subsys_interface *si = &client->si; |
| |
| si->name = client->name; |
| si->subsys = &scif_peer_bus; |
| si->add_dev = scif_add_client_dev; |
| si->remove_dev = scif_remove_client_dev; |
| |
| return subsys_interface_register(&client->si); |
| } |
| EXPORT_SYMBOL_GPL(scif_client_register); |