| /* |
| * cxgb3i_offload.c: Chelsio S3xx iscsi offloaded tcp connection management |
| * |
| * Copyright (C) 2003-2015 Chelsio Communications. All rights reserved. |
| * |
| * This program is distributed in the hope that it will be useful, but WITHOUT |
| * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this |
| * release for licensing terms and conditions. |
| * |
| * Written by: Dimitris Michailidis (dm@chelsio.com) |
| * Karen Xie (kxie@chelsio.com) |
| */ |
| |
| #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ |
| |
| #include <linux/module.h> |
| #include <linux/moduleparam.h> |
| #include <scsi/scsi_host.h> |
| |
| #include "common.h" |
| #include "t3_cpl.h" |
| #include "t3cdev.h" |
| #include "cxgb3_defs.h" |
| #include "cxgb3_ctl_defs.h" |
| #include "cxgb3_offload.h" |
| #include "firmware_exports.h" |
| #include "cxgb3i.h" |
| |
| static unsigned int dbg_level; |
| #include "../libcxgbi.h" |
| |
| #define DRV_MODULE_NAME "cxgb3i" |
| #define DRV_MODULE_DESC "Chelsio T3 iSCSI Driver" |
| #define DRV_MODULE_VERSION "2.0.1-ko" |
| #define DRV_MODULE_RELDATE "Apr. 2015" |
| |
| static char version[] = |
| DRV_MODULE_DESC " " DRV_MODULE_NAME |
| " v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; |
| |
| MODULE_AUTHOR("Chelsio Communications, Inc."); |
| MODULE_DESCRIPTION(DRV_MODULE_DESC); |
| MODULE_VERSION(DRV_MODULE_VERSION); |
| MODULE_LICENSE("GPL"); |
| |
| module_param(dbg_level, uint, 0644); |
| MODULE_PARM_DESC(dbg_level, "debug flag (default=0)"); |
| |
| static int cxgb3i_rcv_win = 256 * 1024; |
| module_param(cxgb3i_rcv_win, int, 0644); |
| MODULE_PARM_DESC(cxgb3i_rcv_win, "TCP receive window in bytes (default=256KB)"); |
| |
| static int cxgb3i_snd_win = 128 * 1024; |
| module_param(cxgb3i_snd_win, int, 0644); |
| MODULE_PARM_DESC(cxgb3i_snd_win, "TCP send window in bytes (default=128KB)"); |
| |
| static int cxgb3i_rx_credit_thres = 10 * 1024; |
| module_param(cxgb3i_rx_credit_thres, int, 0644); |
| MODULE_PARM_DESC(cxgb3i_rx_credit_thres, |
| "RX credits return threshold in bytes (default=10KB)"); |
| |
| static unsigned int cxgb3i_max_connect = 8 * 1024; |
| module_param(cxgb3i_max_connect, uint, 0644); |
| MODULE_PARM_DESC(cxgb3i_max_connect, "Max. # of connections (default=8092)"); |
| |
| static unsigned int cxgb3i_sport_base = 20000; |
| module_param(cxgb3i_sport_base, uint, 0644); |
| MODULE_PARM_DESC(cxgb3i_sport_base, "starting port number (default=20000)"); |
| |
| static void cxgb3i_dev_open(struct t3cdev *); |
| static void cxgb3i_dev_close(struct t3cdev *); |
| static void cxgb3i_dev_event_handler(struct t3cdev *, u32, u32); |
| |
| static struct cxgb3_client t3_client = { |
| .name = DRV_MODULE_NAME, |
| .handlers = cxgb3i_cpl_handlers, |
| .add = cxgb3i_dev_open, |
| .remove = cxgb3i_dev_close, |
| .event_handler = cxgb3i_dev_event_handler, |
| }; |
| |
| static struct scsi_host_template cxgb3i_host_template = { |
| .module = THIS_MODULE, |
| .name = DRV_MODULE_NAME, |
| .proc_name = DRV_MODULE_NAME, |
| .can_queue = CXGB3I_SCSI_HOST_QDEPTH, |
| .queuecommand = iscsi_queuecommand, |
| .change_queue_depth = scsi_change_queue_depth, |
| .sg_tablesize = SG_ALL, |
| .max_sectors = 0xFFFF, |
| .cmd_per_lun = ISCSI_DEF_CMD_PER_LUN, |
| .eh_timed_out = iscsi_eh_cmd_timed_out, |
| .eh_abort_handler = iscsi_eh_abort, |
| .eh_device_reset_handler = iscsi_eh_device_reset, |
| .eh_target_reset_handler = iscsi_eh_recover_target, |
| .target_alloc = iscsi_target_alloc, |
| .dma_boundary = PAGE_SIZE - 1, |
| .this_id = -1, |
| .track_queue_depth = 1, |
| }; |
| |
| static struct iscsi_transport cxgb3i_iscsi_transport = { |
| .owner = THIS_MODULE, |
| .name = DRV_MODULE_NAME, |
| /* owner and name should be set already */ |
| .caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST |
| | CAP_DATADGST | CAP_DIGEST_OFFLOAD | |
| CAP_PADDING_OFFLOAD | CAP_TEXT_NEGO, |
| .attr_is_visible = cxgbi_attr_is_visible, |
| .get_host_param = cxgbi_get_host_param, |
| .set_host_param = cxgbi_set_host_param, |
| /* session management */ |
| .create_session = cxgbi_create_session, |
| .destroy_session = cxgbi_destroy_session, |
| .get_session_param = iscsi_session_get_param, |
| /* connection management */ |
| .create_conn = cxgbi_create_conn, |
| .bind_conn = cxgbi_bind_conn, |
| .destroy_conn = iscsi_tcp_conn_teardown, |
| .start_conn = iscsi_conn_start, |
| .stop_conn = iscsi_conn_stop, |
| .get_conn_param = iscsi_conn_get_param, |
| .set_param = cxgbi_set_conn_param, |
| .get_stats = cxgbi_get_conn_stats, |
| /* pdu xmit req from user space */ |
| .send_pdu = iscsi_conn_send_pdu, |
| /* task */ |
| .init_task = iscsi_tcp_task_init, |
| .xmit_task = iscsi_tcp_task_xmit, |
| .cleanup_task = cxgbi_cleanup_task, |
| /* pdu */ |
| .alloc_pdu = cxgbi_conn_alloc_pdu, |
| .init_pdu = cxgbi_conn_init_pdu, |
| .xmit_pdu = cxgbi_conn_xmit_pdu, |
| .parse_pdu_itt = cxgbi_parse_pdu_itt, |
| /* TCP connect/disconnect */ |
| .get_ep_param = cxgbi_get_ep_param, |
| .ep_connect = cxgbi_ep_connect, |
| .ep_poll = cxgbi_ep_poll, |
| .ep_disconnect = cxgbi_ep_disconnect, |
| /* Error recovery timeout call */ |
| .session_recovery_timedout = iscsi_session_recovery_timedout, |
| }; |
| |
| static struct scsi_transport_template *cxgb3i_stt; |
| |
| /* |
| * CPL (Chelsio Protocol Language) defines a message passing interface between |
| * the host driver and Chelsio asic. |
| * The section below implments CPLs that related to iscsi tcp connection |
| * open/close/abort and data send/receive. |
| */ |
| |
| static int push_tx_frames(struct cxgbi_sock *csk, int req_completion); |
| |
| static void send_act_open_req(struct cxgbi_sock *csk, struct sk_buff *skb, |
| const struct l2t_entry *e) |
| { |
| unsigned int wscale = cxgbi_sock_compute_wscale(csk->rcv_win); |
| struct cpl_act_open_req *req = (struct cpl_act_open_req *)skb->head; |
| |
| skb->priority = CPL_PRIORITY_SETUP; |
| |
| req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); |
| OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, csk->atid)); |
| req->local_port = csk->saddr.sin_port; |
| req->peer_port = csk->daddr.sin_port; |
| req->local_ip = csk->saddr.sin_addr.s_addr; |
| req->peer_ip = csk->daddr.sin_addr.s_addr; |
| |
| req->opt0h = htonl(V_KEEP_ALIVE(1) | F_TCAM_BYPASS | |
| V_WND_SCALE(wscale) | V_MSS_IDX(csk->mss_idx) | |
| V_L2T_IDX(e->idx) | V_TX_CHANNEL(e->smt_idx)); |
| req->opt0l = htonl(V_ULP_MODE(ULP2_MODE_ISCSI) | |
| V_RCV_BUFSIZ(csk->rcv_win >> 10)); |
| |
| log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, |
| "csk 0x%p,%u,0x%lx,%u, %pI4:%u-%pI4:%u, %u,%u,%u.\n", |
| csk, csk->state, csk->flags, csk->atid, |
| &req->local_ip, ntohs(req->local_port), |
| &req->peer_ip, ntohs(req->peer_port), |
| csk->mss_idx, e->idx, e->smt_idx); |
| |
| l2t_send(csk->cdev->lldev, skb, csk->l2t); |
| } |
| |
| static inline void act_open_arp_failure(struct t3cdev *dev, struct sk_buff *skb) |
| { |
| cxgbi_sock_act_open_req_arp_failure(NULL, skb); |
| } |
| |
| /* |
| * CPL connection close request: host -> |
| * |
| * Close a connection by sending a CPL_CLOSE_CON_REQ message and queue it to |
| * the write queue (i.e., after any unsent txt data). |
| */ |
| static void send_close_req(struct cxgbi_sock *csk) |
| { |
| struct sk_buff *skb = csk->cpl_close; |
| struct cpl_close_con_req *req = (struct cpl_close_con_req *)skb->head; |
| unsigned int tid = csk->tid; |
| |
| log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, |
| "csk 0x%p,%u,0x%lx,%u.\n", |
| csk, csk->state, csk->flags, csk->tid); |
| |
| csk->cpl_close = NULL; |
| req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON)); |
| req->wr.wr_lo = htonl(V_WR_TID(tid)); |
| OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid)); |
| req->rsvd = htonl(csk->write_seq); |
| |
| cxgbi_sock_skb_entail(csk, skb); |
| if (csk->state >= CTP_ESTABLISHED) |
| push_tx_frames(csk, 1); |
| } |
| |
| /* |
| * CPL connection abort request: host -> |
| * |
| * Send an ABORT_REQ message. Makes sure we do not send multiple ABORT_REQs |
| * for the same connection and also that we do not try to send a message |
| * after the connection has closed. |
| */ |
| static void abort_arp_failure(struct t3cdev *tdev, struct sk_buff *skb) |
| { |
| struct cpl_abort_req *req = cplhdr(skb); |
| |
| log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, |
| "t3dev 0x%p, tid %u, skb 0x%p.\n", |
| tdev, GET_TID(req), skb); |
| req->cmd = CPL_ABORT_NO_RST; |
| cxgb3_ofld_send(tdev, skb); |
| } |
| |
| static void send_abort_req(struct cxgbi_sock *csk) |
| { |
| struct sk_buff *skb = csk->cpl_abort_req; |
| struct cpl_abort_req *req; |
| |
| if (unlikely(csk->state == CTP_ABORTING || !skb)) |
| return; |
| cxgbi_sock_set_state(csk, CTP_ABORTING); |
| cxgbi_sock_set_flag(csk, CTPF_ABORT_RPL_PENDING); |
| /* Purge the send queue so we don't send anything after an abort. */ |
| cxgbi_sock_purge_write_queue(csk); |
| |
| csk->cpl_abort_req = NULL; |
| req = (struct cpl_abort_req *)skb->head; |
| skb->priority = CPL_PRIORITY_DATA; |
| set_arp_failure_handler(skb, abort_arp_failure); |
| req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ)); |
| req->wr.wr_lo = htonl(V_WR_TID(csk->tid)); |
| OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ, csk->tid)); |
| req->rsvd0 = htonl(csk->snd_nxt); |
| req->rsvd1 = !cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT); |
| req->cmd = CPL_ABORT_SEND_RST; |
| |
| log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, |
| "csk 0x%p,%u,0x%lx,%u, snd_nxt %u, 0x%x.\n", |
| csk, csk->state, csk->flags, csk->tid, csk->snd_nxt, |
| req->rsvd1); |
| |
| l2t_send(csk->cdev->lldev, skb, csk->l2t); |
| } |
| |
| /* |
| * CPL connection abort reply: host -> |
| * |
| * Send an ABORT_RPL message in response of the ABORT_REQ received. |
| */ |
| static void send_abort_rpl(struct cxgbi_sock *csk, int rst_status) |
| { |
| struct sk_buff *skb = csk->cpl_abort_rpl; |
| struct cpl_abort_rpl *rpl = (struct cpl_abort_rpl *)skb->head; |
| |
| log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, |
| "csk 0x%p,%u,0x%lx,%u, status %d.\n", |
| csk, csk->state, csk->flags, csk->tid, rst_status); |
| |
| csk->cpl_abort_rpl = NULL; |
| skb->priority = CPL_PRIORITY_DATA; |
| rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL)); |
| rpl->wr.wr_lo = htonl(V_WR_TID(csk->tid)); |
| OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL, csk->tid)); |
| rpl->cmd = rst_status; |
| cxgb3_ofld_send(csk->cdev->lldev, skb); |
| } |
| |
| /* |
| * CPL connection rx data ack: host -> |
| * Send RX credits through an RX_DATA_ACK CPL message. Returns the number of |
| * credits sent. |
| */ |
| static u32 send_rx_credits(struct cxgbi_sock *csk, u32 credits) |
| { |
| struct sk_buff *skb; |
| struct cpl_rx_data_ack *req; |
| u32 dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1); |
| |
| log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, |
| "csk 0x%p,%u,0x%lx,%u, credit %u, dack %u.\n", |
| csk, csk->state, csk->flags, csk->tid, credits, dack); |
| |
| skb = alloc_wr(sizeof(*req), 0, GFP_ATOMIC); |
| if (!skb) { |
| pr_info("csk 0x%p, credit %u, OOM.\n", csk, credits); |
| return 0; |
| } |
| req = (struct cpl_rx_data_ack *)skb->head; |
| req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); |
| OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, csk->tid)); |
| req->credit_dack = htonl(F_RX_DACK_CHANGE | V_RX_DACK_MODE(1) | |
| V_RX_CREDITS(credits)); |
| skb->priority = CPL_PRIORITY_ACK; |
| cxgb3_ofld_send(csk->cdev->lldev, skb); |
| return credits; |
| } |
| |
| /* |
| * CPL connection tx data: host -> |
| * |
| * Send iscsi PDU via TX_DATA CPL message. Returns the number of |
| * credits sent. |
| * Each TX_DATA consumes work request credit (wrs), so we need to keep track of |
| * how many we've used so far and how many are pending (i.e., yet ack'ed by T3). |
| */ |
| |
| static unsigned int wrlen __read_mostly; |
| static unsigned int skb_wrs[SKB_WR_LIST_SIZE] __read_mostly; |
| |
| static void init_wr_tab(unsigned int wr_len) |
| { |
| int i; |
| |
| if (skb_wrs[1]) /* already initialized */ |
| return; |
| for (i = 1; i < SKB_WR_LIST_SIZE; i++) { |
| int sgl_len = (3 * i) / 2 + (i & 1); |
| |
| sgl_len += 3; |
| skb_wrs[i] = (sgl_len <= wr_len |
| ? 1 : 1 + (sgl_len - 2) / (wr_len - 1)); |
| } |
| wrlen = wr_len * 8; |
| } |
| |
| static inline void make_tx_data_wr(struct cxgbi_sock *csk, struct sk_buff *skb, |
| int len, int req_completion) |
| { |
| struct tx_data_wr *req; |
| struct l2t_entry *l2t = csk->l2t; |
| |
| skb_reset_transport_header(skb); |
| req = __skb_push(skb, sizeof(*req)); |
| req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA) | |
| (req_completion ? F_WR_COMPL : 0)); |
| req->wr_lo = htonl(V_WR_TID(csk->tid)); |
| /* len includes the length of any HW ULP additions */ |
| req->len = htonl(len); |
| /* V_TX_ULP_SUBMODE sets both the mode and submode */ |
| req->flags = htonl(V_TX_ULP_SUBMODE(cxgbi_skcb_ulp_mode(skb)) | |
| V_TX_SHOVE((skb_peek(&csk->write_queue) ? 0 : 1))); |
| req->sndseq = htonl(csk->snd_nxt); |
| req->param = htonl(V_TX_PORT(l2t->smt_idx)); |
| |
| if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) { |
| req->flags |= htonl(V_TX_ACK_PAGES(2) | F_TX_INIT | |
| V_TX_CPU_IDX(csk->rss_qid)); |
| /* sendbuffer is in units of 32KB. */ |
| req->param |= htonl(V_TX_SNDBUF(csk->snd_win >> 15)); |
| cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT); |
| } |
| } |
| |
| /** |
| * push_tx_frames -- start transmit |
| * @c3cn: the offloaded connection |
| * @req_completion: request wr_ack or not |
| * |
| * Prepends TX_DATA_WR or CPL_CLOSE_CON_REQ headers to buffers waiting in a |
| * connection's send queue and sends them on to T3. Must be called with the |
| * connection's lock held. Returns the amount of send buffer space that was |
| * freed as a result of sending queued data to T3. |
| */ |
| |
| static void arp_failure_skb_discard(struct t3cdev *dev, struct sk_buff *skb) |
| { |
| kfree_skb(skb); |
| } |
| |
| static int push_tx_frames(struct cxgbi_sock *csk, int req_completion) |
| { |
| int total_size = 0; |
| struct sk_buff *skb; |
| |
| if (unlikely(csk->state < CTP_ESTABLISHED || |
| csk->state == CTP_CLOSE_WAIT_1 || csk->state >= CTP_ABORTING)) { |
| log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX, |
| "csk 0x%p,%u,0x%lx,%u, in closing state.\n", |
| csk, csk->state, csk->flags, csk->tid); |
| return 0; |
| } |
| |
| while (csk->wr_cred && (skb = skb_peek(&csk->write_queue)) != NULL) { |
| int len = skb->len; /* length before skb_push */ |
| int frags = skb_shinfo(skb)->nr_frags + (len != skb->data_len); |
| int wrs_needed = skb_wrs[frags]; |
| |
| if (wrs_needed > 1 && len + sizeof(struct tx_data_wr) <= wrlen) |
| wrs_needed = 1; |
| |
| WARN_ON(frags >= SKB_WR_LIST_SIZE || wrs_needed < 1); |
| |
| if (csk->wr_cred < wrs_needed) { |
| log_debug(1 << CXGBI_DBG_PDU_TX, |
| "csk 0x%p, skb len %u/%u, frag %u, wr %d<%u.\n", |
| csk, skb->len, skb->data_len, frags, |
| wrs_needed, csk->wr_cred); |
| break; |
| } |
| |
| __skb_unlink(skb, &csk->write_queue); |
| skb->priority = CPL_PRIORITY_DATA; |
| skb->csum = wrs_needed; /* remember this until the WR_ACK */ |
| csk->wr_cred -= wrs_needed; |
| csk->wr_una_cred += wrs_needed; |
| cxgbi_sock_enqueue_wr(csk, skb); |
| |
| log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX, |
| "csk 0x%p, enqueue, skb len %u/%u, frag %u, wr %d, " |
| "left %u, unack %u.\n", |
| csk, skb->len, skb->data_len, frags, skb->csum, |
| csk->wr_cred, csk->wr_una_cred); |
| |
| if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR))) { |
| if ((req_completion && |
| csk->wr_una_cred == wrs_needed) || |
| csk->wr_una_cred >= csk->wr_max_cred / 2) { |
| req_completion = 1; |
| csk->wr_una_cred = 0; |
| } |
| len += cxgbi_ulp_extra_len(cxgbi_skcb_ulp_mode(skb)); |
| make_tx_data_wr(csk, skb, len, req_completion); |
| csk->snd_nxt += len; |
| cxgbi_skcb_clear_flag(skb, SKCBF_TX_NEED_HDR); |
| } |
| total_size += skb->truesize; |
| log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX, |
| "csk 0x%p, tid 0x%x, send skb 0x%p.\n", |
| csk, csk->tid, skb); |
| set_arp_failure_handler(skb, arp_failure_skb_discard); |
| l2t_send(csk->cdev->lldev, skb, csk->l2t); |
| } |
| return total_size; |
| } |
| |
| /* |
| * Process a CPL_ACT_ESTABLISH message: -> host |
| * Updates connection state from an active establish CPL message. Runs with |
| * the connection lock held. |
| */ |
| |
| static inline void free_atid(struct cxgbi_sock *csk) |
| { |
| if (cxgbi_sock_flag(csk, CTPF_HAS_ATID)) { |
| cxgb3_free_atid(csk->cdev->lldev, csk->atid); |
| cxgbi_sock_clear_flag(csk, CTPF_HAS_ATID); |
| cxgbi_sock_put(csk); |
| } |
| } |
| |
| static int do_act_establish(struct t3cdev *tdev, struct sk_buff *skb, void *ctx) |
| { |
| struct cxgbi_sock *csk = ctx; |
| struct cpl_act_establish *req = cplhdr(skb); |
| unsigned int tid = GET_TID(req); |
| unsigned int atid = G_PASS_OPEN_TID(ntohl(req->tos_tid)); |
| u32 rcv_isn = ntohl(req->rcv_isn); /* real RCV_ISN + 1 */ |
| |
| log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, |
| "atid 0x%x,tid 0x%x, csk 0x%p,%u,0x%lx, isn %u.\n", |
| atid, atid, csk, csk->state, csk->flags, rcv_isn); |
| |
| cxgbi_sock_get(csk); |
| cxgbi_sock_set_flag(csk, CTPF_HAS_TID); |
| csk->tid = tid; |
| cxgb3_insert_tid(csk->cdev->lldev, &t3_client, csk, tid); |
| |
| free_atid(csk); |
| |
| csk->rss_qid = G_QNUM(ntohs(skb->csum)); |
| |
| spin_lock_bh(&csk->lock); |
| if (csk->retry_timer.function) { |
| del_timer(&csk->retry_timer); |
| csk->retry_timer.function = NULL; |
| } |
| |
| if (unlikely(csk->state != CTP_ACTIVE_OPEN)) |
| pr_info("csk 0x%p,%u,0x%lx,%u, got EST.\n", |
| csk, csk->state, csk->flags, csk->tid); |
| |
| csk->copied_seq = csk->rcv_wup = csk->rcv_nxt = rcv_isn; |
| if (csk->rcv_win > (M_RCV_BUFSIZ << 10)) |
| csk->rcv_wup -= csk->rcv_win - (M_RCV_BUFSIZ << 10); |
| |
| cxgbi_sock_established(csk, ntohl(req->snd_isn), ntohs(req->tcp_opt)); |
| |
| if (unlikely(cxgbi_sock_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED))) |
| /* upper layer has requested closing */ |
| send_abort_req(csk); |
| else { |
| if (skb_queue_len(&csk->write_queue)) |
| push_tx_frames(csk, 1); |
| cxgbi_conn_tx_open(csk); |
| } |
| |
| spin_unlock_bh(&csk->lock); |
| __kfree_skb(skb); |
| return 0; |
| } |
| |
| /* |
| * Process a CPL_ACT_OPEN_RPL message: -> host |
| * Handle active open failures. |
| */ |
| static int act_open_rpl_status_to_errno(int status) |
| { |
| switch (status) { |
| case CPL_ERR_CONN_RESET: |
| return -ECONNREFUSED; |
| case CPL_ERR_ARP_MISS: |
| return -EHOSTUNREACH; |
| case CPL_ERR_CONN_TIMEDOUT: |
| return -ETIMEDOUT; |
| case CPL_ERR_TCAM_FULL: |
| return -ENOMEM; |
| case CPL_ERR_CONN_EXIST: |
| return -EADDRINUSE; |
| default: |
| return -EIO; |
| } |
| } |
| |
| static void act_open_retry_timer(struct timer_list *t) |
| { |
| struct cxgbi_sock *csk = from_timer(csk, t, retry_timer); |
| struct sk_buff *skb; |
| |
| log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, |
| "csk 0x%p,%u,0x%lx,%u.\n", |
| csk, csk->state, csk->flags, csk->tid); |
| |
| cxgbi_sock_get(csk); |
| spin_lock_bh(&csk->lock); |
| skb = alloc_wr(sizeof(struct cpl_act_open_req), 0, GFP_ATOMIC); |
| if (!skb) |
| cxgbi_sock_fail_act_open(csk, -ENOMEM); |
| else { |
| skb->sk = (struct sock *)csk; |
| set_arp_failure_handler(skb, act_open_arp_failure); |
| send_act_open_req(csk, skb, csk->l2t); |
| } |
| spin_unlock_bh(&csk->lock); |
| cxgbi_sock_put(csk); |
| } |
| |
| static int do_act_open_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx) |
| { |
| struct cxgbi_sock *csk = ctx; |
| struct cpl_act_open_rpl *rpl = cplhdr(skb); |
| |
| pr_info("csk 0x%p,%u,0x%lx,%u, status %u, %pI4:%u-%pI4:%u.\n", |
| csk, csk->state, csk->flags, csk->atid, rpl->status, |
| &csk->saddr.sin_addr.s_addr, ntohs(csk->saddr.sin_port), |
| &csk->daddr.sin_addr.s_addr, ntohs(csk->daddr.sin_port)); |
| |
| if (rpl->status != CPL_ERR_TCAM_FULL && |
| rpl->status != CPL_ERR_CONN_EXIST && |
| rpl->status != CPL_ERR_ARP_MISS) |
| cxgb3_queue_tid_release(tdev, GET_TID(rpl)); |
| |
| cxgbi_sock_get(csk); |
| spin_lock_bh(&csk->lock); |
| if (rpl->status == CPL_ERR_CONN_EXIST && |
| csk->retry_timer.function != act_open_retry_timer) { |
| csk->retry_timer.function = act_open_retry_timer; |
| mod_timer(&csk->retry_timer, jiffies + HZ / 2); |
| } else |
| cxgbi_sock_fail_act_open(csk, |
| act_open_rpl_status_to_errno(rpl->status)); |
| |
| spin_unlock_bh(&csk->lock); |
| cxgbi_sock_put(csk); |
| __kfree_skb(skb); |
| return 0; |
| } |
| |
| /* |
| * Process PEER_CLOSE CPL messages: -> host |
| * Handle peer FIN. |
| */ |
| static int do_peer_close(struct t3cdev *cdev, struct sk_buff *skb, void *ctx) |
| { |
| struct cxgbi_sock *csk = ctx; |
| |
| log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, |
| "csk 0x%p,%u,0x%lx,%u.\n", |
| csk, csk->state, csk->flags, csk->tid); |
| |
| cxgbi_sock_rcv_peer_close(csk); |
| __kfree_skb(skb); |
| return 0; |
| } |
| |
| /* |
| * Process CLOSE_CONN_RPL CPL message: -> host |
| * Process a peer ACK to our FIN. |
| */ |
| static int do_close_con_rpl(struct t3cdev *cdev, struct sk_buff *skb, |
| void *ctx) |
| { |
| struct cxgbi_sock *csk = ctx; |
| struct cpl_close_con_rpl *rpl = cplhdr(skb); |
| |
| log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, |
| "csk 0x%p,%u,0x%lx,%u, snxt %u.\n", |
| csk, csk->state, csk->flags, csk->tid, ntohl(rpl->snd_nxt)); |
| |
| cxgbi_sock_rcv_close_conn_rpl(csk, ntohl(rpl->snd_nxt)); |
| __kfree_skb(skb); |
| return 0; |
| } |
| |
| /* |
| * Process ABORT_REQ_RSS CPL message: -> host |
| * Process abort requests. If we are waiting for an ABORT_RPL we ignore this |
| * request except that we need to reply to it. |
| */ |
| |
| static int abort_status_to_errno(struct cxgbi_sock *csk, int abort_reason, |
| int *need_rst) |
| { |
| switch (abort_reason) { |
| case CPL_ERR_BAD_SYN: /* fall through */ |
| case CPL_ERR_CONN_RESET: |
| return csk->state > CTP_ESTABLISHED ? -EPIPE : -ECONNRESET; |
| case CPL_ERR_XMIT_TIMEDOUT: |
| case CPL_ERR_PERSIST_TIMEDOUT: |
| case CPL_ERR_FINWAIT2_TIMEDOUT: |
| case CPL_ERR_KEEPALIVE_TIMEDOUT: |
| return -ETIMEDOUT; |
| default: |
| return -EIO; |
| } |
| } |
| |
| static int do_abort_req(struct t3cdev *cdev, struct sk_buff *skb, void *ctx) |
| { |
| const struct cpl_abort_req_rss *req = cplhdr(skb); |
| struct cxgbi_sock *csk = ctx; |
| int rst_status = CPL_ABORT_NO_RST; |
| |
| log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, |
| "csk 0x%p,%u,0x%lx,%u.\n", |
| csk, csk->state, csk->flags, csk->tid); |
| |
| if (req->status == CPL_ERR_RTX_NEG_ADVICE || |
| req->status == CPL_ERR_PERSIST_NEG_ADVICE) { |
| goto done; |
| } |
| |
| cxgbi_sock_get(csk); |
| spin_lock_bh(&csk->lock); |
| |
| if (!cxgbi_sock_flag(csk, CTPF_ABORT_REQ_RCVD)) { |
| cxgbi_sock_set_flag(csk, CTPF_ABORT_REQ_RCVD); |
| cxgbi_sock_set_state(csk, CTP_ABORTING); |
| goto out; |
| } |
| |
| cxgbi_sock_clear_flag(csk, CTPF_ABORT_REQ_RCVD); |
| send_abort_rpl(csk, rst_status); |
| |
| if (!cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) { |
| csk->err = abort_status_to_errno(csk, req->status, &rst_status); |
| cxgbi_sock_closed(csk); |
| } |
| |
| out: |
| spin_unlock_bh(&csk->lock); |
| cxgbi_sock_put(csk); |
| done: |
| __kfree_skb(skb); |
| return 0; |
| } |
| |
| /* |
| * Process ABORT_RPL_RSS CPL message: -> host |
| * Process abort replies. We only process these messages if we anticipate |
| * them as the coordination between SW and HW in this area is somewhat lacking |
| * and sometimes we get ABORT_RPLs after we are done with the connection that |
| * originated the ABORT_REQ. |
| */ |
| static int do_abort_rpl(struct t3cdev *cdev, struct sk_buff *skb, void *ctx) |
| { |
| struct cpl_abort_rpl_rss *rpl = cplhdr(skb); |
| struct cxgbi_sock *csk = ctx; |
| |
| log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, |
| "status 0x%x, csk 0x%p, s %u, 0x%lx.\n", |
| rpl->status, csk, csk ? csk->state : 0, |
| csk ? csk->flags : 0UL); |
| /* |
| * Ignore replies to post-close aborts indicating that the abort was |
| * requested too late. These connections are terminated when we get |
| * PEER_CLOSE or CLOSE_CON_RPL and by the time the abort_rpl_rss |
| * arrives the TID is either no longer used or it has been recycled. |
| */ |
| if (rpl->status == CPL_ERR_ABORT_FAILED) |
| goto rel_skb; |
| /* |
| * Sometimes we've already closed the connection, e.g., a post-close |
| * abort races with ABORT_REQ_RSS, the latter frees the connection |
| * expecting the ABORT_REQ will fail with CPL_ERR_ABORT_FAILED, |
| * but FW turns the ABORT_REQ into a regular one and so we get |
| * ABORT_RPL_RSS with status 0 and no connection. |
| */ |
| if (csk) |
| cxgbi_sock_rcv_abort_rpl(csk); |
| rel_skb: |
| __kfree_skb(skb); |
| return 0; |
| } |
| |
| /* |
| * Process RX_ISCSI_HDR CPL message: -> host |
| * Handle received PDUs, the payload could be DDP'ed. If not, the payload |
| * follow after the bhs. |
| */ |
| static int do_iscsi_hdr(struct t3cdev *t3dev, struct sk_buff *skb, void *ctx) |
| { |
| struct cxgbi_sock *csk = ctx; |
| struct cpl_iscsi_hdr *hdr_cpl = cplhdr(skb); |
| struct cpl_iscsi_hdr_norss data_cpl; |
| struct cpl_rx_data_ddp_norss ddp_cpl; |
| unsigned int hdr_len, data_len, status; |
| unsigned int len; |
| int err; |
| |
| log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, |
| "csk 0x%p,%u,0x%lx,%u, skb 0x%p,%u.\n", |
| csk, csk->state, csk->flags, csk->tid, skb, skb->len); |
| |
| spin_lock_bh(&csk->lock); |
| |
| if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) { |
| log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, |
| "csk 0x%p,%u,0x%lx,%u, bad state.\n", |
| csk, csk->state, csk->flags, csk->tid); |
| if (csk->state != CTP_ABORTING) |
| goto abort_conn; |
| else |
| goto discard; |
| } |
| |
| cxgbi_skcb_tcp_seq(skb) = ntohl(hdr_cpl->seq); |
| cxgbi_skcb_flags(skb) = 0; |
| |
| skb_reset_transport_header(skb); |
| __skb_pull(skb, sizeof(struct cpl_iscsi_hdr)); |
| |
| len = hdr_len = ntohs(hdr_cpl->len); |
| /* msg coalesce is off or not enough data received */ |
| if (skb->len <= hdr_len) { |
| pr_err("%s: tid %u, CPL_ISCSI_HDR, skb len %u < %u.\n", |
| csk->cdev->ports[csk->port_id]->name, csk->tid, |
| skb->len, hdr_len); |
| goto abort_conn; |
| } |
| cxgbi_skcb_set_flag(skb, SKCBF_RX_COALESCED); |
| |
| err = skb_copy_bits(skb, skb->len - sizeof(ddp_cpl), &ddp_cpl, |
| sizeof(ddp_cpl)); |
| if (err < 0) { |
| pr_err("%s: tid %u, copy cpl_ddp %u-%zu failed %d.\n", |
| csk->cdev->ports[csk->port_id]->name, csk->tid, |
| skb->len, sizeof(ddp_cpl), err); |
| goto abort_conn; |
| } |
| |
| cxgbi_skcb_set_flag(skb, SKCBF_RX_STATUS); |
| cxgbi_skcb_rx_pdulen(skb) = ntohs(ddp_cpl.len); |
| cxgbi_skcb_rx_ddigest(skb) = ntohl(ddp_cpl.ulp_crc); |
| status = ntohl(ddp_cpl.ddp_status); |
| |
| log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, |
| "csk 0x%p, skb 0x%p,%u, pdulen %u, status 0x%x.\n", |
| csk, skb, skb->len, cxgbi_skcb_rx_pdulen(skb), status); |
| |
| if (status & (1 << CPL_RX_DDP_STATUS_HCRC_SHIFT)) |
| cxgbi_skcb_set_flag(skb, SKCBF_RX_HCRC_ERR); |
| if (status & (1 << CPL_RX_DDP_STATUS_DCRC_SHIFT)) |
| cxgbi_skcb_set_flag(skb, SKCBF_RX_DCRC_ERR); |
| if (status & (1 << CPL_RX_DDP_STATUS_PAD_SHIFT)) |
| cxgbi_skcb_set_flag(skb, SKCBF_RX_PAD_ERR); |
| |
| if (skb->len > (hdr_len + sizeof(ddp_cpl))) { |
| err = skb_copy_bits(skb, hdr_len, &data_cpl, sizeof(data_cpl)); |
| if (err < 0) { |
| pr_err("%s: tid %u, cp %zu/%u failed %d.\n", |
| csk->cdev->ports[csk->port_id]->name, |
| csk->tid, sizeof(data_cpl), skb->len, err); |
| goto abort_conn; |
| } |
| data_len = ntohs(data_cpl.len); |
| log_debug(1 << CXGBI_DBG_DDP | 1 << CXGBI_DBG_PDU_RX, |
| "skb 0x%p, pdu not ddp'ed %u/%u, status 0x%x.\n", |
| skb, data_len, cxgbi_skcb_rx_pdulen(skb), status); |
| len += sizeof(data_cpl) + data_len; |
| } else if (status & (1 << CPL_RX_DDP_STATUS_DDP_SHIFT)) |
| cxgbi_skcb_set_flag(skb, SKCBF_RX_DATA_DDPD); |
| |
| csk->rcv_nxt = ntohl(ddp_cpl.seq) + cxgbi_skcb_rx_pdulen(skb); |
| __pskb_trim(skb, len); |
| __skb_queue_tail(&csk->receive_queue, skb); |
| cxgbi_conn_pdu_ready(csk); |
| |
| spin_unlock_bh(&csk->lock); |
| return 0; |
| |
| abort_conn: |
| send_abort_req(csk); |
| discard: |
| spin_unlock_bh(&csk->lock); |
| __kfree_skb(skb); |
| return 0; |
| } |
| |
| /* |
| * Process TX_DATA_ACK CPL messages: -> host |
| * Process an acknowledgment of WR completion. Advance snd_una and send the |
| * next batch of work requests from the write queue. |
| */ |
| static int do_wr_ack(struct t3cdev *cdev, struct sk_buff *skb, void *ctx) |
| { |
| struct cxgbi_sock *csk = ctx; |
| struct cpl_wr_ack *hdr = cplhdr(skb); |
| |
| log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, |
| "csk 0x%p,%u,0x%lx,%u, cr %u.\n", |
| csk, csk->state, csk->flags, csk->tid, ntohs(hdr->credits)); |
| |
| cxgbi_sock_rcv_wr_ack(csk, ntohs(hdr->credits), ntohl(hdr->snd_una), 1); |
| __kfree_skb(skb); |
| return 0; |
| } |
| |
| /* |
| * for each connection, pre-allocate skbs needed for close/abort requests. So |
| * that we can service the request right away. |
| */ |
| static int alloc_cpls(struct cxgbi_sock *csk) |
| { |
| csk->cpl_close = alloc_wr(sizeof(struct cpl_close_con_req), 0, |
| GFP_KERNEL); |
| if (!csk->cpl_close) |
| return -ENOMEM; |
| csk->cpl_abort_req = alloc_wr(sizeof(struct cpl_abort_req), 0, |
| GFP_KERNEL); |
| if (!csk->cpl_abort_req) |
| goto free_cpl_skbs; |
| |
| csk->cpl_abort_rpl = alloc_wr(sizeof(struct cpl_abort_rpl), 0, |
| GFP_KERNEL); |
| if (!csk->cpl_abort_rpl) |
| goto free_cpl_skbs; |
| |
| return 0; |
| |
| free_cpl_skbs: |
| cxgbi_sock_free_cpl_skbs(csk); |
| return -ENOMEM; |
| } |
| |
| /** |
| * release_offload_resources - release offload resource |
| * @c3cn: the offloaded iscsi tcp connection. |
| * Release resources held by an offload connection (TID, L2T entry, etc.) |
| */ |
| static void l2t_put(struct cxgbi_sock *csk) |
| { |
| struct t3cdev *t3dev = (struct t3cdev *)csk->cdev->lldev; |
| |
| if (csk->l2t) { |
| l2t_release(t3dev, csk->l2t); |
| csk->l2t = NULL; |
| cxgbi_sock_put(csk); |
| } |
| } |
| |
| static void release_offload_resources(struct cxgbi_sock *csk) |
| { |
| struct t3cdev *t3dev = (struct t3cdev *)csk->cdev->lldev; |
| |
| log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, |
| "csk 0x%p,%u,0x%lx,%u.\n", |
| csk, csk->state, csk->flags, csk->tid); |
| |
| csk->rss_qid = 0; |
| cxgbi_sock_free_cpl_skbs(csk); |
| |
| if (csk->wr_cred != csk->wr_max_cred) { |
| cxgbi_sock_purge_wr_queue(csk); |
| cxgbi_sock_reset_wr_list(csk); |
| } |
| l2t_put(csk); |
| if (cxgbi_sock_flag(csk, CTPF_HAS_ATID)) |
| free_atid(csk); |
| else if (cxgbi_sock_flag(csk, CTPF_HAS_TID)) { |
| cxgb3_remove_tid(t3dev, (void *)csk, csk->tid); |
| cxgbi_sock_clear_flag(csk, CTPF_HAS_TID); |
| cxgbi_sock_put(csk); |
| } |
| csk->dst = NULL; |
| csk->cdev = NULL; |
| } |
| |
| static void update_address(struct cxgbi_hba *chba) |
| { |
| if (chba->ipv4addr) { |
| if (chba->vdev && |
| chba->ipv4addr != cxgb3i_get_private_ipv4addr(chba->vdev)) { |
| cxgb3i_set_private_ipv4addr(chba->vdev, chba->ipv4addr); |
| cxgb3i_set_private_ipv4addr(chba->ndev, 0); |
| pr_info("%s set %pI4.\n", |
| chba->vdev->name, &chba->ipv4addr); |
| } else if (chba->ipv4addr != |
| cxgb3i_get_private_ipv4addr(chba->ndev)) { |
| cxgb3i_set_private_ipv4addr(chba->ndev, chba->ipv4addr); |
| pr_info("%s set %pI4.\n", |
| chba->ndev->name, &chba->ipv4addr); |
| } |
| } else if (cxgb3i_get_private_ipv4addr(chba->ndev)) { |
| if (chba->vdev) |
| cxgb3i_set_private_ipv4addr(chba->vdev, 0); |
| cxgb3i_set_private_ipv4addr(chba->ndev, 0); |
| } |
| } |
| |
| static int init_act_open(struct cxgbi_sock *csk) |
| { |
| struct dst_entry *dst = csk->dst; |
| struct cxgbi_device *cdev = csk->cdev; |
| struct t3cdev *t3dev = (struct t3cdev *)cdev->lldev; |
| struct net_device *ndev = cdev->ports[csk->port_id]; |
| struct cxgbi_hba *chba = cdev->hbas[csk->port_id]; |
| struct sk_buff *skb = NULL; |
| |
| log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, |
| "csk 0x%p,%u,0x%lx.\n", csk, csk->state, csk->flags); |
| |
| update_address(chba); |
| if (chba->ipv4addr) |
| csk->saddr.sin_addr.s_addr = chba->ipv4addr; |
| |
| csk->rss_qid = 0; |
| csk->l2t = t3_l2t_get(t3dev, dst, ndev, |
| &csk->daddr.sin_addr.s_addr); |
| if (!csk->l2t) { |
| pr_err("NO l2t available.\n"); |
| return -EINVAL; |
| } |
| cxgbi_sock_get(csk); |
| |
| csk->atid = cxgb3_alloc_atid(t3dev, &t3_client, csk); |
| if (csk->atid < 0) { |
| pr_err("NO atid available.\n"); |
| goto rel_resource; |
| } |
| cxgbi_sock_set_flag(csk, CTPF_HAS_ATID); |
| cxgbi_sock_get(csk); |
| |
| skb = alloc_wr(sizeof(struct cpl_act_open_req), 0, GFP_KERNEL); |
| if (!skb) |
| goto rel_resource; |
| skb->sk = (struct sock *)csk; |
| set_arp_failure_handler(skb, act_open_arp_failure); |
| csk->snd_win = cxgb3i_snd_win; |
| csk->rcv_win = cxgb3i_rcv_win; |
| |
| csk->wr_max_cred = csk->wr_cred = T3C_DATA(t3dev)->max_wrs - 1; |
| csk->wr_una_cred = 0; |
| csk->mss_idx = cxgbi_sock_select_mss(csk, dst_mtu(dst)); |
| cxgbi_sock_reset_wr_list(csk); |
| csk->err = 0; |
| |
| log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, |
| "csk 0x%p,%u,0x%lx, %pI4:%u-%pI4:%u.\n", |
| csk, csk->state, csk->flags, |
| &csk->saddr.sin_addr.s_addr, ntohs(csk->saddr.sin_port), |
| &csk->daddr.sin_addr.s_addr, ntohs(csk->daddr.sin_port)); |
| |
| cxgbi_sock_set_state(csk, CTP_ACTIVE_OPEN); |
| send_act_open_req(csk, skb, csk->l2t); |
| return 0; |
| |
| rel_resource: |
| if (skb) |
| __kfree_skb(skb); |
| return -EINVAL; |
| } |
| |
| cxgb3_cpl_handler_func cxgb3i_cpl_handlers[NUM_CPL_CMDS] = { |
| [CPL_ACT_ESTABLISH] = do_act_establish, |
| [CPL_ACT_OPEN_RPL] = do_act_open_rpl, |
| [CPL_PEER_CLOSE] = do_peer_close, |
| [CPL_ABORT_REQ_RSS] = do_abort_req, |
| [CPL_ABORT_RPL_RSS] = do_abort_rpl, |
| [CPL_CLOSE_CON_RPL] = do_close_con_rpl, |
| [CPL_TX_DMA_ACK] = do_wr_ack, |
| [CPL_ISCSI_HDR] = do_iscsi_hdr, |
| }; |
| |
| /** |
| * cxgb3i_ofld_init - allocate and initialize resources for each adapter found |
| * @cdev: cxgbi adapter |
| */ |
| static int cxgb3i_ofld_init(struct cxgbi_device *cdev) |
| { |
| struct t3cdev *t3dev = (struct t3cdev *)cdev->lldev; |
| struct adap_ports port; |
| struct ofld_page_info rx_page_info; |
| unsigned int wr_len; |
| int rc; |
| |
| if (t3dev->ctl(t3dev, GET_WR_LEN, &wr_len) < 0 || |
| t3dev->ctl(t3dev, GET_PORTS, &port) < 0 || |
| t3dev->ctl(t3dev, GET_RX_PAGE_INFO, &rx_page_info) < 0) { |
| pr_warn("t3 0x%p, offload up, ioctl failed.\n", t3dev); |
| return -EINVAL; |
| } |
| |
| if (cxgb3i_max_connect > CXGBI_MAX_CONN) |
| cxgb3i_max_connect = CXGBI_MAX_CONN; |
| |
| rc = cxgbi_device_portmap_create(cdev, cxgb3i_sport_base, |
| cxgb3i_max_connect); |
| if (rc < 0) |
| return rc; |
| |
| init_wr_tab(wr_len); |
| cdev->csk_release_offload_resources = release_offload_resources; |
| cdev->csk_push_tx_frames = push_tx_frames; |
| cdev->csk_send_abort_req = send_abort_req; |
| cdev->csk_send_close_req = send_close_req; |
| cdev->csk_send_rx_credits = send_rx_credits; |
| cdev->csk_alloc_cpls = alloc_cpls; |
| cdev->csk_init_act_open = init_act_open; |
| |
| pr_info("cdev 0x%p, offload up, added.\n", cdev); |
| return 0; |
| } |
| |
| /* |
| * functions to program the pagepod in h/w |
| */ |
| static inline void ulp_mem_io_set_hdr(struct sk_buff *skb, unsigned int addr) |
| { |
| struct ulp_mem_io *req = (struct ulp_mem_io *)skb->head; |
| |
| memset(req, 0, sizeof(*req)); |
| |
| req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS)); |
| req->cmd_lock_addr = htonl(V_ULP_MEMIO_ADDR(addr >> 5) | |
| V_ULPTX_CMD(ULP_MEM_WRITE)); |
| req->len = htonl(V_ULP_MEMIO_DATA_LEN(IPPOD_SIZE >> 5) | |
| V_ULPTX_NFLITS((IPPOD_SIZE >> 3) + 1)); |
| } |
| |
| static struct cxgbi_ppm *cdev2ppm(struct cxgbi_device *cdev) |
| { |
| return ((struct t3cdev *)cdev->lldev)->ulp_iscsi; |
| } |
| |
| static int ddp_set_map(struct cxgbi_ppm *ppm, struct cxgbi_sock *csk, |
| struct cxgbi_task_tag_info *ttinfo) |
| { |
| unsigned int idx = ttinfo->idx; |
| unsigned int npods = ttinfo->npods; |
| struct scatterlist *sg = ttinfo->sgl; |
| struct cxgbi_pagepod *ppod; |
| struct ulp_mem_io *req; |
| unsigned int sg_off; |
| unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ppm->llimit; |
| int i; |
| |
| for (i = 0; i < npods; i++, idx++, pm_addr += IPPOD_SIZE) { |
| struct sk_buff *skb = alloc_wr(sizeof(struct ulp_mem_io) + |
| IPPOD_SIZE, 0, GFP_ATOMIC); |
| |
| if (!skb) |
| return -ENOMEM; |
| ulp_mem_io_set_hdr(skb, pm_addr); |
| req = (struct ulp_mem_io *)skb->head; |
| ppod = (struct cxgbi_pagepod *)(req + 1); |
| sg_off = i * PPOD_PAGES_MAX; |
| cxgbi_ddp_set_one_ppod(ppod, ttinfo, &sg, |
| &sg_off); |
| skb->priority = CPL_PRIORITY_CONTROL; |
| cxgb3_ofld_send(ppm->lldev, skb); |
| } |
| return 0; |
| } |
| |
| static void ddp_clear_map(struct cxgbi_device *cdev, struct cxgbi_ppm *ppm, |
| struct cxgbi_task_tag_info *ttinfo) |
| { |
| unsigned int idx = ttinfo->idx; |
| unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ppm->llimit; |
| unsigned int npods = ttinfo->npods; |
| int i; |
| |
| log_debug(1 << CXGBI_DBG_DDP, |
| "cdev 0x%p, clear idx %u, npods %u.\n", |
| cdev, idx, npods); |
| |
| for (i = 0; i < npods; i++, idx++, pm_addr += IPPOD_SIZE) { |
| struct sk_buff *skb = alloc_wr(sizeof(struct ulp_mem_io) + |
| IPPOD_SIZE, 0, GFP_ATOMIC); |
| |
| if (!skb) { |
| pr_err("cdev 0x%p, clear ddp, %u,%d/%u, skb OOM.\n", |
| cdev, idx, i, npods); |
| continue; |
| } |
| ulp_mem_io_set_hdr(skb, pm_addr); |
| skb->priority = CPL_PRIORITY_CONTROL; |
| cxgb3_ofld_send(ppm->lldev, skb); |
| } |
| } |
| |
| static int ddp_setup_conn_pgidx(struct cxgbi_sock *csk, |
| unsigned int tid, int pg_idx, bool reply) |
| { |
| struct sk_buff *skb = alloc_wr(sizeof(struct cpl_set_tcb_field), 0, |
| GFP_KERNEL); |
| struct cpl_set_tcb_field *req; |
| u64 val = pg_idx < DDP_PGIDX_MAX ? pg_idx : 0; |
| |
| log_debug(1 << CXGBI_DBG_DDP, |
| "csk 0x%p, tid %u, pg_idx %d.\n", csk, tid, pg_idx); |
| if (!skb) |
| return -ENOMEM; |
| |
| /* set up ulp submode and page size */ |
| req = (struct cpl_set_tcb_field *)skb->head; |
| req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); |
| OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); |
| req->reply = V_NO_REPLY(reply ? 0 : 1); |
| req->cpu_idx = 0; |
| req->word = htons(31); |
| req->mask = cpu_to_be64(0xF0000000); |
| req->val = cpu_to_be64(val << 28); |
| skb->priority = CPL_PRIORITY_CONTROL; |
| |
| cxgb3_ofld_send(csk->cdev->lldev, skb); |
| return 0; |
| } |
| |
| /** |
| * cxgb3i_setup_conn_digest - setup conn. digest setting |
| * @csk: cxgb tcp socket |
| * @tid: connection id |
| * @hcrc: header digest enabled |
| * @dcrc: data digest enabled |
| * @reply: request reply from h/w |
| * set up the iscsi digest settings for a connection identified by tid |
| */ |
| static int ddp_setup_conn_digest(struct cxgbi_sock *csk, unsigned int tid, |
| int hcrc, int dcrc, int reply) |
| { |
| struct sk_buff *skb = alloc_wr(sizeof(struct cpl_set_tcb_field), 0, |
| GFP_KERNEL); |
| struct cpl_set_tcb_field *req; |
| u64 val = (hcrc ? 1 : 0) | (dcrc ? 2 : 0); |
| |
| log_debug(1 << CXGBI_DBG_DDP, |
| "csk 0x%p, tid %u, crc %d,%d.\n", csk, tid, hcrc, dcrc); |
| if (!skb) |
| return -ENOMEM; |
| |
| /* set up ulp submode and page size */ |
| req = (struct cpl_set_tcb_field *)skb->head; |
| req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); |
| OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); |
| req->reply = V_NO_REPLY(reply ? 0 : 1); |
| req->cpu_idx = 0; |
| req->word = htons(31); |
| req->mask = cpu_to_be64(0x0F000000); |
| req->val = cpu_to_be64(val << 24); |
| skb->priority = CPL_PRIORITY_CONTROL; |
| |
| cxgb3_ofld_send(csk->cdev->lldev, skb); |
| return 0; |
| } |
| |
| /** |
| * cxgb3i_ddp_init - initialize the cxgb3 adapter's ddp resource |
| * @cdev: cxgb3i adapter |
| * initialize the ddp pagepod manager for a given adapter |
| */ |
| static int cxgb3i_ddp_init(struct cxgbi_device *cdev) |
| { |
| struct t3cdev *tdev = (struct t3cdev *)cdev->lldev; |
| struct net_device *ndev = cdev->ports[0]; |
| struct cxgbi_tag_format tformat; |
| unsigned int ppmax, tagmask = 0; |
| struct ulp_iscsi_info uinfo; |
| int i, err; |
| |
| err = tdev->ctl(tdev, ULP_ISCSI_GET_PARAMS, &uinfo); |
| if (err < 0) { |
| pr_err("%s, failed to get iscsi param %d.\n", |
| ndev->name, err); |
| return err; |
| } |
| if (uinfo.llimit >= uinfo.ulimit) { |
| pr_warn("T3 %s, iscsi NOT enabled %u ~ %u!\n", |
| ndev->name, uinfo.llimit, uinfo.ulimit); |
| return -EACCES; |
| } |
| |
| ppmax = (uinfo.ulimit - uinfo.llimit + 1) >> PPOD_SIZE_SHIFT; |
| tagmask = cxgbi_tagmask_set(ppmax); |
| |
| pr_info("T3 %s: 0x%x~0x%x, 0x%x, tagmask 0x%x -> 0x%x.\n", |
| ndev->name, uinfo.llimit, uinfo.ulimit, ppmax, uinfo.tagmask, |
| tagmask); |
| |
| memset(&tformat, 0, sizeof(struct cxgbi_tag_format)); |
| for (i = 0; i < 4; i++) |
| tformat.pgsz_order[i] = uinfo.pgsz_factor[i]; |
| cxgbi_tagmask_check(tagmask, &tformat); |
| |
| cxgbi_ddp_ppm_setup(&tdev->ulp_iscsi, cdev, &tformat, ppmax, |
| uinfo.llimit, uinfo.llimit, 0); |
| if (!(cdev->flags & CXGBI_FLAG_DDP_OFF)) { |
| uinfo.tagmask = tagmask; |
| uinfo.ulimit = uinfo.llimit + (ppmax << PPOD_SIZE_SHIFT); |
| |
| err = tdev->ctl(tdev, ULP_ISCSI_SET_PARAMS, &uinfo); |
| if (err < 0) { |
| pr_err("T3 %s fail to set iscsi param %d.\n", |
| ndev->name, err); |
| cdev->flags |= CXGBI_FLAG_DDP_OFF; |
| } |
| err = 0; |
| } |
| |
| cdev->csk_ddp_setup_digest = ddp_setup_conn_digest; |
| cdev->csk_ddp_setup_pgidx = ddp_setup_conn_pgidx; |
| cdev->csk_ddp_set_map = ddp_set_map; |
| cdev->csk_ddp_clear_map = ddp_clear_map; |
| cdev->cdev2ppm = cdev2ppm; |
| cdev->tx_max_size = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD, |
| uinfo.max_txsz - ISCSI_PDU_NONPAYLOAD_LEN); |
| cdev->rx_max_size = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD, |
| uinfo.max_rxsz - ISCSI_PDU_NONPAYLOAD_LEN); |
| |
| return 0; |
| } |
| |
| static void cxgb3i_dev_close(struct t3cdev *t3dev) |
| { |
| struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev); |
| |
| if (!cdev || cdev->flags & CXGBI_FLAG_ADAPTER_RESET) { |
| pr_info("0x%p close, f 0x%x.\n", cdev, cdev ? cdev->flags : 0); |
| return; |
| } |
| |
| cxgbi_device_unregister(cdev); |
| } |
| |
| /** |
| * cxgb3i_dev_open - init a t3 adapter structure and any h/w settings |
| * @t3dev: t3cdev adapter |
| */ |
| static void cxgb3i_dev_open(struct t3cdev *t3dev) |
| { |
| struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev); |
| struct adapter *adapter = tdev2adap(t3dev); |
| int i, err; |
| |
| if (cdev) { |
| pr_info("0x%p, updating.\n", cdev); |
| return; |
| } |
| |
| cdev = cxgbi_device_register(0, adapter->params.nports); |
| if (!cdev) { |
| pr_warn("device 0x%p register failed.\n", t3dev); |
| return; |
| } |
| |
| cdev->flags = CXGBI_FLAG_DEV_T3 | CXGBI_FLAG_IPV4_SET; |
| cdev->lldev = t3dev; |
| cdev->pdev = adapter->pdev; |
| cdev->ports = adapter->port; |
| cdev->nports = adapter->params.nports; |
| cdev->mtus = adapter->params.mtus; |
| cdev->nmtus = NMTUS; |
| cdev->rx_credit_thres = cxgb3i_rx_credit_thres; |
| cdev->skb_tx_rsvd = CXGB3I_TX_HEADER_LEN; |
| cdev->skb_rx_extra = sizeof(struct cpl_iscsi_hdr_norss); |
| cdev->itp = &cxgb3i_iscsi_transport; |
| |
| err = cxgb3i_ddp_init(cdev); |
| if (err) { |
| pr_info("0x%p ddp init failed\n", cdev); |
| goto err_out; |
| } |
| |
| err = cxgb3i_ofld_init(cdev); |
| if (err) { |
| pr_info("0x%p offload init failed\n", cdev); |
| goto err_out; |
| } |
| |
| err = cxgbi_hbas_add(cdev, CXGB3I_MAX_LUN, CXGBI_MAX_CONN, |
| &cxgb3i_host_template, cxgb3i_stt); |
| if (err) |
| goto err_out; |
| |
| for (i = 0; i < cdev->nports; i++) |
| cdev->hbas[i]->ipv4addr = |
| cxgb3i_get_private_ipv4addr(cdev->ports[i]); |
| |
| pr_info("cdev 0x%p, f 0x%x, t3dev 0x%p open, err %d.\n", |
| cdev, cdev ? cdev->flags : 0, t3dev, err); |
| return; |
| |
| err_out: |
| cxgbi_device_unregister(cdev); |
| } |
| |
| static void cxgb3i_dev_event_handler(struct t3cdev *t3dev, u32 event, u32 port) |
| { |
| struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev); |
| |
| log_debug(1 << CXGBI_DBG_TOE, |
| "0x%p, cdev 0x%p, event 0x%x, port 0x%x.\n", |
| t3dev, cdev, event, port); |
| if (!cdev) |
| return; |
| |
| switch (event) { |
| case OFFLOAD_STATUS_DOWN: |
| cdev->flags |= CXGBI_FLAG_ADAPTER_RESET; |
| break; |
| case OFFLOAD_STATUS_UP: |
| cdev->flags &= ~CXGBI_FLAG_ADAPTER_RESET; |
| break; |
| } |
| } |
| |
| /** |
| * cxgb3i_init_module - module init entry point |
| * |
| * initialize any driver wide global data structures and register itself |
| * with the cxgb3 module |
| */ |
| static int __init cxgb3i_init_module(void) |
| { |
| int rc; |
| |
| printk(KERN_INFO "%s", version); |
| |
| rc = cxgbi_iscsi_init(&cxgb3i_iscsi_transport, &cxgb3i_stt); |
| if (rc < 0) |
| return rc; |
| |
| cxgb3_register_client(&t3_client); |
| return 0; |
| } |
| |
| /** |
| * cxgb3i_exit_module - module cleanup/exit entry point |
| * |
| * go through the driver hba list and for each hba, release any resource held. |
| * and unregisters iscsi transport and the cxgb3 module |
| */ |
| static void __exit cxgb3i_exit_module(void) |
| { |
| cxgb3_unregister_client(&t3_client); |
| cxgbi_device_unregister_all(CXGBI_FLAG_DEV_T3); |
| cxgbi_iscsi_cleanup(&cxgb3i_iscsi_transport, &cxgb3i_stt); |
| } |
| |
| module_init(cxgb3i_init_module); |
| module_exit(cxgb3i_exit_module); |