Thomas Gleixner | 09c434b | 2019-05-19 13:08:20 +0100 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0-only |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2 | /* |
| 3 | * Shared Memory Communications over RDMA (SMC-R) and RoCE |
| 4 | * |
| 5 | * AF_SMC protocol family socket handler keeping the AF_INET sock address type |
| 6 | * applies to SOCK_STREAM sockets only |
| 7 | * offers an alternative communication option for TCP-protocol sockets |
| 8 | * applicable with RoCE-cards only |
| 9 | * |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 10 | * Initial restrictions: |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 11 | * - support for alternate links postponed |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 12 | * |
Karsten Graul | aaa4d33 | 2018-03-16 15:06:41 +0100 | [diff] [blame] | 13 | * Copyright IBM Corp. 2016, 2018 |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 14 | * |
| 15 | * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com> |
| 16 | * based on prototype from Frank Blaschka |
| 17 | */ |
| 18 | |
| 19 | #define KMSG_COMPONENT "smc" |
| 20 | #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt |
| 21 | |
| 22 | #include <linux/module.h> |
| 23 | #include <linux/socket.h> |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 24 | #include <linux/workqueue.h> |
Ursula Braun | 5f08318 | 2017-01-09 16:55:22 +0100 | [diff] [blame] | 25 | #include <linux/in.h> |
Ingo Molnar | c3edc40 | 2017-02-02 08:35:14 +0100 | [diff] [blame] | 26 | #include <linux/sched/signal.h> |
Hans Wippel | 4134984 | 2018-06-28 19:05:11 +0200 | [diff] [blame] | 27 | #include <linux/if_vlan.h> |
Ursula Braun | 4ead9c9 | 2019-11-16 17:47:31 +0100 | [diff] [blame] | 28 | #include <linux/rcupdate_wait.h> |
Ursula Braun | b81a5eb | 2020-09-26 12:44:31 +0200 | [diff] [blame] | 29 | #include <linux/ctype.h> |
Ingo Molnar | c3edc40 | 2017-02-02 08:35:14 +0100 | [diff] [blame] | 30 | |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 31 | #include <net/sock.h> |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 32 | #include <net/tcp.h> |
Ursula Braun | f16a7dd | 2017-01-09 16:55:26 +0100 | [diff] [blame] | 33 | #include <net/smc.h> |
Ursula Braun | 9b67e26 | 2018-05-02 16:56:46 +0200 | [diff] [blame] | 34 | #include <asm/ioctls.h> |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 35 | |
Hans Wippel | 64e28b5 | 2019-02-21 13:01:02 +0100 | [diff] [blame] | 36 | #include <net/net_namespace.h> |
| 37 | #include <net/netns/generic.h> |
| 38 | #include "smc_netns.h" |
| 39 | |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 40 | #include "smc.h" |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 41 | #include "smc_clc.h" |
Ursula Braun | 9bf9abe | 2017-01-09 16:55:21 +0100 | [diff] [blame] | 42 | #include "smc_llc.h" |
Ursula Braun | 5f08318 | 2017-01-09 16:55:22 +0100 | [diff] [blame] | 43 | #include "smc_cdc.h" |
Ursula Braun | 0cfdd8f | 2017-01-09 16:55:17 +0100 | [diff] [blame] | 44 | #include "smc_core.h" |
Ursula Braun | a4cf044 | 2017-01-09 16:55:14 +0100 | [diff] [blame] | 45 | #include "smc_ib.h" |
Hans Wippel | 4134984 | 2018-06-28 19:05:11 +0200 | [diff] [blame] | 46 | #include "smc_ism.h" |
Thomas Richter | 6812baa | 2017-01-09 16:55:15 +0100 | [diff] [blame] | 47 | #include "smc_pnet.h" |
Guvenc Gulce | e8372d9 | 2020-12-01 20:20:43 +0100 | [diff] [blame] | 48 | #include "smc_netlink.h" |
Ursula Braun | e6727f3 | 2017-01-09 16:55:23 +0100 | [diff] [blame] | 49 | #include "smc_tx.h" |
Ursula Braun | 952310c | 2017-01-09 16:55:24 +0100 | [diff] [blame] | 50 | #include "smc_rx.h" |
Ursula Braun | b38d732 | 2017-01-09 16:55:25 +0100 | [diff] [blame] | 51 | #include "smc_close.h" |
Guvenc Gulce | e0e4b8f | 2021-06-16 16:52:55 +0200 | [diff] [blame] | 52 | #include "smc_stats.h" |
Tony Lu | 4826260 | 2021-11-01 15:39:12 +0800 | [diff] [blame] | 53 | #include "smc_tracepoint.h" |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 54 | |
Hans Wippel | 72a36a8 | 2019-02-07 15:56:18 +0100 | [diff] [blame] | 55 | static DEFINE_MUTEX(smc_server_lgr_pending); /* serialize link group |
| 56 | * creation on server |
| 57 | */ |
| 58 | static DEFINE_MUTEX(smc_client_lgr_pending); /* serialize link group |
| 59 | * creation on client |
Ursula Braun | 0cfdd8f | 2017-01-09 16:55:17 +0100 | [diff] [blame] | 60 | */ |
| 61 | |
Karsten Graul | 22ef473 | 2020-09-10 18:48:29 +0200 | [diff] [blame] | 62 | struct workqueue_struct *smc_hs_wq; /* wq for handshake work */ |
| 63 | struct workqueue_struct *smc_close_wq; /* wq for close work */ |
| 64 | |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 65 | static void smc_tcp_listen_work(struct work_struct *); |
Ursula Braun | 24ac3a0 | 2018-06-27 17:59:50 +0200 | [diff] [blame] | 66 | static void smc_connect_work(struct work_struct *); |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 67 | |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 68 | static void smc_set_keepalive(struct sock *sk, int val) |
| 69 | { |
| 70 | struct smc_sock *smc = smc_sk(sk); |
| 71 | |
| 72 | smc->clcsock->sk->sk_prot->keepalive(smc->clcsock->sk, val); |
| 73 | } |
| 74 | |
Ursula Braun | f16a7dd | 2017-01-09 16:55:26 +0100 | [diff] [blame] | 75 | static struct smc_hashinfo smc_v4_hashinfo = { |
| 76 | .lock = __RW_LOCK_UNLOCKED(smc_v4_hashinfo.lock), |
| 77 | }; |
| 78 | |
Karsten Graul | aaa4d33 | 2018-03-16 15:06:41 +0100 | [diff] [blame] | 79 | static struct smc_hashinfo smc_v6_hashinfo = { |
| 80 | .lock = __RW_LOCK_UNLOCKED(smc_v6_hashinfo.lock), |
| 81 | }; |
| 82 | |
Ursula Braun | f16a7dd | 2017-01-09 16:55:26 +0100 | [diff] [blame] | 83 | int smc_hash_sk(struct sock *sk) |
| 84 | { |
| 85 | struct smc_hashinfo *h = sk->sk_prot->h.smc_hash; |
| 86 | struct hlist_head *head; |
| 87 | |
| 88 | head = &h->ht; |
| 89 | |
| 90 | write_lock_bh(&h->lock); |
| 91 | sk_add_node(sk, head); |
| 92 | sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); |
| 93 | write_unlock_bh(&h->lock); |
| 94 | |
| 95 | return 0; |
| 96 | } |
| 97 | EXPORT_SYMBOL_GPL(smc_hash_sk); |
| 98 | |
| 99 | void smc_unhash_sk(struct sock *sk) |
| 100 | { |
| 101 | struct smc_hashinfo *h = sk->sk_prot->h.smc_hash; |
| 102 | |
| 103 | write_lock_bh(&h->lock); |
| 104 | if (sk_del_node_init(sk)) |
| 105 | sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); |
| 106 | write_unlock_bh(&h->lock); |
| 107 | } |
| 108 | EXPORT_SYMBOL_GPL(smc_unhash_sk); |
| 109 | |
| 110 | struct proto smc_proto = { |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 111 | .name = "SMC", |
| 112 | .owner = THIS_MODULE, |
| 113 | .keepalive = smc_set_keepalive, |
Ursula Braun | f16a7dd | 2017-01-09 16:55:26 +0100 | [diff] [blame] | 114 | .hash = smc_hash_sk, |
| 115 | .unhash = smc_unhash_sk, |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 116 | .obj_size = sizeof(struct smc_sock), |
Ursula Braun | f16a7dd | 2017-01-09 16:55:26 +0100 | [diff] [blame] | 117 | .h.smc_hash = &smc_v4_hashinfo, |
Paul E. McKenney | 5f0d5a3 | 2017-01-18 02:53:44 -0800 | [diff] [blame] | 118 | .slab_flags = SLAB_TYPESAFE_BY_RCU, |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 119 | }; |
Ursula Braun | f16a7dd | 2017-01-09 16:55:26 +0100 | [diff] [blame] | 120 | EXPORT_SYMBOL_GPL(smc_proto); |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 121 | |
Karsten Graul | aaa4d33 | 2018-03-16 15:06:41 +0100 | [diff] [blame] | 122 | struct proto smc_proto6 = { |
| 123 | .name = "SMC6", |
| 124 | .owner = THIS_MODULE, |
| 125 | .keepalive = smc_set_keepalive, |
| 126 | .hash = smc_hash_sk, |
| 127 | .unhash = smc_unhash_sk, |
| 128 | .obj_size = sizeof(struct smc_sock), |
| 129 | .h.smc_hash = &smc_v6_hashinfo, |
| 130 | .slab_flags = SLAB_TYPESAFE_BY_RCU, |
| 131 | }; |
| 132 | EXPORT_SYMBOL_GPL(smc_proto6); |
| 133 | |
Ursula Braun | f536dff | 2019-10-23 15:44:05 +0200 | [diff] [blame] | 134 | static void smc_restore_fallback_changes(struct smc_sock *smc) |
| 135 | { |
Karsten Graul | 1ad2405 | 2020-07-18 15:06:18 +0200 | [diff] [blame] | 136 | if (smc->clcsock->file) { /* non-accepted sockets have no file yet */ |
| 137 | smc->clcsock->file->private_data = smc->sk.sk_socket; |
| 138 | smc->clcsock->file = NULL; |
| 139 | } |
Ursula Braun | f536dff | 2019-10-23 15:44:05 +0200 | [diff] [blame] | 140 | } |
| 141 | |
Ursula Braun | 39f41f3 | 2019-06-27 15:04:52 +0200 | [diff] [blame] | 142 | static int __smc_release(struct smc_sock *smc) |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 143 | { |
Ursula Braun | 39f41f3 | 2019-06-27 15:04:52 +0200 | [diff] [blame] | 144 | struct sock *sk = &smc->sk; |
Ursula Braun | b38d732 | 2017-01-09 16:55:25 +0100 | [diff] [blame] | 145 | int rc = 0; |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 146 | |
Ursula Braun | 51f1de7 | 2018-01-26 09:28:48 +0100 | [diff] [blame] | 147 | if (!smc->use_fallback) { |
Ursula Braun | b38d732 | 2017-01-09 16:55:25 +0100 | [diff] [blame] | 148 | rc = smc_close_active(smc); |
| 149 | sock_set_flag(sk, SOCK_DEAD); |
| 150 | sk->sk_shutdown |= SHUTDOWN_MASK; |
Ursula Braun | b03faa1 | 2019-02-07 15:56:15 +0100 | [diff] [blame] | 151 | } else { |
Dust Li | e5d5aad | 2021-11-10 15:02:34 +0800 | [diff] [blame] | 152 | if (sk->sk_state != SMC_CLOSED) { |
| 153 | if (sk->sk_state != SMC_LISTEN && |
| 154 | sk->sk_state != SMC_INIT) |
| 155 | sock_put(sk); /* passive closing */ |
| 156 | if (sk->sk_state == SMC_LISTEN) { |
| 157 | /* wake up clcsock accept */ |
| 158 | rc = kernel_sock_shutdown(smc->clcsock, |
| 159 | SHUT_RDWR); |
| 160 | } |
| 161 | sk->sk_state = SMC_CLOSED; |
| 162 | sk->sk_state_change(sk); |
Ursula Braun | b03faa1 | 2019-02-07 15:56:15 +0100 | [diff] [blame] | 163 | } |
Ursula Braun | f536dff | 2019-10-23 15:44:05 +0200 | [diff] [blame] | 164 | smc_restore_fallback_changes(smc); |
Ursula Braun | b38d732 | 2017-01-09 16:55:25 +0100 | [diff] [blame] | 165 | } |
Cong Wang | 26d92e9 | 2019-01-05 23:45:26 -0800 | [diff] [blame] | 166 | |
| 167 | sk->sk_prot->unhash(sk); |
| 168 | |
Ursula Braun | b03faa1 | 2019-02-07 15:56:15 +0100 | [diff] [blame] | 169 | if (sk->sk_state == SMC_CLOSED) { |
| 170 | if (smc->clcsock) { |
Karsten Graul | fd57770 | 2019-04-11 11:17:30 +0200 | [diff] [blame] | 171 | release_sock(sk); |
| 172 | smc_clcsock_release(smc); |
| 173 | lock_sock(sk); |
Myungho Jung | 78abe3d | 2018-12-18 09:02:25 -0800 | [diff] [blame] | 174 | } |
Ursula Braun | b03faa1 | 2019-02-07 15:56:15 +0100 | [diff] [blame] | 175 | if (!smc->use_fallback) |
| 176 | smc_conn_free(&smc->conn); |
Ursula Braun | 51f1de7 | 2018-01-26 09:28:48 +0100 | [diff] [blame] | 177 | } |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 178 | |
Ursula Braun | 39f41f3 | 2019-06-27 15:04:52 +0200 | [diff] [blame] | 179 | return rc; |
| 180 | } |
| 181 | |
| 182 | static int smc_release(struct socket *sock) |
| 183 | { |
| 184 | struct sock *sk = sock->sk; |
| 185 | struct smc_sock *smc; |
| 186 | int rc = 0; |
| 187 | |
| 188 | if (!sk) |
| 189 | goto out; |
| 190 | |
Ursula Braun | 81cf4f4 | 2019-10-21 16:13:15 +0200 | [diff] [blame] | 191 | sock_hold(sk); /* sock_put below */ |
Ursula Braun | 39f41f3 | 2019-06-27 15:04:52 +0200 | [diff] [blame] | 192 | smc = smc_sk(sk); |
| 193 | |
| 194 | /* cleanup for a dangling non-blocking connect */ |
| 195 | if (smc->connect_nonblock && sk->sk_state == SMC_INIT) |
| 196 | tcp_abort(smc->clcsock->sk, ECONNABORTED); |
D. Wythe | 5c15b31 | 2021-12-15 20:29:21 +0800 | [diff] [blame] | 197 | |
| 198 | if (cancel_work_sync(&smc->connect_work)) |
| 199 | sock_put(&smc->sk); /* sock_hold in smc_connect for passive closing */ |
Ursula Braun | 39f41f3 | 2019-06-27 15:04:52 +0200 | [diff] [blame] | 200 | |
| 201 | if (sk->sk_state == SMC_LISTEN) |
| 202 | /* smc_close_non_accepted() is called and acquires |
| 203 | * sock lock for child sockets again |
| 204 | */ |
| 205 | lock_sock_nested(sk, SINGLE_DEPTH_NESTING); |
| 206 | else |
| 207 | lock_sock(sk); |
| 208 | |
| 209 | rc = __smc_release(smc); |
| 210 | |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 211 | /* detach socket */ |
| 212 | sock_orphan(sk); |
| 213 | sock->sk = NULL; |
| 214 | release_sock(sk); |
| 215 | |
Ursula Braun | 81cf4f4 | 2019-10-21 16:13:15 +0200 | [diff] [blame] | 216 | sock_put(sk); /* sock_hold above */ |
Ursula Braun | 51f1de7 | 2018-01-26 09:28:48 +0100 | [diff] [blame] | 217 | sock_put(sk); /* final sock_put */ |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 218 | out: |
Ursula Braun | b38d732 | 2017-01-09 16:55:25 +0100 | [diff] [blame] | 219 | return rc; |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 220 | } |
| 221 | |
| 222 | static void smc_destruct(struct sock *sk) |
| 223 | { |
| 224 | if (sk->sk_state != SMC_CLOSED) |
| 225 | return; |
| 226 | if (!sock_flag(sk, SOCK_DEAD)) |
| 227 | return; |
| 228 | |
| 229 | sk_refcnt_debug_dec(sk); |
| 230 | } |
| 231 | |
Karsten Graul | aaa4d33 | 2018-03-16 15:06:41 +0100 | [diff] [blame] | 232 | static struct sock *smc_sock_alloc(struct net *net, struct socket *sock, |
| 233 | int protocol) |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 234 | { |
| 235 | struct smc_sock *smc; |
Karsten Graul | aaa4d33 | 2018-03-16 15:06:41 +0100 | [diff] [blame] | 236 | struct proto *prot; |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 237 | struct sock *sk; |
| 238 | |
Karsten Graul | aaa4d33 | 2018-03-16 15:06:41 +0100 | [diff] [blame] | 239 | prot = (protocol == SMCPROTO_SMC6) ? &smc_proto6 : &smc_proto; |
| 240 | sk = sk_alloc(net, PF_SMC, GFP_KERNEL, prot, 0); |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 241 | if (!sk) |
| 242 | return NULL; |
| 243 | |
| 244 | sock_init_data(sock, sk); /* sets sk_refcnt to 1 */ |
| 245 | sk->sk_state = SMC_INIT; |
| 246 | sk->sk_destruct = smc_destruct; |
Karsten Graul | aaa4d33 | 2018-03-16 15:06:41 +0100 | [diff] [blame] | 247 | sk->sk_protocol = protocol; |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 248 | smc = smc_sk(sk); |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 249 | INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work); |
Ursula Braun | 24ac3a0 | 2018-06-27 17:59:50 +0200 | [diff] [blame] | 250 | INIT_WORK(&smc->connect_work, smc_connect_work); |
Eric Dumazet | be7f3e5 | 2018-05-17 03:54:21 -0700 | [diff] [blame] | 251 | INIT_DELAYED_WORK(&smc->conn.tx_work, smc_tx_work); |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 252 | INIT_LIST_HEAD(&smc->accept_q); |
| 253 | spin_lock_init(&smc->accept_q_lock); |
Eric Dumazet | be7f3e5 | 2018-05-17 03:54:21 -0700 | [diff] [blame] | 254 | spin_lock_init(&smc->conn.send_lock); |
Ursula Braun | f16a7dd | 2017-01-09 16:55:26 +0100 | [diff] [blame] | 255 | sk->sk_prot->hash(sk); |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 256 | sk_refcnt_debug_inc(sk); |
Myungho Jung | 78abe3d | 2018-12-18 09:02:25 -0800 | [diff] [blame] | 257 | mutex_init(&smc->clcsock_release_lock); |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 258 | |
| 259 | return sk; |
| 260 | } |
| 261 | |
| 262 | static int smc_bind(struct socket *sock, struct sockaddr *uaddr, |
| 263 | int addr_len) |
| 264 | { |
| 265 | struct sockaddr_in *addr = (struct sockaddr_in *)uaddr; |
| 266 | struct sock *sk = sock->sk; |
| 267 | struct smc_sock *smc; |
| 268 | int rc; |
| 269 | |
| 270 | smc = smc_sk(sk); |
| 271 | |
| 272 | /* replicate tests from inet_bind(), to be safe wrt. future changes */ |
| 273 | rc = -EINVAL; |
| 274 | if (addr_len < sizeof(struct sockaddr_in)) |
| 275 | goto out; |
| 276 | |
| 277 | rc = -EAFNOSUPPORT; |
Karsten Graul | aaa4d33 | 2018-03-16 15:06:41 +0100 | [diff] [blame] | 278 | if (addr->sin_family != AF_INET && |
| 279 | addr->sin_family != AF_INET6 && |
| 280 | addr->sin_family != AF_UNSPEC) |
| 281 | goto out; |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 282 | /* accept AF_UNSPEC (mapped to AF_INET) only if s_addr is INADDR_ANY */ |
Karsten Graul | aaa4d33 | 2018-03-16 15:06:41 +0100 | [diff] [blame] | 283 | if (addr->sin_family == AF_UNSPEC && |
| 284 | addr->sin_addr.s_addr != htonl(INADDR_ANY)) |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 285 | goto out; |
| 286 | |
| 287 | lock_sock(sk); |
| 288 | |
| 289 | /* Check if socket is already active */ |
| 290 | rc = -EINVAL; |
Ursula Braun | cd20636 | 2019-08-02 10:47:50 +0200 | [diff] [blame] | 291 | if (sk->sk_state != SMC_INIT || smc->connect_nonblock) |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 292 | goto out_rel; |
| 293 | |
| 294 | smc->clcsock->sk->sk_reuse = sk->sk_reuse; |
| 295 | rc = kernel_bind(smc->clcsock, uaddr, addr_len); |
| 296 | |
| 297 | out_rel: |
| 298 | release_sock(sk); |
| 299 | out: |
| 300 | return rc; |
| 301 | } |
| 302 | |
| 303 | static void smc_copy_sock_settings(struct sock *nsk, struct sock *osk, |
| 304 | unsigned long mask) |
| 305 | { |
| 306 | /* options we don't get control via setsockopt for */ |
| 307 | nsk->sk_type = osk->sk_type; |
| 308 | nsk->sk_sndbuf = osk->sk_sndbuf; |
| 309 | nsk->sk_rcvbuf = osk->sk_rcvbuf; |
| 310 | nsk->sk_sndtimeo = osk->sk_sndtimeo; |
| 311 | nsk->sk_rcvtimeo = osk->sk_rcvtimeo; |
| 312 | nsk->sk_mark = osk->sk_mark; |
| 313 | nsk->sk_priority = osk->sk_priority; |
| 314 | nsk->sk_rcvlowat = osk->sk_rcvlowat; |
| 315 | nsk->sk_bound_dev_if = osk->sk_bound_dev_if; |
| 316 | nsk->sk_err = osk->sk_err; |
| 317 | |
| 318 | nsk->sk_flags &= ~mask; |
| 319 | nsk->sk_flags |= osk->sk_flags & mask; |
| 320 | } |
| 321 | |
| 322 | #define SK_FLAGS_SMC_TO_CLC ((1UL << SOCK_URGINLINE) | \ |
| 323 | (1UL << SOCK_KEEPOPEN) | \ |
| 324 | (1UL << SOCK_LINGER) | \ |
| 325 | (1UL << SOCK_BROADCAST) | \ |
| 326 | (1UL << SOCK_TIMESTAMP) | \ |
| 327 | (1UL << SOCK_DBG) | \ |
| 328 | (1UL << SOCK_RCVTSTAMP) | \ |
| 329 | (1UL << SOCK_RCVTSTAMPNS) | \ |
| 330 | (1UL << SOCK_LOCALROUTE) | \ |
| 331 | (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \ |
| 332 | (1UL << SOCK_RXQ_OVFL) | \ |
| 333 | (1UL << SOCK_WIFI_STATUS) | \ |
| 334 | (1UL << SOCK_NOFCS) | \ |
Deepa Dinamani | 9718475 | 2019-02-02 07:34:51 -0800 | [diff] [blame] | 335 | (1UL << SOCK_FILTER_LOCKED) | \ |
| 336 | (1UL << SOCK_TSTAMP_NEW)) |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 337 | /* copy only relevant settings and flags of SOL_SOCKET level from smc to |
| 338 | * clc socket (since smc is not called for these options from net/core) |
| 339 | */ |
| 340 | static void smc_copy_sock_settings_to_clc(struct smc_sock *smc) |
| 341 | { |
| 342 | smc_copy_sock_settings(smc->clcsock->sk, &smc->sk, SK_FLAGS_SMC_TO_CLC); |
| 343 | } |
| 344 | |
| 345 | #define SK_FLAGS_CLC_TO_SMC ((1UL << SOCK_URGINLINE) | \ |
| 346 | (1UL << SOCK_KEEPOPEN) | \ |
| 347 | (1UL << SOCK_LINGER) | \ |
| 348 | (1UL << SOCK_DBG)) |
| 349 | /* copy only settings and flags relevant for smc from clc to smc socket */ |
| 350 | static void smc_copy_sock_settings_to_smc(struct smc_sock *smc) |
| 351 | { |
| 352 | smc_copy_sock_settings(&smc->sk, smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC); |
| 353 | } |
| 354 | |
Karsten Graul | b924754 | 2020-04-29 17:10:41 +0200 | [diff] [blame] | 355 | /* register the new rmb on all links */ |
Karsten Graul | 7562a13 | 2020-05-01 12:48:01 +0200 | [diff] [blame] | 356 | static int smcr_lgr_reg_rmbs(struct smc_link *link, |
Karsten Graul | b924754 | 2020-04-29 17:10:41 +0200 | [diff] [blame] | 357 | struct smc_buf_desc *rmb_desc) |
| 358 | { |
Karsten Graul | 7562a13 | 2020-05-01 12:48:01 +0200 | [diff] [blame] | 359 | struct smc_link_group *lgr = link->lgr; |
| 360 | int i, rc = 0; |
Karsten Graul | b924754 | 2020-04-29 17:10:41 +0200 | [diff] [blame] | 361 | |
Karsten Graul | d550066 | 2020-05-01 12:48:05 +0200 | [diff] [blame] | 362 | rc = smc_llc_flow_initiate(lgr, SMC_LLC_FLOW_RKEY); |
| 363 | if (rc) |
| 364 | return rc; |
| 365 | /* protect against parallel smc_llc_cli_rkey_exchange() and |
| 366 | * parallel smcr_link_reg_rmb() |
| 367 | */ |
| 368 | mutex_lock(&lgr->llc_conf_mutex); |
Karsten Graul | b924754 | 2020-04-29 17:10:41 +0200 | [diff] [blame] | 369 | for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { |
Karsten Graul | 741a49a | 2020-07-18 15:06:16 +0200 | [diff] [blame] | 370 | if (!smc_link_active(&lgr->lnk[i])) |
Karsten Graul | b924754 | 2020-04-29 17:10:41 +0200 | [diff] [blame] | 371 | continue; |
Karsten Graul | 7562a13 | 2020-05-01 12:48:01 +0200 | [diff] [blame] | 372 | rc = smcr_link_reg_rmb(&lgr->lnk[i], rmb_desc); |
Karsten Graul | b924754 | 2020-04-29 17:10:41 +0200 | [diff] [blame] | 373 | if (rc) |
Karsten Graul | 7562a13 | 2020-05-01 12:48:01 +0200 | [diff] [blame] | 374 | goto out; |
Karsten Graul | b924754 | 2020-04-29 17:10:41 +0200 | [diff] [blame] | 375 | } |
Karsten Graul | 7562a13 | 2020-05-01 12:48:01 +0200 | [diff] [blame] | 376 | |
| 377 | /* exchange confirm_rkey msg with peer */ |
| 378 | rc = smc_llc_do_confirm_rkey(link, rmb_desc); |
| 379 | if (rc) { |
| 380 | rc = -EFAULT; |
| 381 | goto out; |
| 382 | } |
| 383 | rmb_desc->is_conf_rkey = true; |
| 384 | out: |
Karsten Graul | d550066 | 2020-05-01 12:48:05 +0200 | [diff] [blame] | 385 | mutex_unlock(&lgr->llc_conf_mutex); |
| 386 | smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); |
Karsten Graul | 7562a13 | 2020-05-01 12:48:01 +0200 | [diff] [blame] | 387 | return rc; |
Karsten Graul | b924754 | 2020-04-29 17:10:41 +0200 | [diff] [blame] | 388 | } |
| 389 | |
| 390 | static int smcr_clnt_conf_first_link(struct smc_sock *smc) |
Ursula Braun | 9bf9abe | 2017-01-09 16:55:21 +0100 | [diff] [blame] | 391 | { |
Karsten Graul | 387707f | 2020-04-29 17:10:40 +0200 | [diff] [blame] | 392 | struct smc_link *link = smc->conn.lnk; |
Karsten Graul | 0fb0b02 | 2020-04-30 15:55:43 +0200 | [diff] [blame] | 393 | struct smc_llc_qentry *qentry; |
Ursula Braun | 9bf9abe | 2017-01-09 16:55:21 +0100 | [diff] [blame] | 394 | int rc; |
| 395 | |
Ursula Braun | 9bf9abe | 2017-01-09 16:55:21 +0100 | [diff] [blame] | 396 | /* receive CONFIRM LINK request from server over RoCE fabric */ |
Karsten Graul | 0fb0b02 | 2020-04-30 15:55:43 +0200 | [diff] [blame] | 397 | qentry = smc_llc_wait(link->lgr, NULL, SMC_LLC_WAIT_TIME, |
| 398 | SMC_LLC_CONFIRM_LINK); |
| 399 | if (!qentry) { |
Ursula Braun | 9bf9abe | 2017-01-09 16:55:21 +0100 | [diff] [blame] | 400 | struct smc_clc_msg_decline dclc; |
| 401 | |
| 402 | rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc), |
Ursula Braun | 2b59f58 | 2018-11-22 10:26:39 +0100 | [diff] [blame] | 403 | SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT); |
Ursula Braun | 9ed2855 | 2018-11-22 10:26:37 +0100 | [diff] [blame] | 404 | return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc; |
Ursula Braun | 9bf9abe | 2017-01-09 16:55:21 +0100 | [diff] [blame] | 405 | } |
Karsten Graul | 649758f | 2020-05-04 14:18:48 +0200 | [diff] [blame] | 406 | smc_llc_save_peer_uid(qentry); |
Karsten Graul | 0fb0b02 | 2020-04-30 15:55:43 +0200 | [diff] [blame] | 407 | rc = smc_llc_eval_conf_link(qentry, SMC_LLC_REQ); |
| 408 | smc_llc_flow_qentry_del(&link->lgr->llc_flow_lcl); |
| 409 | if (rc) |
Karsten Graul | 75d320d | 2018-03-01 13:51:31 +0100 | [diff] [blame] | 410 | return SMC_CLC_DECL_RMBE_EC; |
| 411 | |
Ursula Braun | 9bf9abe | 2017-01-09 16:55:21 +0100 | [diff] [blame] | 412 | rc = smc_ib_modify_qp_rts(link); |
| 413 | if (rc) |
Karsten Graul | 603cc14 | 2018-07-25 16:35:32 +0200 | [diff] [blame] | 414 | return SMC_CLC_DECL_ERR_RDYLNK; |
Ursula Braun | 9bf9abe | 2017-01-09 16:55:21 +0100 | [diff] [blame] | 415 | |
| 416 | smc_wr_remember_qp_attr(link); |
Ursula Braun | 652a1e4 | 2017-07-28 13:56:17 +0200 | [diff] [blame] | 417 | |
Karsten Graul | 7562a13 | 2020-05-01 12:48:01 +0200 | [diff] [blame] | 418 | if (smcr_link_reg_rmb(link, smc->conn.rmb_desc)) |
Karsten Graul | 603cc14 | 2018-07-25 16:35:32 +0200 | [diff] [blame] | 419 | return SMC_CLC_DECL_ERR_REGRMB; |
Ursula Braun | 652a1e4 | 2017-07-28 13:56:17 +0200 | [diff] [blame] | 420 | |
Karsten Graul | 0fb0b02 | 2020-04-30 15:55:43 +0200 | [diff] [blame] | 421 | /* confirm_rkey is implicit on 1st contact */ |
| 422 | smc->conn.rmb_desc->is_conf_rkey = true; |
| 423 | |
Ursula Braun | 9bf9abe | 2017-01-09 16:55:21 +0100 | [diff] [blame] | 424 | /* send CONFIRM LINK response over RoCE fabric */ |
Ursula Braun | 947541f | 2018-07-25 16:35:30 +0200 | [diff] [blame] | 425 | rc = smc_llc_send_confirm_link(link, SMC_LLC_RESP); |
Ursula Braun | 9bf9abe | 2017-01-09 16:55:21 +0100 | [diff] [blame] | 426 | if (rc < 0) |
Karsten Graul | 603cc14 | 2018-07-25 16:35:32 +0200 | [diff] [blame] | 427 | return SMC_CLC_DECL_TIMEOUT_CL; |
Ursula Braun | 9bf9abe | 2017-01-09 16:55:21 +0100 | [diff] [blame] | 428 | |
Karsten Graul | 0fb0b02 | 2020-04-30 15:55:43 +0200 | [diff] [blame] | 429 | smc_llc_link_active(link); |
Karsten Graul | 0a99be4 | 2020-05-05 15:01:20 +0200 | [diff] [blame] | 430 | smcr_lgr_set_type(link->lgr, SMC_LGR_SINGLE); |
Karsten Graul | 0fb0b02 | 2020-04-30 15:55:43 +0200 | [diff] [blame] | 431 | |
| 432 | /* optional 2nd link, receive ADD LINK request from server */ |
| 433 | qentry = smc_llc_wait(link->lgr, NULL, SMC_LLC_WAIT_TIME, |
| 434 | SMC_LLC_ADD_LINK); |
| 435 | if (!qentry) { |
Karsten Graul | 52bedf3 | 2018-03-01 13:51:32 +0100 | [diff] [blame] | 436 | struct smc_clc_msg_decline dclc; |
| 437 | |
| 438 | rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc), |
Ursula Braun | 2b59f58 | 2018-11-22 10:26:39 +0100 | [diff] [blame] | 439 | SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT); |
Karsten Graul | 0fb0b02 | 2020-04-30 15:55:43 +0200 | [diff] [blame] | 440 | if (rc == -EAGAIN) |
| 441 | rc = 0; /* no DECLINE received, go with one link */ |
| 442 | return rc; |
Karsten Graul | 52bedf3 | 2018-03-01 13:51:32 +0100 | [diff] [blame] | 443 | } |
Karsten Graul | 0fb0b02 | 2020-04-30 15:55:43 +0200 | [diff] [blame] | 444 | smc_llc_flow_qentry_clr(&link->lgr->llc_flow_lcl); |
Karsten Graul | b1570a8 | 2020-05-03 14:38:42 +0200 | [diff] [blame] | 445 | smc_llc_cli_add_link(link, qentry); |
Karsten Graul | 75d320d | 2018-03-01 13:51:31 +0100 | [diff] [blame] | 446 | return 0; |
Ursula Braun | 9bf9abe | 2017-01-09 16:55:21 +0100 | [diff] [blame] | 447 | } |
| 448 | |
Karsten Graul | e5c4744 | 2021-10-16 11:37:45 +0200 | [diff] [blame] | 449 | static bool smc_isascii(char *hostname) |
| 450 | { |
| 451 | int i; |
| 452 | |
| 453 | for (i = 0; i < SMC_MAX_HOSTNAME_LEN; i++) |
| 454 | if (!isascii(hostname[i])) |
| 455 | return false; |
| 456 | return true; |
| 457 | } |
| 458 | |
| 459 | static void smc_conn_save_peer_info_fce(struct smc_sock *smc, |
| 460 | struct smc_clc_msg_accept_confirm *clc) |
| 461 | { |
| 462 | struct smc_clc_msg_accept_confirm_v2 *clc_v2 = |
| 463 | (struct smc_clc_msg_accept_confirm_v2 *)clc; |
| 464 | struct smc_clc_first_contact_ext *fce; |
| 465 | int clc_v2_len; |
| 466 | |
| 467 | if (clc->hdr.version == SMC_V1 || |
| 468 | !(clc->hdr.typev2 & SMC_FIRST_CONTACT_MASK)) |
| 469 | return; |
| 470 | |
| 471 | if (smc->conn.lgr->is_smcd) { |
| 472 | memcpy(smc->conn.lgr->negotiated_eid, clc_v2->d1.eid, |
| 473 | SMC_MAX_EID_LEN); |
| 474 | clc_v2_len = offsetofend(struct smc_clc_msg_accept_confirm_v2, |
| 475 | d1); |
| 476 | } else { |
| 477 | memcpy(smc->conn.lgr->negotiated_eid, clc_v2->r1.eid, |
| 478 | SMC_MAX_EID_LEN); |
| 479 | clc_v2_len = offsetofend(struct smc_clc_msg_accept_confirm_v2, |
| 480 | r1); |
| 481 | } |
| 482 | fce = (struct smc_clc_first_contact_ext *)(((u8 *)clc_v2) + clc_v2_len); |
| 483 | smc->conn.lgr->peer_os = fce->os_type; |
| 484 | smc->conn.lgr->peer_smc_release = fce->release; |
| 485 | if (smc_isascii(fce->hostname)) |
| 486 | memcpy(smc->conn.lgr->peer_hostname, fce->hostname, |
| 487 | SMC_MAX_HOSTNAME_LEN); |
| 488 | } |
| 489 | |
Hans Wippel | 4134984 | 2018-06-28 19:05:11 +0200 | [diff] [blame] | 490 | static void smcr_conn_save_peer_info(struct smc_sock *smc, |
| 491 | struct smc_clc_msg_accept_confirm *clc) |
Ursula Braun | 0cfdd8f | 2017-01-09 16:55:17 +0100 | [diff] [blame] | 492 | { |
Ursula Braun | 3d9725a | 2020-09-10 18:48:23 +0200 | [diff] [blame] | 493 | int bufsize = smc_uncompress_bufsize(clc->r0.rmbe_size); |
Hans Wippel | 95d8d263 | 2018-05-18 09:34:13 +0200 | [diff] [blame] | 494 | |
Ursula Braun | 3d9725a | 2020-09-10 18:48:23 +0200 | [diff] [blame] | 495 | smc->conn.peer_rmbe_idx = clc->r0.rmbe_idx; |
| 496 | smc->conn.local_tx_ctrl.token = ntohl(clc->r0.rmbe_alert_token); |
Hans Wippel | 95d8d263 | 2018-05-18 09:34:13 +0200 | [diff] [blame] | 497 | smc->conn.peer_rmbe_size = bufsize; |
Ursula Braun | cd6851f | 2017-01-09 16:55:18 +0100 | [diff] [blame] | 498 | atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size); |
Hans Wippel | 95d8d263 | 2018-05-18 09:34:13 +0200 | [diff] [blame] | 499 | smc->conn.tx_off = bufsize * (smc->conn.peer_rmbe_idx - 1); |
Ursula Braun | 0cfdd8f | 2017-01-09 16:55:17 +0100 | [diff] [blame] | 500 | } |
| 501 | |
Hans Wippel | 4134984 | 2018-06-28 19:05:11 +0200 | [diff] [blame] | 502 | static void smcd_conn_save_peer_info(struct smc_sock *smc, |
| 503 | struct smc_clc_msg_accept_confirm *clc) |
| 504 | { |
Ursula Braun | 3d9725a | 2020-09-10 18:48:23 +0200 | [diff] [blame] | 505 | int bufsize = smc_uncompress_bufsize(clc->d0.dmbe_size); |
Hans Wippel | 4134984 | 2018-06-28 19:05:11 +0200 | [diff] [blame] | 506 | |
Ursula Braun | 3d9725a | 2020-09-10 18:48:23 +0200 | [diff] [blame] | 507 | smc->conn.peer_rmbe_idx = clc->d0.dmbe_idx; |
| 508 | smc->conn.peer_token = clc->d0.token; |
Hans Wippel | 4134984 | 2018-06-28 19:05:11 +0200 | [diff] [blame] | 509 | /* msg header takes up space in the buffer */ |
| 510 | smc->conn.peer_rmbe_size = bufsize - sizeof(struct smcd_cdc_msg); |
| 511 | atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size); |
| 512 | smc->conn.tx_off = bufsize * smc->conn.peer_rmbe_idx; |
| 513 | } |
| 514 | |
| 515 | static void smc_conn_save_peer_info(struct smc_sock *smc, |
| 516 | struct smc_clc_msg_accept_confirm *clc) |
| 517 | { |
| 518 | if (smc->conn.lgr->is_smcd) |
| 519 | smcd_conn_save_peer_info(smc, clc); |
| 520 | else |
| 521 | smcr_conn_save_peer_info(smc, clc); |
Karsten Graul | e5c4744 | 2021-10-16 11:37:45 +0200 | [diff] [blame] | 522 | smc_conn_save_peer_info_fce(smc, clc); |
Hans Wippel | 4134984 | 2018-06-28 19:05:11 +0200 | [diff] [blame] | 523 | } |
| 524 | |
Ursula Braun | 0cfdd8f | 2017-01-09 16:55:17 +0100 | [diff] [blame] | 525 | static void smc_link_save_peer_info(struct smc_link *link, |
Karsten Graul | e5c4744 | 2021-10-16 11:37:45 +0200 | [diff] [blame] | 526 | struct smc_clc_msg_accept_confirm *clc, |
| 527 | struct smc_init_info *ini) |
Ursula Braun | 0cfdd8f | 2017-01-09 16:55:17 +0100 | [diff] [blame] | 528 | { |
Ursula Braun | 3d9725a | 2020-09-10 18:48:23 +0200 | [diff] [blame] | 529 | link->peer_qpn = ntoh24(clc->r0.qpn); |
Karsten Graul | e5c4744 | 2021-10-16 11:37:45 +0200 | [diff] [blame] | 530 | memcpy(link->peer_gid, ini->peer_gid, SMC_GID_SIZE); |
| 531 | memcpy(link->peer_mac, ini->peer_mac, sizeof(link->peer_mac)); |
Ursula Braun | 3d9725a | 2020-09-10 18:48:23 +0200 | [diff] [blame] | 532 | link->peer_psn = ntoh24(clc->r0.psn); |
| 533 | link->peer_mtu = clc->r0.qp_mtu; |
Ursula Braun | 0cfdd8f | 2017-01-09 16:55:17 +0100 | [diff] [blame] | 534 | } |
| 535 | |
Guvenc Gulce | e0e4b8f | 2021-06-16 16:52:55 +0200 | [diff] [blame] | 536 | static void smc_stat_inc_fback_rsn_cnt(struct smc_sock *smc, |
| 537 | struct smc_stats_fback *fback_arr) |
| 538 | { |
| 539 | int cnt; |
| 540 | |
| 541 | for (cnt = 0; cnt < SMC_MAX_FBACK_RSN_CNT; cnt++) { |
| 542 | if (fback_arr[cnt].fback_code == smc->fallback_rsn) { |
| 543 | fback_arr[cnt].count++; |
| 544 | break; |
| 545 | } |
| 546 | if (!fback_arr[cnt].fback_code) { |
| 547 | fback_arr[cnt].fback_code = smc->fallback_rsn; |
| 548 | fback_arr[cnt].count++; |
| 549 | break; |
| 550 | } |
| 551 | } |
| 552 | } |
| 553 | |
| 554 | static void smc_stat_fallback(struct smc_sock *smc) |
| 555 | { |
Guvenc Gulce | 194730a | 2021-06-16 16:52:58 +0200 | [diff] [blame] | 556 | struct net *net = sock_net(&smc->sk); |
| 557 | |
| 558 | mutex_lock(&net->smc.mutex_fback_rsn); |
Guvenc Gulce | e0e4b8f | 2021-06-16 16:52:55 +0200 | [diff] [blame] | 559 | if (smc->listen_smc) { |
Guvenc Gulce | 194730a | 2021-06-16 16:52:58 +0200 | [diff] [blame] | 560 | smc_stat_inc_fback_rsn_cnt(smc, net->smc.fback_rsn->srv); |
| 561 | net->smc.fback_rsn->srv_fback_cnt++; |
Guvenc Gulce | e0e4b8f | 2021-06-16 16:52:55 +0200 | [diff] [blame] | 562 | } else { |
Guvenc Gulce | 194730a | 2021-06-16 16:52:58 +0200 | [diff] [blame] | 563 | smc_stat_inc_fback_rsn_cnt(smc, net->smc.fback_rsn->clnt); |
| 564 | net->smc.fback_rsn->clnt_fback_cnt++; |
Guvenc Gulce | e0e4b8f | 2021-06-16 16:52:55 +0200 | [diff] [blame] | 565 | } |
Guvenc Gulce | 194730a | 2021-06-16 16:52:58 +0200 | [diff] [blame] | 566 | mutex_unlock(&net->smc.mutex_fback_rsn); |
Guvenc Gulce | e0e4b8f | 2021-06-16 16:52:55 +0200 | [diff] [blame] | 567 | } |
| 568 | |
| 569 | static void smc_switch_to_fallback(struct smc_sock *smc, int reason_code) |
Ursula Braun | 07603b2 | 2019-04-11 11:17:32 +0200 | [diff] [blame] | 570 | { |
Wen Gu | 2153bd1 | 2021-11-13 15:33:35 +0800 | [diff] [blame] | 571 | wait_queue_head_t *smc_wait = sk_sleep(&smc->sk); |
| 572 | wait_queue_head_t *clc_wait = sk_sleep(smc->clcsock->sk); |
| 573 | unsigned long flags; |
| 574 | |
Ursula Braun | 07603b2 | 2019-04-11 11:17:32 +0200 | [diff] [blame] | 575 | smc->use_fallback = true; |
Guvenc Gulce | e0e4b8f | 2021-06-16 16:52:55 +0200 | [diff] [blame] | 576 | smc->fallback_rsn = reason_code; |
| 577 | smc_stat_fallback(smc); |
Tony Lu | 4826260 | 2021-11-01 15:39:12 +0800 | [diff] [blame] | 578 | trace_smc_switch_to_fallback(smc, reason_code); |
Ursula Braun | 07603b2 | 2019-04-11 11:17:32 +0200 | [diff] [blame] | 579 | if (smc->sk.sk_socket && smc->sk.sk_socket->file) { |
| 580 | smc->clcsock->file = smc->sk.sk_socket->file; |
| 581 | smc->clcsock->file->private_data = smc->clcsock; |
Ursula Braun | 67f562e | 2020-02-14 08:58:59 +0100 | [diff] [blame] | 582 | smc->clcsock->wq.fasync_list = |
| 583 | smc->sk.sk_socket->wq.fasync_list; |
Wen Gu | 2153bd1 | 2021-11-13 15:33:35 +0800 | [diff] [blame] | 584 | |
| 585 | /* There may be some entries remaining in |
| 586 | * smc socket->wq, which should be removed |
| 587 | * to clcsocket->wq during the fallback. |
| 588 | */ |
| 589 | spin_lock_irqsave(&smc_wait->lock, flags); |
Wen Gu | 7a61432 | 2021-11-22 20:32:53 +0800 | [diff] [blame] | 590 | spin_lock_nested(&clc_wait->lock, SINGLE_DEPTH_NESTING); |
Wen Gu | 2153bd1 | 2021-11-13 15:33:35 +0800 | [diff] [blame] | 591 | list_splice_init(&smc_wait->head, &clc_wait->head); |
| 592 | spin_unlock(&clc_wait->lock); |
| 593 | spin_unlock_irqrestore(&smc_wait->lock, flags); |
Ursula Braun | 07603b2 | 2019-04-11 11:17:32 +0200 | [diff] [blame] | 594 | } |
| 595 | } |
| 596 | |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 597 | /* fall back during connect */ |
Karsten Graul | 603cc14 | 2018-07-25 16:35:32 +0200 | [diff] [blame] | 598 | static int smc_connect_fallback(struct smc_sock *smc, int reason_code) |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 599 | { |
Guvenc Gulce | e0e4b8f | 2021-06-16 16:52:55 +0200 | [diff] [blame] | 600 | smc_switch_to_fallback(smc, reason_code); |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 601 | smc_copy_sock_settings_to_clc(smc); |
Ursula Braun | 50717a3 | 2019-04-12 12:57:23 +0200 | [diff] [blame] | 602 | smc->connect_nonblock = 0; |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 603 | if (smc->sk.sk_state == SMC_INIT) |
| 604 | smc->sk.sk_state = SMC_ACTIVE; |
| 605 | return 0; |
| 606 | } |
| 607 | |
| 608 | /* decline and fall back during connect */ |
Ursula Braun | e8d726c8 | 2020-09-26 12:44:32 +0200 | [diff] [blame] | 609 | static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code, |
| 610 | u8 version) |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 611 | { |
Guvenc Gulce | 194730a | 2021-06-16 16:52:58 +0200 | [diff] [blame] | 612 | struct net *net = sock_net(&smc->sk); |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 613 | int rc; |
| 614 | |
Ursula Braun | e1bbdd5 | 2018-07-05 16:15:30 +0200 | [diff] [blame] | 615 | if (reason_code < 0) { /* error, fallback is not possible */ |
Guvenc Gulce | 194730a | 2021-06-16 16:52:58 +0200 | [diff] [blame] | 616 | this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt); |
Ursula Braun | e1bbdd5 | 2018-07-05 16:15:30 +0200 | [diff] [blame] | 617 | if (smc->sk.sk_state == SMC_INIT) |
| 618 | sock_put(&smc->sk); /* passive closing */ |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 619 | return reason_code; |
Ursula Braun | e1bbdd5 | 2018-07-05 16:15:30 +0200 | [diff] [blame] | 620 | } |
Karsten Graul | 603cc14 | 2018-07-25 16:35:32 +0200 | [diff] [blame] | 621 | if (reason_code != SMC_CLC_DECL_PEERDECL) { |
Ursula Braun | e8d726c8 | 2020-09-26 12:44:32 +0200 | [diff] [blame] | 622 | rc = smc_clc_send_decline(smc, reason_code, version); |
Ursula Braun | e1bbdd5 | 2018-07-05 16:15:30 +0200 | [diff] [blame] | 623 | if (rc < 0) { |
Guvenc Gulce | 194730a | 2021-06-16 16:52:58 +0200 | [diff] [blame] | 624 | this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt); |
Ursula Braun | e1bbdd5 | 2018-07-05 16:15:30 +0200 | [diff] [blame] | 625 | if (smc->sk.sk_state == SMC_INIT) |
| 626 | sock_put(&smc->sk); /* passive closing */ |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 627 | return rc; |
Ursula Braun | e1bbdd5 | 2018-07-05 16:15:30 +0200 | [diff] [blame] | 628 | } |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 629 | } |
Karsten Graul | 603cc14 | 2018-07-25 16:35:32 +0200 | [diff] [blame] | 630 | return smc_connect_fallback(smc, reason_code); |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 631 | } |
| 632 | |
Karsten Graul | 8cf3f3e | 2020-12-01 20:20:36 +0100 | [diff] [blame] | 633 | static void smc_conn_abort(struct smc_sock *smc, int local_first) |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 634 | { |
Ursula Braun | 5ac54d8 | 2020-09-10 18:48:21 +0200 | [diff] [blame] | 635 | if (local_first) |
Ursula Braun | 51e3dfa | 2020-02-25 16:34:36 +0100 | [diff] [blame] | 636 | smc_lgr_cleanup_early(&smc->conn); |
| 637 | else |
| 638 | smc_conn_free(&smc->conn); |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 639 | } |
| 640 | |
| 641 | /* check if there is a rdma device available for this connection. */ |
| 642 | /* called for connect and listen */ |
Karsten Graul | 228bae0 | 2019-04-12 12:57:28 +0200 | [diff] [blame] | 643 | static int smc_find_rdma_device(struct smc_sock *smc, struct smc_init_info *ini) |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 644 | { |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 645 | /* PNET table look up: search active ib_device and port |
| 646 | * within same PNETID that also contains the ethernet device |
| 647 | * used for the internal TCP socket |
| 648 | */ |
Karsten Graul | bc36d2f | 2019-04-12 12:57:26 +0200 | [diff] [blame] | 649 | smc_pnet_find_roce_resource(smc->clcsock->sk, ini); |
Karsten Graul | 42042db | 2021-10-16 11:37:44 +0200 | [diff] [blame] | 650 | if (!ini->check_smcrv2 && !ini->ib_dev) |
| 651 | return SMC_CLC_DECL_NOSMCRDEV; |
| 652 | if (ini->check_smcrv2 && !ini->smcrv2.ib_dev_v2) |
Karsten Graul | 9aa68d2 | 2019-04-12 12:57:29 +0200 | [diff] [blame] | 653 | return SMC_CLC_DECL_NOSMCRDEV; |
Karsten Graul | bc36d2f | 2019-04-12 12:57:26 +0200 | [diff] [blame] | 654 | return 0; |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 655 | } |
| 656 | |
Hans Wippel | 4134984 | 2018-06-28 19:05:11 +0200 | [diff] [blame] | 657 | /* check if there is an ISM device available for this connection. */ |
| 658 | /* called for connect and listen */ |
Karsten Graul | 228bae0 | 2019-04-12 12:57:28 +0200 | [diff] [blame] | 659 | static int smc_find_ism_device(struct smc_sock *smc, struct smc_init_info *ini) |
Hans Wippel | 4134984 | 2018-06-28 19:05:11 +0200 | [diff] [blame] | 660 | { |
| 661 | /* Find ISM device with same PNETID as connecting interface */ |
Karsten Graul | bc36d2f | 2019-04-12 12:57:26 +0200 | [diff] [blame] | 662 | smc_pnet_find_ism_resource(smc->clcsock->sk, ini); |
Ursula Braun | 3fc6493 | 2020-09-26 12:44:23 +0200 | [diff] [blame] | 663 | if (!ini->ism_dev[0]) |
Karsten Graul | 9aa68d2 | 2019-04-12 12:57:29 +0200 | [diff] [blame] | 664 | return SMC_CLC_DECL_NOSMCDDEV; |
Ursula Braun | 8caaccf | 2020-09-26 12:44:25 +0200 | [diff] [blame] | 665 | else |
| 666 | ini->ism_chid[0] = smc_ism_get_chid(ini->ism_dev[0]); |
Hans Wippel | 4134984 | 2018-06-28 19:05:11 +0200 | [diff] [blame] | 667 | return 0; |
| 668 | } |
| 669 | |
Karsten Graul | 839d696 | 2020-10-02 17:09:26 +0200 | [diff] [blame] | 670 | /* is chid unique for the ism devices that are already determined? */ |
| 671 | static bool smc_find_ism_v2_is_unique_chid(u16 chid, struct smc_init_info *ini, |
| 672 | int cnt) |
| 673 | { |
| 674 | int i = (!ini->ism_dev[0]) ? 1 : 0; |
| 675 | |
| 676 | for (; i < cnt; i++) |
| 677 | if (ini->ism_chid[i] == chid) |
| 678 | return false; |
| 679 | return true; |
| 680 | } |
| 681 | |
Ursula Braun | d70bf4f | 2020-09-26 12:44:27 +0200 | [diff] [blame] | 682 | /* determine possible V2 ISM devices (either without PNETID or with PNETID plus |
| 683 | * PNETID matching net_device) |
| 684 | */ |
| 685 | static int smc_find_ism_v2_device_clnt(struct smc_sock *smc, |
| 686 | struct smc_init_info *ini) |
| 687 | { |
| 688 | int rc = SMC_CLC_DECL_NOSMCDDEV; |
| 689 | struct smcd_dev *smcd; |
| 690 | int i = 1; |
Karsten Graul | 839d696 | 2020-10-02 17:09:26 +0200 | [diff] [blame] | 691 | u16 chid; |
Ursula Braun | d70bf4f | 2020-09-26 12:44:27 +0200 | [diff] [blame] | 692 | |
| 693 | if (smcd_indicated(ini->smc_type_v1)) |
| 694 | rc = 0; /* already initialized for V1 */ |
| 695 | mutex_lock(&smcd_dev_list.mutex); |
| 696 | list_for_each_entry(smcd, &smcd_dev_list.list, list) { |
| 697 | if (smcd->going_away || smcd == ini->ism_dev[0]) |
| 698 | continue; |
Karsten Graul | 839d696 | 2020-10-02 17:09:26 +0200 | [diff] [blame] | 699 | chid = smc_ism_get_chid(smcd); |
| 700 | if (!smc_find_ism_v2_is_unique_chid(chid, ini, i)) |
| 701 | continue; |
Ursula Braun | d70bf4f | 2020-09-26 12:44:27 +0200 | [diff] [blame] | 702 | if (!smc_pnet_is_pnetid_set(smcd->pnetid) || |
| 703 | smc_pnet_is_ndev_pnetid(sock_net(&smc->sk), smcd->pnetid)) { |
| 704 | ini->ism_dev[i] = smcd; |
Karsten Graul | 839d696 | 2020-10-02 17:09:26 +0200 | [diff] [blame] | 705 | ini->ism_chid[i] = chid; |
Ursula Braun | d70bf4f | 2020-09-26 12:44:27 +0200 | [diff] [blame] | 706 | ini->is_smcd = true; |
| 707 | rc = 0; |
| 708 | i++; |
| 709 | if (i > SMC_MAX_ISM_DEVS) |
| 710 | break; |
| 711 | } |
| 712 | } |
| 713 | mutex_unlock(&smcd_dev_list.mutex); |
| 714 | ini->ism_offered_cnt = i - 1; |
| 715 | if (!ini->ism_dev[0] && !ini->ism_dev[1]) |
| 716 | ini->smcd_version = 0; |
| 717 | |
| 718 | return rc; |
| 719 | } |
| 720 | |
Hans Wippel | 4134984 | 2018-06-28 19:05:11 +0200 | [diff] [blame] | 721 | /* Check for VLAN ID and register it on ISM device just for CLC handshake */ |
| 722 | static int smc_connect_ism_vlan_setup(struct smc_sock *smc, |
Karsten Graul | bc36d2f | 2019-04-12 12:57:26 +0200 | [diff] [blame] | 723 | struct smc_init_info *ini) |
Hans Wippel | 4134984 | 2018-06-28 19:05:11 +0200 | [diff] [blame] | 724 | { |
Ursula Braun | 3fc6493 | 2020-09-26 12:44:23 +0200 | [diff] [blame] | 725 | if (ini->vlan_id && smc_ism_get_vlan(ini->ism_dev[0], ini->vlan_id)) |
Karsten Graul | 7a62725a | 2019-04-12 12:57:30 +0200 | [diff] [blame] | 726 | return SMC_CLC_DECL_ISMVLANERR; |
Hans Wippel | 4134984 | 2018-06-28 19:05:11 +0200 | [diff] [blame] | 727 | return 0; |
| 728 | } |
| 729 | |
Ursula Braun | d70bf4f | 2020-09-26 12:44:27 +0200 | [diff] [blame] | 730 | static int smc_find_proposal_devices(struct smc_sock *smc, |
| 731 | struct smc_init_info *ini) |
| 732 | { |
| 733 | int rc = 0; |
| 734 | |
| 735 | /* check if there is an ism device available */ |
Karsten Graul | 42042db | 2021-10-16 11:37:44 +0200 | [diff] [blame] | 736 | if (!(ini->smcd_version & SMC_V1) || |
| 737 | smc_find_ism_device(smc, ini) || |
| 738 | smc_connect_ism_vlan_setup(smc, ini)) |
| 739 | ini->smcd_version &= ~SMC_V1; |
| 740 | /* else ISM V1 is supported for this connection */ |
| 741 | |
| 742 | /* check if there is an rdma device available */ |
| 743 | if (!(ini->smcr_version & SMC_V1) || |
| 744 | smc_find_rdma_device(smc, ini)) |
| 745 | ini->smcr_version &= ~SMC_V1; |
| 746 | /* else RDMA is supported for this connection */ |
| 747 | |
| 748 | ini->smc_type_v1 = smc_indicated_type(ini->smcd_version & SMC_V1, |
| 749 | ini->smcr_version & SMC_V1); |
| 750 | |
| 751 | /* check if there is an ism v2 device available */ |
| 752 | if (!(ini->smcd_version & SMC_V2) || |
| 753 | !smc_ism_is_v2_capable() || |
| 754 | smc_find_ism_v2_device_clnt(smc, ini)) |
| 755 | ini->smcd_version &= ~SMC_V2; |
| 756 | |
| 757 | /* check if there is an rdma v2 device available */ |
| 758 | ini->check_smcrv2 = true; |
| 759 | ini->smcrv2.saddr = smc->clcsock->sk->sk_rcv_saddr; |
| 760 | if (!(ini->smcr_version & SMC_V2) || |
| 761 | smc->clcsock->sk->sk_family != AF_INET || |
| 762 | !smc_clc_ueid_count() || |
| 763 | smc_find_rdma_device(smc, ini)) |
| 764 | ini->smcr_version &= ~SMC_V2; |
| 765 | ini->check_smcrv2 = false; |
| 766 | |
| 767 | ini->smc_type_v2 = smc_indicated_type(ini->smcd_version & SMC_V2, |
| 768 | ini->smcr_version & SMC_V2); |
Ursula Braun | d70bf4f | 2020-09-26 12:44:27 +0200 | [diff] [blame] | 769 | |
| 770 | /* if neither ISM nor RDMA are supported, fallback */ |
Karsten Graul | 42042db | 2021-10-16 11:37:44 +0200 | [diff] [blame] | 771 | if (ini->smc_type_v1 == SMC_TYPE_N && ini->smc_type_v2 == SMC_TYPE_N) |
Ursula Braun | d70bf4f | 2020-09-26 12:44:27 +0200 | [diff] [blame] | 772 | rc = SMC_CLC_DECL_NOSMCDEV; |
| 773 | |
| 774 | return rc; |
| 775 | } |
| 776 | |
Hans Wippel | 4134984 | 2018-06-28 19:05:11 +0200 | [diff] [blame] | 777 | /* cleanup temporary VLAN ID registration used for CLC handshake. If ISM is |
| 778 | * used, the VLAN ID will be registered again during the connection setup. |
| 779 | */ |
Ursula Braun | d70bf4f | 2020-09-26 12:44:27 +0200 | [diff] [blame] | 780 | static int smc_connect_ism_vlan_cleanup(struct smc_sock *smc, |
Karsten Graul | bc36d2f | 2019-04-12 12:57:26 +0200 | [diff] [blame] | 781 | struct smc_init_info *ini) |
Hans Wippel | 4134984 | 2018-06-28 19:05:11 +0200 | [diff] [blame] | 782 | { |
Ursula Braun | d70bf4f | 2020-09-26 12:44:27 +0200 | [diff] [blame] | 783 | if (!smcd_indicated(ini->smc_type_v1)) |
Hans Wippel | 4134984 | 2018-06-28 19:05:11 +0200 | [diff] [blame] | 784 | return 0; |
Ursula Braun | 3fc6493 | 2020-09-26 12:44:23 +0200 | [diff] [blame] | 785 | if (ini->vlan_id && smc_ism_put_vlan(ini->ism_dev[0], ini->vlan_id)) |
Hans Wippel | 4134984 | 2018-06-28 19:05:11 +0200 | [diff] [blame] | 786 | return SMC_CLC_DECL_CNFERR; |
| 787 | return 0; |
| 788 | } |
| 789 | |
Ursula Braun | a7c9c5f | 2020-09-26 12:44:30 +0200 | [diff] [blame] | 790 | #define SMC_CLC_MAX_ACCEPT_LEN \ |
| 791 | (sizeof(struct smc_clc_msg_accept_confirm_v2) + \ |
Ursula Braun | b81a5eb | 2020-09-26 12:44:31 +0200 | [diff] [blame] | 792 | sizeof(struct smc_clc_first_contact_ext) + \ |
Ursula Braun | a7c9c5f | 2020-09-26 12:44:30 +0200 | [diff] [blame] | 793 | sizeof(struct smc_clc_msg_trail)) |
| 794 | |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 795 | /* CLC handshake during connect */ |
Ursula Braun | d70bf4f | 2020-09-26 12:44:27 +0200 | [diff] [blame] | 796 | static int smc_connect_clc(struct smc_sock *smc, |
Ursula Braun | a7c9c5f | 2020-09-26 12:44:30 +0200 | [diff] [blame] | 797 | struct smc_clc_msg_accept_confirm_v2 *aclc2, |
Karsten Graul | bc36d2f | 2019-04-12 12:57:26 +0200 | [diff] [blame] | 798 | struct smc_init_info *ini) |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 799 | { |
| 800 | int rc = 0; |
| 801 | |
| 802 | /* do inband token exchange */ |
Ursula Braun | d70bf4f | 2020-09-26 12:44:27 +0200 | [diff] [blame] | 803 | rc = smc_clc_send_proposal(smc, ini); |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 804 | if (rc) |
| 805 | return rc; |
| 806 | /* receive SMC Accept CLC message */ |
Ursula Braun | a7c9c5f | 2020-09-26 12:44:30 +0200 | [diff] [blame] | 807 | return smc_clc_wait_msg(smc, aclc2, SMC_CLC_MAX_ACCEPT_LEN, |
| 808 | SMC_CLC_ACCEPT, CLC_WAIT_TIME); |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 809 | } |
| 810 | |
Karsten Graul | b4ba465 | 2021-10-16 11:37:50 +0200 | [diff] [blame] | 811 | void smc_fill_gid_list(struct smc_link_group *lgr, |
| 812 | struct smc_gidlist *gidlist, |
| 813 | struct smc_ib_device *known_dev, u8 *known_gid) |
Karsten Graul | e5c4744 | 2021-10-16 11:37:45 +0200 | [diff] [blame] | 814 | { |
| 815 | struct smc_init_info *alt_ini = NULL; |
| 816 | |
| 817 | memset(gidlist, 0, sizeof(*gidlist)); |
| 818 | memcpy(gidlist->list[gidlist->len++], known_gid, SMC_GID_SIZE); |
| 819 | |
| 820 | alt_ini = kzalloc(sizeof(*alt_ini), GFP_KERNEL); |
| 821 | if (!alt_ini) |
| 822 | goto out; |
| 823 | |
| 824 | alt_ini->vlan_id = lgr->vlan_id; |
| 825 | alt_ini->check_smcrv2 = true; |
| 826 | alt_ini->smcrv2.saddr = lgr->saddr; |
| 827 | smc_pnet_find_alt_roce(lgr, alt_ini, known_dev); |
| 828 | |
| 829 | if (!alt_ini->smcrv2.ib_dev_v2) |
| 830 | goto out; |
| 831 | |
| 832 | memcpy(gidlist->list[gidlist->len++], alt_ini->smcrv2.ib_gid_v2, |
| 833 | SMC_GID_SIZE); |
| 834 | |
| 835 | out: |
| 836 | kfree(alt_ini); |
| 837 | } |
| 838 | |
| 839 | static int smc_connect_rdma_v2_prepare(struct smc_sock *smc, |
| 840 | struct smc_clc_msg_accept_confirm *aclc, |
| 841 | struct smc_init_info *ini) |
| 842 | { |
| 843 | struct smc_clc_msg_accept_confirm_v2 *clc_v2 = |
| 844 | (struct smc_clc_msg_accept_confirm_v2 *)aclc; |
| 845 | struct smc_clc_first_contact_ext *fce = |
| 846 | (struct smc_clc_first_contact_ext *) |
| 847 | (((u8 *)clc_v2) + sizeof(*clc_v2)); |
| 848 | |
| 849 | if (!ini->first_contact_peer || aclc->hdr.version == SMC_V1) |
| 850 | return 0; |
| 851 | |
| 852 | if (fce->v2_direct) { |
| 853 | memcpy(ini->smcrv2.nexthop_mac, &aclc->r0.lcl.mac, ETH_ALEN); |
| 854 | ini->smcrv2.uses_gateway = false; |
| 855 | } else { |
| 856 | if (smc_ib_find_route(smc->clcsock->sk->sk_rcv_saddr, |
| 857 | smc_ib_gid_to_ipv4(aclc->r0.lcl.gid), |
| 858 | ini->smcrv2.nexthop_mac, |
| 859 | &ini->smcrv2.uses_gateway)) |
| 860 | return SMC_CLC_DECL_NOROUTE; |
| 861 | if (!ini->smcrv2.uses_gateway) { |
| 862 | /* mismatch: peer claims indirect, but its direct */ |
| 863 | return SMC_CLC_DECL_NOINDIRECT; |
| 864 | } |
| 865 | } |
| 866 | return 0; |
| 867 | } |
| 868 | |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 869 | /* setup for RDMA connection of client */ |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 870 | static int smc_connect_rdma(struct smc_sock *smc, |
| 871 | struct smc_clc_msg_accept_confirm *aclc, |
Karsten Graul | bc36d2f | 2019-04-12 12:57:26 +0200 | [diff] [blame] | 872 | struct smc_init_info *ini) |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 873 | { |
Karsten Graul | 0fb0b02 | 2020-04-30 15:55:43 +0200 | [diff] [blame] | 874 | int i, reason_code = 0; |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 875 | struct smc_link *link; |
Karsten Graul | e5c4744 | 2021-10-16 11:37:45 +0200 | [diff] [blame] | 876 | u8 *eid = NULL; |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 877 | |
Karsten Graul | bc36d2f | 2019-04-12 12:57:26 +0200 | [diff] [blame] | 878 | ini->is_smcd = false; |
Ursula Braun | 3d9725a | 2020-09-10 18:48:23 +0200 | [diff] [blame] | 879 | ini->ib_clcqpn = ntoh24(aclc->r0.qpn); |
Ursula Braun | f1eb02f | 2020-09-26 12:44:20 +0200 | [diff] [blame] | 880 | ini->first_contact_peer = aclc->hdr.typev2 & SMC_FIRST_CONTACT_MASK; |
Karsten Graul | e5c4744 | 2021-10-16 11:37:45 +0200 | [diff] [blame] | 881 | memcpy(ini->peer_systemid, aclc->r0.lcl.id_for_peer, SMC_SYSTEMID_LEN); |
| 882 | memcpy(ini->peer_gid, aclc->r0.lcl.gid, SMC_GID_SIZE); |
| 883 | memcpy(ini->peer_mac, aclc->r0.lcl.mac, ETH_ALEN); |
| 884 | |
| 885 | reason_code = smc_connect_rdma_v2_prepare(smc, aclc, ini); |
| 886 | if (reason_code) |
| 887 | return reason_code; |
Karsten Graul | bc36d2f | 2019-04-12 12:57:26 +0200 | [diff] [blame] | 888 | |
Hans Wippel | 72a36a8 | 2019-02-07 15:56:18 +0100 | [diff] [blame] | 889 | mutex_lock(&smc_client_lgr_pending); |
Karsten Graul | 7a62725a | 2019-04-12 12:57:30 +0200 | [diff] [blame] | 890 | reason_code = smc_conn_create(smc, ini); |
| 891 | if (reason_code) { |
Hans Wippel | 72a36a8 | 2019-02-07 15:56:18 +0100 | [diff] [blame] | 892 | mutex_unlock(&smc_client_lgr_pending); |
| 893 | return reason_code; |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 894 | } |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 895 | |
| 896 | smc_conn_save_peer_info(smc, aclc); |
| 897 | |
Ursula Braun | 5ac54d8 | 2020-09-10 18:48:21 +0200 | [diff] [blame] | 898 | if (ini->first_contact_local) { |
Karsten Graul | 0fb0b02 | 2020-04-30 15:55:43 +0200 | [diff] [blame] | 899 | link = smc->conn.lnk; |
| 900 | } else { |
| 901 | /* set link that was assigned by server */ |
| 902 | link = NULL; |
| 903 | for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { |
| 904 | struct smc_link *l = &smc->conn.lgr->lnk[i]; |
| 905 | |
Ursula Braun | 3d9725a | 2020-09-10 18:48:23 +0200 | [diff] [blame] | 906 | if (l->peer_qpn == ntoh24(aclc->r0.qpn) && |
| 907 | !memcmp(l->peer_gid, &aclc->r0.lcl.gid, |
| 908 | SMC_GID_SIZE) && |
Karsten Graul | e5c4744 | 2021-10-16 11:37:45 +0200 | [diff] [blame] | 909 | (aclc->hdr.version > SMC_V1 || |
| 910 | !memcmp(l->peer_mac, &aclc->r0.lcl.mac, |
| 911 | sizeof(l->peer_mac)))) { |
Karsten Graul | 0fb0b02 | 2020-04-30 15:55:43 +0200 | [diff] [blame] | 912 | link = l; |
| 913 | break; |
| 914 | } |
| 915 | } |
Karsten Graul | c60a2ce | 2020-10-07 22:57:41 +0200 | [diff] [blame] | 916 | if (!link) { |
| 917 | reason_code = SMC_CLC_DECL_NOSRVLINK; |
| 918 | goto connect_abort; |
| 919 | } |
Guvenc Gulce | 64513d2 | 2021-08-09 11:05:57 +0200 | [diff] [blame] | 920 | smc_switch_link_and_count(&smc->conn, link); |
Karsten Graul | 0fb0b02 | 2020-04-30 15:55:43 +0200 | [diff] [blame] | 921 | } |
| 922 | |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 923 | /* create send buffer and rmb */ |
Karsten Graul | c60a2ce | 2020-10-07 22:57:41 +0200 | [diff] [blame] | 924 | if (smc_buf_create(smc, false)) { |
| 925 | reason_code = SMC_CLC_DECL_MEM; |
| 926 | goto connect_abort; |
| 927 | } |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 928 | |
Ursula Braun | 5ac54d8 | 2020-09-10 18:48:21 +0200 | [diff] [blame] | 929 | if (ini->first_contact_local) |
Karsten Graul | e5c4744 | 2021-10-16 11:37:45 +0200 | [diff] [blame] | 930 | smc_link_save_peer_info(link, aclc, ini); |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 931 | |
Karsten Graul | c60a2ce | 2020-10-07 22:57:41 +0200 | [diff] [blame] | 932 | if (smc_rmb_rtoken_handling(&smc->conn, link, aclc)) { |
| 933 | reason_code = SMC_CLC_DECL_ERR_RTOK; |
| 934 | goto connect_abort; |
| 935 | } |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 936 | |
| 937 | smc_close_init(smc); |
| 938 | smc_rx_init(smc); |
| 939 | |
Ursula Braun | 5ac54d8 | 2020-09-10 18:48:21 +0200 | [diff] [blame] | 940 | if (ini->first_contact_local) { |
Karsten Graul | c60a2ce | 2020-10-07 22:57:41 +0200 | [diff] [blame] | 941 | if (smc_ib_ready_link(link)) { |
| 942 | reason_code = SMC_CLC_DECL_ERR_RDYLNK; |
| 943 | goto connect_abort; |
| 944 | } |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 945 | } else { |
Karsten Graul | c60a2ce | 2020-10-07 22:57:41 +0200 | [diff] [blame] | 946 | if (smcr_lgr_reg_rmbs(link, smc->conn.rmb_desc)) { |
| 947 | reason_code = SMC_CLC_DECL_ERR_REGRMB; |
| 948 | goto connect_abort; |
| 949 | } |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 950 | } |
| 951 | smc_rmb_sync_sg_for_device(&smc->conn); |
| 952 | |
Karsten Graul | e5c4744 | 2021-10-16 11:37:45 +0200 | [diff] [blame] | 953 | if (aclc->hdr.version > SMC_V1) { |
| 954 | struct smc_clc_msg_accept_confirm_v2 *clc_v2 = |
| 955 | (struct smc_clc_msg_accept_confirm_v2 *)aclc; |
| 956 | |
| 957 | eid = clc_v2->r1.eid; |
| 958 | if (ini->first_contact_local) |
| 959 | smc_fill_gid_list(link->lgr, &ini->smcrv2.gidlist, |
| 960 | link->smcibdev, link->gid); |
| 961 | } |
| 962 | |
Ursula Braun | a7c9c5f | 2020-09-26 12:44:30 +0200 | [diff] [blame] | 963 | reason_code = smc_clc_send_confirm(smc, ini->first_contact_local, |
Karsten Graul | e5c4744 | 2021-10-16 11:37:45 +0200 | [diff] [blame] | 964 | aclc->hdr.version, eid, ini); |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 965 | if (reason_code) |
Karsten Graul | c60a2ce | 2020-10-07 22:57:41 +0200 | [diff] [blame] | 966 | goto connect_abort; |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 967 | |
| 968 | smc_tx_init(smc); |
| 969 | |
Ursula Braun | 5ac54d8 | 2020-09-10 18:48:21 +0200 | [diff] [blame] | 970 | if (ini->first_contact_local) { |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 971 | /* QP confirmation over RoCE fabric */ |
Karsten Graul | 0fb0b02 | 2020-04-30 15:55:43 +0200 | [diff] [blame] | 972 | smc_llc_flow_initiate(link->lgr, SMC_LLC_FLOW_ADD_LINK); |
Karsten Graul | b924754 | 2020-04-29 17:10:41 +0200 | [diff] [blame] | 973 | reason_code = smcr_clnt_conf_first_link(smc); |
Karsten Graul | 0fb0b02 | 2020-04-30 15:55:43 +0200 | [diff] [blame] | 974 | smc_llc_flow_stop(link->lgr, &link->lgr->llc_flow_lcl); |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 975 | if (reason_code) |
Karsten Graul | c60a2ce | 2020-10-07 22:57:41 +0200 | [diff] [blame] | 976 | goto connect_abort; |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 977 | } |
Hans Wippel | 72a36a8 | 2019-02-07 15:56:18 +0100 | [diff] [blame] | 978 | mutex_unlock(&smc_client_lgr_pending); |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 979 | |
| 980 | smc_copy_sock_settings_to_clc(smc); |
Ursula Braun | 50717a3 | 2019-04-12 12:57:23 +0200 | [diff] [blame] | 981 | smc->connect_nonblock = 0; |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 982 | if (smc->sk.sk_state == SMC_INIT) |
| 983 | smc->sk.sk_state = SMC_ACTIVE; |
| 984 | |
| 985 | return 0; |
Karsten Graul | c60a2ce | 2020-10-07 22:57:41 +0200 | [diff] [blame] | 986 | connect_abort: |
Karsten Graul | 8cf3f3e | 2020-12-01 20:20:36 +0100 | [diff] [blame] | 987 | smc_conn_abort(smc, ini->first_contact_local); |
Karsten Graul | c60a2ce | 2020-10-07 22:57:41 +0200 | [diff] [blame] | 988 | mutex_unlock(&smc_client_lgr_pending); |
| 989 | smc->connect_nonblock = 0; |
| 990 | |
| 991 | return reason_code; |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 992 | } |
| 993 | |
Ursula Braun | a7c9c5f | 2020-09-26 12:44:30 +0200 | [diff] [blame] | 994 | /* The server has chosen one of the proposed ISM devices for the communication. |
| 995 | * Determine from the CHID of the received CLC ACCEPT the ISM device chosen. |
| 996 | */ |
| 997 | static int |
| 998 | smc_v2_determine_accepted_chid(struct smc_clc_msg_accept_confirm_v2 *aclc, |
| 999 | struct smc_init_info *ini) |
| 1000 | { |
| 1001 | int i; |
| 1002 | |
| 1003 | for (i = 0; i < ini->ism_offered_cnt + 1; i++) { |
Karsten Graul | e5c4744 | 2021-10-16 11:37:45 +0200 | [diff] [blame] | 1004 | if (ini->ism_chid[i] == ntohs(aclc->d1.chid)) { |
Ursula Braun | a7c9c5f | 2020-09-26 12:44:30 +0200 | [diff] [blame] | 1005 | ini->ism_selected = i; |
| 1006 | return 0; |
| 1007 | } |
| 1008 | } |
| 1009 | |
| 1010 | return -EPROTO; |
| 1011 | } |
| 1012 | |
Hans Wippel | 4134984 | 2018-06-28 19:05:11 +0200 | [diff] [blame] | 1013 | /* setup for ISM connection of client */ |
| 1014 | static int smc_connect_ism(struct smc_sock *smc, |
| 1015 | struct smc_clc_msg_accept_confirm *aclc, |
Karsten Graul | bc36d2f | 2019-04-12 12:57:26 +0200 | [diff] [blame] | 1016 | struct smc_init_info *ini) |
Hans Wippel | 4134984 | 2018-06-28 19:05:11 +0200 | [diff] [blame] | 1017 | { |
Karsten Graul | fa08666 | 2021-09-14 10:35:05 +0200 | [diff] [blame] | 1018 | u8 *eid = NULL; |
Hans Wippel | 4134984 | 2018-06-28 19:05:11 +0200 | [diff] [blame] | 1019 | int rc = 0; |
| 1020 | |
Karsten Graul | bc36d2f | 2019-04-12 12:57:26 +0200 | [diff] [blame] | 1021 | ini->is_smcd = true; |
Ursula Braun | f1eb02f | 2020-09-26 12:44:20 +0200 | [diff] [blame] | 1022 | ini->first_contact_peer = aclc->hdr.typev2 & SMC_FIRST_CONTACT_MASK; |
Karsten Graul | bc36d2f | 2019-04-12 12:57:26 +0200 | [diff] [blame] | 1023 | |
Ursula Braun | a7c9c5f | 2020-09-26 12:44:30 +0200 | [diff] [blame] | 1024 | if (aclc->hdr.version == SMC_V2) { |
| 1025 | struct smc_clc_msg_accept_confirm_v2 *aclc_v2 = |
| 1026 | (struct smc_clc_msg_accept_confirm_v2 *)aclc; |
| 1027 | |
| 1028 | rc = smc_v2_determine_accepted_chid(aclc_v2, ini); |
| 1029 | if (rc) |
| 1030 | return rc; |
| 1031 | } |
| 1032 | ini->ism_peer_gid[ini->ism_selected] = aclc->d0.gid; |
| 1033 | |
Hans Wippel | 72a36a8 | 2019-02-07 15:56:18 +0100 | [diff] [blame] | 1034 | /* there is only one lgr role for SMC-D; use server lock */ |
| 1035 | mutex_lock(&smc_server_lgr_pending); |
Karsten Graul | 7a62725a | 2019-04-12 12:57:30 +0200 | [diff] [blame] | 1036 | rc = smc_conn_create(smc, ini); |
| 1037 | if (rc) { |
Hans Wippel | 72a36a8 | 2019-02-07 15:56:18 +0100 | [diff] [blame] | 1038 | mutex_unlock(&smc_server_lgr_pending); |
Karsten Graul | 7a62725a | 2019-04-12 12:57:30 +0200 | [diff] [blame] | 1039 | return rc; |
Hans Wippel | 72a36a8 | 2019-02-07 15:56:18 +0100 | [diff] [blame] | 1040 | } |
Hans Wippel | 4134984 | 2018-06-28 19:05:11 +0200 | [diff] [blame] | 1041 | |
| 1042 | /* Create send and receive buffers */ |
Karsten Graul | 72b7f6c | 2020-07-26 20:34:28 +0200 | [diff] [blame] | 1043 | rc = smc_buf_create(smc, true); |
Karsten Graul | c60a2ce | 2020-10-07 22:57:41 +0200 | [diff] [blame] | 1044 | if (rc) { |
| 1045 | rc = (rc == -ENOSPC) ? SMC_CLC_DECL_MAX_DMB : SMC_CLC_DECL_MEM; |
| 1046 | goto connect_abort; |
| 1047 | } |
Hans Wippel | 4134984 | 2018-06-28 19:05:11 +0200 | [diff] [blame] | 1048 | |
| 1049 | smc_conn_save_peer_info(smc, aclc); |
| 1050 | smc_close_init(smc); |
| 1051 | smc_rx_init(smc); |
| 1052 | smc_tx_init(smc); |
| 1053 | |
Karsten Graul | fa08666 | 2021-09-14 10:35:05 +0200 | [diff] [blame] | 1054 | if (aclc->hdr.version > SMC_V1) { |
| 1055 | struct smc_clc_msg_accept_confirm_v2 *clc_v2 = |
| 1056 | (struct smc_clc_msg_accept_confirm_v2 *)aclc; |
| 1057 | |
Karsten Graul | e5c4744 | 2021-10-16 11:37:45 +0200 | [diff] [blame] | 1058 | eid = clc_v2->d1.eid; |
Karsten Graul | fa08666 | 2021-09-14 10:35:05 +0200 | [diff] [blame] | 1059 | } |
| 1060 | |
Ursula Braun | a7c9c5f | 2020-09-26 12:44:30 +0200 | [diff] [blame] | 1061 | rc = smc_clc_send_confirm(smc, ini->first_contact_local, |
Karsten Graul | e5c4744 | 2021-10-16 11:37:45 +0200 | [diff] [blame] | 1062 | aclc->hdr.version, eid, NULL); |
Hans Wippel | 4134984 | 2018-06-28 19:05:11 +0200 | [diff] [blame] | 1063 | if (rc) |
Karsten Graul | c60a2ce | 2020-10-07 22:57:41 +0200 | [diff] [blame] | 1064 | goto connect_abort; |
Hans Wippel | 72a36a8 | 2019-02-07 15:56:18 +0100 | [diff] [blame] | 1065 | mutex_unlock(&smc_server_lgr_pending); |
Hans Wippel | 4134984 | 2018-06-28 19:05:11 +0200 | [diff] [blame] | 1066 | |
| 1067 | smc_copy_sock_settings_to_clc(smc); |
Ursula Braun | 50717a3 | 2019-04-12 12:57:23 +0200 | [diff] [blame] | 1068 | smc->connect_nonblock = 0; |
Hans Wippel | 4134984 | 2018-06-28 19:05:11 +0200 | [diff] [blame] | 1069 | if (smc->sk.sk_state == SMC_INIT) |
| 1070 | smc->sk.sk_state = SMC_ACTIVE; |
| 1071 | |
| 1072 | return 0; |
Karsten Graul | c60a2ce | 2020-10-07 22:57:41 +0200 | [diff] [blame] | 1073 | connect_abort: |
Karsten Graul | 8cf3f3e | 2020-12-01 20:20:36 +0100 | [diff] [blame] | 1074 | smc_conn_abort(smc, ini->first_contact_local); |
Karsten Graul | c60a2ce | 2020-10-07 22:57:41 +0200 | [diff] [blame] | 1075 | mutex_unlock(&smc_server_lgr_pending); |
| 1076 | smc->connect_nonblock = 0; |
| 1077 | |
| 1078 | return rc; |
Hans Wippel | 4134984 | 2018-06-28 19:05:11 +0200 | [diff] [blame] | 1079 | } |
| 1080 | |
Ursula Braun | d70bf4f | 2020-09-26 12:44:27 +0200 | [diff] [blame] | 1081 | /* check if received accept type and version matches a proposed one */ |
| 1082 | static int smc_connect_check_aclc(struct smc_init_info *ini, |
| 1083 | struct smc_clc_msg_accept_confirm *aclc) |
| 1084 | { |
Karsten Graul | 42042db | 2021-10-16 11:37:44 +0200 | [diff] [blame] | 1085 | if (aclc->hdr.typev1 != SMC_TYPE_R && |
| 1086 | aclc->hdr.typev1 != SMC_TYPE_D) |
Ursula Braun | d70bf4f | 2020-09-26 12:44:27 +0200 | [diff] [blame] | 1087 | return SMC_CLC_DECL_MODEUNSUPP; |
| 1088 | |
Karsten Graul | 42042db | 2021-10-16 11:37:44 +0200 | [diff] [blame] | 1089 | if (aclc->hdr.version >= SMC_V2) { |
| 1090 | if ((aclc->hdr.typev1 == SMC_TYPE_R && |
| 1091 | !smcr_indicated(ini->smc_type_v2)) || |
| 1092 | (aclc->hdr.typev1 == SMC_TYPE_D && |
| 1093 | !smcd_indicated(ini->smc_type_v2))) |
| 1094 | return SMC_CLC_DECL_MODEUNSUPP; |
| 1095 | } else { |
| 1096 | if ((aclc->hdr.typev1 == SMC_TYPE_R && |
| 1097 | !smcr_indicated(ini->smc_type_v1)) || |
| 1098 | (aclc->hdr.typev1 == SMC_TYPE_D && |
| 1099 | !smcd_indicated(ini->smc_type_v1))) |
| 1100 | return SMC_CLC_DECL_MODEUNSUPP; |
| 1101 | } |
| 1102 | |
Ursula Braun | d70bf4f | 2020-09-26 12:44:27 +0200 | [diff] [blame] | 1103 | return 0; |
| 1104 | } |
| 1105 | |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1106 | /* perform steps before actually connecting */ |
| 1107 | static int __smc_connect(struct smc_sock *smc) |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 1108 | { |
Guvenc Gulce | 49407ae | 2020-12-01 20:20:42 +0100 | [diff] [blame] | 1109 | u8 version = smc_ism_is_v2_capable() ? SMC_V2 : SMC_V1; |
Ursula Braun | a7c9c5f | 2020-09-26 12:44:30 +0200 | [diff] [blame] | 1110 | struct smc_clc_msg_accept_confirm_v2 *aclc2; |
| 1111 | struct smc_clc_msg_accept_confirm *aclc; |
Ursula Braun | 3fc6493 | 2020-09-26 12:44:23 +0200 | [diff] [blame] | 1112 | struct smc_init_info *ini = NULL; |
Ursula Braun | a7c9c5f | 2020-09-26 12:44:30 +0200 | [diff] [blame] | 1113 | u8 *buf = NULL; |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 1114 | int rc = 0; |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 1115 | |
Ursula Braun | ee9dfbe | 2018-04-26 17:18:21 +0200 | [diff] [blame] | 1116 | if (smc->use_fallback) |
Karsten Graul | 603cc14 | 2018-07-25 16:35:32 +0200 | [diff] [blame] | 1117 | return smc_connect_fallback(smc, smc->fallback_rsn); |
Ursula Braun | ee9dfbe | 2018-04-26 17:18:21 +0200 | [diff] [blame] | 1118 | |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1119 | /* if peer has not signalled SMC-capability, fall back */ |
| 1120 | if (!tcp_sk(smc->clcsock->sk)->syn_smc) |
Karsten Graul | 603cc14 | 2018-07-25 16:35:32 +0200 | [diff] [blame] | 1121 | return smc_connect_fallback(smc, SMC_CLC_DECL_PEERNOSMC); |
Ursula Braun | c5c1cc9 | 2017-10-25 11:01:46 +0200 | [diff] [blame] | 1122 | |
Ursula Braun | e8d726c8 | 2020-09-26 12:44:32 +0200 | [diff] [blame] | 1123 | /* IPSec connections opt out of SMC optimizations */ |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1124 | if (using_ipsec(smc)) |
Ursula Braun | e8d726c8 | 2020-09-26 12:44:32 +0200 | [diff] [blame] | 1125 | return smc_connect_decline_fallback(smc, SMC_CLC_DECL_IPSEC, |
| 1126 | version); |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 1127 | |
Ursula Braun | 3fc6493 | 2020-09-26 12:44:23 +0200 | [diff] [blame] | 1128 | ini = kzalloc(sizeof(*ini), GFP_KERNEL); |
| 1129 | if (!ini) |
Ursula Braun | e8d726c8 | 2020-09-26 12:44:32 +0200 | [diff] [blame] | 1130 | return smc_connect_decline_fallback(smc, SMC_CLC_DECL_MEM, |
| 1131 | version); |
Ursula Braun | 3fc6493 | 2020-09-26 12:44:23 +0200 | [diff] [blame] | 1132 | |
Karsten Graul | 42042db | 2021-10-16 11:37:44 +0200 | [diff] [blame] | 1133 | ini->smcd_version = SMC_V1 | SMC_V2; |
| 1134 | ini->smcr_version = SMC_V1 | SMC_V2; |
Ursula Braun | d70bf4f | 2020-09-26 12:44:27 +0200 | [diff] [blame] | 1135 | ini->smc_type_v1 = SMC_TYPE_B; |
Karsten Graul | 42042db | 2021-10-16 11:37:44 +0200 | [diff] [blame] | 1136 | ini->smc_type_v2 = SMC_TYPE_B; |
Ursula Braun | d70bf4f | 2020-09-26 12:44:27 +0200 | [diff] [blame] | 1137 | |
Karsten Graul | fba7e8e | 2019-04-12 12:57:27 +0200 | [diff] [blame] | 1138 | /* get vlan id from IP device */ |
Ursula Braun | 3fc6493 | 2020-09-26 12:44:23 +0200 | [diff] [blame] | 1139 | if (smc_vlan_by_tcpsk(smc->clcsock, ini)) { |
Ursula Braun | d70bf4f | 2020-09-26 12:44:27 +0200 | [diff] [blame] | 1140 | ini->smcd_version &= ~SMC_V1; |
Karsten Graul | 42042db | 2021-10-16 11:37:44 +0200 | [diff] [blame] | 1141 | ini->smcr_version = 0; |
Ursula Braun | d70bf4f | 2020-09-26 12:44:27 +0200 | [diff] [blame] | 1142 | ini->smc_type_v1 = SMC_TYPE_N; |
| 1143 | if (!ini->smcd_version) { |
| 1144 | rc = SMC_CLC_DECL_GETVLANERR; |
| 1145 | goto fallback; |
| 1146 | } |
Ursula Braun | 3fc6493 | 2020-09-26 12:44:23 +0200 | [diff] [blame] | 1147 | } |
Hans Wippel | 4134984 | 2018-06-28 19:05:11 +0200 | [diff] [blame] | 1148 | |
Ursula Braun | d70bf4f | 2020-09-26 12:44:27 +0200 | [diff] [blame] | 1149 | rc = smc_find_proposal_devices(smc, ini); |
| 1150 | if (rc) |
| 1151 | goto fallback; |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 1152 | |
Ursula Braun | a7c9c5f | 2020-09-26 12:44:30 +0200 | [diff] [blame] | 1153 | buf = kzalloc(SMC_CLC_MAX_ACCEPT_LEN, GFP_KERNEL); |
| 1154 | if (!buf) { |
| 1155 | rc = SMC_CLC_DECL_MEM; |
| 1156 | goto fallback; |
| 1157 | } |
| 1158 | aclc2 = (struct smc_clc_msg_accept_confirm_v2 *)buf; |
| 1159 | aclc = (struct smc_clc_msg_accept_confirm *)aclc2; |
| 1160 | |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1161 | /* perform CLC handshake */ |
Ursula Braun | a7c9c5f | 2020-09-26 12:44:30 +0200 | [diff] [blame] | 1162 | rc = smc_connect_clc(smc, aclc2, ini); |
Ursula Braun | d70bf4f | 2020-09-26 12:44:27 +0200 | [diff] [blame] | 1163 | if (rc) |
| 1164 | goto vlan_cleanup; |
| 1165 | |
| 1166 | /* check if smc modes and versions of CLC proposal and accept match */ |
Ursula Braun | a7c9c5f | 2020-09-26 12:44:30 +0200 | [diff] [blame] | 1167 | rc = smc_connect_check_aclc(ini, aclc); |
Karsten Graul | 0530bd6 | 2020-11-18 22:40:37 +0100 | [diff] [blame] | 1168 | version = aclc->hdr.version == SMC_V1 ? SMC_V1 : SMC_V2; |
Ursula Braun | d70bf4f | 2020-09-26 12:44:27 +0200 | [diff] [blame] | 1169 | if (rc) |
| 1170 | goto vlan_cleanup; |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 1171 | |
Hans Wippel | 4134984 | 2018-06-28 19:05:11 +0200 | [diff] [blame] | 1172 | /* depending on previous steps, connect using rdma or ism */ |
Karsten Graul | 42042db | 2021-10-16 11:37:44 +0200 | [diff] [blame] | 1173 | if (aclc->hdr.typev1 == SMC_TYPE_R) { |
| 1174 | ini->smcr_version = version; |
Ursula Braun | a7c9c5f | 2020-09-26 12:44:30 +0200 | [diff] [blame] | 1175 | rc = smc_connect_rdma(smc, aclc, ini); |
Karsten Graul | 42042db | 2021-10-16 11:37:44 +0200 | [diff] [blame] | 1176 | } else if (aclc->hdr.typev1 == SMC_TYPE_D) { |
| 1177 | ini->smcd_version = version; |
Ursula Braun | a7c9c5f | 2020-09-26 12:44:30 +0200 | [diff] [blame] | 1178 | rc = smc_connect_ism(smc, aclc, ini); |
Karsten Graul | 42042db | 2021-10-16 11:37:44 +0200 | [diff] [blame] | 1179 | } |
Ursula Braun | d70bf4f | 2020-09-26 12:44:27 +0200 | [diff] [blame] | 1180 | if (rc) |
| 1181 | goto vlan_cleanup; |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 1182 | |
Guvenc Gulce | 194730a | 2021-06-16 16:52:58 +0200 | [diff] [blame] | 1183 | SMC_STAT_CLNT_SUCC_INC(sock_net(smc->clcsock->sk), aclc); |
Ursula Braun | d70bf4f | 2020-09-26 12:44:27 +0200 | [diff] [blame] | 1184 | smc_connect_ism_vlan_cleanup(smc, ini); |
Ursula Braun | a7c9c5f | 2020-09-26 12:44:30 +0200 | [diff] [blame] | 1185 | kfree(buf); |
Ursula Braun | 3fc6493 | 2020-09-26 12:44:23 +0200 | [diff] [blame] | 1186 | kfree(ini); |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1187 | return 0; |
Ursula Braun | d70bf4f | 2020-09-26 12:44:27 +0200 | [diff] [blame] | 1188 | |
| 1189 | vlan_cleanup: |
| 1190 | smc_connect_ism_vlan_cleanup(smc, ini); |
Ursula Braun | a7c9c5f | 2020-09-26 12:44:30 +0200 | [diff] [blame] | 1191 | kfree(buf); |
Ursula Braun | d70bf4f | 2020-09-26 12:44:27 +0200 | [diff] [blame] | 1192 | fallback: |
| 1193 | kfree(ini); |
Ursula Braun | e8d726c8 | 2020-09-26 12:44:32 +0200 | [diff] [blame] | 1194 | return smc_connect_decline_fallback(smc, rc, version); |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 1195 | } |
| 1196 | |
Ursula Braun | 24ac3a0 | 2018-06-27 17:59:50 +0200 | [diff] [blame] | 1197 | static void smc_connect_work(struct work_struct *work) |
| 1198 | { |
| 1199 | struct smc_sock *smc = container_of(work, struct smc_sock, |
| 1200 | connect_work); |
Ursula Braun | 50717a3 | 2019-04-12 12:57:23 +0200 | [diff] [blame] | 1201 | long timeo = smc->sk.sk_sndtimeo; |
| 1202 | int rc = 0; |
Ursula Braun | 24ac3a0 | 2018-06-27 17:59:50 +0200 | [diff] [blame] | 1203 | |
Ursula Braun | 50717a3 | 2019-04-12 12:57:23 +0200 | [diff] [blame] | 1204 | if (!timeo) |
| 1205 | timeo = MAX_SCHEDULE_TIMEOUT; |
| 1206 | lock_sock(smc->clcsock->sk); |
Ursula Braun | 24ac3a0 | 2018-06-27 17:59:50 +0200 | [diff] [blame] | 1207 | if (smc->clcsock->sk->sk_err) { |
| 1208 | smc->sk.sk_err = smc->clcsock->sk->sk_err; |
Ursula Braun | 50717a3 | 2019-04-12 12:57:23 +0200 | [diff] [blame] | 1209 | } else if ((1 << smc->clcsock->sk->sk_state) & |
Wen Gu | f3a3a0f | 2021-10-28 15:13:47 +0800 | [diff] [blame] | 1210 | (TCPF_SYN_SENT | TCPF_SYN_RECV)) { |
Ursula Braun | 50717a3 | 2019-04-12 12:57:23 +0200 | [diff] [blame] | 1211 | rc = sk_stream_wait_connect(smc->clcsock->sk, &timeo); |
| 1212 | if ((rc == -EPIPE) && |
| 1213 | ((1 << smc->clcsock->sk->sk_state) & |
| 1214 | (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT))) |
| 1215 | rc = 0; |
Ursula Braun | 24ac3a0 | 2018-06-27 17:59:50 +0200 | [diff] [blame] | 1216 | } |
Ursula Braun | 50717a3 | 2019-04-12 12:57:23 +0200 | [diff] [blame] | 1217 | release_sock(smc->clcsock->sk); |
| 1218 | lock_sock(&smc->sk); |
| 1219 | if (rc != 0 || smc->sk.sk_err) { |
| 1220 | smc->sk.sk_state = SMC_CLOSED; |
| 1221 | if (rc == -EPIPE || rc == -EAGAIN) |
| 1222 | smc->sk.sk_err = EPIPE; |
| 1223 | else if (signal_pending(current)) |
| 1224 | smc->sk.sk_err = -sock_intr_errno(timeo); |
Ursula Braun | 6d6dd52 | 2019-11-12 16:03:41 +0100 | [diff] [blame] | 1225 | sock_put(&smc->sk); /* passive closing */ |
Ursula Braun | 24ac3a0 | 2018-06-27 17:59:50 +0200 | [diff] [blame] | 1226 | goto out; |
| 1227 | } |
| 1228 | |
| 1229 | rc = __smc_connect(smc); |
| 1230 | if (rc < 0) |
| 1231 | smc->sk.sk_err = -rc; |
| 1232 | |
| 1233 | out: |
Ursula Braun | 07603b2 | 2019-04-11 11:17:32 +0200 | [diff] [blame] | 1234 | if (!sock_flag(&smc->sk, SOCK_DEAD)) { |
| 1235 | if (smc->sk.sk_err) { |
| 1236 | smc->sk.sk_state_change(&smc->sk); |
| 1237 | } else { /* allow polling before and after fallback decision */ |
| 1238 | smc->clcsock->sk->sk_write_space(smc->clcsock->sk); |
| 1239 | smc->sk.sk_write_space(&smc->sk); |
| 1240 | } |
| 1241 | } |
Ursula Braun | 24ac3a0 | 2018-06-27 17:59:50 +0200 | [diff] [blame] | 1242 | release_sock(&smc->sk); |
| 1243 | } |
| 1244 | |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 1245 | static int smc_connect(struct socket *sock, struct sockaddr *addr, |
| 1246 | int alen, int flags) |
| 1247 | { |
| 1248 | struct sock *sk = sock->sk; |
| 1249 | struct smc_sock *smc; |
| 1250 | int rc = -EINVAL; |
| 1251 | |
| 1252 | smc = smc_sk(sk); |
| 1253 | |
| 1254 | /* separate smc parameter checking to be safe */ |
| 1255 | if (alen < sizeof(addr->sa_family)) |
| 1256 | goto out_err; |
Karsten Graul | aaa4d33 | 2018-03-16 15:06:41 +0100 | [diff] [blame] | 1257 | if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6) |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 1258 | goto out_err; |
| 1259 | |
| 1260 | lock_sock(sk); |
| 1261 | switch (sk->sk_state) { |
| 1262 | default: |
| 1263 | goto out; |
| 1264 | case SMC_ACTIVE: |
| 1265 | rc = -EISCONN; |
| 1266 | goto out; |
| 1267 | case SMC_INIT: |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 1268 | break; |
| 1269 | } |
| 1270 | |
| 1271 | smc_copy_sock_settings_to_clc(smc); |
Ursula Braun | c5c1cc9 | 2017-10-25 11:01:46 +0200 | [diff] [blame] | 1272 | tcp_sk(smc->clcsock->sk)->syn_smc = 1; |
Ursula Braun | 50717a3 | 2019-04-12 12:57:23 +0200 | [diff] [blame] | 1273 | if (smc->connect_nonblock) { |
| 1274 | rc = -EALREADY; |
| 1275 | goto out; |
| 1276 | } |
| 1277 | rc = kernel_connect(smc->clcsock, addr, alen, flags); |
| 1278 | if (rc && rc != -EINPROGRESS) |
| 1279 | goto out; |
Ursula Braun | 301428e | 2019-10-29 12:41:26 +0100 | [diff] [blame] | 1280 | |
| 1281 | sock_hold(&smc->sk); /* sock put in passive closing */ |
Ursula Braun | 8643474 | 2019-12-12 22:35:58 +0100 | [diff] [blame] | 1282 | if (smc->use_fallback) |
| 1283 | goto out; |
Ursula Braun | 24ac3a0 | 2018-06-27 17:59:50 +0200 | [diff] [blame] | 1284 | if (flags & O_NONBLOCK) { |
Karsten Graul | 22ef473 | 2020-09-10 18:48:29 +0200 | [diff] [blame] | 1285 | if (queue_work(smc_hs_wq, &smc->connect_work)) |
Ursula Braun | 50717a3 | 2019-04-12 12:57:23 +0200 | [diff] [blame] | 1286 | smc->connect_nonblock = 1; |
Ursula Braun | 24ac3a0 | 2018-06-27 17:59:50 +0200 | [diff] [blame] | 1287 | rc = -EINPROGRESS; |
| 1288 | } else { |
Ursula Braun | 24ac3a0 | 2018-06-27 17:59:50 +0200 | [diff] [blame] | 1289 | rc = __smc_connect(smc); |
| 1290 | if (rc < 0) |
| 1291 | goto out; |
| 1292 | else |
| 1293 | rc = 0; /* success cases including fallback */ |
| 1294 | } |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 1295 | |
| 1296 | out: |
| 1297 | release_sock(sk); |
| 1298 | out_err: |
| 1299 | return rc; |
| 1300 | } |
| 1301 | |
| 1302 | static int smc_clcsock_accept(struct smc_sock *lsmc, struct smc_sock **new_smc) |
| 1303 | { |
Ursula Braun | 3163c50 | 2018-01-24 10:28:12 +0100 | [diff] [blame] | 1304 | struct socket *new_clcsock = NULL; |
| 1305 | struct sock *lsk = &lsmc->sk; |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 1306 | struct sock *new_sk; |
Myungho Jung | 78abe3d | 2018-12-18 09:02:25 -0800 | [diff] [blame] | 1307 | int rc = -EINVAL; |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 1308 | |
Ursula Braun | 3163c50 | 2018-01-24 10:28:12 +0100 | [diff] [blame] | 1309 | release_sock(lsk); |
Karsten Graul | aaa4d33 | 2018-03-16 15:06:41 +0100 | [diff] [blame] | 1310 | new_sk = smc_sock_alloc(sock_net(lsk), NULL, lsk->sk_protocol); |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 1311 | if (!new_sk) { |
| 1312 | rc = -ENOMEM; |
Ursula Braun | 3163c50 | 2018-01-24 10:28:12 +0100 | [diff] [blame] | 1313 | lsk->sk_err = ENOMEM; |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 1314 | *new_smc = NULL; |
Ursula Braun | 3163c50 | 2018-01-24 10:28:12 +0100 | [diff] [blame] | 1315 | lock_sock(lsk); |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 1316 | goto out; |
| 1317 | } |
| 1318 | *new_smc = smc_sk(new_sk); |
| 1319 | |
Myungho Jung | 78abe3d | 2018-12-18 09:02:25 -0800 | [diff] [blame] | 1320 | mutex_lock(&lsmc->clcsock_release_lock); |
| 1321 | if (lsmc->clcsock) |
Ursula Braun | a60a2b1 | 2020-09-10 18:48:20 +0200 | [diff] [blame] | 1322 | rc = kernel_accept(lsmc->clcsock, &new_clcsock, SOCK_NONBLOCK); |
Myungho Jung | 78abe3d | 2018-12-18 09:02:25 -0800 | [diff] [blame] | 1323 | mutex_unlock(&lsmc->clcsock_release_lock); |
Ursula Braun | 3163c50 | 2018-01-24 10:28:12 +0100 | [diff] [blame] | 1324 | lock_sock(lsk); |
Ursula Braun | a60a2b1 | 2020-09-10 18:48:20 +0200 | [diff] [blame] | 1325 | if (rc < 0 && rc != -EAGAIN) |
Ursula Braun | 3163c50 | 2018-01-24 10:28:12 +0100 | [diff] [blame] | 1326 | lsk->sk_err = -rc; |
Ursula Braun | 35a6b17 | 2018-01-24 10:28:13 +0100 | [diff] [blame] | 1327 | if (rc < 0 || lsk->sk_state == SMC_CLOSED) { |
Ursula Braun | f61bca5 | 2019-04-11 11:17:34 +0200 | [diff] [blame] | 1328 | new_sk->sk_prot->unhash(new_sk); |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 1329 | if (new_clcsock) |
| 1330 | sock_release(new_clcsock); |
| 1331 | new_sk->sk_state = SMC_CLOSED; |
| 1332 | sock_set_flag(new_sk, SOCK_DEAD); |
Ursula Braun | 51f1de7 | 2018-01-26 09:28:48 +0100 | [diff] [blame] | 1333 | sock_put(new_sk); /* final */ |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 1334 | *new_smc = NULL; |
| 1335 | goto out; |
| 1336 | } |
| 1337 | |
Ursula Braun | a60a2b1 | 2020-09-10 18:48:20 +0200 | [diff] [blame] | 1338 | /* new clcsock has inherited the smc listen-specific sk_data_ready |
| 1339 | * function; switch it back to the original sk_data_ready function |
| 1340 | */ |
| 1341 | new_clcsock->sk->sk_data_ready = lsmc->clcsk_data_ready; |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 1342 | (*new_smc)->clcsock = new_clcsock; |
| 1343 | out: |
| 1344 | return rc; |
| 1345 | } |
| 1346 | |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 1347 | /* add a just created sock to the accept queue of the listen sock as |
| 1348 | * candidate for a following socket accept call from user space |
| 1349 | */ |
| 1350 | static void smc_accept_enqueue(struct sock *parent, struct sock *sk) |
| 1351 | { |
| 1352 | struct smc_sock *par = smc_sk(parent); |
| 1353 | |
Ursula Braun | 51f1de7 | 2018-01-26 09:28:48 +0100 | [diff] [blame] | 1354 | sock_hold(sk); /* sock_put in smc_accept_unlink () */ |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 1355 | spin_lock(&par->accept_q_lock); |
| 1356 | list_add_tail(&smc_sk(sk)->accept_q, &par->accept_q); |
| 1357 | spin_unlock(&par->accept_q_lock); |
| 1358 | sk_acceptq_added(parent); |
| 1359 | } |
| 1360 | |
| 1361 | /* remove a socket from the accept queue of its parental listening socket */ |
| 1362 | static void smc_accept_unlink(struct sock *sk) |
| 1363 | { |
| 1364 | struct smc_sock *par = smc_sk(sk)->listen_smc; |
| 1365 | |
| 1366 | spin_lock(&par->accept_q_lock); |
| 1367 | list_del_init(&smc_sk(sk)->accept_q); |
| 1368 | spin_unlock(&par->accept_q_lock); |
| 1369 | sk_acceptq_removed(&smc_sk(sk)->listen_smc->sk); |
Ursula Braun | 51f1de7 | 2018-01-26 09:28:48 +0100 | [diff] [blame] | 1370 | sock_put(sk); /* sock_hold in smc_accept_enqueue */ |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 1371 | } |
| 1372 | |
| 1373 | /* remove a sock from the accept queue to bind it to a new socket created |
| 1374 | * for a socket accept call from user space |
| 1375 | */ |
Ursula Braun | b38d732 | 2017-01-09 16:55:25 +0100 | [diff] [blame] | 1376 | struct sock *smc_accept_dequeue(struct sock *parent, |
| 1377 | struct socket *new_sock) |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 1378 | { |
| 1379 | struct smc_sock *isk, *n; |
| 1380 | struct sock *new_sk; |
| 1381 | |
| 1382 | list_for_each_entry_safe(isk, n, &smc_sk(parent)->accept_q, accept_q) { |
| 1383 | new_sk = (struct sock *)isk; |
| 1384 | |
| 1385 | smc_accept_unlink(new_sk); |
| 1386 | if (new_sk->sk_state == SMC_CLOSED) { |
Ursula Braun | f61bca5 | 2019-04-11 11:17:34 +0200 | [diff] [blame] | 1387 | new_sk->sk_prot->unhash(new_sk); |
Ursula Braun | 127f497 | 2018-01-26 09:28:49 +0100 | [diff] [blame] | 1388 | if (isk->clcsock) { |
| 1389 | sock_release(isk->clcsock); |
| 1390 | isk->clcsock = NULL; |
| 1391 | } |
Ursula Braun | 51f1de7 | 2018-01-26 09:28:48 +0100 | [diff] [blame] | 1392 | sock_put(new_sk); /* final */ |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 1393 | continue; |
| 1394 | } |
Ursula Braun | 07603b2 | 2019-04-11 11:17:32 +0200 | [diff] [blame] | 1395 | if (new_sock) { |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 1396 | sock_graft(new_sk, new_sock); |
Ursula Braun | 07603b2 | 2019-04-11 11:17:32 +0200 | [diff] [blame] | 1397 | if (isk->use_fallback) { |
| 1398 | smc_sk(new_sk)->clcsock->file = new_sock->file; |
| 1399 | isk->clcsock->file->private_data = isk->clcsock; |
| 1400 | } |
| 1401 | } |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 1402 | return new_sk; |
| 1403 | } |
| 1404 | return NULL; |
| 1405 | } |
| 1406 | |
| 1407 | /* clean up for a created but never accepted sock */ |
Ursula Braun | b38d732 | 2017-01-09 16:55:25 +0100 | [diff] [blame] | 1408 | void smc_close_non_accepted(struct sock *sk) |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 1409 | { |
| 1410 | struct smc_sock *smc = smc_sk(sk); |
| 1411 | |
Ursula Braun | 81cf4f4 | 2019-10-21 16:13:15 +0200 | [diff] [blame] | 1412 | sock_hold(sk); /* sock_put below */ |
Ursula Braun | b38d732 | 2017-01-09 16:55:25 +0100 | [diff] [blame] | 1413 | lock_sock(sk); |
| 1414 | if (!sk->sk_lingertime) |
| 1415 | /* wait for peer closing */ |
| 1416 | sk->sk_lingertime = SMC_MAX_STREAM_WAIT_TIMEOUT; |
Ursula Braun | 39f41f3 | 2019-06-27 15:04:52 +0200 | [diff] [blame] | 1417 | __smc_release(smc); |
Ursula Braun | b38d732 | 2017-01-09 16:55:25 +0100 | [diff] [blame] | 1418 | release_sock(sk); |
Ursula Braun | 81cf4f4 | 2019-10-21 16:13:15 +0200 | [diff] [blame] | 1419 | sock_put(sk); /* sock_hold above */ |
Ursula Braun | 51f1de7 | 2018-01-26 09:28:48 +0100 | [diff] [blame] | 1420 | sock_put(sk); /* final sock_put */ |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 1421 | } |
| 1422 | |
Karsten Graul | b924754 | 2020-04-29 17:10:41 +0200 | [diff] [blame] | 1423 | static int smcr_serv_conf_first_link(struct smc_sock *smc) |
Ursula Braun | 9bf9abe | 2017-01-09 16:55:21 +0100 | [diff] [blame] | 1424 | { |
Karsten Graul | 387707f | 2020-04-29 17:10:40 +0200 | [diff] [blame] | 1425 | struct smc_link *link = smc->conn.lnk; |
Karsten Graul | 4667bb4 | 2020-04-30 15:55:42 +0200 | [diff] [blame] | 1426 | struct smc_llc_qentry *qentry; |
Ursula Braun | 9bf9abe | 2017-01-09 16:55:21 +0100 | [diff] [blame] | 1427 | int rc; |
| 1428 | |
Karsten Graul | 7562a13 | 2020-05-01 12:48:01 +0200 | [diff] [blame] | 1429 | if (smcr_link_reg_rmb(link, smc->conn.rmb_desc)) |
Karsten Graul | 603cc14 | 2018-07-25 16:35:32 +0200 | [diff] [blame] | 1430 | return SMC_CLC_DECL_ERR_REGRMB; |
Ursula Braun | 652a1e4 | 2017-07-28 13:56:17 +0200 | [diff] [blame] | 1431 | |
Ursula Braun | 9bf9abe | 2017-01-09 16:55:21 +0100 | [diff] [blame] | 1432 | /* send CONFIRM LINK request to client over the RoCE fabric */ |
Ursula Braun | 947541f | 2018-07-25 16:35:30 +0200 | [diff] [blame] | 1433 | rc = smc_llc_send_confirm_link(link, SMC_LLC_REQ); |
Ursula Braun | 9bf9abe | 2017-01-09 16:55:21 +0100 | [diff] [blame] | 1434 | if (rc < 0) |
Karsten Graul | 603cc14 | 2018-07-25 16:35:32 +0200 | [diff] [blame] | 1435 | return SMC_CLC_DECL_TIMEOUT_CL; |
Ursula Braun | 9bf9abe | 2017-01-09 16:55:21 +0100 | [diff] [blame] | 1436 | |
| 1437 | /* receive CONFIRM LINK response from client over the RoCE fabric */ |
Karsten Graul | 4667bb4 | 2020-04-30 15:55:42 +0200 | [diff] [blame] | 1438 | qentry = smc_llc_wait(link->lgr, link, SMC_LLC_WAIT_TIME, |
| 1439 | SMC_LLC_CONFIRM_LINK); |
| 1440 | if (!qentry) { |
Ursula Braun | 9bf9abe | 2017-01-09 16:55:21 +0100 | [diff] [blame] | 1441 | struct smc_clc_msg_decline dclc; |
| 1442 | |
| 1443 | rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc), |
Ursula Braun | 2b59f58 | 2018-11-22 10:26:39 +0100 | [diff] [blame] | 1444 | SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT); |
Ursula Braun | 9ed2855 | 2018-11-22 10:26:37 +0100 | [diff] [blame] | 1445 | return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc; |
Ursula Braun | 9bf9abe | 2017-01-09 16:55:21 +0100 | [diff] [blame] | 1446 | } |
Karsten Graul | 649758f | 2020-05-04 14:18:48 +0200 | [diff] [blame] | 1447 | smc_llc_save_peer_uid(qentry); |
Karsten Graul | 4667bb4 | 2020-04-30 15:55:42 +0200 | [diff] [blame] | 1448 | rc = smc_llc_eval_conf_link(qentry, SMC_LLC_RESP); |
| 1449 | smc_llc_flow_qentry_del(&link->lgr->llc_flow_lcl); |
| 1450 | if (rc) |
Karsten Graul | 75d320d | 2018-03-01 13:51:31 +0100 | [diff] [blame] | 1451 | return SMC_CLC_DECL_RMBE_EC; |
| 1452 | |
Karsten Graul | 4667bb4 | 2020-04-30 15:55:42 +0200 | [diff] [blame] | 1453 | /* confirm_rkey is implicit on 1st contact */ |
| 1454 | smc->conn.rmb_desc->is_conf_rkey = true; |
Karsten Graul | 52bedf3 | 2018-03-01 13:51:32 +0100 | [diff] [blame] | 1455 | |
Karsten Graul | 00a049c | 2020-04-29 17:10:49 +0200 | [diff] [blame] | 1456 | smc_llc_link_active(link); |
Karsten Graul | 0a99be4 | 2020-05-05 15:01:20 +0200 | [diff] [blame] | 1457 | smcr_lgr_set_type(link->lgr, SMC_LGR_SINGLE); |
Karsten Graul | 52bedf3 | 2018-03-01 13:51:32 +0100 | [diff] [blame] | 1458 | |
Karsten Graul | 4667bb4 | 2020-04-30 15:55:42 +0200 | [diff] [blame] | 1459 | /* initial contact - try to establish second link */ |
Karsten Graul | b4ba465 | 2021-10-16 11:37:50 +0200 | [diff] [blame] | 1460 | smc_llc_srv_add_link(link, NULL); |
Karsten Graul | 75d320d | 2018-03-01 13:51:31 +0100 | [diff] [blame] | 1461 | return 0; |
Ursula Braun | 9bf9abe | 2017-01-09 16:55:21 +0100 | [diff] [blame] | 1462 | } |
| 1463 | |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1464 | /* listen worker: finish */ |
| 1465 | static void smc_listen_out(struct smc_sock *new_smc) |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 1466 | { |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 1467 | struct smc_sock *lsmc = new_smc->listen_smc; |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 1468 | struct sock *newsmcsk = &new_smc->sk; |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 1469 | |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 1470 | if (lsmc->sk.sk_state == SMC_LISTEN) { |
Karsten Graul | fd57770 | 2019-04-11 11:17:30 +0200 | [diff] [blame] | 1471 | lock_sock_nested(&lsmc->sk, SINGLE_DEPTH_NESTING); |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 1472 | smc_accept_enqueue(&lsmc->sk, newsmcsk); |
Karsten Graul | fd57770 | 2019-04-11 11:17:30 +0200 | [diff] [blame] | 1473 | release_sock(&lsmc->sk); |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 1474 | } else { /* no longer listening */ |
| 1475 | smc_close_non_accepted(newsmcsk); |
| 1476 | } |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 1477 | |
| 1478 | /* Wake up accept */ |
| 1479 | lsmc->sk.sk_data_ready(&lsmc->sk); |
| 1480 | sock_put(&lsmc->sk); /* sock_hold in smc_tcp_listen_work */ |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1481 | } |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 1482 | |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1483 | /* listen worker: finish in state connected */ |
| 1484 | static void smc_listen_out_connected(struct smc_sock *new_smc) |
| 1485 | { |
| 1486 | struct sock *newsmcsk = &new_smc->sk; |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 1487 | |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1488 | sk_refcnt_debug_inc(newsmcsk); |
| 1489 | if (newsmcsk->sk_state == SMC_INIT) |
| 1490 | newsmcsk->sk_state = SMC_ACTIVE; |
| 1491 | |
| 1492 | smc_listen_out(new_smc); |
| 1493 | } |
| 1494 | |
| 1495 | /* listen worker: finish in error state */ |
| 1496 | static void smc_listen_out_err(struct smc_sock *new_smc) |
| 1497 | { |
| 1498 | struct sock *newsmcsk = &new_smc->sk; |
Guvenc Gulce | 194730a | 2021-06-16 16:52:58 +0200 | [diff] [blame] | 1499 | struct net *net = sock_net(newsmcsk); |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1500 | |
Guvenc Gulce | 194730a | 2021-06-16 16:52:58 +0200 | [diff] [blame] | 1501 | this_cpu_inc(net->smc.smc_stats->srv_hshake_err_cnt); |
Ursula Braun | 51f1de7 | 2018-01-26 09:28:48 +0100 | [diff] [blame] | 1502 | if (newsmcsk->sk_state == SMC_INIT) |
| 1503 | sock_put(&new_smc->sk); /* passive closing */ |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 1504 | newsmcsk->sk_state = SMC_CLOSED; |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1505 | |
| 1506 | smc_listen_out(new_smc); |
| 1507 | } |
| 1508 | |
| 1509 | /* listen worker: decline and fall back if possible */ |
| 1510 | static void smc_listen_decline(struct smc_sock *new_smc, int reason_code, |
Karsten Graul | 4a9baf4 | 2020-10-23 20:48:28 +0200 | [diff] [blame] | 1511 | int local_first, u8 version) |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1512 | { |
| 1513 | /* RDMA setup failed, switch back to TCP */ |
Karsten Graul | 8cf3f3e | 2020-12-01 20:20:36 +0100 | [diff] [blame] | 1514 | smc_conn_abort(new_smc, local_first); |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1515 | if (reason_code < 0) { /* error, no fallback possible */ |
| 1516 | smc_listen_out_err(new_smc); |
| 1517 | return; |
| 1518 | } |
Guvenc Gulce | e0e4b8f | 2021-06-16 16:52:55 +0200 | [diff] [blame] | 1519 | smc_switch_to_fallback(new_smc, reason_code); |
Karsten Graul | 603cc14 | 2018-07-25 16:35:32 +0200 | [diff] [blame] | 1520 | if (reason_code && reason_code != SMC_CLC_DECL_PEERDECL) { |
Ursula Braun | e8d726c8 | 2020-09-26 12:44:32 +0200 | [diff] [blame] | 1521 | if (smc_clc_send_decline(new_smc, reason_code, version) < 0) { |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1522 | smc_listen_out_err(new_smc); |
| 1523 | return; |
| 1524 | } |
| 1525 | } |
| 1526 | smc_listen_out_connected(new_smc); |
| 1527 | } |
| 1528 | |
Ursula Braun | 5c21c4c | 2020-09-26 12:44:29 +0200 | [diff] [blame] | 1529 | /* listen worker: version checking */ |
| 1530 | static int smc_listen_v2_check(struct smc_sock *new_smc, |
| 1531 | struct smc_clc_msg_proposal *pclc, |
| 1532 | struct smc_init_info *ini) |
| 1533 | { |
| 1534 | struct smc_clc_smcd_v2_extension *pclc_smcd_v2_ext; |
| 1535 | struct smc_clc_v2_extension *pclc_v2_ext; |
Karsten Graul | 3752404 | 2020-10-31 19:19:38 +0100 | [diff] [blame] | 1536 | int rc = SMC_CLC_DECL_PEERNOSMC; |
Ursula Braun | 5c21c4c | 2020-09-26 12:44:29 +0200 | [diff] [blame] | 1537 | |
| 1538 | ini->smc_type_v1 = pclc->hdr.typev1; |
| 1539 | ini->smc_type_v2 = pclc->hdr.typev2; |
Karsten Graul | e49300a | 2021-10-16 11:37:46 +0200 | [diff] [blame] | 1540 | ini->smcd_version = smcd_indicated(ini->smc_type_v1) ? SMC_V1 : 0; |
| 1541 | ini->smcr_version = smcr_indicated(ini->smc_type_v1) ? SMC_V1 : 0; |
| 1542 | if (pclc->hdr.version > SMC_V1) { |
| 1543 | if (smcd_indicated(ini->smc_type_v2)) |
| 1544 | ini->smcd_version |= SMC_V2; |
| 1545 | if (smcr_indicated(ini->smc_type_v2)) |
| 1546 | ini->smcr_version |= SMC_V2; |
Karsten Graul | 3752404 | 2020-10-31 19:19:38 +0100 | [diff] [blame] | 1547 | } |
Karsten Graul | e49300a | 2021-10-16 11:37:46 +0200 | [diff] [blame] | 1548 | if (!(ini->smcd_version & SMC_V2) && !(ini->smcr_version & SMC_V2)) { |
| 1549 | rc = SMC_CLC_DECL_PEERNOSMC; |
Ursula Braun | 5c21c4c | 2020-09-26 12:44:29 +0200 | [diff] [blame] | 1550 | goto out; |
| 1551 | } |
| 1552 | pclc_v2_ext = smc_get_clc_v2_ext(pclc); |
| 1553 | if (!pclc_v2_ext) { |
| 1554 | ini->smcd_version &= ~SMC_V2; |
Karsten Graul | e49300a | 2021-10-16 11:37:46 +0200 | [diff] [blame] | 1555 | ini->smcr_version &= ~SMC_V2; |
Karsten Graul | 3752404 | 2020-10-31 19:19:38 +0100 | [diff] [blame] | 1556 | rc = SMC_CLC_DECL_NOV2EXT; |
Ursula Braun | 5c21c4c | 2020-09-26 12:44:29 +0200 | [diff] [blame] | 1557 | goto out; |
| 1558 | } |
| 1559 | pclc_smcd_v2_ext = smc_get_clc_smcd_v2_ext(pclc_v2_ext); |
Karsten Graul | e49300a | 2021-10-16 11:37:46 +0200 | [diff] [blame] | 1560 | if (ini->smcd_version & SMC_V2) { |
| 1561 | if (!smc_ism_is_v2_capable()) { |
| 1562 | ini->smcd_version &= ~SMC_V2; |
| 1563 | rc = SMC_CLC_DECL_NOISM2SUPP; |
| 1564 | } else if (!pclc_smcd_v2_ext) { |
| 1565 | ini->smcd_version &= ~SMC_V2; |
| 1566 | rc = SMC_CLC_DECL_NOV2DEXT; |
| 1567 | } else if (!pclc_v2_ext->hdr.eid_cnt && |
| 1568 | !pclc_v2_ext->hdr.flag.seid) { |
| 1569 | ini->smcd_version &= ~SMC_V2; |
| 1570 | rc = SMC_CLC_DECL_NOUEID; |
| 1571 | } |
| 1572 | } |
| 1573 | if (ini->smcr_version & SMC_V2) { |
| 1574 | if (!pclc_v2_ext->hdr.eid_cnt) { |
| 1575 | ini->smcr_version &= ~SMC_V2; |
| 1576 | rc = SMC_CLC_DECL_NOUEID; |
| 1577 | } |
Karsten Graul | 3752404 | 2020-10-31 19:19:38 +0100 | [diff] [blame] | 1578 | } |
Ursula Braun | 5c21c4c | 2020-09-26 12:44:29 +0200 | [diff] [blame] | 1579 | |
| 1580 | out: |
Karsten Graul | e49300a | 2021-10-16 11:37:46 +0200 | [diff] [blame] | 1581 | if (!ini->smcd_version && !ini->smcr_version) |
Karsten Graul | 3752404 | 2020-10-31 19:19:38 +0100 | [diff] [blame] | 1582 | return rc; |
Ursula Braun | 5c21c4c | 2020-09-26 12:44:29 +0200 | [diff] [blame] | 1583 | |
| 1584 | return 0; |
| 1585 | } |
| 1586 | |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1587 | /* listen worker: check prefixes */ |
Karsten Graul | 5988669 | 2019-04-12 12:57:25 +0200 | [diff] [blame] | 1588 | static int smc_listen_prfx_check(struct smc_sock *new_smc, |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1589 | struct smc_clc_msg_proposal *pclc) |
| 1590 | { |
| 1591 | struct smc_clc_msg_proposal_prefix *pclc_prfx; |
| 1592 | struct socket *newclcsock = new_smc->clcsock; |
| 1593 | |
Ursula Braun | 5c21c4c | 2020-09-26 12:44:29 +0200 | [diff] [blame] | 1594 | if (pclc->hdr.typev1 == SMC_TYPE_N) |
| 1595 | return 0; |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1596 | pclc_prfx = smc_clc_proposal_get_prefix(pclc); |
| 1597 | if (smc_clc_prfx_match(newclcsock, pclc_prfx)) |
Karsten Graul | 5988669 | 2019-04-12 12:57:25 +0200 | [diff] [blame] | 1598 | return SMC_CLC_DECL_DIFFPREFIX; |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1599 | |
| 1600 | return 0; |
| 1601 | } |
| 1602 | |
| 1603 | /* listen worker: initialize connection and buffers */ |
| 1604 | static int smc_listen_rdma_init(struct smc_sock *new_smc, |
Karsten Graul | 7a62725a | 2019-04-12 12:57:30 +0200 | [diff] [blame] | 1605 | struct smc_init_info *ini) |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1606 | { |
Karsten Graul | 7a62725a | 2019-04-12 12:57:30 +0200 | [diff] [blame] | 1607 | int rc; |
| 1608 | |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1609 | /* allocate connection / link group */ |
Karsten Graul | 7a62725a | 2019-04-12 12:57:30 +0200 | [diff] [blame] | 1610 | rc = smc_conn_create(new_smc, ini); |
| 1611 | if (rc) |
| 1612 | return rc; |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1613 | |
| 1614 | /* create send buffer and rmb */ |
Hans Wippel | c6ba7c9 | 2018-06-28 19:05:07 +0200 | [diff] [blame] | 1615 | if (smc_buf_create(new_smc, false)) |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1616 | return SMC_CLC_DECL_MEM; |
| 1617 | |
| 1618 | return 0; |
| 1619 | } |
| 1620 | |
Hans Wippel | 4134984 | 2018-06-28 19:05:11 +0200 | [diff] [blame] | 1621 | /* listen worker: initialize connection and buffers for SMC-D */ |
| 1622 | static int smc_listen_ism_init(struct smc_sock *new_smc, |
Karsten Graul | 7a62725a | 2019-04-12 12:57:30 +0200 | [diff] [blame] | 1623 | struct smc_init_info *ini) |
Hans Wippel | 4134984 | 2018-06-28 19:05:11 +0200 | [diff] [blame] | 1624 | { |
Karsten Graul | 7a62725a | 2019-04-12 12:57:30 +0200 | [diff] [blame] | 1625 | int rc; |
Hans Wippel | 4134984 | 2018-06-28 19:05:11 +0200 | [diff] [blame] | 1626 | |
Karsten Graul | 7a62725a | 2019-04-12 12:57:30 +0200 | [diff] [blame] | 1627 | rc = smc_conn_create(new_smc, ini); |
| 1628 | if (rc) |
| 1629 | return rc; |
Hans Wippel | 4134984 | 2018-06-28 19:05:11 +0200 | [diff] [blame] | 1630 | |
Hans Wippel | 4134984 | 2018-06-28 19:05:11 +0200 | [diff] [blame] | 1631 | /* Create send and receive buffers */ |
Karsten Graul | 72b7f6c | 2020-07-26 20:34:28 +0200 | [diff] [blame] | 1632 | rc = smc_buf_create(new_smc, true); |
| 1633 | if (rc) { |
Karsten Graul | 8cf3f3e | 2020-12-01 20:20:36 +0100 | [diff] [blame] | 1634 | smc_conn_abort(new_smc, ini->first_contact_local); |
Karsten Graul | 72b7f6c | 2020-07-26 20:34:28 +0200 | [diff] [blame] | 1635 | return (rc == -ENOSPC) ? SMC_CLC_DECL_MAX_DMB : |
| 1636 | SMC_CLC_DECL_MEM; |
Hans Wippel | 4134984 | 2018-06-28 19:05:11 +0200 | [diff] [blame] | 1637 | } |
| 1638 | |
| 1639 | return 0; |
| 1640 | } |
| 1641 | |
Ursula Braun | 5c21c4c | 2020-09-26 12:44:29 +0200 | [diff] [blame] | 1642 | static bool smc_is_already_selected(struct smcd_dev *smcd, |
| 1643 | struct smc_init_info *ini, |
| 1644 | int matches) |
| 1645 | { |
| 1646 | int i; |
| 1647 | |
| 1648 | for (i = 0; i < matches; i++) |
| 1649 | if (smcd == ini->ism_dev[i]) |
| 1650 | return true; |
| 1651 | |
| 1652 | return false; |
| 1653 | } |
| 1654 | |
| 1655 | /* check for ISM devices matching proposed ISM devices */ |
| 1656 | static void smc_check_ism_v2_match(struct smc_init_info *ini, |
| 1657 | u16 proposed_chid, u64 proposed_gid, |
| 1658 | unsigned int *matches) |
| 1659 | { |
| 1660 | struct smcd_dev *smcd; |
| 1661 | |
| 1662 | list_for_each_entry(smcd, &smcd_dev_list.list, list) { |
| 1663 | if (smcd->going_away) |
| 1664 | continue; |
| 1665 | if (smc_is_already_selected(smcd, ini, *matches)) |
| 1666 | continue; |
| 1667 | if (smc_ism_get_chid(smcd) == proposed_chid && |
| 1668 | !smc_ism_cantalk(proposed_gid, ISM_RESERVED_VLANID, smcd)) { |
| 1669 | ini->ism_peer_gid[*matches] = proposed_gid; |
| 1670 | ini->ism_dev[*matches] = smcd; |
| 1671 | (*matches)++; |
| 1672 | break; |
| 1673 | } |
| 1674 | } |
| 1675 | } |
| 1676 | |
Karsten Graul | 3752404 | 2020-10-31 19:19:38 +0100 | [diff] [blame] | 1677 | static void smc_find_ism_store_rc(u32 rc, struct smc_init_info *ini) |
| 1678 | { |
| 1679 | if (!ini->rc) |
| 1680 | ini->rc = rc; |
| 1681 | } |
| 1682 | |
Ursula Braun | 5c21c4c | 2020-09-26 12:44:29 +0200 | [diff] [blame] | 1683 | static void smc_find_ism_v2_device_serv(struct smc_sock *new_smc, |
| 1684 | struct smc_clc_msg_proposal *pclc, |
| 1685 | struct smc_init_info *ini) |
| 1686 | { |
| 1687 | struct smc_clc_smcd_v2_extension *smcd_v2_ext; |
| 1688 | struct smc_clc_v2_extension *smc_v2_ext; |
| 1689 | struct smc_clc_msg_smcd *pclc_smcd; |
| 1690 | unsigned int matches = 0; |
Karsten Graul | f29fa00 | 2020-10-07 22:57:43 +0200 | [diff] [blame] | 1691 | u8 smcd_version; |
Ursula Braun | 5c21c4c | 2020-09-26 12:44:29 +0200 | [diff] [blame] | 1692 | u8 *eid = NULL; |
Karsten Graul | 3752404 | 2020-10-31 19:19:38 +0100 | [diff] [blame] | 1693 | int i, rc; |
Ursula Braun | 5c21c4c | 2020-09-26 12:44:29 +0200 | [diff] [blame] | 1694 | |
| 1695 | if (!(ini->smcd_version & SMC_V2) || !smcd_indicated(ini->smc_type_v2)) |
Karsten Graul | f29fa00 | 2020-10-07 22:57:43 +0200 | [diff] [blame] | 1696 | goto not_found; |
Ursula Braun | 5c21c4c | 2020-09-26 12:44:29 +0200 | [diff] [blame] | 1697 | |
| 1698 | pclc_smcd = smc_get_clc_msg_smcd(pclc); |
| 1699 | smc_v2_ext = smc_get_clc_v2_ext(pclc); |
| 1700 | smcd_v2_ext = smc_get_clc_smcd_v2_ext(smc_v2_ext); |
Ursula Braun | 5c21c4c | 2020-09-26 12:44:29 +0200 | [diff] [blame] | 1701 | |
| 1702 | mutex_lock(&smcd_dev_list.mutex); |
| 1703 | if (pclc_smcd->ism.chid) |
| 1704 | /* check for ISM device matching proposed native ISM device */ |
| 1705 | smc_check_ism_v2_match(ini, ntohs(pclc_smcd->ism.chid), |
| 1706 | ntohll(pclc_smcd->ism.gid), &matches); |
| 1707 | for (i = 1; i <= smc_v2_ext->hdr.ism_gid_cnt; i++) { |
| 1708 | /* check for ISM devices matching proposed non-native ISM |
| 1709 | * devices |
| 1710 | */ |
| 1711 | smc_check_ism_v2_match(ini, |
| 1712 | ntohs(smcd_v2_ext->gidchid[i - 1].chid), |
| 1713 | ntohll(smcd_v2_ext->gidchid[i - 1].gid), |
| 1714 | &matches); |
| 1715 | } |
| 1716 | mutex_unlock(&smcd_dev_list.mutex); |
| 1717 | |
Karsten Graul | e49300a | 2021-10-16 11:37:46 +0200 | [diff] [blame] | 1718 | if (!ini->ism_dev[0]) { |
| 1719 | smc_find_ism_store_rc(SMC_CLC_DECL_NOSMCD2DEV, ini); |
Ursula Braun | 5c21c4c | 2020-09-26 12:44:29 +0200 | [diff] [blame] | 1720 | goto not_found; |
Karsten Graul | e49300a | 2021-10-16 11:37:46 +0200 | [diff] [blame] | 1721 | } |
Karsten Graul | fa08666 | 2021-09-14 10:35:05 +0200 | [diff] [blame] | 1722 | |
Karsten Graul | 11a26c5 | 2021-09-14 10:35:06 +0200 | [diff] [blame] | 1723 | smc_ism_get_system_eid(&eid); |
Karsten Graul | fa08666 | 2021-09-14 10:35:05 +0200 | [diff] [blame] | 1724 | if (!smc_clc_match_eid(ini->negotiated_eid, smc_v2_ext, |
| 1725 | smcd_v2_ext->system_eid, eid)) |
| 1726 | goto not_found; |
Ursula Braun | 5c21c4c | 2020-09-26 12:44:29 +0200 | [diff] [blame] | 1727 | |
| 1728 | /* separate - outside the smcd_dev_list.lock */ |
Karsten Graul | f29fa00 | 2020-10-07 22:57:43 +0200 | [diff] [blame] | 1729 | smcd_version = ini->smcd_version; |
Ursula Braun | 5c21c4c | 2020-09-26 12:44:29 +0200 | [diff] [blame] | 1730 | for (i = 0; i < matches; i++) { |
| 1731 | ini->smcd_version = SMC_V2; |
| 1732 | ini->is_smcd = true; |
| 1733 | ini->ism_selected = i; |
Karsten Graul | 3752404 | 2020-10-31 19:19:38 +0100 | [diff] [blame] | 1734 | rc = smc_listen_ism_init(new_smc, ini); |
| 1735 | if (rc) { |
| 1736 | smc_find_ism_store_rc(rc, ini); |
Ursula Braun | 5c21c4c | 2020-09-26 12:44:29 +0200 | [diff] [blame] | 1737 | /* try next active ISM device */ |
| 1738 | continue; |
Karsten Graul | 3752404 | 2020-10-31 19:19:38 +0100 | [diff] [blame] | 1739 | } |
Ursula Braun | 5c21c4c | 2020-09-26 12:44:29 +0200 | [diff] [blame] | 1740 | return; /* matching and usable V2 ISM device found */ |
| 1741 | } |
Karsten Graul | f29fa00 | 2020-10-07 22:57:43 +0200 | [diff] [blame] | 1742 | /* no V2 ISM device could be initialized */ |
| 1743 | ini->smcd_version = smcd_version; /* restore original value */ |
Karsten Graul | fa08666 | 2021-09-14 10:35:05 +0200 | [diff] [blame] | 1744 | ini->negotiated_eid[0] = 0; |
Ursula Braun | 5c21c4c | 2020-09-26 12:44:29 +0200 | [diff] [blame] | 1745 | |
| 1746 | not_found: |
| 1747 | ini->smcd_version &= ~SMC_V2; |
| 1748 | ini->ism_dev[0] = NULL; |
| 1749 | ini->is_smcd = false; |
| 1750 | } |
| 1751 | |
| 1752 | static void smc_find_ism_v1_device_serv(struct smc_sock *new_smc, |
| 1753 | struct smc_clc_msg_proposal *pclc, |
| 1754 | struct smc_init_info *ini) |
Ursula Braun | 7affc80 | 2020-09-26 12:44:21 +0200 | [diff] [blame] | 1755 | { |
| 1756 | struct smc_clc_msg_smcd *pclc_smcd = smc_get_clc_msg_smcd(pclc); |
Karsten Graul | 3752404 | 2020-10-31 19:19:38 +0100 | [diff] [blame] | 1757 | int rc = 0; |
Ursula Braun | 7affc80 | 2020-09-26 12:44:21 +0200 | [diff] [blame] | 1758 | |
Ursula Braun | 5c21c4c | 2020-09-26 12:44:29 +0200 | [diff] [blame] | 1759 | /* check if ISM V1 is available */ |
| 1760 | if (!(ini->smcd_version & SMC_V1) || !smcd_indicated(ini->smc_type_v1)) |
Ursula Braun | 7affc80 | 2020-09-26 12:44:21 +0200 | [diff] [blame] | 1761 | goto not_found; |
| 1762 | ini->is_smcd = true; /* prepare ISM check */ |
Ursula Braun | 8c3dca3 | 2020-09-26 12:44:28 +0200 | [diff] [blame] | 1763 | ini->ism_peer_gid[0] = ntohll(pclc_smcd->ism.gid); |
Karsten Graul | 3752404 | 2020-10-31 19:19:38 +0100 | [diff] [blame] | 1764 | rc = smc_find_ism_device(new_smc, ini); |
| 1765 | if (rc) |
Ursula Braun | 7affc80 | 2020-09-26 12:44:21 +0200 | [diff] [blame] | 1766 | goto not_found; |
Ursula Braun | 5c21c4c | 2020-09-26 12:44:29 +0200 | [diff] [blame] | 1767 | ini->ism_selected = 0; |
Karsten Graul | 3752404 | 2020-10-31 19:19:38 +0100 | [diff] [blame] | 1768 | rc = smc_listen_ism_init(new_smc, ini); |
| 1769 | if (!rc) |
Ursula Braun | 5c21c4c | 2020-09-26 12:44:29 +0200 | [diff] [blame] | 1770 | return; /* V1 ISM device found */ |
Ursula Braun | 7affc80 | 2020-09-26 12:44:21 +0200 | [diff] [blame] | 1771 | |
| 1772 | not_found: |
Karsten Graul | 3752404 | 2020-10-31 19:19:38 +0100 | [diff] [blame] | 1773 | smc_find_ism_store_rc(rc, ini); |
Karsten Graul | e49300a | 2021-10-16 11:37:46 +0200 | [diff] [blame] | 1774 | ini->smcd_version &= ~SMC_V1; |
Ursula Braun | 3fc6493 | 2020-09-26 12:44:23 +0200 | [diff] [blame] | 1775 | ini->ism_dev[0] = NULL; |
Ursula Braun | 7affc80 | 2020-09-26 12:44:21 +0200 | [diff] [blame] | 1776 | ini->is_smcd = false; |
| 1777 | } |
| 1778 | |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1779 | /* listen worker: register buffers */ |
Ursula Braun | 5ac54d8 | 2020-09-10 18:48:21 +0200 | [diff] [blame] | 1780 | static int smc_listen_rdma_reg(struct smc_sock *new_smc, bool local_first) |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1781 | { |
Karsten Graul | b924754 | 2020-04-29 17:10:41 +0200 | [diff] [blame] | 1782 | struct smc_connection *conn = &new_smc->conn; |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1783 | |
Ursula Braun | 5ac54d8 | 2020-09-10 18:48:21 +0200 | [diff] [blame] | 1784 | if (!local_first) { |
Karsten Graul | 7562a13 | 2020-05-01 12:48:01 +0200 | [diff] [blame] | 1785 | if (smcr_lgr_reg_rmbs(conn->lnk, conn->rmb_desc)) |
Karsten Graul | c7674c0 | 2018-11-22 10:26:43 +0100 | [diff] [blame] | 1786 | return SMC_CLC_DECL_ERR_REGRMB; |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1787 | } |
| 1788 | smc_rmb_sync_sg_for_device(&new_smc->conn); |
| 1789 | |
| 1790 | return 0; |
| 1791 | } |
| 1792 | |
Karsten Graul | e49300a | 2021-10-16 11:37:46 +0200 | [diff] [blame] | 1793 | static void smc_find_rdma_v2_device_serv(struct smc_sock *new_smc, |
| 1794 | struct smc_clc_msg_proposal *pclc, |
| 1795 | struct smc_init_info *ini) |
| 1796 | { |
| 1797 | struct smc_clc_v2_extension *smc_v2_ext; |
| 1798 | u8 smcr_version; |
| 1799 | int rc; |
| 1800 | |
| 1801 | if (!(ini->smcr_version & SMC_V2) || !smcr_indicated(ini->smc_type_v2)) |
| 1802 | goto not_found; |
| 1803 | |
| 1804 | smc_v2_ext = smc_get_clc_v2_ext(pclc); |
| 1805 | if (!smc_clc_match_eid(ini->negotiated_eid, smc_v2_ext, NULL, NULL)) |
| 1806 | goto not_found; |
| 1807 | |
| 1808 | /* prepare RDMA check */ |
| 1809 | memcpy(ini->peer_systemid, pclc->lcl.id_for_peer, SMC_SYSTEMID_LEN); |
| 1810 | memcpy(ini->peer_gid, smc_v2_ext->roce, SMC_GID_SIZE); |
| 1811 | memcpy(ini->peer_mac, pclc->lcl.mac, ETH_ALEN); |
| 1812 | ini->check_smcrv2 = true; |
| 1813 | ini->smcrv2.clc_sk = new_smc->clcsock->sk; |
| 1814 | ini->smcrv2.saddr = new_smc->clcsock->sk->sk_rcv_saddr; |
| 1815 | ini->smcrv2.daddr = smc_ib_gid_to_ipv4(smc_v2_ext->roce); |
| 1816 | rc = smc_find_rdma_device(new_smc, ini); |
| 1817 | if (rc) { |
| 1818 | smc_find_ism_store_rc(rc, ini); |
| 1819 | goto not_found; |
| 1820 | } |
| 1821 | if (!ini->smcrv2.uses_gateway) |
| 1822 | memcpy(ini->smcrv2.nexthop_mac, pclc->lcl.mac, ETH_ALEN); |
| 1823 | |
| 1824 | smcr_version = ini->smcr_version; |
| 1825 | ini->smcr_version = SMC_V2; |
| 1826 | rc = smc_listen_rdma_init(new_smc, ini); |
| 1827 | if (!rc) |
| 1828 | rc = smc_listen_rdma_reg(new_smc, ini->first_contact_local); |
| 1829 | if (!rc) |
| 1830 | return; |
| 1831 | ini->smcr_version = smcr_version; |
| 1832 | smc_find_ism_store_rc(rc, ini); |
| 1833 | |
| 1834 | not_found: |
| 1835 | ini->smcr_version &= ~SMC_V2; |
| 1836 | ini->check_smcrv2 = false; |
| 1837 | } |
| 1838 | |
Ursula Braun | 5c21c4c | 2020-09-26 12:44:29 +0200 | [diff] [blame] | 1839 | static int smc_find_rdma_v1_device_serv(struct smc_sock *new_smc, |
| 1840 | struct smc_clc_msg_proposal *pclc, |
| 1841 | struct smc_init_info *ini) |
Ursula Braun | 7affc80 | 2020-09-26 12:44:21 +0200 | [diff] [blame] | 1842 | { |
| 1843 | int rc; |
| 1844 | |
Karsten Graul | e49300a | 2021-10-16 11:37:46 +0200 | [diff] [blame] | 1845 | if (!(ini->smcr_version & SMC_V1) || !smcr_indicated(ini->smc_type_v1)) |
Ursula Braun | 7affc80 | 2020-09-26 12:44:21 +0200 | [diff] [blame] | 1846 | return SMC_CLC_DECL_NOSMCDEV; |
| 1847 | |
| 1848 | /* prepare RDMA check */ |
Karsten Graul | e49300a | 2021-10-16 11:37:46 +0200 | [diff] [blame] | 1849 | memcpy(ini->peer_systemid, pclc->lcl.id_for_peer, SMC_SYSTEMID_LEN); |
| 1850 | memcpy(ini->peer_gid, pclc->lcl.gid, SMC_GID_SIZE); |
| 1851 | memcpy(ini->peer_mac, pclc->lcl.mac, ETH_ALEN); |
Ursula Braun | 7affc80 | 2020-09-26 12:44:21 +0200 | [diff] [blame] | 1852 | rc = smc_find_rdma_device(new_smc, ini); |
| 1853 | if (rc) { |
| 1854 | /* no RDMA device found */ |
Karsten Graul | e49300a | 2021-10-16 11:37:46 +0200 | [diff] [blame] | 1855 | return SMC_CLC_DECL_NOSMCDEV; |
Ursula Braun | 7affc80 | 2020-09-26 12:44:21 +0200 | [diff] [blame] | 1856 | } |
| 1857 | rc = smc_listen_rdma_init(new_smc, ini); |
| 1858 | if (rc) |
| 1859 | return rc; |
| 1860 | return smc_listen_rdma_reg(new_smc, ini->first_contact_local); |
| 1861 | } |
| 1862 | |
| 1863 | /* determine the local device matching to proposal */ |
| 1864 | static int smc_listen_find_device(struct smc_sock *new_smc, |
| 1865 | struct smc_clc_msg_proposal *pclc, |
| 1866 | struct smc_init_info *ini) |
| 1867 | { |
Karsten Graul | e49300a | 2021-10-16 11:37:46 +0200 | [diff] [blame] | 1868 | int prfx_rc; |
Ursula Braun | 5c21c4c | 2020-09-26 12:44:29 +0200 | [diff] [blame] | 1869 | |
| 1870 | /* check for ISM device matching V2 proposed device */ |
| 1871 | smc_find_ism_v2_device_serv(new_smc, pclc, ini); |
| 1872 | if (ini->ism_dev[0]) |
Ursula Braun | 7affc80 | 2020-09-26 12:44:21 +0200 | [diff] [blame] | 1873 | return 0; |
Ursula Braun | 5c21c4c | 2020-09-26 12:44:29 +0200 | [diff] [blame] | 1874 | |
Karsten Graul | e49300a | 2021-10-16 11:37:46 +0200 | [diff] [blame] | 1875 | /* check for matching IP prefix and subnet length (V1) */ |
| 1876 | prfx_rc = smc_listen_prfx_check(new_smc, pclc); |
| 1877 | if (prfx_rc) |
| 1878 | smc_find_ism_store_rc(prfx_rc, ini); |
Ursula Braun | 5c21c4c | 2020-09-26 12:44:29 +0200 | [diff] [blame] | 1879 | |
| 1880 | /* get vlan id from IP device */ |
| 1881 | if (smc_vlan_by_tcpsk(new_smc->clcsock, ini)) |
Karsten Graul | 3752404 | 2020-10-31 19:19:38 +0100 | [diff] [blame] | 1882 | return ini->rc ?: SMC_CLC_DECL_GETVLANERR; |
Ursula Braun | 5c21c4c | 2020-09-26 12:44:29 +0200 | [diff] [blame] | 1883 | |
| 1884 | /* check for ISM device matching V1 proposed device */ |
Karsten Graul | e49300a | 2021-10-16 11:37:46 +0200 | [diff] [blame] | 1885 | if (!prfx_rc) |
| 1886 | smc_find_ism_v1_device_serv(new_smc, pclc, ini); |
Ursula Braun | 5c21c4c | 2020-09-26 12:44:29 +0200 | [diff] [blame] | 1887 | if (ini->ism_dev[0]) |
| 1888 | return 0; |
| 1889 | |
Karsten Graul | e49300a | 2021-10-16 11:37:46 +0200 | [diff] [blame] | 1890 | if (!smcr_indicated(pclc->hdr.typev1) && |
| 1891 | !smcr_indicated(pclc->hdr.typev2)) |
Karsten Graul | 3752404 | 2020-10-31 19:19:38 +0100 | [diff] [blame] | 1892 | /* skip RDMA and decline */ |
| 1893 | return ini->rc ?: SMC_CLC_DECL_NOSMCDDEV; |
Ursula Braun | 7affc80 | 2020-09-26 12:44:21 +0200 | [diff] [blame] | 1894 | |
Karsten Graul | e49300a | 2021-10-16 11:37:46 +0200 | [diff] [blame] | 1895 | /* check if RDMA V2 is available */ |
| 1896 | smc_find_rdma_v2_device_serv(new_smc, pclc, ini); |
| 1897 | if (ini->smcrv2.ib_dev_v2) |
| 1898 | return 0; |
Karsten Graul | 3752404 | 2020-10-31 19:19:38 +0100 | [diff] [blame] | 1899 | |
Karsten Graul | e49300a | 2021-10-16 11:37:46 +0200 | [diff] [blame] | 1900 | /* check if RDMA V1 is available */ |
| 1901 | if (!prfx_rc) { |
| 1902 | int rc; |
| 1903 | |
| 1904 | rc = smc_find_rdma_v1_device_serv(new_smc, pclc, ini); |
| 1905 | smc_find_ism_store_rc(rc, ini); |
| 1906 | return (!rc) ? 0 : ini->rc; |
| 1907 | } |
| 1908 | return SMC_CLC_DECL_NOSMCDEV; |
Ursula Braun | 7affc80 | 2020-09-26 12:44:21 +0200 | [diff] [blame] | 1909 | } |
| 1910 | |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1911 | /* listen worker: finish RDMA setup */ |
Ursula Braun | 1ca52fc | 2018-09-18 15:46:35 +0200 | [diff] [blame] | 1912 | static int smc_listen_rdma_finish(struct smc_sock *new_smc, |
| 1913 | struct smc_clc_msg_accept_confirm *cclc, |
Karsten Graul | e49300a | 2021-10-16 11:37:46 +0200 | [diff] [blame] | 1914 | bool local_first, |
| 1915 | struct smc_init_info *ini) |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1916 | { |
Karsten Graul | 387707f | 2020-04-29 17:10:40 +0200 | [diff] [blame] | 1917 | struct smc_link *link = new_smc->conn.lnk; |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1918 | int reason_code = 0; |
| 1919 | |
Ursula Braun | 5ac54d8 | 2020-09-10 18:48:21 +0200 | [diff] [blame] | 1920 | if (local_first) |
Karsten Graul | e49300a | 2021-10-16 11:37:46 +0200 | [diff] [blame] | 1921 | smc_link_save_peer_info(link, cclc, ini); |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1922 | |
Ursula Braun | 0c881ad | 2020-09-10 18:48:25 +0200 | [diff] [blame] | 1923 | if (smc_rmb_rtoken_handling(&new_smc->conn, link, cclc)) |
| 1924 | return SMC_CLC_DECL_ERR_RTOK; |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1925 | |
Ursula Braun | 5ac54d8 | 2020-09-10 18:48:21 +0200 | [diff] [blame] | 1926 | if (local_first) { |
Ursula Braun | 0c881ad | 2020-09-10 18:48:25 +0200 | [diff] [blame] | 1927 | if (smc_ib_ready_link(link)) |
| 1928 | return SMC_CLC_DECL_ERR_RDYLNK; |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1929 | /* QP confirmation over RoCE fabric */ |
Karsten Graul | 4667bb4 | 2020-04-30 15:55:42 +0200 | [diff] [blame] | 1930 | smc_llc_flow_initiate(link->lgr, SMC_LLC_FLOW_ADD_LINK); |
Karsten Graul | b924754 | 2020-04-29 17:10:41 +0200 | [diff] [blame] | 1931 | reason_code = smcr_serv_conf_first_link(new_smc); |
Karsten Graul | 4667bb4 | 2020-04-30 15:55:42 +0200 | [diff] [blame] | 1932 | smc_llc_flow_stop(link->lgr, &link->lgr->llc_flow_lcl); |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1933 | } |
Ursula Braun | 1ca52fc | 2018-09-18 15:46:35 +0200 | [diff] [blame] | 1934 | return reason_code; |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1935 | } |
| 1936 | |
Ursula Braun | 7affc80 | 2020-09-26 12:44:21 +0200 | [diff] [blame] | 1937 | /* setup for connection of server */ |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1938 | static void smc_listen_work(struct work_struct *work) |
| 1939 | { |
| 1940 | struct smc_sock *new_smc = container_of(work, struct smc_sock, |
| 1941 | smc_listen_work); |
| 1942 | struct socket *newclcsock = new_smc->clcsock; |
Ursula Braun | a7c9c5f | 2020-09-26 12:44:30 +0200 | [diff] [blame] | 1943 | struct smc_clc_msg_accept_confirm *cclc; |
Ursula Braun | 6bb14e48 | 2020-09-10 18:48:22 +0200 | [diff] [blame] | 1944 | struct smc_clc_msg_proposal_area *buf; |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1945 | struct smc_clc_msg_proposal *pclc; |
Ursula Braun | 3fc6493 | 2020-09-26 12:44:23 +0200 | [diff] [blame] | 1946 | struct smc_init_info *ini = NULL; |
Karsten Graul | e49300a | 2021-10-16 11:37:46 +0200 | [diff] [blame] | 1947 | u8 proposal_version = SMC_V1; |
| 1948 | u8 accept_version; |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1949 | int rc = 0; |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1950 | |
Karsten Graul | fd57770 | 2019-04-11 11:17:30 +0200 | [diff] [blame] | 1951 | if (new_smc->listen_smc->sk.sk_state != SMC_LISTEN) |
| 1952 | return smc_listen_out_err(new_smc); |
| 1953 | |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1954 | if (new_smc->use_fallback) { |
| 1955 | smc_listen_out_connected(new_smc); |
| 1956 | return; |
| 1957 | } |
| 1958 | |
| 1959 | /* check if peer is smc capable */ |
| 1960 | if (!tcp_sk(newclcsock->sk)->syn_smc) { |
Guvenc Gulce | e0e4b8f | 2021-06-16 16:52:55 +0200 | [diff] [blame] | 1961 | smc_switch_to_fallback(new_smc, SMC_CLC_DECL_PEERNOSMC); |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1962 | smc_listen_out_connected(new_smc); |
| 1963 | return; |
| 1964 | } |
| 1965 | |
| 1966 | /* do inband token exchange - |
| 1967 | * wait for and receive SMC Proposal CLC message |
| 1968 | */ |
Ursula Braun | 6bb14e48 | 2020-09-10 18:48:22 +0200 | [diff] [blame] | 1969 | buf = kzalloc(sizeof(*buf), GFP_KERNEL); |
| 1970 | if (!buf) { |
| 1971 | rc = SMC_CLC_DECL_MEM; |
| 1972 | goto out_decl; |
| 1973 | } |
| 1974 | pclc = (struct smc_clc_msg_proposal *)buf; |
| 1975 | rc = smc_clc_wait_msg(new_smc, pclc, sizeof(*buf), |
Karsten Graul | 228bae0 | 2019-04-12 12:57:28 +0200 | [diff] [blame] | 1976 | SMC_CLC_PROPOSAL, CLC_WAIT_TIME); |
Karsten Graul | 9aa68d2 | 2019-04-12 12:57:29 +0200 | [diff] [blame] | 1977 | if (rc) |
| 1978 | goto out_decl; |
Karsten Graul | e49300a | 2021-10-16 11:37:46 +0200 | [diff] [blame] | 1979 | |
| 1980 | if (pclc->hdr.version > SMC_V1) |
| 1981 | proposal_version = SMC_V2; |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1982 | |
Ursula Braun | e8d726c8 | 2020-09-26 12:44:32 +0200 | [diff] [blame] | 1983 | /* IPSec connections opt out of SMC optimizations */ |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1984 | if (using_ipsec(new_smc)) { |
Karsten Graul | 9aa68d2 | 2019-04-12 12:57:29 +0200 | [diff] [blame] | 1985 | rc = SMC_CLC_DECL_IPSEC; |
| 1986 | goto out_decl; |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 1987 | } |
| 1988 | |
Ursula Braun | 3fc6493 | 2020-09-26 12:44:23 +0200 | [diff] [blame] | 1989 | ini = kzalloc(sizeof(*ini), GFP_KERNEL); |
| 1990 | if (!ini) { |
| 1991 | rc = SMC_CLC_DECL_MEM; |
| 1992 | goto out_decl; |
| 1993 | } |
| 1994 | |
Ursula Braun | 5c21c4c | 2020-09-26 12:44:29 +0200 | [diff] [blame] | 1995 | /* initial version checking */ |
| 1996 | rc = smc_listen_v2_check(new_smc, pclc, ini); |
| 1997 | if (rc) |
Karsten Graul | 9aa68d2 | 2019-04-12 12:57:29 +0200 | [diff] [blame] | 1998 | goto out_decl; |
Karsten Graul | fba7e8e | 2019-04-12 12:57:27 +0200 | [diff] [blame] | 1999 | |
Hans Wippel | 72a36a8 | 2019-02-07 15:56:18 +0100 | [diff] [blame] | 2000 | mutex_lock(&smc_server_lgr_pending); |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 2001 | smc_close_init(new_smc); |
| 2002 | smc_rx_init(new_smc); |
| 2003 | smc_tx_init(new_smc); |
| 2004 | |
Ursula Braun | 7affc80 | 2020-09-26 12:44:21 +0200 | [diff] [blame] | 2005 | /* determine ISM or RoCE device used for connection */ |
Ursula Braun | 3fc6493 | 2020-09-26 12:44:23 +0200 | [diff] [blame] | 2006 | rc = smc_listen_find_device(new_smc, pclc, ini); |
Ursula Braun | 7affc80 | 2020-09-26 12:44:21 +0200 | [diff] [blame] | 2007 | if (rc) |
| 2008 | goto out_unlock; |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 2009 | |
| 2010 | /* send SMC Accept CLC message */ |
Karsten Graul | e49300a | 2021-10-16 11:37:46 +0200 | [diff] [blame] | 2011 | accept_version = ini->is_smcd ? ini->smcd_version : ini->smcr_version; |
Ursula Braun | a7c9c5f | 2020-09-26 12:44:30 +0200 | [diff] [blame] | 2012 | rc = smc_clc_send_accept(new_smc, ini->first_contact_local, |
Karsten Graul | e49300a | 2021-10-16 11:37:46 +0200 | [diff] [blame] | 2013 | accept_version, ini->negotiated_eid); |
Karsten Graul | 9aa68d2 | 2019-04-12 12:57:29 +0200 | [diff] [blame] | 2014 | if (rc) |
| 2015 | goto out_unlock; |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 2016 | |
Hans Wippel | 62c7139 | 2019-02-07 15:56:17 +0100 | [diff] [blame] | 2017 | /* SMC-D does not need this lock any more */ |
Ursula Braun | 3fc6493 | 2020-09-26 12:44:23 +0200 | [diff] [blame] | 2018 | if (ini->is_smcd) |
Hans Wippel | 72a36a8 | 2019-02-07 15:56:18 +0100 | [diff] [blame] | 2019 | mutex_unlock(&smc_server_lgr_pending); |
Hans Wippel | 62c7139 | 2019-02-07 15:56:17 +0100 | [diff] [blame] | 2020 | |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 2021 | /* receive SMC Confirm CLC message */ |
Karsten Graul | 9047a617dc | 2020-10-07 22:57:42 +0200 | [diff] [blame] | 2022 | memset(buf, 0, sizeof(*buf)); |
| 2023 | cclc = (struct smc_clc_msg_accept_confirm *)buf; |
| 2024 | rc = smc_clc_wait_msg(new_smc, cclc, sizeof(*buf), |
Karsten Graul | 228bae0 | 2019-04-12 12:57:28 +0200 | [diff] [blame] | 2025 | SMC_CLC_CONFIRM, CLC_WAIT_TIME); |
| 2026 | if (rc) { |
Ursula Braun | 3fc6493 | 2020-09-26 12:44:23 +0200 | [diff] [blame] | 2027 | if (!ini->is_smcd) |
Karsten Graul | 9aa68d2 | 2019-04-12 12:57:29 +0200 | [diff] [blame] | 2028 | goto out_unlock; |
| 2029 | goto out_decl; |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 2030 | } |
| 2031 | |
| 2032 | /* finish worker */ |
Ursula Braun | 3fc6493 | 2020-09-26 12:44:23 +0200 | [diff] [blame] | 2033 | if (!ini->is_smcd) { |
Ursula Braun | a7c9c5f | 2020-09-26 12:44:30 +0200 | [diff] [blame] | 2034 | rc = smc_listen_rdma_finish(new_smc, cclc, |
Karsten Graul | e49300a | 2021-10-16 11:37:46 +0200 | [diff] [blame] | 2035 | ini->first_contact_local, ini); |
Hans Wippel | 62c7139 | 2019-02-07 15:56:17 +0100 | [diff] [blame] | 2036 | if (rc) |
Ursula Braun | 0c881ad | 2020-09-10 18:48:25 +0200 | [diff] [blame] | 2037 | goto out_unlock; |
| 2038 | mutex_unlock(&smc_server_lgr_pending); |
Ursula Braun | 1ca52fc | 2018-09-18 15:46:35 +0200 | [diff] [blame] | 2039 | } |
Ursula Braun | a7c9c5f | 2020-09-26 12:44:30 +0200 | [diff] [blame] | 2040 | smc_conn_save_peer_info(new_smc, cclc); |
Hans Wippel | 3b2dec2 | 2018-05-18 09:34:18 +0200 | [diff] [blame] | 2041 | smc_listen_out_connected(new_smc); |
Guvenc Gulce | 194730a | 2021-06-16 16:52:58 +0200 | [diff] [blame] | 2042 | SMC_STAT_SERV_SUCC_INC(sock_net(newclcsock->sk), ini); |
Ursula Braun | ac67936 | 2020-09-17 22:46:02 +0200 | [diff] [blame] | 2043 | goto out_free; |
Karsten Graul | 9aa68d2 | 2019-04-12 12:57:29 +0200 | [diff] [blame] | 2044 | |
| 2045 | out_unlock: |
| 2046 | mutex_unlock(&smc_server_lgr_pending); |
| 2047 | out_decl: |
Karsten Graul | 4a9baf4 | 2020-10-23 20:48:28 +0200 | [diff] [blame] | 2048 | smc_listen_decline(new_smc, rc, ini ? ini->first_contact_local : 0, |
Karsten Graul | e49300a | 2021-10-16 11:37:46 +0200 | [diff] [blame] | 2049 | proposal_version); |
Ursula Braun | ac67936 | 2020-09-17 22:46:02 +0200 | [diff] [blame] | 2050 | out_free: |
Ursula Braun | 3fc6493 | 2020-09-26 12:44:23 +0200 | [diff] [blame] | 2051 | kfree(ini); |
Ursula Braun | 6bb14e48 | 2020-09-10 18:48:22 +0200 | [diff] [blame] | 2052 | kfree(buf); |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 2053 | } |
| 2054 | |
| 2055 | static void smc_tcp_listen_work(struct work_struct *work) |
| 2056 | { |
| 2057 | struct smc_sock *lsmc = container_of(work, struct smc_sock, |
| 2058 | tcp_listen_work); |
Ursula Braun | 3163c50 | 2018-01-24 10:28:12 +0100 | [diff] [blame] | 2059 | struct sock *lsk = &lsmc->sk; |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 2060 | struct smc_sock *new_smc; |
| 2061 | int rc = 0; |
| 2062 | |
Ursula Braun | 3163c50 | 2018-01-24 10:28:12 +0100 | [diff] [blame] | 2063 | lock_sock(lsk); |
| 2064 | while (lsk->sk_state == SMC_LISTEN) { |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 2065 | rc = smc_clcsock_accept(lsmc, &new_smc); |
Ursula Braun | a60a2b1 | 2020-09-10 18:48:20 +0200 | [diff] [blame] | 2066 | if (rc) /* clcsock accept queue empty or error */ |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 2067 | goto out; |
| 2068 | if (!new_smc) |
| 2069 | continue; |
| 2070 | |
| 2071 | new_smc->listen_smc = lsmc; |
Ursula Braun | ee9dfbe | 2018-04-26 17:18:21 +0200 | [diff] [blame] | 2072 | new_smc->use_fallback = lsmc->use_fallback; |
Karsten Graul | 603cc14 | 2018-07-25 16:35:32 +0200 | [diff] [blame] | 2073 | new_smc->fallback_rsn = lsmc->fallback_rsn; |
Ursula Braun | 3163c50 | 2018-01-24 10:28:12 +0100 | [diff] [blame] | 2074 | sock_hold(lsk); /* sock_put in smc_listen_work */ |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 2075 | INIT_WORK(&new_smc->smc_listen_work, smc_listen_work); |
| 2076 | smc_copy_sock_settings_to_smc(new_smc); |
Ursula Braun | bd58c7e | 2018-08-08 14:13:20 +0200 | [diff] [blame] | 2077 | new_smc->sk.sk_sndbuf = lsmc->sk.sk_sndbuf; |
| 2078 | new_smc->sk.sk_rcvbuf = lsmc->sk.sk_rcvbuf; |
Ursula Braun | 51f1de7 | 2018-01-26 09:28:48 +0100 | [diff] [blame] | 2079 | sock_hold(&new_smc->sk); /* sock_put in passive closing */ |
Karsten Graul | 22ef473 | 2020-09-10 18:48:29 +0200 | [diff] [blame] | 2080 | if (!queue_work(smc_hs_wq, &new_smc->smc_listen_work)) |
Ursula Braun | 51f1de7 | 2018-01-26 09:28:48 +0100 | [diff] [blame] | 2081 | sock_put(&new_smc->sk); |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 2082 | } |
| 2083 | |
| 2084 | out: |
Ursula Braun | 3163c50 | 2018-01-24 10:28:12 +0100 | [diff] [blame] | 2085 | release_sock(lsk); |
Ursula Braun | a60a2b1 | 2020-09-10 18:48:20 +0200 | [diff] [blame] | 2086 | sock_put(&lsmc->sk); /* sock_hold in smc_clcsock_data_ready() */ |
| 2087 | } |
| 2088 | |
| 2089 | static void smc_clcsock_data_ready(struct sock *listen_clcsock) |
| 2090 | { |
| 2091 | struct smc_sock *lsmc; |
| 2092 | |
| 2093 | lsmc = (struct smc_sock *) |
| 2094 | ((uintptr_t)listen_clcsock->sk_user_data & ~SK_USER_DATA_NOCOPY); |
| 2095 | if (!lsmc) |
| 2096 | return; |
| 2097 | lsmc->clcsk_data_ready(listen_clcsock); |
| 2098 | if (lsmc->sk.sk_state == SMC_LISTEN) { |
| 2099 | sock_hold(&lsmc->sk); /* sock_put in smc_tcp_listen_work() */ |
Karsten Graul | 22ef473 | 2020-09-10 18:48:29 +0200 | [diff] [blame] | 2100 | if (!queue_work(smc_hs_wq, &lsmc->tcp_listen_work)) |
Ursula Braun | a60a2b1 | 2020-09-10 18:48:20 +0200 | [diff] [blame] | 2101 | sock_put(&lsmc->sk); |
| 2102 | } |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 2103 | } |
| 2104 | |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2105 | static int smc_listen(struct socket *sock, int backlog) |
| 2106 | { |
| 2107 | struct sock *sk = sock->sk; |
| 2108 | struct smc_sock *smc; |
| 2109 | int rc; |
| 2110 | |
| 2111 | smc = smc_sk(sk); |
| 2112 | lock_sock(sk); |
| 2113 | |
| 2114 | rc = -EINVAL; |
Ursula Braun | cd20636 | 2019-08-02 10:47:50 +0200 | [diff] [blame] | 2115 | if ((sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) || |
| 2116 | smc->connect_nonblock) |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2117 | goto out; |
| 2118 | |
| 2119 | rc = 0; |
| 2120 | if (sk->sk_state == SMC_LISTEN) { |
| 2121 | sk->sk_max_ack_backlog = backlog; |
| 2122 | goto out; |
| 2123 | } |
| 2124 | /* some socket options are handled in core, so we could not apply |
| 2125 | * them to the clc socket -- copy smc socket options to clc socket |
| 2126 | */ |
| 2127 | smc_copy_sock_settings_to_clc(smc); |
Ursula Braun | ee9dfbe | 2018-04-26 17:18:21 +0200 | [diff] [blame] | 2128 | if (!smc->use_fallback) |
| 2129 | tcp_sk(smc->clcsock->sk)->syn_smc = 1; |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2130 | |
Ursula Braun | a60a2b1 | 2020-09-10 18:48:20 +0200 | [diff] [blame] | 2131 | /* save original sk_data_ready function and establish |
| 2132 | * smc-specific sk_data_ready function |
| 2133 | */ |
| 2134 | smc->clcsk_data_ready = smc->clcsock->sk->sk_data_ready; |
| 2135 | smc->clcsock->sk->sk_data_ready = smc_clcsock_data_ready; |
| 2136 | smc->clcsock->sk->sk_user_data = |
| 2137 | (void *)((uintptr_t)smc | SK_USER_DATA_NOCOPY); |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2138 | rc = kernel_listen(smc->clcsock, backlog); |
Guo DaXing | 9ebb0c4 | 2021-11-24 13:32:38 +0100 | [diff] [blame] | 2139 | if (rc) { |
| 2140 | smc->clcsock->sk->sk_data_ready = smc->clcsk_data_ready; |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2141 | goto out; |
Guo DaXing | 9ebb0c4 | 2021-11-24 13:32:38 +0100 | [diff] [blame] | 2142 | } |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2143 | sk->sk_max_ack_backlog = backlog; |
| 2144 | sk->sk_ack_backlog = 0; |
| 2145 | sk->sk_state = SMC_LISTEN; |
| 2146 | |
| 2147 | out: |
| 2148 | release_sock(sk); |
| 2149 | return rc; |
| 2150 | } |
| 2151 | |
| 2152 | static int smc_accept(struct socket *sock, struct socket *new_sock, |
David Howells | cdfbabf | 2017-03-09 08:09:05 +0000 | [diff] [blame] | 2153 | int flags, bool kern) |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2154 | { |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 2155 | struct sock *sk = sock->sk, *nsk; |
| 2156 | DECLARE_WAITQUEUE(wait, current); |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2157 | struct smc_sock *lsmc; |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 2158 | long timeo; |
| 2159 | int rc = 0; |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2160 | |
| 2161 | lsmc = smc_sk(sk); |
Ursula Braun | 51f1de7 | 2018-01-26 09:28:48 +0100 | [diff] [blame] | 2162 | sock_hold(sk); /* sock_put below */ |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2163 | lock_sock(sk); |
| 2164 | |
| 2165 | if (lsmc->sk.sk_state != SMC_LISTEN) { |
| 2166 | rc = -EINVAL; |
Ursula Braun | abb190f | 2018-04-26 17:18:23 +0200 | [diff] [blame] | 2167 | release_sock(sk); |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2168 | goto out; |
| 2169 | } |
| 2170 | |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 2171 | /* Wait for an incoming connection */ |
| 2172 | timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK); |
| 2173 | add_wait_queue_exclusive(sk_sleep(sk), &wait); |
| 2174 | while (!(nsk = smc_accept_dequeue(sk, new_sock))) { |
| 2175 | set_current_state(TASK_INTERRUPTIBLE); |
| 2176 | if (!timeo) { |
| 2177 | rc = -EAGAIN; |
| 2178 | break; |
| 2179 | } |
| 2180 | release_sock(sk); |
| 2181 | timeo = schedule_timeout(timeo); |
| 2182 | /* wakeup by sk_data_ready in smc_listen_work() */ |
| 2183 | sched_annotate_sleep(); |
| 2184 | lock_sock(sk); |
| 2185 | if (signal_pending(current)) { |
| 2186 | rc = sock_intr_errno(timeo); |
| 2187 | break; |
| 2188 | } |
| 2189 | } |
| 2190 | set_current_state(TASK_RUNNING); |
| 2191 | remove_wait_queue(sk_sleep(sk), &wait); |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2192 | |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 2193 | if (!rc) |
| 2194 | rc = sock_error(nsk); |
Ursula Braun | abb190f | 2018-04-26 17:18:23 +0200 | [diff] [blame] | 2195 | release_sock(sk); |
| 2196 | if (rc) |
| 2197 | goto out; |
| 2198 | |
| 2199 | if (lsmc->sockopt_defer_accept && !(flags & O_NONBLOCK)) { |
| 2200 | /* wait till data arrives on the socket */ |
| 2201 | timeo = msecs_to_jiffies(lsmc->sockopt_defer_accept * |
| 2202 | MSEC_PER_SEC); |
| 2203 | if (smc_sk(nsk)->use_fallback) { |
| 2204 | struct sock *clcsk = smc_sk(nsk)->clcsock->sk; |
| 2205 | |
| 2206 | lock_sock(clcsk); |
| 2207 | if (skb_queue_empty(&clcsk->sk_receive_queue)) |
| 2208 | sk_wait_data(clcsk, &timeo, NULL); |
| 2209 | release_sock(clcsk); |
| 2210 | } else if (!atomic_read(&smc_sk(nsk)->conn.bytes_to_rcv)) { |
| 2211 | lock_sock(nsk); |
Stefan Raspl | b51fa1b | 2018-05-03 18:12:37 +0200 | [diff] [blame] | 2212 | smc_rx_wait(smc_sk(nsk), &timeo, smc_rx_data_available); |
Ursula Braun | abb190f | 2018-04-26 17:18:23 +0200 | [diff] [blame] | 2213 | release_sock(nsk); |
| 2214 | } |
| 2215 | } |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2216 | |
| 2217 | out: |
Ursula Braun | 51f1de7 | 2018-01-26 09:28:48 +0100 | [diff] [blame] | 2218 | sock_put(sk); /* sock_hold above */ |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2219 | return rc; |
| 2220 | } |
| 2221 | |
| 2222 | static int smc_getname(struct socket *sock, struct sockaddr *addr, |
Denys Vlasenko | 9b2c45d | 2018-02-12 20:00:20 +0100 | [diff] [blame] | 2223 | int peer) |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2224 | { |
| 2225 | struct smc_sock *smc; |
| 2226 | |
Ursula Braun | b38d732 | 2017-01-09 16:55:25 +0100 | [diff] [blame] | 2227 | if (peer && (sock->sk->sk_state != SMC_ACTIVE) && |
| 2228 | (sock->sk->sk_state != SMC_APPCLOSEWAIT1)) |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2229 | return -ENOTCONN; |
| 2230 | |
| 2231 | smc = smc_sk(sock->sk); |
| 2232 | |
Denys Vlasenko | 9b2c45d | 2018-02-12 20:00:20 +0100 | [diff] [blame] | 2233 | return smc->clcsock->ops->getname(smc->clcsock, addr, peer); |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2234 | } |
| 2235 | |
| 2236 | static int smc_sendmsg(struct socket *sock, struct msghdr *msg, size_t len) |
| 2237 | { |
| 2238 | struct sock *sk = sock->sk; |
| 2239 | struct smc_sock *smc; |
| 2240 | int rc = -EPIPE; |
| 2241 | |
| 2242 | smc = smc_sk(sk); |
| 2243 | lock_sock(sk); |
Ursula Braun | b38d732 | 2017-01-09 16:55:25 +0100 | [diff] [blame] | 2244 | if ((sk->sk_state != SMC_ACTIVE) && |
| 2245 | (sk->sk_state != SMC_APPCLOSEWAIT1) && |
| 2246 | (sk->sk_state != SMC_INIT)) |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2247 | goto out; |
Ursula Braun | ee9dfbe | 2018-04-26 17:18:21 +0200 | [diff] [blame] | 2248 | |
| 2249 | if (msg->msg_flags & MSG_FASTOPEN) { |
Ursula Braun | cd20636 | 2019-08-02 10:47:50 +0200 | [diff] [blame] | 2250 | if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) { |
Guvenc Gulce | e0e4b8f | 2021-06-16 16:52:55 +0200 | [diff] [blame] | 2251 | smc_switch_to_fallback(smc, SMC_CLC_DECL_OPTUNSUPP); |
Ursula Braun | ee9dfbe | 2018-04-26 17:18:21 +0200 | [diff] [blame] | 2252 | } else { |
| 2253 | rc = -EINVAL; |
| 2254 | goto out; |
| 2255 | } |
| 2256 | } |
| 2257 | |
Guvenc Gulce | e0e4b8f | 2021-06-16 16:52:55 +0200 | [diff] [blame] | 2258 | if (smc->use_fallback) { |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2259 | rc = smc->clcsock->ops->sendmsg(smc->clcsock, msg, len); |
Guvenc Gulce | e0e4b8f | 2021-06-16 16:52:55 +0200 | [diff] [blame] | 2260 | } else { |
Ursula Braun | e6727f3 | 2017-01-09 16:55:23 +0100 | [diff] [blame] | 2261 | rc = smc_tx_sendmsg(smc, msg, len); |
Guvenc Gulce | e0e4b8f | 2021-06-16 16:52:55 +0200 | [diff] [blame] | 2262 | SMC_STAT_TX_PAYLOAD(smc, len, rc); |
| 2263 | } |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2264 | out: |
| 2265 | release_sock(sk); |
| 2266 | return rc; |
| 2267 | } |
| 2268 | |
| 2269 | static int smc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, |
| 2270 | int flags) |
| 2271 | { |
| 2272 | struct sock *sk = sock->sk; |
| 2273 | struct smc_sock *smc; |
| 2274 | int rc = -ENOTCONN; |
| 2275 | |
| 2276 | smc = smc_sk(sk); |
| 2277 | lock_sock(sk); |
Karsten Graul | 51c5aba | 2019-01-30 18:51:04 +0100 | [diff] [blame] | 2278 | if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) { |
| 2279 | /* socket was connected before, no more data to read */ |
| 2280 | rc = 0; |
| 2281 | goto out; |
| 2282 | } |
Ursula Braun | b38d732 | 2017-01-09 16:55:25 +0100 | [diff] [blame] | 2283 | if ((sk->sk_state == SMC_INIT) || |
| 2284 | (sk->sk_state == SMC_LISTEN) || |
| 2285 | (sk->sk_state == SMC_CLOSED)) |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2286 | goto out; |
| 2287 | |
Ursula Braun | b38d732 | 2017-01-09 16:55:25 +0100 | [diff] [blame] | 2288 | if (sk->sk_state == SMC_PEERFINCLOSEWAIT) { |
| 2289 | rc = 0; |
| 2290 | goto out; |
| 2291 | } |
| 2292 | |
Stefan Raspl | 9014db2 | 2018-05-03 18:12:39 +0200 | [diff] [blame] | 2293 | if (smc->use_fallback) { |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2294 | rc = smc->clcsock->ops->recvmsg(smc->clcsock, msg, len, flags); |
Stefan Raspl | 9014db2 | 2018-05-03 18:12:39 +0200 | [diff] [blame] | 2295 | } else { |
| 2296 | msg->msg_namelen = 0; |
| 2297 | rc = smc_rx_recvmsg(smc, msg, NULL, len, flags); |
Guvenc Gulce | e0e4b8f | 2021-06-16 16:52:55 +0200 | [diff] [blame] | 2298 | SMC_STAT_RX_PAYLOAD(smc, rc, rc); |
Stefan Raspl | 9014db2 | 2018-05-03 18:12:39 +0200 | [diff] [blame] | 2299 | } |
Ursula Braun | b38d732 | 2017-01-09 16:55:25 +0100 | [diff] [blame] | 2300 | |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2301 | out: |
| 2302 | release_sock(sk); |
| 2303 | return rc; |
| 2304 | } |
| 2305 | |
Al Viro | ade994f | 2017-07-03 00:01:49 -0400 | [diff] [blame] | 2306 | static __poll_t smc_accept_poll(struct sock *parent) |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 2307 | { |
Ursula Braun | 8dce278 | 2018-01-26 09:28:47 +0100 | [diff] [blame] | 2308 | struct smc_sock *isk = smc_sk(parent); |
Al Viro | 63e2480 | 2018-02-01 10:02:53 -0500 | [diff] [blame] | 2309 | __poll_t mask = 0; |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 2310 | |
Ursula Braun | 8dce278 | 2018-01-26 09:28:47 +0100 | [diff] [blame] | 2311 | spin_lock(&isk->accept_q_lock); |
| 2312 | if (!list_empty(&isk->accept_q)) |
Linus Torvalds | a9a0884 | 2018-02-11 14:34:03 -0800 | [diff] [blame] | 2313 | mask = EPOLLIN | EPOLLRDNORM; |
Ursula Braun | 8dce278 | 2018-01-26 09:28:47 +0100 | [diff] [blame] | 2314 | spin_unlock(&isk->accept_q_lock); |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 2315 | |
Ursula Braun | 8dce278 | 2018-01-26 09:28:47 +0100 | [diff] [blame] | 2316 | return mask; |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 2317 | } |
| 2318 | |
Linus Torvalds | a11e1d4 | 2018-06-28 09:43:44 -0700 | [diff] [blame] | 2319 | static __poll_t smc_poll(struct file *file, struct socket *sock, |
| 2320 | poll_table *wait) |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2321 | { |
| 2322 | struct sock *sk = sock->sk; |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2323 | struct smc_sock *smc; |
Ursula Braun | 50717a3 | 2019-04-12 12:57:23 +0200 | [diff] [blame] | 2324 | __poll_t mask = 0; |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2325 | |
Ursula Braun | 8dce278 | 2018-01-26 09:28:47 +0100 | [diff] [blame] | 2326 | if (!sk) |
Linus Torvalds | a9a0884 | 2018-02-11 14:34:03 -0800 | [diff] [blame] | 2327 | return EPOLLNVAL; |
Ursula Braun | 8dce278 | 2018-01-26 09:28:47 +0100 | [diff] [blame] | 2328 | |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2329 | smc = smc_sk(sock->sk); |
Ursula Braun | 648a5a7a | 2018-09-18 15:46:34 +0200 | [diff] [blame] | 2330 | if (smc->use_fallback) { |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 2331 | /* delegate to CLC child sock */ |
Linus Torvalds | a11e1d4 | 2018-06-28 09:43:44 -0700 | [diff] [blame] | 2332 | mask = smc->clcsock->ops->poll(file, smc->clcsock, wait); |
Ursula Braun | 784813a | 2018-05-02 16:53:56 +0200 | [diff] [blame] | 2333 | sk->sk_err = smc->clcsock->sk->sk_err; |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2334 | } else { |
Linus Torvalds | 410da1e | 2018-07-03 09:53:43 -0700 | [diff] [blame] | 2335 | if (sk->sk_state != SMC_CLOSED) |
Karsten Graul | 89ab066 | 2018-10-23 13:40:39 +0200 | [diff] [blame] | 2336 | sock_poll_wait(file, sock, wait); |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 2337 | if (sk->sk_err) |
Linus Torvalds | a9a0884 | 2018-02-11 14:34:03 -0800 | [diff] [blame] | 2338 | mask |= EPOLLERR; |
Ursula Braun | b38d732 | 2017-01-09 16:55:25 +0100 | [diff] [blame] | 2339 | if ((sk->sk_shutdown == SHUTDOWN_MASK) || |
| 2340 | (sk->sk_state == SMC_CLOSED)) |
Linus Torvalds | a9a0884 | 2018-02-11 14:34:03 -0800 | [diff] [blame] | 2341 | mask |= EPOLLHUP; |
Ursula Braun | 8dce278 | 2018-01-26 09:28:47 +0100 | [diff] [blame] | 2342 | if (sk->sk_state == SMC_LISTEN) { |
| 2343 | /* woken up by sk_data_ready in smc_listen_work() */ |
Ursula Braun | 50717a3 | 2019-04-12 12:57:23 +0200 | [diff] [blame] | 2344 | mask |= smc_accept_poll(sk); |
| 2345 | } else if (smc->use_fallback) { /* as result of connect_work()*/ |
| 2346 | mask |= smc->clcsock->ops->poll(file, smc->clcsock, |
| 2347 | wait); |
| 2348 | sk->sk_err = smc->clcsock->sk->sk_err; |
Ursula Braun | 8dce278 | 2018-01-26 09:28:47 +0100 | [diff] [blame] | 2349 | } else { |
Ursula Braun | 50717a3 | 2019-04-12 12:57:23 +0200 | [diff] [blame] | 2350 | if ((sk->sk_state != SMC_INIT && |
| 2351 | atomic_read(&smc->conn.sndbuf_space)) || |
Ursula Braun | 8dce278 | 2018-01-26 09:28:47 +0100 | [diff] [blame] | 2352 | sk->sk_shutdown & SEND_SHUTDOWN) { |
Linus Torvalds | a9a0884 | 2018-02-11 14:34:03 -0800 | [diff] [blame] | 2353 | mask |= EPOLLOUT | EPOLLWRNORM; |
Ursula Braun | 8dce278 | 2018-01-26 09:28:47 +0100 | [diff] [blame] | 2354 | } else { |
| 2355 | sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); |
| 2356 | set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); |
| 2357 | } |
| 2358 | if (atomic_read(&smc->conn.bytes_to_rcv)) |
Linus Torvalds | a9a0884 | 2018-02-11 14:34:03 -0800 | [diff] [blame] | 2359 | mask |= EPOLLIN | EPOLLRDNORM; |
Ursula Braun | 8dce278 | 2018-01-26 09:28:47 +0100 | [diff] [blame] | 2360 | if (sk->sk_shutdown & RCV_SHUTDOWN) |
Linus Torvalds | a9a0884 | 2018-02-11 14:34:03 -0800 | [diff] [blame] | 2361 | mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP; |
Ursula Braun | 8dce278 | 2018-01-26 09:28:47 +0100 | [diff] [blame] | 2362 | if (sk->sk_state == SMC_APPCLOSEWAIT1) |
Linus Torvalds | a9a0884 | 2018-02-11 14:34:03 -0800 | [diff] [blame] | 2363 | mask |= EPOLLIN; |
Karsten Graul | 71d117f | 2018-09-18 15:46:37 +0200 | [diff] [blame] | 2364 | if (smc->conn.urg_state == SMC_URG_VALID) |
| 2365 | mask |= EPOLLPRI; |
Ursula Braun | 8dce278 | 2018-01-26 09:28:47 +0100 | [diff] [blame] | 2366 | } |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2367 | } |
| 2368 | |
| 2369 | return mask; |
| 2370 | } |
| 2371 | |
| 2372 | static int smc_shutdown(struct socket *sock, int how) |
| 2373 | { |
| 2374 | struct sock *sk = sock->sk; |
Tony Lu | bacb6c1 | 2021-11-26 10:41:35 +0800 | [diff] [blame] | 2375 | bool do_shutdown = true; |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2376 | struct smc_sock *smc; |
| 2377 | int rc = -EINVAL; |
Tony Lu | bacb6c1 | 2021-11-26 10:41:35 +0800 | [diff] [blame] | 2378 | int old_state; |
Ursula Braun | b38d732 | 2017-01-09 16:55:25 +0100 | [diff] [blame] | 2379 | int rc1 = 0; |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2380 | |
| 2381 | smc = smc_sk(sk); |
| 2382 | |
| 2383 | if ((how < SHUT_RD) || (how > SHUT_RDWR)) |
Ursula Braun | b38d732 | 2017-01-09 16:55:25 +0100 | [diff] [blame] | 2384 | return rc; |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2385 | |
| 2386 | lock_sock(sk); |
| 2387 | |
| 2388 | rc = -ENOTCONN; |
Ursula Braun | caa21e1 | 2018-08-08 14:13:19 +0200 | [diff] [blame] | 2389 | if ((sk->sk_state != SMC_ACTIVE) && |
Ursula Braun | b38d732 | 2017-01-09 16:55:25 +0100 | [diff] [blame] | 2390 | (sk->sk_state != SMC_PEERCLOSEWAIT1) && |
| 2391 | (sk->sk_state != SMC_PEERCLOSEWAIT2) && |
| 2392 | (sk->sk_state != SMC_APPCLOSEWAIT1) && |
| 2393 | (sk->sk_state != SMC_APPCLOSEWAIT2) && |
| 2394 | (sk->sk_state != SMC_APPFINCLOSEWAIT)) |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2395 | goto out; |
| 2396 | if (smc->use_fallback) { |
| 2397 | rc = kernel_sock_shutdown(smc->clcsock, how); |
| 2398 | sk->sk_shutdown = smc->clcsock->sk->sk_shutdown; |
| 2399 | if (sk->sk_shutdown == SHUTDOWN_MASK) |
| 2400 | sk->sk_state = SMC_CLOSED; |
Ursula Braun | b38d732 | 2017-01-09 16:55:25 +0100 | [diff] [blame] | 2401 | goto out; |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2402 | } |
Ursula Braun | b38d732 | 2017-01-09 16:55:25 +0100 | [diff] [blame] | 2403 | switch (how) { |
| 2404 | case SHUT_RDWR: /* shutdown in both directions */ |
Tony Lu | bacb6c1 | 2021-11-26 10:41:35 +0800 | [diff] [blame] | 2405 | old_state = sk->sk_state; |
Ursula Braun | b38d732 | 2017-01-09 16:55:25 +0100 | [diff] [blame] | 2406 | rc = smc_close_active(smc); |
Tony Lu | bacb6c1 | 2021-11-26 10:41:35 +0800 | [diff] [blame] | 2407 | if (old_state == SMC_ACTIVE && |
| 2408 | sk->sk_state == SMC_PEERCLOSEWAIT1) |
| 2409 | do_shutdown = false; |
Ursula Braun | b38d732 | 2017-01-09 16:55:25 +0100 | [diff] [blame] | 2410 | break; |
| 2411 | case SHUT_WR: |
| 2412 | rc = smc_close_shutdown_write(smc); |
| 2413 | break; |
| 2414 | case SHUT_RD: |
Ursula Braun | 1255fcb | 2018-04-19 15:56:40 +0200 | [diff] [blame] | 2415 | rc = 0; |
| 2416 | /* nothing more to do because peer is not involved */ |
Ursula Braun | b38d732 | 2017-01-09 16:55:25 +0100 | [diff] [blame] | 2417 | break; |
| 2418 | } |
Tony Lu | bacb6c1 | 2021-11-26 10:41:35 +0800 | [diff] [blame] | 2419 | if (do_shutdown && smc->clcsock) |
Ursula Braun | 1255fcb | 2018-04-19 15:56:40 +0200 | [diff] [blame] | 2420 | rc1 = kernel_sock_shutdown(smc->clcsock, how); |
Ursula Braun | b38d732 | 2017-01-09 16:55:25 +0100 | [diff] [blame] | 2421 | /* map sock_shutdown_cmd constants to sk_shutdown value range */ |
| 2422 | sk->sk_shutdown |= how + 1; |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2423 | |
| 2424 | out: |
| 2425 | release_sock(sk); |
Ursula Braun | b38d732 | 2017-01-09 16:55:25 +0100 | [diff] [blame] | 2426 | return rc ? rc : rc1; |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2427 | } |
| 2428 | |
| 2429 | static int smc_setsockopt(struct socket *sock, int level, int optname, |
Christoph Hellwig | a7b75c5 | 2020-07-23 08:09:07 +0200 | [diff] [blame] | 2430 | sockptr_t optval, unsigned int optlen) |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2431 | { |
| 2432 | struct sock *sk = sock->sk; |
| 2433 | struct smc_sock *smc; |
Ursula Braun | 01d2f7e | 2018-04-26 17:18:22 +0200 | [diff] [blame] | 2434 | int val, rc; |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2435 | |
Cong Wang | 8621436 | 2021-05-05 12:40:48 -0700 | [diff] [blame] | 2436 | if (level == SOL_TCP && optname == TCP_ULP) |
| 2437 | return -EOPNOTSUPP; |
| 2438 | |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2439 | smc = smc_sk(sk); |
| 2440 | |
| 2441 | /* generic setsockopts reaching us here always apply to the |
| 2442 | * CLC socket |
| 2443 | */ |
Christoph Hellwig | a44d9e7 | 2020-07-17 08:23:31 +0200 | [diff] [blame] | 2444 | if (unlikely(!smc->clcsock->ops->setsockopt)) |
| 2445 | rc = -EOPNOTSUPP; |
| 2446 | else |
| 2447 | rc = smc->clcsock->ops->setsockopt(smc->clcsock, level, optname, |
| 2448 | optval, optlen); |
Ursula Braun | ee9dfbe | 2018-04-26 17:18:21 +0200 | [diff] [blame] | 2449 | if (smc->clcsock->sk->sk_err) { |
| 2450 | sk->sk_err = smc->clcsock->sk->sk_err; |
Alexander Aring | e3ae236 | 2021-06-27 18:48:21 -0400 | [diff] [blame] | 2451 | sk_error_report(sk); |
Ursula Braun | ee9dfbe | 2018-04-26 17:18:21 +0200 | [diff] [blame] | 2452 | } |
Ursula Braun | ee9dfbe | 2018-04-26 17:18:21 +0200 | [diff] [blame] | 2453 | |
Ursula Braun | 01d2f7e | 2018-04-26 17:18:22 +0200 | [diff] [blame] | 2454 | if (optlen < sizeof(int)) |
Wei Yongjun | 3dc9f55 | 2018-05-31 02:31:22 +0000 | [diff] [blame] | 2455 | return -EINVAL; |
Christoph Hellwig | a7b75c5 | 2020-07-23 08:09:07 +0200 | [diff] [blame] | 2456 | if (copy_from_sockptr(&val, optval, sizeof(int))) |
Ursula Braun | ac0107e | 2018-07-18 15:22:50 +0200 | [diff] [blame] | 2457 | return -EFAULT; |
Ursula Braun | 01d2f7e | 2018-04-26 17:18:22 +0200 | [diff] [blame] | 2458 | |
Ursula Braun | ee9dfbe | 2018-04-26 17:18:21 +0200 | [diff] [blame] | 2459 | lock_sock(sk); |
Ursula Braun | 8643474 | 2019-12-12 22:35:58 +0100 | [diff] [blame] | 2460 | if (rc || smc->use_fallback) |
| 2461 | goto out; |
Ursula Braun | ee9dfbe | 2018-04-26 17:18:21 +0200 | [diff] [blame] | 2462 | switch (optname) { |
Ursula Braun | ee9dfbe | 2018-04-26 17:18:21 +0200 | [diff] [blame] | 2463 | case TCP_FASTOPEN: |
| 2464 | case TCP_FASTOPEN_CONNECT: |
| 2465 | case TCP_FASTOPEN_KEY: |
| 2466 | case TCP_FASTOPEN_NO_COOKIE: |
| 2467 | /* option not supported by SMC */ |
Ursula Braun | 8204df7 | 2019-11-15 12:39:30 +0100 | [diff] [blame] | 2468 | if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) { |
Guvenc Gulce | e0e4b8f | 2021-06-16 16:52:55 +0200 | [diff] [blame] | 2469 | smc_switch_to_fallback(smc, SMC_CLC_DECL_OPTUNSUPP); |
Ursula Braun | ee9dfbe | 2018-04-26 17:18:21 +0200 | [diff] [blame] | 2470 | } else { |
Ursula Braun | 8643474 | 2019-12-12 22:35:58 +0100 | [diff] [blame] | 2471 | rc = -EINVAL; |
Ursula Braun | ee9dfbe | 2018-04-26 17:18:21 +0200 | [diff] [blame] | 2472 | } |
| 2473 | break; |
Ursula Braun | 01d2f7e | 2018-04-26 17:18:22 +0200 | [diff] [blame] | 2474 | case TCP_NODELAY: |
Ursula Braun | f9cedf1 | 2019-08-02 10:16:38 +0200 | [diff] [blame] | 2475 | if (sk->sk_state != SMC_INIT && |
| 2476 | sk->sk_state != SMC_LISTEN && |
| 2477 | sk->sk_state != SMC_CLOSED) { |
Guvenc Gulce | e0e4b8f | 2021-06-16 16:52:55 +0200 | [diff] [blame] | 2478 | if (val) { |
Guvenc Gulce | 194730a | 2021-06-16 16:52:58 +0200 | [diff] [blame] | 2479 | SMC_STAT_INC(smc, ndly_cnt); |
Karsten Graul | 22ef473 | 2020-09-10 18:48:29 +0200 | [diff] [blame] | 2480 | mod_delayed_work(smc->conn.lgr->tx_wq, |
| 2481 | &smc->conn.tx_work, 0); |
Guvenc Gulce | e0e4b8f | 2021-06-16 16:52:55 +0200 | [diff] [blame] | 2482 | } |
Ursula Braun | 01d2f7e | 2018-04-26 17:18:22 +0200 | [diff] [blame] | 2483 | } |
| 2484 | break; |
| 2485 | case TCP_CORK: |
Ursula Braun | f9cedf1 | 2019-08-02 10:16:38 +0200 | [diff] [blame] | 2486 | if (sk->sk_state != SMC_INIT && |
| 2487 | sk->sk_state != SMC_LISTEN && |
| 2488 | sk->sk_state != SMC_CLOSED) { |
Guvenc Gulce | e0e4b8f | 2021-06-16 16:52:55 +0200 | [diff] [blame] | 2489 | if (!val) { |
Guvenc Gulce | 194730a | 2021-06-16 16:52:58 +0200 | [diff] [blame] | 2490 | SMC_STAT_INC(smc, cork_cnt); |
Karsten Graul | 22ef473 | 2020-09-10 18:48:29 +0200 | [diff] [blame] | 2491 | mod_delayed_work(smc->conn.lgr->tx_wq, |
| 2492 | &smc->conn.tx_work, 0); |
Guvenc Gulce | e0e4b8f | 2021-06-16 16:52:55 +0200 | [diff] [blame] | 2493 | } |
Ursula Braun | 01d2f7e | 2018-04-26 17:18:22 +0200 | [diff] [blame] | 2494 | } |
| 2495 | break; |
Ursula Braun | abb190f | 2018-04-26 17:18:23 +0200 | [diff] [blame] | 2496 | case TCP_DEFER_ACCEPT: |
| 2497 | smc->sockopt_defer_accept = val; |
| 2498 | break; |
Ursula Braun | ee9dfbe | 2018-04-26 17:18:21 +0200 | [diff] [blame] | 2499 | default: |
| 2500 | break; |
| 2501 | } |
Ursula Braun | 8643474 | 2019-12-12 22:35:58 +0100 | [diff] [blame] | 2502 | out: |
Ursula Braun | ee9dfbe | 2018-04-26 17:18:21 +0200 | [diff] [blame] | 2503 | release_sock(sk); |
| 2504 | |
| 2505 | return rc; |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2506 | } |
| 2507 | |
| 2508 | static int smc_getsockopt(struct socket *sock, int level, int optname, |
| 2509 | char __user *optval, int __user *optlen) |
| 2510 | { |
| 2511 | struct smc_sock *smc; |
| 2512 | |
| 2513 | smc = smc_sk(sock->sk); |
| 2514 | /* socket options apply to the CLC socket */ |
Christoph Hellwig | a44d9e7 | 2020-07-17 08:23:31 +0200 | [diff] [blame] | 2515 | if (unlikely(!smc->clcsock->ops->getsockopt)) |
| 2516 | return -EOPNOTSUPP; |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2517 | return smc->clcsock->ops->getsockopt(smc->clcsock, level, optname, |
| 2518 | optval, optlen); |
| 2519 | } |
| 2520 | |
| 2521 | static int smc_ioctl(struct socket *sock, unsigned int cmd, |
| 2522 | unsigned long arg) |
| 2523 | { |
Stefan Raspl | de8474eb | 2018-05-23 16:38:11 +0200 | [diff] [blame] | 2524 | union smc_host_cursor cons, urg; |
| 2525 | struct smc_connection *conn; |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2526 | struct smc_sock *smc; |
Ursula Braun | 9b67e26 | 2018-05-02 16:56:46 +0200 | [diff] [blame] | 2527 | int answ; |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2528 | |
| 2529 | smc = smc_sk(sock->sk); |
Stefan Raspl | de8474eb | 2018-05-23 16:38:11 +0200 | [diff] [blame] | 2530 | conn = &smc->conn; |
Ursula Braun | 1992d99 | 2018-07-16 13:56:52 +0200 | [diff] [blame] | 2531 | lock_sock(&smc->sk); |
Ursula Braun | 7311d66 | 2018-08-08 14:13:21 +0200 | [diff] [blame] | 2532 | if (smc->use_fallback) { |
| 2533 | if (!smc->clcsock) { |
| 2534 | release_sock(&smc->sk); |
| 2535 | return -EBADF; |
| 2536 | } |
| 2537 | answ = smc->clcsock->ops->ioctl(smc->clcsock, cmd, arg); |
| 2538 | release_sock(&smc->sk); |
| 2539 | return answ; |
| 2540 | } |
Ursula Braun | 9b67e26 | 2018-05-02 16:56:46 +0200 | [diff] [blame] | 2541 | switch (cmd) { |
| 2542 | case SIOCINQ: /* same as FIONREAD */ |
Ursula Braun | 1992d99 | 2018-07-16 13:56:52 +0200 | [diff] [blame] | 2543 | if (smc->sk.sk_state == SMC_LISTEN) { |
| 2544 | release_sock(&smc->sk); |
Ursula Braun | 9b67e26 | 2018-05-02 16:56:46 +0200 | [diff] [blame] | 2545 | return -EINVAL; |
Ursula Braun | 1992d99 | 2018-07-16 13:56:52 +0200 | [diff] [blame] | 2546 | } |
Ursula Braun | 2351abe | 2018-05-23 16:38:09 +0200 | [diff] [blame] | 2547 | if (smc->sk.sk_state == SMC_INIT || |
| 2548 | smc->sk.sk_state == SMC_CLOSED) |
| 2549 | answ = 0; |
| 2550 | else |
| 2551 | answ = atomic_read(&smc->conn.bytes_to_rcv); |
Ursula Braun | 9b67e26 | 2018-05-02 16:56:46 +0200 | [diff] [blame] | 2552 | break; |
| 2553 | case SIOCOUTQ: |
| 2554 | /* output queue size (not send + not acked) */ |
Ursula Braun | 1992d99 | 2018-07-16 13:56:52 +0200 | [diff] [blame] | 2555 | if (smc->sk.sk_state == SMC_LISTEN) { |
| 2556 | release_sock(&smc->sk); |
Ursula Braun | 9b67e26 | 2018-05-02 16:56:46 +0200 | [diff] [blame] | 2557 | return -EINVAL; |
Ursula Braun | 1992d99 | 2018-07-16 13:56:52 +0200 | [diff] [blame] | 2558 | } |
Ursula Braun | 2351abe | 2018-05-23 16:38:09 +0200 | [diff] [blame] | 2559 | if (smc->sk.sk_state == SMC_INIT || |
| 2560 | smc->sk.sk_state == SMC_CLOSED) |
| 2561 | answ = 0; |
| 2562 | else |
| 2563 | answ = smc->conn.sndbuf_desc->len - |
Ursula Braun | 9b67e26 | 2018-05-02 16:56:46 +0200 | [diff] [blame] | 2564 | atomic_read(&smc->conn.sndbuf_space); |
| 2565 | break; |
| 2566 | case SIOCOUTQNSD: |
| 2567 | /* output queue size (not send only) */ |
Ursula Braun | 1992d99 | 2018-07-16 13:56:52 +0200 | [diff] [blame] | 2568 | if (smc->sk.sk_state == SMC_LISTEN) { |
| 2569 | release_sock(&smc->sk); |
Ursula Braun | 9b67e26 | 2018-05-02 16:56:46 +0200 | [diff] [blame] | 2570 | return -EINVAL; |
Ursula Braun | 1992d99 | 2018-07-16 13:56:52 +0200 | [diff] [blame] | 2571 | } |
Ursula Braun | 2351abe | 2018-05-23 16:38:09 +0200 | [diff] [blame] | 2572 | if (smc->sk.sk_state == SMC_INIT || |
| 2573 | smc->sk.sk_state == SMC_CLOSED) |
| 2574 | answ = 0; |
| 2575 | else |
| 2576 | answ = smc_tx_prepared_sends(&smc->conn); |
Ursula Braun | 9b67e26 | 2018-05-02 16:56:46 +0200 | [diff] [blame] | 2577 | break; |
Stefan Raspl | de8474eb | 2018-05-23 16:38:11 +0200 | [diff] [blame] | 2578 | case SIOCATMARK: |
Ursula Braun | 1992d99 | 2018-07-16 13:56:52 +0200 | [diff] [blame] | 2579 | if (smc->sk.sk_state == SMC_LISTEN) { |
| 2580 | release_sock(&smc->sk); |
Stefan Raspl | de8474eb | 2018-05-23 16:38:11 +0200 | [diff] [blame] | 2581 | return -EINVAL; |
Ursula Braun | 1992d99 | 2018-07-16 13:56:52 +0200 | [diff] [blame] | 2582 | } |
Stefan Raspl | de8474eb | 2018-05-23 16:38:11 +0200 | [diff] [blame] | 2583 | if (smc->sk.sk_state == SMC_INIT || |
| 2584 | smc->sk.sk_state == SMC_CLOSED) { |
| 2585 | answ = 0; |
| 2586 | } else { |
Stefan Raspl | bac6de7 | 2018-07-23 13:53:09 +0200 | [diff] [blame] | 2587 | smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn); |
| 2588 | smc_curs_copy(&urg, &conn->urg_curs, conn); |
Stefan Raspl | de8474eb | 2018-05-23 16:38:11 +0200 | [diff] [blame] | 2589 | answ = smc_curs_diff(conn->rmb_desc->len, |
| 2590 | &cons, &urg) == 1; |
| 2591 | } |
| 2592 | break; |
Ursula Braun | 9b67e26 | 2018-05-02 16:56:46 +0200 | [diff] [blame] | 2593 | default: |
Ursula Braun | 1992d99 | 2018-07-16 13:56:52 +0200 | [diff] [blame] | 2594 | release_sock(&smc->sk); |
Ursula Braun | 9b67e26 | 2018-05-02 16:56:46 +0200 | [diff] [blame] | 2595 | return -ENOIOCTLCMD; |
| 2596 | } |
Ursula Braun | 1992d99 | 2018-07-16 13:56:52 +0200 | [diff] [blame] | 2597 | release_sock(&smc->sk); |
Ursula Braun | 9b67e26 | 2018-05-02 16:56:46 +0200 | [diff] [blame] | 2598 | |
| 2599 | return put_user(answ, (int __user *)arg); |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2600 | } |
| 2601 | |
| 2602 | static ssize_t smc_sendpage(struct socket *sock, struct page *page, |
| 2603 | int offset, size_t size, int flags) |
| 2604 | { |
| 2605 | struct sock *sk = sock->sk; |
| 2606 | struct smc_sock *smc; |
| 2607 | int rc = -EPIPE; |
| 2608 | |
| 2609 | smc = smc_sk(sk); |
| 2610 | lock_sock(sk); |
Stefan Raspl | bda27ff | 2018-05-03 17:57:39 +0200 | [diff] [blame] | 2611 | if (sk->sk_state != SMC_ACTIVE) { |
| 2612 | release_sock(sk); |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2613 | goto out; |
Stefan Raspl | bda27ff | 2018-05-03 17:57:39 +0200 | [diff] [blame] | 2614 | } |
| 2615 | release_sock(sk); |
Guvenc Gulce | e0e4b8f | 2021-06-16 16:52:55 +0200 | [diff] [blame] | 2616 | if (smc->use_fallback) { |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2617 | rc = kernel_sendpage(smc->clcsock, page, offset, |
| 2618 | size, flags); |
Guvenc Gulce | e0e4b8f | 2021-06-16 16:52:55 +0200 | [diff] [blame] | 2619 | } else { |
Guvenc Gulce | 194730a | 2021-06-16 16:52:58 +0200 | [diff] [blame] | 2620 | SMC_STAT_INC(smc, sendpage_cnt); |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2621 | rc = sock_no_sendpage(sock, page, offset, size, flags); |
Guvenc Gulce | e0e4b8f | 2021-06-16 16:52:55 +0200 | [diff] [blame] | 2622 | } |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2623 | |
| 2624 | out: |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2625 | return rc; |
| 2626 | } |
| 2627 | |
Stefan Raspl | 9014db2 | 2018-05-03 18:12:39 +0200 | [diff] [blame] | 2628 | /* Map the affected portions of the rmbe into an spd, note the number of bytes |
| 2629 | * to splice in conn->splice_pending, and press 'go'. Delays consumer cursor |
| 2630 | * updates till whenever a respective page has been fully processed. |
| 2631 | * Note that subsequent recv() calls have to wait till all splice() processing |
| 2632 | * completed. |
| 2633 | */ |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2634 | static ssize_t smc_splice_read(struct socket *sock, loff_t *ppos, |
| 2635 | struct pipe_inode_info *pipe, size_t len, |
Stefan Raspl | 9014db2 | 2018-05-03 18:12:39 +0200 | [diff] [blame] | 2636 | unsigned int flags) |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2637 | { |
| 2638 | struct sock *sk = sock->sk; |
| 2639 | struct smc_sock *smc; |
| 2640 | int rc = -ENOTCONN; |
| 2641 | |
| 2642 | smc = smc_sk(sk); |
| 2643 | lock_sock(sk); |
Karsten Graul | 51c5aba | 2019-01-30 18:51:04 +0100 | [diff] [blame] | 2644 | if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) { |
| 2645 | /* socket was connected before, no more data to read */ |
| 2646 | rc = 0; |
| 2647 | goto out; |
| 2648 | } |
Stefan Raspl | 9014db2 | 2018-05-03 18:12:39 +0200 | [diff] [blame] | 2649 | if (sk->sk_state == SMC_INIT || |
| 2650 | sk->sk_state == SMC_LISTEN || |
| 2651 | sk->sk_state == SMC_CLOSED) |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2652 | goto out; |
Stefan Raspl | 9014db2 | 2018-05-03 18:12:39 +0200 | [diff] [blame] | 2653 | |
| 2654 | if (sk->sk_state == SMC_PEERFINCLOSEWAIT) { |
| 2655 | rc = 0; |
| 2656 | goto out; |
| 2657 | } |
| 2658 | |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2659 | if (smc->use_fallback) { |
| 2660 | rc = smc->clcsock->ops->splice_read(smc->clcsock, ppos, |
| 2661 | pipe, len, flags); |
| 2662 | } else { |
Stefan Raspl | 9014db2 | 2018-05-03 18:12:39 +0200 | [diff] [blame] | 2663 | if (*ppos) { |
| 2664 | rc = -ESPIPE; |
| 2665 | goto out; |
| 2666 | } |
| 2667 | if (flags & SPLICE_F_NONBLOCK) |
| 2668 | flags = MSG_DONTWAIT; |
| 2669 | else |
| 2670 | flags = 0; |
Guvenc Gulce | 194730a | 2021-06-16 16:52:58 +0200 | [diff] [blame] | 2671 | SMC_STAT_INC(smc, splice_cnt); |
Stefan Raspl | 9014db2 | 2018-05-03 18:12:39 +0200 | [diff] [blame] | 2672 | rc = smc_rx_recvmsg(smc, NULL, pipe, len, flags); |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2673 | } |
| 2674 | out: |
| 2675 | release_sock(sk); |
Stefan Raspl | 9014db2 | 2018-05-03 18:12:39 +0200 | [diff] [blame] | 2676 | |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2677 | return rc; |
| 2678 | } |
| 2679 | |
| 2680 | /* must look like tcp */ |
| 2681 | static const struct proto_ops smc_sock_ops = { |
| 2682 | .family = PF_SMC, |
| 2683 | .owner = THIS_MODULE, |
| 2684 | .release = smc_release, |
| 2685 | .bind = smc_bind, |
| 2686 | .connect = smc_connect, |
| 2687 | .socketpair = sock_no_socketpair, |
| 2688 | .accept = smc_accept, |
| 2689 | .getname = smc_getname, |
Linus Torvalds | a11e1d4 | 2018-06-28 09:43:44 -0700 | [diff] [blame] | 2690 | .poll = smc_poll, |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2691 | .ioctl = smc_ioctl, |
| 2692 | .listen = smc_listen, |
| 2693 | .shutdown = smc_shutdown, |
| 2694 | .setsockopt = smc_setsockopt, |
| 2695 | .getsockopt = smc_getsockopt, |
| 2696 | .sendmsg = smc_sendmsg, |
| 2697 | .recvmsg = smc_recvmsg, |
| 2698 | .mmap = sock_no_mmap, |
| 2699 | .sendpage = smc_sendpage, |
| 2700 | .splice_read = smc_splice_read, |
| 2701 | }; |
| 2702 | |
| 2703 | static int smc_create(struct net *net, struct socket *sock, int protocol, |
| 2704 | int kern) |
| 2705 | { |
Karsten Graul | aaa4d33 | 2018-03-16 15:06:41 +0100 | [diff] [blame] | 2706 | int family = (protocol == SMCPROTO_SMC6) ? PF_INET6 : PF_INET; |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2707 | struct smc_sock *smc; |
| 2708 | struct sock *sk; |
| 2709 | int rc; |
| 2710 | |
| 2711 | rc = -ESOCKTNOSUPPORT; |
| 2712 | if (sock->type != SOCK_STREAM) |
| 2713 | goto out; |
| 2714 | |
| 2715 | rc = -EPROTONOSUPPORT; |
Karsten Graul | aaa4d33 | 2018-03-16 15:06:41 +0100 | [diff] [blame] | 2716 | if (protocol != SMCPROTO_SMC && protocol != SMCPROTO_SMC6) |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2717 | goto out; |
| 2718 | |
| 2719 | rc = -ENOBUFS; |
| 2720 | sock->ops = &smc_sock_ops; |
Karsten Graul | aaa4d33 | 2018-03-16 15:06:41 +0100 | [diff] [blame] | 2721 | sk = smc_sock_alloc(net, sock, protocol); |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2722 | if (!sk) |
| 2723 | goto out; |
| 2724 | |
| 2725 | /* create internal TCP socket for CLC handshake and fallback */ |
| 2726 | smc = smc_sk(sk); |
Ursula Braun | a046d57 | 2017-01-09 16:55:16 +0100 | [diff] [blame] | 2727 | smc->use_fallback = false; /* assume rdma capability first */ |
Karsten Graul | 603cc14 | 2018-07-25 16:35:32 +0200 | [diff] [blame] | 2728 | smc->fallback_rsn = 0; |
Karsten Graul | aaa4d33 | 2018-03-16 15:06:41 +0100 | [diff] [blame] | 2729 | rc = sock_create_kern(net, family, SOCK_STREAM, IPPROTO_TCP, |
| 2730 | &smc->clcsock); |
Davide Caratti | a5dcb73 | 2018-02-28 12:44:09 +0100 | [diff] [blame] | 2731 | if (rc) { |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2732 | sk_common_release(sk); |
Davide Caratti | a5dcb73 | 2018-02-28 12:44:09 +0100 | [diff] [blame] | 2733 | goto out; |
| 2734 | } |
Ursula Braun | cd6851f | 2017-01-09 16:55:18 +0100 | [diff] [blame] | 2735 | smc->sk.sk_sndbuf = max(smc->clcsock->sk->sk_sndbuf, SMC_BUF_MIN_SIZE); |
| 2736 | smc->sk.sk_rcvbuf = max(smc->clcsock->sk->sk_rcvbuf, SMC_BUF_MIN_SIZE); |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2737 | |
| 2738 | out: |
| 2739 | return rc; |
| 2740 | } |
| 2741 | |
| 2742 | static const struct net_proto_family smc_sock_family_ops = { |
| 2743 | .family = PF_SMC, |
| 2744 | .owner = THIS_MODULE, |
| 2745 | .create = smc_create, |
| 2746 | }; |
| 2747 | |
Hans Wippel | 64e28b5 | 2019-02-21 13:01:02 +0100 | [diff] [blame] | 2748 | unsigned int smc_net_id; |
| 2749 | |
| 2750 | static __net_init int smc_net_init(struct net *net) |
| 2751 | { |
| 2752 | return smc_pnet_net_init(net); |
| 2753 | } |
| 2754 | |
| 2755 | static void __net_exit smc_net_exit(struct net *net) |
| 2756 | { |
| 2757 | smc_pnet_net_exit(net); |
| 2758 | } |
| 2759 | |
Guvenc Gulce | 194730a | 2021-06-16 16:52:58 +0200 | [diff] [blame] | 2760 | static __net_init int smc_net_stat_init(struct net *net) |
| 2761 | { |
| 2762 | return smc_stats_init(net); |
| 2763 | } |
| 2764 | |
| 2765 | static void __net_exit smc_net_stat_exit(struct net *net) |
| 2766 | { |
| 2767 | smc_stats_exit(net); |
| 2768 | } |
| 2769 | |
Hans Wippel | 64e28b5 | 2019-02-21 13:01:02 +0100 | [diff] [blame] | 2770 | static struct pernet_operations smc_net_ops = { |
| 2771 | .init = smc_net_init, |
| 2772 | .exit = smc_net_exit, |
| 2773 | .id = &smc_net_id, |
| 2774 | .size = sizeof(struct smc_net), |
| 2775 | }; |
| 2776 | |
Guvenc Gulce | 194730a | 2021-06-16 16:52:58 +0200 | [diff] [blame] | 2777 | static struct pernet_operations smc_net_stat_ops = { |
| 2778 | .init = smc_net_stat_init, |
| 2779 | .exit = smc_net_stat_exit, |
| 2780 | }; |
| 2781 | |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2782 | static int __init smc_init(void) |
| 2783 | { |
| 2784 | int rc; |
| 2785 | |
Hans Wippel | 64e28b5 | 2019-02-21 13:01:02 +0100 | [diff] [blame] | 2786 | rc = register_pernet_subsys(&smc_net_ops); |
| 2787 | if (rc) |
| 2788 | return rc; |
| 2789 | |
Guvenc Gulce | 194730a | 2021-06-16 16:52:58 +0200 | [diff] [blame] | 2790 | rc = register_pernet_subsys(&smc_net_stat_ops); |
| 2791 | if (rc) |
| 2792 | return rc; |
| 2793 | |
Ursula Braun | 201091e | 2020-09-26 12:44:24 +0200 | [diff] [blame] | 2794 | smc_ism_init(); |
Ursula Braun | b81a5eb | 2020-09-26 12:44:31 +0200 | [diff] [blame] | 2795 | smc_clc_init(); |
Ursula Braun | 201091e | 2020-09-26 12:44:24 +0200 | [diff] [blame] | 2796 | |
Guvenc Gulce | e8372d9 | 2020-12-01 20:20:43 +0100 | [diff] [blame] | 2797 | rc = smc_nl_init(); |
Thomas Richter | 6812baa | 2017-01-09 16:55:15 +0100 | [diff] [blame] | 2798 | if (rc) |
YueHaibing | 8c33bf1 | 2019-06-26 17:47:50 +0200 | [diff] [blame] | 2799 | goto out_pernet_subsys; |
Thomas Richter | 6812baa | 2017-01-09 16:55:15 +0100 | [diff] [blame] | 2800 | |
Guvenc Gulce | e8372d9 | 2020-12-01 20:20:43 +0100 | [diff] [blame] | 2801 | rc = smc_pnet_init(); |
| 2802 | if (rc) |
| 2803 | goto out_nl; |
| 2804 | |
Karsten Graul | 22ef473 | 2020-09-10 18:48:29 +0200 | [diff] [blame] | 2805 | rc = -ENOMEM; |
| 2806 | smc_hs_wq = alloc_workqueue("smc_hs_wq", 0, 0); |
| 2807 | if (!smc_hs_wq) |
| 2808 | goto out_pnet; |
| 2809 | |
| 2810 | smc_close_wq = alloc_workqueue("smc_close_wq", 0, 0); |
| 2811 | if (!smc_close_wq) |
| 2812 | goto out_alloc_hs_wq; |
| 2813 | |
Ursula Braun | 6dabd40 | 2019-11-16 17:47:29 +0100 | [diff] [blame] | 2814 | rc = smc_core_init(); |
| 2815 | if (rc) { |
| 2816 | pr_err("%s: smc_core_init fails with %d\n", __func__, rc); |
Guvenc Gulce | 194730a | 2021-06-16 16:52:58 +0200 | [diff] [blame] | 2817 | goto out_alloc_wqs; |
Ursula Braun | 6dabd40 | 2019-11-16 17:47:29 +0100 | [diff] [blame] | 2818 | } |
| 2819 | |
Ursula Braun | 9bf9abe | 2017-01-09 16:55:21 +0100 | [diff] [blame] | 2820 | rc = smc_llc_init(); |
| 2821 | if (rc) { |
| 2822 | pr_err("%s: smc_llc_init fails with %d\n", __func__, rc); |
Ursula Braun | 6dabd40 | 2019-11-16 17:47:29 +0100 | [diff] [blame] | 2823 | goto out_core; |
Ursula Braun | 9bf9abe | 2017-01-09 16:55:21 +0100 | [diff] [blame] | 2824 | } |
| 2825 | |
Ursula Braun | 5f08318 | 2017-01-09 16:55:22 +0100 | [diff] [blame] | 2826 | rc = smc_cdc_init(); |
| 2827 | if (rc) { |
| 2828 | pr_err("%s: smc_cdc_init fails with %d\n", __func__, rc); |
Ursula Braun | 6dabd40 | 2019-11-16 17:47:29 +0100 | [diff] [blame] | 2829 | goto out_core; |
Ursula Braun | 5f08318 | 2017-01-09 16:55:22 +0100 | [diff] [blame] | 2830 | } |
| 2831 | |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2832 | rc = proto_register(&smc_proto, 1); |
| 2833 | if (rc) { |
Karsten Graul | aaa4d33 | 2018-03-16 15:06:41 +0100 | [diff] [blame] | 2834 | pr_err("%s: proto_register(v4) fails with %d\n", __func__, rc); |
Ursula Braun | 6dabd40 | 2019-11-16 17:47:29 +0100 | [diff] [blame] | 2835 | goto out_core; |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2836 | } |
| 2837 | |
Karsten Graul | aaa4d33 | 2018-03-16 15:06:41 +0100 | [diff] [blame] | 2838 | rc = proto_register(&smc_proto6, 1); |
| 2839 | if (rc) { |
| 2840 | pr_err("%s: proto_register(v6) fails with %d\n", __func__, rc); |
| 2841 | goto out_proto; |
| 2842 | } |
| 2843 | |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2844 | rc = sock_register(&smc_sock_family_ops); |
| 2845 | if (rc) { |
| 2846 | pr_err("%s: sock_register fails with %d\n", __func__, rc); |
Karsten Graul | aaa4d33 | 2018-03-16 15:06:41 +0100 | [diff] [blame] | 2847 | goto out_proto6; |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2848 | } |
Ursula Braun | f16a7dd | 2017-01-09 16:55:26 +0100 | [diff] [blame] | 2849 | INIT_HLIST_HEAD(&smc_v4_hashinfo.ht); |
Karsten Graul | aaa4d33 | 2018-03-16 15:06:41 +0100 | [diff] [blame] | 2850 | INIT_HLIST_HEAD(&smc_v6_hashinfo.ht); |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2851 | |
Ursula Braun | a4cf044 | 2017-01-09 16:55:14 +0100 | [diff] [blame] | 2852 | rc = smc_ib_register_client(); |
| 2853 | if (rc) { |
| 2854 | pr_err("%s: ib_register fails with %d\n", __func__, rc); |
| 2855 | goto out_sock; |
| 2856 | } |
| 2857 | |
Ursula Braun | c5c1cc9 | 2017-10-25 11:01:46 +0200 | [diff] [blame] | 2858 | static_branch_enable(&tcp_have_smc); |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2859 | return 0; |
| 2860 | |
Ursula Braun | a4cf044 | 2017-01-09 16:55:14 +0100 | [diff] [blame] | 2861 | out_sock: |
| 2862 | sock_unregister(PF_SMC); |
Karsten Graul | aaa4d33 | 2018-03-16 15:06:41 +0100 | [diff] [blame] | 2863 | out_proto6: |
| 2864 | proto_unregister(&smc_proto6); |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2865 | out_proto: |
| 2866 | proto_unregister(&smc_proto); |
Ursula Braun | 6dabd40 | 2019-11-16 17:47:29 +0100 | [diff] [blame] | 2867 | out_core: |
| 2868 | smc_core_exit(); |
Karsten Graul | 22ef473 | 2020-09-10 18:48:29 +0200 | [diff] [blame] | 2869 | out_alloc_wqs: |
| 2870 | destroy_workqueue(smc_close_wq); |
| 2871 | out_alloc_hs_wq: |
| 2872 | destroy_workqueue(smc_hs_wq); |
Thomas Richter | 6812baa | 2017-01-09 16:55:15 +0100 | [diff] [blame] | 2873 | out_pnet: |
| 2874 | smc_pnet_exit(); |
Guvenc Gulce | e8372d9 | 2020-12-01 20:20:43 +0100 | [diff] [blame] | 2875 | out_nl: |
| 2876 | smc_nl_exit(); |
YueHaibing | 8c33bf1 | 2019-06-26 17:47:50 +0200 | [diff] [blame] | 2877 | out_pernet_subsys: |
| 2878 | unregister_pernet_subsys(&smc_net_ops); |
| 2879 | |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2880 | return rc; |
| 2881 | } |
| 2882 | |
| 2883 | static void __exit smc_exit(void) |
| 2884 | { |
Ursula Braun | c5c1cc9 | 2017-10-25 11:01:46 +0200 | [diff] [blame] | 2885 | static_branch_disable(&tcp_have_smc); |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2886 | sock_unregister(PF_SMC); |
Ursula Braun | 6dabd40 | 2019-11-16 17:47:29 +0100 | [diff] [blame] | 2887 | smc_core_exit(); |
| 2888 | smc_ib_unregister_client(); |
Karsten Graul | 22ef473 | 2020-09-10 18:48:29 +0200 | [diff] [blame] | 2889 | destroy_workqueue(smc_close_wq); |
| 2890 | destroy_workqueue(smc_hs_wq); |
Karsten Graul | aaa4d33 | 2018-03-16 15:06:41 +0100 | [diff] [blame] | 2891 | proto_unregister(&smc_proto6); |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2892 | proto_unregister(&smc_proto); |
Thomas Richter | 6812baa | 2017-01-09 16:55:15 +0100 | [diff] [blame] | 2893 | smc_pnet_exit(); |
Guvenc Gulce | e8372d9 | 2020-12-01 20:20:43 +0100 | [diff] [blame] | 2894 | smc_nl_exit(); |
Karsten Graul | fa08666 | 2021-09-14 10:35:05 +0200 | [diff] [blame] | 2895 | smc_clc_exit(); |
Guvenc Gulce | 194730a | 2021-06-16 16:52:58 +0200 | [diff] [blame] | 2896 | unregister_pernet_subsys(&smc_net_stat_ops); |
Hans Wippel | 64e28b5 | 2019-02-21 13:01:02 +0100 | [diff] [blame] | 2897 | unregister_pernet_subsys(&smc_net_ops); |
Ursula Braun | 4ead9c9 | 2019-11-16 17:47:31 +0100 | [diff] [blame] | 2898 | rcu_barrier(); |
Ursula Braun | ac71387 | 2017-01-09 16:55:13 +0100 | [diff] [blame] | 2899 | } |
| 2900 | |
| 2901 | module_init(smc_init); |
| 2902 | module_exit(smc_exit); |
| 2903 | |
| 2904 | MODULE_AUTHOR("Ursula Braun <ubraun@linux.vnet.ibm.com>"); |
| 2905 | MODULE_DESCRIPTION("smc socket address family"); |
| 2906 | MODULE_LICENSE("GPL"); |
| 2907 | MODULE_ALIAS_NETPROTO(PF_SMC); |