Dave Watson | 3c4d755 | 2017-06-14 11:37:39 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved. |
| 3 | * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved. |
| 4 | * |
| 5 | * This software is available to you under a choice of one of two |
| 6 | * licenses. You may choose to be licensed under the terms of the GNU |
| 7 | * General Public License (GPL) Version 2, available from the file |
| 8 | * COPYING in the main directory of this source tree, or the |
| 9 | * OpenIB.org BSD license below: |
| 10 | * |
| 11 | * Redistribution and use in source and binary forms, with or |
| 12 | * without modification, are permitted provided that the following |
| 13 | * conditions are met: |
| 14 | * |
| 15 | * - Redistributions of source code must retain the above |
| 16 | * copyright notice, this list of conditions and the following |
| 17 | * disclaimer. |
| 18 | * |
| 19 | * - Redistributions in binary form must reproduce the above |
| 20 | * copyright notice, this list of conditions and the following |
| 21 | * disclaimer in the documentation and/or other materials |
| 22 | * provided with the distribution. |
| 23 | * |
| 24 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 25 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 26 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| 27 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
| 28 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
| 29 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
| 30 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 31 | * SOFTWARE. |
| 32 | */ |
| 33 | |
| 34 | #ifndef _TLS_OFFLOAD_H |
| 35 | #define _TLS_OFFLOAD_H |
| 36 | |
| 37 | #include <linux/types.h> |
Dmitry V. Levin | b9f3eb4 | 2017-11-14 06:30:11 +0300 | [diff] [blame] | 38 | #include <asm/byteorder.h> |
Vakul Garg | a54667f | 2018-01-31 21:34:37 +0530 | [diff] [blame] | 39 | #include <linux/crypto.h> |
Dmitry V. Levin | b9f3eb4 | 2017-11-14 06:30:11 +0300 | [diff] [blame] | 40 | #include <linux/socket.h> |
| 41 | #include <linux/tcp.h> |
Jakub Kicinski | 79ffe60 | 2019-11-05 14:24:35 -0800 | [diff] [blame] | 42 | #include <linux/mutex.h> |
Jakub Kicinski | 2e36117 | 2019-06-05 14:11:39 -0700 | [diff] [blame] | 43 | #include <linux/netdevice.h> |
Jakub Kicinski | 15a7dea | 2019-08-30 12:25:47 +0200 | [diff] [blame] | 44 | #include <linux/rcupdate.h> |
Daniel Borkmann | d829e9c | 2018-10-13 02:45:59 +0200 | [diff] [blame] | 45 | |
Jakub Kicinski | d26b698 | 2019-10-04 16:19:24 -0700 | [diff] [blame] | 46 | #include <net/net_namespace.h> |
Dmitry V. Levin | b9f3eb4 | 2017-11-14 06:30:11 +0300 | [diff] [blame] | 47 | #include <net/tcp.h> |
Dave Watson | c46234e | 2018-03-22 10:10:35 -0700 | [diff] [blame] | 48 | #include <net/strparser.h> |
Vakul Garg | a42055e | 2018-09-21 09:46:13 +0530 | [diff] [blame] | 49 | #include <crypto/aead.h> |
Dave Watson | 3c4d755 | 2017-06-14 11:37:39 -0700 | [diff] [blame] | 50 | #include <uapi/linux/tls.h> |
| 51 | |
Jakub Kicinski | 5879031 | 2022-07-07 18:03:13 -0700 | [diff] [blame] | 52 | struct tls_rec; |
Dave Watson | 3c4d755 | 2017-06-14 11:37:39 -0700 | [diff] [blame] | 53 | |
| 54 | /* Maximum data size carried in a TLS record */ |
| 55 | #define TLS_MAX_PAYLOAD_SIZE ((size_t)1 << 14) |
| 56 | |
| 57 | #define TLS_HEADER_SIZE 5 |
| 58 | #define TLS_NONCE_OFFSET TLS_HEADER_SIZE |
| 59 | |
| 60 | #define TLS_CRYPTO_INFO_READY(info) ((info)->cipher_type) |
| 61 | |
Dave Watson | 3c4d755 | 2017-06-14 11:37:39 -0700 | [diff] [blame] | 62 | #define TLS_AAD_SPACE_SIZE 13 |
Atul Gupta | dd0bed1 | 2018-03-31 21:41:52 +0530 | [diff] [blame] | 63 | |
Sabrina Dubroca | bee6b7b | 2023-10-09 22:50:44 +0200 | [diff] [blame] | 64 | #define TLS_MAX_IV_SIZE 16 |
Sabrina Dubroca | 1c1cb31 | 2023-10-09 22:50:45 +0200 | [diff] [blame] | 65 | #define TLS_MAX_SALT_SIZE 4 |
Jakub Kicinski | a8340cc | 2022-04-07 20:38:19 -0700 | [diff] [blame] | 66 | #define TLS_TAG_SIZE 16 |
Jakub Kicinski | 89fec47 | 2019-06-10 21:40:00 -0700 | [diff] [blame] | 67 | #define TLS_MAX_REC_SEQ_SIZE 8 |
Jakub Kicinski | 50a07aa | 2022-07-07 18:03:10 -0700 | [diff] [blame] | 68 | #define TLS_MAX_AAD_SIZE TLS_AAD_SPACE_SIZE |
Vakul Garg | f295b3a | 2019-03-20 02:03:36 +0000 | [diff] [blame] | 69 | |
Tianjia Zhang | 128cfb8 | 2021-09-28 14:28:43 +0800 | [diff] [blame] | 70 | /* For CCM mode, the full 16-bytes of IV is made of '4' fields of given sizes. |
Vakul Garg | f295b3a | 2019-03-20 02:03:36 +0000 | [diff] [blame] | 71 | * |
| 72 | * IV[16] = b0[1] || implicit nonce[4] || explicit nonce[8] || length[3] |
| 73 | * |
| 74 | * The field 'length' is encoded in field 'b0' as '(length width - 1)'. |
| 75 | * Hence b0 contains (3 - 1) = 2. |
| 76 | */ |
| 77 | #define TLS_AES_CCM_IV_B0_BYTE 2 |
Tianjia Zhang | 128cfb8 | 2021-09-28 14:28:43 +0800 | [diff] [blame] | 78 | #define TLS_SM4_CCM_IV_B0_BYTE 2 |
Vakul Garg | f295b3a | 2019-03-20 02:03:36 +0000 | [diff] [blame] | 79 | |
Boris Pismenny | 4799ac8 | 2018-07-13 14:33:43 +0300 | [diff] [blame] | 80 | enum { |
| 81 | TLS_BASE, |
| 82 | TLS_SW, |
Boris Pismenny | 4799ac8 | 2018-07-13 14:33:43 +0300 | [diff] [blame] | 83 | TLS_HW, |
Boris Pismenny | 4799ac8 | 2018-07-13 14:33:43 +0300 | [diff] [blame] | 84 | TLS_HW_RECORD, |
| 85 | TLS_NUM_CONFIG, |
| 86 | }; |
| 87 | |
Vakul Garg | a42055e | 2018-09-21 09:46:13 +0530 | [diff] [blame] | 88 | struct tx_work { |
| 89 | struct delayed_work work; |
| 90 | struct sock *sk; |
| 91 | }; |
| 92 | |
| 93 | struct tls_sw_context_tx { |
| 94 | struct crypto_aead *aead_send; |
| 95 | struct crypto_wait async_wait; |
| 96 | struct tx_work tx_work; |
| 97 | struct tls_rec *open_rec; |
Vakul Garg | 9932a29 | 2018-09-24 15:35:56 +0530 | [diff] [blame] | 98 | struct list_head tx_list; |
Vakul Garg | a42055e | 2018-09-21 09:46:13 +0530 | [diff] [blame] | 99 | atomic_t encrypt_pending; |
Jakub Kicinski | 5c5458e | 2019-10-06 21:09:31 -0700 | [diff] [blame] | 100 | u8 async_capable:1; |
Vakul Garg | a42055e | 2018-09-21 09:46:13 +0530 | [diff] [blame] | 101 | |
| 102 | #define BIT_TX_SCHEDULED 0 |
John Fastabend | f87e62d | 2019-07-19 10:29:16 -0700 | [diff] [blame] | 103 | #define BIT_TX_CLOSING 1 |
Vakul Garg | a42055e | 2018-09-21 09:46:13 +0530 | [diff] [blame] | 104 | unsigned long tx_bitmask; |
Dave Watson | 3c4d755 | 2017-06-14 11:37:39 -0700 | [diff] [blame] | 105 | }; |
| 106 | |
Jakub Kicinski | 84c61fe | 2022-07-22 16:50:33 -0700 | [diff] [blame] | 107 | struct tls_strparser { |
| 108 | struct sock *sk; |
| 109 | |
| 110 | u32 mark : 8; |
| 111 | u32 stopped : 1; |
| 112 | u32 copy_mode : 1; |
Jakub Kicinski | eca9bfa | 2023-05-16 18:50:41 -0700 | [diff] [blame] | 113 | u32 mixed_decrypted : 1; |
Sabrina Dubroca | 0844370 | 2024-04-24 12:25:47 +0200 | [diff] [blame] | 114 | |
| 115 | bool msg_ready; |
Jakub Kicinski | 84c61fe | 2022-07-22 16:50:33 -0700 | [diff] [blame] | 116 | |
| 117 | struct strp_msg stm; |
| 118 | |
| 119 | struct sk_buff *anchor; |
| 120 | struct work_struct work; |
| 121 | }; |
| 122 | |
Boris Pismenny | f66de3e | 2018-04-30 10:16:15 +0300 | [diff] [blame] | 123 | struct tls_sw_context_rx { |
| 124 | struct crypto_aead *aead_recv; |
| 125 | struct crypto_wait async_wait; |
Vakul Garg | 692d7b5 | 2019-01-16 10:40:16 +0000 | [diff] [blame] | 126 | struct sk_buff_head rx_list; /* list of decrypted 'data' records */ |
Boris Pismenny | f66de3e | 2018-04-30 10:16:15 +0300 | [diff] [blame] | 127 | void (*saved_data_ready)(struct sock *sk); |
John Fastabend | 924ad65 | 2018-10-13 02:46:00 +0200 | [diff] [blame] | 128 | |
Jakub Kicinski | 4cbc325 | 2022-07-14 22:22:25 -0700 | [diff] [blame] | 129 | u8 reader_present; |
Jakub Kicinski | 5c5458e | 2019-10-06 21:09:31 -0700 | [diff] [blame] | 130 | u8 async_capable:1; |
Jakub Kicinski | 8852779 | 2022-07-05 16:59:24 -0700 | [diff] [blame] | 131 | u8 zc_capable:1; |
Jakub Kicinski | 4cbc325 | 2022-07-14 22:22:25 -0700 | [diff] [blame] | 132 | u8 reader_contended:1; |
Jakub Kicinski | 84c61fe | 2022-07-22 16:50:33 -0700 | [diff] [blame] | 133 | |
| 134 | struct tls_strparser strp; |
| 135 | |
Vakul Garg | 94524d8 | 2018-08-29 15:26:55 +0530 | [diff] [blame] | 136 | atomic_t decrypt_pending; |
Jakub Kicinski | c618db2 | 2022-07-14 22:22:33 -0700 | [diff] [blame] | 137 | struct sk_buff_head async_hold; |
Jakub Kicinski | 4cbc325 | 2022-07-14 22:22:25 -0700 | [diff] [blame] | 138 | struct wait_queue_head wq; |
Vakul Garg | 94524d8 | 2018-08-29 15:26:55 +0530 | [diff] [blame] | 139 | }; |
| 140 | |
Ilya Lesokhin | e8f6979 | 2018-04-30 10:16:16 +0300 | [diff] [blame] | 141 | struct tls_record_info { |
| 142 | struct list_head list; |
| 143 | u32 end_seq; |
| 144 | int len; |
| 145 | int num_frags; |
| 146 | skb_frag_t frags[MAX_SKB_FRAGS]; |
| 147 | }; |
| 148 | |
Sabrina Dubroca | 9f0c824 | 2023-10-09 22:50:54 +0200 | [diff] [blame] | 149 | #define TLS_DRIVER_STATE_SIZE_TX 16 |
Boris Pismenny | d80a1b9d | 2018-07-13 14:33:39 +0300 | [diff] [blame] | 150 | struct tls_offload_context_tx { |
Ilya Lesokhin | e8f6979 | 2018-04-30 10:16:16 +0300 | [diff] [blame] | 151 | struct crypto_aead *aead_send; |
| 152 | spinlock_t lock; /* protects records list */ |
| 153 | struct list_head records_list; |
| 154 | struct tls_record_info *open_record; |
| 155 | struct tls_record_info *retransmit_hint; |
| 156 | u64 hint_record_sn; |
| 157 | u64 unacked_record_sn; |
| 158 | |
| 159 | struct scatterlist sg_tx_data[MAX_SKB_FRAGS]; |
| 160 | void (*sk_destruct)(struct sock *sk); |
Tariq Toukan | 7adc91e | 2022-07-27 12:43:42 +0300 | [diff] [blame] | 161 | struct work_struct destruct_work; |
| 162 | struct tls_context *ctx; |
Ilya Lesokhin | e8f6979 | 2018-04-30 10:16:16 +0300 | [diff] [blame] | 163 | /* The TLS layer reserves room for driver specific state |
| 164 | * Currently the belief is that there is not enough |
| 165 | * driver specific state to justify another layer of indirection |
| 166 | */ |
Sabrina Dubroca | 9f0c824 | 2023-10-09 22:50:54 +0200 | [diff] [blame] | 167 | u8 driver_state[TLS_DRIVER_STATE_SIZE_TX] __aligned(8); |
Ilya Lesokhin | e8f6979 | 2018-04-30 10:16:16 +0300 | [diff] [blame] | 168 | }; |
| 169 | |
Jakub Kicinski | e52972c | 2019-06-04 12:00:12 -0700 | [diff] [blame] | 170 | enum tls_context_flags { |
Maxim Mikityanskiy | c55dcdd | 2021-06-01 15:08:00 +0300 | [diff] [blame] | 171 | /* tls_device_down was called after the netdev went down, device state |
| 172 | * was released, and kTLS works in software, even though rx_conf is |
| 173 | * still TLS_HW (needed for transition). |
| 174 | */ |
| 175 | TLS_RX_DEV_DEGRADED = 0, |
Jakub Kicinski | 5018007 | 2019-06-10 21:40:09 -0700 | [diff] [blame] | 176 | /* Unlike RX where resync is driven entirely by the core in TX only |
| 177 | * the driver knows when things went out of sync, so we need the flag |
| 178 | * to be atomic. |
| 179 | */ |
| 180 | TLS_TX_SYNC_SCHED = 1, |
Maxim Mikityanskiy | 025cc2f | 2020-11-25 14:18:10 -0800 | [diff] [blame] | 181 | /* tls_dev_del was called for the RX side, device state was released, |
| 182 | * but tls_ctx->netdev might still be kept, because TX-side driver |
| 183 | * resources might not be released yet. Used to prevent the second |
| 184 | * tls_dev_del call in tls_device_down if it happens simultaneously. |
| 185 | */ |
| 186 | TLS_RX_DEV_CLOSED = 2, |
Jakub Kicinski | e52972c | 2019-06-04 12:00:12 -0700 | [diff] [blame] | 187 | }; |
| 188 | |
Dave Watson | dbe4255 | 2018-03-22 10:10:06 -0700 | [diff] [blame] | 189 | struct cipher_context { |
Sabrina Dubroca | 1c1cb31 | 2023-10-09 22:50:45 +0200 | [diff] [blame] | 190 | char iv[TLS_MAX_IV_SIZE + TLS_MAX_SALT_SIZE]; |
Sabrina Dubroca | 6d5029e | 2023-10-09 22:50:43 +0200 | [diff] [blame] | 191 | char rec_seq[TLS_MAX_REC_SEQ_SIZE]; |
Dave Watson | dbe4255 | 2018-03-22 10:10:06 -0700 | [diff] [blame] | 192 | }; |
| 193 | |
Sabrina Dubroca | 86029d1 | 2018-09-12 17:44:42 +0200 | [diff] [blame] | 194 | union tls_crypto_context { |
| 195 | struct tls_crypto_info info; |
Dave Watson | fb99bce | 2019-01-30 21:58:05 +0000 | [diff] [blame] | 196 | union { |
| 197 | struct tls12_crypto_info_aes_gcm_128 aes_gcm_128; |
| 198 | struct tls12_crypto_info_aes_gcm_256 aes_gcm_256; |
Vadim Fedorenko | 923c40c | 2020-11-24 18:24:47 +0300 | [diff] [blame] | 199 | struct tls12_crypto_info_chacha20_poly1305 chacha20_poly1305; |
Tianjia Zhang | 39d8fb9 | 2021-10-25 21:04:39 +0800 | [diff] [blame] | 200 | struct tls12_crypto_info_sm4_gcm sm4_gcm; |
| 201 | struct tls12_crypto_info_sm4_ccm sm4_ccm; |
Dave Watson | fb99bce | 2019-01-30 21:58:05 +0000 | [diff] [blame] | 202 | }; |
Sabrina Dubroca | 86029d1 | 2018-09-12 17:44:42 +0200 | [diff] [blame] | 203 | }; |
| 204 | |
Vakul Garg | 4509de1 | 2019-02-14 07:11:35 +0000 | [diff] [blame] | 205 | struct tls_prot_info { |
| 206 | u16 version; |
| 207 | u16 cipher_type; |
| 208 | u16 prepend_size; |
| 209 | u16 tag_size; |
| 210 | u16 overhead_size; |
| 211 | u16 iv_size; |
Vakul Garg | f295b3a | 2019-03-20 02:03:36 +0000 | [diff] [blame] | 212 | u16 salt_size; |
Vakul Garg | 4509de1 | 2019-02-14 07:11:35 +0000 | [diff] [blame] | 213 | u16 rec_seq_size; |
| 214 | u16 aad_size; |
| 215 | u16 tail_size; |
| 216 | }; |
| 217 | |
Dave Watson | 3c4d755 | 2017-06-14 11:37:39 -0700 | [diff] [blame] | 218 | struct tls_context { |
Jakub Kicinski | f0aaa2c | 2019-06-03 15:17:04 -0700 | [diff] [blame] | 219 | /* read-only cache line */ |
Vakul Garg | 4509de1 | 2019-02-14 07:11:35 +0000 | [diff] [blame] | 220 | struct tls_prot_info prot_info; |
| 221 | |
Boris Pismenny | f66de3e | 2018-04-30 10:16:15 +0300 | [diff] [blame] | 222 | u8 tx_conf:3; |
| 223 | u8 rx_conf:3; |
Boris Pismenny | c1318b3 | 2022-05-18 12:27:31 +0300 | [diff] [blame] | 224 | u8 zerocopy_sendfile:1; |
Jakub Kicinski | 8852779 | 2022-07-05 16:59:24 -0700 | [diff] [blame] | 225 | u8 rx_no_pad:1; |
Ilya Lesokhin | 6d88207 | 2017-11-13 10:22:45 +0200 | [diff] [blame] | 226 | |
Jakub Kicinski | f0aaa2c | 2019-06-03 15:17:04 -0700 | [diff] [blame] | 227 | int (*push_pending_record)(struct sock *sk, int flags); |
| 228 | void (*sk_write_space)(struct sock *sk); |
| 229 | |
| 230 | void *priv_ctx_tx; |
| 231 | void *priv_ctx_rx; |
| 232 | |
Maxim Mikityanskiy | 94ce3b6 | 2022-08-10 11:16:02 +0300 | [diff] [blame] | 233 | struct net_device __rcu *netdev; |
Jakub Kicinski | f0aaa2c | 2019-06-03 15:17:04 -0700 | [diff] [blame] | 234 | |
| 235 | /* rw cache line */ |
Dave Watson | dbe4255 | 2018-03-22 10:10:06 -0700 | [diff] [blame] | 236 | struct cipher_context tx; |
Dave Watson | c46234e | 2018-03-22 10:10:35 -0700 | [diff] [blame] | 237 | struct cipher_context rx; |
Dave Watson | 3c4d755 | 2017-06-14 11:37:39 -0700 | [diff] [blame] | 238 | |
| 239 | struct scatterlist *partially_sent_record; |
| 240 | u16 partially_sent_offset; |
Vakul Garg | a42055e | 2018-09-21 09:46:13 +0530 | [diff] [blame] | 241 | |
David Howells | e117dcf | 2023-05-22 13:11:17 +0100 | [diff] [blame] | 242 | bool splicing_pages; |
Daniel Borkmann | d829e9c | 2018-10-13 02:45:59 +0200 | [diff] [blame] | 243 | bool pending_open_record_frags; |
Jakub Kicinski | 79ffe60 | 2019-11-05 14:24:35 -0800 | [diff] [blame] | 244 | |
| 245 | struct mutex tx_lock; /* protects partially_sent_* fields and |
| 246 | * per-type TX fields |
| 247 | */ |
Jakub Kicinski | f0aaa2c | 2019-06-03 15:17:04 -0700 | [diff] [blame] | 248 | unsigned long flags; |
Dave Watson | 3c4d755 | 2017-06-14 11:37:39 -0700 | [diff] [blame] | 249 | |
Jakub Kicinski | f0aaa2c | 2019-06-03 15:17:04 -0700 | [diff] [blame] | 250 | /* cache cold stuff */ |
John Fastabend | 32857cf | 2019-07-19 10:29:18 -0700 | [diff] [blame] | 251 | struct proto *sk_proto; |
Maxim Mikityanskiy | c55dcdd | 2021-06-01 15:08:00 +0300 | [diff] [blame] | 252 | struct sock *sk; |
John Fastabend | 32857cf | 2019-07-19 10:29:18 -0700 | [diff] [blame] | 253 | |
Boris Pismenny | 4799ac8 | 2018-07-13 14:33:43 +0300 | [diff] [blame] | 254 | void (*sk_destruct)(struct sock *sk); |
Jakub Kicinski | f0aaa2c | 2019-06-03 15:17:04 -0700 | [diff] [blame] | 255 | |
| 256 | union tls_crypto_context crypto_send; |
| 257 | union tls_crypto_context crypto_recv; |
| 258 | |
| 259 | struct list_head list; |
| 260 | refcount_t refcount; |
Jakub Kicinski | 15a7dea | 2019-08-30 12:25:47 +0200 | [diff] [blame] | 261 | struct rcu_head rcu; |
Dave Watson | 3c4d755 | 2017-06-14 11:37:39 -0700 | [diff] [blame] | 262 | }; |
| 263 | |
Jakub Kicinski | da68b4a | 2019-04-25 12:32:03 -0700 | [diff] [blame] | 264 | enum tls_offload_ctx_dir { |
| 265 | TLS_OFFLOAD_CTX_DIR_RX, |
| 266 | TLS_OFFLOAD_CTX_DIR_TX, |
| 267 | }; |
| 268 | |
| 269 | struct tlsdev_ops { |
| 270 | int (*tls_dev_add)(struct net_device *netdev, struct sock *sk, |
| 271 | enum tls_offload_ctx_dir direction, |
| 272 | struct tls_crypto_info *crypto_info, |
| 273 | u32 start_offload_tcp_sn); |
| 274 | void (*tls_dev_del)(struct net_device *netdev, |
| 275 | struct tls_context *ctx, |
| 276 | enum tls_offload_ctx_dir direction); |
Dirk van der Merwe | b5d9a83 | 2019-07-08 19:53:13 -0700 | [diff] [blame] | 277 | int (*tls_dev_resync)(struct net_device *netdev, |
| 278 | struct sock *sk, u32 seq, u8 *rcd_sn, |
| 279 | enum tls_offload_ctx_dir direction); |
Jakub Kicinski | da68b4a | 2019-04-25 12:32:03 -0700 | [diff] [blame] | 280 | }; |
| 281 | |
Jakub Kicinski | f953d33b | 2019-06-10 21:40:02 -0700 | [diff] [blame] | 282 | enum tls_offload_sync_type { |
| 283 | TLS_OFFLOAD_SYNC_TYPE_DRIVER_REQ = 0, |
| 284 | TLS_OFFLOAD_SYNC_TYPE_CORE_NEXT_HINT = 1, |
Boris Pismenny | ed9b764 | 2020-06-08 19:11:38 +0300 | [diff] [blame] | 285 | TLS_OFFLOAD_SYNC_TYPE_DRIVER_REQ_ASYNC = 2, |
Jakub Kicinski | f953d33b | 2019-06-10 21:40:02 -0700 | [diff] [blame] | 286 | }; |
| 287 | |
| 288 | #define TLS_DEVICE_RESYNC_NH_START_IVAL 2 |
| 289 | #define TLS_DEVICE_RESYNC_NH_MAX_IVAL 128 |
| 290 | |
Boris Pismenny | ed9b764 | 2020-06-08 19:11:38 +0300 | [diff] [blame] | 291 | #define TLS_DEVICE_RESYNC_ASYNC_LOGMAX 13 |
| 292 | struct tls_offload_resync_async { |
| 293 | atomic64_t req; |
Tariq Toukan | 138559b | 2020-11-15 15:14:48 +0200 | [diff] [blame] | 294 | u16 loglen; |
| 295 | u16 rcd_delta; |
Boris Pismenny | ed9b764 | 2020-06-08 19:11:38 +0300 | [diff] [blame] | 296 | u32 log[TLS_DEVICE_RESYNC_ASYNC_LOGMAX]; |
| 297 | }; |
| 298 | |
Sabrina Dubroca | 9f0c824 | 2023-10-09 22:50:54 +0200 | [diff] [blame] | 299 | #define TLS_DRIVER_STATE_SIZE_RX 8 |
Boris Pismenny | 4799ac8 | 2018-07-13 14:33:43 +0300 | [diff] [blame] | 300 | struct tls_offload_context_rx { |
| 301 | /* sw must be the first member of tls_offload_context_rx */ |
| 302 | struct tls_sw_context_rx sw; |
Jakub Kicinski | f953d33b | 2019-06-10 21:40:02 -0700 | [diff] [blame] | 303 | enum tls_offload_sync_type resync_type; |
| 304 | /* this member is set regardless of resync_type, to avoid branches */ |
| 305 | u8 resync_nh_reset:1; |
| 306 | /* CORE_NEXT_HINT-only member, but use the hole here */ |
| 307 | u8 resync_nh_do_now:1; |
| 308 | union { |
| 309 | /* TLS_OFFLOAD_SYNC_TYPE_DRIVER_REQ */ |
| 310 | struct { |
| 311 | atomic64_t resync_req; |
| 312 | }; |
| 313 | /* TLS_OFFLOAD_SYNC_TYPE_CORE_NEXT_HINT */ |
| 314 | struct { |
| 315 | u32 decrypted_failed; |
| 316 | u32 decrypted_tgt; |
| 317 | } resync_nh; |
Boris Pismenny | ed9b764 | 2020-06-08 19:11:38 +0300 | [diff] [blame] | 318 | /* TLS_OFFLOAD_SYNC_TYPE_DRIVER_REQ_ASYNC */ |
| 319 | struct { |
| 320 | struct tls_offload_resync_async *resync_async; |
| 321 | }; |
Jakub Kicinski | f953d33b | 2019-06-10 21:40:02 -0700 | [diff] [blame] | 322 | }; |
Boris Pismenny | 4799ac8 | 2018-07-13 14:33:43 +0300 | [diff] [blame] | 323 | /* The TLS layer reserves room for driver specific state |
| 324 | * Currently the belief is that there is not enough |
| 325 | * driver specific state to justify another layer of indirection |
| 326 | */ |
Sabrina Dubroca | 9f0c824 | 2023-10-09 22:50:54 +0200 | [diff] [blame] | 327 | u8 driver_state[TLS_DRIVER_STATE_SIZE_RX] __aligned(8); |
Boris Pismenny | 4799ac8 | 2018-07-13 14:33:43 +0300 | [diff] [blame] | 328 | }; |
| 329 | |
Boris Pismenny | d80a1b9d | 2018-07-13 14:33:39 +0300 | [diff] [blame] | 330 | struct tls_record_info *tls_get_record(struct tls_offload_context_tx *context, |
Ilya Lesokhin | e8f6979 | 2018-04-30 10:16:16 +0300 | [diff] [blame] | 331 | u32 seq, u64 *p_record_sn); |
| 332 | |
| 333 | static inline bool tls_record_is_start_marker(struct tls_record_info *rec) |
| 334 | { |
| 335 | return rec->len == 0; |
| 336 | } |
| 337 | |
| 338 | static inline u32 tls_record_start_seq(struct tls_record_info *rec) |
| 339 | { |
| 340 | return rec->end_seq - rec->len; |
| 341 | } |
| 342 | |
Boris Pismenny | 4799ac8 | 2018-07-13 14:33:43 +0300 | [diff] [blame] | 343 | struct sk_buff * |
| 344 | tls_validate_xmit_skb(struct sock *sk, struct net_device *dev, |
| 345 | struct sk_buff *skb); |
Maxim Mikityanskiy | c55dcdd | 2021-06-01 15:08:00 +0300 | [diff] [blame] | 346 | struct sk_buff * |
| 347 | tls_validate_xmit_skb_sw(struct sock *sk, struct net_device *dev, |
| 348 | struct sk_buff *skb); |
Boris Pismenny | 4799ac8 | 2018-07-13 14:33:43 +0300 | [diff] [blame] | 349 | |
Jakub Kicinski | ed3c9a2 | 2023-06-13 13:50:06 -0700 | [diff] [blame] | 350 | static inline bool tls_is_skb_tx_device_offloaded(const struct sk_buff *skb) |
Ilya Lesokhin | e8f6979 | 2018-04-30 10:16:16 +0300 | [diff] [blame] | 351 | { |
Jakub Kicinski | ed3c9a2 | 2023-06-13 13:50:06 -0700 | [diff] [blame] | 352 | #ifdef CONFIG_TLS_DEVICE |
| 353 | struct sock *sk = skb->sk; |
| 354 | |
| 355 | return sk && sk_fullsock(sk) && |
Boris Pismenny | 4799ac8 | 2018-07-13 14:33:43 +0300 | [diff] [blame] | 356 | (smp_load_acquire(&sk->sk_validate_xmit_skb) == |
| 357 | &tls_validate_xmit_skb); |
| 358 | #else |
| 359 | return false; |
| 360 | #endif |
Ilya Lesokhin | e8f6979 | 2018-04-30 10:16:16 +0300 | [diff] [blame] | 361 | } |
| 362 | |
Vakul Garg | 4509de1 | 2019-02-14 07:11:35 +0000 | [diff] [blame] | 363 | static inline struct tls_context *tls_get_ctx(const struct sock *sk) |
| 364 | { |
Eric Dumazet | 58169ec | 2024-03-29 14:49:31 +0000 | [diff] [blame] | 365 | const struct inet_connection_sock *icsk = inet_csk(sk); |
Vakul Garg | 4509de1 | 2019-02-14 07:11:35 +0000 | [diff] [blame] | 366 | |
Jakub Kicinski | 15a7dea | 2019-08-30 12:25:47 +0200 | [diff] [blame] | 367 | /* Use RCU on icsk_ulp_data only for sock diag code, |
| 368 | * TLS data path doesn't need rcu_dereference(). |
| 369 | */ |
| 370 | return (__force void *)icsk->icsk_ulp_data; |
Vakul Garg | 4509de1 | 2019-02-14 07:11:35 +0000 | [diff] [blame] | 371 | } |
| 372 | |
Boris Pismenny | f66de3e | 2018-04-30 10:16:15 +0300 | [diff] [blame] | 373 | static inline struct tls_sw_context_rx *tls_sw_ctx_rx( |
Dave Watson | 3c4d755 | 2017-06-14 11:37:39 -0700 | [diff] [blame] | 374 | const struct tls_context *tls_ctx) |
| 375 | { |
Boris Pismenny | f66de3e | 2018-04-30 10:16:15 +0300 | [diff] [blame] | 376 | return (struct tls_sw_context_rx *)tls_ctx->priv_ctx_rx; |
| 377 | } |
| 378 | |
| 379 | static inline struct tls_sw_context_tx *tls_sw_ctx_tx( |
| 380 | const struct tls_context *tls_ctx) |
| 381 | { |
| 382 | return (struct tls_sw_context_tx *)tls_ctx->priv_ctx_tx; |
Dave Watson | 3c4d755 | 2017-06-14 11:37:39 -0700 | [diff] [blame] | 383 | } |
| 384 | |
Boris Pismenny | d80a1b9d | 2018-07-13 14:33:39 +0300 | [diff] [blame] | 385 | static inline struct tls_offload_context_tx * |
| 386 | tls_offload_ctx_tx(const struct tls_context *tls_ctx) |
Dave Watson | 3c4d755 | 2017-06-14 11:37:39 -0700 | [diff] [blame] | 387 | { |
Boris Pismenny | d80a1b9d | 2018-07-13 14:33:39 +0300 | [diff] [blame] | 388 | return (struct tls_offload_context_tx *)tls_ctx->priv_ctx_tx; |
Dave Watson | 3c4d755 | 2017-06-14 11:37:39 -0700 | [diff] [blame] | 389 | } |
| 390 | |
John Fastabend | 0608c69 | 2018-12-20 11:35:35 -0800 | [diff] [blame] | 391 | static inline bool tls_sw_has_ctx_tx(const struct sock *sk) |
| 392 | { |
| 393 | struct tls_context *ctx = tls_get_ctx(sk); |
| 394 | |
| 395 | if (!ctx) |
| 396 | return false; |
| 397 | return !!tls_sw_ctx_tx(ctx); |
| 398 | } |
| 399 | |
John Fastabend | e91de6a | 2020-05-29 16:06:59 -0700 | [diff] [blame] | 400 | static inline bool tls_sw_has_ctx_rx(const struct sock *sk) |
| 401 | { |
| 402 | struct tls_context *ctx = tls_get_ctx(sk); |
| 403 | |
| 404 | if (!ctx) |
| 405 | return false; |
| 406 | return !!tls_sw_ctx_rx(ctx); |
| 407 | } |
| 408 | |
Boris Pismenny | 4799ac8 | 2018-07-13 14:33:43 +0300 | [diff] [blame] | 409 | static inline struct tls_offload_context_rx * |
| 410 | tls_offload_ctx_rx(const struct tls_context *tls_ctx) |
| 411 | { |
| 412 | return (struct tls_offload_context_rx *)tls_ctx->priv_ctx_rx; |
| 413 | } |
| 414 | |
Jakub Kicinski | 2e36117 | 2019-06-05 14:11:39 -0700 | [diff] [blame] | 415 | static inline void *__tls_driver_ctx(struct tls_context *tls_ctx, |
| 416 | enum tls_offload_ctx_dir direction) |
| 417 | { |
| 418 | if (direction == TLS_OFFLOAD_CTX_DIR_TX) |
| 419 | return tls_offload_ctx_tx(tls_ctx)->driver_state; |
| 420 | else |
| 421 | return tls_offload_ctx_rx(tls_ctx)->driver_state; |
| 422 | } |
| 423 | |
| 424 | static inline void * |
| 425 | tls_driver_ctx(const struct sock *sk, enum tls_offload_ctx_dir direction) |
| 426 | { |
| 427 | return __tls_driver_ctx(tls_get_ctx(sk), direction); |
| 428 | } |
Jakub Kicinski | 2e36117 | 2019-06-05 14:11:39 -0700 | [diff] [blame] | 429 | |
Boris Pismenny | ed9b764 | 2020-06-08 19:11:38 +0300 | [diff] [blame] | 430 | #define RESYNC_REQ BIT(0) |
| 431 | #define RESYNC_REQ_ASYNC BIT(1) |
Boris Pismenny | 4799ac8 | 2018-07-13 14:33:43 +0300 | [diff] [blame] | 432 | /* The TLS context is valid until sk_destruct is called */ |
| 433 | static inline void tls_offload_rx_resync_request(struct sock *sk, __be32 seq) |
| 434 | { |
| 435 | struct tls_context *tls_ctx = tls_get_ctx(sk); |
| 436 | struct tls_offload_context_rx *rx_ctx = tls_offload_ctx_rx(tls_ctx); |
| 437 | |
Boris Pismenny | ed9b764 | 2020-06-08 19:11:38 +0300 | [diff] [blame] | 438 | atomic64_set(&rx_ctx->resync_req, ((u64)ntohl(seq) << 32) | RESYNC_REQ); |
| 439 | } |
| 440 | |
| 441 | /* Log all TLS record header TCP sequences in [seq, seq+len] */ |
| 442 | static inline void |
| 443 | tls_offload_rx_resync_async_request_start(struct sock *sk, __be32 seq, u16 len) |
| 444 | { |
| 445 | struct tls_context *tls_ctx = tls_get_ctx(sk); |
| 446 | struct tls_offload_context_rx *rx_ctx = tls_offload_ctx_rx(tls_ctx); |
| 447 | |
| 448 | atomic64_set(&rx_ctx->resync_async->req, ((u64)ntohl(seq) << 32) | |
Colin Ian King | a6ed3eb | 2020-06-30 15:27:46 +0100 | [diff] [blame] | 449 | ((u64)len << 16) | RESYNC_REQ | RESYNC_REQ_ASYNC); |
Boris Pismenny | ed9b764 | 2020-06-08 19:11:38 +0300 | [diff] [blame] | 450 | rx_ctx->resync_async->loglen = 0; |
Tariq Toukan | 138559b | 2020-11-15 15:14:48 +0200 | [diff] [blame] | 451 | rx_ctx->resync_async->rcd_delta = 0; |
Boris Pismenny | ed9b764 | 2020-06-08 19:11:38 +0300 | [diff] [blame] | 452 | } |
| 453 | |
| 454 | static inline void |
| 455 | tls_offload_rx_resync_async_request_end(struct sock *sk, __be32 seq) |
| 456 | { |
| 457 | struct tls_context *tls_ctx = tls_get_ctx(sk); |
| 458 | struct tls_offload_context_rx *rx_ctx = tls_offload_ctx_rx(tls_ctx); |
| 459 | |
| 460 | atomic64_set(&rx_ctx->resync_async->req, |
| 461 | ((u64)ntohl(seq) << 32) | RESYNC_REQ); |
Boris Pismenny | 4799ac8 | 2018-07-13 14:33:43 +0300 | [diff] [blame] | 462 | } |
| 463 | |
Jakub Kicinski | f953d33b | 2019-06-10 21:40:02 -0700 | [diff] [blame] | 464 | static inline void |
| 465 | tls_offload_rx_resync_set_type(struct sock *sk, enum tls_offload_sync_type type) |
| 466 | { |
| 467 | struct tls_context *tls_ctx = tls_get_ctx(sk); |
| 468 | |
| 469 | tls_offload_ctx_rx(tls_ctx)->resync_type = type; |
| 470 | } |
Boris Pismenny | 4799ac8 | 2018-07-13 14:33:43 +0300 | [diff] [blame] | 471 | |
Jakub Kicinski | 5018007 | 2019-06-10 21:40:09 -0700 | [diff] [blame] | 472 | /* Driver's seq tracking has to be disabled until resync succeeded */ |
| 473 | static inline bool tls_offload_tx_resync_pending(struct sock *sk) |
| 474 | { |
| 475 | struct tls_context *tls_ctx = tls_get_ctx(sk); |
| 476 | bool ret; |
| 477 | |
| 478 | ret = test_bit(TLS_TX_SYNC_SCHED, &tls_ctx->flags); |
| 479 | smp_mb__after_atomic(); |
| 480 | return ret; |
| 481 | } |
| 482 | |
Dirk van der Merwe | b9727d7 | 2019-06-05 14:11:40 -0700 | [diff] [blame] | 483 | struct sk_buff *tls_encrypt_skb(struct sk_buff *skb); |
Dave Watson | 3c4d755 | 2017-06-14 11:37:39 -0700 | [diff] [blame] | 484 | |
Jakub Kicinski | be2fbc15 | 2019-09-02 21:31:05 -0700 | [diff] [blame] | 485 | #ifdef CONFIG_TLS_DEVICE |
Jakub Kicinski | 8d5a49e | 2019-12-17 14:12:01 -0800 | [diff] [blame] | 486 | void tls_device_sk_destruct(struct sock *sk); |
Jakub Kicinski | 8538d29 | 2019-10-04 16:19:22 -0700 | [diff] [blame] | 487 | void tls_offload_tx_resync_request(struct sock *sk, u32 got_seq, u32 exp_seq); |
Jakub Kicinski | 8d5a49e | 2019-12-17 14:12:01 -0800 | [diff] [blame] | 488 | |
| 489 | static inline bool tls_is_sk_rx_device_offloaded(struct sock *sk) |
| 490 | { |
| 491 | if (!sk_fullsock(sk) || |
| 492 | smp_load_acquire(&sk->sk_destruct) != tls_device_sk_destruct) |
| 493 | return false; |
| 494 | return tls_get_ctx(sk)->rx_conf == TLS_HW; |
| 495 | } |
Jakub Kicinski | be2fbc15 | 2019-09-02 21:31:05 -0700 | [diff] [blame] | 496 | #endif |
Dave Watson | 3c4d755 | 2017-06-14 11:37:39 -0700 | [diff] [blame] | 497 | #endif /* _TLS_OFFLOAD_H */ |