Antoine Tenart | 301422e | 2018-07-13 16:51:37 +0200 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0 |
Antoine Ténart | 1b44c5a | 2017-05-24 16:10:34 +0200 | [diff] [blame] | 2 | /* |
| 3 | * Copyright (C) 2017 Marvell |
| 4 | * |
| 5 | * Antoine Tenart <antoine.tenart@free-electrons.com> |
Antoine Ténart | 1b44c5a | 2017-05-24 16:10:34 +0200 | [diff] [blame] | 6 | */ |
| 7 | |
| 8 | #include <linux/dma-mapping.h> |
| 9 | #include <linux/spinlock.h> |
| 10 | |
| 11 | #include "safexcel.h" |
| 12 | |
| 13 | int safexcel_init_ring_descriptors(struct safexcel_crypto_priv *priv, |
Ofer Heifetz | 18e0e95 | 2018-06-28 17:15:36 +0200 | [diff] [blame] | 14 | struct safexcel_desc_ring *cdr, |
| 15 | struct safexcel_desc_ring *rdr) |
Antoine Ténart | 1b44c5a | 2017-05-24 16:10:34 +0200 | [diff] [blame] | 16 | { |
Pascal van Leeuwen | 098e51e | 2019-12-11 17:32:35 +0100 | [diff] [blame] | 17 | int i; |
| 18 | struct safexcel_command_desc *cdesc; |
| 19 | dma_addr_t atok; |
| 20 | |
| 21 | /* Actual command descriptor ring */ |
Pascal van Leeuwen | 84ca4e5 | 2019-09-18 08:42:39 +0200 | [diff] [blame] | 22 | cdr->offset = priv->config.cd_offset; |
Antoine Ténart | 1b44c5a | 2017-05-24 16:10:34 +0200 | [diff] [blame] | 23 | cdr->base = dmam_alloc_coherent(priv->dev, |
| 24 | cdr->offset * EIP197_DEFAULT_RING_SIZE, |
| 25 | &cdr->base_dma, GFP_KERNEL); |
| 26 | if (!cdr->base) |
| 27 | return -ENOMEM; |
| 28 | cdr->write = cdr->base; |
Ofer Heifetz | 9744fec | 2018-06-28 17:21:57 +0200 | [diff] [blame] | 29 | cdr->base_end = cdr->base + cdr->offset * (EIP197_DEFAULT_RING_SIZE - 1); |
Antoine Ténart | 1b44c5a | 2017-05-24 16:10:34 +0200 | [diff] [blame] | 30 | cdr->read = cdr->base; |
| 31 | |
Pascal van Leeuwen | 098e51e | 2019-12-11 17:32:35 +0100 | [diff] [blame] | 32 | /* Command descriptor shadow ring for storing additional token data */ |
| 33 | cdr->shoffset = priv->config.cdsh_offset; |
| 34 | cdr->shbase = dmam_alloc_coherent(priv->dev, |
| 35 | cdr->shoffset * |
| 36 | EIP197_DEFAULT_RING_SIZE, |
| 37 | &cdr->shbase_dma, GFP_KERNEL); |
| 38 | if (!cdr->shbase) |
| 39 | return -ENOMEM; |
| 40 | cdr->shwrite = cdr->shbase; |
| 41 | cdr->shbase_end = cdr->shbase + cdr->shoffset * |
| 42 | (EIP197_DEFAULT_RING_SIZE - 1); |
| 43 | |
| 44 | /* |
| 45 | * Populate command descriptors with physical pointers to shadow descs. |
| 46 | * Note that we only need to do this once if we don't overwrite them. |
| 47 | */ |
| 48 | cdesc = cdr->base; |
| 49 | atok = cdr->shbase_dma; |
| 50 | for (i = 0; i < EIP197_DEFAULT_RING_SIZE; i++) { |
| 51 | cdesc->atok_lo = lower_32_bits(atok); |
| 52 | cdesc->atok_hi = upper_32_bits(atok); |
| 53 | cdesc = (void *)cdesc + cdr->offset; |
| 54 | atok += cdr->shoffset; |
| 55 | } |
| 56 | |
Pascal van Leeuwen | 84ca4e5 | 2019-09-18 08:42:39 +0200 | [diff] [blame] | 57 | rdr->offset = priv->config.rd_offset; |
Pascal van Leeuwen | 098e51e | 2019-12-11 17:32:35 +0100 | [diff] [blame] | 58 | /* Use shoffset for result token offset here */ |
| 59 | rdr->shoffset = priv->config.res_offset; |
Antoine Ténart | 1b44c5a | 2017-05-24 16:10:34 +0200 | [diff] [blame] | 60 | rdr->base = dmam_alloc_coherent(priv->dev, |
| 61 | rdr->offset * EIP197_DEFAULT_RING_SIZE, |
| 62 | &rdr->base_dma, GFP_KERNEL); |
| 63 | if (!rdr->base) |
| 64 | return -ENOMEM; |
| 65 | rdr->write = rdr->base; |
Ofer Heifetz | 9744fec | 2018-06-28 17:21:57 +0200 | [diff] [blame] | 66 | rdr->base_end = rdr->base + rdr->offset * (EIP197_DEFAULT_RING_SIZE - 1); |
Antoine Ténart | 1b44c5a | 2017-05-24 16:10:34 +0200 | [diff] [blame] | 67 | rdr->read = rdr->base; |
| 68 | |
| 69 | return 0; |
| 70 | } |
| 71 | |
| 72 | inline int safexcel_select_ring(struct safexcel_crypto_priv *priv) |
| 73 | { |
| 74 | return (atomic_inc_return(&priv->ring_used) % priv->config.rings); |
| 75 | } |
| 76 | |
Pascal van Leeuwen | 098e51e | 2019-12-11 17:32:35 +0100 | [diff] [blame] | 77 | static void *safexcel_ring_next_cwptr(struct safexcel_crypto_priv *priv, |
| 78 | struct safexcel_desc_ring *ring, |
| 79 | bool first, |
| 80 | struct safexcel_token **atoken) |
Antoine Ténart | 1b44c5a | 2017-05-24 16:10:34 +0200 | [diff] [blame] | 81 | { |
| 82 | void *ptr = ring->write; |
| 83 | |
Pascal van Leeuwen | 098e51e | 2019-12-11 17:32:35 +0100 | [diff] [blame] | 84 | if (first) |
| 85 | *atoken = ring->shwrite; |
| 86 | |
| 87 | if ((ring->write == ring->read - ring->offset) || |
| 88 | (ring->read == ring->base && ring->write == ring->base_end)) |
| 89 | return ERR_PTR(-ENOMEM); |
| 90 | |
| 91 | if (ring->write == ring->base_end) { |
| 92 | ring->write = ring->base; |
| 93 | ring->shwrite = ring->shbase; |
| 94 | } else { |
| 95 | ring->write += ring->offset; |
| 96 | ring->shwrite += ring->shoffset; |
| 97 | } |
| 98 | |
| 99 | return ptr; |
| 100 | } |
| 101 | |
| 102 | static void *safexcel_ring_next_rwptr(struct safexcel_crypto_priv *priv, |
| 103 | struct safexcel_desc_ring *ring, |
| 104 | struct result_data_desc **rtoken) |
| 105 | { |
| 106 | void *ptr = ring->write; |
| 107 | |
| 108 | /* Result token at relative offset shoffset */ |
| 109 | *rtoken = ring->write + ring->shoffset; |
| 110 | |
Ofer Heifetz | 9744fec | 2018-06-28 17:21:57 +0200 | [diff] [blame] | 111 | if ((ring->write == ring->read - ring->offset) || |
| 112 | (ring->read == ring->base && ring->write == ring->base_end)) |
Antoine Ténart | 1b44c5a | 2017-05-24 16:10:34 +0200 | [diff] [blame] | 113 | return ERR_PTR(-ENOMEM); |
| 114 | |
Antoine Ténart | 1b44c5a | 2017-05-24 16:10:34 +0200 | [diff] [blame] | 115 | if (ring->write == ring->base_end) |
| 116 | ring->write = ring->base; |
Ofer Heifetz | 9744fec | 2018-06-28 17:21:57 +0200 | [diff] [blame] | 117 | else |
| 118 | ring->write += ring->offset; |
Antoine Ténart | 1b44c5a | 2017-05-24 16:10:34 +0200 | [diff] [blame] | 119 | |
Antoine Ténart | 1b44c5a | 2017-05-24 16:10:34 +0200 | [diff] [blame] | 120 | return ptr; |
| 121 | } |
| 122 | |
| 123 | void *safexcel_ring_next_rptr(struct safexcel_crypto_priv *priv, |
Ofer Heifetz | 18e0e95 | 2018-06-28 17:15:36 +0200 | [diff] [blame] | 124 | struct safexcel_desc_ring *ring) |
Antoine Ténart | 1b44c5a | 2017-05-24 16:10:34 +0200 | [diff] [blame] | 125 | { |
| 126 | void *ptr = ring->read; |
| 127 | |
Ofer Heifetz | 9744fec | 2018-06-28 17:21:57 +0200 | [diff] [blame] | 128 | if (ring->write == ring->read) |
Antoine Ténart | 1b44c5a | 2017-05-24 16:10:34 +0200 | [diff] [blame] | 129 | return ERR_PTR(-ENOENT); |
| 130 | |
Antoine Ténart | 1b44c5a | 2017-05-24 16:10:34 +0200 | [diff] [blame] | 131 | if (ring->read == ring->base_end) |
| 132 | ring->read = ring->base; |
Ofer Heifetz | 9744fec | 2018-06-28 17:21:57 +0200 | [diff] [blame] | 133 | else |
| 134 | ring->read += ring->offset; |
Antoine Ténart | 1b44c5a | 2017-05-24 16:10:34 +0200 | [diff] [blame] | 135 | |
Antoine Ténart | 1b44c5a | 2017-05-24 16:10:34 +0200 | [diff] [blame] | 136 | return ptr; |
| 137 | } |
| 138 | |
Ofer Heifetz | 9744fec | 2018-06-28 17:21:57 +0200 | [diff] [blame] | 139 | inline void *safexcel_ring_curr_rptr(struct safexcel_crypto_priv *priv, |
| 140 | int ring) |
| 141 | { |
| 142 | struct safexcel_desc_ring *rdr = &priv->ring[ring].rdr; |
| 143 | |
| 144 | return rdr->read; |
| 145 | } |
| 146 | |
| 147 | inline int safexcel_ring_first_rdr_index(struct safexcel_crypto_priv *priv, |
| 148 | int ring) |
| 149 | { |
| 150 | struct safexcel_desc_ring *rdr = &priv->ring[ring].rdr; |
| 151 | |
| 152 | return (rdr->read - rdr->base) / rdr->offset; |
| 153 | } |
| 154 | |
| 155 | inline int safexcel_ring_rdr_rdesc_index(struct safexcel_crypto_priv *priv, |
| 156 | int ring, |
| 157 | struct safexcel_result_desc *rdesc) |
| 158 | { |
| 159 | struct safexcel_desc_ring *rdr = &priv->ring[ring].rdr; |
| 160 | |
| 161 | return ((void *)rdesc - rdr->base) / rdr->offset; |
| 162 | } |
| 163 | |
Antoine Ténart | 1b44c5a | 2017-05-24 16:10:34 +0200 | [diff] [blame] | 164 | void safexcel_ring_rollback_wptr(struct safexcel_crypto_priv *priv, |
Ofer Heifetz | 18e0e95 | 2018-06-28 17:15:36 +0200 | [diff] [blame] | 165 | struct safexcel_desc_ring *ring) |
Antoine Ténart | 1b44c5a | 2017-05-24 16:10:34 +0200 | [diff] [blame] | 166 | { |
Ofer Heifetz | 9744fec | 2018-06-28 17:21:57 +0200 | [diff] [blame] | 167 | if (ring->write == ring->read) |
Antoine Ténart | 1b44c5a | 2017-05-24 16:10:34 +0200 | [diff] [blame] | 168 | return; |
| 169 | |
Pascal van Leeuwen | 098e51e | 2019-12-11 17:32:35 +0100 | [diff] [blame] | 170 | if (ring->write == ring->base) { |
Ofer Heifetz | 9744fec | 2018-06-28 17:21:57 +0200 | [diff] [blame] | 171 | ring->write = ring->base_end; |
Pascal van Leeuwen | 098e51e | 2019-12-11 17:32:35 +0100 | [diff] [blame] | 172 | ring->shwrite = ring->shbase_end; |
| 173 | } else { |
Antoine Ténart | 1b44c5a | 2017-05-24 16:10:34 +0200 | [diff] [blame] | 174 | ring->write -= ring->offset; |
Pascal van Leeuwen | 098e51e | 2019-12-11 17:32:35 +0100 | [diff] [blame] | 175 | ring->shwrite -= ring->shoffset; |
| 176 | } |
Antoine Ténart | 1b44c5a | 2017-05-24 16:10:34 +0200 | [diff] [blame] | 177 | } |
| 178 | |
| 179 | struct safexcel_command_desc *safexcel_add_cdesc(struct safexcel_crypto_priv *priv, |
| 180 | int ring_id, |
| 181 | bool first, bool last, |
| 182 | dma_addr_t data, u32 data_len, |
| 183 | u32 full_data_len, |
Pascal van Leeuwen | 098e51e | 2019-12-11 17:32:35 +0100 | [diff] [blame] | 184 | dma_addr_t context, |
| 185 | struct safexcel_token **atoken) |
| 186 | { |
Antoine Ténart | 1b44c5a | 2017-05-24 16:10:34 +0200 | [diff] [blame] | 187 | struct safexcel_command_desc *cdesc; |
Antoine Ténart | 1b44c5a | 2017-05-24 16:10:34 +0200 | [diff] [blame] | 188 | |
Pascal van Leeuwen | 098e51e | 2019-12-11 17:32:35 +0100 | [diff] [blame] | 189 | cdesc = safexcel_ring_next_cwptr(priv, &priv->ring[ring_id].cdr, |
| 190 | first, atoken); |
Antoine Ténart | 1b44c5a | 2017-05-24 16:10:34 +0200 | [diff] [blame] | 191 | if (IS_ERR(cdesc)) |
| 192 | return cdesc; |
| 193 | |
Antoine Ténart | 1b44c5a | 2017-05-24 16:10:34 +0200 | [diff] [blame] | 194 | cdesc->particle_size = data_len; |
Pascal van Leeuwen | 098e51e | 2019-12-11 17:32:35 +0100 | [diff] [blame] | 195 | cdesc->rsvd0 = 0; |
| 196 | cdesc->last_seg = last; |
| 197 | cdesc->first_seg = first; |
| 198 | cdesc->additional_cdata_size = 0; |
| 199 | cdesc->rsvd1 = 0; |
Antoine Ténart | 1b44c5a | 2017-05-24 16:10:34 +0200 | [diff] [blame] | 200 | cdesc->data_lo = lower_32_bits(data); |
| 201 | cdesc->data_hi = upper_32_bits(data); |
| 202 | |
Pascal van Leeuwen | 098e51e | 2019-12-11 17:32:35 +0100 | [diff] [blame] | 203 | if (first) { |
Pascal van Leeuwen | 3e45088 | 2019-08-30 09:52:30 +0200 | [diff] [blame] | 204 | /* |
| 205 | * Note that the length here MUST be >0 or else the EIP(1)97 |
| 206 | * may hang. Newer EIP197 firmware actually incorporates this |
| 207 | * fix already, but that doesn't help the EIP97 and we may |
| 208 | * also be running older firmware. |
| 209 | */ |
| 210 | cdesc->control_data.packet_length = full_data_len ?: 1; |
Antoine Ténart | 1b44c5a | 2017-05-24 16:10:34 +0200 | [diff] [blame] | 211 | cdesc->control_data.options = EIP197_OPTION_MAGIC_VALUE | |
| 212 | EIP197_OPTION_64BIT_CTX | |
Pascal van Leeuwen | 098e51e | 2019-12-11 17:32:35 +0100 | [diff] [blame] | 213 | EIP197_OPTION_CTX_CTRL_IN_CMD | |
| 214 | EIP197_OPTION_RC_AUTO; |
| 215 | cdesc->control_data.type = EIP197_TYPE_BCLA; |
| 216 | cdesc->control_data.context_lo = lower_32_bits(context) | |
| 217 | EIP197_CONTEXT_SMALL; |
Antoine Ténart | 1b44c5a | 2017-05-24 16:10:34 +0200 | [diff] [blame] | 218 | cdesc->control_data.context_hi = upper_32_bits(context); |
Antoine Ténart | 1b44c5a | 2017-05-24 16:10:34 +0200 | [diff] [blame] | 219 | } |
| 220 | |
| 221 | return cdesc; |
| 222 | } |
| 223 | |
| 224 | struct safexcel_result_desc *safexcel_add_rdesc(struct safexcel_crypto_priv *priv, |
| 225 | int ring_id, |
| 226 | bool first, bool last, |
| 227 | dma_addr_t data, u32 len) |
| 228 | { |
| 229 | struct safexcel_result_desc *rdesc; |
Pascal van Leeuwen | 098e51e | 2019-12-11 17:32:35 +0100 | [diff] [blame] | 230 | struct result_data_desc *rtoken; |
Antoine Ténart | 1b44c5a | 2017-05-24 16:10:34 +0200 | [diff] [blame] | 231 | |
Pascal van Leeuwen | 098e51e | 2019-12-11 17:32:35 +0100 | [diff] [blame] | 232 | rdesc = safexcel_ring_next_rwptr(priv, &priv->ring[ring_id].rdr, |
| 233 | &rtoken); |
Antoine Ténart | 1b44c5a | 2017-05-24 16:10:34 +0200 | [diff] [blame] | 234 | if (IS_ERR(rdesc)) |
| 235 | return rdesc; |
| 236 | |
Antoine Ténart | 1b44c5a | 2017-05-24 16:10:34 +0200 | [diff] [blame] | 237 | rdesc->particle_size = len; |
Pascal van Leeuwen | 098e51e | 2019-12-11 17:32:35 +0100 | [diff] [blame] | 238 | rdesc->rsvd0 = 0; |
Pascal van Leeuwen | bd03b02 | 2020-09-08 08:10:45 +0200 | [diff] [blame] | 239 | rdesc->descriptor_overflow = 1; /* assume error */ |
| 240 | rdesc->buffer_overflow = 1; /* assume error */ |
Pascal van Leeuwen | 098e51e | 2019-12-11 17:32:35 +0100 | [diff] [blame] | 241 | rdesc->last_seg = last; |
| 242 | rdesc->first_seg = first; |
| 243 | rdesc->result_size = EIP197_RD64_RESULT_SIZE; |
| 244 | rdesc->rsvd1 = 0; |
Antoine Ténart | 1b44c5a | 2017-05-24 16:10:34 +0200 | [diff] [blame] | 245 | rdesc->data_lo = lower_32_bits(data); |
| 246 | rdesc->data_hi = upper_32_bits(data); |
| 247 | |
Pascal van Leeuwen | bd03b02 | 2020-09-08 08:10:45 +0200 | [diff] [blame] | 248 | /* Clear length in result token */ |
Pascal van Leeuwen | 098e51e | 2019-12-11 17:32:35 +0100 | [diff] [blame] | 249 | rtoken->packet_length = 0; |
Pascal van Leeuwen | bd03b02 | 2020-09-08 08:10:45 +0200 | [diff] [blame] | 250 | /* Assume errors - HW will clear if not the case */ |
| 251 | rtoken->error_code = 0x7fff; |
Pascal van Leeuwen | 098e51e | 2019-12-11 17:32:35 +0100 | [diff] [blame] | 252 | |
Antoine Ténart | 1b44c5a | 2017-05-24 16:10:34 +0200 | [diff] [blame] | 253 | return rdesc; |
| 254 | } |