blob: 921039e56f87f9360156074233c9e38828bc1612 [file] [log] [blame]
Kim Phillips9c4a7962008-06-23 19:50:15 +08001/*
2 * talitos - Freescale Integrated Security Engine (SEC) device driver
3 *
Kim Phillips5228f0f2011-07-15 11:21:38 +08004 * Copyright (c) 2008-2011 Freescale Semiconductor, Inc.
Kim Phillips9c4a7962008-06-23 19:50:15 +08005 *
6 * Scatterlist Crypto API glue code copied from files with the following:
7 * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
8 *
9 * Crypto algorithm registration code copied from hifn driver:
10 * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
11 * All rights reserved.
12 *
13 * This program is free software; you can redistribute it and/or modify
14 * it under the terms of the GNU General Public License as published by
15 * the Free Software Foundation; either version 2 of the License, or
16 * (at your option) any later version.
17 *
18 * This program is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
22 *
23 * You should have received a copy of the GNU General Public License
24 * along with this program; if not, write to the Free Software
25 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
26 */
27
28#include <linux/kernel.h>
29#include <linux/module.h>
30#include <linux/mod_devicetable.h>
31#include <linux/device.h>
32#include <linux/interrupt.h>
33#include <linux/crypto.h>
34#include <linux/hw_random.h>
35#include <linux/of_platform.h>
36#include <linux/dma-mapping.h>
37#include <linux/io.h>
38#include <linux/spinlock.h>
39#include <linux/rtnetlink.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090040#include <linux/slab.h>
Kim Phillips9c4a7962008-06-23 19:50:15 +080041
42#include <crypto/algapi.h>
43#include <crypto/aes.h>
Lee Nipper3952f172008-07-10 18:29:18 +080044#include <crypto/des.h>
Kim Phillips9c4a7962008-06-23 19:50:15 +080045#include <crypto/sha.h>
Lee Nipper497f2e62010-05-19 19:20:36 +100046#include <crypto/md5.h>
Kim Phillips9c4a7962008-06-23 19:50:15 +080047#include <crypto/aead.h>
48#include <crypto/authenc.h>
Lee Nipper4de9d0b2009-03-29 15:52:32 +080049#include <crypto/skcipher.h>
Lee Nipperacbf7c622010-05-19 19:19:33 +100050#include <crypto/hash.h>
51#include <crypto/internal/hash.h>
Lee Nipper4de9d0b2009-03-29 15:52:32 +080052#include <crypto/scatterwalk.h>
Kim Phillips9c4a7962008-06-23 19:50:15 +080053
54#include "talitos.h"
55
56#define TALITOS_TIMEOUT 100000
57#define TALITOS_MAX_DATA_LEN 65535
58
59#define DESC_TYPE(desc_hdr) ((be32_to_cpu(desc_hdr) >> 3) & 0x1f)
60#define PRIMARY_EU(desc_hdr) ((be32_to_cpu(desc_hdr) >> 28) & 0xf)
61#define SECONDARY_EU(desc_hdr) ((be32_to_cpu(desc_hdr) >> 16) & 0xf)
62
63/* descriptor pointer entry */
64struct talitos_ptr {
65 __be16 len; /* length */
66 u8 j_extent; /* jump to sg link table and/or extent */
67 u8 eptr; /* extended address */
68 __be32 ptr; /* address */
69};
70
Lee Nipper497f2e62010-05-19 19:20:36 +100071static const struct talitos_ptr zero_entry = {
72 .len = 0,
73 .j_extent = 0,
74 .eptr = 0,
75 .ptr = 0
76};
77
Kim Phillips9c4a7962008-06-23 19:50:15 +080078/* descriptor */
79struct talitos_desc {
80 __be32 hdr; /* header high bits */
81 __be32 hdr_lo; /* header low bits */
82 struct talitos_ptr ptr[7]; /* ptr/len pair array */
83};
84
85/**
86 * talitos_request - descriptor submission request
87 * @desc: descriptor pointer (kernel virtual)
88 * @dma_desc: descriptor's physical bus address
89 * @callback: whom to call when descriptor processing is done
90 * @context: caller context (optional)
91 */
92struct talitos_request {
93 struct talitos_desc *desc;
94 dma_addr_t dma_desc;
95 void (*callback) (struct device *dev, struct talitos_desc *desc,
96 void *context, int error);
97 void *context;
98};
99
Kim Phillips4b9926282009-08-13 11:50:38 +1000100/* per-channel fifo management */
101struct talitos_channel {
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800102 void __iomem *reg;
103
Kim Phillips4b9926282009-08-13 11:50:38 +1000104 /* request fifo */
105 struct talitos_request *fifo;
106
107 /* number of requests pending in channel h/w fifo */
108 atomic_t submit_count ____cacheline_aligned;
109
110 /* request submission (head) lock */
111 spinlock_t head_lock ____cacheline_aligned;
112 /* index to next free descriptor request */
113 int head;
114
115 /* request release (tail) lock */
116 spinlock_t tail_lock ____cacheline_aligned;
117 /* index to next in-progress/done descriptor request */
118 int tail;
119};
120
Kim Phillips9c4a7962008-06-23 19:50:15 +0800121struct talitos_private {
122 struct device *dev;
Grant Likely2dc11582010-08-06 09:25:50 -0600123 struct platform_device *ofdev;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800124 void __iomem *reg;
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800125 int irq[2];
Kim Phillips9c4a7962008-06-23 19:50:15 +0800126
Horia Geanta511d63c2012-03-30 17:49:53 +0300127 /* SEC global registers lock */
128 spinlock_t reg_lock ____cacheline_aligned;
129
Kim Phillips9c4a7962008-06-23 19:50:15 +0800130 /* SEC version geometry (from device tree node) */
131 unsigned int num_channels;
132 unsigned int chfifo_len;
133 unsigned int exec_units;
134 unsigned int desc_types;
135
Lee Nipperf3c85bc2008-07-30 16:26:57 +0800136 /* SEC Compatibility info */
137 unsigned long features;
138
Kim Phillips9c4a7962008-06-23 19:50:15 +0800139 /*
140 * length of the request fifo
141 * fifo_len is chfifo_len rounded up to next power of 2
142 * so we can use bitwise ops to wrap
143 */
144 unsigned int fifo_len;
145
Kim Phillips4b9926282009-08-13 11:50:38 +1000146 struct talitos_channel *chan;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800147
Kim Phillips4b9926282009-08-13 11:50:38 +1000148 /* next channel to be assigned next incoming descriptor */
149 atomic_t last_chan ____cacheline_aligned;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800150
151 /* request callback tasklet */
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800152 struct tasklet_struct done_task[2];
Kim Phillips9c4a7962008-06-23 19:50:15 +0800153
154 /* list of registered algorithms */
155 struct list_head alg_list;
156
157 /* hwrng device */
158 struct hwrng rng;
159};
160
Lee Nipperf3c85bc2008-07-30 16:26:57 +0800161/* .features flag */
162#define TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT 0x00000001
Kim Phillipsfe5720e2008-10-12 20:33:14 +0800163#define TALITOS_FTR_HW_AUTH_CHECK 0x00000002
Kim Phillips60f208d72010-05-19 19:21:53 +1000164#define TALITOS_FTR_SHA224_HWINIT 0x00000004
Lee Nipper79b3a412011-11-21 16:13:25 +0800165#define TALITOS_FTR_HMAC_OK 0x00000008
Lee Nipperf3c85bc2008-07-30 16:26:57 +0800166
Kim Phillips81eb0242009-08-13 11:51:51 +1000167static void to_talitos_ptr(struct talitos_ptr *talitos_ptr, dma_addr_t dma_addr)
168{
169 talitos_ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr));
Kim Phillipsa7524472010-09-23 15:56:38 +0800170 talitos_ptr->eptr = upper_32_bits(dma_addr);
Kim Phillips81eb0242009-08-13 11:51:51 +1000171}
172
Kim Phillips9c4a7962008-06-23 19:50:15 +0800173/*
174 * map virtual single (contiguous) pointer to h/w descriptor pointer
175 */
176static void map_single_talitos_ptr(struct device *dev,
177 struct talitos_ptr *talitos_ptr,
178 unsigned short len, void *data,
179 unsigned char extent,
180 enum dma_data_direction dir)
181{
Kim Phillips81eb0242009-08-13 11:51:51 +1000182 dma_addr_t dma_addr = dma_map_single(dev, data, len, dir);
183
Kim Phillips9c4a7962008-06-23 19:50:15 +0800184 talitos_ptr->len = cpu_to_be16(len);
Kim Phillips81eb0242009-08-13 11:51:51 +1000185 to_talitos_ptr(talitos_ptr, dma_addr);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800186 talitos_ptr->j_extent = extent;
187}
188
189/*
190 * unmap bus single (contiguous) h/w descriptor pointer
191 */
192static void unmap_single_talitos_ptr(struct device *dev,
193 struct talitos_ptr *talitos_ptr,
194 enum dma_data_direction dir)
195{
196 dma_unmap_single(dev, be32_to_cpu(talitos_ptr->ptr),
197 be16_to_cpu(talitos_ptr->len), dir);
198}
199
200static int reset_channel(struct device *dev, int ch)
201{
202 struct talitos_private *priv = dev_get_drvdata(dev);
203 unsigned int timeout = TALITOS_TIMEOUT;
204
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800205 setbits32(priv->chan[ch].reg + TALITOS_CCCR, TALITOS_CCCR_RESET);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800206
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800207 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) & TALITOS_CCCR_RESET)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800208 && --timeout)
209 cpu_relax();
210
211 if (timeout == 0) {
212 dev_err(dev, "failed to reset channel %d\n", ch);
213 return -EIO;
214 }
215
Kim Phillips81eb0242009-08-13 11:51:51 +1000216 /* set 36-bit addressing, done writeback enable and done IRQ enable */
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800217 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE |
Kim Phillips81eb0242009-08-13 11:51:51 +1000218 TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800219
Kim Phillipsfe5720e2008-10-12 20:33:14 +0800220 /* and ICCR writeback, if available */
221 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800222 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
Kim Phillipsfe5720e2008-10-12 20:33:14 +0800223 TALITOS_CCCR_LO_IWSE);
224
Kim Phillips9c4a7962008-06-23 19:50:15 +0800225 return 0;
226}
227
228static int reset_device(struct device *dev)
229{
230 struct talitos_private *priv = dev_get_drvdata(dev);
231 unsigned int timeout = TALITOS_TIMEOUT;
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800232 u32 mcr = TALITOS_MCR_SWR;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800233
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800234 setbits32(priv->reg + TALITOS_MCR, mcr);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800235
236 while ((in_be32(priv->reg + TALITOS_MCR) & TALITOS_MCR_SWR)
237 && --timeout)
238 cpu_relax();
239
Kim Phillips2cdba3c2011-12-12 14:59:11 -0600240 if (priv->irq[1]) {
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800241 mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3;
242 setbits32(priv->reg + TALITOS_MCR, mcr);
243 }
244
Kim Phillips9c4a7962008-06-23 19:50:15 +0800245 if (timeout == 0) {
246 dev_err(dev, "failed to reset device\n");
247 return -EIO;
248 }
249
250 return 0;
251}
252
253/*
254 * Reset and initialize the device
255 */
256static int init_device(struct device *dev)
257{
258 struct talitos_private *priv = dev_get_drvdata(dev);
259 int ch, err;
260
261 /*
262 * Master reset
263 * errata documentation: warning: certain SEC interrupts
264 * are not fully cleared by writing the MCR:SWR bit,
265 * set bit twice to completely reset
266 */
267 err = reset_device(dev);
268 if (err)
269 return err;
270
271 err = reset_device(dev);
272 if (err)
273 return err;
274
275 /* reset channels */
276 for (ch = 0; ch < priv->num_channels; ch++) {
277 err = reset_channel(dev, ch);
278 if (err)
279 return err;
280 }
281
282 /* enable channel done and error interrupts */
283 setbits32(priv->reg + TALITOS_IMR, TALITOS_IMR_INIT);
284 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS_IMR_LO_INIT);
285
Kim Phillipsfe5720e2008-10-12 20:33:14 +0800286 /* disable integrity check error interrupts (use writeback instead) */
287 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
288 setbits32(priv->reg + TALITOS_MDEUICR_LO,
289 TALITOS_MDEUICR_LO_ICE);
290
Kim Phillips9c4a7962008-06-23 19:50:15 +0800291 return 0;
292}
293
294/**
295 * talitos_submit - submits a descriptor to the device for processing
296 * @dev: the SEC device to be used
Kim Phillips5228f0f2011-07-15 11:21:38 +0800297 * @ch: the SEC device channel to be used
Kim Phillips9c4a7962008-06-23 19:50:15 +0800298 * @desc: the descriptor to be processed by the device
299 * @callback: whom to call when processing is complete
300 * @context: a handle for use by caller (optional)
301 *
302 * desc must contain valid dma-mapped (bus physical) address pointers.
303 * callback must check err and feedback in descriptor header
304 * for device processing status.
305 */
Kim Phillips5228f0f2011-07-15 11:21:38 +0800306static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
Kim Phillips9c4a7962008-06-23 19:50:15 +0800307 void (*callback)(struct device *dev,
308 struct talitos_desc *desc,
309 void *context, int error),
310 void *context)
311{
312 struct talitos_private *priv = dev_get_drvdata(dev);
313 struct talitos_request *request;
Kim Phillips5228f0f2011-07-15 11:21:38 +0800314 unsigned long flags;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800315 int head;
316
Kim Phillips4b9926282009-08-13 11:50:38 +1000317 spin_lock_irqsave(&priv->chan[ch].head_lock, flags);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800318
Kim Phillips4b9926282009-08-13 11:50:38 +1000319 if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) {
Kim Phillipsec6644d2008-07-17 20:16:40 +0800320 /* h/w fifo is full */
Kim Phillips4b9926282009-08-13 11:50:38 +1000321 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800322 return -EAGAIN;
323 }
324
Kim Phillips4b9926282009-08-13 11:50:38 +1000325 head = priv->chan[ch].head;
326 request = &priv->chan[ch].fifo[head];
Kim Phillipsec6644d2008-07-17 20:16:40 +0800327
Kim Phillips9c4a7962008-06-23 19:50:15 +0800328 /* map descriptor and save caller data */
329 request->dma_desc = dma_map_single(dev, desc, sizeof(*desc),
330 DMA_BIDIRECTIONAL);
331 request->callback = callback;
332 request->context = context;
333
334 /* increment fifo head */
Kim Phillips4b9926282009-08-13 11:50:38 +1000335 priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800336
337 smp_wmb();
338 request->desc = desc;
339
340 /* GO! */
341 wmb();
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800342 out_be32(priv->chan[ch].reg + TALITOS_FF,
343 upper_32_bits(request->dma_desc));
344 out_be32(priv->chan[ch].reg + TALITOS_FF_LO,
Kim Phillipsa7524472010-09-23 15:56:38 +0800345 lower_32_bits(request->dma_desc));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800346
Kim Phillips4b9926282009-08-13 11:50:38 +1000347 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800348
349 return -EINPROGRESS;
350}
351
352/*
353 * process what was done, notify callback of error if not
354 */
355static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
356{
357 struct talitos_private *priv = dev_get_drvdata(dev);
358 struct talitos_request *request, saved_req;
359 unsigned long flags;
360 int tail, status;
361
Kim Phillips4b9926282009-08-13 11:50:38 +1000362 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800363
Kim Phillips4b9926282009-08-13 11:50:38 +1000364 tail = priv->chan[ch].tail;
365 while (priv->chan[ch].fifo[tail].desc) {
366 request = &priv->chan[ch].fifo[tail];
Kim Phillips9c4a7962008-06-23 19:50:15 +0800367
368 /* descriptors with their done bits set don't get the error */
369 rmb();
Lee Nipperca38a812008-12-20 17:09:25 +1100370 if ((request->desc->hdr & DESC_HDR_DONE) == DESC_HDR_DONE)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800371 status = 0;
Lee Nipperca38a812008-12-20 17:09:25 +1100372 else
Kim Phillips9c4a7962008-06-23 19:50:15 +0800373 if (!error)
374 break;
375 else
376 status = error;
377
378 dma_unmap_single(dev, request->dma_desc,
Kim Phillipse938e462009-03-29 15:53:23 +0800379 sizeof(struct talitos_desc),
380 DMA_BIDIRECTIONAL);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800381
382 /* copy entries so we can call callback outside lock */
383 saved_req.desc = request->desc;
384 saved_req.callback = request->callback;
385 saved_req.context = request->context;
386
387 /* release request entry in fifo */
388 smp_wmb();
389 request->desc = NULL;
390
391 /* increment fifo tail */
Kim Phillips4b9926282009-08-13 11:50:38 +1000392 priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800393
Kim Phillips4b9926282009-08-13 11:50:38 +1000394 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
Kim Phillipsec6644d2008-07-17 20:16:40 +0800395
Kim Phillips4b9926282009-08-13 11:50:38 +1000396 atomic_dec(&priv->chan[ch].submit_count);
Kim Phillipsec6644d2008-07-17 20:16:40 +0800397
Kim Phillips9c4a7962008-06-23 19:50:15 +0800398 saved_req.callback(dev, saved_req.desc, saved_req.context,
399 status);
400 /* channel may resume processing in single desc error case */
401 if (error && !reset_ch && status == error)
402 return;
Kim Phillips4b9926282009-08-13 11:50:38 +1000403 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
404 tail = priv->chan[ch].tail;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800405 }
406
Kim Phillips4b9926282009-08-13 11:50:38 +1000407 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800408}
409
410/*
411 * process completed requests for channels that have done status
412 */
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800413#define DEF_TALITOS_DONE(name, ch_done_mask) \
414static void talitos_done_##name(unsigned long data) \
415{ \
416 struct device *dev = (struct device *)data; \
417 struct talitos_private *priv = dev_get_drvdata(dev); \
Horia Geanta511d63c2012-03-30 17:49:53 +0300418 unsigned long flags; \
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800419 \
420 if (ch_done_mask & 1) \
421 flush_channel(dev, 0, 0, 0); \
422 if (priv->num_channels == 1) \
423 goto out; \
424 if (ch_done_mask & (1 << 2)) \
425 flush_channel(dev, 1, 0, 0); \
426 if (ch_done_mask & (1 << 4)) \
427 flush_channel(dev, 2, 0, 0); \
428 if (ch_done_mask & (1 << 6)) \
429 flush_channel(dev, 3, 0, 0); \
430 \
431out: \
432 /* At this point, all completed channels have been processed */ \
433 /* Unmask done interrupts for channels completed later on. */ \
Horia Geanta511d63c2012-03-30 17:49:53 +0300434 spin_lock_irqsave(&priv->reg_lock, flags); \
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800435 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
436 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS_IMR_LO_INIT); \
Horia Geanta511d63c2012-03-30 17:49:53 +0300437 spin_unlock_irqrestore(&priv->reg_lock, flags); \
Kim Phillips9c4a7962008-06-23 19:50:15 +0800438}
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800439DEF_TALITOS_DONE(4ch, TALITOS_ISR_4CHDONE)
440DEF_TALITOS_DONE(ch0_2, TALITOS_ISR_CH_0_2_DONE)
441DEF_TALITOS_DONE(ch1_3, TALITOS_ISR_CH_1_3_DONE)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800442
443/*
444 * locate current (offending) descriptor
445 */
Kim Phillips3e721ae2011-10-21 15:20:28 +0200446static u32 current_desc_hdr(struct device *dev, int ch)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800447{
448 struct talitos_private *priv = dev_get_drvdata(dev);
Kim Phillips4b9926282009-08-13 11:50:38 +1000449 int tail = priv->chan[ch].tail;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800450 dma_addr_t cur_desc;
451
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800452 cur_desc = in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800453
Kim Phillips4b9926282009-08-13 11:50:38 +1000454 while (priv->chan[ch].fifo[tail].dma_desc != cur_desc) {
Kim Phillips9c4a7962008-06-23 19:50:15 +0800455 tail = (tail + 1) & (priv->fifo_len - 1);
Kim Phillips4b9926282009-08-13 11:50:38 +1000456 if (tail == priv->chan[ch].tail) {
Kim Phillips9c4a7962008-06-23 19:50:15 +0800457 dev_err(dev, "couldn't locate current descriptor\n");
Kim Phillips3e721ae2011-10-21 15:20:28 +0200458 return 0;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800459 }
460 }
461
Kim Phillips3e721ae2011-10-21 15:20:28 +0200462 return priv->chan[ch].fifo[tail].desc->hdr;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800463}
464
465/*
466 * user diagnostics; report root cause of error based on execution unit status
467 */
Kim Phillips3e721ae2011-10-21 15:20:28 +0200468static void report_eu_error(struct device *dev, int ch, u32 desc_hdr)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800469{
470 struct talitos_private *priv = dev_get_drvdata(dev);
471 int i;
472
Kim Phillips3e721ae2011-10-21 15:20:28 +0200473 if (!desc_hdr)
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800474 desc_hdr = in_be32(priv->chan[ch].reg + TALITOS_DESCBUF);
Kim Phillips3e721ae2011-10-21 15:20:28 +0200475
476 switch (desc_hdr & DESC_HDR_SEL0_MASK) {
Kim Phillips9c4a7962008-06-23 19:50:15 +0800477 case DESC_HDR_SEL0_AFEU:
478 dev_err(dev, "AFEUISR 0x%08x_%08x\n",
479 in_be32(priv->reg + TALITOS_AFEUISR),
480 in_be32(priv->reg + TALITOS_AFEUISR_LO));
481 break;
482 case DESC_HDR_SEL0_DEU:
483 dev_err(dev, "DEUISR 0x%08x_%08x\n",
484 in_be32(priv->reg + TALITOS_DEUISR),
485 in_be32(priv->reg + TALITOS_DEUISR_LO));
486 break;
487 case DESC_HDR_SEL0_MDEUA:
488 case DESC_HDR_SEL0_MDEUB:
489 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
490 in_be32(priv->reg + TALITOS_MDEUISR),
491 in_be32(priv->reg + TALITOS_MDEUISR_LO));
492 break;
493 case DESC_HDR_SEL0_RNG:
494 dev_err(dev, "RNGUISR 0x%08x_%08x\n",
495 in_be32(priv->reg + TALITOS_RNGUISR),
496 in_be32(priv->reg + TALITOS_RNGUISR_LO));
497 break;
498 case DESC_HDR_SEL0_PKEU:
499 dev_err(dev, "PKEUISR 0x%08x_%08x\n",
500 in_be32(priv->reg + TALITOS_PKEUISR),
501 in_be32(priv->reg + TALITOS_PKEUISR_LO));
502 break;
503 case DESC_HDR_SEL0_AESU:
504 dev_err(dev, "AESUISR 0x%08x_%08x\n",
505 in_be32(priv->reg + TALITOS_AESUISR),
506 in_be32(priv->reg + TALITOS_AESUISR_LO));
507 break;
508 case DESC_HDR_SEL0_CRCU:
509 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
510 in_be32(priv->reg + TALITOS_CRCUISR),
511 in_be32(priv->reg + TALITOS_CRCUISR_LO));
512 break;
513 case DESC_HDR_SEL0_KEU:
514 dev_err(dev, "KEUISR 0x%08x_%08x\n",
515 in_be32(priv->reg + TALITOS_KEUISR),
516 in_be32(priv->reg + TALITOS_KEUISR_LO));
517 break;
518 }
519
Kim Phillips3e721ae2011-10-21 15:20:28 +0200520 switch (desc_hdr & DESC_HDR_SEL1_MASK) {
Kim Phillips9c4a7962008-06-23 19:50:15 +0800521 case DESC_HDR_SEL1_MDEUA:
522 case DESC_HDR_SEL1_MDEUB:
523 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
524 in_be32(priv->reg + TALITOS_MDEUISR),
525 in_be32(priv->reg + TALITOS_MDEUISR_LO));
526 break;
527 case DESC_HDR_SEL1_CRCU:
528 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
529 in_be32(priv->reg + TALITOS_CRCUISR),
530 in_be32(priv->reg + TALITOS_CRCUISR_LO));
531 break;
532 }
533
534 for (i = 0; i < 8; i++)
535 dev_err(dev, "DESCBUF 0x%08x_%08x\n",
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800536 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i),
537 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800538}
539
540/*
541 * recover from error interrupts
542 */
Kim Phillips5e718a02011-12-12 14:59:12 -0600543static void talitos_error(struct device *dev, u32 isr, u32 isr_lo)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800544{
Kim Phillips9c4a7962008-06-23 19:50:15 +0800545 struct talitos_private *priv = dev_get_drvdata(dev);
546 unsigned int timeout = TALITOS_TIMEOUT;
547 int ch, error, reset_dev = 0, reset_ch = 0;
Kim Phillips40405f12008-10-12 20:19:35 +0800548 u32 v, v_lo;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800549
550 for (ch = 0; ch < priv->num_channels; ch++) {
551 /* skip channels without errors */
552 if (!(isr & (1 << (ch * 2 + 1))))
553 continue;
554
555 error = -EINVAL;
556
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800557 v = in_be32(priv->chan[ch].reg + TALITOS_CCPSR);
558 v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800559
560 if (v_lo & TALITOS_CCPSR_LO_DOF) {
561 dev_err(dev, "double fetch fifo overflow error\n");
562 error = -EAGAIN;
563 reset_ch = 1;
564 }
565 if (v_lo & TALITOS_CCPSR_LO_SOF) {
566 /* h/w dropped descriptor */
567 dev_err(dev, "single fetch fifo overflow error\n");
568 error = -EAGAIN;
569 }
570 if (v_lo & TALITOS_CCPSR_LO_MDTE)
571 dev_err(dev, "master data transfer error\n");
572 if (v_lo & TALITOS_CCPSR_LO_SGDLZ)
573 dev_err(dev, "s/g data length zero error\n");
574 if (v_lo & TALITOS_CCPSR_LO_FPZ)
575 dev_err(dev, "fetch pointer zero error\n");
576 if (v_lo & TALITOS_CCPSR_LO_IDH)
577 dev_err(dev, "illegal descriptor header error\n");
578 if (v_lo & TALITOS_CCPSR_LO_IEU)
579 dev_err(dev, "invalid execution unit error\n");
580 if (v_lo & TALITOS_CCPSR_LO_EU)
Kim Phillips3e721ae2011-10-21 15:20:28 +0200581 report_eu_error(dev, ch, current_desc_hdr(dev, ch));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800582 if (v_lo & TALITOS_CCPSR_LO_GB)
583 dev_err(dev, "gather boundary error\n");
584 if (v_lo & TALITOS_CCPSR_LO_GRL)
585 dev_err(dev, "gather return/length error\n");
586 if (v_lo & TALITOS_CCPSR_LO_SB)
587 dev_err(dev, "scatter boundary error\n");
588 if (v_lo & TALITOS_CCPSR_LO_SRL)
589 dev_err(dev, "scatter return/length error\n");
590
591 flush_channel(dev, ch, error, reset_ch);
592
593 if (reset_ch) {
594 reset_channel(dev, ch);
595 } else {
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800596 setbits32(priv->chan[ch].reg + TALITOS_CCCR,
Kim Phillips9c4a7962008-06-23 19:50:15 +0800597 TALITOS_CCCR_CONT);
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800598 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0);
599 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
Kim Phillips9c4a7962008-06-23 19:50:15 +0800600 TALITOS_CCCR_CONT) && --timeout)
601 cpu_relax();
602 if (timeout == 0) {
603 dev_err(dev, "failed to restart channel %d\n",
604 ch);
605 reset_dev = 1;
606 }
607 }
608 }
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800609 if (reset_dev || isr & ~TALITOS_ISR_4CHERR || isr_lo) {
Kim Phillips9c4a7962008-06-23 19:50:15 +0800610 dev_err(dev, "done overflow, internal time out, or rngu error: "
611 "ISR 0x%08x_%08x\n", isr, isr_lo);
612
613 /* purge request queues */
614 for (ch = 0; ch < priv->num_channels; ch++)
615 flush_channel(dev, ch, -EIO, 1);
616
617 /* reset and reinitialize the device */
618 init_device(dev);
619 }
620}
621
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800622#define DEF_TALITOS_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \
623static irqreturn_t talitos_interrupt_##name(int irq, void *data) \
624{ \
625 struct device *dev = data; \
626 struct talitos_private *priv = dev_get_drvdata(dev); \
627 u32 isr, isr_lo; \
Horia Geanta511d63c2012-03-30 17:49:53 +0300628 unsigned long flags; \
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800629 \
Horia Geanta511d63c2012-03-30 17:49:53 +0300630 spin_lock_irqsave(&priv->reg_lock, flags); \
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800631 isr = in_be32(priv->reg + TALITOS_ISR); \
632 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \
633 /* Acknowledge interrupt */ \
634 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
635 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \
636 \
Horia Geanta511d63c2012-03-30 17:49:53 +0300637 if (unlikely(isr & ch_err_mask || isr_lo)) { \
638 spin_unlock_irqrestore(&priv->reg_lock, flags); \
639 talitos_error(dev, isr & ch_err_mask, isr_lo); \
640 } \
641 else { \
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800642 if (likely(isr & ch_done_mask)) { \
643 /* mask further done interrupts. */ \
644 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
645 /* done_task will unmask done interrupts at exit */ \
646 tasklet_schedule(&priv->done_task[tlet]); \
647 } \
Horia Geanta511d63c2012-03-30 17:49:53 +0300648 spin_unlock_irqrestore(&priv->reg_lock, flags); \
649 } \
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800650 \
651 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \
652 IRQ_NONE; \
Kim Phillips9c4a7962008-06-23 19:50:15 +0800653}
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800654DEF_TALITOS_INTERRUPT(4ch, TALITOS_ISR_4CHDONE, TALITOS_ISR_4CHERR, 0)
655DEF_TALITOS_INTERRUPT(ch0_2, TALITOS_ISR_CH_0_2_DONE, TALITOS_ISR_CH_0_2_ERR, 0)
656DEF_TALITOS_INTERRUPT(ch1_3, TALITOS_ISR_CH_1_3_DONE, TALITOS_ISR_CH_1_3_ERR, 1)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800657
658/*
659 * hwrng
660 */
661static int talitos_rng_data_present(struct hwrng *rng, int wait)
662{
663 struct device *dev = (struct device *)rng->priv;
664 struct talitos_private *priv = dev_get_drvdata(dev);
665 u32 ofl;
666 int i;
667
668 for (i = 0; i < 20; i++) {
669 ofl = in_be32(priv->reg + TALITOS_RNGUSR_LO) &
670 TALITOS_RNGUSR_LO_OFL;
671 if (ofl || !wait)
672 break;
673 udelay(10);
674 }
675
676 return !!ofl;
677}
678
679static int talitos_rng_data_read(struct hwrng *rng, u32 *data)
680{
681 struct device *dev = (struct device *)rng->priv;
682 struct talitos_private *priv = dev_get_drvdata(dev);
683
684 /* rng fifo requires 64-bit accesses */
685 *data = in_be32(priv->reg + TALITOS_RNGU_FIFO);
686 *data = in_be32(priv->reg + TALITOS_RNGU_FIFO_LO);
687
688 return sizeof(u32);
689}
690
691static int talitos_rng_init(struct hwrng *rng)
692{
693 struct device *dev = (struct device *)rng->priv;
694 struct talitos_private *priv = dev_get_drvdata(dev);
695 unsigned int timeout = TALITOS_TIMEOUT;
696
697 setbits32(priv->reg + TALITOS_RNGURCR_LO, TALITOS_RNGURCR_LO_SR);
698 while (!(in_be32(priv->reg + TALITOS_RNGUSR_LO) & TALITOS_RNGUSR_LO_RD)
699 && --timeout)
700 cpu_relax();
701 if (timeout == 0) {
702 dev_err(dev, "failed to reset rng hw\n");
703 return -ENODEV;
704 }
705
706 /* start generating */
707 setbits32(priv->reg + TALITOS_RNGUDSR_LO, 0);
708
709 return 0;
710}
711
712static int talitos_register_rng(struct device *dev)
713{
714 struct talitos_private *priv = dev_get_drvdata(dev);
715
716 priv->rng.name = dev_driver_string(dev),
717 priv->rng.init = talitos_rng_init,
718 priv->rng.data_present = talitos_rng_data_present,
719 priv->rng.data_read = talitos_rng_data_read,
720 priv->rng.priv = (unsigned long)dev;
721
722 return hwrng_register(&priv->rng);
723}
724
725static void talitos_unregister_rng(struct device *dev)
726{
727 struct talitos_private *priv = dev_get_drvdata(dev);
728
729 hwrng_unregister(&priv->rng);
730}
731
732/*
733 * crypto alg
734 */
735#define TALITOS_CRA_PRIORITY 3000
736#define TALITOS_MAX_KEY_SIZE 64
Lee Nipper3952f172008-07-10 18:29:18 +0800737#define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
Lee Nipper70bcaca2008-07-03 19:08:46 +0800738
Lee Nipper497f2e62010-05-19 19:20:36 +1000739#define MD5_BLOCK_SIZE 64
Kim Phillips9c4a7962008-06-23 19:50:15 +0800740
741struct talitos_ctx {
742 struct device *dev;
Kim Phillips5228f0f2011-07-15 11:21:38 +0800743 int ch;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800744 __be32 desc_hdr_template;
745 u8 key[TALITOS_MAX_KEY_SIZE];
Lee Nipper70bcaca2008-07-03 19:08:46 +0800746 u8 iv[TALITOS_MAX_IV_LENGTH];
Kim Phillips9c4a7962008-06-23 19:50:15 +0800747 unsigned int keylen;
748 unsigned int enckeylen;
749 unsigned int authkeylen;
750 unsigned int authsize;
751};
752
Lee Nipper497f2e62010-05-19 19:20:36 +1000753#define HASH_MAX_BLOCK_SIZE SHA512_BLOCK_SIZE
754#define TALITOS_MDEU_MAX_CONTEXT_SIZE TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512
755
756struct talitos_ahash_req_ctx {
Kim Phillips60f208d72010-05-19 19:21:53 +1000757 u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
Lee Nipper497f2e62010-05-19 19:20:36 +1000758 unsigned int hw_context_size;
759 u8 buf[HASH_MAX_BLOCK_SIZE];
760 u8 bufnext[HASH_MAX_BLOCK_SIZE];
Kim Phillips60f208d72010-05-19 19:21:53 +1000761 unsigned int swinit;
Lee Nipper497f2e62010-05-19 19:20:36 +1000762 unsigned int first;
763 unsigned int last;
764 unsigned int to_hash_later;
Lee Nipper5e833bc2010-06-16 15:29:15 +1000765 u64 nbuf;
Lee Nipper497f2e62010-05-19 19:20:36 +1000766 struct scatterlist bufsl[2];
767 struct scatterlist *psrc;
768};
769
Lee Nipper56af8cd2009-03-29 15:50:50 +0800770static int aead_setauthsize(struct crypto_aead *authenc,
771 unsigned int authsize)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800772{
773 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
774
775 ctx->authsize = authsize;
776
777 return 0;
778}
779
Lee Nipper56af8cd2009-03-29 15:50:50 +0800780static int aead_setkey(struct crypto_aead *authenc,
781 const u8 *key, unsigned int keylen)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800782{
783 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
784 struct rtattr *rta = (void *)key;
785 struct crypto_authenc_key_param *param;
786 unsigned int authkeylen;
787 unsigned int enckeylen;
788
789 if (!RTA_OK(rta, keylen))
790 goto badkey;
791
792 if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM)
793 goto badkey;
794
795 if (RTA_PAYLOAD(rta) < sizeof(*param))
796 goto badkey;
797
798 param = RTA_DATA(rta);
799 enckeylen = be32_to_cpu(param->enckeylen);
800
801 key += RTA_ALIGN(rta->rta_len);
802 keylen -= RTA_ALIGN(rta->rta_len);
803
804 if (keylen < enckeylen)
805 goto badkey;
806
807 authkeylen = keylen - enckeylen;
808
809 if (keylen > TALITOS_MAX_KEY_SIZE)
810 goto badkey;
811
812 memcpy(&ctx->key, key, keylen);
813
814 ctx->keylen = keylen;
815 ctx->enckeylen = enckeylen;
816 ctx->authkeylen = authkeylen;
817
818 return 0;
819
820badkey:
821 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
822 return -EINVAL;
823}
824
825/*
Lee Nipper56af8cd2009-03-29 15:50:50 +0800826 * talitos_edesc - s/w-extended descriptor
Kim Phillips9c4a7962008-06-23 19:50:15 +0800827 * @src_nents: number of segments in input scatterlist
828 * @dst_nents: number of segments in output scatterlist
829 * @dma_len: length of dma mapped link_tbl space
830 * @dma_link_tbl: bus physical address of link_tbl
831 * @desc: h/w descriptor
832 * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1)
833 *
834 * if decrypting (with authcheck), or either one of src_nents or dst_nents
835 * is greater than 1, an integrity check value is concatenated to the end
836 * of link_tbl data
837 */
Lee Nipper56af8cd2009-03-29 15:50:50 +0800838struct talitos_edesc {
Kim Phillips9c4a7962008-06-23 19:50:15 +0800839 int src_nents;
840 int dst_nents;
Lee Nipper4de9d0b2009-03-29 15:52:32 +0800841 int src_is_chained;
842 int dst_is_chained;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800843 int dma_len;
844 dma_addr_t dma_link_tbl;
845 struct talitos_desc desc;
846 struct talitos_ptr link_tbl[0];
847};
848
Lee Nipper4de9d0b2009-03-29 15:52:32 +0800849static int talitos_map_sg(struct device *dev, struct scatterlist *sg,
850 unsigned int nents, enum dma_data_direction dir,
851 int chained)
852{
853 if (unlikely(chained))
854 while (sg) {
855 dma_map_sg(dev, sg, 1, dir);
856 sg = scatterwalk_sg_next(sg);
857 }
858 else
859 dma_map_sg(dev, sg, nents, dir);
860 return nents;
861}
862
863static void talitos_unmap_sg_chain(struct device *dev, struct scatterlist *sg,
864 enum dma_data_direction dir)
865{
866 while (sg) {
867 dma_unmap_sg(dev, sg, 1, dir);
868 sg = scatterwalk_sg_next(sg);
869 }
870}
871
872static void talitos_sg_unmap(struct device *dev,
873 struct talitos_edesc *edesc,
874 struct scatterlist *src,
875 struct scatterlist *dst)
876{
877 unsigned int src_nents = edesc->src_nents ? : 1;
878 unsigned int dst_nents = edesc->dst_nents ? : 1;
879
880 if (src != dst) {
881 if (edesc->src_is_chained)
882 talitos_unmap_sg_chain(dev, src, DMA_TO_DEVICE);
883 else
884 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
885
Lee Nipper497f2e62010-05-19 19:20:36 +1000886 if (dst) {
887 if (edesc->dst_is_chained)
888 talitos_unmap_sg_chain(dev, dst,
889 DMA_FROM_DEVICE);
890 else
891 dma_unmap_sg(dev, dst, dst_nents,
892 DMA_FROM_DEVICE);
893 }
Lee Nipper4de9d0b2009-03-29 15:52:32 +0800894 } else
895 if (edesc->src_is_chained)
896 talitos_unmap_sg_chain(dev, src, DMA_BIDIRECTIONAL);
897 else
898 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
899}
900
Kim Phillips9c4a7962008-06-23 19:50:15 +0800901static void ipsec_esp_unmap(struct device *dev,
Lee Nipper56af8cd2009-03-29 15:50:50 +0800902 struct talitos_edesc *edesc,
Kim Phillips9c4a7962008-06-23 19:50:15 +0800903 struct aead_request *areq)
904{
905 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6], DMA_FROM_DEVICE);
906 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[3], DMA_TO_DEVICE);
907 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
908 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE);
909
910 dma_unmap_sg(dev, areq->assoc, 1, DMA_TO_DEVICE);
911
Lee Nipper4de9d0b2009-03-29 15:52:32 +0800912 talitos_sg_unmap(dev, edesc, areq->src, areq->dst);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800913
914 if (edesc->dma_len)
915 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
916 DMA_BIDIRECTIONAL);
917}
918
919/*
920 * ipsec_esp descriptor callbacks
921 */
922static void ipsec_esp_encrypt_done(struct device *dev,
923 struct talitos_desc *desc, void *context,
924 int err)
925{
926 struct aead_request *areq = context;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800927 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
928 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
Kim Phillips19bbbc62009-03-29 15:53:59 +0800929 struct talitos_edesc *edesc;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800930 struct scatterlist *sg;
931 void *icvdata;
932
Kim Phillips19bbbc62009-03-29 15:53:59 +0800933 edesc = container_of(desc, struct talitos_edesc, desc);
934
Kim Phillips9c4a7962008-06-23 19:50:15 +0800935 ipsec_esp_unmap(dev, edesc, areq);
936
937 /* copy the generated ICV to dst */
938 if (edesc->dma_len) {
939 icvdata = &edesc->link_tbl[edesc->src_nents +
Lee Nipperf3c85bc2008-07-30 16:26:57 +0800940 edesc->dst_nents + 2];
Kim Phillips9c4a7962008-06-23 19:50:15 +0800941 sg = sg_last(areq->dst, edesc->dst_nents);
942 memcpy((char *)sg_virt(sg) + sg->length - ctx->authsize,
943 icvdata, ctx->authsize);
944 }
945
946 kfree(edesc);
947
948 aead_request_complete(areq, err);
949}
950
Kim Phillipsfe5720e2008-10-12 20:33:14 +0800951static void ipsec_esp_decrypt_swauth_done(struct device *dev,
Kim Phillipse938e462009-03-29 15:53:23 +0800952 struct talitos_desc *desc,
953 void *context, int err)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800954{
955 struct aead_request *req = context;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800956 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
957 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
Kim Phillips19bbbc62009-03-29 15:53:59 +0800958 struct talitos_edesc *edesc;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800959 struct scatterlist *sg;
960 void *icvdata;
961
Kim Phillips19bbbc62009-03-29 15:53:59 +0800962 edesc = container_of(desc, struct talitos_edesc, desc);
963
Kim Phillips9c4a7962008-06-23 19:50:15 +0800964 ipsec_esp_unmap(dev, edesc, req);
965
966 if (!err) {
967 /* auth check */
968 if (edesc->dma_len)
969 icvdata = &edesc->link_tbl[edesc->src_nents +
Lee Nipperf3c85bc2008-07-30 16:26:57 +0800970 edesc->dst_nents + 2];
Kim Phillips9c4a7962008-06-23 19:50:15 +0800971 else
972 icvdata = &edesc->link_tbl[0];
973
974 sg = sg_last(req->dst, edesc->dst_nents ? : 1);
975 err = memcmp(icvdata, (char *)sg_virt(sg) + sg->length -
976 ctx->authsize, ctx->authsize) ? -EBADMSG : 0;
977 }
978
979 kfree(edesc);
980
981 aead_request_complete(req, err);
982}
983
Kim Phillipsfe5720e2008-10-12 20:33:14 +0800984static void ipsec_esp_decrypt_hwauth_done(struct device *dev,
Kim Phillipse938e462009-03-29 15:53:23 +0800985 struct talitos_desc *desc,
986 void *context, int err)
Kim Phillipsfe5720e2008-10-12 20:33:14 +0800987{
988 struct aead_request *req = context;
Kim Phillips19bbbc62009-03-29 15:53:59 +0800989 struct talitos_edesc *edesc;
990
991 edesc = container_of(desc, struct talitos_edesc, desc);
Kim Phillipsfe5720e2008-10-12 20:33:14 +0800992
993 ipsec_esp_unmap(dev, edesc, req);
994
995 /* check ICV auth status */
Kim Phillipse938e462009-03-29 15:53:23 +0800996 if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) !=
997 DESC_HDR_LO_ICCR1_PASS))
998 err = -EBADMSG;
Kim Phillipsfe5720e2008-10-12 20:33:14 +0800999
1000 kfree(edesc);
1001
1002 aead_request_complete(req, err);
1003}
1004
Kim Phillips9c4a7962008-06-23 19:50:15 +08001005/*
1006 * convert scatterlist to SEC h/w link table format
1007 * stop at cryptlen bytes
1008 */
Lee Nipper70bcaca2008-07-03 19:08:46 +08001009static int sg_to_link_tbl(struct scatterlist *sg, int sg_count,
Kim Phillips9c4a7962008-06-23 19:50:15 +08001010 int cryptlen, struct talitos_ptr *link_tbl_ptr)
1011{
Lee Nipper70bcaca2008-07-03 19:08:46 +08001012 int n_sg = sg_count;
1013
1014 while (n_sg--) {
Kim Phillips81eb0242009-08-13 11:51:51 +10001015 to_talitos_ptr(link_tbl_ptr, sg_dma_address(sg));
Kim Phillips9c4a7962008-06-23 19:50:15 +08001016 link_tbl_ptr->len = cpu_to_be16(sg_dma_len(sg));
1017 link_tbl_ptr->j_extent = 0;
1018 link_tbl_ptr++;
1019 cryptlen -= sg_dma_len(sg);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001020 sg = scatterwalk_sg_next(sg);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001021 }
1022
Lee Nipper70bcaca2008-07-03 19:08:46 +08001023 /* adjust (decrease) last one (or two) entry's len to cryptlen */
Kim Phillips9c4a7962008-06-23 19:50:15 +08001024 link_tbl_ptr--;
Kim Phillipsc0e741d2008-07-17 20:20:59 +08001025 while (be16_to_cpu(link_tbl_ptr->len) <= (-cryptlen)) {
Lee Nipper70bcaca2008-07-03 19:08:46 +08001026 /* Empty this entry, and move to previous one */
1027 cryptlen += be16_to_cpu(link_tbl_ptr->len);
1028 link_tbl_ptr->len = 0;
1029 sg_count--;
1030 link_tbl_ptr--;
1031 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08001032 link_tbl_ptr->len = cpu_to_be16(be16_to_cpu(link_tbl_ptr->len)
1033 + cryptlen);
1034
1035 /* tag end of link table */
1036 link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
Lee Nipper70bcaca2008-07-03 19:08:46 +08001037
1038 return sg_count;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001039}
1040
1041/*
1042 * fill in and submit ipsec_esp descriptor
1043 */
Lee Nipper56af8cd2009-03-29 15:50:50 +08001044static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
Kim Phillips9c4a7962008-06-23 19:50:15 +08001045 u8 *giv, u64 seq,
1046 void (*callback) (struct device *dev,
1047 struct talitos_desc *desc,
1048 void *context, int error))
1049{
1050 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
1051 struct talitos_ctx *ctx = crypto_aead_ctx(aead);
1052 struct device *dev = ctx->dev;
1053 struct talitos_desc *desc = &edesc->desc;
1054 unsigned int cryptlen = areq->cryptlen;
1055 unsigned int authsize = ctx->authsize;
Kim Phillipse41256f2009-08-13 11:49:06 +10001056 unsigned int ivsize = crypto_aead_ivsize(aead);
Kim Phillipsfa86a262008-07-17 20:20:06 +08001057 int sg_count, ret;
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001058 int sg_link_tbl_len;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001059
1060 /* hmac key */
1061 map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key,
1062 0, DMA_TO_DEVICE);
1063 /* hmac data */
Kim Phillipse41256f2009-08-13 11:49:06 +10001064 map_single_talitos_ptr(dev, &desc->ptr[1], areq->assoclen + ivsize,
1065 sg_virt(areq->assoc), 0, DMA_TO_DEVICE);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001066 /* cipher iv */
Kim Phillips9c4a7962008-06-23 19:50:15 +08001067 map_single_talitos_ptr(dev, &desc->ptr[2], ivsize, giv ?: areq->iv, 0,
1068 DMA_TO_DEVICE);
1069
1070 /* cipher key */
1071 map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen,
1072 (char *)&ctx->key + ctx->authkeylen, 0,
1073 DMA_TO_DEVICE);
1074
1075 /*
1076 * cipher in
1077 * map and adjust cipher len to aead request cryptlen.
1078 * extent is bytes of HMAC postpended to ciphertext,
1079 * typically 12 for ipsec
1080 */
1081 desc->ptr[4].len = cpu_to_be16(cryptlen);
1082 desc->ptr[4].j_extent = authsize;
1083
Kim Phillipse938e462009-03-29 15:53:23 +08001084 sg_count = talitos_map_sg(dev, areq->src, edesc->src_nents ? : 1,
1085 (areq->src == areq->dst) ? DMA_BIDIRECTIONAL
1086 : DMA_TO_DEVICE,
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001087 edesc->src_is_chained);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001088
1089 if (sg_count == 1) {
Kim Phillips81eb0242009-08-13 11:51:51 +10001090 to_talitos_ptr(&desc->ptr[4], sg_dma_address(areq->src));
Kim Phillips9c4a7962008-06-23 19:50:15 +08001091 } else {
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001092 sg_link_tbl_len = cryptlen;
1093
Kim Phillips962a9c92009-03-29 15:54:30 +08001094 if (edesc->desc.hdr & DESC_HDR_MODE1_MDEU_CICV)
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001095 sg_link_tbl_len = cryptlen + authsize;
Kim Phillipse938e462009-03-29 15:53:23 +08001096
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001097 sg_count = sg_to_link_tbl(areq->src, sg_count, sg_link_tbl_len,
Lee Nipper70bcaca2008-07-03 19:08:46 +08001098 &edesc->link_tbl[0]);
1099 if (sg_count > 1) {
1100 desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP;
Kim Phillips81eb0242009-08-13 11:51:51 +10001101 to_talitos_ptr(&desc->ptr[4], edesc->dma_link_tbl);
Kim Phillipse938e462009-03-29 15:53:23 +08001102 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1103 edesc->dma_len,
1104 DMA_BIDIRECTIONAL);
Lee Nipper70bcaca2008-07-03 19:08:46 +08001105 } else {
1106 /* Only one segment now, so no link tbl needed */
Kim Phillips81eb0242009-08-13 11:51:51 +10001107 to_talitos_ptr(&desc->ptr[4],
1108 sg_dma_address(areq->src));
Lee Nipper70bcaca2008-07-03 19:08:46 +08001109 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08001110 }
1111
1112 /* cipher out */
1113 desc->ptr[5].len = cpu_to_be16(cryptlen);
1114 desc->ptr[5].j_extent = authsize;
1115
Kim Phillipse938e462009-03-29 15:53:23 +08001116 if (areq->src != areq->dst)
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001117 sg_count = talitos_map_sg(dev, areq->dst,
1118 edesc->dst_nents ? : 1,
1119 DMA_FROM_DEVICE,
1120 edesc->dst_is_chained);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001121
1122 if (sg_count == 1) {
Kim Phillips81eb0242009-08-13 11:51:51 +10001123 to_talitos_ptr(&desc->ptr[5], sg_dma_address(areq->dst));
Kim Phillips9c4a7962008-06-23 19:50:15 +08001124 } else {
1125 struct talitos_ptr *link_tbl_ptr =
Lee Nipperf3c85bc2008-07-30 16:26:57 +08001126 &edesc->link_tbl[edesc->src_nents + 1];
Kim Phillips9c4a7962008-06-23 19:50:15 +08001127
Kim Phillips81eb0242009-08-13 11:51:51 +10001128 to_talitos_ptr(&desc->ptr[5], edesc->dma_link_tbl +
1129 (edesc->src_nents + 1) *
1130 sizeof(struct talitos_ptr));
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001131 sg_count = sg_to_link_tbl(areq->dst, sg_count, cryptlen,
1132 link_tbl_ptr);
1133
Lee Nipperf3c85bc2008-07-30 16:26:57 +08001134 /* Add an entry to the link table for ICV data */
Kim Phillips9c4a7962008-06-23 19:50:15 +08001135 link_tbl_ptr += sg_count - 1;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001136 link_tbl_ptr->j_extent = 0;
Lee Nipperf3c85bc2008-07-30 16:26:57 +08001137 sg_count++;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001138 link_tbl_ptr++;
1139 link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
1140 link_tbl_ptr->len = cpu_to_be16(authsize);
1141
1142 /* icv data follows link tables */
Kim Phillips81eb0242009-08-13 11:51:51 +10001143 to_talitos_ptr(link_tbl_ptr, edesc->dma_link_tbl +
1144 (edesc->src_nents + edesc->dst_nents + 2) *
1145 sizeof(struct talitos_ptr));
Kim Phillips9c4a7962008-06-23 19:50:15 +08001146 desc->ptr[5].j_extent |= DESC_PTR_LNKTBL_JUMP;
1147 dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl,
1148 edesc->dma_len, DMA_BIDIRECTIONAL);
1149 }
1150
1151 /* iv out */
1152 map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv, 0,
1153 DMA_FROM_DEVICE);
1154
Kim Phillips5228f0f2011-07-15 11:21:38 +08001155 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
Kim Phillipsfa86a262008-07-17 20:20:06 +08001156 if (ret != -EINPROGRESS) {
1157 ipsec_esp_unmap(dev, edesc, areq);
1158 kfree(edesc);
1159 }
1160 return ret;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001161}
1162
Kim Phillips9c4a7962008-06-23 19:50:15 +08001163/*
1164 * derive number of elements in scatterlist
1165 */
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001166static int sg_count(struct scatterlist *sg_list, int nbytes, int *chained)
Kim Phillips9c4a7962008-06-23 19:50:15 +08001167{
1168 struct scatterlist *sg = sg_list;
1169 int sg_nents = 0;
1170
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001171 *chained = 0;
1172 while (nbytes > 0) {
Kim Phillips9c4a7962008-06-23 19:50:15 +08001173 sg_nents++;
1174 nbytes -= sg->length;
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001175 if (!sg_is_last(sg) && (sg + 1)->length == 0)
1176 *chained = 1;
1177 sg = scatterwalk_sg_next(sg);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001178 }
1179
1180 return sg_nents;
1181}
1182
Lee Nipper497f2e62010-05-19 19:20:36 +10001183/**
1184 * sg_copy_end_to_buffer - Copy end data from SG list to a linear buffer
1185 * @sgl: The SG list
1186 * @nents: Number of SG entries
1187 * @buf: Where to copy to
1188 * @buflen: The number of bytes to copy
1189 * @skip: The number of bytes to skip before copying.
1190 * Note: skip + buflen should equal SG total size.
1191 *
1192 * Returns the number of copied bytes.
1193 *
1194 **/
1195static size_t sg_copy_end_to_buffer(struct scatterlist *sgl, unsigned int nents,
1196 void *buf, size_t buflen, unsigned int skip)
1197{
1198 unsigned int offset = 0;
1199 unsigned int boffset = 0;
1200 struct sg_mapping_iter miter;
1201 unsigned long flags;
1202 unsigned int sg_flags = SG_MITER_ATOMIC;
1203 size_t total_buffer = buflen + skip;
1204
1205 sg_flags |= SG_MITER_FROM_SG;
1206
1207 sg_miter_start(&miter, sgl, nents, sg_flags);
1208
1209 local_irq_save(flags);
1210
1211 while (sg_miter_next(&miter) && offset < total_buffer) {
1212 unsigned int len;
1213 unsigned int ignore;
1214
1215 if ((offset + miter.length) > skip) {
1216 if (offset < skip) {
1217 /* Copy part of this segment */
1218 ignore = skip - offset;
1219 len = miter.length - ignore;
Lee Nipper72600422010-07-19 14:11:24 +08001220 if (boffset + len > buflen)
1221 len = buflen - boffset;
Lee Nipper497f2e62010-05-19 19:20:36 +10001222 memcpy(buf + boffset, miter.addr + ignore, len);
1223 } else {
Lee Nipper72600422010-07-19 14:11:24 +08001224 /* Copy all of this segment (up to buflen) */
Lee Nipper497f2e62010-05-19 19:20:36 +10001225 len = miter.length;
Lee Nipper72600422010-07-19 14:11:24 +08001226 if (boffset + len > buflen)
1227 len = buflen - boffset;
Lee Nipper497f2e62010-05-19 19:20:36 +10001228 memcpy(buf + boffset, miter.addr, len);
1229 }
1230 boffset += len;
1231 }
1232 offset += miter.length;
1233 }
1234
1235 sg_miter_stop(&miter);
1236
1237 local_irq_restore(flags);
1238 return boffset;
1239}
1240
Kim Phillips9c4a7962008-06-23 19:50:15 +08001241/*
Lee Nipper56af8cd2009-03-29 15:50:50 +08001242 * allocate and map the extended descriptor
Kim Phillips9c4a7962008-06-23 19:50:15 +08001243 */
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001244static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
1245 struct scatterlist *src,
1246 struct scatterlist *dst,
Lee Nipper497f2e62010-05-19 19:20:36 +10001247 int hash_result,
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001248 unsigned int cryptlen,
1249 unsigned int authsize,
1250 int icv_stashing,
1251 u32 cryptoflags)
Kim Phillips9c4a7962008-06-23 19:50:15 +08001252{
Lee Nipper56af8cd2009-03-29 15:50:50 +08001253 struct talitos_edesc *edesc;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001254 int src_nents, dst_nents, alloc_len, dma_len;
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001255 int src_chained, dst_chained = 0;
1256 gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
Kim Phillips586725f2008-07-17 20:19:18 +08001257 GFP_ATOMIC;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001258
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001259 if (cryptlen + authsize > TALITOS_MAX_DATA_LEN) {
1260 dev_err(dev, "length exceeds h/w max limit\n");
Kim Phillips9c4a7962008-06-23 19:50:15 +08001261 return ERR_PTR(-EINVAL);
1262 }
1263
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001264 src_nents = sg_count(src, cryptlen + authsize, &src_chained);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001265 src_nents = (src_nents == 1) ? 0 : src_nents;
1266
Lee Nipper497f2e62010-05-19 19:20:36 +10001267 if (hash_result) {
1268 dst_nents = 0;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001269 } else {
Lee Nipper497f2e62010-05-19 19:20:36 +10001270 if (dst == src) {
1271 dst_nents = src_nents;
1272 } else {
1273 dst_nents = sg_count(dst, cryptlen + authsize,
1274 &dst_chained);
1275 dst_nents = (dst_nents == 1) ? 0 : dst_nents;
1276 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08001277 }
1278
1279 /*
1280 * allocate space for base edesc plus the link tables,
Lee Nipperf3c85bc2008-07-30 16:26:57 +08001281 * allowing for two separate entries for ICV and generated ICV (+ 2),
Kim Phillips9c4a7962008-06-23 19:50:15 +08001282 * and the ICV data itself
1283 */
Lee Nipper56af8cd2009-03-29 15:50:50 +08001284 alloc_len = sizeof(struct talitos_edesc);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001285 if (src_nents || dst_nents) {
Lee Nipperf3c85bc2008-07-30 16:26:57 +08001286 dma_len = (src_nents + dst_nents + 2) *
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001287 sizeof(struct talitos_ptr) + authsize;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001288 alloc_len += dma_len;
1289 } else {
1290 dma_len = 0;
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001291 alloc_len += icv_stashing ? authsize : 0;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001292 }
1293
Kim Phillips586725f2008-07-17 20:19:18 +08001294 edesc = kmalloc(alloc_len, GFP_DMA | flags);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001295 if (!edesc) {
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001296 dev_err(dev, "could not allocate edescriptor\n");
Kim Phillips9c4a7962008-06-23 19:50:15 +08001297 return ERR_PTR(-ENOMEM);
1298 }
1299
1300 edesc->src_nents = src_nents;
1301 edesc->dst_nents = dst_nents;
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001302 edesc->src_is_chained = src_chained;
1303 edesc->dst_is_chained = dst_chained;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001304 edesc->dma_len = dma_len;
Lee Nipper497f2e62010-05-19 19:20:36 +10001305 if (dma_len)
1306 edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0],
1307 edesc->dma_len,
1308 DMA_BIDIRECTIONAL);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001309
1310 return edesc;
1311}
1312
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001313static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq,
1314 int icv_stashing)
1315{
1316 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1317 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1318
Lee Nipper497f2e62010-05-19 19:20:36 +10001319 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, 0,
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001320 areq->cryptlen, ctx->authsize, icv_stashing,
1321 areq->base.flags);
1322}
1323
Lee Nipper56af8cd2009-03-29 15:50:50 +08001324static int aead_encrypt(struct aead_request *req)
Kim Phillips9c4a7962008-06-23 19:50:15 +08001325{
1326 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1327 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
Lee Nipper56af8cd2009-03-29 15:50:50 +08001328 struct talitos_edesc *edesc;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001329
1330 /* allocate extended descriptor */
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001331 edesc = aead_edesc_alloc(req, 0);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001332 if (IS_ERR(edesc))
1333 return PTR_ERR(edesc);
1334
1335 /* set encrypt */
Lee Nipper70bcaca2008-07-03 19:08:46 +08001336 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001337
1338 return ipsec_esp(edesc, req, NULL, 0, ipsec_esp_encrypt_done);
1339}
1340
Lee Nipper56af8cd2009-03-29 15:50:50 +08001341static int aead_decrypt(struct aead_request *req)
Kim Phillips9c4a7962008-06-23 19:50:15 +08001342{
1343 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1344 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1345 unsigned int authsize = ctx->authsize;
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001346 struct talitos_private *priv = dev_get_drvdata(ctx->dev);
Lee Nipper56af8cd2009-03-29 15:50:50 +08001347 struct talitos_edesc *edesc;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001348 struct scatterlist *sg;
1349 void *icvdata;
1350
1351 req->cryptlen -= authsize;
1352
1353 /* allocate extended descriptor */
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001354 edesc = aead_edesc_alloc(req, 1);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001355 if (IS_ERR(edesc))
1356 return PTR_ERR(edesc);
1357
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001358 if ((priv->features & TALITOS_FTR_HW_AUTH_CHECK) &&
Kim Phillipse938e462009-03-29 15:53:23 +08001359 ((!edesc->src_nents && !edesc->dst_nents) ||
1360 priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) {
Kim Phillips9c4a7962008-06-23 19:50:15 +08001361
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001362 /* decrypt and check the ICV */
Kim Phillipse938e462009-03-29 15:53:23 +08001363 edesc->desc.hdr = ctx->desc_hdr_template |
1364 DESC_HDR_DIR_INBOUND |
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001365 DESC_HDR_MODE1_MDEU_CICV;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001366
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001367 /* reset integrity check result bits */
1368 edesc->desc.hdr_lo = 0;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001369
Kim Phillipse938e462009-03-29 15:53:23 +08001370 return ipsec_esp(edesc, req, NULL, 0,
1371 ipsec_esp_decrypt_hwauth_done);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001372
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001373 }
Kim Phillipse938e462009-03-29 15:53:23 +08001374
1375 /* Have to check the ICV with software */
1376 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1377
1378 /* stash incoming ICV for later cmp with ICV generated by the h/w */
1379 if (edesc->dma_len)
1380 icvdata = &edesc->link_tbl[edesc->src_nents +
1381 edesc->dst_nents + 2];
1382 else
1383 icvdata = &edesc->link_tbl[0];
1384
1385 sg = sg_last(req->src, edesc->src_nents ? : 1);
1386
1387 memcpy(icvdata, (char *)sg_virt(sg) + sg->length - ctx->authsize,
1388 ctx->authsize);
1389
1390 return ipsec_esp(edesc, req, NULL, 0, ipsec_esp_decrypt_swauth_done);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001391}
1392
Lee Nipper56af8cd2009-03-29 15:50:50 +08001393static int aead_givencrypt(struct aead_givcrypt_request *req)
Kim Phillips9c4a7962008-06-23 19:50:15 +08001394{
1395 struct aead_request *areq = &req->areq;
1396 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1397 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
Lee Nipper56af8cd2009-03-29 15:50:50 +08001398 struct talitos_edesc *edesc;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001399
1400 /* allocate extended descriptor */
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001401 edesc = aead_edesc_alloc(areq, 0);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001402 if (IS_ERR(edesc))
1403 return PTR_ERR(edesc);
1404
1405 /* set encrypt */
Lee Nipper70bcaca2008-07-03 19:08:46 +08001406 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001407
1408 memcpy(req->giv, ctx->iv, crypto_aead_ivsize(authenc));
Kim Phillipsba954872008-09-14 13:41:19 -07001409 /* avoid consecutive packets going out with same IV */
1410 *(__be64 *)req->giv ^= cpu_to_be64(req->seq);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001411
1412 return ipsec_esp(edesc, areq, req->giv, req->seq,
1413 ipsec_esp_encrypt_done);
1414}
1415
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001416static int ablkcipher_setkey(struct crypto_ablkcipher *cipher,
1417 const u8 *key, unsigned int keylen)
1418{
1419 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001420
1421 memcpy(&ctx->key, key, keylen);
1422 ctx->keylen = keylen;
1423
1424 return 0;
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001425}
1426
1427static void common_nonsnoop_unmap(struct device *dev,
1428 struct talitos_edesc *edesc,
1429 struct ablkcipher_request *areq)
1430{
1431 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1432 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
1433 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE);
1434
1435 talitos_sg_unmap(dev, edesc, areq->src, areq->dst);
1436
1437 if (edesc->dma_len)
1438 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1439 DMA_BIDIRECTIONAL);
1440}
1441
1442static void ablkcipher_done(struct device *dev,
1443 struct talitos_desc *desc, void *context,
1444 int err)
1445{
1446 struct ablkcipher_request *areq = context;
Kim Phillips19bbbc62009-03-29 15:53:59 +08001447 struct talitos_edesc *edesc;
1448
1449 edesc = container_of(desc, struct talitos_edesc, desc);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001450
1451 common_nonsnoop_unmap(dev, edesc, areq);
1452
1453 kfree(edesc);
1454
1455 areq->base.complete(&areq->base, err);
1456}
1457
1458static int common_nonsnoop(struct talitos_edesc *edesc,
1459 struct ablkcipher_request *areq,
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001460 void (*callback) (struct device *dev,
1461 struct talitos_desc *desc,
1462 void *context, int error))
1463{
1464 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1465 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1466 struct device *dev = ctx->dev;
1467 struct talitos_desc *desc = &edesc->desc;
1468 unsigned int cryptlen = areq->nbytes;
1469 unsigned int ivsize;
1470 int sg_count, ret;
1471
1472 /* first DWORD empty */
1473 desc->ptr[0].len = 0;
Kim Phillips81eb0242009-08-13 11:51:51 +10001474 to_talitos_ptr(&desc->ptr[0], 0);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001475 desc->ptr[0].j_extent = 0;
1476
1477 /* cipher iv */
1478 ivsize = crypto_ablkcipher_ivsize(cipher);
Kim Phillipsfebec542011-07-15 11:21:39 +08001479 map_single_talitos_ptr(dev, &desc->ptr[1], ivsize, areq->info, 0,
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001480 DMA_TO_DEVICE);
1481
1482 /* cipher key */
1483 map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
1484 (char *)&ctx->key, 0, DMA_TO_DEVICE);
1485
1486 /*
1487 * cipher in
1488 */
1489 desc->ptr[3].len = cpu_to_be16(cryptlen);
1490 desc->ptr[3].j_extent = 0;
1491
1492 sg_count = talitos_map_sg(dev, areq->src, edesc->src_nents ? : 1,
1493 (areq->src == areq->dst) ? DMA_BIDIRECTIONAL
1494 : DMA_TO_DEVICE,
1495 edesc->src_is_chained);
1496
1497 if (sg_count == 1) {
Kim Phillips81eb0242009-08-13 11:51:51 +10001498 to_talitos_ptr(&desc->ptr[3], sg_dma_address(areq->src));
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001499 } else {
1500 sg_count = sg_to_link_tbl(areq->src, sg_count, cryptlen,
1501 &edesc->link_tbl[0]);
1502 if (sg_count > 1) {
Kim Phillips81eb0242009-08-13 11:51:51 +10001503 to_talitos_ptr(&desc->ptr[3], edesc->dma_link_tbl);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001504 desc->ptr[3].j_extent |= DESC_PTR_LNKTBL_JUMP;
Kim Phillipse938e462009-03-29 15:53:23 +08001505 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1506 edesc->dma_len,
1507 DMA_BIDIRECTIONAL);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001508 } else {
1509 /* Only one segment now, so no link tbl needed */
Kim Phillips81eb0242009-08-13 11:51:51 +10001510 to_talitos_ptr(&desc->ptr[3],
1511 sg_dma_address(areq->src));
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001512 }
1513 }
1514
1515 /* cipher out */
1516 desc->ptr[4].len = cpu_to_be16(cryptlen);
1517 desc->ptr[4].j_extent = 0;
1518
1519 if (areq->src != areq->dst)
1520 sg_count = talitos_map_sg(dev, areq->dst,
1521 edesc->dst_nents ? : 1,
1522 DMA_FROM_DEVICE,
1523 edesc->dst_is_chained);
1524
1525 if (sg_count == 1) {
Kim Phillips81eb0242009-08-13 11:51:51 +10001526 to_talitos_ptr(&desc->ptr[4], sg_dma_address(areq->dst));
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001527 } else {
1528 struct talitos_ptr *link_tbl_ptr =
1529 &edesc->link_tbl[edesc->src_nents + 1];
1530
Kim Phillips81eb0242009-08-13 11:51:51 +10001531 to_talitos_ptr(&desc->ptr[4], edesc->dma_link_tbl +
1532 (edesc->src_nents + 1) *
1533 sizeof(struct talitos_ptr));
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001534 desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP;
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001535 sg_count = sg_to_link_tbl(areq->dst, sg_count, cryptlen,
1536 link_tbl_ptr);
1537 dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl,
1538 edesc->dma_len, DMA_BIDIRECTIONAL);
1539 }
1540
1541 /* iv out */
1542 map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv, 0,
1543 DMA_FROM_DEVICE);
1544
1545 /* last DWORD empty */
1546 desc->ptr[6].len = 0;
Kim Phillips81eb0242009-08-13 11:51:51 +10001547 to_talitos_ptr(&desc->ptr[6], 0);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001548 desc->ptr[6].j_extent = 0;
1549
Kim Phillips5228f0f2011-07-15 11:21:38 +08001550 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001551 if (ret != -EINPROGRESS) {
1552 common_nonsnoop_unmap(dev, edesc, areq);
1553 kfree(edesc);
1554 }
1555 return ret;
1556}
1557
Kim Phillipse938e462009-03-29 15:53:23 +08001558static struct talitos_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request *
1559 areq)
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001560{
1561 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1562 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1563
Lee Nipper497f2e62010-05-19 19:20:36 +10001564 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, 0,
1565 areq->nbytes, 0, 0, areq->base.flags);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001566}
1567
1568static int ablkcipher_encrypt(struct ablkcipher_request *areq)
1569{
1570 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1571 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1572 struct talitos_edesc *edesc;
1573
1574 /* allocate extended descriptor */
1575 edesc = ablkcipher_edesc_alloc(areq);
1576 if (IS_ERR(edesc))
1577 return PTR_ERR(edesc);
1578
1579 /* set encrypt */
1580 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1581
Kim Phillipsfebec542011-07-15 11:21:39 +08001582 return common_nonsnoop(edesc, areq, ablkcipher_done);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001583}
1584
1585static int ablkcipher_decrypt(struct ablkcipher_request *areq)
1586{
1587 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1588 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1589 struct talitos_edesc *edesc;
1590
1591 /* allocate extended descriptor */
1592 edesc = ablkcipher_edesc_alloc(areq);
1593 if (IS_ERR(edesc))
1594 return PTR_ERR(edesc);
1595
1596 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1597
Kim Phillipsfebec542011-07-15 11:21:39 +08001598 return common_nonsnoop(edesc, areq, ablkcipher_done);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001599}
1600
Lee Nipper497f2e62010-05-19 19:20:36 +10001601static void common_nonsnoop_hash_unmap(struct device *dev,
1602 struct talitos_edesc *edesc,
1603 struct ahash_request *areq)
1604{
1605 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1606
1607 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1608
1609 /* When using hashctx-in, must unmap it. */
1610 if (edesc->desc.ptr[1].len)
1611 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1],
1612 DMA_TO_DEVICE);
1613
1614 if (edesc->desc.ptr[2].len)
1615 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2],
1616 DMA_TO_DEVICE);
1617
1618 talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL);
1619
1620 if (edesc->dma_len)
1621 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1622 DMA_BIDIRECTIONAL);
1623
1624}
1625
1626static void ahash_done(struct device *dev,
1627 struct talitos_desc *desc, void *context,
1628 int err)
1629{
1630 struct ahash_request *areq = context;
1631 struct talitos_edesc *edesc =
1632 container_of(desc, struct talitos_edesc, desc);
1633 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1634
1635 if (!req_ctx->last && req_ctx->to_hash_later) {
1636 /* Position any partial block for next update/final/finup */
1637 memcpy(req_ctx->buf, req_ctx->bufnext, req_ctx->to_hash_later);
Lee Nipper5e833bc2010-06-16 15:29:15 +10001638 req_ctx->nbuf = req_ctx->to_hash_later;
Lee Nipper497f2e62010-05-19 19:20:36 +10001639 }
1640 common_nonsnoop_hash_unmap(dev, edesc, areq);
1641
1642 kfree(edesc);
1643
1644 areq->base.complete(&areq->base, err);
1645}
1646
1647static int common_nonsnoop_hash(struct talitos_edesc *edesc,
1648 struct ahash_request *areq, unsigned int length,
1649 void (*callback) (struct device *dev,
1650 struct talitos_desc *desc,
1651 void *context, int error))
1652{
1653 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1654 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1655 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1656 struct device *dev = ctx->dev;
1657 struct talitos_desc *desc = &edesc->desc;
1658 int sg_count, ret;
1659
1660 /* first DWORD empty */
1661 desc->ptr[0] = zero_entry;
1662
Kim Phillips60f208d72010-05-19 19:21:53 +10001663 /* hash context in */
1664 if (!req_ctx->first || req_ctx->swinit) {
Lee Nipper497f2e62010-05-19 19:20:36 +10001665 map_single_talitos_ptr(dev, &desc->ptr[1],
1666 req_ctx->hw_context_size,
1667 (char *)req_ctx->hw_context, 0,
1668 DMA_TO_DEVICE);
Kim Phillips60f208d72010-05-19 19:21:53 +10001669 req_ctx->swinit = 0;
Lee Nipper497f2e62010-05-19 19:20:36 +10001670 } else {
1671 desc->ptr[1] = zero_entry;
1672 /* Indicate next op is not the first. */
1673 req_ctx->first = 0;
1674 }
1675
1676 /* HMAC key */
1677 if (ctx->keylen)
1678 map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
1679 (char *)&ctx->key, 0, DMA_TO_DEVICE);
1680 else
1681 desc->ptr[2] = zero_entry;
1682
1683 /*
1684 * data in
1685 */
1686 desc->ptr[3].len = cpu_to_be16(length);
1687 desc->ptr[3].j_extent = 0;
1688
1689 sg_count = talitos_map_sg(dev, req_ctx->psrc,
1690 edesc->src_nents ? : 1,
1691 DMA_TO_DEVICE,
1692 edesc->src_is_chained);
1693
1694 if (sg_count == 1) {
1695 to_talitos_ptr(&desc->ptr[3], sg_dma_address(req_ctx->psrc));
1696 } else {
1697 sg_count = sg_to_link_tbl(req_ctx->psrc, sg_count, length,
1698 &edesc->link_tbl[0]);
1699 if (sg_count > 1) {
1700 desc->ptr[3].j_extent |= DESC_PTR_LNKTBL_JUMP;
1701 to_talitos_ptr(&desc->ptr[3], edesc->dma_link_tbl);
1702 dma_sync_single_for_device(ctx->dev,
1703 edesc->dma_link_tbl,
1704 edesc->dma_len,
1705 DMA_BIDIRECTIONAL);
1706 } else {
1707 /* Only one segment now, so no link tbl needed */
1708 to_talitos_ptr(&desc->ptr[3],
1709 sg_dma_address(req_ctx->psrc));
1710 }
1711 }
1712
1713 /* fifth DWORD empty */
1714 desc->ptr[4] = zero_entry;
1715
1716 /* hash/HMAC out -or- hash context out */
1717 if (req_ctx->last)
1718 map_single_talitos_ptr(dev, &desc->ptr[5],
1719 crypto_ahash_digestsize(tfm),
1720 areq->result, 0, DMA_FROM_DEVICE);
1721 else
1722 map_single_talitos_ptr(dev, &desc->ptr[5],
1723 req_ctx->hw_context_size,
1724 req_ctx->hw_context, 0, DMA_FROM_DEVICE);
1725
1726 /* last DWORD empty */
1727 desc->ptr[6] = zero_entry;
1728
Kim Phillips5228f0f2011-07-15 11:21:38 +08001729 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
Lee Nipper497f2e62010-05-19 19:20:36 +10001730 if (ret != -EINPROGRESS) {
1731 common_nonsnoop_hash_unmap(dev, edesc, areq);
1732 kfree(edesc);
1733 }
1734 return ret;
1735}
1736
1737static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq,
1738 unsigned int nbytes)
1739{
1740 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1741 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1742 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1743
1744 return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, 1,
1745 nbytes, 0, 0, areq->base.flags);
1746}
1747
1748static int ahash_init(struct ahash_request *areq)
1749{
1750 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1751 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1752
1753 /* Initialize the context */
Lee Nipper5e833bc2010-06-16 15:29:15 +10001754 req_ctx->nbuf = 0;
Kim Phillips60f208d72010-05-19 19:21:53 +10001755 req_ctx->first = 1; /* first indicates h/w must init its context */
1756 req_ctx->swinit = 0; /* assume h/w init of context */
Lee Nipper497f2e62010-05-19 19:20:36 +10001757 req_ctx->hw_context_size =
1758 (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
1759 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
1760 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
1761
1762 return 0;
1763}
1764
Kim Phillips60f208d72010-05-19 19:21:53 +10001765/*
1766 * on h/w without explicit sha224 support, we initialize h/w context
1767 * manually with sha224 constants, and tell it to run sha256.
1768 */
1769static int ahash_init_sha224_swinit(struct ahash_request *areq)
1770{
1771 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1772
1773 ahash_init(areq);
1774 req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/
1775
Kim Phillipsa7524472010-09-23 15:56:38 +08001776 req_ctx->hw_context[0] = SHA224_H0;
1777 req_ctx->hw_context[1] = SHA224_H1;
1778 req_ctx->hw_context[2] = SHA224_H2;
1779 req_ctx->hw_context[3] = SHA224_H3;
1780 req_ctx->hw_context[4] = SHA224_H4;
1781 req_ctx->hw_context[5] = SHA224_H5;
1782 req_ctx->hw_context[6] = SHA224_H6;
1783 req_ctx->hw_context[7] = SHA224_H7;
Kim Phillips60f208d72010-05-19 19:21:53 +10001784
1785 /* init 64-bit count */
1786 req_ctx->hw_context[8] = 0;
1787 req_ctx->hw_context[9] = 0;
1788
1789 return 0;
1790}
1791
Lee Nipper497f2e62010-05-19 19:20:36 +10001792static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
1793{
1794 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1795 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1796 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1797 struct talitos_edesc *edesc;
1798 unsigned int blocksize =
1799 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1800 unsigned int nbytes_to_hash;
1801 unsigned int to_hash_later;
Lee Nipper5e833bc2010-06-16 15:29:15 +10001802 unsigned int nsg;
Lee Nipper497f2e62010-05-19 19:20:36 +10001803 int chained;
1804
Lee Nipper5e833bc2010-06-16 15:29:15 +10001805 if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) {
1806 /* Buffer up to one whole block */
Lee Nipper497f2e62010-05-19 19:20:36 +10001807 sg_copy_to_buffer(areq->src,
1808 sg_count(areq->src, nbytes, &chained),
Lee Nipper5e833bc2010-06-16 15:29:15 +10001809 req_ctx->buf + req_ctx->nbuf, nbytes);
1810 req_ctx->nbuf += nbytes;
Lee Nipper497f2e62010-05-19 19:20:36 +10001811 return 0;
1812 }
1813
Lee Nipper5e833bc2010-06-16 15:29:15 +10001814 /* At least (blocksize + 1) bytes are available to hash */
1815 nbytes_to_hash = nbytes + req_ctx->nbuf;
1816 to_hash_later = nbytes_to_hash & (blocksize - 1);
1817
1818 if (req_ctx->last)
1819 to_hash_later = 0;
1820 else if (to_hash_later)
1821 /* There is a partial block. Hash the full block(s) now */
1822 nbytes_to_hash -= to_hash_later;
1823 else {
1824 /* Keep one block buffered */
1825 nbytes_to_hash -= blocksize;
1826 to_hash_later = blocksize;
1827 }
1828
1829 /* Chain in any previously buffered data */
1830 if (req_ctx->nbuf) {
1831 nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1;
1832 sg_init_table(req_ctx->bufsl, nsg);
1833 sg_set_buf(req_ctx->bufsl, req_ctx->buf, req_ctx->nbuf);
1834 if (nsg > 1)
1835 scatterwalk_sg_chain(req_ctx->bufsl, 2, areq->src);
Lee Nipper497f2e62010-05-19 19:20:36 +10001836 req_ctx->psrc = req_ctx->bufsl;
Lee Nipper5e833bc2010-06-16 15:29:15 +10001837 } else
Lee Nipper497f2e62010-05-19 19:20:36 +10001838 req_ctx->psrc = areq->src;
Lee Nipper497f2e62010-05-19 19:20:36 +10001839
Lee Nipper5e833bc2010-06-16 15:29:15 +10001840 if (to_hash_later) {
1841 int nents = sg_count(areq->src, nbytes, &chained);
1842 sg_copy_end_to_buffer(areq->src, nents,
1843 req_ctx->bufnext,
1844 to_hash_later,
1845 nbytes - to_hash_later);
Lee Nipper497f2e62010-05-19 19:20:36 +10001846 }
Lee Nipper5e833bc2010-06-16 15:29:15 +10001847 req_ctx->to_hash_later = to_hash_later;
Lee Nipper497f2e62010-05-19 19:20:36 +10001848
Lee Nipper5e833bc2010-06-16 15:29:15 +10001849 /* Allocate extended descriptor */
Lee Nipper497f2e62010-05-19 19:20:36 +10001850 edesc = ahash_edesc_alloc(areq, nbytes_to_hash);
1851 if (IS_ERR(edesc))
1852 return PTR_ERR(edesc);
1853
1854 edesc->desc.hdr = ctx->desc_hdr_template;
1855
1856 /* On last one, request SEC to pad; otherwise continue */
1857 if (req_ctx->last)
1858 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;
1859 else
1860 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;
1861
Kim Phillips60f208d72010-05-19 19:21:53 +10001862 /* request SEC to INIT hash. */
1863 if (req_ctx->first && !req_ctx->swinit)
Lee Nipper497f2e62010-05-19 19:20:36 +10001864 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;
1865
1866 /* When the tfm context has a keylen, it's an HMAC.
1867 * A first or last (ie. not middle) descriptor must request HMAC.
1868 */
1869 if (ctx->keylen && (req_ctx->first || req_ctx->last))
1870 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;
1871
1872 return common_nonsnoop_hash(edesc, areq, nbytes_to_hash,
1873 ahash_done);
1874}
1875
1876static int ahash_update(struct ahash_request *areq)
1877{
1878 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1879
1880 req_ctx->last = 0;
1881
1882 return ahash_process_req(areq, areq->nbytes);
1883}
1884
1885static int ahash_final(struct ahash_request *areq)
1886{
1887 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1888
1889 req_ctx->last = 1;
1890
1891 return ahash_process_req(areq, 0);
1892}
1893
1894static int ahash_finup(struct ahash_request *areq)
1895{
1896 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1897
1898 req_ctx->last = 1;
1899
1900 return ahash_process_req(areq, areq->nbytes);
1901}
1902
1903static int ahash_digest(struct ahash_request *areq)
1904{
1905 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
Kim Phillips60f208d72010-05-19 19:21:53 +10001906 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
Lee Nipper497f2e62010-05-19 19:20:36 +10001907
Kim Phillips60f208d72010-05-19 19:21:53 +10001908 ahash->init(areq);
Lee Nipper497f2e62010-05-19 19:20:36 +10001909 req_ctx->last = 1;
1910
1911 return ahash_process_req(areq, areq->nbytes);
1912}
1913
Lee Nipper79b3a412011-11-21 16:13:25 +08001914struct keyhash_result {
1915 struct completion completion;
1916 int err;
1917};
1918
1919static void keyhash_complete(struct crypto_async_request *req, int err)
1920{
1921 struct keyhash_result *res = req->data;
1922
1923 if (err == -EINPROGRESS)
1924 return;
1925
1926 res->err = err;
1927 complete(&res->completion);
1928}
1929
1930static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen,
1931 u8 *hash)
1932{
1933 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
1934
1935 struct scatterlist sg[1];
1936 struct ahash_request *req;
1937 struct keyhash_result hresult;
1938 int ret;
1939
1940 init_completion(&hresult.completion);
1941
1942 req = ahash_request_alloc(tfm, GFP_KERNEL);
1943 if (!req)
1944 return -ENOMEM;
1945
1946 /* Keep tfm keylen == 0 during hash of the long key */
1947 ctx->keylen = 0;
1948 ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
1949 keyhash_complete, &hresult);
1950
1951 sg_init_one(&sg[0], key, keylen);
1952
1953 ahash_request_set_crypt(req, sg, hash, keylen);
1954 ret = crypto_ahash_digest(req);
1955 switch (ret) {
1956 case 0:
1957 break;
1958 case -EINPROGRESS:
1959 case -EBUSY:
1960 ret = wait_for_completion_interruptible(
1961 &hresult.completion);
1962 if (!ret)
1963 ret = hresult.err;
1964 break;
1965 default:
1966 break;
1967 }
1968 ahash_request_free(req);
1969
1970 return ret;
1971}
1972
1973static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
1974 unsigned int keylen)
1975{
1976 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
1977 unsigned int blocksize =
1978 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1979 unsigned int digestsize = crypto_ahash_digestsize(tfm);
1980 unsigned int keysize = keylen;
1981 u8 hash[SHA512_DIGEST_SIZE];
1982 int ret;
1983
1984 if (keylen <= blocksize)
1985 memcpy(ctx->key, key, keysize);
1986 else {
1987 /* Must get the hash of the long key */
1988 ret = keyhash(tfm, key, keylen, hash);
1989
1990 if (ret) {
1991 crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
1992 return -EINVAL;
1993 }
1994
1995 keysize = digestsize;
1996 memcpy(ctx->key, hash, digestsize);
1997 }
1998
1999 ctx->keylen = keysize;
2000
2001 return 0;
2002}
2003
2004
Kim Phillips9c4a7962008-06-23 19:50:15 +08002005struct talitos_alg_template {
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002006 u32 type;
2007 union {
2008 struct crypto_alg crypto;
Lee Nipperacbf7c622010-05-19 19:19:33 +10002009 struct ahash_alg hash;
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002010 } alg;
Kim Phillips9c4a7962008-06-23 19:50:15 +08002011 __be32 desc_hdr_template;
2012};
2013
2014static struct talitos_alg_template driver_algs[] = {
Lee Nipper56af8cd2009-03-29 15:50:50 +08002015 /* AEAD algorithms. These use a single-pass ipsec_esp descriptor */
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002016 { .type = CRYPTO_ALG_TYPE_AEAD,
2017 .alg.crypto = {
Lee Nipper56af8cd2009-03-29 15:50:50 +08002018 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2019 .cra_driver_name = "authenc-hmac-sha1-cbc-aes-talitos",
2020 .cra_blocksize = AES_BLOCK_SIZE,
2021 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2022 .cra_type = &crypto_aead_type,
2023 .cra_aead = {
2024 .setkey = aead_setkey,
2025 .setauthsize = aead_setauthsize,
2026 .encrypt = aead_encrypt,
2027 .decrypt = aead_decrypt,
2028 .givencrypt = aead_givencrypt,
2029 .geniv = "<built-in>",
2030 .ivsize = AES_BLOCK_SIZE,
2031 .maxauthsize = SHA1_DIGEST_SIZE,
2032 }
2033 },
Kim Phillips9c4a7962008-06-23 19:50:15 +08002034 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2035 DESC_HDR_SEL0_AESU |
2036 DESC_HDR_MODE0_AESU_CBC |
2037 DESC_HDR_SEL1_MDEUA |
2038 DESC_HDR_MODE1_MDEU_INIT |
2039 DESC_HDR_MODE1_MDEU_PAD |
2040 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
Lee Nipper70bcaca2008-07-03 19:08:46 +08002041 },
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002042 { .type = CRYPTO_ALG_TYPE_AEAD,
2043 .alg.crypto = {
Lee Nipper56af8cd2009-03-29 15:50:50 +08002044 .cra_name = "authenc(hmac(sha1),cbc(des3_ede))",
2045 .cra_driver_name = "authenc-hmac-sha1-cbc-3des-talitos",
2046 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2047 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2048 .cra_type = &crypto_aead_type,
2049 .cra_aead = {
2050 .setkey = aead_setkey,
2051 .setauthsize = aead_setauthsize,
2052 .encrypt = aead_encrypt,
2053 .decrypt = aead_decrypt,
2054 .givencrypt = aead_givencrypt,
2055 .geniv = "<built-in>",
2056 .ivsize = DES3_EDE_BLOCK_SIZE,
2057 .maxauthsize = SHA1_DIGEST_SIZE,
2058 }
2059 },
Lee Nipper70bcaca2008-07-03 19:08:46 +08002060 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2061 DESC_HDR_SEL0_DEU |
2062 DESC_HDR_MODE0_DEU_CBC |
2063 DESC_HDR_MODE0_DEU_3DES |
2064 DESC_HDR_SEL1_MDEUA |
2065 DESC_HDR_MODE1_MDEU_INIT |
2066 DESC_HDR_MODE1_MDEU_PAD |
2067 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
Lee Nipper3952f172008-07-10 18:29:18 +08002068 },
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002069 { .type = CRYPTO_ALG_TYPE_AEAD,
2070 .alg.crypto = {
Lee Nipper56af8cd2009-03-29 15:50:50 +08002071 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2072 .cra_driver_name = "authenc-hmac-sha256-cbc-aes-talitos",
2073 .cra_blocksize = AES_BLOCK_SIZE,
2074 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2075 .cra_type = &crypto_aead_type,
2076 .cra_aead = {
2077 .setkey = aead_setkey,
2078 .setauthsize = aead_setauthsize,
2079 .encrypt = aead_encrypt,
2080 .decrypt = aead_decrypt,
2081 .givencrypt = aead_givencrypt,
2082 .geniv = "<built-in>",
2083 .ivsize = AES_BLOCK_SIZE,
2084 .maxauthsize = SHA256_DIGEST_SIZE,
2085 }
2086 },
Lee Nipper3952f172008-07-10 18:29:18 +08002087 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2088 DESC_HDR_SEL0_AESU |
2089 DESC_HDR_MODE0_AESU_CBC |
2090 DESC_HDR_SEL1_MDEUA |
2091 DESC_HDR_MODE1_MDEU_INIT |
2092 DESC_HDR_MODE1_MDEU_PAD |
2093 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2094 },
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002095 { .type = CRYPTO_ALG_TYPE_AEAD,
2096 .alg.crypto = {
Lee Nipper56af8cd2009-03-29 15:50:50 +08002097 .cra_name = "authenc(hmac(sha256),cbc(des3_ede))",
2098 .cra_driver_name = "authenc-hmac-sha256-cbc-3des-talitos",
2099 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2100 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2101 .cra_type = &crypto_aead_type,
2102 .cra_aead = {
2103 .setkey = aead_setkey,
2104 .setauthsize = aead_setauthsize,
2105 .encrypt = aead_encrypt,
2106 .decrypt = aead_decrypt,
2107 .givencrypt = aead_givencrypt,
2108 .geniv = "<built-in>",
2109 .ivsize = DES3_EDE_BLOCK_SIZE,
2110 .maxauthsize = SHA256_DIGEST_SIZE,
2111 }
2112 },
Lee Nipper3952f172008-07-10 18:29:18 +08002113 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2114 DESC_HDR_SEL0_DEU |
2115 DESC_HDR_MODE0_DEU_CBC |
2116 DESC_HDR_MODE0_DEU_3DES |
2117 DESC_HDR_SEL1_MDEUA |
2118 DESC_HDR_MODE1_MDEU_INIT |
2119 DESC_HDR_MODE1_MDEU_PAD |
2120 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2121 },
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002122 { .type = CRYPTO_ALG_TYPE_AEAD,
2123 .alg.crypto = {
Lee Nipper56af8cd2009-03-29 15:50:50 +08002124 .cra_name = "authenc(hmac(md5),cbc(aes))",
2125 .cra_driver_name = "authenc-hmac-md5-cbc-aes-talitos",
2126 .cra_blocksize = AES_BLOCK_SIZE,
2127 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2128 .cra_type = &crypto_aead_type,
2129 .cra_aead = {
2130 .setkey = aead_setkey,
2131 .setauthsize = aead_setauthsize,
2132 .encrypt = aead_encrypt,
2133 .decrypt = aead_decrypt,
2134 .givencrypt = aead_givencrypt,
2135 .geniv = "<built-in>",
2136 .ivsize = AES_BLOCK_SIZE,
2137 .maxauthsize = MD5_DIGEST_SIZE,
2138 }
2139 },
Lee Nipper3952f172008-07-10 18:29:18 +08002140 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2141 DESC_HDR_SEL0_AESU |
2142 DESC_HDR_MODE0_AESU_CBC |
2143 DESC_HDR_SEL1_MDEUA |
2144 DESC_HDR_MODE1_MDEU_INIT |
2145 DESC_HDR_MODE1_MDEU_PAD |
2146 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2147 },
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002148 { .type = CRYPTO_ALG_TYPE_AEAD,
2149 .alg.crypto = {
Lee Nipper56af8cd2009-03-29 15:50:50 +08002150 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2151 .cra_driver_name = "authenc-hmac-md5-cbc-3des-talitos",
2152 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2153 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2154 .cra_type = &crypto_aead_type,
2155 .cra_aead = {
2156 .setkey = aead_setkey,
2157 .setauthsize = aead_setauthsize,
2158 .encrypt = aead_encrypt,
2159 .decrypt = aead_decrypt,
2160 .givencrypt = aead_givencrypt,
2161 .geniv = "<built-in>",
2162 .ivsize = DES3_EDE_BLOCK_SIZE,
2163 .maxauthsize = MD5_DIGEST_SIZE,
2164 }
2165 },
Lee Nipper3952f172008-07-10 18:29:18 +08002166 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2167 DESC_HDR_SEL0_DEU |
2168 DESC_HDR_MODE0_DEU_CBC |
2169 DESC_HDR_MODE0_DEU_3DES |
2170 DESC_HDR_SEL1_MDEUA |
2171 DESC_HDR_MODE1_MDEU_INIT |
2172 DESC_HDR_MODE1_MDEU_PAD |
2173 DESC_HDR_MODE1_MDEU_MD5_HMAC,
Lee Nipper4de9d0b2009-03-29 15:52:32 +08002174 },
2175 /* ABLKCIPHER algorithms. */
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002176 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2177 .alg.crypto = {
Lee Nipper4de9d0b2009-03-29 15:52:32 +08002178 .cra_name = "cbc(aes)",
2179 .cra_driver_name = "cbc-aes-talitos",
2180 .cra_blocksize = AES_BLOCK_SIZE,
2181 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2182 CRYPTO_ALG_ASYNC,
2183 .cra_type = &crypto_ablkcipher_type,
2184 .cra_ablkcipher = {
2185 .setkey = ablkcipher_setkey,
2186 .encrypt = ablkcipher_encrypt,
2187 .decrypt = ablkcipher_decrypt,
2188 .geniv = "eseqiv",
2189 .min_keysize = AES_MIN_KEY_SIZE,
2190 .max_keysize = AES_MAX_KEY_SIZE,
2191 .ivsize = AES_BLOCK_SIZE,
2192 }
2193 },
2194 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2195 DESC_HDR_SEL0_AESU |
2196 DESC_HDR_MODE0_AESU_CBC,
2197 },
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002198 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2199 .alg.crypto = {
Lee Nipper4de9d0b2009-03-29 15:52:32 +08002200 .cra_name = "cbc(des3_ede)",
2201 .cra_driver_name = "cbc-3des-talitos",
2202 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2203 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2204 CRYPTO_ALG_ASYNC,
2205 .cra_type = &crypto_ablkcipher_type,
2206 .cra_ablkcipher = {
2207 .setkey = ablkcipher_setkey,
2208 .encrypt = ablkcipher_encrypt,
2209 .decrypt = ablkcipher_decrypt,
2210 .geniv = "eseqiv",
2211 .min_keysize = DES3_EDE_KEY_SIZE,
2212 .max_keysize = DES3_EDE_KEY_SIZE,
2213 .ivsize = DES3_EDE_BLOCK_SIZE,
2214 }
2215 },
2216 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2217 DESC_HDR_SEL0_DEU |
2218 DESC_HDR_MODE0_DEU_CBC |
2219 DESC_HDR_MODE0_DEU_3DES,
Lee Nipper497f2e62010-05-19 19:20:36 +10002220 },
2221 /* AHASH algorithms. */
2222 { .type = CRYPTO_ALG_TYPE_AHASH,
2223 .alg.hash = {
2224 .init = ahash_init,
2225 .update = ahash_update,
2226 .final = ahash_final,
2227 .finup = ahash_finup,
2228 .digest = ahash_digest,
2229 .halg.digestsize = MD5_DIGEST_SIZE,
2230 .halg.base = {
2231 .cra_name = "md5",
2232 .cra_driver_name = "md5-talitos",
2233 .cra_blocksize = MD5_BLOCK_SIZE,
2234 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2235 CRYPTO_ALG_ASYNC,
2236 .cra_type = &crypto_ahash_type
2237 }
2238 },
2239 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2240 DESC_HDR_SEL0_MDEUA |
2241 DESC_HDR_MODE0_MDEU_MD5,
2242 },
2243 { .type = CRYPTO_ALG_TYPE_AHASH,
2244 .alg.hash = {
2245 .init = ahash_init,
2246 .update = ahash_update,
2247 .final = ahash_final,
2248 .finup = ahash_finup,
2249 .digest = ahash_digest,
2250 .halg.digestsize = SHA1_DIGEST_SIZE,
2251 .halg.base = {
2252 .cra_name = "sha1",
2253 .cra_driver_name = "sha1-talitos",
2254 .cra_blocksize = SHA1_BLOCK_SIZE,
2255 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2256 CRYPTO_ALG_ASYNC,
2257 .cra_type = &crypto_ahash_type
2258 }
2259 },
2260 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2261 DESC_HDR_SEL0_MDEUA |
2262 DESC_HDR_MODE0_MDEU_SHA1,
2263 },
2264 { .type = CRYPTO_ALG_TYPE_AHASH,
2265 .alg.hash = {
2266 .init = ahash_init,
2267 .update = ahash_update,
2268 .final = ahash_final,
2269 .finup = ahash_finup,
2270 .digest = ahash_digest,
Kim Phillips60f208d72010-05-19 19:21:53 +10002271 .halg.digestsize = SHA224_DIGEST_SIZE,
2272 .halg.base = {
2273 .cra_name = "sha224",
2274 .cra_driver_name = "sha224-talitos",
2275 .cra_blocksize = SHA224_BLOCK_SIZE,
2276 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2277 CRYPTO_ALG_ASYNC,
2278 .cra_type = &crypto_ahash_type
2279 }
2280 },
2281 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2282 DESC_HDR_SEL0_MDEUA |
2283 DESC_HDR_MODE0_MDEU_SHA224,
2284 },
2285 { .type = CRYPTO_ALG_TYPE_AHASH,
2286 .alg.hash = {
2287 .init = ahash_init,
2288 .update = ahash_update,
2289 .final = ahash_final,
2290 .finup = ahash_finup,
2291 .digest = ahash_digest,
Lee Nipper497f2e62010-05-19 19:20:36 +10002292 .halg.digestsize = SHA256_DIGEST_SIZE,
2293 .halg.base = {
2294 .cra_name = "sha256",
2295 .cra_driver_name = "sha256-talitos",
2296 .cra_blocksize = SHA256_BLOCK_SIZE,
2297 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2298 CRYPTO_ALG_ASYNC,
2299 .cra_type = &crypto_ahash_type
2300 }
2301 },
2302 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2303 DESC_HDR_SEL0_MDEUA |
2304 DESC_HDR_MODE0_MDEU_SHA256,
2305 },
2306 { .type = CRYPTO_ALG_TYPE_AHASH,
2307 .alg.hash = {
2308 .init = ahash_init,
2309 .update = ahash_update,
2310 .final = ahash_final,
2311 .finup = ahash_finup,
2312 .digest = ahash_digest,
2313 .halg.digestsize = SHA384_DIGEST_SIZE,
2314 .halg.base = {
2315 .cra_name = "sha384",
2316 .cra_driver_name = "sha384-talitos",
2317 .cra_blocksize = SHA384_BLOCK_SIZE,
2318 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2319 CRYPTO_ALG_ASYNC,
2320 .cra_type = &crypto_ahash_type
2321 }
2322 },
2323 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2324 DESC_HDR_SEL0_MDEUB |
2325 DESC_HDR_MODE0_MDEUB_SHA384,
2326 },
2327 { .type = CRYPTO_ALG_TYPE_AHASH,
2328 .alg.hash = {
2329 .init = ahash_init,
2330 .update = ahash_update,
2331 .final = ahash_final,
2332 .finup = ahash_finup,
2333 .digest = ahash_digest,
2334 .halg.digestsize = SHA512_DIGEST_SIZE,
2335 .halg.base = {
2336 .cra_name = "sha512",
2337 .cra_driver_name = "sha512-talitos",
2338 .cra_blocksize = SHA512_BLOCK_SIZE,
2339 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2340 CRYPTO_ALG_ASYNC,
2341 .cra_type = &crypto_ahash_type
2342 }
2343 },
2344 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2345 DESC_HDR_SEL0_MDEUB |
2346 DESC_HDR_MODE0_MDEUB_SHA512,
2347 },
Lee Nipper79b3a412011-11-21 16:13:25 +08002348 { .type = CRYPTO_ALG_TYPE_AHASH,
2349 .alg.hash = {
2350 .init = ahash_init,
2351 .update = ahash_update,
2352 .final = ahash_final,
2353 .finup = ahash_finup,
2354 .digest = ahash_digest,
2355 .setkey = ahash_setkey,
2356 .halg.digestsize = MD5_DIGEST_SIZE,
2357 .halg.base = {
2358 .cra_name = "hmac(md5)",
2359 .cra_driver_name = "hmac-md5-talitos",
2360 .cra_blocksize = MD5_BLOCK_SIZE,
2361 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2362 CRYPTO_ALG_ASYNC,
2363 .cra_type = &crypto_ahash_type
2364 }
2365 },
2366 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2367 DESC_HDR_SEL0_MDEUA |
2368 DESC_HDR_MODE0_MDEU_MD5,
2369 },
2370 { .type = CRYPTO_ALG_TYPE_AHASH,
2371 .alg.hash = {
2372 .init = ahash_init,
2373 .update = ahash_update,
2374 .final = ahash_final,
2375 .finup = ahash_finup,
2376 .digest = ahash_digest,
2377 .setkey = ahash_setkey,
2378 .halg.digestsize = SHA1_DIGEST_SIZE,
2379 .halg.base = {
2380 .cra_name = "hmac(sha1)",
2381 .cra_driver_name = "hmac-sha1-talitos",
2382 .cra_blocksize = SHA1_BLOCK_SIZE,
2383 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2384 CRYPTO_ALG_ASYNC,
2385 .cra_type = &crypto_ahash_type
2386 }
2387 },
2388 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2389 DESC_HDR_SEL0_MDEUA |
2390 DESC_HDR_MODE0_MDEU_SHA1,
2391 },
2392 { .type = CRYPTO_ALG_TYPE_AHASH,
2393 .alg.hash = {
2394 .init = ahash_init,
2395 .update = ahash_update,
2396 .final = ahash_final,
2397 .finup = ahash_finup,
2398 .digest = ahash_digest,
2399 .setkey = ahash_setkey,
2400 .halg.digestsize = SHA224_DIGEST_SIZE,
2401 .halg.base = {
2402 .cra_name = "hmac(sha224)",
2403 .cra_driver_name = "hmac-sha224-talitos",
2404 .cra_blocksize = SHA224_BLOCK_SIZE,
2405 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2406 CRYPTO_ALG_ASYNC,
2407 .cra_type = &crypto_ahash_type
2408 }
2409 },
2410 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2411 DESC_HDR_SEL0_MDEUA |
2412 DESC_HDR_MODE0_MDEU_SHA224,
2413 },
2414 { .type = CRYPTO_ALG_TYPE_AHASH,
2415 .alg.hash = {
2416 .init = ahash_init,
2417 .update = ahash_update,
2418 .final = ahash_final,
2419 .finup = ahash_finup,
2420 .digest = ahash_digest,
2421 .setkey = ahash_setkey,
2422 .halg.digestsize = SHA256_DIGEST_SIZE,
2423 .halg.base = {
2424 .cra_name = "hmac(sha256)",
2425 .cra_driver_name = "hmac-sha256-talitos",
2426 .cra_blocksize = SHA256_BLOCK_SIZE,
2427 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2428 CRYPTO_ALG_ASYNC,
2429 .cra_type = &crypto_ahash_type
2430 }
2431 },
2432 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2433 DESC_HDR_SEL0_MDEUA |
2434 DESC_HDR_MODE0_MDEU_SHA256,
2435 },
2436 { .type = CRYPTO_ALG_TYPE_AHASH,
2437 .alg.hash = {
2438 .init = ahash_init,
2439 .update = ahash_update,
2440 .final = ahash_final,
2441 .finup = ahash_finup,
2442 .digest = ahash_digest,
2443 .setkey = ahash_setkey,
2444 .halg.digestsize = SHA384_DIGEST_SIZE,
2445 .halg.base = {
2446 .cra_name = "hmac(sha384)",
2447 .cra_driver_name = "hmac-sha384-talitos",
2448 .cra_blocksize = SHA384_BLOCK_SIZE,
2449 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2450 CRYPTO_ALG_ASYNC,
2451 .cra_type = &crypto_ahash_type
2452 }
2453 },
2454 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2455 DESC_HDR_SEL0_MDEUB |
2456 DESC_HDR_MODE0_MDEUB_SHA384,
2457 },
2458 { .type = CRYPTO_ALG_TYPE_AHASH,
2459 .alg.hash = {
2460 .init = ahash_init,
2461 .update = ahash_update,
2462 .final = ahash_final,
2463 .finup = ahash_finup,
2464 .digest = ahash_digest,
2465 .setkey = ahash_setkey,
2466 .halg.digestsize = SHA512_DIGEST_SIZE,
2467 .halg.base = {
2468 .cra_name = "hmac(sha512)",
2469 .cra_driver_name = "hmac-sha512-talitos",
2470 .cra_blocksize = SHA512_BLOCK_SIZE,
2471 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2472 CRYPTO_ALG_ASYNC,
2473 .cra_type = &crypto_ahash_type
2474 }
2475 },
2476 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2477 DESC_HDR_SEL0_MDEUB |
2478 DESC_HDR_MODE0_MDEUB_SHA512,
2479 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08002480};
2481
2482struct talitos_crypto_alg {
2483 struct list_head entry;
2484 struct device *dev;
Lee Nipperacbf7c622010-05-19 19:19:33 +10002485 struct talitos_alg_template algt;
Kim Phillips9c4a7962008-06-23 19:50:15 +08002486};
2487
2488static int talitos_cra_init(struct crypto_tfm *tfm)
2489{
2490 struct crypto_alg *alg = tfm->__crt_alg;
Kim Phillips19bbbc62009-03-29 15:53:59 +08002491 struct talitos_crypto_alg *talitos_alg;
Kim Phillips9c4a7962008-06-23 19:50:15 +08002492 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
Kim Phillips5228f0f2011-07-15 11:21:38 +08002493 struct talitos_private *priv;
Kim Phillips9c4a7962008-06-23 19:50:15 +08002494
Lee Nipper497f2e62010-05-19 19:20:36 +10002495 if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_AHASH)
2496 talitos_alg = container_of(__crypto_ahash_alg(alg),
2497 struct talitos_crypto_alg,
2498 algt.alg.hash);
2499 else
2500 talitos_alg = container_of(alg, struct talitos_crypto_alg,
2501 algt.alg.crypto);
Kim Phillips19bbbc62009-03-29 15:53:59 +08002502
Kim Phillips9c4a7962008-06-23 19:50:15 +08002503 /* update context with ptr to dev */
2504 ctx->dev = talitos_alg->dev;
Kim Phillips19bbbc62009-03-29 15:53:59 +08002505
Kim Phillips5228f0f2011-07-15 11:21:38 +08002506 /* assign SEC channel to tfm in round-robin fashion */
2507 priv = dev_get_drvdata(ctx->dev);
2508 ctx->ch = atomic_inc_return(&priv->last_chan) &
2509 (priv->num_channels - 1);
2510
Kim Phillips9c4a7962008-06-23 19:50:15 +08002511 /* copy descriptor header template value */
Lee Nipperacbf7c622010-05-19 19:19:33 +10002512 ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template;
Kim Phillips9c4a7962008-06-23 19:50:15 +08002513
Kim Phillips602dba52011-07-15 11:21:39 +08002514 /* select done notification */
2515 ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY;
2516
Lee Nipper497f2e62010-05-19 19:20:36 +10002517 return 0;
2518}
2519
2520static int talitos_cra_init_aead(struct crypto_tfm *tfm)
2521{
2522 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2523
2524 talitos_cra_init(tfm);
Kim Phillips9c4a7962008-06-23 19:50:15 +08002525
2526 /* random first IV */
Lee Nipper70bcaca2008-07-03 19:08:46 +08002527 get_random_bytes(ctx->iv, TALITOS_MAX_IV_LENGTH);
Kim Phillips9c4a7962008-06-23 19:50:15 +08002528
2529 return 0;
2530}
2531
Lee Nipper497f2e62010-05-19 19:20:36 +10002532static int talitos_cra_init_ahash(struct crypto_tfm *tfm)
2533{
2534 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2535
2536 talitos_cra_init(tfm);
2537
2538 ctx->keylen = 0;
2539 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
2540 sizeof(struct talitos_ahash_req_ctx));
2541
2542 return 0;
2543}
2544
Kim Phillips9c4a7962008-06-23 19:50:15 +08002545/*
2546 * given the alg's descriptor header template, determine whether descriptor
2547 * type and primary/secondary execution units required match the hw
2548 * capabilities description provided in the device tree node.
2549 */
2550static int hw_supports(struct device *dev, __be32 desc_hdr_template)
2551{
2552 struct talitos_private *priv = dev_get_drvdata(dev);
2553 int ret;
2554
2555 ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) &&
2556 (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units);
2557
2558 if (SECONDARY_EU(desc_hdr_template))
2559 ret = ret && (1 << SECONDARY_EU(desc_hdr_template)
2560 & priv->exec_units);
2561
2562 return ret;
2563}
2564
Grant Likely2dc11582010-08-06 09:25:50 -06002565static int talitos_remove(struct platform_device *ofdev)
Kim Phillips9c4a7962008-06-23 19:50:15 +08002566{
2567 struct device *dev = &ofdev->dev;
2568 struct talitos_private *priv = dev_get_drvdata(dev);
2569 struct talitos_crypto_alg *t_alg, *n;
2570 int i;
2571
2572 list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
Lee Nipperacbf7c622010-05-19 19:19:33 +10002573 switch (t_alg->algt.type) {
2574 case CRYPTO_ALG_TYPE_ABLKCIPHER:
2575 case CRYPTO_ALG_TYPE_AEAD:
2576 crypto_unregister_alg(&t_alg->algt.alg.crypto);
2577 break;
2578 case CRYPTO_ALG_TYPE_AHASH:
2579 crypto_unregister_ahash(&t_alg->algt.alg.hash);
2580 break;
2581 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08002582 list_del(&t_alg->entry);
2583 kfree(t_alg);
2584 }
2585
2586 if (hw_supports(dev, DESC_HDR_SEL0_RNG))
2587 talitos_unregister_rng(dev);
2588
Kim Phillips4b9926282009-08-13 11:50:38 +10002589 for (i = 0; i < priv->num_channels; i++)
Kim Phillips0b798242010-09-23 15:56:08 +08002590 kfree(priv->chan[i].fifo);
Kim Phillips9c4a7962008-06-23 19:50:15 +08002591
Kim Phillips4b9926282009-08-13 11:50:38 +10002592 kfree(priv->chan);
Kim Phillips9c4a7962008-06-23 19:50:15 +08002593
Kim Phillipsc3e337f2011-11-21 16:13:27 +08002594 for (i = 0; i < 2; i++)
Kim Phillips2cdba3c2011-12-12 14:59:11 -06002595 if (priv->irq[i]) {
Kim Phillipsc3e337f2011-11-21 16:13:27 +08002596 free_irq(priv->irq[i], dev);
2597 irq_dispose_mapping(priv->irq[i]);
2598 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08002599
Kim Phillipsc3e337f2011-11-21 16:13:27 +08002600 tasklet_kill(&priv->done_task[0]);
Kim Phillips2cdba3c2011-12-12 14:59:11 -06002601 if (priv->irq[1])
Kim Phillipsc3e337f2011-11-21 16:13:27 +08002602 tasklet_kill(&priv->done_task[1]);
Kim Phillips9c4a7962008-06-23 19:50:15 +08002603
2604 iounmap(priv->reg);
2605
2606 dev_set_drvdata(dev, NULL);
2607
2608 kfree(priv);
2609
2610 return 0;
2611}
2612
2613static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
2614 struct talitos_alg_template
2615 *template)
2616{
Kim Phillips60f208d72010-05-19 19:21:53 +10002617 struct talitos_private *priv = dev_get_drvdata(dev);
Kim Phillips9c4a7962008-06-23 19:50:15 +08002618 struct talitos_crypto_alg *t_alg;
2619 struct crypto_alg *alg;
2620
2621 t_alg = kzalloc(sizeof(struct talitos_crypto_alg), GFP_KERNEL);
2622 if (!t_alg)
2623 return ERR_PTR(-ENOMEM);
2624
Lee Nipperacbf7c622010-05-19 19:19:33 +10002625 t_alg->algt = *template;
2626
2627 switch (t_alg->algt.type) {
2628 case CRYPTO_ALG_TYPE_ABLKCIPHER:
Lee Nipper497f2e62010-05-19 19:20:36 +10002629 alg = &t_alg->algt.alg.crypto;
2630 alg->cra_init = talitos_cra_init;
2631 break;
Lee Nipperacbf7c622010-05-19 19:19:33 +10002632 case CRYPTO_ALG_TYPE_AEAD:
2633 alg = &t_alg->algt.alg.crypto;
Lee Nipper497f2e62010-05-19 19:20:36 +10002634 alg->cra_init = talitos_cra_init_aead;
Lee Nipperacbf7c622010-05-19 19:19:33 +10002635 break;
2636 case CRYPTO_ALG_TYPE_AHASH:
2637 alg = &t_alg->algt.alg.hash.halg.base;
Lee Nipper497f2e62010-05-19 19:20:36 +10002638 alg->cra_init = talitos_cra_init_ahash;
Lee Nipper79b3a412011-11-21 16:13:25 +08002639 if (!(priv->features & TALITOS_FTR_HMAC_OK) &&
Kim Phillips0b2730d2011-12-12 14:59:10 -06002640 !strncmp(alg->cra_name, "hmac", 4)) {
2641 kfree(t_alg);
Lee Nipper79b3a412011-11-21 16:13:25 +08002642 return ERR_PTR(-ENOTSUPP);
Kim Phillips0b2730d2011-12-12 14:59:10 -06002643 }
Kim Phillips60f208d72010-05-19 19:21:53 +10002644 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
Lee Nipper79b3a412011-11-21 16:13:25 +08002645 (!strcmp(alg->cra_name, "sha224") ||
2646 !strcmp(alg->cra_name, "hmac(sha224)"))) {
Kim Phillips60f208d72010-05-19 19:21:53 +10002647 t_alg->algt.alg.hash.init = ahash_init_sha224_swinit;
2648 t_alg->algt.desc_hdr_template =
2649 DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2650 DESC_HDR_SEL0_MDEUA |
2651 DESC_HDR_MODE0_MDEU_SHA256;
2652 }
Lee Nipper497f2e62010-05-19 19:20:36 +10002653 break;
Kim Phillips1d119112010-09-23 15:55:27 +08002654 default:
2655 dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type);
2656 return ERR_PTR(-EINVAL);
Lee Nipperacbf7c622010-05-19 19:19:33 +10002657 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08002658
Kim Phillips9c4a7962008-06-23 19:50:15 +08002659 alg->cra_module = THIS_MODULE;
Kim Phillips9c4a7962008-06-23 19:50:15 +08002660 alg->cra_priority = TALITOS_CRA_PRIORITY;
Kim Phillips9c4a7962008-06-23 19:50:15 +08002661 alg->cra_alignmask = 0;
Kim Phillips9c4a7962008-06-23 19:50:15 +08002662 alg->cra_ctxsize = sizeof(struct talitos_ctx);
Nikos Mavrogiannopoulosd912bb72011-11-01 13:39:56 +01002663 alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY;
Kim Phillips9c4a7962008-06-23 19:50:15 +08002664
Kim Phillips9c4a7962008-06-23 19:50:15 +08002665 t_alg->dev = dev;
2666
2667 return t_alg;
2668}
2669
Kim Phillipsc3e337f2011-11-21 16:13:27 +08002670static int talitos_probe_irq(struct platform_device *ofdev)
2671{
2672 struct device *dev = &ofdev->dev;
2673 struct device_node *np = ofdev->dev.of_node;
2674 struct talitos_private *priv = dev_get_drvdata(dev);
2675 int err;
2676
2677 priv->irq[0] = irq_of_parse_and_map(np, 0);
Kim Phillips2cdba3c2011-12-12 14:59:11 -06002678 if (!priv->irq[0]) {
Kim Phillipsc3e337f2011-11-21 16:13:27 +08002679 dev_err(dev, "failed to map irq\n");
2680 return -EINVAL;
2681 }
2682
2683 priv->irq[1] = irq_of_parse_and_map(np, 1);
2684
2685 /* get the primary irq line */
Kim Phillips2cdba3c2011-12-12 14:59:11 -06002686 if (!priv->irq[1]) {
Kim Phillipsc3e337f2011-11-21 16:13:27 +08002687 err = request_irq(priv->irq[0], talitos_interrupt_4ch, 0,
2688 dev_driver_string(dev), dev);
2689 goto primary_out;
2690 }
2691
2692 err = request_irq(priv->irq[0], talitos_interrupt_ch0_2, 0,
2693 dev_driver_string(dev), dev);
2694 if (err)
2695 goto primary_out;
2696
2697 /* get the secondary irq line */
2698 err = request_irq(priv->irq[1], talitos_interrupt_ch1_3, 0,
2699 dev_driver_string(dev), dev);
2700 if (err) {
2701 dev_err(dev, "failed to request secondary irq\n");
2702 irq_dispose_mapping(priv->irq[1]);
Kim Phillips2cdba3c2011-12-12 14:59:11 -06002703 priv->irq[1] = 0;
Kim Phillipsc3e337f2011-11-21 16:13:27 +08002704 }
2705
2706 return err;
2707
2708primary_out:
2709 if (err) {
2710 dev_err(dev, "failed to request primary irq\n");
2711 irq_dispose_mapping(priv->irq[0]);
Kim Phillips2cdba3c2011-12-12 14:59:11 -06002712 priv->irq[0] = 0;
Kim Phillipsc3e337f2011-11-21 16:13:27 +08002713 }
2714
2715 return err;
2716}
2717
Grant Likely1c48a5c2011-02-17 02:43:24 -07002718static int talitos_probe(struct platform_device *ofdev)
Kim Phillips9c4a7962008-06-23 19:50:15 +08002719{
2720 struct device *dev = &ofdev->dev;
Grant Likely61c7a082010-04-13 16:12:29 -07002721 struct device_node *np = ofdev->dev.of_node;
Kim Phillips9c4a7962008-06-23 19:50:15 +08002722 struct talitos_private *priv;
2723 const unsigned int *prop;
2724 int i, err;
2725
2726 priv = kzalloc(sizeof(struct talitos_private), GFP_KERNEL);
2727 if (!priv)
2728 return -ENOMEM;
2729
2730 dev_set_drvdata(dev, priv);
2731
2732 priv->ofdev = ofdev;
2733
Horia Geanta511d63c2012-03-30 17:49:53 +03002734 spin_lock_init(&priv->reg_lock);
2735
Kim Phillipsc3e337f2011-11-21 16:13:27 +08002736 err = talitos_probe_irq(ofdev);
2737 if (err)
2738 goto err_out;
2739
Kim Phillips2cdba3c2011-12-12 14:59:11 -06002740 if (!priv->irq[1]) {
Kim Phillipsc3e337f2011-11-21 16:13:27 +08002741 tasklet_init(&priv->done_task[0], talitos_done_4ch,
2742 (unsigned long)dev);
2743 } else {
2744 tasklet_init(&priv->done_task[0], talitos_done_ch0_2,
2745 (unsigned long)dev);
2746 tasklet_init(&priv->done_task[1], talitos_done_ch1_3,
2747 (unsigned long)dev);
2748 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08002749
Kim Phillipsfe5720e2008-10-12 20:33:14 +08002750 INIT_LIST_HEAD(&priv->alg_list);
2751
Kim Phillips9c4a7962008-06-23 19:50:15 +08002752 priv->reg = of_iomap(np, 0);
2753 if (!priv->reg) {
2754 dev_err(dev, "failed to of_iomap\n");
2755 err = -ENOMEM;
2756 goto err_out;
2757 }
2758
2759 /* get SEC version capabilities from device tree */
2760 prop = of_get_property(np, "fsl,num-channels", NULL);
2761 if (prop)
2762 priv->num_channels = *prop;
2763
2764 prop = of_get_property(np, "fsl,channel-fifo-len", NULL);
2765 if (prop)
2766 priv->chfifo_len = *prop;
2767
2768 prop = of_get_property(np, "fsl,exec-units-mask", NULL);
2769 if (prop)
2770 priv->exec_units = *prop;
2771
2772 prop = of_get_property(np, "fsl,descriptor-types-mask", NULL);
2773 if (prop)
2774 priv->desc_types = *prop;
2775
2776 if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len ||
2777 !priv->exec_units || !priv->desc_types) {
2778 dev_err(dev, "invalid property data in device tree node\n");
2779 err = -EINVAL;
2780 goto err_out;
2781 }
2782
Lee Nipperf3c85bc2008-07-30 16:26:57 +08002783 if (of_device_is_compatible(np, "fsl,sec3.0"))
2784 priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT;
2785
Kim Phillipsfe5720e2008-10-12 20:33:14 +08002786 if (of_device_is_compatible(np, "fsl,sec2.1"))
Kim Phillips60f208d72010-05-19 19:21:53 +10002787 priv->features |= TALITOS_FTR_HW_AUTH_CHECK |
Lee Nipper79b3a412011-11-21 16:13:25 +08002788 TALITOS_FTR_SHA224_HWINIT |
2789 TALITOS_FTR_HMAC_OK;
Kim Phillipsfe5720e2008-10-12 20:33:14 +08002790
Kim Phillips4b9926282009-08-13 11:50:38 +10002791 priv->chan = kzalloc(sizeof(struct talitos_channel) *
2792 priv->num_channels, GFP_KERNEL);
2793 if (!priv->chan) {
2794 dev_err(dev, "failed to allocate channel management space\n");
Kim Phillips9c4a7962008-06-23 19:50:15 +08002795 err = -ENOMEM;
2796 goto err_out;
2797 }
2798
Kim Phillipsc3e337f2011-11-21 16:13:27 +08002799 for (i = 0; i < priv->num_channels; i++) {
2800 priv->chan[i].reg = priv->reg + TALITOS_CH_STRIDE * (i + 1);
Kim Phillips2cdba3c2011-12-12 14:59:11 -06002801 if (!priv->irq[1] || !(i & 1))
Kim Phillipsc3e337f2011-11-21 16:13:27 +08002802 priv->chan[i].reg += TALITOS_CH_BASE_OFFSET;
2803 }
Kim Phillipsad42d5f2011-11-21 16:13:27 +08002804
Kim Phillips9c4a7962008-06-23 19:50:15 +08002805 for (i = 0; i < priv->num_channels; i++) {
Kim Phillips4b9926282009-08-13 11:50:38 +10002806 spin_lock_init(&priv->chan[i].head_lock);
2807 spin_lock_init(&priv->chan[i].tail_lock);
Kim Phillips9c4a7962008-06-23 19:50:15 +08002808 }
2809
2810 priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);
2811
2812 for (i = 0; i < priv->num_channels; i++) {
Kim Phillips4b9926282009-08-13 11:50:38 +10002813 priv->chan[i].fifo = kzalloc(sizeof(struct talitos_request) *
2814 priv->fifo_len, GFP_KERNEL);
2815 if (!priv->chan[i].fifo) {
Kim Phillips9c4a7962008-06-23 19:50:15 +08002816 dev_err(dev, "failed to allocate request fifo %d\n", i);
2817 err = -ENOMEM;
2818 goto err_out;
2819 }
2820 }
2821
Kim Phillipsec6644d2008-07-17 20:16:40 +08002822 for (i = 0; i < priv->num_channels; i++)
Kim Phillips4b9926282009-08-13 11:50:38 +10002823 atomic_set(&priv->chan[i].submit_count,
2824 -(priv->chfifo_len - 1));
Kim Phillips9c4a7962008-06-23 19:50:15 +08002825
Kim Phillips81eb0242009-08-13 11:51:51 +10002826 dma_set_mask(dev, DMA_BIT_MASK(36));
2827
Kim Phillips9c4a7962008-06-23 19:50:15 +08002828 /* reset and initialize the h/w */
2829 err = init_device(dev);
2830 if (err) {
2831 dev_err(dev, "failed to initialize device\n");
2832 goto err_out;
2833 }
2834
2835 /* register the RNG, if available */
2836 if (hw_supports(dev, DESC_HDR_SEL0_RNG)) {
2837 err = talitos_register_rng(dev);
2838 if (err) {
2839 dev_err(dev, "failed to register hwrng: %d\n", err);
2840 goto err_out;
2841 } else
2842 dev_info(dev, "hwrng\n");
2843 }
2844
2845 /* register crypto algorithms the device supports */
Kim Phillips9c4a7962008-06-23 19:50:15 +08002846 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
2847 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) {
2848 struct talitos_crypto_alg *t_alg;
Lee Nipperacbf7c622010-05-19 19:19:33 +10002849 char *name = NULL;
Kim Phillips9c4a7962008-06-23 19:50:15 +08002850
2851 t_alg = talitos_alg_alloc(dev, &driver_algs[i]);
2852 if (IS_ERR(t_alg)) {
2853 err = PTR_ERR(t_alg);
Kim Phillips0b2730d2011-12-12 14:59:10 -06002854 if (err == -ENOTSUPP)
Lee Nipper79b3a412011-11-21 16:13:25 +08002855 continue;
Kim Phillips9c4a7962008-06-23 19:50:15 +08002856 goto err_out;
2857 }
2858
Lee Nipperacbf7c622010-05-19 19:19:33 +10002859 switch (t_alg->algt.type) {
2860 case CRYPTO_ALG_TYPE_ABLKCIPHER:
2861 case CRYPTO_ALG_TYPE_AEAD:
2862 err = crypto_register_alg(
2863 &t_alg->algt.alg.crypto);
2864 name = t_alg->algt.alg.crypto.cra_driver_name;
2865 break;
2866 case CRYPTO_ALG_TYPE_AHASH:
2867 err = crypto_register_ahash(
2868 &t_alg->algt.alg.hash);
2869 name =
2870 t_alg->algt.alg.hash.halg.base.cra_driver_name;
2871 break;
2872 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08002873 if (err) {
2874 dev_err(dev, "%s alg registration failed\n",
Lee Nipperacbf7c622010-05-19 19:19:33 +10002875 name);
Kim Phillips9c4a7962008-06-23 19:50:15 +08002876 kfree(t_alg);
Kim Phillips5b859b6e2011-11-21 16:13:26 +08002877 } else
Kim Phillips9c4a7962008-06-23 19:50:15 +08002878 list_add_tail(&t_alg->entry, &priv->alg_list);
Kim Phillips9c4a7962008-06-23 19:50:15 +08002879 }
2880 }
Kim Phillips5b859b6e2011-11-21 16:13:26 +08002881 if (!list_empty(&priv->alg_list))
2882 dev_info(dev, "%s algorithms registered in /proc/crypto\n",
2883 (char *)of_get_property(np, "compatible", NULL));
Kim Phillips9c4a7962008-06-23 19:50:15 +08002884
2885 return 0;
2886
2887err_out:
2888 talitos_remove(ofdev);
Kim Phillips9c4a7962008-06-23 19:50:15 +08002889
2890 return err;
2891}
2892
Márton Németh6c3f9752010-01-17 21:54:01 +11002893static const struct of_device_id talitos_match[] = {
Kim Phillips9c4a7962008-06-23 19:50:15 +08002894 {
2895 .compatible = "fsl,sec2.0",
2896 },
2897 {},
2898};
2899MODULE_DEVICE_TABLE(of, talitos_match);
2900
Grant Likely1c48a5c2011-02-17 02:43:24 -07002901static struct platform_driver talitos_driver = {
Grant Likely40182942010-04-13 16:13:02 -07002902 .driver = {
2903 .name = "talitos",
2904 .owner = THIS_MODULE,
2905 .of_match_table = talitos_match,
2906 },
Kim Phillips9c4a7962008-06-23 19:50:15 +08002907 .probe = talitos_probe,
Al Viro596f1032008-11-22 17:34:24 +00002908 .remove = talitos_remove,
Kim Phillips9c4a7962008-06-23 19:50:15 +08002909};
2910
Axel Lin741e8c22011-11-26 21:26:19 +08002911module_platform_driver(talitos_driver);
Kim Phillips9c4a7962008-06-23 19:50:15 +08002912
2913MODULE_LICENSE("GPL");
2914MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>");
2915MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");