arch/arm64/crypto/polyval-ce-core.S - linux - Git at Google

 /* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Implementation of POLYVAL using ARMv8 Crypto Extensions.
  *
  * Copyright 2021 Google LLC
  */
 /*
  * This is an efficient implementation of POLYVAL using ARMv8 Crypto Extensions
  * It works on 8 blocks at a time, by precomputing the first 8 keys powers h^8,
  * ..., h^1 in the POLYVAL finite field. This precomputation allows us to split
  * finite field multiplication into two steps.
  *
  * In the first step, we consider h^i, m_i as normal polynomials of degree less
  * than 128. We then compute p(x) = h^8m_0 + ... + h^1m_7 where multiplication
  * is simply polynomial multiplication.
  *
  * In the second step, we compute the reduction of p(x) modulo the finite field
  * modulus g(x) = x^128 + x^127 + x^126 + x^121 + 1.
  *
  * This two step process is equivalent to computing h^8m_0 + ... + h^1m_7 where
  * multiplication is finite field multiplication. The advantage is that the
  * two-step process  only requires 1 finite field reduction for every 8
  * polynomial multiplications. Further parallelism is gained by interleaving the
  * multiplications and polynomial reductions.
  */

 #include <linux/linkage.h>
 #define STRIDE_BLOCKS 8

 KEY_POWERS	.req	x0
 MSG		.req	x1
 BLOCKS_LEFT	.req	x2
 ACCUMULATOR	.req	x3
 KEY_START	.req	x10
 EXTRA_BYTES	.req	x11
 TMP	.req	x13

 M0	.req	v0
 M1	.req	v1
 M2	.req	v2
 M3	.req	v3
 M4	.req	v4
 M5	.req	v5
 M6	.req	v6
 M7	.req	v7
 KEY8	.req	v8
 KEY7	.req	v9
 KEY6	.req	v10
 KEY5	.req	v11
 KEY4	.req	v12
 KEY3	.req	v13
 KEY2	.req	v14
 KEY1	.req	v15
 PL	.req	v16
 PH	.req	v17
 TMP_V	.req	v18
 LO	.req	v20
 MI	.req	v21
 HI	.req	v22
 SUM	.req	v23
 GSTAR	.req	v24

 	.text

 	.arch	armv8-a+crypto
 	.align	4

 .Lgstar:
 	.quad	0xc200000000000000, 0xc200000000000000

 /*
  * Computes the product of two 128-bit polynomials in X and Y and XORs the
  * components of the 256-bit product into LO, MI, HI.
  *
  * Given:
  *  X = [X_1 : X_0]
  *  Y = [Y_1 : Y_0]
  *
  * We compute:
  *  LO += X_0 * Y_0
  *  MI += (X_0 + X_1) * (Y_0 + Y_1)
  *  HI += X_1 * Y_1
  *
  * Later, the 256-bit result can be extracted as:
  *   [HI_1 : HI_0 + HI_1 + MI_1 + LO_1 : LO_1 + HI_0 + MI_0 + LO_0 : LO_0]
  * This step is done when computing the polynomial reduction for efficiency
  * reasons.
  *
  * Karatsuba multiplication is used instead of Schoolbook multiplication because
  * it was found to be slightly faster on ARM64 CPUs.
  *
  */
 .macro karatsuba1 X Y
 	X .req \X
 	Y .req \Y
 	ext	v25.16b, X.16b, X.16b, #8
 	ext	v26.16b, Y.16b, Y.16b, #8
 	eor	v25.16b, v25.16b, X.16b
 	eor	v26.16b, v26.16b, Y.16b
 	pmull2	v28.1q, X.2d, Y.2d
 	pmull	v29.1q, X.1d, Y.1d
 	pmull	v27.1q, v25.1d, v26.1d
 	eor	HI.16b, HI.16b, v28.16b
 	eor	LO.16b, LO.16b, v29.16b
 	eor	MI.16b, MI.16b, v27.16b
 	.unreq X
 	.unreq Y
 .endm

 /*
  * Same as karatsuba1, except overwrites HI, LO, MI rather than XORing into
  * them.
  */
 .macro karatsuba1_store X Y
 	X .req \X
 	Y .req \Y
 	ext	v25.16b, X.16b, X.16b, #8
 	ext	v26.16b, Y.16b, Y.16b, #8
 	eor	v25.16b, v25.16b, X.16b
 	eor	v26.16b, v26.16b, Y.16b
 	pmull2	HI.1q, X.2d, Y.2d
 	pmull	LO.1q, X.1d, Y.1d
 	pmull	MI.1q, v25.1d, v26.1d
 	.unreq X
 	.unreq Y
 .endm

 /*
  * Computes the 256-bit polynomial represented by LO, HI, MI. Stores
  * the result in PL, PH.
  * [PH : PL] =
  *   [HI_1 : HI_1 + HI_0 + MI_1 + LO_1 : HI_0 + MI_0 + LO_1 + LO_0 : LO_0]
  */
 .macro karatsuba2
 	// v4 = [HI_1 + MI_1 : HI_0 + MI_0]
 	eor	v4.16b, HI.16b, MI.16b
 	// v4 = [HI_1 + MI_1 + LO_1 : HI_0 + MI_0 + LO_0]
 	eor	v4.16b, v4.16b, LO.16b
 	// v5 = [HI_0 : LO_1]
 	ext	v5.16b, LO.16b, HI.16b, #8
 	// v4 = [HI_1 + HI_0 + MI_1 + LO_1 : HI_0 + MI_0 + LO_1 + LO_0]
 	eor	v4.16b, v4.16b, v5.16b
 	// HI = [HI_0 : HI_1]
 	ext	HI.16b, HI.16b, HI.16b, #8
 	// LO = [LO_0 : LO_1]
 	ext	LO.16b, LO.16b, LO.16b, #8
 	// PH = [HI_1 : HI_1 + HI_0 + MI_1 + LO_1]
 	ext	PH.16b, v4.16b, HI.16b, #8
 	// PL = [HI_0 + MI_0 + LO_1 + LO_0 : LO_0]
 	ext	PL.16b, LO.16b, v4.16b, #8
 .endm

 /*
  * Computes the 128-bit reduction of PH : PL. Stores the result in dest.
  *
  * This macro computes p(x) mod g(x) where p(x) is in montgomery form and g(x) =
  * x^128 + x^127 + x^126 + x^121 + 1.
  *
  * We have a 256-bit polynomial PH : PL = P_3 : P_2 : P_1 : P_0 that is the
  * product of two 128-bit polynomials in Montgomery form.  We need to reduce it
  * mod g(x).  Also, since polynomials in Montgomery form have an "extra" factor
  * of x^128, this product has two extra factors of x^128.  To get it back into
  * Montgomery form, we need to remove one of these factors by dividing by x^128.
  *
  * To accomplish both of these goals, we add multiples of g(x) that cancel out
  * the low 128 bits P_1 : P_0, leaving just the high 128 bits. Since the low
  * bits are zero, the polynomial division by x^128 can be done by right
  * shifting.
  *
  * Since the only nonzero term in the low 64 bits of g(x) is the constant term,
  * the multiple of g(x) needed to cancel out P_0 is P_0 * g(x).  The CPU can
  * only do 64x64 bit multiplications, so split P_0 * g(x) into x^128 * P_0 +
  * x^64 * g*(x) * P_0 + P_0, where g*(x) is bits 64-127 of g(x).  Adding this to
  * the original polynomial gives P_3 : P_2 + P_0 + T_1 : P_1 + T_0 : 0, where T
  * = T_1 : T_0 = g*(x) * P_0.  Thus, bits 0-63 got "folded" into bits 64-191.
  *
  * Repeating this same process on the next 64 bits "folds" bits 64-127 into bits
  * 128-255, giving the answer in bits 128-255. This time, we need to cancel P_1
  * + T_0 in bits 64-127. The multiple of g(x) required is (P_1 + T_0) * g(x) *
  * x^64. Adding this to our previous computation gives P_3 + P_1 + T_0 + V_1 :
  * P_2 + P_0 + T_1 + V_0 : 0 : 0, where V = V_1 : V_0 = g*(x) * (P_1 + T_0).
  *
  * So our final computation is:
  *   T = T_1 : T_0 = g*(x) * P_0
  *   V = V_1 : V_0 = g*(x) * (P_1 + T_0)
  *   p(x) / x^{128} mod g(x) = P_3 + P_1 + T_0 + V_1 : P_2 + P_0 + T_1 + V_0
  *
  * The implementation below saves a XOR instruction by computing P_1 + T_0 : P_0
  * + T_1 and XORing into dest, rather than separately XORing P_1 : P_0 and T_0 :
  * T_1 into dest.  This allows us to reuse P_1 + T_0 when computing V.
  */
 .macro montgomery_reduction dest
 	DEST .req \dest
 	// TMP_V = T_1 : T_0 = P_0 * g*(x)
 	pmull	TMP_V.1q, PL.1d, GSTAR.1d
 	// TMP_V = T_0 : T_1
 	ext	TMP_V.16b, TMP_V.16b, TMP_V.16b, #8
 	// TMP_V = P_1 + T_0 : P_0 + T_1
 	eor	TMP_V.16b, PL.16b, TMP_V.16b
 	// PH = P_3 + P_1 + T_0 : P_2 + P_0 + T_1
 	eor	PH.16b, PH.16b, TMP_V.16b
 	// TMP_V = V_1 : V_0 = (P_1 + T_0) * g*(x)
 	pmull2	TMP_V.1q, TMP_V.2d, GSTAR.2d
 	eor	DEST.16b, PH.16b, TMP_V.16b
 	.unreq DEST
 .endm

 /*
  * Compute Polyval on 8 blocks.
  *
  * If reduce is set, also computes the montgomery reduction of the
  * previous full_stride call and XORs with the first message block.
  * (m_0 + REDUCE(PL, PH))h^8 + ... + m_7h^1.
  * I.e., the first multiplication uses m_0 + REDUCE(PL, PH) instead of m_0.
  *
  * Sets PL, PH.
  */
 .macro full_stride reduce
 	eor		LO.16b, LO.16b, LO.16b
 	eor		MI.16b, MI.16b, MI.16b
 	eor		HI.16b, HI.16b, HI.16b

 	ld1		{M0.16b, M1.16b, M2.16b, M3.16b}, [MSG], #64
 	ld1		{M4.16b, M5.16b, M6.16b, M7.16b}, [MSG], #64

 	karatsuba1 M7 KEY1
 	.if \reduce
 	pmull	TMP_V.1q, PL.1d, GSTAR.1d
 	.endif

 	karatsuba1 M6 KEY2
 	.if \reduce
 	ext	TMP_V.16b, TMP_V.16b, TMP_V.16b, #8
 	.endif

 	karatsuba1 M5 KEY3
 	.if \reduce
 	eor	TMP_V.16b, PL.16b, TMP_V.16b
 	.endif

 	karatsuba1 M4 KEY4
 	.if \reduce
 	eor	PH.16b, PH.16b, TMP_V.16b
 	.endif

 	karatsuba1 M3 KEY5
 	.if \reduce
 	pmull2	TMP_V.1q, TMP_V.2d, GSTAR.2d
 	.endif

 	karatsuba1 M2 KEY6
 	.if \reduce
 	eor	SUM.16b, PH.16b, TMP_V.16b
 	.endif

 	karatsuba1 M1 KEY7
 	eor	M0.16b, M0.16b, SUM.16b

 	karatsuba1 M0 KEY8
 	karatsuba2
 .endm

 /*
  * Handle any extra blocks after full_stride loop.
  */
 .macro partial_stride
 	add	KEY_POWERS, KEY_START, #(STRIDE_BLOCKS << 4)
 	sub	KEY_POWERS, KEY_POWERS, BLOCKS_LEFT, lsl #4
 	ld1	{KEY1.16b}, [KEY_POWERS], #16

 	ld1	{TMP_V.16b}, [MSG], #16
 	eor	SUM.16b, SUM.16b, TMP_V.16b
 	karatsuba1_store KEY1 SUM
 	sub	BLOCKS_LEFT, BLOCKS_LEFT, #1

 	tst	BLOCKS_LEFT, #4
 	beq	.Lpartial4BlocksDone
 	ld1	{M0.16b, M1.16b,  M2.16b, M3.16b}, [MSG], #64
 	ld1	{KEY8.16b, KEY7.16b, KEY6.16b,	KEY5.16b}, [KEY_POWERS], #64
 	karatsuba1 M0 KEY8
 	karatsuba1 M1 KEY7
 	karatsuba1 M2 KEY6
 	karatsuba1 M3 KEY5
 .Lpartial4BlocksDone:
 	tst	BLOCKS_LEFT, #2
 	beq	.Lpartial2BlocksDone
 	ld1	{M0.16b, M1.16b}, [MSG], #32
 	ld1	{KEY8.16b, KEY7.16b}, [KEY_POWERS], #32
 	karatsuba1 M0 KEY8
 	karatsuba1 M1 KEY7
 .Lpartial2BlocksDone:
 	tst	BLOCKS_LEFT, #1
 	beq	.LpartialDone
 	ld1	{M0.16b}, [MSG], #16
 	ld1	{KEY8.16b}, [KEY_POWERS], #16
 	karatsuba1 M0 KEY8
 .LpartialDone:
 	karatsuba2
 	montgomery_reduction SUM
 .endm

 /*
  * Perform montgomery multiplication in GF(2^128) and store result in op1.
  *
  * Computes op1*op2*x^{-128} mod x^128 + x^127 + x^126 + x^121 + 1
  * If op1, op2 are in montgomery form, this computes the montgomery
  * form of op1*op2.
  *
  * void pmull_polyval_mul(u8 *op1, const u8 *op2);
  */
 SYM_FUNC_START(pmull_polyval_mul)
 	adr	TMP, .Lgstar
 	ld1	{GSTAR.2d}, [TMP]
 	ld1	{v0.16b}, [x0]
 	ld1	{v1.16b}, [x1]
 	karatsuba1_store v0 v1
 	karatsuba2
 	montgomery_reduction SUM
 	st1	{SUM.16b}, [x0]
 	ret
 SYM_FUNC_END(pmull_polyval_mul)

 /*
  * Perform polynomial evaluation as specified by POLYVAL.  This computes:
  *	h^n * accumulator + h^n * m_0 + ... + h^1 * m_{n-1}
  * where n=nblocks, h is the hash key, and m_i are the message blocks.
  *
  * x0 - pointer to precomputed key powers h^8 ... h^1
  * x1 - pointer to message blocks
  * x2 - number of blocks to hash
  * x3 - pointer to accumulator
  *
  * void pmull_polyval_update(const struct polyval_ctx *ctx, const u8 *in,
  *			     size_t nblocks, u8 *accumulator);
  */
 SYM_FUNC_START(pmull_polyval_update)
 	adr	TMP, .Lgstar
 	mov	KEY_START, KEY_POWERS
 	ld1	{GSTAR.2d}, [TMP]
 	ld1	{SUM.16b}, [ACCUMULATOR]
 	subs	BLOCKS_LEFT, BLOCKS_LEFT, #STRIDE_BLOCKS
 	blt .LstrideLoopExit
 	ld1	{KEY8.16b, KEY7.16b, KEY6.16b, KEY5.16b}, [KEY_POWERS], #64
 	ld1	{KEY4.16b, KEY3.16b, KEY2.16b, KEY1.16b}, [KEY_POWERS], #64
 	full_stride 0
 	subs	BLOCKS_LEFT, BLOCKS_LEFT, #STRIDE_BLOCKS
 	blt .LstrideLoopExitReduce
 .LstrideLoop:
 	full_stride 1
 	subs	BLOCKS_LEFT, BLOCKS_LEFT, #STRIDE_BLOCKS
 	bge	.LstrideLoop
 .LstrideLoopExitReduce:
 	montgomery_reduction SUM
 .LstrideLoopExit:
 	adds	BLOCKS_LEFT, BLOCKS_LEFT, #STRIDE_BLOCKS
 	beq	.LskipPartial
 	partial_stride
 .LskipPartial:
 	st1	{SUM.16b}, [ACCUMULATOR]
 	ret
 SYM_FUNC_END(pmull_polyval_update)
	/* SPDX-License-Identifier: GPL-2.0 */
	/*
	* Implementation of POLYVAL using ARMv8 Crypto Extensions.
	*
	* Copyright 2021 Google LLC
	*/
	/*
	* This is an efficient implementation of POLYVAL using ARMv8 Crypto Extensions
	* It works on 8 blocks at a time, by precomputing the first 8 keys powers h^8,
	* ..., h^1 in the POLYVAL finite field. This precomputation allows us to split
	* finite field multiplication into two steps.
	*
	* In the first step, we consider h^i, m_i as normal polynomials of degree less
	* than 128. We then compute p(x) = h^8m_0 + ... + h^1m_7 where multiplication
	* is simply polynomial multiplication.
	*
	* In the second step, we compute the reduction of p(x) modulo the finite field
	* modulus g(x) = x^128 + x^127 + x^126 + x^121 + 1.
	*
	* This two step process is equivalent to computing h^8m_0 + ... + h^1m_7 where
	* multiplication is finite field multiplication. The advantage is that the
	* two-step process only requires 1 finite field reduction for every 8
	* polynomial multiplications. Further parallelism is gained by interleaving the
	* multiplications and polynomial reductions.
	*/

	#include <linux/linkage.h>
	#define STRIDE_BLOCKS 8

	KEY_POWERS .req x0
	MSG .req x1
	BLOCKS_LEFT .req x2
	ACCUMULATOR .req x3
	KEY_START .req x10
	EXTRA_BYTES .req x11
	TMP .req x13

	M0 .req v0
	M1 .req v1
	M2 .req v2
	M3 .req v3
	M4 .req v4
	M5 .req v5
	M6 .req v6
	M7 .req v7
	KEY8 .req v8
	KEY7 .req v9
	KEY6 .req v10
	KEY5 .req v11
	KEY4 .req v12
	KEY3 .req v13
	KEY2 .req v14
	KEY1 .req v15
	PL .req v16
	PH .req v17
	TMP_V .req v18
	LO .req v20
	MI .req v21
	HI .req v22
	SUM .req v23
	GSTAR .req v24

	.text

	.arch armv8-a+crypto
	.align 4

	.Lgstar:
	.quad 0xc200000000000000, 0xc200000000000000

	/*
	* Computes the product of two 128-bit polynomials in X and Y and XORs the
	* components of the 256-bit product into LO, MI, HI.
	*
	* Given:
	* X = [X_1 : X_0]
	* Y = [Y_1 : Y_0]
	*
	* We compute:
	* LO += X_0 * Y_0
	* MI += (X_0 + X_1) * (Y_0 + Y_1)
	* HI += X_1 * Y_1
	*
	* Later, the 256-bit result can be extracted as:
	* [HI_1 : HI_0 + HI_1 + MI_1 + LO_1 : LO_1 + HI_0 + MI_0 + LO_0 : LO_0]
	* This step is done when computing the polynomial reduction for efficiency
	* reasons.
	*
	* Karatsuba multiplication is used instead of Schoolbook multiplication because
	* it was found to be slightly faster on ARM64 CPUs.
	*
	*/
	.macro karatsuba1 X Y
	X .req \X
	Y .req \Y
	ext v25.16b, X.16b, X.16b, #8
	ext v26.16b, Y.16b, Y.16b, #8
	eor v25.16b, v25.16b, X.16b
	eor v26.16b, v26.16b, Y.16b
	pmull2 v28.1q, X.2d, Y.2d
	pmull v29.1q, X.1d, Y.1d
	pmull v27.1q, v25.1d, v26.1d
	eor HI.16b, HI.16b, v28.16b
	eor LO.16b, LO.16b, v29.16b
	eor MI.16b, MI.16b, v27.16b
	.unreq X
	.unreq Y
	.endm

	/*
	* Same as karatsuba1, except overwrites HI, LO, MI rather than XORing into
	* them.
	*/
	.macro karatsuba1_store X Y
	X .req \X
	Y .req \Y
	ext v25.16b, X.16b, X.16b, #8
	ext v26.16b, Y.16b, Y.16b, #8
	eor v25.16b, v25.16b, X.16b
	eor v26.16b, v26.16b, Y.16b
	pmull2 HI.1q, X.2d, Y.2d
	pmull LO.1q, X.1d, Y.1d
	pmull MI.1q, v25.1d, v26.1d
	.unreq X
	.unreq Y
	.endm

	/*
	* Computes the 256-bit polynomial represented by LO, HI, MI. Stores
	* the result in PL, PH.
	* [PH : PL] =
	* [HI_1 : HI_1 + HI_0 + MI_1 + LO_1 : HI_0 + MI_0 + LO_1 + LO_0 : LO_0]
	*/
	.macro karatsuba2
	// v4 = [HI_1 + MI_1 : HI_0 + MI_0]
	eor v4.16b, HI.16b, MI.16b
	// v4 = [HI_1 + MI_1 + LO_1 : HI_0 + MI_0 + LO_0]
	eor v4.16b, v4.16b, LO.16b
	// v5 = [HI_0 : LO_1]
	ext v5.16b, LO.16b, HI.16b, #8
	// v4 = [HI_1 + HI_0 + MI_1 + LO_1 : HI_0 + MI_0 + LO_1 + LO_0]
	eor v4.16b, v4.16b, v5.16b
	// HI = [HI_0 : HI_1]
	ext HI.16b, HI.16b, HI.16b, #8
	// LO = [LO_0 : LO_1]
	ext LO.16b, LO.16b, LO.16b, #8
	// PH = [HI_1 : HI_1 + HI_0 + MI_1 + LO_1]
	ext PH.16b, v4.16b, HI.16b, #8
	// PL = [HI_0 + MI_0 + LO_1 + LO_0 : LO_0]
	ext PL.16b, LO.16b, v4.16b, #8
	.endm

	/*
	* Computes the 128-bit reduction of PH : PL. Stores the result in dest.
	*
	* This macro computes p(x) mod g(x) where p(x) is in montgomery form and g(x) =
	* x^128 + x^127 + x^126 + x^121 + 1.
	*
	* We have a 256-bit polynomial PH : PL = P_3 : P_2 : P_1 : P_0 that is the
	* product of two 128-bit polynomials in Montgomery form. We need to reduce it
	* mod g(x). Also, since polynomials in Montgomery form have an "extra" factor
	* of x^128, this product has two extra factors of x^128. To get it back into
	* Montgomery form, we need to remove one of these factors by dividing by x^128.
	*
	* To accomplish both of these goals, we add multiples of g(x) that cancel out
	* the low 128 bits P_1 : P_0, leaving just the high 128 bits. Since the low
	* bits are zero, the polynomial division by x^128 can be done by right
	* shifting.
	*
	* Since the only nonzero term in the low 64 bits of g(x) is the constant term,
	* the multiple of g(x) needed to cancel out P_0 is P_0 * g(x). The CPU can
	* only do 64x64 bit multiplications, so split P_0 * g(x) into x^128 * P_0 +
	* x^64 * g(x) P_0 + P_0, where g*(x) is bits 64-127 of g(x). Adding this to
	* the original polynomial gives P_3 : P_2 + P_0 + T_1 : P_1 + T_0 : 0, where T
	* = T_1 : T_0 = g(x) P_0. Thus, bits 0-63 got "folded" into bits 64-191.
	*
	* Repeating this same process on the next 64 bits "folds" bits 64-127 into bits
	* 128-255, giving the answer in bits 128-255. This time, we need to cancel P_1
	* + T_0 in bits 64-127. The multiple of g(x) required is (P_1 + T_0) * g(x) *
	* x^64. Adding this to our previous computation gives P_3 + P_1 + T_0 + V_1 :
	* P_2 + P_0 + T_1 + V_0 : 0 : 0, where V = V_1 : V_0 = g(x) (P_1 + T_0).
	*
	* So our final computation is:
	* T = T_1 : T_0 = g(x) P_0
	* V = V_1 : V_0 = g(x) (P_1 + T_0)
	* p(x) / x^{128} mod g(x) = P_3 + P_1 + T_0 + V_1 : P_2 + P_0 + T_1 + V_0
	*
	* The implementation below saves a XOR instruction by computing P_1 + T_0 : P_0
	* + T_1 and XORing into dest, rather than separately XORing P_1 : P_0 and T_0 :
	* T_1 into dest. This allows us to reuse P_1 + T_0 when computing V.
	*/
	.macro montgomery_reduction dest
	DEST .req \dest
	// TMP_V = T_1 : T_0 = P_0 * g*(x)
	pmull TMP_V.1q, PL.1d, GSTAR.1d
	// TMP_V = T_0 : T_1
	ext TMP_V.16b, TMP_V.16b, TMP_V.16b, #8
	// TMP_V = P_1 + T_0 : P_0 + T_1
	eor TMP_V.16b, PL.16b, TMP_V.16b
	// PH = P_3 + P_1 + T_0 : P_2 + P_0 + T_1
	eor PH.16b, PH.16b, TMP_V.16b
	// TMP_V = V_1 : V_0 = (P_1 + T_0) * g*(x)
	pmull2 TMP_V.1q, TMP_V.2d, GSTAR.2d
	eor DEST.16b, PH.16b, TMP_V.16b
	.unreq DEST
	.endm

	/*
	* Compute Polyval on 8 blocks.
	*
	* If reduce is set, also computes the montgomery reduction of the
	* previous full_stride call and XORs with the first message block.
	* (m_0 + REDUCE(PL, PH))h^8 + ... + m_7h^1.
	* I.e., the first multiplication uses m_0 + REDUCE(PL, PH) instead of m_0.
	*
	* Sets PL, PH.
	*/
	.macro full_stride reduce
	eor LO.16b, LO.16b, LO.16b
	eor MI.16b, MI.16b, MI.16b
	eor HI.16b, HI.16b, HI.16b

	ld1 {M0.16b, M1.16b, M2.16b, M3.16b}, [MSG], #64
	ld1 {M4.16b, M5.16b, M6.16b, M7.16b}, [MSG], #64

	karatsuba1 M7 KEY1
	.if \reduce
	pmull TMP_V.1q, PL.1d, GSTAR.1d
	.endif

	karatsuba1 M6 KEY2
	.if \reduce
	ext TMP_V.16b, TMP_V.16b, TMP_V.16b, #8
	.endif

	karatsuba1 M5 KEY3
	.if \reduce
	eor TMP_V.16b, PL.16b, TMP_V.16b
	.endif

	karatsuba1 M4 KEY4
	.if \reduce
	eor PH.16b, PH.16b, TMP_V.16b
	.endif

	karatsuba1 M3 KEY5
	.if \reduce
	pmull2 TMP_V.1q, TMP_V.2d, GSTAR.2d
	.endif

	karatsuba1 M2 KEY6
	.if \reduce
	eor SUM.16b, PH.16b, TMP_V.16b
	.endif

	karatsuba1 M1 KEY7
	eor M0.16b, M0.16b, SUM.16b

	karatsuba1 M0 KEY8
	karatsuba2
	.endm

	/*
	* Handle any extra blocks after full_stride loop.
	*/
	.macro partial_stride
	add KEY_POWERS, KEY_START, #(STRIDE_BLOCKS << 4)
	sub KEY_POWERS, KEY_POWERS, BLOCKS_LEFT, lsl #4
	ld1 {KEY1.16b}, [KEY_POWERS], #16

	ld1 {TMP_V.16b}, [MSG], #16
	eor SUM.16b, SUM.16b, TMP_V.16b
	karatsuba1_store KEY1 SUM
	sub BLOCKS_LEFT, BLOCKS_LEFT, #1

	tst BLOCKS_LEFT, #4
	beq .Lpartial4BlocksDone
	ld1 {M0.16b, M1.16b, M2.16b, M3.16b}, [MSG], #64
	ld1 {KEY8.16b, KEY7.16b, KEY6.16b, KEY5.16b}, [KEY_POWERS], #64
	karatsuba1 M0 KEY8
	karatsuba1 M1 KEY7
	karatsuba1 M2 KEY6
	karatsuba1 M3 KEY5
	.Lpartial4BlocksDone:
	tst BLOCKS_LEFT, #2
	beq .Lpartial2BlocksDone
	ld1 {M0.16b, M1.16b}, [MSG], #32
	ld1 {KEY8.16b, KEY7.16b}, [KEY_POWERS], #32
	karatsuba1 M0 KEY8
	karatsuba1 M1 KEY7
	.Lpartial2BlocksDone:
	tst BLOCKS_LEFT, #1
	beq .LpartialDone
	ld1 {M0.16b}, [MSG], #16
	ld1 {KEY8.16b}, [KEY_POWERS], #16
	karatsuba1 M0 KEY8
	.LpartialDone:
	karatsuba2
	montgomery_reduction SUM
	.endm

	/*
	* Perform montgomery multiplication in GF(2^128) and store result in op1.
	*
	* Computes op1op2x^{-128} mod x^128 + x^127 + x^126 + x^121 + 1
	* If op1, op2 are in montgomery form, this computes the montgomery
	* form of op1*op2.
	*
	* void pmull_polyval_mul(u8 op1, const u8 op2);
	*/
	SYM_FUNC_START(pmull_polyval_mul)
	adr TMP, .Lgstar
	ld1 {GSTAR.2d}, [TMP]
	ld1 {v0.16b}, [x0]
	ld1 {v1.16b}, [x1]
	karatsuba1_store v0 v1
	karatsuba2
	montgomery_reduction SUM
	st1 {SUM.16b}, [x0]
	ret
	SYM_FUNC_END(pmull_polyval_mul)

	/*
	* Perform polynomial evaluation as specified by POLYVAL. This computes:
	* h^n * accumulator + h^n * m_0 + ... + h^1 * m_{n-1}
	* where n=nblocks, h is the hash key, and m_i are the message blocks.
	*
	* x0 - pointer to precomputed key powers h^8 ... h^1
	* x1 - pointer to message blocks
	* x2 - number of blocks to hash
	* x3 - pointer to accumulator
	*
	* void pmull_polyval_update(const struct polyval_ctx ctx, const u8 in,
	* size_t nblocks, u8 *accumulator);
	*/
	SYM_FUNC_START(pmull_polyval_update)
	adr TMP, .Lgstar
	mov KEY_START, KEY_POWERS
	ld1 {GSTAR.2d}, [TMP]
	ld1 {SUM.16b}, [ACCUMULATOR]
	subs BLOCKS_LEFT, BLOCKS_LEFT, #STRIDE_BLOCKS
	blt .LstrideLoopExit
	ld1 {KEY8.16b, KEY7.16b, KEY6.16b, KEY5.16b}, [KEY_POWERS], #64
	ld1 {KEY4.16b, KEY3.16b, KEY2.16b, KEY1.16b}, [KEY_POWERS], #64
	full_stride 0
	subs BLOCKS_LEFT, BLOCKS_LEFT, #STRIDE_BLOCKS
	blt .LstrideLoopExitReduce
	.LstrideLoop:
	full_stride 1
	subs BLOCKS_LEFT, BLOCKS_LEFT, #STRIDE_BLOCKS
	bge .LstrideLoop
	.LstrideLoopExitReduce:
	montgomery_reduction SUM
	.LstrideLoopExit:
	adds BLOCKS_LEFT, BLOCKS_LEFT, #STRIDE_BLOCKS
	beq .LskipPartial
	partial_stride
	.LskipPartial:
	st1 {SUM.16b}, [ACCUMULATOR]
	ret
	SYM_FUNC_END(pmull_polyval_update)