/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * SM4 Cipher Algorithm for ARMv8 with Crypto Extensions
 * as specified in
 * https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html
 *
 * Copyright (C) 2022, Alibaba Group.
 * Copyright (C) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
 */

#include <linux/linkage.h>
#include <asm/assembler.h>

.arch	armv8-a+crypto

.irp b, 0, 1, 2, 3, 4, 5, 6, 7, 16, 20, 24, 25, 26, 27, 28, 29, 30, 31
	.set .Lv\b\().4s, \b
.endr

.macro sm4e, vd, vn
	.inst 0xcec08400 | (.L\vn << 5) | .L\vd
.endm

.macro sm4ekey, vd, vn, vm
	.inst 0xce60c800 | (.L\vm << 16) | (.L\vn << 5) | .L\vd
.endm

/* Register macros */

#define RTMP0	v16
#define RTMP1	v17
#define RTMP2	v18
#define RTMP3	v19

#define RIV	v20

/* Helper macros. */

#define PREPARE                                       \
	ld1		{v24.16b-v27.16b}, [x0], #64; \
	ld1		{v28.16b-v31.16b}, [x0];

#define SM4_CRYPT_BLK(b0)                           \
	rev32		b0.16b, b0.16b;             \
	sm4e		b0.4s, v24.4s;              \
	sm4e		b0.4s, v25.4s;              \
	sm4e		b0.4s, v26.4s;              \
	sm4e		b0.4s, v27.4s;              \
	sm4e		b0.4s, v28.4s;              \
	sm4e		b0.4s, v29.4s;              \
	sm4e		b0.4s, v30.4s;              \
	sm4e		b0.4s, v31.4s;              \
	rev64		b0.4s, b0.4s;               \
	ext		b0.16b, b0.16b, b0.16b, #8; \
	rev32		b0.16b, b0.16b;

#define SM4_CRYPT_BLK4(b0, b1, b2, b3)              \
	rev32		b0.16b, b0.16b;             \
	rev32		b1.16b, b1.16b;             \
	rev32		b2.16b, b2.16b;             \
	rev32		b3.16b, b3.16b;             \
	sm4e		b0.4s, v24.4s;              \
	sm4e		b1.4s, v24.4s;              \
	sm4e		b2.4s, v24.4s;              \
	sm4e		b3.4s, v24.4s;              \
	sm4e		b0.4s, v25.4s;              \
	sm4e		b1.4s, v25.4s;              \
	sm4e		b2.4s, v25.4s;              \
	sm4e		b3.4s, v25.4s;              \
	sm4e		b0.4s, v26.4s;              \
	sm4e		b1.4s, v26.4s;              \
	sm4e		b2.4s, v26.4s;              \
	sm4e		b3.4s, v26.4s;              \
	sm4e		b0.4s, v27.4s;              \
	sm4e		b1.4s, v27.4s;              \
	sm4e		b2.4s, v27.4s;              \
	sm4e		b3.4s, v27.4s;              \
	sm4e		b0.4s, v28.4s;              \
	sm4e		b1.4s, v28.4s;              \
	sm4e		b2.4s, v28.4s;              \
	sm4e		b3.4s, v28.4s;              \
	sm4e		b0.4s, v29.4s;              \
	sm4e		b1.4s, v29.4s;              \
	sm4e		b2.4s, v29.4s;              \
	sm4e		b3.4s, v29.4s;              \
	sm4e		b0.4s, v30.4s;              \
	sm4e		b1.4s, v30.4s;              \
	sm4e		b2.4s, v30.4s;              \
	sm4e		b3.4s, v30.4s;              \
	sm4e		b0.4s, v31.4s;              \
	sm4e		b1.4s, v31.4s;              \
	sm4e		b2.4s, v31.4s;              \
	sm4e		b3.4s, v31.4s;              \
	rev64		b0.4s, b0.4s;               \
	rev64		b1.4s, b1.4s;               \
	rev64		b2.4s, b2.4s;               \
	rev64		b3.4s, b3.4s;               \
	ext		b0.16b, b0.16b, b0.16b, #8; \
	ext		b1.16b, b1.16b, b1.16b, #8; \
	ext		b2.16b, b2.16b, b2.16b, #8; \
	ext		b3.16b, b3.16b, b3.16b, #8; \
	rev32		b0.16b, b0.16b;             \
	rev32		b1.16b, b1.16b;             \
	rev32		b2.16b, b2.16b;             \
	rev32		b3.16b, b3.16b;

#define SM4_CRYPT_BLK8(b0, b1, b2, b3, b4, b5, b6, b7) \
	rev32		b0.16b, b0.16b;             \
	rev32		b1.16b, b1.16b;             \
	rev32		b2.16b, b2.16b;             \
	rev32		b3.16b, b3.16b;             \
	rev32		b4.16b, b4.16b;             \
	rev32		b5.16b, b5.16b;             \
	rev32		b6.16b, b6.16b;             \
	rev32		b7.16b, b7.16b;             \
	sm4e		b0.4s, v24.4s;              \
	sm4e		b1.4s, v24.4s;              \
	sm4e		b2.4s, v24.4s;              \
	sm4e		b3.4s, v24.4s;              \
	sm4e		b4.4s, v24.4s;              \
	sm4e		b5.4s, v24.4s;              \
	sm4e		b6.4s, v24.4s;              \
	sm4e		b7.4s, v24.4s;              \
	sm4e		b0.4s, v25.4s;              \
	sm4e		b1.4s, v25.4s;              \
	sm4e		b2.4s, v25.4s;              \
	sm4e		b3.4s, v25.4s;              \
	sm4e		b4.4s, v25.4s;              \
	sm4e		b5.4s, v25.4s;              \
	sm4e		b6.4s, v25.4s;              \
	sm4e		b7.4s, v25.4s;              \
	sm4e		b0.4s, v26.4s;              \
	sm4e		b1.4s, v26.4s;              \
	sm4e		b2.4s, v26.4s;              \
	sm4e		b3.4s, v26.4s;              \
	sm4e		b4.4s, v26.4s;              \
	sm4e		b5.4s, v26.4s;              \
	sm4e		b6.4s, v26.4s;              \
	sm4e		b7.4s, v26.4s;              \
	sm4e		b0.4s, v27.4s;              \
	sm4e		b1.4s, v27.4s;              \
	sm4e		b2.4s, v27.4s;              \
	sm4e		b3.4s, v27.4s;              \
	sm4e		b4.4s, v27.4s;              \
	sm4e		b5.4s, v27.4s;              \
	sm4e		b6.4s, v27.4s;              \
	sm4e		b7.4s, v27.4s;              \
	sm4e		b0.4s, v28.4s;              \
	sm4e		b1.4s, v28.4s;              \
	sm4e		b2.4s, v28.4s;              \
	sm4e		b3.4s, v28.4s;              \
	sm4e		b4.4s, v28.4s;              \
	sm4e		b5.4s, v28.4s;              \
	sm4e		b6.4s, v28.4s;              \
	sm4e		b7.4s, v28.4s;              \
	sm4e		b0.4s, v29.4s;              \
	sm4e		b1.4s, v29.4s;              \
	sm4e		b2.4s, v29.4s;              \
	sm4e		b3.4s, v29.4s;              \
	sm4e		b4.4s, v29.4s;              \
	sm4e		b5.4s, v29.4s;              \
	sm4e		b6.4s, v29.4s;              \
	sm4e		b7.4s, v29.4s;              \
	sm4e		b0.4s, v30.4s;              \
	sm4e		b1.4s, v30.4s;              \
	sm4e		b2.4s, v30.4s;              \
	sm4e		b3.4s, v30.4s;              \
	sm4e		b4.4s, v30.4s;              \
	sm4e		b5.4s, v30.4s;              \
	sm4e		b6.4s, v30.4s;              \
	sm4e		b7.4s, v30.4s;              \
	sm4e		b0.4s, v31.4s;              \
	sm4e		b1.4s, v31.4s;              \
	sm4e		b2.4s, v31.4s;              \
	sm4e		b3.4s, v31.4s;              \
	sm4e		b4.4s, v31.4s;              \
	sm4e		b5.4s, v31.4s;              \
	sm4e		b6.4s, v31.4s;              \
	sm4e		b7.4s, v31.4s;              \
	rev64		b0.4s, b0.4s;               \
	rev64		b1.4s, b1.4s;               \
	rev64		b2.4s, b2.4s;               \
	rev64		b3.4s, b3.4s;               \
	rev64		b4.4s, b4.4s;               \
	rev64		b5.4s, b5.4s;               \
	rev64		b6.4s, b6.4s;               \
	rev64		b7.4s, b7.4s;               \
	ext		b0.16b, b0.16b, b0.16b, #8; \
	ext		b1.16b, b1.16b, b1.16b, #8; \
	ext		b2.16b, b2.16b, b2.16b, #8; \
	ext		b3.16b, b3.16b, b3.16b, #8; \
	ext		b4.16b, b4.16b, b4.16b, #8; \
	ext		b5.16b, b5.16b, b5.16b, #8; \
	ext		b6.16b, b6.16b, b6.16b, #8; \
	ext		b7.16b, b7.16b, b7.16b, #8; \
	rev32		b0.16b, b0.16b;             \
	rev32		b1.16b, b1.16b;             \
	rev32		b2.16b, b2.16b;             \
	rev32		b3.16b, b3.16b;             \
	rev32		b4.16b, b4.16b;             \
	rev32		b5.16b, b5.16b;             \
	rev32		b6.16b, b6.16b;             \
	rev32		b7.16b, b7.16b;


.align 3
SYM_FUNC_START(sm4_ce_expand_key)
	/* input:
	 *   x0: 128-bit key
	 *   x1: rkey_enc
	 *   x2: rkey_dec
	 *   x3: fk array
	 *   x4: ck array
	 */
	ld1		{v0.16b}, [x0];
	rev32		v0.16b, v0.16b;
	ld1		{v1.16b}, [x3];
	/* load ck */
	ld1		{v24.16b-v27.16b}, [x4], #64;
	ld1		{v28.16b-v31.16b}, [x4];

	/* input ^ fk */
	eor		v0.16b, v0.16b, v1.16b;

	sm4ekey		v0.4s, v0.4s, v24.4s;
	sm4ekey		v1.4s, v0.4s, v25.4s;
	sm4ekey		v2.4s, v1.4s, v26.4s;
	sm4ekey		v3.4s, v2.4s, v27.4s;
	sm4ekey		v4.4s, v3.4s, v28.4s;
	sm4ekey		v5.4s, v4.4s, v29.4s;
	sm4ekey		v6.4s, v5.4s, v30.4s;
	sm4ekey		v7.4s, v6.4s, v31.4s;

	st1		{v0.16b-v3.16b}, [x1], #64;
	st1		{v4.16b-v7.16b}, [x1];
	rev64		v7.4s, v7.4s;
	rev64		v6.4s, v6.4s;
	rev64		v5.4s, v5.4s;
	rev64		v4.4s, v4.4s;
	rev64		v3.4s, v3.4s;
	rev64		v2.4s, v2.4s;
	rev64		v1.4s, v1.4s;
	rev64		v0.4s, v0.4s;
	ext		v7.16b, v7.16b, v7.16b, #8;
	ext		v6.16b, v6.16b, v6.16b, #8;
	ext		v5.16b, v5.16b, v5.16b, #8;
	ext		v4.16b, v4.16b, v4.16b, #8;
	ext		v3.16b, v3.16b, v3.16b, #8;
	ext		v2.16b, v2.16b, v2.16b, #8;
	ext		v1.16b, v1.16b, v1.16b, #8;
	ext		v0.16b, v0.16b, v0.16b, #8;
	st1		{v7.16b}, [x2], #16;
	st1		{v6.16b}, [x2], #16;
	st1		{v5.16b}, [x2], #16;
	st1		{v4.16b}, [x2], #16;
	st1		{v3.16b}, [x2], #16;
	st1		{v2.16b}, [x2], #16;
	st1		{v1.16b}, [x2], #16;
	st1		{v0.16b}, [x2];

	ret;
SYM_FUNC_END(sm4_ce_expand_key)

.align 3
SYM_FUNC_START(sm4_ce_crypt_block)
	/* input:
	 *   x0: round key array, CTX
	 *   x1: dst
	 *   x2: src
	 */
	PREPARE;

	ld1		{v0.16b}, [x2];
	SM4_CRYPT_BLK(v0);
	st1		{v0.16b}, [x1];

	ret;
SYM_FUNC_END(sm4_ce_crypt_block)

.align 3
SYM_FUNC_START(sm4_ce_crypt)
	/* input:
	 *   x0: round key array, CTX
	 *   x1: dst
	 *   x2: src
	 *   w3: nblocks
	 */
	PREPARE;

.Lcrypt_loop_blk:
	sub		w3, w3, #8;
	tbnz		w3, #31, .Lcrypt_tail8;

	ld1		{v0.16b-v3.16b}, [x2], #64;
	ld1		{v4.16b-v7.16b}, [x2], #64;

	SM4_CRYPT_BLK8(v0, v1, v2, v3, v4, v5, v6, v7);

	st1		{v0.16b-v3.16b}, [x1], #64;
	st1		{v4.16b-v7.16b}, [x1], #64;

	cbz		w3, .Lcrypt_end;
	b		.Lcrypt_loop_blk;

.Lcrypt_tail8:
	add		w3, w3, #8;
	cmp		w3, #4;
	blt		.Lcrypt_tail4;

	sub		w3, w3, #4;

	ld1		{v0.16b-v3.16b}, [x2], #64;
	SM4_CRYPT_BLK4(v0, v1, v2, v3);
	st1		{v0.16b-v3.16b}, [x1], #64;

	cbz		w3, .Lcrypt_end;

.Lcrypt_tail4:
	sub		w3, w3, #1;

	ld1		{v0.16b}, [x2], #16;
	SM4_CRYPT_BLK(v0);
	st1		{v0.16b}, [x1], #16;

	cbnz		w3, .Lcrypt_tail4;

.Lcrypt_end:
	ret;
SYM_FUNC_END(sm4_ce_crypt)

.align 3
SYM_FUNC_START(sm4_ce_cbc_enc)
	/* input:
	 *   x0: round key array, CTX
	 *   x1: dst
	 *   x2: src
	 *   x3: iv (big endian, 128 bit)
	 *   w4: nblocks
	 */
	PREPARE;

	ld1		{RIV.16b}, [x3];

.Lcbc_enc_loop:
	sub		w4, w4, #1;

	ld1		{RTMP0.16b}, [x2], #16;
	eor		RIV.16b, RIV.16b, RTMP0.16b;

	SM4_CRYPT_BLK(RIV);

	st1		{RIV.16b}, [x1], #16;

	cbnz		w4, .Lcbc_enc_loop;

	/* store new IV */
	st1		{RIV.16b}, [x3];

	ret;
SYM_FUNC_END(sm4_ce_cbc_enc)

.align 3
SYM_FUNC_START(sm4_ce_cbc_dec)
	/* input:
	 *   x0: round key array, CTX
	 *   x1: dst
	 *   x2: src
	 *   x3: iv (big endian, 128 bit)
	 *   w4: nblocks
	 */
	PREPARE;

	ld1		{RIV.16b}, [x3];

.Lcbc_loop_blk:
	sub		w4, w4, #8;
	tbnz		w4, #31, .Lcbc_tail8;

	ld1		{v0.16b-v3.16b}, [x2], #64;
	ld1		{v4.16b-v7.16b}, [x2];

	SM4_CRYPT_BLK8(v0, v1, v2, v3, v4, v5, v6, v7);

	sub		x2, x2, #64;
	eor		v0.16b, v0.16b, RIV.16b;
	ld1		{RTMP0.16b-RTMP3.16b}, [x2], #64;
	eor		v1.16b, v1.16b, RTMP0.16b;
	eor		v2.16b, v2.16b, RTMP1.16b;
	eor		v3.16b, v3.16b, RTMP2.16b;
	st1		{v0.16b-v3.16b}, [x1], #64;

	eor		v4.16b, v4.16b, RTMP3.16b;
	ld1		{RTMP0.16b-RTMP3.16b}, [x2], #64;
	eor		v5.16b, v5.16b, RTMP0.16b;
	eor		v6.16b, v6.16b, RTMP1.16b;
	eor		v7.16b, v7.16b, RTMP2.16b;

	mov		RIV.16b, RTMP3.16b;
	st1		{v4.16b-v7.16b}, [x1], #64;

	cbz		w4, .Lcbc_end;
	b		.Lcbc_loop_blk;

.Lcbc_tail8:
	add		w4, w4, #8;
	cmp		w4, #4;
	blt		.Lcbc_tail4;

	sub		w4, w4, #4;

	ld1		{v0.16b-v3.16b}, [x2];

	SM4_CRYPT_BLK4(v0, v1, v2, v3);

	eor		v0.16b, v0.16b, RIV.16b;
	ld1		{RTMP0.16b-RTMP3.16b}, [x2], #64;
	eor		v1.16b, v1.16b, RTMP0.16b;
	eor		v2.16b, v2.16b, RTMP1.16b;
	eor		v3.16b, v3.16b, RTMP2.16b;

	mov		RIV.16b, RTMP3.16b;
	st1		{v0.16b-v3.16b}, [x1], #64;

	cbz		w4, .Lcbc_end;

.Lcbc_tail4:
	sub		w4, w4, #1;

	ld1		{v0.16b}, [x2];

	SM4_CRYPT_BLK(v0);

	eor		v0.16b, v0.16b, RIV.16b;
	ld1		{RIV.16b}, [x2], #16;
	st1		{v0.16b}, [x1], #16;

	cbnz		w4, .Lcbc_tail4;

.Lcbc_end:
	/* store new IV */
	st1		{RIV.16b}, [x3];

	ret;
SYM_FUNC_END(sm4_ce_cbc_dec)

.align 3
SYM_FUNC_START(sm4_ce_cfb_enc)
	/* input:
	 *   x0: round key array, CTX
	 *   x1: dst
	 *   x2: src
	 *   x3: iv (big endian, 128 bit)
	 *   w4: nblocks
	 */
	PREPARE;

	ld1		{RIV.16b}, [x3];

.Lcfb_enc_loop:
	sub		w4, w4, #1;

	SM4_CRYPT_BLK(RIV);

	ld1		{RTMP0.16b}, [x2], #16;
	eor		RIV.16b, RIV.16b, RTMP0.16b;
	st1		{RIV.16b}, [x1], #16;

	cbnz		w4, .Lcfb_enc_loop;

	/* store new IV */
	st1		{RIV.16b}, [x3];

	ret;
SYM_FUNC_END(sm4_ce_cfb_enc)

.align 3
SYM_FUNC_START(sm4_ce_cfb_dec)
	/* input:
	 *   x0: round key array, CTX
	 *   x1: dst
	 *   x2: src
	 *   x3: iv (big endian, 128 bit)
	 *   w4: nblocks
	 */
	PREPARE;

	ld1		{v0.16b}, [x3];

.Lcfb_loop_blk:
	sub		w4, w4, #8;
	tbnz		w4, #31, .Lcfb_tail8;

	ld1		{v1.16b, v2.16b, v3.16b}, [x2], #48;
	ld1		{v4.16b-v7.16b}, [x2];

	SM4_CRYPT_BLK8(v0, v1, v2, v3, v4, v5, v6, v7);

	sub		x2, x2, #48;
	ld1		{RTMP0.16b-RTMP3.16b}, [x2], #64;
	eor		v0.16b, v0.16b, RTMP0.16b;
	eor		v1.16b, v1.16b, RTMP1.16b;
	eor		v2.16b, v2.16b, RTMP2.16b;
	eor		v3.16b, v3.16b, RTMP3.16b;
	st1		{v0.16b-v3.16b}, [x1], #64;

	ld1		{RTMP0.16b-RTMP3.16b}, [x2], #64;
	eor		v4.16b, v4.16b, RTMP0.16b;
	eor		v5.16b, v5.16b, RTMP1.16b;
	eor		v6.16b, v6.16b, RTMP2.16b;
	eor		v7.16b, v7.16b, RTMP3.16b;
	st1		{v4.16b-v7.16b}, [x1], #64;

	mov		v0.16b, RTMP3.16b;

	cbz		w4, .Lcfb_end;
	b		.Lcfb_loop_blk;

.Lcfb_tail8:
	add		w4, w4, #8;
	cmp		w4, #4;
	blt		.Lcfb_tail4;

	sub		w4, w4, #4;

	ld1		{v1.16b, v2.16b, v3.16b}, [x2];

	SM4_CRYPT_BLK4(v0, v1, v2, v3);

	ld1		{RTMP0.16b-RTMP3.16b}, [x2], #64;
	eor		v0.16b, v0.16b, RTMP0.16b;
	eor		v1.16b, v1.16b, RTMP1.16b;
	eor		v2.16b, v2.16b, RTMP2.16b;
	eor		v3.16b, v3.16b, RTMP3.16b;
	st1		{v0.16b-v3.16b}, [x1], #64;

	mov		v0.16b, RTMP3.16b;

	cbz		w4, .Lcfb_end;

.Lcfb_tail4:
	sub		w4, w4, #1;

	SM4_CRYPT_BLK(v0);

	ld1		{RTMP0.16b}, [x2], #16;
	eor		v0.16b, v0.16b, RTMP0.16b;
	st1		{v0.16b}, [x1], #16;

	mov		v0.16b, RTMP0.16b;

	cbnz		w4, .Lcfb_tail4;

.Lcfb_end:
	/* store new IV */
	st1		{v0.16b}, [x3];

	ret;
SYM_FUNC_END(sm4_ce_cfb_dec)

.align 3
SYM_FUNC_START(sm4_ce_ctr_enc)
	/* input:
	 *   x0: round key array, CTX
	 *   x1: dst
	 *   x2: src
	 *   x3: ctr (big endian, 128 bit)
	 *   w4: nblocks
	 */
	PREPARE;

	ldp		x7, x8, [x3];
	rev		x7, x7;
	rev		x8, x8;

.Lctr_loop_blk:
	sub		w4, w4, #8;
	tbnz		w4, #31, .Lctr_tail8;

#define inc_le128(vctr)                     \
	mov		vctr.d[1], x8;      \
	mov		vctr.d[0], x7;      \
	adds		x8, x8, #1;         \
	adc		x7, x7, xzr;        \
	rev64		vctr.16b, vctr.16b;

	/* construct CTRs */
	inc_le128(v0);			/* +0 */
	inc_le128(v1);			/* +1 */
	inc_le128(v2);			/* +2 */
	inc_le128(v3);			/* +3 */
	inc_le128(v4);			/* +4 */
	inc_le128(v5);			/* +5 */
	inc_le128(v6);			/* +6 */
	inc_le128(v7);			/* +7 */

	SM4_CRYPT_BLK8(v0, v1, v2, v3, v4, v5, v6, v7);

	ld1		{RTMP0.16b-RTMP3.16b}, [x2], #64;
	eor		v0.16b, v0.16b, RTMP0.16b;
	eor		v1.16b, v1.16b, RTMP1.16b;
	eor		v2.16b, v2.16b, RTMP2.16b;
	eor		v3.16b, v3.16b, RTMP3.16b;
	st1		{v0.16b-v3.16b}, [x1], #64;

	ld1		{RTMP0.16b-RTMP3.16b}, [x2], #64;
	eor		v4.16b, v4.16b, RTMP0.16b;
	eor		v5.16b, v5.16b, RTMP1.16b;
	eor		v6.16b, v6.16b, RTMP2.16b;
	eor		v7.16b, v7.16b, RTMP3.16b;
	st1		{v4.16b-v7.16b}, [x1], #64;

	cbz		w4, .Lctr_end;
	b		.Lctr_loop_blk;

.Lctr_tail8:
	add		w4, w4, #8;
	cmp		w4, #4;
	blt		.Lctr_tail4;

	sub		w4, w4, #4;

	/* construct CTRs */
	inc_le128(v0);			/* +0 */
	inc_le128(v1);			/* +1 */
	inc_le128(v2);			/* +2 */
	inc_le128(v3);			/* +3 */

	SM4_CRYPT_BLK4(v0, v1, v2, v3);

	ld1		{RTMP0.16b-RTMP3.16b}, [x2], #64;
	eor		v0.16b, v0.16b, RTMP0.16b;
	eor		v1.16b, v1.16b, RTMP1.16b;
	eor		v2.16b, v2.16b, RTMP2.16b;
	eor		v3.16b, v3.16b, RTMP3.16b;
	st1		{v0.16b-v3.16b}, [x1], #64;

	cbz		w4, .Lctr_end;

.Lctr_tail4:
	sub		w4, w4, #1;

	/* construct CTRs */
	inc_le128(v0);

	SM4_CRYPT_BLK(v0);

	ld1		{RTMP0.16b}, [x2], #16;
	eor		v0.16b, v0.16b, RTMP0.16b;
	st1		{v0.16b}, [x1], #16;

	cbnz		w4, .Lctr_tail4;

.Lctr_end:
	/* store new CTR */
	rev		x7, x7;
	rev		x8, x8;
	stp		x7, x8, [x3];

	ret;
SYM_FUNC_END(sm4_ce_ctr_enc)