arch/x86/crypto/sha512-mb/sha512_mb_mgr_submit_avx2.S - linux - Git at Google

 /*
  * Buffer submit code for multi buffer SHA512 algorithm
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
  *
  * GPL LICENSE SUMMARY
  *
  * Copyright(c) 2016 Intel Corporation.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
  * published by the Free Software Foundation.
  *
  * This program is distributed in the hope that it will be useful, but
  * WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
  * Contact Information:
  *     Megha Dey <megha.dey@linux.intel.com>
  *
  * BSD LICENSE
  *
  * Copyright(c) 2016 Intel Corporation.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  *
  *   * Redistributions of source code must retain the above copyright
  *     notice, this list of conditions and the following disclaimer.
  *   * Redistributions in binary form must reproduce the above copyright
  *     notice, this list of conditions and the following disclaimer in
  *     the documentation and/or other materials provided with the
  *     distribution.
  *   * Neither the name of Intel Corporation nor the names of its
  *     contributors may be used to endorse or promote products derived
  *     from this software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */

 #include <linux/linkage.h>
 #include <asm/frame.h>
 #include "sha512_mb_mgr_datastruct.S"

 .extern sha512_x4_avx2

 #define arg1    %rdi
 #define arg2    %rsi

 #define idx             %rdx
 #define last_len        %rdx

 #define size_offset     %rcx
 #define tmp2            %rcx

 # Common definitions
 #define state   arg1
 #define job     arg2
 #define len2    arg2
 #define p2      arg2

 #define p               %r11
 #define start_offset    %r11

 #define unused_lanes    %rbx

 #define job_rax         %rax
 #define len             %rax

 #define lane            %r12
 #define tmp3            %r12
 #define lens3           %r12

 #define extra_blocks    %r8
 #define lens0           %r8

 #define tmp             %r9
 #define lens1           %r9

 #define lane_data       %r10
 #define lens2           %r10

 #define DWORD_len %eax

 # JOB* sha512_mb_mgr_submit_avx2(MB_MGR *state, JOB *job)
 # arg 1 : rcx : state
 # arg 2 : rdx : job
 ENTRY(sha512_mb_mgr_submit_avx2)
 	FRAME_BEGIN
 	push	%rbx
 	push	%r12

         mov     _unused_lanes(state), unused_lanes
         movzb     %bl,lane
         shr     $8, unused_lanes
         imul    $_LANE_DATA_size, lane,lane_data
         movl    $STS_BEING_PROCESSED, _status(job)
 	lea     _ldata(state, lane_data), lane_data
         mov     unused_lanes, _unused_lanes(state)
         movl    _len(job),  DWORD_len

 	mov     job, _job_in_lane(lane_data)
         movl    DWORD_len,_lens+4(state , lane, 8)

 	# Load digest words from result_digest
 	vmovdqu	_result_digest+0*16(job), %xmm0
 	vmovdqu _result_digest+1*16(job), %xmm1
 	vmovdqu	_result_digest+2*16(job), %xmm2
         vmovdqu	_result_digest+3*16(job), %xmm3

 	vmovq    %xmm0, _args_digest(state, lane, 8)
 	vpextrq  $1, %xmm0, _args_digest+1*32(state , lane, 8)
 	vmovq    %xmm1, _args_digest+2*32(state , lane, 8)
 	vpextrq  $1, %xmm1, _args_digest+3*32(state , lane, 8)
 	vmovq    %xmm2, _args_digest+4*32(state , lane, 8)
 	vpextrq  $1, %xmm2, _args_digest+5*32(state , lane, 8)
 	vmovq    %xmm3, _args_digest+6*32(state , lane, 8)
 	vpextrq  $1, %xmm3, _args_digest+7*32(state , lane, 8)

 	mov     _buffer(job), p
 	mov     p, _args_data_ptr(state, lane, 8)

 	cmp     $0xFF, unused_lanes
 	jne     return_null

 start_loop:

 	# Find min length
 	mov     _lens+0*8(state),lens0
 	mov     lens0,idx
 	mov     _lens+1*8(state),lens1
 	cmp     idx,lens1
 	cmovb   lens1, idx
 	mov     _lens+2*8(state),lens2
 	cmp     idx,lens2
 	cmovb   lens2,idx
 	mov     _lens+3*8(state),lens3
 	cmp     idx,lens3
 	cmovb   lens3,idx
 	mov     idx,len2
 	and     $0xF,idx
 	and     $~0xFF,len2
 	jz      len_is_0

 	sub     len2,lens0
 	sub     len2,lens1
 	sub     len2,lens2
 	sub     len2,lens3
 	shr     $32,len2
 	mov     lens0, _lens + 0*8(state)
 	mov     lens1, _lens + 1*8(state)
 	mov     lens2, _lens + 2*8(state)
 	mov     lens3, _lens + 3*8(state)

 	# "state" and "args" are the same address, arg1
 	# len is arg2
 	call    sha512_x4_avx2
 	# state and idx are intact

 len_is_0:

 	# process completed job "idx"
 	imul    $_LANE_DATA_size, idx, lane_data
 	lea     _ldata(state, lane_data), lane_data

 	mov     _job_in_lane(lane_data), job_rax
 	mov     _unused_lanes(state), unused_lanes
 	movq    $0, _job_in_lane(lane_data)
 	movl    $STS_COMPLETED, _status(job_rax)
 	shl     $8, unused_lanes
 	or      idx, unused_lanes
 	mov     unused_lanes, _unused_lanes(state)

 	movl	$0xFFFFFFFF,_lens+4(state,idx,8)
 	vmovq    _args_digest+0*32(state , idx, 8), %xmm0
 	vpinsrq  $1, _args_digest+1*32(state , idx, 8), %xmm0, %xmm0
 	vmovq    _args_digest+2*32(state , idx, 8), %xmm1
 	vpinsrq  $1, _args_digest+3*32(state , idx, 8), %xmm1, %xmm1
 	vmovq    _args_digest+4*32(state , idx, 8), %xmm2
 	vpinsrq  $1, _args_digest+5*32(state , idx, 8), %xmm2, %xmm2
 	vmovq    _args_digest+6*32(state , idx, 8), %xmm3
 	vpinsrq  $1, _args_digest+7*32(state , idx, 8), %xmm3, %xmm3

 	vmovdqu  %xmm0, _result_digest + 0*16(job_rax)
 	vmovdqu  %xmm1, _result_digest + 1*16(job_rax)
 	vmovdqu  %xmm2, _result_digest + 2*16(job_rax)
 	vmovdqu  %xmm3, _result_digest + 3*16(job_rax)

 return:
 	pop	%r12
 	pop	%rbx
 	FRAME_END
 	ret

 return_null:
 	xor     job_rax, job_rax
 	jmp     return
 ENDPROC(sha512_mb_mgr_submit_avx2)

 /* UNUSED?
 .section	.rodata.cst16, "aM", @progbits, 16
 .align 16
 H0:     .int  0x6a09e667
 H1:     .int  0xbb67ae85
 H2:     .int  0x3c6ef372
 H3:     .int  0xa54ff53a
 H4:     .int  0x510e527f
 H5:     .int  0x9b05688c
 H6:     .int  0x1f83d9ab
 H7:     .int  0x5be0cd19
 */
	/*
	* Buffer submit code for multi buffer SHA512 algorithm
	*
	* This file is provided under a dual BSD/GPLv2 license. When using or
	* redistributing this file, you may do so under either license.
	*
	* GPL LICENSE SUMMARY
	*
	* Copyright(c) 2016 Intel Corporation.
	*
	* This program is free software; you can redistribute it and/or modify
	* it under the terms of version 2 of the GNU General Public License as
	* published by the Free Software Foundation.
	*
	* This program is distributed in the hope that it will be useful, but
	* WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* Contact Information:
	* Megha Dey <megha.dey@linux.intel.com>
	*
	* BSD LICENSE
	*
	* Copyright(c) 2016 Intel Corporation.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* * Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* * Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	* * Neither the name of Intel Corporation nor the names of its
	* contributors may be used to endorse or promote products derived
	* from this software without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	*/

	#include <linux/linkage.h>
	#include <asm/frame.h>
	#include "sha512_mb_mgr_datastruct.S"

	.extern sha512_x4_avx2

	#define arg1 %rdi
	#define arg2 %rsi

	#define idx %rdx
	#define last_len %rdx

	#define size_offset %rcx
	#define tmp2 %rcx

	# Common definitions
	#define state arg1
	#define job arg2
	#define len2 arg2
	#define p2 arg2

	#define p %r11
	#define start_offset %r11

	#define unused_lanes %rbx

	#define job_rax %rax
	#define len %rax

	#define lane %r12
	#define tmp3 %r12
	#define lens3 %r12

	#define extra_blocks %r8
	#define lens0 %r8

	#define tmp %r9
	#define lens1 %r9

	#define lane_data %r10
	#define lens2 %r10

	#define DWORD_len %eax

	# JOB* sha512_mb_mgr_submit_avx2(MB_MGR state, JOB job)
	# arg 1 : rcx : state
	# arg 2 : rdx : job
	ENTRY(sha512_mb_mgr_submit_avx2)
	FRAME_BEGIN
	push %rbx
	push %r12

	mov _unused_lanes(state), unused_lanes
	movzb %bl,lane
	shr $8, unused_lanes
	imul $_LANE_DATA_size, lane,lane_data
	movl $STS_BEING_PROCESSED, _status(job)
	lea _ldata(state, lane_data), lane_data
	mov unused_lanes, _unused_lanes(state)
	movl _len(job), DWORD_len

	mov job, _job_in_lane(lane_data)
	movl DWORD_len,_lens+4(state , lane, 8)

	# Load digest words from result_digest
	vmovdqu _result_digest+0*16(job), %xmm0
	vmovdqu _result_digest+1*16(job), %xmm1
	vmovdqu _result_digest+2*16(job), %xmm2
	vmovdqu _result_digest+3*16(job), %xmm3

	vmovq %xmm0, _args_digest(state, lane, 8)
	vpextrq $1, %xmm0, _args_digest+1*32(state , lane, 8)
	vmovq %xmm1, _args_digest+2*32(state , lane, 8)
	vpextrq $1, %xmm1, _args_digest+3*32(state , lane, 8)
	vmovq %xmm2, _args_digest+4*32(state , lane, 8)
	vpextrq $1, %xmm2, _args_digest+5*32(state , lane, 8)
	vmovq %xmm3, _args_digest+6*32(state , lane, 8)
	vpextrq $1, %xmm3, _args_digest+7*32(state , lane, 8)

	mov _buffer(job), p
	mov p, _args_data_ptr(state, lane, 8)

	cmp $0xFF, unused_lanes
	jne return_null

	start_loop:

	# Find min length
	mov _lens+0*8(state),lens0
	mov lens0,idx
	mov _lens+1*8(state),lens1
	cmp idx,lens1
	cmovb lens1, idx
	mov _lens+2*8(state),lens2
	cmp idx,lens2
	cmovb lens2,idx
	mov _lens+3*8(state),lens3
	cmp idx,lens3
	cmovb lens3,idx
	mov idx,len2
	and $0xF,idx
	and $~0xFF,len2
	jz len_is_0

	sub len2,lens0
	sub len2,lens1
	sub len2,lens2
	sub len2,lens3
	shr $32,len2
	mov lens0, _lens + 0*8(state)
	mov lens1, _lens + 1*8(state)
	mov lens2, _lens + 2*8(state)
	mov lens3, _lens + 3*8(state)

	# "state" and "args" are the same address, arg1
	# len is arg2
	call sha512_x4_avx2
	# state and idx are intact

	len_is_0:

	# process completed job "idx"
	imul $_LANE_DATA_size, idx, lane_data
	lea _ldata(state, lane_data), lane_data

	mov _job_in_lane(lane_data), job_rax
	mov _unused_lanes(state), unused_lanes
	movq $0, _job_in_lane(lane_data)
	movl $STS_COMPLETED, _status(job_rax)
	shl $8, unused_lanes
	or idx, unused_lanes
	mov unused_lanes, _unused_lanes(state)

	movl $0xFFFFFFFF,_lens+4(state,idx,8)
	vmovq _args_digest+0*32(state , idx, 8), %xmm0
	vpinsrq $1, _args_digest+1*32(state , idx, 8), %xmm0, %xmm0
	vmovq _args_digest+2*32(state , idx, 8), %xmm1
	vpinsrq $1, _args_digest+3*32(state , idx, 8), %xmm1, %xmm1
	vmovq _args_digest+4*32(state , idx, 8), %xmm2
	vpinsrq $1, _args_digest+5*32(state , idx, 8), %xmm2, %xmm2
	vmovq _args_digest+6*32(state , idx, 8), %xmm3
	vpinsrq $1, _args_digest+7*32(state , idx, 8), %xmm3, %xmm3

	vmovdqu %xmm0, _result_digest + 0*16(job_rax)
	vmovdqu %xmm1, _result_digest + 1*16(job_rax)
	vmovdqu %xmm2, _result_digest + 2*16(job_rax)
	vmovdqu %xmm3, _result_digest + 3*16(job_rax)

	return:
	pop %r12
	pop %rbx
	FRAME_END
	ret

	return_null:
	xor job_rax, job_rax
	jmp return
	ENDPROC(sha512_mb_mgr_submit_avx2)

	/* UNUSED?
	.section .rodata.cst16, "aM", @progbits, 16
	.align 16
	H0: .int 0x6a09e667
	H1: .int 0xbb67ae85
	H2: .int 0x3c6ef372
	H3: .int 0xa54ff53a
	H4: .int 0x510e527f
	H5: .int 0x9b05688c
	H6: .int 0x1f83d9ab
	H7: .int 0x5be0cd19
	*/