#!/usr/bin/env perl

# Specific modes implementations for SPARC Architecture 2011. There

# is T4 dependency though, an ASI value that is not specified in the

# Architecture Manual. But as SPARC universe is rather monocultural,

# we imply that processor capable of executing crypto instructions

# can handle the ASI in question as well. This means that we ought to

# keep eyes open when new processors emerge...

#

# As for above mentioned ASI. It's so called "block initializing

# store" which cancels "read" in "read-update-write" on cache lines.

# This is "cooperative" optimization, as it reduces overall pressure

# on memory interface. Benefits can't be observed/quantified with

# usual benchmarks, on the contrary you can notice that single-thread

# performance for parallelizable modes is ~1.5% worse for largest

# block sizes [though few percent better for not so long ones]. All

# this based on suggestions from David Miller.

sub asm_init {		# to be called with @ARGV as argument

    for (@_)		{ $::abibits=64 if (/\-m64/ || /\-xarch\=v9/); }

    if ($::abibits==64)	{ $::bias=2047; $::frame=192; $::size_t_cc="%xcc"; }

    else		{ $::bias=0;    $::frame=112; $::size_t_cc="%icc"; }

}

# unified interface

my ($inp,$out,$len,$key,$ivec)=map("%i$_",(0..5));

# local variables

my ($ileft,$iright,$ooff,$omask,$ivoff,$blk_init)=map("%l$_",(0..7));

sub alg_cbc_encrypt_implement {

my ($alg,$bits) = @_;

$::code.=<<___;

.globl	${alg}${bits}_t4_cbc_encrypt

.align	32

${alg}${bits}_t4_cbc_encrypt:

	save		%sp, -$::frame, %sp

	sub		$inp, $out, $blk_init	! $inp!=$out

___

$::code.=<<___ if (!$::evp);

	andcc		$ivec, 7, $ivoff

	alignaddr	$ivec, %g0, $ivec

	ldd		[$ivec + 0], %f0	! load ivec

	bz,pt		%icc, 1f

	ldd		[$ivec + 8], %f2

	ldd		[$ivec + 16], %f4

	faligndata	%f0, %f2, %f0

	faligndata	%f2, %f4, %f2

1:

___

$::code.=<<___ if ($::evp);

	ld		[$ivec + 0], %f0

	ld		[$ivec + 4], %f1

	ld		[$ivec + 8], %f2

	ld		[$ivec + 12], %f3

___

$::code.=<<___;

	prefetch	[$inp], 20

	prefetch	[$inp + 63], 20

	call		_${alg}${bits}_load_enckey

	and		$inp, 7, $ileft

	andn		$inp, 7, $inp

	sll		$ileft, 3, $ileft

	mov		64, $iright

	mov		0xff, $omask

	sub		$iright, $ileft, $iright

	and		$out, 7, $ooff

	cmp		$len, 127

	movrnz		$ooff, 0, $blk_init		! if (	$out&7 ||

	movleu		$::size_t_cc, 0, $blk_init	!	$len<128 ||

	brnz,pn		$blk_init, .L${bits}cbc_enc_blk	!	$inp==$out)

	srl		$omask, $ooff, $omask

	alignaddrl	$out, %g0, $out

	srlx		$len, 4, $len

	prefetch	[$out], 22

.L${bits}_cbc_enc_loop:

	ldx		[$inp + 0], %o0

	brz,pt		$ileft, 4f

	ldx		[$inp + 8], %o1

	ldx		[$inp + 16], %o2

	sllx		%o0, $ileft, %o0

	srlx		%o1, $iright, %g1

	sllx		%o1, $ileft, %o1

	or		%g1, %o0, %o0

	srlx		%o2, $iright, %o2

	or		%o2, %o1, %o1

4:

	xor		%g4, %o0, %o0		! ^= rk[0]

	xor		%g5, %o1, %o1

	movxtod		%o0, %f12

	movxtod		%o1, %f14

	fxor		%f12, %f0, %f0		! ^= ivec

	fxor		%f14, %f2, %f2

	prefetch	[$out + 63], 22

	prefetch	[$inp + 16+63], 20

	call		_${alg}${bits}_encrypt_1x

	add		$inp, 16, $inp

	brnz,pn		$ooff, 2f

	sub		$len, 1, $len

	std		%f0, [$out + 0]

	std		%f2, [$out + 8]

	brnz,pt		$len, .L${bits}_cbc_enc_loop

	add		$out, 16, $out

___

$::code.=<<___ if ($::evp);

	st		%f0, [$ivec + 0]

	st		%f1, [$ivec + 4]

	st		%f2, [$ivec + 8]

	st		%f3, [$ivec + 12]

___

$::code.=<<___ if (!$::evp);

	brnz,pn		$ivoff, 3f

	nop

	std		%f0, [$ivec + 0]	! write out ivec

	std		%f2, [$ivec + 8]

___

$::code.=<<___;

	ret

	restore

.align	16

2:	ldxa		[$inp]0x82, %o0		! avoid read-after-write hazard

						! and ~3x deterioration

						! in inp==out case

	faligndata	%f0, %f0, %f4		! handle unaligned output

	faligndata	%f0, %f2, %f6

	faligndata	%f2, %f2, %f8

	stda		%f4, [$out + $omask]0xc0	! partial store

	std		%f6, [$out + 8]

	add		$out, 16, $out

	orn		%g0, $omask, $omask

	stda		%f8, [$out + $omask]0xc0	! partial store

	brnz,pt		$len, .L${bits}_cbc_enc_loop+4

	orn		%g0, $omask, $omask

___

$::code.=<<___ if ($::evp);

	st		%f0, [$ivec + 0]

	st		%f1, [$ivec + 4]

	st		%f2, [$ivec + 8]

	st		%f3, [$ivec + 12]

___

$::code.=<<___ if (!$::evp);

	brnz,pn		$ivoff, 3f

	nop

	std		%f0, [$ivec + 0]	! write out ivec

	std		%f2, [$ivec + 8]

	ret

	restore

.align	16

3:	alignaddrl	$ivec, $ivoff, %g0	! handle unaligned ivec

	mov		0xff, $omask

	srl		$omask, $ivoff, $omask

	faligndata	%f0, %f0, %f4

	faligndata	%f0, %f2, %f6

	faligndata	%f2, %f2, %f8

	stda		%f4, [$ivec + $omask]0xc0

	std		%f6, [$ivec + 8]

	add		$ivec, 16, $ivec

	orn		%g0, $omask, $omask

	stda		%f8, [$ivec + $omask]0xc0

___

$::code.=<<___;

	ret

	restore

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

.align	32

.L${bits}cbc_enc_blk:

	add	$out, $len, $blk_init

	and	$blk_init, 63, $blk_init	! tail

	sub	$len, $blk_init, $len

	add	$blk_init, 15, $blk_init	! round up to 16n

	srlx	$len, 4, $len

	srl	$blk_init, 4, $blk_init

.L${bits}_cbc_enc_blk_loop:

	ldx		[$inp + 0], %o0

	brz,pt		$ileft, 5f

	ldx		[$inp + 8], %o1

	ldx		[$inp + 16], %o2

	sllx		%o0, $ileft, %o0

	srlx		%o1, $iright, %g1

	sllx		%o1, $ileft, %o1

	or		%g1, %o0, %o0

	srlx		%o2, $iright, %o2

	or		%o2, %o1, %o1

5:

	xor		%g4, %o0, %o0		! ^= rk[0]

	xor		%g5, %o1, %o1

	movxtod		%o0, %f12

	movxtod		%o1, %f14

	fxor		%f12, %f0, %f0		! ^= ivec

	fxor		%f14, %f2, %f2

	prefetch	[$inp + 16+63], 20

	call		_${alg}${bits}_encrypt_1x

	add		$inp, 16, $inp

	sub		$len, 1, $len

	stda		%f0, [$out]0xe2		! ASI_BLK_INIT, T4-specific

	add		$out, 8, $out

	stda		%f2, [$out]0xe2		! ASI_BLK_INIT, T4-specific

	brnz,pt		$len, .L${bits}_cbc_enc_blk_loop

	add		$out, 8, $out

	membar		#StoreLoad|#StoreStore

	brnz,pt		$blk_init, .L${bits}_cbc_enc_loop

	mov		$blk_init, $len

___

$::code.=<<___ if ($::evp);

	st		%f0, [$ivec + 0]

	st		%f1, [$ivec + 4]

	st		%f2, [$ivec + 8]

	st		%f3, [$ivec + 12]

___

$::code.=<<___ if (!$::evp);

	brnz,pn		$ivoff, 3b

	nop

	std		%f0, [$ivec + 0]	! write out ivec

	std		%f2, [$ivec + 8]

___

$::code.=<<___;

	ret

	restore

.type	${alg}${bits}_t4_cbc_encrypt,#function

.size	${alg}${bits}_t4_cbc_encrypt,.-${alg}${bits}_t4_cbc_encrypt

___

}

sub alg_cbc_decrypt_implement {

my ($alg,$bits) = @_;

$::code.=<<___;

.globl	${alg}${bits}_t4_cbc_decrypt

.align	32

${alg}${bits}_t4_cbc_decrypt:

	save		%sp, -$::frame, %sp

	sub		$inp, $out, $blk_init	! $inp!=$out

___

$::code.=<<___ if (!$::evp);

	andcc		$ivec, 7, $ivoff

	alignaddr	$ivec, %g0, $ivec

	ldd		[$ivec + 0], %f12	! load ivec

	bz,pt		%icc, 1f

	ldd		[$ivec + 8], %f14

	ldd		[$ivec + 16], %f0

	faligndata	%f12, %f14, %f12

	faligndata	%f14, %f0, %f14

1:

___

$::code.=<<___ if ($::evp);

	ld		[$ivec + 0], %f12	! load ivec

	ld		[$ivec + 4], %f13

	ld		[$ivec + 8], %f14

	ld		[$ivec + 12], %f15

___

$::code.=<<___;

	prefetch	[$inp], 20

	prefetch	[$inp + 63], 20

	call		_${alg}${bits}_load_deckey

	and		$inp, 7, $ileft

	andn		$inp, 7, $inp

	sll		$ileft, 3, $ileft

	mov		64, $iright

	mov		0xff, $omask

	sub		$iright, $ileft, $iright

	and		$out, 7, $ooff

	cmp		$len, 255

	movrnz		$ooff, 0, $blk_init		! if (	$out&7 ||

	movleu		$::size_t_cc, 0, $blk_init	!	$len<256 ||

	brnz,pn		$blk_init, .L${bits}cbc_dec_blk	!	$inp==$out)

	srl		$omask, $ooff, $omask

	andcc		$len, 16, %g0		! is number of blocks even?

	srlx		$len, 4, $len

	alignaddrl	$out, %g0, $out

	bz		%icc, .L${bits}_cbc_dec_loop2x

	prefetch	[$out], 22

.L${bits}_cbc_dec_loop:

	ldx		[$inp + 0], %o0

	brz,pt		$ileft, 4f

	ldx		[$inp + 8], %o1

	ldx		[$inp + 16], %o2

	sllx		%o0, $ileft, %o0

	srlx		%o1, $iright, %g1

	sllx		%o1, $ileft, %o1

	or		%g1, %o0, %o0

	srlx		%o2, $iright, %o2

	or		%o2, %o1, %o1

4:

	xor		%g4, %o0, %o2		! ^= rk[0]

	xor		%g5, %o1, %o3

	movxtod		%o2, %f0

	movxtod		%o3, %f2

	prefetch	[$out + 63], 22

	prefetch	[$inp + 16+63], 20

	call		_${alg}${bits}_decrypt_1x

	add		$inp, 16, $inp

	fxor		%f12, %f0, %f0		! ^= ivec

	fxor		%f14, %f2, %f2

	movxtod		%o0, %f12

	movxtod		%o1, %f14

	brnz,pn		$ooff, 2f

	sub		$len, 1, $len

	std		%f0, [$out + 0]

	std		%f2, [$out + 8]

	brnz,pt		$len, .L${bits}_cbc_dec_loop2x

	add		$out, 16, $out

___

$::code.=<<___ if ($::evp);

	st		%f12, [$ivec + 0]

	st		%f13, [$ivec + 4]

	st		%f14, [$ivec + 8]

	st		%f15, [$ivec + 12]

___

$::code.=<<___ if (!$::evp);

	brnz,pn		$ivoff, .L${bits}_cbc_dec_unaligned_ivec

	nop

	std		%f12, [$ivec + 0]	! write out ivec

	std		%f14, [$ivec + 8]

___

$::code.=<<___;

	ret

	restore

.align	16

2:	ldxa		[$inp]0x82, %o0		! avoid read-after-write hazard

						! and ~3x deterioration

						! in inp==out case

	faligndata	%f0, %f0, %f4		! handle unaligned output

	faligndata	%f0, %f2, %f6

	faligndata	%f2, %f2, %f8

	stda		%f4, [$out + $omask]0xc0	! partial store

	std		%f6, [$out + 8]

	add		$out, 16, $out

	orn		%g0, $omask, $omask

	stda		%f8, [$out + $omask]0xc0	! partial store

	brnz,pt		$len, .L${bits}_cbc_dec_loop2x+4

	orn		%g0, $omask, $omask

___

$::code.=<<___ if ($::evp);

	st		%f12, [$ivec + 0]

	st		%f13, [$ivec + 4]

	st		%f14, [$ivec + 8]

	st		%f15, [$ivec + 12]

___

$::code.=<<___ if (!$::evp);

	brnz,pn		$ivoff, .L${bits}_cbc_dec_unaligned_ivec

	nop

	std		%f12, [$ivec + 0]	! write out ivec

	std		%f14, [$ivec + 8]

___

$::code.=<<___;

	ret

	restore

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

.align	32

.L${bits}_cbc_dec_loop2x:

	ldx		[$inp + 0], %o0

	ldx		[$inp + 8], %o1

	ldx		[$inp + 16], %o2

	brz,pt		$ileft, 4f

	ldx		[$inp + 24], %o3

	ldx		[$inp + 32], %o4

	sllx		%o0, $ileft, %o0

	srlx		%o1, $iright, %g1

	or		%g1, %o0, %o0

	sllx		%o1, $ileft, %o1

	srlx		%o2, $iright, %g1

	or		%g1, %o1, %o1

	sllx		%o2, $ileft, %o2

	srlx		%o3, $iright, %g1

	or		%g1, %o2, %o2

	sllx		%o3, $ileft, %o3

	srlx		%o4, $iright, %o4

	or		%o4, %o3, %o3

4:

	xor		%g4, %o0, %o4		! ^= rk[0]

	xor		%g5, %o1, %o5

	movxtod		%o4, %f0

	movxtod		%o5, %f2

	xor		%g4, %o2, %o4

	xor		%g5, %o3, %o5

	movxtod		%o4, %f4

	movxtod		%o5, %f6

	prefetch	[$out + 63], 22

	prefetch	[$inp + 32+63], 20

	call		_${alg}${bits}_decrypt_2x

	add		$inp, 32, $inp

	movxtod		%o0, %f8

	movxtod		%o1, %f10

	fxor		%f12, %f0, %f0		! ^= ivec

	fxor		%f14, %f2, %f2

	movxtod		%o2, %f12

	movxtod		%o3, %f14

	fxor		%f8, %f4, %f4

	fxor		%f10, %f6, %f6

	brnz,pn		$ooff, 2f

	sub		$len, 2, $len

	std		%f0, [$out + 0]

	std		%f2, [$out + 8]

	std		%f4, [$out + 16]

	std		%f6, [$out + 24]

	brnz,pt		$len, .L${bits}_cbc_dec_loop2x

	add		$out, 32, $out

___

$::code.=<<___ if ($::evp);

	st		%f12, [$ivec + 0]

	st		%f13, [$ivec + 4]

	st		%f14, [$ivec + 8]

	st		%f15, [$ivec + 12]

___

$::code.=<<___ if (!$::evp);

	brnz,pn		$ivoff, .L${bits}_cbc_dec_unaligned_ivec

	nop

	std		%f12, [$ivec + 0]	! write out ivec

	std		%f14, [$ivec + 8]

___

$::code.=<<___;

	ret

	restore

.align	16

2:	ldxa		[$inp]0x82, %o0		! avoid read-after-write hazard

						! and ~3x deterioration

						! in inp==out case

	faligndata	%f0, %f0, %f8		! handle unaligned output

	faligndata	%f0, %f2, %f0

	faligndata	%f2, %f4, %f2

	faligndata	%f4, %f6, %f4

	faligndata	%f6, %f6, %f6

	stda		%f8, [$out + $omask]0xc0	! partial store

	std		%f0, [$out + 8]

	std		%f2, [$out + 16]

	std		%f4, [$out + 24]

	add		$out, 32, $out

	orn		%g0, $omask, $omask

	stda		%f6, [$out + $omask]0xc0	! partial store

	brnz,pt		$len, .L${bits}_cbc_dec_loop2x+4

	orn		%g0, $omask, $omask

___

$::code.=<<___ if ($::evp);

	st		%f12, [$ivec + 0]

	st		%f13, [$ivec + 4]

	st		%f14, [$ivec + 8]

	st		%f15, [$ivec + 12]

___

$::code.=<<___ if (!$::evp);

	brnz,pn		$ivoff, .L${bits}_cbc_dec_unaligned_ivec

	nop

	std		%f12, [$ivec + 0]	! write out ivec

	std		%f14, [$ivec + 8]

	ret

	restore

.align	16

.L${bits}_cbc_dec_unaligned_ivec:

	alignaddrl	$ivec, $ivoff, %g0	! handle unaligned ivec

	mov		0xff, $omask

	srl		$omask, $ivoff, $omask

	faligndata	%f12, %f12, %f0

	faligndata	%f12, %f14, %f2

	faligndata	%f14, %f14, %f4

	stda		%f0, [$ivec + $omask]0xc0

	std		%f2, [$ivec + 8]

	add		$ivec, 16, $ivec

	orn		%g0, $omask, $omask

	stda		%f4, [$ivec + $omask]0xc0

___

$::code.=<<___;

	ret

	restore

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

.align	32

.L${bits}cbc_dec_blk:

	add	$out, $len, $blk_init

	and	$blk_init, 63, $blk_init	! tail

	sub	$len, $blk_init, $len

	add	$blk_init, 15, $blk_init	! round up to 16n

	srlx	$len, 4, $len

	srl	$blk_init, 4, $blk_init

	sub	$len, 1, $len

	add	$blk_init, 1, $blk_init

.L${bits}_cbc_dec_blk_loop2x:

	ldx		[$inp + 0], %o0

	ldx		[$inp + 8], %o1

	ldx		[$inp + 16], %o2

	brz,pt		$ileft, 5f

	ldx		[$inp + 24], %o3

	ldx		[$inp + 32], %o4

	sllx		%o0, $ileft, %o0

	srlx		%o1, $iright, %g1

	or		%g1, %o0, %o0