#!/usr/bin/env perl

# ====================================================================

# Written by David S. Miller <[email protected]> and Andy Polyakov

# <[email protected]>. The module is licensed under 2-clause BSD

# license. November 2012. All rights reserved.

# ====================================================================

######################################################################

# Montgomery squaring-n-multiplication module for SPARC T4.

#

# The module consists of three parts:

#

# 1) collection of "single-op" subroutines that perform single

#    operation, Montgomery squaring or multiplication, on 512-,

#    1024-, 1536- and 2048-bit operands;

# 2) collection of "multi-op" subroutines that perform 5 squaring and

#    1 multiplication operations on operands of above lengths;

# 3) fall-back and helper VIS3 subroutines.

#

# RSA sign is dominated by multi-op subroutine, while RSA verify and

# DSA - by single-op. Special note about 4096-bit RSA verify result.

# Operands are too long for dedicated hardware and it's handled by

# VIS3 code, which is why you don't see any improvement. It's surely

# possible to improve it [by deploying 'mpmul' instruction], maybe in

# the future...

#

# Performance improvement.

#

# 64-bit process, VIS3:

#                   sign    verify    sign/s verify/s

# rsa 1024 bits 0.000628s 0.000028s   1592.4  35434.4

# rsa 2048 bits 0.003282s 0.000106s    304.7   9438.3

# rsa 4096 bits 0.025866s 0.000340s     38.7   2940.9

# dsa 1024 bits 0.000301s 0.000332s   3323.7   3013.9

# dsa 2048 bits 0.001056s 0.001233s    946.9    810.8

#

# 64-bit process, this module:

#                   sign    verify    sign/s verify/s

# rsa 1024 bits 0.000256s 0.000016s   3904.4  61411.9

# rsa 2048 bits 0.000946s 0.000029s   1056.8  34292.7

# rsa 4096 bits 0.005061s 0.000340s    197.6   2940.5

# dsa 1024 bits 0.000176s 0.000195s   5674.7   5130.5

# dsa 2048 bits 0.000296s 0.000354s   3383.2   2827.6

#

######################################################################

# 32-bit process, VIS3:

#                   sign    verify    sign/s verify/s

# rsa 1024 bits 0.000665s 0.000028s   1504.8  35233.3

# rsa 2048 bits 0.003349s 0.000106s    298.6   9433.4

# rsa 4096 bits 0.025959s 0.000341s     38.5   2934.8

# dsa 1024 bits 0.000320s 0.000341s   3123.3   2929.6

# dsa 2048 bits 0.001101s 0.001260s    908.2    793.4

#

# 32-bit process, this module:

#                   sign    verify    sign/s verify/s

# rsa 1024 bits 0.000301s 0.000017s   3317.1  60240.0

# rsa 2048 bits 0.001034s 0.000030s    966.9  33812.7

# rsa 4096 bits 0.005244s 0.000341s    190.7   2935.4

# dsa 1024 bits 0.000201s 0.000205s   4976.1   4879.2

# dsa 2048 bits 0.000328s 0.000360s   3051.1   2774.2

#

# 32-bit code is prone to performance degradation as interrupt rate

# dispatched to CPU executing the code grows. This is because in

# standard process of handling interrupt in 32-bit process context

# upper halves of most integer registers used as input or output are

# zeroed. This renders result invalid, and operation has to be re-run.

# If CPU is "bothered" with timer interrupts only, the penalty is

# hardly measurable. But in order to mitigate this problem for higher

# interrupt rates contemporary Linux kernel recognizes biased stack

# even in 32-bit process context and preserves full register contents.

# See http://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/commit/?id=517ffce4e1a03aea979fe3a18a3dd1761a24fafb

# for details.

$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;

push(@INC,"${dir}","${dir}../../perlasm");

require "sparcv9_modes.pl";

$code.=<<___;

#include "sparc_arch.h"

#ifdef	__arch64__

.register	%g2,#scratch

.register	%g3,#scratch

#endif

.section	".text",#alloc,#execinstr

#ifdef	__PIC__

SPARC_PIC_THUNK(%g1)

#endif

___

########################################################################

# Register layout for mont[mul|sqr] instructions.

# For details see "Oracle SPARC Architecture 2011" manual at

# http://www.oracle.com/technetwork/server-storage/sun-sparc-enterprise/documentation/.

#

my @R=map("%f".2*$_,(0..11,30,31,12..29));

my @N=(map("%l$_",(0..7)),map("%o$_",(0..5))); @N=(@N,@N,@N[0..3]);

my @A=(@N[0..13],@R[14..31]);

my @B=(map("%i$_",(0..5)),map("%l$_",(0..7))); @B=(@B,@B,map("%o$_",(0..3)));

########################################################################

# int bn_mul_mont_t4_$NUM(u64 *rp,const u64 *ap,const u64 *bp,

#			  const u64 *np,const BN_ULONG *n0);

#

sub generate_bn_mul_mont_t4() {

my $NUM=shift;

my ($rp,$ap,$bp,$np,$sentinel)=map("%g$_",(1..5));

$code.=<<___;

.globl	bn_mul_mont_t4_$NUM

.align	32

bn_mul_mont_t4_$NUM:

#ifdef	__arch64__

	mov	0,$sentinel

	mov	-128,%g4

#elif defined(SPARCV9_64BIT_STACK)

	SPARC_LOAD_ADDRESS_LEAF(OPENSSL_sparcv9cap_P,%g1,%g5)

	ld	[%g1+0],%g1	! OPENSSL_sparcv9_P[0]

	mov	-2047,%g4

	and	%g1,SPARCV9_64BIT_STACK,%g1

	movrz	%g1,0,%g4

	mov	-1,$sentinel

	add	%g4,-128,%g4

#else

	mov	-1,$sentinel

	mov	-128,%g4

#endif

	sllx	$sentinel,32,$sentinel

	save	%sp,%g4,%sp

#ifndef	__arch64__

	save	%sp,-128,%sp	! warm it up

	save	%sp,-128,%sp

	save	%sp,-128,%sp

	save	%sp,-128,%sp

	save	%sp,-128,%sp

	save	%sp,-128,%sp

	restore

	restore

	restore

	restore

	restore

	restore

#endif

	and	%sp,1,%g4

	or	$sentinel,%fp,%fp

	or	%g4,$sentinel,$sentinel

	! copy arguments to global registers

	mov	%i0,$rp

	mov	%i1,$ap

	mov	%i2,$bp

	mov	%i3,$np

	ld	[%i4+0],%f1	! load *n0

	ld	[%i4+4],%f0

	fsrc2	%f0,%f60

___

# load ap[$NUM] ########################################################

$code.=<<___;

	save	%sp,-128,%sp;		or	$sentinel,%fp,%fp

___

for($i=0; $i<14 && $i<$NUM; $i++) {

my $lo=$i<13?@A[$i+1]:"%o7";

$code.=<<___;

	ld	[$ap+$i*8+0],$lo

	ld	[$ap+$i*8+4],@A[$i]

	sllx	@A[$i],32,@A[$i]

	or	$lo,@A[$i],@A[$i]

___

}

for(; $i<$NUM; $i++) {

my ($hi,$lo)=("%f".2*($i%4),"%f".(2*($i%4)+1));

$code.=<<___;

	ld	[$ap+$i*8+0],$lo

	ld	[$ap+$i*8+4],$hi

	fsrc2	$hi,@A[$i]

___

}

# load np[$NUM] ########################################################

$code.=<<___;

	save	%sp,-128,%sp;		or	$sentinel,%fp,%fp

___

for($i=0; $i<14 && $i<$NUM; $i++) {

my $lo=$i<13?@N[$i+1]:"%o7";

$code.=<<___;

	ld	[$np+$i*8+0],$lo

	ld	[$np+$i*8+4],@N[$i]

	sllx	@N[$i],32,@N[$i]

	or	$lo,@N[$i],@N[$i]

___

}

$code.=<<___;

	save	%sp,-128,%sp;		or	$sentinel,%fp,%fp

___

for(; $i<28 && $i<$NUM; $i++) {

my $lo=$i<27?@N[$i+1]:"%o7";

$code.=<<___;

	ld	[$np+$i*8+0],$lo

	ld	[$np+$i*8+4],@N[$i]

	sllx	@N[$i],32,@N[$i]

	or	$lo,@N[$i],@N[$i]

___

}

$code.=<<___;

	save	%sp,-128,%sp;		or	$sentinel,%fp,%fp

___

for(; $i<$NUM; $i++) {

my $lo=($i<$NUM-1)?@N[$i+1]:"%o7";

$code.=<<___;

	ld	[$np+$i*8+0],$lo

	ld	[$np+$i*8+4],@N[$i]

	sllx	@N[$i],32,@N[$i]

	or	$lo,@N[$i],@N[$i]

___

}

$code.=<<___;

	cmp	$ap,$bp

	be	SIZE_T_CC,.Lmsquare_$NUM

	nop

___

# load bp[$NUM] ########################################################

$code.=<<___;

	save	%sp,-128,%sp;		or	$sentinel,%fp,%fp

___

for($i=0; $i<14 && $i<$NUM; $i++) {

my $lo=$i<13?@B[$i+1]:"%o7";

$code.=<<___;

	ld	[$bp+$i*8+0],$lo

	ld	[$bp+$i*8+4],@B[$i]

	sllx	@B[$i],32,@B[$i]

	or	$lo,@B[$i],@B[$i]

___

}

$code.=<<___;

	save	%sp,-128,%sp;		or	$sentinel,%fp,%fp

___

for(; $i<$NUM; $i++) {

my $lo=($i<$NUM-1)?@B[$i+1]:"%o7";

$code.=<<___;

	ld	[$bp+$i*8+0],$lo

	ld	[$bp+$i*8+4],@B[$i]

	sllx	@B[$i],32,@B[$i]

	or	$lo,@B[$i],@B[$i]

___

}

# magic ################################################################

$code.=<<___;

	.word	0x81b02920+$NUM-1	! montmul	$NUM-1

.Lmresume_$NUM:

	fbu,pn	%fcc3,.Lmabort_$NUM

#ifndef	__arch64__

	and	%fp,$sentinel,$sentinel

	brz,pn	$sentinel,.Lmabort_$NUM

#endif

	nop

#ifdef	__arch64__

	restore

	restore

	restore

	restore

	restore

#else

	restore;		and	%fp,$sentinel,$sentinel

	restore;		and	%fp,$sentinel,$sentinel

	restore;		and	%fp,$sentinel,$sentinel

	restore;		and	%fp,$sentinel,$sentinel

	 brz,pn	$sentinel,.Lmabort1_$NUM

	restore

#endif

___

# save tp[$NUM] ########################################################

for($i=0; $i<14 && $i<$NUM; $i++) {

$code.=<<___;

	movxtod	@A[$i],@R[$i]

___

}

$code.=<<___;

#ifdef	__arch64__

	restore

#else

	 and	%fp,$sentinel,$sentinel

	restore

	 and	$sentinel,1,%o7

	 and	%fp,$sentinel,$sentinel

	 srl	%fp,0,%fp		! just in case?

	 or	%o7,$sentinel,$sentinel

	brz,a,pn $sentinel,.Lmdone_$NUM

	mov	0,%i0		! return failure

#endif

___

for($i=0; $i<12 && $i<$NUM; $i++) {

@R[$i] =~ /%f([0-9]+)/;

my $lo = "%f".($1+1);

$code.=<<___;

	st	$lo,[$rp+$i*8+0]

	st	@R[$i],[$rp+$i*8+4]

___

}

for(; $i<$NUM; $i++) {

my ($hi,$lo)=("%f".2*($i%4),"%f".(2*($i%4)+1));

$code.=<<___;

	fsrc2	@R[$i],$hi

	st	$lo,[$rp+$i*8+0]

	st	$hi,[$rp+$i*8+4]

___

}

$code.=<<___;

	mov	1,%i0		! return success

.Lmdone_$NUM:

	ret

	restore

.Lmabort_$NUM:

	restore

	restore

	restore

	restore

	restore

.Lmabort1_$NUM:

	restore

	mov	0,%i0		! return failure

	ret

	restore

.align	32

.Lmsquare_$NUM:

	save	%sp,-128,%sp;		or	$sentinel,%fp,%fp

	save	%sp,-128,%sp;		or	$sentinel,%fp,%fp

	.word   0x81b02940+$NUM-1	! montsqr	$NUM-1

	ba	.Lmresume_$NUM

	nop

.type	bn_mul_mont_t4_$NUM, #function

.size	bn_mul_mont_t4_$NUM, .-bn_mul_mont_t4_$NUM

___

}

for ($i=8;$i<=32;$i+=8) {

	&generate_bn_mul_mont_t4($i);

}

########################################################################

#

sub load_ccr {

my ($ptbl,$pwr,$ccr,$skip_wr)=@_;

$code.=<<___;

	srl	$pwr,	2,	%o4

	and	$pwr,	3,	%o5

	and	%o4,	7,	%o4

	sll	%o5,	3,	%o5	! offset within first cache line

	add	%o5,	$ptbl,	$ptbl	! of the pwrtbl

	or	%g0,	1,	%o5

	sll	%o5,	%o4,	$ccr

___

$code.=<<___	if (!$skip_wr);

	wr	$ccr,	%g0,	%ccr

___

}

sub load_b_pair {

my ($pwrtbl,$B0,$B1)=@_;

$code.=<<___;

	ldx	[$pwrtbl+0*32],	$B0

	ldx	[$pwrtbl+8*32],	$B1

	ldx	[$pwrtbl+1*32],	%o4

	ldx	[$pwrtbl+9*32],	%o5

	movvs	%icc,	%o4,	$B0

	ldx	[$pwrtbl+2*32],	%o4

	movvs	%icc,	%o5,	$B1

	ldx	[$pwrtbl+10*32],%o5

	move	%icc,	%o4,	$B0

	ldx	[$pwrtbl+3*32],	%o4

	move	%icc,	%o5,	$B1

	ldx	[$pwrtbl+11*32],%o5

	movneg	%icc,	%o4,	$B0

	ldx	[$pwrtbl+4*32],	%o4

	movneg	%icc,	%o5,	$B1

	ldx	[$pwrtbl+12*32],%o5

	movcs	%xcc,	%o4,	$B0

	ldx	[$pwrtbl+5*32],%o4

	movcs	%xcc,	%o5,	$B1

	ldx	[$pwrtbl+13*32],%o5

	movvs	%xcc,	%o4,	$B0

	ldx	[$pwrtbl+6*32],	%o4

	movvs	%xcc,	%o5,	$B1

	ldx	[$pwrtbl+14*32],%o5

	move	%xcc,	%o4,	$B0

	ldx	[$pwrtbl+7*32],	%o4

	move	%xcc,	%o5,	$B1

	ldx	[$pwrtbl+15*32],%o5

	movneg	%xcc,	%o4,	$B0

	add	$pwrtbl,16*32,	$pwrtbl

	movneg	%xcc,	%o5,	$B1

___

}

sub load_b {

my ($pwrtbl,$Bi)=@_;

$code.=<<___;

	ldx	[$pwrtbl+0*32],	$Bi

	ldx	[$pwrtbl+1*32],	%o4

	ldx	[$pwrtbl+2*32],	%o5

	movvs	%icc,	%o4,	$Bi

	ldx	[$pwrtbl+3*32],	%o4

	move	%icc,	%o5,	$Bi

	ldx	[$pwrtbl+4*32],	%o5

	movneg	%icc,	%o4,	$Bi

	ldx	[$pwrtbl+5*32],	%o4

	movcs	%xcc,	%o5,	$Bi

	ldx	[$pwrtbl+6*32],	%o5

	movvs	%xcc,	%o4,	$Bi

	ldx	[$pwrtbl+7*32],	%o4

	move	%xcc,	%o5,	$Bi

	add	$pwrtbl,8*32,	$pwrtbl

	movneg	%xcc,	%o4,	$Bi

___

}

########################################################################

# int bn_pwr5_mont_t4_$NUM(u64 *tp,const u64 *np,const BN_ULONG *n0,

#			   const u64 *pwrtbl,int pwr,int stride);

#

sub generate_bn_pwr5_mont_t4() {

my $NUM=shift;

my ($tp,$np,$pwrtbl,$pwr,$sentinel)=map("%g$_",(1..5));

$code.=<<___;

.globl	bn_pwr5_mont_t4_$NUM

.align	32

bn_pwr5_mont_t4_$NUM:

#ifdef	__arch64__

	mov	0,$sentinel

	mov	-128,%g4

#elif defined(SPARCV9_64BIT_STACK)

	SPARC_LOAD_ADDRESS_LEAF(OPENSSL_sparcv9cap_P,%g1,%g5)

	ld	[%g1+0],%g1	! OPENSSL_sparcv9_P[0]

	mov	-2047,%g4

	and	%g1,SPARCV9_64BIT_STACK,%g1

	movrz	%g1,0,%g4

	mov	-1,$sentinel

	add	%g4,-128,%g4

#else

	mov	-1,$sentinel

	mov	-128,%g4

#endif

	sllx	$sentinel,32,$sentinel

	save	%sp,%g4,%sp

#ifndef	__arch64__

	save	%sp,-128,%sp	! warm it up

	save	%sp,-128,%sp

	save	%sp,-128,%sp

	save	%sp,-128,%sp

	save	%sp,-128,%sp

	save	%sp,-128,%sp

	restore

	restore

	restore

	restore

	restore

	restore

#endif

	and	%sp,1,%g4

	or	$sentinel,%fp,%fp

	or	%g4,$sentinel,$sentinel

	! copy arguments to global registers

	mov	%i0,$tp

	mov	%i1,$np

	ld	[%i2+0],%f1	! load *n0

	ld	[%i2+4],%f0

	mov	%i3,$pwrtbl

	srl	%i4,%g0,%i4	! pack last arguments

	sllx	%i5,32,$pwr

	or	%i4,$pwr,$pwr

	fsrc2	%f0,%f60

___

# load tp[$NUM] ########################################################

$code.=<<___;

	save	%sp,-128,%sp;		or	$sentinel,%fp,%fp

___

for($i=0; $i<14 && $i<$NUM; $i++) {

$code.=<<___;

	ldx	[$tp+$i*8],@A[$i]

___

}

for(; $i<$NUM; $i++) {

$code.=<<___;

	ldd	[$tp+$i*8],@A[$i]

___

}

# load np[$NUM] ########################################################

$code.=<<___;

	save	%sp,-128,%sp;		or	$sentinel,%fp,%fp

___

for($i=0; $i<14 && $i<$NUM; $i++) {

$code.=<<___;

	ldx	[$np+$i*8],@N[$i]

___

}

$code.=<<___;

	save	%sp,-128,%sp;		or	$sentinel,%fp,%fp

___

for(; $i<28 && $i<$NUM; $i++) {

$code.=<<___;

	ldx	[$np+$i*8],@N[$i]

___

}

$code.=<<___;

	save	%sp,-128,%sp;		or	$sentinel,%fp,%fp

___

for(; $i<$NUM; $i++) {

$code.=<<___;

	ldx	[$np+$i*8],@N[$i]

___

}

# load pwrtbl[pwr] ########################################################

$code.=<<___;

	save	%sp,-128,%sp;		or	$sentinel,%fp,%fp

	srlx	$pwr,	32,	%o4		! unpack $pwr

	srl	$pwr,	%g0,	%o5

	sub	%o4,	5,	%o4

	mov	$pwrtbl,	%o7