[thirdparty/openssl.git] / crypto / modes / asm / ghash-s390x.pl

#! /usr/bin/env perl
# Copyright 2010-2016 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the Apache License 2.0 (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html


# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================

# September 2010.
#
# The module implements "4-bit" GCM GHASH function and underlying
# single multiplication operation in GF(2^128). "4-bit" means that it
# uses 256 bytes per-key table [+128 bytes shared table]. Performance
# was measured to be ~18 cycles per processed byte on z10, which is
# almost 40% better than gcc-generated code. It should be noted that
# 18 cycles is worse result than expected: loop is scheduled for 12
# and the result should be close to 12. In the lack of instruction-
# level profiling data it's impossible to tell why...

# November 2010.
#
# Adapt for -m31 build. If kernel supports what's called "highgprs"
# feature on Linux [see /proc/cpuinfo], it's possible to use 64-bit
# instructions and achieve "64-bit" performance even in 31-bit legacy
# application context. The feature is not specific to any particular
# processor, as long as it's "z-CPU". Latter implies that the code
# remains z/Architecture specific. On z990 it was measured to perform
# 2.8x better than 32-bit code generated by gcc 4.3.

# March 2011.
#
# Support for hardware KIMD-GHASH is verified to produce correct
# result and therefore is engaged. On z196 it was measured to process
# 8KB buffer ~7 faster than software implementation. It's not as
# impressive for smaller buffer sizes and for smallest 16-bytes buffer
# it's actually almost 2 times slower. Which is the reason why
# KIMD-GHASH is not used in gcm_gmult_4bit.

# $output is the last argument if it looks like a file (it has an extension)
# $flavour is the first argument if it doesn't look like a file
$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;

if ($flavour =~ /3[12]/) {
	$SIZE_T=4;
	$g="";
} else {
	$SIZE_T=8;
	$g="g";
}

$output and open STDOUT,">$output";

$softonly=0;

$Zhi="%r0";
$Zlo="%r1";

$Xi="%r2";	# argument block
$Htbl="%r3";
$inp="%r4";
$len="%r5";

$rem0="%r6";	# variables
$rem1="%r7";
$nlo="%r8";
$nhi="%r9";
$xi="%r10";
$cnt="%r11";
$tmp="%r12";
$x78="%r13";
$rem_4bit="%r14";

$sp="%r15";

$code.=<<___;
#include "s390x_arch.h"

.text

.globl	gcm_gmult_4bit
.align	32
gcm_gmult_4bit:
___
$code.=<<___ if(!$softonly && 0);	# hardware is slow for single block...
	larl	%r1,OPENSSL_s390xcap_P
	lghi	%r0,0
	lg	%r1,S390X_KIMD+8(%r1)	# load second word of kimd capabilities
					#  vector
	tmhh	%r1,0x4000	# check for function 65
	jz	.Lsoft_gmult
	stg	%r0,16($sp)	# arrange 16 bytes of zero input
	stg	%r0,24($sp)
	lghi	%r0,S390X_GHASH	# function 65
	la	%r1,0($Xi)	# H lies right after Xi in gcm128_context
	la	$inp,16($sp)
	lghi	$len,16
	.long	0xb93e0004	# kimd %r0,$inp
	brc	1,.-4		# pay attention to "partial completion"
	br	%r14
.align	32
.Lsoft_gmult:
___
$code.=<<___;
	stm${g}	%r6,%r14,6*$SIZE_T($sp)

	aghi	$Xi,-1
	lghi	$len,1
	lghi	$x78,`0xf<<3`
	larl	$rem_4bit,rem_4bit

	lg	$Zlo,8+1($Xi)		# Xi
	j	.Lgmult_shortcut
.type	gcm_gmult_4bit,\@function
.size	gcm_gmult_4bit,(.-gcm_gmult_4bit)

.globl	gcm_ghash_4bit
.align	32
gcm_ghash_4bit:
___
$code.=<<___ if(!$softonly);
	larl	%r1,OPENSSL_s390xcap_P
	lg	%r0,S390X_KIMD+8(%r1)	# load second word of kimd capabilities
					#  vector
	tmhh	%r0,0x4000	# check for function 65
	jz	.Lsoft_ghash
	lghi	%r0,S390X_GHASH	# function 65
	la	%r1,0($Xi)	# H lies right after Xi in gcm128_context
	.long	0xb93e0004	# kimd %r0,$inp
	brc	1,.-4		# pay attention to "partial completion"
	br	%r14
.align	32
.Lsoft_ghash:
___
$code.=<<___ if ($flavour =~ /3[12]/);
	llgfr	$len,$len
___
$code.=<<___;
	stm${g}	%r6,%r14,6*$SIZE_T($sp)

	aghi	$Xi,-1
	srlg	$len,$len,4
	lghi	$x78,`0xf<<3`
	larl	$rem_4bit,rem_4bit

	lg	$Zlo,8+1($Xi)		# Xi
	lg	$Zhi,0+1($Xi)
	lghi	$tmp,0
.Louter:
	xg	$Zhi,0($inp)		# Xi ^= inp
	xg	$Zlo,8($inp)
	xgr	$Zhi,$tmp
	stg	$Zlo,8+1($Xi)
	stg	$Zhi,0+1($Xi)

.Lgmult_shortcut:
	lghi	$tmp,0xf0
	sllg	$nlo,$Zlo,4
	srlg	$xi,$Zlo,8		# extract second byte
	ngr	$nlo,$tmp
	lgr	$nhi,$Zlo
	lghi	$cnt,14
	ngr	$nhi,$tmp

	lg	$Zlo,8($nlo,$Htbl)
	lg	$Zhi,0($nlo,$Htbl)

	sllg	$nlo,$xi,4
	sllg	$rem0,$Zlo,3
	ngr	$nlo,$tmp
	ngr	$rem0,$x78
	ngr	$xi,$tmp

	sllg	$tmp,$Zhi,60
	srlg	$Zlo,$Zlo,4
	srlg	$Zhi,$Zhi,4
	xg	$Zlo,8($nhi,$Htbl)
	xg	$Zhi,0($nhi,$Htbl)
	lgr	$nhi,$xi
	sllg	$rem1,$Zlo,3
	xgr	$Zlo,$tmp
	ngr	$rem1,$x78
	sllg	$tmp,$Zhi,60
	j	.Lghash_inner
.align	16
.Lghash_inner:
	srlg	$Zlo,$Zlo,4
	srlg	$Zhi,$Zhi,4
	xg	$Zlo,8($nlo,$Htbl)
	llgc	$xi,0($cnt,$Xi)
	xg	$Zhi,0($nlo,$Htbl)
	sllg	$nlo,$xi,4
	xg	$Zhi,0($rem0,$rem_4bit)
	nill	$nlo,0xf0
	sllg	$rem0,$Zlo,3
	xgr	$Zlo,$tmp
	ngr	$rem0,$x78
	nill	$xi,0xf0

	sllg	$tmp,$Zhi,60
	srlg	$Zlo,$Zlo,4
	srlg	$Zhi,$Zhi,4
	xg	$Zlo,8($nhi,$Htbl)
	xg	$Zhi,0($nhi,$Htbl)
	lgr	$nhi,$xi
	xg	$Zhi,0($rem1,$rem_4bit)
	sllg	$rem1,$Zlo,3
	xgr	$Zlo,$tmp
	ngr	$rem1,$x78
	sllg	$tmp,$Zhi,60
	brct	$cnt,.Lghash_inner

	srlg	$Zlo,$Zlo,4
	srlg	$Zhi,$Zhi,4
	xg	$Zlo,8($nlo,$Htbl)
	xg	$Zhi,0($nlo,$Htbl)
	sllg	$xi,$Zlo,3
	xg	$Zhi,0($rem0,$rem_4bit)
	xgr	$Zlo,$tmp
	ngr	$xi,$x78

	sllg	$tmp,$Zhi,60
	srlg	$Zlo,$Zlo,4
	srlg	$Zhi,$Zhi,4
	xg	$Zlo,8($nhi,$Htbl)
	xg	$Zhi,0($nhi,$Htbl)
	xgr	$Zlo,$tmp
	xg	$Zhi,0($rem1,$rem_4bit)

	lg	$tmp,0($xi,$rem_4bit)
	la	$inp,16($inp)
	sllg	$tmp,$tmp,4		# correct last rem_4bit[rem]
	brctg	$len,.Louter

	xgr	$Zhi,$tmp
	stg	$Zlo,8+1($Xi)
	stg	$Zhi,0+1($Xi)
	lm${g}	%r6,%r14,6*$SIZE_T($sp)
	br	%r14
.type	gcm_ghash_4bit,\@function
.size	gcm_ghash_4bit,(.-gcm_ghash_4bit)

.align	64
rem_4bit:
	.long	`0x0000<<12`,0,`0x1C20<<12`,0,`0x3840<<12`,0,`0x2460<<12`,0
	.long	`0x7080<<12`,0,`0x6CA0<<12`,0,`0x48C0<<12`,0,`0x54E0<<12`,0
	.long	`0xE100<<12`,0,`0xFD20<<12`,0,`0xD940<<12`,0,`0xC560<<12`,0
	.long	`0x9180<<12`,0,`0x8DA0<<12`,0,`0xA9C0<<12`,0,`0xB5E0<<12`,0
.type	rem_4bit,\@object
.size	rem_4bit,(.-rem_4bit)
.string	"GHASH for s390x, CRYPTOGAMS by <appro\@openssl.org>"
___

$code =~ s/\`([^\`]*)\`/eval $1/gem;
print $code;
close STDOUT or die "error closing STDOUT: $!";
Commit	Line	Data
6aa36e8e RS	1	#! /usr/bin/env perl
	2	# Copyright 2010-2016 The OpenSSL Project Authors. All Rights Reserved.
	3	#
81cae8ce	4	# Licensed under the Apache License 2.0 (the "License"). You may not use
6aa36e8e RS	5	# this file except in compliance with the License. You can obtain a copy
	6	# in the file LICENSE in the source distribution or at
	7	# https://www.openssl.org/source/license.html
	8
7d1f55e9 AP	9
	10	# ====================================================================
	11	# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
	12	# project. The module is, however, dual licensed under OpenSSL and
	13	# CRYPTOGAMS licenses depending on where you obtain it. For further
	14	# details see http://www.openssl.org/~appro/cryptogams/.
	15	# ====================================================================
	16
	17	# September 2010.
8986e372 AP	18	#
	19	# The module implements "4-bit" GCM GHASH function and underlying
	20	# single multiplication operation in GF(2^128). "4-bit" means that it
	21	# uses 256 bytes per-key table [+128 bytes shared table]. Performance
	22	# was measured to be ~18 cycles per processed byte on z10, which is
	23	# almost 40% better than gcc-generated code. It should be noted that
	24	# 18 cycles is worse result than expected: loop is scheduled for 12
	25	# and the result should be close to 12. In the lack of instruction-
	26	# level profiling data it's impossible to tell why...
7d1f55e9	27
e822c756 AP	28	# November 2010.
	29	#
	30	# Adapt for -m31 build. If kernel supports what's called "highgprs"
	31	# feature on Linux [see /proc/cpuinfo], it's possible to use 64-bit
	32	# instructions and achieve "64-bit" performance even in 31-bit legacy
	33	# application context. The feature is not specific to any particular
	34	# processor, as long as it's "z-CPU". Latter implies that the code
	35	# remains z/Architecture specific. On z990 it was measured to perform
	36	# 2.8x better than 32-bit code generated by gcc 4.3.
	37
0ab8fd58 AP	38	# March 2011.
	39	#
	40	# Support for hardware KIMD-GHASH is verified to produce correct
	41	# result and therefore is engaged. On z196 it was measured to process
	42	# 8KB buffer ~7 faster than software implementation. It's not as
	43	# impressive for smaller buffer sizes and for smallest 16-bytes buffer
	44	# it's actually almost 2 times slower. Which is the reason why
	45	# KIMD-GHASH is not used in gcm_gmult_4bit.
	46
1aa89a7a RL	47	# $output is the last argument if it looks like a file (it has an extension)
	48	# $flavour is the first argument if it doesn't look like a file
	49	$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m\|\.\w+$\| ? pop : undef;
	50	$flavour = $#ARGV >= 0 && $ARGV[0] !~ m\|\.\| ? shift : undef;
e822c756 AP	51
	52	if ($flavour =~ /3[12]/) {
	53	$SIZE_T=4;
	54	$g="";
	55	} else {
	56	$SIZE_T=8;
	57	$g="g";
	58	}
	59
1aa89a7a	60	$output and open STDOUT,">$output";
7d1f55e9	61
0ab8fd58	62	$softonly=0;
8986e372	63
7d1f55e9 AP	64	$Zhi="%r0";
	65	$Zlo="%r1";
	66
	67	$Xi="%r2"; # argument block
	68	$Htbl="%r3";
	69	$inp="%r4";
	70	$len="%r5";
	71
	72	$rem0="%r6"; # variables
	73	$rem1="%r7";
	74	$nlo="%r8";
	75	$nhi="%r9";
	76	$xi="%r10";
	77	$cnt="%r11";
	78	$tmp="%r12";
	79	$x78="%r13";
	80	$rem_4bit="%r14";
	81
	82	$sp="%r15";
	83
	84	$code.=<<___;
bc4e831c PS	85	#include "s390x_arch.h"
bc4e831c PS	86
7d1f55e9 AP	87	.text
	88
	89	.globl gcm_gmult_4bit
	90	.align 32
	91	gcm_gmult_4bit:
8986e372	92	___
0ab8fd58	93	$code.=<<___ if(!$softonly && 0); # hardware is slow for single block...
8986e372	94	larl %r1,OPENSSL_s390xcap_P
8986e372	95	lghi %r0,0
bc4e831c PS	96	lg %r1,S390X_KIMD+8(%r1) # load second word of kimd capabilities
bc4e831c PS	97	# vector
8986e372 AP	98	tmhh %r1,0x4000 # check for function 65
	99	jz .Lsoft_gmult
	100	stg %r0,16($sp) # arrange 16 bytes of zero input
	101	stg %r0,24($sp)
bc4e831c	102	lghi %r0,S390X_GHASH # function 65
8986e372 AP	103	la %r1,0($Xi) # H lies right after Xi in gcm128_context
	104	la $inp,16($sp)
	105	lghi $len,16
	106	.long 0xb93e0004 # kimd %r0,$inp
	107	brc 1,.-4 # pay attention to "partial completion"
	108	br %r14
	109	.align 32
	110	.Lsoft_gmult:
	111	___
	112	$code.=<<___;
e822c756	113	stm${g} %r6,%r14,6*$SIZE_T($sp)
7d1f55e9 AP	114
	115	aghi $Xi,-1
	116	lghi $len,1
	117	lghi $x78,`0xf<<3`
	118	larl $rem_4bit,rem_4bit
	119
	120	lg $Zlo,8+1($Xi) # Xi
	121	j .Lgmult_shortcut
	122	.type gcm_gmult_4bit,\@function
	123	.size gcm_gmult_4bit,(.-gcm_gmult_4bit)
	124
	125	.globl gcm_ghash_4bit
	126	.align 32
	127	gcm_ghash_4bit:
8986e372 AP	128	___
	129	$code.=<<___ if(!$softonly);
	130	larl %r1,OPENSSL_s390xcap_P
bc4e831c PS	131	lg %r0,S390X_KIMD+8(%r1) # load second word of kimd capabilities
bc4e831c PS	132	# vector
af1d6387	133	tmhh %r0,0x4000 # check for function 65
8986e372	134	jz .Lsoft_ghash
bc4e831c	135	lghi %r0,S390X_GHASH # function 65
8986e372 AP	136	la %r1,0($Xi) # H lies right after Xi in gcm128_context
	137	.long 0xb93e0004 # kimd %r0,$inp
	138	brc 1,.-4 # pay attention to "partial completion"
	139	br %r14
	140	.align 32
	141	.Lsoft_ghash:
	142	___
26e6bac1	143	$code.=<<___ if ($flavour =~ /3[12]/);
e822c756 AP	144	llgfr $len,$len
e822c756 AP	145	___
8986e372	146	$code.=<<___;
e822c756	147	stm${g} %r6,%r14,6*$SIZE_T($sp)
7d1f55e9 AP	148
	149	aghi $Xi,-1
	150	srlg $len,$len,4
	151	lghi $x78,`0xf<<3`
	152	larl $rem_4bit,rem_4bit
	153
	154	lg $Zlo,8+1($Xi) # Xi
	155	lg $Zhi,0+1($Xi)
8986e372	156	lghi $tmp,0
7d1f55e9	157	.Louter:
609b0852	158	xg $Zhi,0($inp) # Xi ^= inp
8986e372 AP	159	xg $Zlo,8($inp)
8986e372 AP	160	xgr $Zhi,$tmp
7d1f55e9 AP	161	stg $Zlo,8+1($Xi)
	162	stg $Zhi,0+1($Xi)
	163
	164	.Lgmult_shortcut:
8986e372 AP	165	lghi $tmp,0xf0
	166	sllg $nlo,$Zlo,4
	167	srlg $xi,$Zlo,8 # extract second byte
	168	ngr $nlo,$tmp
7d1f55e9	169	lgr $nhi,$Zlo
7d1f55e9	170	lghi $cnt,14
8986e372	171	ngr $nhi,$tmp
7d1f55e9 AP	172
	173	lg $Zlo,8($nlo,$Htbl)
	174	lg $Zhi,0($nlo,$Htbl)
	175
	176	sllg $nlo,$xi,4
7d1f55e9	177	sllg $rem0,$Zlo,3
8986e372	178	ngr $nlo,$tmp
7d1f55e9	179	ngr $rem0,$x78
8986e372 AP	180	ngr $xi,$tmp
8986e372 AP	181
7d1f55e9	182	sllg $tmp,$Zhi,60
8986e372	183	srlg $Zlo,$Zlo,4
7d1f55e9	184	srlg $Zhi,$Zhi,4
8986e372	185	xg $Zlo,8($nhi,$Htbl)
7d1f55e9 AP	186	xg $Zhi,0($nhi,$Htbl)
	187	lgr $nhi,$xi
	188	sllg $rem1,$Zlo,3
8986e372 AP	189	xgr $Zlo,$tmp
8986e372 AP	190	ngr $rem1,$x78
d162584b	191	sllg $tmp,$Zhi,60
8986e372 AP	192	j .Lghash_inner
8986e372 AP	193	.align 16
7d1f55e9 AP	194	.Lghash_inner:
7d1f55e9 AP	195	srlg $Zlo,$Zlo,4
7d1f55e9	196	srlg $Zhi,$Zhi,4
d162584b	197	xg $Zlo,8($nlo,$Htbl)
7d1f55e9	198	llgc $xi,0($cnt,$Xi)
7d1f55e9 AP	199	xg $Zhi,0($nlo,$Htbl)
7d1f55e9 AP	200	sllg $nlo,$xi,4
8986e372	201	xg $Zhi,0($rem0,$rem_4bit)
7d1f55e9	202	nill $nlo,0xf0
8986e372 AP	203	sllg $rem0,$Zlo,3
8986e372 AP	204	xgr $Zlo,$tmp
7d1f55e9	205	ngr $rem0,$x78
8986e372 AP	206	nill $xi,0xf0
8986e372 AP	207
7d1f55e9	208	sllg $tmp,$Zhi,60
8986e372	209	srlg $Zlo,$Zlo,4
7d1f55e9	210	srlg $Zhi,$Zhi,4
8986e372	211	xg $Zlo,8($nhi,$Htbl)
7d1f55e9 AP	212	xg $Zhi,0($nhi,$Htbl)
7d1f55e9 AP	213	lgr $nhi,$xi
8986e372 AP	214	xg $Zhi,0($rem1,$rem_4bit)
	215	sllg $rem1,$Zlo,3
	216	xgr $Zlo,$tmp
	217	ngr $rem1,$x78
d162584b	218	sllg $tmp,$Zhi,60
7d1f55e9 AP	219	brct $cnt,.Lghash_inner
	220
	221	srlg $Zlo,$Zlo,4
8986e372	222	srlg $Zhi,$Zhi,4
7d1f55e9	223	xg $Zlo,8($nlo,$Htbl)
8986e372 AP	224	xg $Zhi,0($nlo,$Htbl)
8986e372 AP	225	sllg $xi,$Zlo,3
7d1f55e9 AP	226	xg $Zhi,0($rem0,$rem_4bit)
7d1f55e9 AP	227	xgr $Zlo,$tmp
8986e372	228	ngr $xi,$x78
7d1f55e9	229
7d1f55e9	230	sllg $tmp,$Zhi,60
8986e372	231	srlg $Zlo,$Zlo,4
7d1f55e9	232	srlg $Zhi,$Zhi,4
8986e372	233	xg $Zlo,8($nhi,$Htbl)
7d1f55e9	234	xg $Zhi,0($nhi,$Htbl)
8986e372 AP	235	xgr $Zlo,$tmp
8986e372 AP	236	xg $Zhi,0($rem1,$rem_4bit)
7d1f55e9	237
8986e372	238	lg $tmp,0($xi,$rem_4bit)
7d1f55e9	239	la $inp,16($inp)
8986e372	240	sllg $tmp,$tmp,4 # correct last rem_4bit[rem]
7d1f55e9 AP	241	brctg $len,.Louter
7d1f55e9 AP	242
8986e372	243	xgr $Zhi,$tmp
7d1f55e9 AP	244	stg $Zlo,8+1($Xi)
7d1f55e9 AP	245	stg $Zhi,0+1($Xi)
e822c756	246	lm${g} %r6,%r14,6*$SIZE_T($sp)
7d1f55e9 AP	247	br %r14
	248	.type gcm_ghash_4bit,\@function
	249	.size gcm_ghash_4bit,(.-gcm_ghash_4bit)
	250
	251	.align 64
	252	rem_4bit:
8986e372 AP	253	.long `0x0000<<12`,0,`0x1C20<<12`,0,`0x3840<<12`,0,`0x2460<<12`,0
	254	.long `0x7080<<12`,0,`0x6CA0<<12`,0,`0x48C0<<12`,0,`0x54E0<<12`,0
	255	.long `0xE100<<12`,0,`0xFD20<<12`,0,`0xD940<<12`,0,`0xC560<<12`,0
	256	.long `0x9180<<12`,0,`0x8DA0<<12`,0,`0xA9C0<<12`,0,`0xB5E0<<12`,0
7d1f55e9 AP	257	.type rem_4bit,\@object
	258	.size rem_4bit,(.-rem_4bit)
	259	.string "GHASH for s390x, CRYPTOGAMS by <appro\@openssl.org>"
	260	___
	261
	262	$code =~ s/\`([^\`]*)\`/eval $1/gem;
	263	print $code;
a21314db	264	close STDOUT or die "error closing STDOUT: $!";