[thirdparty/openssl.git] / crypto / modes / asm / ghash-s390x.pl

#! /usr/bin/env perl
# Copyright 2010-2016 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the Apache License 2.0 (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html


# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================

# September 2010.
#
# The module implements "4-bit" GCM GHASH function and underlying
# single multiplication operation in GF(2^128). "4-bit" means that it
# uses 256 bytes per-key table [+128 bytes shared table]. Performance
# was measured to be ~18 cycles per processed byte on z10, which is
# almost 40% better than gcc-generated code. It should be noted that
# 18 cycles is worse result than expected: loop is scheduled for 12
# and the result should be close to 12. In the lack of instruction-
# level profiling data it's impossible to tell why...

# November 2010.
#
# Adapt for -m31 build. If kernel supports what's called "highgprs"
# feature on Linux [see /proc/cpuinfo], it's possible to use 64-bit
# instructions and achieve "64-bit" performance even in 31-bit legacy
# application context. The feature is not specific to any particular
# processor, as long as it's "z-CPU". Latter implies that the code
# remains z/Architecture specific. On z990 it was measured to perform
# 2.8x better than 32-bit code generated by gcc 4.3.

# March 2011.
#
# Support for hardware KIMD-GHASH is verified to produce correct
# result and therefore is engaged. On z196 it was measured to process
# 8KB buffer ~7 faster than software implementation. It's not as
# impressive for smaller buffer sizes and for smallest 16-bytes buffer
# it's actually almost 2 times slower. Which is the reason why
# KIMD-GHASH is not used in gcm_gmult_4bit.

$flavour = shift;

if ($flavour =~ /3[12]/) {
	$SIZE_T=4;
	$g="";
} else {
	$SIZE_T=8;
	$g="g";
}

while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {}
open STDOUT,">$output";

$softonly=0;

$Zhi="%r0";
$Zlo="%r1";

$Xi="%r2";	# argument block
$Htbl="%r3";
$inp="%r4";
$len="%r5";

$rem0="%r6";	# variables
$rem1="%r7";
$nlo="%r8";
$nhi="%r9";
$xi="%r10";
$cnt="%r11";
$tmp="%r12";
$x78="%r13";
$rem_4bit="%r14";

$sp="%r15";

$code.=<<___;
#include "s390x_arch.h"

.text

.globl	gcm_gmult_4bit
.align	32
gcm_gmult_4bit:
___
$code.=<<___ if(!$softonly && 0);	# hardware is slow for single block...
	larl	%r1,OPENSSL_s390xcap_P
	lghi	%r0,0
	lg	%r1,S390X_KIMD+8(%r1)	# load second word of kimd capabilities
					#  vector
	tmhh	%r1,0x4000	# check for function 65
	jz	.Lsoft_gmult
	stg	%r0,16($sp)	# arrange 16 bytes of zero input
	stg	%r0,24($sp)
	lghi	%r0,S390X_GHASH	# function 65
	la	%r1,0($Xi)	# H lies right after Xi in gcm128_context
	la	$inp,16($sp)
	lghi	$len,16
	.long	0xb93e0004	# kimd %r0,$inp
	brc	1,.-4		# pay attention to "partial completion"
	br	%r14
.align	32
.Lsoft_gmult:
___
$code.=<<___;
	stm${g}	%r6,%r14,6*$SIZE_T($sp)

	aghi	$Xi,-1
	lghi	$len,1
	lghi	$x78,`0xf<<3`
	larl	$rem_4bit,rem_4bit

	lg	$Zlo,8+1($Xi)		# Xi
	j	.Lgmult_shortcut
.type	gcm_gmult_4bit,\@function
.size	gcm_gmult_4bit,(.-gcm_gmult_4bit)

.globl	gcm_ghash_4bit
.align	32
gcm_ghash_4bit:
___
$code.=<<___ if(!$softonly);
	larl	%r1,OPENSSL_s390xcap_P
	lg	%r0,S390X_KIMD+8(%r1)	# load second word of kimd capabilities
					#  vector
	tmhh	%r0,0x4000	# check for function 65
	jz	.Lsoft_ghash
	lghi	%r0,S390X_GHASH	# function 65
	la	%r1,0($Xi)	# H lies right after Xi in gcm128_context
	.long	0xb93e0004	# kimd %r0,$inp
	brc	1,.-4		# pay attention to "partial completion"
	br	%r14
.align	32
.Lsoft_ghash:
___
$code.=<<___ if ($flavour =~ /3[12]/);
	llgfr	$len,$len
___
$code.=<<___;
	stm${g}	%r6,%r14,6*$SIZE_T($sp)

	aghi	$Xi,-1
	srlg	$len,$len,4
	lghi	$x78,`0xf<<3`
	larl	$rem_4bit,rem_4bit

	lg	$Zlo,8+1($Xi)		# Xi
	lg	$Zhi,0+1($Xi)
	lghi	$tmp,0
.Louter:
	xg	$Zhi,0($inp)		# Xi ^= inp
	xg	$Zlo,8($inp)
	xgr	$Zhi,$tmp
	stg	$Zlo,8+1($Xi)
	stg	$Zhi,0+1($Xi)

.Lgmult_shortcut:
	lghi	$tmp,0xf0
	sllg	$nlo,$Zlo,4
	srlg	$xi,$Zlo,8		# extract second byte
	ngr	$nlo,$tmp
	lgr	$nhi,$Zlo
	lghi	$cnt,14
	ngr	$nhi,$tmp

	lg	$Zlo,8($nlo,$Htbl)
	lg	$Zhi,0($nlo,$Htbl)

	sllg	$nlo,$xi,4
	sllg	$rem0,$Zlo,3
	ngr	$nlo,$tmp
	ngr	$rem0,$x78
	ngr	$xi,$tmp

	sllg	$tmp,$Zhi,60
	srlg	$Zlo,$Zlo,4
	srlg	$Zhi,$Zhi,4
	xg	$Zlo,8($nhi,$Htbl)
	xg	$Zhi,0($nhi,$Htbl)
	lgr	$nhi,$xi
	sllg	$rem1,$Zlo,3
	xgr	$Zlo,$tmp
	ngr	$rem1,$x78
	sllg	$tmp,$Zhi,60
	j	.Lghash_inner
.align	16
.Lghash_inner:
	srlg	$Zlo,$Zlo,4
	srlg	$Zhi,$Zhi,4
	xg	$Zlo,8($nlo,$Htbl)
	llgc	$xi,0($cnt,$Xi)
	xg	$Zhi,0($nlo,$Htbl)
	sllg	$nlo,$xi,4
	xg	$Zhi,0($rem0,$rem_4bit)
	nill	$nlo,0xf0
	sllg	$rem0,$Zlo,3
	xgr	$Zlo,$tmp
	ngr	$rem0,$x78
	nill	$xi,0xf0

	sllg	$tmp,$Zhi,60
	srlg	$Zlo,$Zlo,4
	srlg	$Zhi,$Zhi,4
	xg	$Zlo,8($nhi,$Htbl)
	xg	$Zhi,0($nhi,$Htbl)
	lgr	$nhi,$xi
	xg	$Zhi,0($rem1,$rem_4bit)
	sllg	$rem1,$Zlo,3
	xgr	$Zlo,$tmp
	ngr	$rem1,$x78
	sllg	$tmp,$Zhi,60
	brct	$cnt,.Lghash_inner

	srlg	$Zlo,$Zlo,4
	srlg	$Zhi,$Zhi,4
	xg	$Zlo,8($nlo,$Htbl)
	xg	$Zhi,0($nlo,$Htbl)
	sllg	$xi,$Zlo,3
	xg	$Zhi,0($rem0,$rem_4bit)
	xgr	$Zlo,$tmp
	ngr	$xi,$x78

	sllg	$tmp,$Zhi,60
	srlg	$Zlo,$Zlo,4
	srlg	$Zhi,$Zhi,4
	xg	$Zlo,8($nhi,$Htbl)
	xg	$Zhi,0($nhi,$Htbl)
	xgr	$Zlo,$tmp
	xg	$Zhi,0($rem1,$rem_4bit)

	lg	$tmp,0($xi,$rem_4bit)
	la	$inp,16($inp)
	sllg	$tmp,$tmp,4		# correct last rem_4bit[rem]
	brctg	$len,.Louter

	xgr	$Zhi,$tmp
	stg	$Zlo,8+1($Xi)
	stg	$Zhi,0+1($Xi)
	lm${g}	%r6,%r14,6*$SIZE_T($sp)
	br	%r14
.type	gcm_ghash_4bit,\@function
.size	gcm_ghash_4bit,(.-gcm_ghash_4bit)

.align	64
rem_4bit:
	.long	`0x0000<<12`,0,`0x1C20<<12`,0,`0x3840<<12`,0,`0x2460<<12`,0
	.long	`0x7080<<12`,0,`0x6CA0<<12`,0,`0x48C0<<12`,0,`0x54E0<<12`,0
	.long	`0xE100<<12`,0,`0xFD20<<12`,0,`0xD940<<12`,0,`0xC560<<12`,0
	.long	`0x9180<<12`,0,`0x8DA0<<12`,0,`0xA9C0<<12`,0,`0xB5E0<<12`,0
.type	rem_4bit,\@object
.size	rem_4bit,(.-rem_4bit)
.string	"GHASH for s390x, CRYPTOGAMS by <appro\@openssl.org>"
___

$code =~ s/\`([^\`]*)\`/eval $1/gem;
print $code;
close STDOUT;
Commit	Line	Data
6aa36e8e RS	1	#! /usr/bin/env perl
	2	# Copyright 2010-2016 The OpenSSL Project Authors. All Rights Reserved.
	3	#
81cae8ce	4	# Licensed under the Apache License 2.0 (the "License"). You may not use
6aa36e8e RS	5	# this file except in compliance with the License. You can obtain a copy
	6	# in the file LICENSE in the source distribution or at
	7	# https://www.openssl.org/source/license.html
	8
7d1f55e9 AP	9
	10	# ====================================================================
	11	# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
	12	# project. The module is, however, dual licensed under OpenSSL and
	13	# CRYPTOGAMS licenses depending on where you obtain it. For further
	14	# details see http://www.openssl.org/~appro/cryptogams/.
	15	# ====================================================================
	16
	17	# September 2010.
8986e372 AP	18	#
	19	# The module implements "4-bit" GCM GHASH function and underlying
	20	# single multiplication operation in GF(2^128). "4-bit" means that it
	21	# uses 256 bytes per-key table [+128 bytes shared table]. Performance
	22	# was measured to be ~18 cycles per processed byte on z10, which is
	23	# almost 40% better than gcc-generated code. It should be noted that
	24	# 18 cycles is worse result than expected: loop is scheduled for 12
	25	# and the result should be close to 12. In the lack of instruction-
	26	# level profiling data it's impossible to tell why...
7d1f55e9	27
e822c756 AP	28	# November 2010.
	29	#
	30	# Adapt for -m31 build. If kernel supports what's called "highgprs"
	31	# feature on Linux [see /proc/cpuinfo], it's possible to use 64-bit
	32	# instructions and achieve "64-bit" performance even in 31-bit legacy
	33	# application context. The feature is not specific to any particular
	34	# processor, as long as it's "z-CPU". Latter implies that the code
	35	# remains z/Architecture specific. On z990 it was measured to perform
	36	# 2.8x better than 32-bit code generated by gcc 4.3.
	37
0ab8fd58 AP	38	# March 2011.
	39	#
	40	# Support for hardware KIMD-GHASH is verified to produce correct
	41	# result and therefore is engaged. On z196 it was measured to process
	42	# 8KB buffer ~7 faster than software implementation. It's not as
	43	# impressive for smaller buffer sizes and for smallest 16-bytes buffer
	44	# it's actually almost 2 times slower. Which is the reason why
	45	# KIMD-GHASH is not used in gcm_gmult_4bit.
	46
e822c756 AP	47	$flavour = shift;
	48
	49	if ($flavour =~ /3[12]/) {
	50	$SIZE_T=4;
	51	$g="";
	52	} else {
	53	$SIZE_T=8;
	54	$g="g";
	55	}
	56
a5aa63a4	57	while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {}
7d1f55e9 AP	58	open STDOUT,">$output";
7d1f55e9 AP	59
0ab8fd58	60	$softonly=0;
8986e372	61
7d1f55e9 AP	62	$Zhi="%r0";
	63	$Zlo="%r1";
	64
	65	$Xi="%r2"; # argument block
	66	$Htbl="%r3";
	67	$inp="%r4";
	68	$len="%r5";
	69
	70	$rem0="%r6"; # variables
	71	$rem1="%r7";
	72	$nlo="%r8";
	73	$nhi="%r9";
	74	$xi="%r10";
	75	$cnt="%r11";
	76	$tmp="%r12";
	77	$x78="%r13";
	78	$rem_4bit="%r14";
	79
	80	$sp="%r15";
	81
	82	$code.=<<___;
bc4e831c PS	83	#include "s390x_arch.h"
bc4e831c PS	84
7d1f55e9 AP	85	.text
	86
	87	.globl gcm_gmult_4bit
	88	.align 32
	89	gcm_gmult_4bit:
8986e372	90	___
0ab8fd58	91	$code.=<<___ if(!$softonly && 0); # hardware is slow for single block...
8986e372	92	larl %r1,OPENSSL_s390xcap_P
8986e372	93	lghi %r0,0
bc4e831c PS	94	lg %r1,S390X_KIMD+8(%r1) # load second word of kimd capabilities
bc4e831c PS	95	# vector
8986e372 AP	96	tmhh %r1,0x4000 # check for function 65
	97	jz .Lsoft_gmult
	98	stg %r0,16($sp) # arrange 16 bytes of zero input
	99	stg %r0,24($sp)
bc4e831c	100	lghi %r0,S390X_GHASH # function 65
8986e372 AP	101	la %r1,0($Xi) # H lies right after Xi in gcm128_context
	102	la $inp,16($sp)
	103	lghi $len,16
	104	.long 0xb93e0004 # kimd %r0,$inp
	105	brc 1,.-4 # pay attention to "partial completion"
	106	br %r14
	107	.align 32
	108	.Lsoft_gmult:
	109	___
	110	$code.=<<___;
e822c756	111	stm${g} %r6,%r14,6*$SIZE_T($sp)
7d1f55e9 AP	112
	113	aghi $Xi,-1
	114	lghi $len,1
	115	lghi $x78,`0xf<<3`
	116	larl $rem_4bit,rem_4bit
	117
	118	lg $Zlo,8+1($Xi) # Xi
	119	j .Lgmult_shortcut
	120	.type gcm_gmult_4bit,\@function
	121	.size gcm_gmult_4bit,(.-gcm_gmult_4bit)
	122
	123	.globl gcm_ghash_4bit
	124	.align 32
	125	gcm_ghash_4bit:
8986e372 AP	126	___
	127	$code.=<<___ if(!$softonly);
	128	larl %r1,OPENSSL_s390xcap_P
bc4e831c PS	129	lg %r0,S390X_KIMD+8(%r1) # load second word of kimd capabilities
bc4e831c PS	130	# vector
af1d6387	131	tmhh %r0,0x4000 # check for function 65
8986e372	132	jz .Lsoft_ghash
bc4e831c	133	lghi %r0,S390X_GHASH # function 65
8986e372 AP	134	la %r1,0($Xi) # H lies right after Xi in gcm128_context
	135	.long 0xb93e0004 # kimd %r0,$inp
	136	brc 1,.-4 # pay attention to "partial completion"
	137	br %r14
	138	.align 32
	139	.Lsoft_ghash:
	140	___
26e6bac1	141	$code.=<<___ if ($flavour =~ /3[12]/);
e822c756 AP	142	llgfr $len,$len
e822c756 AP	143	___
8986e372	144	$code.=<<___;
e822c756	145	stm${g} %r6,%r14,6*$SIZE_T($sp)
7d1f55e9 AP	146
	147	aghi $Xi,-1
	148	srlg $len,$len,4
	149	lghi $x78,`0xf<<3`
	150	larl $rem_4bit,rem_4bit
	151
	152	lg $Zlo,8+1($Xi) # Xi
	153	lg $Zhi,0+1($Xi)
8986e372	154	lghi $tmp,0
7d1f55e9	155	.Louter:
609b0852	156	xg $Zhi,0($inp) # Xi ^= inp
8986e372 AP	157	xg $Zlo,8($inp)
8986e372 AP	158	xgr $Zhi,$tmp
7d1f55e9 AP	159	stg $Zlo,8+1($Xi)
	160	stg $Zhi,0+1($Xi)
	161
	162	.Lgmult_shortcut:
8986e372 AP	163	lghi $tmp,0xf0
	164	sllg $nlo,$Zlo,4
	165	srlg $xi,$Zlo,8 # extract second byte
	166	ngr $nlo,$tmp
7d1f55e9	167	lgr $nhi,$Zlo
7d1f55e9	168	lghi $cnt,14
8986e372	169	ngr $nhi,$tmp
7d1f55e9 AP	170
	171	lg $Zlo,8($nlo,$Htbl)
	172	lg $Zhi,0($nlo,$Htbl)
	173
	174	sllg $nlo,$xi,4
7d1f55e9	175	sllg $rem0,$Zlo,3
8986e372	176	ngr $nlo,$tmp
7d1f55e9	177	ngr $rem0,$x78
8986e372 AP	178	ngr $xi,$tmp
8986e372 AP	179
7d1f55e9	180	sllg $tmp,$Zhi,60
8986e372	181	srlg $Zlo,$Zlo,4
7d1f55e9	182	srlg $Zhi,$Zhi,4
8986e372	183	xg $Zlo,8($nhi,$Htbl)
7d1f55e9 AP	184	xg $Zhi,0($nhi,$Htbl)
	185	lgr $nhi,$xi
	186	sllg $rem1,$Zlo,3
8986e372 AP	187	xgr $Zlo,$tmp
8986e372 AP	188	ngr $rem1,$x78
d162584b	189	sllg $tmp,$Zhi,60
8986e372 AP	190	j .Lghash_inner
8986e372 AP	191	.align 16
7d1f55e9 AP	192	.Lghash_inner:
7d1f55e9 AP	193	srlg $Zlo,$Zlo,4
7d1f55e9	194	srlg $Zhi,$Zhi,4
d162584b	195	xg $Zlo,8($nlo,$Htbl)
7d1f55e9	196	llgc $xi,0($cnt,$Xi)
7d1f55e9 AP	197	xg $Zhi,0($nlo,$Htbl)
7d1f55e9 AP	198	sllg $nlo,$xi,4
8986e372	199	xg $Zhi,0($rem0,$rem_4bit)
7d1f55e9	200	nill $nlo,0xf0
8986e372 AP	201	sllg $rem0,$Zlo,3
8986e372 AP	202	xgr $Zlo,$tmp
7d1f55e9	203	ngr $rem0,$x78
8986e372 AP	204	nill $xi,0xf0
8986e372 AP	205
7d1f55e9	206	sllg $tmp,$Zhi,60
8986e372	207	srlg $Zlo,$Zlo,4
7d1f55e9	208	srlg $Zhi,$Zhi,4
8986e372	209	xg $Zlo,8($nhi,$Htbl)
7d1f55e9 AP	210	xg $Zhi,0($nhi,$Htbl)
7d1f55e9 AP	211	lgr $nhi,$xi
8986e372 AP	212	xg $Zhi,0($rem1,$rem_4bit)
	213	sllg $rem1,$Zlo,3
	214	xgr $Zlo,$tmp
	215	ngr $rem1,$x78
d162584b	216	sllg $tmp,$Zhi,60
7d1f55e9 AP	217	brct $cnt,.Lghash_inner
	218
	219	srlg $Zlo,$Zlo,4
8986e372	220	srlg $Zhi,$Zhi,4
7d1f55e9	221	xg $Zlo,8($nlo,$Htbl)
8986e372 AP	222	xg $Zhi,0($nlo,$Htbl)
8986e372 AP	223	sllg $xi,$Zlo,3
7d1f55e9 AP	224	xg $Zhi,0($rem0,$rem_4bit)
7d1f55e9 AP	225	xgr $Zlo,$tmp
8986e372	226	ngr $xi,$x78
7d1f55e9	227
7d1f55e9	228	sllg $tmp,$Zhi,60
8986e372	229	srlg $Zlo,$Zlo,4
7d1f55e9	230	srlg $Zhi,$Zhi,4
8986e372	231	xg $Zlo,8($nhi,$Htbl)
7d1f55e9	232	xg $Zhi,0($nhi,$Htbl)
8986e372 AP	233	xgr $Zlo,$tmp
8986e372 AP	234	xg $Zhi,0($rem1,$rem_4bit)
7d1f55e9	235
8986e372	236	lg $tmp,0($xi,$rem_4bit)
7d1f55e9	237	la $inp,16($inp)
8986e372	238	sllg $tmp,$tmp,4 # correct last rem_4bit[rem]
7d1f55e9 AP	239	brctg $len,.Louter
7d1f55e9 AP	240
8986e372	241	xgr $Zhi,$tmp
7d1f55e9 AP	242	stg $Zlo,8+1($Xi)
7d1f55e9 AP	243	stg $Zhi,0+1($Xi)
e822c756	244	lm${g} %r6,%r14,6*$SIZE_T($sp)
7d1f55e9 AP	245	br %r14
	246	.type gcm_ghash_4bit,\@function
	247	.size gcm_ghash_4bit,(.-gcm_ghash_4bit)
	248
	249	.align 64
	250	rem_4bit:
8986e372 AP	251	.long `0x0000<<12`,0,`0x1C20<<12`,0,`0x3840<<12`,0,`0x2460<<12`,0
	252	.long `0x7080<<12`,0,`0x6CA0<<12`,0,`0x48C0<<12`,0,`0x54E0<<12`,0
	253	.long `0xE100<<12`,0,`0xFD20<<12`,0,`0xD940<<12`,0,`0xC560<<12`,0
	254	.long `0x9180<<12`,0,`0x8DA0<<12`,0,`0xA9C0<<12`,0,`0xB5E0<<12`,0
7d1f55e9 AP	255	.type rem_4bit,\@object
	256	.size rem_4bit,(.-rem_4bit)
	257	.string "GHASH for s390x, CRYPTOGAMS by <appro\@openssl.org>"
	258	___
	259
	260	$code =~ s/\`([^\`]*)\`/eval $1/gem;
	261	print $code;
	262	close STDOUT;