Copyright year updates

[thirdparty/openssl.git] / crypto / sm4 / asm / vpsm4-armv8.pl

#! /usr/bin/env perl
# Copyright 2020-2024 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the Apache License 2.0 (the "License").  You may not use
# this file except in compliance with the License.  You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html

#
# This module implements SM4 with ASIMD on aarch64
#
# Feb 2022
#

# $output is the last argument if it looks like a file (it has an extension)
# $flavour is the first argument if it doesn't look like a file
$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;

$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
die "can't locate arm-xlate.pl";

open OUT,"| \"$^X\" $xlate $flavour \"$output\""
    or die "can't call $xlate: $!";
*STDOUT=*OUT;

$prefix="vpsm4";
my @vtmp=map("v$_",(0..3));
my @qtmp=map("q$_",(0..3));
my @data=map("v$_",(4..7));
my @datax=map("v$_",(8..11));
my ($rk0,$rk1)=("v12","v13");
my ($rka,$rkb)=("v14","v15");
my @vtmpx=map("v$_",(12..15));
my @sbox=map("v$_",(16..31));
my ($inp,$outp,$blocks,$rks)=("x0","x1","w2","x3");
my ($tmpw,$tmp,$wtmp0,$wtmp1,$wtmp2)=("w6","x6","w7","w8","w9");
my ($xtmp1,$xtmp2)=("x8","x9");
my ($ptr,$counter)=("x10","w11");
my ($word0,$word1,$word2,$word3)=("w12","w13","w14","w15");

sub rev32() {
        my $dst = shift;
        my $src = shift;

        if ($src and ("$src" ne "$dst")) {
$code.=<<___;
#ifndef __AARCH64EB__
        rev32   $dst.16b,$src.16b
#else
        mov     $dst.16b,$src.16b
#endif
___
        } else {
$code.=<<___;
#ifndef __AARCH64EB__
        rev32   $dst.16b,$dst.16b
#endif
___
        }
}

sub rev32_armeb() {
        my $dst = shift;
        my $src = shift;

        if ($src and ("$src" ne "$dst")) {
$code.=<<___;
#ifdef __AARCH64EB__
        rev32   $dst.16b,$src.16b
#else
        mov     $dst.16b,$src.16b
#endif
___
        } else {
$code.=<<___;
#ifdef __AARCH64EB__
        rev32   $dst.16b,$dst.16b
#endif
___
        }
}

sub rbit() {
        my $dst = shift;
        my $src = shift;
        my $std = shift;

        if ($src and ("$src" ne "$dst")) {
                if ($std eq "_gb") {
$code.=<<___;
                        rbit $dst.16b,$src.16b
___
                } else {
$code.=<<___;
                        mov $dst.16b,$src.16b
___
                }
        } else {
                if ($std eq "_gb") {
$code.=<<___;
                        rbit $dst.16b,$src.16b
___
                }
        }
}

sub transpose() {
        my ($dat0,$dat1,$dat2,$dat3,$vt0,$vt1,$vt2,$vt3) = @_;

$code.=<<___;
        zip1    $vt0.4s,$dat0.4s,$dat1.4s
        zip2    $vt1.4s,$dat0.4s,$dat1.4s
        zip1    $vt2.4s,$dat2.4s,$dat3.4s
        zip2    $vt3.4s,$dat2.4s,$dat3.4s
        zip1    $dat0.2d,$vt0.2d,$vt2.2d
        zip2    $dat1.2d,$vt0.2d,$vt2.2d
        zip1    $dat2.2d,$vt1.2d,$vt3.2d
        zip2    $dat3.2d,$vt1.2d,$vt3.2d
___
}

# sbox operations for 4-lane of words
sub sbox() {
        my $dat = shift;

$code.=<<___;
        movi    @vtmp[0].16b,#64
        movi    @vtmp[1].16b,#128
        movi    @vtmp[2].16b,#192
        sub     @vtmp[0].16b,$dat.16b,@vtmp[0].16b
        sub     @vtmp[1].16b,$dat.16b,@vtmp[1].16b
        sub     @vtmp[2].16b,$dat.16b,@vtmp[2].16b
        tbl     $dat.16b,{@sbox[0].16b,@sbox[1].16b,@sbox[2].16b,@sbox[3].16b},$dat.16b
        tbl     @vtmp[0].16b,{@sbox[4].16b,@sbox[5].16b,@sbox[6].16b,@sbox[7].16b},@vtmp[0].16b
        tbl     @vtmp[1].16b,{@sbox[8].16b,@sbox[9].16b,@sbox[10].16b,@sbox[11].16b},@vtmp[1].16b
        tbl     @vtmp[2].16b,{@sbox[12].16b,@sbox[13].16b,@sbox[14].16b,@sbox[15].16b},@vtmp[2].16b
        add     @vtmp[0].2d,@vtmp[0].2d,@vtmp[1].2d
        add     @vtmp[2].2d,@vtmp[2].2d,$dat.2d
        add     $dat.2d,@vtmp[0].2d,@vtmp[2].2d

        ushr    @vtmp[0].4s,$dat.4s,32-2
        sli     @vtmp[0].4s,$dat.4s,2
        ushr    @vtmp[2].4s,$dat.4s,32-10
        eor     @vtmp[1].16b,@vtmp[0].16b,$dat.16b
        sli     @vtmp[2].4s,$dat.4s,10
        eor     @vtmp[1].16b,@vtmp[2].16b,$vtmp[1].16b
        ushr    @vtmp[0].4s,$dat.4s,32-18
        sli     @vtmp[0].4s,$dat.4s,18
        ushr    @vtmp[2].4s,$dat.4s,32-24
        eor     @vtmp[1].16b,@vtmp[0].16b,$vtmp[1].16b
        sli     @vtmp[2].4s,$dat.4s,24
        eor     $dat.16b,@vtmp[2].16b,@vtmp[1].16b
___
}

# sbox operation for 8-lane of words
sub sbox_double() {
        my $dat = shift;
        my $datx = shift;

$code.=<<___;
        movi    @vtmp[3].16b,#64
        sub     @vtmp[0].16b,$dat.16b,@vtmp[3].16b
        sub     @vtmp[1].16b,@vtmp[0].16b,@vtmp[3].16b
        sub     @vtmp[2].16b,@vtmp[1].16b,@vtmp[3].16b
        tbl     $dat.16b,{@sbox[0].16b,@sbox[1].16b,@sbox[2].16b,@sbox[3].16b},$dat.16b
        tbl     @vtmp[0].16b,{@sbox[4].16b,@sbox[5].16b,@sbox[6].16b,@sbox[7].16b},@vtmp[0].16b
        tbl     @vtmp[1].16b,{@sbox[8].16b,@sbox[9].16b,@sbox[10].16b,@sbox[11].16b},@vtmp[1].16b
        tbl     @vtmp[2].16b,{@sbox[12].16b,@sbox[13].16b,@sbox[14].16b,@sbox[15].16b},@vtmp[2].16b
        add     @vtmp[1].2d,@vtmp[0].2d,@vtmp[1].2d
        add     $dat.2d,@vtmp[2].2d,$dat.2d
        add     $dat.2d,@vtmp[1].2d,$dat.2d

        sub     @vtmp[0].16b,$datx.16b,@vtmp[3].16b
        sub     @vtmp[1].16b,@vtmp[0].16b,@vtmp[3].16b
        sub     @vtmp[2].16b,@vtmp[1].16b,@vtmp[3].16b
        tbl     $datx.16b,{@sbox[0].16b,@sbox[1].16b,@sbox[2].16b,@sbox[3].16b},$datx.16b
        tbl     @vtmp[0].16b,{@sbox[4].16b,@sbox[5].16b,@sbox[6].16b,@sbox[7].16b},@vtmp[0].16b
        tbl     @vtmp[1].16b,{@sbox[8].16b,@sbox[9].16b,@sbox[10].16b,@sbox[11].16b},@vtmp[1].16b
        tbl     @vtmp[2].16b,{@sbox[12].16b,@sbox[13].16b,@sbox[14].16b,@sbox[15].16b},@vtmp[2].16b
        add     @vtmp[1].2d,@vtmp[0].2d,@vtmp[1].2d
        add     $datx.2d,@vtmp[2].2d,$datx.2d
        add     $datx.2d,@vtmp[1].2d,$datx.2d

        ushr    @vtmp[0].4s,$dat.4s,32-2
        sli     @vtmp[0].4s,$dat.4s,2
        ushr    @vtmp[2].4s,$datx.4s,32-2
        eor     @vtmp[1].16b,@vtmp[0].16b,$dat.16b
        sli     @vtmp[2].4s,$datx.4s,2

        ushr    @vtmp[0].4s,$dat.4s,32-10
        eor     @vtmp[3].16b,@vtmp[2].16b,$datx.16b
        sli     @vtmp[0].4s,$dat.4s,10
        ushr    @vtmp[2].4s,$datx.4s,32-10
        eor     @vtmp[1].16b,@vtmp[0].16b,$vtmp[1].16b
        sli     @vtmp[2].4s,$datx.4s,10

        ushr    @vtmp[0].4s,$dat.4s,32-18
        eor     @vtmp[3].16b,@vtmp[2].16b,$vtmp[3].16b
        sli     @vtmp[0].4s,$dat.4s,18
        ushr    @vtmp[2].4s,$datx.4s,32-18
        eor     @vtmp[1].16b,@vtmp[0].16b,$vtmp[1].16b
        sli     @vtmp[2].4s,$datx.4s,18

        ushr    @vtmp[0].4s,$dat.4s,32-24
        eor     @vtmp[3].16b,@vtmp[2].16b,$vtmp[3].16b
        sli     @vtmp[0].4s,$dat.4s,24
        ushr    @vtmp[2].4s,$datx.4s,32-24
        eor     $dat.16b,@vtmp[0].16b,@vtmp[1].16b
        sli     @vtmp[2].4s,$datx.4s,24
        eor     $datx.16b,@vtmp[2].16b,@vtmp[3].16b
___
}

# sbox operation for one single word
sub sbox_1word () {
        my $word = shift;

$code.=<<___;
        movi    @vtmp[1].16b,#64
        movi    @vtmp[2].16b,#128
        movi    @vtmp[3].16b,#192
        mov     @vtmp[0].s[0],$word

        sub     @vtmp[1].16b,@vtmp[0].16b,@vtmp[1].16b
        sub     @vtmp[2].16b,@vtmp[0].16b,@vtmp[2].16b
        sub     @vtmp[3].16b,@vtmp[0].16b,@vtmp[3].16b

        tbl     @vtmp[0].16b,{@sbox[0].16b,@sbox[1].16b,@sbox[2].16b,@sbox[3].16b},@vtmp[0].16b
        tbl     @vtmp[1].16b,{@sbox[4].16b,@sbox[5].16b,@sbox[6].16b,@sbox[7].16b},@vtmp[1].16b
        tbl     @vtmp[2].16b,{@sbox[8].16b,@sbox[9].16b,@sbox[10].16b,@sbox[11].16b},@vtmp[2].16b
        tbl     @vtmp[3].16b,{@sbox[12].16b,@sbox[13].16b,@sbox[14].16b,@sbox[15].16b},@vtmp[3].16b

        mov     $word,@vtmp[0].s[0]
        mov     $wtmp0,@vtmp[1].s[0]
        mov     $wtmp2,@vtmp[2].s[0]
        add     $wtmp0,$word,$wtmp0
        mov     $word,@vtmp[3].s[0]
        add     $wtmp0,$wtmp0,$wtmp2
        add     $wtmp0,$wtmp0,$word

        eor     $word,$wtmp0,$wtmp0,ror #32-2
        eor     $word,$word,$wtmp0,ror #32-10
        eor     $word,$word,$wtmp0,ror #32-18
        eor     $word,$word,$wtmp0,ror #32-24
___
}

# sm4 for one block of data, in scalar registers word0/word1/word2/word3
sub sm4_1blk () {
        my $kptr = shift;

$code.=<<___;
        ldp     $wtmp0,$wtmp1,[$kptr],8
        // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0)
        eor     $tmpw,$word2,$word3
        eor     $wtmp2,$wtmp0,$word1
        eor     $tmpw,$tmpw,$wtmp2
___
        &sbox_1word($tmpw);
$code.=<<___;
        eor     $word0,$word0,$tmpw
        // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1)
        eor     $tmpw,$word2,$word3
        eor     $wtmp2,$word0,$wtmp1
        eor     $tmpw,$tmpw,$wtmp2
___
        &sbox_1word($tmpw);
$code.=<<___;
        ldp     $wtmp0,$wtmp1,[$kptr],8
        eor     $word1,$word1,$tmpw
        // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2)
        eor     $tmpw,$word0,$word1
        eor     $wtmp2,$wtmp0,$word3
        eor     $tmpw,$tmpw,$wtmp2
___
        &sbox_1word($tmpw);
$code.=<<___;
        eor     $word2,$word2,$tmpw
        // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3)
        eor     $tmpw,$word0,$word1
        eor     $wtmp2,$word2,$wtmp1
        eor     $tmpw,$tmpw,$wtmp2
___
        &sbox_1word($tmpw);
$code.=<<___;
        eor     $word3,$word3,$tmpw
___
}

# sm4 for 4-lanes of data, in neon registers data0/data1/data2/data3
sub sm4_4blks () {
        my $kptr = shift;

$code.=<<___;
        ldp     $wtmp0,$wtmp1,[$kptr],8
        dup     $rk0.4s,$wtmp0
        dup     $rk1.4s,$wtmp1

        // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0)
        eor     $rka.16b,@data[2].16b,@data[3].16b
        eor     $rk0.16b,@data[1].16b,$rk0.16b
        eor     $rk0.16b,$rka.16b,$rk0.16b
___
        &sbox($rk0);
$code.=<<___;
        eor     @data[0].16b,@data[0].16b,$rk0.16b

        // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1)
        eor     $rka.16b,$rka.16b,@data[0].16b
        eor     $rk1.16b,$rka.16b,$rk1.16b
___
        &sbox($rk1);
$code.=<<___;
        ldp     $wtmp0,$wtmp1,[$kptr],8
        eor     @data[1].16b,@data[1].16b,$rk1.16b

        dup     $rk0.4s,$wtmp0
        dup     $rk1.4s,$wtmp1

        // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2)
        eor     $rka.16b,@data[0].16b,@data[1].16b
        eor     $rk0.16b,@data[3].16b,$rk0.16b
        eor     $rk0.16b,$rka.16b,$rk0.16b
___
        &sbox($rk0);
$code.=<<___;
        eor     @data[2].16b,@data[2].16b,$rk0.16b

        // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3)
        eor     $rka.16b,$rka.16b,@data[2].16b
        eor     $rk1.16b,$rka.16b,$rk1.16b
___
        &sbox($rk1);
$code.=<<___;
        eor     @data[3].16b,@data[3].16b,$rk1.16b
___
}

# sm4 for 8 lanes of data, in neon registers
# data0/data1/data2/data3 datax0/datax1/datax2/datax3
sub sm4_8blks () {
        my $kptr = shift;

$code.=<<___;
        ldp     $wtmp0,$wtmp1,[$kptr],8
        // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0)
        dup     $rk0.4s,$wtmp0
        eor     $rka.16b,@data[2].16b,@data[3].16b
        eor     $rkb.16b,@datax[2].16b,@datax[3].16b
        eor     @vtmp[0].16b,@data[1].16b,$rk0.16b
        eor     @vtmp[1].16b,@datax[1].16b,$rk0.16b
        eor     $rk0.16b,$rka.16b,@vtmp[0].16b
        eor     $rk1.16b,$rkb.16b,@vtmp[1].16b
___
        &sbox_double($rk0,$rk1);
$code.=<<___;
        eor     @data[0].16b,@data[0].16b,$rk0.16b
        eor     @datax[0].16b,@datax[0].16b,$rk1.16b

        // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1)
        dup     $rk1.4s,$wtmp1
        eor     $rka.16b,$rka.16b,@data[0].16b
        eor     $rkb.16b,$rkb.16b,@datax[0].16b
        eor     $rk0.16b,$rka.16b,$rk1.16b
        eor     $rk1.16b,$rkb.16b,$rk1.16b
___
        &sbox_double($rk0,$rk1);
$code.=<<___;
        ldp     $wtmp0,$wtmp1,[$kptr],8
        eor     @data[1].16b,@data[1].16b,$rk0.16b
        eor     @datax[1].16b,@datax[1].16b,$rk1.16b

        // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2)
        dup     $rk0.4s,$wtmp0
        eor     $rka.16b,@data[0].16b,@data[1].16b
        eor     $rkb.16b,@datax[0].16b,@datax[1].16b
        eor     @vtmp[0].16b,@data[3].16b,$rk0.16b
        eor     @vtmp[1].16b,@datax[3].16b,$rk0.16b
        eor     $rk0.16b,$rka.16b,@vtmp[0].16b
        eor     $rk1.16b,$rkb.16b,@vtmp[1].16b
___
        &sbox_double($rk0,$rk1);
$code.=<<___;
        eor     @data[2].16b,@data[2].16b,$rk0.16b
        eor     @datax[2].16b,@datax[2].16b,$rk1.16b

        // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3)
        dup     $rk1.4s,$wtmp1
        eor     $rka.16b,$rka.16b,@data[2].16b
        eor     $rkb.16b,$rkb.16b,@datax[2].16b
        eor     $rk0.16b,$rka.16b,$rk1.16b
        eor     $rk1.16b,$rkb.16b,$rk1.16b
___
        &sbox_double($rk0,$rk1);
$code.=<<___;
        eor     @data[3].16b,@data[3].16b,$rk0.16b
        eor     @datax[3].16b,@datax[3].16b,$rk1.16b
___
}

sub encrypt_1blk_norev() {
        my $dat = shift;

$code.=<<___;
        mov     $ptr,$rks
        mov     $counter,#8
        mov     $word0,$dat.s[0]
        mov     $word1,$dat.s[1]
        mov     $word2,$dat.s[2]
        mov     $word3,$dat.s[3]
10:
___
        &sm4_1blk($ptr);
$code.=<<___;
        subs    $counter,$counter,#1
        b.ne    10b
        mov     $dat.s[0],$word3
        mov     $dat.s[1],$word2
        mov     $dat.s[2],$word1
        mov     $dat.s[3],$word0
___
}

sub encrypt_1blk() {
        my $dat = shift;

        &encrypt_1blk_norev($dat);
        &rev32($dat,$dat);
}

sub encrypt_4blks() {
$code.=<<___;
        mov     $ptr,$rks
        mov     $counter,#8
10:
___
        &sm4_4blks($ptr);
$code.=<<___;
        subs    $counter,$counter,#1
        b.ne    10b
___
        &rev32(@vtmp[3],@data[0]);
        &rev32(@vtmp[2],@data[1]);
        &rev32(@vtmp[1],@data[2]);
        &rev32(@vtmp[0],@data[3]);
}

sub encrypt_8blks() {
$code.=<<___;
        mov     $ptr,$rks
        mov     $counter,#8
10:
___
        &sm4_8blks($ptr);
$code.=<<___;
        subs    $counter,$counter,#1
        b.ne    10b
___
        &rev32(@vtmp[3],@data[0]);
        &rev32(@vtmp[2],@data[1]);
        &rev32(@vtmp[1],@data[2]);
        &rev32(@vtmp[0],@data[3]);
        &rev32(@data[3],@datax[0]);
        &rev32(@data[2],@datax[1]);
        &rev32(@data[1],@datax[2]);
        &rev32(@data[0],@datax[3]);
}

sub load_sbox () {
        my $data = shift;

$code.=<<___;
        adr     $ptr,.Lsbox
        ld1     {@sbox[0].16b,@sbox[1].16b,@sbox[2].16b,@sbox[3].16b},[$ptr],#64
        ld1     {@sbox[4].16b,@sbox[5].16b,@sbox[6].16b,@sbox[7].16b},[$ptr],#64
        ld1     {@sbox[8].16b,@sbox[9].16b,@sbox[10].16b,@sbox[11].16b},[$ptr],#64
        ld1     {@sbox[12].16b,@sbox[13].16b,@sbox[14].16b,@sbox[15].16b},[$ptr]
___
}


sub mov_reg_to_vec() {
        my $src0 = shift;
        my $src1 = shift;
        my $desv = shift;
$code.=<<___;
        mov $desv.d[0],$src0
        mov $desv.d[1],$src1
___
        &rev32_armeb($desv,$desv);
}

sub mov_vec_to_reg() {
        my $srcv = shift;
        my $des0 = shift;
        my $des1 = shift;
$code.=<<___;
        mov $des0,$srcv.d[0]
        mov $des1,$srcv.d[1]
___
}

sub compute_tweak() {
        my $src0 = shift;
        my $src1 = shift;
        my $des0 = shift;
        my $des1 = shift;
$code.=<<___;
        mov $wtmp0,0x87
        extr    $xtmp2,$src1,$src1,#32
        extr    $des1,$src1,$src0,#63
        and     $wtmp1,$wtmp0,$wtmp2,asr#31
        eor     $des0,$xtmp1,$src0,lsl#1
___
}

sub compute_tweak_vec() {
        my $src = shift;
        my $des = shift;
        my $std = shift;
        &rbit(@vtmp[2],$src,$std);
$code.=<<___;
        ldr  @qtmp[0], .Lxts_magic
        shl  $des.16b, @vtmp[2].16b, #1
        ext  @vtmp[1].16b, @vtmp[2].16b, @vtmp[2].16b,#15
        ushr @vtmp[1].16b, @vtmp[1].16b, #7
        mul  @vtmp[1].16b, @vtmp[1].16b, @vtmp[0].16b
        eor  $des.16b, $des.16b, @vtmp[1].16b
___
        &rbit($des,$des,$std);
}

$code=<<___;
#include "arm_arch.h"
.arch   armv8-a
.text

.type   _vpsm4_consts,%object
.align  7
_vpsm4_consts:
.Lsbox:
        .byte 0xD6,0x90,0xE9,0xFE,0xCC,0xE1,0x3D,0xB7,0x16,0xB6,0x14,0xC2,0x28,0xFB,0x2C,0x05
        .byte 0x2B,0x67,0x9A,0x76,0x2A,0xBE,0x04,0xC3,0xAA,0x44,0x13,0x26,0x49,0x86,0x06,0x99
        .byte 0x9C,0x42,0x50,0xF4,0x91,0xEF,0x98,0x7A,0x33,0x54,0x0B,0x43,0xED,0xCF,0xAC,0x62
        .byte 0xE4,0xB3,0x1C,0xA9,0xC9,0x08,0xE8,0x95,0x80,0xDF,0x94,0xFA,0x75,0x8F,0x3F,0xA6
        .byte 0x47,0x07,0xA7,0xFC,0xF3,0x73,0x17,0xBA,0x83,0x59,0x3C,0x19,0xE6,0x85,0x4F,0xA8
        .byte 0x68,0x6B,0x81,0xB2,0x71,0x64,0xDA,0x8B,0xF8,0xEB,0x0F,0x4B,0x70,0x56,0x9D,0x35
        .byte 0x1E,0x24,0x0E,0x5E,0x63,0x58,0xD1,0xA2,0x25,0x22,0x7C,0x3B,0x01,0x21,0x78,0x87
        .byte 0xD4,0x00,0x46,0x57,0x9F,0xD3,0x27,0x52,0x4C,0x36,0x02,0xE7,0xA0,0xC4,0xC8,0x9E
        .byte 0xEA,0xBF,0x8A,0xD2,0x40,0xC7,0x38,0xB5,0xA3,0xF7,0xF2,0xCE,0xF9,0x61,0x15,0xA1
        .byte 0xE0,0xAE,0x5D,0xA4,0x9B,0x34,0x1A,0x55,0xAD,0x93,0x32,0x30,0xF5,0x8C,0xB1,0xE3
        .byte 0x1D,0xF6,0xE2,0x2E,0x82,0x66,0xCA,0x60,0xC0,0x29,0x23,0xAB,0x0D,0x53,0x4E,0x6F
        .byte 0xD5,0xDB,0x37,0x45,0xDE,0xFD,0x8E,0x2F,0x03,0xFF,0x6A,0x72,0x6D,0x6C,0x5B,0x51
        .byte 0x8D,0x1B,0xAF,0x92,0xBB,0xDD,0xBC,0x7F,0x11,0xD9,0x5C,0x41,0x1F,0x10,0x5A,0xD8
        .byte 0x0A,0xC1,0x31,0x88,0xA5,0xCD,0x7B,0xBD,0x2D,0x74,0xD0,0x12,0xB8,0xE5,0xB4,0xB0
        .byte 0x89,0x69,0x97,0x4A,0x0C,0x96,0x77,0x7E,0x65,0xB9,0xF1,0x09,0xC5,0x6E,0xC6,0x84
        .byte 0x18,0xF0,0x7D,0xEC,0x3A,0xDC,0x4D,0x20,0x79,0xEE,0x5F,0x3E,0xD7,0xCB,0x39,0x48
.Lck:
        .long 0x00070E15, 0x1C232A31, 0x383F464D, 0x545B6269
        .long 0x70777E85, 0x8C939AA1, 0xA8AFB6BD, 0xC4CBD2D9
        .long 0xE0E7EEF5, 0xFC030A11, 0x181F262D, 0x343B4249
        .long 0x50575E65, 0x6C737A81, 0x888F969D, 0xA4ABB2B9
        .long 0xC0C7CED5, 0xDCE3EAF1, 0xF8FF060D, 0x141B2229
        .long 0x30373E45, 0x4C535A61, 0x686F767D, 0x848B9299
        .long 0xA0A7AEB5, 0xBCC3CAD1, 0xD8DFE6ED, 0xF4FB0209
        .long 0x10171E25, 0x2C333A41, 0x484F565D, 0x646B7279
.Lfk:
        .quad 0x56aa3350a3b1bac6,0xb27022dc677d9197
.Lshuffles:
        .quad 0x0B0A090807060504,0x030201000F0E0D0C
.Lxts_magic:
        .quad 0x0101010101010187,0x0101010101010101

.size   _vpsm4_consts,.-_vpsm4_consts
___

{{{
my ($key,$keys,$enc)=("x0","x1","w2");
my ($pointer,$schedules,$wtmp,$roundkey)=("x5","x6","w7","w8");
my ($vkey,$vfk,$vmap)=("v5","v6","v7");
$code.=<<___;
.type   _vpsm4_set_key,%function
.align  4
_vpsm4_set_key:
        AARCH64_VALID_CALL_TARGET
        ld1     {$vkey.4s},[$key]
___
        &load_sbox();
        &rev32($vkey,$vkey);
$code.=<<___;
        adr     $pointer,.Lshuffles
        ld1     {$vmap.2d},[$pointer]
        adr     $pointer,.Lfk
        ld1     {$vfk.2d},[$pointer]
        eor     $vkey.16b,$vkey.16b,$vfk.16b
        mov     $schedules,#32
        adr     $pointer,.Lck
        movi    @vtmp[0].16b,#64
        cbnz    $enc,1f
        add     $keys,$keys,124
1:
        mov     $wtmp,$vkey.s[1]
        ldr     $roundkey,[$pointer],#4
        eor     $roundkey,$roundkey,$wtmp
        mov     $wtmp,$vkey.s[2]
        eor     $roundkey,$roundkey,$wtmp
        mov     $wtmp,$vkey.s[3]
        eor     $roundkey,$roundkey,$wtmp
        // sbox lookup
        mov     @data[0].s[0],$roundkey
        tbl     @vtmp[1].16b,{@sbox[0].16b,@sbox[1].16b,@sbox[2].16b,@sbox[3].16b},@data[0].16b
        sub     @data[0].16b,@data[0].16b,@vtmp[0].16b
        tbx     @vtmp[1].16b,{@sbox[4].16b,@sbox[5].16b,@sbox[6].16b,@sbox[7].16b},@data[0].16b
        sub     @data[0].16b,@data[0].16b,@vtmp[0].16b
        tbx     @vtmp[1].16b,{@sbox[8].16b,@sbox[9].16b,@sbox[10].16b,@sbox[11].16b},@data[0].16b
        sub     @data[0].16b,@data[0].16b,@vtmp[0].16b
        tbx     @vtmp[1].16b,{@sbox[12].16b,@sbox[13].16b,@sbox[14].16b,@sbox[15].16b},@data[0].16b
        mov     $wtmp,@vtmp[1].s[0]
        eor     $roundkey,$wtmp,$wtmp,ror #19
        eor     $roundkey,$roundkey,$wtmp,ror #9
        mov     $wtmp,$vkey.s[0]
        eor     $roundkey,$roundkey,$wtmp
        mov     $vkey.s[0],$roundkey
        cbz     $enc,2f
        str     $roundkey,[$keys],#4
        b       3f
2:
        str     $roundkey,[$keys],#-4
3:
        tbl     $vkey.16b,{$vkey.16b},$vmap.16b
        subs    $schedules,$schedules,#1
        b.ne    1b
        ret
.size   _vpsm4_set_key,.-_vpsm4_set_key
___
}}}


{{{
$code.=<<___;
.type   _vpsm4_enc_4blks,%function
.align  4
_vpsm4_enc_4blks:
        AARCH64_VALID_CALL_TARGET
___
        &encrypt_4blks();
$code.=<<___;
        ret
.size   _vpsm4_enc_4blks,.-_vpsm4_enc_4blks
___
}}}

{{{
$code.=<<___;
.type   _vpsm4_enc_8blks,%function
.align  4
_vpsm4_enc_8blks:
        AARCH64_VALID_CALL_TARGET
___
        &encrypt_8blks();
$code.=<<___;
        ret
.size   _vpsm4_enc_8blks,.-_vpsm4_enc_8blks
___
}}}


{{{
my ($key,$keys)=("x0","x1");
$code.=<<___;
.globl  ${prefix}_set_encrypt_key
.type   ${prefix}_set_encrypt_key,%function
.align  5
${prefix}_set_encrypt_key:
        AARCH64_SIGN_LINK_REGISTER
        stp     x29,x30,[sp,#-16]!
        mov     w2,1
        bl      _vpsm4_set_key
        ldp     x29,x30,[sp],#16
        AARCH64_VALIDATE_LINK_REGISTER
        ret
.size   ${prefix}_set_encrypt_key,.-${prefix}_set_encrypt_key
___
}}}

{{{
my ($key,$keys)=("x0","x1");
$code.=<<___;
.globl  ${prefix}_set_decrypt_key
.type   ${prefix}_set_decrypt_key,%function
.align  5
${prefix}_set_decrypt_key:
        AARCH64_SIGN_LINK_REGISTER
        stp     x29,x30,[sp,#-16]!
        mov     w2,0
        bl      _vpsm4_set_key
        ldp     x29,x30,[sp],#16
        AARCH64_VALIDATE_LINK_REGISTER
        ret
.size   ${prefix}_set_decrypt_key,.-${prefix}_set_decrypt_key
___
}}}

{{{
sub gen_block () {
        my $dir = shift;
        my ($inp,$outp,$rk)=map("x$_",(0..2));

$code.=<<___;
.globl  ${prefix}_${dir}crypt
.type   ${prefix}_${dir}crypt,%function
.align  5
${prefix}_${dir}crypt:
        AARCH64_VALID_CALL_TARGET
        ld1     {@data[0].4s},[$inp]
___
        &load_sbox();
        &rev32(@data[0],@data[0]);
$code.=<<___;
        mov     $rks,x2
___
        &encrypt_1blk(@data[0]);
$code.=<<___;
        st1     {@data[0].4s},[$outp]
        ret
.size   ${prefix}_${dir}crypt,.-${prefix}_${dir}crypt
___
}
&gen_block("en");
&gen_block("de");
}}}

{{{
my ($enc) = ("w4");
my @dat=map("v$_",(16..23));

$code.=<<___;
.globl  ${prefix}_ecb_encrypt
.type   ${prefix}_ecb_encrypt,%function
.align  5
${prefix}_ecb_encrypt:
        AARCH64_SIGN_LINK_REGISTER
        // convert length into blocks
        lsr     x2,x2,4
        stp     d8,d9,[sp,#-80]!
        stp     d10,d11,[sp,#16]
        stp     d12,d13,[sp,#32]
        stp     d14,d15,[sp,#48]
        stp     x29,x30,[sp,#64]
___
        &load_sbox();
$code.=<<___;
.Lecb_8_blocks_process:
        cmp     $blocks,#8
        b.lt    .Lecb_4_blocks_process
        ld4     {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$inp],#64
        ld4     {@datax[0].4s,$datax[1].4s,@datax[2].4s,@datax[3].4s},[$inp],#64
___
        &rev32(@data[0],@data[0]);
        &rev32(@data[1],@data[1]);
        &rev32(@data[2],@data[2]);
        &rev32(@data[3],@data[3]);
        &rev32(@datax[0],@datax[0]);
        &rev32(@datax[1],@datax[1]);
        &rev32(@datax[2],@datax[2]);
        &rev32(@datax[3],@datax[3]);
$code.=<<___;
        bl      _vpsm4_enc_8blks
        st4     {@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s,@vtmp[3].4s},[$outp],#64
        st4     {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$outp],#64
        subs    $blocks,$blocks,#8
        b.gt    .Lecb_8_blocks_process
        b       100f
.Lecb_4_blocks_process:
        cmp     $blocks,#4
        b.lt    1f
        ld4     {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$inp],#64
___
        &rev32(@data[0],@data[0]);
        &rev32(@data[1],@data[1]);
        &rev32(@data[2],@data[2]);
        &rev32(@data[3],@data[3]);
$code.=<<___;
        bl      _vpsm4_enc_4blks
        st4     {@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s,@vtmp[3].4s},[$outp],#64
        sub     $blocks,$blocks,#4
1:
        // process last block
        cmp     $blocks,#1
        b.lt    100f
        b.gt    1f
        ld1     {@data[0].4s},[$inp]
___
        &rev32(@data[0],@data[0]);
        &encrypt_1blk(@data[0]);
$code.=<<___;
        st1     {@data[0].4s},[$outp]
        b       100f
1:      // process last 2 blocks
        ld4     {@data[0].s,@data[1].s,@data[2].s,@data[3].s}[0],[$inp],#16
        ld4     {@data[0].s,@data[1].s,@data[2].s,@data[3].s}[1],[$inp],#16
        cmp     $blocks,#2
        b.gt    1f
___
        &rev32(@data[0],@data[0]);
        &rev32(@data[1],@data[1]);
        &rev32(@data[2],@data[2]);
        &rev32(@data[3],@data[3]);
$code.=<<___;
        bl      _vpsm4_enc_4blks
        st4     {@vtmp[0].s-@vtmp[3].s}[0],[$outp],#16
        st4     {@vtmp[0].s-@vtmp[3].s}[1],[$outp]
        b       100f
1:      // process last 3 blocks
        ld4     {@data[0].s,@data[1].s,@data[2].s,@data[3].s}[2],[$inp],#16
___
        &rev32(@data[0],@data[0]);
        &rev32(@data[1],@data[1]);
        &rev32(@data[2],@data[2]);
        &rev32(@data[3],@data[3]);
$code.=<<___;
        bl      _vpsm4_enc_4blks
        st4     {@vtmp[0].s-@vtmp[3].s}[0],[$outp],#16
        st4     {@vtmp[0].s-@vtmp[3].s}[1],[$outp],#16
        st4     {@vtmp[0].s-@vtmp[3].s}[2],[$outp]
100:
        ldp     d10,d11,[sp,#16]
        ldp     d12,d13,[sp,#32]
        ldp     d14,d15,[sp,#48]
        ldp     x29,x30,[sp,#64]
        ldp     d8,d9,[sp],#80
        AARCH64_VALIDATE_LINK_REGISTER
        ret
.size   ${prefix}_ecb_encrypt,.-${prefix}_ecb_encrypt
___
}}}

{{{
my ($len,$ivp,$enc)=("x2","x4","w5");
my $ivec0=("v3");
my $ivec1=("v15");

$code.=<<___;
.globl  ${prefix}_cbc_encrypt
.type   ${prefix}_cbc_encrypt,%function
.align  5
${prefix}_cbc_encrypt:
        AARCH64_VALID_CALL_TARGET
        lsr     $len,$len,4
___
        &load_sbox();
$code.=<<___;
        cbz     $enc,.Ldec
        ld1     {$ivec0.4s},[$ivp]
.Lcbc_4_blocks_enc:
        cmp     $blocks,#4
        b.lt    1f
        ld1     {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$inp],#64
        eor     @data[0].16b,@data[0].16b,$ivec0.16b
___
        &rev32(@data[1],@data[1]);
        &rev32(@data[0],@data[0]);
        &rev32(@data[2],@data[2]);
        &rev32(@data[3],@data[3]);
        &encrypt_1blk_norev(@data[0]);
$code.=<<___;
        eor     @data[1].16b,@data[1].16b,@data[0].16b
___
        &encrypt_1blk_norev(@data[1]);
        &rev32(@data[0],@data[0]);

$code.=<<___;
        eor     @data[2].16b,@data[2].16b,@data[1].16b
___
        &encrypt_1blk_norev(@data[2]);
        &rev32(@data[1],@data[1]);
$code.=<<___;
        eor     @data[3].16b,@data[3].16b,@data[2].16b
___
        &encrypt_1blk_norev(@data[3]);
        &rev32(@data[2],@data[2]);
        &rev32(@data[3],@data[3]);
$code.=<<___;
        orr     $ivec0.16b,@data[3].16b,@data[3].16b
        st1     {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$outp],#64
        subs    $blocks,$blocks,#4
        b.ne    .Lcbc_4_blocks_enc
        b       2f
1:
        subs    $blocks,$blocks,#1
        b.lt    2f
        ld1     {@data[0].4s},[$inp],#16
        eor     $ivec0.16b,$ivec0.16b,@data[0].16b
___
        &rev32($ivec0,$ivec0);
        &encrypt_1blk($ivec0);
$code.=<<___;
        st1     {$ivec0.4s},[$outp],#16
        b       1b
2:
        // save back IV
        st1     {$ivec0.4s},[$ivp]
        ret

.Ldec:
        // decryption mode starts
        AARCH64_SIGN_LINK_REGISTER
        stp     d8,d9,[sp,#-80]!
        stp     d10,d11,[sp,#16]
        stp     d12,d13,[sp,#32]
        stp     d14,d15,[sp,#48]
        stp     x29,x30,[sp,#64]
.Lcbc_8_blocks_dec:
        cmp     $blocks,#8
        b.lt    1f
        ld4     {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$inp]
        add     $ptr,$inp,#64
        ld4     {@datax[0].4s,@datax[1].4s,@datax[2].4s,@datax[3].4s},[$ptr]
___
        &rev32(@data[0],@data[0]);
        &rev32(@data[1],@data[1]);
        &rev32(@data[2],@data[2]);
        &rev32(@data[3],$data[3]);
        &rev32(@datax[0],@datax[0]);
        &rev32(@datax[1],@datax[1]);
        &rev32(@datax[2],@datax[2]);
        &rev32(@datax[3],$datax[3]);
$code.=<<___;
        bl      _vpsm4_enc_8blks
___
        &transpose(@vtmp,@datax);
        &transpose(@data,@datax);
$code.=<<___;
        ld1     {$ivec1.4s},[$ivp]
        ld1     {@datax[0].4s,@datax[1].4s,@datax[2].4s,@datax[3].4s},[$inp],#64
        // note ivec1 and vtmpx[3] are reusing the same register
        // care needs to be taken to avoid conflict
        eor     @vtmp[0].16b,@vtmp[0].16b,$ivec1.16b
        ld1     {@vtmpx[0].4s,@vtmpx[1].4s,@vtmpx[2].4s,@vtmpx[3].4s},[$inp],#64
        eor     @vtmp[1].16b,@vtmp[1].16b,@datax[0].16b
        eor     @vtmp[2].16b,@vtmp[2].16b,@datax[1].16b
        eor     @vtmp[3].16b,$vtmp[3].16b,@datax[2].16b
        // save back IV
        st1     {$vtmpx[3].4s}, [$ivp]
        eor     @data[0].16b,@data[0].16b,$datax[3].16b
        eor     @data[1].16b,@data[1].16b,@vtmpx[0].16b
        eor     @data[2].16b,@data[2].16b,@vtmpx[1].16b
        eor     @data[3].16b,$data[3].16b,@vtmpx[2].16b
        st1     {@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s,@vtmp[3].4s},[$outp],#64
        st1     {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$outp],#64
        subs    $blocks,$blocks,#8
        b.gt    .Lcbc_8_blocks_dec
        b.eq    100f
1:
        ld1     {$ivec1.4s},[$ivp]
.Lcbc_4_blocks_dec:
        cmp     $blocks,#4
        b.lt    1f
        ld4     {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$inp]
___
        &rev32(@data[0],@data[0]);
        &rev32(@data[1],@data[1]);
        &rev32(@data[2],@data[2]);
        &rev32(@data[3],$data[3]);
$code.=<<___;
        bl      _vpsm4_enc_4blks
        ld1     {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$inp],#64
___
        &transpose(@vtmp,@datax);
$code.=<<___;
        eor     @vtmp[0].16b,@vtmp[0].16b,$ivec1.16b
        eor     @vtmp[1].16b,@vtmp[1].16b,@data[0].16b
        orr     $ivec1.16b,@data[3].16b,@data[3].16b
        eor     @vtmp[2].16b,@vtmp[2].16b,@data[1].16b
        eor     @vtmp[3].16b,$vtmp[3].16b,@data[2].16b
        st1     {@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s,@vtmp[3].4s},[$outp],#64
        subs    $blocks,$blocks,#4
        b.gt    .Lcbc_4_blocks_dec
        // save back IV
        st1     {@data[3].4s}, [$ivp]
        b       100f
1:      // last block
        subs    $blocks,$blocks,#1
        b.lt    100f
        b.gt    1f
        ld1     {@data[0].4s},[$inp],#16
        // save back IV
        st1     {$data[0].4s}, [$ivp]
___
        &rev32(@datax[0],@data[0]);
        &encrypt_1blk(@datax[0]);
$code.=<<___;
        eor     @datax[0].16b,@datax[0].16b,$ivec1.16b
        st1     {@datax[0].4s},[$outp],#16
        b       100f
1:      // last two blocks
        ld4     {@data[0].s,@data[1].s,@data[2].s,@data[3].s}[0],[$inp]
        add     $ptr,$inp,#16
        ld4     {@data[0].s,@data[1].s,@data[2].s,@data[3].s}[1],[$ptr],#16
        subs    $blocks,$blocks,1
        b.gt    1f
___
        &rev32(@data[0],@data[0]);
        &rev32(@data[1],@data[1]);
        &rev32(@data[2],@data[2]);
        &rev32(@data[3],@data[3]);
$code.=<<___;
        bl      _vpsm4_enc_4blks
        ld1     {@data[0].4s,@data[1].4s},[$inp],#32
___
        &transpose(@vtmp,@datax);
$code.=<<___;
        eor     @vtmp[0].16b,@vtmp[0].16b,$ivec1.16b
        eor     @vtmp[1].16b,@vtmp[1].16b,@data[0].16b
        st1     {@vtmp[0].4s,@vtmp[1].4s},[$outp],#32
        // save back IV
        st1     {@data[1].4s}, [$ivp]
        b       100f
1:      // last 3 blocks
        ld4     {@data[0].s,@data[1].s,@data[2].s,@data[3].s}[2],[$ptr]
___
        &rev32(@data[0],@data[0]);
        &rev32(@data[1],@data[1]);
        &rev32(@data[2],@data[2]);
        &rev32(@data[3],@data[3]);
$code.=<<___;
        bl      _vpsm4_enc_4blks
        ld1     {@data[0].4s,@data[1].4s,@data[2].4s},[$inp],#48
___
        &transpose(@vtmp,@datax);
$code.=<<___;
        eor     @vtmp[0].16b,@vtmp[0].16b,$ivec1.16b
        eor     @vtmp[1].16b,@vtmp[1].16b,@data[0].16b
        eor     @vtmp[2].16b,@vtmp[2].16b,@data[1].16b
        st1     {@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s},[$outp],#48
        // save back IV
        st1     {@data[2].4s}, [$ivp]
100:
        ldp     d10,d11,[sp,#16]
        ldp     d12,d13,[sp,#32]
        ldp     d14,d15,[sp,#48]
        ldp     x29,x30,[sp,#64]
        ldp     d8,d9,[sp],#80
        AARCH64_VALIDATE_LINK_REGISTER
        ret
.size   ${prefix}_cbc_encrypt,.-${prefix}_cbc_encrypt
___
}}}

{{{
my ($ivp)=("x4");
my ($ctr)=("w5");
my $ivec=("v3");

$code.=<<___;
.globl  ${prefix}_ctr32_encrypt_blocks
.type   ${prefix}_ctr32_encrypt_blocks,%function
.align  5
${prefix}_ctr32_encrypt_blocks:
        AARCH64_VALID_CALL_TARGET
        ld1     {$ivec.4s},[$ivp]
___
        &rev32($ivec,$ivec);
        &load_sbox();
$code.=<<___;
        cmp     $blocks,#1
        b.ne    1f
        // fast processing for one single block without
        // context saving overhead
___
        &encrypt_1blk($ivec);
$code.=<<___;
        ld1     {@data[0].4s},[$inp]
        eor     @data[0].16b,@data[0].16b,$ivec.16b
        st1     {@data[0].4s},[$outp]
        ret
1:
        AARCH64_SIGN_LINK_REGISTER
        stp     d8,d9,[sp,#-80]!
        stp     d10,d11,[sp,#16]
        stp     d12,d13,[sp,#32]
        stp     d14,d15,[sp,#48]
        stp     x29,x30,[sp,#64]
        mov     $word0,$ivec.s[0]
        mov     $word1,$ivec.s[1]
        mov     $word2,$ivec.s[2]
        mov     $ctr,$ivec.s[3]
.Lctr32_4_blocks_process:
        cmp     $blocks,#4
        b.lt    1f
        dup     @data[0].4s,$word0
        dup     @data[1].4s,$word1
        dup     @data[2].4s,$word2
        mov     @data[3].s[0],$ctr
        add     $ctr,$ctr,#1
        mov     $data[3].s[1],$ctr
        add     $ctr,$ctr,#1
        mov     @data[3].s[2],$ctr
        add     $ctr,$ctr,#1
        mov     @data[3].s[3],$ctr
        add     $ctr,$ctr,#1
        cmp     $blocks,#8
        b.ge    .Lctr32_8_blocks_process
        bl      _vpsm4_enc_4blks
        ld4     {@vtmpx[0].4s,@vtmpx[1].4s,@vtmpx[2].4s,@vtmpx[3].4s},[$inp],#64
        eor     @vtmp[0].16b,@vtmp[0].16b,@vtmpx[0].16b
        eor     @vtmp[1].16b,@vtmp[1].16b,@vtmpx[1].16b
        eor     @vtmp[2].16b,@vtmp[2].16b,@vtmpx[2].16b
        eor     @vtmp[3].16b,@vtmp[3].16b,@vtmpx[3].16b
        st4     {@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s,@vtmp[3].4s},[$outp],#64
        subs    $blocks,$blocks,#4
        b.ne    .Lctr32_4_blocks_process
        b       100f
.Lctr32_8_blocks_process:
        dup     @datax[0].4s,$word0
        dup     @datax[1].4s,$word1
        dup     @datax[2].4s,$word2
        mov     @datax[3].s[0],$ctr
        add     $ctr,$ctr,#1
        mov     $datax[3].s[1],$ctr
        add     $ctr,$ctr,#1
        mov     @datax[3].s[2],$ctr
        add     $ctr,$ctr,#1
        mov     @datax[3].s[3],$ctr
        add     $ctr,$ctr,#1
        bl      _vpsm4_enc_8blks
        ld4     {@vtmpx[0].4s,@vtmpx[1].4s,@vtmpx[2].4s,@vtmpx[3].4s},[$inp],#64
        ld4     {@datax[0].4s,@datax[1].4s,@datax[2].4s,@datax[3].4s},[$inp],#64
        eor     @vtmp[0].16b,@vtmp[0].16b,@vtmpx[0].16b
        eor     @vtmp[1].16b,@vtmp[1].16b,@vtmpx[1].16b
        eor     @vtmp[2].16b,@vtmp[2].16b,@vtmpx[2].16b
        eor     @vtmp[3].16b,@vtmp[3].16b,@vtmpx[3].16b
        eor     @data[0].16b,@data[0].16b,@datax[0].16b
        eor     @data[1].16b,@data[1].16b,@datax[1].16b
        eor     @data[2].16b,@data[2].16b,@datax[2].16b
        eor     @data[3].16b,@data[3].16b,@datax[3].16b
        st4     {@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s,@vtmp[3].4s},[$outp],#64
        st4     {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$outp],#64
        subs    $blocks,$blocks,#8
        b.ne    .Lctr32_4_blocks_process
        b       100f
1:      // last block processing
        subs    $blocks,$blocks,#1
        b.lt    100f
        b.gt    1f
        mov     $ivec.s[0],$word0
        mov     $ivec.s[1],$word1
        mov     $ivec.s[2],$word2
        mov     $ivec.s[3],$ctr
___
        &encrypt_1blk($ivec);
$code.=<<___;
        ld1     {@data[0].4s},[$inp]
        eor     @data[0].16b,@data[0].16b,$ivec.16b
        st1     {@data[0].4s},[$outp]
        b       100f
1:      // last 2 blocks processing
        dup     @data[0].4s,$word0
        dup     @data[1].4s,$word1
        dup     @data[2].4s,$word2
        mov     @data[3].s[0],$ctr
        add     $ctr,$ctr,#1
        mov     @data[3].s[1],$ctr
        subs    $blocks,$blocks,#1
        b.ne    1f
        bl      _vpsm4_enc_4blks
        ld4     {@vtmpx[0].s,@vtmpx[1].s,@vtmpx[2].s,@vtmpx[3].s}[0],[$inp],#16
        ld4     {@vtmpx[0].s,@vtmpx[1].s,@vtmpx[2].s,@vtmpx[3].s}[1],[$inp],#16
        eor     @vtmp[0].16b,@vtmp[0].16b,@vtmpx[0].16b
        eor     @vtmp[1].16b,@vtmp[1].16b,@vtmpx[1].16b
        eor     @vtmp[2].16b,@vtmp[2].16b,@vtmpx[2].16b
        eor     @vtmp[3].16b,@vtmp[3].16b,@vtmpx[3].16b
        st4     {@vtmp[0].s,@vtmp[1].s,@vtmp[2].s,@vtmp[3].s}[0],[$outp],#16
        st4     {@vtmp[0].s,@vtmp[1].s,@vtmp[2].s,@vtmp[3].s}[1],[$outp],#16
        b       100f
1:      // last 3 blocks processing
        add     $ctr,$ctr,#1
        mov     @data[3].s[2],$ctr
        bl      _vpsm4_enc_4blks
        ld4     {@vtmpx[0].s,@vtmpx[1].s,@vtmpx[2].s,@vtmpx[3].s}[0],[$inp],#16
        ld4     {@vtmpx[0].s,@vtmpx[1].s,@vtmpx[2].s,@vtmpx[3].s}[1],[$inp],#16
        ld4     {@vtmpx[0].s,@vtmpx[1].s,@vtmpx[2].s,@vtmpx[3].s}[2],[$inp],#16
        eor     @vtmp[0].16b,@vtmp[0].16b,@vtmpx[0].16b
        eor     @vtmp[1].16b,@vtmp[1].16b,@vtmpx[1].16b
        eor     @vtmp[2].16b,@vtmp[2].16b,@vtmpx[2].16b
        eor     @vtmp[3].16b,@vtmp[3].16b,@vtmpx[3].16b
        st4     {@vtmp[0].s,@vtmp[1].s,@vtmp[2].s,@vtmp[3].s}[0],[$outp],#16
        st4     {@vtmp[0].s,@vtmp[1].s,@vtmp[2].s,@vtmp[3].s}[1],[$outp],#16
        st4     {@vtmp[0].s,@vtmp[1].s,@vtmp[2].s,@vtmp[3].s}[2],[$outp],#16
100:
        ldp     d10,d11,[sp,#16]
        ldp     d12,d13,[sp,#32]
        ldp     d14,d15,[sp,#48]
        ldp     x29,x30,[sp,#64]
        ldp     d8,d9,[sp],#80
        AARCH64_VALIDATE_LINK_REGISTER
        ret
.size   ${prefix}_ctr32_encrypt_blocks,.-${prefix}_ctr32_encrypt_blocks
___
}}}

{{{
my ($blocks,$len)=("x2","x2");
my $ivp=("x5");
my @twx=map("x$_",(12..27));
my ($rks1,$rks2)=("x26","x27");
my $lastBlk=("x26");
my $enc=("w28");
my $remain=("x29");

my @tweak=@datax;

sub gen_xts_cipher() {
        my $std = shift;
$code.=<<___;
.globl  ${prefix}_xts_encrypt${std}
.type   ${prefix}_xts_encrypt${std},%function
.align  5
${prefix}_xts_encrypt${std}:
        AARCH64_SIGN_LINK_REGISTER
        stp     x15, x16, [sp, #-0x10]!
        stp     x17, x18, [sp, #-0x10]!
        stp     x19, x20, [sp, #-0x10]!
        stp     x21, x22, [sp, #-0x10]!
        stp     x23, x24, [sp, #-0x10]!
        stp     x25, x26, [sp, #-0x10]!
        stp     x27, x28, [sp, #-0x10]!
        stp     x29, x30, [sp, #-0x10]!
        stp     d8, d9, [sp, #-0x10]!
        stp     d10, d11, [sp, #-0x10]!
        stp     d12, d13, [sp, #-0x10]!
        stp     d14, d15, [sp, #-0x10]!
        mov     $rks1,x3
        mov     $rks2,x4
        mov     $enc,w6
        ld1     {@tweak[0].4s}, [$ivp]
        mov     $rks,$rks2
___
        &load_sbox();
        &rev32(@tweak[0],@tweak[0]);
        &encrypt_1blk(@tweak[0]);
$code.=<<___;
        mov     $rks,$rks1
        and     $remain,$len,#0x0F
        // convert length into blocks
        lsr     $blocks,$len,4
        cmp     $blocks,#1
        b.lt .return${std}

        cmp $remain,0
        // If the encryption/decryption Length is N times of 16,
        // the all blocks are encrypted/decrypted in .xts_encrypt_blocks${std}
        b.eq .xts_encrypt_blocks${std}

        // If the encryption/decryption length is not N times of 16,
        // the last two blocks are encrypted/decrypted in .last_2blks_tweak${std} or .only_2blks_tweak${std}
        // the other blocks are encrypted/decrypted in .xts_encrypt_blocks${std}
        subs $blocks,$blocks,#1
        b.eq .only_2blks_tweak${std}
.xts_encrypt_blocks${std}:
___
        &rbit(@tweak[0],@tweak[0],$std);
        &rev32_armeb(@tweak[0],@tweak[0]);
        &mov_vec_to_reg(@tweak[0],@twx[0],@twx[1]);
        &compute_tweak(@twx[0],@twx[1],@twx[2],@twx[3]);
        &compute_tweak(@twx[2],@twx[3],@twx[4],@twx[5]);
        &compute_tweak(@twx[4],@twx[5],@twx[6],@twx[7]);
        &compute_tweak(@twx[6],@twx[7],@twx[8],@twx[9]);
        &compute_tweak(@twx[8],@twx[9],@twx[10],@twx[11]);
        &compute_tweak(@twx[10],@twx[11],@twx[12],@twx[13]);
        &compute_tweak(@twx[12],@twx[13],@twx[14],@twx[15]);
$code.=<<___;
.Lxts_8_blocks_process${std}:
        cmp     $blocks,#8
        b.lt    .Lxts_4_blocks_process${std}
___
        &mov_reg_to_vec(@twx[0],@twx[1],@vtmp[0]);
        &mov_reg_to_vec(@twx[2],@twx[3],@vtmp[1]);
        &mov_reg_to_vec(@twx[4],@twx[5],@vtmp[2]);
        &mov_reg_to_vec(@twx[6],@twx[7],@vtmp[3]);
        &mov_reg_to_vec(@twx[8],@twx[9],@vtmpx[0]);
        &mov_reg_to_vec(@twx[10],@twx[11],@vtmpx[1]);
        &mov_reg_to_vec(@twx[12],@twx[13],@vtmpx[2]);
        &mov_reg_to_vec(@twx[14],@twx[15],@vtmpx[3]);
$code.=<<___;
        ld1 {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$inp],#64
___
        &rbit(@vtmp[0],@vtmp[0],$std);
        &rbit(@vtmp[1],@vtmp[1],$std);
        &rbit(@vtmp[2],@vtmp[2],$std);
        &rbit(@vtmp[3],@vtmp[3],$std);
$code.=<<___;
        eor @data[0].16b, @data[0].16b, @vtmp[0].16b
        eor @data[1].16b, @data[1].16b, @vtmp[1].16b
        eor @data[2].16b, @data[2].16b, @vtmp[2].16b
        eor @data[3].16b, @data[3].16b, @vtmp[3].16b
        ld1     {@datax[0].4s,$datax[1].4s,@datax[2].4s,@datax[3].4s},[$inp],#64
___
        &rbit(@vtmpx[0],@vtmpx[0],$std);
        &rbit(@vtmpx[1],@vtmpx[1],$std);
        &rbit(@vtmpx[2],@vtmpx[2],$std);
        &rbit(@vtmpx[3],@vtmpx[3],$std);
$code.=<<___;
        eor @datax[0].16b, @datax[0].16b, @vtmpx[0].16b
        eor @datax[1].16b, @datax[1].16b, @vtmpx[1].16b
        eor @datax[2].16b, @datax[2].16b, @vtmpx[2].16b
        eor @datax[3].16b, @datax[3].16b, @vtmpx[3].16b
___
        &rev32(@data[0],@data[0]);
        &rev32(@data[1],@data[1]);
        &rev32(@data[2],@data[2]);
        &rev32(@data[3],@data[3]);
        &rev32(@datax[0],@datax[0]);
        &rev32(@datax[1],@datax[1]);
        &rev32(@datax[2],@datax[2]);
        &rev32(@datax[3],@datax[3]);
        &transpose(@data,@vtmp);
        &transpose(@datax,@vtmp);
$code.=<<___;
        bl      _${prefix}_enc_8blks
___
        &transpose(@vtmp,@datax);
        &transpose(@data,@datax);

        &mov_reg_to_vec(@twx[0],@twx[1],@vtmpx[0]);
        &compute_tweak(@twx[14],@twx[15],@twx[0],@twx[1]);
        &mov_reg_to_vec(@twx[2],@twx[3],@vtmpx[1]);
        &compute_tweak(@twx[0],@twx[1],@twx[2],@twx[3]);
        &mov_reg_to_vec(@twx[4],@twx[5],@vtmpx[2]);
        &compute_tweak(@twx[2],@twx[3],@twx[4],@twx[5]);
        &mov_reg_to_vec(@twx[6],@twx[7],@vtmpx[3]);
        &compute_tweak(@twx[4],@twx[5],@twx[6],@twx[7]);
        &mov_reg_to_vec(@twx[8],@twx[9],@tweak[0]);
        &compute_tweak(@twx[6],@twx[7],@twx[8],@twx[9]);
        &mov_reg_to_vec(@twx[10],@twx[11],@tweak[1]);
        &compute_tweak(@twx[8],@twx[9],@twx[10],@twx[11]);
        &mov_reg_to_vec(@twx[12],@twx[13],@tweak[2]);
        &compute_tweak(@twx[10],@twx[11],@twx[12],@twx[13]);
        &mov_reg_to_vec(@twx[14],@twx[15],@tweak[3]);
        &compute_tweak(@twx[12],@twx[13],@twx[14],@twx[15]);
$code.=<<___;
        eor @vtmp[0].16b, @vtmp[0].16b, @vtmpx[0].16b
        eor @vtmp[1].16b, @vtmp[1].16b, @vtmpx[1].16b
        eor @vtmp[2].16b, @vtmp[2].16b, @vtmpx[2].16b
        eor @vtmp[3].16b, @vtmp[3].16b, @vtmpx[3].16b
        eor @data[0].16b, @data[0].16b, @tweak[0].16b
        eor @data[1].16b, @data[1].16b, @tweak[1].16b
        eor @data[2].16b, @data[2].16b, @tweak[2].16b
        eor @data[3].16b, @data[3].16b, @tweak[3].16b

        // save the last tweak
        st1     {@tweak[3].4s},[$ivp]
        st1     {@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s,@vtmp[3].4s},[$outp],#64
        st1     {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$outp],#64
        subs    $blocks,$blocks,#8
        b.gt    .Lxts_8_blocks_process${std}
        b       100f
.Lxts_4_blocks_process${std}:
___
        &mov_reg_to_vec(@twx[0],@twx[1],@tweak[0]);
        &mov_reg_to_vec(@twx[2],@twx[3],@tweak[1]);
        &mov_reg_to_vec(@twx[4],@twx[5],@tweak[2]);
        &mov_reg_to_vec(@twx[6],@twx[7],@tweak[3]);
$code.=<<___;
        cmp     $blocks,#4
        b.lt    1f
        ld1     {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$inp],#64
___
        &rbit(@tweak[0],@tweak[0],$std);
        &rbit(@tweak[1],@tweak[1],$std);
        &rbit(@tweak[2],@tweak[2],$std);
        &rbit(@tweak[3],@tweak[3],$std);
$code.=<<___;
        eor @data[0].16b, @data[0].16b, @tweak[0].16b
        eor @data[1].16b, @data[1].16b, @tweak[1].16b
        eor @data[2].16b, @data[2].16b, @tweak[2].16b
        eor @data[3].16b, @data[3].16b, @tweak[3].16b
___
        &rev32(@data[0],@data[0]);
        &rev32(@data[1],@data[1]);
        &rev32(@data[2],@data[2]);
        &rev32(@data[3],@data[3]);
        &transpose(@data,@vtmp);
$code.=<<___;
        bl      _${prefix}_enc_4blks
___
        &transpose(@vtmp,@data);
$code.=<<___;
        eor @vtmp[0].16b, @vtmp[0].16b, @tweak[0].16b
        eor @vtmp[1].16b, @vtmp[1].16b, @tweak[1].16b
        eor @vtmp[2].16b, @vtmp[2].16b, @tweak[2].16b
        eor @vtmp[3].16b, @vtmp[3].16b, @tweak[3].16b
        st1     {@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s,@vtmp[3].4s},[$outp],#64
        sub     $blocks,$blocks,#4
___
        &mov_reg_to_vec(@twx[8],@twx[9],@tweak[0]);
        &mov_reg_to_vec(@twx[10],@twx[11],@tweak[1]);
        &mov_reg_to_vec(@twx[12],@twx[13],@tweak[2]);
$code.=<<___;
        // save the last tweak
        st1     {@tweak[3].4s},[$ivp]
1:
        // process last block
        cmp     $blocks,#1
        b.lt    100f
        b.gt    1f
        ld1     {@data[0].4s},[$inp],#16
___
        &rbit(@tweak[0],@tweak[0],$std);
$code.=<<___;
        eor @data[0].16b, @data[0].16b, @tweak[0].16b
___
        &rev32(@data[0],@data[0]);
        &encrypt_1blk(@data[0]);
$code.=<<___;
        eor @data[0].16b, @data[0].16b, @tweak[0].16b
        st1     {@data[0].4s},[$outp],#16
        // save the last tweak
        st1     {@tweak[0].4s},[$ivp]
        b       100f
1:  // process last 2 blocks
        cmp     $blocks,#2
        b.gt    1f
        ld1     {@data[0].4s,@data[1].4s},[$inp],#32
___
        &rbit(@tweak[0],@tweak[0],$std);
        &rbit(@tweak[1],@tweak[1],$std);
$code.=<<___;
        eor @data[0].16b, @data[0].16b, @tweak[0].16b
        eor @data[1].16b, @data[1].16b, @tweak[1].16b
___
        &rev32(@data[0],@data[0]);
        &rev32(@data[1],@data[1]);
        &transpose(@data,@vtmp);
$code.=<<___;
        bl      _${prefix}_enc_4blks
___
        &transpose(@vtmp,@data);
$code.=<<___;
        eor @vtmp[0].16b, @vtmp[0].16b, @tweak[0].16b
        eor @vtmp[1].16b, @vtmp[1].16b, @tweak[1].16b
        st1     {@vtmp[0].4s,@vtmp[1].4s},[$outp],#32
        // save the last tweak
        st1     {@tweak[1].4s},[$ivp]
        b       100f
1:  // process last 3 blocks
        ld1     {@data[0].4s,@data[1].4s,@data[2].4s},[$inp],#48
___
        &rbit(@tweak[0],@tweak[0],$std);
        &rbit(@tweak[1],@tweak[1],$std);
        &rbit(@tweak[2],@tweak[2],$std);
$code.=<<___;
        eor @data[0].16b, @data[0].16b, @tweak[0].16b
        eor @data[1].16b, @data[1].16b, @tweak[1].16b
        eor @data[2].16b, @data[2].16b, @tweak[2].16b
___
        &rev32(@data[0],@data[0]);
        &rev32(@data[1],@data[1]);
        &rev32(@data[2],@data[2]);
        &transpose(@data,@vtmp);
$code.=<<___;
        bl      _${prefix}_enc_4blks
___
        &transpose(@vtmp,@data);
$code.=<<___;
        eor @vtmp[0].16b, @vtmp[0].16b, @tweak[0].16b
        eor @vtmp[1].16b, @vtmp[1].16b, @tweak[1].16b
        eor @vtmp[2].16b, @vtmp[2].16b, @tweak[2].16b
        st1     {@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s},[$outp],#48
        // save the last tweak
        st1     {@tweak[2].4s},[$ivp]
100:
        cmp $remain,0
        b.eq .return${std}

// This branch calculates the last two tweaks, 
// while the encryption/decryption length is larger than 32
.last_2blks_tweak${std}:
        ld1     {@tweak[0].4s},[$ivp]
___
        &rev32_armeb(@tweak[0],@tweak[0]);
        &compute_tweak_vec(@tweak[0],@tweak[1],$std);
        &compute_tweak_vec(@tweak[1],@tweak[2],$std);
$code.=<<___;
        b .check_dec${std}


// This branch calculates the last two tweaks, 
// while the encryption/decryption length is equal to 32, who only need two tweaks
.only_2blks_tweak${std}:
        mov @tweak[1].16b,@tweak[0].16b
___
        &rev32_armeb(@tweak[1],@tweak[1]);
        &compute_tweak_vec(@tweak[1],@tweak[2],$std);
$code.=<<___;
        b .check_dec${std}


// Determine whether encryption or decryption is required.
// The last two tweaks need to be swapped for decryption.
.check_dec${std}:
        // encryption:1 decryption:0
        cmp $enc,1
        b.eq .process_last_2blks${std}
        mov @vtmp[0].16B,@tweak[1].16b
        mov @tweak[1].16B,@tweak[2].16b
        mov @tweak[2].16B,@vtmp[0].16b

.process_last_2blks${std}:
___
        &rev32_armeb(@tweak[1],@tweak[1]);
        &rev32_armeb(@tweak[2],@tweak[2]);
$code.=<<___;
        ld1     {@data[0].4s},[$inp],#16
        eor @data[0].16b, @data[0].16b, @tweak[1].16b
___
        &rev32(@data[0],@data[0]);
        &encrypt_1blk(@data[0]);
$code.=<<___;
        eor @data[0].16b, @data[0].16b, @tweak[1].16b
        st1     {@data[0].4s},[$outp],#16

        sub $lastBlk,$outp,16
        .loop${std}:
                subs $remain,$remain,1
                ldrb    $wtmp0,[$lastBlk,$remain]
                ldrb    $wtmp1,[$inp,$remain]
                strb    $wtmp1,[$lastBlk,$remain]
                strb    $wtmp0,[$outp,$remain]
        b.gt .loop${std}
        ld1             {@data[0].4s}, [$lastBlk]       
        eor @data[0].16b, @data[0].16b, @tweak[2].16b
___
        &rev32(@data[0],@data[0]);
        &encrypt_1blk(@data[0]);
$code.=<<___;
        eor @data[0].16b, @data[0].16b, @tweak[2].16b
        st1             {@data[0].4s}, [$lastBlk]
.return${std}:
        ldp             d14, d15, [sp], #0x10
        ldp             d12, d13, [sp], #0x10
        ldp             d10, d11, [sp], #0x10
        ldp             d8, d9, [sp], #0x10
        ldp             x29, x30, [sp], #0x10
        ldp             x27, x28, [sp], #0x10
        ldp             x25, x26, [sp], #0x10
        ldp             x23, x24, [sp], #0x10
        ldp             x21, x22, [sp], #0x10
        ldp             x19, x20, [sp], #0x10
        ldp             x17, x18, [sp], #0x10
        ldp             x15, x16, [sp], #0x10
        AARCH64_VALIDATE_LINK_REGISTER
        ret
.size   ${prefix}_xts_encrypt${std},.-${prefix}_xts_encrypt${std}
___
} # end of gen_xts_cipher
&gen_xts_cipher("_gb");
&gen_xts_cipher("");
}}}
########################################
open SELF,$0;
while(<SELF>) {
        next if (/^#!/);
        last if (!s/^#/\/\// and !/^$/);
        print;
}
close SELF;

foreach(split("\n",$code)) {
        s/\`([^\`]*)\`/eval($1)/ge;
        print $_,"\n";
}

close STDOUT or die "error closing STDOUT: $!";