From: Jerry Shih Date: Mon, 25 Sep 2023 01:14:48 +0000 (+0800) Subject: riscv: Provide vector crypto implementation of AES-GCM mode. X-Git-Tag: openssl-3.3.0-alpha1~736 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=d056e90ee58a039263b843e8fa330fa71b4d4835;p=thirdparty%2Fopenssl.git riscv: Provide vector crypto implementation of AES-GCM mode. To accelerate the performance of the AES-GCM mode, in this patch, we have the specialized multi-block implementations for AES-128-GCM, AES-192-GCM and AES-256-GCM. Signed-off-by: Phoebe Chen Signed-off-by: Jerry Shih Reviewed-by: Tomas Mraz Reviewed-by: Paul Dale Reviewed-by: Hugo Landau (Merged from https://github.com/openssl/openssl/pull/21923) --- diff --git a/crypto/modes/asm/aes-gcm-riscv64-zvbb-zvkg-zvkned.pl b/crypto/modes/asm/aes-gcm-riscv64-zvbb-zvkg-zvkned.pl new file mode 100644 index 00000000000..4bfb15228e6 --- /dev/null +++ b/crypto/modes/asm/aes-gcm-riscv64-zvbb-zvkg-zvkned.pl @@ -0,0 +1,975 @@ +#! /usr/bin/env perl +# This file is dual-licensed, meaning that you can use it under your +# choice of either of the following two licenses: +# +# Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You can obtain +# a copy in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html +# +# or +# +# Copyright (c) 2023, Jerry Shih +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# - RV64I +# - RISC-V Vector ('V') with VLEN >= 128 +# - RISC-V Vector Basic Bit-manipulation extension ('Zvbb') +# - RISC-V Vector GCM/GMAC extension ('Zvkg') +# - RISC-V Vector AES Block Cipher extension ('Zvkned') +# - RISC-V Zicclsm(Main memory supports misaligned loads/stores) + +# Reference: https://github.com/riscv/riscv-crypto/issues/192#issuecomment-1270447575 +# +# Assume we have 12 GCM blocks and we try to parallelize GCM computation for 4 blocks. +# Tag = M0*H^12 + M1*H^11 + M2*H^10 + M3*H^9 + +# M4*H^8 + M5*H^7 + M6*H^6 + M7*H^5 + +# M8*H^4 + M9*H^3 + M10*H^2 + M11*H^1 +# We could rewrite the formula into: +# T0 = 0 +# T1 = (T0+M1)*H^4 T2 = (T0+M2)*H^4 T3 = (T0+M3)*H^4 T4 = (T0+M4)*H^4 +# T5 = (T1+M5)*H^4 T6 = (T2+M6)*H^4 T7 = (T3+M7)*H^4 T8 = (T4+M8)*H^4 +# T9 = (T5+M9)*H^4 T10 = (T6+M10)*H^3 T11 = (T7+M11)*H^2 T12 = (T8+M12)*H^1 +# +# We will multiply with [H^4, H^4, H^4, H^4] in each steps except the last iteration. +# The last iteration will multiply with [H^4, H^3, H^2, H^1]. + +use strict; +use warnings; + +use FindBin qw($Bin); +use lib "$Bin"; +use lib "$Bin/../../perlasm"; +use riscv; + +# $output is the last argument if it looks like a file (it has an extension) +# $flavour is the first argument if it doesn't look like a file +my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$output and open STDOUT,">$output"; + +my $code=<<___; +.text +___ + +{ +my ($INP, $OUTP, $LEN, $KEYP, $IVP, $XIP) = ("a0", "a1", "a2", "a3", "a4", "a5"); +my ($T0, $T1, $T2, $T3) = ("t0", "t1", "t2", "t3"); +my ($PADDING_LEN32) = ("t4"); +my ($LEN32) = ("t5"); +my ($CTR) = ("t6"); +my ($FULL_BLOCK_LEN32) = ("a6"); +my ($ORIGINAL_LEN32) = ("a7"); +my ($PROCESSED_LEN) = ("a0"); +my ($CTR_MASK) = ("v0"); +my ($INPUT_PADDING_MASK) = ("v0"); +my ($V0, $V1, $V2, $V3, $V4, $V5, $V6, $V7, + $V8, $V9, $V10, $V11, $V12, $V13, $V14, $V15, + $V16, $V17, $V18, $V19, $V20, $V21, $V22, $V23, + $V24, $V25, $V26, $V27, $V28, $V29, $V30, $V31, +) = map("v$_",(0..31)); + +# Do aes-128 enc. +sub aes_128_cipher_body { + my $code=<<___; + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + @{[vaesz_vs $V28, $V1]} + @{[vaesem_vs $V28, $V2]} + @{[vaesem_vs $V28, $V3]} + @{[vaesem_vs $V28, $V4]} + @{[vaesem_vs $V28, $V5]} + @{[vaesem_vs $V28, $V6]} + @{[vaesem_vs $V28, $V7]} + @{[vaesem_vs $V28, $V8]} + @{[vaesem_vs $V28, $V9]} + @{[vaesem_vs $V28, $V10]} + @{[vaesef_vs $V28, $V11]} +___ + + return $code; +} + +# Do aes-192 enc. +sub aes_192_cipher_body { + my $TMP_REG = shift; + + my $code=<<___; + # Load key 4 + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + addi $TMP_REG, $KEYP, 48 + @{[vle32_v $V11, $TMP_REG]} + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + @{[vaesz_vs $V28, $V1]} + @{[vaesem_vs $V28, $V2]} + @{[vaesem_vs $V28, $V3]} + @{[vaesem_vs $V28, $V11]} + # Load key 8 + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + addi $TMP_REG, $KEYP, 112 + @{[vle32_v $V11, $TMP_REG]} + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + @{[vaesem_vs $V28, $V4]} + @{[vaesem_vs $V28, $V5]} + @{[vaesem_vs $V28, $V6]} + @{[vaesem_vs $V28, $V11]} + # Load key 13 + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + addi $TMP_REG, $KEYP, 192 + @{[vle32_v $V11, $TMP_REG]} + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + @{[vaesem_vs $V28, $V7]} + @{[vaesem_vs $V28, $V8]} + @{[vaesem_vs $V28, $V9]} + @{[vaesem_vs $V28, $V10]} + @{[vaesef_vs $V28, $V11]} +___ + + return $code; +} + +# Do aes-256 enc. +sub aes_256_cipher_body { + my $TMP_REG = shift; + + my $code=<<___; + # Load key 3 + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + addi $TMP_REG, $KEYP, 32 + @{[vle32_v $V11, $TMP_REG]} + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + @{[vaesz_vs $V28, $V1]} + @{[vaesem_vs $V28, $V2]} + @{[vaesem_vs $V28, $V11]} + # Load key 6 + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + addi $TMP_REG, $KEYP, 80 + @{[vle32_v $V11, $TMP_REG]} + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + @{[vaesem_vs $V28, $V3]} + @{[vaesem_vs $V28, $V4]} + @{[vaesem_vs $V28, $V11]} + # Load key 9 + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + addi $TMP_REG, $KEYP, 128 + @{[vle32_v $V11, $TMP_REG]} + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + @{[vaesem_vs $V28, $V5]} + @{[vaesem_vs $V28, $V6]} + @{[vaesem_vs $V28, $V11]} + # Load key 12 + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + addi $TMP_REG, $KEYP, 176 + @{[vle32_v $V11, $TMP_REG]} + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + @{[vaesem_vs $V28, $V7]} + @{[vaesem_vs $V28, $V8]} + @{[vaesem_vs $V28, $V11]} + # Load key 15 + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + addi $TMP_REG, $KEYP, 224 + @{[vle32_v $V11, $TMP_REG]} + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + @{[vaesem_vs $V28, $V9]} + @{[vaesem_vs $V28, $V10]} + @{[vaesef_vs $V28, $V11]} +___ + + return $code; +} + +sub handle_padding_in_first_round { + my $TMP_REG = shift; + + my $code=<<___; + bnez $PADDING_LEN32, 1f + + ## without padding + # Store ciphertext/plaintext + @{[vse32_v $V28, $OUTP]} + j 2f + + ## with padding +1: + # Store ciphertext/plaintext using mask + @{[vse32_v $V28, $OUTP, $INPUT_PADDING_MASK]} + + # Fill zero for the padding blocks + @{[vsetvli "zero", $PADDING_LEN32, "e32", "m4", "tu", "ma"]} + @{[vmv_v_i $V28, 0]} + + # We have used mask register for `INPUT_PADDING_MASK` before. We need to + # setup the ctr mask back. + # ctr mask : [000100010001....] + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e8", "m1", "ta", "ma"]} + li $TMP_REG, 0b10001000 + @{[vmv_v_x $CTR_MASK, $TMP_REG]} +2: + +___ + + return $code; +} + + +# Do aes-128 enc for first round. +sub aes_128_first_round { + my $PTR_OFFSET_REG = shift; + my $TMP_REG = shift; + + my $code=<<___; + # Load all 11 aes round keys to v1-v11 registers. + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vle32_v $V1, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V2, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V3, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V4, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V5, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V6, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V7, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V8, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V9, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V10, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V11, $KEYP]} + + # We already have the ciphertext/plaintext and ctr data for the first round. + @{[aes_128_cipher_body]} + + # Compute AES ctr result. + @{[vxor_vv $V28, $V28, $V24]} + + @{[handle_padding_in_first_round $TMP_REG]} + + add $INP, $INP, $PTR_OFFSET_REG + add $OUTP, $OUTP, $PTR_OFFSET_REG +___ + + return $code; +} + +# Do aes-192 enc for first round. +sub aes_192_first_round { + my $PTR_OFFSET_REG = shift; + my $TMP_REG = shift; + + my $code=<<___; + # We run out of 32 vector registers, so we just preserve some round keys + # and load the remaining round keys inside the aes body. + # We keep the round keys for: + # 1, 2, 3, 5, 6, 7, 9, 10, 11 and 12th keys. + # The following keys will be loaded in the aes body: + # 4, 8 and 13th keys. + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + # key 1 + @{[vle32_v $V1, $KEYP]} + # key 2 + addi $TMP_REG, $KEYP, 16 + @{[vle32_v $V2, $TMP_REG]} + # key 3 + addi $TMP_REG, $KEYP, 32 + @{[vle32_v $V3, $TMP_REG]} + # key 5 + addi $TMP_REG, $KEYP, 64 + @{[vle32_v $V4, $TMP_REG]} + # key 6 + addi $TMP_REG, $KEYP, 80 + @{[vle32_v $V5, $TMP_REG]} + # key 7 + addi $TMP_REG, $KEYP, 96 + @{[vle32_v $V6, $TMP_REG]} + # key 9 + addi $TMP_REG, $KEYP, 128 + @{[vle32_v $V7, $TMP_REG]} + # key 10 + addi $TMP_REG, $KEYP, 144 + @{[vle32_v $V8, $TMP_REG]} + # key 11 + addi $TMP_REG, $KEYP, 160 + @{[vle32_v $V9, $TMP_REG]} + # key 12 + addi $TMP_REG, $KEYP, 176 + @{[vle32_v $V10, $TMP_REG]} + + # We already have the ciphertext/plaintext and ctr data for the first round. + @{[aes_192_cipher_body $TMP_REG]} + + # Compute AES ctr result. + @{[vxor_vv $V28, $V28, $V24]} + + @{[handle_padding_in_first_round $TMP_REG]} + + add $INP, $INP, $PTR_OFFSET_REG + add $OUTP, $OUTP, $PTR_OFFSET_REG +___ + + return $code; +} + +# Do aes-256 enc for first round. +sub aes_256_first_round { + my $PTR_OFFSET_REG = shift; + my $TMP_REG = shift; + + my $code=<<___; + # We run out of 32 vector registers, so we just preserve some round keys + # and load the remaining round keys inside the aes body. + # We keep the round keys for: + # 1, 2, 4, 5, 7, 8, 10, 11, 13 and 14th keys. + # The following keys will be loaded in the aes body: + # 3, 6, 9, 12 and 15th keys. + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + # key 1 + @{[vle32_v $V1, $KEYP]} + # key 2 + addi $TMP_REG, $KEYP, 16 + @{[vle32_v $V2, $TMP_REG]} + # key 4 + addi $TMP_REG, $KEYP, 48 + @{[vle32_v $V3, $TMP_REG]} + # key 5 + addi $TMP_REG, $KEYP, 64 + @{[vle32_v $V4, $TMP_REG]} + # key 7 + addi $TMP_REG, $KEYP, 96 + @{[vle32_v $V5, $TMP_REG]} + # key 8 + addi $TMP_REG, $KEYP, 112 + @{[vle32_v $V6, $TMP_REG]} + # key 10 + addi $TMP_REG, $KEYP, 144 + @{[vle32_v $V7, $TMP_REG]} + # key 11 + addi $TMP_REG, $KEYP, 160 + @{[vle32_v $V8, $TMP_REG]} + # key 13 + addi $TMP_REG, $KEYP, 192 + @{[vle32_v $V9, $TMP_REG]} + # key 14 + addi $TMP_REG, $KEYP, 208 + @{[vle32_v $V10, $TMP_REG]} + + # We already have the ciphertext/plaintext and ctr data for the first round. + @{[aes_256_cipher_body $TMP_REG]} + + # Compute AES ctr result. + @{[vxor_vv $V28, $V28, $V24]} + + @{[handle_padding_in_first_round $TMP_REG]} + + add $INP, $INP, $PTR_OFFSET_REG + add $OUTP, $OUTP, $PTR_OFFSET_REG +___ + + return $code; +} + +sub aes_gcm_init { + my $code=<<___; + # Compute the AES-GCM full-block e32 length for `LMUL=4`. We will handle + # the multiple AES-GCM blocks at the same time within `LMUL=4` register. + # The AES-GCM's SEW is e32 and EGW is 128 bits. + # FULL_BLOCK_LEN32 = (VLEN*LMUL)/(EGW) * (EGW/SEW) = (VLEN*4)/(32*4) * 4 + # = (VLEN*4)/32 + # We could get the block_num using the VL value of `vsetvli with e32, m4`. + @{[vsetvli $FULL_BLOCK_LEN32, "zero", "e32", "m4", "ta", "ma"]} + # If `LEN32 % FULL_BLOCK_LEN32` is not equal to zero, we could fill the + # zero padding data to make sure we could always handle FULL_BLOCK_LEN32 + # blocks for all iterations. + + ## Prepare the H^n multiplier in v16 for GCM multiplier. The `n` is the gcm + ## block number in a LMUL=4 register group. + ## n = ((VLEN*LMUL)/(32*4)) = ((VLEN*4)/(32*4)) + ## = (VLEN/32) + ## We could use vsetvli with `e32, m1` to compute the `n` number. + @{[vsetvli $T0, "zero", "e32", "m1", "ta", "ma"]} + + # The H is at `gcm128_context.Htable[0]`(addr(Xi)+16*2). + addi $T1, $XIP, 32 + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vle32_v $V31, $T1]} + + # Compute the H^n + li $T1, 1 +1: + @{[vgmul_vv $V31, $V31]} + slli $T1, $T1, 1 + bltu $T1, $T0, 1b + + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + @{[vmv_v_i $V16, 0]} + @{[vaesz_vs $V16, $V31]} + + #### Load plaintext into v24 and handle padding. We also load the init tag + #### data into v20 and prepare the AES ctr input data into v12 and v28. + @{[vmv_v_i $V20, 0]} + + ## Prepare the AES ctr input data into v12. + # Setup ctr input mask. + # ctr mask : [000100010001....] + # Note: The actual vl should be `FULL_BLOCK_LEN32/4 * 2`, but we just use + # `FULL_BLOCK_LEN32` here. + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e8", "m1", "ta", "ma"]} + li $T0, 0b10001000 + @{[vmv_v_x $CTR_MASK, $T0]} + # Load IV. + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vle32_v $V31, $IVP]} + # Convert the big-endian counter into little-endian. + @{[vsetivli "zero", 4, "e32", "m1", "ta", "mu"]} + @{[vrev8_v $V31, $V31, $CTR_MASK]} + # Splat the `single block of IV` to v12 + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + @{[vmv_v_i $V12, 0]} + @{[vaesz_vs $V12, $V31]} + # Prepare the ctr counter into v8 + # v8: [x, x, x, 0, x, x, x, 1, x, x, x, 2, ...] + @{[viota_m $V8, $CTR_MASK, $CTR_MASK]} + # Merge IV and ctr counter into v12. + # v12:[x, x, x, count+0, x, x, x, count+1, ...] + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "mu"]} + @{[vadd_vv $V12, $V12, $V8, $CTR_MASK]} + + li $PADDING_LEN32, 0 + # Get the SEW32 size in the first round. + # If we have the non-zero value for `LEN32&(FULL_BLOCK_LEN32-1)`, then + # we will have the leading padding zero. + addi $T0, $FULL_BLOCK_LEN32, -1 + and $T0, $T0, $LEN32 + beqz $T0, 1f + + ## with padding + sub $LEN32, $LEN32, $T0 + sub $PADDING_LEN32, $FULL_BLOCK_LEN32, $T0 + # padding block size + srli $T1, $PADDING_LEN32, 2 + # padding byte size + slli $T2, $PADDING_LEN32, 2 + + # Adjust the ctr counter to make the counter start from `counter+0` for the + # first non-padding block. + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "mu"]} + @{[vsub_vx $V12, $V12, $T1, $CTR_MASK]} + # Prepare the AES ctr input into v28. + # The ctr data uses big-endian form. + @{[vmv_v_v $V28, $V12]} + @{[vrev8_v $V28, $V28, $CTR_MASK]} + + # Prepare the mask for input loading in the first round. We use + # `VL=FULL_BLOCK_LEN32` with the mask in the first round. + # Adjust input ptr. + sub $INP, $INP, $T2 + # Adjust output ptr. + sub $OUTP, $OUTP, $T2 + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e16", "m2", "ta", "ma"]} + @{[vid_v $V2]} + # We don't use the pseudo instruction `vmsgeu` here. Use `vmsgtu` instead. + # The original code is: + # vmsgeu.vx $INPUT_PADDING_MASK, $V2, $PADDING_LEN32 + addi $T0, $PADDING_LEN32, -1 + @{[vmsgtu_vx $INPUT_PADDING_MASK, $V2, $T0]} + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + @{[vmv_v_i $V24, 0]} + # Load the input for length FULL_BLOCK_LEN32 with mask. + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "mu"]} + @{[vle32_v $V24, $INP, $INPUT_PADDING_MASK]} + + # Load the init `Xi` data to v20 with preceding zero padding. + # Adjust Xi ptr. + sub $T0, $XIP, $T2 + # Load for length `zero-padding-e32-length + 4`. + addi $T1, $PADDING_LEN32, 4 + @{[vsetvli "zero", $T1, "e32", "m4", "tu", "mu"]} + @{[vle32_v $V20, $T0, $INPUT_PADDING_MASK]} + j 2f + +1: + ## without padding + sub $LEN32, $LEN32, $FULL_BLOCK_LEN32 + + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + @{[vle32_v $V24, $INP]} + + # Load the init Xi data to v20. + @{[vsetivli "zero", 4, "e32", "m1", "tu", "ma"]} + @{[vle32_v $V20, $XIP]} + + # Prepare the AES ctr input into v28. + # The ctr data uses big-endian form. + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "mu"]} + @{[vmv_v_v $V28, $V12]} + @{[vrev8_v $V28, $V28, $CTR_MASK]} +2: +___ + + return $code; +} + +sub prepare_input_and_ctr { + my $PTR_OFFSET_REG = shift; + + my $code=<<___; + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "mu"]} + # Increase ctr in v12. + @{[vadd_vx $V12, $V12, $CTR, $CTR_MASK]} + sub $LEN32, $LEN32, $FULL_BLOCK_LEN32 + # Load plaintext into v24 + @{[vsetvli "zero", "zero", "e32", "m4", "ta", "ma"]} + @{[vle32_v $V24, $INP]} + # Prepare the AES ctr input into v28. + # The ctr data uses big-endian form. + @{[vmv_v_v $V28, $V12]} + add $INP, $INP, $PTR_OFFSET_REG + @{[vsetvli "zero", "zero", "e32", "m4", "ta", "mu"]} + @{[vrev8_v $V28, $V28, $CTR_MASK]} +___ + + return $code; +} + +# Store the current CTR back to IV buffer. +sub store_current_ctr { + my $code=<<___; + @{[vsetivli "zero", 4, "e32", "m4", "ta", "ma"]} + # Update current ctr value to v12 + @{[vadd_vx $V12, $V12, $CTR, $CTR_MASK]} + # Convert ctr to big-endian counter. + @{[vrev8_v $V12, $V12, $CTR_MASK]} + @{[vse32_v $V12, $IVP, $CTR_MASK]} +___ + + return $code; +} + +# Compute the final tag into v0 from the partial tag v20. +sub compute_final_tag { + my $TMP_REG = shift; + + my $code=<<___; + # The H is at `gcm128_context.Htable[0]` (addr(Xi)+16*2). + # Load H to v1 + addi $TMP_REG, $XIP, 32 + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vle32_v $V1, $TMP_REG]} + # Multiply H for each partial tag and XOR them together. + # Handle 1st partial tag + @{[vmv_v_v $V0, $V20]} + @{[vgmul_vv $V0, $V1]} + # Handle 2nd to N-th partial tags + li $TMP_REG, 4 +1: + @{[vsetivli "zero", 4, "e32", "m4", "ta", "ma"]} + @{[vslidedown_vx $V4, $V20, $TMP_REG]} + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vghsh_vv $V0, $V1, $V4]} + addi $TMP_REG, $TMP_REG, 4 + blt $TMP_REG, $FULL_BLOCK_LEN32, 1b +___ + + return $code; +} + +################################################################################ +# size_t rv64i_zvbb_zvkg_zvkned_aes_gcm_encrypt(const unsigned char *in, +# unsigned char *out, size_t len, +# const void *key, +# unsigned char ivec[16], u64 *Xi); +{ +$code .= <<___; +.p2align 3 +.globl rv64i_zvbb_zvkg_zvkned_aes_gcm_encrypt +.type rv64i_zvbb_zvkg_zvkned_aes_gcm_encrypt,\@function +rv64i_zvbb_zvkg_zvkned_aes_gcm_encrypt: + srli $T0, $LEN, 4 + beqz $T0, .Lenc_end + slli $LEN32, $T0, 2 + + mv $ORIGINAL_LEN32, $LEN32 + + @{[aes_gcm_init]} + + # Load number of rounds + lwu $T0, 240($KEYP) + li $T1, 14 + li $T2, 12 + li $T3, 10 + + beq $T0, $T1, aes_gcm_enc_blocks_256 + beq $T0, $T2, aes_gcm_enc_blocks_192 + beq $T0, $T3, aes_gcm_enc_blocks_128 + +.Lenc_end: + li $PROCESSED_LEN, 0 + ret + +.size rv64i_zvbb_zvkg_zvkned_aes_gcm_encrypt,.-rv64i_zvbb_zvkg_zvkned_aes_gcm_encrypt +___ + +$code .= <<___; +.p2align 3 +aes_gcm_enc_blocks_128: + srli $CTR, $FULL_BLOCK_LEN32, 2 + slli $T0, $FULL_BLOCK_LEN32, 2 + + @{[aes_128_first_round $T0, $T1]} + + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + +.Lenc_blocks_128: + # Compute the partial tags. + # The partial tags will multiply with [H^n, H^n, ..., H^n] + # [tag0, tag1, ...] = + # ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n] + # We will skip the [H^n, H^n, ..., H^n] multiplication for the last round. + beqz $LEN32, .Lenc_blocks_128_end + @{[vghsh_vv $V20, $V16, $V28]} + + @{[prepare_input_and_ctr $T0]} + + @{[aes_128_cipher_body]} + + # Compute AES ctr ciphertext result. + @{[vxor_vv $V28, $V28, $V24]} + + # Store ciphertext + @{[vse32_v $V28, $OUTP]} + add $OUTP, $OUTP, $T0 + + j .Lenc_blocks_128 +.Lenc_blocks_128_end: + + # Add ciphertext into partial tag + @{[vxor_vv $V20, $V20, $V28]} + + @{[store_current_ctr]} + + @{[compute_final_tag $T1]} + + # Save the final tag + @{[vse32_v $V0, $XIP]} + + # return the processed size. + slli $PROCESSED_LEN, $ORIGINAL_LEN32, 2 + ret +.size aes_gcm_enc_blocks_128,.-aes_gcm_enc_blocks_128 +___ + +$code .= <<___; +.p2align 3 +aes_gcm_enc_blocks_192: + srli $CTR, $FULL_BLOCK_LEN32, 2 + slli $T0, $FULL_BLOCK_LEN32, 2 + + @{[aes_192_first_round $T0, $T1]} + + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + +.Lenc_blocks_192: + # Compute the partial tags. + # The partial tags will multiply with [H^n, H^n, ..., H^n] + # [tag0, tag1, ...] = + # ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n] + # We will skip the [H^n, H^n, ..., H^n] multiplication for the last round. + beqz $LEN32, .Lenc_blocks_192_end + @{[vghsh_vv $V20, $V16, $V28]} + + @{[prepare_input_and_ctr $T0]} + + @{[aes_192_cipher_body $T1]} + + # Compute AES ctr ciphertext result. + @{[vxor_vv $V28, $V28, $V24]} + + # Store ciphertext + @{[vse32_v $V28, $OUTP]} + add $OUTP, $OUTP, $T0 + + j .Lenc_blocks_192 +.Lenc_blocks_192_end: + + # Add ciphertext into partial tag + @{[vxor_vv $V20, $V20, $V28]} + + @{[store_current_ctr]} + + @{[compute_final_tag $T1]} + + # Save the final tag + @{[vse32_v $V0, $XIP]} + + # return the processed size. + slli $PROCESSED_LEN, $ORIGINAL_LEN32, 2 + ret +.size aes_gcm_enc_blocks_192,.-aes_gcm_enc_blocks_192 +___ + +$code .= <<___; +.p2align 3 +aes_gcm_enc_blocks_256: + srli $CTR, $FULL_BLOCK_LEN32, 2 + slli $T0, $FULL_BLOCK_LEN32, 2 + + @{[aes_256_first_round $T0, $T1]} + + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + +.Lenc_blocks_256: + # Compute the partial tags. + # The partial tags will multiply with [H^n, H^n, ..., H^n] + # [tag0, tag1, ...] = + # ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n] + # We will skip the [H^n, H^n, ..., H^n] multiplication for the last round. + beqz $LEN32, .Lenc_blocks_256_end + @{[vghsh_vv $V20, $V16, $V28]} + + @{[prepare_input_and_ctr $T0]} + + @{[aes_256_cipher_body $T1]} + + # Compute AES ctr ciphertext result. + @{[vxor_vv $V28, $V28, $V24]} + + # Store ciphertext + @{[vse32_v $V28, $OUTP]} + add $OUTP, $OUTP, $T0 + + j .Lenc_blocks_256 +.Lenc_blocks_256_end: + + # Add ciphertext into partial tag + @{[vxor_vv $V20, $V20, $V28]} + + @{[store_current_ctr]} + + @{[compute_final_tag $T1]} + + # Save the final tag + @{[vse32_v $V0, $XIP]} + + # return the processed size. + slli $PROCESSED_LEN, $ORIGINAL_LEN32, 2 + ret +.size aes_gcm_enc_blocks_256,.-aes_gcm_enc_blocks_256 +___ + +} + +################################################################################ +# size_t rv64i_zvbb_zvkg_zvkned_aes_gcm_decrypt(const unsigned char *in, +# unsigned char *out, size_t len, +# const void *key, +# unsigned char ivec[16], u64 *Xi); +{ +$code .= <<___; +.p2align 3 +.globl rv64i_zvbb_zvkg_zvkned_aes_gcm_decrypt +.type rv64i_zvbb_zvkg_zvkned_aes_gcm_decrypt,\@function +rv64i_zvbb_zvkg_zvkned_aes_gcm_decrypt: + srli $T0, $LEN, 4 + beqz $T0, .Ldec_end + slli $LEN32, $T0, 2 + + mv $ORIGINAL_LEN32, $LEN32 + + @{[aes_gcm_init]} + + # Load number of rounds + lwu $T0, 240($KEYP) + li $T1, 14 + li $T2, 12 + li $T3, 10 + + beq $T0, $T1, aes_gcm_dec_blocks_256 + beq $T0, $T2, aes_gcm_dec_blocks_192 + beq $T0, $T3, aes_gcm_dec_blocks_128 + +.Ldec_end: + li $PROCESSED_LEN, 0 + ret +.size rv64i_zvbb_zvkg_zvkned_aes_gcm_decrypt,.-rv64i_zvbb_zvkg_zvkned_aes_gcm_decrypt +___ + +$code .= <<___; +.p2align 3 +aes_gcm_dec_blocks_128: + srli $CTR, $FULL_BLOCK_LEN32, 2 + slli $T0, $FULL_BLOCK_LEN32, 2 + + @{[aes_128_first_round $T0, $T1]} + + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + +.Ldec_blocks_128: + # Compute the partial tags. + # The partial tags will multiply with [H^n, H^n, ..., H^n] + # [tag0, tag1, ...] = + # ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n] + # We will skip the [H^n, H^n, ..., H^n] multiplication for the last round. + beqz $LEN32, .Ldec_blocks_256_end + @{[vghsh_vv $V20, $V16, $V24]} + + @{[prepare_input_and_ctr $T0]} + + @{[aes_128_cipher_body]} + + # Compute AES ctr plaintext result. + @{[vxor_vv $V28, $V28, $V24]} + + # Store plaintext + @{[vse32_v $V28, $OUTP]} + add $OUTP, $OUTP, $T0 + + j .Ldec_blocks_128 +.Ldec_blocks_128_end: + + # Add ciphertext into partial tag + @{[vxor_vv $V20, $V20, $V24]} + + @{[store_current_ctr]} + + @{[compute_final_tag $T1]} + + # Save the final tag + @{[vse32_v $V0, $XIP]} + + # return the processed size. + slli $PROCESSED_LEN, $ORIGINAL_LEN32, 2 + ret +.size aes_gcm_dec_blocks_128,.-aes_gcm_dec_blocks_128 +___ + +$code .= <<___; +.p2align 3 +aes_gcm_dec_blocks_192: + srli $CTR, $FULL_BLOCK_LEN32, 2 + slli $T0, $FULL_BLOCK_LEN32, 2 + + @{[aes_192_first_round $T0, $T1]} + + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + +.Ldec_blocks_192: + # Compute the partial tags. + # The partial tags will multiply with [H^n, H^n, ..., H^n] + # [tag0, tag1, ...] = + # ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n] + # We will skip the [H^n, H^n, ..., H^n] multiplication for the last round. + beqz $LEN32, .Ldec_blocks_192_end + @{[vghsh_vv $V20, $V16, $V24]} + + @{[prepare_input_and_ctr $T0]} + + @{[aes_192_cipher_body $T1]} + + # Compute AES ctr plaintext result. + @{[vxor_vv $V28, $V28, $V24]} + + # Store plaintext + @{[vse32_v $V28, $OUTP]} + add $OUTP, $OUTP, $T0 + + j .Ldec_blocks_192 +.Ldec_blocks_192_end: + + # Add ciphertext into partial tag + @{[vxor_vv $V20, $V20, $V24]} + + @{[store_current_ctr]} + + @{[compute_final_tag $T1]} + + # Save the final tag + @{[vse32_v $V0, $XIP]} + + # return the processed size. + slli $PROCESSED_LEN, $ORIGINAL_LEN32, 2 + ret +.size aes_gcm_dec_blocks_192,.-aes_gcm_dec_blocks_192 +___ + +$code .= <<___; +.p2align 3 +aes_gcm_dec_blocks_256: + srli $CTR, $FULL_BLOCK_LEN32, 2 + slli $T0, $FULL_BLOCK_LEN32, 2 + + @{[aes_256_first_round $T0, $T1]} + + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + +.Ldec_blocks_256: + # Compute the partial tags. + # The partial tags will multiply with [H^n, H^n, ..., H^n] + # [tag0, tag1, ...] = + # ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n] + # We will skip the [H^n, H^n, ..., H^n] multiplication for the last round. + beqz $LEN32, .Ldec_blocks_256_end + @{[vghsh_vv $V20, $V16, $V24]} + + @{[prepare_input_and_ctr $T0]} + + @{[aes_256_cipher_body $T1]} + + # Compute AES ctr plaintext result. + @{[vxor_vv $V28, $V28, $V24]} + + # Store plaintext + @{[vse32_v $V28, $OUTP]} + add $OUTP, $OUTP, $T0 + + j .Ldec_blocks_256 +.Ldec_blocks_256_end: + + # Add ciphertext into partial tag + @{[vxor_vv $V20, $V20, $V24]} + + @{[store_current_ctr]} + + @{[compute_final_tag $T1]} + + # Save the final tag + @{[vse32_v $V0, $XIP]} + + # return the processed size. + slli $PROCESSED_LEN, $ORIGINAL_LEN32, 2 + ret +.size aes_gcm_dec_blocks_256,.-aes_gcm_dec_blocks_256 +___ + +} +} + +print $code; + +close STDOUT or die "error closing STDOUT: $!"; diff --git a/crypto/modes/build.info b/crypto/modes/build.info index 9ebb5cc7a8c..b2c526dc552 100644 --- a/crypto/modes/build.info +++ b/crypto/modes/build.info @@ -43,7 +43,7 @@ IF[{- !$disabled{asm} -}] $MODESASM_c64xplus=ghash-c64xplus.s $MODESDEF_c64xplus=GHASH_ASM - $MODESASM_riscv64=ghash-riscv64.s ghash-riscv64-zvkb-zvbc.s ghash-riscv64-zvkg.s + $MODESASM_riscv64=ghash-riscv64.s ghash-riscv64-zvkb-zvbc.s ghash-riscv64-zvkg.s aes-gcm-riscv64-zvkb-zvkg-zvkned.s $MODESDEF_riscv64=GHASH_ASM # Now that we have defined all the arch specific variables, use the @@ -93,3 +93,4 @@ GENERATE[ghash-c64xplus.S]=asm/ghash-c64xplus.pl GENERATE[ghash-riscv64.s]=asm/ghash-riscv64.pl GENERATE[ghash-riscv64-zvkb-zvbc.s]=asm/ghash-riscv64-zvkb-zvbc.pl GENERATE[ghash-riscv64-zvkg.s]=asm/ghash-riscv64-zvkg.pl +GENERATE[aes-gcm-riscv64-zvbb-zvkg-zvkned.s]=asm/aes-gcm-riscv64-zvbb-zvkg-zvkned.pl diff --git a/include/crypto/aes_platform.h b/include/crypto/aes_platform.h index 13d94436289..749515d57d6 100644 --- a/include/crypto/aes_platform.h +++ b/include/crypto/aes_platform.h @@ -475,6 +475,26 @@ void rv64i_zvkb_zvkned_ctr32_encrypt_blocks(const unsigned char *in, const void *key, const unsigned char ivec[16]); +size_t rv64i_zvbb_zvkg_zvkned_aes_gcm_encrypt(const unsigned char *in, + unsigned char *out, size_t len, + const void *key, + unsigned char ivec[16], u64 *Xi); + +size_t rv64i_zvbb_zvkg_zvkned_aes_gcm_decrypt(const unsigned char *in, + unsigned char *out, size_t len, + const void *key, + unsigned char ivec[16], u64 *Xi); + +void gcm_ghash_rv64i_zvkg(u64 Xi[2], const u128 Htable[16], const u8 *inp, + size_t len); + +# define AES_GCM_ENC_BYTES 64 +# define AES_GCM_DEC_BYTES 64 +# define AES_gcm_encrypt rv64i_zvbb_zvkg_zvkned_aes_gcm_encrypt +# define AES_gcm_decrypt rv64i_zvbb_zvkg_zvkned_aes_gcm_decrypt +# define AES_GCM_ASM(ctx) (ctx->ctr == rv64i_zvbb_zvkned_ctr32_encrypt_blocks && \ + ctx->gcm.funcs.ghash == gcm_ghash_rv64i_zvkg) + # elif defined(OPENSSL_CPUID_OBJ) && defined(__riscv) && __riscv_xlen == 32 /* RISC-V 32 support */ # include "riscv_arch.h" diff --git a/providers/implementations/ciphers/cipher_aes_gcm_hw_rv64i.inc b/providers/implementations/ciphers/cipher_aes_gcm_hw_rv64i.inc index b4a6749d3f9..883d5d918f6 100644 --- a/providers/implementations/ciphers/cipher_aes_gcm_hw_rv64i.inc +++ b/providers/implementations/ciphers/cipher_aes_gcm_hw_rv64i.inc @@ -8,10 +8,13 @@ */ /*- - * RISC-V 64 ZKND ZKNE support for AES GCM. + * RISC-V 64 support for AES GCM. * This file is included by cipher_aes_gcm_hw.c */ +/*- + * RISC-V 64 ZKND and ZKNE support for AES GCM. + */ static int rv64i_zknd_zkne_gcm_initkey(PROV_GCM_CTX *ctx, const unsigned char *key, size_t keylen) { @@ -33,21 +36,25 @@ static const PROV_GCM_HW rv64i_zknd_zkne_gcm = { /*- * RISC-V RV64 ZVKNED support for AES GCM. - * This file is included by cipher_aes_gcm_hw.c */ - static int rv64i_zvkned_gcm_initkey(PROV_GCM_CTX *ctx, const unsigned char *key, size_t keylen) { PROV_AES_GCM_CTX *actx = (PROV_AES_GCM_CTX *)ctx; AES_KEY *ks = &actx->ks.ks; - /* Zvkned only supports 128 and 256 bit keys for key schedule generation. */ + + /* + * Zvkned only supports 128 and 256 bit keys for key schedule generation. + * For AES-192 case, we could fallback to `AES_set_encrypt_key`. + */ if (keylen * 8 == 128 || keylen * 8 == 256) { GCM_HW_SET_KEY_CTR_FN(ks, rv64i_zvkned_set_encrypt_key, rv64i_zvkned_encrypt, NULL); } else { - GCM_HW_SET_KEY_CTR_FN(ks, AES_set_encrypt_key, rv64i_zvkned_encrypt, NULL); + GCM_HW_SET_KEY_CTR_FN(ks, AES_set_encrypt_key, + rv64i_zvkned_encrypt, NULL); } + return 1; } @@ -60,12 +67,52 @@ static const PROV_GCM_HW rv64i_zvkned_gcm = { ossl_gcm_one_shot }; -const PROV_GCM_HW *ossl_prov_aes_hw_gcm(size_t keybits) -{ - if (RISCV_HAS_ZVKNED() && riscv_vlen() >= 128) - return &rv64i_zvkned_gcm; - else if (RISCV_HAS_ZKND_AND_ZKNE()) - return &rv64i_zknd_zkne_gcm; - else - return &aes_gcm; +/*- + * RISC-V RV64 ZVBB, ZVKG and ZVKNED support for AES GCM. + */ +static int rv64i_zvbb_zvkg_zvkned_gcm_initkey(PROV_GCM_CTX *ctx, + const unsigned char *key, + size_t keylen) { + PROV_AES_GCM_CTX *actx = (PROV_AES_GCM_CTX *)ctx; + AES_KEY *ks = &actx->ks.ks; + + /* + * Zvkned only supports 128 and 256 bit keys for key schedule generation. + * For AES-192 case, we could fallback to `AES_set_encrypt_key`. + */ + if (keylen * 8 == 128 || keylen * 8 == 256) { + GCM_HW_SET_KEY_CTR_FN(ks, rv64i_zvkned_set_encrypt_key, + rv64i_zvkned_encrypt, + rv64i_zvbb_zvkned_ctr32_encrypt_blocks); + } else { + GCM_HW_SET_KEY_CTR_FN(ks, AES_set_encrypt_key, + rv64i_zvkned_encrypt, + rv64i_zvbb_zvkned_ctr32_encrypt_blocks); + } + + return 1; +} + +static const PROV_GCM_HW rv64i_zvbb_zvkg_zvkned_gcm = { + rv64i_zvbb_zvkg_zvkned_gcm_initkey, + ossl_gcm_setiv, + ossl_gcm_aad_update, + generic_aes_gcm_cipher_update, + ossl_gcm_cipher_final, + ossl_gcm_one_shot +}; + +const PROV_GCM_HW *ossl_prov_aes_hw_gcm(size_t keybits) { + if (RISCV_HAS_ZVKNED()) { + if (RISCV_HAS_ZVBB() && RISCV_HAS_ZVKG() && riscv_vlen() >= 128) { + return &rv64i_zvbb_zvkg_zvkned_gcm; + } + return &rv64i_zvkned_gcm; + } + + if (RISCV_HAS_ZKND_AND_ZKNE()) { + return &rv64i_zknd_zkne_gcm; + } + + return &aes_gcm; }