From: Min Zhou <zhoumin@loongson.cn>
Date: Thu, 7 Sep 2023 03:07:53 +0000 (+0800)
Subject: LoongArch64 assembly pack: add ChaCha20 modules
X-Git-Tag: openssl-3.2.0-alpha2~110
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=9a41a3c6a453a90d6c6cf106480a4a72b08b14f9;p=thirdparty%2Fopenssl.git

LoongArch64 assembly pack: add ChaCha20 modules

This assembly implementation for ChaCha20 includes three code paths:
scalar path, 128-bit LSX path and 256-bit LASX path. We prefer the
LASX path or LSX path if the hardware and system support these
extensions.

There are 32 vector registers avaialable in the LSX and LASX
extensions. So, we can load the 16 initial states and the 16
intermediate states of ChaCha into the 32 vector registers for
calculating in the implementation. The test results on the 3A5000
and 3A6000 show that this assembly implementation significantly
improves the performance of ChaCha20 on LoongArch based machines.
The detailed test results are as following.

Test with:
$ openssl speed -evp chacha20

3A5000
type               16 bytes     64 bytes    256 bytes    1024 bytes    8192 bytes   16384 bytes
C code           178484.53k   282789.93k   311793.70k    322234.99k    324405.93k    324659.88k
assembly code    223152.28k   407863.65k   989520.55k   2049192.96k   2127248.70k   2131749.55k
                   +25%         +44%         +217%        +536%         +556%         +557%

3A6000
type               16 bytes     64 bytes     256 bytes    1024 bytes    8192 bytes   16384 bytes
C code           214945.33k   310041.75k    340724.22k    349949.27k    352925.01k    353140.74k
assembly code    299151.34k   492766.34k   2070166.02k   4300909.91k   4473978.88k   4499084.63k
                   +39%         +59%         +508%         +1129%        +1168%        +1174%

Signed-off-by: Min Zhou <zhoumin@loongson.cn>

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Paul Dale <pauli@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/21998)
---

diff --git a/crypto/chacha/asm/chacha-loongarch64.pl b/crypto/chacha/asm/chacha-loongarch64.pl
new file mode 100644
index 00000000000..ea9cc7ecce2
--- /dev/null
+++ b/crypto/chacha/asm/chacha-loongarch64.pl
@@ -0,0 +1,1413 @@
+#! /usr/bin/env perl
+# Author: Min Zhou <zhoumin@loongson.cn>
+# Copyright 2023 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the OpenSSL license (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+use strict;
+
+my $code;
+
+# Here is the scalar register layout for LoongArch.
+my ($zero,$ra,$tp,$sp,$fp)=map("\$r$_",(0..3,22));
+my ($a0,$a1,$a2,$a3,$a4,$a5,$a6,$a7)=map("\$r$_",(4..11));
+my ($t0,$t1,$t2,$t3,$t4,$t5,$t6,$t7,$t8,$x)=map("\$r$_",(12..21));
+my ($s0,$s1,$s2,$s3,$s4,$s5,$s6,$s7,$s8)=map("\$r$_",(23..31));
+
+# Here is the 128-bit vector register layout for LSX extension.
+my ($vr0,$vr1,$vr2,$vr3,$vr4,$vr5,$vr6,$vr7,$vr8,$vr9,$vr10,
+    $vr11,$vr12,$vr13,$vr14,$vr15,$vr16,$vr17,$vr18,$vr19,
+    $vr20,$vr21,$vr22,$vr23,$vr24,$vr25,$vr26,$vr27,$vr28,
+    $vr29,$vr30,$vr31)=map("\$vr$_",(0..31));
+
+# Here is the 256-bit vector register layout for LASX extension.
+my ($xr0,$xr1,$xr2,$xr3,$xr4,$xr5,$xr6,$xr7,$xr8,$xr9,$xr10,
+    $xr11,$xr12,$xr13,$xr14,$xr15,$xr16,$xr17,$xr18,$xr19,
+    $xr20,$xr21,$xr22,$xr23,$xr24,$xr25,$xr26,$xr27,$xr28,
+    $xr29,$xr30,$xr31)=map("\$xr$_",(0..31));
+
+my $output;
+for (@ARGV) {	$output=$_ if (/\w[\w\-]*\.\w+$/);	}
+open STDOUT,">$output";
+
+# Input parameter block
+my ($out, $inp, $len, $key, $counter) = ($a0, $a1, $a2, $a3, $a4);
+
+$code .= <<EOF;
+#include "loongarch_arch.h"
+
+.text
+
+.extern OPENSSL_loongarch_hwcap_P
+
+.align 6
+.Lsigma:
+.ascii	"expand 32-byte k"
+.Linc8x:
+.long	0,1,2,3,4,5,6,7
+.Linc4x:
+.long	0,1,2,3
+
+.globl	ChaCha20_ctr32
+.type	ChaCha20_ctr32 function
+
+.align 6
+ChaCha20_ctr32:
+	# $a0 = arg #1 (out pointer)
+	# $a1 = arg #2 (inp pointer)
+	# $a2 = arg #3 (len)
+	# $a3 = arg #4 (key array)
+	# $a4 = arg #5 (counter array)
+
+	beqz		$len,.Lno_data
+	la.pcrel	$t0,OPENSSL_loongarch_hwcap_P
+	ld.w		$t0,$t0,0
+
+	andi		$t1,$t0,LOONGARCH_HWCAP_LASX
+	bnez		$t1,.LChaCha20_8x
+
+	andi		$t2,$t0,LOONGARCH_HWCAP_LSX
+	bnez		$t2,.LChaCha20_4x
+
+	b			.LChaCha20_1x
+
+EOF
+
+########################################################################
+# Scalar code path that handles all lengths.
+{
+# Load the initial states in array @x[*] and update directly
+my @x = ($t0, $t1, $t2, $t3, $t4, $t5, $t6, $t7,
+         $s0, $s1, $s2, $s3, $s4, $s5, $s6, $s7);
+
+sub ROUND {
+	my ($a0,$b0,$c0,$d0) = @_;
+	my ($a1,$b1,$c1,$d1) = map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
+	my ($a2,$b2,$c2,$d2) = map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
+	my ($a3,$b3,$c3,$d3) = map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
+
+$code .= <<EOF;
+	add.w		@x[$a0],@x[$a0],@x[$b0]
+	xor			@x[$d0],@x[$d0],@x[$a0]
+	rotri.w		@x[$d0],@x[$d0],16      # rotate left 16 bits
+	add.w		@x[$a1],@x[$a1],@x[$b1]
+	xor			@x[$d1],@x[$d1],@x[$a1]
+	rotri.w		@x[$d1],@x[$d1],16
+
+	add.w		@x[$c0],@x[$c0],@x[$d0]
+	xor			@x[$b0],@x[$b0],@x[$c0]
+	rotri.w		@x[$b0],@x[$b0],20      # rotate left 12 bits
+	add.w		@x[$c1],@x[$c1],@x[$d1]
+	xor			@x[$b1],@x[$b1],@x[$c1]
+	rotri.w		@x[$b1],@x[$b1],20
+
+	add.w		@x[$a0],@x[$a0],@x[$b0]
+	xor			@x[$d0],@x[$d0],@x[$a0]
+	rotri.w		@x[$d0],@x[$d0],24      # rotate left 8 bits
+	add.w		@x[$a1],@x[$a1],@x[$b1]
+	xor			@x[$d1],@x[$d1],@x[$a1]
+	rotri.w		@x[$d1],@x[$d1],24
+
+	add.w		@x[$c0],@x[$c0],@x[$d0]
+	xor			@x[$b0],@x[$b0],@x[$c0]
+	rotri.w		@x[$b0],@x[$b0],25      # rotate left 7 bits
+	add.w		@x[$c1],@x[$c1],@x[$d1]
+	xor			@x[$b1],@x[$b1],@x[$c1]
+	rotri.w		@x[$b1],@x[$b1],25
+
+	add.w		@x[$a2],@x[$a2],@x[$b2]
+	xor			@x[$d2],@x[$d2],@x[$a2]
+	rotri.w		@x[$d2],@x[$d2],16
+	add.w		@x[$a3],@x[$a3],@x[$b3]
+	xor			@x[$d3],@x[$d3],@x[$a3]
+	rotri.w		@x[$d3],@x[$d3],16
+
+	add.w		@x[$c2],@x[$c2],@x[$d2]
+	xor			@x[$b2],@x[$b2],@x[$c2]
+	rotri.w		@x[$b2],@x[$b2],20
+	add.w		@x[$c3],@x[$c3],@x[$d3]
+	xor			@x[$b3],@x[$b3],@x[$c3]
+	rotri.w		@x[$b3],@x[$b3],20
+
+	add.w		@x[$a2],@x[$a2],@x[$b2]
+	xor			@x[$d2],@x[$d2],@x[$a2]
+	rotri.w		@x[$d2],@x[$d2],24
+	add.w		@x[$a3],@x[$a3],@x[$b3]
+	xor			@x[$d3],@x[$d3],@x[$a3]
+	rotri.w		@x[$d3],@x[$d3],24
+
+	add.w		@x[$c2],@x[$c2],@x[$d2]
+	xor			@x[$b2],@x[$b2],@x[$c2]
+	rotri.w		@x[$b2],@x[$b2],25
+	add.w		@x[$c3],@x[$c3],@x[$d3]
+	xor			@x[$b3],@x[$b3],@x[$c3]
+	rotri.w		@x[$b3],@x[$b3],25
+
+EOF
+}
+
+$code .= <<EOF;
+.align 6
+.LChaCha20_1x:
+	addi.d		$sp,$sp,-256
+	st.d		$s0,$sp,0
+	st.d		$s1,$sp,8
+	st.d		$s2,$sp,16
+	st.d		$s3,$sp,24
+	st.d		$s4,$sp,32
+	st.d		$s5,$sp,40
+	st.d		$s6,$sp,48
+	st.d		$s7,$sp,56
+	st.d		$s8,$sp,64
+
+	# Save the initial block counter in $s8
+	ld.w		$s8,$counter,0
+	b			.Loop_outer_1x
+
+.align 5
+.Loop_outer_1x:
+	# Load constants
+	la.local	$t8,.Lsigma
+	ld.w		@x[0],$t8,0		  # 'expa'
+	ld.w		@x[1],$t8,4		  # 'nd 3'
+	ld.w		@x[2],$t8,8		  # '2-by'
+	ld.w		@x[3],$t8,12	  # 'te k'
+
+	# Load key
+	ld.w		@x[4],$key,4*0
+	ld.w		@x[5],$key,4*1
+	ld.w		@x[6],$key,4*2
+	ld.w		@x[7],$key,4*3
+	ld.w		@x[8],$key,4*4
+	ld.w		@x[9],$key,4*5
+	ld.w		@x[10],$key,4*6
+	ld.w		@x[11],$key,4*7
+
+	# Load block counter
+	move		@x[12],$s8
+
+	# Load nonce
+	ld.w		@x[13],$counter,4*1
+	ld.w		@x[14],$counter,4*2
+	ld.w		@x[15],$counter,4*3
+
+	# Update states in \@x[*] for 20 rounds
+	ori			$t8,$zero,10
+	b			.Loop_1x
+
+.align 5
+.Loop_1x:
+EOF
+
+&ROUND (0, 4, 8, 12);
+&ROUND (0, 5, 10, 15);
+
+$code .= <<EOF;
+	addi.w		$t8,$t8,-1
+	bnez		$t8,.Loop_1x
+
+	# Get the final states by adding the initial states
+	la.local	$t8,.Lsigma
+	ld.w		$a7,$t8,4*0
+	ld.w		$a6,$t8,4*1
+	ld.w		$a5,$t8,4*2
+	add.w		@x[0],@x[0],$a7
+	add.w		@x[1],@x[1],$a6
+	add.w		@x[2],@x[2],$a5
+	ld.w		$a7,$t8,4*3
+	add.w		@x[3],@x[3],$a7
+
+	ld.w		$t8,$key,4*0
+	ld.w		$a7,$key,4*1
+	ld.w		$a6,$key,4*2
+	ld.w		$a5,$key,4*3
+	add.w		@x[4],@x[4],$t8
+	add.w		@x[5],@x[5],$a7
+	add.w		@x[6],@x[6],$a6
+	add.w		@x[7],@x[7],$a5
+
+	ld.w		$t8,$key,4*4
+	ld.w		$a7,$key,4*5
+	ld.w		$a6,$key,4*6
+	ld.w		$a5,$key,4*7
+	add.w		@x[8],@x[8],$t8
+	add.w		@x[9],@x[9],$a7
+	add.w		@x[10],@x[10],$a6
+	add.w		@x[11],@x[11],$a5
+
+	add.w		@x[12],@x[12],$s8
+
+	ld.w		$t8,$counter,4*1
+	ld.w		$a7,$counter,4*2
+	ld.w		$a6,$counter,4*3
+	add.w		@x[13],@x[13],$t8
+	add.w		@x[14],@x[14],$a7
+	add.w		@x[15],@x[15],$a6
+
+	ori			$t8,$zero,64
+	bltu		$len,$t8,.Ltail_1x
+
+	# Get the encrypted message by xor states with plaintext
+	ld.w		$t8,$inp,4*0
+	ld.w		$a7,$inp,4*1
+	ld.w		$a6,$inp,4*2
+	ld.w		$a5,$inp,4*3
+	xor			$t8,$t8,@x[0]
+	xor			$a7,$a7,@x[1]
+	xor			$a6,$a6,@x[2]
+	xor			$a5,$a5,@x[3]
+	st.w		$t8,$out,4*0
+	st.w		$a7,$out,4*1
+	st.w		$a6,$out,4*2
+	st.w		$a5,$out,4*3
+
+	ld.w		$t8,$inp,4*4
+	ld.w		$a7,$inp,4*5
+	ld.w		$a6,$inp,4*6
+	ld.w		$a5,$inp,4*7
+	xor			$t8,$t8,@x[4]
+	xor			$a7,$a7,@x[5]
+	xor			$a6,$a6,@x[6]
+	xor			$a5,$a5,@x[7]
+	st.w		$t8,$out,4*4
+	st.w		$a7,$out,4*5
+	st.w		$a6,$out,4*6
+	st.w		$a5,$out,4*7
+
+	ld.w		$t8,$inp,4*8
+	ld.w		$a7,$inp,4*9
+	ld.w		$a6,$inp,4*10
+	ld.w		$a5,$inp,4*11
+	xor			$t8,$t8,@x[8]
+	xor			$a7,$a7,@x[9]
+	xor			$a6,$a6,@x[10]
+	xor			$a5,$a5,@x[11]
+	st.w		$t8,$out,4*8
+	st.w		$a7,$out,4*9
+	st.w		$a6,$out,4*10
+	st.w		$a5,$out,4*11
+
+	ld.w		$t8,$inp,4*12
+	ld.w		$a7,$inp,4*13
+	ld.w		$a6,$inp,4*14
+	ld.w		$a5,$inp,4*15
+	xor			$t8,$t8,@x[12]
+	xor			$a7,$a7,@x[13]
+	xor			$a6,$a6,@x[14]
+	xor			$a5,$a5,@x[15]
+	st.w		$t8,$out,4*12
+	st.w		$a7,$out,4*13
+	st.w		$a6,$out,4*14
+	st.w		$a5,$out,4*15
+
+	addi.d		$len,$len,-64
+	beqz		$len,.Ldone_1x
+	addi.d		$inp,$inp,64
+	addi.d		$out,$out,64
+	addi.w		$s8,$s8,1
+	b			.Loop_outer_1x
+
+.align 4
+.Ltail_1x:
+	# Handle the tail for 1x (1 <= tail_len <= 63)
+	addi.d		$a7,$sp,72
+	st.w		@x[0],$a7,4*0
+	st.w		@x[1],$a7,4*1
+	st.w		@x[2],$a7,4*2
+	st.w		@x[3],$a7,4*3
+	st.w		@x[4],$a7,4*4
+	st.w		@x[5],$a7,4*5
+	st.w		@x[6],$a7,4*6
+	st.w		@x[7],$a7,4*7
+	st.w		@x[8],$a7,4*8
+	st.w		@x[9],$a7,4*9
+	st.w		@x[10],$a7,4*10
+	st.w		@x[11],$a7,4*11
+	st.w		@x[12],$a7,4*12
+	st.w		@x[13],$a7,4*13
+	st.w		@x[14],$a7,4*14
+	st.w		@x[15],$a7,4*15
+
+	move		$t8,$zero
+
+.Loop_tail_1x:
+	# Xor input with states byte by byte
+	ldx.bu		$a6,$inp,$t8
+	ldx.bu		$a5,$a7,$t8
+	xor			$a6,$a6,$a5
+	stx.b		$a6,$out,$t8
+	addi.w		$t8,$t8,1
+	addi.d		$len,$len,-1
+	bnez		$len,.Loop_tail_1x
+	b			.Ldone_1x
+
+.Ldone_1x:
+	ld.d		$s0,$sp,0
+	ld.d		$s1,$sp,8
+	ld.d		$s2,$sp,16
+	ld.d		$s3,$sp,24
+	ld.d		$s4,$sp,32
+	ld.d		$s5,$sp,40
+	ld.d		$s6,$sp,48
+	ld.d		$s7,$sp,56
+	ld.d		$s8,$sp,64
+	addi.d		$sp,$sp,256
+
+	b			.Lend
+
+EOF
+}
+
+########################################################################
+# 128-bit LSX code path that handles all lengths.
+{
+# Load the initial states in array @x[*] and update directly.
+my @x = ($vr0, $vr1, $vr2, $vr3, $vr4, $vr5, $vr6, $vr7,
+         $vr8, $vr9, $vr10, $vr11, $vr12, $vr13, $vr14, $vr15);
+
+# Save the initial states in array @y[*]
+my @y = ($vr16, $vr17, $vr18, $vr19, $vr20, $vr21, $vr22, $vr23,
+         $vr24, $vr25, $vr26, $vr27, $vr28, $vr29, $vr30, $vr31);
+
+sub ROUND_4x {
+	my ($a0,$b0,$c0,$d0) = @_;
+	my ($a1,$b1,$c1,$d1) = map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
+	my ($a2,$b2,$c2,$d2) = map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
+	my ($a3,$b3,$c3,$d3) = map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
+
+$code .= <<EOF;
+	vadd.w		@x[$a0],@x[$a0],@x[$b0]
+	vxor.v		@x[$d0],@x[$d0],@x[$a0]
+	vrotri.w	@x[$d0],@x[$d0],16      # rotate left 16 bits
+	vadd.w		@x[$a1],@x[$a1],@x[$b1]
+	vxor.v		@x[$d1],@x[$d1],@x[$a1]
+	vrotri.w	@x[$d1],@x[$d1],16
+
+	vadd.w		@x[$c0],@x[$c0],@x[$d0]
+	vxor.v		@x[$b0],@x[$b0],@x[$c0]
+	vrotri.w	@x[$b0],@x[$b0],20      # rotate left 12 bits
+	vadd.w		@x[$c1],@x[$c1],@x[$d1]
+	vxor.v		@x[$b1],@x[$b1],@x[$c1]
+	vrotri.w	@x[$b1],@x[$b1],20
+
+	vadd.w		@x[$a0],@x[$a0],@x[$b0]
+	vxor.v		@x[$d0],@x[$d0],@x[$a0]
+	vrotri.w	@x[$d0],@x[$d0],24      # rotate left 8 bits
+	vadd.w		@x[$a1],@x[$a1],@x[$b1]
+	vxor.v		@x[$d1],@x[$d1],@x[$a1]
+	vrotri.w	@x[$d1],@x[$d1],24
+
+	vadd.w		@x[$c0],@x[$c0],@x[$d0]
+	vxor.v		@x[$b0],@x[$b0],@x[$c0]
+	vrotri.w	@x[$b0],@x[$b0],25      # rotate left 7 bits
+	vadd.w		@x[$c1],@x[$c1],@x[$d1]
+	vxor.v		@x[$b1],@x[$b1],@x[$c1]
+	vrotri.w	@x[$b1],@x[$b1],25
+
+	vadd.w		@x[$a2],@x[$a2],@x[$b2]
+	vxor.v		@x[$d2],@x[$d2],@x[$a2]
+	vrotri.w	@x[$d2],@x[$d2],16
+	vadd.w		@x[$a3],@x[$a3],@x[$b3]
+	vxor.v		@x[$d3],@x[$d3],@x[$a3]
+	vrotri.w	@x[$d3],@x[$d3],16
+
+	vadd.w		@x[$c2],@x[$c2],@x[$d2]
+	vxor.v		@x[$b2],@x[$b2],@x[$c2]
+	vrotri.w	@x[$b2],@x[$b2],20
+	vadd.w		@x[$c3],@x[$c3],@x[$d3]
+	vxor.v		@x[$b3],@x[$b3],@x[$c3]
+	vrotri.w	@x[$b3],@x[$b3],20
+
+	vadd.w		@x[$a2],@x[$a2],@x[$b2]
+	vxor.v		@x[$d2],@x[$d2],@x[$a2]
+	vrotri.w	@x[$d2],@x[$d2],24
+	vadd.w		@x[$a3],@x[$a3],@x[$b3]
+	vxor.v		@x[$d3],@x[$d3],@x[$a3]
+	vrotri.w	@x[$d3],@x[$d3],24
+
+	vadd.w		@x[$c2],@x[$c2],@x[$d2]
+	vxor.v		@x[$b2],@x[$b2],@x[$c2]
+	vrotri.w	@x[$b2],@x[$b2],25
+	vadd.w		@x[$c3],@x[$c3],@x[$d3]
+	vxor.v		@x[$b3],@x[$b3],@x[$c3]
+	vrotri.w	@x[$b3],@x[$b3],25
+
+EOF
+}
+
+$code .= <<EOF;
+.align 6
+.LChaCha20_4x:
+	ori			$t3,$zero,64
+	bleu		$len,$t3,.LChaCha20_1x  # goto 1x when len <= 64
+
+	addi.d		$sp,$sp,-128
+
+	# Save the initial block counter in $t4
+	ld.w		$t4,$counter,0
+	b			.Loop_outer_4x
+
+.align 5
+.Loop_outer_4x:
+	# Load constant
+	la.local		$t8,.Lsigma
+	vldrepl.w		@x[0],$t8,4*0		# 'expa'
+	vldrepl.w		@x[1],$t8,4*1		# 'nd 3'
+	vldrepl.w		@x[2],$t8,4*2		# '2-by'
+	vldrepl.w		@x[3],$t8,4*3		# 'te k'
+
+	# Load key
+	vldrepl.w		@x[4],$key,4*0
+	vldrepl.w		@x[5],$key,4*1
+	vldrepl.w		@x[6],$key,4*2
+	vldrepl.w		@x[7],$key,4*3
+	vldrepl.w		@x[8],$key,4*4
+	vldrepl.w		@x[9],$key,4*5
+	vldrepl.w		@x[10],$key,4*6
+	vldrepl.w		@x[11],$key,4*7
+
+	# Load block counter
+	vreplgr2vr.w	@x[12],$t4
+
+	# Load nonce
+	vldrepl.w		@x[13],$counter,4*1
+	vldrepl.w		@x[14],$counter,4*2
+	vldrepl.w		@x[15],$counter,4*3
+
+	# Get the correct block counter for each block
+	la.local		$t8,.Linc4x
+	vld				@y[0],$t8,0
+	vadd.w			@x[12],@x[12],@y[0]
+
+	# Copy the initial states from \@x[*] to \@y[*]
+	vori.b			@y[0],@x[0],0
+	vori.b			@y[1],@x[1],0
+	vori.b			@y[2],@x[2],0
+	vori.b			@y[3],@x[3],0
+	vori.b			@y[4],@x[4],0
+	vori.b			@y[5],@x[5],0
+	vori.b			@y[6],@x[6],0
+	vori.b			@y[7],@x[7],0
+	vori.b			@y[8],@x[8],0
+	vori.b			@y[9],@x[9],0
+	vori.b			@y[10],@x[10],0
+	vori.b			@y[11],@x[11],0
+	vori.b			@y[12],@x[12],0
+	vori.b			@y[13],@x[13],0
+	vori.b			@y[14],@x[14],0
+	vori.b			@y[15],@x[15],0
+
+	# Update states in \@x[*] for 20 rounds
+	ori				$t8,$zero,10
+	b				.Loop_4x
+
+.align 5
+.Loop_4x:
+EOF
+
+&ROUND_4x (0, 4, 8, 12);
+&ROUND_4x (0, 5, 10, 15);
+
+$code .= <<EOF;
+	addi.w		$t8,$t8,-1
+	bnez		$t8,.Loop_4x
+
+	# Get the final states by adding the initial states
+	vadd.w		@x[0],@x[0],@y[0]
+	vadd.w		@x[1],@x[1],@y[1]
+	vadd.w		@x[2],@x[2],@y[2]
+	vadd.w		@x[3],@x[3],@y[3]
+	vadd.w		@x[4],@x[4],@y[4]
+	vadd.w		@x[5],@x[5],@y[5]
+	vadd.w		@x[6],@x[6],@y[6]
+	vadd.w		@x[7],@x[7],@y[7]
+	vadd.w		@x[8],@x[8],@y[8]
+	vadd.w		@x[9],@x[9],@y[9]
+	vadd.w		@x[10],@x[10],@y[10]
+	vadd.w		@x[11],@x[11],@y[11]
+	vadd.w		@x[12],@x[12],@y[12]
+	vadd.w		@x[13],@x[13],@y[13]
+	vadd.w		@x[14],@x[14],@y[14]
+	vadd.w		@x[15],@x[15],@y[15]
+
+	# Get the transpose of \@x[*] and save them in \@x[*]
+	vilvl.w		@y[0],@x[1],@x[0]
+	vilvh.w		@y[1],@x[1],@x[0]
+	vilvl.w		@y[2],@x[3],@x[2]
+	vilvh.w		@y[3],@x[3],@x[2]
+	vilvl.w		@y[4],@x[5],@x[4]
+	vilvh.w		@y[5],@x[5],@x[4]
+	vilvl.w		@y[6],@x[7],@x[6]
+	vilvh.w		@y[7],@x[7],@x[6]
+	vilvl.w		@y[8],@x[9],@x[8]
+	vilvh.w		@y[9],@x[9],@x[8]
+	vilvl.w		@y[10],@x[11],@x[10]
+	vilvh.w		@y[11],@x[11],@x[10]
+	vilvl.w		@y[12],@x[13],@x[12]
+	vilvh.w		@y[13],@x[13],@x[12]
+	vilvl.w		@y[14],@x[15],@x[14]
+	vilvh.w		@y[15],@x[15],@x[14]
+
+	vilvl.d		@x[0],@y[2],@y[0]
+	vilvh.d		@x[1],@y[2],@y[0]
+	vilvl.d		@x[2],@y[3],@y[1]
+	vilvh.d		@x[3],@y[3],@y[1]
+	vilvl.d		@x[4],@y[6],@y[4]
+	vilvh.d		@x[5],@y[6],@y[4]
+	vilvl.d		@x[6],@y[7],@y[5]
+	vilvh.d		@x[7],@y[7],@y[5]
+	vilvl.d		@x[8],@y[10],@y[8]
+	vilvh.d		@x[9],@y[10],@y[8]
+	vilvl.d		@x[10],@y[11],@y[9]
+	vilvh.d		@x[11],@y[11],@y[9]
+	vilvl.d		@x[12],@y[14],@y[12]
+	vilvh.d		@x[13],@y[14],@y[12]
+	vilvl.d		@x[14],@y[15],@y[13]
+	vilvh.d		@x[15],@y[15],@y[13]
+EOF
+
+# Adjust the order of elements in @x[*] for ease of use.
+@x = (@x[0],@x[4],@x[8],@x[12],@x[1],@x[5],@x[9],@x[13],
+      @x[2],@x[6],@x[10],@x[14],@x[3],@x[7],@x[11],@x[15]);
+
+$code .= <<EOF;
+	ori			$t8,$zero,64*4
+	bltu		$len,$t8,.Ltail_4x
+
+	# Get the encrypted message by xor states with plaintext
+	vld			@y[0],$inp,16*0
+	vld			@y[1],$inp,16*1
+	vld			@y[2],$inp,16*2
+	vld			@y[3],$inp,16*3
+	vxor.v		@y[0],@y[0],@x[0]
+	vxor.v		@y[1],@y[1],@x[1]
+	vxor.v		@y[2],@y[2],@x[2]
+	vxor.v		@y[3],@y[3],@x[3]
+	vst			@y[0],$out,16*0
+	vst			@y[1],$out,16*1
+	vst			@y[2],$out,16*2
+	vst			@y[3],$out,16*3
+
+	vld			@y[0],$inp,16*4
+	vld			@y[1],$inp,16*5
+	vld			@y[2],$inp,16*6
+	vld			@y[3],$inp,16*7
+	vxor.v		@y[0],@y[0],@x[4]
+	vxor.v		@y[1],@y[1],@x[5]
+	vxor.v		@y[2],@y[2],@x[6]
+	vxor.v		@y[3],@y[3],@x[7]
+	vst			@y[0],$out,16*4
+	vst			@y[1],$out,16*5
+	vst			@y[2],$out,16*6
+	vst			@y[3],$out,16*7
+
+	vld			@y[0],$inp,16*8
+	vld			@y[1],$inp,16*9
+	vld			@y[2],$inp,16*10
+	vld			@y[3],$inp,16*11
+	vxor.v		@y[0],@y[0],@x[8]
+	vxor.v		@y[1],@y[1],@x[9]
+	vxor.v		@y[2],@y[2],@x[10]
+	vxor.v		@y[3],@y[3],@x[11]
+	vst			@y[0],$out,16*8
+	vst			@y[1],$out,16*9
+	vst			@y[2],$out,16*10
+	vst			@y[3],$out,16*11
+
+	vld			@y[0],$inp,16*12
+	vld			@y[1],$inp,16*13
+	vld			@y[2],$inp,16*14
+	vld			@y[3],$inp,16*15
+	vxor.v		@y[0],@y[0],@x[12]
+	vxor.v		@y[1],@y[1],@x[13]
+	vxor.v		@y[2],@y[2],@x[14]
+	vxor.v		@y[3],@y[3],@x[15]
+	vst			@y[0],$out,16*12
+	vst			@y[1],$out,16*13
+	vst			@y[2],$out,16*14
+	vst			@y[3],$out,16*15
+
+	addi.d		$len,$len,-64*4
+	beqz		$len,.Ldone_4x
+	addi.d		$inp,$inp,64*4
+	addi.d		$out,$out,64*4
+	addi.w		$t4,$t4,4
+	b			.Loop_outer_4x
+
+.Ltail_4x:
+	# Handle the tail for 4x (1 <= tail_len <= 255)
+	ori			$t8,$zero,192
+	bgeu		$len,$t8,.L192_or_more4x
+	ori			$t8,$zero,128
+	bgeu		$len,$t8,.L128_or_more4x
+	ori			$t8,$zero,64
+	bgeu		$len,$t8,.L64_or_more4x
+
+	vst			@x[0],$sp,16*0
+	vst			@x[1],$sp,16*1
+	vst			@x[2],$sp,16*2
+	vst			@x[3],$sp,16*3
+	move		$t8,$zero
+	b			.Loop_tail_4x
+
+.align 5
+.L64_or_more4x:
+	vld			@y[0],$inp,16*0
+	vld			@y[1],$inp,16*1
+	vld			@y[2],$inp,16*2
+	vld			@y[3],$inp,16*3
+	vxor.v		@y[0],@y[0],@x[0]
+	vxor.v		@y[1],@y[1],@x[1]
+	vxor.v		@y[2],@y[2],@x[2]
+	vxor.v		@y[3],@y[3],@x[3]
+	vst			@y[0],$out,16*0
+	vst			@y[1],$out,16*1
+	vst			@y[2],$out,16*2
+	vst			@y[3],$out,16*3
+
+	addi.d		$len,$len,-64
+	beqz		$len,.Ldone_4x
+	addi.d		$inp,$inp,64
+	addi.d		$out,$out,64
+	vst			@x[4],$sp,16*0
+	vst			@x[5],$sp,16*1
+	vst			@x[6],$sp,16*2
+	vst			@x[7],$sp,16*3
+	move		$t8,$zero
+	b			.Loop_tail_4x
+
+.align 5
+.L128_or_more4x:
+	vld			@y[0],$inp,16*0
+	vld			@y[1],$inp,16*1
+	vld			@y[2],$inp,16*2
+	vld			@y[3],$inp,16*3
+	vxor.v		@y[0],@y[0],@x[0]
+	vxor.v		@y[1],@y[1],@x[1]
+	vxor.v		@y[2],@y[2],@x[2]
+	vxor.v		@y[3],@y[3],@x[3]
+	vst			@y[0],$out,16*0
+	vst			@y[1],$out,16*1
+	vst			@y[2],$out,16*2
+	vst			@y[3],$out,16*3
+
+	vld			@y[0],$inp,16*4
+	vld			@y[1],$inp,16*5
+	vld			@y[2],$inp,16*6
+	vld			@y[3],$inp,16*7
+	vxor.v		@y[0],@y[0],@x[4]
+	vxor.v		@y[1],@y[1],@x[5]
+	vxor.v		@y[2],@y[2],@x[6]
+	vxor.v		@y[3],@y[3],@x[7]
+	vst			@y[0],$out,16*4
+	vst			@y[1],$out,16*5
+	vst			@y[2],$out,16*6
+	vst			@y[3],$out,16*7
+
+	addi.d		$len,$len,-128
+	beqz		$len,.Ldone_4x
+	addi.d		$inp,$inp,128
+	addi.d		$out,$out,128
+	vst			@x[8],$sp,16*0
+	vst			@x[9],$sp,16*1
+	vst			@x[10],$sp,16*2
+	vst			@x[11],$sp,16*3
+	move		$t8,$zero
+	b			.Loop_tail_4x
+
+.align 5
+.L192_or_more4x:
+	vld			@y[0],$inp,16*0
+	vld			@y[1],$inp,16*1
+	vld			@y[2],$inp,16*2
+	vld			@y[3],$inp,16*3
+	vxor.v		@y[0],@y[0],@x[0]
+	vxor.v		@y[1],@y[1],@x[1]
+	vxor.v		@y[2],@y[2],@x[2]
+	vxor.v		@y[3],@y[3],@x[3]
+	vst			@y[0],$out,16*0
+	vst			@y[1],$out,16*1
+	vst			@y[2],$out,16*2
+	vst			@y[3],$out,16*3
+
+	vld			@y[0],$inp,16*4
+	vld			@y[1],$inp,16*5
+	vld			@y[2],$inp,16*6
+	vld			@y[3],$inp,16*7
+	vxor.v		@y[0],@y[0],@x[4]
+	vxor.v		@y[1],@y[1],@x[5]
+	vxor.v		@y[2],@y[2],@x[6]
+	vxor.v		@y[3],@y[3],@x[7]
+	vst			@y[0],$out,16*4
+	vst			@y[1],$out,16*5
+	vst			@y[2],$out,16*6
+	vst			@y[3],$out,16*7
+
+	vld			@y[0],$inp,16*8
+	vld			@y[1],$inp,16*9
+	vld			@y[2],$inp,16*10
+	vld			@y[3],$inp,16*11
+	vxor.v		@y[0],@y[0],@x[8]
+	vxor.v		@y[1],@y[1],@x[9]
+	vxor.v		@y[2],@y[2],@x[10]
+	vxor.v		@y[3],@y[3],@x[11]
+	vst			@y[0],$out,16*8
+	vst			@y[1],$out,16*9
+	vst			@y[2],$out,16*10
+	vst			@y[3],$out,16*11
+
+	addi.d		$len,$len,-192
+	beqz		$len,.Ldone_4x
+	addi.d		$inp,$inp,192
+	addi.d		$out,$out,192
+	vst			@x[12],$sp,16*0
+	vst			@x[13],$sp,16*1
+	vst			@x[14],$sp,16*2
+	vst			@x[15],$sp,16*3
+	move		$t8,$zero
+	b			.Loop_tail_4x
+
+.Loop_tail_4x:
+	# Xor input with states byte by byte
+	ldx.bu		$t5,$inp,$t8
+	ldx.bu		$t6,$sp,$t8
+	xor			$t5,$t5,$t6
+	stx.b		$t5,$out,$t8
+	addi.w		$t8,$t8,1
+	addi.d		$len,$len,-1
+	bnez		$len,.Loop_tail_4x
+	b			.Ldone_4x
+
+.Ldone_4x:
+	addi.d		$sp,$sp,128
+	b			.Lend
+
+EOF
+}
+
+########################################################################
+# 256-bit LASX code path that handles all lengths.
+{
+# Load the initial states in array @x[*] and update directly.
+my @x = ($xr0, $xr1, $xr2, $xr3, $xr4, $xr5, $xr6, $xr7,
+         $xr8, $xr9, $xr10, $xr11, $xr12, $xr13, $xr14, $xr15);
+
+# Save the initial states in array @y[*]
+my @y = ($xr16, $xr17, $xr18, $xr19, $xr20, $xr21, $xr22, $xr23,
+         $xr24, $xr25, $xr26, $xr27, $xr28, $xr29, $xr30, $xr31);
+
+sub ROUND_8x {
+	my ($a0,$b0,$c0,$d0) = @_;
+	my ($a1,$b1,$c1,$d1) = map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
+	my ($a2,$b2,$c2,$d2) = map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
+	my ($a3,$b3,$c3,$d3) = map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
+
+$code .= <<EOF;
+	xvadd.w		@x[$a0],@x[$a0],@x[$b0]
+	xvxor.v		@x[$d0],@x[$d0],@x[$a0]
+	xvrotri.w	@x[$d0],@x[$d0],16      # rotate left 16 bits
+	xvadd.w		@x[$a1],@x[$a1],@x[$b1]
+	xvxor.v		@x[$d1],@x[$d1],@x[$a1]
+	xvrotri.w	@x[$d1],@x[$d1],16
+
+	xvadd.w		@x[$c0],@x[$c0],@x[$d0]
+	xvxor.v		@x[$b0],@x[$b0],@x[$c0]
+	xvrotri.w	@x[$b0],@x[$b0],20      # rotate left 12 bits
+	xvadd.w		@x[$c1],@x[$c1],@x[$d1]
+	xvxor.v		@x[$b1],@x[$b1],@x[$c1]
+	xvrotri.w	@x[$b1],@x[$b1],20
+
+	xvadd.w		@x[$a0],@x[$a0],@x[$b0]
+	xvxor.v		@x[$d0],@x[$d0],@x[$a0]
+	xvrotri.w	@x[$d0],@x[$d0],24      # rotate left 8 bits
+	xvadd.w		@x[$a1],@x[$a1],@x[$b1]
+	xvxor.v		@x[$d1],@x[$d1],@x[$a1]
+	xvrotri.w	@x[$d1],@x[$d1],24
+
+	xvadd.w		@x[$c0],@x[$c0],@x[$d0]
+	xvxor.v		@x[$b0],@x[$b0],@x[$c0]
+	xvrotri.w	@x[$b0],@x[$b0],25      # rotate left 7 bits
+	xvadd.w		@x[$c1],@x[$c1],@x[$d1]
+	xvxor.v		@x[$b1],@x[$b1],@x[$c1]
+	xvrotri.w	@x[$b1],@x[$b1],25
+
+	xvadd.w		@x[$a2],@x[$a2],@x[$b2]
+	xvxor.v		@x[$d2],@x[$d2],@x[$a2]
+	xvrotri.w	@x[$d2],@x[$d2],16
+	xvadd.w		@x[$a3],@x[$a3],@x[$b3]
+	xvxor.v		@x[$d3],@x[$d3],@x[$a3]
+	xvrotri.w	@x[$d3],@x[$d3],16
+
+	xvadd.w		@x[$c2],@x[$c2],@x[$d2]
+	xvxor.v		@x[$b2],@x[$b2],@x[$c2]
+	xvrotri.w	@x[$b2],@x[$b2],20
+	xvadd.w		@x[$c3],@x[$c3],@x[$d3]
+	xvxor.v		@x[$b3],@x[$b3],@x[$c3]
+	xvrotri.w	@x[$b3],@x[$b3],20
+
+	xvadd.w		@x[$a2],@x[$a2],@x[$b2]
+	xvxor.v		@x[$d2],@x[$d2],@x[$a2]
+	xvrotri.w	@x[$d2],@x[$d2],24
+	xvadd.w		@x[$a3],@x[$a3],@x[$b3]
+	xvxor.v		@x[$d3],@x[$d3],@x[$a3]
+	xvrotri.w	@x[$d3],@x[$d3],24
+
+	xvadd.w		@x[$c2],@x[$c2],@x[$d2]
+	xvxor.v		@x[$b2],@x[$b2],@x[$c2]
+	xvrotri.w	@x[$b2],@x[$b2],25
+	xvadd.w		@x[$c3],@x[$c3],@x[$d3]
+	xvxor.v		@x[$b3],@x[$b3],@x[$c3]
+	xvrotri.w	@x[$b3],@x[$b3],25
+
+EOF
+}
+
+$code .= <<EOF;
+.align 6
+.LChaCha20_8x:
+	ori			$t3,$zero,64
+	bleu		$len,$t3,.LChaCha20_1x  # goto 1x when len <= 64
+
+	addi.d		$sp,$sp,-128
+
+	# Save the initial block counter in $t4
+	ld.w		$t4,$counter,0
+	b			.Loop_outer_8x
+
+.align 5
+.Loop_outer_8x:
+	# Load constant
+	la.local		$t8,.Lsigma
+	xvldrepl.w		@x[0],$t8,4*0		# 'expa'
+	xvldrepl.w		@x[1],$t8,4*1		# 'nd 3'
+	xvldrepl.w		@x[2],$t8,4*2		# '2-by'
+	xvldrepl.w		@x[3],$t8,4*3		# 'te k'
+
+	# Load key
+	xvldrepl.w		@x[4],$key,4*0
+	xvldrepl.w		@x[5],$key,4*1
+	xvldrepl.w		@x[6],$key,4*2
+	xvldrepl.w		@x[7],$key,4*3
+	xvldrepl.w		@x[8],$key,4*4
+	xvldrepl.w		@x[9],$key,4*5
+	xvldrepl.w		@x[10],$key,4*6
+	xvldrepl.w		@x[11],$key,4*7
+
+	# Load block counter
+	xvreplgr2vr.w	@x[12],$t4
+
+	# Load nonce
+	xvldrepl.w		@x[13],$counter,4*1
+	xvldrepl.w		@x[14],$counter,4*2
+	xvldrepl.w		@x[15],$counter,4*3
+
+	# Get the correct block counter for each block
+	la.local		$t8,.Linc8x
+	xvld			@y[0],$t8,0
+	xvadd.w			@x[12],@x[12],@y[0]
+
+	# Copy the initial states from \@x[*] to \@y[*]
+	xvori.b			@y[0],@x[0],0
+	xvori.b			@y[1],@x[1],0
+	xvori.b			@y[2],@x[2],0
+	xvori.b			@y[3],@x[3],0
+	xvori.b			@y[4],@x[4],0
+	xvori.b			@y[5],@x[5],0
+	xvori.b			@y[6],@x[6],0
+	xvori.b			@y[7],@x[7],0
+	xvori.b			@y[8],@x[8],0
+	xvori.b			@y[9],@x[9],0
+	xvori.b			@y[10],@x[10],0
+	xvori.b			@y[11],@x[11],0
+	xvori.b			@y[12],@x[12],0
+	xvori.b			@y[13],@x[13],0
+	xvori.b			@y[14],@x[14],0
+	xvori.b			@y[15],@x[15],0
+
+	# Update states in \@x[*] for 20 rounds
+	ori				$t8,$zero,10
+	b				.Loop_8x
+
+.align 5
+.Loop_8x:
+EOF
+
+&ROUND_8x (0, 4, 8, 12);
+&ROUND_8x (0, 5, 10, 15);
+
+$code .= <<EOF;
+	addi.w		$t8,$t8,-1
+	bnez		$t8,.Loop_8x
+
+	# Get the final states by adding the initial states
+	xvadd.w		@x[0],@x[0],@y[0]
+	xvadd.w		@x[1],@x[1],@y[1]
+	xvadd.w		@x[2],@x[2],@y[2]
+	xvadd.w		@x[3],@x[3],@y[3]
+	xvadd.w		@x[4],@x[4],@y[4]
+	xvadd.w		@x[5],@x[5],@y[5]
+	xvadd.w		@x[6],@x[6],@y[6]
+	xvadd.w		@x[7],@x[7],@y[7]
+	xvadd.w		@x[8],@x[8],@y[8]
+	xvadd.w		@x[9],@x[9],@y[9]
+	xvadd.w		@x[10],@x[10],@y[10]
+	xvadd.w		@x[11],@x[11],@y[11]
+	xvadd.w		@x[12],@x[12],@y[12]
+	xvadd.w		@x[13],@x[13],@y[13]
+	xvadd.w		@x[14],@x[14],@y[14]
+	xvadd.w		@x[15],@x[15],@y[15]
+
+	# Get the transpose of \@x[*] and save them in \@y[*]
+	xvilvl.w	@y[0],@x[1],@x[0]
+	xvilvh.w	@y[1],@x[1],@x[0]
+	xvilvl.w	@y[2],@x[3],@x[2]
+	xvilvh.w	@y[3],@x[3],@x[2]
+	xvilvl.w	@y[4],@x[5],@x[4]
+	xvilvh.w	@y[5],@x[5],@x[4]
+	xvilvl.w	@y[6],@x[7],@x[6]
+	xvilvh.w	@y[7],@x[7],@x[6]
+	xvilvl.w	@y[8],@x[9],@x[8]
+	xvilvh.w	@y[9],@x[9],@x[8]
+	xvilvl.w	@y[10],@x[11],@x[10]
+	xvilvh.w	@y[11],@x[11],@x[10]
+	xvilvl.w	@y[12],@x[13],@x[12]
+	xvilvh.w	@y[13],@x[13],@x[12]
+	xvilvl.w	@y[14],@x[15],@x[14]
+	xvilvh.w	@y[15],@x[15],@x[14]
+
+	xvilvl.d	@x[0],@y[2],@y[0]
+	xvilvh.d	@x[1],@y[2],@y[0]
+	xvilvl.d	@x[2],@y[3],@y[1]
+	xvilvh.d	@x[3],@y[3],@y[1]
+	xvilvl.d	@x[4],@y[6],@y[4]
+	xvilvh.d	@x[5],@y[6],@y[4]
+	xvilvl.d	@x[6],@y[7],@y[5]
+	xvilvh.d	@x[7],@y[7],@y[5]
+	xvilvl.d	@x[8],@y[10],@y[8]
+	xvilvh.d	@x[9],@y[10],@y[8]
+	xvilvl.d	@x[10],@y[11],@y[9]
+	xvilvh.d	@x[11],@y[11],@y[9]
+	xvilvl.d	@x[12],@y[14],@y[12]
+	xvilvh.d	@x[13],@y[14],@y[12]
+	xvilvl.d	@x[14],@y[15],@y[13]
+	xvilvh.d	@x[15],@y[15],@y[13]
+
+	xvori.b		@y[0],@x[4],0
+	xvpermi.q	@y[0],@x[0],0x20
+	xvori.b		@y[1],@x[5],0
+	xvpermi.q	@y[1],@x[1],0x20
+	xvori.b		@y[2],@x[6],0
+	xvpermi.q	@y[2],@x[2],0x20
+	xvori.b		@y[3],@x[7],0
+	xvpermi.q	@y[3],@x[3],0x20
+	xvori.b		@y[4],@x[4],0
+	xvpermi.q	@y[4],@x[0],0x31
+	xvori.b		@y[5],@x[5],0
+	xvpermi.q	@y[5],@x[1],0x31
+	xvori.b		@y[6],@x[6],0
+	xvpermi.q	@y[6],@x[2],0x31
+	xvori.b		@y[7],@x[7],0
+	xvpermi.q	@y[7],@x[3],0x31
+	xvori.b		@y[8],@x[12],0
+	xvpermi.q	@y[8],@x[8],0x20
+	xvori.b		@y[9],@x[13],0
+	xvpermi.q	@y[9],@x[9],0x20
+	xvori.b		@y[10],@x[14],0
+	xvpermi.q	@y[10],@x[10],0x20
+	xvori.b		@y[11],@x[15],0
+	xvpermi.q	@y[11],@x[11],0x20
+	xvori.b		@y[12],@x[12],0
+	xvpermi.q	@y[12],@x[8],0x31
+	xvori.b		@y[13],@x[13],0
+	xvpermi.q	@y[13],@x[9],0x31
+	xvori.b		@y[14],@x[14],0
+	xvpermi.q	@y[14],@x[10],0x31
+	xvori.b		@y[15],@x[15],0
+	xvpermi.q	@y[15],@x[11],0x31
+
+EOF
+
+# Adjust the order of elements in @y[*] for ease of use.
+@y = (@y[0],@y[8],@y[1],@y[9],@y[2],@y[10],@y[3],@y[11],
+      @y[4],@y[12],@y[5],@y[13],@y[6],@y[14],@y[7],@y[15]);
+
+$code .= <<EOF;
+	ori			$t8,$zero,64*8
+	bltu		$len,$t8,.Ltail_8x
+
+	# Get the encrypted message by xor states with plaintext
+	xvld		@x[0],$inp,32*0
+	xvld		@x[1],$inp,32*1
+	xvld		@x[2],$inp,32*2
+	xvld		@x[3],$inp,32*3
+	xvxor.v		@x[0],@x[0],@y[0]
+	xvxor.v		@x[1],@x[1],@y[1]
+	xvxor.v		@x[2],@x[2],@y[2]
+	xvxor.v		@x[3],@x[3],@y[3]
+	xvst		@x[0],$out,32*0
+	xvst		@x[1],$out,32*1
+	xvst		@x[2],$out,32*2
+	xvst		@x[3],$out,32*3
+
+	xvld		@x[0],$inp,32*4
+	xvld		@x[1],$inp,32*5
+	xvld		@x[2],$inp,32*6
+	xvld		@x[3],$inp,32*7
+	xvxor.v		@x[0],@x[0],@y[4]
+	xvxor.v		@x[1],@x[1],@y[5]
+	xvxor.v		@x[2],@x[2],@y[6]
+	xvxor.v		@x[3],@x[3],@y[7]
+	xvst		@x[0],$out,32*4
+	xvst		@x[1],$out,32*5
+	xvst		@x[2],$out,32*6
+	xvst		@x[3],$out,32*7
+
+	xvld		@x[0],$inp,32*8
+	xvld		@x[1],$inp,32*9
+	xvld		@x[2],$inp,32*10
+	xvld		@x[3],$inp,32*11
+	xvxor.v		@x[0],@x[0],@y[8]
+	xvxor.v		@x[1],@x[1],@y[9]
+	xvxor.v		@x[2],@x[2],@y[10]
+	xvxor.v		@x[3],@x[3],@y[11]
+	xvst		@x[0],$out,32*8
+	xvst		@x[1],$out,32*9
+	xvst		@x[2],$out,32*10
+	xvst		@x[3],$out,32*11
+
+	xvld		@x[0],$inp,32*12
+	xvld		@x[1],$inp,32*13
+	xvld		@x[2],$inp,32*14
+	xvld		@x[3],$inp,32*15
+	xvxor.v		@x[0],@x[0],@y[12]
+	xvxor.v		@x[1],@x[1],@y[13]
+	xvxor.v		@x[2],@x[2],@y[14]
+	xvxor.v		@x[3],@x[3],@y[15]
+	xvst		@x[0],$out,32*12
+	xvst		@x[1],$out,32*13
+	xvst		@x[2],$out,32*14
+	xvst		@x[3],$out,32*15
+
+	addi.d		$len,$len,-64*8
+	beqz		$len,.Ldone_8x
+	addi.d		$inp,$inp,64*8
+	addi.d		$out,$out,64*8
+	addi.w		$t4,$t4,8
+	b			.Loop_outer_8x
+
+.Ltail_8x:
+	# Handle the tail for 8x (1 <= tail_len <= 511)
+	ori			$t8,$zero,448
+	bgeu		$len,$t8,.L448_or_more8x
+	ori			$t8,$zero,384
+	bgeu		$len,$t8,.L384_or_more8x
+	ori			$t8,$zero,320
+	bgeu		$len,$t8,.L320_or_more8x
+	ori			$t8,$zero,256
+	bgeu		$len,$t8,.L256_or_more8x
+	ori			$t8,$zero,192
+	bgeu		$len,$t8,.L192_or_more8x
+	ori			$t8,$zero,128
+	bgeu		$len,$t8,.L128_or_more8x
+	ori			$t8,$zero,64
+	bgeu		$len,$t8,.L64_or_more8x
+
+	xvst		@y[0],$sp,32*0
+	xvst		@y[1],$sp,32*1
+	move		$t8,$zero
+	b			.Loop_tail_8x
+
+.align 5
+.L64_or_more8x:
+	xvld		@x[0],$inp,32*0
+	xvld		@x[1],$inp,32*1
+	xvxor.v		@x[0],@x[0],@y[0]
+	xvxor.v		@x[1],@x[1],@y[1]
+	xvst		@x[0],$out,32*0
+	xvst		@x[1],$out,32*1
+
+	addi.d		$len,$len,-64
+	beqz		$len,.Ldone_8x
+	addi.d		$inp,$inp,64
+	addi.d		$out,$out,64
+	xvst		@y[2],$sp,32*0
+	xvst		@y[3],$sp,32*1
+	move		$t8,$zero
+	b			.Loop_tail_8x
+
+.align 5
+.L128_or_more8x:
+	xvld		@x[0],$inp,32*0
+	xvld		@x[1],$inp,32*1
+	xvld		@x[2],$inp,32*2
+	xvld		@x[3],$inp,32*3
+	xvxor.v		@x[0],@x[0],@y[0]
+	xvxor.v		@x[1],@x[1],@y[1]
+	xvxor.v		@x[2],@x[2],@y[2]
+	xvxor.v		@x[3],@x[3],@y[3]
+	xvst		@x[0],$out,32*0
+	xvst		@x[1],$out,32*1
+	xvst		@x[2],$out,32*2
+	xvst		@x[3],$out,32*3
+
+	addi.d		$len,$len,-128
+	beqz		$len,.Ldone_8x
+	addi.d		$inp,$inp,128
+	addi.d		$out,$out,128
+	xvst		@y[4],$sp,32*0
+	xvst		@y[5],$sp,32*1
+	move		$t8,$zero
+	b			.Loop_tail_8x
+
+.align 5
+.L192_or_more8x:
+	xvld		@x[0],$inp,32*0
+	xvld		@x[1],$inp,32*1
+	xvld		@x[2],$inp,32*2
+	xvld		@x[3],$inp,32*3
+	xvxor.v		@x[0],@x[0],@y[0]
+	xvxor.v		@x[1],@x[1],@y[1]
+	xvxor.v		@x[2],@x[2],@y[2]
+	xvxor.v		@x[3],@x[3],@y[3]
+	xvst		@x[0],$out,32*0
+	xvst		@x[1],$out,32*1
+	xvst		@x[2],$out,32*2
+	xvst		@x[3],$out,32*3
+
+	xvld		@x[0],$inp,32*4
+	xvld		@x[1],$inp,32*5
+	xvxor.v		@x[0],@x[0],@y[4]
+	xvxor.v		@x[1],@x[1],@y[5]
+	xvst		@x[0],$out,32*4
+	xvst		@x[1],$out,32*5
+
+	addi.d		$len,$len,-192
+	beqz		$len,.Ldone_8x
+	addi.d		$inp,$inp,192
+	addi.d		$out,$out,192
+	xvst		@y[6],$sp,32*0
+	xvst		@y[7],$sp,32*1
+	move		$t8,$zero
+	b			.Loop_tail_8x
+
+.align 5
+.L256_or_more8x:
+	xvld		@x[0],$inp,32*0
+	xvld		@x[1],$inp,32*1
+	xvld		@x[2],$inp,32*2
+	xvld		@x[3],$inp,32*3
+	xvxor.v		@x[0],@x[0],@y[0]
+	xvxor.v		@x[1],@x[1],@y[1]
+	xvxor.v		@x[2],@x[2],@y[2]
+	xvxor.v		@x[3],@x[3],@y[3]
+	xvst		@x[0],$out,32*0
+	xvst		@x[1],$out,32*1
+	xvst		@x[2],$out,32*2
+	xvst		@x[3],$out,32*3
+
+	xvld		@x[0],$inp,32*4
+	xvld		@x[1],$inp,32*5
+	xvld		@x[2],$inp,32*6
+	xvld		@x[3],$inp,32*7
+	xvxor.v		@x[0],@x[0],@y[4]
+	xvxor.v		@x[1],@x[1],@y[5]
+	xvxor.v		@x[2],@x[2],@y[6]
+	xvxor.v		@x[3],@x[3],@y[7]
+	xvst		@x[0],$out,32*4
+	xvst		@x[1],$out,32*5
+	xvst		@x[2],$out,32*6
+	xvst		@x[3],$out,32*7
+
+	addi.d		$len,$len,-256
+	beqz		$len,.Ldone_8x
+	addi.d		$inp,$inp,256
+	addi.d		$out,$out,256
+	xvst		@y[8],$sp,32*0
+	xvst		@y[9],$sp,32*1
+	move		$t8,$zero
+	b			.Loop_tail_8x
+
+.align 5
+.L320_or_more8x:
+	xvld		@x[0],$inp,32*0
+	xvld		@x[1],$inp,32*1
+	xvld		@x[2],$inp,32*2
+	xvld		@x[3],$inp,32*3
+	xvxor.v		@x[0],@x[0],@y[0]
+	xvxor.v		@x[1],@x[1],@y[1]
+	xvxor.v		@x[2],@x[2],@y[2]
+	xvxor.v		@x[3],@x[3],@y[3]
+	xvst		@x[0],$out,32*0
+	xvst		@x[1],$out,32*1
+	xvst		@x[2],$out,32*2
+	xvst		@x[3],$out,32*3
+
+	xvld		@x[0],$inp,32*4
+	xvld		@x[1],$inp,32*5
+	xvld		@x[2],$inp,32*6
+	xvld		@x[3],$inp,32*7
+	xvxor.v		@x[0],@x[0],@y[4]
+	xvxor.v		@x[1],@x[1],@y[5]
+	xvxor.v		@x[2],@x[2],@y[6]
+	xvxor.v		@x[3],@x[3],@y[7]
+	xvst		@x[0],$out,32*4
+	xvst		@x[1],$out,32*5
+	xvst		@x[2],$out,32*6
+	xvst		@x[3],$out,32*7
+
+	xvld		@x[0],$inp,32*8
+	xvld		@x[1],$inp,32*9
+	xvxor.v		@x[0],@x[0],@y[8]
+	xvxor.v		@x[1],@x[1],@y[9]
+	xvst		@x[0],$out,32*8
+	xvst		@x[1],$out,32*9
+
+	addi.d		$len,$len,-320
+	beqz		$len,.Ldone_8x
+	addi.d		$inp,$inp,320
+	addi.d		$out,$out,320
+	xvst		@y[10],$sp,32*0
+	xvst		@y[11],$sp,32*1
+	move		$t8,$zero
+	b			.Loop_tail_8x
+
+.align 5
+.L384_or_more8x:
+	xvld		@x[0],$inp,32*0
+	xvld		@x[1],$inp,32*1
+	xvld		@x[2],$inp,32*2
+	xvld		@x[3],$inp,32*3
+	xvxor.v		@x[0],@x[0],@y[0]
+	xvxor.v		@x[1],@x[1],@y[1]
+	xvxor.v		@x[2],@x[2],@y[2]
+	xvxor.v		@x[3],@x[3],@y[3]
+	xvst		@x[0],$out,32*0
+	xvst		@x[1],$out,32*1
+	xvst		@x[2],$out,32*2
+	xvst		@x[3],$out,32*3
+
+	xvld		@x[0],$inp,32*4
+	xvld		@x[1],$inp,32*5
+	xvld		@x[2],$inp,32*6
+	xvld		@x[3],$inp,32*7
+	xvxor.v		@x[0],@x[0],@y[4]
+	xvxor.v		@x[1],@x[1],@y[5]
+	xvxor.v		@x[2],@x[2],@y[6]
+	xvxor.v		@x[3],@x[3],@y[7]
+	xvst		@x[0],$out,32*4
+	xvst		@x[1],$out,32*5
+	xvst		@x[2],$out,32*6
+	xvst		@x[3],$out,32*7
+
+	xvld		@x[0],$inp,32*8
+	xvld		@x[1],$inp,32*9
+	xvld		@x[2],$inp,32*10
+	xvld		@x[3],$inp,32*11
+	xvxor.v		@x[0],@x[0],@y[8]
+	xvxor.v		@x[1],@x[1],@y[9]
+	xvxor.v		@x[2],@x[2],@y[10]
+	xvxor.v		@x[3],@x[3],@y[11]
+	xvst		@x[0],$out,32*8
+	xvst		@x[1],$out,32*9
+	xvst		@x[2],$out,32*10
+	xvst		@x[3],$out,32*11
+
+	addi.d		$len,$len,-384
+	beqz		$len,.Ldone_8x
+	addi.d		$inp,$inp,384
+	addi.d		$out,$out,384
+	xvst		@y[12],$sp,32*0
+	xvst		@y[13],$sp,32*1
+	move		$t8,$zero
+	b			.Loop_tail_8x
+
+.align 5
+.L448_or_more8x:
+	xvld		@x[0],$inp,32*0
+	xvld		@x[1],$inp,32*1
+	xvld		@x[2],$inp,32*2
+	xvld		@x[3],$inp,32*3
+	xvxor.v		@x[0],@x[0],@y[0]
+	xvxor.v		@x[1],@x[1],@y[1]
+	xvxor.v		@x[2],@x[2],@y[2]
+	xvxor.v		@x[3],@x[3],@y[3]
+	xvst		@x[0],$out,32*0
+	xvst		@x[1],$out,32*1
+	xvst		@x[2],$out,32*2
+	xvst		@x[3],$out,32*3
+
+	xvld		@x[0],$inp,32*4
+	xvld		@x[1],$inp,32*5
+	xvld		@x[2],$inp,32*6
+	xvld		@x[3],$inp,32*7
+	xvxor.v		@x[0],@x[0],@y[4]
+	xvxor.v		@x[1],@x[1],@y[5]
+	xvxor.v		@x[2],@x[2],@y[6]
+	xvxor.v		@x[3],@x[3],@y[7]
+	xvst		@x[0],$out,32*4
+	xvst		@x[1],$out,32*5
+	xvst		@x[2],$out,32*6
+	xvst		@x[3],$out,32*7
+
+	xvld		@x[0],$inp,32*8
+	xvld		@x[1],$inp,32*9
+	xvld		@x[2],$inp,32*10
+	xvld		@x[3],$inp,32*11
+	xvxor.v		@x[0],@x[0],@y[8]
+	xvxor.v		@x[1],@x[1],@y[9]
+	xvxor.v		@x[2],@x[2],@y[10]
+	xvxor.v		@x[3],@x[3],@y[11]
+	xvst		@x[0],$out,32*8
+	xvst		@x[1],$out,32*9
+	xvst		@x[2],$out,32*10
+	xvst		@x[3],$out,32*11
+
+	xvld		@x[0],$inp,32*12
+	xvld		@x[1],$inp,32*13
+	xvxor.v		@x[0],@x[0],@y[12]
+	xvxor.v		@x[1],@x[1],@y[13]
+	xvst		@x[0],$out,32*12
+	xvst		@x[1],$out,32*13
+
+	addi.d		$len,$len,-448
+	beqz		$len,.Ldone_8x
+	addi.d		$inp,$inp,448
+	addi.d		$out,$out,448
+	xvst		@y[14],$sp,32*0
+	xvst		@y[15],$sp,32*1
+	move		$t8,$zero
+	b			.Loop_tail_8x
+
+.Loop_tail_8x:
+	# Xor input with states byte by byte
+	ldx.bu		$t5,$inp,$t8
+	ldx.bu		$t6,$sp,$t8
+	xor			$t5,$t5,$t6
+	stx.b		$t5,$out,$t8
+	addi.w		$t8,$t8,1
+	addi.d		$len,$len,-1
+	bnez		$len,.Loop_tail_8x
+	b			.Ldone_8x
+
+.Ldone_8x:
+	addi.d		$sp,$sp,128
+	b			.Lend
+
+EOF
+}
+
+$code .= <<EOF;
+.Lno_data:
+.Lend:
+	jr	$ra
+.size ChaCha20_ctr32,.-ChaCha20_ctr32
+EOF
+
+$code =~ s/\`([^\`]*)\`/eval($1)/gem;
+
+print $code;
+
+close STDOUT;
diff --git a/crypto/chacha/build.info b/crypto/chacha/build.info
index 1fa1a716f51..24593e6c130 100644
--- a/crypto/chacha/build.info
+++ b/crypto/chacha/build.info
@@ -12,6 +12,8 @@ IF[{- !$disabled{asm} -}]
   $CHACHAASM_armv4=chacha-armv4.S
   $CHACHAASM_aarch64=chacha-armv8.S chacha-armv8-sve.S
 
+  $CHACHAASM_loongarch64=chacha-loongarch64.S
+
   $CHACHAASM_ppc32=chacha_ppc.c chacha-ppc.s
   IF[{- $target{sys_id} ne "AIX" && $target{sys_id} ne "MACOSX" -}]
     $CHACHAASM_ppc32=chacha_ppc.c chacha-ppc.s chachap10-ppc.s
@@ -39,8 +41,10 @@ GENERATE[chacha-armv8.S]=asm/chacha-armv8.pl
 GENERATE[chacha-armv8-sve.S]=asm/chacha-armv8-sve.pl
 INCLUDE[chacha-armv8.o]=..
 INCLUDE[chacha-armv8-sve.o]=..
+INCLUDE[chacha-loongarch64.o]=..
 INCLUDE[chacha-s390x.o]=..
 GENERATE[chacha-c64xplus.S]=asm/chacha-c64xplus.pl
 GENERATE[chacha-s390x.S]=asm/chacha-s390x.pl
 GENERATE[chacha-ia64.S]=asm/chacha-ia64.pl
 GENERATE[chacha-ia64.s]=chacha-ia64.S
+GENERATE[chacha-loongarch64.S]=asm/chacha-loongarch64.pl