chacha: Move dispatching from assembly to Rust (merge BoringSSL b3cda5c)

briansmith · Jan 17, 2025 · 3f2a80f · 3f2a80f
2 parents 683e9bc + b3cda5c
commit 3f2a80f
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 39 deletions.
diff --git a/crypto/chacha/asm/chacha-armv4.pl b/crypto/chacha/asm/chacha-armv4.pl
@@ -196,41 +196,14 @@ sub ROUND {
 .long	0x61707865,0x3320646e,0x79622d32,0x6b206574	@ endian-neutral
 .Lone:
 .long	1,0,0,0
-#if __ARM_MAX_ARCH__>=7
-.extern OPENSSL_armcap_P
-.hidden OPENSSL_armcap_P
-.LOPENSSL_armcap:
-.word   OPENSSL_armcap_P-.Lsigma
-#else
-.word	-1
-#endif
 
-.globl	ChaCha20_ctr32
-.type	ChaCha20_ctr32,%function
+.globl	ChaCha20_ctr32_nohw
+.type	ChaCha20_ctr32_nohw,%function
 .align	5
-ChaCha20_ctr32:
-.LChaCha20_ctr32:
+ChaCha20_ctr32_nohw:
 	ldr	r12,[sp,#0]		@ pull pointer to counter and nonce
 	stmdb	sp!,{r0-r2,r4-r11,lr}
 	adr	r14,.Lsigma
-	cmp	r2,#0			@ len==0?
-#ifdef	__thumb2__
-	itt	eq
-#endif
-	addeq	sp,sp,#4*3
-	beq	.Lno_data
-#if __ARM_MAX_ARCH__>=7
-	cmp	r2,#192			@ test len
-	bls	.Lshort
-	ldr	r4,[r14,#32]
-	ldr	r4,[r14,r4]
-# ifdef	__APPLE__
-	ldr	r4,[r4]
-# endif
-	tst	r4,#ARMV7_NEON
-	bne	.LChaCha20_neon
-.Lshort:
-#endif
 	ldmia	r12,{r4-r7}		@ load counter and nonce
 	sub	sp,sp,#4*(16)		@ off-load area
 	stmdb	sp!,{r4-r7}		@ copy counter and nonce
@@ -623,9 +596,8 @@ sub ROUND {
 
 .Ldone:
 	add	sp,sp,#4*(32+3)
-.Lno_data:
 	ldmia	sp!,{r4-r11,pc}
-.size	ChaCha20_ctr32,.-ChaCha20_ctr32
+.size	ChaCha20_ctr32_nohw,.-ChaCha20_ctr32_nohw
 ___
 
 {{{
@@ -667,12 +639,12 @@ sub NEONROUND {
 .arch	armv7-a
 .fpu	neon
 
-.type	ChaCha20_neon,%function
+.globl	ChaCha20_ctr32_neon
+.type	ChaCha20_ctr32_neon,%function
 .align	5
-ChaCha20_neon:
+ChaCha20_ctr32_neon:
 	ldr		r12,[sp,#0]		@ pull pointer to counter and nonce
 	stmdb		sp!,{r0-r2,r4-r11,lr}
-.LChaCha20_neon:
 	adr		r14,.Lsigma
 	vstmdb		sp!,{d8-d15}		@ ABI spec says so
 	stmdb		sp!,{r0-r3}
@@ -1147,7 +1119,7 @@ sub NEONROUND {
 	vldmia		sp,{d8-d15}
 	add		sp,sp,#4*(16+3)
 	ldmia		sp!,{r4-r11,pc}
-.size	ChaCha20_neon,.-ChaCha20_neon
+.size	ChaCha20_ctr32_neon,.-ChaCha20_ctr32_neon
 #endif
 ___
 }}}

diff --git a/src/aead/chacha.rs b/src/aead/chacha.rs
@@ -101,9 +101,20 @@ impl Key {
                         self, counter, in_out, cpu)
                 }
             } else if #[cfg(all(target_arch = "arm", target_endian = "little"))] {
-                chacha20_ctr32_ffi!(
-                    unsafe { (0, cpu::Features, &mut [u8]) => ChaCha20_ctr32 },
-                    self, counter, in_out.copy_within(), cpu)
+                use cpu::{GetFeature as _, arm::Neon};
+                const NEON_MIN_LEN: usize = 192 + 1;
+                if in_out.len() >= NEON_MIN_LEN {
+                    if let Some(cpu) = cpu.get_feature() {
+                        return chacha20_ctr32_ffi!(
+                            unsafe { (NEON_MIN_LEN, Neon, &mut [u8]) => ChaCha20_ctr32_neon },
+                            self, counter, in_out.copy_within(), cpu);
+                    }
+                }
+                if in_out.len() >= 1 {
+                    chacha20_ctr32_ffi!(
+                        unsafe { (1, cpu::Features, &mut [u8]) => ChaCha20_ctr32_nohw },
+                        self, counter, in_out.copy_within(), cpu)
+                }
             } else if #[cfg(target_arch = "x86")] {
                 chacha20_ctr32_ffi!(
                     unsafe { (0, cpu::Features, &mut [u8]) => ChaCha20_ctr32 },