From 7e03acf2be20bf3f27147f8533ba5f863d7691da Mon Sep 17 00:00:00 2001 From: Andy Polyakov Date: Wed, 11 Jun 2014 00:08:03 +0200 Subject: [PATCH] Engage ARMv8 AES support [from HEAD]. --- Configure | 4 +- TABLE | 43 ++--------------- crypto/aes/Makefile | 4 ++ crypto/evp/e_aes.c | 113 +++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 122 insertions(+), 42 deletions(-) diff --git a/Configure b/Configure index e53c023743..4c063a0cbc 100755 --- a/Configure +++ b/Configure @@ -136,8 +136,8 @@ my $alpha_asm="alphacpuid.o:bn_asm.o alpha-mont.o:::::sha1-alpha.o:::::::ghash-a my $mips64_asm=":bn-mips.o mips-mont.o::aes_cbc.o aes-mips.o:::sha1-mips.o sha256-mips.o sha512-mips.o::::::::"; my $mips32_asm=$mips64_asm; $mips32_asm =~ s/\s*sha512\-mips\.o//; my $s390x_asm="s390xcap.o s390xcpuid.o:bn-s390x.o s390x-mont.o s390x-gf2m.o::aes-s390x.o aes-ctr.o aes-xts.o:::sha1-s390x.o sha256-s390x.o sha512-s390x.o::rc4-s390x.o:::::ghash-s390x.o:"; -my $armv4_asm="armcap.o armv4cpuid.o:bn_asm.o armv4-mont.o armv4-gf2m.o::aes_cbc.o aes-armv4.o bsaes-armv7.o:::sha1-armv4-large.o sha256-armv4.o sha512-armv4.o:::::::ghash-armv4.o::void"; -my $aarch64_asm="armcap.o arm64cpuid.o mem_clr.o::::::sha1-armv8.o sha256-armv8.o sha512-armv8.o::::::::"; +my $armv4_asm="armcap.o armv4cpuid.o:bn_asm.o armv4-mont.o armv4-gf2m.o::aes_cbc.o aes-armv4.o bsaes-armv7.o aesv8-armx.o:::sha1-armv4-large.o sha256-armv4.o sha512-armv4.o:::::::ghash-armv4.o::void"; +my $aarch64_asm="armcap.o arm64cpuid.o mem_clr.o:::aes_core.o aes_cbc.o aesv8-armx.o:::sha1-armv8.o sha256-armv8.o sha512-armv8.o::::::::"; my $parisc11_asm="pariscid.o:bn_asm.o parisc-mont.o::aes_core.o aes_cbc.o aes-parisc.o:::sha1-parisc.o sha256-parisc.o sha512-parisc.o::rc4-parisc.o:::::ghash-parisc.o::32"; my $parisc20_asm="pariscid.o:pa-risc2W.o parisc-mont.o::aes_core.o aes_cbc.o aes-parisc.o:::sha1-parisc.o sha256-parisc.o sha512-parisc.o::rc4-parisc.o:::::ghash-parisc.o::64"; my $ppc64_asm="ppccpuid.o ppccap.o:bn-ppc.o ppc-mont.o ppc64-mont.o::aes_core.o aes_cbc.o aes-ppc.o vpaes-ppc.o:::sha1-ppc.o sha256-ppc.o sha512-ppc.o::::::::"; diff --git a/TABLE b/TABLE index d10cb85c87..71bd7de1de 100644 --- a/TABLE +++ b/TABLE @@ -1100,7 +1100,7 @@ $bn_ops = BN_LLONG RC4_CHAR RC4_CHUNK DES_INT DES_UNROLL BF_PTR $cpuid_obj = armcap.o armv4cpuid.o $bn_obj = bn_asm.o armv4-mont.o armv4-gf2m.o $des_obj = -$aes_obj = aes_cbc.o aes-armv4.o bsaes-armv7.o +$aes_obj = aes_cbc.o aes-armv4.o bsaes-armv7.o aesv8-armx.o $bf_obj = $md5_obj = $sha1_obj = sha1-armv4-large.o sha256-armv4.o sha512-armv4.o @@ -1652,7 +1652,7 @@ $multilib = *** debug-VC-WIN32 $cc = cl -$cflags = -W3 -WX -Gs0 -GF -Gy -Zi -nologo -DOPENSSL_SYSNAME_WIN32 -DWIN32_LEAN_AND_MEAN -DL_ENDIAN -D_CRT_SECURE_NO_DEPRECATE +$cflags = -W3 -Gs0 -GF -Gy -Zi -nologo -DOPENSSL_SYSNAME_WIN32 -DWIN32_LEAN_AND_MEAN -DL_ENDIAN -D_CRT_SECURE_NO_DEPRECATE $unistd = $thread_cflag = $sys_id = WIN32 @@ -1883,7 +1883,7 @@ $multilib = *** debug-ben-debug-64-clang $cc = clang -$cflags = -Wall -pedantic -DPEDANTIC -Wno-long-long -Wsign-compare -Wmissing-prototypes -Wshadow -Wformat -Werror -DCRYPTO_MDEBUG_ALL -DCRYPTO_MDEBUG_ABORT -DREF_CHECK -DOPENSSL_NO_DEPRECATED -fsanitize=undefined -Wno-error=overlength-strings -Wno-error=extended-offsetof -Wno-error=language-extension-token -Wstrict-overflow -Qunused-arguments -DBN_DEBUG -DCONF_DEBUG -DDEBUG_SAFESTACK -DDEBUG_UNUSED -g3 -O3 -pipe +$cflags = -Wall -pedantic -DPEDANTIC -Wno-long-long -Wsign-compare -Wmissing-prototypes -Wshadow -Wformat -Werror -DCRYPTO_MDEBUG_ALL -DCRYPTO_MDEBUG_ABORT -DREF_CHECK -DOPENSSL_NO_DEPRECATED -Wno-error=overlength-strings -Wno-error=extended-offsetof -Qunused-arguments -DBN_DEBUG -DCONF_DEBUG -DDEBUG_SAFESTACK -DDEBUG_UNUSED -g3 -O3 -pipe $unistd = $thread_cflag = -pthread -D_THREAD_SAFE -D_REENTRANT $sys_id = @@ -2244,39 +2244,6 @@ $ranlib = $arflags = $multilib = -*** debug-erbridge -$cc = gcc -$cflags = -Wall -pedantic -DPEDANTIC -Wno-long-long -Wsign-compare -Wmissing-prototypes -Wshadow -Wformat -Werror -DCRYPTO_MDEBUG_ALL -DCRYPTO_MDEBUG_ABORT -DREF_CHECK -DOPENSSL_NO_DEPRECATED -DBN_DEBUG -DCONF_DEBUG -DCRYPTO_MDEBUG -m64 -DL_ENDIAN -DTERMIO -g -$unistd = -$thread_cflag = -D_REENTRANT -$sys_id = -$lflags = -ldl -$bn_ops = SIXTY_FOUR_BIT_LONG RC4_CHUNK DES_INT DES_UNROLL -$cpuid_obj = x86_64cpuid.o -$bn_obj = x86_64-gcc.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o rsaz_exp.o rsaz-x86_64.o rsaz-avx2.o -$des_obj = -$aes_obj = aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o aesni-sha256-x86_64.o aesni-mb-x86_64.o -$bf_obj = -$md5_obj = md5-x86_64.o -$sha1_obj = sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o sha1-mb-x86_64.o sha256-mb-x86_64.o -$cast_obj = -$rc4_obj = rc4-x86_64.o rc4-md5-x86_64.o -$rmd160_obj = -$rc5_obj = -$wp_obj = wp-x86_64.o -$cmll_obj = cmll-x86_64.o cmll_misc.o -$modes_obj = ghash-x86_64.o aesni-gcm-x86_64.o -$engines_obj = e_padlock-x86_64.o -$perlasm_scheme = elf -$dso_scheme = dlfcn -$shared_target= linux-shared -$shared_cflag = -fPIC -$shared_ldflag = -m64 -$shared_extension = .so.$(SHLIB_MAJOR).$(SHLIB_MINOR) -$ranlib = -$arflags = -$multilib = 64 - *** debug-geoff32 $cc = gcc $cflags = -DBN_DEBUG -DBN_DEBUG_RAND -DBN_STRICT -DPURIFY -DOPENSSL_NO_DEPRECATED -DOPENSSL_NO_ASM -DOPENSSL_NO_INLINE_ASM -DL_ENDIAN -DTERMIO -DPEDANTIC -O1 -ggdb2 -Wall -Werror -Wundef -pedantic -Wshadow -Wpointer-arith -Wbad-function-cast -Wcast-align -Wsign-compare -Wmissing-prototypes -Wmissing-declarations -Wno-long-long @@ -4004,7 +3971,7 @@ $bn_ops = SIXTY_FOUR_BIT_LONG RC4_CHAR RC4_CHUNK DES_INT DES_UNROLL BF_PTR $cpuid_obj = armcap.o arm64cpuid.o mem_clr.o $bn_obj = $des_obj = -$aes_obj = +$aes_obj = aes_core.o aes_cbc.o aesv8-armx.o $bf_obj = $md5_obj = $sha1_obj = sha1-armv8.o sha256-armv8.o sha512-armv8.o @@ -4202,7 +4169,7 @@ $bn_ops = BN_LLONG RC4_CHAR RC4_CHUNK DES_INT DES_UNROLL BF_PTR $cpuid_obj = armcap.o armv4cpuid.o $bn_obj = bn_asm.o armv4-mont.o armv4-gf2m.o $des_obj = -$aes_obj = aes_cbc.o aes-armv4.o bsaes-armv7.o +$aes_obj = aes_cbc.o aes-armv4.o bsaes-armv7.o aesv8-armx.o $bf_obj = $md5_obj = $sha1_obj = sha1-armv4-large.o sha256-armv4.o sha512-armv4.o diff --git a/crypto/aes/Makefile b/crypto/aes/Makefile index ce9bb925fd..e9fa404baf 100644 --- a/crypto/aes/Makefile +++ b/crypto/aes/Makefile @@ -86,6 +86,10 @@ aes-parisc.s: asm/aes-parisc.pl aes-mips.S: asm/aes-mips.pl $(PERL) asm/aes-mips.pl $(PERLASM_SCHEME) $@ +aesv8-armx.S: asm/aesv8-armx.pl + $(PERL) asm/aesv8-armx.pl $(PERLASM_SCHEME) $@ +aesv8-armx.o: aesv8-armx.S + # GNU make "catch all" aes-%.S: asm/aes-%.pl; $(PERL) $< $(PERLASM_SCHEME) > $@ aes-armv4.o: aes-armv4.S diff --git a/crypto/evp/e_aes.c b/crypto/evp/e_aes.c index 4740dab17a..04111ae50a 100644 --- a/crypto/evp/e_aes.c +++ b/crypto/evp/e_aes.c @@ -900,13 +900,38 @@ const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \ { return &aes_##keylen##_##mode; } #endif -#if defined(AES_ASM) && defined(BSAES_ASM) && (defined(__arm__) || defined(__arm)) +#if defined(OPENSSL_CPUID_OBJ) && (defined(__arm__) || defined(__arm) || defined(__aarch64__)) #include "arm_arch.h" #if __ARM_ARCH__>=7 -#define BSAES_CAPABLE (OPENSSL_armcap_P & ARMV7_NEON) +# if defined(BSAES_ASM) +# define BSAES_CAPABLE (OPENSSL_armcap_P & ARMV7_NEON) +# endif +# define HWAES_CAPABLE (OPENSSL_armcap_P & ARMV8_AES) +# define HWAES_set_encrypt_key aes_v8_set_encrypt_key +# define HWAES_set_decrypt_key aes_v8_set_decrypt_key +# define HWAES_encrypt aes_v8_encrypt +# define HWAES_decrypt aes_v8_decrypt +# define HWAES_cbc_encrypt aes_v8_cbc_encrypt +# define HWAES_ctr32_encrypt_blocks aes_v8_ctr32_encrypt_blocks #endif #endif +#if defined(HWAES_CAPABLE) +int HWAES_set_encrypt_key(const unsigned char *userKey, const int bits, + AES_KEY *key); +int HWAES_set_decrypt_key(const unsigned char *userKey, const int bits, + AES_KEY *key); +void HWAES_encrypt(const unsigned char *in, unsigned char *out, + const AES_KEY *key); +void HWAES_decrypt(const unsigned char *in, unsigned char *out, + const AES_KEY *key); +void HWAES_cbc_encrypt(const unsigned char *in, unsigned char *out, + size_t length, const AES_KEY *key, + unsigned char *ivec, const int enc); +void HWAES_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out, + size_t len, const AES_KEY *key, const unsigned char ivec[16]); +#endif + #define BLOCK_CIPHER_generic_pack(nid,keylen,flags) \ BLOCK_CIPHER_generic(nid,keylen,16,16,cbc,cbc,CBC,flags|EVP_CIPH_FLAG_DEFAULT_ASN1) \ BLOCK_CIPHER_generic(nid,keylen,16,0,ecb,ecb,ECB,flags|EVP_CIPH_FLAG_DEFAULT_ASN1) \ @@ -925,6 +950,19 @@ static int aes_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, mode = ctx->cipher->flags & EVP_CIPH_MODE; if ((mode == EVP_CIPH_ECB_MODE || mode == EVP_CIPH_CBC_MODE) && !enc) +#ifdef HWAES_CAPABLE + if (HWAES_CAPABLE) + { + ret = HWAES_set_decrypt_key(key,ctx->key_len*8,&dat->ks.ks); + dat->block = (block128_f)HWAES_decrypt; + dat->stream.cbc = NULL; +#ifdef HWAES_cbc_encrypt + if (mode==EVP_CIPH_CBC_MODE) + dat->stream.cbc = (cbc128_f)HWAES_cbc_encrypt; +#endif + } + else +#endif #ifdef BSAES_CAPABLE if (BSAES_CAPABLE && mode==EVP_CIPH_CBC_MODE) { @@ -953,6 +991,26 @@ static int aes_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, NULL; } else +#ifdef HWAES_CAPABLE + if (HWAES_CAPABLE) + { + ret = HWAES_set_encrypt_key(key,ctx->key_len*8,&dat->ks.ks); + dat->block = (block128_f)HWAES_encrypt; + dat->stream.cbc = NULL; +#ifdef HWAES_cbc_encrypt + if (mode==EVP_CIPH_CBC_MODE) + dat->stream.cbc = (cbc128_f)HWAES_cbc_encrypt; + else +#endif +#ifdef HWAES_ctr32_encrypt_blocks + if (mode==EVP_CIPH_CTR_MODE) + dat->stream.ctr = (ctr128_f)HWAES_ctr32_encrypt_blocks; + else +#endif + (void)0; /* terminate potentially open 'else' */ + } + else +#endif #ifdef BSAES_CAPABLE if (BSAES_CAPABLE && mode==EVP_CIPH_CTR_MODE) { @@ -1240,6 +1298,21 @@ static int aes_gcm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, return 1; if (key) { do { +#ifdef HWAES_CAPABLE + if (HWAES_CAPABLE) + { + HWAES_set_encrypt_key(key,ctx->key_len*8,&gctx->ks.ks); + CRYPTO_gcm128_init(&gctx->gcm,&gctx->ks, + (block128_f)HWAES_encrypt); +#ifdef HWAES_ctr32_encrypt_blocks + gctx->ctr = (ctr128_f)HWAES_ctr32_encrypt_blocks; +#else + gctx->ctr = NULL; +#endif + break; + } + else +#endif #ifdef BSAES_CAPABLE if (BSAES_CAPABLE) { @@ -1635,6 +1708,29 @@ static int aes_xts_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, xctx->stream = NULL; #endif /* key_len is two AES keys */ +#ifdef HWAES_CAPABLE + if (HWAES_CAPABLE) + { + if (enc) + { + HWAES_set_encrypt_key(key, ctx->key_len * 4, &xctx->ks1.ks); + xctx->xts.block1 = (block128_f)HWAES_encrypt; + } + else + { + HWAES_set_decrypt_key(key, ctx->key_len * 4, &xctx->ks1.ks); + xctx->xts.block1 = (block128_f)HWAES_decrypt; + } + + HWAES_set_encrypt_key(key + ctx->key_len/2, + ctx->key_len * 4, &xctx->ks2.ks); + xctx->xts.block2 = (block128_f)HWAES_encrypt; + + xctx->xts.key1 = &xctx->ks1; + break; + } + else +#endif #ifdef BSAES_CAPABLE if (BSAES_CAPABLE) xctx->stream = enc ? bsaes_xts_encrypt : bsaes_xts_decrypt; @@ -1776,6 +1872,19 @@ static int aes_ccm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, return 1; if (key) do { +#ifdef HWAES_CAPABLE + if (HWAES_CAPABLE) + { + HWAES_set_encrypt_key(key,ctx->key_len*8,&cctx->ks.ks); + + CRYPTO_ccm128_init(&cctx->ccm, cctx->M, cctx->L, + &cctx->ks, (block128_f)HWAES_encrypt); + cctx->str = NULL; + cctx->key_set = 1; + break; + } + else +#endif #ifdef VPAES_CAPABLE if (VPAES_CAPABLE) { -- 2.40.0