From 993adc0531cca2df70a2b663b0fcc78e3a977154 Mon Sep 17 00:00:00 2001 From: Andy Polyakov Date: Mon, 17 Oct 2011 17:10:54 +0000 Subject: [PATCH] Engage bsaes-x86_64.pl, bit-sliced AES. --- Configure | 5 +-- TABLE | 22 ++++++------ crypto/aes/Makefile | 2 ++ crypto/evp/e_aes.c | 84 +++++++++++++++++++++++++++++++++------------ 4 files changed, 78 insertions(+), 35 deletions(-) diff --git a/Configure b/Configure index ed8c4c0bda..b624f35407 100755 --- a/Configure +++ b/Configure @@ -127,7 +127,7 @@ my $x86_asm="x86cpuid.o:bn-586.o co-586.o x86-mont.o x86-gf2m.o:des-586.o crypt5 my $x86_elf_asm="$x86_asm:elf"; -my $x86_64_asm="x86_64cpuid.o:x86_64-gcc.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o modexp512-x86_64.o::aes-x86_64.o vpaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o::md5-x86_64.o:sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o::rc4-x86_64.o rc4-md5-x86_64.o:::wp-x86_64.o:cmll-x86_64.o cmll_misc.o:ghash-x86_64.o:e_padlock-x86_64.o"; +my $x86_64_asm="x86_64cpuid.o:x86_64-gcc.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o modexp512-x86_64.o::aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o::md5-x86_64.o:sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o::rc4-x86_64.o rc4-md5-x86_64.o:::wp-x86_64.o:cmll-x86_64.o cmll_misc.o:ghash-x86_64.o:e_padlock-x86_64.o"; my $ia64_asm="ia64cpuid.o:bn-ia64.o ia64-mont.o::aes_core.o aes_cbc.o aes-ia64.o::md5-ia64.o:sha1-ia64.o sha256-ia64.o sha512-ia64.o::rc4-ia64.o rc4_skey.o:::::ghash-ia64.o::void"; my $sparcv9_asm="sparcv9cap.o sparccpuid.o:bn-sparcv9.o sparcv9-mont.o sparcv9a-mont.o:des_enc-sparc.o fcrypt_b.o:aes_core.o aes_cbc.o aes-sparcv9.o:::sha1-sparcv9.o sha256-sparcv9.o sha512-sparcv9.o:::::::ghash-sparcv9.o::void"; my $sparcv8_asm=":sparcv8.o:des_enc-sparc.o fcrypt_b.o:::::::::::::void"; @@ -1555,8 +1555,9 @@ if ($aes_obj =~ /\.o$/) # module implements AES_ctr32_encrypt... $cflags.=" -DAES_CTR_ASM" if ($aes_obj =~ s/\s*aes_ctr\.o//); $aes_obj =~ s/\s*(vpaes|aesni)\-x86\.o// if ($no_sse2); - $aes_obj =~ s/\s*vpaes-\w*\.o// if ($fipscanisterinternal eq "y"); + $aes_obj =~ s/\s*(vp|bs)aes-\w*\.o// if ($fipscanisterinternal eq "y"); $cflags.=" -DVPAES_ASM" if ($aes_obj =~ m/vpaes/); + $cflags.=" -DBSAES_ASM" if ($aes_obj =~ m/bsaes/); } else { $aes_obj=$aes_enc; diff --git a/TABLE b/TABLE index 47b7af934b..7a95ee52e5 100644 --- a/TABLE +++ b/TABLE @@ -308,7 +308,7 @@ $bn_ops = SIXTY_FOUR_BIT_LONG RC4_CHUNK DES_INT DES_UNROLL $cpuid_obj = x86_64cpuid.o $bn_obj = x86_64-gcc.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o modexp512-x86_64.o $des_obj = -$aes_obj = aes-x86_64.o vpaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o +$aes_obj = aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o $bf_obj = $md5_obj = md5-x86_64.o $sha1_obj = sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o @@ -803,7 +803,7 @@ $bn_ops = SIXTY_FOUR_BIT RC4_CHUNK_LL DES_INT EXPORT_VAR_AS_FN $cpuid_obj = x86_64cpuid.o $bn_obj = bn_asm.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o modexp512-x86_64.o $des_obj = -$aes_obj = aes-x86_64.o vpaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o +$aes_obj = aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o $bf_obj = $md5_obj = md5-x86_64.o $sha1_obj = sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o @@ -1463,7 +1463,7 @@ $bn_ops = SIXTY_FOUR_BIT_LONG RC4_CHAR RC4_CHUNK DES_INT DES_UNROLL $cpuid_obj = x86_64cpuid.o $bn_obj = x86_64-gcc.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o modexp512-x86_64.o $des_obj = -$aes_obj = aes-x86_64.o vpaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o +$aes_obj = aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o $bf_obj = $md5_obj = md5-x86_64.o $sha1_obj = sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o @@ -1628,7 +1628,7 @@ $bn_ops = SIXTY_FOUR_BIT RC4_CHUNK_LL DES_INT EXPORT_VAR_AS_FN $cpuid_obj = x86_64cpuid.o $bn_obj = bn_asm.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o modexp512-x86_64.o $des_obj = -$aes_obj = aes-x86_64.o vpaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o +$aes_obj = aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o $bf_obj = $md5_obj = md5-x86_64.o $sha1_obj = sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o @@ -2420,7 +2420,7 @@ $bn_ops = SIXTY_FOUR_BIT_LONG RC4_CHUNK DES_INT DES_UNROLL $cpuid_obj = x86_64cpuid.o $bn_obj = x86_64-gcc.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o modexp512-x86_64.o $des_obj = -$aes_obj = aes-x86_64.o vpaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o +$aes_obj = aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o $bf_obj = $md5_obj = md5-x86_64.o $sha1_obj = sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o @@ -2618,7 +2618,7 @@ $bn_ops = SIXTY_FOUR_BIT_LONG RC4_CHUNK DES_INT DES_UNROLL $cpuid_obj = x86_64cpuid.o $bn_obj = x86_64-gcc.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o modexp512-x86_64.o $des_obj = -$aes_obj = aes-x86_64.o vpaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o +$aes_obj = aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o $bf_obj = $md5_obj = md5-x86_64.o $sha1_obj = sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o @@ -2684,7 +2684,7 @@ $bn_ops = SIXTY_FOUR_BIT_LONG RC4_CHUNK DES_INT DES_UNROLL $cpuid_obj = x86_64cpuid.o $bn_obj = x86_64-gcc.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o modexp512-x86_64.o $des_obj = -$aes_obj = aes-x86_64.o vpaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o +$aes_obj = aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o $bf_obj = $md5_obj = md5-x86_64.o $sha1_obj = sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o @@ -4235,7 +4235,7 @@ $bn_ops = SIXTY_FOUR_BIT_LONG RC4_CHUNK DES_INT DES_UNROLL $cpuid_obj = x86_64cpuid.o $bn_obj = x86_64-gcc.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o modexp512-x86_64.o $des_obj = -$aes_obj = aes-x86_64.o vpaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o +$aes_obj = aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o $bf_obj = $md5_obj = md5-x86_64.o $sha1_obj = sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o @@ -4400,7 +4400,7 @@ $bn_ops = SIXTY_FOUR_BIT RC4_CHUNK_LL DES_INT EXPORT_VAR_AS_FN $cpuid_obj = x86_64cpuid.o $bn_obj = x86_64-gcc.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o modexp512-x86_64.o $des_obj = -$aes_obj = aes-x86_64.o vpaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o +$aes_obj = aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o $bf_obj = $md5_obj = md5-x86_64.o $sha1_obj = sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o @@ -5390,7 +5390,7 @@ $bn_ops = SIXTY_FOUR_BIT_LONG RC4_CHUNK DES_INT DES_UNROLL $cpuid_obj = x86_64cpuid.o $bn_obj = x86_64-gcc.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o modexp512-x86_64.o $des_obj = -$aes_obj = aes-x86_64.o vpaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o +$aes_obj = aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o $bf_obj = $md5_obj = md5-x86_64.o $sha1_obj = sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o @@ -5423,7 +5423,7 @@ $bn_ops = SIXTY_FOUR_BIT_LONG RC4_CHUNK DES_INT DES_UNROLL $cpuid_obj = x86_64cpuid.o $bn_obj = x86_64-gcc.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o modexp512-x86_64.o $des_obj = -$aes_obj = aes-x86_64.o vpaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o +$aes_obj = aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o $bf_obj = $md5_obj = md5-x86_64.o $sha1_obj = sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o diff --git a/crypto/aes/Makefile b/crypto/aes/Makefile index 78d5984140..86e97f335e 100644 --- a/crypto/aes/Makefile +++ b/crypto/aes/Makefile @@ -59,6 +59,8 @@ aes-x86_64.s: asm/aes-x86_64.pl $(PERL) asm/aes-x86_64.pl $(PERLASM_SCHEME) > $@ vpaes-x86_64.s: asm/vpaes-x86_64.pl $(PERL) asm/vpaes-x86_64.pl $(PERLASM_SCHEME) > $@ +bsaes-x86_64.s: asm/bsaes-x86_64.pl + $(PERL) asm/bsaes-x86_64.pl $(PERLASM_SCHEME) > $@ aesni-x86_64.s: asm/aesni-x86_64.pl $(PERL) asm/aesni-x86_64.pl $(PERLASM_SCHEME) > $@ aesni-sha1-x86_64.s: asm/aesni-sha1-x86_64.pl diff --git a/crypto/evp/e_aes.c b/crypto/evp/e_aes.c index 4b94b2a840..53d8316484 100644 --- a/crypto/evp/e_aes.c +++ b/crypto/evp/e_aes.c @@ -66,11 +66,17 @@ typedef struct AES_KEY ks; void (*block)(const unsigned char *in, unsigned char *out, const AES_KEY *key); + union { void (*cbc)(const unsigned char *in, unsigned char *out, size_t length, const AES_KEY *key, unsigned char *ivec, int enc); + void (*ctr)(const unsigned char *in, + unsigned char *out, + size_t blocks, const AES_KEY *key, + const unsigned char ivec[16]); + } stream; } EVP_AES_KEY; @@ -123,6 +129,11 @@ void vpaes_cbc_encrypt(const unsigned char *in, const AES_KEY *key, unsigned char *ivec, int enc); #endif +#ifdef BSAES_ASM +void bsaes_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out, + size_t len, const AES_KEY *key, + const unsigned char ivec[16]); +#endif #if defined(AES_ASM) && !defined(I386_ONLY) && ( \ ((defined(__i386) || defined(__i386__) || \ @@ -136,6 +147,9 @@ extern unsigned int OPENSSL_ia32cap_P[2]; #ifdef VPAES_ASM #define VPAES_CAPABLE (OPENSSL_ia32cap_P[1]&(1<<(41-32))) #endif +#ifdef BSAES_ASM +#define BSAES_CAPABLE VPAES_CAPABLE +#endif /* * AES-NI section */ @@ -690,40 +704,63 @@ const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \ static int aes_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key, const unsigned char *iv, int enc) { - int ret; + int ret, mode; EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data; - if (((ctx->cipher->flags & EVP_CIPH_MODE) == EVP_CIPH_ECB_MODE - || (ctx->cipher->flags & EVP_CIPH_MODE) == EVP_CIPH_CBC_MODE) + mode = ctx->cipher->flags & EVP_CIPH_MODE; + if ((mode == EVP_CIPH_ECB_MODE || mode == EVP_CIPH_CBC_MODE) && !enc) #ifdef VPAES_CAPABLE if (VPAES_CAPABLE) { ret = vpaes_set_decrypt_key(key,ctx->key_len*8,&dat->ks); - dat->block = vpaes_decrypt; - dat->cbc = vpaes_cbc_encrypt; + dat->block = vpaes_decrypt; + dat->stream.cbc = mode==EVP_CIPH_CBC_MODE ? + vpaes_cbc_encrypt:NULL; } else #endif { ret = AES_set_decrypt_key(key,ctx->key_len*8,&dat->ks); - dat->block = AES_decrypt; - dat->cbc = AES_cbc_encrypt; + dat->block = AES_decrypt; + dat->stream.cbc = mode==EVP_CIPH_CBC_MODE ? + AES_cbc_encrypt:NULL; } else +#ifdef BSAES_CAPABLE + if (BSAES_CAPABLE && mode==EVP_CIPH_CTR_MODE) + { + ret = AES_set_encrypt_key(key,ctx->key_len*8,&dat->ks); + dat->block = AES_encrypt; + dat->stream.ctr = bsaes_ctr32_encrypt_blocks; + } + else +#endif #ifdef VPAES_CAPABLE if (VPAES_CAPABLE) { ret = vpaes_set_encrypt_key(key,ctx->key_len*8,&dat->ks); - dat->block = vpaes_encrypt; - dat->cbc = vpaes_cbc_encrypt; + dat->block = vpaes_encrypt; + dat->stream.cbc = mode==EVP_CIPH_CBC_MODE ? + vpaes_cbc_encrypt:NULL; } else #endif { ret = AES_set_encrypt_key(key,ctx->key_len*8,&dat->ks); - dat->block = AES_encrypt; - dat->cbc = AES_cbc_encrypt; + dat->block = AES_encrypt; + dat->stream.cbc = mode==EVP_CIPH_CBC_MODE ? + AES_cbc_encrypt:NULL; +#ifdef AES_CTR_ASM + if (mode==EVP_CIPH_CTR_MODE) + { + void AES_ctr32_encrypt(const unsigned char *in, + unsigned char *out, + size_t blocks, const AES_KEY *key, + const unsigned char ivec[AES_BLOCK_SIZE]); + dat->stream.ctr = AES_ctr32_encrypt; + } +#endif } if(ret < 0) @@ -740,7 +777,14 @@ static int aes_cbc_cipher(EVP_CIPHER_CTX *ctx,unsigned char *out, { EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data; - (*dat->cbc)(in,out,len,&dat->ks,ctx->iv,ctx->encrypt); + if (dat->stream.cbc) + (*dat->stream.cbc)(in,out,len,&dat->ks,ctx->iv,ctx->encrypt); + else if (ctx->encrypt) + CRYPTO_cbc128_encrypt(in,out,len,&dat->ks,ctx->iv, + (block128_f)dat->block); + else + CRYPTO_cbc128_encrypt(in,out,len,&dat->ks,ctx->iv, + (block128_f)dat->block); return 1; } @@ -824,17 +868,13 @@ static int aes_ctr_cipher (EVP_CIPHER_CTX *ctx, unsigned char *out, { unsigned int num = ctx->num; EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data; -#ifdef AES_CTR_ASM - void AES_ctr32_encrypt(const unsigned char *in, unsigned char *out, - size_t blocks, const AES_KEY *key, - const unsigned char ivec[AES_BLOCK_SIZE]); - CRYPTO_ctr128_encrypt_ctr32(in,out,len,&dat->ks, - ctx->iv,ctx->buf,&num,(ctr128_f)AES_ctr32_encrypt); -#else - CRYPTO_ctr128_encrypt(in,out,len,&dat->ks, - ctx->iv,ctx->buf,&num,(block128_f)dat->block); -#endif + if (dat->stream.ctr) + CRYPTO_ctr128_encrypt_ctr32(in,out,len,&dat->ks, + ctx->iv,ctx->buf,&num,(ctr128_f)dat->stream.ctr); + else + CRYPTO_ctr128_encrypt(in,out,len,&dat->ks, + ctx->iv,ctx->buf,&num,(block128_f)dat->block); ctx->num = (size_t)num; return 1; } -- 2.40.0