Fix "random" ECDSA signature failures when using mbedTLS

It turns out that mbedtls_mpi_size() does no always return what
I expected for the r and s values of ECDSA signatures.
We now rely on mbedtls_mpi_size(&d_ctx.grp.P), as P is fixed for
the group anyway, so we shouldn't have any suprise here.

diff --git a/pdns/mbedtlssigners.cc b/pdns/mbedtlssigners.cc
index 330d0430fc1269b1743ba5d45939749a3ea49a6b..eefd412a8ba82537fc0275ddbe07a984e8d13e4d 100644 (file)
--- a/pdns/mbedtlssigners.cc
+++ b/pdns/mbedtlssigners.cc
@@ -621,13 +621,12 @@ std::string MbedECDSADNSCryptoKeyEngine::sign(const std::string& msg) const
   }
 
   /* SEC1: 4.1.3 Signing Operation */
-  const size_t rSize = mbedtls_mpi_size(&r);
-  const size_t sSize = mbedtls_mpi_size(&s);
-  const size_t sigLen = rSize + sSize;
+  const size_t mpiLen = mbedtls_mpi_size(&d_ctx.grp.P);
+  const size_t sigLen = mpiLen * 2;
 
   unsigned char sig[sigLen];
 
-  ret = mbedtls_mpi_write_binary(&r, sig, rSize);
+  ret = mbedtls_mpi_write_binary(&r, sig, mpiLen);
 
   if (ret != 0) {
     mbedtls_mpi_free(&r);
@@ -635,7 +634,7 @@ std::string MbedECDSADNSCryptoKeyEngine::sign(const std::string& msg) const
     throw runtime_error("Error converting ECDSA signature part R to binary");
   }
 
-  ret = mbedtls_mpi_write_binary(&s, sig + rSize, sSize);
+  ret = mbedtls_mpi_write_binary(&s, sig + mpiLen, mpiLen);
 
   if (ret != 0) {
     mbedtls_mpi_free(&r);