From: Martin Storsjo <martin@martin.st>
Date: Fri, 19 Jul 2013 12:40:25 +0000 (+0300)
Subject: Don't use non-public gnutls functions
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=8e3064207fa7535baad07fd06b65630ec8b31a08;p=rtmpdump

Don't use non-public gnutls functions

The internal functions gnutls_calc_dh_secret and gnutls_calc_dh_key
have changed signature in recent gnutls versions - don't use them
but manually do roughly the same as what they used to do before
using only public API functions.
---

diff --git a/librtmp/dh.h b/librtmp/dh.h
index e29587b..5fc3f32 100644
--- a/librtmp/dh.h
+++ b/librtmp/dh.h
@@ -79,6 +79,7 @@ static int MDH_compute_key(uint8_t *secret, size_t len, MP_t pub, MDH *dh)
 #elif defined(USE_GNUTLS)
 #include <gmp.h>
 #include <nettle/bignum.h>
+#include <gnutls/crypto.h>
 typedef mpz_ptr MP_t;
 #define MP_new(m)	m = malloc(sizeof(*m)); mpz_init2(m, 1)
 #define MP_set_w(mpi, w)	mpz_set_ui(mpi, w)
@@ -104,21 +105,62 @@ typedef struct MDH {
 #define	MDH_new()	calloc(1,sizeof(MDH))
 #define MDH_free(dh)	do {MP_free(((MDH*)(dh))->p); MP_free(((MDH*)(dh))->g); MP_free(((MDH*)(dh))->pub_key); MP_free(((MDH*)(dh))->priv_key); free(dh);} while(0)
 
-extern MP_t gnutls_calc_dh_secret(MP_t *priv, MP_t g, MP_t p);
-extern MP_t gnutls_calc_dh_key(MP_t y, MP_t x, MP_t p);
+static int MDH_generate_key(MDH *dh)
+{
+  int num_bytes;
+  uint32_t seed;
+  gmp_randstate_t rs;
+
+  num_bytes = (mpz_sizeinbase(dh->p, 2) + 7) / 8 - 1;
+  if (num_bytes <= 0 || num_bytes > 18000)
+    return 0;
+
+  dh->priv_key = calloc(1, sizeof(*dh->priv_key));
+  if (!dh->priv_key)
+    return 0;
+  mpz_init2(dh->priv_key, 1);
+  gnutls_rnd(GNUTLS_RND_RANDOM, &seed, sizeof(seed));
+  gmp_randinit_mt(rs);
+  gmp_randseed_ui(rs, seed);
+  mpz_urandomb(dh->priv_key, rs, num_bytes);
+  gmp_randclear(rs);
+
+  dh->pub_key = calloc(1, sizeof(*dh->pub_key));
+  if (!dh->pub_key)
+    return 0;
+  mpz_init2(dh->pub_key, 1);
+  if (!dh->pub_key) {
+    mpz_clear(dh->priv_key);
+    free(dh->priv_key);
+    return 0;
+  }
+
+  mpz_powm(dh->pub_key, dh->g, dh->priv_key, dh->p);
+
+  return 1;
+}
 
-#define MDH_generate_key(dh)	(dh->pub_key = gnutls_calc_dh_secret(&dh->priv_key, dh->g, dh->p))
 static int MDH_compute_key(uint8_t *secret, size_t len, MP_t pub, MDH *dh)
 {
-  MP_t sec = gnutls_calc_dh_key(pub, dh->priv_key, dh->p);
-  if (sec)
-    {
-	  MP_setbin(sec, secret, len);
-	  MP_free(sec);
-	  return 0;
-	}
-  else
+  mpz_ptr k;
+  int num_bytes;
+
+  num_bytes = (mpz_sizeinbase(dh->p, 2) + 7) / 8;
+  if (num_bytes <= 0 || num_bytes > 18000)
+    return -1;
+
+  k = calloc(1, sizeof(*k));
+  if (!k)
     return -1;
+  mpz_init2(k, 1);
+
+  mpz_powm(k, pub, dh->priv_key, dh->p);
+  nettle_mpz_get_str_256(len, secret, k);
+  mpz_clear(k);
+  free(k);
+
+  /* return the length of the shared secret key like DH_compute_key */
+  return len;
 }
 
 #else /* USE_OPENSSL */