Don't use non-public gnutls functions

The internal functions gnutls_calc_dh_secret and gnutls_calc_dh_key
have changed signature in recent gnutls versions - don't use them
but manually do roughly the same as what they used to do before
using only public API functions.

diff --git a/librtmp/dh.h b/librtmp/dh.h
index e29587bcb3f5d0b93cfa83f66d6aa6ccf27fcb6e..5fc3f32e8ec9cbf31958444d1f39e0f2c5ea0f17 100644 (file)
--- a/librtmp/dh.h
+++ b/librtmp/dh.h
@@ -79,6 +79,7 @@ static int MDH_compute_key(uint8_t *secret, size_t len, MP_t pub, MDH *dh)
 #elif defined(USE_GNUTLS)
 #include <gmp.h>
 #include <nettle/bignum.h>
+#include <gnutls/crypto.h>
 typedef mpz_ptr MP_t;
 #define MP_new(m)      m = malloc(sizeof(*m)); mpz_init2(m, 1)
 #define MP_set_w(mpi, w)       mpz_set_ui(mpi, w)
@@ -104,21 +105,62 @@ typedef struct MDH {
 #define        MDH_new()       calloc(1,sizeof(MDH))
 #define MDH_free(dh)   do {MP_free(((MDH*)(dh))->p); MP_free(((MDH*)(dh))->g); MP_free(((MDH*)(dh))->pub_key); MP_free(((MDH*)(dh))->priv_key); free(dh);} while(0)
 
-extern MP_t gnutls_calc_dh_secret(MP_t *priv, MP_t g, MP_t p);
-extern MP_t gnutls_calc_dh_key(MP_t y, MP_t x, MP_t p);
+static int MDH_generate_key(MDH *dh)
+{
+  int num_bytes;
+  uint32_t seed;
+  gmp_randstate_t rs;
+
+  num_bytes = (mpz_sizeinbase(dh->p, 2) + 7) / 8 - 1;
+  if (num_bytes <= 0 || num_bytes > 18000)
+    return 0;
+
+  dh->priv_key = calloc(1, sizeof(*dh->priv_key));
+  if (!dh->priv_key)
+    return 0;
+  mpz_init2(dh->priv_key, 1);
+  gnutls_rnd(GNUTLS_RND_RANDOM, &seed, sizeof(seed));
+  gmp_randinit_mt(rs);
+  gmp_randseed_ui(rs, seed);
+  mpz_urandomb(dh->priv_key, rs, num_bytes);
+  gmp_randclear(rs);
+
+  dh->pub_key = calloc(1, sizeof(*dh->pub_key));
+  if (!dh->pub_key)
+    return 0;
+  mpz_init2(dh->pub_key, 1);
+  if (!dh->pub_key) {
+    mpz_clear(dh->priv_key);
+    free(dh->priv_key);
+    return 0;
+  }
+
+  mpz_powm(dh->pub_key, dh->g, dh->priv_key, dh->p);
+
+  return 1;
+}
 
-#define MDH_generate_key(dh)   (dh->pub_key = gnutls_calc_dh_secret(&dh->priv_key, dh->g, dh->p))
 static int MDH_compute_key(uint8_t *secret, size_t len, MP_t pub, MDH *dh)
 {
-  MP_t sec = gnutls_calc_dh_key(pub, dh->priv_key, dh->p);
-  if (sec)
-    {
-         MP_setbin(sec, secret, len);
-         MP_free(sec);
-         return 0;
-       }
-  else
+  mpz_ptr k;
+  int num_bytes;
+
+  num_bytes = (mpz_sizeinbase(dh->p, 2) + 7) / 8;
+  if (num_bytes <= 0 || num_bytes > 18000)
+    return -1;
+
+  k = calloc(1, sizeof(*k));
+  if (!k)
     return -1;
+  mpz_init2(k, 1);
+
+  mpz_powm(k, pub, dh->priv_key, dh->p);
+  nettle_mpz_get_str_256(len, secret, k);
+  mpz_clear(k);
+  free(k);
+
+  /* return the length of the shared secret key like DH_compute_key */
+  return len;
 }
 
 #else /* USE_OPENSSL */