Fix unsigned overflow in rd_variance_adjustment.

Found with clang -fsanitize=integer

Change-Id: I2538e7483cb2d5f06bceecbd3326bdd88bfecfa1

diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c
index 8ba211017ab48048a3f2ba55628e4c1db64cf084..d87d1641846d4cadb6915414c0a992114fd7d66e 100644 (file)
--- a/vp10/encoder/rdopt.c
+++ b/vp10/encoder/rdopt.c
@@ -3017,9 +3017,9 @@ static void rd_variance_adjustment(VP10_COMP *cpi,
       ? (source_variance - recon_variance)
       : (recon_variance - source_variance);
 
-    var_error = (200 * source_variance * recon_variance) /
-      ((source_variance * source_variance) +
-       (recon_variance * recon_variance));
+    var_error = ((int64_t)200 * source_variance * recon_variance) /
+      (((int64_t)source_variance * source_variance) +
+       ((int64_t)recon_variance * recon_variance));
     var_error = 100 - var_error;
   }
 

diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c
index ae1446417b35597e815ca58f7e09c72fb61a26d2..2a6b707036b8081e46171c923ba4a43a76a41971 100644 (file)
--- a/vp9/encoder/vp9_rdopt.c
+++ b/vp9/encoder/vp9_rdopt.c
@@ -2831,9 +2831,9 @@ static void rd_variance_adjustment(VP9_COMP *cpi,
       ? (source_variance - recon_variance)
       : (recon_variance - source_variance);
 
-    var_error = (200 * source_variance * recon_variance) /
-      ((source_variance * source_variance) +
-       (recon_variance * recon_variance));
+    var_error = ((int64_t)200 * source_variance * recon_variance) /
+      (((int64_t)source_variance * source_variance) +
+       ((int64_t)recon_variance * recon_variance));
     var_error = 100 - var_error;
   }