From: Ronald S. Bultje <rbultje@google.com>
Date: Wed, 15 Feb 2012 16:30:36 +0000 (-0800)
Subject: Fix overflows in dual prediction mode selection.
X-Git-Tag: v1.3.0~1217^2~380^2~52
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=0930dde2496e911d24c930751b08c71ae722bd89;p=libvpx

Fix overflows in dual prediction mode selection.

Change-Id: I265ad46e01a307bca21e6223725e4055f5e08648
---

diff --git a/vp8/encoder/encodeframe.c b/vp8/encoder/encodeframe.c
index 7ad51df07..62366964f 100644
--- a/vp8/encoder/encodeframe.c
+++ b/vp8/encoder/encodeframe.c
@@ -1348,6 +1348,7 @@ void vp8_encode_frame(VP8_COMP *cpi)
     {
         int frame_type, pred_type;
         int redo = 0;
+        int single_diff, dual_diff, hybrid_diff;
 
         /*
          * This code does a single RD pass over the whole frame assuming
@@ -1384,27 +1385,27 @@ void vp8_encode_frame(VP8_COMP *cpi)
         cpi->common.dual_pred_mode = pred_type;
         encode_frame_internal(cpi);
 
-        cpi->rd_single_diff /= cpi->common.MBs;
-        cpi->rd_prediction_type_threshes[frame_type][0] += cpi->rd_single_diff;
+        single_diff = cpi->rd_single_diff / cpi->common.MBs;
+        cpi->rd_prediction_type_threshes[frame_type][0] += single_diff;
         cpi->rd_prediction_type_threshes[frame_type][0] >>= 1;
-        cpi->rd_dual_diff   /= cpi->common.MBs;
-        cpi->rd_prediction_type_threshes[frame_type][1] += cpi->rd_dual_diff;
+        dual_diff   = cpi->rd_dual_diff   / cpi->common.MBs;
+        cpi->rd_prediction_type_threshes[frame_type][1] += dual_diff;
         cpi->rd_prediction_type_threshes[frame_type][1] >>= 1;
-        cpi->rd_hybrid_diff /= cpi->common.MBs;
-        cpi->rd_prediction_type_threshes[frame_type][2] += cpi->rd_hybrid_diff;
+        hybrid_diff = cpi->rd_hybrid_diff / cpi->common.MBs;
+        cpi->rd_prediction_type_threshes[frame_type][2] += hybrid_diff;
         cpi->rd_prediction_type_threshes[frame_type][2] >>= 1;
 
         /* FIXME make "100" (the threshold at which to re-encode the
          * current frame) a commandline option. */
         if (cpi->common.dual_pred_mode == SINGLE_PREDICTION_ONLY &&
-            (cpi->rd_dual_diff >= 100 || cpi->rd_hybrid_diff >= 100))
+            (dual_diff >= 100 || hybrid_diff >= 100))
         {
             redo = 1;
             cpi->common.dual_pred_mode = cpi->rd_dual_diff > cpi->rd_hybrid_diff ?
                         DUAL_PREDICTION_ONLY : HYBRID_PREDICTION;
         }
         else if (cpi->common.dual_pred_mode == DUAL_PREDICTION_ONLY &&
-                 (cpi->rd_single_diff >= 100 || cpi->rd_hybrid_diff >= 100))
+                 (single_diff >= 100 || hybrid_diff >= 100))
         {
             redo = 1;
             cpi->common.dual_pred_mode = cpi->rd_single_diff > cpi->rd_hybrid_diff ?
@@ -1430,7 +1431,7 @@ void vp8_encode_frame(VP8_COMP *cpi)
             {
                 cpi->common.dual_pred_mode = DUAL_PREDICTION_ONLY;
             }
-            else if (cpi->rd_single_diff >= 100 || cpi->rd_dual_diff >= 100)
+            else if (single_diff >= 100 || dual_diff >= 100)
             {
                 redo = 1;
                 cpi->common.dual_pred_mode = cpi->rd_single_diff > cpi->rd_dual_diff ?
diff --git a/vp8/encoder/onyx_int.h b/vp8/encoder/onyx_int.h
index 03bd3ffba..b107ca716 100644
--- a/vp8/encoder/onyx_int.h
+++ b/vp8/encoder/onyx_int.h
@@ -353,7 +353,7 @@ typedef struct VP8_COMP
     int rd_thresh_mult[MAX_MODES];
     int rd_baseline_thresh[MAX_MODES];
     int rd_threshes[MAX_MODES];
-    int rd_single_diff, rd_dual_diff, rd_hybrid_diff;
+    int64_t rd_single_diff, rd_dual_diff, rd_hybrid_diff;
     int rd_prediction_type_threshes[4][NB_PREDICTION_TYPES];
     int dual_pred_count[DUAL_PRED_CONTEXTS];
     int single_pred_count[DUAL_PRED_CONTEXTS];
diff --git a/vp8/encoder/rdopt.c b/vp8/encoder/rdopt.c
index 69dd9adc7..1d593fbee 100644
--- a/vp8/encoder/rdopt.c
+++ b/vp8/encoder/rdopt.c
@@ -3142,9 +3142,18 @@ void vp8_rd_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, int
 
     rd_update_mvcount(cpi, x, &frame_best_ref_mv[xd->mode_info_context->mbmi.ref_frame]);
 
-    *best_single_rd_diff = best_rd - best_single_rd;
-    *best_dual_rd_diff   = best_rd - best_dual_rd;
-    *best_hybrid_rd_diff = best_rd - best_hybrid_rd;
+    if (best_single_rd == INT_MAX)
+        *best_single_rd_diff = INT_MIN;
+    else
+        *best_single_rd_diff = best_rd - best_single_rd;
+    if (best_dual_rd == INT_MAX)
+        *best_dual_rd_diff = INT_MIN;
+    else
+        *best_dual_rd_diff   = best_rd - best_dual_rd;
+    if (best_hybrid_rd == INT_MAX)
+        *best_hybrid_rd_diff = INT_MIN;
+    else
+        *best_hybrid_rd_diff = best_rd - best_hybrid_rd;
 }
 
 void vp8_rd_pick_intra_mode(VP8_COMP *cpi, MACROBLOCK *x, int *rate_)