Fix integer overflow issue in rtc coding flow intra mode search

The overflow issue affects a variable that is only used in inter
mode. This commit fixes the ioc warning triggered in the intra
mode. It does not affect the compression performance.

Change-Id: I593d1b5650599de07f3e68176dd1442c6cb7bdbc

diff --git a/vp9/encoder/vp9_pickmode.c b/vp9/encoder/vp9_pickmode.c
index 2479b6e73c48ce63bf49247c151754ad311291ea..944a34e3c814aed811c52ebc33daf64668b0a1e6 100644 (file)
--- a/vp9/encoder/vp9_pickmode.c
+++ b/vp9/encoder/vp9_pickmode.c
@@ -658,7 +658,8 @@ static void block_yrd(VP9_COMP *cpi, MACROBLOCK *x, int *rate, int64_t *dist,
   block = 0;
   *rate = 0;
   *dist = 0;
-  *sse = (*sse << 6) >> shift;
+  if (*sse < INT64_MAX)
+    *sse = (*sse << 6) >> shift;
   for (r = 0; r < max_blocks_high; r += block_step) {
     for (c = 0; c < num_4x4_w; c += block_step) {
       if (c < max_blocks_wide) {
@@ -912,6 +913,8 @@ static void estimate_block_intra(int plane, int block, BLOCK_SIZE plane_bsize,
   // TODO(jingning): This needs further refactoring.
   block_yrd(cpi, x, &rate, &dist, &is_skippable, &this_sse, 0,
             bsize_tx, MIN(tx_size, TX_16X16));
+  // this_sse is a dummy variable here. Its value should remain INT64_MAX.
+  assert(this_sse == INT64_MAX);
   x->skip_txfm[0] = is_skippable;
   rate += vp9_cost_bit(vp9_get_skip_prob(&cpi->common, xd), is_skippable);