From: Alex Converse <aconverse@google.com>
Date: Mon, 9 May 2016 18:21:20 +0000 (-0700)
Subject: pickmode: Fix a pair of unsigned overflows.
X-Git-Tag: v1.6.0~146^2
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=d8a18186ba4482ff121aa526fbf6df31f9d90d35;p=libvpx

pickmode: Fix a pair of unsigned overflows.

block_variance: This operates on 8x8s and would be safe with a int32 *
int32 to uint32 multiply, but this is potentially unsafe for 12-bit
input. Unfortunately the code already segfaults on 12-bit input:
https://bugs.chromium.org/p/webm/issues/detail?id=1223

calculate_variance: This operates on up to a 32x32 of 8x8s and can
overflow even with 8-bit input (log2((256*32*32)**2) == 36).

BUG=https://bugs.chromium.org/p/webm/issues/detail?id=1220

Change-Id: I1ca4ff6092db9a7580da371ee9a21f403fdadc40
---

diff --git a/vp9/encoder/vp9_pickmode.c b/vp9/encoder/vp9_pickmode.c
index 98fc603cb..d2ecafc0a 100644
--- a/vp9/encoder/vp9_pickmode.c
+++ b/vp9/encoder/vp9_pickmode.c
@@ -244,7 +244,7 @@ static void block_variance(const uint8_t *src, int src_stride,
                     &sse8x8[k], &sum8x8[k]);
       *sse += sse8x8[k];
       *sum += sum8x8[k];
-      var8x8[k] = sse8x8[k] - (((unsigned int)sum8x8[k] * sum8x8[k]) >> 6);
+      var8x8[k] = sse8x8[k] - (uint32_t)(((int64_t)sum8x8[k] * sum8x8[k]) >> 6);
       k++;
     }
   }
@@ -265,7 +265,7 @@ static void calculate_variance(int bw, int bh, TX_SIZE tx_size,
           sse_i[(i + 1) * nw + j] + sse_i[(i + 1) * nw + j + 1];
       sum_o[k] = sum_i[i * nw + j] + sum_i[i * nw + j + 1] +
           sum_i[(i + 1) * nw + j] + sum_i[(i + 1) * nw + j + 1];
-      var_o[k] = sse_o[k] - (((unsigned int)sum_o[k] * sum_o[k]) >>
+      var_o[k] = sse_o[k] - (uint32_t)(((int64_t)sum_o[k] * sum_o[k]) >>
           (b_width_log2_lookup[unit_size] +
               b_height_log2_lookup[unit_size] + 6));
       k++;