Fix an integer overflow issue in restoration

https://bugs.chromium.org/p/webm/issues/detail?id=1306

Change-Id: Icd11d373ff08954121c097728e4c7791791e223f

diff --git a/av1/common/restoration.c b/av1/common/restoration.c
index 97c89e12fec3bda6bb29dded80fec7352ea9f6ba..8f0a212aa6bb12a5de1a0960ad9a7e16dd206a4f 100644 (file)
--- a/av1/common/restoration.c
+++ b/av1/common/restoration.c
@@ -218,15 +218,15 @@ static void loop_bilateral_filter_tile(uint8_t *data, int tile_idx, int width,
 
     for (i = 0; i < (v_end - v_start); ++i) {
       for (j = 0; j < (h_end - h_start); ++j) {
-        int x, y;
-        int flsum = 0, wtsum = 0, wt;
+        int x, y, wt;
+        int64_t flsum = 0, wtsum = 0;
         uint8_t *data_p2 = data_p + j - RESTORATION_HALFWIN * stride;
         for (y = -RESTORATION_HALFWIN; y <= RESTORATION_HALFWIN; ++y) {
           for (x = -RESTORATION_HALFWIN; x <= RESTORATION_HALFWIN; ++x) {
             wt = (int)wx_lut[y + RESTORATION_HALFWIN][x + RESTORATION_HALFWIN] *
                  (int)wr_lut_[data_p2[x] - data_p[j]];
-            wtsum += wt;
-            flsum += wt * data_p2[x];
+            wtsum += (int64_t)wt;
+            flsum += (int64_t)wt * data_p2[x];
           }
           data_p2 += stride;
         }
@@ -392,15 +392,15 @@ static void loop_bilateral_filter_tile_highbd(uint16_t *data, int tile_idx,
 
     for (i = 0; i < (v_end - v_start); ++i) {
       for (j = 0; j < (h_end - h_start); ++j) {
-        int x, y;
-        int flsum = 0, wtsum = 0, wt;
+        int x, y, wt;
+        int64_t flsum = 0, wtsum = 0;
         uint16_t *data_p2 = data_p + j - RESTORATION_HALFWIN * stride;
         for (y = -RESTORATION_HALFWIN; y <= RESTORATION_HALFWIN; ++y) {
           for (x = -RESTORATION_HALFWIN; x <= RESTORATION_HALFWIN; ++x) {
             wt = (int)wx_lut[y + RESTORATION_HALFWIN][x + RESTORATION_HALFWIN] *
                  (int)wr_lut_[data_p2[x] - data_p[j]];
-            wtsum += wt;
-            flsum += wt * data_p2[x];
+            wtsum += (int64_t)wt;
+            flsum += (int64_t)wt * data_p2[x];
           }
           data_p2 += stride;
         }