From: James Zern <jzern@google.com>
Date: Wed, 22 Mar 2017 06:29:12 +0000 (-0700)
Subject: vp9_rdopt: correct size to vpx_sum_squares_2d_i16
X-Git-Tag: v1.7.0~601^2
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=5661cd8ff4ab35d4eac487f6b67484fd60a10ce4;p=libvpx

vp9_rdopt: correct size to vpx_sum_squares_2d_i16

the current implementations expect pixel size, not the block type

BUG=webm:1392

Change-Id: Ib91e9f30a1f56e13566b1fb76f089dae9bb50cdc
---

diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c
index 0432eebd5..bbfc2dc4b 100644
--- a/vp9/encoder/vp9_rdopt.c
+++ b/vp9/encoder/vp9_rdopt.c
@@ -515,7 +515,8 @@ static int64_t sum_squares_visible(const MACROBLOCKD *xd,
                                              pd->subsampling_y, blk_row);
   if (tx_bsize == BLOCK_4X4 ||
       (b4x4s_to_right_edge >= tx_4x4_w && b4x4s_to_bottom_edge >= tx_4x4_h)) {
-    sse = (int64_t)vpx_sum_squares_2d_i16(diff, diff_stride, tx_bsize);
+    assert(tx_4x4_w == tx_4x4_h);
+    sse = (int64_t)vpx_sum_squares_2d_i16(diff, diff_stride, tx_4x4_w << 2);
   } else {
     int r, c;
     int max_r = VPXMIN(b4x4s_to_bottom_edge, tx_4x4_h);
@@ -525,7 +526,8 @@ static int64_t sum_squares_visible(const MACROBLOCKD *xd,
     for (r = 0; r < max_r; ++r) {
       // Skip visiting the sub blocks that are wholly within the UMV.
       for (c = 0; c < max_c; ++c) {
-        sse += (int64_t)vpx_sum_squares_2d_i16(diff, diff_stride, BLOCK_4X4);
+        sse += (int64_t)vpx_sum_squares_2d_i16(
+            diff + r * diff_stride * 4 + c * 4, diff_stride, 4);
       }
     }
   }
diff --git a/vpx_dsp/x86/sum_squares_sse2.c b/vpx_dsp/x86/sum_squares_sse2.c
index bc5362e10..026d0ca2f 100644
--- a/vpx_dsp/x86/sum_squares_sse2.c
+++ b/vpx_dsp/x86/sum_squares_sse2.c
@@ -123,6 +123,7 @@ uint64_t vpx_sum_squares_2d_i16_sse2(const int16_t *src, int stride, int size) {
     return vpx_sum_squares_2d_i16_4x4_sse2(src, stride);
   } else {
     // Generic case
+    assert(size % 8 == 0);
     return vpx_sum_squares_2d_i16_nxn_sse2(src, stride, size);
   }
 }