From 5661cd8ff4ab35d4eac487f6b67484fd60a10ce4 Mon Sep 17 00:00:00 2001 From: James Zern Date: Tue, 21 Mar 2017 23:29:12 -0700 Subject: [PATCH] vp9_rdopt: correct size to vpx_sum_squares_2d_i16 the current implementations expect pixel size, not the block type BUG=webm:1392 Change-Id: Ib91e9f30a1f56e13566b1fb76f089dae9bb50cdc --- vp9/encoder/vp9_rdopt.c | 6 ++++-- vpx_dsp/x86/sum_squares_sse2.c | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c index 0432eebd5..bbfc2dc4b 100644 --- a/vp9/encoder/vp9_rdopt.c +++ b/vp9/encoder/vp9_rdopt.c @@ -515,7 +515,8 @@ static int64_t sum_squares_visible(const MACROBLOCKD *xd, pd->subsampling_y, blk_row); if (tx_bsize == BLOCK_4X4 || (b4x4s_to_right_edge >= tx_4x4_w && b4x4s_to_bottom_edge >= tx_4x4_h)) { - sse = (int64_t)vpx_sum_squares_2d_i16(diff, diff_stride, tx_bsize); + assert(tx_4x4_w == tx_4x4_h); + sse = (int64_t)vpx_sum_squares_2d_i16(diff, diff_stride, tx_4x4_w << 2); } else { int r, c; int max_r = VPXMIN(b4x4s_to_bottom_edge, tx_4x4_h); @@ -525,7 +526,8 @@ static int64_t sum_squares_visible(const MACROBLOCKD *xd, for (r = 0; r < max_r; ++r) { // Skip visiting the sub blocks that are wholly within the UMV. for (c = 0; c < max_c; ++c) { - sse += (int64_t)vpx_sum_squares_2d_i16(diff, diff_stride, BLOCK_4X4); + sse += (int64_t)vpx_sum_squares_2d_i16( + diff + r * diff_stride * 4 + c * 4, diff_stride, 4); } } } diff --git a/vpx_dsp/x86/sum_squares_sse2.c b/vpx_dsp/x86/sum_squares_sse2.c index bc5362e10..026d0ca2f 100644 --- a/vpx_dsp/x86/sum_squares_sse2.c +++ b/vpx_dsp/x86/sum_squares_sse2.c @@ -123,6 +123,7 @@ uint64_t vpx_sum_squares_2d_i16_sse2(const int16_t *src, int stride, int size) { return vpx_sum_squares_2d_i16_4x4_sse2(src, stride); } else { // Generic case + assert(size % 8 == 0); return vpx_sum_squares_2d_i16_nxn_sse2(src, stride, size); } } -- 2.40.0