From 5041aa0fbe225631547066211ee1c02a553ed1ae Mon Sep 17 00:00:00 2001 From: Jingning Han Date: Thu, 12 Feb 2015 12:08:30 -0800 Subject: [PATCH] Fix ioc issue in block_rd_txfm Force 64-bit precision in the intermediate steps. Change-Id: I666113d9adcef8975da201d5aa1a13b783d09594 --- vp9/encoder/vp9_rdopt.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c index 79d118fce..53a5dba19 100644 --- a/vp9/encoder/vp9_rdopt.c +++ b/vp9/encoder/vp9_rdopt.c @@ -509,8 +509,9 @@ static void block_rd_txfm(int plane, int block, BLOCK_SIZE plane_bsize, args->sse = x->bsse[(plane << 2) + (block >> (tx_size << 1))] << 4; args->dist = args->sse; if (x->plane[plane].eobs[block]) { - int64_t dc_correct = coeff[0] * coeff[0] - - (coeff[0] - dqcoeff[0]) * (coeff[0] - dqcoeff[0]); + const int64_t orig_sse = (int64_t)coeff[0] * coeff[0]; + const int64_t resd_sse = coeff[0] - dqcoeff[0]; + int64_t dc_correct = orig_sse - resd_sse * resd_sse; #if CONFIG_VP9_HIGHBITDEPTH dc_correct >>= ((xd->bd - 8) * 2); #endif -- 2.40.0