From 431cdc33eee902bbc1c4272a8551e3734fe4a16f Mon Sep 17 00:00:00 2001 From: Deb Mukherjee Date: Fri, 3 Oct 2014 10:22:21 -0700 Subject: [PATCH] Prevent negative cost for highbitdepth Adds proper scaling for highbitdepth in a rdopt cost. Change-Id: I066694799a7f491b830945ef1c66eb202071c355 --- vp9/encoder/vp9_rdopt.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c index b14ff0822..0ae0d5757 100644 --- a/vp9/encoder/vp9_rdopt.c +++ b/vp9/encoder/vp9_rdopt.c @@ -491,7 +491,9 @@ static void block_rd_txfm(int plane, int block, BLOCK_SIZE plane_bsize, if (x->plane[plane].eobs[block]) { int64_t dc_correct = coeff[0] * coeff[0] - (coeff[0] - dqcoeff[0]) * (coeff[0] - dqcoeff[0]); - +#if CONFIG_VP9_HIGHBITDEPTH + dc_correct >>= ((xd->bd - 8) * 2); +#endif if (tx_size != TX_32X32) dc_correct >>= 2; -- 2.40.0