From 3e3ca7faa6c9e68be3058e175187f382f050f8da Mon Sep 17 00:00:00 2001 From: Jingning Han Date: Thu, 28 Jun 2018 12:06:12 -0700 Subject: [PATCH] Avoid operation on INT64_MAX value If the rate cost returns as INT_MAX, directly set the rdcost as INT64_MAX. Change-Id: I3ea1963aff10040dd9cef805beed9aebeedb93bc --- vp9/encoder/vp9_encodeframe.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/vp9/encoder/vp9_encodeframe.c b/vp9/encoder/vp9_encodeframe.c index f6fcd9d33..8b426451c 100644 --- a/vp9/encoder/vp9_encodeframe.c +++ b/vp9/encoder/vp9_encodeframe.c @@ -1967,13 +1967,14 @@ static void rd_pick_sb_modes(VP9_COMP *cpi, TileDataEnc *tile_data, vp9_caq_select_segment(cpi, x, bsize, mi_row, mi_col, rd_cost->rate); } - rd_cost->rdcost = RDCOST(x->rdmult, x->rddiv, rd_cost->rate, rd_cost->dist); - x->rdmult = orig_rdmult; // TODO(jingning) The rate-distortion optimization flow needs to be // refactored to provide proper exit/return handle. - if (rd_cost->rate == INT_MAX) rd_cost->rdcost = INT64_MAX; + if (rd_cost->rate == INT_MAX) + rd_cost->rdcost = INT64_MAX; + else + rd_cost->rdcost = RDCOST(x->rdmult, x->rddiv, rd_cost->rate, rd_cost->dist); ctx->rate = rd_cost->rate; ctx->dist = rd_cost->dist; -- 2.40.0