From 215084b31c3e5683c75a558911172d8f34661333 Mon Sep 17 00:00:00 2001 From: James Zern Date: Tue, 5 Mar 2013 12:02:28 -0800 Subject: [PATCH] vp8: clamp probability values fixes out of bounds reads on vp8_prob_cost table. present since: 217591f Added rate-targeted temporal scalability v0.9.7-p1-71-g217591f Change-Id: I9194b773098a381f1e3f67bd3307f05df27c24fd --- vp8/encoder/bitstream.c | 6 ++++++ vp8/encoder/onyx_if.c | 2 ++ 2 files changed, 8 insertions(+) diff --git a/vp8/encoder/bitstream.c b/vp8/encoder/bitstream.c index e666b6c7e..8f94171e6 100644 --- a/vp8/encoder/bitstream.c +++ b/vp8/encoder/bitstream.c @@ -980,6 +980,12 @@ void vp8_calc_ref_frame_costs(int *ref_frame_cost, int prob_garf ) { + assert(prob_intra >= 0); + assert(prob_intra <= 255); + assert(prob_last >= 0); + assert(prob_last <= 255); + assert(prob_garf >= 0); + assert(prob_garf <= 255); ref_frame_cost[INTRA_FRAME] = vp8_cost_zero(prob_intra); ref_frame_cost[LAST_FRAME] = vp8_cost_one(prob_intra) + vp8_cost_zero(prob_last); diff --git a/vp8/encoder/onyx_if.c b/vp8/encoder/onyx_if.c index 92f981857..555a9e4bc 100644 --- a/vp8/encoder/onyx_if.c +++ b/vp8/encoder/onyx_if.c @@ -2816,6 +2816,8 @@ static void update_rd_ref_frame_probs(VP8_COMP *cpi) if (cpi->common.refresh_alt_ref_frame) { cpi->prob_intra_coded += 40; + if (cpi->prob_intra_coded > 255) + cpi->prob_intra_coded = 255; cpi->prob_last_coded = 200; cpi->prob_gf_coded = 1; } -- 2.40.0