vp8: clamp probability values

fixes out of bounds reads on vp8_prob_cost table.
present since:
217591f Added rate-targeted temporal scalability
v0.9.7-p1-71-g217591f

Change-Id: I9194b773098a381f1e3f67bd3307f05df27c24fd

diff --git a/vp8/encoder/bitstream.c b/vp8/encoder/bitstream.c
index e666b6c7e1c86c04cf9b3b4cdbb6f1260bfcaa0c..8f94171e692e9a5a3374d6f352b1f0b23424dd47 100644 (file)
--- a/vp8/encoder/bitstream.c
+++ b/vp8/encoder/bitstream.c
@@ -980,6 +980,12 @@ void vp8_calc_ref_frame_costs(int *ref_frame_cost,
                               int prob_garf
                              )
 {
+    assert(prob_intra >= 0);
+    assert(prob_intra <= 255);
+    assert(prob_last >= 0);
+    assert(prob_last <= 255);
+    assert(prob_garf >= 0);
+    assert(prob_garf <= 255);
     ref_frame_cost[INTRA_FRAME]   = vp8_cost_zero(prob_intra);
     ref_frame_cost[LAST_FRAME]    = vp8_cost_one(prob_intra)
                                     + vp8_cost_zero(prob_last);

diff --git a/vp8/encoder/onyx_if.c b/vp8/encoder/onyx_if.c
index 92f9818577436368e5e665c480e9da8c92256179..555a9e4bcade5b3b76d5c693ba22108b66ebb86b 100644 (file)
--- a/vp8/encoder/onyx_if.c
+++ b/vp8/encoder/onyx_if.c
@@ -2816,6 +2816,8 @@ static void update_rd_ref_frame_probs(VP8_COMP *cpi)
         if (cpi->common.refresh_alt_ref_frame)
         {
             cpi->prob_intra_coded += 40;
+            if (cpi->prob_intra_coded > 255)
+                cpi->prob_intra_coded = 255;
             cpi->prob_last_coded = 200;
             cpi->prob_gf_coded = 1;
         }