From 343352b556f5f61833174c08a35d697d280933e3 Mon Sep 17 00:00:00 2001 From: Jerome Jiang Date: Wed, 13 Jun 2018 12:55:17 -0700 Subject: [PATCH] vp8: remove assertion in tree coder. Cast the counter to uint64_t in case it overflows. The assert was to prevent c[0] * Pfac being overflow beyong unsigned int since Pfac could be 2^8. Thus c[0] needs to be smaller than 2^24. In VP9, the assert was removed and c[0] was casted to uint64_t. Bug: 805277 Change-Id: Ic46a3c5b4af2f267de4e32c1518b64e8d6e9d856 --- vp8/common/treecoder.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vp8/common/treecoder.c b/vp8/common/treecoder.c index 9feb40a5a..8a94cdeec 100644 --- a/vp8/common/treecoder.c +++ b/vp8/common/treecoder.c @@ -12,6 +12,7 @@ #include #include "vp8/common/treecoder.h" +#include "vpx/vpx_integer.h" static void tree2tok(struct vp8_token_struct *const p, vp8_tree t, int i, int v, int L) { @@ -89,10 +90,9 @@ void vp8_tree_probs_from_distribution(int n, /* n = size of alphabet */ const unsigned int *const c = branch_ct[t]; const unsigned int tot = c[0] + c[1]; - assert(tot < (1 << 24)); /* no overflow below */ - if (tot) { - const unsigned int p = ((c[0] * Pfac) + (rd ? tot >> 1 : 0)) / tot; + const unsigned int p = + (unsigned int)(((uint64_t)c[0] * Pfac) + (rd ? tot >> 1 : 0)) / tot; probs[t] = p < 256 ? (p ? p : 1) : 255; /* agree w/old version for now */ } else { probs[t] = vp8_prob_half; -- 2.40.0