Fix VP8 encoder UBSAN/IOC errors

1. vp8/decoder/dboolhuff.c
2. vp8/decoder/dboolhuff.h
3. vp8/encoder/bitstream.c
4. vp8/encoder/boolhuff.h
5. vp8/encoder/rdopt.c

BUG=https://bugs.chromium.org/p/webm/issues/detail?id=1218

Change-Id: I5d315d63fd7aeaee6f3bd79178e593f3db38a6b1

diff --git a/vp8/decoder/dboolhuff.c b/vp8/decoder/dboolhuff.c
index 8a7e33205b9dddf6bcef418e859df5359770d0f1..5cdd2a249156020ca2bb6f464e447a7f54924f16 100644 (file)
--- a/vp8/decoder/dboolhuff.c
+++ b/vp8/decoder/dboolhuff.c
@@ -44,7 +44,7 @@ void vp8dx_bool_decoder_fill(BOOL_DECODER *br)
     int shift = VP8_BD_VALUE_SIZE - CHAR_BIT - (count + CHAR_BIT);
     size_t bytes_left = br->user_buffer_end - bufptr;
     size_t bits_left = bytes_left * CHAR_BIT;
-    int x = (int)(shift + CHAR_BIT - bits_left);
+    int x = shift + CHAR_BIT - (int)bits_left;
     int loop_end = 0;
     unsigned char decrypted[sizeof(VP8_BD_VALUE) + 1];
 

diff --git a/vp8/decoder/dboolhuff.h b/vp8/decoder/dboolhuff.h
index cc9eaaf4399bcb51e6e834fa442789401adeeac7..1b1bbf868ec062524aae551300abf7393b12b87b 100644 (file)
--- a/vp8/decoder/dboolhuff.h
+++ b/vp8/decoder/dboolhuff.h
@@ -83,7 +83,7 @@ static int vp8dx_decode_bool(BOOL_DECODER *br, int probability) {
     }
 
     {
-        register unsigned int shift = vp8_norm[range];
+        register int shift = vp8_norm[range];
         range <<= shift;
         value <<= shift;
         count -= shift;

diff --git a/vp8/encoder/bitstream.c b/vp8/encoder/bitstream.c
index f3d91b5528bc18be11cff361e1c73551ba25e0ea..3196422c28d0d1ab67959920d5825c90419fd5ca 100644 (file)
--- a/vp8/encoder/bitstream.c
+++ b/vp8/encoder/bitstream.c
@@ -163,7 +163,7 @@ void vp8_pack_tokens(vp8_writer *w, const TOKENEXTRA *p, int xcount)
 {
     const TOKENEXTRA *stop = p + xcount;
     unsigned int split;
-    unsigned int shift;
+    int shift;
     int count = w->count;
     unsigned int range = w->range;
     unsigned int lowvalue = w->lowvalue;

diff --git a/vp8/encoder/boolhuff.h b/vp8/encoder/boolhuff.h
index 7c012a8296963a408d7213631ab1cdd135f69ded..e66a2dbd8e6d0a7fe259a85bbd3711b7fa28df62 100644 (file)
--- a/vp8/encoder/boolhuff.h
+++ b/vp8/encoder/boolhuff.h
@@ -65,7 +65,7 @@ static void vp8_encode_bool(BOOL_CODER *br, int bit, int probability)
     int count = br->count;
     unsigned int range = br->range;
     unsigned int lowvalue = br->lowvalue;
-    register unsigned int shift;
+    register int shift;
 
 #ifdef VP8_ENTROPY_STATS
 #if defined(SECTIONBITS_OUTPUT)

diff --git a/vp8/encoder/rdopt.c b/vp8/encoder/rdopt.c
index 9063cea76632c3f34c8c412dcf07e4eff0023896..6507ae9f13abe0109efae671ef8dbea6f732b29a 100644 (file)
--- a/vp8/encoder/rdopt.c
+++ b/vp8/encoder/rdopt.c
@@ -1899,7 +1899,8 @@ static int calculate_final_rd_costs(int this_rd,
                     int prob_skip_cost;
 
                     prob_skip_cost = vp8_cost_bit(cpi->prob_skip_false, 1);
-                    prob_skip_cost -= vp8_cost_bit(cpi->prob_skip_false, 0);
+                    prob_skip_cost -=
+                        (int)vp8_cost_bit(cpi->prob_skip_false, 0);
                     rd->rate2 += prob_skip_cost;
                     *other_cost += prob_skip_cost;
                 }