From: hui su Date: Tue, 29 Mar 2016 23:28:28 +0000 (-0700) Subject: Set block size upper bound for Palette mode X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=cbb8be769df5dfbf53b70a11981bf53e3c40b418;p=libvpx Set block size upper bound for Palette mode Avoid buffer overflow in case of such new experiments as 128 x 128 superblock size. Change-Id: Ib775f3925a85fc87227c0ddd9b6a6110a12ef196 --- diff --git a/vp10/common/entropymode.h b/vp10/common/entropymode.h index 8219dc5e0..b14fbfde3 100644 --- a/vp10/common/entropymode.h +++ b/vp10/common/entropymode.h @@ -34,6 +34,7 @@ extern "C" { #define PALETTE_MAX_SIZE 8 #define PALETTE_BLOCK_SIZES (BLOCK_LARGEST - BLOCK_8X8 + 1) #define PALETTE_Y_MODE_CONTEXTS 3 +#define PALETTE_MAX_BLOCK_SIZE (64 * 64) struct VP10Common; diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c index 328e70c75..481a17e04 100644 --- a/vp10/encoder/rdopt.c +++ b/vp10/encoder/rdopt.c @@ -1823,6 +1823,9 @@ static int rd_pick_palette_intra_sby(VP10_COMP *cpi, MACROBLOCK *x, mbmi->ext_intra_mode_info.use_ext_intra_mode[0] = 0; #endif // CONFIG_EXT_INTRA + if (rows * cols > PALETTE_MAX_BLOCK_SIZE) + return 0; + for (n = colors > PALETTE_MAX_SIZE ? PALETTE_MAX_SIZE : colors; n >= 2; --n) { for (i = 0; i < n; ++i) @@ -3552,6 +3555,9 @@ static void rd_pick_palette_intra_sbuv(VP10_COMP *cpi, MACROBLOCK *x, const uint8_t *const src_u = x->plane[1].src.buf; const uint8_t *const src_v = x->plane[2].src.buf; + if (rows * cols > PALETTE_MAX_BLOCK_SIZE) + return; + #if CONFIG_EXT_INTRA mbmi->ext_intra_mode_info.use_ext_intra_mode[1] = 0; #endif // CONFIG_EXT_INTRA