From cbb8be769df5dfbf53b70a11981bf53e3c40b418 Mon Sep 17 00:00:00 2001 From: hui su Date: Tue, 29 Mar 2016 16:28:28 -0700 Subject: [PATCH] Set block size upper bound for Palette mode Avoid buffer overflow in case of such new experiments as 128 x 128 superblock size. Change-Id: Ib775f3925a85fc87227c0ddd9b6a6110a12ef196 --- vp10/common/entropymode.h | 1 + vp10/encoder/rdopt.c | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/vp10/common/entropymode.h b/vp10/common/entropymode.h index 8219dc5e0..b14fbfde3 100644 --- a/vp10/common/entropymode.h +++ b/vp10/common/entropymode.h @@ -34,6 +34,7 @@ extern "C" { #define PALETTE_MAX_SIZE 8 #define PALETTE_BLOCK_SIZES (BLOCK_LARGEST - BLOCK_8X8 + 1) #define PALETTE_Y_MODE_CONTEXTS 3 +#define PALETTE_MAX_BLOCK_SIZE (64 * 64) struct VP10Common; diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c index 328e70c75..481a17e04 100644 --- a/vp10/encoder/rdopt.c +++ b/vp10/encoder/rdopt.c @@ -1823,6 +1823,9 @@ static int rd_pick_palette_intra_sby(VP10_COMP *cpi, MACROBLOCK *x, mbmi->ext_intra_mode_info.use_ext_intra_mode[0] = 0; #endif // CONFIG_EXT_INTRA + if (rows * cols > PALETTE_MAX_BLOCK_SIZE) + return 0; + for (n = colors > PALETTE_MAX_SIZE ? PALETTE_MAX_SIZE : colors; n >= 2; --n) { for (i = 0; i < n; ++i) @@ -3552,6 +3555,9 @@ static void rd_pick_palette_intra_sbuv(VP10_COMP *cpi, MACROBLOCK *x, const uint8_t *const src_u = x->plane[1].src.buf; const uint8_t *const src_v = x->plane[2].src.buf; + if (rows * cols > PALETTE_MAX_BLOCK_SIZE) + return; + #if CONFIG_EXT_INTRA mbmi->ext_intra_mode_info.use_ext_intra_mode[1] = 0; #endif // CONFIG_EXT_INTRA -- 2.40.0