Set block size upper bound for Palette mode

Avoid buffer overflow in case of such new experiments as
128 x 128 superblock size.

Change-Id: Ib775f3925a85fc87227c0ddd9b6a6110a12ef196

diff --git a/vp10/common/entropymode.h b/vp10/common/entropymode.h
index 8219dc5e003d1a032526d349ef4aeefd34ccf7d4..b14fbfde32fd8f37e935069e846a432ae49e580a 100644 (file)
--- a/vp10/common/entropymode.h
+++ b/vp10/common/entropymode.h
@@ -34,6 +34,7 @@ extern "C" {
 #define PALETTE_MAX_SIZE 8
 #define PALETTE_BLOCK_SIZES (BLOCK_LARGEST - BLOCK_8X8 + 1)
 #define PALETTE_Y_MODE_CONTEXTS 3
+#define PALETTE_MAX_BLOCK_SIZE (64 * 64)
 
 struct VP10Common;
 

diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c
index 328e70c75b66b18500dc408492b2e5c621c73974..481a17e04b3f5d74cad196f76484be968af6e5b0 100644 (file)
--- a/vp10/encoder/rdopt.c
+++ b/vp10/encoder/rdopt.c
@@ -1823,6 +1823,9 @@ static int rd_pick_palette_intra_sby(VP10_COMP *cpi, MACROBLOCK *x,
     mbmi->ext_intra_mode_info.use_ext_intra_mode[0] = 0;
 #endif  // CONFIG_EXT_INTRA
 
+    if (rows * cols > PALETTE_MAX_BLOCK_SIZE)
+      return 0;
+
     for (n = colors > PALETTE_MAX_SIZE ? PALETTE_MAX_SIZE : colors;
         n >= 2; --n) {
       for (i = 0; i < n; ++i)
@@ -3552,6 +3555,9 @@ static void rd_pick_palette_intra_sbuv(VP10_COMP *cpi, MACROBLOCK *x,
   const uint8_t *const src_u = x->plane[1].src.buf;
   const uint8_t *const src_v = x->plane[2].src.buf;
 
+  if (rows * cols > PALETTE_MAX_BLOCK_SIZE)
+    return;
+
 #if CONFIG_EXT_INTRA
   mbmi->ext_intra_mode_info.use_ext_intra_mode[1] = 0;
 #endif  // CONFIG_EXT_INTRA