From: hui su Date: Thu, 16 Apr 2015 18:18:58 +0000 (-0700) Subject: Fix palette expt asan failure X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=8c00c7a9cd7f5773f0bedc23d5018c62c587c50c;p=libvpx Fix palette expt asan failure Account for 422 video format. Change-Id: Ic5af661720fc5fa7142210d907dd25e1e79ff653 --- diff --git a/vp9/common/vp9_reconintra.c b/vp9/common/vp9_reconintra.c index a35af897f..7b83b809b 100644 --- a/vp9/common/vp9_reconintra.c +++ b/vp9/common/vp9_reconintra.c @@ -1386,7 +1386,7 @@ void vp9_predict_intra_block(const MACROBLOCKD *xd, int block_idx, int bwl_in, if (!filterflag) { #endif // CONFIG_FILTERINTRA #if CONFIG_PALETTE - if (xd->mi[0].src_mi->mbmi.palette_enabled[plane !=0 ]) { + if (xd->mi[0].src_mi->mbmi.palette_enabled[plane !=0]) { uint8_t *palette = xd->mi[0].src_mi->mbmi.palette_colors + plane * PALETTE_MAX_SIZE; int bs = 4 * (1 << tx_size); @@ -1394,9 +1394,9 @@ void vp9_predict_intra_block(const MACROBLOCKD *xd, int block_idx, int bwl_in, int r, c; uint8_t *map = NULL; - if (xd->plane[1].subsampling_x && xd->plane[1].subsampling_y) + if (xd->plane[1].subsampling_x || xd->plane[1].subsampling_y) map = xd->plane[plane != 0].color_index_map; - else if (!xd->plane[1].subsampling_x && !xd->plane[1].subsampling_y) + else map = xd->plane[0].color_index_map; for (r = 0; r < bs; r++) { diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c index 7a90d112d..0070962fe 100644 --- a/vp9/encoder/vp9_rdopt.c +++ b/vp9/encoder/vp9_rdopt.c @@ -2459,7 +2459,7 @@ static int64_t rd_pick_intra_sbuv_mode(VP9_COMP *cpi, MACROBLOCK *x, #if CONFIG_PALETTE if (xd->mi[0].src_mi->mbmi.sb_type >= BLOCK_8X8 && - xd->plane[1].subsampling_x && xd->plane[1].subsampling_y && + (xd->plane[1].subsampling_x || xd->plane[1].subsampling_y) && cpi->common.allow_palette_mode) { int colors_u = vp9_count_colors(src_u, src_stride, rows, cols); int colors_v = vp9_count_colors(src_v, src_stride, rows, cols);