From: Yunqing Wang Date: Fri, 2 Nov 2012 20:06:51 +0000 (-0700) Subject: Fix eobs data type X-Git-Tag: v1.3.0~1217^2~118^2~11^2 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=d41b0e6498a8ef72a002bf72a6928666b7758c53;p=libvpx Fix eobs data type The block sizes for decoding tokens are up to 16x16, which means eobs is within [0, 256]. Using (signed) char is not enough. Changed eobs data type to unsigned short to fix the problem. Change-Id: I88a7d3098e1f1604c336d6adb88ffec971fb03a6 --- diff --git a/vp9/common/blockd.h b/vp9/common/blockd.h index ec010e9b1..0b1fd46b4 100644 --- a/vp9/common/blockd.h +++ b/vp9/common/blockd.h @@ -280,7 +280,7 @@ typedef struct macroblockd { DECLARE_ALIGNED(16, unsigned char, predictor[384]); DECLARE_ALIGNED(16, short, qcoeff[400]); DECLARE_ALIGNED(16, short, dqcoeff[400]); - DECLARE_ALIGNED(16, char, eobs[25]); + DECLARE_ALIGNED(16, unsigned short, eobs[25]); /* 16 Y blocks, 4 U, 4 V, 1 DC 2nd order block, each with 16 entries. */ BLOCKD block[25]; diff --git a/vp9/common/rtcd_defs.sh b/vp9/common/rtcd_defs.sh index f909c1898..3a37b24d0 100644 --- a/vp9/common/rtcd_defs.sh +++ b/vp9/common/rtcd_defs.sh @@ -43,13 +43,13 @@ specialize vp9_dequantize_b mmx prototype void vp9_dequantize_b_2x2 "struct blockd *x" specialize vp9_dequantize_b_2x2 -prototype void vp9_dequant_dc_idct_add_y_block_8x8 "short *q, short *dq, unsigned char *pre, unsigned char *dst, int stride, char *eobs, short *dc, struct macroblockd *xd" +prototype void vp9_dequant_dc_idct_add_y_block_8x8 "short *q, short *dq, unsigned char *pre, unsigned char *dst, int stride, unsigned short *eobs, short *dc, struct macroblockd *xd" specialize vp9_dequant_dc_idct_add_y_block_8x8 -prototype void vp9_dequant_idct_add_y_block_8x8 "short *q, short *dq, unsigned char *pre, unsigned char *dst, int stride, char *eobs, struct macroblockd *xd" +prototype void vp9_dequant_idct_add_y_block_8x8 "short *q, short *dq, unsigned char *pre, unsigned char *dst, int stride, unsigned short *eobs, struct macroblockd *xd" specialize vp9_dequant_idct_add_y_block_8x8 -prototype void vp9_dequant_idct_add_uv_block_8x8 "short *q, short *dq, unsigned char *pre, unsigned char *dstu, unsigned char *dstv, int stride, char *eobs, struct macroblockd *xd" +prototype void vp9_dequant_idct_add_uv_block_8x8 "short *q, short *dq, unsigned char *pre, unsigned char *dstu, unsigned char *dstv, int stride, unsigned short *eobs, struct macroblockd *xd" specialize vp9_dequant_idct_add_uv_block_8x8 prototype void vp9_dequant_idct_add_16x16 "short *input, short *dq, unsigned char *pred, unsigned char *dest, int pitch, int stride" @@ -61,13 +61,13 @@ specialize vp9_dequant_idct_add prototype void vp9_dequant_dc_idct_add "short *input, short *dq, unsigned char *pred, unsigned char *dest, int pitch, int stride, int Dc" specialize vp9_dequant_dc_idct_add -prototype void vp9_dequant_dc_idct_add_y_block "short *q, short *dq, unsigned char *pre, unsigned char *dst, int stride, char *eobs, short *dc" +prototype void vp9_dequant_dc_idct_add_y_block "short *q, short *dq, unsigned char *pre, unsigned char *dst, int stride, unsigned short *eobs, short *dc" specialize vp9_dequant_dc_idct_add_y_block mmx -prototype void vp9_dequant_idct_add_y_block "short *q, short *dq, unsigned char *pre, unsigned char *dst, int stride, char *eobs" +prototype void vp9_dequant_idct_add_y_block "short *q, short *dq, unsigned char *pre, unsigned char *dst, int stride, unsigned short *eobs" specialize vp9_dequant_idct_add_y_block mmx -prototype void vp9_dequant_idct_add_uv_block "short *q, short *dq, unsigned char *pre, unsigned char *dstu, unsigned char *dstv, int stride, char *eobs" +prototype void vp9_dequant_idct_add_uv_block "short *q, short *dq, unsigned char *pre, unsigned char *dstu, unsigned char *dstv, int stride, unsigned short *eobs" specialize vp9_dequant_idct_add_uv_block mmx # diff --git a/vp9/decoder/arm/armv6/idct_blk_v6.c b/vp9/decoder/arm/armv6/idct_blk_v6.c index fe9b9035e..e430f2995 100644 --- a/vp9/decoder/arm/armv6/idct_blk_v6.c +++ b/vp9/decoder/arm/armv6/idct_blk_v6.c @@ -12,9 +12,10 @@ #include "vp9/common/idct.h" #include "vp9/decoder/dequantize.h" -void vp8_dequant_dc_idct_add_y_block_v6 -(short *q, short *dq, unsigned char *pre, - unsigned char *dst, int stride, char *eobs, short *dc) { +void vp8_dequant_dc_idct_add_y_block_v6(short *q, short *dq, + unsigned char *pre, + unsigned char *dst, int stride, + unsigned short *eobs, short *dc) { int i; for (i = 0; i < 4; i++) { @@ -46,9 +47,9 @@ void vp8_dequant_dc_idct_add_y_block_v6 } } -void vp8_dequant_idct_add_y_block_v6 -(short *q, short *dq, unsigned char *pre, - unsigned char *dst, int stride, char *eobs) { +void vp8_dequant_idct_add_y_block_v6(short *q, short *dq, unsigned char *pre, + unsigned char *dst, int stride, + unsigned short *eobs) { int i; for (i = 0; i < 4; i++) { @@ -87,9 +88,9 @@ void vp8_dequant_idct_add_y_block_v6 } } -void vp8_dequant_idct_add_uv_block_v6 -(short *q, short *dq, unsigned char *pre, - unsigned char *dstu, unsigned char *dstv, int stride, char *eobs) { +void vp8_dequant_idct_add_uv_block_v6(short *q, short *dq, unsigned char *pre, + unsigned char *dstu, unsigned char *dstv, + int stride, unsigned short *eobs) { int i; for (i = 0; i < 2; i++) { diff --git a/vp9/decoder/arm/neon/idct_blk_neon.c b/vp9/decoder/arm/neon/idct_blk_neon.c index fb5d298df..5711e86e8 100644 --- a/vp9/decoder/arm/neon/idct_blk_neon.c +++ b/vp9/decoder/arm/neon/idct_blk_neon.c @@ -27,9 +27,10 @@ void idct_dequant_0_2x_neon (short *q, short dq, unsigned char *pre, int pitch, unsigned char *dst, int stride); -void vp8_dequant_dc_idct_add_y_block_neon -(short *q, short *dq, unsigned char *pre, - unsigned char *dst, int stride, char *eobs, short *dc) { +void vp8_dequant_dc_idct_add_y_block_neon(short *q, short *dq, + unsigned char *pre, + unsigned char *dst, int stride, + unsigned short *eobs, short *dc) { int i; for (i = 0; i < 4; i++) { @@ -51,9 +52,9 @@ void vp8_dequant_dc_idct_add_y_block_neon } } -void vp8_dequant_idct_add_y_block_neon -(short *q, short *dq, unsigned char *pre, - unsigned char *dst, int stride, char *eobs) { +void vp8_dequant_idct_add_y_block_neon(short *q, short *dq, unsigned char *pre, + unsigned char *dst, int stride, + unsigned short *eobs) { int i; for (i = 0; i < 4; i++) { @@ -74,9 +75,11 @@ void vp8_dequant_idct_add_y_block_neon } } -void vp8_dequant_idct_add_uv_block_neon -(short *q, short *dq, unsigned char *pre, - unsigned char *dstu, unsigned char *dstv, int stride, char *eobs) { +void vp8_dequant_idct_add_uv_block_neon(short *q, short *dq, + unsigned char *pre, + unsigned char *dstu, + unsigned char *dstv, int stride, + unsigned short *eobs) { if (((short *)eobs)[0] & 0xfefe) idct_dequant_full_2x_neon(q, dq, pre, dstu, 8, stride); else diff --git a/vp9/decoder/dequantize.h b/vp9/decoder/dequantize.h index 912061f28..560c4a417 100644 --- a/vp9/decoder/dequantize.h +++ b/vp9/decoder/dequantize.h @@ -25,17 +25,20 @@ extern void vp9_dequant_dc_idct_add_lossless_c(short *input, short *dq, extern void vp9_dequant_dc_idct_add_y_block_lossless_c(short *q, short *dq, unsigned char *pre, unsigned char *dst, - int stride, char *eobs, + int stride, + unsigned short *eobs, short *dc); extern void vp9_dequant_idct_add_y_block_lossless_c(short *q, short *dq, unsigned char *pre, unsigned char *dst, - int stride, char *eobs); + int stride, + unsigned short *eobs); extern void vp9_dequant_idct_add_uv_block_lossless_c(short *q, short *dq, unsigned char *pre, unsigned char *dst_u, unsigned char *dst_v, - int stride, char *eobs); + int stride, + unsigned short *eobs); #endif typedef void (*vp9_dequant_idct_add_fn_t)(short *input, short *dq, @@ -44,12 +47,13 @@ typedef void(*vp9_dequant_dc_idct_add_fn_t)(short *input, short *dq, unsigned char *pred, unsigned char *output, int pitch, int stride, int dc); typedef void(*vp9_dequant_dc_idct_add_y_block_fn_t)(short *q, short *dq, - unsigned char *pre, unsigned char *dst, int stride, char *eobs, short *dc); + unsigned char *pre, unsigned char *dst, int stride, unsigned short *eobs, + short *dc); typedef void(*vp9_dequant_idct_add_y_block_fn_t)(short *q, short *dq, - unsigned char *pre, unsigned char *dst, int stride, char *eobs); + unsigned char *pre, unsigned char *dst, int stride, unsigned short *eobs); typedef void(*vp9_dequant_idct_add_uv_block_fn_t)(short *q, short *dq, unsigned char *pre, unsigned char *dst_u, unsigned char *dst_v, int stride, - char *eobs); + unsigned short *eobs); void vp9_ht_dequant_idct_add_c(TX_TYPE tx_type, short *input, short *dq, unsigned char *pred, unsigned char *dest, @@ -66,12 +70,14 @@ void vp9_ht_dequant_idct_add_16x16_c(TX_TYPE tx_type, short *input, short *dq, #if CONFIG_SUPERBLOCKS void vp9_dequant_dc_idct_add_y_block_8x8_inplace_c(short *q, short *dq, unsigned char *dst, - int stride, char *eobs, + int stride, + unsigned short *eobs, short *dc, MACROBLOCKD *xd); void vp9_dequant_idct_add_uv_block_8x8_inplace_c(short *q, short *dq, unsigned char *dstu, unsigned char *dstv, - int stride, char *eobs, + int stride, + unsigned short *eobs, MACROBLOCKD *xd); #endif diff --git a/vp9/decoder/detokenize.c b/vp9/decoder/detokenize.c index 58f5044ea..62511f0b4 100644 --- a/vp9/decoder/detokenize.c +++ b/vp9/decoder/detokenize.c @@ -419,7 +419,7 @@ int vp9_decode_mb_tokens_16x16(VP9D_COMP *pbi, MACROBLOCKD *xd, ENTROPY_CONTEXT* const A = (ENTROPY_CONTEXT *)xd->above_context; ENTROPY_CONTEXT* const L = (ENTROPY_CONTEXT *)xd->left_context; - char* const eobs = xd->eobs; + unsigned short* const eobs = xd->eobs; PLANE_TYPE type; int c, i, eobtotal = 0, seg_eob; const int segment_id = xd->mode_info_context->mbmi.segment_id; @@ -482,7 +482,7 @@ int vp9_decode_mb_tokens_8x8(VP9D_COMP *pbi, MACROBLOCKD *xd, ENTROPY_CONTEXT *const A = (ENTROPY_CONTEXT *)xd->above_context; ENTROPY_CONTEXT *const L = (ENTROPY_CONTEXT *)xd->left_context; - char *const eobs = xd->eobs; + unsigned short *const eobs = xd->eobs; PLANE_TYPE type; int c, i, eobtotal = 0, seg_eob; const int segment_id = xd->mode_info_context->mbmi.segment_id; @@ -576,7 +576,7 @@ int vp9_decode_mb_tokens(VP9D_COMP *dx, MACROBLOCKD *xd, ENTROPY_CONTEXT *const A = (ENTROPY_CONTEXT *)xd->above_context; ENTROPY_CONTEXT *const L = (ENTROPY_CONTEXT *)xd->left_context; - char *const eobs = xd->eobs; + unsigned short *const eobs = xd->eobs; const int *scan = vp9_default_zig_zag1d; PLANE_TYPE type; int c, i, eobtotal = 0, seg_eob = 16; diff --git a/vp9/decoder/idct_blk.c b/vp9/decoder/idct_blk.c index e9605fc96..88c30abf7 100644 --- a/vp9/decoder/idct_blk.c +++ b/vp9/decoder/idct_blk.c @@ -31,7 +31,7 @@ void vp9_dc_only_idct_add_lossless_c(short input_dc, unsigned char *pred_ptr, void vp9_dequant_dc_idct_add_y_block_c(short *q, short *dq, unsigned char *pre, unsigned char *dst, - int stride, char *eobs, + int stride, unsigned short *eobs, short *dc) { int i, j; @@ -56,7 +56,7 @@ void vp9_dequant_dc_idct_add_y_block_c(short *q, short *dq, void vp9_dequant_idct_add_y_block_c(short *q, short *dq, unsigned char *pre, unsigned char *dst, - int stride, char *eobs) { + int stride, unsigned short *eobs) { int i, j; for (i = 0; i < 4; i++) { @@ -80,7 +80,7 @@ void vp9_dequant_idct_add_y_block_c(short *q, short *dq, void vp9_dequant_idct_add_uv_block_c(short *q, short *dq, unsigned char *pre, unsigned char *dstu, unsigned char *dstv, - int stride, char *eobs) { + int stride, unsigned short *eobs) { int i, j; for (i = 0; i < 2; i++) { @@ -124,7 +124,8 @@ void vp9_dequant_idct_add_uv_block_c(short *q, short *dq, unsigned char *pre, void vp9_dequant_dc_idct_add_y_block_8x8_c(short *q, short *dq, unsigned char *pre, unsigned char *dst, - int stride, char *eobs, short *dc, + int stride, unsigned short *eobs, + short *dc, MACROBLOCKD *xd) { vp9_dequant_dc_idct_add_8x8_c(q, dq, pre, dst, 16, stride, dc[0]); vp9_dequant_dc_idct_add_8x8_c(&q[64], dq, pre + 8, dst + 8, 16, stride, dc[1]); @@ -137,7 +138,8 @@ void vp9_dequant_dc_idct_add_y_block_8x8_c(short *q, short *dq, #if CONFIG_SUPERBLOCKS void vp9_dequant_dc_idct_add_y_block_8x8_inplace_c(short *q, short *dq, unsigned char *dst, - int stride, char *eobs, + int stride, + unsigned short *eobs, short *dc, MACROBLOCKD *xd) { vp9_dequant_dc_idct_add_8x8_c(q, dq, dst, dst, stride, stride, dc[0]); vp9_dequant_dc_idct_add_8x8_c(&q[64], dq, dst + 8, @@ -152,7 +154,7 @@ void vp9_dequant_dc_idct_add_y_block_8x8_inplace_c(short *q, short *dq, void vp9_dequant_idct_add_y_block_8x8_c(short *q, short *dq, unsigned char *pre, unsigned char *dst, - int stride, char *eobs, + int stride, unsigned short *eobs, MACROBLOCKD *xd) { unsigned char *origdest = dst; unsigned char *origpred = pre; @@ -170,7 +172,7 @@ void vp9_dequant_idct_add_uv_block_8x8_c(short *q, short *dq, unsigned char *pre, unsigned char *dstu, unsigned char *dstv, - int stride, char *eobs, + int stride, unsigned short *eobs, MACROBLOCKD *xd) { vp9_dequant_idct_add_8x8_c(q, dq, pre, dstu, 8, stride); @@ -184,7 +186,8 @@ void vp9_dequant_idct_add_uv_block_8x8_c(short *q, short *dq, void vp9_dequant_idct_add_uv_block_8x8_inplace_c(short *q, short *dq, unsigned char *dstu, unsigned char *dstv, - int stride, char *eobs, + int stride, + unsigned short *eobs, MACROBLOCKD *xd) { vp9_dequant_idct_add_8x8_c(q, dq, dstu, dstu, stride, stride); @@ -198,7 +201,8 @@ void vp9_dequant_idct_add_uv_block_8x8_inplace_c(short *q, short *dq, void vp9_dequant_dc_idct_add_y_block_lossless_c(short *q, short *dq, unsigned char *pre, unsigned char *dst, - int stride, char *eobs, + int stride, + unsigned short *eobs, short *dc) { int i, j; @@ -223,7 +227,7 @@ void vp9_dequant_dc_idct_add_y_block_lossless_c(short *q, short *dq, void vp9_dequant_idct_add_y_block_lossless_c(short *q, short *dq, unsigned char *pre, unsigned char *dst, - int stride, char *eobs) { + int stride, unsigned short *eobs) { int i, j; for (i = 0; i < 4; i++) { @@ -249,7 +253,8 @@ void vp9_dequant_idct_add_uv_block_lossless_c(short *q, short *dq, unsigned char *pre, unsigned char *dstu, unsigned char *dstv, - int stride, char *eobs) { + int stride, + unsigned short *eobs) { int i, j; for (i = 0; i < 2; i++) { diff --git a/vp9/decoder/x86/idct_blk_mmx.c b/vp9/decoder/x86/idct_blk_mmx.c index 189a846de..6e5473106 100644 --- a/vp9/decoder/x86/idct_blk_mmx.c +++ b/vp9/decoder/x86/idct_blk_mmx.c @@ -15,7 +15,8 @@ void vp9_dequant_dc_idct_add_y_block_mmx(short *q, short *dq, unsigned char *pre, unsigned char *dst, - int stride, char *eobs, short *dc) { + int stride, unsigned short *eobs, + short *dc) { int i; for (i = 0; i < 4; i++) { @@ -53,7 +54,7 @@ void vp9_dequant_dc_idct_add_y_block_mmx(short *q, short *dq, void vp9_dequant_idct_add_y_block_mmx(short *q, short *dq, unsigned char *pre, unsigned char *dst, - int stride, char *eobs) { + int stride, unsigned short *eobs) { int i; for (i = 0; i < 4; i++) { @@ -96,7 +97,7 @@ void vp9_dequant_idct_add_uv_block_mmx(short *q, short *dq, unsigned char *pre, unsigned char *dstu, unsigned char *dstv, - int stride, char *eobs) { + int stride, unsigned short *eobs) { int i; for (i = 0; i < 2; i++) { diff --git a/vp9/decoder/x86/idct_blk_sse2.c b/vp9/decoder/x86/idct_blk_sse2.c index bc3c5d663..5914c16bc 100644 --- a/vp9/decoder/x86/idct_blk_sse2.c +++ b/vp9/decoder/x86/idct_blk_sse2.c @@ -31,7 +31,8 @@ void vp9_idct_dequant_full_2x_sse2(short *q, short *dq, void vp9_dequant_dc_idct_add_y_block_sse2(short *q, short *dq, unsigned char *pre, unsigned char *dst, - int stride, char *eobs, short *dc) { + int stride, unsigned short *eobs, + short *dc) { int i; for (i = 0; i < 4; i++) { @@ -57,7 +58,7 @@ void vp9_dequant_dc_idct_add_y_block_sse2(short *q, short *dq, void vp9_dequant_idct_add_y_block_sse2(short *q, short *dq, unsigned char *pre, unsigned char *dst, - int stride, char *eobs) { + int stride, unsigned short *eobs) { int i; for (i = 0; i < 4; i++) { @@ -82,7 +83,7 @@ void vp9_dequant_idct_add_uv_block_sse2(short *q, short *dq, unsigned char *pre, unsigned char *dstu, unsigned char *dstv, - int stride, char *eobs) { + int stride, unsigned short *eobs) { if (((short *)(eobs))[0] & 0xfefe) vp9_idct_dequant_full_2x_sse2(q, dq, pre, dstu, stride, 8); else