From: Angie Chiang Date: Fri, 1 Apr 2016 23:42:11 +0000 (-0700) Subject: Fit dct's stage range into 32-bit when bitdepth is 12 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=39b3c025fae2bac6049d3b92b0cc5fe3eab8b6b7;p=libvpx Fit dct's stage range into 32-bit when bitdepth is 12 Change-Id: I807e60c6dcacc50c087adcbdb1df022f8541efc5 --- diff --git a/vp10/common/vp10_fwd_txfm2d_cfg.h b/vp10/common/vp10_fwd_txfm2d_cfg.h index 3c0a9065e..2f60baae5 100644 --- a/vp10/common/vp10_fwd_txfm2d_cfg.h +++ b/vp10/common/vp10_fwd_txfm2d_cfg.h @@ -12,16 +12,17 @@ #define VP10_FWD_TXFM2D_CFG_H_ #include "vp10/common/vp10_fwd_txfm1d.h" // ---------------- config fwd_dct_dct_4 ---------------- -static const int8_t fwd_shift_dct_dct_4[3] = {4, 0, -2}; +static const int8_t fwd_shift_dct_dct_4[3] = {2, 0, 0}; static const int8_t fwd_stage_range_col_dct_dct_4[4] = {15, 16, 17, 17}; static const int8_t fwd_stage_range_row_dct_dct_4[4] = {17, 18, 18, 18}; -static const int8_t fwd_cos_bit_col_dct_dct_4[4] = {15, 15, 15, 15}; -static const int8_t fwd_cos_bit_row_dct_dct_4[4] = {15, 14, 14, 14}; +static const int8_t fwd_cos_bit_col_dct_dct_4[4] = {13, 13, 13, 13}; +static const int8_t fwd_cos_bit_row_dct_dct_4[4] = {13, 13, 13, 13}; static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_dct_4 = { - 4, // .txfm_size - 4, // .stage_num_col - 4, // .stage_num_row + 4, // .txfm_size + 4, // .stage_num_col + 4, // .stage_num_row + // 0, // .log_scale fwd_shift_dct_dct_4, // .shift fwd_stage_range_col_dct_dct_4, // .stage_range_col fwd_stage_range_row_dct_dct_4, // .stage_range_row @@ -31,16 +32,17 @@ static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_dct_4 = { TXFM_TYPE_DCT4}; // .txfm_type_row // ---------------- config fwd_dct_dct_8 ---------------- -static const int8_t fwd_shift_dct_dct_8[3] = {5, -3, -1}; +static const int8_t fwd_shift_dct_dct_8[3] = {3, -1, -1}; static const int8_t fwd_stage_range_col_dct_dct_8[6] = {16, 17, 18, 19, 19, 19}; -static const int8_t fwd_stage_range_row_dct_dct_8[6] = {16, 17, 18, 18, 18, 18}; -static const int8_t fwd_cos_bit_col_dct_dct_8[6] = {15, 15, 14, 13, 13, 13}; -static const int8_t fwd_cos_bit_row_dct_dct_8[6] = {15, 15, 14, 14, 14, 14}; +static const int8_t fwd_stage_range_row_dct_dct_8[6] = {18, 19, 20, 20, 20, 20}; +static const int8_t fwd_cos_bit_col_dct_dct_8[6] = {13, 13, 13, 13, 13, 13}; +static const int8_t fwd_cos_bit_row_dct_dct_8[6] = {13, 13, 12, 12, 12, 12}; static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_dct_8 = { - 8, // .txfm_size - 6, // .stage_num_col - 6, // .stage_num_row + 8, // .txfm_size + 6, // .stage_num_col + 6, // .stage_num_row + // 0, // .log_scale fwd_shift_dct_dct_8, // .shift fwd_stage_range_col_dct_dct_8, // .stage_range_col fwd_stage_range_row_dct_dct_8, // .stage_range_row @@ -50,20 +52,21 @@ static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_dct_8 = { TXFM_TYPE_DCT8}; // .txfm_type_row // ---------------- config fwd_dct_dct_16 ---------------- -static const int8_t fwd_shift_dct_dct_16[3] = {4, -3, -1}; +static const int8_t fwd_shift_dct_dct_16[3] = {2, -2, 0}; static const int8_t fwd_stage_range_col_dct_dct_16[8] = {15, 16, 17, 18, 19, 19, 19, 19}; -static const int8_t fwd_stage_range_row_dct_dct_16[8] = {16, 17, 18, 19, - 19, 19, 19, 19}; -static const int8_t fwd_cos_bit_col_dct_dct_16[8] = {15, 15, 15, 14, - 13, 13, 13, 13}; -static const int8_t fwd_cos_bit_row_dct_dct_16[8] = {15, 15, 14, 13, +static const int8_t fwd_stage_range_row_dct_dct_16[8] = {17, 18, 19, 20, + 20, 20, 20, 20}; +static const int8_t fwd_cos_bit_col_dct_dct_16[8] = {13, 13, 13, 13, 13, 13, 13, 13}; +static const int8_t fwd_cos_bit_row_dct_dct_16[8] = {13, 13, 13, 12, + 12, 12, 12, 12}; static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_dct_16 = { - 16, // .txfm_size - 8, // .stage_num_col - 8, // .stage_num_row + 16, // .txfm_size + 8, // .stage_num_col + 8, // .stage_num_row + // 0, // .log_scale fwd_shift_dct_dct_16, // .shift fwd_stage_range_col_dct_dct_16, // .stage_range_col fwd_stage_range_row_dct_dct_16, // .stage_range_row @@ -73,20 +76,21 @@ static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_dct_16 = { TXFM_TYPE_DCT16}; // .txfm_type_row // ---------------- config fwd_dct_dct_32 ---------------- -static const int8_t fwd_shift_dct_dct_32[3] = {3, -3, -1}; -static const int8_t fwd_stage_range_col_dct_dct_32[10] = {14, 15, 16, 17, 18, - 19, 19, 19, 19, 19}; -static const int8_t fwd_stage_range_row_dct_dct_32[10] = {16, 17, 18, 19, 20, +static const int8_t fwd_shift_dct_dct_32[3] = {2, -3, 0}; +static const int8_t fwd_stage_range_col_dct_dct_32[10] = {15, 16, 17, 18, 19, 20, 20, 20, 20, 20}; -static const int8_t fwd_cos_bit_col_dct_dct_32[10] = {15, 15, 15, 15, 14, - 13, 13, 13, 13, 13}; -static const int8_t fwd_cos_bit_row_dct_dct_32[10] = {15, 15, 14, 13, 12, +static const int8_t fwd_stage_range_row_dct_dct_32[10] = {17, 18, 19, 20, 21, + 21, 21, 21, 21, 21}; +static const int8_t fwd_cos_bit_col_dct_dct_32[10] = {13, 13, 13, 13, 13, 12, 12, 12, 12, 12}; +static const int8_t fwd_cos_bit_row_dct_dct_32[10] = {13, 13, 13, 12, 11, + 11, 11, 11, 11, 11}; static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_dct_32 = { - 32, // .txfm_size - 10, // .stage_num_col - 10, // .stage_num_row + 32, // .txfm_size + 10, // .stage_num_col + 10, // .stage_num_row + // 0, // .log_scale fwd_shift_dct_dct_32, // .shift fwd_stage_range_col_dct_dct_32, // .stage_range_col fwd_stage_range_row_dct_dct_32, // .stage_range_row diff --git a/vp10/common/vp10_inv_txfm2d_cfg.h b/vp10/common/vp10_inv_txfm2d_cfg.h index ee965ba23..897e21cd4 100644 --- a/vp10/common/vp10_inv_txfm2d_cfg.h +++ b/vp10/common/vp10_inv_txfm2d_cfg.h @@ -12,16 +12,17 @@ #define VP10_INV_TXFM2D_CFG_H_ #include "vp10/common/vp10_inv_txfm1d.h" // ---------------- config inv_dct_dct_4 ---------------- -static const int8_t inv_shift_dct_dct_4[2] = {1, -5}; -static const int8_t inv_stage_range_col_dct_dct_4[4] = {17, 17, 16, 16}; -static const int8_t inv_stage_range_row_dct_dct_4[4] = {16, 16, 16, 16}; -static const int8_t inv_cos_bit_col_dct_dct_4[4] = {15, 15, 15, 15}; -static const int8_t inv_cos_bit_row_dct_dct_4[4] = {15, 15, 15, 15}; +static const int8_t inv_shift_dct_dct_4[2] = {0, -4}; +static const int8_t inv_stage_range_col_dct_dct_4[4] = {18, 18, 17, 17}; +static const int8_t inv_stage_range_row_dct_dct_4[4] = {18, 18, 18, 18}; +static const int8_t inv_cos_bit_col_dct_dct_4[4] = {13, 13, 13, 13}; +static const int8_t inv_cos_bit_row_dct_dct_4[4] = {13, 13, 13, 13}; static const TXFM_2D_CFG inv_txfm_2d_cfg_dct_dct_4 = { - 4, // .txfm_size - 4, // .stage_num_col - 4, // .stage_num_row + 4, // .txfm_size + 4, // .stage_num_col + 4, // .stage_num_row + // 0, // .log_scale inv_shift_dct_dct_4, // .shift inv_stage_range_col_dct_dct_4, // .stage_range_col inv_stage_range_row_dct_dct_4, // .stage_range_row @@ -31,16 +32,17 @@ static const TXFM_2D_CFG inv_txfm_2d_cfg_dct_dct_4 = { TXFM_TYPE_DCT4}; // .txfm_type_row // ---------------- config inv_dct_dct_8 ---------------- -static const int8_t inv_shift_dct_dct_8[2] = {0, -5}; -static const int8_t inv_stage_range_col_dct_dct_8[6] = {17, 17, 17, 17, 16, 16}; -static const int8_t inv_stage_range_row_dct_dct_8[6] = {17, 17, 17, 17, 17, 17}; -static const int8_t inv_cos_bit_col_dct_dct_8[6] = {15, 15, 15, 15, 15, 15}; -static const int8_t inv_cos_bit_row_dct_dct_8[6] = {15, 15, 15, 15, 15, 15}; +static const int8_t inv_shift_dct_dct_8[2] = {1, -6}; +static const int8_t inv_stage_range_col_dct_dct_8[6] = {20, 20, 20, 20, 19, 19}; +static const int8_t inv_stage_range_row_dct_dct_8[6] = {19, 19, 19, 19, 19, 19}; +static const int8_t inv_cos_bit_col_dct_dct_8[6] = {12, 12, 12, 12, 12, 13}; +static const int8_t inv_cos_bit_row_dct_dct_8[6] = {13, 13, 13, 13, 13, 13}; static const TXFM_2D_CFG inv_txfm_2d_cfg_dct_dct_8 = { - 8, // .txfm_size - 6, // .stage_num_col - 6, // .stage_num_row + 8, // .txfm_size + 6, // .stage_num_col + 6, // .stage_num_row + // 0, // .log_scale inv_shift_dct_dct_8, // .shift inv_stage_range_col_dct_dct_8, // .stage_range_col inv_stage_range_row_dct_dct_8, // .stage_range_row @@ -50,20 +52,21 @@ static const TXFM_2D_CFG inv_txfm_2d_cfg_dct_dct_8 = { TXFM_TYPE_DCT8}; // .txfm_type_row // ---------------- config inv_dct_dct_16 ---------------- -static const int8_t inv_shift_dct_dct_16[2] = {0, -6}; -static const int8_t inv_stage_range_col_dct_dct_16[8] = {18, 18, 18, 18, - 18, 18, 17, 17}; -static const int8_t inv_stage_range_row_dct_dct_16[8] = {18, 18, 18, 18, - 18, 18, 18, 18}; -static const int8_t inv_cos_bit_col_dct_dct_16[8] = {14, 14, 14, 14, - 14, 14, 14, 15}; -static const int8_t inv_cos_bit_row_dct_dct_16[8] = {14, 14, 14, 14, - 14, 14, 14, 14}; +static const int8_t inv_shift_dct_dct_16[2] = {-1, -5}; +static const int8_t inv_stage_range_col_dct_dct_16[8] = {19, 19, 19, 19, + 19, 19, 18, 18}; +static const int8_t inv_stage_range_row_dct_dct_16[8] = {20, 20, 20, 20, + 20, 20, 20, 20}; +static const int8_t inv_cos_bit_col_dct_dct_16[8] = {13, 13, 13, 13, + 13, 13, 13, 13}; +static const int8_t inv_cos_bit_row_dct_dct_16[8] = {12, 12, 12, 12, + 12, 12, 12, 12}; static const TXFM_2D_CFG inv_txfm_2d_cfg_dct_dct_16 = { - 16, // .txfm_size - 8, // .stage_num_col - 8, // .stage_num_row + 16, // .txfm_size + 8, // .stage_num_col + 8, // .stage_num_row + // 0, // .log_scale inv_shift_dct_dct_16, // .shift inv_stage_range_col_dct_dct_16, // .stage_range_col inv_stage_range_row_dct_dct_16, // .stage_range_row @@ -74,19 +77,20 @@ static const TXFM_2D_CFG inv_txfm_2d_cfg_dct_dct_16 = { // ---------------- config inv_dct_dct_32 ---------------- static const int8_t inv_shift_dct_dct_32[2] = {-1, -6}; -static const int8_t inv_stage_range_col_dct_dct_32[10] = {18, 18, 18, 18, 18, - 18, 18, 18, 17, 17}; -static const int8_t inv_stage_range_row_dct_dct_32[10] = {19, 19, 19, 19, 19, - 19, 19, 19, 19, 19}; -static const int8_t inv_cos_bit_col_dct_dct_32[10] = {14, 14, 14, 14, 14, - 14, 14, 14, 14, 15}; -static const int8_t inv_cos_bit_row_dct_dct_32[10] = {13, 13, 13, 13, 13, - 13, 13, 13, 13, 13}; +static const int8_t inv_stage_range_col_dct_dct_32[10] = {20, 20, 20, 20, 20, + 20, 20, 20, 19, 19}; +static const int8_t inv_stage_range_row_dct_dct_32[10] = {21, 21, 21, 21, 21, + 21, 21, 21, 21, 21}; +static const int8_t inv_cos_bit_col_dct_dct_32[10] = {12, 12, 12, 12, 12, + 12, 12, 12, 12, 13}; +static const int8_t inv_cos_bit_row_dct_dct_32[10] = {11, 11, 11, 11, 11, + 11, 11, 11, 11, 11}; static const TXFM_2D_CFG inv_txfm_2d_cfg_dct_dct_32 = { - 32, // .txfm_size - 10, // .stage_num_col - 10, // .stage_num_row + 32, // .txfm_size + 10, // .stage_num_col + 10, // .stage_num_row + // 0, // .log_scale inv_shift_dct_dct_32, // .shift inv_stage_range_col_dct_dct_32, // .stage_range_col inv_stage_range_row_dct_dct_32, // .stage_range_row