Fit dct's stage range into 32-bit when bitdepth is 12

Change-Id: I807e60c6dcacc50c087adcbdb1df022f8541efc5

diff --git a/vp10/common/vp10_fwd_txfm2d_cfg.h b/vp10/common/vp10_fwd_txfm2d_cfg.h
index 3c0a9065ed4e6182b4bcc5f7a449441bd9dd9f7d..2f60baae57e33f729dbaede16371e4d2a8d72bb2 100644 (file)
--- a/vp10/common/vp10_fwd_txfm2d_cfg.h
+++ b/vp10/common/vp10_fwd_txfm2d_cfg.h
@@ -12,16 +12,17 @@
 #define VP10_FWD_TXFM2D_CFG_H_
 #include "vp10/common/vp10_fwd_txfm1d.h"
 //  ---------------- config fwd_dct_dct_4 ----------------
-static const int8_t fwd_shift_dct_dct_4[3] = {4, 0, -2};
+static const int8_t fwd_shift_dct_dct_4[3] = {2, 0, 0};
 static const int8_t fwd_stage_range_col_dct_dct_4[4] = {15, 16, 17, 17};
 static const int8_t fwd_stage_range_row_dct_dct_4[4] = {17, 18, 18, 18};
-static const int8_t fwd_cos_bit_col_dct_dct_4[4] = {15, 15, 15, 15};
-static const int8_t fwd_cos_bit_row_dct_dct_4[4] = {15, 14, 14, 14};
+static const int8_t fwd_cos_bit_col_dct_dct_4[4] = {13, 13, 13, 13};
+static const int8_t fwd_cos_bit_row_dct_dct_4[4] = {13, 13, 13, 13};
 
 static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_dct_4 = {
-    4,                              // .txfm_size
-    4,                              // .stage_num_col
-    4,                              // .stage_num_row
+    4,  // .txfm_size
+    4,  // .stage_num_col
+    4,  // .stage_num_row
+    // 0,  // .log_scale
     fwd_shift_dct_dct_4,            // .shift
     fwd_stage_range_col_dct_dct_4,  // .stage_range_col
     fwd_stage_range_row_dct_dct_4,  // .stage_range_row
@@ -31,16 +32,17 @@ static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_dct_4 = {
     TXFM_TYPE_DCT4};                // .txfm_type_row
 
 //  ---------------- config fwd_dct_dct_8 ----------------
-static const int8_t fwd_shift_dct_dct_8[3] = {5, -3, -1};
+static const int8_t fwd_shift_dct_dct_8[3] = {3, -1, -1};
 static const int8_t fwd_stage_range_col_dct_dct_8[6] = {16, 17, 18, 19, 19, 19};
-static const int8_t fwd_stage_range_row_dct_dct_8[6] = {16, 17, 18, 18, 18, 18};
-static const int8_t fwd_cos_bit_col_dct_dct_8[6] = {15, 15, 14, 13, 13, 13};
-static const int8_t fwd_cos_bit_row_dct_dct_8[6] = {15, 15, 14, 14, 14, 14};
+static const int8_t fwd_stage_range_row_dct_dct_8[6] = {18, 19, 20, 20, 20, 20};
+static const int8_t fwd_cos_bit_col_dct_dct_8[6] = {13, 13, 13, 13, 13, 13};
+static const int8_t fwd_cos_bit_row_dct_dct_8[6] = {13, 13, 12, 12, 12, 12};
 
 static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_dct_8 = {
-    8,                              // .txfm_size
-    6,                              // .stage_num_col
-    6,                              // .stage_num_row
+    8,  // .txfm_size
+    6,  // .stage_num_col
+    6,  // .stage_num_row
+    // 0,  // .log_scale
     fwd_shift_dct_dct_8,            // .shift
     fwd_stage_range_col_dct_dct_8,  // .stage_range_col
     fwd_stage_range_row_dct_dct_8,  // .stage_range_row
@@ -50,20 +52,21 @@ static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_dct_8 = {
     TXFM_TYPE_DCT8};                // .txfm_type_row
 
 //  ---------------- config fwd_dct_dct_16 ----------------
-static const int8_t fwd_shift_dct_dct_16[3] = {4, -3, -1};
+static const int8_t fwd_shift_dct_dct_16[3] = {2, -2, 0};
 static const int8_t fwd_stage_range_col_dct_dct_16[8] = {15, 16, 17, 18,
                                                          19, 19, 19, 19};
-static const int8_t fwd_stage_range_row_dct_dct_16[8] = {16, 17, 18, 19,
-                                                         19, 19, 19, 19};
-static const int8_t fwd_cos_bit_col_dct_dct_16[8] = {15, 15, 15, 14,
-                                                     13, 13, 13, 13};
-static const int8_t fwd_cos_bit_row_dct_dct_16[8] = {15, 15, 14, 13,
+static const int8_t fwd_stage_range_row_dct_dct_16[8] = {17, 18, 19, 20,
+                                                         20, 20, 20, 20};
+static const int8_t fwd_cos_bit_col_dct_dct_16[8] = {13, 13, 13, 13,
                                                      13, 13, 13, 13};
+static const int8_t fwd_cos_bit_row_dct_dct_16[8] = {13, 13, 13, 12,
+                                                     12, 12, 12, 12};
 
 static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_dct_16 = {
-    16,                              // .txfm_size
-    8,                               // .stage_num_col
-    8,                               // .stage_num_row
+    16,  // .txfm_size
+    8,   // .stage_num_col
+    8,   // .stage_num_row
+    // 0,  // .log_scale
     fwd_shift_dct_dct_16,            // .shift
     fwd_stage_range_col_dct_dct_16,  // .stage_range_col
     fwd_stage_range_row_dct_dct_16,  // .stage_range_row
@@ -73,20 +76,21 @@ static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_dct_16 = {
     TXFM_TYPE_DCT16};                // .txfm_type_row
 
 //  ---------------- config fwd_dct_dct_32 ----------------
-static const int8_t fwd_shift_dct_dct_32[3] = {3, -3, -1};
-static const int8_t fwd_stage_range_col_dct_dct_32[10] = {14, 15, 16, 17, 18,
-                                                          19, 19, 19, 19, 19};
-static const int8_t fwd_stage_range_row_dct_dct_32[10] = {16, 17, 18, 19, 20,
+static const int8_t fwd_shift_dct_dct_32[3] = {2, -3, 0};
+static const int8_t fwd_stage_range_col_dct_dct_32[10] = {15, 16, 17, 18, 19,
                                                           20, 20, 20, 20, 20};
-static const int8_t fwd_cos_bit_col_dct_dct_32[10] = {15, 15, 15, 15, 14,
-                                                      13, 13, 13, 13, 13};
-static const int8_t fwd_cos_bit_row_dct_dct_32[10] = {15, 15, 14, 13, 12,
+static const int8_t fwd_stage_range_row_dct_dct_32[10] = {17, 18, 19, 20, 21,
+                                                          21, 21, 21, 21, 21};
+static const int8_t fwd_cos_bit_col_dct_dct_32[10] = {13, 13, 13, 13, 13,
                                                       12, 12, 12, 12, 12};
+static const int8_t fwd_cos_bit_row_dct_dct_32[10] = {13, 13, 13, 12, 11,
+                                                      11, 11, 11, 11, 11};
 
 static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_dct_32 = {
-    32,                              // .txfm_size
-    10,                              // .stage_num_col
-    10,                              // .stage_num_row
+    32,  // .txfm_size
+    10,  // .stage_num_col
+    10,  // .stage_num_row
+    // 0,  // .log_scale
     fwd_shift_dct_dct_32,            // .shift
     fwd_stage_range_col_dct_dct_32,  // .stage_range_col
     fwd_stage_range_row_dct_dct_32,  // .stage_range_row

diff --git a/vp10/common/vp10_inv_txfm2d_cfg.h b/vp10/common/vp10_inv_txfm2d_cfg.h
index ee965ba23260496bce0439abb57a3d57e2dd8e30..897e21cd42838f088fc5e121a6fb2a8e5a93daf7 100644 (file)
--- a/vp10/common/vp10_inv_txfm2d_cfg.h
+++ b/vp10/common/vp10_inv_txfm2d_cfg.h
@@ -12,16 +12,17 @@
 #define VP10_INV_TXFM2D_CFG_H_
 #include "vp10/common/vp10_inv_txfm1d.h"
 //  ---------------- config inv_dct_dct_4 ----------------
-static const int8_t inv_shift_dct_dct_4[2] = {1, -5};
-static const int8_t inv_stage_range_col_dct_dct_4[4] = {17, 17, 16, 16};
-static const int8_t inv_stage_range_row_dct_dct_4[4] = {16, 16, 16, 16};
-static const int8_t inv_cos_bit_col_dct_dct_4[4] = {15, 15, 15, 15};
-static const int8_t inv_cos_bit_row_dct_dct_4[4] = {15, 15, 15, 15};
+static const int8_t inv_shift_dct_dct_4[2] = {0, -4};
+static const int8_t inv_stage_range_col_dct_dct_4[4] = {18, 18, 17, 17};
+static const int8_t inv_stage_range_row_dct_dct_4[4] = {18, 18, 18, 18};
+static const int8_t inv_cos_bit_col_dct_dct_4[4] = {13, 13, 13, 13};
+static const int8_t inv_cos_bit_row_dct_dct_4[4] = {13, 13, 13, 13};
 
 static const TXFM_2D_CFG inv_txfm_2d_cfg_dct_dct_4 = {
-    4,                              // .txfm_size
-    4,                              // .stage_num_col
-    4,                              // .stage_num_row
+    4,  // .txfm_size
+    4,  // .stage_num_col
+    4,  // .stage_num_row
+    // 0,  // .log_scale
     inv_shift_dct_dct_4,            // .shift
     inv_stage_range_col_dct_dct_4,  // .stage_range_col
     inv_stage_range_row_dct_dct_4,  // .stage_range_row
@@ -31,16 +32,17 @@ static const TXFM_2D_CFG inv_txfm_2d_cfg_dct_dct_4 = {
     TXFM_TYPE_DCT4};                // .txfm_type_row
 
 //  ---------------- config inv_dct_dct_8 ----------------
-static const int8_t inv_shift_dct_dct_8[2] = {0, -5};
-static const int8_t inv_stage_range_col_dct_dct_8[6] = {17, 17, 17, 17, 16, 16};
-static const int8_t inv_stage_range_row_dct_dct_8[6] = {17, 17, 17, 17, 17, 17};
-static const int8_t inv_cos_bit_col_dct_dct_8[6] = {15, 15, 15, 15, 15, 15};
-static const int8_t inv_cos_bit_row_dct_dct_8[6] = {15, 15, 15, 15, 15, 15};
+static const int8_t inv_shift_dct_dct_8[2] = {1, -6};
+static const int8_t inv_stage_range_col_dct_dct_8[6] = {20, 20, 20, 20, 19, 19};
+static const int8_t inv_stage_range_row_dct_dct_8[6] = {19, 19, 19, 19, 19, 19};
+static const int8_t inv_cos_bit_col_dct_dct_8[6] = {12, 12, 12, 12, 12, 13};
+static const int8_t inv_cos_bit_row_dct_dct_8[6] = {13, 13, 13, 13, 13, 13};
 
 static const TXFM_2D_CFG inv_txfm_2d_cfg_dct_dct_8 = {
-    8,                              // .txfm_size
-    6,                              // .stage_num_col
-    6,                              // .stage_num_row
+    8,  // .txfm_size
+    6,  // .stage_num_col
+    6,  // .stage_num_row
+    // 0,  // .log_scale
     inv_shift_dct_dct_8,            // .shift
     inv_stage_range_col_dct_dct_8,  // .stage_range_col
     inv_stage_range_row_dct_dct_8,  // .stage_range_row
@@ -50,20 +52,21 @@ static const TXFM_2D_CFG inv_txfm_2d_cfg_dct_dct_8 = {
     TXFM_TYPE_DCT8};                // .txfm_type_row
 
 //  ---------------- config inv_dct_dct_16 ----------------
-static const int8_t inv_shift_dct_dct_16[2] = {0, -6};
-static const int8_t inv_stage_range_col_dct_dct_16[8] = {18, 18, 18, 18,
-                                                         18, 18, 17, 17};
-static const int8_t inv_stage_range_row_dct_dct_16[8] = {18, 18, 18, 18,
-                                                         18, 18, 18, 18};
-static const int8_t inv_cos_bit_col_dct_dct_16[8] = {14, 14, 14, 14,
-                                                     14, 14, 14, 15};
-static const int8_t inv_cos_bit_row_dct_dct_16[8] = {14, 14, 14, 14,
-                                                     14, 14, 14, 14};
+static const int8_t inv_shift_dct_dct_16[2] = {-1, -5};
+static const int8_t inv_stage_range_col_dct_dct_16[8] = {19, 19, 19, 19,
+                                                         19, 19, 18, 18};
+static const int8_t inv_stage_range_row_dct_dct_16[8] = {20, 20, 20, 20,
+                                                         20, 20, 20, 20};
+static const int8_t inv_cos_bit_col_dct_dct_16[8] = {13, 13, 13, 13,
+                                                     13, 13, 13, 13};
+static const int8_t inv_cos_bit_row_dct_dct_16[8] = {12, 12, 12, 12,
+                                                     12, 12, 12, 12};
 
 static const TXFM_2D_CFG inv_txfm_2d_cfg_dct_dct_16 = {
-    16,                              // .txfm_size
-    8,                               // .stage_num_col
-    8,                               // .stage_num_row
+    16,  // .txfm_size
+    8,   // .stage_num_col
+    8,   // .stage_num_row
+    // 0,  // .log_scale
     inv_shift_dct_dct_16,            // .shift
     inv_stage_range_col_dct_dct_16,  // .stage_range_col
     inv_stage_range_row_dct_dct_16,  // .stage_range_row
@@ -74,19 +77,20 @@ static const TXFM_2D_CFG inv_txfm_2d_cfg_dct_dct_16 = {
 
 //  ---------------- config inv_dct_dct_32 ----------------
 static const int8_t inv_shift_dct_dct_32[2] = {-1, -6};
-static const int8_t inv_stage_range_col_dct_dct_32[10] = {18, 18, 18, 18, 18,
-                                                          18, 18, 18, 17, 17};
-static const int8_t inv_stage_range_row_dct_dct_32[10] = {19, 19, 19, 19, 19,
-                                                          19, 19, 19, 19, 19};
-static const int8_t inv_cos_bit_col_dct_dct_32[10] = {14, 14, 14, 14, 14,
-                                                      14, 14, 14, 14, 15};
-static const int8_t inv_cos_bit_row_dct_dct_32[10] = {13, 13, 13, 13, 13,
-                                                      13, 13, 13, 13, 13};
+static const int8_t inv_stage_range_col_dct_dct_32[10] = {20, 20, 20, 20, 20,
+                                                          20, 20, 20, 19, 19};
+static const int8_t inv_stage_range_row_dct_dct_32[10] = {21, 21, 21, 21, 21,
+                                                          21, 21, 21, 21, 21};
+static const int8_t inv_cos_bit_col_dct_dct_32[10] = {12, 12, 12, 12, 12,
+                                                      12, 12, 12, 12, 13};
+static const int8_t inv_cos_bit_row_dct_dct_32[10] = {11, 11, 11, 11, 11,
+                                                      11, 11, 11, 11, 11};
 
 static const TXFM_2D_CFG inv_txfm_2d_cfg_dct_dct_32 = {
-    32,                              // .txfm_size
-    10,                              // .stage_num_col
-    10,                              // .stage_num_row
+    32,  // .txfm_size
+    10,  // .stage_num_col
+    10,  // .stage_num_row
+    // 0,  // .log_scale
     inv_shift_dct_dct_32,            // .shift
     inv_stage_range_col_dct_dct_32,  // .stage_range_col
     inv_stage_range_row_dct_dct_32,  // .stage_range_row