From: Angie Chiang <angiebird@google.com>
Date: Sat, 2 Apr 2016 00:29:05 +0000 (-0700)
Subject: Change dct32x32's range
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=f1060f5bc4d4f31230de5886a20c0049f3ff47e6;p=libvpx

Change dct32x32's range

Bitdepth 10/12:
Fit coefficient range into 32 bits
Fit codfficient * const range into 32 bits

Bitdepth 8:
Fit coefficient range into 16 bits
Fit codfficient * constant range into 32 bits

Change-Id: I50b5a3132e8a9f5155c971ab0f6eb52876d2b5ca
---

diff --git a/vp10/common/idct.c b/vp10/common/idct.c
index b06a5e9e3..0eec3b436 100644
--- a/vp10/common/idct.c
+++ b/vp10/common/idct.c
@@ -24,11 +24,7 @@ int get_tx_scale(const MACROBLOCKD *const xd, const TX_TYPE tx_type,
   (void) tx_type;
 #if CONFIG_VP9_HIGHBITDEPTH
   if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
-    if (xd->bd == BITDEPTH_10) {
-      return 0;
-    } else {
-      return tx_size == TX_32X32;
-    }
+    return tx_size == TX_32X32;
   }
 #else
   (void)xd;
diff --git a/vp10/common/vp10_fwd_txfm2d_cfg.h b/vp10/common/vp10_fwd_txfm2d_cfg.h
index 2f60baae5..10050b74f 100644
--- a/vp10/common/vp10_fwd_txfm2d_cfg.h
+++ b/vp10/common/vp10_fwd_txfm2d_cfg.h
@@ -76,21 +76,21 @@ static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_dct_16 = {
     TXFM_TYPE_DCT16};                // .txfm_type_row
 
 //  ---------------- config fwd_dct_dct_32 ----------------
-static const int8_t fwd_shift_dct_dct_32[3] = {2, -3, 0};
+static const int8_t fwd_shift_dct_dct_32[3] = {2, -4, 0};
 static const int8_t fwd_stage_range_col_dct_dct_32[10] = {15, 16, 17, 18, 19,
                                                           20, 20, 20, 20, 20};
-static const int8_t fwd_stage_range_row_dct_dct_32[10] = {17, 18, 19, 20, 21,
-                                                          21, 21, 21, 21, 21};
+static const int8_t fwd_stage_range_row_dct_dct_32[10] = {16, 17, 18, 19, 20,
+                                                          20, 20, 20, 20, 20};
 static const int8_t fwd_cos_bit_col_dct_dct_32[10] = {13, 13, 13, 13, 13,
                                                       12, 12, 12, 12, 12};
-static const int8_t fwd_cos_bit_row_dct_dct_32[10] = {13, 13, 13, 12, 11,
-                                                      11, 11, 11, 11, 11};
+static const int8_t fwd_cos_bit_row_dct_dct_32[10] = {13, 13, 13, 13, 12,
+                                                      12, 12, 12, 12, 12};
 
 static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_dct_32 = {
     32,  // .txfm_size
     10,  // .stage_num_col
     10,  // .stage_num_row
-    // 0,  // .log_scale
+    // 1,  // .log_scale
     fwd_shift_dct_dct_32,            // .shift
     fwd_stage_range_col_dct_dct_32,  // .stage_range_col
     fwd_stage_range_row_dct_dct_32,  // .stage_range_row
diff --git a/vp10/common/vp10_inv_txfm2d_cfg.h b/vp10/common/vp10_inv_txfm2d_cfg.h
index 897e21cd4..f1bb2c748 100644
--- a/vp10/common/vp10_inv_txfm2d_cfg.h
+++ b/vp10/common/vp10_inv_txfm2d_cfg.h
@@ -76,21 +76,21 @@ static const TXFM_2D_CFG inv_txfm_2d_cfg_dct_dct_16 = {
     TXFM_TYPE_DCT16};                // .txfm_type_row
 
 //  ---------------- config inv_dct_dct_32 ----------------
-static const int8_t inv_shift_dct_dct_32[2] = {-1, -6};
-static const int8_t inv_stage_range_col_dct_dct_32[10] = {20, 20, 20, 20, 20,
-                                                          20, 20, 20, 19, 19};
-static const int8_t inv_stage_range_row_dct_dct_32[10] = {21, 21, 21, 21, 21,
-                                                          21, 21, 21, 21, 21};
-static const int8_t inv_cos_bit_col_dct_dct_32[10] = {12, 12, 12, 12, 12,
-                                                      12, 12, 12, 12, 13};
-static const int8_t inv_cos_bit_row_dct_dct_32[10] = {11, 11, 11, 11, 11,
-                                                      11, 11, 11, 11, 11};
+static const int8_t inv_shift_dct_dct_32[2] = {-1, -5};
+static const int8_t inv_stage_range_col_dct_dct_32[10] = {19, 19, 19, 19, 19,
+                                                          19, 19, 19, 18, 18};
+static const int8_t inv_stage_range_row_dct_dct_32[10] = {20, 20, 20, 20, 20,
+                                                          20, 20, 20, 20, 20};
+static const int8_t inv_cos_bit_col_dct_dct_32[10] = {13, 13, 13, 13, 13,
+                                                      13, 13, 13, 13, 13};
+static const int8_t inv_cos_bit_row_dct_dct_32[10] = {12, 12, 12, 12, 12,
+                                                      12, 12, 12, 12, 12};
 
 static const TXFM_2D_CFG inv_txfm_2d_cfg_dct_dct_32 = {
     32,  // .txfm_size
     10,  // .stage_num_col
     10,  // .stage_num_row
-    // 0,  // .log_scale
+    // 1,  // .log_scale
     inv_shift_dct_dct_32,            // .shift
     inv_stage_range_col_dct_dct_32,  // .stage_range_col
     inv_stage_range_row_dct_dct_32,  // .stage_range_row