Fixes in 64x64 transform

Extends quantizer range and some other fixes.

Change-Id: Ia9adf7848e772783365d6501efd07585bff80c15
stdhd: +0.166%, turns positive a little
derf: -0.036% (very few blocks use the 64x64 mode)

diff --git a/vp9/common/vp9_entropy.c b/vp9/common/vp9_entropy.c
index fd4f4338e9459ad6cc0f8072d69304cbe9ffe82f..7a41a5945a89deade12cf641c17fcef8fe729474 100644 (file)
--- a/vp9/common/vp9_entropy.c
+++ b/vp9/common/vp9_entropy.c
@@ -21,8 +21,12 @@ const vp9_prob vp9_cat3_prob[] = { 173, 148, 140 };
 const vp9_prob vp9_cat4_prob[] = { 176, 155, 140, 135 };
 const vp9_prob vp9_cat5_prob[] = { 180, 157, 141, 134, 130 };
 const vp9_prob vp9_cat6_prob[] = {
+#if TX_64X64
+    255,
+#endif
     254, 254, 254, 252, 249, 243, 230, 196, 177, 153, 140, 133, 130, 129
 };
+
 #if CONFIG_VP9_HIGHBITDEPTH
 const vp9_prob vp9_cat1_prob_high10[] = { 159 };
 const vp9_prob vp9_cat2_prob_high10[] = { 165, 145 };
@@ -30,19 +34,26 @@ const vp9_prob vp9_cat3_prob_high10[] = { 173, 148, 140 };
 const vp9_prob vp9_cat4_prob_high10[] = { 176, 155, 140, 135 };
 const vp9_prob vp9_cat5_prob_high10[] = { 180, 157, 141, 134, 130 };
 const vp9_prob vp9_cat6_prob_high10[] = {
+#if TX_64X64
+    255,
+#endif
     255, 255, 254, 254, 254, 252, 249, 243,
     230, 196, 177, 153, 140, 133, 130, 129
 };
+
 const vp9_prob vp9_cat1_prob_high12[] = { 159 };
 const vp9_prob vp9_cat2_prob_high12[] = { 165, 145 };
 const vp9_prob vp9_cat3_prob_high12[] = { 173, 148, 140 };
 const vp9_prob vp9_cat4_prob_high12[] = { 176, 155, 140, 135 };
 const vp9_prob vp9_cat5_prob_high12[] = { 180, 157, 141, 134, 130 };
 const vp9_prob vp9_cat6_prob_high12[] = {
+#if TX_64X64
+    255,
+#endif
     255, 255, 255, 255, 254, 254, 254, 252, 249,
     243, 230, 196, 177, 153, 140, 133, 130, 129
 };
-#endif
+#endif  // CONFIG_VP9_HIGHBITDEPTH
 
 const uint8_t vp9_coefband_trans_8x8plus[MAX_NUM_COEFS] = {
   0, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4,

diff --git a/vp9/common/vp9_entropy.h b/vp9/common/vp9_entropy.h
index 7e1dceeddb02509dcd1e534ae91f7902f45fda0a..1766afb3ae93a2587be2927d41f7ab13e99b8f09 100644 (file)
--- a/vp9/common/vp9_entropy.h
+++ b/vp9/common/vp9_entropy.h
@@ -50,13 +50,35 @@ DECLARE_ALIGNED(16, extern const uint8_t, vp9_pt_energy_class[ENTROPY_TOKENS]);
 #define CAT5_MIN_VAL   35
 #define CAT6_MIN_VAL   67
 
+#if CONFIG_TX64X64
+#define DCT_MAX_VALUE              32768
+#define NUM_CAT6_BITS                 15
+#else
+#define DCT_MAX_VALUE              16384
+#define NUM_CAT6_BITS                 14
+#endif  // CONFIG_TX64X64
+
+#if CONFIG_VP9_HIGHBITDEPTH
+#if CONFIG_TX64X64
+#define DCT_MAX_VALUE_HIGH10      131072
+#define DCT_MAX_VALUE_HIGH12      524288
+#define NUM_CAT6_BITS_HIGH10          17
+#define NUM_CAT6_BITS_HIGH12          19
+#else
+#define DCT_MAX_VALUE_HIGH10       65536
+#define DCT_MAX_VALUE_HIGH12      262144
+#define NUM_CAT6_BITS_HIGH10          16
+#define NUM_CAT6_BITS_HIGH12          18
+#endif  // CONFIG_TX64X64
+#endif  // CONFIG_VP9_HIGHBITDEPTH
+
 // Extra bit probabilities.
 DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat1_prob[1]);
 DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat2_prob[2]);
 DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat3_prob[3]);
 DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat4_prob[4]);
 DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat5_prob[5]);
-DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat6_prob[14]);
+DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat6_prob[NUM_CAT6_BITS]);
 
 #if CONFIG_VP9_HIGHBITDEPTH
 DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat1_prob_high10[1]);
@@ -64,13 +86,15 @@ DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat2_prob_high10[2]);
 DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat3_prob_high10[3]);
 DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat4_prob_high10[4]);
 DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat5_prob_high10[5]);
-DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat6_prob_high10[16]);
+DECLARE_ALIGNED(16, extern const uint8_t,
+                vp9_cat6_prob_high10[NUM_CAT6_BITS_HIGH10]);
 DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat1_prob_high12[1]);
 DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat2_prob_high12[2]);
 DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat3_prob_high12[3]);
 DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat4_prob_high12[4]);
 DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat5_prob_high12[5]);
-DECLARE_ALIGNED(16, extern const uint8_t, vp9_cat6_prob_high12[18]);
+DECLARE_ALIGNED(16, extern const uint8_t,
+                vp9_cat6_prob_high12[NUM_CAT6_BITS_HIGH12]);
 #endif  // CONFIG_VP9_HIGHBITDEPTH
 
 #define EOB_MODEL_TOKEN 3
@@ -90,22 +114,6 @@ extern const vp9_extra_bit vp9_extra_bits_high10[ENTROPY_TOKENS];
 extern const vp9_extra_bit vp9_extra_bits_high12[ENTROPY_TOKENS];
 #endif  // CONFIG_VP9_HIGHBITDEPTH
 
-#if CONFIG_TX64X64
-#define DCT_MAX_VALUE           32768
-#else
-#define DCT_MAX_VALUE           16384
-#endif  // CONFIG_TX64X64
-
-#if CONFIG_VP9_HIGHBITDEPTH
-#if CONFIG_TX64X64
-#define DCT_MAX_VALUE_HIGH10   131072
-#define DCT_MAX_VALUE_HIGH12   524288
-#else
-#define DCT_MAX_VALUE_HIGH10    65536
-#define DCT_MAX_VALUE_HIGH12   262144
-#endif  // CONFIG_TX64X64
-#endif  // CONFIG_VP9_HIGHBITDEPTH
-
 /* Coefficients are predicted via a 3-dimensional probability table. */
 
 #define REF_TYPES 2  // intra=0, inter=1

diff --git a/vp9/common/vp9_entropymode.c b/vp9/common/vp9_entropymode.c
index 9c1ff4b0a9ee09aea66e601c20a036b4ab0cf999..506b0c842fbcb57b416d1f50479e4d7d3c9f77db 100644 (file)
--- a/vp9/common/vp9_entropymode.c
+++ b/vp9/common/vp9_entropymode.c
@@ -266,8 +266,8 @@ static const vp9_prob default_single_ref_p[REF_CONTEXTS][2] = {
 
 static const struct tx_probs default_tx_probs = {
 #if CONFIG_TX64X64
-  { { 3, 3, 136, 37 },
-    { 3, 5, 52,  13 } },
+  { { 1, 3, 136, 48 },
+    { 1, 5, 52,  24 } },
 #endif
 
   { { 3, 136, 37 },
@@ -298,7 +298,7 @@ void tx_counts_to_branch_counts_64x64(const unsigned int *tx_count_64x64p,
   ct_64x64p[3][0] = tx_count_64x64p[TX_32X32];
   ct_64x64p[3][1] = tx_count_64x64p[TX_64X64];
 }
-#endif
+#endif  // CONFIG_TX64X64
 
 void tx_counts_to_branch_counts_32x32(const unsigned int *tx_count_32x32p,
                                       unsigned int (*ct_32x32p)[2]) {

diff --git a/vp9/common/vp9_scan.c b/vp9/common/vp9_scan.c
index 4667edf0f00d78204e0ef901429e9056ec95c0b1..d9ccdaf7135519fe876c5428c2c51841b3da754f 100644 (file)
--- a/vp9/common/vp9_scan.c
+++ b/vp9/common/vp9_scan.c
@@ -2075,7 +2075,7 @@ DECLARE_ALIGNED(16, static const int16_t,
   4026, 4089, 3901, 3964, 3838, 3901, 3964, 4027,
   3775, 3838, 4027, 4090, 3902, 3965, 3965, 4028,
   3839, 3902, 4028, 4091, 3966, 4029, 3903, 3966,
-  4029, 4092, 3967, 4030, 4030, 4093, 4031, 4094,
+  4029, 4092, 3967, 4030, 4030, 4093, 4031, 4094, 0, 0,
 };
 #endif  // CONFIG_TX64X64
 

diff --git a/vp9/decoder/vp9_detokenize.c b/vp9/decoder/vp9_detokenize.c
index cb3874830611f5923e922c9d287441e498d16c31..c8202347954628694e94f832bf5b803eeaee3e9a 100644 (file)
--- a/vp9/decoder/vp9_detokenize.c
+++ b/vp9/decoder/vp9_detokenize.c
@@ -171,20 +171,22 @@ static int decode_coefs(VP9_COMMON *cm, const MACROBLOCKD *xd, PLANE_TYPE type,
 #if CONFIG_VP9_HIGHBITDEPTH
           switch (cm->bit_depth) {
             case VPX_BITS_8:
-              val = CAT6_MIN_VAL + read_coeff(cat6_prob, 14, r);
+              val = CAT6_MIN_VAL + read_coeff(cat6_prob, NUM_CAT6_BITS, r);
               break;
             case VPX_BITS_10:
-              val = CAT6_MIN_VAL + read_coeff(cat6_prob, 16, r);
+              val = CAT6_MIN_VAL +
+                    read_coeff(cat6_prob, NUM_CAT6_BITS_HIGH10, r);
               break;
             case VPX_BITS_12:
-              val = CAT6_MIN_VAL + read_coeff(cat6_prob, 18, r);
+              val = CAT6_MIN_VAL +
+                    read_coeff(cat6_prob, NUM_CAT6_BITS_HIGH12, r);
               break;
             default:
               assert(0);
               return -1;
           }
 #else
-          val = CAT6_MIN_VAL + read_coeff(cat6_prob, 14, r);
+          val = CAT6_MIN_VAL + read_coeff(cat6_prob, NUM_CAT6_BITS, r);
 #endif
           break;
       }

diff --git a/vp9/encoder/vp9_encodemb.c b/vp9/encoder/vp9_encodemb.c
index 16e81ca3b31d14d8a18f96eb09dc1fc30b90a274..3268bbdcf26d5bae38b40ff356d0eb5854a18e8f 100644 (file)
--- a/vp9/encoder/vp9_encodemb.c
+++ b/vp9/encoder/vp9_encodemb.c
@@ -702,7 +702,9 @@ static void encode_block(int plane, int block, BLOCK_SIZE plane_bsize,
   dst = &pd->dst.buf[4 * j * pd->dst.stride + 4 * i];
   a = &ctx->ta[plane][i];
   l = &ctx->tl[plane][j];
+#if CONFIG_TX64X64
   if (plane) assert(tx_size != TX_64X64);
+#endif
 
   // TODO(jingning): per transformed block zero forcing only enabled for
   // luma component. will integrate chroma components as well.

diff --git a/vp9/encoder/vp9_tokenize.c b/vp9/encoder/vp9_tokenize.c
index 34f71e9caa975723a42f686485c20754194fd829..53784ebac2cc318a1186687b0d0ca93041c65843 100644 (file)
--- a/vp9/encoder/vp9_tokenize.c
+++ b/vp9/encoder/vp9_tokenize.c
@@ -67,21 +67,21 @@ const vp9_tree_index vp9_coef_con_tree[TREE_SIZE(ENTROPY_TOKENS)] = {
   -CATEGORY5_TOKEN, -CATEGORY6_TOKEN   // 7 = CAT_FIVE
 };
 
-static vp9_tree_index cat1[2], cat2[4], cat3[6], cat4[8], cat5[10], cat6[28];
-
+static vp9_tree_index cat1[2], cat2[4], cat3[6], cat4[8], cat5[10],
+                      cat6[NUM_CAT6_BITS * 2];
 #if CONFIG_VP9_HIGHBITDEPTH
 static vp9_tree_index cat1_high10[2];
 static vp9_tree_index cat2_high10[4];
 static vp9_tree_index cat3_high10[6];
 static vp9_tree_index cat4_high10[8];
 static vp9_tree_index cat5_high10[10];
-static vp9_tree_index cat6_high10[32];
+static vp9_tree_index cat6_high10[NUM_CAT6_BITS_HIGH10 * 2];
 static vp9_tree_index cat1_high12[2];
 static vp9_tree_index cat2_high12[4];
 static vp9_tree_index cat3_high12[6];
 static vp9_tree_index cat4_high12[8];
 static vp9_tree_index cat5_high12[10];
-static vp9_tree_index cat6_high12[36];
+static vp9_tree_index cat6_high12[NUM_CAT6_BITS_HIGH12 * 2];
 #endif
 
 static void init_bit_tree(vp9_tree_index *p, int n) {
@@ -101,20 +101,20 @@ static void init_bit_trees() {
   init_bit_tree(cat3, 3);
   init_bit_tree(cat4, 4);
   init_bit_tree(cat5, 5);
-  init_bit_tree(cat6, 14);
+  init_bit_tree(cat6, NUM_CAT6_BITS);
 #if CONFIG_VP9_HIGHBITDEPTH
   init_bit_tree(cat1_high10, 1);
   init_bit_tree(cat2_high10, 2);
   init_bit_tree(cat3_high10, 3);
   init_bit_tree(cat4_high10, 4);
   init_bit_tree(cat5_high10, 5);
-  init_bit_tree(cat6_high10, 16);
+  init_bit_tree(cat6_high10, NUM_CAT6_BITS_HIGH10);
   init_bit_tree(cat1_high12, 1);
   init_bit_tree(cat2_high12, 2);
   init_bit_tree(cat3_high12, 3);
   init_bit_tree(cat4_high12, 4);
   init_bit_tree(cat5_high12, 5);
-  init_bit_tree(cat6_high12, 18);
+  init_bit_tree(cat6_high12, NUM_CAT6_BITS_HIGH12);
 #endif
 }
 
@@ -129,7 +129,7 @@ const vp9_extra_bit vp9_extra_bits[ENTROPY_TOKENS] = {
   {cat3, vp9_cat3_prob, 3,  CAT3_MIN_VAL},   // CATEGORY3_TOKEN
   {cat4, vp9_cat4_prob, 4,  CAT4_MIN_VAL},   // CATEGORY4_TOKEN
   {cat5, vp9_cat5_prob, 5,  CAT5_MIN_VAL},   // CATEGORY5_TOKEN
-  {cat6, vp9_cat6_prob, 14, CAT6_MIN_VAL},   // CATEGORY6_TOKEN
+  {cat6, vp9_cat6_prob, NUM_CAT6_BITS, CAT6_MIN_VAL},   // CATEGORY6_TOKEN
   {0, 0, 0, 0}                               // EOB_TOKEN
 };
 
@@ -145,7 +145,8 @@ const vp9_extra_bit vp9_extra_bits_high10[ENTROPY_TOKENS] = {
   {cat3_high10, vp9_cat3_prob_high10, 3,  CAT3_MIN_VAL},   // CATEGORY3_TOKEN
   {cat4_high10, vp9_cat4_prob_high10, 4,  CAT4_MIN_VAL},   // CATEGORY4_TOKEN
   {cat5_high10, vp9_cat5_prob_high10, 5,  CAT5_MIN_VAL},   // CATEGORY5_TOKEN
-  {cat6_high10, vp9_cat6_prob_high10, 16, CAT6_MIN_VAL},   // CATEGORY6_TOKEN
+  {cat6_high10, vp9_cat6_prob_high10, NUM_CAT6_BITS_HIGH10, CAT6_MIN_VAL},
+                                                           // CATEGORY6_TOKEN
   {0, 0, 0, 0}                                             // EOB_TOKEN
 };
 const vp9_extra_bit vp9_extra_bits_high12[ENTROPY_TOKENS] = {
@@ -159,7 +160,8 @@ const vp9_extra_bit vp9_extra_bits_high12[ENTROPY_TOKENS] = {
   {cat3_high12, vp9_cat3_prob_high12, 3,  CAT3_MIN_VAL},   // CATEGORY3_TOKEN
   {cat4_high12, vp9_cat4_prob_high12, 4,  CAT4_MIN_VAL},   // CATEGORY4_TOKEN
   {cat5_high12, vp9_cat5_prob_high12, 5,  CAT5_MIN_VAL},   // CATEGORY5_TOKEN
-  {cat6_high12, vp9_cat6_prob_high12, 18, CAT6_MIN_VAL},   // CATEGORY6_TOKEN
+  {cat6_high12, vp9_cat6_prob_high12, NUM_CAT6_BITS_HIGH12, CAT6_MIN_VAL},
+                                                           // CATEGORY6_TOKEN
   {0, 0, 0, 0}                                             // EOB_TOKEN
 };
 #endif