Misc. ext_tx fixes/enhancements

derflr: +1.732% (8-bit)

Change-Id: I9c04c8249646ff96eacacfa1dcb0bd118c04e84a

diff --git a/vp10/common/blockd.h b/vp10/common/blockd.h
index 597cf125cef5c01e1986beee2e581e75dbbfb075..ce4bd39269f764b40af9147a8cf4ac1a098643c5 100644 (file)
--- a/vp10/common/blockd.h
+++ b/vp10/common/blockd.h
@@ -231,14 +231,14 @@ static const TX_TYPE intra_mode_to_tx_type_lookup[INTRA_MODES] = {
 
 static TX_TYPE ext_tx_to_txtype[EXT_TX_TYPES] = {
   DCT_DCT,
-  ADST_ADST,
-  FLIPADST_FLIPADST,
-  ADST_FLIPADST,
-  FLIPADST_ADST,
   ADST_DCT,
   DCT_ADST,
+  ADST_ADST,
   FLIPADST_DCT,
   DCT_FLIPADST,
+  FLIPADST_FLIPADST,
+  ADST_FLIPADST,
+  FLIPADST_ADST,
   DST_DST,
   DST_DCT,
   DCT_DST,

diff --git a/vp10/common/entropymode.c b/vp10/common/entropymode.c
index c6384231e90ca9c798297776aa73cd43109a01ba..b8c735058c3a64cede3e27ab42f08d5d49bd8d7c 100644 (file)
--- a/vp10/common/entropymode.c
+++ b/vp10/common/entropymode.c
@@ -334,9 +334,9 @@ const vpx_tree_index vp10_ext_tx_tree[TREE_SIZE(EXT_TX_TYPES)] = {
 };
 
 static const vpx_prob default_ext_tx_prob[EXT_TX_SIZES][EXT_TX_TYPES - 1] = {
-  { 216, 20, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128 },
-  { 192, 20, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128 },
-  { 168, 20, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128 },
+  { 160, 20, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128 },
+  { 144, 20, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128 },
+  { 128, 20, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128 },
 };
 #endif  // CONFIG_EXT_TX
 

diff --git a/vp10/common/idct.c b/vp10/common/idct.c
index 9e1f870073ab6dcc87f2cb44a786a71ddc9d448f..5656b06e7d899b13a7f210adfaad7147c62931ee 100644 (file)
--- a/vp10/common/idct.c
+++ b/vp10/common/idct.c
@@ -20,10 +20,10 @@
 #if CONFIG_EXT_TX
 void idst4_c(const tran_low_t *input, tran_low_t *output) {
   static const int N = 4;
-  static const int sinvalue_lookup_table[] = {
-    9630, 15582
+  // {sin(pi/5), sin(pi*2/5)} * sqrt(2/5) * sqrt(2)
+  static const int32_t sinvalue_lookup[] = {
+    141124871, 228344838,
   };
-  static const int mult = 14654;  // sqrt(4/5)
   int i, j;
   for (i = 0; i < N; i++) {
     int64_t sum = 0;
@@ -36,20 +36,19 @@ void idst4_c(const tran_low_t *input, tran_low_t *output) {
       }
       idx = idx > N + 1 - idx ? N + 1 - idx : idx;
       if (idx == 0) continue;
-      idx--;
-      sum += (int64_t)input[j] * sinvalue_lookup_table[idx] * (sign ? -1 : 1);
+      sum += (int64_t)input[j] * sinvalue_lookup[idx - 1] * (sign ? -1 : 1);
     }
-    sum = (sum * mult) >> (2 * DCT_CONST_BITS);
+    sum = ROUND_POWER_OF_TWO(sum, (2 * DCT_CONST_BITS));
     output[i] = WRAPLOW(sum, 8);
   }
 }
 
 void idst8_c(const tran_low_t *input, tran_low_t *output) {
   static const int N = 8;
-  static const int sinvalue_lookup_table[] = {
-    5604, 10531, 14189, 16135
+  // {sin(pi/9), sin(pi*2/9), ..., sin(pi*4/9)} * sqrt(2/9) * 2
+  static const int32_t sinvalue_lookup[] = {
+    86559612, 162678858, 219176632, 249238470
   };
-  static const int mult = 15447;  // 2*sqrt(2/9)
   int i, j;
   for (i = 0; i < N; i++) {
     int64_t sum = 0;
@@ -62,21 +61,20 @@ void idst8_c(const tran_low_t *input, tran_low_t *output) {
       }
       idx = idx > N + 1 - idx ? N + 1 - idx : idx;
       if (idx == 0) continue;
-      idx--;
-      sum += (int64_t)input[j] * sinvalue_lookup_table[idx] * (sign ? -1 : 1);
+      sum += (int64_t)input[j] * sinvalue_lookup[idx - 1] * (sign ? -1 : 1);
     }
-    sum = (sum * mult) >> (2 * DCT_CONST_BITS);
+    sum = ROUND_POWER_OF_TWO(sum, (2 * DCT_CONST_BITS));
     output[i] = WRAPLOW(sum, 8);
   }
 }
 
 void idst16_c(const tran_low_t *input, tran_low_t *output) {
   static const int N = 16;
-  static const int sinvalue_lookup_table[] = {
-    3011,  5919,  8625, 11038,
-    13075, 14666, 15759, 16314
+  // {sin(pi/17), sin(pi*2/17, ..., sin(pi*8/17)} * sqrt(2/17) * 2 * sqrt(2)
+  static const int32_t sinvalue_lookup[] = {
+    47852167, 94074787, 137093803, 175444254,
+    207820161, 233119001, 250479254, 259309736
   };
-  static const int mult = 15895;  // 2*sqrt(4/17)
   int i, j;
   for (i = 0; i < N; i++) {
     int64_t sum = 0;
@@ -89,10 +87,9 @@ void idst16_c(const tran_low_t *input, tran_low_t *output) {
       }
       idx = idx > N + 1 - idx ? N + 1 - idx : idx;
       if (idx == 0) continue;
-      idx--;
-      sum += (int64_t)input[j] * sinvalue_lookup_table[idx] * (sign ? -1 : 1);
+      sum += (int64_t)input[j] * sinvalue_lookup[idx - 1] * (sign ? -1 : 1);
     }
-    sum = (sum * mult) >> (2 * DCT_CONST_BITS);
+    sum = ROUND_POWER_OF_TWO(sum, (2 * DCT_CONST_BITS));
     output[i] = WRAPLOW(sum, 8);
   }
 }
@@ -100,10 +97,10 @@ void idst16_c(const tran_low_t *input, tran_low_t *output) {
 #if CONFIG_VP9_HIGHBITDEPTH
 void highbd_idst4_c(const tran_low_t *input, tran_low_t *output, int bd) {
   static const int N = 4;
-  static const int sinvalue_lookup_table[] = {
-    9630, 15582
+  // {sin(pi/5), sin(pi*2/5)} * sqrt(2/5) * sqrt(2)
+  static const int32_t sinvalue_lookup[] = {
+    141124871, 228344838,
   };
-  static const int mult = 14654;  // sqrt(4/5)
   int i, j;
   (void) bd;
   for (i = 0; i < N; i++) {
@@ -117,20 +114,19 @@ void highbd_idst4_c(const tran_low_t *input, tran_low_t *output, int bd) {
       }
       idx = idx > N + 1 - idx ? N + 1 - idx : idx;
       if (idx == 0) continue;
-      idx--;
-      sum += (int64_t)input[j] * sinvalue_lookup_table[idx] * (sign ? -1 : 1);
+      sum += (int64_t)input[j] * sinvalue_lookup[idx - 1] * (sign ? -1 : 1);
     }
-    sum = (sum * mult) >> (2 * DCT_CONST_BITS);
+    sum = ROUND_POWER_OF_TWO(sum, (2 * DCT_CONST_BITS));
     output[i] = WRAPLOW(sum, bd);
   }
 }
 
 void highbd_idst8_c(const tran_low_t *input, tran_low_t *output, int bd) {
   static const int N = 8;
-  static const int sinvalue_lookup_table[] = {
-    5604, 10531, 14189, 16135
+  // {sin(pi/9), sin(pi*2/9), ..., sin(pi*4/9)} * sqrt(2/9) * 2
+  static const int32_t sinvalue_lookup[] = {
+    86559612, 162678858, 219176632, 249238470
   };
-  static const int mult = 15447;  // 2*sqrt(2/9)
   int i, j;
   (void) bd;
   for (i = 0; i < N; i++) {
@@ -144,21 +140,20 @@ void highbd_idst8_c(const tran_low_t *input, tran_low_t *output, int bd) {
       }
       idx = idx > N + 1 - idx ? N + 1 - idx : idx;
       if (idx == 0) continue;
-      idx--;
-      sum += (int64_t)input[j] * sinvalue_lookup_table[idx] * (sign ? -1 : 1);
+      sum += (int64_t)input[j] * sinvalue_lookup[idx - 1] * (sign ? -1 : 1);
     }
-    sum = (sum * mult) >> (2 * DCT_CONST_BITS);
+    sum = ROUND_POWER_OF_TWO(sum, (2 * DCT_CONST_BITS));
     output[i] = WRAPLOW(sum, bd);
   }
 }
 
 void highbd_idst16_c(const tran_low_t *input, tran_low_t *output, int bd) {
   static const int N = 16;
-  static const int sinvalue_lookup_table[] = {
-    3011,  5919,  8625, 11038,
-    13075, 14666, 15759, 16314
+  // {sin(pi/17), sin(pi*2/17, ..., sin(pi*8/17)} * sqrt(2/17) * 2 * sqrt(2)
+  static const int32_t sinvalue_lookup[] = {
+    47852167, 94074787, 137093803, 175444254,
+    207820161, 233119001, 250479254, 259309736
   };
-  static const int mult = 15895;  // 2*sqrt(4/17)
   int i, j;
   (void) bd;
   for (i = 0; i < N; i++) {
@@ -172,10 +167,9 @@ void highbd_idst16_c(const tran_low_t *input, tran_low_t *output, int bd) {
       }
       idx = idx > N + 1 - idx ? N + 1 - idx : idx;
       if (idx == 0) continue;
-      idx--;
-      sum += (int64_t)input[j] * sinvalue_lookup_table[idx] * (sign ? -1 : 1);
+      sum += (int64_t)input[j] * sinvalue_lookup[idx - 1] * (sign ? -1 : 1);
     }
-    sum = (sum * mult) >> (2 * DCT_CONST_BITS);
+    sum = ROUND_POWER_OF_TWO(sum, (2 * DCT_CONST_BITS));
     output[i] = WRAPLOW(sum, bd);
   }
 }

diff --git a/vp10/encoder/dct.c b/vp10/encoder/dct.c
index cb1ede2531141a0f79447bfea5f6e88d8bf7e476..f183167819168bc50036e23ec56c6e665ceb70ce 100644 (file)
--- a/vp10/encoder/dct.c
+++ b/vp10/encoder/dct.c
@@ -23,10 +23,10 @@
 #if CONFIG_EXT_TX
 void fdst4(const tran_low_t *input, tran_low_t *output) {
   static const int N = 4;
-  static const int sinvalue_lookup_table[] = {
-    9630, 15582
+  // {sin(pi/5), sin(pi*2/5)} * sqrt(2/5) * sqrt(2)
+  static const int32_t sinvalue_lookup[] = {
+    141124871, 228344838,
   };
-  static const int mult = 14654;  // sqrt(4/5)
   int i, j;
   for (i = 0; i < N; i++) {
     int64_t sum = 0;
@@ -40,18 +40,18 @@ void fdst4(const tran_low_t *input, tran_low_t *output) {
       idx = idx > N + 1 - idx ? N + 1 - idx : idx;
       if (idx == 0) continue;
       idx--;
-      sum += (int64_t)input[j] * sinvalue_lookup_table[idx] * (sign ? -1 : 1);
+      sum += (int64_t)input[j] * sinvalue_lookup[idx] * (sign ? -1 : 1);
     }
-    output[i] = (sum * mult) >> (2 * DCT_CONST_BITS);
+    output[i] = ROUND_POWER_OF_TWO(sum, (2 * DCT_CONST_BITS));
   }
 }
 
 void fdst8(const tran_low_t *input, tran_low_t *output) {
   static const int N = 8;
-  static const int sinvalue_lookup_table[] = {
-    5604, 10531, 14189, 16135
+  // {sin(pi/9), sin(pi*2/9), ..., sin(pi*4/9)} * sqrt(2/9) * 2
+  static const int sinvalue_lookup[] = {
+    86559612, 162678858, 219176632, 249238470
   };
-  static const int mult = 15447;  // 2*sqrt(2/9)
   int i, j;
   for (i = 0; i < N; i++) {
     int64_t sum = 0;
@@ -65,19 +65,19 @@ void fdst8(const tran_low_t *input, tran_low_t *output) {
       idx = idx > N + 1 - idx ? N + 1 - idx : idx;
       if (idx == 0) continue;
       idx--;
-      sum += (int64_t)input[j] * sinvalue_lookup_table[idx] * (sign ? -1 : 1);
+      sum += (int64_t)input[j] * sinvalue_lookup[idx] * (sign ? -1 : 1);
     }
-    output[i] = (sum * mult) >> (2 * DCT_CONST_BITS);
+    output[i] = ROUND_POWER_OF_TWO(sum, (2 * DCT_CONST_BITS));
   }
 }
 
 void fdst16(const tran_low_t *input, tran_low_t *output) {
   static const int N = 16;
-  static const int sinvalue_lookup_table[] = {
-    3011,  5919,  8625, 11038,
-    13075, 14666, 15759, 16314
+  // {sin(pi/17), sin(pi*2/17, ..., sin(pi*8/17)} * sqrt(2/17) * 2 * sqrt(2)
+  static const int sinvalue_lookup[] = {
+    47852167, 94074787, 137093803, 175444254,
+    207820161, 233119001, 250479254, 259309736
   };
-  static const int mult = 15895;  // 2*sqrt(4/17)
   int i, j;
   for (i = 0; i < N; i++) {
     int64_t sum = 0;
@@ -91,9 +91,9 @@ void fdst16(const tran_low_t *input, tran_low_t *output) {
       idx = idx > N + 1 - idx ? N + 1 - idx : idx;
       if (idx == 0) continue;
       idx--;
-      sum += (int64_t)input[j] * sinvalue_lookup_table[idx] * (sign ? -1 : 1);
+      sum += (int64_t)input[j] * sinvalue_lookup[idx] * (sign ? -1 : 1);
     }
-    output[i] = (sum * mult) >> (2 * DCT_CONST_BITS);
+    output[i] = ROUND_POWER_OF_TWO(sum, (2 * DCT_CONST_BITS));
   }
 }
 #endif  // CONFIG_EXT_TX

diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c
index 9b27bc79598c5c7393554b6bf375480b02fa6b93..0c2f58f43f6c9d78e0b1e51743fde4efae59f4bd 100644 (file)
--- a/vp10/encoder/rdopt.c
+++ b/vp10/encoder/rdopt.c
@@ -54,7 +54,7 @@
 #define NEW_MV_DISCOUNT_FACTOR  8
 
 #if CONFIG_EXT_TX
-const double ext_tx_th = 0.99;
+const double ext_tx_th = 0.98;
 #endif
 
 typedef struct {
@@ -2736,7 +2736,7 @@ static int64_t handle_inter_mode(VP10_COMP *cpi, MACROBLOCK *x,
       int64_t distortion_y_tx;
       int dummy;
       int64_t best_rdcost_tx = INT64_MAX;
-      int best_ext_tx = NORM;
+      int best_ext_tx = -1;
 
       for (i = NORM; i < EXT_TX_TYPES; i++) {
         mbmi->ext_txfrm = i;
@@ -2747,7 +2747,8 @@ static int64_t handle_inter_mode(VP10_COMP *cpi, MACROBLOCK *x,
         rdcost_tx = RDCOST(x->rdmult, x->rddiv, rate_y_tx, distortion_y_tx);
         rdcost_tx = MIN(rdcost_tx, RDCOST(x->rdmult, x->rddiv, 0, *psse));
         assert(rdcost_tx >= 0);
-        if (rdcost_tx < best_rdcost_tx * ext_tx_th) {
+        if (rdcost_tx <
+            (best_ext_tx == NORM ? ext_tx_th : 1) * best_rdcost_tx) {
           best_ext_tx = i;
           best_rdcost_tx = rdcost_tx;
         }