Reduce intra transform set

Reduce transform set for intra for 8x8 and smalller to 7 from 12.
Also fixes an issue with prob updates.

Enocder Speed-up about 8-10%

Coding efficiency very little change.
lowres: -2.996 (from -3.055 before)
midres: -2.482 (from -2.552 before)

Change-Id: I4ba50ff967521b33c748fe423bd92f7cf4105ebc

diff --git a/vp10/common/blockd.h b/vp10/common/blockd.h
index fbcba090173b67a48d5ed7e488adc429eb373073..b19aa8c343fcfa642f3f6f953c9780a443e7f801 100644 (file)
--- a/vp10/common/blockd.h
+++ b/vp10/common/blockd.h
@@ -400,18 +400,20 @@ static const int num_ext_tx_set_inter[EXT_TX_SETS_INTER] = {
   1, 16, 12, 2
 };
 static const int num_ext_tx_set_intra[EXT_TX_SETS_INTRA] = {
-  1, 12, 5
+  1, 7, 5
 };
 
 #if EXT_TX_SIZES == 4
 static INLINE int get_ext_tx_set(TX_SIZE tx_size, BLOCK_SIZE bs,
                                  int is_inter) {
   if (tx_size > TX_32X32 || bs < BLOCK_8X8) return 0;
-  if (tx_size == TX_32X32)
-    return is_inter ? 3 - 2 * USE_MSKTX_FOR_32X32 : 0;
 #if USE_REDUCED_TXSET_FOR_16X16
+  if (tx_size == TX_32X32)
+    return is_inter ? 3 - USE_MSKTX_FOR_32X32 : 0;
   return (tx_size == TX_16X16 ? 2 : 1);
 #else
+  if (tx_size == TX_32X32)
+    return is_inter ? 3 - 2 * USE_MSKTX_FOR_32X32 : 0;
   return (tx_size == TX_16X16 && !is_inter ? 2 : 1);
 #endif  // USE_REDUCED_TXSET_FOR_16X16
 }
@@ -424,8 +426,8 @@ static const int use_intra_ext_tx_for_txsize[EXT_TX_SETS_INTRA][TX_SIZES] = {
 
 static const int use_inter_ext_tx_for_txsize[EXT_TX_SETS_INTER][TX_SIZES] = {
   { 0, 0, 0, 0, },  // unused
-  { 1, 1, 1, USE_MSKTX_FOR_32X32, },
-  { 0, 0, 0, 0, },
+  { 1, 1, (!USE_REDUCED_TXSET_FOR_16X16), USE_MSKTX_FOR_32X32, },
+  { 0, 0, USE_REDUCED_TXSET_FOR_16X16, 0, },
   { 0, 0, 0, (!USE_MSKTX_FOR_32X32), },
 };
 
@@ -436,7 +438,11 @@ static INLINE int get_ext_tx_set(TX_SIZE tx_size, BLOCK_SIZE bs,
   (void) is_inter;
   if (tx_size > TX_32X32 || bs < BLOCK_8X8) return 0;
   if (tx_size == TX_32X32) return 0;
-  return tx_size == TX_16X16 ? 2 : 1;
+#if USE_REDUCED_TXSET_FOR_16X16
+  return (tx_size == TX_16X16 ? 2 : 1);
+#else
+  return (tx_size == TX_16X16 && !is_inter ? 2 : 1);
+#endif  // USE_REDUCED_TXSET_FOR_16X16
 }
 
 static const int use_intra_ext_tx_for_txsize[EXT_TX_SETS_INTRA][TX_SIZES] = {
@@ -447,8 +453,8 @@ static const int use_intra_ext_tx_for_txsize[EXT_TX_SETS_INTRA][TX_SIZES] = {
 
 static const int use_inter_ext_tx_for_txsize[EXT_TX_SETS_INTER][TX_SIZES] = {
   { 0, 0, 0, 0, },  // unused
-  { 1, 1, 0, 0, },
-  { 0, 0, 1, 0, },
+  { 1, 1, (!USE_REDUCED_TXSET_FOR_16X16), 0, },
+  { 0, 0, USE_REDUCED_TXSET_FOR_16X16, 0, },
   { 0, 0, 0, 1, },
 };
 #endif  // EXT_TX_SIZES == 4
@@ -456,7 +462,7 @@ static const int use_inter_ext_tx_for_txsize[EXT_TX_SETS_INTER][TX_SIZES] = {
 // Transform types used in each intra set
 static const int ext_tx_used_intra[EXT_TX_SETS_INTRA][TX_TYPES] = {
   {1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
-  {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0},
+  {1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0},
   {1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0},
 };
 

diff --git a/vp10/common/entropymode.c b/vp10/common/entropymode.c
index f1c8e30d0118902a4d1954ff8ff8fd6a8837e2c7..e622ebcb77a613fdaac5902d42f10c0c49e7f25b 100644 (file)
--- a/vp10/common/entropymode.c
+++ b/vp10/common/entropymode.c
@@ -976,13 +976,8 @@ const vpx_tree_index vp10_ext_tx_intra_tree[EXT_TX_SETS_INTRA]
     -DCT_DCT, 4,
     6, 8,
     -V_DCT, -H_DCT,
-    10, 16,
-    12, 14,
+    -ADST_ADST, 10,
     -ADST_DCT, -DCT_ADST,
-    -FLIPADST_DCT, -DCT_FLIPADST,
-    18, 20,
-    -ADST_ADST, -FLIPADST_FLIPADST,
-    -ADST_FLIPADST, -FLIPADST_ADST,
   }, {
     -IDTX, 2,
     -DCT_DCT, 4,
@@ -1041,50 +1036,50 @@ default_intra_ext_tx_prob[EXT_TX_SETS_INTRA][EXT_TX_SIZES]
     },
   }, {
     {
-      {   8, 176,  32, 128, 128, 128, 128, 128, 128, 128, 128, },
-      {  10,  28,  32, 128, 176, 192, 208, 128, 128, 128, 128, },
-      {  10,  28,  32, 128, 176, 192,  48, 128, 128, 128, 128, },
-      {   9, 160,  32, 128, 128, 128, 128, 128, 128, 128, 128, },
-      {   8,  28,  32, 128,  96, 128, 128, 128, 160, 192, 128, },
-      {   7,  28,  32, 128, 160, 176, 192, 128, 128, 128, 128, },
-      {   7,  20,  32, 128, 160, 176,  64, 128, 128, 128, 128, },
-      {  10,  23,  32, 128, 160, 176,  64, 128, 128, 128, 128, },
-      {   8,  29,  32, 128, 160, 176, 192, 128, 128, 128, 128, },
-      {   3,  20,  32, 128,  96, 128, 128, 128, 160, 192, 128, },
+      {   8, 224,  32, 128, 64, 128, },
+      {  10,  32,  32, 128, 16, 192, },
+      {  10,  32,  32, 128, 16,  64, },
+      {   9, 200,  32, 128, 64, 128, },
+      {   8,   8,  32, 128, 224, 128, },
+      {  10,  32,  32, 128, 16, 192, },
+      {  10,  32,  32, 128, 16,  64, },
+      {  10,  23,  32, 128, 80, 176, },
+      {  10,  23,  32, 128, 80, 176, },
+      {  10,  32,  32, 128, 16,  64, },
     }, {
-      {   2, 176,  32, 128, 128, 128, 128, 128, 128, 128, 128, },
-      {   4,  28,  32, 128, 176, 192, 208, 128, 128, 128, 128, },
-      {   4,  28,  32, 128, 176, 192,  48, 128, 128, 128, 128, },
-      {   8, 160,  32, 128, 128, 128, 128, 128, 128, 128, 128, },
-      {   2,  28,  32, 128,  96, 128, 128, 128, 160, 192, 128, },
-      {   3,  28,  32, 128, 160, 176, 192, 128, 128, 128, 128, },
-      {   3,  26,  32, 128, 160, 176,  64, 128, 128, 128, 128, },
-      {   9,  24,  32, 128, 160, 176,  64, 128, 128, 128, 128, },
-      {   5,  24,  32, 128, 160, 176, 192, 128, 128, 128, 128, },
-      {   2,  25,  32, 128,  96, 128, 128, 128, 160, 192, 128, },
+      {   8, 224, 32, 128,  64, 128, },
+      {  10,  32, 32, 128,  16, 192, },
+      {  10,  32, 32, 128,  16,  64, },
+      {   9, 200, 32, 128,  64, 128, },
+      {   8,   8, 32, 128, 224, 128, },
+      {  10,  32, 32, 128,  16, 192, },
+      {  10,  32, 32, 128,  16,  64, },
+      {  10,  23, 32, 128,  80, 176, },
+      {  10,  23, 32, 128,  80, 176, },
+      {  10,  32, 32, 128,  16,  64, },
     }, {
-      {   2, 176,  32, 128, 128, 128, 128, 128, 128, 128, 128, },
-      {   1,  28,  32, 128, 176, 192, 208, 128, 128, 128, 128, },
-      {   1,  28,  32, 128, 176, 192,  48, 128, 128, 128, 128, },
-      {   4, 160,  32, 128, 128, 128, 128, 128, 128, 128, 128, },
-      {   2,  28,  32, 128,  96, 128, 128, 128, 160, 192, 128, },
-      {   2,  28,  32, 128, 160, 176, 192, 128, 128, 128, 128, },
-      {   3,  29,  32, 128, 160, 176,  64, 128, 128, 128, 128, },
-      {   4,  27,  32, 128, 160, 176,  64, 128, 128, 128, 128, },
-      {   2,  34,  32, 128, 160, 176, 192, 128, 128, 128, 128, },
-      {   1,  25,  32, 128,  96, 128, 128, 128, 160, 192, 128, },
+      {   8, 224, 32, 128,  64, 128, },
+      {  10,  32, 32, 128,  16, 192, },
+      {  10,  32, 32, 128,  16,  64, },
+      {   9, 200, 32, 128,  64, 128, },
+      {   8,   8, 32, 128, 224, 128, },
+      {  10,  32, 32, 128,  16, 192, },
+      {  10,  32, 32, 128,  16,  64, },
+      {  10,  23, 32, 128,  80, 176, },
+      {  10,  23, 32, 128,  80, 176, },
+      {  10,  32, 32, 128,  16,  64, },
 #if EXT_TX_SIZES == 4
     }, {
-      {   2, 176,  32, 128, 128, 128, 128, 128, 128, 128, 128, },
-      {   1,  12,  32, 128, 160, 176, 192, 128, 128, 128, 128, },
-      {   1,  17,  32, 128, 160, 176,  64, 128, 128, 128, 128, },
-      {   4,  41,  32, 128, 128, 128, 128, 128, 128, 128, 128, },
-      {   2,  17,  32, 128,  96, 128, 128, 128, 160, 192, 128, },
-      {   2,  14,  32, 128, 160, 176, 192, 128, 128, 128, 128, },
-      {   3,  19,  32, 128, 160, 176,  64, 128, 128, 128, 128, },
-      {   4,  27,  32, 128, 160, 176,  64, 128, 128, 128, 128, },
-      {   2,  34,  32, 128, 160, 176, 192, 128, 128, 128, 128, },
-      {   1,  15,  32, 128,  96, 128, 128, 128, 160, 192, 128, },
+      {   8, 224, 32, 128,  64, 128, },
+      {  10,  32, 32, 128,  16, 192, },
+      {  10,  32, 32, 128,  16,  64, },
+      {   9, 200, 32, 128,  64, 128, },
+      {   8,   8, 32, 128, 224, 128, },
+      {  10,  32, 32, 128,  16, 192, },
+      {  10,  32, 32, 128,  16,  64, },
+      {  10,  23, 32, 128,  80, 176, },
+      {  10,  23, 32, 128,  80, 176, },
+      {  10,  32, 32, 128,  16,  64, },
 #endif
     },
   }, {