Limit the transform block partition depth

Limit the recursive transform block partition depth to 2. For a
32x32 transform block unit, one can maximally go down to 8x8 transform
block size.

Change-Id: I2caa92bb2eee64762b7ecca8920259f7c50fb0aa

diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c
index 916832b459641cb9926f9dcb50673d2911ed3614..f0b37ce4cbfc31099a8fb38b2a54c94247df2a56 100644 (file)
--- a/av1/decoder/decodemv.c
+++ b/av1/decoder/decodemv.c
@@ -257,8 +257,8 @@ static int read_segment_id(aom_reader *r,
 #if CONFIG_VAR_TX
 static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd,
                                MB_MODE_INFO *mbmi, FRAME_COUNTS *counts,
-                               TX_SIZE tx_size, int blk_row, int blk_col,
-                               aom_reader *r) {
+                               TX_SIZE tx_size, int depth, int blk_row,
+                               int blk_col, aom_reader *r) {
   int is_split = 0;
   const int tx_row = blk_row >> 1;
   const int tx_col = blk_col >> 1;
@@ -275,6 +275,19 @@ static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd,
 
   if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
 
+  if (depth == 2) {
+    int idx, idy;
+    inter_tx_size[0][0] = tx_size;
+    for (idy = 0; idy < num_4x4_blocks_high_txsize_lookup[tx_size] / 2; ++idy)
+      for (idx = 0; idx < num_4x4_blocks_wide_txsize_lookup[tx_size] / 2; ++idx)
+        inter_tx_size[idy][idx] = tx_size;
+    mbmi->tx_size = tx_size;
+    if (counts) ++counts->txfm_partition[ctx][0];
+    txfm_partition_update(xd->above_txfm_context + tx_col,
+                          xd->left_txfm_context + tx_row, tx_size);
+    return;
+  }
+
   is_split = aom_read(r, cm->fc->txfm_partition_prob[ctx], ACCT_STR);
 
   if (is_split) {
@@ -297,8 +310,8 @@ static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd,
     for (i = 0; i < 4; ++i) {
       int offsetr = blk_row + ((i >> 1) << bsl);
       int offsetc = blk_col + ((i & 0x01) << bsl);
-      read_tx_size_vartx(cm, xd, mbmi, counts, tx_size - 1, offsetr, offsetc,
-                         r);
+      read_tx_size_vartx(cm, xd, mbmi, counts, tx_size - 1, depth + 1, offsetr,
+                         offsetc, r);
     }
   } else {
     int idx, idy;
@@ -1755,8 +1768,8 @@ static void read_inter_frame_mode_info(AV1Decoder *const pbi,
 #endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
         for (idy = 0; idy < height; idy += bs)
           for (idx = 0; idx < width; idx += bs)
-            read_tx_size_vartx(cm, xd, mbmi, xd->counts, max_tx_size, idy, idx,
-                               r);
+            read_tx_size_vartx(cm, xd, mbmi, xd->counts, max_tx_size,
+                               height != width, idy, idx, r);
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
       }
 #endif

diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index e0fb7ecb0afe873039689430f84df6d69bd3a34a..92e98b2c37fdd2b7daf98615c6cf4a592320af5b 100644 (file)
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -352,7 +352,8 @@ static int prob_diff_update_savings(const aom_tree_index *tree,
 #if CONFIG_VAR_TX
 static void write_tx_size_vartx(const AV1_COMMON *cm, const MACROBLOCKD *xd,
                                 const MB_MODE_INFO *mbmi, TX_SIZE tx_size,
-                                int blk_row, int blk_col, aom_writer *w) {
+                                int depth, int blk_row, int blk_col,
+                                aom_writer *w) {
   const int tx_row = blk_row >> 1;
   const int tx_col = blk_col >> 1;
   int max_blocks_high = num_4x4_blocks_high_lookup[mbmi->sb_type];
@@ -365,6 +366,12 @@ static void write_tx_size_vartx(const AV1_COMMON *cm, const MACROBLOCKD *xd,
 
   if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
 
+  if (depth == 2) {
+    txfm_partition_update(xd->above_txfm_context + tx_col,
+                          xd->left_txfm_context + tx_row, tx_size);
+    return;
+  }
+
   if (tx_size == mbmi->inter_tx_size[tx_row][tx_col]) {
     aom_write(w, 0, cm->fc->txfm_partition_prob[ctx]);
     txfm_partition_update(xd->above_txfm_context + tx_col,
@@ -386,7 +393,8 @@ static void write_tx_size_vartx(const AV1_COMMON *cm, const MACROBLOCKD *xd,
     for (i = 0; i < 4; ++i) {
       int offsetr = blk_row + ((i >> 1) << bsl);
       int offsetc = blk_col + ((i & 0x01) << bsl);
-      write_tx_size_vartx(cm, xd, mbmi, tx_size - 1, offsetr, offsetc, w);
+      write_tx_size_vartx(cm, xd, mbmi, tx_size - 1, depth + 1, offsetr,
+                          offsetc, w);
     }
   }
 }
@@ -1216,7 +1224,8 @@ static void pack_inter_mode_mvs(AV1_COMP *cpi, const MODE_INFO *mi,
 #endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
         for (idy = 0; idy < height; idy += bs)
           for (idx = 0; idx < width; idx += bs)
-            write_tx_size_vartx(cm, xd, mbmi, max_tx_size, idy, idx, w);
+            write_tx_size_vartx(cm, xd, mbmi, max_tx_size, height != width, idy,
+                                idx, w);
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
       }
 #endif  // CONFIG_EXT_TX && CONFIG_RECT_TX

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 627352b513f13d247e603c04eeebc0ea459d9f93..1351281d3eb49aba67bbc4e0f907a38934224a94 100644 (file)
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -3020,11 +3020,11 @@ void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size,
 
 static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row,
                             int blk_col, int plane, int block, TX_SIZE tx_size,
-                            BLOCK_SIZE plane_bsize, ENTROPY_CONTEXT *ta,
-                            ENTROPY_CONTEXT *tl, TXFM_CONTEXT *tx_above,
-                            TXFM_CONTEXT *tx_left, int *rate, int64_t *dist,
-                            int64_t *bsse, int *skip, int64_t ref_best_rd,
-                            int *is_cost_valid) {
+                            int depth, BLOCK_SIZE plane_bsize,
+                            ENTROPY_CONTEXT *ta, ENTROPY_CONTEXT *tl,
+                            TXFM_CONTEXT *tx_above, TXFM_CONTEXT *tx_left,
+                            int *rate, int64_t *dist, int64_t *bsse, int *skip,
+                            int64_t ref_best_rd, int *is_cost_valid) {
   MACROBLOCKD *const xd = &x->e_mbd;
   MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
   struct macroblock_plane *const p = &x->plane[plane];
@@ -3116,13 +3116,13 @@ static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row,
       *skip = 0;
     }
 
-    if (tx_size > TX_4X4)
+    if (tx_size > TX_4X4 && depth < 2)
       *rate += av1_cost_bit(cpi->common.fc->txfm_partition_prob[ctx], 0);
     this_rd = RDCOST(x->rdmult, x->rddiv, *rate, *dist);
     tmp_eob = p->eobs[block];
   }
 
-  if (tx_size > TX_4X4) {
+  if (tx_size > TX_4X4 && depth < 2) {
     BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
     int bsl = b_height_log2_lookup[bsize];
     int sub_step = num_4x4_blocks_txsize_lookup[tx_size - 1];
@@ -3141,9 +3141,10 @@ static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row,
       int offsetr = (i >> 1) << bsl;
       int offsetc = (i & 0x01) << bsl;
       select_tx_block(cpi, x, blk_row + offsetr, blk_col + offsetc, plane,
-                      block + i * sub_step, tx_size - 1, plane_bsize, ta, tl,
-                      tx_above, tx_left, &this_rate, &this_dist, &this_bsse,
-                      &this_skip, ref_best_rd - tmp_rd, &this_cost_valid);
+                      block + i * sub_step, tx_size - 1, depth + 1, plane_bsize,
+                      ta, tl, tx_above, tx_left, &this_rate, &this_dist,
+                      &this_bsse, &this_skip, ref_best_rd - tmp_rd,
+                      &this_cost_valid);
       sum_rate += this_rate;
       sum_dist += this_dist;
       sum_bsse += this_bsse;
@@ -3219,9 +3220,10 @@ static void inter_block_yrd(const AV1_COMP *cpi, MACROBLOCK *x, int *rate,
     for (idy = 0; idy < mi_height; idy += bh) {
       for (idx = 0; idx < mi_width; idx += bh) {
         select_tx_block(cpi, x, idy, idx, 0, block,
-                        max_txsize_lookup[plane_bsize], plane_bsize, ctxa, ctxl,
-                        tx_above, tx_left, &pnrate, &pndist, &pnsse, &pnskip,
-                        ref_best_rd - this_rd, &is_cost_valid);
+                        max_txsize_lookup[plane_bsize], mi_height != mi_width,
+                        plane_bsize, ctxa, ctxl, tx_above, tx_left, &pnrate,
+                        &pndist, &pnsse, &pnskip, ref_best_rd - this_rd,
+                        &is_cost_valid);
         *rate += pnrate;
         *distortion += pndist;
         *sse += pnsse;
@@ -7586,7 +7588,7 @@ static int64_t handle_inter_mode(
       // Y cost and distortion
       av1_subtract_plane(x, bsize, 0);
 #if CONFIG_VAR_TX
-      if (cm->tx_mode == TX_MODE_SELECT || xd->lossless[mbmi->segment_id]) {
+      if (cm->tx_mode == TX_MODE_SELECT && !xd->lossless[mbmi->segment_id]) {
         select_tx_type_yrd(cpi, x, rate_y, &distortion_y, &skippable_y, psse,
                            bsize, ref_best_rd);
       } else {