From: Jingning Han Date: Fri, 21 Oct 2016 17:14:36 +0000 (-0700) Subject: Limit the transform block partition depth X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=94d5bfccdd5dbd0b04d41592f41031d6cc561aab;p=libvpx Limit the transform block partition depth Limit the recursive transform block partition depth to 2. For a 32x32 transform block unit, one can maximally go down to 8x8 transform block size. Change-Id: I2caa92bb2eee64762b7ecca8920259f7c50fb0aa --- diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c index 916832b45..f0b37ce4c 100644 --- a/av1/decoder/decodemv.c +++ b/av1/decoder/decodemv.c @@ -257,8 +257,8 @@ static int read_segment_id(aom_reader *r, #if CONFIG_VAR_TX static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd, MB_MODE_INFO *mbmi, FRAME_COUNTS *counts, - TX_SIZE tx_size, int blk_row, int blk_col, - aom_reader *r) { + TX_SIZE tx_size, int depth, int blk_row, + int blk_col, aom_reader *r) { int is_split = 0; const int tx_row = blk_row >> 1; const int tx_col = blk_col >> 1; @@ -275,6 +275,19 @@ static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd, if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return; + if (depth == 2) { + int idx, idy; + inter_tx_size[0][0] = tx_size; + for (idy = 0; idy < num_4x4_blocks_high_txsize_lookup[tx_size] / 2; ++idy) + for (idx = 0; idx < num_4x4_blocks_wide_txsize_lookup[tx_size] / 2; ++idx) + inter_tx_size[idy][idx] = tx_size; + mbmi->tx_size = tx_size; + if (counts) ++counts->txfm_partition[ctx][0]; + txfm_partition_update(xd->above_txfm_context + tx_col, + xd->left_txfm_context + tx_row, tx_size); + return; + } + is_split = aom_read(r, cm->fc->txfm_partition_prob[ctx], ACCT_STR); if (is_split) { @@ -297,8 +310,8 @@ static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd, for (i = 0; i < 4; ++i) { int offsetr = blk_row + ((i >> 1) << bsl); int offsetc = blk_col + ((i & 0x01) << bsl); - read_tx_size_vartx(cm, xd, mbmi, counts, tx_size - 1, offsetr, offsetc, - r); + read_tx_size_vartx(cm, xd, mbmi, counts, tx_size - 1, depth + 1, offsetr, + offsetc, r); } } else { int idx, idy; @@ -1755,8 +1768,8 @@ static void read_inter_frame_mode_info(AV1Decoder *const pbi, #endif // CONFIG_EXT_TX && CONFIG_RECT_TX for (idy = 0; idy < height; idy += bs) for (idx = 0; idx < width; idx += bs) - read_tx_size_vartx(cm, xd, mbmi, xd->counts, max_tx_size, idy, idx, - r); + read_tx_size_vartx(cm, xd, mbmi, xd->counts, max_tx_size, + height != width, idy, idx, r); #if CONFIG_EXT_TX && CONFIG_RECT_TX } #endif diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c index e0fb7ecb0..92e98b2c3 100644 --- a/av1/encoder/bitstream.c +++ b/av1/encoder/bitstream.c @@ -352,7 +352,8 @@ static int prob_diff_update_savings(const aom_tree_index *tree, #if CONFIG_VAR_TX static void write_tx_size_vartx(const AV1_COMMON *cm, const MACROBLOCKD *xd, const MB_MODE_INFO *mbmi, TX_SIZE tx_size, - int blk_row, int blk_col, aom_writer *w) { + int depth, int blk_row, int blk_col, + aom_writer *w) { const int tx_row = blk_row >> 1; const int tx_col = blk_col >> 1; int max_blocks_high = num_4x4_blocks_high_lookup[mbmi->sb_type]; @@ -365,6 +366,12 @@ static void write_tx_size_vartx(const AV1_COMMON *cm, const MACROBLOCKD *xd, if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return; + if (depth == 2) { + txfm_partition_update(xd->above_txfm_context + tx_col, + xd->left_txfm_context + tx_row, tx_size); + return; + } + if (tx_size == mbmi->inter_tx_size[tx_row][tx_col]) { aom_write(w, 0, cm->fc->txfm_partition_prob[ctx]); txfm_partition_update(xd->above_txfm_context + tx_col, @@ -386,7 +393,8 @@ static void write_tx_size_vartx(const AV1_COMMON *cm, const MACROBLOCKD *xd, for (i = 0; i < 4; ++i) { int offsetr = blk_row + ((i >> 1) << bsl); int offsetc = blk_col + ((i & 0x01) << bsl); - write_tx_size_vartx(cm, xd, mbmi, tx_size - 1, offsetr, offsetc, w); + write_tx_size_vartx(cm, xd, mbmi, tx_size - 1, depth + 1, offsetr, + offsetc, w); } } } @@ -1216,7 +1224,8 @@ static void pack_inter_mode_mvs(AV1_COMP *cpi, const MODE_INFO *mi, #endif // CONFIG_EXT_TX && CONFIG_RECT_TX for (idy = 0; idy < height; idy += bs) for (idx = 0; idx < width; idx += bs) - write_tx_size_vartx(cm, xd, mbmi, max_tx_size, idy, idx, w); + write_tx_size_vartx(cm, xd, mbmi, max_tx_size, height != width, idy, + idx, w); #if CONFIG_EXT_TX && CONFIG_RECT_TX } #endif // CONFIG_EXT_TX && CONFIG_RECT_TX diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c index 627352b51..1351281d3 100644 --- a/av1/encoder/rdopt.c +++ b/av1/encoder/rdopt.c @@ -3020,11 +3020,11 @@ void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size, static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row, int blk_col, int plane, int block, TX_SIZE tx_size, - BLOCK_SIZE plane_bsize, ENTROPY_CONTEXT *ta, - ENTROPY_CONTEXT *tl, TXFM_CONTEXT *tx_above, - TXFM_CONTEXT *tx_left, int *rate, int64_t *dist, - int64_t *bsse, int *skip, int64_t ref_best_rd, - int *is_cost_valid) { + int depth, BLOCK_SIZE plane_bsize, + ENTROPY_CONTEXT *ta, ENTROPY_CONTEXT *tl, + TXFM_CONTEXT *tx_above, TXFM_CONTEXT *tx_left, + int *rate, int64_t *dist, int64_t *bsse, int *skip, + int64_t ref_best_rd, int *is_cost_valid) { MACROBLOCKD *const xd = &x->e_mbd; MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; struct macroblock_plane *const p = &x->plane[plane]; @@ -3116,13 +3116,13 @@ static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row, *skip = 0; } - if (tx_size > TX_4X4) + if (tx_size > TX_4X4 && depth < 2) *rate += av1_cost_bit(cpi->common.fc->txfm_partition_prob[ctx], 0); this_rd = RDCOST(x->rdmult, x->rddiv, *rate, *dist); tmp_eob = p->eobs[block]; } - if (tx_size > TX_4X4) { + if (tx_size > TX_4X4 && depth < 2) { BLOCK_SIZE bsize = txsize_to_bsize[tx_size]; int bsl = b_height_log2_lookup[bsize]; int sub_step = num_4x4_blocks_txsize_lookup[tx_size - 1]; @@ -3141,9 +3141,10 @@ static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row, int offsetr = (i >> 1) << bsl; int offsetc = (i & 0x01) << bsl; select_tx_block(cpi, x, blk_row + offsetr, blk_col + offsetc, plane, - block + i * sub_step, tx_size - 1, plane_bsize, ta, tl, - tx_above, tx_left, &this_rate, &this_dist, &this_bsse, - &this_skip, ref_best_rd - tmp_rd, &this_cost_valid); + block + i * sub_step, tx_size - 1, depth + 1, plane_bsize, + ta, tl, tx_above, tx_left, &this_rate, &this_dist, + &this_bsse, &this_skip, ref_best_rd - tmp_rd, + &this_cost_valid); sum_rate += this_rate; sum_dist += this_dist; sum_bsse += this_bsse; @@ -3219,9 +3220,10 @@ static void inter_block_yrd(const AV1_COMP *cpi, MACROBLOCK *x, int *rate, for (idy = 0; idy < mi_height; idy += bh) { for (idx = 0; idx < mi_width; idx += bh) { select_tx_block(cpi, x, idy, idx, 0, block, - max_txsize_lookup[plane_bsize], plane_bsize, ctxa, ctxl, - tx_above, tx_left, &pnrate, &pndist, &pnsse, &pnskip, - ref_best_rd - this_rd, &is_cost_valid); + max_txsize_lookup[plane_bsize], mi_height != mi_width, + plane_bsize, ctxa, ctxl, tx_above, tx_left, &pnrate, + &pndist, &pnsse, &pnskip, ref_best_rd - this_rd, + &is_cost_valid); *rate += pnrate; *distortion += pndist; *sse += pnsse; @@ -7586,7 +7588,7 @@ static int64_t handle_inter_mode( // Y cost and distortion av1_subtract_plane(x, bsize, 0); #if CONFIG_VAR_TX - if (cm->tx_mode == TX_MODE_SELECT || xd->lossless[mbmi->segment_id]) { + if (cm->tx_mode == TX_MODE_SELECT && !xd->lossless[mbmi->segment_id]) { select_tx_type_yrd(cpi, x, rate_y, &distortion_y, &skippable_y, psse, bsize, ref_best_rd); } else {