From: Ronald S. Bultje Date: Mon, 22 Oct 2012 18:49:00 +0000 (-0700) Subject: 8x8 transform support in splitmv. X-Git-Tag: v1.3.0~1217^2~200^2~1 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=84ea002eb1e2e8276589ce24370d53e5d46013b4;p=libvpx 8x8 transform support in splitmv. For splitmv, where partitioning is 8x16, 16x8 or 8x8, this patch uses the 8x8 transform (instead of the 4x4) if txfm_mode is ALLOW_8X8 or ALLOW_16X16. For TX_MODE_SELECT, splitmv can indicate which of the 2 transform sizes (4x4 or 8x8) it wants to use. Gains (with hybridtx4x4/8x8/16x16 and tx_select experiments enabled) on derf: +0.9%, HD: +0.4%, STD/HD: +0.8% (SSIM or overall PSNR, both metrics show similar improvements). Change-Id: Ide954b000b415548ed92a7ac78e24f36e60fcf06 --- diff --git a/vp8/decoder/decodemv.c b/vp8/decoder/decodemv.c index 72ff126f2..064d16426 100644 --- a/vp8/decoder/decodemv.c +++ b/vp8/decoder/decodemv.c @@ -1285,10 +1285,12 @@ static void read_mb_modes_mv(VP8D_COMP *pbi, MODE_INFO *mi, MB_MODE_INFO *mbmi, #if CONFIG_TX_SELECT if (cm->txfm_mode == TX_MODE_SELECT && mbmi->mb_skip_coeff == 0 && ((mbmi->ref_frame == INTRA_FRAME && mbmi->mode <= I8X8_PRED) || - (mbmi->ref_frame != INTRA_FRAME && mbmi->mode != SPLITMV))) { + (mbmi->ref_frame != INTRA_FRAME && !(mbmi->mode == SPLITMV && + mbmi->partitioning == PARTITIONING_4X4)))) { // FIXME(rbultje) code ternary symbol once all experiments are merged mbmi->txfm_size = vp8_read(bc, cm->prob_tx[0]); - if (mbmi->txfm_size != TX_4X4 && mbmi->mode != I8X8_PRED) + if (mbmi->txfm_size != TX_4X4 && mbmi->mode != I8X8_PRED && + mbmi->mode != SPLITMV) mbmi->txfm_size += vp8_read(bc, cm->prob_tx[1]); } else #endif @@ -1297,8 +1299,9 @@ static void read_mb_modes_mv(VP8D_COMP *pbi, MODE_INFO *mi, MB_MODE_INFO *mbmi, (mbmi->ref_frame != INTRA_FRAME && mbmi->mode != SPLITMV))) { mbmi->txfm_size = TX_16X16; } else if (cm->txfm_mode >= ALLOW_8X8 && - ((mbmi->ref_frame == INTRA_FRAME && mbmi->mode != B_PRED) || - (mbmi->ref_frame != INTRA_FRAME && mbmi->mode != SPLITMV))) { + (!(mbmi->ref_frame == INTRA_FRAME && mbmi->mode == B_PRED) && + !(mbmi->ref_frame != INTRA_FRAME && mbmi->mode == SPLITMV && + mbmi->partitioning == PARTITIONING_4X4))) { mbmi->txfm_size = TX_8X8; } else { mbmi->txfm_size = TX_4X4; diff --git a/vp8/decoder/decodframe.c b/vp8/decoder/decodframe.c index 01739c0db..fe36a9db8 100644 --- a/vp8/decoder/decodframe.c +++ b/vp8/decoder/decodframe.c @@ -404,10 +404,16 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd, #endif } } else if (mode == SPLITMV) { - DEQUANT_INVOKE(&pbi->dequant, idct_add_y_block) - (xd->qcoeff, xd->block[0].dequant, - xd->predictor, xd->dst.y_buffer, - xd->dst.y_stride, xd->eobs); + if (tx_size == TX_8X8) { + vp8_dequant_idct_add_y_block_8x8_c(xd->qcoeff, xd->block[0].dequant, + xd->predictor, xd->dst.y_buffer, + xd->dst.y_stride, xd->eobs, xd); + } else { + DEQUANT_INVOKE(&pbi->dequant, + idct_add_y_block)(xd->qcoeff, xd->block[0].dequant, + xd->predictor, xd->dst.y_buffer, + xd->dst.y_stride, xd->eobs); + } } else { BLOCKD *b = &xd->block[24]; @@ -515,8 +521,10 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd, if (!xd->mode_info_context->mbmi.encoded_as_sb) { #endif if ((tx_size == TX_8X8 && - xd->mode_info_context->mbmi.mode != I8X8_PRED) - || tx_size == TX_16X16) + xd->mode_info_context->mbmi.mode != I8X8_PRED && + xd->mode_info_context->mbmi.mode != SPLITMV) + || tx_size == TX_16X16 + ) DEQUANT_INVOKE(&pbi->dequant, idct_add_uv_block_8x8) // (xd->qcoeff + 16 * 16, xd->block[16].dequant, xd->predictor + 16 * 16, xd->dst.u_buffer, xd->dst.v_buffer, diff --git a/vp8/decoder/detokenize.c b/vp8/decoder/detokenize.c index 85f213470..063fc2da6 100644 --- a/vp8/decoder/detokenize.c +++ b/vp8/decoder/detokenize.c @@ -537,7 +537,8 @@ int vp8_decode_mb_tokens_8x8(VP8D_COMP *pbi, MACROBLOCKD *xd, TX_TYPE tx_type = DCT_DCT; #endif - int bufthred = (xd->mode_info_context->mbmi.mode == I8X8_PRED) ? 16 : 24; + int bufthred = (xd->mode_info_context->mbmi.mode == I8X8_PRED || + xd->mode_info_context->mbmi.mode == SPLITMV) ? 16 : 24; if (xd->mode_info_context->mbmi.mode != B_PRED && xd->mode_info_context->mbmi.mode != SPLITMV && xd->mode_info_context->mbmi.mode != I8X8_PRED) { diff --git a/vp8/encoder/bitstream.c b/vp8/encoder/bitstream.c index f74f85fad..e2db43ff4 100644 --- a/vp8/encoder/bitstream.c +++ b/vp8/encoder/bitstream.c @@ -1225,7 +1225,8 @@ static void pack_inter_mode_mvs(VP8_COMP *const cpi, vp8_writer *const bc) { #if CONFIG_TX_SELECT if (((rf == INTRA_FRAME && mode <= I8X8_PRED) || - (rf != INTRA_FRAME && mode != SPLITMV)) && + (rf != INTRA_FRAME && !(mode == SPLITMV && + mi->partitioning == PARTITIONING_4X4))) && pc->txfm_mode == TX_MODE_SELECT && !((pc->mb_no_coeff_skip && mi->mb_skip_coeff) || (segfeature_active(xd, segment_id, SEG_LVL_EOB) && @@ -1233,7 +1234,7 @@ static void pack_inter_mode_mvs(VP8_COMP *const cpi, vp8_writer *const bc) { TX_SIZE sz = mi->txfm_size; // FIXME(rbultje) code ternary symbol once all experiments are merged vp8_write(bc, sz != TX_4X4, pc->prob_tx[0]); - if (sz != TX_4X4 && mode != I8X8_PRED) + if (sz != TX_4X4 && mode != I8X8_PRED && mode != SPLITMV) vp8_write(bc, sz != TX_8X8, pc->prob_tx[1]); } #endif diff --git a/vp8/encoder/encodeframe.c b/vp8/encoder/encodeframe.c index adfbfc79b..8f662d5d0 100644 --- a/vp8/encoder/encodeframe.c +++ b/vp8/encoder/encodeframe.c @@ -2167,7 +2167,9 @@ void vp8cx_encode_inter_macroblock (VP8_COMP *cpi, MACROBLOCK *x, if (mbmi->mode != B_PRED && mbmi->mode != I8X8_PRED && mbmi->mode != SPLITMV) { cpi->txfm_count[mbmi->txfm_size]++; - } else if (mbmi->mode == I8X8_PRED) { + } else if (mbmi->mode == I8X8_PRED || + (mbmi->mode == SPLITMV && + mbmi->partitioning != PARTITIONING_4X4)) { cpi->txfm_count_8x8p[mbmi->txfm_size]++; } } else @@ -2175,8 +2177,10 @@ void vp8cx_encode_inter_macroblock (VP8_COMP *cpi, MACROBLOCK *x, if (mbmi->mode != B_PRED && mbmi->mode != I8X8_PRED && mbmi->mode != SPLITMV && cpi->common.txfm_mode >= ALLOW_16X16) { mbmi->txfm_size = TX_16X16; - } else if (mbmi->mode != B_PRED && mbmi->mode != SPLITMV && - cpi->common.txfm_mode >= ALLOW_8X8) { + } else if (mbmi->mode != B_PRED && + !(mbmi->mode == SPLITMV && + mbmi->partitioning == PARTITIONING_4X4) && + cpi->common.txfm_mode >= ALLOW_8X8) { mbmi->txfm_size = TX_8X8; } else { mbmi->txfm_size = TX_4X4; diff --git a/vp8/encoder/encodemb.c b/vp8/encoder/encodemb.c index dc54d05a2..ab321d8b0 100644 --- a/vp8/encoder/encodemb.c +++ b/vp8/encoder/encodemb.c @@ -638,6 +638,7 @@ void vp8_optimize_mby_8x8(MACROBLOCK *x, const VP8_ENCODER_RTCD *rtcd) { ENTROPY_CONTEXT_PLANES t_above, t_left; ENTROPY_CONTEXT *ta; ENTROPY_CONTEXT *tl; + int has_2nd_order = x->e_mbd.mode_info_context->mbmi.mode != SPLITMV; if (!x->e_mbd.above_context || !x->e_mbd.left_context) return; @@ -647,7 +648,7 @@ void vp8_optimize_mby_8x8(MACROBLOCK *x, const VP8_ENCODER_RTCD *rtcd) { ta = (ENTROPY_CONTEXT *)&t_above; tl = (ENTROPY_CONTEXT *)&t_left; - type = PLANE_TYPE_Y_NO_DC; + type = has_2nd_order ? PLANE_TYPE_Y_NO_DC : PLANE_TYPE_Y_WITH_DC; for (b = 0; b < 16; b += 4) { optimize_b(x, b, type, ta + vp8_block2above_8x8[b], tl + vp8_block2left_8x8[b], @@ -657,8 +658,11 @@ void vp8_optimize_mby_8x8(MACROBLOCK *x, const VP8_ENCODER_RTCD *rtcd) { } // 8x8 always have 2nd roder haar block - check_reset_8x8_2nd_coeffs(&x->e_mbd, - ta + vp8_block2above_8x8[24], tl + vp8_block2left_8x8[24]); + if (has_2nd_order) { + check_reset_8x8_2nd_coeffs(&x->e_mbd, + ta + vp8_block2above_8x8[24], + tl + vp8_block2left_8x8[24]); + } } void vp8_optimize_mbuv_8x8(MACROBLOCK *x, const VP8_ENCODER_RTCD *rtcd) { @@ -898,11 +902,25 @@ void vp8_encode_inter16x16(const VP8_ENCODER_RTCD *rtcd, MACROBLOCK *x) { optimize_mb_16x16(x, rtcd); vp8_inverse_transform_mb_16x16(IF_RTCD(&rtcd->common->idct), xd); } else if (tx_size == TX_8X8) { - vp8_transform_mb_8x8(x); - vp8_quantize_mb_8x8(x); - if (x->optimize) - optimize_mb_8x8(x, rtcd); - vp8_inverse_transform_mb_8x8(IF_RTCD(&rtcd->common->idct), xd); + if (xd->mode_info_context->mbmi.mode == SPLITMV) { + assert(xd->mode_info_context->mbmi.partitioning != PARTITIONING_4X4); + vp8_transform_mby_8x8(x); + vp8_transform_mbuv_4x4(x); + vp8_quantize_mby_8x8(x); + vp8_quantize_mbuv_4x4(x); + if (x->optimize) { + vp8_optimize_mby_8x8(x, rtcd); + vp8_optimize_mbuv_4x4(x, rtcd); + } + vp8_inverse_transform_mby_8x8(IF_RTCD(&rtcd->common->idct), xd); + vp8_inverse_transform_mbuv_4x4(IF_RTCD(&rtcd->common->idct), xd); + } else { + vp8_transform_mb_8x8(x); + vp8_quantize_mb_8x8(x); + if (x->optimize) + optimize_mb_8x8(x, rtcd); + vp8_inverse_transform_mb_8x8(IF_RTCD(&rtcd->common->idct), xd); + } } else { transform_mb_4x4(x); vp8_quantize_mb_4x4(x); diff --git a/vp8/encoder/rdopt.c b/vp8/encoder/rdopt.c index fe6c03081..aadadbcb1 100644 --- a/vp8/encoder/rdopt.c +++ b/vp8/encoder/rdopt.c @@ -2221,12 +2221,22 @@ static int64_t encode_inter_mb_segment_8x8(MACROBLOCK *x, int which_label, int *labelyrate, int *distortion, + int64_t *otherrd, ENTROPY_CONTEXT *ta, ENTROPY_CONTEXT *tl, const VP8_ENCODER_RTCD *rtcd) { int i, j; MACROBLOCKD *xd = &x->e_mbd; const int iblock[4] = { 0, 1, 4, 5 }; + int othercost = 0, otherdist = 0; + ENTROPY_CONTEXT_PLANES tac, tlc; + ENTROPY_CONTEXT *tacp = (ENTROPY_CONTEXT *) &tac, + *tlcp = (ENTROPY_CONTEXT *) &tlc; + + if (otherrd) { + memcpy(&tac, ta, sizeof(ENTROPY_CONTEXT_PLANES)); + memcpy(&tlc, tl, sizeof(ENTROPY_CONTEXT_PLANES)); + } *distortion = 0; *labelyrate = 0; @@ -2234,8 +2244,9 @@ static int64_t encode_inter_mb_segment_8x8(MACROBLOCK *x, int ib = vp8_i8x8_block[i]; if (labels[ib] == which_label) { - BLOCKD *bd = &xd->block[ib]; - BLOCK *be = &x->block[ib]; + int idx = (ib & 8) + ((ib & 2) << 1); + BLOCKD *bd = &xd->block[ib], *bd2 = &xd->block[idx]; + BLOCK *be = &x->block[ib], *be2 = &x->block[idx]; int thisdistortion; vp8_build_inter_predictors4b(xd, bd, 16); @@ -2243,24 +2254,66 @@ static int64_t encode_inter_mb_segment_8x8(MACROBLOCK *x, vp8_build_2nd_inter_predictors4b(xd, bd, 16); vp8_subtract_4b_c(be, bd, 16); - for (j = 0; j < 4; j += 2) { - bd = &xd->block[ib + iblock[j]]; - be = &x->block[ib + iblock[j]]; - x->vp8_short_fdct8x4(be->src_diff, be->coeff, 32); - x->quantize_b_4x4_pair(be, be + 1, bd, bd + 1); - thisdistortion = vp8_block_error_c(be->coeff, bd->dqcoeff, 32); + if (xd->mode_info_context->mbmi.txfm_size == TX_4X4) { + if (otherrd) { + x->vp8_short_fdct8x8(be->src_diff, be2->coeff, 32); + x->quantize_b_8x8(be2, bd2); + thisdistortion = vp8_block_error_c(be2->coeff, bd2->dqcoeff, 64); + otherdist += thisdistortion; + othercost += cost_coeffs(x, bd2, PLANE_TYPE_Y_WITH_DC, + tacp + vp8_block2above_8x8[idx], + tlcp + vp8_block2left_8x8[idx], TX_8X8); + } + for (j = 0; j < 4; j += 2) { + bd = &xd->block[ib + iblock[j]]; + be = &x->block[ib + iblock[j]]; + x->vp8_short_fdct8x4(be->src_diff, be->coeff, 32); + x->quantize_b_4x4_pair(be, be + 1, bd, bd + 1); + thisdistortion = vp8_block_error_c(be->coeff, bd->dqcoeff, 32); + *distortion += thisdistortion; + *labelyrate += cost_coeffs(x, bd, PLANE_TYPE_Y_WITH_DC, + ta + vp8_block2above[ib + iblock[j]], + tl + vp8_block2left[ib + iblock[j]], + TX_4X4); + *labelyrate += cost_coeffs(x, bd + 1, PLANE_TYPE_Y_WITH_DC, + ta + vp8_block2above[ib + iblock[j] + 1], + tl + vp8_block2left[ib + iblock[j]], + TX_4X4); + } + } else /* 8x8 */ { + if (otherrd) { + for (j = 0; j < 4; j += 2) { + BLOCKD *bd3 = &xd->block[ib + iblock[j]]; + BLOCK *be3 = &x->block[ib + iblock[j]]; + x->vp8_short_fdct8x4(be3->src_diff, be3->coeff, 32); + x->quantize_b_4x4_pair(be3, be3 + 1, bd3, bd3 + 1); + thisdistortion = vp8_block_error_c(be3->coeff, bd3->dqcoeff, 32); + otherdist += thisdistortion; + othercost += cost_coeffs(x, bd3, PLANE_TYPE_Y_WITH_DC, + tacp + vp8_block2above[ib + iblock[j]], + tlcp + vp8_block2left[ib + iblock[j]], + TX_4X4); + othercost += cost_coeffs(x, bd3 + 1, PLANE_TYPE_Y_WITH_DC, + tacp + vp8_block2above[ib + iblock[j] + 1], + tlcp + vp8_block2left[ib + iblock[j]], + TX_4X4); + } + } + x->vp8_short_fdct8x8(be->src_diff, be2->coeff, 32); + x->quantize_b_8x8(be2, bd2); + thisdistortion = vp8_block_error_c(be2->coeff, bd2->dqcoeff, 64); *distortion += thisdistortion; - *labelyrate += cost_coeffs(x, bd, PLANE_TYPE_Y_WITH_DC, - ta + vp8_block2above[ib + iblock[j]], - tl + vp8_block2left[ib + iblock[j]], TX_4X4); - *labelyrate += cost_coeffs(x, bd + 1, PLANE_TYPE_Y_WITH_DC, - ta + vp8_block2above[ib + iblock[j] + 1], - tl + vp8_block2left[ib + iblock[j]], - TX_4X4); + *labelyrate += cost_coeffs(x, bd2, PLANE_TYPE_Y_WITH_DC, + ta + vp8_block2above_8x8[idx], + tl + vp8_block2left_8x8[idx], TX_8X8); } } } *distortion >>= 2; + if (otherrd) { + othercost >>= 2; + *otherrd = RDCOST(x->rdmult, x->rddiv, othercost, otherdist); + } return RDCOST(x->rdmult, x->rddiv, *labelyrate, *distortion); } @@ -2273,6 +2326,7 @@ typedef struct { int64_t segment_rd; SPLITMV_PARTITIONING_TYPE segment_num; + TX_SIZE txfm_size; int r; int d; int segment_yrate; @@ -2298,11 +2352,14 @@ int mv_check_bounds(MACROBLOCK *x, int_mv *mv) { return r; } -static void rd_check_segment(VP8_COMP *cpi, MACROBLOCK *x, - BEST_SEG_INFO *bsi, - SPLITMV_PARTITIONING_TYPE segmentation, - int_mv seg_mvs[16 /* n_blocks */] - [MAX_REF_FRAMES - 1]) { +static void rd_check_segment_txsize(VP8_COMP *cpi, MACROBLOCK *x, + BEST_SEG_INFO *bsi, + SPLITMV_PARTITIONING_TYPE segmentation, + TX_SIZE tx_size, int64_t *otherrds, + int64_t *rds, int *completed, + /* 16 = n_blocks */ + int_mv seg_mvs[16 /* n_blocks */] + [MAX_REF_FRAMES - 1]) { int i, j; int const *labels; int br = 0, bd = 0; @@ -2310,12 +2367,12 @@ static void rd_check_segment(VP8_COMP *cpi, MACROBLOCK *x, MB_MODE_INFO * mbmi = &x->e_mbd.mode_info_context->mbmi; int label_count; - int64_t this_segment_rd = 0; + int64_t this_segment_rd = 0, other_segment_rd; int label_mv_thresh; int rate = 0; int sbr = 0, sbd = 0; int segmentyrate = 0; - uint8_t best_eobs[16]; + uint8_t best_eobs[16] = { 0 }; vp8_variance_fn_ptr_t *v_fn_ptr; @@ -2348,16 +2405,18 @@ static void rd_check_segment(VP8_COMP *cpi, MACROBLOCK *x, rate += vp8_cost_mv_ref(cpi, SPLITMV, bsi->mdcounts); this_segment_rd += RDCOST(x->rdmult, x->rddiv, rate, 0); br += rate; + other_segment_rd = this_segment_rd; - for (i = 0; i < label_count; i++) { + mbmi->txfm_size = tx_size; + for (i = 0; i < label_count && this_segment_rd < bsi->segment_rd; i++) { int_mv mode_mv[B_MODE_COUNT], second_mode_mv[B_MODE_COUNT]; - int64_t best_label_rd = INT64_MAX; + int64_t best_label_rd = INT64_MAX, best_other_rd = INT64_MAX; B_PREDICTION_MODE mode_selected = ZERO4X4; int bestlabelyrate = 0; // search for the best motion vector on this segment for (this_mode = LEFT4X4; this_mode <= NEW4X4; this_mode ++) { - int64_t this_rd; + int64_t this_rd, other_rd; int distortion; int labelyrate; ENTROPY_CONTEXT_PLANES t_above_s, t_left_s; @@ -2385,16 +2444,17 @@ static void rd_check_segment(VP8_COMP *cpi, MACROBLOCK *x, break; if (cpi->compressor_speed) { - if (segmentation == BLOCK_8X16 || segmentation == BLOCK_16X8) { + if (segmentation == PARTITIONING_8X16 || + segmentation == PARTITIONING_16X8) { bsi->mvp.as_int = bsi->sv_mvp[i].as_int; - if (i == 1 && segmentation == BLOCK_16X8) + if (i == 1 && segmentation == PARTITIONING_16X8) bsi->mvp.as_int = bsi->sv_mvp[2].as_int; step_param = bsi->sv_istep[i]; } // use previous block's result as next block's MV predictor. - if (segmentation == BLOCK_4X4 && i > 0) { + if (segmentation == PARTITIONING_4X4 && i > 0) { bsi->mvp.as_int = x->e_mbd.block[i - 1].bmi.as_mv.first.as_int; if (i == 4 || i == 8 || i == 12) bsi->mvp.as_int = x->e_mbd.block[i - 4].bmi.as_mv.first.as_int; @@ -2483,10 +2543,11 @@ static void rd_check_segment(VP8_COMP *cpi, MACROBLOCK *x, this_rd = encode_inter_mb_segment(x, labels, i, &labelyrate, &distortion, ta_s, tl_s, IF_RTCD(&cpi->rtcd)); + other_rd = this_rd; } else { this_rd = encode_inter_mb_segment_8x8(x, labels, i, &labelyrate, - &distortion, ta_s, tl_s, - IF_RTCD(&cpi->rtcd)); + &distortion, &other_rd, + ta_s, tl_s, IF_RTCD(&cpi->rtcd)); } this_rd += RDCOST(x->rdmult, x->rddiv, rate, 0); rate += labelyrate; @@ -2497,9 +2558,20 @@ static void rd_check_segment(VP8_COMP *cpi, MACROBLOCK *x, bestlabelyrate = labelyrate; mode_selected = this_mode; best_label_rd = this_rd; - for (j = 0; j < 16; j++) - if (labels[j] == i) - best_eobs[j] = x->e_mbd.block[j].eob; + if (x->e_mbd.mode_info_context->mbmi.txfm_size == TX_4X4) { + for (j = 0; j < 16; j++) + if (labels[j] == i) + best_eobs[j] = x->e_mbd.block[j].eob; + } else { + for (j = 0; j < 4; j++) { + int ib = vp8_i8x8_block[j], idx = j * 4; + + if (labels[ib] == i) + best_eobs[idx] = x->e_mbd.block[idx].eob; + } + } + if (other_rd < best_other_rd) + best_other_rd = other_rd; vpx_memcpy(ta_b, ta_s, sizeof(ENTROPY_CONTEXT_PLANES)); vpx_memcpy(tl_b, tl_s, sizeof(ENTROPY_CONTEXT_PLANES)); @@ -2511,19 +2583,18 @@ static void rd_check_segment(VP8_COMP *cpi, MACROBLOCK *x, vpx_memcpy(tl, tl_b, sizeof(ENTROPY_CONTEXT_PLANES)); labels2mode(x, labels, i, mode_selected, &mode_mv[mode_selected], - &second_mode_mv[mode_selected], seg_mvs[i], bsi->ref_mv, - bsi->second_ref_mv, XMVCOST); + &second_mode_mv[mode_selected], seg_mvs[i], + bsi->ref_mv, bsi->second_ref_mv, XMVCOST); br += sbr; bd += sbd; segmentyrate += bestlabelyrate; this_segment_rd += best_label_rd; - - if (this_segment_rd >= bsi->segment_rd) { - break; - } - - + other_segment_rd += best_other_rd; + if (rds) + rds[i] = this_segment_rd; + if (otherrds) + rds[i] = other_segment_rd; } /* for each label */ if (this_segment_rd < bsi->segment_rd) { @@ -2532,6 +2603,7 @@ static void rd_check_segment(VP8_COMP *cpi, MACROBLOCK *x, bsi->segment_yrate = segmentyrate; bsi->segment_rd = this_segment_rd; bsi->segment_num = segmentation; + bsi->txfm_size = mbmi->txfm_size; // store everything needed to come back to this!! for (i = 0; i < 16; i++) { @@ -2544,6 +2616,112 @@ static void rd_check_segment(VP8_COMP *cpi, MACROBLOCK *x, bsi->eobs[i] = best_eobs[i]; } } + + if (completed) { + *completed = i; + } +} + +static void rd_check_segment(VP8_COMP *cpi, MACROBLOCK *x, + BEST_SEG_INFO *bsi, + unsigned int segmentation, + /* 16 = n_blocks */ + int_mv seg_mvs[16][MAX_REF_FRAMES - 1], + int64_t txfm_cache[NB_TXFM_MODES]) { +#if CONFIG_TX_SELECT + int i, n, c = vp8_mbsplit_count[segmentation]; + + if (segmentation == PARTITIONING_4X4) { + int64_t rd[16]; + + rd_check_segment_txsize(cpi, x, bsi, segmentation, TX_4X4, NULL, + rd, &n, seg_mvs); + if (n == c) { + for (i = 0; i < NB_TXFM_MODES; i++) { + if (rd[c - 1] < txfm_cache[i]) + txfm_cache[i] = rd[c - 1]; + } + } + } else { + int64_t diff, base_rd; + int cost4x4 = vp8_cost_bit(cpi->common.prob_tx[0], 0); + int cost8x8 = vp8_cost_bit(cpi->common.prob_tx[0], 1); + + if (cpi->common.txfm_mode == TX_MODE_SELECT) { + int64_t rd4x4[4], rd8x8[4]; + int n4x4, n8x8, nmin; + BEST_SEG_INFO bsi4x4, bsi8x8; + + /* factor in cost of cost4x4/8x8 in decision */ + vpx_memcpy(&bsi4x4, bsi, sizeof(*bsi)); + vpx_memcpy(&bsi8x8, bsi, sizeof(*bsi)); + rd_check_segment_txsize(cpi, x, &bsi4x4, segmentation, + TX_4X4, NULL, rd4x4, &n4x4, seg_mvs); + rd_check_segment_txsize(cpi, x, &bsi8x8, segmentation, + TX_8X8, NULL, rd8x8, &n8x8, seg_mvs); + if (bsi4x4.segment_num == segmentation) { + bsi4x4.segment_rd += RDCOST(x->rdmult, x->rddiv, cost4x4, 0); + if (bsi4x4.segment_rd < bsi->segment_rd) + vpx_memcpy(bsi, &bsi4x4, sizeof(*bsi)); + } + if (bsi8x8.segment_num == segmentation) { + bsi8x8.segment_rd += RDCOST(x->rdmult, x->rddiv, cost8x8, 0); + if (bsi8x8.segment_rd < bsi->segment_rd) + vpx_memcpy(bsi, &bsi8x8, sizeof(*bsi)); + } + n = n4x4 > n8x8 ? n4x4 : n8x8; + if (n == c) { + nmin = n4x4 < n8x8 ? n4x4 : n8x8; + diff = rd8x8[nmin - 1] - rd4x4[nmin - 1]; + if (n == n4x4) { + base_rd = rd4x4[c - 1]; + } else { + base_rd = rd8x8[c - 1] - diff; + } + } + } else { + int64_t rd[4], otherrd[4]; + + if (cpi->common.txfm_mode == ONLY_4X4) { + rd_check_segment_txsize(cpi, x, bsi, segmentation, TX_4X4, otherrd, + rd, &n, seg_mvs); + if (n == c) { + base_rd = rd[c - 1]; + diff = otherrd[c - 1] - rd[c - 1]; + } + } else /* use 8x8 transform */ { + rd_check_segment_txsize(cpi, x, bsi, segmentation, TX_8X8, otherrd, + rd, &n, seg_mvs); + if (n == c) { + diff = rd[c - 1] - otherrd[c - 1]; + base_rd = otherrd[c - 1]; + } + } + } + + if (n == c) { + if (base_rd < txfm_cache[ONLY_4X4]) { + txfm_cache[ONLY_4X4] = base_rd; + } + if (base_rd + diff < txfm_cache[1]) { + txfm_cache[ALLOW_8X8] = txfm_cache[ALLOW_16X16] = base_rd + diff; + } + if (diff < 0) { + base_rd += diff + RDCOST(x->rdmult, x->rddiv, cost8x8, 0); + } else { + base_rd += RDCOST(x->rdmult, x->rddiv, cost4x4, 0); + } + if (base_rd < txfm_cache[TX_MODE_SELECT]) { + txfm_cache[TX_MODE_SELECT] = base_rd; + } + } + } +#else + rd_check_segment_txsize(cpi, x, bsi, segmentation, + (segmentation == PARTITIONING_4X4 || + cpi->common.txfm_mode == ONLY_4X4) ? TX_4X4 : TX_8X8, + NULL, NULL, NULL, seg_mvs); +#endif } static __inline @@ -2570,12 +2748,15 @@ static int rd_pick_best_mbsegmentation(VP8_COMP *cpi, MACROBLOCK *x, int *skippable, int mvthresh, int_mv seg_mvs[NB_PARTITIONINGS] [16 /* n_blocks */] - [MAX_REF_FRAMES - 1]) { + [MAX_REF_FRAMES - 1], + int64_t txfm_cache[NB_TXFM_MODES]) { int i; BEST_SEG_INFO bsi; MB_MODE_INFO * mbmi = &x->e_mbd.mode_info_context->mbmi; vpx_memset(&bsi, 0, sizeof(bsi)); + for (i = 0; i < NB_TXFM_MODES; i++) + txfm_cache[i] = INT64_MAX; bsi.segment_rd = best_rd; bsi.ref_mv = best_ref_mv; @@ -2583,6 +2764,7 @@ static int rd_pick_best_mbsegmentation(VP8_COMP *cpi, MACROBLOCK *x, bsi.mvp.as_int = best_ref_mv->as_int; bsi.mvthresh = mvthresh; bsi.mdcounts = mdcounts; + bsi.txfm_size = TX_4X4; for (i = 0; i < 16; i++) bsi.modes[i] = ZERO4X4; @@ -2591,19 +2773,18 @@ static int rd_pick_best_mbsegmentation(VP8_COMP *cpi, MACROBLOCK *x, /* for now, we will keep the original segmentation order when in best quality mode */ rd_check_segment(cpi, x, &bsi, PARTITIONING_16X8, - seg_mvs[PARTITIONING_16X8]); + seg_mvs[PARTITIONING_16X8], txfm_cache); rd_check_segment(cpi, x, &bsi, PARTITIONING_8X16, - seg_mvs[PARTITIONING_8X16]); + seg_mvs[PARTITIONING_8X16], txfm_cache); rd_check_segment(cpi, x, &bsi, PARTITIONING_8X8, - seg_mvs[PARTITIONING_8X8]); + seg_mvs[PARTITIONING_8X8], txfm_cache); rd_check_segment(cpi, x, &bsi, PARTITIONING_4X4, - seg_mvs[PARTITIONING_4X4]); + seg_mvs[PARTITIONING_4X4], txfm_cache); } else { int sr; rd_check_segment(cpi, x, &bsi, PARTITIONING_8X8, - seg_mvs[PARTITIONING_8X8]); - + seg_mvs[PARTITIONING_8X8], txfm_cache); if (bsi.segment_rd < best_rd) { int tmp_col_min = x->mv_col_min; @@ -2621,6 +2802,7 @@ static int rd_pick_best_mbsegmentation(VP8_COMP *cpi, MACROBLOCK *x, /* Use 8x8 result as 16x8/8x16's predictor MV. Adjust search range * according to the closeness of 2 MV. */ + /* block 8X16 */ sr = MAXF((abs(bsi.sv_mvp[0].as_mv.row - bsi.sv_mvp[2].as_mv.row)) >> 3, (abs(bsi.sv_mvp[0].as_mv.col - bsi.sv_mvp[2].as_mv.col)) >> 3); vp8_cal_step_param(sr, &bsi.sv_istep[0]); @@ -2630,8 +2812,9 @@ static int rd_pick_best_mbsegmentation(VP8_COMP *cpi, MACROBLOCK *x, vp8_cal_step_param(sr, &bsi.sv_istep[1]); rd_check_segment(cpi, x, &bsi, PARTITIONING_8X16, - seg_mvs[PARTITIONING_8X16]); + seg_mvs[PARTITIONING_8X16], txfm_cache); + /* block 16X8 */ sr = MAXF((abs(bsi.sv_mvp[0].as_mv.row - bsi.sv_mvp[1].as_mv.row)) >> 3, (abs(bsi.sv_mvp[0].as_mv.col - bsi.sv_mvp[1].as_mv.col)) >> 3); vp8_cal_step_param(sr, &bsi.sv_istep[0]); @@ -2641,15 +2824,16 @@ static int rd_pick_best_mbsegmentation(VP8_COMP *cpi, MACROBLOCK *x, vp8_cal_step_param(sr, &bsi.sv_istep[1]); rd_check_segment(cpi, x, &bsi, PARTITIONING_16X8, - seg_mvs[PARTITIONING_16X8]); + seg_mvs[PARTITIONING_16X8], txfm_cache); /* If 8x8 is better than 16x8/8x16, then do 4x4 search */ /* Not skip 4x4 if speed=0 (good quality) */ - if (cpi->sf.no_skip_block4x4_search || bsi.segment_num == BLOCK_8X8) { + if (cpi->sf.no_skip_block4x4_search || + bsi.segment_num == PARTITIONING_8X8) { /* || (sv_segment_rd8x8-bsi.segment_rd) < sv_segment_rd8x8>>5) */ bsi.mvp.as_int = bsi.sv_mvp[0].as_int; rd_check_segment(cpi, x, &bsi, PARTITIONING_4X4, - seg_mvs[PARTITIONING_4X4]); + seg_mvs[PARTITIONING_4X4], txfm_cache); } /* restore UMV window */ @@ -2673,9 +2857,12 @@ static int rd_pick_best_mbsegmentation(VP8_COMP *cpi, MACROBLOCK *x, *returntotrate = bsi.r; *returndistortion = bsi.d; *returnyrate = bsi.segment_yrate; - *skippable = mby_is_skippable_4x4(&x->e_mbd, 0); + *skippable = bsi.txfm_size == TX_4X4 ? + mby_is_skippable_4x4(&x->e_mbd, 0) : + mby_is_skippable_8x8(&x->e_mbd, 0); /* save partitions */ + mbmi->txfm_size = bsi.txfm_size; mbmi->partitioning = bsi.segment_num; x->partition_info->count = vp8_mbsplit_count[bsi.segment_num]; @@ -3745,12 +3932,12 @@ void vp8_rd_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, int (mbmi->ref_frame == GOLDEN_FRAME) ? cpi->rd_threshes[THR_NEWG] : this_rd_thresh; - mbmi->txfm_size = TX_4X4; // FIXME use 8x8 in case of 8x8/8x16/16x8 tmp_rd = rd_pick_best_mbsegmentation(cpi, x, &best_ref_mv, second_ref, best_yrd, mdcounts, &rate, &rate_y, &distortion, &skippable, - this_rd_thresh, seg_mvs); + this_rd_thresh, seg_mvs, + txfm_cache); rate2 += rate; distortion2 += distortion; @@ -4146,7 +4333,7 @@ void vp8_rd_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, int if (!mode_excluded && this_rd != INT64_MAX) { for (i = 0; i < NB_TXFM_MODES; i++) { int64_t adj_rd; - if (this_mode != B_PRED && this_mode != SPLITMV) { + if (this_mode != B_PRED) { adj_rd = this_rd + txfm_cache[i] - txfm_cache[cm->txfm_mode]; } else { adj_rd = this_rd; diff --git a/vp8/encoder/tokenize.c b/vp8/encoder/tokenize.c index d46637a3e..2c2d0bde2 100644 --- a/vp8/encoder/tokenize.c +++ b/vp8/encoder/tokenize.c @@ -619,7 +619,8 @@ void vp8_tokenize_mb(VP8_COMP *cpi, xd->mode_info_context->mbmi.mb_skip_coeff = mb_is_skippable_16x16(xd); break; case TX_8X8: - if (xd->mode_info_context->mbmi.mode == I8X8_PRED) + if (xd->mode_info_context->mbmi.mode == I8X8_PRED || + xd->mode_info_context->mbmi.mode == SPLITMV) xd->mode_info_context->mbmi.mb_skip_coeff = mb_is_skippable_8x8_4x4uv(xd, 0); else xd->mode_info_context->mbmi.mb_skip_coeff = mb_is_skippable_8x8(xd, has_y2_block); @@ -695,7 +696,8 @@ void vp8_tokenize_mb(VP8_COMP *cpi, *(A + vp8_block2above_8x8[b] + 1) = *(A + vp8_block2above_8x8[b]); *(L + vp8_block2left_8x8[b] + 1) = *(L + vp8_block2left_8x8[b]); } - if (xd->mode_info_context->mbmi.mode == I8X8_PRED) { + if (xd->mode_info_context->mbmi.mode == I8X8_PRED || + xd->mode_info_context->mbmi.mode == SPLITMV) { tokenize1st_order_chroma_4x4(xd, t, cpi, dry_run); } else { for (b = 16; b < 24; b += 4) { @@ -1308,7 +1310,8 @@ void vp8_stuff_mb(VP8_COMP *cpi, MACROBLOCKD *xd, TOKENEXTRA **t, int dry_run) { if (tx_size == TX_16X16) { vp8_stuff_mb_16x16(cpi, xd, t, dry_run); } else if (tx_size == TX_8X8) { - if (xd->mode_info_context->mbmi.mode == I8X8_PRED) { + if (xd->mode_info_context->mbmi.mode == I8X8_PRED || + xd->mode_info_context->mbmi.mode == SPLITMV) { vp8_stuff_mb_8x8_4x4uv(cpi, xd, t, dry_run); } else { vp8_stuff_mb_8x8(cpi, xd, t, dry_run);