From: Jingning Han Date: Sat, 17 Oct 2015 01:21:11 +0000 (-0700) Subject: Make loop filter support recursive transform block partitioning X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=94266f4f34206291d35551096d8271540a6bed0c;p=libvpx Make loop filter support recursive transform block partitioning This commit allows the loop filter to account for the recursive transform block partition when selecting the filter and mask. Change-Id: I62b6c2dcc0497cbe1f264b03c46163f55d2c9752 --- diff --git a/vp10/common/loopfilter.c b/vp10/common/loopfilter.c index 9f4d0f6e8..2eced346d 100644 --- a/vp10/common/loopfilter.c +++ b/vp10/common/loopfilter.c @@ -1224,61 +1224,104 @@ void vp10_filter_block_plane_non420(VP10_COMMON *cm, !blk_row : 1; const int skip_this_r = skip_this && !block_edge_above; +#if CONFIG_VAR_TX + TX_SIZE tx_size = (plane->plane_type == PLANE_TYPE_UV) + ? get_uv_tx_size(mbmi, plane) : mbmi->tx_size; +#else const TX_SIZE tx_size = (plane->plane_type == PLANE_TYPE_UV) - ? get_uv_tx_size(&mi[0].mbmi, plane) - : mi[0].mbmi.tx_size; + ? get_uv_tx_size(mbmi, plane) + : mbmi->tx_size; +#endif + const int skip_border_4x4_c = ss_x && mi_col + c == cm->mi_cols - 1; const int skip_border_4x4_r = ss_y && mi_row + r == cm->mi_rows - 1; + TX_SIZE tx_size_c = tx_size; + TX_SIZE tx_size_r = tx_size; + + int tx_size_mask = 0; // Filter level can vary per MI if (!(lfl[(r << 3) + (c >> ss_x)] = get_filter_level(&cm->lf_info, mbmi))) continue; + if (tx_size == TX_32X32) + tx_size_mask = 3; + else if (tx_size == TX_16X16) + tx_size_mask = 1; + else + tx_size_mask = 0; + +#if CONFIG_VAR_TX + if (is_inter_block(mbmi) && !mbmi->skip) + tx_size = (plane->plane_type == PLANE_TYPE_UV) ? + get_uv_tx_size_impl(mbmi->inter_tx_size[blk_row * 8 + blk_col], + sb_type, ss_x, ss_y) : + mbmi->inter_tx_size[blk_row * 8 + blk_col]; + + tx_size_r = VPXMIN(tx_size, cm->above_txfm_context[mi_col + c]); + tx_size_c = VPXMIN(tx_size, cm->left_txfm_context[(mi_row + r) & 0x07]); + + cm->above_txfm_context[mi_col + c] = tx_size; + cm->left_txfm_context[(mi_row + r) & 0x07] = tx_size; +#endif + // Build masks based on the transform size of each block - if (tx_size == TX_32X32) { - if (!skip_this_c && ((c >> ss_x) & 3) == 0) { + // handle vertical mask + if (tx_size_c == TX_32X32) { + if (!skip_this_c && ((c >> ss_x) & tx_size_mask) == 0) { if (!skip_border_4x4_c) mask_16x16_c |= 1 << (c >> ss_x); else mask_8x8_c |= 1 << (c >> ss_x); } - if (!skip_this_r && ((r >> ss_y) & 3) == 0) { - if (!skip_border_4x4_r) - mask_16x16[r] |= 1 << (c >> ss_x); - else - mask_8x8[r] |= 1 << (c >> ss_x); - } - } else if (tx_size == TX_16X16) { - if (!skip_this_c && ((c >> ss_x) & 1) == 0) { + } else if (tx_size_c == TX_16X16) { + if (!skip_this_c && ((c >> ss_x) & tx_size_mask) == 0) { if (!skip_border_4x4_c) mask_16x16_c |= 1 << (c >> ss_x); else mask_8x8_c |= 1 << (c >> ss_x); } - if (!skip_this_r && ((r >> ss_y) & 1) == 0) { - if (!skip_border_4x4_r) - mask_16x16[r] |= 1 << (c >> ss_x); - else - mask_8x8[r] |= 1 << (c >> ss_x); - } } else { // force 8x8 filtering on 32x32 boundaries - if (!skip_this_c) { - if (tx_size == TX_8X8 || ((c >> ss_x) & 3) == 0) + if (!skip_this_c && ((c >> ss_x) & tx_size_mask) == 0) { + if (tx_size_c == TX_8X8 || ((c >> ss_x) & 3) == 0) mask_8x8_c |= 1 << (c >> ss_x); else mask_4x4_c |= 1 << (c >> ss_x); } - if (!skip_this_r) { - if (tx_size == TX_8X8 || ((r >> ss_y) & 3) == 0) + if (!skip_this && tx_size_c < TX_8X8 && !skip_border_4x4_c && + ((c >> ss_x) & tx_size_mask) == 0) + mask_4x4_int[r] |= 1 << (c >> ss_x); + } + + // set horizontal mask + if (tx_size_r == TX_32X32) { + if (!skip_this_r && ((r >> ss_y) & tx_size_mask) == 0) { + if (!skip_border_4x4_r) + mask_16x16[r] |= 1 << (c >> ss_x); + else + mask_8x8[r] |= 1 << (c >> ss_x); + } + } else if (tx_size_r == TX_16X16) { + if (!skip_this_r && ((r >> ss_y) & tx_size_mask) == 0) { + if (!skip_border_4x4_r) + mask_16x16[r] |= 1 << (c >> ss_x); + else + mask_8x8[r] |= 1 << (c >> ss_x); + } + } else { + // force 8x8 filtering on 32x32 boundaries + if (!skip_this_r && ((r >> ss_y) & tx_size_mask) == 0) { + if (tx_size_r == TX_8X8 || ((r >> ss_y) & 3) == 0) mask_8x8[r] |= 1 << (c >> ss_x); else mask_4x4[r] |= 1 << (c >> ss_x); } - if (!skip_this && tx_size < TX_8X8 && !skip_border_4x4_c) + if (!skip_this && tx_size_r < TX_8X8 && !skip_border_4x4_c && + ((r >> ss_y) & tx_size_mask) == 0) mask_4x4_int[r] |= 1 << (c >> ss_x); } } @@ -1592,9 +1635,14 @@ void vp10_loop_filter_rows(YV12_BUFFER_CONFIG *frame_buffer, path = LF_PATH_SLOW; #endif +#if CONFIG_VAR_TX + memset(cm->above_txfm_context, TX_SIZES, cm->mi_cols); +#endif for (mi_row = start; mi_row < stop; mi_row += MI_BLOCK_SIZE) { MODE_INFO **mi = cm->mi_grid_visible + mi_row * cm->mi_stride; - +#if CONFIG_VAR_TX + memset(cm->left_txfm_context, TX_SIZES, 8); +#endif for (mi_col = 0; mi_col < cm->mi_cols; mi_col += MI_BLOCK_SIZE) { int plane; diff --git a/vp10/common/onyxc_int.h b/vp10/common/onyxc_int.h index 8e41e9a89..1ef92484e 100644 --- a/vp10/common/onyxc_int.h +++ b/vp10/common/onyxc_int.h @@ -303,6 +303,7 @@ typedef struct VP10Common { ENTROPY_CONTEXT *above_context; #if CONFIG_VAR_TX TXFM_CONTEXT *above_txfm_context; + TXFM_CONTEXT left_txfm_context[8]; #endif int above_context_alloc_cols; diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c index 8d9393582..942303b89 100644 --- a/vp10/encoder/rdopt.c +++ b/vp10/encoder/rdopt.c @@ -514,7 +514,7 @@ static void block_rd_txfm(int plane, int block, int blk_row, int blk_col, int dst_stride = xd->plane[plane].dst.stride; unsigned int tmp_sse; PREDICTION_MODE mode = (plane == 0) ? - get_y_mode(xd->mi[0], block) : mbmi->uv_mode; + get_y_mode(xd->mi[0], block) : mbmi->uv_mode; #if CONFIG_VP9_HIGHBITDEPTH vp10_encode_block_intra(plane, block, blk_row, blk_col,