changed loop filter for MBs using 8x8 transform

This commit added a set of loop filter functions for macroblocks
using 8x8 transform. First we turned off the regular loop filtering
on 4x4 block boundaries that do not exist in macroblocks using 8x8
transform. Second, we change to use the same loop filter(mask and
7 tap filter) that used for macroblock edge filtering.

Change-Id: I3a00460b7674ced116917d86812ffc32578c1d3a

diff --git a/vp8/common/alloccommon.c b/vp8/common/alloccommon.c
index 4457c8821c7433da4732cbf3333f2c2d37bbe192..e643eff09016aecb97a14551e90bd78719d27fe0 100644 (file)
--- a/vp8/common/alloccommon.c
+++ b/vp8/common/alloccommon.c
@@ -153,10 +153,10 @@ void vp8_setup_version(VP8_COMMON *cm)
         if (!CONFIG_EXPERIMENTAL)
             vpx_internal_error(&cm->error, VPX_CODEC_UNSUP_BITSTREAM,
                                "Bitstream was created by an experimental "
-                               "encoder");        
+                               "encoder");
         cm->experimental = 1;
     }
-    
+
     switch (cm->version & 0x3)
     {
     case 0:

diff --git a/vp8/common/loopfilter.c b/vp8/common/loopfilter.c
index 052d00e0f9ea82c4b2da31895f4b1c5e6da66c24..677f7c6e40a675b74557f9602db928a0c90f55f7 100644 (file)
--- a/vp8/common/loopfilter.c
+++ b/vp8/common/loopfilter.c
@@ -22,6 +22,8 @@ typedef unsigned char uc;
 
 prototype_loopfilter(vp8_loop_filter_horizontal_edge_c);
 prototype_loopfilter(vp8_loop_filter_vertical_edge_c);
+
+
 prototype_loopfilter(vp8_mbloop_filter_horizontal_edge_c);
 prototype_loopfilter(vp8_mbloop_filter_vertical_edge_c);
 
@@ -72,6 +74,16 @@ void vp8_loop_filter_bh_c(unsigned char *y_ptr, unsigned char *u_ptr,
         vp8_loop_filter_horizontal_edge_c(v_ptr + 4 * uv_stride, uv_stride, lfi->blim, lfi->lim, lfi->hev_thr, 1);
 }
 
+#if CONFIG_T8X8
+void vp8_loop_filter_bh8x8_c(unsigned char *y_ptr, unsigned char *u_ptr,
+                          unsigned char *v_ptr, int y_stride, int uv_stride,
+                          loop_filter_info *lfi)
+{
+    vp8_mbloop_filter_horizontal_edge_c(
+        y_ptr + 8 * y_stride, y_stride, lfi->blim, lfi->lim, lfi->hev_thr, 2);
+}
+#endif
+
 void vp8_loop_filter_bhs_c(unsigned char *y_ptr, int y_stride,
                            const unsigned char *blimit)
 {
@@ -96,6 +108,17 @@ void vp8_loop_filter_bv_c(unsigned char *y_ptr, unsigned char *u_ptr,
         vp8_loop_filter_vertical_edge_c(v_ptr + 4, uv_stride, lfi->blim, lfi->lim, lfi->hev_thr, 1);
 }
 
+#if CONFIG_T8X8
+void vp8_loop_filter_bv8x8_c(unsigned char *y_ptr, unsigned char *u_ptr,
+                          unsigned char *v_ptr, int y_stride, int uv_stride,
+                          loop_filter_info *lfi)
+{
+    vp8_mbloop_filter_vertical_edge_c(
+        y_ptr + 8, y_stride, lfi->blim, lfi->lim, lfi->hev_thr, 2);
+}
+
+#endif
+
 void vp8_loop_filter_bvs_c(unsigned char *y_ptr, int y_stride,
                            const unsigned char *blimit)
 {
@@ -328,7 +351,9 @@ void vp8_loop_filter_frame
             const int mode_index = lfi_n->mode_lf_lut[mode_info_context->mbmi.mode];
             const int seg = mode_info_context->mbmi.segment_id;
             const int ref_frame = mode_info_context->mbmi.ref_frame;
-
+#if CONFIG_T8X8
+            int tx_type = mode_info_context->mbmi.txfm_size;
+#endif
             filter_level = lfi_n->lvl[seg][ref_frame][mode_index];
 
             if (filter_level)
@@ -350,8 +375,17 @@ void vp8_loop_filter_frame
                         (y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
 #endif
                     if (!skip_lf)
-                        LF_INVOKE(&cm->rtcd.loopfilter, normal_b_v)
-                        (y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
+                    {
+#if CONFIG_T8X8
+                        if(tx_type == TX_8X8)
+                            vp8_loop_filter_bv8x8_c
+                            (y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
+                        else
+#endif
+                            LF_INVOKE(&cm->rtcd.loopfilter, normal_b_v)
+                            (y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
+
+                    }
 
                     /* don't apply across umv border */
                     if (mb_row > 0)
@@ -362,9 +396,18 @@ void vp8_loop_filter_frame
                         LF_INVOKE(&cm->rtcd.loopfilter, normal_mb_h)
                         (y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
 #endif
+
                     if (!skip_lf)
-                        LF_INVOKE(&cm->rtcd.loopfilter, normal_b_h)
-                        (y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
+                    {
+#if CONFIG_T8X8
+                        if(tx_type == TX_8X8)
+                            vp8_loop_filter_bh8x8_c
+                            (y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
+                        else
+#endif
+                            LF_INVOKE(&cm->rtcd.loopfilter, normal_b_h)
+                            (y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
+                    }
                 }
                 else
                 {
@@ -448,6 +491,9 @@ void vp8_loop_filter_frame_yonly
             const int mode_index = lfi_n->mode_lf_lut[mode_info_context->mbmi.mode];
             const int seg = mode_info_context->mbmi.segment_id;
             const int ref_frame = mode_info_context->mbmi.ref_frame;
+#if CONFIG_T8X8
+            int tx_type = mode_info_context->mbmi.txfm_size;
+#endif
 
             filter_level = lfi_n->lvl[seg][ref_frame][mode_index];
 
@@ -469,10 +515,17 @@ void vp8_loop_filter_frame_yonly
                         LF_INVOKE(&cm->rtcd.loopfilter, normal_mb_v)
                         (y_ptr, 0, 0, post->y_stride, 0, &lfi);
 #endif
-
                     if (!skip_lf)
-                        LF_INVOKE(&cm->rtcd.loopfilter, normal_b_v)
-                        (y_ptr, 0, 0, post->y_stride, 0, &lfi);
+                    {
+#if CONFIG_T8X8
+                        if(tx_type == TX_8X8)
+                            vp8_loop_filter_bv8x8_c
+                            (y_ptr, 0, 0, post->y_stride, 0, &lfi);
+                        else
+#endif
+                            LF_INVOKE(&cm->rtcd.loopfilter, normal_b_v)
+                            (y_ptr, 0, 0, post->y_stride, 0, &lfi);
+                    }
 
                     /* don't apply across umv border */
                     if (mb_row > 0)
@@ -484,8 +537,16 @@ void vp8_loop_filter_frame_yonly
                         (y_ptr, 0, 0, post->y_stride, 0, &lfi);
 #endif
                     if (!skip_lf)
-                        LF_INVOKE(&cm->rtcd.loopfilter, normal_b_h)
-                        (y_ptr, 0, 0, post->y_stride, 0, &lfi);
+                    {
+#if CONFIG_T8X8
+                        if(tx_type == TX_8X8)
+                            vp8_loop_filter_bh8x8_c
+                            (y_ptr, 0, 0, post->y_stride, 0, &lfi);
+                        else
+#endif
+                            LF_INVOKE(&cm->rtcd.loopfilter, normal_b_h)
+                            (y_ptr, 0, 0, post->y_stride, 0, &lfi);
+                    }
                 }
                 else
                 {

diff --git a/vp8/common/loopfilter_filters.c b/vp8/common/loopfilter_filters.c
index 72ba31eaa908fddde055ef7c9e9c7840cb479282..afd0fde28afffbeea737e9ceae9111635107fc37 100644 (file)
--- a/vp8/common/loopfilter_filters.c
+++ b/vp8/common/loopfilter_filters.c
@@ -10,6 +10,7 @@
 
 
 #include <stdlib.h>
+#include "vpx_config.h"
 #include "loopfilter.h"
 #include "onyxc_int.h"