changed UV plane loop filtering for TX_8X8

In commit 9a1d73d, loop filtering was added for UV 4x4 boundaries
when TX_8X8 is used by a MB. This commit further refined the decision
to be based on the actual transform used for the UV planes. When
UV planes use 4x4 transform, i.e. when prediction mode used is either
I8X8_PRED or SPLITMV, UV planes are filtered on 4x4 boundaries, and no
filtering is applied on 4x4 block boundaries when UV planes use 8X8
transform.

Change-Id: Ibb404face0a1d129b4b4abaf67c55d82e8df8bec

diff --git a/vp9/common/vp9_loopfilter.c b/vp9/common/vp9_loopfilter.c
index a7973bea6b2b95a17651faa081ae03e381524450..54658ebc5d06988b9ff7cd4ab8830a64830766d9 100644 (file)
--- a/vp9/common/vp9_loopfilter.c
+++ b/vp9/common/vp9_loopfilter.c
@@ -232,11 +232,11 @@ void vp9_loop_filter_frame(VP9_COMMON *cm,
   /* vp9_filter each macro block */
   for (mb_row = 0; mb_row < cm->mb_rows; mb_row++) {
     for (mb_col = 0; mb_col < cm->mb_cols; mb_col++) {
-      const int mode_index = lfi_n->mode_lf_lut[mode_info_context->mbmi.mode];
+      const MB_PREDICTION_MODE mode = mode_info_context->mbmi.mode;
+      const int mode_index = lfi_n->mode_lf_lut[mode];
       const int seg = mode_info_context->mbmi.segment_id;
       const int ref_frame = mode_info_context->mbmi.ref_frame;
       const int filter_level = lfi_n->lvl[seg][ref_frame][mode_index];
-
       if (filter_level) {
         const int skip_lf = mb_lf_skip(&mode_info_context->mbmi);
         const int tx_size = mode_info_context->mbmi.txfm_size;
@@ -255,19 +255,24 @@ void vp9_loop_filter_frame(VP9_COMMON *cm,
 #if CONFIG_WIDERLPF
             if (tx_size >= TX_16X16)
               vp9_lpf_mbv_w(y_ptr, u_ptr, v_ptr, post->y_stride,
-                                post->uv_stride, &lfi);
+                            post->uv_stride, &lfi);
             else
 #endif
               vp9_loop_filter_mbv(y_ptr, u_ptr, v_ptr, post->y_stride,
-                                post->uv_stride, &lfi);
+                                  post->uv_stride, &lfi);
           }
           if (!skip_lf) {
-            if (tx_size >= TX_8X8)
-              vp9_loop_filter_bv8x8(y_ptr, u_ptr, v_ptr, post->y_stride,
-                                    post->uv_stride, &lfi);
-            else
+            if (tx_size >= TX_8X8) {
+              if (tx_size == TX_8X8 && (mode == I8X8_PRED || mode == SPLITMV))
+                vp9_loop_filter_bv8x8(y_ptr, u_ptr, v_ptr, post->y_stride,
+                                      post->uv_stride, &lfi);
+              else
+                vp9_loop_filter_bv8x8(y_ptr, NULL, NULL, post->y_stride,
+                                      post->uv_stride, &lfi);
+            } else {
               vp9_loop_filter_bv(y_ptr, u_ptr, v_ptr, post->y_stride,
                                  post->uv_stride, &lfi);
+            }
 
           }
           /* don't apply across umv border */
@@ -279,19 +284,24 @@ void vp9_loop_filter_frame(VP9_COMMON *cm,
 #if CONFIG_WIDERLPF
             if (tx_size >= TX_16X16)
               vp9_lpf_mbh_w(y_ptr, u_ptr, v_ptr, post->y_stride,
-                                post->uv_stride, &lfi);
+                            post->uv_stride, &lfi);
             else
 #endif
               vp9_loop_filter_mbh(y_ptr, u_ptr, v_ptr, post->y_stride,
-                                post->uv_stride, &lfi);
+                                  post->uv_stride, &lfi);
           }
           if (!skip_lf) {
-            if (tx_size >= TX_8X8)
-              vp9_loop_filter_bh8x8(y_ptr, u_ptr, v_ptr, post->y_stride,
-                                  post->uv_stride, &lfi);
-            else
+            if (tx_size >= TX_8X8) {
+              if (tx_size == TX_8X8 && (mode == I8X8_PRED || mode == SPLITMV))
+                vp9_loop_filter_bh8x8(y_ptr, u_ptr, v_ptr, post->y_stride,
+                                      post->uv_stride, &lfi);
+              else
+                vp9_loop_filter_bh8x8(y_ptr, NULL, NULL, post->y_stride,
+                                      post->uv_stride, &lfi);
+            } else {
               vp9_loop_filter_bh(y_ptr, u_ptr, v_ptr, post->y_stride,
                                  post->uv_stride, &lfi);
+            }
           }
         } else {
           // FIXME: Not 8x8 aware