Disallow wide loopfilter on some chroma borders

Don't do the 15 tap filter if there aren't 8 pixels below/right of the
edge.

Change-Id: I62f16437c1d9ba59b6901a5fe71ddb2f472da344

diff --git a/vp9/common/vp9_loopfilter.c b/vp9/common/vp9_loopfilter.c
index 31acb27faf3c93d03e29fe092476e9dbaef9b7b0..f623c21db5680bf31c459742a212f9f36bd2fc63 100644 (file)
--- a/vp9/common/vp9_loopfilter.c
+++ b/vp9/common/vp9_loopfilter.c
@@ -299,6 +299,7 @@ static void filter_block_plane(VP9_COMMON *cm, MACROBLOCKD *xd,
       const TX_SIZE tx_size = plane ? get_uv_tx_size(&mi[c].mbmi)
                                     : mi[c].mbmi.txfm_size;
       const int skip_border_4x4_c = ss_x && mi_col + c == cm->mi_cols - 1;
+      const int skip_border_4x4_r = ss_y && mi_row + r == cm->mi_rows - 1;
 
       // Filter level can vary per MI
       if (!build_lfi(cm, &mi[c].mbmi,
@@ -307,15 +308,31 @@ static void filter_block_plane(VP9_COMMON *cm, MACROBLOCKD *xd,
 
       // Build masks based on the transform size of each block
       if (tx_size == TX_32X32) {
-        if (!skip_this_c && ((c >> ss_x) & 3) == 0)
-          mask_16x16_c |= 1 << (c >> ss_x);
-        if (!skip_this_r && ((r >> ss_y) & 3) == 0)
-          mask_16x16[r] |= 1 << (c >> ss_x);
+        if (!skip_this_c && ((c >> ss_x) & 3) == 0) {
+          if (!skip_border_4x4_c)
+            mask_16x16_c |= 1 << (c >> ss_x);
+          else
+            mask_8x8_c |= 1 << (c >> ss_x);
+        }
+        if (!skip_this_r && ((r >> ss_y) & 3) == 0) {
+          if (!skip_border_4x4_r)
+            mask_16x16[r] |= 1 << (c >> ss_x);
+          else
+            mask_8x8[r] |= 1 << (c >> ss_x);
+        }
       } else if (tx_size == TX_16X16) {
-        if (!skip_this_c && ((c >> ss_x) & 1) == 0)
-          mask_16x16_c |= 1 << (c >> ss_x);
-        if (!skip_this_r && ((r >> ss_y) & 1) == 0)
-          mask_16x16[r] |= 1 << (c >> ss_x);
+        if (!skip_this_c && ((c >> ss_x) & 1) == 0) {
+          if (!skip_border_4x4_c)
+            mask_16x16_c |= 1 << (c >> ss_x);
+          else
+            mask_8x8_c |= 1 << (c >> ss_x);
+        }
+        if (!skip_this_r && ((r >> ss_y) & 1) == 0) {
+          if (!skip_border_4x4_r)
+            mask_16x16[r] |= 1 << (c >> ss_x);
+          else
+            mask_8x8[r] |= 1 << (c >> ss_x);
+        }
       } else {
         // force 8x8 filtering on 32x32 boundaries
         if (!skip_this_c) {