From: Jingning Han <jingning@google.com>
Date: Fri, 13 May 2016 23:15:43 +0000 (-0700)
Subject: Properly handle 2D filter boundary extension
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=14dd5538e96c890cfd4e67d0813be07c24c669c7;p=libvpx

Properly handle 2D filter boundary extension

The amount of border extension needed in the first stage inter
filtering is decided by the length of the second stage filter
kernel.

Change-Id: Icddbc58c02234d5df09ff0eeebcf166ffe689203
---

diff --git a/vp10/common/vp10_convolve.c b/vp10/common/vp10_convolve.c
index 6514a60e8..6ffb42531 100644
--- a/vp10/common/vp10_convolve.c
+++ b/vp10/common/vp10_convolve.c
@@ -142,13 +142,21 @@ void vp10_convolve(const uint8_t *src, int src_stride, uint8_t *dst,
                  MAX_BLOCK_WIDTH];
     int temp_stride = MAX_BLOCK_WIDTH;
 #if CONFIG_DUAL_FILTER
-    InterpFilterParams filter_params =
+    InterpFilterParams filter_params_x =
         vp10_get_interp_filter_params(interp_filter[1 + 2 * ref_idx]);
+    InterpFilterParams filter_params_y =
+        vp10_get_interp_filter_params(interp_filter[0 + 2 * ref_idx]);
+    InterpFilterParams filter_params = filter_params_x;
+
+    // The filter size implies the required number of reference pixels for
+    // the second stage filtering. It is possible that the two directions
+    // require different filter sizes.
+    int filter_size = filter_params_y.taps;
 #else
     InterpFilterParams filter_params =
         vp10_get_interp_filter_params(interp_filter);
-#endif
     int filter_size = filter_params.taps;
+#endif
     int intermediate_height =
         (((h - 1) * y_step_q4 + subpel_y_q4) >> SUBPEL_BITS) + filter_size;
 
@@ -159,7 +167,7 @@ void vp10_convolve(const uint8_t *src, int src_stride, uint8_t *dst,
                    subpel_x_q4, x_step_q4, 0);
 
 #if CONFIG_DUAL_FILTER
-    filter_params = vp10_get_interp_filter_params(interp_filter[2 * ref_idx]);
+    filter_params = filter_params_y;
 #else
     filter_params = vp10_get_interp_filter_params(interp_filter);
 #endif