Properly handle 2D filter boundary extension

The amount of border extension needed in the first stage inter
filtering is decided by the length of the second stage filter
kernel.

Change-Id: Icddbc58c02234d5df09ff0eeebcf166ffe689203

diff --git a/vp10/common/vp10_convolve.c b/vp10/common/vp10_convolve.c
index 6514a60e84d2abeb13dd9332c2ecdeb7e3b23893..6ffb4253133978b64ada33855fb6729aa4e7771d 100644 (file)
--- a/vp10/common/vp10_convolve.c
+++ b/vp10/common/vp10_convolve.c
@@ -142,13 +142,21 @@ void vp10_convolve(const uint8_t *src, int src_stride, uint8_t *dst,
                  MAX_BLOCK_WIDTH];
     int temp_stride = MAX_BLOCK_WIDTH;
 #if CONFIG_DUAL_FILTER
-    InterpFilterParams filter_params =
+    InterpFilterParams filter_params_x =
         vp10_get_interp_filter_params(interp_filter[1 + 2 * ref_idx]);
+    InterpFilterParams filter_params_y =
+        vp10_get_interp_filter_params(interp_filter[0 + 2 * ref_idx]);
+    InterpFilterParams filter_params = filter_params_x;
+
+    // The filter size implies the required number of reference pixels for
+    // the second stage filtering. It is possible that the two directions
+    // require different filter sizes.
+    int filter_size = filter_params_y.taps;
 #else
     InterpFilterParams filter_params =
         vp10_get_interp_filter_params(interp_filter);
-#endif
     int filter_size = filter_params.taps;
+#endif
     int intermediate_height =
         (((h - 1) * y_step_q4 + subpel_y_q4) >> SUBPEL_BITS) + filter_size;
 
@@ -159,7 +167,7 @@ void vp10_convolve(const uint8_t *src, int src_stride, uint8_t *dst,
                    subpel_x_q4, x_step_q4, 0);
 
 #if CONFIG_DUAL_FILTER
-    filter_params = vp10_get_interp_filter_params(interp_filter[2 * ref_idx]);
+    filter_params = filter_params_y;
 #else
     filter_params = vp10_get_interp_filter_params(interp_filter);
 #endif