Fix intermediate height in convolve

intermediate_height for horizontal filtering must be at least 8
pixels to be able to do vertical filtering correctly. Currently
it can be less for small block and y_step_q4 sizes.

Change-Id: I2ee28b0591b2041c2fa9844d0ae2ff8a1a59cc21

diff --git a/vp9/common/vp9_convolve.c b/vp9/common/vp9_convolve.c
index 46ae50349f3e6d5591ca120fb555473048ce0ed8..914afa70282c58fa993dcd612273e32a7a18d321 100644 (file)
--- a/vp9/common/vp9_convolve.c
+++ b/vp9/common/vp9_convolve.c
@@ -217,12 +217,13 @@ static void convolve_c(const uint8_t *src, int src_stride,
    * h == 64, taps == 8.
    */
   uint8_t temp[64 * 135];
-  int intermediate_height = ((h * y_step_q4) >> 4) + taps - 1;
+  int intermediate_height = MAX(((h * y_step_q4) >> 4), 1) + taps - 1;
 
   assert(w <= 64);
   assert(h <= 64);
   assert(taps <= 8);
   assert(y_step_q4 <= 32);
+  assert(x_step_q4 <= 32);
 
   if (intermediate_height < h)
     intermediate_height = h;
@@ -246,12 +247,13 @@ static void convolve_avg_c(const uint8_t *src, int src_stride,
    * h == 64, taps == 8.
    */
   uint8_t temp[64 * 135];
-  int intermediate_height = ((h * y_step_q4) >> 4) + taps - 1;
+  int intermediate_height = MAX(((h * y_step_q4) >> 4), 1) + taps - 1;
 
   assert(w <= 64);
   assert(h <= 64);
   assert(taps <= 8);
   assert(y_step_q4 <= 32);
+  assert(x_step_q4 <= 32);
 
   if (intermediate_height < h)
     intermediate_height = h;