From: Tero Rintaluoma <teror@google.com>
Date: Fri, 5 Jul 2013 10:53:36 +0000 (+0300)
Subject: Fix intermediate height in convolve
X-Git-Tag: v1.3.0~914^2
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=18303b1263170e33369a2925b191844ea0ff0b05;p=libvpx

Fix intermediate height in convolve

intermediate_height for horizontal filtering must be at least 8
pixels to be able to do vertical filtering correctly. Currently
it can be less for small block and y_step_q4 sizes.

Change-Id: I2ee28b0591b2041c2fa9844d0ae2ff8a1a59cc21
---

diff --git a/vp9/common/vp9_convolve.c b/vp9/common/vp9_convolve.c
index 46ae50349..914afa702 100644
--- a/vp9/common/vp9_convolve.c
+++ b/vp9/common/vp9_convolve.c
@@ -217,12 +217,13 @@ static void convolve_c(const uint8_t *src, int src_stride,
    * h == 64, taps == 8.
    */
   uint8_t temp[64 * 135];
-  int intermediate_height = ((h * y_step_q4) >> 4) + taps - 1;
+  int intermediate_height = MAX(((h * y_step_q4) >> 4), 1) + taps - 1;
 
   assert(w <= 64);
   assert(h <= 64);
   assert(taps <= 8);
   assert(y_step_q4 <= 32);
+  assert(x_step_q4 <= 32);
 
   if (intermediate_height < h)
     intermediate_height = h;
@@ -246,12 +247,13 @@ static void convolve_avg_c(const uint8_t *src, int src_stride,
    * h == 64, taps == 8.
    */
   uint8_t temp[64 * 135];
-  int intermediate_height = ((h * y_step_q4) >> 4) + taps - 1;
+  int intermediate_height = MAX(((h * y_step_q4) >> 4), 1) + taps - 1;
 
   assert(w <= 64);
   assert(h <= 64);
   assert(taps <= 8);
   assert(y_step_q4 <= 32);
+  assert(x_step_q4 <= 32);
 
   if (intermediate_height < h)
     intermediate_height = h;