Fix forward transform bit range limits

Change-Id: I13c0ecff8c58a0571d9de4bc5fbbebe72533ccdb

diff --git a/vp10/encoder/dct.c b/vp10/encoder/dct.c
index ba0b50ab2b19a8d30716139a8ebab6169be7c679..219fd17fc74d753a08629b27ca4d20704c1e8296 100644 (file)
--- a/vp10/encoder/dct.c
+++ b/vp10/encoder/dct.c
@@ -204,7 +204,7 @@ static void fdct4(const tran_low_t *input, tran_low_t *output) {
   tran_low_t step[4];
 
   // stage 0
-  range_check(input, 4, 11);
+  range_check(input, 4, 14);
 
   // stage 1
   output[0] = input[0] + input[3];
@@ -212,7 +212,7 @@ static void fdct4(const tran_low_t *input, tran_low_t *output) {
   output[2] = input[1] - input[2];
   output[3] = input[0] - input[3];
 
-  range_check(output, 4, 12);
+  range_check(output, 4, 15);
 
   // stage 2
   temp = output[0] * cospi_16_64 + output[1] * cospi_16_64;
@@ -224,7 +224,7 @@ static void fdct4(const tran_low_t *input, tran_low_t *output) {
   temp = output[3] * cospi_24_64 + output[2] * -cospi_8_64;
   step[3] = (tran_low_t)fdct_round_shift(temp);
 
-  range_check(step, 4, 13);
+  range_check(step, 4, 16);
 
   // stage 3
   output[0] = step[0];
@@ -232,7 +232,7 @@ static void fdct4(const tran_low_t *input, tran_low_t *output) {
   output[2] = step[1];
   output[3] = step[3];
 
-  range_check(output, 4, 13);
+  range_check(output, 4, 16);
 }
 
 static void fdct8(const tran_low_t *input, tran_low_t *output) {
@@ -240,7 +240,7 @@ static void fdct8(const tran_low_t *input, tran_low_t *output) {
   tran_low_t step[8];
 
   // stage 0
-  range_check(input, 8, 12);
+  range_check(input, 8, 13);
 
   // stage 1
   output[0] = input[0] + input[7];
@@ -252,7 +252,7 @@ static void fdct8(const tran_low_t *input, tran_low_t *output) {
   output[6] = input[1] - input[6];
   output[7] = input[0] - input[7];
 
-  range_check(output, 8, 13);
+  range_check(output, 8, 14);
 
   // stage 2
   step[0] = output[0] + output[3];
@@ -266,7 +266,7 @@ static void fdct8(const tran_low_t *input, tran_low_t *output) {
   step[6] = (tran_low_t)fdct_round_shift(temp);
   step[7] = output[7];
 
-  range_check(step, 8, 14);
+  range_check(step, 8, 15);
 
   // stage 3
   temp = step[0] * cospi_16_64 + step[1] * cospi_16_64;
@@ -282,7 +282,7 @@ static void fdct8(const tran_low_t *input, tran_low_t *output) {
   output[6] = step[7] - step[6];
   output[7] = step[7] + step[6];
 
-  range_check(output, 8, 14);
+  range_check(output, 8, 16);
 
   // stage 4
   step[0] = output[0];
@@ -298,7 +298,7 @@ static void fdct8(const tran_low_t *input, tran_low_t *output) {
   temp = output[7] * cospi_28_64 + output[4] * -cospi_4_64;
   step[7] = (tran_low_t)fdct_round_shift(temp);
 
-  range_check(step, 8, 14);
+  range_check(step, 8, 16);
 
   // stage 5
   output[0] = step[0];
@@ -310,7 +310,7 @@ static void fdct8(const tran_low_t *input, tran_low_t *output) {
   output[6] = step[3];
   output[7] = step[7];
 
-  range_check(output, 8, 14);
+  range_check(output, 8, 16);
 }
 
 static void fdct16(const tran_low_t *input, tran_low_t *output) {