moved scaling from dequantization to inverse transform for T8x8

Previously, the scaling related to extended quantize range happens in
dequantization stage, which implies the coefficients form forward
transform are in different scale(4x) from dequantization coefficients
This worked fine when there was not distortion computation done based
on 8x8 transform, but it completely wracked the distortion estimation
based on transform coefficients and dequantized transform coefficients
introduced in commit f64725a00 for macroblocks using 8x8 transform.
This commit fixed the issue by moving the scaling into the stage of
inverse 8x8 transform.

TODO: Test&Verify the transform/quantization pipeline accuracy.

Change-Id: Iff77b36a965c2a6b247e59b9c59df93eba5d60e2

diff --git a/vp8/common/idctllm.c b/vp8/common/idctllm.c
index c113868987db44ead567d2e3c2acf059ea3df14c..22c9baa46ff70de0d8b83fc60551d3ccfc4979b6 100644 (file)
--- a/vp8/common/idctllm.c
+++ b/vp8/common/idctllm.c
@@ -211,7 +211,8 @@ void vp8_short_idct8x8_1_c(short *input, short *output, int pitch)
     short *op = output;
     short *orig_op = output;
     int shortpitch = pitch >> 1;
-    a1 = ((input[0] + 4) >> 3);
+    //a1 = ((input[0] + 4) >> 3);
+    a1 = ((input[0] + 16) >> 5);
     for (b = 0; b < 4; b++)
     {
         for (i = 0; i < 4; i++)
@@ -228,7 +229,8 @@ void vp8_short_idct8x8_1_c(short *input, short *output, int pitch)
 
 void vp8_dc_only_idct_add_8x8_c(short input_dc, unsigned char *pred_ptr, unsigned char *dst_ptr, int pitch, int stride)
 {
-    int a1 = ((input_dc + 4) >> 3);
+    //int a1 = ((input_dc + 4) >> 3);
+    int a1 = ((input_dc + 16) >> 5);
     int r, c, b;
     unsigned char *orig_pred = pred_ptr;
     unsigned char *orig_dst = dst_ptr;
@@ -394,7 +396,7 @@ void vp8_short_idct8x8_c(short *coefs, short *block, int pitch)
     {
         for (j = 0; j < TX_DIM; j++)
         {
-             X[i * TX_DIM + j] = (int)coefs[i * TX_DIM + j];
+             X[i * TX_DIM + j] = (int)(coefs[i * TX_DIM + j]+2)>>2;
         }
     }
   for (i = 0; i < 8; i++)

diff --git a/vp8/decoder/dequantize.c b/vp8/decoder/dequantize.c
index 83bf8a76c572ffbec8fcf7aeb94bf3af097cccdf..c48f5c23fab4d77185a9f338baac307247b044d2 100644 (file)
--- a/vp8/decoder/dequantize.c
+++ b/vp8/decoder/dequantize.c
@@ -131,7 +131,7 @@ void vp8_dequantize_b_2x2_c(BLOCKD *d)
 
     for (i = 0; i < 16; i++)
     {
-        DQ[i] = (short)((Q[i] * DQC[i]+2)>>2);
+        DQ[i] = (short)((Q[i] * DQC[i]));
     }
 #ifdef DEC_DEBUG
     if (dec_debug) {
@@ -164,12 +164,12 @@ void vp8_dequant_idct_add_8x8_c(short *input, short *dq, unsigned char *pred,
     }
 #endif
 
-    input[0]= (input[0] * dq[0]+2)>>2;
+    input[0]= input[0] * dq[0];
 
     // recover quantizer for 4 4x4 blocks
     for (i = 1; i < 64; i++)
     {
-      input[i]=(input[i] * dq[1]+2)>>2;
+      input[i]=input[i] * dq[1];
     }
 #ifdef DEC_DEBUG
     if (dec_debug) {
@@ -262,7 +262,7 @@ void vp8_dequant_dc_idct_add_8x8_c(short *input, short *dq, unsigned char *pred,
 #endif
     for (i = 1; i < 64; i++)
     {
-        input[i]=(input[i] * dq[1]+2)>>2;
+        input[i]=input[i] * dq[1];
     }
 
 #ifdef DEC_DEBUG

diff --git a/vp8/encoder/encodemb.c b/vp8/encoder/encodemb.c
index 270bdf585a5cf677a3f1e346ad3b4e92f660ff63..b8f92c9706d66ad4224776177fe43442d886fc8d 100644 (file)
--- a/vp8/encoder/encodemb.c
+++ b/vp8/encoder/encodemb.c
@@ -1035,7 +1035,7 @@ void optimize_b_8x8(MACROBLOCK *mb, int i, int type,
             final_eob = i;
         rc = vp8_default_zig_zag1d_8x8[i];
         qcoeff_ptr[rc] = x;
-        dqcoeff_ptr[rc] = (x * dequant_ptr[rc!=0]+2)>>2;
+        dqcoeff_ptr[rc] = (x * dequant_ptr[rc!=0]);
 
         next = tokens[i][best].next;
         best = (best_mask[best] >> i) & 1;

diff --git a/vp8/encoder/onyx_if.c b/vp8/encoder/onyx_if.c
index 8d170b18a69f25a10cd80984414b6eb2c169f694..c2f7e92e7bcf0ad2eadd591a050ed9db7a718a83 100644 (file)
--- a/vp8/encoder/onyx_if.c
+++ b/vp8/encoder/onyx_if.c
@@ -3411,7 +3411,6 @@ static void Pass1Encode(VP8_COMP *cpi, unsigned long *size, unsigned char *dest,
     scale_and_extend_source(cpi->un_scaled_source, cpi);
     vp8_first_pass(cpi);
 }
-
 //#define WRITE_RECON_BUFFER 1
 #if WRITE_RECON_BUFFER
 void write_cx_frame_to_file(YV12_BUFFER_CONFIG *frame, int this_frame)

diff --git a/vp8/encoder/quantize.c b/vp8/encoder/quantize.c
index 300c407e73168e3890e05afb546889cf98f92ef8..3b22e57b2dfa150c53d61114d12b00731a518539 100644 (file)
--- a/vp8/encoder/quantize.c
+++ b/vp8/encoder/quantize.c
@@ -566,7 +566,6 @@ void vp8_regular_quantize_b_2x2(BLOCK *b, BLOCKD *d)
       qcoeff_ptr[rc]  = x;                         // write to destination
       //dqcoeff_ptr[rc] = x * dequant_ptr[rc]/q2nd;        // dequantized value
       dqcoeff_ptr[rc] = x * dequant_ptr[rc];        // dequantized value
-      dqcoeff_ptr[rc] = (dqcoeff_ptr[rc]+2)>>2;
 
 
       if (y)
@@ -627,7 +626,6 @@ void vp8_regular_quantize_b_8x8(BLOCK *b, BLOCKD *d)
       qcoeff_ptr[rc]  = x;                         // write to destination
       //dqcoeff_ptr[rc] = x * dequant_ptr[rc!=0] / q1st;        // dequantized value
       dqcoeff_ptr[rc] = x * dequant_ptr[rc!=0];        // dequantized value
-      dqcoeff_ptr[rc] = (dqcoeff_ptr[rc]+2)>>2;
 
       if (y)
       {

diff --git a/vp8/encoder/rdopt.c b/vp8/encoder/rdopt.c
index 1d593fbee0968b6196666bd594111574fbfdbd60..c7bd8fc9965e6bc0a6069ea10ae3800df2bbaf68 100644 (file)
--- a/vp8/encoder/rdopt.c
+++ b/vp8/encoder/rdopt.c
@@ -777,11 +777,7 @@ static void macro_block_yrd_8x8( MACROBLOCK *mb,
     mb->e_mbd.dqcoeff[192] = 0;
     d = ENCODEMB_INVOKE(&rtcd->encodemb, mberr)(mb, 0) << 2;
 
-#if CONFIG_EXTEND_QRANGE
-    d += ENCODEMB_INVOKE(rtcd, berr)(mb_y2->coeff, x_y2->dqcoeff)<<2;
-#else
-    d += ENCODEMB_INVOKE(&rtcd->encodemb, berr)(mb_y2->coeff, x_y2->dqcoeff);
-#endif
+    d += ENCODEMB_INVOKE(&rtcd->encodemb, berr)(mb_y2->coeff, x_y2->dqcoeff)<<2;
 
     *Distortion = (d >> 4);
     // rate