Possible bug in vp8cx_encode_intra_macro_block

vp8_pick_intra4x4mby_modes uses the passed in distortion
for an early breakout.  The best distortion was never saved
and the distortion for TM_PRED was always used.

Change-Id: Idbaf73027408a4bba26601713725191a5d7b325e

diff --git a/vp8/encoder/encodeframe.c b/vp8/encoder/encodeframe.c
index 793191d241a92470e3df25b1e825bc8b1b5794c7..7ca27a624fdcf6fe09dc444233ab82f53705056f 100644 (file)
--- a/vp8/encoder/encodeframe.c
+++ b/vp8/encoder/encodeframe.c
@@ -1211,13 +1211,15 @@ int vp8cx_encode_intra_macro_block(VP8_COMP *cpi, MACROBLOCK *x, TOKENEXTRA **t)
 #endif
     {
 
-        int rate2, distortion2;
+        int rate2, best_distortion;
         MB_PREDICTION_MODE mode, best_mode = DC_PRED;
         int this_rd;
         Error16x16 = INT_MAX;
 
         for (mode = DC_PRED; mode <= TM_PRED; mode ++)
         {
+            int distortion2;
+
             x->e_mbd.mode_info_context->mbmi.mode = mode;
             vp8_build_intra_predictors_mby_ptr(&x->e_mbd);
             distortion2 = VARIANCE_INVOKE(&cpi->rtcd.variance, get16x16prederror)(x->src.y_buffer, x->src.y_stride, x->e_mbd.predictor, 16, 0x7fffffff);
@@ -1228,15 +1230,16 @@ int vp8cx_encode_intra_macro_block(VP8_COMP *cpi, MACROBLOCK *x, TOKENEXTRA **t)
             {
                 Error16x16 = this_rd;
                 best_mode = mode;
+                best_distortion = distortion2;
             }
         }
 
-        vp8_pick_intra4x4mby_modes(IF_RTCD(&cpi->rtcd), x, &rate2, &distortion2);
+        vp8_pick_intra4x4mby_modes(IF_RTCD(&cpi->rtcd), x, &rate2, &best_distortion);
 
-        if (distortion2 == INT_MAX)
+        if (best_distortion == INT_MAX)
             Error4x4 = INT_MAX;
         else
-            Error4x4 = RD_ESTIMATE(x->rdmult, x->rddiv, rate2, distortion2);
+            Error4x4 = RD_ESTIMATE(x->rdmult, x->rddiv, rate2, best_distortion);
 
         if (Error4x4 < Error16x16)
         {