From: Hui Su <huisu@google.com>
Date: Fri, 31 Jul 2020 05:48:09 +0000 (-0700)
Subject: Assign correct values for zcoeff_blk in sub8x8 RDO
X-Git-Tag: v1.10.0-rc1~91
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=bdbf8725243c7eec4c8d0c59310a94bc002c81e2;p=libvpx

Assign correct values for zcoeff_blk in sub8x8 RDO

This fixes a lossless encoding bug as reported in the issue tracker.
Coding performance change is neutral.

BUG=webm:1700

Change-Id: I0f034b16b57e917e722709a7e9addef864b83d27
---

diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c
index 39b99d50c..37de4e483 100644
--- a/vp9/encoder/vp9_rdopt.c
+++ b/vp9/encoder/vp9_rdopt.c
@@ -4443,6 +4443,7 @@ void vp9_rd_pick_inter_mode_sub8x8(VP9_COMP *cpi, TileDataEnc *tile_data,
               tmp_best_sse = total_sse;
               tmp_best_skippable = skippable;
               tmp_best_mbmode = *mi;
+              x->sum_y_eobs[TX_4X4] = 0;
               for (i = 0; i < 4; i++) {
                 tmp_best_bmodes[i] = xd->mi[0]->bmi[i];
                 x->zcoeff_blk[TX_4X4][i] = !x->plane[0].eobs[i];
@@ -4476,6 +4477,11 @@ void vp9_rd_pick_inter_mode_sub8x8(VP9_COMP *cpi, TileDataEnc *tile_data,
             &rate, &rate_y, &distortion, &skippable, &total_sse,
             (int)this_rd_thresh, seg_mvs, bsi, 0, mi_row, mi_col);
         if (tmp_rd == INT64_MAX) continue;
+        x->sum_y_eobs[TX_4X4] = 0;
+        for (i = 0; i < 4; i++) {
+          x->zcoeff_blk[TX_4X4][i] = !x->plane[0].eobs[i];
+          x->sum_y_eobs[TX_4X4] += x->plane[0].eobs[i];
+        }
       } else {
         total_sse = tmp_best_sse;
         rate = tmp_best_rate;