From: Jingning Han <jingning@google.com>
Date: Tue, 27 Oct 2015 02:39:30 +0000 (-0700)
Subject: Fix early termination flag in recursive transform block search
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=236623cf2c9e001a6defb713e4b912d270e5a5c9;p=libvpx

Fix early termination flag in recursive transform block search

Properly reset the early termination flag in the recursive transform
block partitioning rate-distortion optimization scheme.

Change-Id: Ibfe918f21f11dcb1ec267c09f954c635305cc95a
---

diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c
index e567d04c4..abbf0ee44 100644
--- a/vp10/encoder/rdopt.c
+++ b/vp10/encoder/rdopt.c
@@ -1614,7 +1614,7 @@ static void select_tx_block(const VP10_COMP *cpi, MACROBLOCK *x,
     int64_t tmp_rd = 0;
 
     --bsl;
-    for (i = 0; i < 4; ++i) {
+    for (i = 0; i < 4 && this_cost_valid; ++i) {
       int offsetr = (i >> 1) << bsl;
       int offsetc = (i & 0x01) << bsl;
       select_tx_block(cpi, x, blk_row + offsetr, blk_col + offsetc,
@@ -1644,11 +1644,15 @@ static void select_tx_block(const VP10_COMP *cpi, MACROBLOCK *x,
       for (idx = 0; idx < (1 << tx_size) / 2; ++idx)
         mbmi->inter_tx_size[tx_idx + (idy << 3) + idx] = tx_size;
     mbmi->tx_size = tx_size;
+    if (this_rd == INT64_MAX)
+      *is_cost_valid = 0;
   } else {
     *rate = sum_rate;
     *dist = sum_dist;
     *bsse = sum_bsse;
     *skip = all_skip;
+    if (sum_rd == INT64_MAX)
+      *is_cost_valid = 0;
   }
 }