Fix early termination flag in recursive transform block search

Properly reset the early termination flag in the recursive transform
block partitioning rate-distortion optimization scheme.

Change-Id: Ibfe918f21f11dcb1ec267c09f954c635305cc95a

diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c
index e567d04c4f5e876386fb2f87ed3737c6675e10d1..abbf0ee44be4ad49b7a6241833add481e7ecd5ab 100644 (file)
--- a/vp10/encoder/rdopt.c
+++ b/vp10/encoder/rdopt.c
@@ -1614,7 +1614,7 @@ static void select_tx_block(const VP10_COMP *cpi, MACROBLOCK *x,
     int64_t tmp_rd = 0;
 
     --bsl;
-    for (i = 0; i < 4; ++i) {
+    for (i = 0; i < 4 && this_cost_valid; ++i) {
       int offsetr = (i >> 1) << bsl;
       int offsetc = (i & 0x01) << bsl;
       select_tx_block(cpi, x, blk_row + offsetr, blk_col + offsetc,
@@ -1644,11 +1644,15 @@ static void select_tx_block(const VP10_COMP *cpi, MACROBLOCK *x,
       for (idx = 0; idx < (1 << tx_size) / 2; ++idx)
         mbmi->inter_tx_size[tx_idx + (idy << 3) + idx] = tx_size;
     mbmi->tx_size = tx_size;
+    if (this_rd == INT64_MAX)
+      *is_cost_valid = 0;
   } else {
     *rate = sum_rate;
     *dist = sum_dist;
     *bsse = sum_bsse;
     *skip = all_skip;
+    if (sum_rd == INT64_MAX)
+      *is_cost_valid = 0;
   }
 }