Align rate-distortion cost metric for chroma compoments

This commit aligns the rate-distortion metrics for both luma and
chroma components in super transform rate-distortion optimization.
It improves the coding gains due to var-tx and supertx experiments
by 0.2% for high resolution test sets.

Change-Id: Ib89d99e29cb5ee27b1f867e301954d4164d8b364

diff --git a/vp10/encoder/encodeframe.c b/vp10/encoder/encodeframe.c
index 41a697f1e8c7f40a967309f0c6df7816db3fec87..898b18f5b3d8be8b3dca5374141dfeeda2d0c6f2 100644 (file)
--- a/vp10/encoder/encodeframe.c
+++ b/vp10/encoder/encodeframe.c
@@ -5187,16 +5187,36 @@ static void rd_supertx_sb(VP10_COMP *cpi, ThreadData *td,
   dist_uv = 0;
   sse_uv = 0;
   for (plane = 1; plane < MAX_MB_PLANE; ++plane) {
+#if CONFIG_VAR_TX
+    ENTROPY_CONTEXT ctxa[16], ctxl[16];
+    const struct macroblockd_plane *const pd = &xd->plane[plane];
+    int coeff_ctx = 1;
+
+    this_rate = 0;
+    this_dist = 0;
+    pnsse = 0;
+    pnskip = 1;
+
     tx_size = max_txsize_lookup[bsize];
     tx_size = get_uv_tx_size_impl(tx_size, bsize,
                                   cm->subsampling_x, cm->subsampling_y);
+    vp10_get_entropy_contexts(bsize, tx_size, pd, ctxa, ctxl);
+    coeff_ctx = combine_entropy_contexts(ctxa[0], ctxl[0]);
+
     vp10_subtract_plane(x, bsize, plane);
-    vp10_txfm_rd_in_plane_supertx(x,
-#if CONFIG_VAR_TX
-                                  cpi,
-#endif
-                                  &this_rate, &this_dist, &pnskip, &pnsse,
+    vp10_tx_block_rd_b(cpi, x, tx_size,
+                       0, 0, plane, 0,
+                       get_plane_block_size(bsize, pd), coeff_ctx,
+                       &this_rate, &this_dist, &pnsse, &pnskip);
+#else
+    tx_size = max_txsize_lookup[bsize];
+    tx_size = get_uv_tx_size_impl(tx_size, bsize,
+                                  cm->subsampling_x, cm->subsampling_y);
+    vp10_subtract_plane(x, bsize, plane);
+    vp10_txfm_rd_in_plane_supertx(x, &this_rate, &this_dist, &pnskip, &pnsse,
                                   INT64_MAX, plane, bsize, tx_size, 0);
+#endif  // CONFIG_VAR_TX
+
     rate_uv += this_rate;
     dist_uv += this_dist;
     sse_uv += pnsse;