Fix a bug in ml_prune_rect_partition()

The quantization step size should be scaled properly for high bit depth
settings.

This only affects speed 0.
Encoder speed change is almost neutral.
There is a small coding gain of 0.09%.

Change-Id: I96b2bae03a53ce8ccd6428e3a050cfe18e06a024

diff --git a/vp9/encoder/vp9_encodeframe.c b/vp9/encoder/vp9_encodeframe.c
index ba1d71bf70f44f842a49c0e27bebbb9c541e5a08..9864de075d8d368e1c9949ac1255e62ad9d953ef 100644 (file)
--- a/vp9/encoder/vp9_encodeframe.c
+++ b/vp9/encoder/vp9_encodeframe.c
@@ -3288,7 +3288,12 @@ static void ml_prune_rect_partition(VP9_COMP *const cpi, MACROBLOCK *const x,
   {
     const int64_t none_rdcost = pc_tree->none.rdcost;
     const VP9_COMMON *const cm = &cpi->common;
+#if CONFIG_VP9_HIGHBITDEPTH
+    const int dc_q =
+        vp9_dc_quant(cm->base_qindex, 0, cm->bit_depth) >> (x->e_mbd.bd - 8);
+#else
     const int dc_q = vp9_dc_quant(cm->base_qindex, 0, cm->bit_depth);
+#endif  // CONFIG_VP9_HIGHBITDEPTH
     int feature_index = 0;
     unsigned int block_var = 0;
     unsigned int sub_block_var[4] = { 0 };