From 209f830d976aff4f140e79b28d21230436ac0d47 Mon Sep 17 00:00:00 2001 From: Jean-Marc Valin Date: Sat, 8 Oct 2016 14:10:39 -0400 Subject: [PATCH] Fix deringing level choice for 10-bit and 12-bit Making sure we never exceed a base level of 63 Change-Id: I821254b8d970446bd40fdd6e4d7073c69760a86d --- av1/encoder/pickdering.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/av1/encoder/pickdering.c b/av1/encoder/pickdering.c index cc5528bea..4ef83cdd4 100644 --- a/av1/encoder/pickdering.c +++ b/av1/encoder/pickdering.c @@ -90,8 +90,12 @@ int av1_dering_search(YV12_BUFFER_CONFIG *frame, const YV12_BUFFER_CONFIG *ref, adjusted on a 64x64 basis. We use a threshold of the form T = a*Q^b, where a and b are derived empirically trying to optimize rate-distortion at different quantizer settings. */ - best_level = (int)floor( - .5 + .45 * pow(av1_ac_quant(cm->base_qindex, 0, cm->bit_depth), 0.6)); + best_level = AOMMIN( + MAX_DERING_LEVEL - 1, + (int)floor(.5 + + .45 * pow(av1_ac_quant(cm->base_qindex, 0, cm->bit_depth) >> + (cm->bit_depth - 8), + 0.6))); for (sbr = 0; sbr < nvsb; sbr++) { for (sbc = 0; sbc < nhsb; sbc++) { int nvb, nhb; -- 2.50.0