From 26c8303472b837e301d789ba569eae01955cf7f6 Mon Sep 17 00:00:00 2001 From: Loren Merritt Date: Wed, 18 Jan 2012 15:47:07 -0800 Subject: [PATCH] Fix trellis 2 + subme >= 8 Trellis didn't return a boolean value as it was supposed to. Regression in r2143-5. --- common/x86/trellis-64.asm | 7 +++---- encoder/rdo.c | 3 ++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/common/x86/trellis-64.asm b/common/x86/trellis-64.asm index f0683bd8..15438357 100644 --- a/common/x86/trellis-64.asm +++ b/common/x86/trellis-64.asm @@ -261,9 +261,7 @@ cglobal %1, 4,15,9 inc iiq jle .writeback_loop -%if dc mov eax, 1 -%endif .return: ADD rsp, pad RET @@ -724,6 +722,7 @@ TRELLIS_COEF0 1 mov [nodes_curq + node_cabac_state(node_ctx) + (coeff_abs_level1_offs>>2)], r11b ; delayed from x264_cabac_size_decision2 %endif %if %1 && node_ctx == 7 + mov r6d, levelgt1_ctxm mov [nodes_curq + node_cabac_state(node_ctx) + coeff_abs_levelgt1_offs-6], r10b %endif mov r6d, [nodes_prevq + node_level_idx(j)] @@ -786,8 +785,8 @@ TRELLIS_COEF0 1 add bitsd, r5d ; bs_size_ue_big from COEFN_SUFFIX ; n.cabac_state[levelgt1_ctx] %if j == 7 ; && compiling support for 4:2:2 - mov r5d, levelgt1_ctxm - %define coeff_abs_levelgt1_offs r5 + mov r6d, levelgt1_ctxm + %define coeff_abs_levelgt1_offs r6 %endif %if j == 7 movzx r10, byte [nodes_prevq + node_cabac_state(j) + coeff_abs_levelgt1_offs-6] ; -6 because node only stores ctx 8 and 9 diff --git a/encoder/rdo.c b/encoder/rdo.c index 5f2e48a6..f0fe1632 100644 --- a/encoder/rdo.c +++ b/encoder/rdo.c @@ -691,7 +691,8 @@ int quant_trellis_cabac( x264_t *h, dctcoef *dct, { int cost_sig = x264_cabac_size_decision_noup2( &cabac_state_sig[0], 1 ) + x264_cabac_size_decision_noup2( &cabac_state_last[0], 1 ); - return dct[0] = trellis_dc_shortcut( orig_coefs[0], quant_coefs[0], unquant_mf[0], coef_weight2[0], lambda2, cabac_state, cost_sig ); + dct[0] = trellis_dc_shortcut( orig_coefs[0], quant_coefs[0], unquant_mf[0], coef_weight2[0], lambda2, cabac_state, cost_sig ); + return !!dct[0]; } #if HAVE_MMX && ARCH_X86_64 -- 2.40.0