From 347d01811538f89fac6ffb34af3adf87dc945822 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Matthias=20R=C3=A4ncker?= Date: Mon, 17 Sep 2018 13:57:17 +0200 Subject: [PATCH] Fix stack corruption with x86 and --enable-pic MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit x86inc.asm's cglobal macro is frequently used to declare more arguments than the function actually has. Normally, this is done to aquire an alias to a register that would correspond to that positional function argument if it existed. This is safe when used in this manner. In the case fixed here, however, the alias is used to temporarily store adresses obtained through the GOT in memory. Because those extra arguments don't actually exist, those stores corrupt the callers stack frame. SSE2/VpxHBDSubpelVarianceTest.Ref is a test that may fail as a result. To simply fix the space allocated to actual arguments that have been loaded into registers already is reused. This avoids having to allocate extra space for local variables. Also removed duplicate code while at it. Signed-off-by: Matthias Räncker Change-Id: I505281ecaa6be586185fe6a2d34d62bdf40c839f --- .../x86/highbd_subpel_variance_impl_sse2.asm | 46 +++++++------------ vpx_dsp/x86/subpel_variance_sse2.asm | 45 +++++++----------- 2 files changed, 34 insertions(+), 57 deletions(-) diff --git a/vpx_dsp/x86/highbd_subpel_variance_impl_sse2.asm b/vpx_dsp/x86/highbd_subpel_variance_impl_sse2.asm index e1f9657df..db4aaf4ea 100644 --- a/vpx_dsp/x86/highbd_subpel_variance_impl_sse2.asm +++ b/vpx_dsp/x86/highbd_subpel_variance_impl_sse2.asm @@ -111,45 +111,33 @@ SECTION .text cglobal highbd_sub_pixel_avg_variance%1xh, 7, 7, 13, src, src_stride, \ x_offset, y_offset, \ dst, dst_stride, \ - sec, sec_stride, height, sse, \ - g_bilin_filter, g_pw_8 + sec, sec_stride, height, sse %define block_height dword heightm %define sec_str sec_stridemp - - ; Store bilin_filter and pw_8 location in stack - %if GET_GOT_DEFINED == 1 - GET_GOT eax - add esp, 4 ; restore esp - %endif - - lea ecx, [GLOBAL(bilin_filter_m)] - mov g_bilin_filterm, ecx - - lea ecx, [GLOBAL(pw_8)] - mov g_pw_8m, ecx - - LOAD_IF_USED 0, 1 ; load eax, ecx back %else cglobal highbd_sub_pixel_variance%1xh, 7, 7, 13, src, src_stride, \ x_offset, y_offset, \ - dst, dst_stride, height, sse, \ - g_bilin_filter, g_pw_8 + dst, dst_stride, height, sse %define block_height heightd + %endif - ; Store bilin_filter and pw_8 location in stack - %if GET_GOT_DEFINED == 1 - GET_GOT eax - add esp, 4 ; restore esp - %endif + ; reuse argument stack space + %define g_bilin_filterm x_offsetm + %define g_pw_8m y_offsetm - lea ecx, [GLOBAL(bilin_filter_m)] - mov g_bilin_filterm, ecx + ; Store bilin_filter and pw_8 location in stack + %if GET_GOT_DEFINED == 1 + GET_GOT eax + add esp, 4 ; restore esp + %endif - lea ecx, [GLOBAL(pw_8)] - mov g_pw_8m, ecx + lea ecx, [GLOBAL(bilin_filter_m)] + mov g_bilin_filterm, ecx - LOAD_IF_USED 0, 1 ; load eax, ecx back - %endif + lea ecx, [GLOBAL(pw_8)] + mov g_pw_8m, ecx + + LOAD_IF_USED 0, 1 ; load eax, ecx back %else %if %2 == 1 ; avg cglobal highbd_sub_pixel_avg_variance%1xh, 7, 7, 13, src, src_stride, \ diff --git a/vpx_dsp/x86/subpel_variance_sse2.asm b/vpx_dsp/x86/subpel_variance_sse2.asm index d938c1da4..88967a3f1 100644 --- a/vpx_dsp/x86/subpel_variance_sse2.asm +++ b/vpx_dsp/x86/subpel_variance_sse2.asm @@ -132,44 +132,33 @@ SECTION .text %if %2 == 1 ; avg cglobal sub_pixel_avg_variance%1xh, 7, 7, 13, src, src_stride, \ x_offset, y_offset, dst, dst_stride, \ - sec, sec_stride, height, sse, \ - g_bilin_filter, g_pw_8 + sec, sec_stride, height, sse %define block_height dword heightm %define sec_str sec_stridemp - - ;Store bilin_filter and pw_8 location in stack - %if GET_GOT_DEFINED == 1 - GET_GOT eax - add esp, 4 ; restore esp - %endif - - lea ecx, [GLOBAL(bilin_filter_m)] - mov g_bilin_filterm, ecx - - lea ecx, [GLOBAL(pw_8)] - mov g_pw_8m, ecx - - LOAD_IF_USED 0, 1 ; load eax, ecx back %else cglobal sub_pixel_variance%1xh, 7, 7, 13, src, src_stride, \ x_offset, y_offset, dst, dst_stride, \ - height, sse, g_bilin_filter, g_pw_8 + height, sse %define block_height heightd + %endif + + ; reuse argument stack space + %define g_bilin_filterm x_offsetm + %define g_pw_8m y_offsetm - ;Store bilin_filter and pw_8 location in stack - %if GET_GOT_DEFINED == 1 - GET_GOT eax - add esp, 4 ; restore esp - %endif + ;Store bilin_filter and pw_8 location in stack + %if GET_GOT_DEFINED == 1 + GET_GOT eax + add esp, 4 ; restore esp + %endif - lea ecx, [GLOBAL(bilin_filter_m)] - mov g_bilin_filterm, ecx + lea ecx, [GLOBAL(bilin_filter_m)] + mov g_bilin_filterm, ecx - lea ecx, [GLOBAL(pw_8)] - mov g_pw_8m, ecx + lea ecx, [GLOBAL(pw_8)] + mov g_pw_8m, ecx - LOAD_IF_USED 0, 1 ; load eax, ecx back - %endif + LOAD_IF_USED 0, 1 ; load eax, ecx back %else %if %2 == 1 ; avg cglobal sub_pixel_avg_variance%1xh, 7, 7, 13, src, src_stride, \ -- 2.40.0