From: DRC Date: Sat, 28 May 2016 21:42:44 +0000 (-0500) Subject: 64-bit AVX2 impl. of RGB->YCC color conversion X-Git-Tag: 1.5.90~134 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=426d787c4c635d7ebefc3f261961245d51750c1f;p=libjpeg-turbo 64-bit AVX2 impl. of RGB->YCC color conversion --- diff --git a/BUILDING.md b/BUILDING.md index 5322873..d5ea22a 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -15,11 +15,12 @@ Build Requirements - NASM or YASM (if building x86 or x86-64 SIMD extensions) * If using NASM, 0.98, or 2.01 or later is required for an x86 build (0.99 and 2.00 do not work properly with libjpeg-turbo's x86 SIMD code.) - * If using NASM, 2.00 or later is required for an x86-64 build. - * If using NASM, 2.07 or later (except 2.11.08) is required for an x86-64 + * If using NASM, 2.10 or later is required for an x86-64 build. + * If using NASM, 2.10 or later (except 2.11.08) is required for an x86-64 Mac build (2.11.08 does not work properly with libjpeg-turbo's x86-64 SIMD code when building macho64 objects.) NASM or YASM can be obtained from [MacPorts](http://www.macports.org/). + * If using YASM, 1.2.0 or later is required for an x86-64 build. The binary RPMs released by the NASM project do not work on older Linux systems, such as Red Hat Enterprise Linux 4. On such systems, you can @@ -205,7 +206,7 @@ Add --host x86_64-apple-darwin NASM=/opt/local/bin/nasm -to the `configure` command line. NASM 2.07 or later from MacPorts must be +to the `configure` command line. NASM 2.10 or later from MacPorts must be installed. @@ -226,7 +227,7 @@ Add CFLAGS='-mmacosx-version-min=10.5 -O3' \ LDFLAGS='-mmacosx-version-min=10.5' -to the `configure` command line. NASM 2.07 or later from MacPorts must be +to the `configure` command line. NASM 2.10 or later from MacPorts must be installed. @@ -462,7 +463,8 @@ Build Requirements - [NASM](http://www.nasm.us) or [YASM](http://yasm.tortall.net) * If using NASM, 0.98 or later is required for an x86 build. - * If using NASM, 2.05 or later is required for an x86-64 build. + * If using NASM, 2.10 or later is required for an x86-64 build. + * If using YASM, 1.2.0 or later is required for an x86-64 build. * nasm.exe/yasm.exe should be in your `PATH`. - Microsoft Visual C++ 2005 or later diff --git a/jversion.h b/jversion.h index 6ce663d..d0993ba 100644 --- a/jversion.h +++ b/jversion.h @@ -38,6 +38,7 @@ #define JCOPYRIGHT "Copyright (C) 2009-2016 D. R. Commander\n" \ "Copyright (C) 2011-2016 Siarhei Siamashka\n" \ "Copyright (C) 2015-2016 Matthieu Darbois\n" \ + "Copyright (C) 2015 Intel Corporation\n" \ "Copyright (C) 2015 Google, Inc.\n" \ "Copyright (C) 2013-2014 MIPS Technologies, Inc.\n" \ "Copyright (C) 2013 Linaro Limited\n" \ diff --git a/simd/CMakeLists.txt b/simd/CMakeLists.txt index 2738c77..2422439 100755 --- a/simd/CMakeLists.txt +++ b/simd/CMakeLists.txt @@ -25,7 +25,7 @@ if(SIMD_X86_64) jchuff-sse2-64 jcsample-sse2-64 jdcolor-sse2-64 jdmerge-sse2-64 jdsample-sse2-64 jfdctfst-sse2-64 jfdctint-sse2-64 jidctflt-sse2-64 jidctfst-sse2-64 jidctint-sse2-64 jidctred-sse2-64 jquantf-sse2-64 - jquanti-sse2-64) + jquanti-sse2-64 jccolor-avx2-64) message(STATUS "Building x86_64 SIMD extensions") else() set(SIMD_BASENAMES jsimdcpu jfdctflt-3dn jidctflt-3dn jquant-3dn jccolor-mmx diff --git a/simd/Makefile.am b/simd/Makefile.am index a908385..80af295 100644 --- a/simd/Makefile.am +++ b/simd/Makefile.am @@ -18,12 +18,14 @@ libsimd_la_SOURCES = jsimd_x86_64.c jsimd.h jsimdcfg.inc.h jsimdext.inc \ jcsample-sse2-64.asm jdcolor-sse2-64.asm jdmerge-sse2-64.asm \ jdsample-sse2-64.asm jfdctfst-sse2-64.asm jfdctint-sse2-64.asm \ jidctflt-sse2-64.asm jidctfst-sse2-64.asm jidctint-sse2-64.asm \ - jidctred-sse2-64.asm jquantf-sse2-64.asm jquanti-sse2-64.asm + jidctred-sse2-64.asm jquantf-sse2-64.asm jquanti-sse2-64.asm \ + jccolor-avx2-64.asm jccolor-sse2-64.lo: jccolext-sse2-64.asm jcgray-sse2-64.lo: jcgryext-sse2-64.asm jdcolor-sse2-64.lo: jdcolext-sse2-64.asm jdmerge-sse2-64.lo: jdmrgext-sse2-64.asm +jccolor-avx2-64.lo: jccolext-avx2-64.asm endif diff --git a/simd/jccolext-avx2-64.asm b/simd/jccolext-avx2-64.asm new file mode 100644 index 0000000..0c9b984 --- /dev/null +++ b/simd/jccolext-avx2-64.asm @@ -0,0 +1,512 @@ +; +; jccolext.asm - colorspace conversion (64-bit AVX2) +; +; Copyright (C) 2009, 2016, D. R. Commander. +; Copyright (C) 2015, Intel Corporation. +; +; Based on the x86 SIMD extension for IJG JPEG library +; Copyright (C) 1999-2006, MIYASAKA Masaru. +; For conditions of distribution and use, see copyright notice in jsimdext.inc +; +; This file should be assembled with NASM (Netwide Assembler), +; can *not* be assembled with Microsoft's MASM or any compatible +; assembler (including Borland's Turbo Assembler). +; NASM is available from http://nasm.sourceforge.net/ or +; http://sourceforge.net/project/showfiles.php?group_id=6208 +; +; [TAB8] + +%include "jcolsamp.inc" + +; -------------------------------------------------------------------------- +; +; Convert some rows of samples to the output colorspace. +; +; GLOBAL(void) +; jsimd_rgb_ycc_convert_avx2 (JDIMENSION img_width, +; JSAMPARRAY input_buf, JSAMPIMAGE output_buf, +; JDIMENSION output_row, int num_rows); +; + +; r10d = JDIMENSION img_width +; r11 = JSAMPARRAY input_buf +; r12 = JSAMPIMAGE output_buf +; r13d = JDIMENSION output_row +; r14d = int num_rows + +%define wk(i) rbp-(WK_NUM-(i))*SIZEOF_YMMWORD ; ymmword wk[WK_NUM] +%define WK_NUM 8 + + align 32 + + global EXTN(jsimd_rgb_ycc_convert_avx2) + +EXTN(jsimd_rgb_ycc_convert_avx2): + push rbp + mov rax, rsp ; rax = original rbp + sub rsp, byte 4 + and rsp, byte (-SIZEOF_YMMWORD) ; align to 256 bits + mov [rsp], rax + mov rbp, rsp ; rbp = aligned rbp + lea rsp, [wk(0)] + collect_args 5 + push rbx + + mov ecx, r10d + test rcx, rcx + jz near .return + + push rcx + + mov rsi, r12 + mov ecx, r13d + mov rdi, JSAMPARRAY [rsi+0*SIZEOF_JSAMPARRAY] + mov rbx, JSAMPARRAY [rsi+1*SIZEOF_JSAMPARRAY] + mov rdx, JSAMPARRAY [rsi+2*SIZEOF_JSAMPARRAY] + lea rdi, [rdi+rcx*SIZEOF_JSAMPROW] + lea rbx, [rbx+rcx*SIZEOF_JSAMPROW] + lea rdx, [rdx+rcx*SIZEOF_JSAMPROW] + + pop rcx + + mov rsi, r11 + mov eax, r14d + test rax, rax + jle near .return +.rowloop: + push rdx + push rbx + push rdi + push rsi + push rcx ; col + + mov rsi, JSAMPROW [rsi] ; inptr + mov rdi, JSAMPROW [rdi] ; outptr0 + mov rbx, JSAMPROW [rbx] ; outptr1 + mov rdx, JSAMPROW [rdx] ; outptr2 + + cmp rcx, byte SIZEOF_YMMWORD + jae near .columnloop + +%if RGB_PIXELSIZE == 3 ; --------------- + +.column_ld1: + push rax + push rdx + lea rcx, [rcx+rcx*2] ; imul ecx,RGB_PIXELSIZE + test cl, SIZEOF_BYTE + jz short .column_ld2 + sub rcx, byte SIZEOF_BYTE + movzx rax, BYTE [rsi+rcx] +.column_ld2: + test cl, SIZEOF_WORD + jz short .column_ld4 + sub rcx, byte SIZEOF_WORD + movzx rdx, WORD [rsi+rcx] + shl rax, WORD_BIT + or rax, rdx +.column_ld4: + vmovd xmmA, eax + pop rdx + pop rax + test cl, SIZEOF_DWORD + jz short .column_ld8 + sub rcx, byte SIZEOF_DWORD + vmovd xmmF, XMM_DWORD [rsi+rcx] + vpslldq xmmA, xmmA, SIZEOF_DWORD + vpor xmmA, xmmA, xmmF +.column_ld8: + test cl, SIZEOF_MMWORD + jz short .column_ld16 + sub rcx, byte SIZEOF_MMWORD + vmovq xmmB, XMM_MMWORD [rsi+rcx] + vpslldq xmmA, xmmA, SIZEOF_MMWORD + vpor xmmA, xmmA, xmmB +.column_ld16: + test cl, SIZEOF_XMMWORD + jz short .column_ld32 + sub rcx, byte SIZEOF_XMMWORD + vmovdqu xmmB, XMM_MMWORD [rsi+rcx] + vperm2i128 ymmA, ymmA, ymmA, 1 + vpor ymmA, ymmB +.column_ld32: + test cl, SIZEOF_YMMWORD + jz short .column_ld64 + sub rcx, byte SIZEOF_YMMWORD + vmovdqa ymmF, ymmA + vmovdqu ymmA, YMMWORD [rsi+0*SIZEOF_YMMWORD] +.column_ld64: + test cl, 2*SIZEOF_YMMWORD + mov rcx, SIZEOF_YMMWORD + jz short .rgb_ycc_cnv + vmovdqa ymmB, ymmA + vmovdqu ymmA, YMMWORD [rsi+0*SIZEOF_YMMWORD] + vmovdqu ymmF, YMMWORD [rsi+1*SIZEOF_YMMWORD] + jmp short .rgb_ycc_cnv + +.columnloop: + vmovdqu ymmA, YMMWORD [rsi+0*SIZEOF_YMMWORD] + vmovdqu ymmF, YMMWORD [rsi+1*SIZEOF_YMMWORD] + vmovdqu ymmB, YMMWORD [rsi+2*SIZEOF_YMMWORD] + +.rgb_ycc_cnv: + + vmovdqu ymmC, ymmA + vinserti128 ymmA, ymmF, xmmA, 0 + vinserti128 ymmC, ymmC, xmmB, 0 + vinserti128 ymmB, ymmB, xmmF, 0 + vperm2i128 ymmF, ymmC, ymmC, 1 + + vmovdqa ymmG, ymmA + vpslldq ymmA, ymmA, 8 + vpsrldq ymmG, ymmG, 8 + + vpunpckhbw ymmA, ymmA, ymmF + vpslldq ymmF, ymmF, 8 + + vpunpcklbw ymmG, ymmG, ymmB + vpunpckhbw ymmF, ymmF, ymmB + + vmovdqa ymmD, ymmA + vpslldq ymmA, ymmA, 8 + vpsrldq ymmD, ymmD, 8 + + vpunpckhbw ymmA, ymmA, ymmG + vpslldq ymmG, ymmG, 8 + + vpunpcklbw ymmD, ymmD, ymmF + vpunpckhbw ymmG, ymmG, ymmF + + vmovdqa ymmE, ymmA + vpslldq ymmA, ymmA, 8 + vpsrldq ymmE, ymmE, 8 + + vpunpckhbw ymmA, ymmA, ymmD + vpslldq ymmD, ymmD, 8 + + vpunpcklbw ymmE, ymmE, ymmG + vpunpckhbw ymmD, ymmD, ymmG + + vpxor ymmH, ymmH, ymmH + + vmovdqa ymmC, ymmA + vpunpcklbw ymmA, ymmA, ymmH + vpunpckhbw ymmC, ymmC, ymmH + + vmovdqa ymmB, ymmE + vpunpcklbw ymmE, ymmE, ymmH + vpunpckhbw ymmB, ymmB, ymmH + + vmovdqa ymmF, ymmD + vpunpcklbw ymmD, ymmD, ymmH + vpunpckhbw ymmF, ymmF, ymmH + +%else ; RGB_PIXELSIZE == 4 ; ----------- + +.column_ld1: + test cl, SIZEOF_XMMWORD/16 + jz short .column_ld2 + sub rcx, byte SIZEOF_XMMWORD/16 + vmovd xmmA, XMM_DWORD [rsi+rcx*RGB_PIXELSIZE] +.column_ld2: + test cl, SIZEOF_XMMWORD/8 + jz short .column_ld4 + sub rcx, byte SIZEOF_XMMWORD/8 + vmovq xmmE, XMM_MMWORD [rsi+rcx*RGB_PIXELSIZE] + vpslldq xmmA, xmmA, SIZEOF_MMWORD + vpor xmmA, xmmA, xmmE +.column_ld4: + test cl, SIZEOF_XMMWORD/4 + jz short .column_ld8 + sub rcx, byte SIZEOF_XMMWORD/4 + vmovdqa xmmE, xmmA + vperm2i128 ymmE, ymmE, ymmE, 1 + vmovdqu xmmA, XMMWORD [rsi+rcx*RGB_PIXELSIZE] + vpor ymmA, ymmA, ymmE +.column_ld8: + test cl, SIZEOF_XMMWORD/2 + jz short .column_ld16 + sub rcx, byte SIZEOF_XMMWORD/2 + vmovdqa ymmE, ymmA + vmovdqu ymmA, YMMWORD [rsi+rcx*RGB_PIXELSIZE] +.column_ld16: + test cl, SIZEOF_XMMWORD + mov rcx, SIZEOF_YMMWORD + jz short .rgb_ycc_cnv + vmovdqa ymmF, ymmA + vmovdqa ymmH, ymmE + vmovdqu ymmA, YMMWORD [rsi+0*SIZEOF_YMMWORD] + vmovdqu ymmE, YMMWORD [rsi+1*SIZEOF_YMMWORD] + jmp short .rgb_ycc_cnv + +.columnloop: + vmovdqu ymmA, YMMWORD [rsi+0*SIZEOF_YMMWORD] + vmovdqu ymmE, YMMWORD [rsi+1*SIZEOF_YMMWORD] + vmovdqu ymmF, YMMWORD [rsi+2*SIZEOF_YMMWORD] + vmovdqu ymmH, YMMWORD [rsi+3*SIZEOF_YMMWORD] + +.rgb_ycc_cnv: + vmovdqa ymmB, ymmA + vinserti128 ymmA, ymmA, xmmF, 1 + vperm2i128 ymmF, ymmB, ymmF, 0x31 + + vmovdqa ymmB, ymmE + vinserti128 ymmE, ymmE, xmmH, 1 + vperm2i128 ymmH, ymmB, ymmH, 0x31 + + vmovdqa ymmB, ymmE + vmovdqa ymmE, ymmF + vmovdqa ymmF, ymmB + + vmovdqa ymmD, ymmA + vpunpcklbw ymmA, ymmA, ymmE + vpunpckhbw ymmD, ymmD, ymmE + + vmovdqa ymmC, ymmF + vpunpcklbw ymmF, ymmF, ymmH + vpunpckhbw ymmC, ymmC, ymmH + + vmovdqa ymmB, ymmA + vpunpcklwd ymmA, ymmA, ymmF + vpunpckhwd ymmB, ymmB, ymmF + + vmovdqa ymmG, ymmD + vpunpcklwd ymmD, ymmD, ymmC + vpunpckhwd ymmG, ymmG, ymmC + + vmovdqa ymmE, ymmA + vpunpcklbw ymmA, ymmA, ymmD + vpunpckhbw ymmE, ymmE, ymmD + + vmovdqa ymmH, ymmB + vpunpcklbw ymmB, ymmB, ymmG + vpunpckhbw ymmH, ymmH, ymmG + + vpxor ymmF, ymmF, ymmF + + vmovdqa ymmC, ymmA + vpunpcklbw ymmA, ymmA, ymmF + vpunpckhbw ymmC, ymmC, ymmF + + vmovdqa ymmD, ymmB + vpunpcklbw ymmB, ymmB, ymmF + vpunpckhbw ymmD, ymmD, ymmF + + vmovdqa ymmG, ymmE + vpunpcklbw ymmE, ymmE, ymmF + vpunpckhbw ymmG, ymmG, ymmF + + vpunpcklbw ymmF, ymmF, ymmH + vpunpckhbw ymmH, ymmH, ymmH + vpsrlw ymmF, ymmF, BYTE_BIT + vpsrlw ymmH, ymmH, BYTE_BIT + +%endif ; RGB_PIXELSIZE ; --------------- + + ; ymm0=R(02468ACE)=RE, ymm2=G(02468ACE)=GE, ymm4=B(02468ACE)=BE + ; ymm1=R(13579BDF)=RO, ymm3=G(13579BDF)=GO, ymm5=B(13579BDF)=BO + + ; (Original) + ; Y = 0.29900 * R + 0.58700 * G + 0.11400 * B + ; Cb = -0.16874 * R - 0.33126 * G + 0.50000 * B + CENTERJSAMPLE + ; Cr = 0.50000 * R - 0.41869 * G - 0.08131 * B + CENTERJSAMPLE + ; + ; (This implementation) + ; Y = 0.29900 * R + 0.33700 * G + 0.11400 * B + 0.25000 * G + ; Cb = -0.16874 * R - 0.33126 * G + 0.50000 * B + CENTERJSAMPLE + ; Cr = 0.50000 * R - 0.41869 * G - 0.08131 * B + CENTERJSAMPLE + + vmovdqa YMMWORD [wk(0)], ymm0 ; wk(0)=RE + vmovdqa YMMWORD [wk(1)], ymm1 ; wk(1)=RO + vmovdqa YMMWORD [wk(2)], ymm4 ; wk(2)=BE + vmovdqa YMMWORD [wk(3)], ymm5 ; wk(3)=BO + + vmovdqa ymm6, ymm1 + vpunpcklwd ymm1, ymm1, ymm3 + vpunpckhwd ymm6, ymm6, ymm3 + vmovdqa ymm7, ymm1 + vmovdqa ymm4, ymm6 + vpmaddwd ymm1, ymm1, [rel PW_F0299_F0337] ; ymm1=ROL*FIX(0.299)+GOL*FIX(0.337) + vpmaddwd ymm6, ymm6, [rel PW_F0299_F0337] ; ymm6=ROH*FIX(0.299)+GOH*FIX(0.337) + vpmaddwd ymm7, ymm7, [rel PW_MF016_MF033] ; ymm7=ROL*-FIX(0.168)+GOL*-FIX(0.331) + vpmaddwd ymm4, ymm4, [rel PW_MF016_MF033] ; ymm4=ROH*-FIX(0.168)+GOH*-FIX(0.331) + + vmovdqa YMMWORD [wk(4)], ymm1 ; wk(4)=ROL*FIX(0.299)+GOL*FIX(0.337) + vmovdqa YMMWORD [wk(5)], ymm6 ; wk(5)=ROH*FIX(0.299)+GOH*FIX(0.337) + + vpxor ymm1, ymm1, ymm1 + vpxor ymm6, ymm6, ymm6 + vpunpcklwd ymm1, ymm1, ymm5 ; ymm1=BOL + vpunpckhwd ymm6, ymm6, ymm5 ; ymm6=BOH + vpsrld ymm1, ymm1, 1 ; ymm1=BOL*FIX(0.500) + vpsrld ymm6, ymm6, 1 ; ymm6=BOH*FIX(0.500) + + vmovdqa ymm5, [rel PD_ONEHALFM1_CJ] ; ymm5=[PD_ONEHALFM1_CJ] + + vpaddd ymm7, ymm7, ymm1 + vpaddd ymm4, ymm4, ymm6 + vpaddd ymm7, ymm7, ymm5 + vpaddd ymm4, ymm4, ymm5 + vpsrld ymm7, ymm7, SCALEBITS ; ymm7=CbOL + vpsrld ymm4, ymm4, SCALEBITS ; ymm4=CbOH + vpackssdw ymm7, ymm7, ymm4 ; ymm7=CbO + + vmovdqa ymm1, YMMWORD [wk(2)] ; ymm1=BE + + vmovdqa ymm6, ymm0 + vpunpcklwd ymm0, ymm0, ymm2 + vpunpckhwd ymm6, ymm6, ymm2 + vmovdqa ymm5, ymm0 + vmovdqa ymm4, ymm6 + vpmaddwd ymm0, ymm0, [rel PW_F0299_F0337] ; ymm0=REL*FIX(0.299)+GEL*FIX(0.337) + vpmaddwd ymm6, ymm6, [rel PW_F0299_F0337] ; ymm6=REH*FIX(0.299)+GEH*FIX(0.337) + vpmaddwd ymm5, ymm5, [rel PW_MF016_MF033] ; ymm5=REL*-FIX(0.168)+GEL*-FIX(0.331) + vpmaddwd ymm4, ymm4, [rel PW_MF016_MF033] ; ymm4=REH*-FIX(0.168)+GEH*-FIX(0.331) + + vmovdqa YMMWORD [wk(6)], ymm0 ; wk(6)=REL*FIX(0.299)+GEL*FIX(0.337) + vmovdqa YMMWORD [wk(7)], ymm6 ; wk(7)=REH*FIX(0.299)+GEH*FIX(0.337) + + vpxor ymm0, ymm0, ymm0 + vpxor ymm6, ymm6, ymm6 + vpunpcklwd ymm0, ymm0, ymm1 ; ymm0=BEL + vpunpckhwd ymm6, ymm6, ymm1 ; ymm6=BEH + vpsrld ymm0, ymm0, 1 ; ymm0=BEL*FIX(0.500) + vpsrld ymm6, ymm6, 1 ; ymm6=BEH*FIX(0.500) + + + vmovdqa ymm1, [rel PD_ONEHALFM1_CJ] ; ymm1=[PD_ONEHALFM1_CJ] + + vpaddd ymm5, ymm5, ymm0 + vpaddd ymm4, ymm4, ymm6 + vpaddd ymm5, ymm5, ymm1 + vpaddd ymm4, ymm4, ymm1 + vpsrld ymm5, ymm5, SCALEBITS ; ymm5=CbEL + vpsrld ymm4, ymm4, SCALEBITS ; ymm4=CbEH + vpackssdw ymm5, ymm5, ymm4 ; ymm5=CbE + + vpsllw ymm7, ymm7, BYTE_BIT + vpor ymm5, ymm5, ymm7 ; ymm5=Cb + vmovdqu YMMWORD [rbx], ymm5 ; Save Cb + + vmovdqa ymm0, YMMWORD [wk(3)] ; ymm0=BO + vmovdqa ymm6, YMMWORD [wk(2)] ; ymm6=BE + vmovdqa ymm1, YMMWORD [wk(1)] ; ymm1=RO + + vmovdqa ymm4, ymm0 + vpunpcklwd ymm0, ymm0, ymm3 + vpunpckhwd ymm4, ymm4, ymm3 + vmovdqa ymm7, ymm0 + vmovdqa ymm5, ymm4 + vpmaddwd ymm0, ymm0, [rel PW_F0114_F0250] ; ymm0=BOL*FIX(0.114)+GOL*FIX(0.250) + vpmaddwd ymm4, ymm4, [rel PW_F0114_F0250] ; ymm4=BOH*FIX(0.114)+GOH*FIX(0.250) + vpmaddwd ymm7, ymm7, [rel PW_MF008_MF041] ; ymm7=BOL*-FIX(0.081)+GOL*-FIX(0.418) + vpmaddwd ymm5, ymm5, [rel PW_MF008_MF041] ; ymm5=BOH*-FIX(0.081)+GOH*-FIX(0.418) + + vmovdqa ymm3, [rel PD_ONEHALF] ; ymm3=[PD_ONEHALF] + + vpaddd ymm0, ymm0, YMMWORD [wk(4)] + vpaddd ymm4, ymm4, YMMWORD [wk(5)] + vpaddd ymm0, ymm0, ymm3 + vpaddd ymm4, ymm4, ymm3 + vpsrld ymm0, ymm0, SCALEBITS ; ymm0=YOL + vpsrld ymm4, ymm4, SCALEBITS ; ymm4=YOH + vpackssdw ymm0, ymm0, ymm4 ; ymm0=YO + + vpxor ymm3, ymm3, ymm3 + vpxor ymm4, ymm4, ymm4 + vpunpcklwd ymm3, ymm3, ymm1 ; ymm3=ROL + vpunpckhwd ymm4, ymm4, ymm1 ; ymm4=ROH + vpsrld ymm3, ymm3, 1 ; ymm3=ROL*FIX(0.500) + vpsrld ymm4, ymm4, 1 ; ymm4=ROH*FIX(0.500) + + vmovdqa ymm1, [rel PD_ONEHALFM1_CJ] ; ymm1=[PD_ONEHALFM1_CJ] + + vpaddd ymm7, ymm7, ymm3 + vpaddd ymm5, ymm5, ymm4 + vpaddd ymm7, ymm7, ymm1 + vpaddd ymm5, ymm5, ymm1 + vpsrld ymm7, ymm7, SCALEBITS ; ymm7=CrOL + vpsrld ymm5, ymm5, SCALEBITS ; ymm5=CrOH + vpackssdw ymm7, ymm7, ymm5 ; ymm7=CrO + + vmovdqa ymm3, YMMWORD [wk(0)] ; ymm3=RE + + vmovdqa ymm4, ymm6 + vpunpcklwd ymm6, ymm6, ymm2 + vpunpckhwd ymm4, ymm4, ymm2 + vmovdqa ymm1, ymm6 + vmovdqa ymm5, ymm4 + vpmaddwd ymm6, ymm6, [rel PW_F0114_F0250] ; ymm6=BEL*FIX(0.114)+GEL*FIX(0.250) + vpmaddwd ymm4, ymm4, [rel PW_F0114_F0250] ; ymm4=BEH*FIX(0.114)+GEH*FIX(0.250) + vpmaddwd ymm1, ymm1, [rel PW_MF008_MF041] ; ymm1=BEL*-FIX(0.081)+GEL*-FIX(0.418) + vpmaddwd ymm5, ymm5, [rel PW_MF008_MF041] ; ymm5=BEH*-FIX(0.081)+GEH*-FIX(0.418) + + vmovdqa ymm2, [rel PD_ONEHALF] ; ymm2=[PD_ONEHALF] + + vpaddd ymm6, ymm6, YMMWORD [wk(6)] + vpaddd ymm4, ymm4, YMMWORD [wk(7)] + vpaddd ymm6, ymm6, ymm2 + vpaddd ymm4, ymm4, ymm2 + vpsrld ymm6, ymm6, SCALEBITS ; ymm6=YEL + vpsrld ymm4, ymm4, SCALEBITS ; ymm4=YEH + vpackssdw ymm6, ymm6, ymm4 ; ymm6=YE + + vpsllw ymm0, ymm0, BYTE_BIT + vpor ymm6, ymm6, ymm0 ; ymm6=Y + vmovdqu YMMWORD [rdi], ymm6 ; Save Y + + vpxor ymm2, ymm2, ymm2 + vpxor ymm4, ymm4, ymm4 + vpunpcklwd ymm2, ymm2, ymm3 ; ymm2=REL + vpunpckhwd ymm4, ymm4, ymm3 ; ymm4=REH + vpsrld ymm2, ymm2, 1 ; ymm2=REL*FIX(0.500) + vpsrld ymm4, ymm4, 1 ; ymm4=REH*FIX(0.500) + + vmovdqa ymm0, [rel PD_ONEHALFM1_CJ] ; ymm0=[PD_ONEHALFM1_CJ] + + vpaddd ymm1, ymm1, ymm2 + vpaddd ymm5, ymm5, ymm4 + vpaddd ymm1, ymm1, ymm0 + vpaddd ymm5, ymm5, ymm0 + vpsrld ymm1, ymm1, SCALEBITS ; ymm1=CrEL + vpsrld ymm5, ymm5, SCALEBITS ; ymm5=CrEH + vpackssdw ymm1, ymm1, ymm5 ; ymm1=CrE + + vpsllw ymm7, ymm7, BYTE_BIT + vpor ymm1, ymm1, ymm7 ; ymm1=Cr + vmovdqu YMMWORD [rdx], ymm1 ; Save Cr + + sub rcx, byte SIZEOF_YMMWORD + add rsi, RGB_PIXELSIZE*SIZEOF_YMMWORD ; inptr + add rdi, byte SIZEOF_YMMWORD ; outptr0 + add rbx, byte SIZEOF_YMMWORD ; outptr1 + add rdx, byte SIZEOF_YMMWORD ; outptr2 + cmp rcx, byte SIZEOF_YMMWORD + jae near .columnloop + test rcx, rcx + jnz near .column_ld1 + + pop rcx ; col + pop rsi + pop rdi + pop rbx + pop rdx + + add rsi, byte SIZEOF_JSAMPROW ; input_buf + add rdi, byte SIZEOF_JSAMPROW + add rbx, byte SIZEOF_JSAMPROW + add rdx, byte SIZEOF_JSAMPROW + dec rax ; num_rows + jg near .rowloop + +.return: + pop rbx + uncollect_args 5 + mov rsp, rbp ; rsp <- aligned rbp + pop rsp ; rsp <- original rbp + pop rbp + ret + +; For some reason, the OS X linker does not honor the request to align the +; segment unless we do this. + align 32 diff --git a/simd/jccolor-avx2-64.asm b/simd/jccolor-avx2-64.asm new file mode 100644 index 0000000..9250fb9 --- /dev/null +++ b/simd/jccolor-avx2-64.asm @@ -0,0 +1,122 @@ +; +; jccolor.asm - colorspace conversion (64-bit AVX2) +; +; Copyright (C) 2009, 2016, D. R. Commander. +; Copyright (C) 2015, Intel Corporation. +; +; Based on the x86 SIMD extension for IJG JPEG library +; Copyright (C) 1999-2006, MIYASAKA Masaru. +; For conditions of distribution and use, see copyright notice in jsimdext.inc +; +; This file should be assembled with NASM (Netwide Assembler), +; can *not* be assembled with Microsoft's MASM or any compatible +; assembler (including Borland's Turbo Assembler). +; NASM is available from http://nasm.sourceforge.net/ or +; http://sourceforge.net/project/showfiles.php?group_id=6208 +; +; [TAB8] + +%include "jsimdext.inc" + +; -------------------------------------------------------------------------- + +%define SCALEBITS 16 + +F_0_081 equ 5329 ; FIX(0.08131) +F_0_114 equ 7471 ; FIX(0.11400) +F_0_168 equ 11059 ; FIX(0.16874) +F_0_250 equ 16384 ; FIX(0.25000) +F_0_299 equ 19595 ; FIX(0.29900) +F_0_331 equ 21709 ; FIX(0.33126) +F_0_418 equ 27439 ; FIX(0.41869) +F_0_587 equ 38470 ; FIX(0.58700) +F_0_337 equ (F_0_587 - F_0_250) ; FIX(0.58700) - FIX(0.25000) + +; -------------------------------------------------------------------------- + SECTION SEG_CONST + + alignz 32 + global EXTN(jconst_rgb_ycc_convert_avx2) + +EXTN(jconst_rgb_ycc_convert_avx2): + +PW_F0299_F0337 times 8 dw F_0_299, F_0_337 +PW_F0114_F0250 times 8 dw F_0_114, F_0_250 +PW_MF016_MF033 times 8 dw -F_0_168,-F_0_331 +PW_MF008_MF041 times 8 dw -F_0_081,-F_0_418 +PD_ONEHALFM1_CJ times 8 dd (1 << (SCALEBITS-1)) - 1 + (CENTERJSAMPLE << SCALEBITS) +PD_ONEHALF times 8 dd (1 << (SCALEBITS-1)) + + alignz 32 + +; -------------------------------------------------------------------------- + SECTION SEG_TEXT + BITS 64 + +%include "jccolext-avx2-64.asm" + +%undef RGB_RED +%undef RGB_GREEN +%undef RGB_BLUE +%undef RGB_PIXELSIZE +%define RGB_RED EXT_RGB_RED +%define RGB_GREEN EXT_RGB_GREEN +%define RGB_BLUE EXT_RGB_BLUE +%define RGB_PIXELSIZE EXT_RGB_PIXELSIZE +%define jsimd_rgb_ycc_convert_avx2 jsimd_extrgb_ycc_convert_avx2 +%include "jccolext-avx2-64.asm" + +%undef RGB_RED +%undef RGB_GREEN +%undef RGB_BLUE +%undef RGB_PIXELSIZE +%define RGB_RED EXT_RGBX_RED +%define RGB_GREEN EXT_RGBX_GREEN +%define RGB_BLUE EXT_RGBX_BLUE +%define RGB_PIXELSIZE EXT_RGBX_PIXELSIZE +%define jsimd_rgb_ycc_convert_avx2 jsimd_extrgbx_ycc_convert_avx2 +%include "jccolext-avx2-64.asm" + +%undef RGB_RED +%undef RGB_GREEN +%undef RGB_BLUE +%undef RGB_PIXELSIZE +%define RGB_RED EXT_BGR_RED +%define RGB_GREEN EXT_BGR_GREEN +%define RGB_BLUE EXT_BGR_BLUE +%define RGB_PIXELSIZE EXT_BGR_PIXELSIZE +%define jsimd_rgb_ycc_convert_avx2 jsimd_extbgr_ycc_convert_avx2 +%include "jccolext-avx2-64.asm" + +%undef RGB_RED +%undef RGB_GREEN +%undef RGB_BLUE +%undef RGB_PIXELSIZE +%define RGB_RED EXT_BGRX_RED +%define RGB_GREEN EXT_BGRX_GREEN +%define RGB_BLUE EXT_BGRX_BLUE +%define RGB_PIXELSIZE EXT_BGRX_PIXELSIZE +%define jsimd_rgb_ycc_convert_avx2 jsimd_extbgrx_ycc_convert_avx2 +%include "jccolext-avx2-64.asm" + +%undef RGB_RED +%undef RGB_GREEN +%undef RGB_BLUE +%undef RGB_PIXELSIZE +%define RGB_RED EXT_XBGR_RED +%define RGB_GREEN EXT_XBGR_GREEN +%define RGB_BLUE EXT_XBGR_BLUE +%define RGB_PIXELSIZE EXT_XBGR_PIXELSIZE +%define jsimd_rgb_ycc_convert_avx2 jsimd_extxbgr_ycc_convert_avx2 +%include "jccolext-avx2-64.asm" + +%undef RGB_RED +%undef RGB_GREEN +%undef RGB_BLUE +%undef RGB_PIXELSIZE +%define RGB_RED EXT_XRGB_RED +%define RGB_GREEN EXT_XRGB_GREEN +%define RGB_BLUE EXT_XRGB_BLUE +%define RGB_PIXELSIZE EXT_XRGB_PIXELSIZE +%define jsimd_rgb_ycc_convert_avx2 jsimd_extxrgb_ycc_convert_avx2 +%include "jccolext-avx2-64.asm" diff --git a/simd/jcolsamp.inc b/simd/jcolsamp.inc index 3be446e..3fa6214 100644 --- a/simd/jcolsamp.inc +++ b/simd/jcolsamp.inc @@ -2,6 +2,7 @@ ; jcolsamp.inc - private declarations for color conversion & up/downsampling ; ; Copyright 2009 Pierre Ossman for Cendio AB +; Copyright (C) 2015, Intel Corporation. ; ; Based on the x86 SIMD extension for IJG JPEG library ; Copyright (C) 1999-2006, MIYASAKA Masaru. @@ -18,21 +19,29 @@ %define mmB mm1 %define xmmA xmm0 %define xmmB xmm1 +%define ymmA ymm0 +%define ymmB ymm1 %elif RGB_GREEN == 0 %define mmA mm2 %define mmB mm3 %define xmmA xmm2 %define xmmB xmm3 +%define ymmA ymm2 +%define ymmB ymm3 %elif RGB_BLUE == 0 %define mmA mm4 %define mmB mm5 %define xmmA xmm4 %define xmmB xmm5 +%define ymmA ymm4 +%define ymmB ymm5 %else %define mmA mm6 %define mmB mm7 %define xmmA xmm6 %define xmmB xmm7 +%define ymmA ymm6 +%define ymmB ymm7 %endif %if RGB_RED == 1 @@ -40,21 +49,29 @@ %define mmD mm1 %define xmmC xmm0 %define xmmD xmm1 +%define ymmC ymm0 +%define ymmD ymm1 %elif RGB_GREEN == 1 %define mmC mm2 %define mmD mm3 %define xmmC xmm2 %define xmmD xmm3 +%define ymmC ymm2 +%define ymmD ymm3 %elif RGB_BLUE == 1 %define mmC mm4 %define mmD mm5 %define xmmC xmm4 %define xmmD xmm5 +%define ymmC ymm4 +%define ymmD ymm5 %else %define mmC mm6 %define mmD mm7 %define xmmC xmm6 %define xmmD xmm7 +%define ymmC ymm6 +%define ymmD ymm7 %endif %if RGB_RED == 2 @@ -62,21 +79,29 @@ %define mmF mm1 %define xmmE xmm0 %define xmmF xmm1 +%define ymmE ymm0 +%define ymmF ymm1 %elif RGB_GREEN == 2 %define mmE mm2 %define mmF mm3 %define xmmE xmm2 %define xmmF xmm3 +%define ymmE ymm2 +%define ymmF ymm3 %elif RGB_BLUE == 2 %define mmE mm4 %define mmF mm5 %define xmmE xmm4 %define xmmF xmm5 +%define ymmE ymm4 +%define ymmF ymm5 %else %define mmE mm6 %define mmF mm7 %define xmmE xmm6 %define xmmF xmm7 +%define ymmE ymm6 +%define ymmF ymm7 %endif %if RGB_RED == 3 @@ -84,21 +109,29 @@ %define mmH mm1 %define xmmG xmm0 %define xmmH xmm1 +%define ymmG ymm0 +%define ymmH ymm1 %elif RGB_GREEN == 3 %define mmG mm2 %define mmH mm3 %define xmmG xmm2 %define xmmH xmm3 +%define ymmG ymm2 +%define ymmH ymm3 %elif RGB_BLUE == 3 %define mmG mm4 %define mmH mm5 %define xmmG xmm4 %define xmmH xmm5 +%define ymmG ymm4 +%define ymmH ymm5 %else %define mmG mm6 %define mmH mm7 %define xmmG xmm6 %define xmmH xmm7 +%define ymmG ymm6 +%define ymmH ymm7 %endif ; -------------------------------------------------------------------------- diff --git a/simd/jsimd.h b/simd/jsimd.h index a10995e..c44e109 100644 --- a/simd/jsimd.h +++ b/simd/jsimd.h @@ -74,6 +74,29 @@ EXTERN(void) jsimd_extxrgb_ycc_convert_sse2 (JDIMENSION img_width, JSAMPARRAY input_buf, JSAMPIMAGE output_buf, JDIMENSION output_row, int num_rows); +extern const int jconst_rgb_ycc_convert_avx2[]; +EXTERN(void) jsimd_rgb_ycc_convert_avx2 + (JDIMENSION img_width, JSAMPARRAY input_buf, JSAMPIMAGE output_buf, + JDIMENSION output_row, int num_rows); +EXTERN(void) jsimd_extrgb_ycc_convert_avx2 + (JDIMENSION img_width, JSAMPARRAY input_buf, JSAMPIMAGE output_buf, + JDIMENSION output_row, int num_rows); +EXTERN(void) jsimd_extrgbx_ycc_convert_avx2 + (JDIMENSION img_width, JSAMPARRAY input_buf, JSAMPIMAGE output_buf, + JDIMENSION output_row, int num_rows); +EXTERN(void) jsimd_extbgr_ycc_convert_avx2 + (JDIMENSION img_width, JSAMPARRAY input_buf, JSAMPIMAGE output_buf, + JDIMENSION output_row, int num_rows); +EXTERN(void) jsimd_extbgrx_ycc_convert_avx2 + (JDIMENSION img_width, JSAMPARRAY input_buf, JSAMPIMAGE output_buf, + JDIMENSION output_row, int num_rows); +EXTERN(void) jsimd_extxbgr_ycc_convert_avx2 + (JDIMENSION img_width, JSAMPARRAY input_buf, JSAMPIMAGE output_buf, + JDIMENSION output_row, int num_rows); +EXTERN(void) jsimd_extxrgb_ycc_convert_avx2 + (JDIMENSION img_width, JSAMPARRAY input_buf, JSAMPIMAGE output_buf, + JDIMENSION output_row, int num_rows); + EXTERN(void) jsimd_rgb_ycc_convert_neon (JDIMENSION img_width, JSAMPARRAY input_buf, JSAMPIMAGE output_buf, JDIMENSION output_row, int num_rows); diff --git a/simd/jsimd_x86_64.c b/simd/jsimd_x86_64.c index b2fe5d3..1621316 100644 --- a/simd/jsimd_x86_64.c +++ b/simd/jsimd_x86_64.c @@ -29,6 +29,7 @@ #define IS_ALIGNED(ptr, order) (((size_t)ptr & ((1 << order) - 1)) == 0) #define IS_ALIGNED_SSE(ptr) (IS_ALIGNED(ptr, 4)) /* 16 byte alignment */ +#define IS_ALIGNED_AVX(ptr) (IS_ALIGNED(ptr, 8)) /* 32 byte alignment */ static unsigned int simd_support = ~0; static unsigned int simd_huffman = 1; @@ -76,6 +77,9 @@ jsimd_can_rgb_ycc (void) if ((RGB_PIXELSIZE != 3) && (RGB_PIXELSIZE != 4)) return 0; + if ((simd_support & JSIMD_AVX2) && + IS_ALIGNED_AVX(jconst_rgb_ycc_convert_avx2)) + return 1; if ((simd_support & JSIMD_SSE2) && IS_ALIGNED_SSE(jconst_rgb_ycc_convert_sse2)) return 1; @@ -134,37 +138,48 @@ jsimd_rgb_ycc_convert (j_compress_ptr cinfo, JSAMPARRAY input_buf, JSAMPIMAGE output_buf, JDIMENSION output_row, int num_rows) { + void (*avx2fct)(JDIMENSION, JSAMPARRAY, JSAMPIMAGE, JDIMENSION, int); void (*sse2fct)(JDIMENSION, JSAMPARRAY, JSAMPIMAGE, JDIMENSION, int); switch(cinfo->in_color_space) { case JCS_EXT_RGB: + avx2fct=jsimd_extrgb_ycc_convert_avx2; sse2fct=jsimd_extrgb_ycc_convert_sse2; break; case JCS_EXT_RGBX: case JCS_EXT_RGBA: + avx2fct=jsimd_extrgbx_ycc_convert_avx2; sse2fct=jsimd_extrgbx_ycc_convert_sse2; break; case JCS_EXT_BGR: + avx2fct=jsimd_extbgr_ycc_convert_avx2; sse2fct=jsimd_extbgr_ycc_convert_sse2; break; case JCS_EXT_BGRX: case JCS_EXT_BGRA: + avx2fct=jsimd_extbgrx_ycc_convert_avx2; sse2fct=jsimd_extbgrx_ycc_convert_sse2; break; case JCS_EXT_XBGR: case JCS_EXT_ABGR: + avx2fct=jsimd_extxbgr_ycc_convert_avx2; sse2fct=jsimd_extxbgr_ycc_convert_sse2; break; case JCS_EXT_XRGB: case JCS_EXT_ARGB: + avx2fct=jsimd_extxrgb_ycc_convert_avx2; sse2fct=jsimd_extxrgb_ycc_convert_sse2; break; default: + avx2fct=jsimd_rgb_ycc_convert_avx2; sse2fct=jsimd_rgb_ycc_convert_sse2; break; } - sse2fct(cinfo->image_width, input_buf, output_buf, output_row, num_rows); + if (simd_support & JSIMD_AVX2) + avx2fct(cinfo->image_width, input_buf, output_buf, output_row, num_rows); + else + sse2fct(cinfo->image_width, input_buf, output_buf, output_row, num_rows); } GLOBAL(void)