From c84804a7409927813430f5bb1b56bb7b8afbcee8 Mon Sep 17 00:00:00 2001 From: Manman Ren Date: Fri, 29 Jun 2012 00:54:35 +0000 Subject: [PATCH] X86: add more GATHER intrinsics in Clang Corrected type for index of _mm256_mask_i32gather_pd from 256-bit to 128-bit Corrected types for src|dst|mask of _mm256_mask_i64gather_ps from 256-bit to 128-bit Support the following intrinsics: _mm_mask_i32gather_epi64, _mm256_mask_i32gather_epi64, _mm_mask_i64gather_epi64, _mm256_mask_i64gather_epi64, _mm_mask_i32gather_epi32, _mm256_mask_i32gather_epi32, _mm_mask_i64gather_epi32, _mm256_mask_i64gather_epi32 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@159403 91177308-0d34-0410-b5e6-96231b3b80d8 --- include/clang/Basic/BuiltinsX86.def | 13 ++++- lib/Headers/avx2intrin.h | 76 ++++++++++++++++++++++++++--- test/CodeGen/avx2-builtins.c | 53 ++++++++++++++++++-- 3 files changed, 131 insertions(+), 11 deletions(-) diff --git a/include/clang/Basic/BuiltinsX86.def b/include/clang/Basic/BuiltinsX86.def index a331cde2d0..4e8d13c758 100644 --- a/include/clang/Basic/BuiltinsX86.def +++ b/include/clang/Basic/BuiltinsX86.def @@ -597,13 +597,22 @@ BUILTIN(__builtin_ia32_psrlv2di, "V2LLiV2LLiV2LLi", "") // GATHER BUILTIN(__builtin_ia32_gatherd_pd, "V2dV2dV2dC*V4iV2dIc", "") -BUILTIN(__builtin_ia32_gatherd_pd256, "V4dV4dV4dC*V8iV4dIc", "") +BUILTIN(__builtin_ia32_gatherd_pd256, "V4dV4dV4dC*V4iV4dIc", "") BUILTIN(__builtin_ia32_gatherq_pd, "V2dV2dV2dC*V2LLiV2dIc", "") BUILTIN(__builtin_ia32_gatherq_pd256, "V4dV4dV4dC*V4LLiV4dIc", "") BUILTIN(__builtin_ia32_gatherd_ps, "V4fV4fV4fC*V4iV4fIc", "") BUILTIN(__builtin_ia32_gatherd_ps256, "V8fV8fV8fC*V8iV8fIc", "") BUILTIN(__builtin_ia32_gatherq_ps, "V4fV4fV4fC*V2LLiV4fIc", "") -BUILTIN(__builtin_ia32_gatherq_ps256, "V8fV8fV8fC*V4LLiV8fIc", "") +BUILTIN(__builtin_ia32_gatherq_ps256, "V4fV4fV4fC*V4LLiV4fIc", "") + +BUILTIN(__builtin_ia32_gatherd_q, "V2LLiV2LLiV2LLiC*V4iV2LLiIc", "") +BUILTIN(__builtin_ia32_gatherd_q256, "V4LLiV4LLiV4LLiC*V4iV4LLiIc", "") +BUILTIN(__builtin_ia32_gatherq_q, "V2LLiV2LLiV2LLiC*V2LLiV2LLiIc", "") +BUILTIN(__builtin_ia32_gatherq_q256, "V4LLiV4LLiV4LLiC*V4LLiV4LLiIc", "") +BUILTIN(__builtin_ia32_gatherd_d, "V4iV4iV4iC*V4iV4iIc", "") +BUILTIN(__builtin_ia32_gatherd_d256, "V8iV8iV8iC*V8iV8iIc", "") +BUILTIN(__builtin_ia32_gatherq_d, "V4iV4iV4iC*V2LLiV4iIc", "") +BUILTIN(__builtin_ia32_gatherq_d256, "V4iV4iV4iC*V4LLiV4iIc", "") // BMI BUILTIN(__builtin_ia32_bextr_u32, "UiUiUi", "") diff --git a/lib/Headers/avx2intrin.h b/lib/Headers/avx2intrin.h index 2a27bef40e..6878fbd80f 100644 --- a/lib/Headers/avx2intrin.h +++ b/lib/Headers/avx2intrin.h @@ -971,10 +971,10 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y) #define _mm256_mask_i32gather_pd(a, m, i, mask, s) __extension__ ({ \ __m256d __a = (a); \ double const *__m = (m); \ - __m256i __i = (i); \ + __m128i __i = (i); \ __m256d __mask = (mask); \ (__m256d)__builtin_ia32_gatherd_pd256((__v4df)__a, (const __v4df *)__m, \ - (__v8si)__i, (__v4df)__mask, (s)); }) + (__v4si)__i, (__v4df)__mask, (s)); }) #define _mm_mask_i64gather_pd(a, m, i, mask, s) __extension__ ({ \ __m128d __a = (a); \ @@ -1017,9 +1017,73 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y) (__v2di)__i, (__v4sf)__mask, (s)); }) #define _mm256_mask_i64gather_ps(a, m, i, mask, s) __extension__ ({ \ - __m256 __a = (a); \ + __m128 __a = (a); \ float const *__m = (m); \ __m256i __i = (i); \ - __m256 __mask = (mask); \ - (__m256)__builtin_ia32_gatherq_ps256((__v8sf)__a, (const __v8sf *)__m, \ - (__v4di)__i, (__v8sf)__mask, (s)); }) + __m128 __mask = (mask); \ + (__m128)__builtin_ia32_gatherq_ps256((__v4sf)__a, (const __v4sf *)__m, \ + (__v4di)__i, (__v4sf)__mask, (s)); }) + +#define _mm_mask_i32gather_epi32(a, m, i, mask, s) __extension__ ({ \ + __m128i __a = (a); \ + int const *__m = (m); \ + __m128i __i = (i); \ + __m128i __mask = (mask); \ + (__m128i)__builtin_ia32_gatherd_d((__v4si)__a, (const __v4si *)__m, \ + (__v4si)__i, (__v4si)__mask, (s)); }) + +#define _mm256_mask_i32gather_epi32(a, m, i, mask, s) __extension__ ({ \ + __m256i __a = (a); \ + int const *__m = (m); \ + __m256i __i = (i); \ + __m256i __mask = (mask); \ + (__m256i)__builtin_ia32_gatherd_d256((__v8si)__a, (const __v8si *)__m, \ + (__v8si)__i, (__v8si)__mask, (s)); }) + +#define _mm_mask_i64gather_epi32(a, m, i, mask, s) __extension__ ({ \ + __m128i __a = (a); \ + int const *__m = (m); \ + __m128i __i = (i); \ + __m128i __mask = (mask); \ + (__m128i)__builtin_ia32_gatherq_d((__v4si)__a, (const __v4si *)__m, \ + (__v2di)__i, (__v4si)__mask, (s)); }) + +#define _mm256_mask_i64gather_epi32(a, m, i, mask, s) __extension__ ({ \ + __m128i __a = (a); \ + int const *__m = (m); \ + __m256i __i = (i); \ + __m128i __mask = (mask); \ + (__m128i)__builtin_ia32_gatherq_d256((__v4si)__a, (const __v4si *)__m, \ + (__v4di)__i, (__v4si)__mask, (s)); }) + +#define _mm_mask_i32gather_epi64(a, m, i, mask, s) __extension__ ({ \ + __m128i __a = (a); \ + int const *__m = (m); \ + __m128i __i = (i); \ + __m128i __mask = (mask); \ + (__m128i)__builtin_ia32_gatherd_q((__v2di)__a, (const __v2di *)__m, \ + (__v4si)__i, (__v2di)__mask, (s)); }) + +#define _mm256_mask_i32gather_epi64(a, m, i, mask, s) __extension__ ({ \ + __m256i __a = (a); \ + int const *__m = (m); \ + __m128i __i = (i); \ + __m256i __mask = (mask); \ + (__m256i)__builtin_ia32_gatherd_q256((__v4di)__a, (const __v4di *)__m, \ + (__v4si)__i, (__v4di)__mask, (s)); }) + +#define _mm_mask_i64gather_epi64(a, m, i, mask, s) __extension__ ({ \ + __m128i __a = (a); \ + int const *__m = (m); \ + __m128i __i = (i); \ + __m128i __mask = (mask); \ + (__m128i)__builtin_ia32_gatherq_q((__v2di)__a, (const __v2di *)__m, \ + (__v2di)__i, (__v2di)__mask, (s)); }) + +#define _mm256_mask_i64gather_epi64(a, m, i, mask, s) __extension__ ({ \ + __m256i __a = (a); \ + int const *__m = (m); \ + __m256i __i = (i); \ + __m256i __mask = (mask); \ + (__m256i)__builtin_ia32_gatherq_q256((__v4di)__a, (const __v4di *)__m, \ + (__v4di)__i, (__v4di)__mask, (s)); }) diff --git a/test/CodeGen/avx2-builtins.c b/test/CodeGen/avx2-builtins.c index c6953b31e2..86cc80a281 100644 --- a/test/CodeGen/avx2-builtins.c +++ b/test/CodeGen/avx2-builtins.c @@ -10,6 +10,11 @@ __m256i test_mm256_mpsadbw_epu8(__m256i x, __m256i y) { return _mm256_mpsadbw_epu8(x, y, 3); } +__m256i test_mm256_sad_epu8(__m256i x, __m256i y) { + // CHECK: @llvm.x86.avx2.psad.bw + return _mm256_sad_epu8(x, y); +} + __m256i test_mm256_abs_epi8(__m256i a) { // CHECK: @llvm.x86.avx2.pabs.b return _mm256_abs_epi8(a); @@ -787,7 +792,7 @@ __m128d test_mm_mask_i32gather_pd(__m128d a, double const *b, __m128i c, return _mm_mask_i32gather_pd(a, b, c, d, 2); } -__m256d test_mm256_mask_i32gather_pd(__m256d a, double const *b, __m256i c, +__m256d test_mm256_mask_i32gather_pd(__m256d a, double const *b, __m128i c, __m256d d) { // CHECK: @llvm.x86.avx2.gather.d.pd.256 return _mm256_mask_i32gather_pd(a, b, c, d, 2); @@ -818,8 +823,50 @@ __m128 test_mm_mask_i64gather_ps(__m128 a, float const *b, __m128i c, // CHECK: @llvm.x86.avx2.gather.q.ps return _mm_mask_i64gather_ps(a, b, c, d, 2); } -__m256 test_mm256_mask_i64gather_ps(__m256 a, float const *b, __m256i c, - __m256 d) { +__m128 test_mm256_mask_i64gather_ps(__m128 a, float const *b, __m256i c, + __m128 d) { // CHECK: @llvm.x86.avx2.gather.q.ps.256 return _mm256_mask_i64gather_ps(a, b, c, d, 2); } + +__m128i test_mm_mask_i32gather_epi32(__m128i a, int const *b, __m128i c, + __m128i d) { + // CHECK: @llvm.x86.avx2.gather.d.d + return _mm_mask_i32gather_epi32(a, b, c, d, 2); +} +__m256i test_mm256_mask_i32gather_epi32(__m256i a, int const *b, __m256i c, + __m256i d) { + // CHECK: @llvm.x86.avx2.gather.d.d.256 + return _mm256_mask_i32gather_epi32(a, b, c, d, 2); +} +__m128i test_mm_mask_i64gather_epi32(__m128i a, int const *b, __m128i c, + __m128i d) { + // CHECK: @llvm.x86.avx2.gather.q.d + return _mm_mask_i64gather_epi32(a, b, c, d, 2); +} +__m128i test_mm256_mask_i64gather_epi32(__m128i a, int const *b, __m256i c, + __m128i d) { + // CHECK: @llvm.x86.avx2.gather.q.d.256 + return _mm256_mask_i64gather_epi32(a, b, c, d, 2); +} + +__m128i test_mm_mask_i32gather_epi64(__m128i a, int const *b, __m128i c, + __m128i d) { + // CHECK: @llvm.x86.avx2.gather.d.q + return _mm_mask_i32gather_epi64(a, b, c, d, 2); +} +__m256i test_mm256_mask_i32gather_epi64(__m256i a, int const *b, __m128i c, + __m256i d) { + // CHECK: @llvm.x86.avx2.gather.d.q.256 + return _mm256_mask_i32gather_epi64(a, b, c, d, 2); +} +__m128i test_mm_mask_i64gather_epi64(__m128i a, int const *b, __m128i c, + __m128i d) { + // CHECK: @llvm.x86.avx2.gather.q.q + return _mm_mask_i64gather_epi64(a, b, c, d, 2); +} +__m256i test_mm256_mask_i64gather_epi64(__m256i a, int const *b, __m256i c, + __m256i d) { + // CHECK: @llvm.x86.avx2.gather.q.q.256 + return _mm256_mask_i64gather_epi64(a, b, c, d, 2); +} -- 2.40.0