From 36363304903286369a63b0498d506a13fe335606 Mon Sep 17 00:00:00 2001 From: Linfeng Zhang Date: Tue, 30 Jan 2018 16:01:41 -0800 Subject: [PATCH] Add vp9_highbd_iht4x4_16_add_neon() BUG=webm:1403 Change-Id: Id9833e985fb70958cf4bde38f8e6303ed83c12f9 --- test/dct_test.cc | 78 +++++++-- .../arm/neon/vp9_highbd_iht4x4_add_neon.c | 162 ++++++++++++++++++ vp9/common/arm/neon/vp9_iht4x4_add_neon.c | 52 +----- vp9/common/arm/neon/vp9_iht_neon.h | 63 +++++++ vp9/common/vp9_rtcd_defs.pl | 2 +- vp9/common/x86/vp9_idct_intrin_sse2.c | 2 - vp9/vp9_common.mk | 2 + vpx_dsp/arm/highbd_idct4x4_add_neon.c | 80 +-------- vpx_dsp/arm/highbd_idct_neon.h | 99 +++++++++++ vpx_dsp/vpx_dsp.mk | 1 + 10 files changed, 394 insertions(+), 147 deletions(-) create mode 100644 vp9/common/arm/neon/vp9_highbd_iht4x4_add_neon.c create mode 100644 vp9/common/arm/neon/vp9_iht_neon.h create mode 100644 vpx_dsp/arm/highbd_idct_neon.h diff --git a/test/dct_test.cc b/test/dct_test.cc index 6873d0542..71b36c7bb 100644 --- a/test/dct_test.cc +++ b/test/dct_test.cc @@ -753,26 +753,66 @@ INSTANTIATE_TEST_CASE_P(C, TransHT, ::testing::ValuesIn(c_ht_tests)); #if !CONFIG_EMULATE_HARDWARE #if HAVE_NEON -INSTANTIATE_TEST_CASE_P( - NEON, TransHT, - ::testing::Values( - make_tuple(&vp9_fht8x8_c, &iht_wrapper, 8, 0, - VPX_BITS_8, 1), - make_tuple(&vp9_fht8x8_c, &iht_wrapper, 8, 1, - VPX_BITS_8, 1), - make_tuple(&vp9_fht8x8_c, &iht_wrapper, 8, 2, - VPX_BITS_8, 1), - make_tuple(&vp9_fht8x8_c, &iht_wrapper, 8, 3, - VPX_BITS_8, 1), - make_tuple(&vp9_fht4x4_c, &iht_wrapper, 4, 0, - VPX_BITS_8, 1), - make_tuple(&vp9_fht4x4_c, &iht_wrapper, 4, 1, - VPX_BITS_8, 1), - make_tuple(&vp9_fht4x4_c, &iht_wrapper, 4, 2, - VPX_BITS_8, 1), - make_tuple(&vp9_fht4x4_c, &iht_wrapper, 4, 3, - VPX_BITS_8, 1))); +const DctParam neon_ht_tests[] = { +#if CONFIG_VP9_HIGHBITDEPTH + make_tuple(&vp9_highbd_fht4x4_c, + &highbd_iht_wrapper, 4, 0, + VPX_BITS_8, 2), + make_tuple(&vp9_highbd_fht4x4_c, + &highbd_iht_wrapper, 4, 1, + VPX_BITS_8, 2), + make_tuple(&vp9_highbd_fht4x4_c, + &highbd_iht_wrapper, 4, 2, + VPX_BITS_8, 2), + make_tuple(&vp9_highbd_fht4x4_c, + &highbd_iht_wrapper, 4, 3, + VPX_BITS_8, 2), + make_tuple(&vp9_highbd_fht4x4_c, + &highbd_iht_wrapper, 4, 0, + VPX_BITS_10, 2), + make_tuple(&vp9_highbd_fht4x4_c, + &highbd_iht_wrapper, 4, 1, + VPX_BITS_10, 2), + make_tuple(&vp9_highbd_fht4x4_c, + &highbd_iht_wrapper, 4, 2, + VPX_BITS_10, 2), + make_tuple(&vp9_highbd_fht4x4_c, + &highbd_iht_wrapper, 4, 3, + VPX_BITS_10, 2), + make_tuple(&vp9_highbd_fht4x4_c, + &highbd_iht_wrapper, 4, 0, + VPX_BITS_12, 2), + make_tuple(&vp9_highbd_fht4x4_c, + &highbd_iht_wrapper, 4, 1, + VPX_BITS_12, 2), + make_tuple(&vp9_highbd_fht4x4_c, + &highbd_iht_wrapper, 4, 2, + VPX_BITS_12, 2), + make_tuple(&vp9_highbd_fht4x4_c, + &highbd_iht_wrapper, 4, 3, + VPX_BITS_12, 2), +#endif // CONFIG_VP9_HIGHBITDEPTH + make_tuple(&vp9_fht8x8_c, &iht_wrapper, 8, 0, + VPX_BITS_8, 1), + make_tuple(&vp9_fht8x8_c, &iht_wrapper, 8, 1, + VPX_BITS_8, 1), + make_tuple(&vp9_fht8x8_c, &iht_wrapper, 8, 2, + VPX_BITS_8, 1), + make_tuple(&vp9_fht8x8_c, &iht_wrapper, 8, 3, + VPX_BITS_8, 1), + + make_tuple(&vp9_fht4x4_c, &iht_wrapper, 4, 0, + VPX_BITS_8, 1), + make_tuple(&vp9_fht4x4_c, &iht_wrapper, 4, 1, + VPX_BITS_8, 1), + make_tuple(&vp9_fht4x4_c, &iht_wrapper, 4, 2, + VPX_BITS_8, 1), + make_tuple(&vp9_fht4x4_c, &iht_wrapper, 4, 3, + VPX_BITS_8, 1) +}; + +INSTANTIATE_TEST_CASE_P(NEON, TransHT, ::testing::ValuesIn(neon_ht_tests)); #endif // HAVE_NEON #if HAVE_SSE2 diff --git a/vp9/common/arm/neon/vp9_highbd_iht4x4_add_neon.c b/vp9/common/arm/neon/vp9_highbd_iht4x4_add_neon.c new file mode 100644 index 000000000..156d542a5 --- /dev/null +++ b/vp9/common/arm/neon/vp9_highbd_iht4x4_add_neon.c @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2018 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include + +#include "./vp9_rtcd.h" +#include "./vpx_config.h" +#include "vp9/common/vp9_common.h" +#include "vp9/common/arm/neon/vp9_iht_neon.h" +#include "vpx_dsp/arm/highbd_idct_neon.h" +#include "vpx_dsp/arm/idct_neon.h" +#include "vpx_dsp/arm/mem_neon.h" +#include "vpx_dsp/txfm_common.h" + +static INLINE void highbd_iadst4(int32x4_t *const io) { + const int32_t sinpis[4] = { sinpi_1_9, sinpi_2_9, sinpi_3_9, sinpi_4_9 }; + const int32x4_t sinpi = vld1q_s32(sinpis); + int32x4_t s[8]; + + s[0] = vmulq_lane_s32(io[0], vget_low_s32(sinpi), 0); + s[1] = vmulq_lane_s32(io[0], vget_low_s32(sinpi), 1); + s[2] = vmulq_lane_s32(io[1], vget_high_s32(sinpi), 0); + s[3] = vmulq_lane_s32(io[2], vget_high_s32(sinpi), 1); + s[4] = vmulq_lane_s32(io[2], vget_low_s32(sinpi), 0); + s[5] = vmulq_lane_s32(io[3], vget_low_s32(sinpi), 1); + s[6] = vmulq_lane_s32(io[3], vget_high_s32(sinpi), 1); + s[7] = vsubq_s32(io[0], io[2]); + s[7] = vaddq_s32(s[7], io[3]); + + s[0] = vaddq_s32(s[0], s[3]); + s[0] = vaddq_s32(s[0], s[5]); + s[1] = vsubq_s32(s[1], s[4]); + s[1] = vsubq_s32(s[1], s[6]); + s[3] = s[2]; + s[2] = vmulq_lane_s32(s[7], vget_high_s32(sinpi), 0); + + io[0] = vaddq_s32(s[0], s[3]); + io[1] = vaddq_s32(s[1], s[3]); + io[2] = s[2]; + io[3] = vaddq_s32(s[0], s[1]); + io[3] = vsubq_s32(io[3], s[3]); + io[0] = vrshrq_n_s32(io[0], DCT_CONST_BITS); + io[1] = vrshrq_n_s32(io[1], DCT_CONST_BITS); + io[2] = vrshrq_n_s32(io[2], DCT_CONST_BITS); + io[3] = vrshrq_n_s32(io[3], DCT_CONST_BITS); +} + +void vp9_highbd_iht4x4_16_add_neon(const tran_low_t *input, uint16_t *dest, + int stride, int tx_type, int bd) { + const int16x8_t max = vdupq_n_s16((1 << bd) - 1); + int16x8_t a[2]; + int32x4_t c[4]; + + c[0] = vld1q_s32(input); + c[1] = vld1q_s32(input + 4); + c[2] = vld1q_s32(input + 8); + c[3] = vld1q_s32(input + 12); + + if (bd == 8) { + a[0] = vcombine_s16(vmovn_s32(c[0]), vmovn_s32(c[1])); + a[1] = vcombine_s16(vmovn_s32(c[2]), vmovn_s32(c[3])); + transpose_s16_4x4q(&a[0], &a[1]); + + switch (tx_type) { + case DCT_DCT: + idct4x4_16_kernel_bd8(a); + a[1] = vcombine_s16(vget_high_s16(a[1]), vget_low_s16(a[1])); + transpose_s16_4x4q(&a[0], &a[1]); + idct4x4_16_kernel_bd8(a); + a[1] = vcombine_s16(vget_high_s16(a[1]), vget_low_s16(a[1])); + break; + + case ADST_DCT: + idct4x4_16_kernel_bd8(a); + a[1] = vcombine_s16(vget_high_s16(a[1]), vget_low_s16(a[1])); + transpose_s16_4x4q(&a[0], &a[1]); + iadst4(a); + break; + + case DCT_ADST: + iadst4(a); + transpose_s16_4x4q(&a[0], &a[1]); + idct4x4_16_kernel_bd8(a); + a[1] = vcombine_s16(vget_high_s16(a[1]), vget_low_s16(a[1])); + break; + + default: + assert(tx_type == ADST_ADST); + iadst4(a); + transpose_s16_4x4q(&a[0], &a[1]); + iadst4(a); + break; + } + a[0] = vrshrq_n_s16(a[0], 4); + a[1] = vrshrq_n_s16(a[1], 4); + } else { + switch (tx_type) { + case DCT_DCT: { + const int32x4_t cospis = vld1q_s32(kCospi32); + + if (bd == 10) { + idct4x4_16_kernel_bd10(cospis, c); + idct4x4_16_kernel_bd10(cospis, c); + } else { + idct4x4_16_kernel_bd12(cospis, c); + idct4x4_16_kernel_bd12(cospis, c); + } + break; + } + + case ADST_DCT: { + const int32x4_t cospis = vld1q_s32(kCospi32); + + if (bd == 10) { + idct4x4_16_kernel_bd10(cospis, c); + } else { + idct4x4_16_kernel_bd12(cospis, c); + } + transpose_s32_4x4(&c[0], &c[1], &c[2], &c[3]); + highbd_iadst4(c); + break; + } + + case DCT_ADST: { + const int32x4_t cospis = vld1q_s32(kCospi32); + + if (bd == 10) { + transpose_s32_4x4(&c[0], &c[1], &c[2], &c[3]); + highbd_iadst4(c); + idct4x4_16_kernel_bd10(cospis, c); + } else { + transpose_s32_4x4(&c[0], &c[1], &c[2], &c[3]); + highbd_iadst4(c); + idct4x4_16_kernel_bd12(cospis, c); + } + break; + } + + default: { + assert(tx_type == ADST_ADST); + transpose_s32_4x4(&c[0], &c[1], &c[2], &c[3]); + highbd_iadst4(c); + transpose_s32_4x4(&c[0], &c[1], &c[2], &c[3]); + highbd_iadst4(c); + break; + } + } + a[0] = vcombine_s16(vqrshrn_n_s32(c[0], 4), vqrshrn_n_s32(c[1], 4)); + a[1] = vcombine_s16(vqrshrn_n_s32(c[2], 4), vqrshrn_n_s32(c[3], 4)); + } + + highbd_idct4x4_1_add_kernel1(&dest, stride, a[0], max); + highbd_idct4x4_1_add_kernel1(&dest, stride, a[1], max); +} diff --git a/vp9/common/arm/neon/vp9_iht4x4_add_neon.c b/vp9/common/arm/neon/vp9_iht4x4_add_neon.c index f469de6c9..4f0a90f21 100644 --- a/vp9/common/arm/neon/vp9_iht4x4_add_neon.c +++ b/vp9/common/arm/neon/vp9_iht4x4_add_neon.c @@ -14,50 +14,11 @@ #include "./vp9_rtcd.h" #include "./vpx_config.h" #include "vp9/common/vp9_common.h" +#include "vp9/common/arm/neon/vp9_iht_neon.h" #include "vpx_dsp/arm/idct_neon.h" #include "vpx_dsp/arm/mem_neon.h" #include "vpx_dsp/txfm_common.h" -static INLINE void iadst4(int16x8_t *const io) { - const int32x4_t c3 = vdupq_n_s32(sinpi_3_9); - int16x4_t c[5], x[4]; - int32x4_t s[8], output[4]; - - c[1] = vdup_n_s16(sinpi_1_9); - c[2] = vdup_n_s16(sinpi_2_9); - c[3] = vdup_n_s16(sinpi_3_9); - c[4] = vdup_n_s16(sinpi_4_9); - - x[0] = vget_low_s16(io[0]); - x[1] = vget_low_s16(io[1]); - x[2] = vget_high_s16(io[0]); - x[3] = vget_high_s16(io[1]); - - s[0] = vmull_s16(c[1], x[0]); - s[1] = vmull_s16(c[2], x[0]); - s[2] = vmull_s16(c[3], x[1]); - s[3] = vmull_s16(c[4], x[2]); - s[4] = vmull_s16(c[1], x[2]); - s[5] = vmull_s16(c[2], x[3]); - s[6] = vmull_s16(c[4], x[3]); - s[7] = vaddl_s16(x[0], x[3]); - s[7] = vsubw_s16(s[7], x[2]); - - s[0] = vaddq_s32(s[0], s[3]); - s[0] = vaddq_s32(s[0], s[5]); - s[1] = vsubq_s32(s[1], s[4]); - s[1] = vsubq_s32(s[1], s[6]); - s[3] = s[2]; - s[2] = vmulq_s32(c3, s[7]); - - output[0] = vaddq_s32(s[0], s[3]); - output[1] = vaddq_s32(s[1], s[3]); - output[2] = s[2]; - output[3] = vaddq_s32(s[0], s[1]); - output[3] = vsubq_s32(output[3], s[3]); - dct_const_round_shift_low_8_dual(output, &io[0], &io[1]); -} - void vp9_iht4x4_16_add_neon(const tran_low_t *input, uint8_t *dest, int stride, int tx_type) { int16x8_t a[2]; @@ -72,7 +33,7 @@ void vp9_iht4x4_16_add_neon(const tran_low_t *input, uint8_t *dest, int stride, transpose_s16_4x4q(&a[0], &a[1]); switch (tx_type) { - case 0: // DCT_DCT + case DCT_DCT: idct4x4_16_kernel_bd8(a); a[1] = vcombine_s16(vget_high_s16(a[1]), vget_low_s16(a[1])); transpose_s16_4x4q(&a[0], &a[1]); @@ -80,27 +41,26 @@ void vp9_iht4x4_16_add_neon(const tran_low_t *input, uint8_t *dest, int stride, a[1] = vcombine_s16(vget_high_s16(a[1]), vget_low_s16(a[1])); break; - case 1: // ADST_DCT + case ADST_DCT: idct4x4_16_kernel_bd8(a); a[1] = vcombine_s16(vget_high_s16(a[1]), vget_low_s16(a[1])); transpose_s16_4x4q(&a[0], &a[1]); iadst4(a); break; - case 2: // DCT_ADST + case DCT_ADST: iadst4(a); transpose_s16_4x4q(&a[0], &a[1]); idct4x4_16_kernel_bd8(a); a[1] = vcombine_s16(vget_high_s16(a[1]), vget_low_s16(a[1])); break; - case 3: // ADST_ADST + default: + assert(tx_type == ADST_ADST); iadst4(a); transpose_s16_4x4q(&a[0], &a[1]); iadst4(a); break; - - default: assert(0); break; } a[0] = vrshrq_n_s16(a[0], 4); diff --git a/vp9/common/arm/neon/vp9_iht_neon.h b/vp9/common/arm/neon/vp9_iht_neon.h new file mode 100644 index 000000000..b222377a7 --- /dev/null +++ b/vp9/common/arm/neon/vp9_iht_neon.h @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2018 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VP9_COMMON_ARM_NEON_VP9_IHT_NEON_H_ +#define VP9_COMMON_ARM_NEON_VP9_IHT_NEON_H_ + +#include + +#include "./vp9_rtcd.h" +#include "./vpx_config.h" +#include "vp9/common/vp9_common.h" +#include "vpx_dsp/arm/idct_neon.h" +#include "vpx_dsp/arm/mem_neon.h" +#include "vpx_dsp/txfm_common.h" + +static INLINE void iadst4(int16x8_t *const io) { + const int32x4_t c3 = vdupq_n_s32(sinpi_3_9); + int16x4_t c[5], x[4]; + int32x4_t s[8], output[4]; + + c[1] = vdup_n_s16(sinpi_1_9); + c[2] = vdup_n_s16(sinpi_2_9); + c[3] = vdup_n_s16(sinpi_3_9); + c[4] = vdup_n_s16(sinpi_4_9); + + x[0] = vget_low_s16(io[0]); + x[1] = vget_low_s16(io[1]); + x[2] = vget_high_s16(io[0]); + x[3] = vget_high_s16(io[1]); + + s[0] = vmull_s16(c[1], x[0]); + s[1] = vmull_s16(c[2], x[0]); + s[2] = vmull_s16(c[3], x[1]); + s[3] = vmull_s16(c[4], x[2]); + s[4] = vmull_s16(c[1], x[2]); + s[5] = vmull_s16(c[2], x[3]); + s[6] = vmull_s16(c[4], x[3]); + s[7] = vaddl_s16(x[0], x[3]); + s[7] = vsubw_s16(s[7], x[2]); + + s[0] = vaddq_s32(s[0], s[3]); + s[0] = vaddq_s32(s[0], s[5]); + s[1] = vsubq_s32(s[1], s[4]); + s[1] = vsubq_s32(s[1], s[6]); + s[3] = s[2]; + s[2] = vmulq_s32(c3, s[7]); + + output[0] = vaddq_s32(s[0], s[3]); + output[1] = vaddq_s32(s[1], s[3]); + output[2] = s[2]; + output[3] = vaddq_s32(s[0], s[1]); + output[3] = vsubq_s32(output[3], s[3]); + dct_const_round_shift_low_8_dual(output, &io[0], &io[1]); +} + +#endif // VP9_COMMON_ARM_NEON_VP9_IHT_NEON_H_ diff --git a/vp9/common/vp9_rtcd_defs.pl b/vp9/common/vp9_rtcd_defs.pl index 2b15b661c..d048857dd 100644 --- a/vp9/common/vp9_rtcd_defs.pl +++ b/vp9/common/vp9_rtcd_defs.pl @@ -103,7 +103,7 @@ if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") { add_proto qw/void vp9_highbd_iht16x16_256_add/, "const tran_low_t *input, uint16_t *output, int pitch, int tx_type, int bd"; if (vpx_config("CONFIG_EMULATE_HARDWARE") ne "yes") { - specialize qw/vp9_highbd_iht4x4_16_add sse4_1/; + specialize qw/vp9_highbd_iht4x4_16_add neon sse4_1/; specialize qw/vp9_highbd_iht8x8_64_add sse4_1/; specialize qw/vp9_highbd_iht16x16_256_add sse4_1/; } diff --git a/vp9/common/x86/vp9_idct_intrin_sse2.c b/vp9/common/x86/vp9_idct_intrin_sse2.c index 95dad919e..8b3109fa9 100644 --- a/vp9/common/x86/vp9_idct_intrin_sse2.c +++ b/vp9/common/x86/vp9_idct_intrin_sse2.c @@ -10,8 +10,6 @@ #include "./vp9_rtcd.h" #include "vpx_dsp/x86/inv_txfm_sse2.h" -#include "vpx_dsp/x86/txfm_common_sse2.h" -#include "vpx_ports/mem.h" void vp9_iht4x4_16_add_sse2(const tran_low_t *input, uint8_t *dest, int stride, int tx_type) { diff --git a/vp9/vp9_common.mk b/vp9/vp9_common.mk index c502df332..721e170bf 100644 --- a/vp9/vp9_common.mk +++ b/vp9/vp9_common.mk @@ -70,6 +70,7 @@ VP9_COMMON_SRCS-$(HAVE_MSA) += common/mips/msa/vp9_idct16x16_msa.c VP9_COMMON_SRCS-$(HAVE_SSE2) += common/x86/vp9_idct_intrin_sse2.c VP9_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/vp9_iht4x4_add_neon.c VP9_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/vp9_iht8x8_add_neon.c +VP9_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/vp9_iht_neon.h ifeq ($(CONFIG_VP9_POSTPROC),yes) VP9_COMMON_SRCS-$(HAVE_MSA) += common/mips/msa/vp9_mfqe_msa.c @@ -81,6 +82,7 @@ VP9_COMMON_SRCS-$(HAVE_DSPR2) += common/mips/dspr2/vp9_itrans4_dspr2.c VP9_COMMON_SRCS-$(HAVE_DSPR2) += common/mips/dspr2/vp9_itrans8_dspr2.c VP9_COMMON_SRCS-$(HAVE_DSPR2) += common/mips/dspr2/vp9_itrans16_dspr2.c else +VP9_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/vp9_highbd_iht4x4_add_neon.c VP9_COMMON_SRCS-$(HAVE_SSE4_1) += common/x86/vp9_highbd_iht4x4_add_sse4.c VP9_COMMON_SRCS-$(HAVE_SSE4_1) += common/x86/vp9_highbd_iht8x8_add_sse4.c VP9_COMMON_SRCS-$(HAVE_SSE4_1) += common/x86/vp9_highbd_iht16x16_add_sse4.c diff --git a/vpx_dsp/arm/highbd_idct4x4_add_neon.c b/vpx_dsp/arm/highbd_idct4x4_add_neon.c index ed232b09d..7be1dad1d 100644 --- a/vpx_dsp/arm/highbd_idct4x4_add_neon.c +++ b/vpx_dsp/arm/highbd_idct4x4_add_neon.c @@ -11,27 +11,10 @@ #include #include "./vpx_dsp_rtcd.h" +#include "vpx_dsp/arm/highbd_idct_neon.h" #include "vpx_dsp/arm/idct_neon.h" #include "vpx_dsp/inv_txfm.h" -static INLINE void highbd_idct4x4_1_add_kernel1(uint16_t **dest, - const int stride, - const int16x8_t res, - const int16x8_t max) { - const uint16x4_t a0 = vld1_u16(*dest); - const uint16x4_t a1 = vld1_u16(*dest + stride); - const int16x8_t a = vreinterpretq_s16_u16(vcombine_u16(a0, a1)); - // Note: In some profile tests, res is quite close to +/-32767. - // We use saturating addition. - const int16x8_t b = vqaddq_s16(res, a); - const int16x8_t c = vminq_s16(b, max); - const uint16x8_t d = vqshluq_n_s16(c, 0); - vst1_u16(*dest, vget_low_u16(d)); - *dest += stride; - vst1_u16(*dest, vget_high_u16(d)); - *dest += stride; -} - // res is in reverse row order static INLINE void highbd_idct4x4_1_add_kernel2(uint16_t **dest, const int stride, @@ -65,67 +48,6 @@ void vpx_highbd_idct4x4_1_add_neon(const tran_low_t *input, uint16_t *dest, highbd_idct4x4_1_add_kernel1(&dest, stride, dc, max); } -static INLINE void idct4x4_16_kernel_bd10(const int32x4_t cospis, - int32x4_t *const a) { - int32x4_t b0, b1, b2, b3; - - transpose_s32_4x4(&a[0], &a[1], &a[2], &a[3]); - b0 = vaddq_s32(a[0], a[2]); - b1 = vsubq_s32(a[0], a[2]); - b0 = vmulq_lane_s32(b0, vget_high_s32(cospis), 0); - b1 = vmulq_lane_s32(b1, vget_high_s32(cospis), 0); - b2 = vmulq_lane_s32(a[1], vget_high_s32(cospis), 1); - b3 = vmulq_lane_s32(a[1], vget_low_s32(cospis), 1); - b2 = vmlsq_lane_s32(b2, a[3], vget_low_s32(cospis), 1); - b3 = vmlaq_lane_s32(b3, a[3], vget_high_s32(cospis), 1); - b0 = vrshrq_n_s32(b0, DCT_CONST_BITS); - b1 = vrshrq_n_s32(b1, DCT_CONST_BITS); - b2 = vrshrq_n_s32(b2, DCT_CONST_BITS); - b3 = vrshrq_n_s32(b3, DCT_CONST_BITS); - a[0] = vaddq_s32(b0, b3); - a[1] = vaddq_s32(b1, b2); - a[2] = vsubq_s32(b1, b2); - a[3] = vsubq_s32(b0, b3); -} - -static INLINE void idct4x4_16_kernel_bd12(const int32x4_t cospis, - int32x4_t *const a) { - int32x4_t b0, b1, b2, b3; - int64x2_t c[12]; - - transpose_s32_4x4(&a[0], &a[1], &a[2], &a[3]); - b0 = vaddq_s32(a[0], a[2]); - b1 = vsubq_s32(a[0], a[2]); - c[0] = vmull_lane_s32(vget_low_s32(b0), vget_high_s32(cospis), 0); - c[1] = vmull_lane_s32(vget_high_s32(b0), vget_high_s32(cospis), 0); - c[2] = vmull_lane_s32(vget_low_s32(b1), vget_high_s32(cospis), 0); - c[3] = vmull_lane_s32(vget_high_s32(b1), vget_high_s32(cospis), 0); - c[4] = vmull_lane_s32(vget_low_s32(a[1]), vget_high_s32(cospis), 1); - c[5] = vmull_lane_s32(vget_high_s32(a[1]), vget_high_s32(cospis), 1); - c[6] = vmull_lane_s32(vget_low_s32(a[1]), vget_low_s32(cospis), 1); - c[7] = vmull_lane_s32(vget_high_s32(a[1]), vget_low_s32(cospis), 1); - c[8] = vmull_lane_s32(vget_low_s32(a[3]), vget_low_s32(cospis), 1); - c[9] = vmull_lane_s32(vget_high_s32(a[3]), vget_low_s32(cospis), 1); - c[10] = vmull_lane_s32(vget_low_s32(a[3]), vget_high_s32(cospis), 1); - c[11] = vmull_lane_s32(vget_high_s32(a[3]), vget_high_s32(cospis), 1); - c[4] = vsubq_s64(c[4], c[8]); - c[5] = vsubq_s64(c[5], c[9]); - c[6] = vaddq_s64(c[6], c[10]); - c[7] = vaddq_s64(c[7], c[11]); - b0 = vcombine_s32(vrshrn_n_s64(c[0], DCT_CONST_BITS), - vrshrn_n_s64(c[1], DCT_CONST_BITS)); - b1 = vcombine_s32(vrshrn_n_s64(c[2], DCT_CONST_BITS), - vrshrn_n_s64(c[3], DCT_CONST_BITS)); - b2 = vcombine_s32(vrshrn_n_s64(c[4], DCT_CONST_BITS), - vrshrn_n_s64(c[5], DCT_CONST_BITS)); - b3 = vcombine_s32(vrshrn_n_s64(c[6], DCT_CONST_BITS), - vrshrn_n_s64(c[7], DCT_CONST_BITS)); - a[0] = vaddq_s32(b0, b3); - a[1] = vaddq_s32(b1, b2); - a[2] = vsubq_s32(b1, b2); - a[3] = vsubq_s32(b0, b3); -} - void vpx_highbd_idct4x4_16_add_neon(const tran_low_t *input, uint16_t *dest, int stride, int bd) { const int16x8_t max = vdupq_n_s16((1 << bd) - 1); diff --git a/vpx_dsp/arm/highbd_idct_neon.h b/vpx_dsp/arm/highbd_idct_neon.h new file mode 100644 index 000000000..92fcb7f3a --- /dev/null +++ b/vpx_dsp/arm/highbd_idct_neon.h @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2018 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VPX_DSP_ARM_HIGHBD_IDCT_NEON_H_ +#define VPX_DSP_ARM_HIGHBD_IDCT_NEON_H_ + +#include + +#include "./vpx_dsp_rtcd.h" +#include "vpx_dsp/arm/idct_neon.h" +#include "vpx_dsp/inv_txfm.h" + +static INLINE void highbd_idct4x4_1_add_kernel1(uint16_t **dest, + const int stride, + const int16x8_t res, + const int16x8_t max) { + const uint16x4_t a0 = vld1_u16(*dest); + const uint16x4_t a1 = vld1_u16(*dest + stride); + const int16x8_t a = vreinterpretq_s16_u16(vcombine_u16(a0, a1)); + // Note: In some profile tests, res is quite close to +/-32767. + // We use saturating addition. + const int16x8_t b = vqaddq_s16(res, a); + const int16x8_t c = vminq_s16(b, max); + const uint16x8_t d = vqshluq_n_s16(c, 0); + vst1_u16(*dest, vget_low_u16(d)); + *dest += stride; + vst1_u16(*dest, vget_high_u16(d)); + *dest += stride; +} + +static INLINE void idct4x4_16_kernel_bd10(const int32x4_t cospis, + int32x4_t *const a) { + int32x4_t b0, b1, b2, b3; + + transpose_s32_4x4(&a[0], &a[1], &a[2], &a[3]); + b0 = vaddq_s32(a[0], a[2]); + b1 = vsubq_s32(a[0], a[2]); + b0 = vmulq_lane_s32(b0, vget_high_s32(cospis), 0); + b1 = vmulq_lane_s32(b1, vget_high_s32(cospis), 0); + b2 = vmulq_lane_s32(a[1], vget_high_s32(cospis), 1); + b3 = vmulq_lane_s32(a[1], vget_low_s32(cospis), 1); + b2 = vmlsq_lane_s32(b2, a[3], vget_low_s32(cospis), 1); + b3 = vmlaq_lane_s32(b3, a[3], vget_high_s32(cospis), 1); + b0 = vrshrq_n_s32(b0, DCT_CONST_BITS); + b1 = vrshrq_n_s32(b1, DCT_CONST_BITS); + b2 = vrshrq_n_s32(b2, DCT_CONST_BITS); + b3 = vrshrq_n_s32(b3, DCT_CONST_BITS); + a[0] = vaddq_s32(b0, b3); + a[1] = vaddq_s32(b1, b2); + a[2] = vsubq_s32(b1, b2); + a[3] = vsubq_s32(b0, b3); +} + +static INLINE void idct4x4_16_kernel_bd12(const int32x4_t cospis, + int32x4_t *const a) { + int32x4_t b0, b1, b2, b3; + int64x2_t c[12]; + + transpose_s32_4x4(&a[0], &a[1], &a[2], &a[3]); + b0 = vaddq_s32(a[0], a[2]); + b1 = vsubq_s32(a[0], a[2]); + c[0] = vmull_lane_s32(vget_low_s32(b0), vget_high_s32(cospis), 0); + c[1] = vmull_lane_s32(vget_high_s32(b0), vget_high_s32(cospis), 0); + c[2] = vmull_lane_s32(vget_low_s32(b1), vget_high_s32(cospis), 0); + c[3] = vmull_lane_s32(vget_high_s32(b1), vget_high_s32(cospis), 0); + c[4] = vmull_lane_s32(vget_low_s32(a[1]), vget_high_s32(cospis), 1); + c[5] = vmull_lane_s32(vget_high_s32(a[1]), vget_high_s32(cospis), 1); + c[6] = vmull_lane_s32(vget_low_s32(a[1]), vget_low_s32(cospis), 1); + c[7] = vmull_lane_s32(vget_high_s32(a[1]), vget_low_s32(cospis), 1); + c[8] = vmull_lane_s32(vget_low_s32(a[3]), vget_low_s32(cospis), 1); + c[9] = vmull_lane_s32(vget_high_s32(a[3]), vget_low_s32(cospis), 1); + c[10] = vmull_lane_s32(vget_low_s32(a[3]), vget_high_s32(cospis), 1); + c[11] = vmull_lane_s32(vget_high_s32(a[3]), vget_high_s32(cospis), 1); + c[4] = vsubq_s64(c[4], c[8]); + c[5] = vsubq_s64(c[5], c[9]); + c[6] = vaddq_s64(c[6], c[10]); + c[7] = vaddq_s64(c[7], c[11]); + b0 = vcombine_s32(vrshrn_n_s64(c[0], DCT_CONST_BITS), + vrshrn_n_s64(c[1], DCT_CONST_BITS)); + b1 = vcombine_s32(vrshrn_n_s64(c[2], DCT_CONST_BITS), + vrshrn_n_s64(c[3], DCT_CONST_BITS)); + b2 = vcombine_s32(vrshrn_n_s64(c[4], DCT_CONST_BITS), + vrshrn_n_s64(c[5], DCT_CONST_BITS)); + b3 = vcombine_s32(vrshrn_n_s64(c[6], DCT_CONST_BITS), + vrshrn_n_s64(c[7], DCT_CONST_BITS)); + a[0] = vaddq_s32(b0, b3); + a[1] = vaddq_s32(b1, b2); + a[2] = vsubq_s32(b1, b2); + a[3] = vsubq_s32(b0, b3); +} + +#endif // VPX_DSP_ARM_HIGHBD_IDCT_NEON_H_ diff --git a/vpx_dsp/vpx_dsp.mk b/vpx_dsp/vpx_dsp.mk index 3b1a873cd..a4a6fa084 100644 --- a/vpx_dsp/vpx_dsp.mk +++ b/vpx_dsp/vpx_dsp.mk @@ -242,6 +242,7 @@ DSP_SRCS-$(HAVE_NEON) += arm/highbd_idct32x32_add_neon.c DSP_SRCS-$(HAVE_NEON) += arm/highbd_idct32x32_34_add_neon.c DSP_SRCS-$(HAVE_NEON) += arm/highbd_idct32x32_135_add_neon.c DSP_SRCS-$(HAVE_NEON) += arm/highbd_idct32x32_1024_add_neon.c +DSP_SRCS-$(HAVE_NEON) += arm/highbd_idct_neon.h DSP_SRCS-$(HAVE_SSE2) += x86/highbd_inv_txfm_sse2.h DSP_SRCS-$(HAVE_SSE2) += x86/highbd_idct4x4_add_sse2.c DSP_SRCS-$(HAVE_SSE2) += x86/highbd_idct8x8_add_sse2.c -- 2.40.0