From 931ed516ba308e1f927afeeea11aa35ce6ee87c3 Mon Sep 17 00:00:00 2001 From: Deb Mukherjee Date: Wed, 17 Sep 2014 16:55:05 -0700 Subject: [PATCH] High bit-depth loop/arf/postproc filter functions Adds high-bitdepth loopfilter, temporal filter and postproc functions Change-Id: I81c8a9176890784686bc4f2af0d550d243b3b2d3 --- test/lpf_8_test.cc | 587 +++++++++ test/test.mk | 1 + vp9/common/vp9_loopfilter.c | 396 +++++- vp9/common/vp9_loopfilter_filters.c | 401 ++++++ vp9/common/vp9_postproc.c | 241 +++- vp9/common/vp9_rtcd_defs.pl | 161 ++- .../x86/vp9_high_loopfilter_intrin_sse2.c | 1119 +++++++++++++++++ vp9/encoder/vp9_temporal_filter.c | 232 +++- vp9/vp9_common.mk | 1 + 9 files changed, 3070 insertions(+), 69 deletions(-) create mode 100644 test/lpf_8_test.cc create mode 100644 vp9/common/x86/vp9_high_loopfilter_intrin_sse2.c diff --git a/test/lpf_8_test.cc b/test/lpf_8_test.cc new file mode 100644 index 000000000..abc410798 --- /dev/null +++ b/test/lpf_8_test.cc @@ -0,0 +1,587 @@ +/* + * Copyright (c) 2014 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include +#include + +#include "third_party/googletest/src/include/gtest/gtest.h" +#include "test/acm_random.h" +#include "test/clear_system_state.h" +#include "test/register_state_check.h" +#include "test/util.h" + +#include "./vpx_config.h" +#include "./vp9_rtcd.h" +#include "vp9/common/vp9_entropy.h" +#include "vpx/vpx_integer.h" + +using libvpx_test::ACMRandom; + +namespace { +// Horizontally and Vertically need 32x32: 8 Coeffs preceeding filtered section +// 16 Coefs within filtered section +// 8 Coeffs following filtered section +const int kNumCoeffs = 1024; + +const int number_of_iterations = 10000; + +#if CONFIG_VP9_HIGHBITDEPTH +typedef void (*loop_op_t)(uint16_t *s, int p, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh, + int count, int bd); +typedef void (*dual_loop_op_t)(uint16_t *s, int p, const uint8_t *blimit0, + const uint8_t *limit0, const uint8_t *thresh0, + const uint8_t *blimit1, const uint8_t *limit1, + const uint8_t *thresh1, int bd); +#else +typedef void (*loop_op_t)(uint8_t *s, int p, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh, + int count); +typedef void (*dual_loop_op_t)(uint8_t *s, int p, const uint8_t *blimit0, + const uint8_t *limit0, const uint8_t *thresh0, + const uint8_t *blimit1, const uint8_t *limit1, + const uint8_t *thresh1); +#endif // CONFIG_VP9_HIGHBITDEPTH + +typedef std::tr1::tuple loop8_param_t; +typedef std::tr1::tuple dualloop8_param_t; + +#if HAVE_SSE2 +#if CONFIG_VP9_HIGHBITDEPTH +void wrapper_vertical_16_sse2(uint16_t *s, int p, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh, + int count, int bd) { + vp9_highbd_lpf_vertical_16_sse2(s, p, blimit, limit, thresh, bd); +} + +void wrapper_vertical_16_c(uint16_t *s, int p, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh, + int count, int bd) { + vp9_highbd_lpf_vertical_16_c(s, p, blimit, limit, thresh, bd); +} + +void wrapper_vertical_16_dual_sse2(uint16_t *s, int p, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh, + int count, int bd) { + vp9_highbd_lpf_vertical_16_dual_sse2(s, p, blimit, limit, thresh, bd); +} + +void wrapper_vertical_16_dual_c(uint16_t *s, int p, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh, + int count, int bd) { + vp9_highbd_lpf_vertical_16_dual_c(s, p, blimit, limit, thresh, bd); +} +#else +void wrapper_vertical_16_sse2(uint8_t *s, int p, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh, + int count) { + vp9_lpf_vertical_16_sse2(s, p, blimit, limit, thresh); +} + +void wrapper_vertical_16_c(uint8_t *s, int p, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh, + int count) { + vp9_lpf_vertical_16_c(s, p, blimit, limit, thresh); +} + +void wrapper_vertical_16_dual_sse2(uint8_t *s, int p, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh, + int count) { + vp9_lpf_vertical_16_dual_sse2(s, p, blimit, limit, thresh); +} + +void wrapper_vertical_16_dual_c(uint8_t *s, int p, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh, + int count) { + vp9_lpf_vertical_16_dual_c(s, p, blimit, limit, thresh); +} +#endif // CONFIG_VP9_HIGHBITDEPTH +#endif // HAVE_SSE2 + +class Loop8Test6Param : public ::testing::TestWithParam { + public: + virtual ~Loop8Test6Param() {} + virtual void SetUp() { + loopfilter_op_ = GET_PARAM(0); + ref_loopfilter_op_ = GET_PARAM(1); + bit_depth_ = GET_PARAM(2); + mask_ = (1 << bit_depth_) - 1; + } + + virtual void TearDown() { libvpx_test::ClearSystemState(); } + + protected: + int bit_depth_; + int mask_; + loop_op_t loopfilter_op_; + loop_op_t ref_loopfilter_op_; +}; + +class Loop8Test9Param : public ::testing::TestWithParam { + public: + virtual ~Loop8Test9Param() {} + virtual void SetUp() { + loopfilter_op_ = GET_PARAM(0); + ref_loopfilter_op_ = GET_PARAM(1); + bit_depth_ = GET_PARAM(2); + mask_ = (1 << bit_depth_) - 1; + } + + virtual void TearDown() { libvpx_test::ClearSystemState(); } + + protected: + int bit_depth_; + int mask_; + dual_loop_op_t loopfilter_op_; + dual_loop_op_t ref_loopfilter_op_; +}; + +TEST_P(Loop8Test6Param, OperationCheck) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + const int count_test_block = number_of_iterations; +#if CONFIG_VP9_HIGHBITDEPTH + int32_t bd = bit_depth_; + DECLARE_ALIGNED_ARRAY(16, uint16_t, s, kNumCoeffs); + DECLARE_ALIGNED_ARRAY(16, uint16_t, ref_s, kNumCoeffs); +#else + DECLARE_ALIGNED_ARRAY(8, uint8_t, s, kNumCoeffs); + DECLARE_ALIGNED_ARRAY(8, uint8_t, ref_s, kNumCoeffs); +#endif // CONFIG_VP9_HIGHBITDEPTH + int err_count_total = 0; + int first_failure = -1; + for (int i = 0; i < count_test_block; ++i) { + int err_count = 0; + uint8_t tmp = rnd.Rand8(); + DECLARE_ALIGNED(16, const uint8_t, blimit[16]) = { + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp + }; + tmp = rnd.Rand8(); + DECLARE_ALIGNED(16, const uint8_t, limit[16]) = { + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp + }; + tmp = rnd.Rand8(); + DECLARE_ALIGNED(16, const uint8_t, thresh[16]) = { + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp + }; + int32_t p = kNumCoeffs/32; + int count = 1; + + uint16_t tmp_s[kNumCoeffs]; + int j = 0; + while (j < kNumCoeffs) { + uint8_t val = rnd.Rand8(); + if (val & 0x80) { // 50% chance to choose a new value. + tmp_s[j] = rnd.Rand16(); + j++; + } else { // 50% chance to repeat previous value in row X times + int k = 0; + while (k++ < ((val & 0x1f) + 1) && j < kNumCoeffs) { + if (j < 1) { + tmp_s[j] = rnd.Rand16(); + } else if (val & 0x20) { // Increment by an value within the limit + tmp_s[j] = (tmp_s[j - 1] + (*limit - 1)); + } else { // Decrement by an value within the limit + tmp_s[j] = (tmp_s[j - 1] - (*limit - 1)); + } + j++; + } + } + } + for (j = 0; j < kNumCoeffs; j++) { + if (i % 2) { + s[j] = tmp_s[j] & mask_; + } else { + s[j] = tmp_s[p * (j % p) + j / p] & mask_; + } + ref_s[j] = s[j]; + } +#if CONFIG_VP9_HIGHBITDEPTH + ref_loopfilter_op_(ref_s + 8 + p * 8, p, blimit, limit, thresh, count, bd); + ASM_REGISTER_STATE_CHECK( + loopfilter_op_(s + 8 + p * 8, p, blimit, limit, thresh, count, bd)); +#else + ref_loopfilter_op_(ref_s+8+p*8, p, blimit, limit, thresh, count); + ASM_REGISTER_STATE_CHECK( + loopfilter_op_(s + 8 + p * 8, p, blimit, limit, thresh, count)); +#endif // CONFIG_VP9_HIGHBITDEPTH + + for (int j = 0; j < kNumCoeffs; ++j) { + err_count += ref_s[j] != s[j]; + } + if (err_count && !err_count_total) { + first_failure = i; + } + err_count_total += err_count; + } + EXPECT_EQ(0, err_count_total) + << "Error: Loop8Test6Param, C output doesn't match SSE2 " + "loopfilter output. " + << "First failed at test case " << first_failure; +} + +TEST_P(Loop8Test6Param, ValueCheck) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + const int count_test_block = number_of_iterations; +#if CONFIG_VP9_HIGHBITDEPTH + const int32_t bd = bit_depth_; + DECLARE_ALIGNED_ARRAY(16, uint16_t, s, kNumCoeffs); + DECLARE_ALIGNED_ARRAY(16, uint16_t, ref_s, kNumCoeffs); +#else + DECLARE_ALIGNED_ARRAY(8, uint8_t, s, kNumCoeffs); + DECLARE_ALIGNED_ARRAY(8, uint8_t, ref_s, kNumCoeffs); +#endif // CONFIG_VP9_HIGHBITDEPTH + int err_count_total = 0; + int first_failure = -1; + for (int i = 0; i < count_test_block; ++i) { + int err_count = 0; + uint8_t tmp = rnd.Rand8(); + DECLARE_ALIGNED(16, const uint8_t, blimit[16]) = { + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp + }; + tmp = rnd.Rand8(); + DECLARE_ALIGNED(16, const uint8_t, limit[16]) = { + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp + }; + tmp = rnd.Rand8(); + DECLARE_ALIGNED(16, const uint8_t, thresh[16]) = { + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp + }; + int32_t p = kNumCoeffs / 32; + int count = 1; + for (int j = 0; j < kNumCoeffs; ++j) { + s[j] = rnd.Rand16() & mask_; + ref_s[j] = s[j]; + } +#if CONFIG_VP9_HIGHBITDEPTH + ref_loopfilter_op_(ref_s + 8 + p * 8, p, blimit, limit, thresh, count, bd); + ASM_REGISTER_STATE_CHECK( + loopfilter_op_(s + 8 + p * 8, p, blimit, limit, thresh, count, bd)); +#else + ref_loopfilter_op_(ref_s+8+p*8, p, blimit, limit, thresh, count); + ASM_REGISTER_STATE_CHECK( + loopfilter_op_(s + 8 + p * 8, p, blimit, limit, thresh, count)); +#endif // CONFIG_VP9_HIGHBITDEPTH + for (int j = 0; j < kNumCoeffs; ++j) { + err_count += ref_s[j] != s[j]; + } + if (err_count && !err_count_total) { + first_failure = i; + } + err_count_total += err_count; + } + EXPECT_EQ(0, err_count_total) + << "Error: Loop8Test6Param, C output doesn't match SSE2 " + "loopfilter output. " + << "First failed at test case " << first_failure; +} + +TEST_P(Loop8Test9Param, OperationCheck) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + const int count_test_block = number_of_iterations; +#if CONFIG_VP9_HIGHBITDEPTH + const int32_t bd = bit_depth_; + DECLARE_ALIGNED_ARRAY(16, uint16_t, s, kNumCoeffs); + DECLARE_ALIGNED_ARRAY(16, uint16_t, ref_s, kNumCoeffs); +#else + DECLARE_ALIGNED_ARRAY(8, uint8_t, s, kNumCoeffs); + DECLARE_ALIGNED_ARRAY(8, uint8_t, ref_s, kNumCoeffs); +#endif // CONFIG_VP9_HIGHBITDEPTH + int err_count_total = 0; + int first_failure = -1; + for (int i = 0; i < count_test_block; ++i) { + int err_count = 0; + uint8_t tmp = rnd.Rand8(); + DECLARE_ALIGNED(16, const uint8_t, blimit0[16]) = { + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp + }; + tmp = rnd.Rand8(); + DECLARE_ALIGNED(16, const uint8_t, limit0[16]) = { + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp + }; + tmp = rnd.Rand8(); + DECLARE_ALIGNED(16, const uint8_t, thresh0[16]) = { + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp + }; + tmp = rnd.Rand8(); + DECLARE_ALIGNED(16, const uint8_t, blimit1[16]) = { + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp + }; + tmp = rnd.Rand8(); + DECLARE_ALIGNED(16, const uint8_t, limit1[16]) = { + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp + }; + tmp = rnd.Rand8(); + DECLARE_ALIGNED(16, const uint8_t, thresh1[16]) = { + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp + }; + int32_t p = kNumCoeffs / 32; + uint16_t tmp_s[kNumCoeffs]; + int j = 0; + const uint8_t limit = *limit0 < *limit1 ? *limit0 : *limit1; + while (j < kNumCoeffs) { + uint8_t val = rnd.Rand8(); + if (val & 0x80) { // 50% chance to choose a new value. + tmp_s[j] = rnd.Rand16(); + j++; + } else { // 50% chance to repeat previous value in row X times. + int k = 0; + while (k++ < ((val & 0x1f) + 1) && j < kNumCoeffs) { + if (j < 1) { + tmp_s[j] = rnd.Rand16(); + } else if (val & 0x20) { // Increment by a value within the limit. + tmp_s[j] = (tmp_s[j - 1] + (limit - 1)); + } else { // Decrement by an value within the limit. + tmp_s[j] = (tmp_s[j - 1] - (limit - 1)); + } + j++; + } + } + } + for (j = 0; j < kNumCoeffs; j++) { + if (i % 2) { + s[j] = tmp_s[j] & mask_; + } else { + s[j] = tmp_s[p * (j % p) + j / p] & mask_; + } + ref_s[j] = s[j]; + } +#if CONFIG_VP9_HIGHBITDEPTH + ref_loopfilter_op_(ref_s + 8 + p * 8, p, blimit0, limit0, thresh0, + blimit1, limit1, thresh1, bd); + ASM_REGISTER_STATE_CHECK( + loopfilter_op_(s + 8 + p * 8, p, blimit0, limit0, thresh0, + blimit1, limit1, thresh1, bd)); +#else + ref_loopfilter_op_(ref_s + 8 + p * 8, p, blimit0, limit0, thresh0, + blimit1, limit1, thresh1); + ASM_REGISTER_STATE_CHECK( + loopfilter_op_(s + 8 + p * 8, p, blimit0, limit0, thresh0, + blimit1, limit1, thresh1)); +#endif // CONFIG_VP9_HIGHBITDEPTH + for (int j = 0; j < kNumCoeffs; ++j) { + err_count += ref_s[j] != s[j]; + } + if (err_count && !err_count_total) { + first_failure = i; + } + err_count_total += err_count; + } + EXPECT_EQ(0, err_count_total) + << "Error: Loop8Test9Param, C output doesn't match SSE2 " + "loopfilter output. " + << "First failed at test case " << first_failure; +} + +TEST_P(Loop8Test9Param, ValueCheck) { + ACMRandom rnd(ACMRandom::DeterministicSeed()); + const int count_test_block = number_of_iterations; +#if CONFIG_VP9_HIGHBITDEPTH + DECLARE_ALIGNED_ARRAY(16, uint16_t, s, kNumCoeffs); + DECLARE_ALIGNED_ARRAY(16, uint16_t, ref_s, kNumCoeffs); +#else + DECLARE_ALIGNED_ARRAY(8, uint8_t, s, kNumCoeffs); + DECLARE_ALIGNED_ARRAY(8, uint8_t, ref_s, kNumCoeffs); +#endif // CONFIG_VP9_HIGHBITDEPTH + int err_count_total = 0; + int first_failure = -1; + for (int i = 0; i < count_test_block; ++i) { + int err_count = 0; + uint8_t tmp = rnd.Rand8(); + DECLARE_ALIGNED(16, const uint8_t, blimit0[16]) = { + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp + }; + tmp = rnd.Rand8(); + DECLARE_ALIGNED(16, const uint8_t, limit0[16]) = { + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp + }; + tmp = rnd.Rand8(); + DECLARE_ALIGNED(16, const uint8_t, thresh0[16]) = { + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp + }; + tmp = rnd.Rand8(); + DECLARE_ALIGNED(16, const uint8_t, blimit1[16]) = { + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp + }; + tmp = rnd.Rand8(); + DECLARE_ALIGNED(16, const uint8_t, limit1[16]) = { + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp + }; + tmp = rnd.Rand8(); + DECLARE_ALIGNED(16, const uint8_t, thresh1[16]) = { + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp, + tmp, tmp, tmp, tmp, tmp, tmp, tmp, tmp + }; + int32_t p = kNumCoeffs / 32; // TODO(pdlf) can we have non-square here? + for (int j = 0; j < kNumCoeffs; ++j) { + s[j] = rnd.Rand16() & mask_; + ref_s[j] = s[j]; + } +#if CONFIG_VP9_HIGHBITDEPTH + const int32_t bd = bit_depth_; + ref_loopfilter_op_(ref_s + 8 + p * 8, p, blimit0, limit0, thresh0, + blimit1, limit1, thresh1, bd); + ASM_REGISTER_STATE_CHECK( + loopfilter_op_(s + 8 + p * 8, p, blimit0, limit0, + thresh0, blimit1, limit1, thresh1, bd)); +#else + ref_loopfilter_op_(ref_s + 8 + p * 8, p, blimit0, limit0, thresh0, + blimit1, limit1, thresh1); + ASM_REGISTER_STATE_CHECK( + loopfilter_op_(s + 8 + p * 8, p, blimit0, limit0, thresh0, + blimit1, limit1, thresh1)); +#endif // CONFIG_VP9_HIGHBITDEPTH + for (int j = 0; j < kNumCoeffs; ++j) { + err_count += ref_s[j] != s[j]; + } + if (err_count && !err_count_total) { + first_failure = i; + } + err_count_total += err_count; + } + EXPECT_EQ(0, err_count_total) + << "Error: Loop8Test9Param, C output doesn't match SSE2" + "loopfilter output. " + << "First failed at test case " << first_failure; +} + +using std::tr1::make_tuple; + +#if HAVE_SSE2 +#if CONFIG_VP9_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P( + SSE2_C_COMPARE_SINGLE, Loop8Test6Param, + ::testing::Values( + make_tuple(&vp9_highbd_lpf_horizontal_4_sse2, + &vp9_highbd_lpf_horizontal_4_c, 8), + make_tuple(&vp9_highbd_lpf_vertical_4_sse2, + &vp9_highbd_lpf_vertical_4_c, 8), + make_tuple(&vp9_highbd_lpf_horizontal_8_sse2, + &vp9_highbd_lpf_horizontal_8_c, 8), + make_tuple(&vp9_highbd_lpf_horizontal_16_sse2, + &vp9_highbd_lpf_horizontal_16_c, 8), + make_tuple(&vp9_highbd_lpf_vertical_8_sse2, + &vp9_highbd_lpf_vertical_8_c, 8), + make_tuple(&wrapper_vertical_16_sse2, + &wrapper_vertical_16_c, 8), + make_tuple(&vp9_highbd_lpf_horizontal_4_sse2, + &vp9_highbd_lpf_horizontal_4_c, 10), + make_tuple(&vp9_highbd_lpf_vertical_4_sse2, + &vp9_highbd_lpf_vertical_4_c, 10), + make_tuple(&vp9_highbd_lpf_horizontal_8_sse2, + &vp9_highbd_lpf_horizontal_8_c, 10), + make_tuple(&vp9_highbd_lpf_horizontal_16_sse2, + &vp9_highbd_lpf_horizontal_16_c, 10), + make_tuple(&vp9_highbd_lpf_vertical_8_sse2, + &vp9_highbd_lpf_vertical_8_c, 10), + make_tuple(&wrapper_vertical_16_sse2, + &wrapper_vertical_16_c, 10), + make_tuple(&vp9_highbd_lpf_horizontal_4_sse2, + &vp9_highbd_lpf_horizontal_4_c, 12), + make_tuple(&vp9_highbd_lpf_vertical_4_sse2, + &vp9_highbd_lpf_vertical_4_c, 12), + make_tuple(&vp9_highbd_lpf_horizontal_8_sse2, + &vp9_highbd_lpf_horizontal_8_c, 12), + make_tuple(&vp9_highbd_lpf_horizontal_16_sse2, + &vp9_highbd_lpf_horizontal_16_c, 12), + make_tuple(&vp9_highbd_lpf_vertical_8_sse2, + &vp9_highbd_lpf_vertical_8_c, 12), + make_tuple(&wrapper_vertical_16_sse2, + &wrapper_vertical_16_c, 12))); +#else +INSTANTIATE_TEST_CASE_P( + SSE2_C_COMPARE_SINGLE, Loop8Test6Param, + ::testing::Values( + make_tuple(&vp9_lpf_horizontal_8_sse2, &vp9_lpf_horizontal_8_c, 8), + make_tuple(&vp9_lpf_horizontal_16_sse2, &vp9_lpf_horizontal_16_c, 8), + make_tuple(&vp9_lpf_vertical_8_sse2, &vp9_lpf_vertical_8_c, 8))); +#endif // CONFIG_VP9_HIGHBITDEPTH +#endif + +#if HAVE_SSE2 +#if CONFIG_VP9_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P( + SSE2_C_COMPARE_DUAL, Loop8Test6Param, + ::testing::Values( + make_tuple(&wrapper_vertical_16_dual_sse2, + &wrapper_vertical_16_dual_c, 8), + make_tuple(&wrapper_vertical_16_dual_sse2, + &wrapper_vertical_16_dual_c, 10), + make_tuple(&wrapper_vertical_16_dual_sse2, + &wrapper_vertical_16_dual_c, 12))); +#else +INSTANTIATE_TEST_CASE_P( + SSE2_C_COMPARE_DUAL, Loop8Test6Param, + ::testing::Values( + make_tuple(&wrapper_vertical_16_sse2, &wrapper_vertical_16_c, 8))); +#endif // CONFIG_VP9_HIGHBITDEPTH +#endif // HAVE_SSE2 + +#if HAVE_SSE2 +#if CONFIG_VP9_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P( + SSE_C_COMPARE_DUAL, Loop8Test9Param, + ::testing::Values( + make_tuple(&vp9_highbd_lpf_horizontal_4_dual_sse2, + &vp9_highbd_lpf_horizontal_4_dual_c, 8), + make_tuple(&vp9_highbd_lpf_horizontal_8_dual_sse2, + &vp9_highbd_lpf_horizontal_8_dual_c, 8), + make_tuple(&vp9_highbd_lpf_vertical_4_dual_sse2, + &vp9_highbd_lpf_vertical_4_dual_c, 8), + make_tuple(&vp9_highbd_lpf_vertical_8_dual_sse2, + &vp9_highbd_lpf_vertical_8_dual_c, 8), + make_tuple(&vp9_highbd_lpf_horizontal_4_dual_sse2, + &vp9_highbd_lpf_horizontal_4_dual_c, 10), + make_tuple(&vp9_highbd_lpf_horizontal_8_dual_sse2, + &vp9_highbd_lpf_horizontal_8_dual_c, 10), + make_tuple(&vp9_highbd_lpf_vertical_4_dual_sse2, + &vp9_highbd_lpf_vertical_4_dual_c, 10), + make_tuple(&vp9_highbd_lpf_vertical_8_dual_sse2, + &vp9_highbd_lpf_vertical_8_dual_c, 10), + make_tuple(&vp9_highbd_lpf_horizontal_4_dual_sse2, + &vp9_highbd_lpf_horizontal_4_dual_c, 12), + make_tuple(&vp9_highbd_lpf_horizontal_8_dual_sse2, + &vp9_highbd_lpf_horizontal_8_dual_c, 12), + make_tuple(&vp9_highbd_lpf_vertical_4_dual_sse2, + &vp9_highbd_lpf_vertical_4_dual_c, 12), + make_tuple(&vp9_highbd_lpf_vertical_8_dual_sse2, + &vp9_highbd_lpf_vertical_8_dual_c, 12))); +#else +INSTANTIATE_TEST_CASE_P( + SSE_C_COMPARE_DUAL, Loop8Test9Param, + ::testing::Values( + make_tuple(&vp9_lpf_horizontal_4_dual_sse2, + &vp9_lpf_horizontal_4_dual_c, 8), + make_tuple(&vp9_lpf_horizontal_8_dual_sse2, + &vp9_lpf_horizontal_8_dual_c, 8), + make_tuple(&vp9_lpf_vertical_4_dual_sse2, + &vp9_lpf_vertical_4_dual_c, 8), + make_tuple(&vp9_lpf_vertical_8_dual_sse2, + &vp9_lpf_vertical_8_dual_c, 8))); +#endif // CONFIG_VP9_HIGHBITDEPTH +#endif +} // namespace diff --git a/test/test.mk b/test/test.mk index abf815cc9..c315b45f0 100644 --- a/test/test.mk +++ b/test/test.mk @@ -128,6 +128,7 @@ LIBVPX_TEST_SRCS-$(CONFIG_VP9_ENCODER) += fdct4x4_test.cc LIBVPX_TEST_SRCS-$(CONFIG_VP9_ENCODER) += fdct8x8_test.cc LIBVPX_TEST_SRCS-$(CONFIG_VP9_ENCODER) += variance_test.cc LIBVPX_TEST_SRCS-$(CONFIG_VP9_ENCODER) += vp9_subtract_test.cc +LIBVPX_TEST_SRCS-$(CONFIG_VP9_ENCODER) += lpf_8_test.cc LIBVPX_TEST_SRCS-$(CONFIG_VP9) += vp9_intrapred_test.cc ifeq ($(CONFIG_VP9_ENCODER),yes) diff --git a/vp9/common/vp9_loopfilter.c b/vp9/common/vp9_loopfilter.c index 4d03c4dcb..a23cefab4 100644 --- a/vp9/common/vp9_loopfilter.c +++ b/vp9/common/vp9_loopfilter.c @@ -392,6 +392,107 @@ static void filter_selectively_vert_row2(PLANE_TYPE plane_type, } } +#if CONFIG_VP9_HIGHBITDEPTH +static void high_filter_selectively_vert_row2(PLANE_TYPE plane_type, + uint16_t *s, int pitch, + unsigned int mask_16x16_l, + unsigned int mask_8x8_l, + unsigned int mask_4x4_l, + unsigned int mask_4x4_int_l, + const loop_filter_info_n *lfi_n, + const uint8_t *lfl, int bd) { + const int mask_shift = plane_type ? 4 : 8; + const int mask_cutoff = plane_type ? 0xf : 0xff; + const int lfl_forward = plane_type ? 4 : 8; + + unsigned int mask_16x16_0 = mask_16x16_l & mask_cutoff; + unsigned int mask_8x8_0 = mask_8x8_l & mask_cutoff; + unsigned int mask_4x4_0 = mask_4x4_l & mask_cutoff; + unsigned int mask_4x4_int_0 = mask_4x4_int_l & mask_cutoff; + unsigned int mask_16x16_1 = (mask_16x16_l >> mask_shift) & mask_cutoff; + unsigned int mask_8x8_1 = (mask_8x8_l >> mask_shift) & mask_cutoff; + unsigned int mask_4x4_1 = (mask_4x4_l >> mask_shift) & mask_cutoff; + unsigned int mask_4x4_int_1 = (mask_4x4_int_l >> mask_shift) & mask_cutoff; + unsigned int mask; + + for (mask = mask_16x16_0 | mask_8x8_0 | mask_4x4_0 | mask_4x4_int_0 | + mask_16x16_1 | mask_8x8_1 | mask_4x4_1 | mask_4x4_int_1; + mask; mask >>= 1) { + const loop_filter_thresh *lfi0 = lfi_n->lfthr + *lfl; + const loop_filter_thresh *lfi1 = lfi_n->lfthr + *(lfl + lfl_forward); + + // TODO(yunqingwang): count in loopfilter functions should be removed. + if (mask & 1) { + if ((mask_16x16_0 | mask_16x16_1) & 1) { + if ((mask_16x16_0 & mask_16x16_1) & 1) { + vp9_highbd_lpf_vertical_16_dual(s, pitch, lfi0->mblim, lfi0->lim, + lfi0->hev_thr, bd); + } else if (mask_16x16_0 & 1) { + vp9_highbd_lpf_vertical_16(s, pitch, lfi0->mblim, lfi0->lim, + lfi0->hev_thr, bd); + } else { + vp9_highbd_lpf_vertical_16(s + 8 *pitch, pitch, lfi1->mblim, + lfi1->lim, lfi1->hev_thr, bd); + } + } + + if ((mask_8x8_0 | mask_8x8_1) & 1) { + if ((mask_8x8_0 & mask_8x8_1) & 1) { + vp9_highbd_lpf_vertical_8_dual(s, pitch, lfi0->mblim, lfi0->lim, + lfi0->hev_thr, lfi1->mblim, lfi1->lim, + lfi1->hev_thr, bd); + } else if (mask_8x8_0 & 1) { + vp9_highbd_lpf_vertical_8(s, pitch, lfi0->mblim, lfi0->lim, + lfi0->hev_thr, 1, bd); + } else { + vp9_highbd_lpf_vertical_8(s + 8 * pitch, pitch, lfi1->mblim, + lfi1->lim, lfi1->hev_thr, 1, bd); + } + } + + if ((mask_4x4_0 | mask_4x4_1) & 1) { + if ((mask_4x4_0 & mask_4x4_1) & 1) { + vp9_highbd_lpf_vertical_4_dual(s, pitch, lfi0->mblim, lfi0->lim, + lfi0->hev_thr, lfi1->mblim, lfi1->lim, + lfi1->hev_thr, bd); + } else if (mask_4x4_0 & 1) { + vp9_highbd_lpf_vertical_4(s, pitch, lfi0->mblim, lfi0->lim, + lfi0->hev_thr, 1, bd); + } else { + vp9_highbd_lpf_vertical_4(s + 8 * pitch, pitch, lfi1->mblim, + lfi1->lim, lfi1->hev_thr, 1, bd); + } + } + + if ((mask_4x4_int_0 | mask_4x4_int_1) & 1) { + if ((mask_4x4_int_0 & mask_4x4_int_1) & 1) { + vp9_highbd_lpf_vertical_4_dual(s + 4, pitch, lfi0->mblim, lfi0->lim, + lfi0->hev_thr, lfi1->mblim, lfi1->lim, + lfi1->hev_thr, bd); + } else if (mask_4x4_int_0 & 1) { + vp9_highbd_lpf_vertical_4(s + 4, pitch, lfi0->mblim, lfi0->lim, + lfi0->hev_thr, 1, bd); + } else { + vp9_highbd_lpf_vertical_4(s + 8 * pitch + 4, pitch, lfi1->mblim, + lfi1->lim, lfi1->hev_thr, 1, bd); + } + } + } + + s += 8; + lfl += 1; + mask_16x16_0 >>= 1; + mask_8x8_0 >>= 1; + mask_4x4_0 >>= 1; + mask_4x4_int_0 >>= 1; + mask_16x16_1 >>= 1; + mask_8x8_1 >>= 1; + mask_4x4_1 >>= 1; + mask_4x4_int_1 >>= 1; + } +} +#endif // CONFIG_VP9_HIGHBITDEPTH + static void filter_selectively_horiz(uint8_t *s, int pitch, unsigned int mask_16x16, unsigned int mask_8x8, @@ -419,7 +520,7 @@ static void filter_selectively_horiz(uint8_t *s, int pitch, } } else if (mask_8x8 & 1) { if ((mask_8x8 & 3) == 3) { - // Next block's thresholds + // Next block's thresholds. const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1); vp9_lpf_horizontal_8_dual(s, pitch, lfi->mblim, lfi->lim, @@ -448,7 +549,7 @@ static void filter_selectively_horiz(uint8_t *s, int pitch, } } else if (mask_4x4 & 1) { if ((mask_4x4 & 3) == 3) { - // Next block's thresholds + // Next block's thresholds. const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1); vp9_lpf_horizontal_4_dual(s, pitch, lfi->mblim, lfi->lim, @@ -488,6 +589,112 @@ static void filter_selectively_horiz(uint8_t *s, int pitch, } } +#if CONFIG_VP9_HIGHBITDEPTH +static void high_filter_selectively_horiz(uint16_t *s, int pitch, + unsigned int mask_16x16, + unsigned int mask_8x8, + unsigned int mask_4x4, + unsigned int mask_4x4_int, + const loop_filter_info_n *lfi_n, + const uint8_t *lfl, int bd) { + unsigned int mask; + int count; + + for (mask = mask_16x16 | mask_8x8 | mask_4x4 | mask_4x4_int; + mask; mask >>= count) { + const loop_filter_thresh *lfi = lfi_n->lfthr + *lfl; + + count = 1; + if (mask & 1) { + if (mask_16x16 & 1) { + if ((mask_16x16 & 3) == 3) { + vp9_highbd_lpf_horizontal_16(s, pitch, lfi->mblim, lfi->lim, + lfi->hev_thr, 2, bd); + count = 2; + } else { + vp9_highbd_lpf_horizontal_16(s, pitch, lfi->mblim, lfi->lim, + lfi->hev_thr, 1, bd); + } + } else if (mask_8x8 & 1) { + if ((mask_8x8 & 3) == 3) { + // Next block's thresholds. + const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1); + + vp9_highbd_lpf_horizontal_8_dual(s, pitch, lfi->mblim, lfi->lim, + lfi->hev_thr, lfin->mblim, lfin->lim, + lfin->hev_thr, bd); + + if ((mask_4x4_int & 3) == 3) { + vp9_highbd_lpf_horizontal_4_dual(s + 4 * pitch, pitch, lfi->mblim, + lfi->lim, lfi->hev_thr, + lfin->mblim, lfin->lim, + lfin->hev_thr, bd); + } else { + if (mask_4x4_int & 1) { + vp9_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, + lfi->lim, lfi->hev_thr, 1, bd); + } else if (mask_4x4_int & 2) { + vp9_highbd_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim, + lfin->lim, lfin->hev_thr, 1, bd); + } + } + count = 2; + } else { + vp9_highbd_lpf_horizontal_8(s, pitch, lfi->mblim, lfi->lim, + lfi->hev_thr, 1, bd); + + if (mask_4x4_int & 1) { + vp9_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, + lfi->lim, lfi->hev_thr, 1, bd); + } + } + } else if (mask_4x4 & 1) { + if ((mask_4x4 & 3) == 3) { + // Next block's thresholds. + const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1); + + vp9_highbd_lpf_horizontal_4_dual(s, pitch, lfi->mblim, lfi->lim, + lfi->hev_thr, lfin->mblim, lfin->lim, + lfin->hev_thr, bd); + if ((mask_4x4_int & 3) == 3) { + vp9_highbd_lpf_horizontal_4_dual(s + 4 * pitch, pitch, lfi->mblim, + lfi->lim, lfi->hev_thr, + lfin->mblim, lfin->lim, + lfin->hev_thr, bd); + } else { + if (mask_4x4_int & 1) { + vp9_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, + lfi->lim, lfi->hev_thr, 1, bd); + } else if (mask_4x4_int & 2) { + vp9_highbd_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim, + lfin->lim, lfin->hev_thr, 1, bd); + } + } + count = 2; + } else { + vp9_highbd_lpf_horizontal_4(s, pitch, lfi->mblim, lfi->lim, + lfi->hev_thr, 1, bd); + + if (mask_4x4_int & 1) { + vp9_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, + lfi->lim, lfi->hev_thr, 1, bd); + } + } + } else if (mask_4x4_int & 1) { + vp9_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim, + lfi->hev_thr, 1, bd); + } + } + s += 8 * count; + lfl += count; + mask_16x16 >>= count; + mask_8x8 >>= count; + mask_4x4 >>= count; + mask_4x4_int >>= count; + } +} +#endif // CONFIG_VP9_HIGHBITDEPTH + // This function ors into the current lfm structure, where to do loop // filters for the specific mi we are looking at. It uses information // including the block_size_type (32x16, 32x32, etc.), the transform size, @@ -903,6 +1110,45 @@ static void filter_selectively_vert(uint8_t *s, int pitch, } } +#if CONFIG_VP9_HIGHBITDEPTH +static void high_filter_selectively_vert(uint16_t *s, int pitch, + unsigned int mask_16x16, + unsigned int mask_8x8, + unsigned int mask_4x4, + unsigned int mask_4x4_int, + const loop_filter_info_n *lfi_n, + const uint8_t *lfl, int bd) { + unsigned int mask; + + for (mask = mask_16x16 | mask_8x8 | mask_4x4 | mask_4x4_int; + mask; mask >>= 1) { + const loop_filter_thresh *lfi = lfi_n->lfthr + *lfl; + + if (mask & 1) { + if (mask_16x16 & 1) { + vp9_highbd_lpf_vertical_16(s, pitch, lfi->mblim, lfi->lim, + lfi->hev_thr, bd); + } else if (mask_8x8 & 1) { + vp9_highbd_lpf_vertical_8(s, pitch, lfi->mblim, lfi->lim, + lfi->hev_thr, 1, bd); + } else if (mask_4x4 & 1) { + vp9_highbd_lpf_vertical_4(s, pitch, lfi->mblim, lfi->lim, + lfi->hev_thr, 1, bd); + } + } + if (mask_4x4_int & 1) + vp9_highbd_lpf_vertical_4(s + 4, pitch, lfi->mblim, lfi->lim, + lfi->hev_thr, 1, bd); + s += 8; + lfl += 1; + mask_16x16 >>= 1; + mask_8x8 >>= 1; + mask_4x4 >>= 1; + mask_4x4_int >>= 1; + } +} +#endif // CONFIG_VP9_HIGHBITDEPTH + static void filter_block_plane_non420(VP9_COMMON *cm, struct macroblockd_plane *plane, MODE_INFO **mi_8x8, @@ -1001,12 +1247,32 @@ static void filter_block_plane_non420(VP9_COMMON *cm, // Disable filtering on the leftmost column border_mask = ~(mi_col == 0); +#if CONFIG_VP9_HIGHBITDEPTH + if (cm->use_highbitdepth) { + high_filter_selectively_vert(CONVERT_TO_SHORTPTR(dst->buf), + dst->stride, + mask_16x16_c & border_mask, + mask_8x8_c & border_mask, + mask_4x4_c & border_mask, + mask_4x4_int[r], + &cm->lf_info, &lfl[r << 3], + (int)cm->bit_depth); + } else { + filter_selectively_vert(dst->buf, dst->stride, + mask_16x16_c & border_mask, + mask_8x8_c & border_mask, + mask_4x4_c & border_mask, + mask_4x4_int[r], + &cm->lf_info, &lfl[r << 3]); + } +#else filter_selectively_vert(dst->buf, dst->stride, mask_16x16_c & border_mask, mask_8x8_c & border_mask, mask_4x4_c & border_mask, mask_4x4_int[r], &cm->lf_info, &lfl[r << 3]); +#endif // CONFIG_VP9_HIGHBITDEPTH dst->buf += 8 * dst->stride; mi_8x8 += row_step_stride; } @@ -1030,13 +1296,32 @@ static void filter_block_plane_non420(VP9_COMMON *cm, mask_8x8_r = mask_8x8[r]; mask_4x4_r = mask_4x4[r]; } - +#if CONFIG_VP9_HIGHBITDEPTH + if (cm->use_highbitdepth) { + high_filter_selectively_horiz(CONVERT_TO_SHORTPTR(dst->buf), + dst->stride, + mask_16x16_r, + mask_8x8_r, + mask_4x4_r, + mask_4x4_int_r, + &cm->lf_info, &lfl[r << 3], + (int)cm->bit_depth); + } else { + filter_selectively_horiz(dst->buf, dst->stride, + mask_16x16_r, + mask_8x8_r, + mask_4x4_r, + mask_4x4_int_r, + &cm->lf_info, &lfl[r << 3]); + } +#else filter_selectively_horiz(dst->buf, dst->stride, mask_16x16_r, mask_8x8_r, mask_4x4_r, mask_4x4_int_r, &cm->lf_info, &lfl[r << 3]); +#endif // CONFIG_VP9_HIGHBITDEPTH dst->buf += 8 * dst->stride; } } @@ -1062,7 +1347,29 @@ void vp9_filter_block_plane(VP9_COMMON *const cm, unsigned int mask_4x4_l = mask_4x4 & 0xffff; unsigned int mask_4x4_int_l = mask_4x4_int & 0xffff; - // Disable filtering on the leftmost column + // Disable filtering on the leftmost column. +#if CONFIG_VP9_HIGHBITDEPTH + if (cm->use_highbitdepth) { + high_filter_selectively_vert_row2(plane->plane_type, + CONVERT_TO_SHORTPTR(dst->buf), + dst->stride, + mask_16x16_l, + mask_8x8_l, + mask_4x4_l, + mask_4x4_int_l, + &cm->lf_info, &lfm->lfl_y[r << 3], + (int)cm->bit_depth); + } else { + filter_selectively_vert_row2(plane->plane_type, + dst->buf, dst->stride, + mask_16x16_l, + mask_8x8_l, + mask_4x4_l, + mask_4x4_int_l, + &cm->lf_info, + &lfm->lfl_y[r << 3]); + } +#else filter_selectively_vert_row2(plane->plane_type, dst->buf, dst->stride, mask_16x16_l, @@ -1070,7 +1377,7 @@ void vp9_filter_block_plane(VP9_COMMON *const cm, mask_4x4_l, mask_4x4_int_l, &cm->lf_info, &lfm->lfl_y[r << 3]); - +#endif // CONFIG_VP9_HIGHBITDEPTH dst->buf += 16 * dst->stride; mask_16x16 >>= 16; mask_8x8 >>= 16; @@ -1100,12 +1407,35 @@ void vp9_filter_block_plane(VP9_COMMON *const cm, mask_4x4_r = mask_4x4 & 0xff; } +#if CONFIG_VP9_HIGHBITDEPTH + if (cm->use_highbitdepth) { + high_filter_selectively_horiz(CONVERT_TO_SHORTPTR(dst->buf), + dst->stride, + mask_16x16_r, + mask_8x8_r, + mask_4x4_r, + mask_4x4_int & 0xff, + &cm->lf_info, + &lfm->lfl_y[r << 3], + (int)cm->bit_depth); + } else { + filter_selectively_horiz(dst->buf, dst->stride, + mask_16x16_r, + mask_8x8_r, + mask_4x4_r, + mask_4x4_int & 0xff, + &cm->lf_info, + &lfm->lfl_y[r << 3]); + } +#else filter_selectively_horiz(dst->buf, dst->stride, mask_16x16_r, mask_8x8_r, mask_4x4_r, mask_4x4_int & 0xff, - &cm->lf_info, &lfm->lfl_y[r << 3]); + &cm->lf_info, + &lfm->lfl_y[r << 3]); +#endif // CONFIG_VP9_HIGHBITDEPTH dst->buf += 8 * dst->stride; mask_16x16 >>= 8; @@ -1135,14 +1465,39 @@ void vp9_filter_block_plane(VP9_COMMON *const cm, unsigned int mask_4x4_l = mask_4x4 & 0xff; unsigned int mask_4x4_int_l = mask_4x4_int & 0xff; - // Disable filtering on the leftmost column + // Disable filtering on the leftmost column. +#if CONFIG_VP9_HIGHBITDEPTH + if (cm->use_highbitdepth) { + high_filter_selectively_vert_row2(plane->plane_type, + CONVERT_TO_SHORTPTR(dst->buf), + dst->stride, + mask_16x16_l, + mask_8x8_l, + mask_4x4_l, + mask_4x4_int_l, + &cm->lf_info, + &lfm->lfl_uv[r << 1], + (int)cm->bit_depth); + } else { + filter_selectively_vert_row2(plane->plane_type, + dst->buf, dst->stride, + mask_16x16_l, + mask_8x8_l, + mask_4x4_l, + mask_4x4_int_l, + &cm->lf_info, + &lfm->lfl_uv[r << 1]); + } +#else filter_selectively_vert_row2(plane->plane_type, dst->buf, dst->stride, mask_16x16_l, mask_8x8_l, mask_4x4_l, mask_4x4_int_l, - &cm->lf_info, &lfm->lfl_uv[r << 1]); + &cm->lf_info, + &lfm->lfl_uv[r << 1]); +#endif // CONFIG_VP9_HIGHBITDEPTH dst->buf += 16 * dst->stride; mask_16x16 >>= 8; @@ -1177,12 +1532,35 @@ void vp9_filter_block_plane(VP9_COMMON *const cm, mask_4x4_r = mask_4x4 & 0xf; } +#if CONFIG_VP9_HIGHBITDEPTH + if (cm->use_highbitdepth) { + high_filter_selectively_horiz(CONVERT_TO_SHORTPTR(dst->buf), + dst->stride, + mask_16x16_r, + mask_8x8_r, + mask_4x4_r, + mask_4x4_int_r, + &cm->lf_info, + &lfm->lfl_uv[r << 1], + (int)cm->bit_depth); + } else { + filter_selectively_horiz(dst->buf, dst->stride, + mask_16x16_r, + mask_8x8_r, + mask_4x4_r, + mask_4x4_int_r, + &cm->lf_info, + &lfm->lfl_uv[r << 1]); + } +#else filter_selectively_horiz(dst->buf, dst->stride, mask_16x16_r, mask_8x8_r, mask_4x4_r, mask_4x4_int_r, - &cm->lf_info, &lfm->lfl_uv[r << 1]); + &cm->lf_info, + &lfm->lfl_uv[r << 1]); +#endif // CONFIG_VP9_HIGHBITDEPTH dst->buf += 8 * dst->stride; mask_16x16 >>= 4; diff --git a/vp9/common/vp9_loopfilter_filters.c b/vp9/common/vp9_loopfilter_filters.c index 25d3311b6..5af52c273 100644 --- a/vp9/common/vp9_loopfilter_filters.c +++ b/vp9/common/vp9_loopfilter_filters.c @@ -17,6 +17,20 @@ static INLINE int8_t signed_char_clamp(int t) { return (int8_t)clamp(t, -128, 127); } +#if CONFIG_VP9_HIGHBITDEPTH +static INLINE int16_t signed_char_clamp_high(int t, int bd) { + switch (bd) { + case 10: + return (int16_t)clamp(t, -128*4, 128*4-1); + case 12: + return (int16_t)clamp(t, -128*16, 128*16-1); + case 8: + default: + return (int16_t)clamp(t, -128, 128-1); + } +} +#endif + // should we apply any filter at all: 11111111 yes, 00000000 no static INLINE int8_t filter_mask(uint8_t limit, uint8_t blimit, uint8_t p3, uint8_t p2, @@ -337,3 +351,390 @@ void vp9_lpf_vertical_16_dual_c(uint8_t *s, int p, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh) { mb_lpf_vertical_edge_w(s, p, blimit, limit, thresh, 16); } + +#if CONFIG_VP9_HIGHBITDEPTH +// Should we apply any filter at all: 11111111 yes, 00000000 no ? +static INLINE int8_t high_filter_mask(uint8_t limit, uint8_t blimit, + uint16_t p3, uint16_t p2, + uint16_t p1, uint16_t p0, + uint16_t q0, uint16_t q1, + uint16_t q2, uint16_t q3, int bd) { + int8_t mask = 0; + int16_t limit16 = (uint16_t)limit << (bd - 8); + int16_t blimit16 = (uint16_t)blimit << (bd - 8); + mask |= (abs(p3 - p2) > limit16) * -1; + mask |= (abs(p2 - p1) > limit16) * -1; + mask |= (abs(p1 - p0) > limit16) * -1; + mask |= (abs(q1 - q0) > limit16) * -1; + mask |= (abs(q2 - q1) > limit16) * -1; + mask |= (abs(q3 - q2) > limit16) * -1; + mask |= (abs(p0 - q0) * 2 + abs(p1 - q1) / 2 > blimit16) * -1; + return ~mask; +} + +static INLINE int8_t high_flat_mask4(uint8_t thresh, + uint16_t p3, uint16_t p2, + uint16_t p1, uint16_t p0, + uint16_t q0, uint16_t q1, + uint16_t q2, uint16_t q3, int bd) { + int8_t mask = 0; + int16_t thresh16 = (uint16_t)thresh << (bd - 8); + mask |= (abs(p1 - p0) > thresh16) * -1; + mask |= (abs(q1 - q0) > thresh16) * -1; + mask |= (abs(p2 - p0) > thresh16) * -1; + mask |= (abs(q2 - q0) > thresh16) * -1; + mask |= (abs(p3 - p0) > thresh16) * -1; + mask |= (abs(q3 - q0) > thresh16) * -1; + return ~mask; +} + +static INLINE int8_t high_flat_mask5(uint8_t thresh, + uint16_t p4, uint16_t p3, + uint16_t p2, uint16_t p1, + uint16_t p0, uint16_t q0, + uint16_t q1, uint16_t q2, + uint16_t q3, uint16_t q4, int bd) { + int8_t mask = ~high_flat_mask4(thresh, p3, p2, p1, p0, q0, q1, q2, q3, bd); + int16_t thresh16 = (uint16_t)thresh << (bd - 8); + mask |= (abs(p4 - p0) > thresh16) * -1; + mask |= (abs(q4 - q0) > thresh16) * -1; + return ~mask; +} + +// Is there high edge variance internal edge: +// 11111111_11111111 yes, 00000000_00000000 no ? +static INLINE int16_t high_hev_mask(uint8_t thresh, uint16_t p1, uint16_t p0, + uint16_t q0, uint16_t q1, int bd) { + int16_t hev = 0; + int16_t thresh16 = (uint16_t)thresh << (bd - 8); + hev |= (abs(p1 - p0) > thresh16) * -1; + hev |= (abs(q1 - q0) > thresh16) * -1; + return hev; +} + +static INLINE void high_filter4(int8_t mask, uint8_t thresh, uint16_t *op1, + uint16_t *op0, uint16_t *oq0, uint16_t *oq1, + int bd) { + int16_t filter1, filter2; + // ^0x80 equivalent to subtracting 0x80 from the values to turn them + // into -128 to +127 instead of 0 to 255. + int shift = bd - 8; + const int16_t ps1 = (int16_t)*op1 - (0x80 << shift); + const int16_t ps0 = (int16_t)*op0 - (0x80 << shift); + const int16_t qs0 = (int16_t)*oq0 - (0x80 << shift); + const int16_t qs1 = (int16_t)*oq1 - (0x80 << shift); + const uint16_t hev = high_hev_mask(thresh, *op1, *op0, *oq0, *oq1, bd); + + // Add outer taps if we have high edge variance. + int16_t filter = signed_char_clamp_high(ps1 - qs1, bd) & hev; + + // Inner taps. + filter = signed_char_clamp_high(filter + 3 * (qs0 - ps0), bd) & mask; + + // Save bottom 3 bits so that we round one side +4 and the other +3 + // if it equals 4 we'll set to adjust by -1 to account for the fact + // we'd round 3 the other way. + filter1 = signed_char_clamp_high(filter + 4, bd) >> 3; + filter2 = signed_char_clamp_high(filter + 3, bd) >> 3; + + *oq0 = signed_char_clamp_high(qs0 - filter1, bd) + (0x80 << shift); + *op0 = signed_char_clamp_high(ps0 + filter2, bd) + (0x80 << shift); + + // Outer tap adjustments. + filter = ROUND_POWER_OF_TWO(filter1, 1) & ~hev; + + *oq1 = signed_char_clamp_high(qs1 - filter, bd) + (0x80 << shift); + *op1 = signed_char_clamp_high(ps1 + filter, bd) + (0x80 << shift); +} + +void vp9_highbd_lpf_horizontal_4_c(uint16_t *s, int p /* pitch */, + const uint8_t *blimit, const uint8_t *limit, + const uint8_t *thresh, int count, int bd) { + int i; + + // loop filter designed to work using chars so that we can make maximum use + // of 8 bit simd instructions. + for (i = 0; i < 8 * count; ++i) { + const uint16_t p3 = s[-4 * p]; + const uint16_t p2 = s[-3 * p]; + const uint16_t p1 = s[-2 * p]; + const uint16_t p0 = s[-p]; + const uint16_t q0 = s[0 * p]; + const uint16_t q1 = s[1 * p]; + const uint16_t q2 = s[2 * p]; + const uint16_t q3 = s[3 * p]; + const int8_t mask = high_filter_mask(*limit, *blimit, + p3, p2, p1, p0, q0, q1, q2, q3, bd); + high_filter4(mask, *thresh, s - 2 * p, s - 1 * p, s, s + 1 * p, bd); + ++s; + } +} + +void vp9_highbd_lpf_horizontal_4_dual_c(uint16_t *s, int p, + const uint8_t *blimit0, + const uint8_t *limit0, + const uint8_t *thresh0, + const uint8_t *blimit1, + const uint8_t *limit1, + const uint8_t *thresh1, + int bd) { + vp9_highbd_lpf_horizontal_4_c(s, p, blimit0, limit0, thresh0, 1, bd); + vp9_highbd_lpf_horizontal_4_c(s + 8, p, blimit1, limit1, thresh1, 1, bd); +} + +void vp9_highbd_lpf_vertical_4_c(uint16_t *s, int pitch, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh, + int count, int bd) { + int i; + + // loop filter designed to work using chars so that we can make maximum use + // of 8 bit simd instructions. + for (i = 0; i < 8 * count; ++i) { + const uint16_t p3 = s[-4], p2 = s[-3], p1 = s[-2], p0 = s[-1]; + const uint16_t q0 = s[0], q1 = s[1], q2 = s[2], q3 = s[3]; + const int8_t mask = high_filter_mask(*limit, *blimit, + p3, p2, p1, p0, q0, q1, q2, q3, bd); + high_filter4(mask, *thresh, s - 2, s - 1, s, s + 1, bd); + s += pitch; + } +} + +void vp9_highbd_lpf_vertical_4_dual_c(uint16_t *s, int pitch, + const uint8_t *blimit0, + const uint8_t *limit0, + const uint8_t *thresh0, + const uint8_t *blimit1, + const uint8_t *limit1, + const uint8_t *thresh1, + int bd) { + vp9_highbd_lpf_vertical_4_c(s, pitch, blimit0, limit0, thresh0, 1, bd); + vp9_highbd_lpf_vertical_4_c(s + 8 * pitch, pitch, blimit1, limit1, + thresh1, 1, bd); +} + +static INLINE void high_filter8(int8_t mask, uint8_t thresh, uint8_t flat, + uint16_t *op3, uint16_t *op2, + uint16_t *op1, uint16_t *op0, + uint16_t *oq0, uint16_t *oq1, + uint16_t *oq2, uint16_t *oq3, int bd) { + if (flat && mask) { + const uint16_t p3 = *op3, p2 = *op2, p1 = *op1, p0 = *op0; + const uint16_t q0 = *oq0, q1 = *oq1, q2 = *oq2, q3 = *oq3; + + // 7-tap filter [1, 1, 1, 2, 1, 1, 1] + *op2 = ROUND_POWER_OF_TWO(p3 + p3 + p3 + 2 * p2 + p1 + p0 + q0, 3); + *op1 = ROUND_POWER_OF_TWO(p3 + p3 + p2 + 2 * p1 + p0 + q0 + q1, 3); + *op0 = ROUND_POWER_OF_TWO(p3 + p2 + p1 + 2 * p0 + q0 + q1 + q2, 3); + *oq0 = ROUND_POWER_OF_TWO(p2 + p1 + p0 + 2 * q0 + q1 + q2 + q3, 3); + *oq1 = ROUND_POWER_OF_TWO(p1 + p0 + q0 + 2 * q1 + q2 + q3 + q3, 3); + *oq2 = ROUND_POWER_OF_TWO(p0 + q0 + q1 + 2 * q2 + q3 + q3 + q3, 3); + } else { + high_filter4(mask, thresh, op1, op0, oq0, oq1, bd); + } +} + +void vp9_highbd_lpf_horizontal_8_c(uint16_t *s, int p, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh, + int count, int bd) { + int i; + + // loop filter designed to work using chars so that we can make maximum use + // of 8 bit simd instructions. + for (i = 0; i < 8 * count; ++i) { + const uint16_t p3 = s[-4 * p], p2 = s[-3 * p], p1 = s[-2 * p], p0 = s[-p]; + const uint16_t q0 = s[0 * p], q1 = s[1 * p], q2 = s[2 * p], q3 = s[3 * p]; + + const int8_t mask = high_filter_mask(*limit, *blimit, + p3, p2, p1, p0, q0, q1, q2, q3, bd); + const int8_t flat = high_flat_mask4(1, p3, p2, p1, p0, q0, q1, q2, q3, bd); + high_filter8(mask, *thresh, flat, + s - 4 * p, s - 3 * p, s - 2 * p, s - 1 * p, + s, s + 1 * p, s + 2 * p, s + 3 * p, bd); + ++s; + } +} + +void vp9_highbd_lpf_horizontal_8_dual_c(uint16_t *s, int p, + const uint8_t *blimit0, + const uint8_t *limit0, + const uint8_t *thresh0, + const uint8_t *blimit1, + const uint8_t *limit1, + const uint8_t *thresh1, + int bd) { + vp9_highbd_lpf_horizontal_8_c(s, p, blimit0, limit0, thresh0, 1, bd); + vp9_highbd_lpf_horizontal_8_c(s + 8, p, blimit1, limit1, thresh1, 1, bd); +} + +void vp9_highbd_lpf_vertical_8_c(uint16_t *s, int pitch, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh, + int count, int bd) { + int i; + + for (i = 0; i < 8 * count; ++i) { + const uint16_t p3 = s[-4], p2 = s[-3], p1 = s[-2], p0 = s[-1]; + const uint16_t q0 = s[0], q1 = s[1], q2 = s[2], q3 = s[3]; + const int8_t mask = high_filter_mask(*limit, *blimit, + p3, p2, p1, p0, q0, q1, q2, q3, bd); + const int8_t flat = high_flat_mask4(1, p3, p2, p1, p0, q0, q1, q2, q3, bd); + high_filter8(mask, *thresh, flat, + s - 4, s - 3, s - 2, s - 1, + s, s + 1, s + 2, s + 3, + bd); + s += pitch; + } +} + +void vp9_highbd_lpf_vertical_8_dual_c(uint16_t *s, int pitch, + const uint8_t *blimit0, + const uint8_t *limit0, + const uint8_t *thresh0, + const uint8_t *blimit1, + const uint8_t *limit1, + const uint8_t *thresh1, + int bd) { + vp9_highbd_lpf_vertical_8_c(s, pitch, blimit0, limit0, thresh0, 1, bd); + vp9_highbd_lpf_vertical_8_c(s + 8 * pitch, pitch, blimit1, limit1, + thresh1, 1, bd); +} + +static INLINE void high_filter16(int8_t mask, uint8_t thresh, + uint8_t flat, uint8_t flat2, + uint16_t *op7, uint16_t *op6, + uint16_t *op5, uint16_t *op4, + uint16_t *op3, uint16_t *op2, + uint16_t *op1, uint16_t *op0, + uint16_t *oq0, uint16_t *oq1, + uint16_t *oq2, uint16_t *oq3, + uint16_t *oq4, uint16_t *oq5, + uint16_t *oq6, uint16_t *oq7, int bd) { + if (flat2 && flat && mask) { + const uint16_t p7 = *op7; + const uint16_t p6 = *op6; + const uint16_t p5 = *op5; + const uint16_t p4 = *op4; + const uint16_t p3 = *op3; + const uint16_t p2 = *op2; + const uint16_t p1 = *op1; + const uint16_t p0 = *op0; + const uint16_t q0 = *oq0; + const uint16_t q1 = *oq1; + const uint16_t q2 = *oq2; + const uint16_t q3 = *oq3; + const uint16_t q4 = *oq4; + const uint16_t q5 = *oq5; + const uint16_t q6 = *oq6; + const uint16_t q7 = *oq7; + + // 15-tap filter [1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1] + *op6 = ROUND_POWER_OF_TWO(p7 * 7 + p6 * 2 + p5 + p4 + p3 + p2 + p1 + p0 + + q0, 4); + *op5 = ROUND_POWER_OF_TWO(p7 * 6 + p6 + p5 * 2 + p4 + p3 + p2 + p1 + p0 + + q0 + q1, 4); + *op4 = ROUND_POWER_OF_TWO(p7 * 5 + p6 + p5 + p4 * 2 + p3 + p2 + p1 + p0 + + q0 + q1 + q2, 4); + *op3 = ROUND_POWER_OF_TWO(p7 * 4 + p6 + p5 + p4 + p3 * 2 + p2 + p1 + p0 + + q0 + q1 + q2 + q3, 4); + *op2 = ROUND_POWER_OF_TWO(p7 * 3 + p6 + p5 + p4 + p3 + p2 * 2 + p1 + p0 + + q0 + q1 + q2 + q3 + q4, 4); + *op1 = ROUND_POWER_OF_TWO(p7 * 2 + p6 + p5 + p4 + p3 + p2 + p1 * 2 + p0 + + q0 + q1 + q2 + q3 + q4 + q5, 4); + *op0 = ROUND_POWER_OF_TWO(p7 + p6 + p5 + p4 + p3 + p2 + p1 + p0 * 2 + + q0 + q1 + q2 + q3 + q4 + q5 + q6, 4); + *oq0 = ROUND_POWER_OF_TWO(p6 + p5 + p4 + p3 + p2 + p1 + p0 + + q0 * 2 + q1 + q2 + q3 + q4 + q5 + q6 + q7, 4); + *oq1 = ROUND_POWER_OF_TWO(p5 + p4 + p3 + p2 + p1 + p0 + + q0 + q1 * 2 + q2 + q3 + q4 + q5 + q6 + q7 * 2, 4); + *oq2 = ROUND_POWER_OF_TWO(p4 + p3 + p2 + p1 + p0 + + q0 + q1 + q2 * 2 + q3 + q4 + q5 + q6 + q7 * 3, 4); + *oq3 = ROUND_POWER_OF_TWO(p3 + p2 + p1 + p0 + + q0 + q1 + q2 + q3 * 2 + q4 + q5 + q6 + q7 * 4, 4); + *oq4 = ROUND_POWER_OF_TWO(p2 + p1 + p0 + + q0 + q1 + q2 + q3 + q4 * 2 + q5 + q6 + q7 * 5, 4); + *oq5 = ROUND_POWER_OF_TWO(p1 + p0 + + q0 + q1 + q2 + q3 + q4 + q5 * 2 + q6 + q7 * 6, 4); + *oq6 = ROUND_POWER_OF_TWO(p0 + + q0 + q1 + q2 + q3 + q4 + q5 + q6 * 2 + q7 * 7, 4); + } else { + high_filter8(mask, thresh, flat, op3, op2, op1, op0, oq0, oq1, oq2, oq3, + bd); + } +} + +void vp9_highbd_lpf_horizontal_16_c(uint16_t *s, int p, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh, + int count, int bd) { + int i; + + // loop filter designed to work using chars so that we can make maximum use + // of 8 bit simd instructions. + for (i = 0; i < 8 * count; ++i) { + const uint16_t p3 = s[-4 * p]; + const uint16_t p2 = s[-3 * p]; + const uint16_t p1 = s[-2 * p]; + const uint16_t p0 = s[-p]; + const uint16_t q0 = s[0 * p]; + const uint16_t q1 = s[1 * p]; + const uint16_t q2 = s[2 * p]; + const uint16_t q3 = s[3 * p]; + const int8_t mask = high_filter_mask(*limit, *blimit, + p3, p2, p1, p0, q0, q1, q2, q3, bd); + const int8_t flat = high_flat_mask4(1, p3, p2, p1, p0, q0, q1, q2, q3, bd); + const int8_t flat2 = high_flat_mask5( + 1, s[-8 * p], s[-7 * p], s[-6 * p], s[-5 * p], p0, + q0, s[4 * p], s[5 * p], s[6 * p], s[7 * p], bd); + + high_filter16(mask, *thresh, flat, flat2, + s - 8 * p, s - 7 * p, s - 6 * p, s - 5 * p, + s - 4 * p, s - 3 * p, s - 2 * p, s - 1 * p, + s, s + 1 * p, s + 2 * p, s + 3 * p, + s + 4 * p, s + 5 * p, s + 6 * p, s + 7 * p, + bd); + ++s; + } +} + +static void high_mb_lpf_vertical_edge_w(uint16_t *s, int p, + const uint8_t *blimit, + const uint8_t *limit, + const uint8_t *thresh, + int count, int bd) { + int i; + + for (i = 0; i < count; ++i) { + const uint16_t p3 = s[-4]; + const uint16_t p2 = s[-3]; + const uint16_t p1 = s[-2]; + const uint16_t p0 = s[-1]; + const uint16_t q0 = s[0]; + const uint16_t q1 = s[1]; + const uint16_t q2 = s[2]; + const uint16_t q3 = s[3]; + const int8_t mask = high_filter_mask(*limit, *blimit, + p3, p2, p1, p0, q0, q1, q2, q3, bd); + const int8_t flat = high_flat_mask4(1, p3, p2, p1, p0, q0, q1, q2, q3, bd); + const int8_t flat2 = high_flat_mask5(1, s[-8], s[-7], s[-6], s[-5], p0, + q0, s[4], s[5], s[6], s[7], bd); + + high_filter16(mask, *thresh, flat, flat2, + s - 8, s - 7, s - 6, s - 5, s - 4, s - 3, s - 2, s - 1, + s, s + 1, s + 2, s + 3, s + 4, s + 5, s + 6, s + 7, + bd); + s += p; + } +} + +void vp9_highbd_lpf_vertical_16_c(uint16_t *s, int p, const uint8_t *blimit, + const uint8_t *limit, const uint8_t *thresh, + int bd) { + high_mb_lpf_vertical_edge_w(s, p, blimit, limit, thresh, 8, bd); +} + +void vp9_highbd_lpf_vertical_16_dual_c(uint16_t *s, int p, + const uint8_t *blimit, + const uint8_t *limit, + const uint8_t *thresh, + int bd) { + high_mb_lpf_vertical_edge_w(s, p, blimit, limit, thresh, 16, bd); +} +#endif // CONFIG_VP9_HIGHBITDEPTH diff --git a/vp9/common/vp9_postproc.c b/vp9/common/vp9_postproc.c index e4e6ce782..abe71da40 100644 --- a/vp9/common/vp9_postproc.c +++ b/vp9/common/vp9_postproc.c @@ -19,6 +19,9 @@ #include "vpx_scale/vpx_scale.h" #include "vpx_scale/yv12config.h" +#if CONFIG_VP9_HIGHBITDEPTH +#include "vp9/common/vp9_common.h" +#endif #include "vp9/common/vp9_onyxc_int.h" #include "vp9/common/vp9_postproc.h" #include "vp9/common/vp9_systemdependent.h" @@ -152,6 +155,84 @@ void vp9_post_proc_down_and_across_c(const uint8_t *src_ptr, } } +#if CONFIG_VP9_HIGHBITDEPTH +void vp9_highbd_post_proc_down_and_across_c(const uint16_t *src_ptr, + uint16_t *dst_ptr, + int src_pixels_per_line, + int dst_pixels_per_line, + int rows, + int cols, + int flimit) { + uint16_t const *p_src; + uint16_t *p_dst; + int row; + int col; + int i; + int v; + int pitch = src_pixels_per_line; + uint16_t d[8]; + + for (row = 0; row < rows; row++) { + // post_proc_down for one row. + p_src = src_ptr; + p_dst = dst_ptr; + + for (col = 0; col < cols; col++) { + int kernel = 4; + int v = p_src[col]; + + for (i = -2; i <= 2; i++) { + if (abs(v - p_src[col + i * pitch]) > flimit) + goto down_skip_convolve; + + kernel += kernel5[2 + i] * p_src[col + i * pitch]; + } + + v = (kernel >> 3); + + down_skip_convolve: + p_dst[col] = v; + } + + /* now post_proc_across */ + p_src = dst_ptr; + p_dst = dst_ptr; + + for (i = 0; i < 8; i++) + d[i] = p_src[i]; + + for (col = 0; col < cols; col++) { + int kernel = 4; + v = p_src[col]; + + d[col & 7] = v; + + for (i = -2; i <= 2; i++) { + if (abs(v - p_src[col + i]) > flimit) + goto across_skip_convolve; + + kernel += kernel5[2 + i] * p_src[col + i]; + } + + d[col & 7] = (kernel >> 3); + + across_skip_convolve: + if (col >= 2) + p_dst[col - 2] = d[(col - 2) & 7]; + } + + /* handle the last two pixels */ + p_dst[col - 2] = d[(col - 2) & 7]; + p_dst[col - 1] = d[(col - 1) & 7]; + + + /* next row */ + src_ptr += pitch; + dst_ptr += dst_pixels_per_line; + } +} +#endif // CONFIG_VP9_HIGHBITDEPTH + static int q2mbl(int x) { if (x < 20) x = 20; @@ -162,10 +243,46 @@ static int q2mbl(int x) { void vp9_mbpost_proc_across_ip_c(uint8_t *src, int pitch, int rows, int cols, int flimit) { int r, c, i; - uint8_t *s = src; uint8_t d[16]; + for (r = 0; r < rows; r++) { + int sumsq = 0; + int sum = 0; + + for (i = -8; i <= 6; i++) { + sumsq += s[i] * s[i]; + sum += s[i]; + d[i + 8] = 0; + } + + for (c = 0; c < cols + 8; c++) { + int x = s[c + 7] - s[c - 8]; + int y = s[c + 7] + s[c - 8]; + + sum += x; + sumsq += x * y; + + d[c & 15] = s[c]; + + if (sumsq * 15 - sum * sum < flimit) { + d[c & 15] = (8 + sum + s[c]) >> 4; + } + + s[c - 8] = d[(c - 8) & 15]; + } + s += pitch; + } +} + +#if CONFIG_VP9_HIGHBITDEPTH +void vp9_highbd_mbpost_proc_across_ip_c(uint16_t *src, int pitch, + int rows, int cols, int flimit) { + int r, c, i; + + uint16_t *s = src; + uint16_t d[16]; + for (r = 0; r < rows; r++) { int sumsq = 0; @@ -196,6 +313,7 @@ void vp9_mbpost_proc_across_ip_c(uint8_t *src, int pitch, s += pitch; } } +#endif // CONFIG_VP9_HIGHBITDEPTH void vp9_mbpost_proc_down_c(uint8_t *dst, int pitch, int rows, int cols, int flimit) { @@ -229,6 +347,40 @@ void vp9_mbpost_proc_down_c(uint8_t *dst, int pitch, } } +#if CONFIG_VP9_HIGHBITDEPTH +void vp9_highbd_mbpost_proc_down_c(uint16_t *dst, int pitch, + int rows, int cols, int flimit) { + int r, c, i; + const int16_t *rv3 = &vp9_rv[63 & rand()]; // NOLINT + + for (c = 0; c < cols; c++) { + uint16_t *s = &dst[c]; + int sumsq = 0; + int sum = 0; + uint16_t d[16]; + const int16_t *rv2 = rv3 + ((c * 17) & 127); + + for (i = -8; i <= 6; i++) { + sumsq += s[i * pitch] * s[i * pitch]; + sum += s[i * pitch]; + } + + for (r = 0; r < rows + 8; r++) { + sumsq += s[7 * pitch] * s[ 7 * pitch] - s[-8 * pitch] * s[-8 * pitch]; + sum += s[7 * pitch] - s[-8 * pitch]; + d[r & 15] = s[0]; + + if (sumsq * 15 - sum * sum < flimit) { + d[r & 15] = (rv2[r & 127] + sum + s[0]) >> 4; + } + + s[-8 * pitch] = d[(r - 8) & 15]; + s += pitch; + } + } +} +#endif // CONFIG_VP9_HIGHBITDEPTH + static void deblock_and_de_macro_block(YV12_BUFFER_CONFIG *source, YV12_BUFFER_CONFIG *post, int q, @@ -239,6 +391,51 @@ static void deblock_and_de_macro_block(YV12_BUFFER_CONFIG *source, (void) low_var_thresh; (void) flag; +#if CONFIG_VP9_HIGHBITDEPTH + if (source->flags & YV12_FLAG_HIGHBITDEPTH) { + vp9_highbd_post_proc_down_and_across(CONVERT_TO_SHORTPTR(source->y_buffer), + CONVERT_TO_SHORTPTR(post->y_buffer), + source->y_stride, post->y_stride, + source->y_height, source->y_width, + ppl); + + vp9_highbd_mbpost_proc_across_ip(CONVERT_TO_SHORTPTR(post->y_buffer), + post->y_stride, post->y_height, + post->y_width, q2mbl(q)); + + vp9_highbd_mbpost_proc_down(CONVERT_TO_SHORTPTR(post->y_buffer), + post->y_stride, post->y_height, + post->y_width, q2mbl(q)); + + vp9_highbd_post_proc_down_and_across(CONVERT_TO_SHORTPTR(source->u_buffer), + CONVERT_TO_SHORTPTR(post->u_buffer), + source->uv_stride, post->uv_stride, + source->uv_height, source->uv_width, + ppl); + vp9_highbd_post_proc_down_and_across(CONVERT_TO_SHORTPTR(source->v_buffer), + CONVERT_TO_SHORTPTR(post->v_buffer), + source->uv_stride, post->uv_stride, + source->uv_height, source->uv_width, + ppl); + } else { + vp9_post_proc_down_and_across(source->y_buffer, post->y_buffer, + source->y_stride, post->y_stride, + source->y_height, source->y_width, ppl); + + vp9_mbpost_proc_across_ip(post->y_buffer, post->y_stride, post->y_height, + post->y_width, q2mbl(q)); + + vp9_mbpost_proc_down(post->y_buffer, post->y_stride, post->y_height, + post->y_width, q2mbl(q)); + + vp9_post_proc_down_and_across(source->u_buffer, post->u_buffer, + source->uv_stride, post->uv_stride, + source->uv_height, source->uv_width, ppl); + vp9_post_proc_down_and_across(source->v_buffer, post->v_buffer, + source->uv_stride, post->uv_stride, + source->uv_height, source->uv_width, ppl); + } +#else vp9_post_proc_down_and_across(source->y_buffer, post->y_buffer, source->y_stride, post->y_stride, source->y_height, source->y_width, ppl); @@ -255,6 +452,7 @@ static void deblock_and_de_macro_block(YV12_BUFFER_CONFIG *source, vp9_post_proc_down_and_across(source->v_buffer, post->v_buffer, source->uv_stride, post->uv_stride, source->uv_height, source->uv_width, ppl); +#endif // CONFIG_VP9_HIGHBITDEPTH } void vp9_deblock(const YV12_BUFFER_CONFIG *src, YV12_BUFFER_CONFIG *dst, @@ -272,9 +470,24 @@ void vp9_deblock(const YV12_BUFFER_CONFIG *src, YV12_BUFFER_CONFIG *dst, const int dst_strides[3] = {dst->y_stride, dst->uv_stride, dst->uv_stride}; for (i = 0; i < MAX_MB_PLANE; ++i) +#if CONFIG_VP9_HIGHBITDEPTH + assert((src->flags & YV12_FLAG_HIGHBITDEPTH) == + (dst->flags & YV12_FLAG_HIGHBITDEPTH)); + if (src->flags & YV12_FLAG_HIGHBITDEPTH) { + vp9_highbd_post_proc_down_and_across(CONVERT_TO_SHORTPTR(srcs[i]), + CONVERT_TO_SHORTPTR(dsts[i]), + src_strides[i], dst_strides[i], + src_heights[i], src_widths[i], ppl); + } else { + vp9_post_proc_down_and_across(srcs[i], dsts[i], + src_strides[i], dst_strides[i], + src_heights[i], src_widths[i], ppl); + } +#else vp9_post_proc_down_and_across(srcs[i], dsts[i], src_strides[i], dst_strides[i], src_heights[i], src_widths[i], ppl); +#endif // CONFIG_VP9_HIGHBITDEPTH } void vp9_denoise(const YV12_BUFFER_CONFIG *src, YV12_BUFFER_CONFIG *dst, @@ -293,15 +506,32 @@ void vp9_denoise(const YV12_BUFFER_CONFIG *src, YV12_BUFFER_CONFIG *dst, for (i = 0; i < MAX_MB_PLANE; ++i) { const int src_stride = src_strides[i]; - const uint8_t *const src = srcs[i] + 2 * src_stride + 2; const int src_width = src_widths[i] - 4; const int src_height = src_heights[i] - 4; - const int dst_stride = dst_strides[i]; - uint8_t *const dst = dsts[i] + 2 * dst_stride + 2; +#if CONFIG_VP9_HIGHBITDEPTH + assert((src->flags & YV12_FLAG_HIGHBITDEPTH) == + (dst->flags & YV12_FLAG_HIGHBITDEPTH)); + if (src->flags & YV12_FLAG_HIGHBITDEPTH) { + const uint16_t *const src = CONVERT_TO_SHORTPTR(srcs[i] + 2 * src_stride + + 2); + uint16_t *const dst = CONVERT_TO_SHORTPTR(dsts[i] + 2 * dst_stride + 2); + vp9_highbd_post_proc_down_and_across(src, dst, src_stride, dst_stride, + src_height, src_width, ppl); + } else { + const uint8_t *const src = srcs[i] + 2 * src_stride + 2; + uint8_t *const dst = dsts[i] + 2 * dst_stride + 2; + + vp9_post_proc_down_and_across(src, dst, src_stride, dst_stride, + src_height, src_width, ppl); + } +#else + const uint8_t *const src = srcs[i] + 2 * src_stride + 2; + uint8_t *const dst = dsts[i] + 2 * dst_stride + 2; vp9_post_proc_down_and_across(src, dst, src_stride, dst_stride, src_height, src_width, ppl); +#endif } } @@ -405,6 +635,9 @@ int vp9_post_proc_frame(struct VP9Common *cm, #if CONFIG_VP9_POSTPROC || CONFIG_INTERNAL_STATS if (vp9_realloc_frame_buffer(&cm->post_proc_buffer, cm->width, cm->height, cm->subsampling_x, cm->subsampling_y, +#if CONFIG_VP9_HIGHBITDEPTH + cm->use_highbitdepth, +#endif VP9_DEC_BORDER_IN_PIXELS, NULL, NULL, NULL) < 0) vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR, "Failed to allocate post-processing buffer"); diff --git a/vp9/common/vp9_rtcd_defs.pl b/vp9/common/vp9_rtcd_defs.pl index 0f52ae15c..0e95141aa 100644 --- a/vp9/common/vp9_rtcd_defs.pl +++ b/vp9/common/vp9_rtcd_defs.pl @@ -450,160 +450,160 @@ if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") { # # Intra prediction # - add_proto qw/void vp9_high_d207_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_d207_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_d207_predictor_4x4/; - add_proto qw/void vp9_high_d45_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_d45_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_d45_predictor_4x4/; - add_proto qw/void vp9_high_d63_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_d63_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_d63_predictor_4x4/; - add_proto qw/void vp9_high_h_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_h_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_h_predictor_4x4/; - add_proto qw/void vp9_high_d117_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_d117_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_d117_predictor_4x4/; - add_proto qw/void vp9_high_d135_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_d135_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_d135_predictor_4x4/; - add_proto qw/void vp9_high_d153_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_d153_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_d153_predictor_4x4/; - add_proto qw/void vp9_high_v_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_v_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_v_predictor_4x4 neon/, "$sse_x86inc"; - add_proto qw/void vp9_high_tm_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_tm_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_tm_predictor_4x4/, "$sse_x86inc"; - add_proto qw/void vp9_high_dc_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_dc_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_dc_predictor_4x4/, "$sse_x86inc"; - add_proto qw/void vp9_high_dc_top_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_dc_top_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_dc_top_predictor_4x4/; - add_proto qw/void vp9_high_dc_left_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_dc_left_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_dc_left_predictor_4x4/; - add_proto qw/void vp9_high_dc_128_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_dc_128_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_dc_128_predictor_4x4/; - add_proto qw/void vp9_high_d207_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_d207_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_d207_predictor_8x8/; - add_proto qw/void vp9_high_d45_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_d45_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_d45_predictor_8x8/; - add_proto qw/void vp9_high_d63_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_d63_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_d63_predictor_8x8/; - add_proto qw/void vp9_high_h_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_h_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_h_predictor_8x8/; - add_proto qw/void vp9_high_d117_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_d117_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_d117_predictor_8x8/; - add_proto qw/void vp9_high_d135_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_d135_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_d135_predictor_8x8/; - add_proto qw/void vp9_high_d153_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_d153_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_d153_predictor_8x8/; - add_proto qw/void vp9_high_v_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_v_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_v_predictor_8x8/, "$sse2_x86inc"; - add_proto qw/void vp9_high_tm_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_tm_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_tm_predictor_8x8/, "$sse2_x86inc"; - add_proto qw/void vp9_high_dc_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_dc_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_dc_predictor_8x8/, "$sse2_x86inc";; - add_proto qw/void vp9_high_dc_top_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_dc_top_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_dc_top_predictor_8x8/; - add_proto qw/void vp9_high_dc_left_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_dc_left_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_dc_left_predictor_8x8/; - add_proto qw/void vp9_high_dc_128_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_dc_128_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_dc_128_predictor_8x8/; - add_proto qw/void vp9_high_d207_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_d207_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_d207_predictor_16x16/; - add_proto qw/void vp9_high_d45_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_d45_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_d45_predictor_16x16/; - add_proto qw/void vp9_high_d63_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_d63_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_d63_predictor_16x16/; - add_proto qw/void vp9_high_h_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_h_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_h_predictor_16x16/; - add_proto qw/void vp9_high_d117_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_d117_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_d117_predictor_16x16/; - add_proto qw/void vp9_high_d135_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_d135_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_d135_predictor_16x16/; - add_proto qw/void vp9_high_d153_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_d153_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_d153_predictor_16x16/; - add_proto qw/void vp9_high_v_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_v_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_v_predictor_16x16 neon/, "$sse2_x86inc"; - add_proto qw/void vp9_high_tm_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_tm_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_tm_predictor_16x16/, "$sse2_x86_64"; - add_proto qw/void vp9_high_dc_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_dc_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_dc_predictor_16x16/, "$sse2_x86inc"; - add_proto qw/void vp9_high_dc_top_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_dc_top_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_dc_top_predictor_16x16/; - add_proto qw/void vp9_high_dc_left_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_dc_left_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_dc_left_predictor_16x16/; - add_proto qw/void vp9_high_dc_128_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_dc_128_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_dc_128_predictor_16x16/; - add_proto qw/void vp9_high_d207_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_d207_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_d207_predictor_32x32/; - add_proto qw/void vp9_high_d45_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_d45_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_d45_predictor_32x32/; - add_proto qw/void vp9_high_d63_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_d63_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_d63_predictor_32x32/; - add_proto qw/void vp9_high_h_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_h_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_h_predictor_32x32/; - add_proto qw/void vp9_high_d117_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_d117_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_d117_predictor_32x32/; - add_proto qw/void vp9_high_d135_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_d135_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_d135_predictor_32x32/; - add_proto qw/void vp9_high_d153_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_d153_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_d153_predictor_32x32/; - add_proto qw/void vp9_high_v_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_v_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_v_predictor_32x32/, "$sse2_x86inc"; - add_proto qw/void vp9_high_tm_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_tm_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_tm_predictor_32x32/, "$sse2_x86_64"; - add_proto qw/void vp9_high_dc_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_dc_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_dc_predictor_32x32/, "$sse2_x86_64"; - add_proto qw/void vp9_high_dc_top_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_dc_top_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_dc_top_predictor_32x32/; - add_proto qw/void vp9_high_dc_left_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_dc_left_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_dc_left_predictor_32x32/; - add_proto qw/void vp9_high_dc_128_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bps"; + add_proto qw/void vp9_high_dc_128_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; specialize qw/vp9_high_dc_128_predictor_32x32/; # @@ -633,6 +633,59 @@ if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") { add_proto qw/void vp9_high_convolve8_avg_vert/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, int bps"; specialize qw/vp9_high_convolve8_avg_vert/, "$sse2_x86_64"; + # + # Loopfilter + # + add_proto qw/void vp9_highbd_lpf_vertical_16/, "uint16_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh, int bd"; + specialize qw/vp9_highbd_lpf_vertical_16 sse2/; + + add_proto qw/void vp9_highbd_lpf_vertical_16_dual/, "uint16_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh, int bd"; + specialize qw/vp9_highbd_lpf_vertical_16_dual sse2/; + + add_proto qw/void vp9_highbd_lpf_vertical_8/, "uint16_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh, int count, int bd"; + specialize qw/vp9_highbd_lpf_vertical_8 sse2/; + + add_proto qw/void vp9_highbd_lpf_vertical_8_dual/, "uint16_t *s, int pitch, const uint8_t *blimit0, const uint8_t *limit0, const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, const uint8_t *thresh1, int bd"; + specialize qw/vp9_highbd_lpf_vertical_8_dual sse2/; + + add_proto qw/void vp9_highbd_lpf_vertical_4/, "uint16_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh, int count, int bd"; + specialize qw/vp9_highbd_lpf_vertical_4 sse2/; + + add_proto qw/void vp9_highbd_lpf_vertical_4_dual/, "uint16_t *s, int pitch, const uint8_t *blimit0, const uint8_t *limit0, const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, const uint8_t *thresh1, int bd"; + specialize qw/vp9_highbd_lpf_vertical_4_dual sse2/; + + add_proto qw/void vp9_highbd_lpf_horizontal_16/, "uint16_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh, int count, int bd"; + specialize qw/vp9_highbd_lpf_horizontal_16 sse2/; + + add_proto qw/void vp9_highbd_lpf_horizontal_8/, "uint16_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh, int count, int bd"; + specialize qw/vp9_highbd_lpf_horizontal_8 sse2/; + + add_proto qw/void vp9_highbd_lpf_horizontal_8_dual/, "uint16_t *s, int pitch, const uint8_t *blimit0, const uint8_t *limit0, const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, const uint8_t *thresh1, int bd"; + specialize qw/vp9_highbd_lpf_horizontal_8_dual sse2/; + + add_proto qw/void vp9_highbd_lpf_horizontal_4/, "uint16_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh, int count, int bd"; + specialize qw/vp9_highbd_lpf_horizontal_4 sse2/; + + add_proto qw/void vp9_highbd_lpf_horizontal_4_dual/, "uint16_t *s, int pitch, const uint8_t *blimit0, const uint8_t *limit0, const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, const uint8_t *thresh1, int bd"; + specialize qw/vp9_highbd_lpf_horizontal_4_dual sse2/; + + # + # post proc + # + if (vpx_config("CONFIG_VP9_POSTPROC") eq "yes") { + add_proto qw/void vp9_highbd_mbpost_proc_down/, "uint16_t *dst, int pitch, int rows, int cols, int flimit"; + specialize qw/vp9_highbd_mbpost_proc_down/; + + add_proto qw/void vp9_highbd_mbpost_proc_across_ip/, "uint16_t *src, int pitch, int rows, int cols, int flimit"; + specialize qw/vp9_highbd_mbpost_proc_across_ip/; + + add_proto qw/void vp9_highbd_post_proc_down_and_across/, "const uint16_t *src_ptr, uint16_t *dst_ptr, int src_pixels_per_line, int dst_pixels_per_line, int rows, int cols, int flimit"; + specialize qw/vp9_highbd_post_proc_down_and_across/; + + add_proto qw/void vp9_highbd_plane_add_noise/, "uint8_t *Start, char *noise, char blackclamp[16], char whiteclamp[16], char bothclamp[16], unsigned int Width, unsigned int Height, int Pitch"; + specialize qw/vp9_highbd_plane_add_noise/; + } + # # dct # @@ -1793,8 +1846,8 @@ if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") { add_proto qw/void vp9_high_fdct32x32_rd/, "const int16_t *input, tran_low_t *output, int stride"; specialize qw/vp9_high_fdct32x32_rd/; - add_proto qw/void vp9_high_temporal_filter_apply/, "uint8_t *frame1, unsigned int stride, uint8_t *frame2, unsigned int block_width, unsigned int block_height, int strength, int filter_weight, unsigned int *accumulator, uint16_t *count"; - specialize qw/vp9_high_temporal_filter_apply/; + add_proto qw/void vp9_highbd_temporal_filter_apply/, "uint8_t *frame1, unsigned int stride, uint8_t *frame2, unsigned int block_width, unsigned int block_height, int strength, int filter_weight, unsigned int *accumulator, uint16_t *count"; + specialize qw/vp9_highbd_temporal_filter_apply/; } # End vp9_high encoder functions diff --git a/vp9/common/x86/vp9_high_loopfilter_intrin_sse2.c b/vp9/common/x86/vp9_high_loopfilter_intrin_sse2.c new file mode 100644 index 000000000..32e4b2012 --- /dev/null +++ b/vp9/common/x86/vp9_high_loopfilter_intrin_sse2.c @@ -0,0 +1,1119 @@ +/* + * Copyright (c) 2014 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include // SSE2 + +#include "./vp9_rtcd.h" +#include "vp9/common/vp9_loopfilter.h" +#include "vpx_ports/emmintrin_compat.h" + +static INLINE __m128i signed_char_clamp_bd_sse2(__m128i value, int bd) { + __m128i ubounded; + __m128i lbounded; + __m128i retval; + + const __m128i zero = _mm_set1_epi16(0); + const __m128i one = _mm_set1_epi16(1); + const __m128i t80 = _mm_slli_epi16(_mm_set1_epi16(0x80), bd - 8); + const __m128i max = _mm_subs_epi16( + _mm_subs_epi16(_mm_slli_epi16(one, bd), one), t80); + const __m128i min = _mm_subs_epi16(zero, t80); + ubounded = _mm_cmpgt_epi16(value, max); + lbounded = _mm_cmplt_epi16(value, min); + retval = _mm_andnot_si128(_mm_or_si128(ubounded, lbounded), value); + ubounded = _mm_and_si128(ubounded, max); + lbounded = _mm_and_si128(lbounded, min); + retval = _mm_or_si128(retval, ubounded); + retval = _mm_or_si128(retval, lbounded); + return retval; +} + +// TODO(debargha, peter): Break up large functions into smaller ones +// in this file. +static void highbd_mb_lpf_horizontal_edge_w_sse2_8(uint16_t *s, + int p, + const uint8_t *_blimit, + const uint8_t *_limit, + const uint8_t *_thresh, + int bd) { + const __m128i zero = _mm_set1_epi16(0); + const __m128i one = _mm_set1_epi16(1); + const __m128i blimit = _mm_slli_epi16( + _mm_unpacklo_epi8( + _mm_load_si128((const __m128i *)_blimit), zero), bd - 8); + const __m128i limit = _mm_slli_epi16( + _mm_unpacklo_epi8(_mm_load_si128((const __m128i *)_limit), zero), bd - 8); + const __m128i thresh = _mm_slli_epi16( + _mm_unpacklo_epi8( + _mm_load_si128((const __m128i *)_thresh), zero), bd - 8); + __m128i q7, p7, q6, p6, q5, p5, q4, p4, q3, p3, q2, p2, q1, p1, q0, p0; + __m128i mask, hev, flat, flat2, abs_p1p0, abs_q1q0; + __m128i ps1, qs1, ps0, qs0; + __m128i abs_p0q0, abs_p1q1, ffff, work; + __m128i filt, work_a, filter1, filter2; + __m128i flat2_q6, flat2_p6, flat2_q5, flat2_p5, flat2_q4, flat2_p4; + __m128i flat2_q3, flat2_p3, flat2_q2, flat2_p2, flat2_q1, flat2_p1; + __m128i flat2_q0, flat2_p0; + __m128i flat_q2, flat_p2, flat_q1, flat_p1, flat_q0, flat_p0; + __m128i pixelFilter_p, pixelFilter_q; + __m128i pixetFilter_p2p1p0, pixetFilter_q2q1q0; + __m128i sum_p7, sum_q7, sum_p3, sum_q3; + __m128i t4, t3, t80, t1; + __m128i eight, four; + + q4 = _mm_load_si128((__m128i *)(s + 4 * p)); + p4 = _mm_load_si128((__m128i *)(s - 5 * p)); + q3 = _mm_load_si128((__m128i *)(s + 3 * p)); + p3 = _mm_load_si128((__m128i *)(s - 4 * p)); + q2 = _mm_load_si128((__m128i *)(s + 2 * p)); + p2 = _mm_load_si128((__m128i *)(s - 3 * p)); + q1 = _mm_load_si128((__m128i *)(s + 1 * p)); + p1 = _mm_load_si128((__m128i *)(s - 2 * p)); + q0 = _mm_load_si128((__m128i *)(s + 0 * p)); + p0 = _mm_load_si128((__m128i *)(s - 1 * p)); + + // high_filter_mask + abs_p1p0 = _mm_or_si128(_mm_subs_epu16(p1, p0), _mm_subs_epu16(p0, p1)); + abs_q1q0 = _mm_or_si128(_mm_subs_epu16(q1, q0), _mm_subs_epu16(q0, q1)); + + ffff = _mm_cmpeq_epi16(abs_p1p0, abs_p1p0); + + abs_p0q0 = _mm_or_si128(_mm_subs_epu16(p0, q0), _mm_subs_epu16(q0, p0)); + abs_p1q1 = _mm_or_si128(_mm_subs_epu16(p1, q1), _mm_subs_epu16(q1, p1)); + + // high_hev_mask (in C code this is actually called from high_filter4) + flat = _mm_max_epi16(abs_p1p0, abs_q1q0); + hev = _mm_subs_epu16(flat, thresh); + hev = _mm_xor_si128(_mm_cmpeq_epi16(hev, zero), ffff); + + abs_p0q0 =_mm_adds_epu16(abs_p0q0, abs_p0q0); // abs(p0 - q0) * 2 + abs_p1q1 = _mm_srli_epi16(abs_p1q1, 1); // abs(p1 - q1) / 2 + mask = _mm_subs_epu16(_mm_adds_epu16(abs_p0q0, abs_p1q1), blimit); + mask = _mm_xor_si128(_mm_cmpeq_epi16(mask, zero), ffff); + mask = _mm_and_si128(mask, _mm_adds_epu16(limit, one)); + work = _mm_max_epi16(_mm_or_si128(_mm_subs_epu16(p1, p0), + _mm_subs_epu16(p0, p1)), + _mm_or_si128(_mm_subs_epu16(q1, q0), + _mm_subs_epu16(q0, q1))); + mask = _mm_max_epi16(work, mask); + work = _mm_max_epi16(_mm_or_si128(_mm_subs_epu16(p2, p1), + _mm_subs_epu16(p1, p2)), + _mm_or_si128(_mm_subs_epu16(q2, q1), + _mm_subs_epu16(q1, q2))); + mask = _mm_max_epi16(work, mask); + work = _mm_max_epi16(_mm_or_si128(_mm_subs_epu16(p3, p2), + _mm_subs_epu16(p2, p3)), + _mm_or_si128(_mm_subs_epu16(q3, q2), + _mm_subs_epu16(q2, q3))); + mask = _mm_max_epi16(work, mask); + + mask = _mm_subs_epu16(mask, limit); + mask = _mm_cmpeq_epi16(mask, zero); // return ~mask + + // lp filter + // high_filter4 + t4 = _mm_set1_epi16(4); + t3 = _mm_set1_epi16(3); + t80 = _mm_slli_epi16(_mm_set1_epi16(0x80), bd - 8); + t1 = _mm_set1_epi16(0x1); + + ps1 = _mm_subs_epi16(p1, t80); + qs1 = _mm_subs_epi16(q1, t80); + ps0 = _mm_subs_epi16(p0, t80); + qs0 = _mm_subs_epi16(q0, t80); + + filt = _mm_and_si128( + signed_char_clamp_bd_sse2(_mm_subs_epi16(ps1, qs1), bd), hev); + work_a = _mm_subs_epi16(qs0, ps0); + filt = _mm_adds_epi16(filt, work_a); + filt = _mm_adds_epi16(filt, work_a); + filt = signed_char_clamp_bd_sse2(_mm_adds_epi16(filt, work_a), bd); + filt = _mm_and_si128(filt, mask); + + filter1 = signed_char_clamp_bd_sse2(_mm_adds_epi16(filt, t4), bd); + filter2 = signed_char_clamp_bd_sse2(_mm_adds_epi16(filt, t3), bd); + + // Filter1 >> 3 + filter1 = _mm_srai_epi16(filter1, 0x3); + filter2 = _mm_srai_epi16(filter2, 0x3); + + qs0 = _mm_adds_epi16( + signed_char_clamp_bd_sse2(_mm_subs_epi16(qs0, filter1), bd), + t80); + ps0 = _mm_adds_epi16( + signed_char_clamp_bd_sse2(_mm_adds_epi16(ps0, filter2), bd), + t80); + filt = _mm_adds_epi16(filter1, t1); + filt = _mm_srai_epi16(filt, 1); + filt = _mm_andnot_si128(hev, filt); + + qs1 = _mm_adds_epi16( + signed_char_clamp_bd_sse2(_mm_subs_epi16(qs1, filt), bd), + t80); + ps1 = _mm_adds_epi16( + signed_char_clamp_bd_sse2(_mm_adds_epi16(ps1, filt), bd), + t80); + // end high_filter4 + // loopfilter done + + // high_flat_mask4 + flat = _mm_max_epi16(_mm_or_si128(_mm_subs_epu16(p2, p0), + _mm_subs_epu16(p0, p2)), + _mm_or_si128(_mm_subs_epu16(p3, p0), + _mm_subs_epu16(p0, p3))); + work = _mm_max_epi16(_mm_or_si128(_mm_subs_epu16(q2, q0), + _mm_subs_epu16(q0, q2)), + _mm_or_si128(_mm_subs_epu16(q3, q0), + _mm_subs_epu16(q0, q3))); + flat = _mm_max_epi16(work, flat); + work = _mm_max_epi16(abs_p1p0, abs_q1q0); + flat = _mm_max_epi16(work, flat); + flat = _mm_subs_epu16(flat, _mm_slli_epi16(one, bd - 8)); + flat = _mm_cmpeq_epi16(flat, zero); + // end flat_mask4 + + // flat & mask = flat && mask (as used in filter8) + // (because, in both vars, each block of 16 either all 1s or all 0s) + flat = _mm_and_si128(flat, mask); + + p5 = _mm_load_si128((__m128i *)(s - 6 * p)); + q5 = _mm_load_si128((__m128i *)(s + 5 * p)); + p6 = _mm_load_si128((__m128i *)(s - 7 * p)); + q6 = _mm_load_si128((__m128i *)(s + 6 * p)); + p7 = _mm_load_si128((__m128i *)(s - 8 * p)); + q7 = _mm_load_si128((__m128i *)(s + 7 * p)); + + // high_flat_mask5 (arguments passed in are p0, q0, p4-p7, q4-q7 + // but referred to as p0-p4 & q0-q4 in fn) + flat2 = _mm_max_epi16(_mm_or_si128(_mm_subs_epu16(p4, p0), + _mm_subs_epu16(p0, p4)), + _mm_or_si128(_mm_subs_epu16(q4, q0), + _mm_subs_epu16(q0, q4))); + + work = _mm_max_epi16(_mm_or_si128(_mm_subs_epu16(p5, p0), + _mm_subs_epu16(p0, p5)), + _mm_or_si128(_mm_subs_epu16(q5, q0), + _mm_subs_epu16(q0, q5))); + flat2 = _mm_max_epi16(work, flat2); + + work = _mm_max_epi16(_mm_or_si128(_mm_subs_epu16(p6, p0), + _mm_subs_epu16(p0, p6)), + _mm_or_si128(_mm_subs_epu16(q6, q0), + _mm_subs_epu16(q0, q6))); + flat2 = _mm_max_epi16(work, flat2); + + work = _mm_max_epi16(_mm_or_si128(_mm_subs_epu16(p7, p0), + _mm_subs_epu16(p0, p7)), + _mm_or_si128(_mm_subs_epu16(q7, q0), + _mm_subs_epu16(q0, q7))); + flat2 = _mm_max_epi16(work, flat2); + + flat2 = _mm_subs_epu16(flat2, _mm_slli_epi16(one, bd - 8)); + flat2 = _mm_cmpeq_epi16(flat2, zero); + flat2 = _mm_and_si128(flat2, flat); // flat2 & flat & mask + // end high_flat_mask5 + + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + // flat and wide flat calculations + eight = _mm_set1_epi16(8); + four = _mm_set1_epi16(4); + + pixelFilter_p = _mm_add_epi16(_mm_add_epi16(p6, p5), + _mm_add_epi16(p4, p3)); + pixelFilter_q = _mm_add_epi16(_mm_add_epi16(q6, q5), + _mm_add_epi16(q4, q3)); + + pixetFilter_p2p1p0 = _mm_add_epi16(p0, _mm_add_epi16(p2, p1)); + pixelFilter_p = _mm_add_epi16(pixelFilter_p, pixetFilter_p2p1p0); + + pixetFilter_q2q1q0 = _mm_add_epi16(q0, _mm_add_epi16(q2, q1)); + pixelFilter_q = _mm_add_epi16(pixelFilter_q, pixetFilter_q2q1q0); + pixelFilter_p = _mm_add_epi16(eight, _mm_add_epi16(pixelFilter_p, + pixelFilter_q)); + pixetFilter_p2p1p0 = _mm_add_epi16(four, + _mm_add_epi16(pixetFilter_p2p1p0, + pixetFilter_q2q1q0)); + flat2_p0 = _mm_srli_epi16(_mm_add_epi16(pixelFilter_p, + _mm_add_epi16(p7, p0)), 4); + flat2_q0 = _mm_srli_epi16(_mm_add_epi16(pixelFilter_p, + _mm_add_epi16(q7, q0)), 4); + flat_p0 = _mm_srli_epi16(_mm_add_epi16(pixetFilter_p2p1p0, + _mm_add_epi16(p3, p0)), 3); + flat_q0 = _mm_srli_epi16(_mm_add_epi16(pixetFilter_p2p1p0, + _mm_add_epi16(q3, q0)), 3); + + sum_p7 = _mm_add_epi16(p7, p7); + sum_q7 = _mm_add_epi16(q7, q7); + sum_p3 = _mm_add_epi16(p3, p3); + sum_q3 = _mm_add_epi16(q3, q3); + + pixelFilter_q = _mm_sub_epi16(pixelFilter_p, p6); + pixelFilter_p = _mm_sub_epi16(pixelFilter_p, q6); + flat2_p1 = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_p, _mm_add_epi16(sum_p7, p1)), 4); + flat2_q1 = _mm_srli_epi16( + _mm_add_epi16(pixelFilter_q, _mm_add_epi16(sum_q7, q1)), 4); + + pixetFilter_q2q1q0 = _mm_sub_epi16(pixetFilter_p2p1p0, p2); + pixetFilter_p2p1p0 = _mm_sub_epi16(pixetFilter_p2p1p0, q2); + flat_p1 = _mm_srli_epi16(_mm_add_epi16(pixetFilter_p2p1p0, + _mm_add_epi16(sum_p3, p1)), 3); + flat_q1 = _mm_srli_epi16(_mm_add_epi16(pixetFilter_q2q1q0, + _mm_add_epi16(sum_q3, q1)), 3); + + sum_p7 = _mm_add_epi16(sum_p7, p7); + sum_q7 = _mm_add_epi16(sum_q7, q7); + sum_p3 = _mm_add_epi16(sum_p3, p3); + sum_q3 = _mm_add_epi16(sum_q3, q3); + + pixelFilter_p = _mm_sub_epi16(pixelFilter_p, q5); + pixelFilter_q = _mm_sub_epi16(pixelFilter_q, p5); + flat2_p2 = _mm_srli_epi16(_mm_add_epi16(pixelFilter_p, + _mm_add_epi16(sum_p7, p2)), 4); + flat2_q2 = _mm_srli_epi16(_mm_add_epi16(pixelFilter_q, + _mm_add_epi16(sum_q7, q2)), 4); + + pixetFilter_p2p1p0 = _mm_sub_epi16(pixetFilter_p2p1p0, q1); + pixetFilter_q2q1q0 = _mm_sub_epi16(pixetFilter_q2q1q0, p1); + flat_p2 = _mm_srli_epi16(_mm_add_epi16(pixetFilter_p2p1p0, + _mm_add_epi16(sum_p3, p2)), 3); + flat_q2 = _mm_srli_epi16(_mm_add_epi16(pixetFilter_q2q1q0, + _mm_add_epi16(sum_q3, q2)), 3); + + sum_p7 = _mm_add_epi16(sum_p7, p7); + sum_q7 = _mm_add_epi16(sum_q7, q7); + pixelFilter_p = _mm_sub_epi16(pixelFilter_p, q4); + pixelFilter_q = _mm_sub_epi16(pixelFilter_q, p4); + flat2_p3 = _mm_srli_epi16(_mm_add_epi16(pixelFilter_p, + _mm_add_epi16(sum_p7, p3)), 4); + flat2_q3 = _mm_srli_epi16(_mm_add_epi16(pixelFilter_q, + _mm_add_epi16(sum_q7, q3)), 4); + + sum_p7 = _mm_add_epi16(sum_p7, p7); + sum_q7 = _mm_add_epi16(sum_q7, q7); + pixelFilter_p = _mm_sub_epi16(pixelFilter_p, q3); + pixelFilter_q = _mm_sub_epi16(pixelFilter_q, p3); + flat2_p4 = _mm_srli_epi16(_mm_add_epi16(pixelFilter_p, + _mm_add_epi16(sum_p7, p4)), 4); + flat2_q4 = _mm_srli_epi16(_mm_add_epi16(pixelFilter_q, + _mm_add_epi16(sum_q7, q4)), 4); + + sum_p7 = _mm_add_epi16(sum_p7, p7); + sum_q7 = _mm_add_epi16(sum_q7, q7); + pixelFilter_p = _mm_sub_epi16(pixelFilter_p, q2); + pixelFilter_q = _mm_sub_epi16(pixelFilter_q, p2); + flat2_p5 = _mm_srli_epi16(_mm_add_epi16(pixelFilter_p, + _mm_add_epi16(sum_p7, p5)), 4); + flat2_q5 = _mm_srli_epi16(_mm_add_epi16(pixelFilter_q, + _mm_add_epi16(sum_q7, q5)), 4); + + sum_p7 = _mm_add_epi16(sum_p7, p7); + sum_q7 = _mm_add_epi16(sum_q7, q7); + pixelFilter_p = _mm_sub_epi16(pixelFilter_p, q1); + pixelFilter_q = _mm_sub_epi16(pixelFilter_q, p1); + flat2_p6 = _mm_srli_epi16(_mm_add_epi16(pixelFilter_p, + _mm_add_epi16(sum_p7, p6)), 4); + flat2_q6 = _mm_srli_epi16(_mm_add_epi16(pixelFilter_q, + _mm_add_epi16(sum_q7, q6)), 4); + + // wide flat + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + // high_filter8 + p2 = _mm_andnot_si128(flat, p2); + // p2 remains unchanged if !(flat && mask) + flat_p2 = _mm_and_si128(flat, flat_p2); + // when (flat && mask) + p2 = _mm_or_si128(p2, flat_p2); // full list of p2 values + q2 = _mm_andnot_si128(flat, q2); + flat_q2 = _mm_and_si128(flat, flat_q2); + q2 = _mm_or_si128(q2, flat_q2); // full list of q2 values + + ps1 = _mm_andnot_si128(flat, ps1); + // p1 takes the value assigned to in in filter4 if !(flat && mask) + flat_p1 = _mm_and_si128(flat, flat_p1); + // when (flat && mask) + p1 = _mm_or_si128(ps1, flat_p1); // full list of p1 values + qs1 = _mm_andnot_si128(flat, qs1); + flat_q1 = _mm_and_si128(flat, flat_q1); + q1 = _mm_or_si128(qs1, flat_q1); // full list of q1 values + + ps0 = _mm_andnot_si128(flat, ps0); + // p0 takes the value assigned to in in filter4 if !(flat && mask) + flat_p0 = _mm_and_si128(flat, flat_p0); + // when (flat && mask) + p0 = _mm_or_si128(ps0, flat_p0); // full list of p0 values + qs0 = _mm_andnot_si128(flat, qs0); + flat_q0 = _mm_and_si128(flat, flat_q0); + q0 = _mm_or_si128(qs0, flat_q0); // full list of q0 values + // end high_filter8 + + // high_filter16 + p6 = _mm_andnot_si128(flat2, p6); + // p6 remains unchanged if !(flat2 && flat && mask) + flat2_p6 = _mm_and_si128(flat2, flat2_p6); + // get values for when (flat2 && flat && mask) + p6 = _mm_or_si128(p6, flat2_p6); // full list of p6 values + q6 = _mm_andnot_si128(flat2, q6); + // q6 remains unchanged if !(flat2 && flat && mask) + flat2_q6 = _mm_and_si128(flat2, flat2_q6); + // get values for when (flat2 && flat && mask) + q6 = _mm_or_si128(q6, flat2_q6); // full list of q6 values + _mm_store_si128((__m128i *)(s - 7 * p), p6); + _mm_store_si128((__m128i *)(s + 6 * p), q6); + + p5 = _mm_andnot_si128(flat2, p5); + // p5 remains unchanged if !(flat2 && flat && mask) + flat2_p5 = _mm_and_si128(flat2, flat2_p5); + // get values for when (flat2 && flat && mask) + p5 = _mm_or_si128(p5, flat2_p5); + // full list of p5 values + q5 = _mm_andnot_si128(flat2, q5); + // q5 remains unchanged if !(flat2 && flat && mask) + flat2_q5 = _mm_and_si128(flat2, flat2_q5); + // get values for when (flat2 && flat && mask) + q5 = _mm_or_si128(q5, flat2_q5); + // full list of q5 values + _mm_store_si128((__m128i *)(s - 6 * p), p5); + _mm_store_si128((__m128i *)(s + 5 * p), q5); + + p4 = _mm_andnot_si128(flat2, p4); + // p4 remains unchanged if !(flat2 && flat && mask) + flat2_p4 = _mm_and_si128(flat2, flat2_p4); + // get values for when (flat2 && flat && mask) + p4 = _mm_or_si128(p4, flat2_p4); // full list of p4 values + q4 = _mm_andnot_si128(flat2, q4); + // q4 remains unchanged if !(flat2 && flat && mask) + flat2_q4 = _mm_and_si128(flat2, flat2_q4); + // get values for when (flat2 && flat && mask) + q4 = _mm_or_si128(q4, flat2_q4); // full list of q4 values + _mm_store_si128((__m128i *)(s - 5 * p), p4); + _mm_store_si128((__m128i *)(s + 4 * p), q4); + + p3 = _mm_andnot_si128(flat2, p3); + // p3 takes value from high_filter8 if !(flat2 && flat && mask) + flat2_p3 = _mm_and_si128(flat2, flat2_p3); + // get values for when (flat2 && flat && mask) + p3 = _mm_or_si128(p3, flat2_p3); // full list of p3 values + q3 = _mm_andnot_si128(flat2, q3); + // q3 takes value from high_filter8 if !(flat2 && flat && mask) + flat2_q3 = _mm_and_si128(flat2, flat2_q3); + // get values for when (flat2 && flat && mask) + q3 = _mm_or_si128(q3, flat2_q3); // full list of q3 values + _mm_store_si128((__m128i *)(s - 4 * p), p3); + _mm_store_si128((__m128i *)(s + 3 * p), q3); + + p2 = _mm_andnot_si128(flat2, p2); + // p2 takes value from high_filter8 if !(flat2 && flat && mask) + flat2_p2 = _mm_and_si128(flat2, flat2_p2); + // get values for when (flat2 && flat && mask) + p2 = _mm_or_si128(p2, flat2_p2); + // full list of p2 values + q2 = _mm_andnot_si128(flat2, q2); + // q2 takes value from high_filter8 if !(flat2 && flat && mask) + flat2_q2 = _mm_and_si128(flat2, flat2_q2); + // get values for when (flat2 && flat && mask) + q2 = _mm_or_si128(q2, flat2_q2); // full list of q2 values + _mm_store_si128((__m128i *)(s - 3 * p), p2); + _mm_store_si128((__m128i *)(s + 2 * p), q2); + + p1 = _mm_andnot_si128(flat2, p1); + // p1 takes value from high_filter8 if !(flat2 && flat && mask) + flat2_p1 = _mm_and_si128(flat2, flat2_p1); + // get values for when (flat2 && flat && mask) + p1 = _mm_or_si128(p1, flat2_p1); // full list of p1 values + q1 = _mm_andnot_si128(flat2, q1); + // q1 takes value from high_filter8 if !(flat2 && flat && mask) + flat2_q1 = _mm_and_si128(flat2, flat2_q1); + // get values for when (flat2 && flat && mask) + q1 = _mm_or_si128(q1, flat2_q1); // full list of q1 values + _mm_store_si128((__m128i *)(s - 2 * p), p1); + _mm_store_si128((__m128i *)(s + 1 * p), q1); + + p0 = _mm_andnot_si128(flat2, p0); + // p0 takes value from high_filter8 if !(flat2 && flat && mask) + flat2_p0 = _mm_and_si128(flat2, flat2_p0); + // get values for when (flat2 && flat && mask) + p0 = _mm_or_si128(p0, flat2_p0); // full list of p0 values + q0 = _mm_andnot_si128(flat2, q0); + // q0 takes value from high_filter8 if !(flat2 && flat && mask) + flat2_q0 = _mm_and_si128(flat2, flat2_q0); + // get values for when (flat2 && flat && mask) + q0 = _mm_or_si128(q0, flat2_q0); // full list of q0 values + _mm_store_si128((__m128i *)(s - 1 * p), p0); + _mm_store_si128((__m128i *)(s - 0 * p), q0); +} + +static void highbd_mb_lpf_horizontal_edge_w_sse2_16(uint16_t *s, + int p, + const uint8_t *_blimit, + const uint8_t *_limit, + const uint8_t *_thresh, + int bd) { + highbd_mb_lpf_horizontal_edge_w_sse2_8(s, p, _blimit, _limit, _thresh, bd); + highbd_mb_lpf_horizontal_edge_w_sse2_8(s + 8, p, _blimit, _limit, _thresh, + bd); +} + +// TODO(yunqingwang): remove count and call these 2 functions(8 or 16) directly. +void vp9_highbd_lpf_horizontal_16_sse2(uint16_t *s, int p, + const uint8_t *_blimit, + const uint8_t *_limit, + const uint8_t *_thresh, + int count, int bd) { + if (count == 1) + highbd_mb_lpf_horizontal_edge_w_sse2_8(s, p, _blimit, _limit, _thresh, bd); + else + highbd_mb_lpf_horizontal_edge_w_sse2_16(s, p, _blimit, _limit, _thresh, bd); +} + +void vp9_highbd_lpf_horizontal_8_sse2(uint16_t *s, int p, + const uint8_t *_blimit, + const uint8_t *_limit, + const uint8_t *_thresh, + int count, int bd) { + DECLARE_ALIGNED_ARRAY(16, uint16_t, flat_op2, 16); + DECLARE_ALIGNED_ARRAY(16, uint16_t, flat_op1, 16); + DECLARE_ALIGNED_ARRAY(16, uint16_t, flat_op0, 16); + DECLARE_ALIGNED_ARRAY(16, uint16_t, flat_oq2, 16); + DECLARE_ALIGNED_ARRAY(16, uint16_t, flat_oq1, 16); + DECLARE_ALIGNED_ARRAY(16, uint16_t, flat_oq0, 16); + const __m128i zero = _mm_set1_epi16(0); + const __m128i blimit = _mm_slli_epi16( + _mm_unpacklo_epi8(_mm_load_si128((const __m128i *)_blimit), zero), + bd - 8); + const __m128i limit = _mm_slli_epi16( + _mm_unpacklo_epi8(_mm_load_si128((const __m128i *)_limit), zero), + bd - 8); + const __m128i thresh = _mm_slli_epi16( + _mm_unpacklo_epi8(_mm_load_si128((const __m128i *)_thresh), zero), + bd - 8); + __m128i mask, hev, flat; + __m128i p3 = _mm_load_si128((__m128i *)(s - 4 * p)); + __m128i q3 = _mm_load_si128((__m128i *)(s + 3 * p)); + __m128i p2 = _mm_load_si128((__m128i *)(s - 3 * p)); + __m128i q2 = _mm_load_si128((__m128i *)(s + 2 * p)); + __m128i p1 = _mm_load_si128((__m128i *)(s - 2 * p)); + __m128i q1 = _mm_load_si128((__m128i *)(s + 1 * p)); + __m128i p0 = _mm_load_si128((__m128i *)(s - 1 * p)); + __m128i q0 = _mm_load_si128((__m128i *)(s + 0 * p)); + const __m128i one = _mm_set1_epi16(1); + const __m128i ffff = _mm_cmpeq_epi16(one, one); + __m128i abs_p1q1, abs_p0q0, abs_q1q0, abs_p1p0, work; + const __m128i four = _mm_set1_epi16(4); + __m128i workp_a, workp_b, workp_shft; + + const __m128i t4 = _mm_set1_epi16(4); + const __m128i t3 = _mm_set1_epi16(3); + const __m128i t80 = _mm_slli_epi16(_mm_set1_epi16(0x80), bd - 8); + const __m128i t1 = _mm_set1_epi16(0x1); + const __m128i ps1 = _mm_subs_epi16(p1, t80); + const __m128i ps0 = _mm_subs_epi16(p0, t80); + const __m128i qs0 = _mm_subs_epi16(q0, t80); + const __m128i qs1 = _mm_subs_epi16(q1, t80); + __m128i filt; + __m128i work_a; + __m128i filter1, filter2; + + (void)count; + + // filter_mask and hev_mask + abs_p1p0 = _mm_or_si128(_mm_subs_epu16(p1, p0), + _mm_subs_epu16(p0, p1)); + abs_q1q0 = _mm_or_si128(_mm_subs_epu16(q1, q0), + _mm_subs_epu16(q0, q1)); + + abs_p0q0 = _mm_or_si128(_mm_subs_epu16(p0, q0), + _mm_subs_epu16(q0, p0)); + abs_p1q1 = _mm_or_si128(_mm_subs_epu16(p1, q1), + _mm_subs_epu16(q1, p1)); + flat = _mm_max_epi16(abs_p1p0, abs_q1q0); + hev = _mm_subs_epu16(flat, thresh); + hev = _mm_xor_si128(_mm_cmpeq_epi16(hev, zero), ffff); + + abs_p0q0 =_mm_adds_epu16(abs_p0q0, abs_p0q0); + abs_p1q1 = _mm_srli_epi16(abs_p1q1, 1); + mask = _mm_subs_epu16(_mm_adds_epu16(abs_p0q0, abs_p1q1), blimit); + mask = _mm_xor_si128(_mm_cmpeq_epi16(mask, zero), ffff); + // mask |= (abs(p0 - q0) * 2 + abs(p1 - q1) / 2 > blimit) * -1; + // So taking maximums continues to work: + mask = _mm_and_si128(mask, _mm_adds_epu16(limit, one)); + mask = _mm_max_epi16(abs_p1p0, mask); + // mask |= (abs(p1 - p0) > limit) * -1; + mask = _mm_max_epi16(abs_q1q0, mask); + // mask |= (abs(q1 - q0) > limit) * -1; + + work = _mm_max_epi16(_mm_or_si128(_mm_subs_epu16(p2, p1), + _mm_subs_epu16(p1, p2)), + _mm_or_si128(_mm_subs_epu16(q2, q1), + _mm_subs_epu16(q1, q2))); + mask = _mm_max_epi16(work, mask); + work = _mm_max_epi16(_mm_or_si128(_mm_subs_epu16(p3, p2), + _mm_subs_epu16(p2, p3)), + _mm_or_si128(_mm_subs_epu16(q3, q2), + _mm_subs_epu16(q2, q3))); + mask = _mm_max_epi16(work, mask); + mask = _mm_subs_epu16(mask, limit); + mask = _mm_cmpeq_epi16(mask, zero); + + // flat_mask4 + flat = _mm_max_epi16(_mm_or_si128(_mm_subs_epu16(p2, p0), + _mm_subs_epu16(p0, p2)), + _mm_or_si128(_mm_subs_epu16(q2, q0), + _mm_subs_epu16(q0, q2))); + work = _mm_max_epi16(_mm_or_si128(_mm_subs_epu16(p3, p0), + _mm_subs_epu16(p0, p3)), + _mm_or_si128(_mm_subs_epu16(q3, q0), + _mm_subs_epu16(q0, q3))); + flat = _mm_max_epi16(work, flat); + flat = _mm_max_epi16(abs_p1p0, flat); + flat = _mm_max_epi16(abs_q1q0, flat); + flat = _mm_subs_epu16(flat, _mm_slli_epi16(one, bd - 8)); + flat = _mm_cmpeq_epi16(flat, zero); + flat = _mm_and_si128(flat, mask); // flat & mask + + // Added before shift for rounding part of ROUND_POWER_OF_TWO + + workp_a = _mm_add_epi16(_mm_add_epi16(p3, p3), _mm_add_epi16(p2, p1)); + workp_a = _mm_add_epi16(_mm_add_epi16(workp_a, four), p0); + workp_b = _mm_add_epi16(_mm_add_epi16(q0, p2), p3); + workp_shft = _mm_srli_epi16(_mm_add_epi16(workp_a, workp_b), 3); + _mm_store_si128((__m128i *)&flat_op2[0], workp_shft); + + workp_b = _mm_add_epi16(_mm_add_epi16(q0, q1), p1); + workp_shft = _mm_srli_epi16(_mm_add_epi16(workp_a, workp_b), 3); + _mm_store_si128((__m128i *)&flat_op1[0], workp_shft); + + workp_a = _mm_add_epi16(_mm_sub_epi16(workp_a, p3), q2); + workp_b = _mm_add_epi16(_mm_sub_epi16(workp_b, p1), p0); + workp_shft = _mm_srli_epi16(_mm_add_epi16(workp_a, workp_b), 3); + _mm_store_si128((__m128i *)&flat_op0[0], workp_shft); + + workp_a = _mm_add_epi16(_mm_sub_epi16(workp_a, p3), q3); + workp_b = _mm_add_epi16(_mm_sub_epi16(workp_b, p0), q0); + workp_shft = _mm_srli_epi16(_mm_add_epi16(workp_a, workp_b), 3); + _mm_store_si128((__m128i *)&flat_oq0[0], workp_shft); + + workp_a = _mm_add_epi16(_mm_sub_epi16(workp_a, p2), q3); + workp_b = _mm_add_epi16(_mm_sub_epi16(workp_b, q0), q1); + workp_shft = _mm_srli_epi16(_mm_add_epi16(workp_a, workp_b), 3); + _mm_store_si128((__m128i *)&flat_oq1[0], workp_shft); + + workp_a = _mm_add_epi16(_mm_sub_epi16(workp_a, p1), q3); + workp_b = _mm_add_epi16(_mm_sub_epi16(workp_b, q1), q2); + workp_shft = _mm_srli_epi16(_mm_add_epi16(workp_a, workp_b), 3); + _mm_store_si128((__m128i *)&flat_oq2[0], workp_shft); + + // lp filter + filt = signed_char_clamp_bd_sse2(_mm_subs_epi16(ps1, qs1), bd); + filt = _mm_and_si128(filt, hev); + work_a = _mm_subs_epi16(qs0, ps0); + filt = _mm_adds_epi16(filt, work_a); + filt = _mm_adds_epi16(filt, work_a); + filt = _mm_adds_epi16(filt, work_a); + // (vp9_filter + 3 * (qs0 - ps0)) & mask + filt = signed_char_clamp_bd_sse2(filt, bd); + filt = _mm_and_si128(filt, mask); + + filter1 = _mm_adds_epi16(filt, t4); + filter2 = _mm_adds_epi16(filt, t3); + + // Filter1 >> 3 + filter1 = signed_char_clamp_bd_sse2(filter1, bd); + filter1 = _mm_srai_epi16(filter1, 3); + + // Filter2 >> 3 + filter2 = signed_char_clamp_bd_sse2(filter2, bd); + filter2 = _mm_srai_epi16(filter2, 3); + + // filt >> 1 + filt = _mm_adds_epi16(filter1, t1); + filt = _mm_srai_epi16(filt, 1); + // filter = ROUND_POWER_OF_TWO(filter1, 1) & ~hev; + filt = _mm_andnot_si128(hev, filt); + + work_a = signed_char_clamp_bd_sse2(_mm_subs_epi16(qs0, filter1), bd); + work_a = _mm_adds_epi16(work_a, t80); + q0 = _mm_load_si128((__m128i *)flat_oq0); + work_a = _mm_andnot_si128(flat, work_a); + q0 = _mm_and_si128(flat, q0); + q0 = _mm_or_si128(work_a, q0); + + work_a = signed_char_clamp_bd_sse2(_mm_subs_epi16(qs1, filt), bd); + work_a = _mm_adds_epi16(work_a, t80); + q1 = _mm_load_si128((__m128i *)flat_oq1); + work_a = _mm_andnot_si128(flat, work_a); + q1 = _mm_and_si128(flat, q1); + q1 = _mm_or_si128(work_a, q1); + + work_a = _mm_loadu_si128((__m128i *)(s + 2 * p)); + q2 = _mm_load_si128((__m128i *)flat_oq2); + work_a = _mm_andnot_si128(flat, work_a); + q2 = _mm_and_si128(flat, q2); + q2 = _mm_or_si128(work_a, q2); + + work_a = signed_char_clamp_bd_sse2(_mm_adds_epi16(ps0, filter2), bd); + work_a = _mm_adds_epi16(work_a, t80); + p0 = _mm_load_si128((__m128i *)flat_op0); + work_a = _mm_andnot_si128(flat, work_a); + p0 = _mm_and_si128(flat, p0); + p0 = _mm_or_si128(work_a, p0); + + work_a = signed_char_clamp_bd_sse2(_mm_adds_epi16(ps1, filt), bd); + work_a = _mm_adds_epi16(work_a, t80); + p1 = _mm_load_si128((__m128i *)flat_op1); + work_a = _mm_andnot_si128(flat, work_a); + p1 = _mm_and_si128(flat, p1); + p1 = _mm_or_si128(work_a, p1); + + work_a = _mm_loadu_si128((__m128i *)(s - 3 * p)); + p2 = _mm_load_si128((__m128i *)flat_op2); + work_a = _mm_andnot_si128(flat, work_a); + p2 = _mm_and_si128(flat, p2); + p2 = _mm_or_si128(work_a, p2); + + _mm_store_si128((__m128i *)(s - 3 * p), p2); + _mm_store_si128((__m128i *)(s - 2 * p), p1); + _mm_store_si128((__m128i *)(s - 1 * p), p0); + _mm_store_si128((__m128i *)(s + 0 * p), q0); + _mm_store_si128((__m128i *)(s + 1 * p), q1); + _mm_store_si128((__m128i *)(s + 2 * p), q2); +} + +void vp9_highbd_lpf_horizontal_8_dual_sse2(uint16_t *s, int p, + const uint8_t *_blimit0, + const uint8_t *_limit0, + const uint8_t *_thresh0, + const uint8_t *_blimit1, + const uint8_t *_limit1, + const uint8_t *_thresh1, + int bd) { + vp9_highbd_lpf_horizontal_8_sse2(s, p, _blimit0, _limit0, _thresh0, 1, bd); + vp9_highbd_lpf_horizontal_8_sse2(s + 8, p, _blimit1, _limit1, _thresh1, + 1, bd); +} + +void vp9_highbd_lpf_horizontal_4_sse2(uint16_t *s, int p, + const uint8_t *_blimit, + const uint8_t *_limit, + const uint8_t *_thresh, + int count, int bd) { + const __m128i zero = _mm_set1_epi16(0); + const __m128i blimit = _mm_slli_epi16( + _mm_unpacklo_epi8( + _mm_load_si128((const __m128i *)_blimit), zero), bd - 8); + const __m128i limit = _mm_slli_epi16( + _mm_unpacklo_epi8( + _mm_load_si128((const __m128i *)_limit), zero), bd - 8); + const __m128i thresh = _mm_slli_epi16( + _mm_unpacklo_epi8( + _mm_load_si128((const __m128i *)_thresh), zero), bd - 8); + __m128i mask, hev, flat; + __m128i p3 = _mm_loadu_si128((__m128i *)(s - 4 * p)); + __m128i p2 = _mm_loadu_si128((__m128i *)(s - 3 * p)); + __m128i p1 = _mm_loadu_si128((__m128i *)(s - 2 * p)); + __m128i p0 = _mm_loadu_si128((__m128i *)(s - 1 * p)); + __m128i q0 = _mm_loadu_si128((__m128i *)(s - 0 * p)); + __m128i q1 = _mm_loadu_si128((__m128i *)(s + 1 * p)); + __m128i q2 = _mm_loadu_si128((__m128i *)(s + 2 * p)); + __m128i q3 = _mm_loadu_si128((__m128i *)(s + 3 * p)); + const __m128i abs_p1p0 = _mm_or_si128(_mm_subs_epu16(p1, p0), + _mm_subs_epu16(p0, p1)); + const __m128i abs_q1q0 = _mm_or_si128(_mm_subs_epu16(q1, q0), + _mm_subs_epu16(q0, q1)); + const __m128i ffff = _mm_cmpeq_epi16(abs_p1p0, abs_p1p0); + const __m128i one = _mm_set1_epi16(1); + __m128i abs_p0q0 = _mm_or_si128(_mm_subs_epu16(p0, q0), + _mm_subs_epu16(q0, p0)); + __m128i abs_p1q1 = _mm_or_si128(_mm_subs_epu16(p1, q1), + _mm_subs_epu16(q1, p1)); + __m128i work; + const __m128i t4 = _mm_set1_epi16(4); + const __m128i t3 = _mm_set1_epi16(3); + const __m128i t80 = _mm_slli_epi16(_mm_set1_epi16(0x80), bd - 8); + const __m128i tff80 = _mm_slli_epi16(_mm_set1_epi16(0xff80), bd - 8); + const __m128i tffe0 = _mm_slli_epi16(_mm_set1_epi16(0xffe0), bd - 8); + const __m128i t1f = _mm_srli_epi16(_mm_set1_epi16(0x1fff), 16 - bd); + // equivalent to shifting 0x1f left by bitdepth - 8 + // and setting new bits to 1 + const __m128i t1 = _mm_set1_epi16(0x1); + const __m128i t7f = _mm_srli_epi16(_mm_set1_epi16(0x7fff), 16 - bd); + // equivalent to shifting 0x7f left by bitdepth - 8 + // and setting new bits to 1 + const __m128i ps1 = _mm_subs_epi16(_mm_loadu_si128((__m128i *)(s - 2 * p)), + t80); + const __m128i ps0 = _mm_subs_epi16(_mm_loadu_si128((__m128i *)(s - 1 * p)), + t80); + const __m128i qs0 = _mm_subs_epi16(_mm_loadu_si128((__m128i *)(s + 0 * p)), + t80); + const __m128i qs1 = _mm_subs_epi16(_mm_loadu_si128((__m128i *)(s + 1 * p)), + t80); + __m128i filt; + __m128i work_a; + __m128i filter1, filter2; + + (void)count; + + // filter_mask and hev_mask + flat = _mm_max_epi16(abs_p1p0, abs_q1q0); + hev = _mm_subs_epu16(flat, thresh); + hev = _mm_xor_si128(_mm_cmpeq_epi16(hev, zero), ffff); + + abs_p0q0 =_mm_adds_epu16(abs_p0q0, abs_p0q0); + abs_p1q1 = _mm_srli_epi16(abs_p1q1, 1); + mask = _mm_subs_epu16(_mm_adds_epu16(abs_p0q0, abs_p1q1), blimit); + mask = _mm_xor_si128(_mm_cmpeq_epi16(mask, zero), ffff); + // mask |= (abs(p0 - q0) * 2 + abs(p1 - q1) / 2 > blimit) * -1; + // So taking maximums continues to work: + mask = _mm_and_si128(mask, _mm_adds_epu16(limit, one)); + mask = _mm_max_epi16(flat, mask); + // mask |= (abs(p1 - p0) > limit) * -1; + // mask |= (abs(q1 - q0) > limit) * -1; + work = _mm_max_epi16(_mm_or_si128(_mm_subs_epu16(p2, p1), + _mm_subs_epu16(p1, p2)), + _mm_or_si128(_mm_subs_epu16(p3, p2), + _mm_subs_epu16(p2, p3))); + mask = _mm_max_epi16(work, mask); + work = _mm_max_epi16(_mm_or_si128(_mm_subs_epu16(q2, q1), + _mm_subs_epu16(q1, q2)), + _mm_or_si128(_mm_subs_epu16(q3, q2), + _mm_subs_epu16(q2, q3))); + mask = _mm_max_epi16(work, mask); + mask = _mm_subs_epu16(mask, limit); + mask = _mm_cmpeq_epi16(mask, zero); + + // filter4 + filt = signed_char_clamp_bd_sse2(_mm_subs_epi16(ps1, qs1), bd); + filt = _mm_and_si128(filt, hev); + work_a = _mm_subs_epi16(qs0, ps0); + filt = _mm_adds_epi16(filt, work_a); + filt = _mm_adds_epi16(filt, work_a); + filt = signed_char_clamp_bd_sse2(_mm_adds_epi16(filt, work_a), bd); + // (vp9_filter + 3 * (qs0 - ps0)) & mask + filt = _mm_and_si128(filt, mask); + + filter1 = signed_char_clamp_bd_sse2(_mm_adds_epi16(filt, t4), bd); + filter2 = signed_char_clamp_bd_sse2(_mm_adds_epi16(filt, t3), bd); + + // Filter1 >> 3 + work_a = _mm_cmpgt_epi16(zero, filter1); // get the values that are <0 + filter1 = _mm_srli_epi16(filter1, 3); + work_a = _mm_and_si128(work_a, tffe0); // sign bits for the values < 0 + filter1 = _mm_and_si128(filter1, t1f); // clamp the range + filter1 = _mm_or_si128(filter1, work_a); // reinsert the sign bits + + // Filter2 >> 3 + work_a = _mm_cmpgt_epi16(zero, filter2); + filter2 = _mm_srli_epi16(filter2, 3); + work_a = _mm_and_si128(work_a, tffe0); + filter2 = _mm_and_si128(filter2, t1f); + filter2 = _mm_or_si128(filter2, work_a); + + // filt >> 1 + filt = _mm_adds_epi16(filter1, t1); + work_a = _mm_cmpgt_epi16(zero, filt); + filt = _mm_srli_epi16(filt, 1); + work_a = _mm_and_si128(work_a, tff80); + filt = _mm_and_si128(filt, t7f); + filt = _mm_or_si128(filt, work_a); + + filt = _mm_andnot_si128(hev, filt); + + q0 = _mm_adds_epi16( + signed_char_clamp_bd_sse2(_mm_subs_epi16(qs0, filter1), bd), t80); + q1 = _mm_adds_epi16( + signed_char_clamp_bd_sse2(_mm_subs_epi16(qs1, filt), bd), t80); + p0 = _mm_adds_epi16( + signed_char_clamp_bd_sse2(_mm_adds_epi16(ps0, filter2), bd), t80); + p1 = _mm_adds_epi16( + signed_char_clamp_bd_sse2(_mm_adds_epi16(ps1, filt), bd), t80); + + _mm_storeu_si128((__m128i *)(s - 2 * p), p1); + _mm_storeu_si128((__m128i *)(s - 1 * p), p0); + _mm_storeu_si128((__m128i *)(s + 0 * p), q0); + _mm_storeu_si128((__m128i *)(s + 1 * p), q1); +} + +void vp9_highbd_lpf_horizontal_4_dual_sse2(uint16_t *s, int p, + const uint8_t *_blimit0, + const uint8_t *_limit0, + const uint8_t *_thresh0, + const uint8_t *_blimit1, + const uint8_t *_limit1, + const uint8_t *_thresh1, + int bd) { + vp9_highbd_lpf_horizontal_4_sse2(s, p, _blimit0, _limit0, _thresh0, 1, bd); + vp9_highbd_lpf_horizontal_4_sse2(s + 8, p, _blimit1, _limit1, _thresh1, 1, + bd); +} + +static INLINE void highbd_transpose(uint16_t *src[], int in_p, + uint16_t *dst[], int out_p, + int num_8x8_to_transpose) { + int idx8x8 = 0; + __m128i p0, p1, p2, p3, p4, p5, p6, p7, x0, x1, x2, x3, x4, x5, x6, x7; + do { + uint16_t *in = src[idx8x8]; + uint16_t *out = dst[idx8x8]; + + p0 = _mm_loadu_si128((__m128i *)(in + 0*in_p)); // 00 01 02 03 04 05 06 07 + p1 = _mm_loadu_si128((__m128i *)(in + 1*in_p)); // 10 11 12 13 14 15 16 17 + p2 = _mm_loadu_si128((__m128i *)(in + 2*in_p)); // 20 21 22 23 24 25 26 27 + p3 = _mm_loadu_si128((__m128i *)(in + 3*in_p)); // 30 31 32 33 34 35 36 37 + p4 = _mm_loadu_si128((__m128i *)(in + 4*in_p)); // 40 41 42 43 44 45 46 47 + p5 = _mm_loadu_si128((__m128i *)(in + 5*in_p)); // 50 51 52 53 54 55 56 57 + p6 = _mm_loadu_si128((__m128i *)(in + 6*in_p)); // 60 61 62 63 64 65 66 67 + p7 = _mm_loadu_si128((__m128i *)(in + 7*in_p)); // 70 71 72 73 74 75 76 77 + // 00 10 01 11 02 12 03 13 + x0 = _mm_unpacklo_epi16(p0, p1); + // 20 30 21 31 22 32 23 33 + x1 = _mm_unpacklo_epi16(p2, p3); + // 40 50 41 51 42 52 43 53 + x2 = _mm_unpacklo_epi16(p4, p5); + // 60 70 61 71 62 72 63 73 + x3 = _mm_unpacklo_epi16(p6, p7); + // 00 10 20 30 01 11 21 31 + x4 = _mm_unpacklo_epi32(x0, x1); + // 40 50 60 70 41 51 61 71 + x5 = _mm_unpacklo_epi32(x2, x3); + // 00 10 20 30 40 50 60 70 + x6 = _mm_unpacklo_epi64(x4, x5); + // 01 11 21 31 41 51 61 71 + x7 = _mm_unpackhi_epi64(x4, x5); + + _mm_storeu_si128((__m128i *)(out + 0*out_p), x6); + // 00 10 20 30 40 50 60 70 + _mm_storeu_si128((__m128i *)(out + 1*out_p), x7); + // 01 11 21 31 41 51 61 71 + + // 02 12 22 32 03 13 23 33 + x4 = _mm_unpackhi_epi32(x0, x1); + // 42 52 62 72 43 53 63 73 + x5 = _mm_unpackhi_epi32(x2, x3); + // 02 12 22 32 42 52 62 72 + x6 = _mm_unpacklo_epi64(x4, x5); + // 03 13 23 33 43 53 63 73 + x7 = _mm_unpackhi_epi64(x4, x5); + + _mm_storeu_si128((__m128i *)(out + 2*out_p), x6); + // 02 12 22 32 42 52 62 72 + _mm_storeu_si128((__m128i *)(out + 3*out_p), x7); + // 03 13 23 33 43 53 63 73 + + // 04 14 05 15 06 16 07 17 + x0 = _mm_unpackhi_epi16(p0, p1); + // 24 34 25 35 26 36 27 37 + x1 = _mm_unpackhi_epi16(p2, p3); + // 44 54 45 55 46 56 47 57 + x2 = _mm_unpackhi_epi16(p4, p5); + // 64 74 65 75 66 76 67 77 + x3 = _mm_unpackhi_epi16(p6, p7); + // 04 14 24 34 05 15 25 35 + x4 = _mm_unpacklo_epi32(x0, x1); + // 44 54 64 74 45 55 65 75 + x5 = _mm_unpacklo_epi32(x2, x3); + // 04 14 24 34 44 54 64 74 + x6 = _mm_unpacklo_epi64(x4, x5); + // 05 15 25 35 45 55 65 75 + x7 = _mm_unpackhi_epi64(x4, x5); + + _mm_storeu_si128((__m128i *)(out + 4*out_p), x6); + // 04 14 24 34 44 54 64 74 + _mm_storeu_si128((__m128i *)(out + 5*out_p), x7); + // 05 15 25 35 45 55 65 75 + + // 06 16 26 36 07 17 27 37 + x4 = _mm_unpackhi_epi32(x0, x1); + // 46 56 66 76 47 57 67 77 + x5 = _mm_unpackhi_epi32(x2, x3); + // 06 16 26 36 46 56 66 76 + x6 = _mm_unpacklo_epi64(x4, x5); + // 07 17 27 37 47 57 67 77 + x7 = _mm_unpackhi_epi64(x4, x5); + + _mm_storeu_si128((__m128i *)(out + 6*out_p), x6); + // 06 16 26 36 46 56 66 76 + _mm_storeu_si128((__m128i *)(out + 7*out_p), x7); + // 07 17 27 37 47 57 67 77 + } while (++idx8x8 < num_8x8_to_transpose); +} + +static INLINE void highbd_transpose8x16(uint16_t *in0, uint16_t *in1, + int in_p, uint16_t *out, int out_p) { + uint16_t *src0[1]; + uint16_t *src1[1]; + uint16_t *dest0[1]; + uint16_t *dest1[1]; + src0[0] = in0; + src1[0] = in1; + dest0[0] = out; + dest1[0] = out + 8; + highbd_transpose(src0, in_p, dest0, out_p, 1); + highbd_transpose(src1, in_p, dest1, out_p, 1); +} + +void vp9_highbd_lpf_vertical_4_sse2(uint16_t *s, int p, + const uint8_t *blimit, + const uint8_t *limit, + const uint8_t *thresh, + int count, int bd) { + DECLARE_ALIGNED_ARRAY(16, uint16_t, t_dst, 8 * 8); + uint16_t *src[1]; + uint16_t *dst[1]; + (void)count; + + // Transpose 8x8 + src[0] = s - 4; + dst[0] = t_dst; + + highbd_transpose(src, p, dst, 8, 1); + + // Loop filtering + vp9_highbd_lpf_horizontal_4_sse2(t_dst + 4 * 8, 8, blimit, limit, thresh, 1, + bd); + + src[0] = t_dst; + dst[0] = s - 4; + + // Transpose back + highbd_transpose(src, 8, dst, p, 1); +} + +void vp9_highbd_lpf_vertical_4_dual_sse2(uint16_t *s, int p, + const uint8_t *blimit0, + const uint8_t *limit0, + const uint8_t *thresh0, + const uint8_t *blimit1, + const uint8_t *limit1, + const uint8_t *thresh1, + int bd) { + DECLARE_ALIGNED_ARRAY(16, uint16_t, t_dst, 16 * 8); + uint16_t *src[2]; + uint16_t *dst[2]; + + // Transpose 8x16 + highbd_transpose8x16(s - 4, s - 4 + p * 8, p, t_dst, 16); + + // Loop filtering + vp9_highbd_lpf_horizontal_4_dual_sse2(t_dst + 4 * 16, 16, blimit0, limit0, + thresh0, blimit1, limit1, thresh1, bd); + src[0] = t_dst; + src[1] = t_dst + 8; + dst[0] = s - 4; + dst[1] = s - 4 + p * 8; + + // Transpose back + highbd_transpose(src, 16, dst, p, 2); +} + +void vp9_highbd_lpf_vertical_8_sse2(uint16_t *s, int p, + const uint8_t *blimit, + const uint8_t *limit, + const uint8_t *thresh, + int count, int bd) { + DECLARE_ALIGNED_ARRAY(16, uint16_t, t_dst, 8 * 8); + uint16_t *src[1]; + uint16_t *dst[1]; + (void)count; + + // Transpose 8x8 + src[0] = s - 4; + dst[0] = t_dst; + + highbd_transpose(src, p, dst, 8, 1); + + // Loop filtering + vp9_highbd_lpf_horizontal_8_sse2(t_dst + 4 * 8, 8, blimit, limit, thresh, 1, + bd); + + src[0] = t_dst; + dst[0] = s - 4; + + // Transpose back + highbd_transpose(src, 8, dst, p, 1); +} + +void vp9_highbd_lpf_vertical_8_dual_sse2(uint16_t *s, int p, + const uint8_t *blimit0, + const uint8_t *limit0, + const uint8_t *thresh0, + const uint8_t *blimit1, + const uint8_t *limit1, + const uint8_t *thresh1, + int bd) { + DECLARE_ALIGNED_ARRAY(16, uint16_t, t_dst, 16 * 8); + uint16_t *src[2]; + uint16_t *dst[2]; + + // Transpose 8x16 + highbd_transpose8x16(s - 4, s - 4 + p * 8, p, t_dst, 16); + + // Loop filtering + vp9_highbd_lpf_horizontal_8_dual_sse2(t_dst + 4 * 16, 16, blimit0, limit0, + thresh0, blimit1, limit1, thresh1, bd); + src[0] = t_dst; + src[1] = t_dst + 8; + + dst[0] = s - 4; + dst[1] = s - 4 + p * 8; + + // Transpose back + highbd_transpose(src, 16, dst, p, 2); +} + +void vp9_highbd_lpf_vertical_16_sse2(uint16_t *s, int p, + const uint8_t *blimit, + const uint8_t *limit, + const uint8_t *thresh, + int bd) { + DECLARE_ALIGNED_ARRAY(16, uint16_t, t_dst, 8 * 16); + uint16_t *src[2]; + uint16_t *dst[2]; + + src[0] = s - 8; + src[1] = s; + dst[0] = t_dst; + dst[1] = t_dst + 8 * 8; + + // Transpose 16x8 + highbd_transpose(src, p, dst, 8, 2); + + // Loop filtering + highbd_mb_lpf_horizontal_edge_w_sse2_8(t_dst + 8 * 8, 8, blimit, limit, + thresh, bd); + src[0] = t_dst; + src[1] = t_dst + 8 * 8; + dst[0] = s - 8; + dst[1] = s; + + // Transpose back + highbd_transpose(src, 8, dst, p, 2); +} + +void vp9_highbd_lpf_vertical_16_dual_sse2(uint16_t *s, + int p, + const uint8_t *blimit, + const uint8_t *limit, + const uint8_t *thresh, + int bd) { + DECLARE_ALIGNED_ARRAY(16, uint16_t, t_dst, 256); + + // Transpose 16x16 + highbd_transpose8x16(s - 8, s - 8 + 8 * p, p, t_dst, 16); + highbd_transpose8x16(s, s + 8 * p, p, t_dst + 8 * 16, 16); + + // Loop filtering + highbd_mb_lpf_horizontal_edge_w_sse2_16(t_dst + 8 * 16, 16, blimit, limit, + thresh, bd); + + // Transpose back + highbd_transpose8x16(t_dst, t_dst + 8 * 16, 16, s - 8, p); + highbd_transpose8x16(t_dst + 8, t_dst + 8 + 8 * 16, 16, s - 8 + 8 * p, p); +} diff --git a/vp9/encoder/vp9_temporal_filter.c b/vp9/encoder/vp9_temporal_filter.c index 6a532b243..6db562315 100644 --- a/vp9/encoder/vp9_temporal_filter.c +++ b/vp9/encoder/vp9_temporal_filter.c @@ -56,6 +56,34 @@ static void temporal_filter_predictors_mb_c(MACROBLOCKD *xd, mv_precision_uv = MV_PRECISION_Q3; } +#if CONFIG_VP9_HIGHBITDEPTH + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + vp9_high_build_inter_predictor(y_mb_ptr, stride, + &pred[0], 16, + &mv, + scale, + 16, 16, + which_mv, + kernel, MV_PRECISION_Q3, x, y, xd->bd); + + vp9_high_build_inter_predictor(u_mb_ptr, uv_stride, + &pred[256], uv_block_width, + &mv, + scale, + uv_block_width, uv_block_height, + which_mv, + kernel, mv_precision_uv, x, y, xd->bd); + + vp9_high_build_inter_predictor(v_mb_ptr, uv_stride, + &pred[512], uv_block_width, + &mv, + scale, + uv_block_width, uv_block_height, + which_mv, + kernel, mv_precision_uv, x, y, xd->bd); + return; + } +#endif // CONFIG_VP9_HIGHBITDEPTH vp9_build_inter_predictor(y_mb_ptr, stride, &pred[0], 16, &mv, @@ -133,6 +161,54 @@ void vp9_temporal_filter_apply_c(uint8_t *frame1, } } +#if CONFIG_VP9_HIGHBITDEPTH +void vp9_highbd_temporal_filter_apply_c(uint8_t *frame1_8, + unsigned int stride, + uint8_t *frame2_8, + unsigned int block_width, + unsigned int block_height, + int strength, + int filter_weight, + unsigned int *accumulator, + uint16_t *count) { + uint16_t *frame1 = CONVERT_TO_SHORTPTR(frame1_8); + uint16_t *frame2 = CONVERT_TO_SHORTPTR(frame2_8); + unsigned int i, j, k; + int modifier; + int byte = 0; + const int rounding = strength > 0 ? 1 << (strength - 1) : 0; + + for (i = 0, k = 0; i < block_height; i++) { + for (j = 0; j < block_width; j++, k++) { + int src_byte = frame1[byte]; + int pixel_value = *frame2++; + + modifier = src_byte - pixel_value; + // This is an integer approximation of: + // float coeff = (3.0 * modifer * modifier) / pow(2, strength); + // modifier = (int)roundf(coeff > 16 ? 0 : 16-coeff); + modifier *= modifier; + modifier *= 3; + modifier += rounding; + modifier >>= strength; + + if (modifier > 16) + modifier = 16; + + modifier = 16 - modifier; + modifier *= filter_weight; + + count[k] += modifier; + accumulator[k] += modifier * pixel_value; + + byte++; + } + + byte += stride - block_width; + } +} +#endif // CONFIG_VP9_HIGHBITDEPTH + static int temporal_filter_find_matching_mb_c(VP9_COMP *cpi, uint8_t *arf_frame_buf, uint8_t *frame_ptr_buf, @@ -209,13 +285,26 @@ static void temporal_filter_iterate_c(VP9_COMP *cpi, MACROBLOCKD *mbd = &cpi->mb.e_mbd; YV12_BUFFER_CONFIG *f = frames[alt_ref_index]; uint8_t *dst1, *dst2; +#if CONFIG_VP9_HIGHBITDEPTH + DECLARE_ALIGNED_ARRAY(16, uint16_t, predictor16, 16 * 16 * 3); + DECLARE_ALIGNED_ARRAY(16, uint8_t, predictor8, 16 * 16 * 3); + uint8_t *predictor; +#else DECLARE_ALIGNED_ARRAY(16, uint8_t, predictor, 16 * 16 * 3); +#endif const int mb_uv_height = 16 >> mbd->plane[1].subsampling_y; const int mb_uv_width = 16 >> mbd->plane[1].subsampling_x; // Save input state uint8_t* input_buffer[MAX_MB_PLANE]; int i; +#if CONFIG_VP9_HIGHBITDEPTH + if (mbd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + predictor = CONVERT_TO_BYTEPTR(predictor16); + } else { + predictor = predictor8; + } +#endif for (i = 0; i < MAX_MB_PLANE; i++) input_buffer[i] = mbd->plane[i].pre[0].buf; @@ -286,6 +375,44 @@ static void temporal_filter_iterate_c(VP9_COMP *cpi, predictor, scale, mb_col * 16, mb_row * 16); +#if CONFIG_VP9_HIGHBITDEPTH + if (mbd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + int adj_strength = strength + 2 * (mbd->bd - 8); + // Apply the filter (YUV) + vp9_highbd_temporal_filter_apply(f->y_buffer + mb_y_offset, + f->y_stride, + predictor, 16, 16, adj_strength, + filter_weight, + accumulator, count); + vp9_highbd_temporal_filter_apply(f->u_buffer + mb_uv_offset, + f->uv_stride, predictor + 256, + mb_uv_width, mb_uv_height, + adj_strength, + filter_weight, accumulator + 256, + count + 256); + vp9_highbd_temporal_filter_apply(f->v_buffer + mb_uv_offset, + f->uv_stride, predictor + 512, + mb_uv_width, mb_uv_height, + adj_strength, filter_weight, + accumulator + 512, count + 512); + } else { + // Apply the filter (YUV) + vp9_temporal_filter_apply(f->y_buffer + mb_y_offset, f->y_stride, + predictor, 16, 16, + strength, filter_weight, + accumulator, count); + vp9_temporal_filter_apply(f->u_buffer + mb_uv_offset, f->uv_stride, + predictor + 256, + mb_uv_width, mb_uv_height, strength, + filter_weight, accumulator + 256, + count + 256); + vp9_temporal_filter_apply(f->v_buffer + mb_uv_offset, f->uv_stride, + predictor + 512, + mb_uv_width, mb_uv_height, strength, + filter_weight, accumulator + 512, + count + 512); + } +#else // Apply the filter (YUV) vp9_temporal_filter_apply(f->y_buffer + mb_y_offset, f->y_stride, predictor, 16, 16, @@ -301,9 +428,108 @@ static void temporal_filter_iterate_c(VP9_COMP *cpi, mb_uv_width, mb_uv_height, strength, filter_weight, accumulator + 512, count + 512); +#endif // CONFIG_VP9_HIGHBITDEPTH } } +#if CONFIG_VP9_HIGHBITDEPTH + if (mbd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { + uint16_t *dst1_16; + uint16_t *dst2_16; + // Normalize filter output to produce AltRef frame + dst1 = cpi->alt_ref_buffer.y_buffer; + dst1_16 = CONVERT_TO_SHORTPTR(dst1); + stride = cpi->alt_ref_buffer.y_stride; + byte = mb_y_offset; + for (i = 0, k = 0; i < 16; i++) { + for (j = 0; j < 16; j++, k++) { + unsigned int pval = accumulator[k] + (count[k] >> 1); + pval *= fixed_divide[count[k]]; + pval >>= 19; + + dst1_16[byte] = (uint16_t)pval; + + // move to next pixel + byte++; + } + + byte += stride - 16; + } + + dst1 = cpi->alt_ref_buffer.u_buffer; + dst2 = cpi->alt_ref_buffer.v_buffer; + dst1_16 = CONVERT_TO_SHORTPTR(dst1); + dst2_16 = CONVERT_TO_SHORTPTR(dst2); + stride = cpi->alt_ref_buffer.uv_stride; + byte = mb_uv_offset; + for (i = 0, k = 256; i < mb_uv_height; i++) { + for (j = 0; j < mb_uv_width; j++, k++) { + int m = k + 256; + + // U + unsigned int pval = accumulator[k] + (count[k] >> 1); + pval *= fixed_divide[count[k]]; + pval >>= 19; + dst1_16[byte] = (uint16_t)pval; + + // V + pval = accumulator[m] + (count[m] >> 1); + pval *= fixed_divide[count[m]]; + pval >>= 19; + dst2_16[byte] = (uint16_t)pval; + + // move to next pixel + byte++; + } + + byte += stride - mb_uv_width; + } + } else { + // Normalize filter output to produce AltRef frame + dst1 = cpi->alt_ref_buffer.y_buffer; + stride = cpi->alt_ref_buffer.y_stride; + byte = mb_y_offset; + for (i = 0, k = 0; i < 16; i++) { + for (j = 0; j < 16; j++, k++) { + unsigned int pval = accumulator[k] + (count[k] >> 1); + pval *= fixed_divide[count[k]]; + pval >>= 19; + + dst1[byte] = (uint8_t)pval; + + // move to next pixel + byte++; + } + byte += stride - 16; + } + + dst1 = cpi->alt_ref_buffer.u_buffer; + dst2 = cpi->alt_ref_buffer.v_buffer; + stride = cpi->alt_ref_buffer.uv_stride; + byte = mb_uv_offset; + for (i = 0, k = 256; i < mb_uv_height; i++) { + for (j = 0; j < mb_uv_width; j++, k++) { + int m = k + 256; + + // U + unsigned int pval = accumulator[k] + (count[k] >> 1); + pval *= fixed_divide[count[k]]; + pval >>= 19; + dst1[byte] = (uint8_t)pval; + + // V + pval = accumulator[m] + (count[m] >> 1); + pval *= fixed_divide[count[m]]; + pval >>= 19; + dst2[byte] = (uint8_t)pval; + + // move to next pixel + byte++; + } + byte += stride - mb_uv_width; + } + } +#else // Normalize filter output to produce AltRef frame dst1 = cpi->alt_ref_buffer.y_buffer; stride = cpi->alt_ref_buffer.y_stride; @@ -347,6 +573,7 @@ static void temporal_filter_iterate_c(VP9_COMP *cpi, } byte += stride - mb_uv_width; } +#endif // CONFIG_VP9_HIGHBITDEPTH mb_y_offset += 16; mb_uv_offset += mb_uv_width; } @@ -467,7 +694,8 @@ void vp9_temporal_filter(VP9_COMP *cpi, int distance) { get_frame_new_buffer(cm)->y_crop_height, get_frame_new_buffer(cm)->y_crop_width, get_frame_new_buffer(cm)->y_crop_height); -#endif +#endif // CONFIG_VP9_HIGHBITDEPTH + for (frame = 0; frame < frames_to_blur; ++frame) { if (cm->mi_cols * MI_SIZE != frames[frame]->y_width || cm->mi_rows * MI_SIZE != frames[frame]->y_height) { @@ -502,7 +730,7 @@ void vp9_temporal_filter(VP9_COMP *cpi, int distance) { frames[0]->y_crop_height, frames[0]->y_crop_width, frames[0]->y_crop_height); -#endif +#endif // CONFIG_VP9_HIGHBITDEPTH } temporal_filter_iterate_c(cpi, frames, frames_to_blur, diff --git a/vp9/vp9_common.mk b/vp9/vp9_common.mk index 07a3be8db..9414120f6 100644 --- a/vp9/vp9_common.mk +++ b/vp9/vp9_common.mk @@ -93,6 +93,7 @@ ifeq ($(CONFIG_VP9_HIGHBITDEPTH),yes) VP9_COMMON_SRCS-$(HAVE_SSE2) += common/x86/vp9_high_intrapred_sse2.asm VP9_COMMON_SRCS-$(HAVE_SSE2) += common/x86/vp9_high_subpixel_8t_sse2.asm VP9_COMMON_SRCS-$(HAVE_SSE2) += common/x86/vp9_high_subpixel_bilinear_sse2.asm +VP9_COMMON_SRCS-$(HAVE_SSE2) += common/x86/vp9_high_loopfilter_intrin_sse2.c endif # common (c) -- 2.40.0