From 766d875b9da5c088d62fa50c37194c6e0dc0861a Mon Sep 17 00:00:00 2001 From: Luc Trudeau Date: Mon, 28 May 2018 15:04:57 -0400 Subject: [PATCH] VSX version of vpx_mbpost_proc_ip MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Low bit depth version only. Passes the VpxMbPostProcAcrossIpTest. VpxMbPostProcAcrossIpTest Speed Test (POWER8 Model 2.1) C time = 188.5ms (±0.2ms), VSX time = 65.2ms (±0.1ms) [2.9x] Change-Id: I1cf72365d94a9d7f1e9323925a87a30e3bd5cfe2 --- test/pp_filter_test.cc | 49 ++++++---- vpx_dsp/ppc/deblock_vsx.c | 184 +++++++++++++++++++++++++++++++---- vpx_dsp/ppc/types_vsx.h | 3 + vpx_dsp/vpx_dsp_rtcd_defs.pl | 2 +- 4 files changed, 202 insertions(+), 36 deletions(-) diff --git a/test/pp_filter_test.cc b/test/pp_filter_test.cc index 13e975dfa..ebed196c8 100644 --- a/test/pp_filter_test.cc +++ b/test/pp_filter_test.cc @@ -164,11 +164,21 @@ TEST_P(VpxPostProcDownAndAcrossMbRowTest, CheckCvsAssembly) { } class VpxMbPostProcAcrossIpTest - : public ::testing::TestWithParam { + : public AbstractBench, + public ::testing::TestWithParam { public: + VpxMbPostProcAcrossIpTest() + : rows(16), cols(16), mbPostProcAcrossIp(GetParam()), + src(Buffer(rows, cols, 8, 8, 17, 8)) {} virtual void TearDown() { libvpx_test::ClearSystemState(); } protected: + const int rows; + const int cols; + const VpxMbPostProcAcrossIpFunc mbPostProcAcrossIp; + Buffer src; + void run(); + void SetCols(unsigned char *s, int rows, int cols, int src_width) { for (int r = 0; r < rows; r++) { for (int c = 0; c < cols; c++) { @@ -197,11 +207,11 @@ class VpxMbPostProcAcrossIpTest } }; -TEST_P(VpxMbPostProcAcrossIpTest, CheckLowFilterOutput) { - const int rows = 16; - const int cols = 16; +void VpxMbPostProcAcrossIpTest::run() { + mbPostProcAcrossIp(src.TopLeftPixel(), src.stride(), rows, cols, q2mbl(0)); +} - Buffer src = Buffer(cols, rows, 8, 8, 17, 8); +TEST_P(VpxMbPostProcAcrossIpTest, CheckLowFilterOutput) { ASSERT_TRUE(src.Init()); src.SetPadding(10); SetCols(src.TopLeftPixel(), rows, cols, src.stride()); @@ -215,15 +225,11 @@ TEST_P(VpxMbPostProcAcrossIpTest, CheckLowFilterOutput) { } TEST_P(VpxMbPostProcAcrossIpTest, CheckMediumFilterOutput) { - const int rows = 16; - const int cols = 16; - - Buffer src = Buffer(cols, rows, 8, 8, 17, 8); ASSERT_TRUE(src.Init()); src.SetPadding(10); SetCols(src.TopLeftPixel(), rows, cols, src.stride()); - static const unsigned char kExpectedOutput[cols] = { + static const unsigned char kExpectedOutput[] = { 2, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 13 }; @@ -232,15 +238,11 @@ TEST_P(VpxMbPostProcAcrossIpTest, CheckMediumFilterOutput) { } TEST_P(VpxMbPostProcAcrossIpTest, CheckHighFilterOutput) { - const int rows = 16; - const int cols = 16; - - Buffer src = Buffer(cols, rows, 8, 8, 17, 8); ASSERT_TRUE(src.Init()); src.SetPadding(10); SetCols(src.TopLeftPixel(), rows, cols, src.stride()); - static const unsigned char kExpectedOutput[cols] = { + static const unsigned char kExpectedOutput[] = { 2, 2, 3, 4, 4, 5, 6, 7, 8, 9, 10, 11, 11, 12, 13, 13 }; @@ -254,9 +256,6 @@ TEST_P(VpxMbPostProcAcrossIpTest, CheckHighFilterOutput) { } TEST_P(VpxMbPostProcAcrossIpTest, CheckCvsAssembly) { - const int rows = 16; - const int cols = 16; - Buffer c_mem = Buffer(cols, rows, 8, 8, 17, 8); ASSERT_TRUE(c_mem.Init()); Buffer asm_mem = Buffer(cols, rows, 8, 8, 17, 8); @@ -279,6 +278,16 @@ TEST_P(VpxMbPostProcAcrossIpTest, CheckCvsAssembly) { } } +TEST_P(VpxMbPostProcAcrossIpTest, DISABLED_Speed) { + ASSERT_TRUE(src.Init()); + src.SetPadding(10); + + SetCols(src.TopLeftPixel(), rows, cols, src.stride()); + + runNTimes(100000); + printMedian("16x16"); +} + class VpxMbPostProcDownTest : public AbstractBench, public ::testing::TestWithParam { @@ -491,6 +500,10 @@ INSTANTIATE_TEST_CASE_P(MSA, VpxMbPostProcDownTest, #endif // HAVE_MSA #if HAVE_VSX + +INSTANTIATE_TEST_CASE_P(VSX, VpxMbPostProcAcrossIpTest, + ::testing::Values(vpx_mbpost_proc_across_ip_vsx)); + INSTANTIATE_TEST_CASE_P(VSX, VpxMbPostProcDownTest, ::testing::Values(vpx_mbpost_proc_down_vsx)); #endif // HAVE_VSX diff --git a/vpx_dsp/ppc/deblock_vsx.c b/vpx_dsp/ppc/deblock_vsx.c index 4a031832c..de19a4601 100644 --- a/vpx_dsp/ppc/deblock_vsx.c +++ b/vpx_dsp/ppc/deblock_vsx.c @@ -19,9 +19,170 @@ static const uint8x16_t load_merge = { 0x00, 0x02, 0x04, 0x06, 0x08, 0x0A, 0x0C, 0x0E, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F }; -static const uint8x16_t mask_merge = { 0x00, 0x01, 0x10, 0x11, 0x04, 0x05, - 0x14, 0x15, 0x08, 0x09, 0x18, 0x19, - 0x0C, 0x0D, 0x1C, 0x1D }; +// C: s[c + 7] +static INLINE int16x8_t next7l_s16(uint8x16_t c) { + static const uint8x16_t next7_perm = { + 0x07, 0x10, 0x08, 0x11, 0x09, 0x12, 0x0A, 0x13, + 0x0B, 0x14, 0x0C, 0x15, 0x0D, 0x16, 0x0E, 0x17, + }; + return (int16x8_t)vec_perm(c, vec_zeros_u8, next7_perm); +} + +// Slide across window and add. +static INLINE int16x8_t slide_sum_s16(int16x8_t x) { + // x = A B C D E F G H + // + // 0 A B C D E F G + const int16x8_t sum1 = vec_add(x, vec_slo(x, vec_splats((int8_t)(2 << 3)))); + // 0 0 A B C D E F + const int16x8_t sum2 = vec_add(vec_slo(x, vec_splats((int8_t)(4 << 3))), + // 0 0 0 A B C D E + vec_slo(x, vec_splats((int8_t)(6 << 3)))); + // 0 0 0 0 A B C D + const int16x8_t sum3 = vec_add(vec_slo(x, vec_splats((int8_t)(8 << 3))), + // 0 0 0 0 0 A B C + vec_slo(x, vec_splats((int8_t)(10 << 3)))); + // 0 0 0 0 0 0 A B + const int16x8_t sum4 = vec_add(vec_slo(x, vec_splats((int8_t)(12 << 3))), + // 0 0 0 0 0 0 0 A + vec_slo(x, vec_splats((int8_t)(14 << 3)))); + return vec_add(vec_add(sum1, sum2), vec_add(sum3, sum4)); +} + +// Slide across window and add. +static INLINE int32x4_t slide_sumsq_s32(int32x4_t xsq_even, int32x4_t xsq_odd) { + // 0 A C E + // + 0 B D F + int32x4_t sumsq_1 = vec_add(vec_slo(xsq_even, vec_splats((int8_t)(4 << 3))), + vec_slo(xsq_odd, vec_splats((int8_t)(4 << 3)))); + // 0 0 A C + // + 0 0 B D + int32x4_t sumsq_2 = vec_add(vec_slo(xsq_even, vec_splats((int8_t)(8 << 3))), + vec_slo(xsq_odd, vec_splats((int8_t)(8 << 3)))); + // 0 0 0 A + // + 0 0 0 B + int32x4_t sumsq_3 = vec_add(vec_slo(xsq_even, vec_splats((int8_t)(12 << 3))), + vec_slo(xsq_odd, vec_splats((int8_t)(12 << 3)))); + sumsq_1 = vec_add(sumsq_1, xsq_even); + sumsq_2 = vec_add(sumsq_2, sumsq_3); + return vec_add(sumsq_1, sumsq_2); +} + +// C: (b + sum + val) >> 4 +static INLINE int16x8_t filter_s16(int16x8_t b, int16x8_t sum, int16x8_t val) { + return vec_sra(vec_add(vec_add(b, sum), val), vec_splats((uint16_t)4)); +} + +// C: sumsq * 15 - sum * sum +static INLINE bool16x8_t mask_s16(int32x4_t sumsq_even, int32x4_t sumsq_odd, + int16x8_t sum, int32x4_t lim) { + static const uint8x16_t mask_merge = { 0x00, 0x01, 0x10, 0x11, 0x04, 0x05, + 0x14, 0x15, 0x08, 0x09, 0x18, 0x19, + 0x0C, 0x0D, 0x1C, 0x1D }; + const int32x4_t sumsq_odd_scaled = + vec_mul(sumsq_odd, vec_splats((int32_t)15)); + const int32x4_t sumsq_even_scaled = + vec_mul(sumsq_even, vec_splats((int32_t)15)); + const int32x4_t thres_odd = vec_sub(sumsq_odd_scaled, vec_mulo(sum, sum)); + const int32x4_t thres_even = vec_sub(sumsq_even_scaled, vec_mule(sum, sum)); + + const bool32x4_t mask_odd = vec_cmplt(thres_odd, lim); + const bool32x4_t mask_even = vec_cmplt(thres_even, lim); + return vec_perm((bool16x8_t)mask_even, (bool16x8_t)mask_odd, mask_merge); +} + +void vpx_mbpost_proc_across_ip_vsx(unsigned char *src, int pitch, int rows, + int cols, int flimit) { + int row, col; + const int32x4_t lim = vec_splats(flimit); + + // 8 columns are processed at a time. + assert(cols % 8 == 0); + + for (row = 0; row < rows; row++) { + // The sum is signed and requires at most 13 bits. + // (8 bits + sign) * 15 (4 bits) + int16x8_t sum; + // The sum of squares requires at most 20 bits. + // (16 bits + sign) * 15 (4 bits) + int32x4_t sumsq_even, sumsq_odd; + + // Fill left context with first col. + int16x8_t left_ctx = vec_splats((int16_t)src[0]); + int16_t s = src[0] * 9; + int32_t ssq = src[0] * src[0] * 9 + 16; + + // Fill the next 6 columns of the sliding window with cols 2 to 7. + for (col = 1; col <= 6; ++col) { + s += src[col]; + ssq += src[col] * src[col]; + } + // Set this sum to every element in the window. + sum = vec_splats(s); + sumsq_even = vec_splats(ssq); + sumsq_odd = vec_splats(ssq); + + for (col = 0; col < cols; col += 8) { + bool16x8_t mask; + int16x8_t filtered, masked; + uint8x16_t out; + + const uint8x16_t val = vec_vsx_ld(0, src + col); + const int16x8_t val_high = unpack_to_s16_h(val); + + // C: s[c + 7] + const int16x8_t right_ctx = (col + 8 == cols) + ? vec_splats((int16_t)src[col + 7]) + : next7l_s16(val); + + // C: x = s[c + 7] - s[c - 8]; + const int16x8_t x = vec_sub(right_ctx, left_ctx); + const int32x4_t xsq_even = + vec_sub(vec_mule(right_ctx, right_ctx), vec_mule(left_ctx, left_ctx)); + const int32x4_t xsq_odd = + vec_sub(vec_mulo(right_ctx, right_ctx), vec_mulo(left_ctx, left_ctx)); + + const int32x4_t sumsq_tmp = slide_sumsq_s32(xsq_even, xsq_odd); + // A C E G + // 0 B D F + // 0 A C E + // 0 0 B D + // 0 0 A C + // 0 0 0 B + // 0 0 0 A + sumsq_even = vec_add(sumsq_even, sumsq_tmp); + // B D F G + // A C E G + // 0 B D F + // 0 A C E + // 0 0 B D + // 0 0 A C + // 0 0 0 B + // 0 0 0 A + sumsq_odd = vec_add(sumsq_odd, vec_add(sumsq_tmp, xsq_odd)); + + sum = vec_add(sum, slide_sum_s16(x)); + + // C: (8 + sum + s[c]) >> 4 + filtered = filter_s16(vec_splats((int16_t)8), sum, val_high); + // C: sumsq * 15 - sum * sum + mask = mask_s16(sumsq_even, sumsq_odd, sum, lim); + masked = vec_sel(val_high, filtered, mask); + + out = vec_perm((uint8x16_t)masked, vec_vsx_ld(0, src + col), load_merge); + vec_vsx_st(out, 0, src + col); + + // Update window sum and square sum + sum = vec_splat(sum, 7); + sumsq_even = vec_splat(sumsq_odd, 3); + sumsq_odd = vec_splat(sumsq_odd, 3); + + // C: s[c - 8] (for next iteration) + left_ctx = val_high; + } + src += pitch; + } +} void vpx_mbpost_proc_down_vsx(uint8_t *dst, int pitch, int rows, int cols, int flimit) { @@ -66,9 +227,6 @@ void vpx_mbpost_proc_down_vsx(uint8_t *dst, int pitch, int rows, int cols, for (row = 0; row < rows; row++) { int32x4_t d15_even, d15_odd, d0_even, d0_odd; - int32x4_t sumsq_odd_scaled, sumsq_even_scaled; - int32x4_t thres_odd, thres_even; - bool32x4_t mask_odd, mask_even; bool16x8_t mask; int16x8_t filtered, masked; uint8x16_t out; @@ -96,19 +254,11 @@ void vpx_mbpost_proc_down_vsx(uint8_t *dst, int pitch, int rows, int cols, sumsq_odd = vec_add(sumsq_odd, vec_sub(d15_odd, d0_odd)); sumsq_even = vec_add(sumsq_even, vec_sub(d15_even, d0_even)); - // C: sumsq * 15 - sum * sum - sumsq_odd_scaled = vec_mul(sumsq_odd, vec_splats((int32_t)15)); - sumsq_even_scaled = vec_mul(sumsq_even, vec_splats((int32_t)15)); - thres_odd = vec_sub(sumsq_odd_scaled, vec_mulo(sum, sum)); - thres_even = vec_sub(sumsq_even_scaled, vec_mule(sum, sum)); - // C: (vpx_rv[(r & 127) + (c & 7)] + sum + s[0]) >> 4 - filtered = vec_add(vec_add(rv, sum), window[8]); - filtered = vec_sra(filtered, vec_splats((uint16_t)4)); + filtered = filter_s16(rv, sum, window[8]); - mask_odd = vec_cmplt(thres_odd, lim); - mask_even = vec_cmplt(thres_even, lim); - mask = vec_perm((bool16x8_t)mask_even, (bool16x8_t)mask_odd, mask_merge); + // C: sumsq * 15 - sum * sum + mask = mask_s16(sumsq_even, sumsq_odd, sum, lim); masked = vec_sel(window[8], filtered, mask); // TODO(ltrudeau) If cols % 16 == 0, we could just process 16 per diff --git a/vpx_dsp/ppc/types_vsx.h b/vpx_dsp/ppc/types_vsx.h index 9e7476b58..ca925f940 100644 --- a/vpx_dsp/ppc/types_vsx.h +++ b/vpx_dsp/ppc/types_vsx.h @@ -67,10 +67,13 @@ static const uint8x16_t xxpermdi3_perm = { 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, #endif #endif +static const uint8x16_t vec_zeros_u8 = { 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 }; static const int16x8_t vec_zeros_s16 = { 0, 0, 0, 0, 0, 0, 0, 0 }; static const int16x8_t vec_ones_s16 = { 1, 1, 1, 1, 1, 1, 1, 1 }; static const uint16x8_t vec_ones_u16 = { 1, 1, 1, 1, 1, 1, 1, 1 }; static const uint32x4_t vec_ones_u32 = { 1, 1, 1, 1 }; +static const int32x4_t vec_zeros_s32 = { 0, 0, 0, 0 }; static const uint16x8_t vec_shift_sign_s16 = { 15, 15, 15, 15, 15, 15, 15, 15 }; static const uint32x4_t vec_shift_sign_s32 = { 31, 31, 31, 31 }; static const uint8x16_t vec_perm64 = { 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, diff --git a/vpx_dsp/vpx_dsp_rtcd_defs.pl b/vpx_dsp/vpx_dsp_rtcd_defs.pl index ff57d87cb..52aabb4fd 100644 --- a/vpx_dsp/vpx_dsp_rtcd_defs.pl +++ b/vpx_dsp/vpx_dsp_rtcd_defs.pl @@ -1601,7 +1601,7 @@ if (vpx_config("CONFIG_POSTPROC") eq "yes" || vpx_config("CONFIG_VP9_POSTPROC") specialize qw/vpx_mbpost_proc_down sse2 neon msa vsx/; add_proto qw/void vpx_mbpost_proc_across_ip/, "unsigned char *dst, int pitch, int rows, int cols,int flimit"; - specialize qw/vpx_mbpost_proc_across_ip sse2 neon msa/; + specialize qw/vpx_mbpost_proc_across_ip sse2 neon msa vsx/; add_proto qw/void vpx_post_proc_down_and_across_mb_row/, "unsigned char *src, unsigned char *dst, int src_pitch, int dst_pitch, int cols, unsigned char *flimits, int size"; specialize qw/vpx_post_proc_down_and_across_mb_row sse2 neon msa/; -- 2.40.0