From: Yi Luo Date: Wed, 1 Mar 2017 00:38:41 +0000 (-0800) Subject: Improve idct32x32_34_add SSSE3 intrinsics performance X-Git-Tag: v1.7.0~661 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=07c48ccfe;p=libvpx Improve idct32x32_34_add SSSE3 intrinsics performance - Split the transform into first half and second half. - Reschedule the instructions to avoid stack spillover. - Function level speed improves ~16%. Change-Id: I166889840d23aa8a273eca00f6fbdae8b4566f35 --- diff --git a/vpx_dsp/x86/inv_txfm_ssse3.c b/vpx_dsp/x86/inv_txfm_ssse3.c index cfa6a732a..e6789a465 100644 --- a/vpx_dsp/x86/inv_txfm_ssse3.c +++ b/vpx_dsp/x86/inv_txfm_ssse3.c @@ -378,18 +378,71 @@ void vpx_idct8x8_12_add_ssse3(const tran_low_t *input, uint8_t *dest, RECON_AND_STORE(dest + 7 * stride, in7); } -static INLINE void idct32_34(const __m128i *in, __m128i *stp1) { +static INLINE void simple_group_butterfly(const __m128i *in, __m128i *out) { + int i = 0; + while (i < 16) { + out[i] = _mm_add_epi16(in[i], in[31 - i]); + out[31 - i] = _mm_sub_epi16(in[i], in[31 - i]); + i++; + } +} + +static INLINE void butterfly(const __m128i x0, const __m128i x1, + const __m128i c0, const __m128i c1, __m128i *y0, + __m128i *y1) { + __m128i tmp0, tmp1, tmp2, tmp3, u0, u1; const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING); - // idct constants for each stage - const __m128i stk1_0 = pair_set_epi16(2 * cospi_31_64, 2 * cospi_31_64); - const __m128i stk1_1 = pair_set_epi16(2 * cospi_1_64, 2 * cospi_1_64); - const __m128i stk1_6 = pair_set_epi16(-2 * cospi_25_64, -2 * cospi_25_64); - const __m128i stk1_7 = pair_set_epi16(2 * cospi_7_64, 2 * cospi_7_64); - const __m128i stk1_8 = pair_set_epi16(2 * cospi_27_64, 2 * cospi_27_64); - const __m128i stk1_9 = pair_set_epi16(2 * cospi_5_64, 2 * cospi_5_64); - const __m128i stk1_14 = pair_set_epi16(-2 * cospi_29_64, -2 * cospi_29_64); - const __m128i stk1_15 = pair_set_epi16(2 * cospi_3_64, 2 * cospi_3_64); + u0 = _mm_unpacklo_epi16(x0, x1); + u1 = _mm_unpackhi_epi16(x0, x1); + + tmp0 = _mm_madd_epi16(u0, c0); + tmp1 = _mm_madd_epi16(u1, c0); + tmp2 = _mm_madd_epi16(u0, c1); + tmp3 = _mm_madd_epi16(u1, c1); + + tmp0 = _mm_add_epi32(tmp0, rounding); + tmp1 = _mm_add_epi32(tmp1, rounding); + tmp2 = _mm_add_epi32(tmp2, rounding); + tmp3 = _mm_add_epi32(tmp3, rounding); + + tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS); + tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS); + tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); + tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS); + + *y0 = _mm_packs_epi32(tmp0, tmp1); + *y1 = _mm_packs_epi32(tmp2, tmp3); +} + +static INLINE void butterfly_self(__m128i *x0, __m128i *x1, const __m128i c0, + const __m128i c1) { + __m128i tmp0, tmp1, tmp2, tmp3, u0, u1; + const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING); + + u0 = _mm_unpacklo_epi16(*x0, *x1); + u1 = _mm_unpackhi_epi16(*x0, *x1); + + tmp0 = _mm_madd_epi16(u0, c0); + tmp1 = _mm_madd_epi16(u1, c0); + tmp2 = _mm_madd_epi16(u0, c1); + tmp3 = _mm_madd_epi16(u1, c1); + + tmp0 = _mm_add_epi32(tmp0, rounding); + tmp1 = _mm_add_epi32(tmp1, rounding); + tmp2 = _mm_add_epi32(tmp2, rounding); + tmp3 = _mm_add_epi32(tmp3, rounding); + + tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS); + tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS); + tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); + tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS); + + *x0 = _mm_packs_epi32(tmp0, tmp1); + *x1 = _mm_packs_epi32(tmp2, tmp3); +} + +static void idct32_34_first_half(const __m128i *in, __m128i *stp1) { const __m128i stk2_0 = pair_set_epi16(2 * cospi_30_64, 2 * cospi_30_64); const __m128i stk2_1 = pair_set_epi16(2 * cospi_2_64, 2 * cospi_2_64); const __m128i stk2_6 = pair_set_epi16(-2 * cospi_26_64, -2 * cospi_26_64); @@ -397,6 +450,94 @@ static INLINE void idct32_34(const __m128i *in, __m128i *stp1) { const __m128i stk3_0 = pair_set_epi16(2 * cospi_28_64, 2 * cospi_28_64); const __m128i stk3_1 = pair_set_epi16(2 * cospi_4_64, 2 * cospi_4_64); + + const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64); + const __m128i stk4_0 = pair_set_epi16(2 * cospi_16_64, 2 * cospi_16_64); + const __m128i stg4_1 = pair_set_epi16(cospi_16_64, -cospi_16_64); + const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64); + const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64); + const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64); + + const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64); + __m128i u0, u1, u2, u3, u4, u5, u6, u7; + __m128i x0, x1, x4, x5, x6, x7; + __m128i v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15; + + // phase 1 + + // 0, 15 + u2 = _mm_mulhrs_epi16(in[2], stk2_1); // stp2_15 + u3 = _mm_mulhrs_epi16(in[6], stk2_7); // stp2_12 + v15 = _mm_add_epi16(u2, u3); + // in[0], in[4] + x0 = _mm_mulhrs_epi16(in[0], stk4_0); // stp1[0] + x7 = _mm_mulhrs_epi16(in[4], stk3_1); // stp1[7] + v0 = _mm_add_epi16(x0, x7); // stp2_0 + stp1[0] = _mm_add_epi16(v0, v15); + stp1[15] = _mm_sub_epi16(v0, v15); + + // in[2], in[6] + u0 = _mm_mulhrs_epi16(in[2], stk2_0); // stp2_8 + u1 = _mm_mulhrs_epi16(in[6], stk2_6); // stp2_11 + butterfly(u0, u2, stg4_4, stg4_5, &u4, &u5); // stp2_9, stp2_14 + butterfly(u1, u3, stg4_6, stg4_4, &u6, &u7); // stp2_10, stp2_13 + + v8 = _mm_add_epi16(u0, u1); + v9 = _mm_add_epi16(u4, u6); + v10 = _mm_sub_epi16(u4, u6); + v11 = _mm_sub_epi16(u0, u1); + v12 = _mm_sub_epi16(u2, u3); + v13 = _mm_sub_epi16(u5, u7); + v14 = _mm_add_epi16(u5, u7); + + butterfly_self(&v10, &v13, stg6_0, stg4_0); + butterfly_self(&v11, &v12, stg6_0, stg4_0); + + // 1, 14 + x1 = _mm_mulhrs_epi16(in[0], stk4_0); // stp1[1], stk4_1 = stk4_0 + // stp1[2] = stp1[0], stp1[3] = stp1[1] + x4 = _mm_mulhrs_epi16(in[4], stk3_0); // stp1[4] + butterfly(x7, x4, stg4_1, stg4_0, &x5, &x6); + v1 = _mm_add_epi16(x1, x6); // stp2_1 + v2 = _mm_add_epi16(x0, x5); // stp2_2 + stp1[1] = _mm_add_epi16(v1, v14); + stp1[14] = _mm_sub_epi16(v1, v14); + + stp1[2] = _mm_add_epi16(v2, v13); + stp1[13] = _mm_sub_epi16(v2, v13); + + v3 = _mm_add_epi16(x1, x4); // stp2_3 + v4 = _mm_sub_epi16(x1, x4); // stp2_4 + + v5 = _mm_sub_epi16(x0, x5); // stp2_5 + + v6 = _mm_sub_epi16(x1, x6); // stp2_6 + v7 = _mm_sub_epi16(x0, x7); // stp2_7 + stp1[3] = _mm_add_epi16(v3, v12); + stp1[12] = _mm_sub_epi16(v3, v12); + + stp1[6] = _mm_add_epi16(v6, v9); + stp1[9] = _mm_sub_epi16(v6, v9); + + stp1[7] = _mm_add_epi16(v7, v8); + stp1[8] = _mm_sub_epi16(v7, v8); + + stp1[4] = _mm_add_epi16(v4, v11); + stp1[11] = _mm_sub_epi16(v4, v11); + + stp1[5] = _mm_add_epi16(v5, v10); + stp1[10] = _mm_sub_epi16(v5, v10); +} + +static void idct32_34_second_half(const __m128i *in, __m128i *stp1) { + const __m128i stk1_0 = pair_set_epi16(2 * cospi_31_64, 2 * cospi_31_64); + const __m128i stk1_1 = pair_set_epi16(2 * cospi_1_64, 2 * cospi_1_64); + const __m128i stk1_6 = pair_set_epi16(-2 * cospi_25_64, -2 * cospi_25_64); + const __m128i stk1_7 = pair_set_epi16(2 * cospi_7_64, 2 * cospi_7_64); + const __m128i stk1_8 = pair_set_epi16(2 * cospi_27_64, 2 * cospi_27_64); + const __m128i stk1_9 = pair_set_epi16(2 * cospi_5_64, 2 * cospi_5_64); + const __m128i stk1_14 = pair_set_epi16(-2 * cospi_29_64, -2 * cospi_29_64); + const __m128i stk1_15 = pair_set_epi16(2 * cospi_3_64, 2 * cospi_3_64); const __m128i stg3_4 = pair_set_epi16(-cospi_4_64, cospi_28_64); const __m128i stg3_5 = pair_set_epi16(cospi_28_64, cospi_4_64); const __m128i stg3_6 = pair_set_epi16(-cospi_28_64, -cospi_4_64); @@ -405,301 +546,83 @@ static INLINE void idct32_34(const __m128i *in, __m128i *stp1) { const __m128i stg3_10 = pair_set_epi16(-cospi_12_64, -cospi_20_64); const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64); - const __m128i stk4_0 = pair_set_epi16(2 * cospi_16_64, 2 * cospi_16_64); - const __m128i stg4_1 = pair_set_epi16(cospi_16_64, -cospi_16_64); const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64); const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64); const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64); const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64); - __m128i stp2_0, stp2_1, stp2_2, stp2_3, stp2_4, stp2_5, stp2_6, stp2_7, - stp2_8, stp2_9, stp2_10, stp2_11, stp2_12, stp2_13, stp2_14, stp2_15, - stp2_16, stp2_17, stp2_18, stp2_19, stp2_20, stp2_21, stp2_22, stp2_23, - stp2_24, stp2_25, stp2_26, stp2_27, stp2_28, stp2_29, stp2_30, stp2_31; - __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; - - /* Stage1 */ - - stp1[16] = _mm_mulhrs_epi16(in[1], stk1_0); - stp1[31] = _mm_mulhrs_epi16(in[1], stk1_1); - - stp1[19] = _mm_mulhrs_epi16(in[7], stk1_6); - stp1[28] = _mm_mulhrs_epi16(in[7], stk1_7); - - stp1[20] = _mm_mulhrs_epi16(in[5], stk1_8); - stp1[27] = _mm_mulhrs_epi16(in[5], stk1_9); - - stp1[23] = _mm_mulhrs_epi16(in[3], stk1_14); - stp1[24] = _mm_mulhrs_epi16(in[3], stk1_15); - - /* Stage2 */ - - stp2_8 = _mm_mulhrs_epi16(in[2], stk2_0); - stp2_15 = _mm_mulhrs_epi16(in[2], stk2_1); - - stp2_11 = _mm_mulhrs_epi16(in[6], stk2_6); - stp2_12 = _mm_mulhrs_epi16(in[6], stk2_7); - - /* Stage3 */ - { - const __m128i lo_17_30 = _mm_unpacklo_epi16(stp1[16], stp1[31]); - const __m128i hi_17_30 = _mm_unpackhi_epi16(stp1[16], stp1[31]); - const __m128i lo_18_29 = _mm_unpacklo_epi16(stp1[19], stp1[28]); - const __m128i hi_18_29 = _mm_unpackhi_epi16(stp1[19], stp1[28]); - - const __m128i lo_21_26 = _mm_unpacklo_epi16(stp1[20], stp1[27]); - const __m128i hi_21_26 = _mm_unpackhi_epi16(stp1[20], stp1[27]); - const __m128i lo_22_25 = _mm_unpacklo_epi16(stp1[23], stp1[24]); - const __m128i hi_22_25 = _mm_unpackhi_epi16(stp1[23], stp1[24]); - - stp1[4] = _mm_mulhrs_epi16(in[4], stk3_0); - stp1[7] = _mm_mulhrs_epi16(in[4], stk3_1); - - MULTIPLICATION_AND_ADD(lo_17_30, hi_17_30, lo_18_29, hi_18_29, stg3_4, - stg3_5, stg3_6, stg3_4, stp1[17], stp1[30], stp1[18], - stp1[29]) - MULTIPLICATION_AND_ADD(lo_21_26, hi_21_26, lo_22_25, hi_22_25, stg3_8, - stg3_9, stg3_10, stg3_8, stp1[21], stp1[26], - stp1[22], stp1[25]) - } - - /* Stage4 */ - { - const __m128i lo_9_14 = _mm_unpacklo_epi16(stp2_8, stp2_15); - const __m128i hi_9_14 = _mm_unpackhi_epi16(stp2_8, stp2_15); - const __m128i lo_10_13 = _mm_unpacklo_epi16(stp2_11, stp2_12); - const __m128i hi_10_13 = _mm_unpackhi_epi16(stp2_11, stp2_12); - - stp1[0] = _mm_mulhrs_epi16(in[0], stk4_0); - stp1[1] = _mm_mulhrs_epi16(in[0], stk4_0); // stk4_1 = stk4_0 - stp1[2] = stp1[0]; - stp1[3] = stp1[1]; - - MULTIPLICATION_AND_ADD(lo_9_14, hi_9_14, lo_10_13, hi_10_13, stg4_4, stg4_5, - stg4_6, stg4_4, stp2_9, stp2_14, stp2_10, stp2_13) - - stp2_16 = _mm_add_epi16(stp1[16], stp1[19]); - stp2_17 = _mm_add_epi16(stp1[17], stp1[18]); - stp2_18 = _mm_sub_epi16(stp1[17], stp1[18]); - stp2_19 = _mm_sub_epi16(stp1[16], stp1[19]); - stp2_20 = _mm_sub_epi16(stp1[23], stp1[20]); - stp2_21 = _mm_sub_epi16(stp1[22], stp1[21]); - stp2_22 = _mm_add_epi16(stp1[22], stp1[21]); - stp2_23 = _mm_add_epi16(stp1[23], stp1[20]); - - stp2_24 = _mm_add_epi16(stp1[24], stp1[27]); - stp2_25 = _mm_add_epi16(stp1[25], stp1[26]); - stp2_26 = _mm_sub_epi16(stp1[25], stp1[26]); - stp2_27 = _mm_sub_epi16(stp1[24], stp1[27]); - stp2_28 = _mm_sub_epi16(stp1[31], stp1[28]); - stp2_29 = _mm_sub_epi16(stp1[30], stp1[29]); - stp2_30 = _mm_add_epi16(stp1[29], stp1[30]); - stp2_31 = _mm_add_epi16(stp1[28], stp1[31]); - } - - /* Stage5 */ - { -// Note: -// #define AVOID_OVERFLOW = 0, code would be faster. But it can't pass -// SingleExtreme test. The MaxSupportedCoeff/MinSupportedCoeff must drop -// to 23198 and -23197, respectively. -#define AVOID_OVERFLOW (1) - -#if AVOID_OVERFLOW - const __m128i lo_6_5 = _mm_unpacklo_epi16(stp1[7], stp1[4]); - const __m128i hi_6_5 = _mm_unpackhi_epi16(stp1[7], stp1[4]); -#endif - const __m128i lo_18_29 = _mm_unpacklo_epi16(stp2_18, stp2_29); - const __m128i hi_18_29 = _mm_unpackhi_epi16(stp2_18, stp2_29); - - const __m128i lo_19_28 = _mm_unpacklo_epi16(stp2_19, stp2_28); - const __m128i hi_19_28 = _mm_unpackhi_epi16(stp2_19, stp2_28); - const __m128i lo_20_27 = _mm_unpacklo_epi16(stp2_20, stp2_27); - const __m128i hi_20_27 = _mm_unpackhi_epi16(stp2_20, stp2_27); - - const __m128i lo_21_26 = _mm_unpacklo_epi16(stp2_21, stp2_26); - const __m128i hi_21_26 = _mm_unpackhi_epi16(stp2_21, stp2_26); - -#if AVOID_OVERFLOW - tmp0 = _mm_madd_epi16(lo_6_5, stg4_1); - tmp1 = _mm_madd_epi16(hi_6_5, stg4_1); - tmp2 = _mm_madd_epi16(lo_6_5, stg4_0); - tmp3 = _mm_madd_epi16(hi_6_5, stg4_0); - - tmp0 = _mm_add_epi32(tmp0, rounding); - tmp1 = _mm_add_epi32(tmp1, rounding); - tmp2 = _mm_add_epi32(tmp2, rounding); - tmp3 = _mm_add_epi32(tmp3, rounding); - - tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS); - tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS); - tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); - tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS); - - stp1[5] = _mm_packs_epi32(tmp0, tmp1); - stp1[6] = _mm_packs_epi32(tmp2, tmp3); -#else - tmp0 = _mm_sub_epi16(stp1[7], stp1[4]); - tmp1 = _mm_adds_epi16(stp1[7], stp1[4]); - stp1[5] = _mm_mulhrs_epi16(tmp0, stk4_0); - stp1[6] = _mm_mulhrs_epi16(tmp1, stk4_0); -#endif - - stp1[8] = _mm_add_epi16(stp2_8, stp2_11); - stp1[9] = _mm_add_epi16(stp2_9, stp2_10); - stp1[10] = _mm_sub_epi16(stp2_9, stp2_10); - stp1[11] = _mm_sub_epi16(stp2_8, stp2_11); - stp1[12] = _mm_sub_epi16(stp2_15, stp2_12); - stp1[13] = _mm_sub_epi16(stp2_14, stp2_13); - stp1[14] = _mm_add_epi16(stp2_14, stp2_13); - stp1[15] = _mm_add_epi16(stp2_15, stp2_12); - - MULTIPLICATION_AND_ADD(lo_18_29, hi_18_29, lo_19_28, hi_19_28, stg4_4, - stg4_5, stg4_4, stg4_5, stp1[18], stp1[29], stp1[19], - stp1[28]) - MULTIPLICATION_AND_ADD(lo_20_27, hi_20_27, lo_21_26, hi_21_26, stg4_6, - stg4_4, stg4_6, stg4_4, stp1[20], stp1[27], stp1[21], - stp1[26]) - - stp1[16] = stp2_16; - stp1[17] = stp2_17; - stp1[22] = stp2_22; - stp1[23] = stp2_23; - stp1[24] = stp2_24; - stp1[25] = stp2_25; - stp1[30] = stp2_30; - stp1[31] = stp2_31; - } - - /* Stage6 */ - { -#if AVOID_OVERFLOW - const __m128i lo_10_13 = _mm_unpacklo_epi16(stp1[10], stp1[13]); - const __m128i hi_10_13 = _mm_unpackhi_epi16(stp1[10], stp1[13]); - const __m128i lo_11_12 = _mm_unpacklo_epi16(stp1[11], stp1[12]); - const __m128i hi_11_12 = _mm_unpackhi_epi16(stp1[11], stp1[12]); -#endif - - stp2_0 = _mm_add_epi16(stp1[0], stp1[7]); - stp2_1 = _mm_add_epi16(stp1[1], stp1[6]); - stp2_2 = _mm_add_epi16(stp1[2], stp1[5]); - stp2_3 = _mm_add_epi16(stp1[3], stp1[4]); - stp2_4 = _mm_sub_epi16(stp1[3], stp1[4]); - stp2_5 = _mm_sub_epi16(stp1[2], stp1[5]); - stp2_6 = _mm_sub_epi16(stp1[1], stp1[6]); - stp2_7 = _mm_sub_epi16(stp1[0], stp1[7]); - - stp2_8 = stp1[8]; - stp2_9 = stp1[9]; - stp2_14 = stp1[14]; - stp2_15 = stp1[15]; - -#if AVOID_OVERFLOW - MULTIPLICATION_AND_ADD(lo_10_13, hi_10_13, lo_11_12, hi_11_12, stg6_0, - stg4_0, stg6_0, stg4_0, stp2_10, stp2_13, stp2_11, - stp2_12) -#else - tmp0 = _mm_add_epi16(stp1[10], stp1[13]); - tmp1 = _mm_sub_epi16(stp1[13], stp1[10]); - tmp2 = _mm_add_epi16(stp1[11], stp1[12]); - tmp3 = _mm_sub_epi16(stp1[12], stp1[11]); - - stp2_10 = _mm_mulhrs_epi16(tmp1, stk4_0); - stp2_13 = _mm_mulhrs_epi16(tmp0, stk4_0); - stp2_11 = _mm_mulhrs_epi16(tmp3, stk4_0); - stp2_12 = _mm_mulhrs_epi16(tmp2, stk4_0); - -#endif - - stp2_16 = _mm_add_epi16(stp1[16], stp1[23]); - stp2_17 = _mm_add_epi16(stp1[17], stp1[22]); - stp2_18 = _mm_add_epi16(stp1[18], stp1[21]); - stp2_19 = _mm_add_epi16(stp1[19], stp1[20]); - stp2_20 = _mm_sub_epi16(stp1[19], stp1[20]); - stp2_21 = _mm_sub_epi16(stp1[18], stp1[21]); - stp2_22 = _mm_sub_epi16(stp1[17], stp1[22]); - stp2_23 = _mm_sub_epi16(stp1[16], stp1[23]); - - stp2_24 = _mm_sub_epi16(stp1[31], stp1[24]); - stp2_25 = _mm_sub_epi16(stp1[30], stp1[25]); - stp2_26 = _mm_sub_epi16(stp1[29], stp1[26]); - stp2_27 = _mm_sub_epi16(stp1[28], stp1[27]); - stp2_28 = _mm_add_epi16(stp1[27], stp1[28]); - stp2_29 = _mm_add_epi16(stp1[26], stp1[29]); - stp2_30 = _mm_add_epi16(stp1[25], stp1[30]); - stp2_31 = _mm_add_epi16(stp1[24], stp1[31]); - } - - /* Stage7 */ - { -#if AVOID_OVERFLOW - const __m128i lo_20_27 = _mm_unpacklo_epi16(stp2_20, stp2_27); - const __m128i hi_20_27 = _mm_unpackhi_epi16(stp2_20, stp2_27); - const __m128i lo_21_26 = _mm_unpacklo_epi16(stp2_21, stp2_26); - const __m128i hi_21_26 = _mm_unpackhi_epi16(stp2_21, stp2_26); - - const __m128i lo_22_25 = _mm_unpacklo_epi16(stp2_22, stp2_25); - const __m128i hi_22_25 = _mm_unpackhi_epi16(stp2_22, stp2_25); - const __m128i lo_23_24 = _mm_unpacklo_epi16(stp2_23, stp2_24); - const __m128i hi_23_24 = _mm_unpackhi_epi16(stp2_23, stp2_24); -#endif - stp1[0] = _mm_add_epi16(stp2_0, stp2_15); - stp1[1] = _mm_add_epi16(stp2_1, stp2_14); - stp1[2] = _mm_add_epi16(stp2_2, stp2_13); - stp1[3] = _mm_add_epi16(stp2_3, stp2_12); - stp1[4] = _mm_add_epi16(stp2_4, stp2_11); - stp1[5] = _mm_add_epi16(stp2_5, stp2_10); - stp1[6] = _mm_add_epi16(stp2_6, stp2_9); - stp1[7] = _mm_add_epi16(stp2_7, stp2_8); - stp1[8] = _mm_sub_epi16(stp2_7, stp2_8); - stp1[9] = _mm_sub_epi16(stp2_6, stp2_9); - stp1[10] = _mm_sub_epi16(stp2_5, stp2_10); - stp1[11] = _mm_sub_epi16(stp2_4, stp2_11); - stp1[12] = _mm_sub_epi16(stp2_3, stp2_12); - stp1[13] = _mm_sub_epi16(stp2_2, stp2_13); - stp1[14] = _mm_sub_epi16(stp2_1, stp2_14); - stp1[15] = _mm_sub_epi16(stp2_0, stp2_15); - - stp1[16] = stp2_16; - stp1[17] = stp2_17; - stp1[18] = stp2_18; - stp1[19] = stp2_19; - -#if AVOID_OVERFLOW - MULTIPLICATION_AND_ADD(lo_20_27, hi_20_27, lo_21_26, hi_21_26, stg6_0, - stg4_0, stg6_0, stg4_0, stp1[20], stp1[27], stp1[21], - stp1[26]) - MULTIPLICATION_AND_ADD(lo_22_25, hi_22_25, lo_23_24, hi_23_24, stg6_0, - stg4_0, stg6_0, stg4_0, stp1[22], stp1[25], stp1[23], - stp1[24]) -#else - tmp0 = _mm_add_epi16(stp2_20, stp2_27); - tmp1 = _mm_sub_epi16(stp2_27, stp2_20); - tmp2 = _mm_add_epi16(stp2_21, stp2_26); - tmp3 = _mm_sub_epi16(stp2_26, stp2_21); - - stp1[20] = _mm_mulhrs_epi16(tmp1, stk4_0); - stp1[27] = _mm_mulhrs_epi16(tmp0, stk4_0); - stp1[21] = _mm_mulhrs_epi16(tmp3, stk4_0); - stp1[26] = _mm_mulhrs_epi16(tmp2, stk4_0); - - tmp0 = _mm_add_epi16(stp2_22, stp2_25); - tmp1 = _mm_sub_epi16(stp2_25, stp2_22); - tmp2 = _mm_add_epi16(stp2_23, stp2_24); - tmp3 = _mm_sub_epi16(stp2_24, stp2_23); - - stp1[22] = _mm_mulhrs_epi16(tmp1, stk4_0); - stp1[25] = _mm_mulhrs_epi16(tmp0, stk4_0); - stp1[23] = _mm_mulhrs_epi16(tmp3, stk4_0); - stp1[24] = _mm_mulhrs_epi16(tmp2, stk4_0); -#endif - - stp1[28] = stp2_28; - stp1[29] = stp2_29; - stp1[30] = stp2_30; - stp1[31] = stp2_31; - } -#undef AVOID_OVERFLOW + __m128i v16, v17, v18, v19, v20, v21, v22, v23; + __m128i v24, v25, v26, v27, v28, v29, v30, v31; + __m128i u16, u17, u18, u19, u20, u21, u22, u23; + __m128i u24, u25, u26, u27, u28, u29, u30, u31; + + v16 = _mm_mulhrs_epi16(in[1], stk1_0); + v31 = _mm_mulhrs_epi16(in[1], stk1_1); + + v19 = _mm_mulhrs_epi16(in[7], stk1_6); + v28 = _mm_mulhrs_epi16(in[7], stk1_7); + + v20 = _mm_mulhrs_epi16(in[5], stk1_8); + v27 = _mm_mulhrs_epi16(in[5], stk1_9); + + v23 = _mm_mulhrs_epi16(in[3], stk1_14); + v24 = _mm_mulhrs_epi16(in[3], stk1_15); + + butterfly(v16, v31, stg3_4, stg3_5, &v17, &v30); + butterfly(v19, v28, stg3_6, stg3_4, &v18, &v29); + butterfly(v20, v27, stg3_8, stg3_9, &v21, &v26); + butterfly(v23, v24, stg3_10, stg3_8, &v22, &v25); + + u16 = _mm_add_epi16(v16, v19); + u17 = _mm_add_epi16(v17, v18); + u18 = _mm_sub_epi16(v17, v18); + u19 = _mm_sub_epi16(v16, v19); + u20 = _mm_sub_epi16(v23, v20); + u21 = _mm_sub_epi16(v22, v21); + u22 = _mm_add_epi16(v22, v21); + u23 = _mm_add_epi16(v23, v20); + u24 = _mm_add_epi16(v24, v27); + u27 = _mm_sub_epi16(v24, v27); + u25 = _mm_add_epi16(v25, v26); + u26 = _mm_sub_epi16(v25, v26); + u28 = _mm_sub_epi16(v31, v28); + u31 = _mm_add_epi16(v28, v31); + u29 = _mm_sub_epi16(v30, v29); + u30 = _mm_add_epi16(v29, v30); + + butterfly_self(&u18, &u29, stg4_4, stg4_5); + butterfly_self(&u19, &u28, stg4_4, stg4_5); + butterfly_self(&u20, &u27, stg4_6, stg4_4); + butterfly_self(&u21, &u26, stg4_6, stg4_4); + + stp1[16] = _mm_add_epi16(u16, u23); + v23 = _mm_sub_epi16(u16, u23); + + stp1[17] = _mm_add_epi16(u17, u22); + v22 = _mm_sub_epi16(u17, u22); + + stp1[18] = _mm_add_epi16(u18, u21); + v21 = _mm_sub_epi16(u18, u21); + + stp1[19] = _mm_add_epi16(u19, u20); + v20 = _mm_sub_epi16(u19, u20); + + v24 = _mm_sub_epi16(u31, u24); + stp1[31] = _mm_add_epi16(u24, u31); + + v25 = _mm_sub_epi16(u30, u25); + stp1[30] = _mm_add_epi16(u25, u30); + + v26 = _mm_sub_epi16(u29, u26); + stp1[29] = _mm_add_epi16(u26, u29); + + v27 = _mm_sub_epi16(u28, u27); + stp1[28] = _mm_add_epi16(u27, u28); + + butterfly(v20, v27, stg6_0, stg4_0, &stp1[20], &stp1[27]); + butterfly(v21, v26, stg6_0, stg4_0, &stp1[21], &stp1[26]); + butterfly(v22, v25, stg6_0, stg4_0, &stp1[22], &stp1[25]); + butterfly(v23, v24, stg6_0, stg4_0, &stp1[23], &stp1[24]); } // Only upper-left 8x8 has non-zero coeff @@ -722,81 +645,20 @@ void vpx_idct32x32_34_add_ssse3(const tran_low_t *input, uint8_t *dest, in[7] = load_input_data(input + 224); array_transpose_8x8(in, in); - idct32_34(in, stp1); + idct32_34_first_half(in, stp1); + idct32_34_second_half(in, stp1); // 1_D: Store 32 intermediate results for each 8x32 block. - col[0] = _mm_add_epi16(stp1[0], stp1[31]); - col[1] = _mm_add_epi16(stp1[1], stp1[30]); - col[2] = _mm_add_epi16(stp1[2], stp1[29]); - col[3] = _mm_add_epi16(stp1[3], stp1[28]); - col[4] = _mm_add_epi16(stp1[4], stp1[27]); - col[5] = _mm_add_epi16(stp1[5], stp1[26]); - col[6] = _mm_add_epi16(stp1[6], stp1[25]); - col[7] = _mm_add_epi16(stp1[7], stp1[24]); - col[8] = _mm_add_epi16(stp1[8], stp1[23]); - col[9] = _mm_add_epi16(stp1[9], stp1[22]); - col[10] = _mm_add_epi16(stp1[10], stp1[21]); - col[11] = _mm_add_epi16(stp1[11], stp1[20]); - col[12] = _mm_add_epi16(stp1[12], stp1[19]); - col[13] = _mm_add_epi16(stp1[13], stp1[18]); - col[14] = _mm_add_epi16(stp1[14], stp1[17]); - col[15] = _mm_add_epi16(stp1[15], stp1[16]); - col[16] = _mm_sub_epi16(stp1[15], stp1[16]); - col[17] = _mm_sub_epi16(stp1[14], stp1[17]); - col[18] = _mm_sub_epi16(stp1[13], stp1[18]); - col[19] = _mm_sub_epi16(stp1[12], stp1[19]); - col[20] = _mm_sub_epi16(stp1[11], stp1[20]); - col[21] = _mm_sub_epi16(stp1[10], stp1[21]); - col[22] = _mm_sub_epi16(stp1[9], stp1[22]); - col[23] = _mm_sub_epi16(stp1[8], stp1[23]); - col[24] = _mm_sub_epi16(stp1[7], stp1[24]); - col[25] = _mm_sub_epi16(stp1[6], stp1[25]); - col[26] = _mm_sub_epi16(stp1[5], stp1[26]); - col[27] = _mm_sub_epi16(stp1[4], stp1[27]); - col[28] = _mm_sub_epi16(stp1[3], stp1[28]); - col[29] = _mm_sub_epi16(stp1[2], stp1[29]); - col[30] = _mm_sub_epi16(stp1[1], stp1[30]); - col[31] = _mm_sub_epi16(stp1[0], stp1[31]); + simple_group_butterfly(stp1, col); for (i = 0; i < 4; i++) { int j; // Transpose 32x8 block to 8x32 block array_transpose_8x8(col + i * 8, in); - idct32_34(in, stp1); + idct32_34_first_half(in, stp1); + idct32_34_second_half(in, stp1); // 2_D: Calculate the results and store them to destination. - in[0] = _mm_add_epi16(stp1[0], stp1[31]); - in[1] = _mm_add_epi16(stp1[1], stp1[30]); - in[2] = _mm_add_epi16(stp1[2], stp1[29]); - in[3] = _mm_add_epi16(stp1[3], stp1[28]); - in[4] = _mm_add_epi16(stp1[4], stp1[27]); - in[5] = _mm_add_epi16(stp1[5], stp1[26]); - in[6] = _mm_add_epi16(stp1[6], stp1[25]); - in[7] = _mm_add_epi16(stp1[7], stp1[24]); - in[8] = _mm_add_epi16(stp1[8], stp1[23]); - in[9] = _mm_add_epi16(stp1[9], stp1[22]); - in[10] = _mm_add_epi16(stp1[10], stp1[21]); - in[11] = _mm_add_epi16(stp1[11], stp1[20]); - in[12] = _mm_add_epi16(stp1[12], stp1[19]); - in[13] = _mm_add_epi16(stp1[13], stp1[18]); - in[14] = _mm_add_epi16(stp1[14], stp1[17]); - in[15] = _mm_add_epi16(stp1[15], stp1[16]); - in[16] = _mm_sub_epi16(stp1[15], stp1[16]); - in[17] = _mm_sub_epi16(stp1[14], stp1[17]); - in[18] = _mm_sub_epi16(stp1[13], stp1[18]); - in[19] = _mm_sub_epi16(stp1[12], stp1[19]); - in[20] = _mm_sub_epi16(stp1[11], stp1[20]); - in[21] = _mm_sub_epi16(stp1[10], stp1[21]); - in[22] = _mm_sub_epi16(stp1[9], stp1[22]); - in[23] = _mm_sub_epi16(stp1[8], stp1[23]); - in[24] = _mm_sub_epi16(stp1[7], stp1[24]); - in[25] = _mm_sub_epi16(stp1[6], stp1[25]); - in[26] = _mm_sub_epi16(stp1[5], stp1[26]); - in[27] = _mm_sub_epi16(stp1[4], stp1[27]); - in[28] = _mm_sub_epi16(stp1[3], stp1[28]); - in[29] = _mm_sub_epi16(stp1[2], stp1[29]); - in[30] = _mm_sub_epi16(stp1[1], stp1[30]); - in[31] = _mm_sub_epi16(stp1[0], stp1[31]); - + simple_group_butterfly(stp1, in); for (j = 0; j < 32; ++j) { // Final rounding and shift in[j] = _mm_adds_epi16(in[j], final_rounding);