ppc: Add vpx_sadnxmx4d_vsx for n,m = {8, 16, 32 ,64}

[libvpx] / vpx_dsp / arm / idct16x16_add_neon.c

diff --git a/vpx_dsp/arm/idct16x16_add_neon.c b/vpx_dsp/arm/idct16x16_add_neon.c
index b2f516f41208054605e20b2f0790585496702760..828fb5f6c71f89446233d4239b4352e719ebe22c 100644 (file)
--- a/vpx_dsp/arm/idct16x16_add_neon.c
+++ b/vpx_dsp/arm/idct16x16_add_neon.c
@@ -62,42 +62,6 @@ static INLINE void idct_cospi_16_16_d(const int16x4_t s0, const int16x4_t s1,
   wrap_low_4x2(t32, d0, d1);
 }
 
-static INLINE void idct16x16_store_pass1(const int16x8_t *const out,
-                                         int16_t *output) {
-  // Save the result into output
-  vst1q_s16(output, out[0]);
-  output += 16;
-  vst1q_s16(output, out[1]);
-  output += 16;
-  vst1q_s16(output, out[2]);
-  output += 16;
-  vst1q_s16(output, out[3]);
-  output += 16;
-  vst1q_s16(output, out[4]);
-  output += 16;
-  vst1q_s16(output, out[5]);
-  output += 16;
-  vst1q_s16(output, out[6]);
-  output += 16;
-  vst1q_s16(output, out[7]);
-  output += 16;
-  vst1q_s16(output, out[8]);
-  output += 16;
-  vst1q_s16(output, out[9]);
-  output += 16;
-  vst1q_s16(output, out[10]);
-  output += 16;
-  vst1q_s16(output, out[11]);
-  output += 16;
-  vst1q_s16(output, out[12]);
-  output += 16;
-  vst1q_s16(output, out[13]);
-  output += 16;
-  vst1q_s16(output, out[14]);
-  output += 16;
-  vst1q_s16(output, out[15]);
-}
-
 static INLINE void idct16x16_add_store(const int16x8_t *const out,
                                        uint8_t *dest, const int stride) {
   // Add the result to dest
@@ -119,15 +83,53 @@ static INLINE void idct16x16_add_store(const int16x8_t *const out,
   idct16x16_add8x1(out[15], &dest, stride);
 }
 
-void idct16x16_256_add_half1d(const void *const input, int16_t *output,
-                              void *const dest, const int stride,
-                              const int highbd_flag) {
+static INLINE void idct16x16_add_store_bd8(int16x8_t *const out, uint16_t *dest,
+                                           const int stride) {
+  // Add the result to dest
+  const int16x8_t max = vdupq_n_s16((1 << 8) - 1);
+  out[0] = vrshrq_n_s16(out[0], 6);
+  out[1] = vrshrq_n_s16(out[1], 6);
+  out[2] = vrshrq_n_s16(out[2], 6);
+  out[3] = vrshrq_n_s16(out[3], 6);
+  out[4] = vrshrq_n_s16(out[4], 6);
+  out[5] = vrshrq_n_s16(out[5], 6);
+  out[6] = vrshrq_n_s16(out[6], 6);
+  out[7] = vrshrq_n_s16(out[7], 6);
+  out[8] = vrshrq_n_s16(out[8], 6);
+  out[9] = vrshrq_n_s16(out[9], 6);
+  out[10] = vrshrq_n_s16(out[10], 6);
+  out[11] = vrshrq_n_s16(out[11], 6);
+  out[12] = vrshrq_n_s16(out[12], 6);
+  out[13] = vrshrq_n_s16(out[13], 6);
+  out[14] = vrshrq_n_s16(out[14], 6);
+  out[15] = vrshrq_n_s16(out[15], 6);
+  highbd_idct16x16_add8x1(out[0], max, &dest, stride);
+  highbd_idct16x16_add8x1(out[1], max, &dest, stride);
+  highbd_idct16x16_add8x1(out[2], max, &dest, stride);
+  highbd_idct16x16_add8x1(out[3], max, &dest, stride);
+  highbd_idct16x16_add8x1(out[4], max, &dest, stride);
+  highbd_idct16x16_add8x1(out[5], max, &dest, stride);
+  highbd_idct16x16_add8x1(out[6], max, &dest, stride);
+  highbd_idct16x16_add8x1(out[7], max, &dest, stride);
+  highbd_idct16x16_add8x1(out[8], max, &dest, stride);
+  highbd_idct16x16_add8x1(out[9], max, &dest, stride);
+  highbd_idct16x16_add8x1(out[10], max, &dest, stride);
+  highbd_idct16x16_add8x1(out[11], max, &dest, stride);
+  highbd_idct16x16_add8x1(out[12], max, &dest, stride);
+  highbd_idct16x16_add8x1(out[13], max, &dest, stride);
+  highbd_idct16x16_add8x1(out[14], max, &dest, stride);
+  highbd_idct16x16_add8x1(out[15], max, &dest, stride);
+}
+
+void vpx_idct16x16_256_add_half1d(const void *const input, int16_t *output,
+                                  void *const dest, const int stride,
+                                  const int highbd_flag) {
   const int16x8_t cospis0 = vld1q_s16(kCospi);
   const int16x8_t cospis1 = vld1q_s16(kCospi + 8);
   const int16x4_t cospi_0_8_16_24 = vget_low_s16(cospis0);
   const int16x4_t cospi_4_12_20N_28 = vget_high_s16(cospis0);
   const int16x4_t cospi_2_30_10_22 = vget_low_s16(cospis1);
-  const int16x4_t cospi_6_26_14_18N = vget_high_s16(cospis1);
+  const int16x4_t cospi_6_26N_14_18N = vget_high_s16(cospis1);
   int16x8_t in[16], step1[16], step2[16], out[16];
 
   // Load input (16x8)
@@ -233,11 +235,11 @@ void idct16x16_256_add_half1d(const void *const input, int16_t *output,
   step2[6] = step1[6];
   step2[7] = step1[7];
   idct_cospi_2_30(step1[8], step1[15], cospi_2_30_10_22, &step2[8], &step2[15]);
-  idct_cospi_14_18(step1[9], step1[14], cospi_6_26_14_18N, &step2[9],
+  idct_cospi_14_18(step1[9], step1[14], cospi_6_26N_14_18N, &step2[9],
                    &step2[14]);
   idct_cospi_10_22(step1[10], step1[13], cospi_2_30_10_22, &step2[10],
                    &step2[13]);
-  idct_cospi_6_26(step1[11], step1[12], cospi_6_26_14_18N, &step2[11],
+  idct_cospi_6_26(step1[11], step1[12], cospi_6_26N_14_18N, &step2[11],
                   &step2[12]);
 
   // stage 3
@@ -314,49 +316,16 @@ void idct16x16_256_add_half1d(const void *const input, int16_t *output,
     idct16x16_store_pass1(out, output);
   } else {
     if (highbd_flag) {
-      // pass 2: add the result to dest.
-      const int16x8_t max = vdupq_n_s16((1 << 8) - 1);
-      uint16_t *destT = dest;
-      out[0] = vrshrq_n_s16(out[0], 6);
-      out[1] = vrshrq_n_s16(out[1], 6);
-      out[2] = vrshrq_n_s16(out[2], 6);
-      out[3] = vrshrq_n_s16(out[3], 6);
-      out[4] = vrshrq_n_s16(out[4], 6);
-      out[5] = vrshrq_n_s16(out[5], 6);
-      out[6] = vrshrq_n_s16(out[6], 6);
-      out[7] = vrshrq_n_s16(out[7], 6);
-      out[8] = vrshrq_n_s16(out[8], 6);
-      out[9] = vrshrq_n_s16(out[9], 6);
-      out[10] = vrshrq_n_s16(out[10], 6);
-      out[11] = vrshrq_n_s16(out[11], 6);
-      out[12] = vrshrq_n_s16(out[12], 6);
-      out[13] = vrshrq_n_s16(out[13], 6);
-      out[14] = vrshrq_n_s16(out[14], 6);
-      out[15] = vrshrq_n_s16(out[15], 6);
-      highbd_idct16x16_add8x1(out[0], max, &destT, stride);
-      highbd_idct16x16_add8x1(out[1], max, &destT, stride);
-      highbd_idct16x16_add8x1(out[2], max, &destT, stride);
-      highbd_idct16x16_add8x1(out[3], max, &destT, stride);
-      highbd_idct16x16_add8x1(out[4], max, &destT, stride);
-      highbd_idct16x16_add8x1(out[5], max, &destT, stride);
-      highbd_idct16x16_add8x1(out[6], max, &destT, stride);
-      highbd_idct16x16_add8x1(out[7], max, &destT, stride);
-      highbd_idct16x16_add8x1(out[8], max, &destT, stride);
-      highbd_idct16x16_add8x1(out[9], max, &destT, stride);
-      highbd_idct16x16_add8x1(out[10], max, &destT, stride);
-      highbd_idct16x16_add8x1(out[11], max, &destT, stride);
-      highbd_idct16x16_add8x1(out[12], max, &destT, stride);
-      highbd_idct16x16_add8x1(out[13], max, &destT, stride);
-      highbd_idct16x16_add8x1(out[14], max, &destT, stride);
-      highbd_idct16x16_add8x1(out[15], max, &destT, stride);
+      idct16x16_add_store_bd8(out, dest, stride);
     } else {
       idct16x16_add_store(out, dest, stride);
     }
   }
 }
 
-static void idct16x16_38_add_half1d(const void *const input, int16_t *output,
-                                    uint8_t *dest, const int stride) {
+void vpx_idct16x16_38_add_half1d(const void *const input, int16_t *const output,
+                                 void *const dest, const int stride,
+                                 const int highbd_flag) {
   const int16x8_t cospis0 = vld1q_s16(kCospi);
   const int16x8_t cospis1 = vld1q_s16(kCospi + 8);
   const int16x8_t cospisd0 = vaddq_s16(cospis0, cospis0);
@@ -507,12 +476,16 @@ static void idct16x16_38_add_half1d(const void *const input, int16_t *output,
   if (output) {
     idct16x16_store_pass1(out, output);
   } else {
-    idct16x16_add_store(out, dest, stride);
+    if (highbd_flag) {
+      idct16x16_add_store_bd8(out, dest, stride);
+    } else {
+      idct16x16_add_store(out, dest, stride);
+    }
   }
 }
 
-static void idct16x16_10_add_half1d_pass1(const tran_low_t *input,
-                                          int16_t *output) {
+void vpx_idct16x16_10_add_half1d_pass1(const tran_low_t *input,
+                                       int16_t *output) {
   const int16x8_t cospis0 = vld1q_s16(kCospi);
   const int16x8_t cospis1 = vld1q_s16(kCospi + 8);
   const int16x8_t cospisd0 = vaddq_s16(cospis0, cospis0);
@@ -665,8 +638,10 @@ static void idct16x16_10_add_half1d_pass1(const tran_low_t *input,
   vst1_s16(output, out[15]);
 }
 
-static void idct16x16_10_add_half1d_pass2(const int16_t *input, int16_t *output,
-                                          uint8_t *dest, const int stride) {
+void vpx_idct16x16_10_add_half1d_pass2(const int16_t *input,
+                                       int16_t *const output, void *const dest,
+                                       const int stride,
+                                       const int highbd_flag) {
   const int16x8_t cospis0 = vld1q_s16(kCospi);
   const int16x8_t cospis1 = vld1q_s16(kCospi + 8);
   const int16x8_t cospisd0 = vaddq_s16(cospis0, cospis0);
@@ -778,27 +753,16 @@ static void idct16x16_10_add_half1d_pass2(const int16_t *input, int16_t *output,
   step2[15] = step1[15];
 
   // stage 7
-  out[0] = vaddq_s16(step2[0], step2[15]);
-  out[1] = vaddq_s16(step2[1], step2[14]);
-  out[2] = vaddq_s16(step2[2], step2[13]);
-  out[3] = vaddq_s16(step2[3], step2[12]);
-  out[4] = vaddq_s16(step2[4], step2[11]);
-  out[5] = vaddq_s16(step2[5], step2[10]);
-  out[6] = vaddq_s16(step2[6], step2[9]);
-  out[7] = vaddq_s16(step2[7], step2[8]);
-  out[8] = vsubq_s16(step2[7], step2[8]);
-  out[9] = vsubq_s16(step2[6], step2[9]);
-  out[10] = vsubq_s16(step2[5], step2[10]);
-  out[11] = vsubq_s16(step2[4], step2[11]);
-  out[12] = vsubq_s16(step2[3], step2[12]);
-  out[13] = vsubq_s16(step2[2], step2[13]);
-  out[14] = vsubq_s16(step2[1], step2[14]);
-  out[15] = vsubq_s16(step2[0], step2[15]);
+  idct16x16_add_stage7(step2, out);
 
   if (output) {
     idct16x16_store_pass1(out, output);
   } else {
-    idct16x16_add_store(out, dest, stride);
+    if (highbd_flag) {
+      idct16x16_add_store_bd8(out, dest, stride);
+    } else {
+      idct16x16_add_store(out, dest, stride);
+    }
   }
 }
 
@@ -808,18 +772,19 @@ void vpx_idct16x16_256_add_neon(const tran_low_t *input, uint8_t *dest,
 
   // pass 1
   // Parallel idct on the upper 8 rows
-  idct16x16_256_add_half1d(input, row_idct_output, dest, stride, 0);
+  vpx_idct16x16_256_add_half1d(input, row_idct_output, dest, stride, 0);
 
   // Parallel idct on the lower 8 rows
-  idct16x16_256_add_half1d(input + 8 * 16, row_idct_output + 8, dest, stride,
-                           0);
+  vpx_idct16x16_256_add_half1d(input + 8 * 16, row_idct_output + 8, dest,
+                               stride, 0);
 
   // pass 2
   // Parallel idct to get the left 8 columns
-  idct16x16_256_add_half1d(row_idct_output, NULL, dest, stride, 0);
+  vpx_idct16x16_256_add_half1d(row_idct_output, NULL, dest, stride, 0);
 
   // Parallel idct to get the right 8 columns
-  idct16x16_256_add_half1d(row_idct_output + 16 * 8, NULL, dest + 8, stride, 0);
+  vpx_idct16x16_256_add_half1d(row_idct_output + 16 * 8, NULL, dest + 8, stride,
+                               0);
 }
 
 void vpx_idct16x16_38_add_neon(const tran_low_t *input, uint8_t *dest,
@@ -828,14 +793,15 @@ void vpx_idct16x16_38_add_neon(const tran_low_t *input, uint8_t *dest,
 
   // pass 1
   // Parallel idct on the upper 8 rows
-  idct16x16_38_add_half1d(input, row_idct_output, dest, stride);
+  vpx_idct16x16_38_add_half1d(input, row_idct_output, dest, stride, 0);
 
   // pass 2
   // Parallel idct to get the left 8 columns
-  idct16x16_38_add_half1d(row_idct_output, NULL, dest, stride);
+  vpx_idct16x16_38_add_half1d(row_idct_output, NULL, dest, stride, 0);
 
   // Parallel idct to get the right 8 columns
-  idct16x16_38_add_half1d(row_idct_output + 16 * 8, NULL, dest + 8, stride);
+  vpx_idct16x16_38_add_half1d(row_idct_output + 16 * 8, NULL, dest + 8, stride,
+                              0);
 }
 
 void vpx_idct16x16_10_add_neon(const tran_low_t *input, uint8_t *dest,
@@ -844,13 +810,13 @@ void vpx_idct16x16_10_add_neon(const tran_low_t *input, uint8_t *dest,
 
   // pass 1
   // Parallel idct on the upper 8 rows
-  idct16x16_10_add_half1d_pass1(input, row_idct_output);
+  vpx_idct16x16_10_add_half1d_pass1(input, row_idct_output);
 
   // pass 2
   // Parallel idct to get the left 8 columns
-  idct16x16_10_add_half1d_pass2(row_idct_output, NULL, dest, stride);
+  vpx_idct16x16_10_add_half1d_pass2(row_idct_output, NULL, dest, stride, 0);
 
   // Parallel idct to get the right 8 columns
-  idct16x16_10_add_half1d_pass2(row_idct_output + 4 * 8, NULL, dest + 8,
-                                stride);
+  vpx_idct16x16_10_add_half1d_pass2(row_idct_output + 4 * 8, NULL, dest + 8,
+                                    stride, 0);
 }