eob = vec_max_across(eob);
*eob_ptr = eob[0] + 1;
}
+
+// Sets the value of a 32-bit integers to 1 when the corresponding value in a is
+// negative.
+static INLINE int32x4_t vec_is_neg(int32x4_t a) {
+ return vec_sr(a, vec_shift_sign_s32);
+}
+
+// DeQuantization function used for 32x32 blocks. Quantized coeff of 32x32
+// blocks are twice as big as for other block sizes. As such, using
+// vec_mladd results in overflow.
+static INLINE int16x8_t dequantize_coeff_32(int16x8_t qcoeff,
+ int16x8_t dequant) {
+ int16x8_t dqcoeff;
+ int32x4_t dqcoeffe = vec_mule(qcoeff, dequant);
+ int32x4_t dqcoeffo = vec_mulo(qcoeff, dequant);
+ // Add 1 if negative to round towards zero because the C uses division.
+ dqcoeffe = vec_add(dqcoeffe, vec_is_neg(dqcoeffe));
+ dqcoeffo = vec_add(dqcoeffo, vec_is_neg(dqcoeffo));
+ dqcoeffe = vec_sra(dqcoeffe, vec_ones_u32);
+ dqcoeffo = vec_sra(dqcoeffo, vec_ones_u32);
+ dqcoeff = vec_pack(dqcoeffe, dqcoeffo);
+ return vec_perm(dqcoeff, dqcoeff, vec_perm_merge);
+}
+
+void vp9_quantize_fp_32x32_vsx(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
+ int skip_block, const int16_t *round_ptr,
+ const int16_t *quant_ptr, tran_low_t *qcoeff_ptr,
+ tran_low_t *dqcoeff_ptr,
+ const int16_t *dequant_ptr, uint16_t *eob_ptr,
+ const int16_t *scan_ptr,
+ const int16_t *iscan_ptr) {
+ // In stage 1, we quantize 16 coeffs (DC + 15 AC)
+ // In stage 2, we loop 42 times and quantize 24 coeffs per iteration
+ // (32 * 32 - 16) / 24 = 42
+ int num_itr = 42;
+ // Offsets are in bytes, 16 coeffs = 32 bytes
+ int off0 = 32;
+ int off1 = 48;
+ int off2 = 64;
+
+ int16x8_t qcoeff0, qcoeff1, dqcoeff0, dqcoeff1, eob;
+ bool16x8_t mask0, mask1, zero_coeff0, zero_coeff1;
+
+ int16x8_t round = vec_vsx_ld(0, round_ptr);
+ int16x8_t quant = vec_vsx_ld(0, quant_ptr);
+ int16x8_t dequant = vec_vsx_ld(0, dequant_ptr);
+ int16x8_t coeff0 = vec_vsx_ld(0, coeff_ptr);
+ int16x8_t coeff1 = vec_vsx_ld(16, coeff_ptr);
+ int16x8_t scan0 = vec_vsx_ld(0, iscan_ptr);
+ int16x8_t scan1 = vec_vsx_ld(16, iscan_ptr);
+ int16x8_t thres = vec_sra(dequant, vec_splats((uint16_t)2));
+ int16x8_t abs_coeff0 = vec_abs(coeff0);
+ int16x8_t abs_coeff1 = vec_abs(coeff1);
+
+ (void)scan_ptr;
+ (void)skip_block;
+ (void)n_coeffs;
+ assert(!skip_block);
+
+ mask0 = vec_cmpge(abs_coeff0, thres);
+ round = vec_sra(vec_add(round, vec_ones_s16), vec_ones_u16);
+ // First set of 8 coeff starts with DC + 7 AC
+ qcoeff0 = vec_madds(vec_vaddshs(abs_coeff0, round), quant, vec_zeros_s16);
+ qcoeff0 = vec_and(qcoeff0, mask0);
+ zero_coeff0 = vec_cmpeq(qcoeff0, vec_zeros_s16);
+ qcoeff0 = vec_sign(qcoeff0, coeff0);
+ vec_vsx_st(qcoeff0, 0, qcoeff_ptr);
+
+ dqcoeff0 = dequantize_coeff_32(qcoeff0, dequant);
+ vec_vsx_st(dqcoeff0, 0, dqcoeff_ptr);
+
+ // Remove DC value from thres, round, quant and dequant
+ thres = vec_splat(thres, 1);
+ round = vec_splat(round, 1);
+ quant = vec_splat(quant, 1);
+ dequant = vec_splat(dequant, 1);
+
+ mask1 = vec_cmpge(abs_coeff1, thres);
+
+ // Second set of 8 coeff starts with (all AC)
+ qcoeff1 =
+ vec_madds(vec_vaddshs(vec_abs(coeff1), round), quant, vec_zeros_s16);
+ qcoeff1 = vec_and(qcoeff1, mask1);
+ zero_coeff1 = vec_cmpeq(qcoeff1, vec_zeros_s16);
+ qcoeff1 = vec_sign(qcoeff1, coeff1);
+ vec_vsx_st(qcoeff1, 16, qcoeff_ptr);
+
+ dqcoeff1 = dequantize_coeff_32(qcoeff1, dequant);
+ vec_vsx_st(dqcoeff1, 16, dqcoeff_ptr);
+
+ eob = vec_max(vec_or(scan0, zero_coeff0), vec_or(scan1, zero_coeff1));
+
+ do {
+ int16x8_t coeff2, abs_coeff2, qcoeff2, dqcoeff2, eob2, scan2;
+ bool16x8_t zero_coeff2, mask2;
+ coeff0 = vec_vsx_ld(off0, coeff_ptr);
+ coeff1 = vec_vsx_ld(off1, coeff_ptr);
+ coeff2 = vec_vsx_ld(off2, coeff_ptr);
+ scan0 = vec_vsx_ld(off0, iscan_ptr);
+ scan1 = vec_vsx_ld(off1, iscan_ptr);
+ scan2 = vec_vsx_ld(off2, iscan_ptr);
+
+ abs_coeff0 = vec_abs(coeff0);
+ abs_coeff1 = vec_abs(coeff1);
+ abs_coeff2 = vec_abs(coeff2);
+
+ qcoeff0 = vec_madds(vec_vaddshs(abs_coeff0, round), quant, vec_zeros_s16);
+ qcoeff1 = vec_madds(vec_vaddshs(abs_coeff1, round), quant, vec_zeros_s16);
+ qcoeff2 = vec_madds(vec_vaddshs(abs_coeff2, round), quant, vec_zeros_s16);
+
+ mask0 = vec_cmpge(abs_coeff0, thres);
+ mask1 = vec_cmpge(abs_coeff1, thres);
+ mask2 = vec_cmpge(abs_coeff2, thres);
+
+ qcoeff0 = vec_and(qcoeff0, mask0);
+ qcoeff1 = vec_and(qcoeff1, mask1);
+ qcoeff2 = vec_and(qcoeff2, mask2);
+
+ zero_coeff0 = vec_cmpeq(qcoeff0, vec_zeros_s16);
+ zero_coeff1 = vec_cmpeq(qcoeff1, vec_zeros_s16);
+ zero_coeff2 = vec_cmpeq(qcoeff2, vec_zeros_s16);
+
+ qcoeff0 = vec_sign(qcoeff0, coeff0);
+ qcoeff1 = vec_sign(qcoeff1, coeff1);
+ qcoeff2 = vec_sign(qcoeff2, coeff2);
+
+ vec_vsx_st(qcoeff0, off0, qcoeff_ptr);
+ vec_vsx_st(qcoeff1, off1, qcoeff_ptr);
+ vec_vsx_st(qcoeff2, off2, qcoeff_ptr);
+
+ dqcoeff0 = dequantize_coeff_32(qcoeff0, dequant);
+ dqcoeff1 = dequantize_coeff_32(qcoeff1, dequant);
+ dqcoeff2 = dequantize_coeff_32(qcoeff2, dequant);
+
+ vec_vsx_st(dqcoeff0, off0, dqcoeff_ptr);
+ vec_vsx_st(dqcoeff1, off1, dqcoeff_ptr);
+ vec_vsx_st(dqcoeff2, off2, dqcoeff_ptr);
+
+ eob = vec_max(eob, vec_or(scan0, zero_coeff0));
+ eob2 = vec_max(vec_or(scan1, zero_coeff1), vec_or(scan2, zero_coeff2));
+ eob = vec_max(eob, eob2);
+
+ off0 += 48;
+ off1 += 48;
+ off2 += 48;
+ num_itr--;
+ } while (num_itr != 0);
+
+ eob = vec_max_across(eob);
+ *eob_ptr = eob[0] + 1;
+}
projects / libvpx / commitdiff