2 * Copyright (c) 2015 The WebM project authors. All Rights Reserved.
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
14 #include <emmintrin.h>
15 #include <smmintrin.h>
17 #include "vpx_dsp/vpx_dsp_common.h"
18 #include "vp9/encoder/vp9_encoder.h"
19 #include "vpx_ports/mem.h"
22 # define __likely__(v) __builtin_expect(v, 1)
23 # define __unlikely__(v) __builtin_expect(v, 0)
25 # define __likely__(v) (v)
26 # define __unlikely__(v) (v)
29 static INLINE int_mv pack_int_mv(int16_t row, int16_t col) {
31 result.as_mv.row = row;
32 result.as_mv.col = col;
36 static INLINE MV_JOINT_TYPE get_mv_joint(const int_mv mv) {
37 // This is simplified from the C implementation to utilise that
38 // x->nmvjointsadcost[1] == x->nmvjointsadcost[2] and
39 // x->nmvjointsadcost[1] == x->nmvjointsadcost[3]
40 return mv.as_int == 0 ? 0 : 1;
43 static INLINE int mv_cost(const int_mv mv,
44 const int *joint_cost, int *const comp_cost[2]) {
45 return joint_cost[get_mv_joint(mv)] +
46 comp_cost[0][mv.as_mv.row] + comp_cost[1][mv.as_mv.col];
49 static int mvsad_err_cost(const MACROBLOCK *x, const int_mv mv, const MV *ref,
51 const int_mv diff = pack_int_mv(mv.as_mv.row - ref->row,
52 mv.as_mv.col - ref->col);
53 return ROUND_POWER_OF_TWO(mv_cost(diff, x->nmvjointsadcost,
54 x->nmvsadcost) * error_per_bit, 8);
57 /*****************************************************************************
58 * This function utilises 3 properties of the cost function lookup tables, *
59 * constructed in using 'cal_nmvjointsadcost' and 'cal_nmvsadcosts' in *
61 * For the joint cost: *
62 * - mvjointsadcost[1] == mvjointsadcost[2] == mvjointsadcost[3] *
63 * For the component costs: *
64 * - For all i: mvsadcost[0][i] == mvsadcost[1][i] *
65 * (Equal costs for both components) *
66 * - For all i: mvsadcost[0][i] == mvsadcost[0][-i] *
67 * (Cost function is even) *
68 * If these do not hold, then this function cannot be used without *
69 * modification, in which case you can revert to using the C implementation, *
70 * which does not rely on these properties. *
71 *****************************************************************************/
72 int vp9_diamond_search_sad_avx(const MACROBLOCK *x,
73 const search_site_config *cfg,
74 MV *ref_mv, MV *best_mv, int search_param,
75 int sad_per_bit, int *num00,
76 const vp9_variance_fn_ptr_t *fn_ptr,
77 const MV *center_mv) {
78 const int_mv maxmv = pack_int_mv(x->mv_row_max, x->mv_col_max);
79 const __m128i v_max_mv_w = _mm_set1_epi32(maxmv.as_int);
80 const int_mv minmv = pack_int_mv(x->mv_row_min, x->mv_col_min);
81 const __m128i v_min_mv_w = _mm_set1_epi32(minmv.as_int);
83 const __m128i v_spb_d = _mm_set1_epi32(sad_per_bit);
85 const __m128i v_joint_cost_0_d = _mm_set1_epi32(x->nmvjointsadcost[0]);
86 const __m128i v_joint_cost_1_d = _mm_set1_epi32(x->nmvjointsadcost[1]);
88 // search_param determines the length of the initial step and hence the number
90 // 0 = initial step (MAX_FIRST_STEP) pel
91 // 1 = (MAX_FIRST_STEP/2) pel,
92 // 2 = (MAX_FIRST_STEP/4) pel...
93 const MV *ss_mv = &cfg->ss_mv[cfg->searches_per_step * search_param];
94 const intptr_t *ss_os = &cfg->ss_os[cfg->searches_per_step * search_param];
95 const int tot_steps = cfg->total_steps - search_param;
97 const int_mv fcenter_mv = pack_int_mv(center_mv->row >> 3,
99 const __m128i vfcmv = _mm_set1_epi32(fcenter_mv.as_int);
101 const int ref_row = clamp(ref_mv->row, minmv.as_mv.row, maxmv.as_mv.row);
102 const int ref_col = clamp(ref_mv->col, minmv.as_mv.col, maxmv.as_mv.col);
104 int_mv bmv = pack_int_mv(ref_row, ref_col);
105 int_mv new_bmv = bmv;
106 __m128i v_bmv_w = _mm_set1_epi32(bmv.as_int);
108 const int what_stride = x->plane[0].src.stride;
109 const int in_what_stride = x->e_mbd.plane[0].pre[0].stride;
110 const uint8_t *const what = x->plane[0].src.buf;
111 const uint8_t *const in_what = x->e_mbd.plane[0].pre[0].buf +
112 ref_row * in_what_stride + ref_col;
114 // Work out the start point for the search
115 const uint8_t *best_address = in_what;
116 const uint8_t *new_best_address = best_address;
118 __m128i v_ba_q = _mm_set1_epi64x((intptr_t)best_address);
120 __m128i v_ba_d = _mm_set1_epi32((intptr_t)best_address);
123 unsigned int best_sad;
129 // Check the prerequisite cost function properties that are easy to check
130 // in an assert. See the function-level documentation for details on all
132 assert(x->nmvjointsadcost[1] == x->nmvjointsadcost[2]);
133 assert(x->nmvjointsadcost[1] == x->nmvjointsadcost[3]);
135 // Check the starting position
136 best_sad = fn_ptr->sdf(what, what_stride, in_what, in_what_stride);
137 best_sad += mvsad_err_cost(x, bmv, &fcenter_mv.as_mv, sad_per_bit);
141 for (i = 0, step = 0; step < tot_steps; step++) {
142 for (j = 0; j < cfg->searches_per_step; j += 4, i += 4) {
154 // Compute the candidate motion vectors
155 const __m128i v_ss_mv_w = _mm_loadu_si128((const __m128i*)&ss_mv[i]);
156 const __m128i v_these_mv_w = _mm_add_epi16(v_bmv_w, v_ss_mv_w);
157 // Clamp them to the search bounds
158 __m128i v_these_mv_clamp_w = v_these_mv_w;
159 v_these_mv_clamp_w = _mm_min_epi16(v_these_mv_clamp_w, v_max_mv_w);
160 v_these_mv_clamp_w = _mm_max_epi16(v_these_mv_clamp_w, v_min_mv_w);
161 // The ones that did not change are inside the search area
162 v_inside_d = _mm_cmpeq_epi32(v_these_mv_clamp_w, v_these_mv_w);
164 // If none of them are inside, then move on
165 if (__likely__(_mm_test_all_zeros(v_inside_d, v_inside_d))) {
169 // The inverse mask indicates which of the MVs are outside
170 v_outside_d = _mm_xor_si128(v_inside_d, _mm_set1_epi8(0xff));
171 // Shift right to keep the sign bit clear, we will use this later
172 // to set the cost to the maximum value.
173 v_outside_d = _mm_srli_epi32(v_outside_d, 1);
175 // Compute the difference MV
176 v_diff_mv_w = _mm_sub_epi16(v_these_mv_clamp_w, vfcmv);
177 // We utilise the fact that the cost function is even, and use the
178 // absolute difference. This allows us to use unsigned indexes later
179 // and reduces cache pressure somewhat as only a half of the table
180 // is ever referenced.
181 v_diff_mv_w = _mm_abs_epi16(v_diff_mv_w);
183 // Compute the SIMD pointer offsets.
185 #if ARCH_X86_64 // sizeof(intptr_t) == 8
187 __m128i v_bo10_q = _mm_loadu_si128((const __m128i*)&ss_os[i+0]);
188 __m128i v_bo32_q = _mm_loadu_si128((const __m128i*)&ss_os[i+2]);
189 // Set the ones falling outside to zero
190 v_bo10_q = _mm_and_si128(v_bo10_q,
191 _mm_cvtepi32_epi64(v_inside_d));
192 v_bo32_q = _mm_and_si128(v_bo32_q,
193 _mm_unpackhi_epi32(v_inside_d, v_inside_d));
194 // Compute the candidate addresses
195 v_blocka[0] = _mm_add_epi64(v_ba_q, v_bo10_q);
196 v_blocka[1] = _mm_add_epi64(v_ba_q, v_bo32_q);
197 #else // ARCH_X86 // sizeof(intptr_t) == 4
198 __m128i v_bo_d = _mm_loadu_si128((const __m128i*)&ss_os[i]);
199 v_bo_d = _mm_and_si128(v_bo_d, v_inside_d);
200 v_blocka[0] = _mm_add_epi32(v_ba_d, v_bo_d);
204 fn_ptr->sdx4df(what, what_stride,
205 (const uint8_t **)&v_blocka[0], in_what_stride,
206 (uint32_t*)&v_sad_d);
208 // Look up the component cost of the residual motion vector
210 const int32_t row0 = _mm_extract_epi16(v_diff_mv_w, 0);
211 const int32_t col0 = _mm_extract_epi16(v_diff_mv_w, 1);
212 const int32_t row1 = _mm_extract_epi16(v_diff_mv_w, 2);
213 const int32_t col1 = _mm_extract_epi16(v_diff_mv_w, 3);
214 const int32_t row2 = _mm_extract_epi16(v_diff_mv_w, 4);
215 const int32_t col2 = _mm_extract_epi16(v_diff_mv_w, 5);
216 const int32_t row3 = _mm_extract_epi16(v_diff_mv_w, 6);
217 const int32_t col3 = _mm_extract_epi16(v_diff_mv_w, 7);
219 // Note: This is a use case for vpgather in AVX2
220 const uint32_t cost0 = x->nmvsadcost[0][row0] + x->nmvsadcost[0][col0];
221 const uint32_t cost1 = x->nmvsadcost[0][row1] + x->nmvsadcost[0][col1];
222 const uint32_t cost2 = x->nmvsadcost[0][row2] + x->nmvsadcost[0][col2];
223 const uint32_t cost3 = x->nmvsadcost[0][row3] + x->nmvsadcost[0][col3];
225 __m128i v_cost_10_d, v_cost_32_d;
227 v_cost_10_d = _mm_cvtsi32_si128(cost0);
228 v_cost_10_d = _mm_insert_epi32(v_cost_10_d, cost1, 1);
230 v_cost_32_d = _mm_cvtsi32_si128(cost2);
231 v_cost_32_d = _mm_insert_epi32(v_cost_32_d, cost3, 1);
233 v_cost_d = _mm_unpacklo_epi64(v_cost_10_d, v_cost_32_d);
236 // Now add in the joint cost
238 const __m128i v_sel_d = _mm_cmpeq_epi32(v_diff_mv_w,
239 _mm_setzero_si128());
240 const __m128i v_joint_cost_d = _mm_blendv_epi8(v_joint_cost_1_d,
243 v_cost_d = _mm_add_epi32(v_cost_d, v_joint_cost_d);
246 // Multiply by sad_per_bit
247 v_cost_d = _mm_mullo_epi32(v_cost_d, v_spb_d);
248 // ROUND_POWER_OF_TWO(v_cost_d, 8)
249 v_cost_d = _mm_add_epi32(v_cost_d, _mm_set1_epi32(0x80));
250 v_cost_d = _mm_srai_epi32(v_cost_d, 8);
251 // Add the cost to the sad
252 v_sad_d = _mm_add_epi32(v_sad_d, v_cost_d);
254 // Make the motion vectors outside the search area have max cost
255 // by or'ing in the comparison mask, this way the minimum search won't
257 v_sad_d = _mm_or_si128(v_sad_d, v_outside_d);
259 // Find the minimum value and index horizontally in v_sad_d
261 // Try speculatively on 16 bits, so we can use the minpos intrinsic
262 const __m128i v_sad_w = _mm_packus_epi32(v_sad_d, v_sad_d);
263 const __m128i v_minp_w = _mm_minpos_epu16(v_sad_w);
265 uint32_t local_best_sad = _mm_extract_epi16(v_minp_w, 0);
266 uint32_t local_best_idx = _mm_extract_epi16(v_minp_w, 1);
268 // If the local best value is not saturated, just use it, otherwise
269 // find the horizontal minimum again the hard way on 32 bits.
270 // This is executed rarely.
271 if (__unlikely__(local_best_sad == 0xffff)) {
272 __m128i v_loval_d, v_hival_d, v_loidx_d, v_hiidx_d, v_sel_d;
275 v_loidx_d = _mm_set_epi32(3, 2, 1, 0);
276 v_hival_d = _mm_srli_si128(v_loval_d, 8);
277 v_hiidx_d = _mm_srli_si128(v_loidx_d, 8);
279 v_sel_d = _mm_cmplt_epi32(v_hival_d, v_loval_d);
281 v_loval_d = _mm_blendv_epi8(v_loval_d, v_hival_d, v_sel_d);
282 v_loidx_d = _mm_blendv_epi8(v_loidx_d, v_hiidx_d, v_sel_d);
283 v_hival_d = _mm_srli_si128(v_loval_d, 4);
284 v_hiidx_d = _mm_srli_si128(v_loidx_d, 4);
286 v_sel_d = _mm_cmplt_epi32(v_hival_d, v_loval_d);
288 v_loval_d = _mm_blendv_epi8(v_loval_d, v_hival_d, v_sel_d);
289 v_loidx_d = _mm_blendv_epi8(v_loidx_d, v_hiidx_d, v_sel_d);
291 local_best_sad = _mm_extract_epi32(v_loval_d, 0);
292 local_best_idx = _mm_extract_epi32(v_loidx_d, 0);
295 // Update the global minimum if the local minimum is smaller
296 if (__likely__(local_best_sad < best_sad)) {
297 new_bmv = ((const int_mv *)&v_these_mv_w)[local_best_idx];
298 new_best_address = ((const uint8_t **)v_blocka)[local_best_idx];
300 best_sad = local_best_sad;
306 best_address = new_best_address;
308 v_bmv_w = _mm_set1_epi32(bmv.as_int);
310 v_ba_q = _mm_set1_epi64x((intptr_t)best_address);
312 v_ba_d = _mm_set1_epi32((intptr_t)best_address);
315 if (__unlikely__(best_address == in_what)) {
320 *best_mv = bmv.as_mv;