granicus.if.org Git - libvpx/blob - vp9/encoder/vp9_quantize.c

   1 /*
   2  *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
   3  *
   4  *  Use of this source code is governed by a BSD-style license
   5  *  that can be found in the LICENSE file in the root of the source
   6  *  tree. An additional intellectual property rights grant can be found
   7  *  in the file PATENTS.  All contributing project authors may
   8  *  be found in the AUTHORS file in the root of the source tree.
   9  */
  10
  11 #include <math.h>
  12
  13 #include "vpx_mem/vpx_mem.h"
  14 #include "vpx_ports/mem.h"
  15
  16 #include "vp9/common/vp9_quant_common.h"
  17 #include "vp9/common/vp9_seg_common.h"
  18
  19 #include "vp9/encoder/vp9_encoder.h"
  20 #include "vp9/encoder/vp9_quantize.h"
  21 #include "vp9/encoder/vp9_rd.h"
  22
  23 void vp9_quantize_dc(const tran_low_t *coeff_ptr,
  24                      int n_coeffs, int skip_block,
  25                      const int16_t *round_ptr, const int16_t quant,
  26                      tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
  27                      const int16_t dequant_ptr, uint16_t *eob_ptr) {
  28   const int rc = 0;
  29   const int coeff = coeff_ptr[rc];
  30   const int coeff_sign = (coeff >> 31);
  31   const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
  32   int tmp, eob = -1;
  33
  34   memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
  35   memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
  36
  37   if (!skip_block) {
  38     tmp = clamp(abs_coeff + round_ptr[rc != 0], INT16_MIN, INT16_MAX);
  39     tmp = (tmp * quant) >> 16;
  40     qcoeff_ptr[rc]  = (tmp ^ coeff_sign) - coeff_sign;
  41     dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr;
  42     if (tmp)
  43       eob = 0;
  44   }
  45   *eob_ptr = eob + 1;
  46 }
  47
  48 #if CONFIG_VP9_HIGHBITDEPTH
  49 void vp9_highbd_quantize_dc(const tran_low_t *coeff_ptr,
  50                             int n_coeffs, int skip_block,
  51                             const int16_t *round_ptr, const int16_t quant,
  52                             tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
  53                             const int16_t dequant_ptr, uint16_t *eob_ptr) {
  54   int eob = -1;
  55
  56   memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
  57   memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
  58
  59   if (!skip_block) {
  60     const int rc = 0;
  61     const int coeff = coeff_ptr[rc];
  62     const int coeff_sign = (coeff >> 31);
  63     const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
  64
  65     const int64_t tmp =
  66         (clamp(abs_coeff + round_ptr[rc != 0], INT32_MIN, INT32_MAX) *
  67          quant) >> 16;
  68     qcoeff_ptr[rc] = (tran_low_t)((tmp ^ coeff_sign) - coeff_sign);
  69     dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr;
  70     if (tmp)
  71       eob = 0;
  72   }
  73   *eob_ptr = eob + 1;
  74 }
  75 #endif
  76
  77 void vp9_quantize_dc_32x32(const tran_low_t *coeff_ptr, int skip_block,
  78                            const int16_t *round_ptr, const int16_t quant,
  79                            tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
  80                            const int16_t dequant_ptr, uint16_t *eob_ptr) {
  81   const int n_coeffs = 1024;
  82   const int rc = 0;
  83   const int coeff = coeff_ptr[rc];
  84   const int coeff_sign = (coeff >> 31);
  85   const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
  86   int tmp, eob = -1;
  87
  88   memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
  89   memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
  90
  91   if (!skip_block) {
  92
  93     tmp = clamp(abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], 1),
  94                 INT16_MIN, INT16_MAX);
  95     tmp = (tmp * quant) >> 15;
  96     qcoeff_ptr[rc]  = (tmp ^ coeff_sign) - coeff_sign;
  97     dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr / 2;
  98     if (tmp)
  99       eob = 0;
 100   }
 101   *eob_ptr = eob + 1;
 102 }
 103
 104 #if CONFIG_VP9_HIGHBITDEPTH
 105 void vp9_highbd_quantize_dc_32x32(const tran_low_t *coeff_ptr,
 106                                   int skip_block,
 107                                   const int16_t *round_ptr,
 108                                   const int16_t quant,
 109                                   tran_low_t *qcoeff_ptr,
 110                                   tran_low_t *dqcoeff_ptr,
 111                                   const int16_t dequant_ptr,
 112                                   uint16_t *eob_ptr) {
 113   const int n_coeffs = 1024;
 114   int eob = -1;
 115
 116   memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
 117   memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
 118
 119   if (!skip_block) {
 120     const int rc = 0;
 121     const int coeff = coeff_ptr[rc];
 122     const int coeff_sign = (coeff >> 31);
 123     const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
 124
 125     const int64_t tmp =
 126         (clamp(abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], 1),
 127                INT32_MIN, INT32_MAX) * quant) >> 15;
 128     qcoeff_ptr[rc] = (tran_low_t)((tmp ^ coeff_sign) - coeff_sign);
 129     dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr / 2;
 130     if (tmp)
 131       eob = 0;
 132   }
 133   *eob_ptr = eob + 1;
 134 }
 135 #endif
 136
 137 void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
 138                        int skip_block,
 139                        const int16_t *zbin_ptr, const int16_t *round_ptr,
 140                        const int16_t *quant_ptr, const int16_t *quant_shift_ptr,
 141                        tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
 142                        const int16_t *dequant_ptr,
 143                        uint16_t *eob_ptr,
 144                        const int16_t *scan, const int16_t *iscan) {
 145   int i, eob = -1;
 146   // TODO(jingning) Decide the need of these arguments after the
 147   // quantization process is completed.
 148   (void)zbin_ptr;
 149   (void)quant_shift_ptr;
 150   (void)iscan;
 151
 152   memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
 153   memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
 154
 155   if (!skip_block) {
 156     // Quantization pass: All coefficients with index >= zero_flag are
 157     // skippable. Note: zero_flag can be zero.
 158     for (i = 0; i < n_coeffs; i++) {
 159       const int rc = scan[i];
 160       const int coeff = coeff_ptr[rc];
 161       const int coeff_sign = (coeff >> 31);
 162       const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
 163
 164       int tmp = clamp(abs_coeff + round_ptr[rc != 0], INT16_MIN, INT16_MAX);
 165       tmp = (tmp * quant_ptr[rc != 0]) >> 16;
 166
 167       qcoeff_ptr[rc] = (tmp ^ coeff_sign) - coeff_sign;
 168       dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0];
 169
 170       if (tmp)
 171         eob = i;
 172     }
 173   }
 174   *eob_ptr = eob + 1;
 175 }
 176
 177 #if CONFIG_VP9_HIGHBITDEPTH
 178 void vp9_highbd_quantize_fp_c(const tran_low_t *coeff_ptr,
 179                               intptr_t count,
 180                               int skip_block,
 181                               const int16_t *zbin_ptr,
 182                               const int16_t *round_ptr,
 183                               const int16_t *quant_ptr,
 184                               const int16_t *quant_shift_ptr,
 185                               tran_low_t *qcoeff_ptr,
 186                               tran_low_t *dqcoeff_ptr,
 187                               const int16_t *dequant_ptr,
 188                               uint16_t *eob_ptr,
 189                               const int16_t *scan,
 190                               const int16_t *iscan) {
 191   int i;
 192   int eob = -1;
 193   // TODO(jingning) Decide the need of these arguments after the
 194   // quantization process is completed.
 195   (void)zbin_ptr;
 196   (void)quant_shift_ptr;
 197   (void)iscan;
 198
 199   memset(qcoeff_ptr, 0, count * sizeof(*qcoeff_ptr));
 200   memset(dqcoeff_ptr, 0, count * sizeof(*dqcoeff_ptr));
 201
 202   if (!skip_block) {
 203     // Quantization pass: All coefficients with index >= zero_flag are
 204     // skippable. Note: zero_flag can be zero.
 205     for (i = 0; i < count; i++) {
 206       const int rc = scan[i];
 207       const int coeff = coeff_ptr[rc];
 208       const int coeff_sign = (coeff >> 31);
 209       const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
 210
 211       const int64_t tmp =
 212           (clamp(abs_coeff + round_ptr[rc != 0], INT32_MIN, INT32_MAX) *
 213            quant_ptr[rc != 0]) >> 16;
 214
 215       qcoeff_ptr[rc] = (tran_low_t)((tmp ^ coeff_sign) - coeff_sign);
 216       dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0];
 217
 218       if (tmp)
 219         eob = i;
 220     }
 221   }
 222   *eob_ptr = eob + 1;
 223 }
 224 #endif
 225
 226 // TODO(jingning) Refactor this file and combine functions with similar
 227 // operations.
 228 void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
 229                              int skip_block,
 230                              const int16_t *zbin_ptr, const int16_t *round_ptr,
 231                              const int16_t *quant_ptr,
 232                              const int16_t *quant_shift_ptr,
 233                              tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
 234                              const int16_t *dequant_ptr,
 235                              uint16_t *eob_ptr,
 236                              const int16_t *scan, const int16_t *iscan) {
 237   int i, eob = -1;
 238   (void)zbin_ptr;
 239   (void)quant_shift_ptr;
 240   (void)iscan;
 241
 242   memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
 243   memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
 244
 245   if (!skip_block) {
 246     for (i = 0; i < n_coeffs; i++) {
 247       const int rc = scan[i];
 248       const int coeff = coeff_ptr[rc];
 249       const int coeff_sign = (coeff >> 31);
 250       int tmp = 0;
 251       int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
 252
 253       if (abs_coeff >= (dequant_ptr[rc != 0] >> 2)) {
 254         abs_coeff += ROUND_POWER_OF_TWO(round_ptr[rc != 0], 1);
 255         abs_coeff = clamp(abs_coeff, INT16_MIN, INT16_MAX);
 256         tmp = (abs_coeff * quant_ptr[rc != 0]) >> 15;
 257         qcoeff_ptr[rc] = (tmp ^ coeff_sign) - coeff_sign;
 258         dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / 2;
 259       }
 260
 261       if (tmp)
 262         eob = i;
 263     }
 264   }
 265   *eob_ptr = eob + 1;
 266 }
 267
 268 #if CONFIG_VP9_HIGHBITDEPTH
 269 void vp9_highbd_quantize_fp_32x32_c(const tran_low_t *coeff_ptr,
 270                                     intptr_t n_coeffs, int skip_block,
 271                                     const int16_t *zbin_ptr,
 272                                     const int16_t *round_ptr,
 273                                     const int16_t *quant_ptr,
 274                                     const int16_t *quant_shift_ptr,
 275                                     tran_low_t *qcoeff_ptr,
 276                                     tran_low_t *dqcoeff_ptr,
 277                                     const int16_t *dequant_ptr,
 278                                     uint16_t *eob_ptr,
 279                                     const int16_t *scan, const int16_t *iscan) {
 280   int i, eob = -1;
 281   (void)zbin_ptr;
 282   (void)quant_shift_ptr;
 283   (void)iscan;
 284
 285   memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
 286   memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
 287
 288   if (!skip_block) {
 289     for (i = 0; i < n_coeffs; i++) {
 290       const int rc = scan[i];
 291       const int coeff = coeff_ptr[rc];
 292       const int coeff_sign = (coeff >> 31);
 293       int64_t tmp = 0;
 294       const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
 295
 296       if (abs_coeff >= (dequant_ptr[rc != 0] >> 2)) {
 297         tmp = clamp(abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], 1),
 298                     INT32_MIN, INT32_MAX);
 299         tmp = (tmp * quant_ptr[rc != 0]) >> 15;
 300         qcoeff_ptr[rc] = (tran_low_t)((tmp ^ coeff_sign) - coeff_sign);
 301         dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / 2;
 302       }
 303
 304       if (tmp)
 305         eob = i;
 306     }
 307   }
 308   *eob_ptr = eob + 1;
 309 }
 310 #endif
 311
 312 void vp9_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
 313                       int skip_block,
 314                       const int16_t *zbin_ptr, const int16_t *round_ptr,
 315                       const int16_t *quant_ptr, const int16_t *quant_shift_ptr,
 316                       tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
 317                       const int16_t *dequant_ptr,
 318                       uint16_t *eob_ptr,
 319                       const int16_t *scan, const int16_t *iscan) {
 320   int i, non_zero_count = (int)n_coeffs, eob = -1;
 321   const int zbins[2] = {zbin_ptr[0], zbin_ptr[1]};
 322   const int nzbins[2] = {zbins[0] * -1, zbins[1] * -1};
 323   (void)iscan;
 324
 325   memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
 326   memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
 327
 328   if (!skip_block) {
 329     // Pre-scan pass
 330     for (i = (int)n_coeffs - 1; i >= 0; i--) {
 331       const int rc = scan[i];
 332       const int coeff = coeff_ptr[rc];
 333
 334       if (coeff < zbins[rc != 0] && coeff > nzbins[rc != 0])
 335         non_zero_count--;
 336       else
 337         break;
 338     }
 339
 340     // Quantization pass: All coefficients with index >= zero_flag are
 341     // skippable. Note: zero_flag can be zero.
 342     for (i = 0; i < non_zero_count; i++) {
 343       const int rc = scan[i];
 344       const int coeff = coeff_ptr[rc];
 345       const int coeff_sign = (coeff >> 31);
 346       const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
 347
 348       if (abs_coeff >= zbins[rc != 0]) {
 349         int tmp = clamp(abs_coeff + round_ptr[rc != 0], INT16_MIN, INT16_MAX);
 350         tmp = ((((tmp * quant_ptr[rc != 0]) >> 16) + tmp) *
 351                   quant_shift_ptr[rc != 0]) >> 16;  // quantization
 352         qcoeff_ptr[rc]  = (tmp ^ coeff_sign) - coeff_sign;
 353         dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0];
 354
 355         if (tmp)
 356           eob = i;
 357       }
 358     }
 359   }
 360   *eob_ptr = eob + 1;
 361 }
 362
 363 #if CONFIG_VP9_HIGHBITDEPTH
 364 void vp9_highbd_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
 365                              int skip_block, const int16_t *zbin_ptr,
 366                              const int16_t *round_ptr, const int16_t *quant_ptr,
 367                              const int16_t *quant_shift_ptr,
 368                              tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
 369                              const int16_t *dequant_ptr,
 370                              uint16_t *eob_ptr, const int16_t *scan,
 371                              const int16_t *iscan) {
 372   int i, non_zero_count = (int)n_coeffs, eob = -1;
 373   const int zbins[2] = {zbin_ptr[0], zbin_ptr[1]};
 374   const int nzbins[2] = {zbins[0] * -1, zbins[1] * -1};
 375   (void)iscan;
 376
 377   memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
 378   memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
 379
 380   if (!skip_block) {
 381     // Pre-scan pass
 382     for (i = (int)n_coeffs - 1; i >= 0; i--) {
 383       const int rc = scan[i];
 384       const int coeff = coeff_ptr[rc];
 385
 386       if (coeff < zbins[rc != 0] && coeff > nzbins[rc != 0])
 387         non_zero_count--;
 388       else
 389         break;
 390     }
 391
 392     // Quantization pass: All coefficients with index >= zero_flag are
 393     // skippable. Note: zero_flag can be zero.
 394     for (i = 0; i < non_zero_count; i++) {
 395       const int rc = scan[i];
 396       const int coeff = coeff_ptr[rc];
 397       const int coeff_sign = (coeff >> 31);
 398       const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
 399
 400       if (abs_coeff >= zbins[rc != 0]) {
 401         int64_t tmp = clamp(abs_coeff + round_ptr[rc != 0],
 402                             INT32_MIN, INT32_MAX);
 403         tmp = ((((tmp * quant_ptr[rc != 0]) >> 16) + tmp) *
 404                   quant_shift_ptr[rc != 0]) >> 16;  // quantization
 405         qcoeff_ptr[rc]  = (tran_low_t)((tmp ^ coeff_sign) - coeff_sign);
 406         dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0];
 407
 408         if (tmp)
 409           eob = i;
 410       }
 411     }
 412   }
 413   *eob_ptr = eob + 1;
 414 }
 415 #endif
 416
 417 void vp9_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
 418                             int skip_block,
 419                             const int16_t *zbin_ptr, const int16_t *round_ptr,
 420                             const int16_t *quant_ptr,
 421                             const int16_t *quant_shift_ptr,
 422                             tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
 423                             const int16_t *dequant_ptr,
 424                             uint16_t *eob_ptr,
 425                             const int16_t *scan, const int16_t *iscan) {
 426   const int zbins[2] = {ROUND_POWER_OF_TWO(zbin_ptr[0], 1),
 427                         ROUND_POWER_OF_TWO(zbin_ptr[1], 1)};
 428   const int nzbins[2] = {zbins[0] * -1, zbins[1] * -1};
 429
 430   int idx = 0;
 431   int idx_arr[1024];
 432   int i, eob = -1;
 433   (void)iscan;
 434
 435   memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
 436   memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
 437
 438   if (!skip_block) {
 439     // Pre-scan pass
 440     for (i = 0; i < n_coeffs; i++) {
 441       const int rc = scan[i];
 442       const int coeff = coeff_ptr[rc];
 443
 444       // If the coefficient is out of the base ZBIN range, keep it for
 445       // quantization.
 446       if (coeff >= zbins[rc != 0] || coeff <= nzbins[rc != 0])
 447         idx_arr[idx++] = i;
 448     }
 449
 450     // Quantization pass: only process the coefficients selected in
 451     // pre-scan pass. Note: idx can be zero.
 452     for (i = 0; i < idx; i++) {
 453       const int rc = scan[idx_arr[i]];
 454       const int coeff = coeff_ptr[rc];
 455       const int coeff_sign = (coeff >> 31);
 456       int tmp;
 457       int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
 458       abs_coeff += ROUND_POWER_OF_TWO(round_ptr[rc != 0], 1);
 459       abs_coeff = clamp(abs_coeff, INT16_MIN, INT16_MAX);
 460       tmp = ((((abs_coeff * quant_ptr[rc != 0]) >> 16) + abs_coeff) *
 461                quant_shift_ptr[rc != 0]) >> 15;
 462
 463       qcoeff_ptr[rc] = (tmp ^ coeff_sign) - coeff_sign;
 464       dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / 2;
 465
 466       if (tmp)
 467         eob = idx_arr[i];
 468     }
 469   }
 470   *eob_ptr = eob + 1;
 471 }
 472
 473 #if CONFIG_VP9_HIGHBITDEPTH
 474 void vp9_highbd_quantize_b_32x32_c(const tran_low_t *coeff_ptr,
 475                                    intptr_t n_coeffs, int skip_block,
 476                                    const int16_t *zbin_ptr,
 477                                    const int16_t *round_ptr,
 478                                    const int16_t *quant_ptr,
 479                                    const int16_t *quant_shift_ptr,
 480                                    tran_low_t *qcoeff_ptr,
 481                                    tran_low_t *dqcoeff_ptr,
 482                                    const int16_t *dequant_ptr,
 483                                    uint16_t *eob_ptr,
 484                                    const int16_t *scan, const int16_t *iscan) {
 485   const int zbins[2] = {ROUND_POWER_OF_TWO(zbin_ptr[0], 1),
 486                         ROUND_POWER_OF_TWO(zbin_ptr[1], 1)};
 487   const int nzbins[2] = {zbins[0] * -1, zbins[1] * -1};
 488
 489   int idx = 0;
 490   int idx_arr[1024];
 491   int i, eob = -1;
 492   (void)iscan;
 493
 494   memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
 495   memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
 496
 497   if (!skip_block) {
 498     // Pre-scan pass
 499     for (i = 0; i < n_coeffs; i++) {
 500       const int rc = scan[i];
 501       const int coeff = coeff_ptr[rc];
 502
 503       // If the coefficient is out of the base ZBIN range, keep it for
 504       // quantization.
 505       if (coeff >= zbins[rc != 0] || coeff <= nzbins[rc != 0])
 506         idx_arr[idx++] = i;
 507     }
 508
 509     // Quantization pass: only process the coefficients selected in
 510     // pre-scan pass. Note: idx can be zero.
 511     for (i = 0; i < idx; i++) {
 512       const int rc = scan[idx_arr[i]];
 513       const int coeff = coeff_ptr[rc];
 514       const int coeff_sign = (coeff >> 31);
 515       const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
 516       int64_t tmp = clamp(abs_coeff +
 517                           ROUND_POWER_OF_TWO(round_ptr[rc != 0], 1),
 518                           INT32_MIN, INT32_MAX);
 519       tmp = ((((tmp * quant_ptr[rc != 0]) >> 16) + tmp) *
 520                quant_shift_ptr[rc != 0]) >> 15;
 521
 522       qcoeff_ptr[rc] = (tran_low_t)((tmp ^ coeff_sign) - coeff_sign);
 523       dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / 2;
 524
 525       if (tmp)
 526         eob = idx_arr[i];
 527     }
 528   }
 529   *eob_ptr = eob + 1;
 530 }
 531 #endif
 532
 533 void vp9_regular_quantize_b_4x4(MACROBLOCK *x, int plane, int block,
 534                                 const int16_t *scan, const int16_t *iscan) {
 535   MACROBLOCKD *const xd = &x->e_mbd;
 536   struct macroblock_plane *p = &x->plane[plane];
 537   struct macroblockd_plane *pd = &xd->plane[plane];
 538
 539 #if CONFIG_VP9_HIGHBITDEPTH
 540   if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
 541     vp9_highbd_quantize_b(BLOCK_OFFSET(p->coeff, block),
 542                           16, x->skip_block,
 543                           p->zbin, p->round, p->quant, p->quant_shift,
 544                           BLOCK_OFFSET(p->qcoeff, block),
 545                           BLOCK_OFFSET(pd->dqcoeff, block),
 546                           pd->dequant, &p->eobs[block],
 547                           scan, iscan);
 548     return;
 549   }
 550 #endif
 551   vp9_quantize_b(BLOCK_OFFSET(p->coeff, block),
 552                  16, x->skip_block,
 553                  p->zbin, p->round, p->quant, p->quant_shift,
 554                  BLOCK_OFFSET(p->qcoeff, block),
 555                  BLOCK_OFFSET(pd->dqcoeff, block),
 556                  pd->dequant, &p->eobs[block], scan, iscan);
 557 }
 558
 559 static void invert_quant(int16_t *quant, int16_t *shift, int d) {
 560   unsigned t;
 561   int l;
 562   t = d;
 563   for (l = 0; t > 1; l++)
 564     t >>= 1;
 565   t = 1 + (1 << (16 + l)) / d;
 566   *quant = (int16_t)(t - (1 << 16));
 567   *shift = 1 << (16 - l);
 568 }
 569
 570 static int get_qzbin_factor(int q, vpx_bit_depth_t bit_depth) {
 571   const int quant = vp9_dc_quant(q, 0, bit_depth);
 572 #if CONFIG_VP9_HIGHBITDEPTH
 573   switch (bit_depth) {
 574     case VPX_BITS_8:
 575       return q == 0 ? 64 : (quant < 148 ? 84 : 80);
 576     case VPX_BITS_10:
 577       return q == 0 ? 64 : (quant < 592 ? 84 : 80);
 578     case VPX_BITS_12:
 579       return q == 0 ? 64 : (quant < 2368 ? 84 : 80);
 580     default:
 581       assert(0 && "bit_depth should be VPX_BITS_8, VPX_BITS_10 or VPX_BITS_12");
 582       return -1;
 583   }
 584 #else
 585   (void) bit_depth;
 586   return q == 0 ? 64 : (quant < 148 ? 84 : 80);
 587 #endif
 588 }
 589
 590 void vp9_init_quantizer(VP9_COMP *cpi) {
 591   VP9_COMMON *const cm = &cpi->common;
 592   QUANTS *const quants = &cpi->quants;
 593   int i, q, quant;
 594
 595   for (q = 0; q < QINDEX_RANGE; q++) {
 596     const int qzbin_factor = get_qzbin_factor(q, cm->bit_depth);
 597     const int qrounding_factor = q == 0 ? 64 : 48;
 598
 599     for (i = 0; i < 2; ++i) {
 600       int qrounding_factor_fp = i == 0 ? 48 : 42;
 601       if (q == 0)
 602         qrounding_factor_fp = 64;
 603
 604       // y
 605       quant = i == 0 ? vp9_dc_quant(q, cm->y_dc_delta_q, cm->bit_depth)
 606                      : vp9_ac_quant(q, 0, cm->bit_depth);
 607       invert_quant(&quants->y_quant[q][i], &quants->y_quant_shift[q][i], quant);
 608       quants->y_quant_fp[q][i] = (1 << 16) / quant;
 609       quants->y_round_fp[q][i] = (qrounding_factor_fp * quant) >> 7;
 610       quants->y_zbin[q][i] = ROUND_POWER_OF_TWO(qzbin_factor * quant, 7);
 611       quants->y_round[q][i] = (qrounding_factor * quant) >> 7;
 612       cpi->y_dequant[q][i] = quant;
 613
 614       // uv
 615       quant = i == 0 ? vp9_dc_quant(q, cm->uv_dc_delta_q, cm->bit_depth)
 616                      : vp9_ac_quant(q, cm->uv_ac_delta_q, cm->bit_depth);
 617       invert_quant(&quants->uv_quant[q][i],
 618                    &quants->uv_quant_shift[q][i], quant);
 619       quants->uv_quant_fp[q][i] = (1 << 16) / quant;
 620       quants->uv_round_fp[q][i] = (qrounding_factor_fp * quant) >> 7;
 621       quants->uv_zbin[q][i] = ROUND_POWER_OF_TWO(qzbin_factor * quant, 7);
 622       quants->uv_round[q][i] = (qrounding_factor * quant) >> 7;
 623       cpi->uv_dequant[q][i] = quant;
 624     }
 625
 626     for (i = 2; i < 8; i++) {
 627       quants->y_quant[q][i] = quants->y_quant[q][1];
 628       quants->y_quant_fp[q][i] = quants->y_quant_fp[q][1];
 629       quants->y_round_fp[q][i] = quants->y_round_fp[q][1];
 630       quants->y_quant_shift[q][i] = quants->y_quant_shift[q][1];
 631       quants->y_zbin[q][i] = quants->y_zbin[q][1];
 632       quants->y_round[q][i] = quants->y_round[q][1];
 633       cpi->y_dequant[q][i] = cpi->y_dequant[q][1];
 634
 635       quants->uv_quant[q][i] = quants->uv_quant[q][1];
 636       quants->uv_quant_fp[q][i] = quants->uv_quant_fp[q][1];
 637       quants->uv_round_fp[q][i] = quants->uv_round_fp[q][1];
 638       quants->uv_quant_shift[q][i] = quants->uv_quant_shift[q][1];
 639       quants->uv_zbin[q][i] = quants->uv_zbin[q][1];
 640       quants->uv_round[q][i] = quants->uv_round[q][1];
 641       cpi->uv_dequant[q][i] = cpi->uv_dequant[q][1];
 642     }
 643   }
 644 }
 645
 646 void vp9_init_plane_quantizers(VP9_COMP *cpi, MACROBLOCK *x) {
 647   const VP9_COMMON *const cm = &cpi->common;
 648   MACROBLOCKD *const xd = &x->e_mbd;
 649   QUANTS *const quants = &cpi->quants;
 650   const int segment_id = xd->mi[0]->mbmi.segment_id;
 651   const int qindex = vp9_get_qindex(&cm->seg, segment_id, cm->base_qindex);
 652   const int rdmult = vp9_compute_rd_mult(cpi, qindex + cm->y_dc_delta_q);
 653   int i;
 654
 655   // Y
 656   x->plane[0].quant = quants->y_quant[qindex];
 657   x->plane[0].quant_fp = quants->y_quant_fp[qindex];
 658   x->plane[0].round_fp = quants->y_round_fp[qindex];
 659   x->plane[0].quant_shift = quants->y_quant_shift[qindex];
 660   x->plane[0].zbin = quants->y_zbin[qindex];
 661   x->plane[0].round = quants->y_round[qindex];
 662   xd->plane[0].dequant = cpi->y_dequant[qindex];
 663
 664   x->plane[0].quant_thred[0] = x->plane[0].zbin[0] * x->plane[0].zbin[0];
 665   x->plane[0].quant_thred[1] = x->plane[0].zbin[1] * x->plane[0].zbin[1];
 666
 667   // UV
 668   for (i = 1; i < 3; i++) {
 669     x->plane[i].quant = quants->uv_quant[qindex];
 670     x->plane[i].quant_fp = quants->uv_quant_fp[qindex];
 671     x->plane[i].round_fp = quants->uv_round_fp[qindex];
 672     x->plane[i].quant_shift = quants->uv_quant_shift[qindex];
 673     x->plane[i].zbin = quants->uv_zbin[qindex];
 674     x->plane[i].round = quants->uv_round[qindex];
 675     xd->plane[i].dequant = cpi->uv_dequant[qindex];
 676
 677     x->plane[i].quant_thred[0] = x->plane[i].zbin[0] * x->plane[i].zbin[0];
 678     x->plane[i].quant_thred[1] = x->plane[i].zbin[1] * x->plane[i].zbin[1];
 679   }
 680
 681   x->skip_block = vp9_segfeature_active(&cm->seg, segment_id, SEG_LVL_SKIP);
 682   x->q_index = qindex;
 683
 684   x->errorperbit = rdmult >> 6;
 685   x->errorperbit += (x->errorperbit == 0);
 686
 687   vp9_initialize_me_consts(cpi, x, x->q_index);
 688 }
 689
 690 void vp9_frame_init_quantizer(VP9_COMP *cpi) {
 691   vp9_init_plane_quantizers(cpi, &cpi->td.mb);
 692 }
 693
 694 void vp9_set_quantizer(VP9_COMMON *cm, int q) {
 695   // quantizer has to be reinitialized with vp9_init_quantizer() if any
 696   // delta_q changes.
 697   cm->base_qindex = q;
 698   cm->y_dc_delta_q = 0;
 699   cm->uv_dc_delta_q = 0;
 700   cm->uv_ac_delta_q = 0;
 701 }
 702
 703 // Table that converts 0-63 Q-range values passed in outside to the Qindex
 704 // range used internally.
 705 static const int quantizer_to_qindex[] = {
 706   0,    4,   8,  12,  16,  20,  24,  28,
 707   32,   36,  40,  44,  48,  52,  56,  60,
 708   64,   68,  72,  76,  80,  84,  88,  92,
 709   96,  100, 104, 108, 112, 116, 120, 124,
 710   128, 132, 136, 140, 144, 148, 152, 156,
 711   160, 164, 168, 172, 176, 180, 184, 188,
 712   192, 196, 200, 204, 208, 212, 216, 220,
 713   224, 228, 232, 236, 240, 244, 249, 255,
 714 };
 715
 716 int vp9_quantizer_to_qindex(int quantizer) {
 717   return quantizer_to_qindex[quantizer];
 718 }
 719
 720 int vp9_qindex_to_quantizer(int qindex) {
 721   int quantizer;
 722
 723   for (quantizer = 0; quantizer < 64; ++quantizer)
 724     if (quantizer_to_qindex[quantizer] >= qindex)
 725       return quantizer;
 726
 727   return 63;
 728 }