granicus.if.org Git - libvpx/blob - vp8/encoder/denoising.c

   1 /*
   2  *  Copyright (c) 2012 The WebM project authors. All Rights Reserved.
   3  *
   4  *  Use of this source code is governed by a BSD-style license
   5  *  that can be found in the LICENSE file in the root of the source
   6  *  tree. An additional intellectual property rights grant can be found
   7  *  in the file PATENTS.  All contributing project authors may
   8  *  be found in the AUTHORS file in the root of the source tree.
   9  */
  10
  11 #include "denoising.h"
  12
  13 #include "vp8/common/reconinter.h"
  14 #include "vpx/vpx_integer.h"
  15 #include "vpx_mem/vpx_mem.h"
  16 #include "vp8_rtcd.h"
  17
  18 static const unsigned int NOISE_MOTION_THRESHOLD = 25 * 25;
  19 /* SSE_DIFF_THRESHOLD is selected as ~95% confidence assuming
  20  * var(noise) ~= 100.
  21  */
  22 static const unsigned int SSE_DIFF_THRESHOLD = 16 * 16 * 20;
  23 static const unsigned int SSE_THRESHOLD = 16 * 16 * 40;
  24 static const unsigned int SSE_THRESHOLD_HIGH = 16 * 16 * 60;
  25
  26 /*
  27  * The filter function was modified to reduce the computational complexity.
  28  * Step 1:
  29  * Instead of applying tap coefficients for each pixel, we calculated the
  30  * pixel adjustments vs. pixel diff value ahead of time.
  31  *     adjustment = filtered_value - current_raw
  32  *                = (filter_coefficient * diff + 128) >> 8
  33  * where
  34  *     filter_coefficient = (255 << 8) / (256 + ((absdiff * 330) >> 3));
  35  *     filter_coefficient += filter_coefficient /
  36  *                           (3 + motion_magnitude_adjustment);
  37  *     filter_coefficient is clamped to 0 ~ 255.
  38  *
  39  * Step 2:
  40  * The adjustment vs. diff curve becomes flat very quick when diff increases.
  41  * This allowed us to use only several levels to approximate the curve without
  42  * changing the filtering algorithm too much.
  43  * The adjustments were further corrected by checking the motion magnitude.
  44  * The levels used are:
  45  * diff       adjustment w/o motion correction   adjustment w/ motion correction
  46  * [-255, -16]           -6                                   -7
  47  * [-15, -8]             -4                                   -5
  48  * [-7, -4]              -3                                   -4
  49  * [-3, 3]               diff                                 diff
  50  * [4, 7]                 3                                    4
  51  * [8, 15]                4                                    5
  52  * [16, 255]              6                                    7
  53  */
  54
  55 int vp8_denoiser_filter_c(unsigned char *mc_running_avg_y, int mc_avg_y_stride,
  56                           unsigned char *running_avg_y, int avg_y_stride,
  57                           unsigned char *sig, int sig_stride,
  58                           unsigned int motion_magnitude,
  59                           int increase_denoising)
  60 {
  61     unsigned char *running_avg_y_start = running_avg_y;
  62     unsigned char *sig_start = sig;
  63     int sum_diff_thresh;
  64     int r, c;
  65     int sum_diff = 0;
  66     int adj_val[3] = {3, 4, 6};
  67     int shift_inc1 = 0;
  68     int shift_inc2 = 1;
  69     /* If motion_magnitude is small, making the denoiser more aggressive by
  70      * increasing the adjustment for each level. Add another increment for
  71      * blocks that are labeled for increase denoising. */
  72     if (motion_magnitude <= MOTION_MAGNITUDE_THRESHOLD)
  73     {
  74       if (increase_denoising) {
  75         shift_inc1 = 1;
  76         shift_inc2 = 2;
  77       }
  78       adj_val[0] += shift_inc2;
  79       adj_val[1] += shift_inc2;
  80       adj_val[2] += shift_inc2;
  81     }
  82
  83     for (r = 0; r < 16; ++r)
  84     {
  85         for (c = 0; c < 16; ++c)
  86         {
  87             int diff = 0;
  88             int adjustment = 0;
  89             int absdiff = 0;
  90
  91             diff = mc_running_avg_y[c] - sig[c];
  92             absdiff = abs(diff);
  93
  94             // When |diff| <= |3 + shift_inc1|, use pixel value from
  95             // last denoised raw.
  96             if (absdiff <= 3 + shift_inc1)
  97             {
  98                 running_avg_y[c] = mc_running_avg_y[c];
  99                 sum_diff += diff;
 100             }
 101             else
 102             {
 103                 if (absdiff >= 4 && absdiff <= 7)
 104                     adjustment = adj_val[0];
 105                 else if (absdiff >= 8 && absdiff <= 15)
 106                     adjustment = adj_val[1];
 107                 else
 108                     adjustment = adj_val[2];
 109
 110                 if (diff > 0)
 111                 {
 112                     if ((sig[c] + adjustment) > 255)
 113                         running_avg_y[c] = 255;
 114                     else
 115                         running_avg_y[c] = sig[c] + adjustment;
 116
 117                     sum_diff += adjustment;
 118                 }
 119                 else
 120                 {
 121                     if ((sig[c] - adjustment) < 0)
 122                         running_avg_y[c] = 0;
 123                     else
 124                         running_avg_y[c] = sig[c] - adjustment;
 125
 126                     sum_diff -= adjustment;
 127                 }
 128             }
 129         }
 130
 131         /* Update pointers for next iteration. */
 132         sig += sig_stride;
 133         mc_running_avg_y += mc_avg_y_stride;
 134         running_avg_y += avg_y_stride;
 135     }
 136
 137     sum_diff_thresh= SUM_DIFF_THRESHOLD;
 138     if (increase_denoising) sum_diff_thresh = SUM_DIFF_THRESHOLD_HIGH;
 139     if (abs(sum_diff) > sum_diff_thresh) {
 140       // Before returning to copy the block (i.e., apply no denoising), check
 141       // if we can still apply some (weaker) temporal filtering to this block,
 142       // that would otherwise not be denoised at all. Simplest is to apply
 143       // an additional adjustment to running_avg_y to bring it closer to sig.
 144       // The adjustment is capped by a maximum delta, and chosen such that
 145       // in most cases the resulting sum_diff will be within the
 146       // accceptable range given by sum_diff_thresh.
 147
 148       // The delta is set by the excess of absolute pixel diff over threshold.
 149       int delta = ((abs(sum_diff) - sum_diff_thresh) >> 8) + 1;
 150       // Only apply the adjustment for max delta up to 3.
 151       if (delta < 4) {
 152         sig -= sig_stride * 16;
 153         mc_running_avg_y -= mc_avg_y_stride * 16;
 154         running_avg_y -= avg_y_stride * 16;
 155         for (r = 0; r < 16; ++r) {
 156           for (c = 0; c < 16; ++c) {
 157             int diff = mc_running_avg_y[c] - sig[c];
 158             int adjustment = abs(diff);
 159             if (adjustment > delta)
 160               adjustment = delta;
 161             if (diff > 0) {
 162               // Bring denoised signal down.
 163               if (running_avg_y[c] - adjustment < 0)
 164                 running_avg_y[c] = 0;
 165               else
 166                 running_avg_y[c] = running_avg_y[c] - adjustment;
 167               sum_diff -= adjustment;
 168             } else if (diff < 0) {
 169               // Bring denoised signal up.
 170               if (running_avg_y[c] + adjustment > 255)
 171                 running_avg_y[c] = 255;
 172               else
 173                 running_avg_y[c] = running_avg_y[c] + adjustment;
 174               sum_diff += adjustment;
 175             }
 176           }
 177           // TODO(marpan): Check here if abs(sum_diff) has gone below the
 178           // threshold sum_diff_thresh, and if so, we can exit the row loop.
 179           sig += sig_stride;
 180           mc_running_avg_y += mc_avg_y_stride;
 181           running_avg_y += avg_y_stride;
 182         }
 183         if (abs(sum_diff) > sum_diff_thresh)
 184           return COPY_BLOCK;
 185       } else {
 186         return COPY_BLOCK;
 187       }
 188     }
 189
 190     vp8_copy_mem16x16(running_avg_y_start, avg_y_stride, sig_start, sig_stride);
 191     return FILTER_BLOCK;
 192 }
 193
 194 int vp8_denoiser_allocate(VP8_DENOISER *denoiser, int width, int height,
 195                           int num_mb_rows, int num_mb_cols)
 196 {
 197     int i;
 198     assert(denoiser);
 199     denoiser->num_mb_cols = num_mb_cols;
 200
 201     for (i = 0; i < MAX_REF_FRAMES; i++)
 202     {
 203         denoiser->yv12_running_avg[i].flags = 0;
 204
 205         if (vp8_yv12_alloc_frame_buffer(&(denoiser->yv12_running_avg[i]), width,
 206                                         height, VP8BORDERINPIXELS)
 207             < 0)
 208         {
 209             vp8_denoiser_free(denoiser);
 210             return 1;
 211         }
 212         vpx_memset(denoiser->yv12_running_avg[i].buffer_alloc, 0,
 213                    denoiser->yv12_running_avg[i].frame_size);
 214
 215     }
 216     denoiser->yv12_mc_running_avg.flags = 0;
 217
 218     if (vp8_yv12_alloc_frame_buffer(&(denoiser->yv12_mc_running_avg), width,
 219                                    height, VP8BORDERINPIXELS) < 0)
 220     {
 221         vp8_denoiser_free(denoiser);
 222         return 1;
 223     }
 224
 225     vpx_memset(denoiser->yv12_mc_running_avg.buffer_alloc, 0,
 226                denoiser->yv12_mc_running_avg.frame_size);
 227
 228     denoiser->denoise_state = vpx_calloc((num_mb_rows * num_mb_cols), 1);
 229     vpx_memset(denoiser->denoise_state, 0, (num_mb_rows * num_mb_cols));
 230
 231     return 0;
 232 }
 233
 234 void vp8_denoiser_free(VP8_DENOISER *denoiser)
 235 {
 236     int i;
 237     assert(denoiser);
 238
 239     for (i = 0; i < MAX_REF_FRAMES ; i++)
 240     {
 241         vp8_yv12_de_alloc_frame_buffer(&denoiser->yv12_running_avg[i]);
 242     }
 243     vp8_yv12_de_alloc_frame_buffer(&denoiser->yv12_mc_running_avg);
 244 }
 245
 246
 247 void vp8_denoiser_denoise_mb(VP8_DENOISER *denoiser,
 248                              MACROBLOCK *x,
 249                              unsigned int best_sse,
 250                              unsigned int zero_mv_sse,
 251                              int recon_yoffset,
 252                              int recon_uvoffset,
 253                              loop_filter_info_n *lfi_n,
 254                              int mb_row,
 255                              int mb_col,
 256                              int block_index)
 257 {
 258     int mv_row;
 259     int mv_col;
 260     unsigned int motion_magnitude2;
 261     unsigned int sse_thresh;
 262     int sse_diff_thresh = 0;
 263     // Spatial loop filter: only applied selectively based on
 264     // temporal filter state of block relative to top/left neighbors.
 265     int apply_spatial_loop_filter = 1;
 266     MV_REFERENCE_FRAME frame = x->best_reference_frame;
 267     MV_REFERENCE_FRAME zero_frame = x->best_zeromv_reference_frame;
 268
 269     enum vp8_denoiser_decision decision = FILTER_BLOCK;
 270
 271     if (zero_frame)
 272     {
 273         YV12_BUFFER_CONFIG *src = &denoiser->yv12_running_avg[frame];
 274         YV12_BUFFER_CONFIG *dst = &denoiser->yv12_mc_running_avg;
 275         YV12_BUFFER_CONFIG saved_pre,saved_dst;
 276         MB_MODE_INFO saved_mbmi;
 277         MACROBLOCKD *filter_xd = &x->e_mbd;
 278         MB_MODE_INFO *mbmi = &filter_xd->mode_info_context->mbmi;
 279         int sse_diff = zero_mv_sse - best_sse;
 280
 281         saved_mbmi = *mbmi;
 282
 283         /* Use the best MV for the compensation. */
 284         mbmi->ref_frame = x->best_reference_frame;
 285         mbmi->mode = x->best_sse_inter_mode;
 286         mbmi->mv = x->best_sse_mv;
 287         mbmi->need_to_clamp_mvs = x->need_to_clamp_best_mvs;
 288         mv_col = x->best_sse_mv.as_mv.col;
 289         mv_row = x->best_sse_mv.as_mv.row;
 290         // Bias to zero_mv if small amount of motion.
 291         // Note sse_diff_thresh is intialized to zero, so this ensures
 292         // we will always choose zero_mv for denoising if
 293         // zero_mv_see <= best_sse (i.e., sse_diff <= 0).
 294         if ((unsigned int)(mv_row * mv_row + mv_col * mv_col)
 295             <= NOISE_MOTION_THRESHOLD)
 296             sse_diff_thresh = (int)SSE_DIFF_THRESHOLD;
 297
 298         if (frame == INTRA_FRAME ||
 299             sse_diff <= sse_diff_thresh)
 300         {
 301             /*
 302              * Handle intra blocks as referring to last frame with zero motion
 303              * and let the absolute pixel difference affect the filter factor.
 304              * Also consider small amount of motion as being random walk due
 305              * to noise, if it doesn't mean that we get a much bigger error.
 306              * Note that any changes to the mode info only affects the
 307              * denoising.
 308              */
 309             mbmi->ref_frame =
 310                     x->best_zeromv_reference_frame;
 311
 312             src = &denoiser->yv12_running_avg[zero_frame];
 313
 314             mbmi->mode = ZEROMV;
 315             mbmi->mv.as_int = 0;
 316             x->best_sse_inter_mode = ZEROMV;
 317             x->best_sse_mv.as_int = 0;
 318             best_sse = zero_mv_sse;
 319         }
 320
 321         saved_pre = filter_xd->pre;
 322         saved_dst = filter_xd->dst;
 323
 324         /* Compensate the running average. */
 325         filter_xd->pre.y_buffer = src->y_buffer + recon_yoffset;
 326         filter_xd->pre.u_buffer = src->u_buffer + recon_uvoffset;
 327         filter_xd->pre.v_buffer = src->v_buffer + recon_uvoffset;
 328         /* Write the compensated running average to the destination buffer. */
 329         filter_xd->dst.y_buffer = dst->y_buffer + recon_yoffset;
 330         filter_xd->dst.u_buffer = dst->u_buffer + recon_uvoffset;
 331         filter_xd->dst.v_buffer = dst->v_buffer + recon_uvoffset;
 332
 333         if (!x->skip)
 334         {
 335             vp8_build_inter_predictors_mb(filter_xd);
 336         }
 337         else
 338         {
 339             vp8_build_inter16x16_predictors_mb(filter_xd,
 340                                                filter_xd->dst.y_buffer,
 341                                                filter_xd->dst.u_buffer,
 342                                                filter_xd->dst.v_buffer,
 343                                                filter_xd->dst.y_stride,
 344                                                filter_xd->dst.uv_stride);
 345         }
 346         filter_xd->pre = saved_pre;
 347         filter_xd->dst = saved_dst;
 348         *mbmi = saved_mbmi;
 349
 350     }
 351
 352     mv_row = x->best_sse_mv.as_mv.row;
 353     mv_col = x->best_sse_mv.as_mv.col;
 354     motion_magnitude2 = mv_row * mv_row + mv_col * mv_col;
 355     sse_thresh = SSE_THRESHOLD;
 356     if (x->increase_denoising) sse_thresh = SSE_THRESHOLD_HIGH;
 357
 358     if (best_sse > sse_thresh || motion_magnitude2
 359            > 8 * NOISE_MOTION_THRESHOLD)
 360     {
 361         decision = COPY_BLOCK;
 362     }
 363
 364     if (decision == FILTER_BLOCK)
 365     {
 366         unsigned char *mc_running_avg_y =
 367             denoiser->yv12_mc_running_avg.y_buffer + recon_yoffset;
 368         int mc_avg_y_stride = denoiser->yv12_mc_running_avg.y_stride;
 369         unsigned char *running_avg_y =
 370             denoiser->yv12_running_avg[INTRA_FRAME].y_buffer + recon_yoffset;
 371         int avg_y_stride = denoiser->yv12_running_avg[INTRA_FRAME].y_stride;
 372
 373         /* Filter. */
 374         decision = vp8_denoiser_filter(mc_running_avg_y, mc_avg_y_stride,
 375                                          running_avg_y, avg_y_stride,
 376                                          x->thismb, 16, motion_magnitude2,
 377                                          x->increase_denoising);
 378         denoiser->denoise_state[block_index] = motion_magnitude2 > 0 ?
 379             kFilterNonZeroMV : kFilterZeroMV;
 380     }
 381     if (decision == COPY_BLOCK)
 382     {
 383         /* No filtering of this block; it differs too much from the predictor,
 384          * or the motion vector magnitude is considered too big.
 385          */
 386         vp8_copy_mem16x16(
 387                 x->thismb, 16,
 388                 denoiser->yv12_running_avg[INTRA_FRAME].y_buffer + recon_yoffset,
 389                 denoiser->yv12_running_avg[INTRA_FRAME].y_stride);
 390         denoiser->denoise_state[block_index] = kNoFilter;
 391     }
 392     // Option to selectively deblock the denoised signal.
 393     if (apply_spatial_loop_filter) {
 394       loop_filter_info lfi;
 395       int apply_filter_col = 0;
 396       int apply_filter_row = 0;
 397       int apply_filter = 0;
 398       int y_stride = denoiser->yv12_running_avg[INTRA_FRAME].y_stride;
 399       int uv_stride =denoiser->yv12_running_avg[INTRA_FRAME].uv_stride;
 400
 401       // Fix filter level to some nominal value for now.
 402       int filter_level = 32;
 403
 404       int hev_index = lfi_n->hev_thr_lut[INTER_FRAME][filter_level];
 405       lfi.mblim = lfi_n->mblim[filter_level];
 406       lfi.blim = lfi_n->blim[filter_level];
 407       lfi.lim = lfi_n->lim[filter_level];
 408       lfi.hev_thr = lfi_n->hev_thr[hev_index];
 409
 410       // Apply filter if there is a difference in the denoiser filter state
 411       // between the current and left/top block, or if non-zero motion vector
 412       // is used for the motion-compensated filtering.
 413       if (mb_col > 0) {
 414         apply_filter_col = !((denoiser->denoise_state[block_index] ==
 415             denoiser->denoise_state[block_index - 1]) &&
 416             denoiser->denoise_state[block_index] != kFilterNonZeroMV);
 417         if (apply_filter_col) {
 418           // Filter left vertical edge.
 419           apply_filter = 1;
 420           vp8_loop_filter_mbv(
 421               denoiser->yv12_running_avg[INTRA_FRAME].y_buffer + recon_yoffset,
 422               NULL, NULL, y_stride, uv_stride, &lfi);
 423         }
 424       }
 425       if (mb_row > 0) {
 426         apply_filter_row = !((denoiser->denoise_state[block_index] ==
 427             denoiser->denoise_state[block_index - denoiser->num_mb_cols]) &&
 428             denoiser->denoise_state[block_index] != kFilterNonZeroMV);
 429         if (apply_filter_row) {
 430           // Filter top horizontal edge.
 431           apply_filter = 1;
 432           vp8_loop_filter_mbh(
 433               denoiser->yv12_running_avg[INTRA_FRAME].y_buffer + recon_yoffset,
 434               NULL, NULL, y_stride, uv_stride, &lfi);
 435         }
 436       }
 437       if (apply_filter) {
 438         // Update the signal block |x|. Pixel changes are only to top and/or
 439         // left boundary pixels: can we avoid full block copy here.
 440         vp8_copy_mem16x16(
 441             denoiser->yv12_running_avg[INTRA_FRAME].y_buffer + recon_yoffset,
 442             y_stride, x->thismb, 16);
 443       }
 444     }
 445 }