Revert "Fix dering filter when using 4:2:2 or 4:4:0 subsampling"

This reverts commit 401204a50bea8b21e5fcd721cd5db513b8f70e72.

Change-Id: Id27eadf679b0df2d2ccfab61155be29979b0b6ba

diff --git a/av1/common/dering.c b/av1/common/dering.c
index 451903161e42d636efa40182ad914cf3c2c5d36b..7643e03f7c3f6aba30c3de9dd5ad88b376014055 100644 (file)
--- a/av1/common/dering.c
+++ b/av1/common/dering.c
@@ -54,10 +54,8 @@ void av1_dering_frame(YV12_BUFFER_CONFIG *frame, AV1_COMMON *cm,
   unsigned char *bskip;
   int dir[OD_DERING_NBLOCKS][OD_DERING_NBLOCKS] = { { 0 } };
   int stride;
-  int bsize_x[3];
-  int bsize_y[3];
-  int dec_x[3];
-  int dec_y[3];
+  int bsize[3];
+  int dec[3];
   int pli;
   int coeff_shift = AOMMAX(cm->bit_depth - 8, 0);
   int nplanes;
@@ -70,17 +68,15 @@ void av1_dering_frame(YV12_BUFFER_CONFIG *frame, AV1_COMMON *cm,
   nhsb = (cm->mi_cols + MAX_MIB_SIZE - 1) / MAX_MIB_SIZE;
   bskip = aom_malloc(sizeof(*bskip) * cm->mi_rows * cm->mi_cols);
   av1_setup_dst_planes(xd->plane, frame, 0, 0);
-  for (pli = 0; pli < nplanes; pli++) {
-    dec_x[pli] = xd->plane[pli].subsampling_x;
-    dec_y[pli] = xd->plane[pli].subsampling_y;
-    bsize_x[pli] = 8 >> dec_x[pli];
-    bsize_y[pli] = 8 >> dec_y[pli];
+  for (pli = 0; pli < 3; pli++) {
+    dec[pli] = xd->plane[pli].subsampling_x;
+    bsize[pli] = 8 >> dec[pli];
   }
-  stride = bsize_x[0] * cm->mi_cols;
-  for (pli = 0; pli < nplanes; pli++) {
+  stride = bsize[0] * cm->mi_cols;
+  for (pli = 0; pli < 3; pli++) {
     src[pli] = aom_malloc(sizeof(*src) * cm->mi_rows * cm->mi_cols * 64);
-    for (r = 0; r < bsize_y[pli] * cm->mi_rows; ++r) {
-      for (c = 0; c < bsize_x[pli] * cm->mi_cols; ++c) {
+    for (r = 0; r < bsize[pli] * cm->mi_rows; ++r) {
+      for (c = 0; c < bsize[pli] * cm->mi_cols; ++c) {
 #if CONFIG_AOM_HIGHBITDEPTH
         if (cm->use_highbitdepth) {
           src[pli][r * stride + c] = CONVERT_TO_SHORTPTR(
@@ -124,29 +120,27 @@ void av1_dering_frame(YV12_BUFFER_CONFIG *frame, AV1_COMMON *cm,
         else
           threshold = level << coeff_shift;
         if (threshold == 0) continue;
-        od_dering(dst, MAX_MIB_SIZE * bsize_x[pli],
-                  &src[pli][sbr * stride * bsize_x[pli] * MAX_MIB_SIZE +
-                            sbc * bsize_x[pli] * MAX_MIB_SIZE],
-                  stride, nhb, nvb, sbc, sbr, nhsb, nvsb, dec_x[pli],
-                  dec_y[pli], dir, pli,
+        od_dering(dst, MAX_MIB_SIZE * bsize[pli],
+                  &src[pli][sbr * stride * bsize[pli] * MAX_MIB_SIZE +
+                            sbc * bsize[pli] * MAX_MIB_SIZE],
+                  stride, nhb, nvb, sbc, sbr, nhsb, nvsb, dec[pli], dir, pli,
                   &bskip[MAX_MIB_SIZE * sbr * cm->mi_cols + MAX_MIB_SIZE * sbc],
                   cm->mi_cols, threshold, coeff_shift);
-        for (r = 0; r < bsize_y[pli] * nvb; ++r) {
-          for (c = 0; c < bsize_x[pli] * nhb; ++c) {
+        for (r = 0; r < bsize[pli] * nvb; ++r) {
+          for (c = 0; c < bsize[pli] * nhb; ++c) {
 #if CONFIG_AOM_HIGHBITDEPTH
             if (cm->use_highbitdepth) {
               CONVERT_TO_SHORTPTR(xd->plane[pli].dst.buf)
               [xd->plane[pli].dst.stride *
-                   (bsize_x[pli] * MAX_MIB_SIZE * sbr + r) +
-               sbc * bsize_x[pli] * MAX_MIB_SIZE + c] =
-                  dst[r * MAX_MIB_SIZE * bsize_x[pli] + c];
+                   (bsize[pli] * MAX_MIB_SIZE * sbr + r) +
+               sbc * bsize[pli] * MAX_MIB_SIZE + c] =
+                  dst[r * MAX_MIB_SIZE * bsize[pli] + c];
             } else {
 #endif
-              xd->plane[pli]
-                  .dst.buf[xd->plane[pli].dst.stride *
-                               (bsize_x[pli] * MAX_MIB_SIZE * sbr + r) +
-                           sbc * bsize_x[pli] * MAX_MIB_SIZE + c] =
-                  dst[r * MAX_MIB_SIZE * bsize_x[pli] + c];
+              xd->plane[pli].dst.buf[xd->plane[pli].dst.stride *
+                                         (bsize[pli] * MAX_MIB_SIZE * sbr + r) +
+                                     sbc * bsize[pli] * MAX_MIB_SIZE + c] =
+                  dst[r * MAX_MIB_SIZE * bsize[pli] + c];
 #if CONFIG_AOM_HIGHBITDEPTH
             }
 #endif

diff --git a/av1/common/od_dering.c b/av1/common/od_dering.c
index 7ed49a4b1a53455ad47de05dec6c672bcd90236b..7aa704f243c25b430c080f4510755d0fe37d374f 100644 (file)
--- a/av1/common/od_dering.c
+++ b/av1/common/od_dering.c
@@ -183,19 +183,6 @@ int od_filter_dering_direction_4x4_c(int16_t *y, int ystride, const int16_t *in,
   return (total_abs + 2) >> 2;
 }
 
-int od_filter_dering_direction_4x8(int16_t *y, int ystride, const int16_t *in,
-                                     int threshold, int dir) {
-  return od_filter_dering_direction_4x4(y, ystride, in, threshold, dir)
-  + od_filter_dering_direction_4x4(y + 4*ystride, ystride,
-                                   in + 4*OD_FILT_BSTRIDE, threshold, dir);
-}
-
-int od_filter_dering_direction_8x4(int16_t *y, int ystride, const int16_t *in,
-                                     int threshold, int dir) {
-  return od_filter_dering_direction_4x4(y, ystride, in, threshold, dir)
-  + od_filter_dering_direction_4x4(y + 4, ystride, in + 4, threshold, dir);
-}
-
 /* Smooth in the direction orthogonal to what was detected. */
 void od_filter_dering_orthogonal_8x8_c(int16_t *y, int ystride,
                                        const int16_t *in, int threshold,
@@ -254,21 +241,6 @@ void od_filter_dering_orthogonal_4x4_c(int16_t *y, int ystride,
   }
 }
 
-void od_filter_dering_orthogonal_4x8(int16_t *y, int ystride,
-                                       const int16_t *in, int threshold,
-                                       int dir) {
-  od_filter_dering_orthogonal_4x4(y, ystride, in, threshold, dir);
-  od_filter_dering_orthogonal_4x4(y + 4*ystride, ystride,
-                                  in + 4*OD_FILT_BSTRIDE, threshold, dir);
-}
-
-void od_filter_dering_orthogonal_8x4(int16_t *y, int ystride,
-                                       const int16_t *in, int threshold,
-                                       int dir) {
-  od_filter_dering_orthogonal_4x4(y, ystride, in, threshold, dir);
-  od_filter_dering_orthogonal_4x4(y + 4, ystride, in + 4, threshold, dir);
-}
-
 /* This table approximates x^0.16 with the index being log2(x). It is clamped
    to [-.5, 3]. The table is computed as:
    round(256*min(3, max(.5, 1.08*(sqrt(2)*2.^([0:17]+8)/256/256).^.16))) */
@@ -292,7 +264,7 @@ static INLINE int od_adjust_thresh(int threshold, int32_t var) {
 
 void od_dering(int16_t *y, int ystride, const od_dering_in *x, int xstride,
                int nhb, int nvb, int sbx, int sby, int nhsb, int nvsb, int xdec,
-               int ydec, int dir[OD_DERING_NBLOCKS][OD_DERING_NBLOCKS], int pli,
+               int dir[OD_DERING_NBLOCKS][OD_DERING_NBLOCKS], int pli,
                unsigned char *bskip, int skip_stride, int threshold,
                int coeff_shift) {
   int i;
@@ -301,35 +273,32 @@ void od_dering(int16_t *y, int ystride, const od_dering_in *x, int xstride,
   int by;
   int16_t inbuf[OD_DERING_INBUF_SIZE];
   int16_t *in;
-  int bsize_x = 3 - xdec;
-  int bsize_y = 3 - ydec;
+  int bsize;
   int32_t var[OD_DERING_NBLOCKS][OD_DERING_NBLOCKS];
   int filter2_thresh[OD_DERING_NBLOCKS][OD_DERING_NBLOCKS];
   od_filter_dering_direction_func filter_dering_direction[OD_DERINGSIZES] = {
-    od_filter_dering_direction_8x8, od_filter_dering_direction_8x4,
-    od_filter_dering_direction_4x8, od_filter_dering_direction_4x4
+    od_filter_dering_direction_4x4, od_filter_dering_direction_8x8
   };
   od_filter_dering_orthogonal_func filter_dering_orthogonal[OD_DERINGSIZES] = {
-    od_filter_dering_orthogonal_8x8, od_filter_dering_orthogonal_8x4,
-    od_filter_dering_orthogonal_4x8, od_filter_dering_orthogonal_4x4
+    od_filter_dering_orthogonal_4x4, od_filter_dering_orthogonal_8x8
   };
-  int filter_idx = xdec*2 + ydec;
+  bsize = 3 - xdec;
   in = inbuf + OD_FILT_BORDER * OD_FILT_BSTRIDE + OD_FILT_BORDER;
   /* We avoid filtering the pixels for which some of the pixels to average
      are outside the frame. We could change the filter instead, but it would
      add special cases for any future vectorization. */
   for (i = 0; i < OD_DERING_INBUF_SIZE; i++) inbuf[i] = OD_DERING_VERY_LARGE;
   for (i = -OD_FILT_BORDER * (sby != 0);
-       i < (nvb << bsize_y) + OD_FILT_BORDER * (sby != nvsb - 1); i++) {
+       i < (nvb << bsize) + OD_FILT_BORDER * (sby != nvsb - 1); i++) {
     for (j = -OD_FILT_BORDER * (sbx != 0);
-         j < (nhb << bsize_x) + OD_FILT_BORDER * (sbx != nhsb - 1); j++) {
+         j < (nhb << bsize) + OD_FILT_BORDER * (sbx != nhsb - 1); j++) {
       in[i * OD_FILT_BSTRIDE + j] = x[i * xstride + j];
     }
   }
   /* Assume deringing filter is sparsely applied, so do one large copy rather
      than small copies later if deringing is skipped. */
-  for (i = 0; i < nvb << bsize_y; i++) {
-    for (j = 0; j < nhb << bsize_x; j++) {
+  for (i = 0; i < nvb << bsize; i++) {
+    for (j = 0; j < nhb << bsize; j++) {
       y[i * ystride + j] = in[i * OD_FILT_BSTRIDE + j];
     }
   }
@@ -347,9 +316,9 @@ void od_dering(int16_t *y, int ystride, const od_dering_in *x, int xstride,
            to be a little bit more aggressive on pure horizontal/vertical
            since the ringing there tends to be directional, so it doesn't
            get removed by the directional filtering. */
-        filter2_thresh[by][bx] = (filter_dering_direction[filter_idx])(
-            &y[(by * ystride << bsize_y) + (bx << bsize_x)], ystride,
-            &in[(by * OD_FILT_BSTRIDE << bsize_y) + (bx << bsize_x)],
+        filter2_thresh[by][bx] = (filter_dering_direction[bsize - OD_LOG_BSIZE0])(
+            &y[(by * ystride << bsize) + (bx << bsize)], ystride,
+            &in[(by * OD_FILT_BSTRIDE << bsize) + (bx << bsize)],
             od_adjust_thresh(threshold, var[by][bx]), dir[by][bx]);
       }
     }
@@ -357,25 +326,25 @@ void od_dering(int16_t *y, int ystride, const od_dering_in *x, int xstride,
     for (by = 0; by < nvb; by++) {
       for (bx = 0; bx < nhb; bx++) {
         if (bskip[by * skip_stride + bx]) continue;
-        filter2_thresh[by][bx] = (filter_dering_direction[filter_idx])(
-            &y[(by * ystride << bsize_y) + (bx << bsize_x)], ystride,
-            &in[(by * OD_FILT_BSTRIDE << bsize_y) + (bx << bsize_x)], threshold,
+        filter2_thresh[by][bx] = (filter_dering_direction[bsize - OD_LOG_BSIZE0])(
+            &y[(by * ystride << bsize) + (bx << bsize)], ystride,
+            &in[(by * OD_FILT_BSTRIDE << bsize) + (bx << bsize)], threshold,
             dir[by][bx]);
       }
     }
   }
-  for (i = 0; i < nvb << bsize_y; i++) {
-    for (j = 0; j < nhb << bsize_x; j++) {
+  for (i = 0; i < nvb << bsize; i++) {
+    for (j = 0; j < nhb << bsize; j++) {
       in[i * OD_FILT_BSTRIDE + j] = y[i * ystride + j];
     }
   }
   for (by = 0; by < nvb; by++) {
     for (bx = 0; bx < nhb; bx++) {
       if (bskip[by * skip_stride + bx] || filter2_thresh[by][bx] == 0) continue;
-      (filter_dering_orthogonal[filter_idx])(
-          &y[(by * ystride << bsize_y) + (bx << bsize_x)], ystride,
-          &in[(by * OD_FILT_BSTRIDE << bsize_y) + (bx << bsize_x)],
-          filter2_thresh[by][bx], dir[by][bx]);
+      (filter_dering_orthogonal[bsize - OD_LOG_BSIZE0])(
+          &y[(by * ystride << bsize) + (bx << bsize)], ystride,
+          &in[(by * OD_FILT_BSTRIDE << bsize) + (bx << bsize)], filter2_thresh[by][bx],
+          dir[by][bx]);
     }
   }
 }

diff --git a/av1/common/od_dering.h b/av1/common/od_dering.h
index fc3a3ef0ce1b195a0a7d2078d9009f0914aaf473..c64439f3d4ee80f27a0d0649bf374dee0bca0c72 100644 (file)
--- a/av1/common/od_dering.h
+++ b/av1/common/od_dering.h
@@ -19,7 +19,7 @@
 typedef int16_t od_dering_in;
 #endif
 
-#define OD_DERINGSIZES (4)
+#define OD_DERINGSIZES (2)
 
 #define OD_DERING_NBLOCKS (OD_BSIZE_MAX / 8)
 
@@ -36,24 +36,16 @@ typedef void (*od_filter_dering_orthogonal_func)(int16_t *y, int ystride,
                                                  int threshold, int dir);
 void od_dering(int16_t *y, int ystride, const od_dering_in *x, int xstride,
                int nvb, int nhb, int sbx, int sby, int nhsb, int nvsb, int xdec,
-               int ydec, int dir[OD_DERING_NBLOCKS][OD_DERING_NBLOCKS], int pli,
+               int dir[OD_DERING_NBLOCKS][OD_DERING_NBLOCKS], int pli,
                unsigned char *bskip, int skip_stride, int threshold,
                int coeff_shift);
 int od_filter_dering_direction_4x4_c(int16_t *y, int ystride, const int16_t *in,
                                      int threshold, int dir);
-int od_filter_dering_direction_4x8(int16_t *y, int ystride, const int16_t *in,
-                                   int threshold, int dir);
-int od_filter_dering_direction_8x4(int16_t *y, int ystride, const int16_t *in,
-                                   int threshold, int dir);
 int od_filter_dering_direction_8x8_c(int16_t *y, int ystride, const int16_t *in,
                                      int threshold, int dir);
 void od_filter_dering_orthogonal_4x4_c(int16_t *y, int ystride,
                                        const int16_t *in, int threshold,
                                        int dir);
-void od_filter_dering_orthogonal_4x8(int16_t *y, int ystride, const int16_t *in,
-                                     int threshold, int dir);
-void od_filter_dering_orthogonal_8x4(int16_t *y, int ystride, const int16_t *in,
-                                     int threshold, int dir);
 void od_filter_dering_orthogonal_8x8_c(int16_t *y, int ystride,
                                        const int16_t *in, int threshold,
                                        int dir);

diff --git a/av1/encoder/pickdering.c b/av1/encoder/pickdering.c
index 6f7767ac01bb81f73101c8b08b2111fb21fc0765..4ef83cdd4a62f9425184b47c418dd70ba1f8f892 100644 (file)
--- a/av1/encoder/pickdering.c
+++ b/av1/encoder/pickdering.c
@@ -115,7 +115,7 @@ int av1_dering_search(YV12_BUFFER_CONFIG *frame, const YV12_BUFFER_CONFIG *ref,
         od_dering(dst, MAX_MIB_SIZE * bsize[0],
                   &src[sbr * stride * bsize[0] * MAX_MIB_SIZE +
                        sbc * bsize[0] * MAX_MIB_SIZE],
-                  cm->mi_cols * bsize[0], nhb, nvb, sbc, sbr, nhsb, nvsb, 0, 0,
+                  cm->mi_cols * bsize[0], nhb, nvb, sbc, sbr, nhsb, nvsb, 0,
                   dir, 0,
                   &bskip[MAX_MIB_SIZE * sbr * cm->mi_cols + MAX_MIB_SIZE * sbc],
                   cm->mi_cols, threshold, coeff_shift);