Adding support for error concealment in multi-threaded decoding

Also includes a couple of error concealment bug fixes:
- the segment_id wasn't properly initialized when missing
- when interpolating and no neighbors are found, set to zero
- clear the qcoef buffer when concealing an MB

Change-Id: Id79c876b41d78b559a2241e9cd0fd2cae6198f49

diff --git a/examples/decode_with_partial_drops.txt b/examples/decode_with_partial_drops.txt
index 30854d669c20883d012a0e9645d1382e78a1915d..7b0d3d2ca8e1b1f9b9f03f901cfb367c6a830fe5 100644 (file)
--- a/examples/decode_with_partial_drops.txt
+++ b/examples/decode_with_partial_drops.txt
@@ -110,7 +110,7 @@ void throw_packets(unsigned char* frame, int* size, int loss_rate,
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ DEC_INIT
 /* Initialize codec */
 flags = VPX_CODEC_USE_ERROR_CONCEALMENT;
-res = vpx_codec_dec_init(&codec, interface, NULL, flags);
+res = vpx_codec_dec_init(&codec, interface, &dec_cfg, flags);
 if(res)
     die_codec(&codec, "Failed to initialize decoder");
 
@@ -123,11 +123,15 @@ which specifies the range or pattern of frames to drop. The parameter is
 parsed as follows:
 
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ USAGE
-if(argc!=4 && argc != 5)
-    die("Usage: %s <infile> <outfile> <N-M|N/M|L,S>\n", argv[0]);
+if(argc < 4 || argc > 6)
+    die("Usage: %s <infile> <outfile> [-t <num threads>] <N-M|N/M|L,S>\n",
+        argv[0]);
 {
     char *nptr;
-    n = strtol(argv[3], &nptr, 0);
+    int arg_num = 3;
+    if (argc == 6 && strncmp(argv[arg_num++], "-t", 2) == 0)
+        dec_cfg.threads = strtol(argv[arg_num++], NULL, 0);
+    n = strtol(argv[arg_num], &nptr, 0);
     mode = (*nptr == '\0' || *nptr == ',') ? 2 : (*nptr == '-') ? 1 : 0;
 
     m = strtol(nptr+1, NULL, 0);
@@ -138,6 +142,7 @@ if(argc!=4 && argc != 5)
 seed = (m > 0) ? m : (unsigned int)time(NULL);
 srand(seed);thrown_frame = 0;
 printf("Seed: %u\n", seed);
+printf("Threads: %d\n", dec_cfg.threads);
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ USAGE
 
 
@@ -181,6 +186,7 @@ int              n, m, mode;
 unsigned int     seed;
 int              thrown=0, kept=0;
 int              thrown_frame=0, kept_frame=0;
+vpx_codec_dec_cfg_t  dec_cfg = {0};
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ EXTRA_VARS
 
 

diff --git a/vp8/decoder/decodframe.c b/vp8/decoder/decodframe.c
index 734ab36fc9467ab4ef2e0715cc4d4ed049a18b98..00e001733f5dca1829a12f654727be5641a89a92 100644 (file)
--- a/vp8/decoder/decodframe.c
+++ b/vp8/decoder/decodframe.c
@@ -239,12 +239,13 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd,
 
 #if CONFIG_ERROR_CONCEALMENT
     if (pbi->ec_enabled &&
-        (mb_idx > pbi->mvs_corrupt_from_mb ||
+        (mb_idx >= pbi->mvs_corrupt_from_mb ||
         vp8dx_bool_error(xd->current_bc)))
     {
         /* MB with corrupt residuals or corrupt mode/motion vectors.
          * Better to use the predictor as reconstruction.
          */
+        vpx_memset(xd->qcoeff, 0, sizeof(xd->qcoeff));
         vp8_conceal_corrupt_mb(xd);
         return;
     }
@@ -474,7 +475,13 @@ static void setup_token_decoder(VP8D_COMP *pbi,
     const unsigned char *partition;
 
     /* Parse number of token partitions to use */
-    pc->multi_token_partition = (TOKEN_PARTITION)vp8_read_literal(&pbi->bc, 2);
+    const TOKEN_PARTITION multi_token_partition =
+            (TOKEN_PARTITION)vp8_read_literal(&pbi->bc, 2);
+    /* Only update the multi_token_partition field if we are sure the value
+     * is correct. */
+    if (!pbi->ec_enabled || !vp8dx_bool_error(&pbi->bc))
+        pc->multi_token_partition = multi_token_partition;
+
     num_part = 1 << pc->multi_token_partition;
 
     /* Set up pointers to the first partition */

diff --git a/vp8/decoder/error_concealment.c b/vp8/decoder/error_concealment.c
index c06bdcf07be50959808ea9167b182fc5f6aad877..4f8f1a66c22304bef3b84677a5081a811b6a76a9 100644 (file)
--- a/vp8/decoder/error_concealment.c
+++ b/vp8/decoder/error_concealment.c
@@ -382,6 +382,7 @@ static void estimate_missing_mvs(MB_OVERLAP *overlaps,
             mi->mbmi.mode = SPLITMV;
             mi->mbmi.uv_mode = DC_PRED;
             mi->mbmi.partitioning = 3;
+            mi->mbmi.segment_id = 0;
             estimate_mb_mvs(block_overlaps,
                                 mi,
                                 mb_to_left_edge,
@@ -563,6 +564,12 @@ static void interpolate_mvs(MACROBLOCKD *mb,
                                                        mb->mb_to_top_edge,
                                                        mb->mb_to_bottom_edge);
             }
+            else
+            {
+                mv->as_int = 0;
+                mi->bmi[row*4 + col].as_mode = NEW4X4;
+                mi->mbmi.need_to_clamp_mvs = 0;
+            }
         }
     }
 }
@@ -597,6 +604,7 @@ void vp8_interpolate_motion(MACROBLOCKD *mb,
     mb->mode_info_context->mbmi.mode = SPLITMV;
     mb->mode_info_context->mbmi.uv_mode = DC_PRED;
     mb->mode_info_context->mbmi.partitioning = 3;
+    mb->mode_info_context->mbmi.segment_id = 0;
 }
 
 void vp8_conceal_corrupt_mb(MACROBLOCKD *xd)

diff --git a/vp8/decoder/threading.c b/vp8/decoder/threading.c
index c1df823f108929bc01f73607caaa8d83f861ef2b..a7af9acfb4850ad3ad09ea109bf0bf40fafd5856 100644 (file)
--- a/vp8/decoder/threading.c
+++ b/vp8/decoder/threading.c
@@ -22,6 +22,9 @@
 #include "detokenize.h"
 #include "vp8/common/reconinter.h"
 #include "reconintra_mt.h"
+#if CONFIG_ERROR_CONCEALMENT
+#include "error_concealment.h"
+#endif
 
 extern void mb_init_dequantizer(VP8D_COMP *pbi, MACROBLOCKD *xd);
 extern void clamp_mvs(MACROBLOCKD *xd);
@@ -151,6 +154,20 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd, int mb_row, int m
         vp8_build_inter_predictors_mb(xd);
     }
 
+#if CONFIG_ERROR_CONCEALMENT
+    if (pbi->ec_enabled &&
+        (mb_row * pbi->common.mb_cols + mb_col >= pbi->mvs_corrupt_from_mb ||
+        vp8dx_bool_error(xd->current_bc)))
+    {
+        /* MB with corrupt residuals or corrupt mode/motion vectors.
+         * Better to use the predictor as reconstruction.
+         */
+        vpx_memset(xd->qcoeff, 0, sizeof(xd->qcoeff));
+        vp8_conceal_corrupt_mb(xd);
+        return;
+    }
+#endif
+
     /* dequantization and idct */
     if (xd->mode_info_context->mbmi.mode != B_PRED && xd->mode_info_context->mbmi.mode != SPLITMV)
     {
@@ -291,12 +308,37 @@ static THREAD_FUNCTION thread_decoding_proc(void *p_data)
 
                         update_blockd_bmi(xd);
 
-                        /* Distance of Mb to the various image edges.
-                         * These are specified to 8th pel as they are always compared to values that are in 1/8th pel units
+                        /* Distance of MB to the various image edges.
+                         * These are specified to 8th pel as they are always
+                         * compared to values that are in 1/8th pel units.
                          */
                         xd->mb_to_left_edge = -((mb_col * 16) << 3);
                         xd->mb_to_right_edge = ((pc->mb_cols - 1 - mb_col) * 16) << 3;
 
+#if CONFIG_ERROR_CONCEALMENT
+                        if (pbi->ec_enabled &&
+                            (xd->mode_info_context->mbmi.ref_frame ==
+                                                                 INTRA_FRAME) &&
+                            vp8dx_bool_error(xd->current_bc))
+                        {
+                            /* We have an intra block with corrupt coefficients,
+                             * better to conceal with an inter block.
+                             * Interpolate MVs from neighboring MBs
+                             *
+                             * Note that for the first mb with corrupt residual
+                             * in a frame, we might not discover that before
+                             * decoding the residual. That happens after this
+                             * check, and therefore no inter concealment will be
+                             * done.
+                             */
+                            vp8_interpolate_motion(xd,
+                                                   mb_row, mb_col,
+                                                   pc->mb_rows, pc->mb_cols,
+                                                   pc->mode_info_stride);
+                        }
+#endif
+
+
                         xd->dst.y_buffer = pc->yv12_fb[dst_fb_idx].y_buffer + recon_yoffset;
                         xd->dst.u_buffer = pc->yv12_fb[dst_fb_idx].u_buffer + recon_uvoffset;
                         xd->dst.v_buffer = pc->yv12_fb[dst_fb_idx].v_buffer + recon_uvoffset;
@@ -772,12 +814,35 @@ void vp8mt_decode_mb_rows( VP8D_COMP *pbi, MACROBLOCKD *xd)
 
                 update_blockd_bmi(xd);
 
-                /* Distance of Mb to the various image edges.
-                 * These are specified to 8th pel as they are always compared to values that are in 1/8th pel units
+                /* Distance of MB to the various image edges.
+                 * These are specified to 8th pel as they are always compared to
+                 * values that are in 1/8th pel units.
                  */
                 xd->mb_to_left_edge = -((mb_col * 16) << 3);
                 xd->mb_to_right_edge = ((pc->mb_cols - 1 - mb_col) * 16) << 3;
 
+#if CONFIG_ERROR_CONCEALMENT
+                if (pbi->ec_enabled &&
+                    (xd->mode_info_context->mbmi.ref_frame == INTRA_FRAME) &&
+                    vp8dx_bool_error(xd->current_bc))
+                {
+                    /* We have an intra block with corrupt coefficients, better
+                     * to conceal with an inter block. Interpolate MVs from
+                     * neighboring MBs
+                     *
+                     * Note that for the first mb with corrupt residual in a
+                     * frame, we might not discover that before decoding the
+                     * residual. That happens after this check, and therefore no
+                     * inter concealment will be done.
+                     */
+                    vp8_interpolate_motion(xd,
+                                           mb_row, mb_col,
+                                           pc->mb_rows, pc->mb_cols,
+                                           pc->mode_info_stride);
+                }
+#endif
+
+
                 xd->dst.y_buffer = pc->yv12_fb[dst_fb_idx].y_buffer + recon_yoffset;
                 xd->dst.u_buffer = pc->yv12_fb[dst_fb_idx].u_buffer + recon_uvoffset;
                 xd->dst.v_buffer = pc->yv12_fb[dst_fb_idx].v_buffer + recon_uvoffset;