multi-res: add drop_frame support

Added drop_frame support in multi-resolution encoder.

If one frame is dropped at a lower-resolution level, the next
upper-resolution level encoder needs to encode that frame
independently without any lower-resolution level motion
information.

Another issue is that if one frame is dropped at some but not all
resolution levels, a frame after that one may use different set
of reference frames at different resolution levels. This reference
frame asynchronization could degrade motion search precision in
upper-resolution level encoding, which uses lower-resolution level
motion result. This change compares the lower-resolution and upper-
resolution level's reference frames. If they are not the same, the
upper-resolution level encoder can not use lower-resolution level
motion result.

Change-Id: I61afa4f313630e75b7cbdd5742e230e8724a988a

diff --git a/vp8/common/blockd.h b/vp8/common/blockd.h
index c715f6547b6863a50ea5605558c7113a541fc639..acef8caa2e99a0fa747bc89a145ff4485dc5953c 100644 (file)
--- a/vp8/common/blockd.h
+++ b/vp8/common/blockd.h
@@ -182,6 +182,9 @@ typedef struct
 typedef struct
 {
     FRAME_TYPE frame_type;
+    int is_frame_dropped;
+    /* The frame number of each reference frames */
+    unsigned int low_res_ref_frames[MAX_REF_FRAMES];
     LOWER_RES_MB_INFO *mb_info;
 } LOWER_RES_FRAME_INFO;
 #endif

diff --git a/vp8/encoder/mr_dissim.c b/vp8/encoder/mr_dissim.c
index 564f963f8fb4fa3950e8ab717b0a02dc0740844f..71218cca146f4c7cffe84853fca247c484be7e23 100644 (file)
--- a/vp8/encoder/mr_dissim.c
+++ b/vp8/encoder/mr_dissim.c
@@ -53,6 +53,7 @@ if(x->mbmi.ref_frame !=INTRA_FRAME)   \
 void vp8_cal_dissimilarity(VP8_COMP *cpi)
 {
     VP8_COMMON *cm = &cpi->common;
+    int i;
 
     /* Note: The first row & first column in mip are outside the frame, which
      * were initialized to all 0.(ref_frame, mode, mv...)
@@ -70,6 +71,13 @@ void vp8_cal_dissimilarity(VP8_COMP *cpi)
 
         store_info->frame_type = cm->frame_type;
 
+        if(cm->frame_type != KEY_FRAME)
+        {
+            store_info->is_frame_dropped = 0;
+            for (i = 1; i < MAX_REF_FRAMES; i++)
+                store_info->low_res_ref_frames[i] = cpi->current_ref_frames[i];
+        }
+
         if(cm->frame_type != KEY_FRAME)
         {
             int mb_row;
@@ -203,3 +211,26 @@ void vp8_cal_dissimilarity(VP8_COMP *cpi)
         }
     }
 }
+
+/* This function is called only when this frame is dropped at current
+   resolution level. */
+void vp8_store_drop_frame_info(VP8_COMP *cpi)
+{
+    /* If the frame is dropped in lower-resolution encoding, this information
+       is passed to higher resolution level so that the encoder knows there
+       is no mode & motion info available.
+     */
+    if (cpi->oxcf.mr_total_resolutions >1
+        && cpi->oxcf.mr_encoder_id < (cpi->oxcf.mr_total_resolutions - 1))
+    {
+        /* Store info for show/no-show frames for supporting alt_ref.
+         * If parent frame is alt_ref, child has one too.
+         */
+        LOWER_RES_FRAME_INFO* store_info
+                      = (LOWER_RES_FRAME_INFO*)cpi->oxcf.mr_low_res_mode_info;
+
+        /* Set frame_type to be INTER_FRAME since we won't drop key frame. */
+        store_info->frame_type = INTER_FRAME;
+        store_info->is_frame_dropped = 1;
+    }
+}

diff --git a/vp8/encoder/mr_dissim.h b/vp8/encoder/mr_dissim.h
index 3d2c2035fc082a38f23d3cc55b58053117ad9ad9..f8cb135d52c3bbf3ade66e3cad748e44be1ab38e 100644 (file)
--- a/vp8/encoder/mr_dissim.h
+++ b/vp8/encoder/mr_dissim.h
@@ -15,5 +15,6 @@
 
 extern void vp8_cal_low_res_mb_cols(VP8_COMP *cpi);
 extern void vp8_cal_dissimilarity(VP8_COMP *cpi);
+extern void vp8_store_drop_frame_info(VP8_COMP *cpi);
 
 #endif

diff --git a/vp8/encoder/onyx_if.c b/vp8/encoder/onyx_if.c
index d50076d3b871116ed8667002ca740a97c360f170..1d8613f381608c91e81762ec0595d1c1bbb8b790 100644 (file)
--- a/vp8/encoder/onyx_if.c
+++ b/vp8/encoder/onyx_if.c
@@ -3014,8 +3014,9 @@ static int recode_loop_test( VP8_COMP *cpi,
     return force_recode;
 }
 
-static void update_reference_frames(VP8_COMMON *cm)
+static void update_reference_frames(VP8_COMP *cpi)
 {
+    VP8_COMMON *cm = &cpi->common;
     YV12_BUFFER_CONFIG *yv12_fb = cm->yv12_fb;
 
     /* At this point the new frame has been encoded.
@@ -3030,6 +3031,11 @@ static void update_reference_frames(VP8_COMMON *cm)
         yv12_fb[cm->alt_fb_idx].flags &= ~VP8_ALTR_FRAME;
 
         cm->alt_fb_idx = cm->gld_fb_idx = cm->new_fb_idx;
+
+#if CONFIG_MULTI_RES_ENCODING
+        cpi->current_ref_frames[GOLDEN_FRAME] = cm->current_video_frame;
+        cpi->current_ref_frames[ALTREF_FRAME] = cm->current_video_frame;
+#endif
     }
     else    /* For non key frames */
     {
@@ -3040,6 +3046,10 @@ static void update_reference_frames(VP8_COMMON *cm)
             cm->yv12_fb[cm->new_fb_idx].flags |= VP8_ALTR_FRAME;
             cm->yv12_fb[cm->alt_fb_idx].flags &= ~VP8_ALTR_FRAME;
             cm->alt_fb_idx = cm->new_fb_idx;
+
+#if CONFIG_MULTI_RES_ENCODING
+            cpi->current_ref_frames[ALTREF_FRAME] = cm->current_video_frame;
+#endif
         }
         else if (cm->copy_buffer_to_arf)
         {
@@ -3052,6 +3062,11 @@ static void update_reference_frames(VP8_COMMON *cm)
                     yv12_fb[cm->lst_fb_idx].flags |= VP8_ALTR_FRAME;
                     yv12_fb[cm->alt_fb_idx].flags &= ~VP8_ALTR_FRAME;
                     cm->alt_fb_idx = cm->lst_fb_idx;
+
+#if CONFIG_MULTI_RES_ENCODING
+                    cpi->current_ref_frames[ALTREF_FRAME] =
+                        cpi->current_ref_frames[LAST_FRAME];
+#endif
                 }
             }
             else /* if (cm->copy_buffer_to_arf == 2) */
@@ -3061,6 +3076,11 @@ static void update_reference_frames(VP8_COMMON *cm)
                     yv12_fb[cm->gld_fb_idx].flags |= VP8_ALTR_FRAME;
                     yv12_fb[cm->alt_fb_idx].flags &= ~VP8_ALTR_FRAME;
                     cm->alt_fb_idx = cm->gld_fb_idx;
+
+#if CONFIG_MULTI_RES_ENCODING
+                    cpi->current_ref_frames[ALTREF_FRAME] =
+                        cpi->current_ref_frames[GOLDEN_FRAME];
+#endif
                 }
             }
         }
@@ -3072,6 +3092,10 @@ static void update_reference_frames(VP8_COMMON *cm)
             cm->yv12_fb[cm->new_fb_idx].flags |= VP8_GOLD_FRAME;
             cm->yv12_fb[cm->gld_fb_idx].flags &= ~VP8_GOLD_FRAME;
             cm->gld_fb_idx = cm->new_fb_idx;
+
+#if CONFIG_MULTI_RES_ENCODING
+            cpi->current_ref_frames[GOLDEN_FRAME] = cm->current_video_frame;
+#endif
         }
         else if (cm->copy_buffer_to_gf)
         {
@@ -3084,6 +3108,11 @@ static void update_reference_frames(VP8_COMMON *cm)
                     yv12_fb[cm->lst_fb_idx].flags |= VP8_GOLD_FRAME;
                     yv12_fb[cm->gld_fb_idx].flags &= ~VP8_GOLD_FRAME;
                     cm->gld_fb_idx = cm->lst_fb_idx;
+
+#if CONFIG_MULTI_RES_ENCODING
+                    cpi->current_ref_frames[GOLDEN_FRAME] =
+                        cpi->current_ref_frames[LAST_FRAME];
+#endif
                 }
             }
             else /* if (cm->copy_buffer_to_gf == 2) */
@@ -3093,6 +3122,11 @@ static void update_reference_frames(VP8_COMMON *cm)
                     yv12_fb[cm->alt_fb_idx].flags |= VP8_GOLD_FRAME;
                     yv12_fb[cm->gld_fb_idx].flags &= ~VP8_GOLD_FRAME;
                     cm->gld_fb_idx = cm->alt_fb_idx;
+
+#if CONFIG_MULTI_RES_ENCODING
+                    cpi->current_ref_frames[GOLDEN_FRAME] =
+                        cpi->current_ref_frames[ALTREF_FRAME];
+#endif
                 }
             }
         }
@@ -3103,6 +3137,10 @@ static void update_reference_frames(VP8_COMMON *cm)
         cm->yv12_fb[cm->new_fb_idx].flags |= VP8_LAST_FRAME;
         cm->yv12_fb[cm->lst_fb_idx].flags &= ~VP8_LAST_FRAME;
         cm->lst_fb_idx = cm->new_fb_idx;
+
+#if CONFIG_MULTI_RES_ENCODING
+        cpi->current_ref_frames[LAST_FRAME] = cm->current_video_frame;
+#endif
     }
 }
 
@@ -3313,8 +3351,28 @@ static void encode_frame_to_data_rate
      */
     if (cpi->oxcf.mr_encoder_id)
     {
-        cm->frame_type =
-            ((LOWER_RES_FRAME_INFO*)cpi->oxcf.mr_low_res_mode_info)->frame_type;
+        LOWER_RES_FRAME_INFO* low_res_frame_info
+                        = (LOWER_RES_FRAME_INFO*)cpi->oxcf.mr_low_res_mode_info;
+
+        cm->frame_type = low_res_frame_info->frame_type;
+
+        if(cm->frame_type != KEY_FRAME)
+        {
+            cpi->mr_low_res_mv_avail = 1;
+            cpi->mr_low_res_mv_avail &= !(low_res_frame_info->is_frame_dropped);
+
+            if (cpi->ref_frame_flags & VP8_LAST_FRAME)
+                cpi->mr_low_res_mv_avail &= (cpi->current_ref_frames[LAST_FRAME]
+                         == low_res_frame_info->low_res_ref_frames[LAST_FRAME]);
+
+            if (cpi->ref_frame_flags & VP8_GOLD_FRAME)
+                cpi->mr_low_res_mv_avail &= (cpi->current_ref_frames[GOLDEN_FRAME]
+                         == low_res_frame_info->low_res_ref_frames[GOLDEN_FRAME]);
+
+            if (cpi->ref_frame_flags & VP8_ALTR_FRAME)
+                cpi->mr_low_res_mv_avail &= (cpi->current_ref_frames[ALTREF_FRAME]
+                         == low_res_frame_info->low_res_ref_frames[ALTREF_FRAME]);
+        }
     }
 #endif
 
@@ -3422,6 +3480,10 @@ static void encode_frame_to_data_rate
             if (cpi->bits_off_target > cpi->oxcf.maximum_buffer_size)
                 cpi->bits_off_target = cpi->oxcf.maximum_buffer_size;
 
+#if CONFIG_MULTI_RES_ENCODING
+            vp8_store_drop_frame_info(cpi);
+#endif
+
             cm->current_video_frame++;
             cpi->frames_since_key++;
 
@@ -3459,6 +3521,10 @@ static void encode_frame_to_data_rate
     /* Decide how big to make the frame */
     if (!vp8_pick_frame_size(cpi))
     {
+        /*TODO: 2 drop_frame and return code could be put together. */
+#if CONFIG_MULTI_RES_ENCODING
+        vp8_store_drop_frame_info(cpi);
+#endif
         cm->current_video_frame++;
         cpi->frames_since_key++;
         return;
@@ -4231,7 +4297,7 @@ static void encode_frame_to_data_rate
         vp8_loopfilter_frame(cpi, cm);
     }
 
-    update_reference_frames(cm);
+    update_reference_frames(cpi);
 
 #if !(CONFIG_REALTIME_ONLY & CONFIG_ONTHEFLY_BITPACKING)
     if (cpi->oxcf.error_resilient_mode)

diff --git a/vp8/encoder/onyx_int.h b/vp8/encoder/onyx_int.h
index 808080f22c199607e9c01ba8272b6249ec4855c1..584cadae6b1d28ebf6b2cfb7169b4fd767877be5 100644 (file)
--- a/vp8/encoder/onyx_int.h
+++ b/vp8/encoder/onyx_int.h
@@ -701,6 +701,10 @@ typedef struct VP8_COMP
 #if CONFIG_MULTI_RES_ENCODING
     /* Number of MBs per row at lower-resolution level */
     int    mr_low_res_mb_cols;
+    /* Indicate if lower-res mv info is available */
+    unsigned char  mr_low_res_mv_avail;
+    /* The frame number of each reference frames */
+    unsigned int current_ref_frames[MAX_REF_FRAMES];
 #endif
 
     struct rd_costs_struct

diff --git a/vp8/encoder/pickinter.c b/vp8/encoder/pickinter.c
index e2e052f2eb6cd46105619d1bf81380be97df8afc..75e262eab2245f41542dc45a8768763592f66afb 100644 (file)
--- a/vp8/encoder/pickinter.c
+++ b/vp8/encoder/pickinter.c
@@ -542,7 +542,7 @@ void vp8_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset,
     int_mv parent_ref_mv;
     MB_PREDICTION_MODE parent_mode = 0;
 
-    if (cpi->oxcf.mr_encoder_id)
+    if (cpi->oxcf.mr_encoder_id && cpi->mr_low_res_mv_avail)
         get_lower_res_motion_info(cpi, xd, &dissim, &parent_ref_frame,
                                   &parent_mode, &parent_ref_mv, mb_row, mb_col);
 #endif
@@ -600,11 +600,14 @@ void vp8_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset,
         x->e_mbd.mode_info_context->mbmi.ref_frame = this_ref_frame;
 
 #if CONFIG_MULTI_RES_ENCODING
-        if (cpi->oxcf.mr_encoder_id)
+        if (cpi->oxcf.mr_encoder_id && cpi->mr_low_res_mv_avail)
         {
-            /* If parent MB is intra, child MB is intra. */
-            if (!parent_ref_frame && this_ref_frame)
-                continue;
+            /* TODO: If parent MB is intra, child MB is intra. This is removed
+             * now since it cause noticeable quality loss for some test clip.
+             * Will come back to evaluate more.
+             * if (!parent_ref_frame && this_ref_frame)
+             *     continue;
+             */
 
             /* If parent MB is inter, and it is unlikely there are multiple
              * objects in parent MB, we use parent ref frame as child MB's
@@ -630,7 +633,7 @@ void vp8_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset,
             }
 
 #if CONFIG_MULTI_RES_ENCODING
-            if (cpi->oxcf.mr_encoder_id)
+            if (cpi->oxcf.mr_encoder_id && cpi->mr_low_res_mv_avail)
             {
                 if (vp8_mode_order[mode_index] == NEARESTMV &&
                     mode_mv[NEARESTMV].as_int ==0)
@@ -783,7 +786,14 @@ void vp8_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset,
             step_param = cpi->sf.first_step + speed_adjust;
 
 #if CONFIG_MULTI_RES_ENCODING
-            if (cpi->oxcf.mr_encoder_id)
+            /* If lower-res drops this frame, then higher-res encoder does
+               motion search without any previous knowledge. Also, since
+               last frame motion info is not stored, then we can not
+               use improved_mv_pred. */
+            if (cpi->oxcf.mr_encoder_id && !cpi->mr_low_res_mv_avail)
+                cpi->sf.improved_mv_pred = 0;
+
+            if (cpi->oxcf.mr_encoder_id && cpi->mr_low_res_mv_avail)
             {
                 /* Use parent MV as predictor. Adjust search range
                  * accordingly.
@@ -827,7 +837,8 @@ void vp8_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset,
             }
 
 #if CONFIG_MULTI_RES_ENCODING
-            if (cpi->oxcf.mr_encoder_id && dissim <= 2 &&
+            if (cpi->oxcf.mr_encoder_id && cpi->mr_low_res_mv_avail &&
+                dissim <= 2 &&
                 MAX(abs(best_ref_mv.as_mv.row - parent_ref_mv.as_mv.row),
                     abs(best_ref_mv.as_mv.col - parent_ref_mv.as_mv.col)) <= 4)
             {
@@ -864,7 +875,10 @@ void vp8_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset,
                  * change the behavior in lowest-resolution encoder.
                  * Will improve it later.
                  */
-                if (!cpi->oxcf.mr_encoder_id)
+                 /* Set step_param to 0 to ensure large-range motion search
+                    when encoder drops this frame at lower-resolution.
+                  */
+                if (!cpi->oxcf.mr_encoder_id || !cpi->mr_low_res_mv_avail)
                     step_param = 0;
 #endif
                     bestsme = vp8_hex_search(x, b, d, &mvp_full, &d->bmi.mv,

diff --git a/vp8/vp8_cx_iface.c b/vp8/vp8_cx_iface.c
index 3a87ad8304221b543905f720ec9835e108b92acb..072314f24ab672f668db99de34c04121a5da0252 100644 (file)
--- a/vp8/vp8_cx_iface.c
+++ b/vp8/vp8_cx_iface.c
@@ -163,14 +163,11 @@ static vpx_codec_err_t validate_config(vpx_codec_alg_priv_t      *ctx,
  * multi-res-encoder.*/
 #if CONFIG_MULTI_RES_ENCODING
     if (ctx->base.enc.total_encoders > 1)
-    {
         RANGE_CHECK_HI(cfg, rc_resize_allowed,     0);
-        RANGE_CHECK_HI(cfg, rc_dropframe_thresh,   0);
-    }
 #else
     RANGE_CHECK_BOOL(cfg, rc_resize_allowed);
-    RANGE_CHECK_HI(cfg,   rc_dropframe_thresh,   100);
 #endif
+    RANGE_CHECK_HI(cfg, rc_dropframe_thresh,   100);
     RANGE_CHECK_HI(cfg, rc_resize_up_thresh,   100);
     RANGE_CHECK_HI(cfg, rc_resize_down_thresh, 100);
 

diff --git a/vp8_multi_resolution_encoder.c b/vp8_multi_resolution_encoder.c
index 497d8f7e1eeb36fc7685362d8b29ce45200eed16..eae36a4da4f4e7384bbea805b8c83b7f543e5989 100644 (file)
--- a/vp8_multi_resolution_encoder.c
+++ b/vp8_multi_resolution_encoder.c
@@ -273,7 +273,7 @@ int main(int argc, char **argv)
     cfg[0].g_w = width;
     cfg[0].g_h = height;
     cfg[0].g_threads = 1;                           /* number of threads used */
-    cfg[0].rc_dropframe_thresh = 0;
+    cfg[0].rc_dropframe_thresh = 30;
     cfg[0].rc_end_usage = VPX_CBR;
     cfg[0].rc_resize_allowed = 0;
     cfg[0].rc_min_quantizer = 4;
@@ -283,7 +283,6 @@ int main(int argc, char **argv)
     cfg[0].rc_buf_initial_sz = 500;
     cfg[0].rc_buf_optimal_sz = 600;
     cfg[0].rc_buf_sz = 1000;
-    //cfg[0].rc_dropframe_thresh = 10;
     cfg[0].g_error_resilient = 1;              /* Enable error resilient mode */
     cfg[0].g_lag_in_frames   = 0;
 
@@ -293,8 +292,8 @@ int main(int argc, char **argv)
      */
     //cfg[0].kf_mode           = VPX_KF_DISABLED;
     cfg[0].kf_mode           = VPX_KF_AUTO;
-    cfg[0].kf_min_dist = 0;
-    cfg[0].kf_max_dist = 150;
+    cfg[0].kf_min_dist = 3000;
+    cfg[0].kf_max_dist = 3000;
 
     cfg[0].rc_target_bitrate = target_bitrate[0];       /* Set target bitrate */
     cfg[0].g_timebase.num = 1;                          /* Set fps */
@@ -366,6 +365,12 @@ int main(int argc, char **argv)
         if(vpx_codec_control(&codec[i], VP8E_SET_STATIC_THRESHOLD, static_thresh))
             die_codec(&codec[i], "Failed to set static threshold");
     }
+    /* Set NOISE_SENSITIVITY to do TEMPORAL_DENOISING */
+    for ( i=0; i< NUM_ENCODERS; i++)
+    {
+        if(vpx_codec_control(&codec[i], VP8E_SET_NOISE_SENSITIVITY, 0))
+            die_codec(&codec[i], "Failed to set noise_sensitivity");
+    }
 
     frame_avail = 1;
     got_data = 0;