From: Jim Bankoski <jimbankoski@google.com>
Date: Mon, 5 Nov 2018 22:57:11 +0000 (-0800)
Subject: WIP:  CONFIG_INSPECTION: fix non show frames
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=50ead0db62a7229d8ba93b9c91676a2094bba966;p=libvpx

WIP:  CONFIG_INSPECTION: fix non show frames

This makes non showable frames - analyzeable and also fixes
display of motion vectors.

Change-Id: Ie286a3d2005a0c5e35d164ed62dd0fe2269b9be6
---

diff --git a/examples/inspect.c b/examples/inspect.c
index 8c132a6da..3ccb6e6b4 100644
--- a/examples/inspect.c
+++ b/examples/inspect.c
@@ -548,22 +548,51 @@ int open_file(char *file) {
   return EXIT_SUCCESS;
 }
 
+VpxDecodeReturn adr;
+int have_frame = 0;
+const unsigned char *frame;
+const unsigned char *end_frame;
+size_t frame_size = 0;
+vpx_codec_iter_t iter = NULL;
 EMSCRIPTEN_KEEPALIVE
 int read_frame() {
-  if (!vpx_video_reader_read_frame(reader)) return EXIT_FAILURE;
+  int got_any_frames = 0;
+  struct vpx_ref_frame ref_dec;
   img = NULL;
-  vpx_codec_iter_t iter = NULL;
-  size_t frame_size = 0;
-  const unsigned char *frame = vpx_video_reader_get_frame(reader, &frame_size);
-  if (vpx_codec_decode(&codec, frame, (unsigned int)frame_size, NULL, 0) !=
-      VPX_CODEC_OK) {
-    die_codec(&codec, "Failed to decode frame.");
+
+  // This loop skips over any frames that are show_existing_frames,  as
+  // there is nothing to analyze.
+  do {
+    if (!have_frame) {
+      if (!vpx_video_reader_read_frame(reader)) return EXIT_FAILURE;
+      frame = vpx_video_reader_get_frame(reader, &frame_size);
+
+      have_frame = 1;
+      end_frame = frame + frame_size;
+    }
+
+    if (vpx_codec_decode(&codec, frame, (unsigned int)frame_size, &adr, 0) !=
+        VPX_CODEC_OK) {
+      die_codec(&codec, "Failed to decode frame.");
+    }
+
+    frame = adr.buf;
+    if (adr.finished_superframes) have_frame = 0;
+  } while (adr.show_existing);
+
+  // ref_dec.idx is the index to the reference buffer idx to VP9_GET_REFERENCE
+  // if its -1 the decoder didn't update any reference buffer and the only
+  // way to see the frame is aom_codec_get_frame.
+  ref_dec.frame_type = adr.idx;
+
+  if (!vpx_codec_control(&codec, VP9_GET_REFERENCE, &ref_dec)) {
+    img = &ref_dec.img;
+    ++frame_count;
+    got_any_frames = 1;
   }
-  img = vpx_codec_get_frame(&codec, &iter);
-  if (img == NULL) {
+  if (!got_any_frames) {
     return EXIT_FAILURE;
   }
-  ++frame_count;
   return EXIT_SUCCESS;
 }
 
diff --git a/tools/build_inspector.sh b/tools/build_inspector.sh
index 3cceb60f3..4619b3436 100755
--- a/tools/build_inspector.sh
+++ b/tools/build_inspector.sh
@@ -32,7 +32,7 @@ if [ ! -d ".inspect" ]; then
     --disable-docs \
     --disable-unit-tests \
     --enable-inspection \
-    --extra-cflags="-D_POSIX_SOURCE"
+    --extra-cflags="-D_POSIX_SOURCE -s ALLOW_MEMORY_GROWTH=1"
   cd ..
 fi
 
@@ -43,6 +43,7 @@ emcc -O3 inspect.bc -o inspect.js \
   -s TOTAL_MEMORY=134217728 \
   -s MODULARIZE=1 \
   -s EXPORT_NAME="'DecoderModule'" \
+  -s ALLOW_MEMORY_GROWTH=1 \
   --post-js "../inspect-post.js" \
   --memory-init-file 0
 cp inspect.js ../inspect.js
diff --git a/vp9/decoder/inspection.c b/vp9/decoder/inspection.c
index b6b38a5a0..2be256933 100644
--- a/vp9/decoder/inspection.c
+++ b/vp9/decoder/inspection.c
@@ -43,6 +43,8 @@ int ifd_inspect(insp_frame_data *fd, void *decoder) {
   fd->frame_type = cm->frame_type;
   fd->base_qindex = cm->base_qindex;
   // TODO(jimbankoski): copy tile data
+  fd->show_existing_frame = cm->show_existing_frame;
+
   // fd->tile_mi_cols = cm->tile_width;
   // fd->tile_mi_rows = cm->tile_height;
 #if CONFIG_ACCOUNTING
@@ -64,8 +66,14 @@ int ifd_inspect(insp_frame_data *fd, void *decoder) {
       // Motion Vectors
       mi->mv[0].row = bmi->mv[0].as_mv.row;
       mi->mv[0].col = bmi->mv[0].as_mv.col;
-      mi->mv[1].row = bmi->mv[1].as_mv.row;
-      mi->mv[1].col = bmi->mv[1].as_mv.col;
+
+      if (bmi->ref_frame[1] == -1) {
+        mi->mv[1].row = 0;
+        mi->mv[1].col = 0;
+      } else {
+        mi->mv[1].row = bmi->mv[1].as_mv.row;
+        mi->mv[1].col = bmi->mv[1].as_mv.col;
+      }
       // Reference Frames
       mi->ref_frame[0] = bmi->ref_frame[0];
       mi->ref_frame[1] = bmi->ref_frame[1];
diff --git a/vp9/decoder/inspection.h b/vp9/decoder/inspection.h
index a3c98076b..304e2eee7 100644
--- a/vp9/decoder/inspection.h
+++ b/vp9/decoder/inspection.h
@@ -58,6 +58,7 @@ struct insp_frame_data {
   int show_frame;
   int frame_type;
   int base_qindex;
+  int show_existing_frame;
   int mi_rows;
   int mi_cols;
   int tile_mi_rows;
diff --git a/vp9/vp9_dx_iface.c b/vp9/vp9_dx_iface.c
index 2bf0d78d2..1f13d8728 100644
--- a/vp9/vp9_dx_iface.c
+++ b/vp9/vp9_dx_iface.c
@@ -279,7 +279,10 @@ static vpx_codec_err_t init_decoder(vpx_codec_alg_priv_t *ctx) {
     set_default_ppflags(&ctx->postproc_cfg);
 
   init_buffer_callbacks(ctx);
-
+#if CONFIG_INSPECTION
+  ctx->frame_count = 0;
+  ctx->which_frame = 0;
+#endif
   return VPX_CODEC_OK;
 }
 
@@ -333,6 +336,68 @@ static vpx_codec_err_t decode_one(vpx_codec_alg_priv_t *ctx,
   return VPX_CODEC_OK;
 }
 
+#if CONFIG_INSPECTION
+// This function enables the inspector to inspect non visible frames.
+static vpx_codec_err_t decoder_inspect(vpx_codec_alg_priv_t *ctx,
+                                       const uint8_t *data, size_t data_sz,
+                                       void *user_priv, int64_t deadline) {
+  vpx_codec_err_t res = VPX_CODEC_OK;
+
+  if (ctx->pbi == NULL) {
+    const vpx_codec_err_t res = init_decoder(ctx);
+    if (res != VPX_CODEC_OK) return res;
+  }
+
+  if (ctx->which_frame >= ctx->frame_count) {
+    int i;
+    const uint8_t *data_start = data;
+    const uint8_t *const data_end = data + data_sz;
+    res = vp9_parse_superframe_index(data, data_sz, ctx->frame_sizes,
+                                     &ctx->frame_count, ctx->decrypt_cb,
+                                     ctx->decrypt_state);
+
+    if (res != VPX_CODEC_OK) return res;
+    if (ctx->svc_decoding && ctx->svc_spatial_layer < ctx->frame_count - 1)
+      ctx->frame_count = ctx->svc_spatial_layer + 1;
+
+    ctx->which_frame = 0;
+
+    // Double check that we don't have problems with the index;
+    for (i = 0; i < ctx->frame_count; ++i) {
+      const uint32_t frame_size = ctx->frame_sizes[i];
+      if (data_start < data || frame_size > (uint32_t)(data_end - data_start)) {
+        set_error_detail(ctx, "Invalid frame size in index");
+        return VPX_CODEC_CORRUPT_FRAME;
+      }
+      data_start += frame_size;
+    }
+  }
+  if (ctx->frame_count == 0) ctx->frame_sizes[0] = data_sz;
+
+  // Decode in serial mode.
+  if (ctx->frame_count >= 0) {
+    int i;
+    vpx_codec_err_t res;
+    VP9_COMMON *const cm = &ctx->pbi->common;
+    VpxDecodeReturn *data2 = (VpxDecodeReturn *)user_priv;
+    const uint8_t *data_start = data;
+
+    res =
+        decode_one(ctx, &data, ctx->frame_sizes[ctx->which_frame], 0, deadline);
+    if (res != VPX_CODEC_OK) return res;
+
+    data2->idx = cm->new_fb_idx;
+
+    data2->buf = data_start + ctx->frame_sizes[ctx->which_frame];
+    data2->show_existing = cm->show_existing_frame;
+    ctx->which_frame++;
+    data2->finished_superframes = (ctx->which_frame >= ctx->frame_count);
+    return res;
+  }
+  return VPX_CODEC_OK;
+}
+#endif
+
 static vpx_codec_err_t decoder_decode(vpx_codec_alg_priv_t *ctx,
                                       const uint8_t *data, unsigned int data_sz,
                                       void *user_priv, long deadline) {
@@ -341,7 +406,11 @@ static vpx_codec_err_t decoder_decode(vpx_codec_alg_priv_t *ctx,
   vpx_codec_err_t res;
   uint32_t frame_sizes[8];
   int frame_count;
-
+#if CONFIG_INSPECTION
+  if (user_priv != 0) {
+    return decoder_inspect(ctx, data, data_sz, user_priv, deadline);
+  }
+#endif
   if (data == NULL && data_sz == 0) {
     ctx->flushed = 1;
     return VPX_CODEC_OK;
@@ -479,7 +548,11 @@ static vpx_codec_err_t ctrl_get_reference(vpx_codec_alg_priv_t *ctx,
   vp9_ref_frame_t *data = va_arg(args, vp9_ref_frame_t *);
 
   if (data) {
+#if CONFIG_INSPECTION
+    const int fb_idx = data->idx;
+#else
     const int fb_idx = ctx->pbi->common.cur_show_frame_fb_idx;
+#endif
     YV12_BUFFER_CONFIG *fb = get_buf_frame(&ctx->pbi->common, fb_idx);
     if (fb == NULL) return VPX_CODEC_ERROR;
     yuvconfig2image(&data->img, fb, NULL);
diff --git a/vp9/vp9_dx_iface.h b/vp9/vp9_dx_iface.h
index 722959737..e20aed93e 100644
--- a/vp9/vp9_dx_iface.h
+++ b/vp9/vp9_dx_iface.h
@@ -51,6 +51,9 @@ struct vpx_codec_alg_priv {
 #if CONFIG_INSPECTION
   vpx_inspect_cb inspect_cb;
   void *inspect_ctx;
+  uint32_t frame_sizes[8];
+  int frame_count;
+  int which_frame;
 #endif
 };
 
diff --git a/vpx/vp8dx.h b/vpx/vp8dx.h
index 5d0e533f1..e9d479fae 100644
--- a/vpx/vp8dx.h
+++ b/vpx/vp8dx.h
@@ -64,6 +64,23 @@ typedef struct vpx_inspect_init {
   void *inspect_ctx;
 } vpx_inspect_init;
 
+/*!\brief Structure to hold decoder return.
+ *
+ * Defines a structure to hold the buffer and return an index
+ * when calling decode from inspect. This enables us to decode
+ * non showable sub frames.
+ */
+typedef struct {
+  /*! Pointer for new position in compressed buffer after decoding 1 OBU. */
+  const unsigned char *buf;
+  /*! Index into reference buffer array to see result of decoding 1 OBU. */
+  int idx;
+  /*! Is a show existing frame. */
+  int show_existing;
+  /*! Whether we've decoded all the superframes. */
+  int finished_superframes;
+} VpxDecodeReturn;
+
 /*!\enum vp8_dec_control_id
  * \brief VP8 decoder control functions
  *