From: punksu <huisu@google.com>
Date: Sat, 20 Dec 2014 06:49:44 +0000 (+0800)
Subject: Allow large tx_size in lossless coding with transform skipping
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=8a41d18bd0bb489c62575144a9b46d70df435ae8;p=libvpx

Allow large tx_size in lossless coding with transform skipping

In lossless coding, tx_size can be larger than 4x4 when transform
skipping is activated. Compared to regular vp9 lossless coding,
performance improvement for derf is about 5%; gain is larger
for screen content videos.

Change-Id: Ib20ece7e117f29fb91543612757302a2400110b4
---

diff --git a/vp9/decoder/vp9_decodeframe.c b/vp9/decoder/vp9_decodeframe.c
index 057a62291..5739cba3e 100644
--- a/vp9/decoder/vp9_decodeframe.c
+++ b/vp9/decoder/vp9_decodeframe.c
@@ -2045,7 +2045,9 @@ static void read_supertx_probs(FRAME_CONTEXT *fc, vp9_reader *r) {
 static int read_compressed_header(VP9Decoder *pbi, const uint8_t *data,
                                   size_t partition_size) {
   VP9_COMMON *const cm = &pbi->common;
+#if !CONFIG_TX_SKIP
   MACROBLOCKD *const xd = &pbi->mb;
+#endif
   FRAME_CONTEXT *const fc = &cm->fc;
   vp9_reader r;
   int k;
@@ -2055,7 +2057,11 @@ static int read_compressed_header(VP9Decoder *pbi, const uint8_t *data,
     vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
                        "Failed to allocate bool decoder 0");
 
+#if CONFIG_TX_SKIP
+  cm->tx_mode = read_tx_mode(&r);
+#else
   cm->tx_mode = xd->lossless ? ONLY_4X4 : read_tx_mode(&r);
+#endif
   if (cm->tx_mode == TX_MODE_SELECT)
     read_tx_mode_probs(&fc->tx_probs, &r);
   read_coef_probs(fc, cm->tx_mode, &r);
diff --git a/vp9/decoder/vp9_decodemv.c b/vp9/decoder/vp9_decodemv.c
index 9fae17723..a81894b5e 100644
--- a/vp9/decoder/vp9_decodemv.c
+++ b/vp9/decoder/vp9_decodemv.c
@@ -188,8 +188,22 @@ static void read_intra_frame_mode_info(VP9_COMMON *const cm,
     int q_idx = vp9_get_qindex(&cm->seg, mbmi->segment_id, cm->base_qindex);
     int try_tx_skip = q_idx <= TX_SKIP_Q_THRESH_INTRA;
     if (try_tx_skip) {
-      mbmi->tx_skip[0] = vp9_read(r, cm->fc.y_tx_skip_prob[0]);
-      mbmi->tx_skip[1] = vp9_read(r, cm->fc.uv_tx_skip_prob[mbmi->tx_skip[0]]);
+      if (xd->lossless) {
+        if (mbmi->tx_size == TX_4X4)
+          mbmi->tx_skip[0] = vp9_read(r, cm->fc.y_tx_skip_prob[0]);
+        else
+          mbmi->tx_skip[0] = 1;
+
+        if (get_uv_tx_size(mbmi, &xd->plane[1]) == TX_4X4)
+          mbmi->tx_skip[1] =
+            vp9_read(r, cm->fc.uv_tx_skip_prob[mbmi->tx_skip[0]]);
+        else
+          mbmi->tx_skip[1] = 1;
+        } else {
+          mbmi->tx_skip[0] = vp9_read(r, cm->fc.y_tx_skip_prob[0]);
+          mbmi->tx_skip[1] =
+            vp9_read(r, cm->fc.uv_tx_skip_prob[mbmi->tx_skip[0]]);
+        }
     } else {
       mbmi->tx_skip[0] = 0;
       mbmi->tx_skip[1] = 0;
@@ -730,12 +744,38 @@ static void read_inter_frame_mode_info(VP9_COMMON *const cm,
     int q_idx = cm->base_qindex;
 #else
     int q_idx = vp9_get_qindex(&cm->seg, mbmi->segment_id, cm->base_qindex);
-#endif
+#endif  // CONFIG_SUPERTX
     int try_tx_skip = inter_block ? q_idx <= TX_SKIP_Q_THRESH_INTER :
                                     q_idx <= TX_SKIP_Q_THRESH_INTRA;
+#if CONFIG_SUPERTX
     if (try_tx_skip) {
-      mbmi->tx_skip[0] = vp9_read(r, cm->fc.y_tx_skip_prob[inter_block]);
-      mbmi->tx_skip[1] = vp9_read(r, cm->fc.uv_tx_skip_prob[mbmi->tx_skip[0]]);
+#else
+    if (try_tx_skip && !mbmi->skip) {
+#endif  // CONFIG_SUPERTX
+      if (xd->lossless) {
+#if CONFIG_SUPERTX
+        if (1)
+#else
+        if (mbmi->tx_size == TX_4X4)
+#endif  // CONFIG_SUPERTX
+          mbmi->tx_skip[0] = vp9_read(r, cm->fc.y_tx_skip_prob[inter_block]);
+        else
+          mbmi->tx_skip[0] = 1;
+
+#if CONFIG_SUPERTX
+        if (1)
+#else
+        if (get_uv_tx_size(mbmi, &xd->plane[1]) == TX_4X4)
+#endif  // CONFIG_SUPERTX
+          mbmi->tx_skip[1] =
+            vp9_read(r, cm->fc.uv_tx_skip_prob[mbmi->tx_skip[0]]);
+        else
+          mbmi->tx_skip[1] = 1;
+      } else {
+        mbmi->tx_skip[0] = vp9_read(r, cm->fc.y_tx_skip_prob[inter_block]);
+        mbmi->tx_skip[1] =
+          vp9_read(r, cm->fc.uv_tx_skip_prob[mbmi->tx_skip[0]]);
+      }
 #if CONFIG_SUPERTX
       if (!cm->frame_parallel_decoding_mode && !supertx_enabled) {
 #else
diff --git a/vp9/encoder/vp9_bitstream.c b/vp9/encoder/vp9_bitstream.c
index 919789ffe..75bde4011 100644
--- a/vp9/encoder/vp9_bitstream.c
+++ b/vp9/encoder/vp9_bitstream.c
@@ -401,12 +401,33 @@ static void pack_inter_mode_mvs(VP9_COMP *cpi, const MODE_INFO *mi,
     int q_idx = cm->base_qindex;
 #else
     int q_idx = vp9_get_qindex(seg, segment_id, cm->base_qindex);
-#endif
+#endif  // CONFIG_SUPERTX
     int try_tx_skip = is_inter ? q_idx <= TX_SKIP_Q_THRESH_INTER :
                                  q_idx <= TX_SKIP_Q_THRESH_INTRA;
+#if CONFIG_SUPERTX
     if (try_tx_skip) {
-      vp9_write(w, mbmi->tx_skip[0], cm->fc.y_tx_skip_prob[is_inter]);
-      vp9_write(w, mbmi->tx_skip[1], cm->fc.uv_tx_skip_prob[mbmi->tx_skip[0]]);
+#else
+    if (try_tx_skip && !skip) {
+#endif  // CONFIG_SUPERTX
+      if (xd->lossless) {
+#if CONFIG_SUPERTX
+        if (1)
+#else
+        if (mbmi->tx_size == TX_4X4)
+#endif  // CONFIG_SUPERTX
+          vp9_write(w, mbmi->tx_skip[0], cm->fc.y_tx_skip_prob[is_inter]);
+#if CONFIG_SUPERTX
+        if (1)
+#else
+        if (get_uv_tx_size(mbmi, &xd->plane[1]) == TX_4X4)
+#endif  // CONFIG_SUPERTX
+          vp9_write(w, mbmi->tx_skip[1],
+                    cm->fc.uv_tx_skip_prob[mbmi->tx_skip[0]]);
+      } else {
+        vp9_write(w, mbmi->tx_skip[0], cm->fc.y_tx_skip_prob[is_inter]);
+        vp9_write(w, mbmi->tx_skip[1],
+                  cm->fc.uv_tx_skip_prob[mbmi->tx_skip[0]]);
+      }
     }
   }
 #endif
@@ -520,8 +541,17 @@ static void write_mb_modes_kf(const VP9_COMMON *cm, const MACROBLOCKD *xd,
     int q_idx = vp9_get_qindex(seg, mbmi->segment_id, cm->base_qindex);
     int try_tx_skip = q_idx <= TX_SKIP_Q_THRESH_INTRA;
     if (try_tx_skip) {
-      vp9_write(w, mbmi->tx_skip[0], cm->fc.y_tx_skip_prob[0]);
-      vp9_write(w, mbmi->tx_skip[1], cm->fc.uv_tx_skip_prob[mbmi->tx_skip[0]]);
+      if (xd->lossless) {
+        if (mbmi->tx_size == TX_4X4)
+          vp9_write(w, mbmi->tx_skip[0], cm->fc.y_tx_skip_prob[0]);
+        if (get_uv_tx_size(mbmi, &xd->plane[1]) == TX_4X4)
+          vp9_write(w, mbmi->tx_skip[1],
+                    cm->fc.uv_tx_skip_prob[mbmi->tx_skip[0]]);
+      } else {
+        vp9_write(w, mbmi->tx_skip[0], cm->fc.y_tx_skip_prob[0]);
+        vp9_write(w, mbmi->tx_skip[1],
+                  cm->fc.uv_tx_skip_prob[mbmi->tx_skip[0]]);
+      }
     }
   }
 #endif
@@ -1453,16 +1483,22 @@ static void write_uncompressed_header(VP9_COMP *cpi,
 
 static size_t write_compressed_header(VP9_COMP *cpi, uint8_t *data) {
   VP9_COMMON *const cm = &cpi->common;
+#if !CONFIG_TX_SKIP
   MACROBLOCKD *const xd = &cpi->mb.e_mbd;
+#endif
   FRAME_CONTEXT *const fc = &cm->fc;
   vp9_writer header_bc;
 
   vp9_start_encode(&header_bc, data);
 
+#if CONFIG_TX_SKIP
+  encode_txfm_probs(cm, &header_bc);
+#else
   if (xd->lossless)
     cm->tx_mode = ONLY_4X4;
   else
     encode_txfm_probs(cm, &header_bc);
+#endif
 
   update_coef_probs(cpi, &header_bc);
   update_skip_probs(cm, &header_bc);
diff --git a/vp9/encoder/vp9_encodeframe.c b/vp9/encoder/vp9_encodeframe.c
index 7b2bf698f..88bfa3118 100644
--- a/vp9/encoder/vp9_encodeframe.c
+++ b/vp9/encoder/vp9_encodeframe.c
@@ -3439,8 +3439,10 @@ static MV_REFERENCE_FRAME get_frame_type(const VP9_COMP *cpi) {
 }
 
 static TX_MODE select_tx_mode(const VP9_COMP *cpi) {
+#if !CONFIG_TX_SKIP
   if (cpi->mb.e_mbd.lossless)
     return ONLY_4X4;
+#endif
   if (cpi->sf.tx_size_search_method == USE_LARGESTALL)
 #if CONFIG_TX64X64
     return ALLOW_64X64;
@@ -4741,7 +4743,11 @@ static void encode_superblock(VP9_COMP *cpi, TOKENEXTRA **t, int output_enabled,
 #endif
       int try_tx_skip = is_inter_block(mbmi) ? q_idx <= TX_SKIP_Q_THRESH_INTER :
                                                q_idx <= TX_SKIP_Q_THRESH_INTRA;
+#if CONFIG_SUPERTX
       if (try_tx_skip) {
+#else
+      if (try_tx_skip && !(mbmi->skip || seg_skip)) {
+#endif
         ++cm->counts.y_tx_skip[is_inter_block(mbmi)][mbmi->tx_skip[0]];
         ++cm->counts.uv_tx_skip[mbmi->tx_skip[0]][mbmi->tx_skip[1]];
       }
diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c
index 6cfb06987..67ab66028 100644
--- a/vp9/encoder/vp9_rdopt.c
+++ b/vp9/encoder/vp9_rdopt.c
@@ -599,6 +599,17 @@ static void txfm_rd_in_plane(MACROBLOCK *x,
   args.best_rd = ref_best_rd;
   args.use_fast_coef_costing = use_fast_coef_casting;
 
+#if CONFIG_TX_SKIP
+  if (xd->lossless && tx_size != TX_4X4 &&
+      !xd->mi[0].src_mi->mbmi.tx_skip[plane != 0]) {
+    *rate       = INT_MAX;
+    *distortion = INT64_MAX;
+    *sse        = INT64_MAX;
+    *skippable  = 0;
+    return;
+  }
+#endif
+
   if (plane == 0)
     xd->mi[0].src_mi->mbmi.tx_size = tx_size;
 
@@ -802,13 +813,19 @@ static void super_block_yrd(VP9_COMP *cpi, MACROBLOCK *x, int *rate,
                             int64_t *psse, BLOCK_SIZE bs,
                             int64_t txfm_cache[TX_MODES],
                             int64_t ref_best_rd) {
+#if !CONFIG_TX_SKIP
   MACROBLOCKD *xd = &x->e_mbd;
+#endif
   int64_t sse;
   int64_t *ret_sse = psse ? psse : &sse;
 
   assert(bs == xd->mi[0].src_mi->mbmi.sb_type);
 
+#if CONFIG_TX_SKIP
+  if (cpi->sf.tx_size_search_method == USE_LARGESTALL) {
+#else
   if (cpi->sf.tx_size_search_method == USE_LARGESTALL || xd->lossless) {
+#endif
     vpx_memset(txfm_cache, 0, TX_MODES * sizeof(int64_t));
     choose_largest_tx_size(cpi, x, rate, distortion, skip, ret_sse, ref_best_rd,
                            bs);