Fix unit test failure for RECT_TX + VAR_TX

Disable rect_tx because we only support 4x4 Walsh-Hadamard transform
in lossless mode.

Fixes failure in ./test_libaom --gtest_filter=*Large*ScreencastQ0/1
Configuration: --enable-experimental --enable-var-tx --enable-rect-tx
 --enable-ref-mv --enable-ext_intra --enable-ext_tx --enable-debug
 --disable-optimizations

Change-Id: Ib6b3494c7dcf7182f1cab9b138388d054851a23d

diff --git a/av1/common/blockd.h b/av1/common/blockd.h
index 94eb089b16d6504efee8c5c4d2896e4ba8f72f5d..266f8fe62c7df1ff97fb2ee1ceaa4bee1931e36d 100644 (file)
--- a/av1/common/blockd.h
+++ b/av1/common/blockd.h
@@ -516,8 +516,10 @@ static INLINE int is_rect_tx_allowed_bsize(BLOCK_SIZE bsize) {
   return LUT[bsize];
 }
 
-static INLINE int is_rect_tx_allowed(const MB_MODE_INFO *mbmi) {
-  return is_inter_block(mbmi) && is_rect_tx_allowed_bsize(mbmi->sb_type);
+static INLINE int is_rect_tx_allowed(const MACROBLOCKD *xd,
+                                     const MB_MODE_INFO *mbmi) {
+  return is_inter_block(mbmi) && is_rect_tx_allowed_bsize(mbmi->sb_type) &&
+         !xd->lossless[mbmi->segment_id];
 }
 
 static INLINE int is_rect_tx(TX_SIZE tx_size) { return tx_size >= TX_SIZES; }

diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c
index 6492f62093160e25e32cb0deb6d0bc642f3e01d9..7be3e752a1ecffa0f6855a55881b03cda46359a7 100644 (file)
--- a/av1/decoder/decodemv.c
+++ b/av1/decoder/decodemv.c
@@ -1633,7 +1633,8 @@ static void read_inter_frame_mode_info(AV1Decoder *const pbi,
       int idx, idy;
       int tx_size_cat = inter_tx_size_cat_lookup[bsize];
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
-      int is_rect_tx_allowed = inter_block && is_rect_tx_allowed_bsize(bsize);
+      int is_rect_tx_allowed = inter_block && is_rect_tx_allowed_bsize(bsize) &&
+                               !xd->lossless[mbmi->segment_id];
       int use_rect_tx = 0;
 
       if (is_rect_tx_allowed) {

diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 763615184067ab710f6ac811795a6dd8b32ee0ae..f9ae154665b936eeb9fca3cb05b15d8e7f86199b 100644 (file)
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -406,7 +406,7 @@ static void write_selected_tx_size(const AV1_COMMON *cm, const MACROBLOCKD *xd,
     const TX_SIZE coded_tx_size = txsize_sqr_up_map[tx_size];
 
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
-    assert(IMPLIES(is_rect_tx(tx_size), is_rect_tx_allowed(mbmi)));
+    assert(IMPLIES(is_rect_tx(tx_size), is_rect_tx_allowed(xd, mbmi)));
     assert(
         IMPLIES(is_rect_tx(tx_size), tx_size == max_txsize_rect_lookup[bsize]));
 #endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
@@ -1132,7 +1132,7 @@ static void pack_inter_mode_mvs(AV1_COMP *cpi, const MODE_INFO *mi,
       int idx, idy;
 
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
-      if (is_rect_tx_allowed(mbmi)) {
+      if (is_rect_tx_allowed(xd, mbmi)) {
         int tx_size_cat = inter_tx_size_cat_lookup[bsize];
 
         aom_write(w, is_rect_tx(mbmi->tx_size),

diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index cfc47188aae6d88378beda76d7ea8517204cb7f3..cc73d4c3fb44e7c42d1121feac6ddd723f8d68d8 100644 (file)
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -5141,14 +5141,14 @@ static void encode_superblock(AV1_COMP *cpi, ThreadData *td, TOKENEXTRA **t,
                                        : intra_tx_size_cat_lookup[bsize];
       const TX_SIZE coded_tx_size = txsize_sqr_up_map[mbmi->tx_size];
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
-      assert(IMPLIES(is_rect_tx(mbmi->tx_size), is_rect_tx_allowed(mbmi)));
+      assert(IMPLIES(is_rect_tx(mbmi->tx_size), is_rect_tx_allowed(xd, mbmi)));
 #endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
 #if CONFIG_VAR_TX
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
-      if (is_rect_tx_allowed(mbmi)) {
+      if (is_rect_tx_allowed(xd, mbmi)) {
         td->counts->rect_tx[tx_size_cat][is_rect_tx(mbmi->tx_size)]++;
       }
-      if (!is_rect_tx_allowed(mbmi) || !is_rect_tx(mbmi->tx_size)) {
+      if (!is_rect_tx_allowed(xd, mbmi) || !is_rect_tx(mbmi->tx_size)) {
 #endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
         if (is_inter)
           tx_partition_count_update(cm, xd, bsize, mi_row, mi_col, td->counts);

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 9721abeb02ba499a1f8dca93267f8ff963243dbd..8b0a7bdca08b4b7178168e36a66e2b142f77e395 100644 (file)
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -1385,14 +1385,14 @@ static int64_t choose_tx_size_fix_type(AV1_COMP *cpi, BLOCK_SIZE bs,
   const int is_inter = is_inter_block(mbmi);
 #if CONFIG_EXT_TX
 #if CONFIG_RECT_TX
-  int evaulate_rect_tx = 0;
+  int evaluate_rect_tx = 0;
 #endif  // CONFIG_RECT_TX
   int ext_tx_set;
 #endif  // CONFIG_EXT_TX
 
   if (tx_select) {
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
-    evaulate_rect_tx = is_rect_tx_allowed(mbmi);
+    evaluate_rect_tx = is_rect_tx_allowed(xd, mbmi);
 #endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
     start_tx = max_tx_size;
     end_tx = 0;
@@ -1400,8 +1400,8 @@ static int64_t choose_tx_size_fix_type(AV1_COMP *cpi, BLOCK_SIZE bs,
     const TX_SIZE chosen_tx_size =
         tx_size_from_tx_mode(bs, cm->tx_mode, is_inter);
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
-    evaulate_rect_tx = is_rect_tx(chosen_tx_size);
-    assert(IMPLIES(evaulate_rect_tx, is_rect_tx_allowed(mbmi)));
+    evaluate_rect_tx = is_rect_tx(chosen_tx_size);
+    assert(IMPLIES(evaluate_rect_tx, is_rect_tx_allowed(xd, mbmi)));
 #endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
     start_tx = chosen_tx_size;
     end_tx = chosen_tx_size;
@@ -1415,7 +1415,7 @@ static int64_t choose_tx_size_fix_type(AV1_COMP *cpi, BLOCK_SIZE bs,
   mbmi->tx_type = tx_type;
 
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
-  if (evaulate_rect_tx) {
+  if (evaluate_rect_tx) {
     const TX_SIZE rect_tx_size = max_txsize_rect_lookup[bs];
     const int ext_tx_set = get_ext_tx_set(rect_tx_size, bs, 1);
     if (ext_tx_used_inter[ext_tx_set][tx_type]) {
@@ -3214,7 +3214,7 @@ static int64_t select_tx_size_fix_type(const AV1_COMP *cpi, MACROBLOCK *x,
   mbmi->tx_type = tx_type;
   inter_block_yrd(cpi, x, rate, dist, skippable, sse, bsize, ref_best_rd);
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
-  if (is_rect_tx_allowed(mbmi)) {
+  if (is_rect_tx_allowed(xd, mbmi)) {
     int rate_rect_tx, skippable_rect_tx = 0;
     int64_t dist_rect_tx, sse_rect_tx, rd, rd_rect_tx;
     int tx_size_cat = inter_tx_size_cat_lookup[bsize];