From 2823173ee05e70a5cf20b442a2bbe9702b490ce6 Mon Sep 17 00:00:00 2001
From: Yaowu Xu <yaowu@google.com>
Date: Mon, 19 Mar 2012 11:03:27 -0700
Subject: [PATCH] enable 8x8 transform for MBs in intra frames

When ac_yquant>171, a key frame is enabled to use 8x8 transform. In
such case, MBs with DC_PRED or TM_PRED are selected to use T8x8. This
change helped the full STD-HD set by ~.1% or so, which is reasonable
considering how often key frame occurs in these encodings.

Change-Id: Id17009ef6327252177b19e6bf0d6628827febaf1
---
 vp8/decoder/decodframe.c  | 31 +++++++++++++++++++++----------
 vp8/encoder/encodeframe.c |  4 ++--
 vp8/encoder/ratectrl.c    |  9 +++++++--
 3 files changed, 30 insertions(+), 14 deletions(-)

diff --git a/vp8/decoder/decodframe.c b/vp8/decoder/decodframe.c
index 91948263b..c1e70a212 100644
--- a/vp8/decoder/decodframe.c
+++ b/vp8/decoder/decodframe.c
@@ -157,22 +157,33 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd,
     int eobtotal = 0;
     MB_PREDICTION_MODE mode;
     int i;
-
     int tx_type;
-    if( pbi->common.txfm_mode==ONLY_4X4 )
+
+    if(pbi->common.frame_type == KEY_FRAME)
     {
-        xd->mode_info_context->mbmi.txfm_size = TX_4X4;
+        if( pbi->common.txfm_mode==ALLOW_8X8 &&
+             (xd->mode_info_context->mbmi.mode == DC_PRED
+            ||xd->mode_info_context->mbmi.mode == TM_PRED))
+            xd->mode_info_context->mbmi.txfm_size = TX_8X8;
+        else
+            xd->mode_info_context->mbmi.txfm_size = TX_4X4;
     }
-    else if( pbi->common.txfm_mode == ALLOW_8X8 )
+    else
     {
-        if( xd->mode_info_context->mbmi.mode ==B_PRED
-            ||xd->mode_info_context->mbmi.mode ==I8X8_PRED
-            ||xd->mode_info_context->mbmi.mode ==SPLITMV)
+        if( pbi->common.txfm_mode==ONLY_4X4 )
+        {
             xd->mode_info_context->mbmi.txfm_size = TX_4X4;
-        else
-            xd->mode_info_context->mbmi.txfm_size = TX_8X8;
+        }
+        else if( pbi->common.txfm_mode == ALLOW_8X8 )
+        {
+            if( xd->mode_info_context->mbmi.mode ==B_PRED
+                ||xd->mode_info_context->mbmi.mode ==I8X8_PRED
+                ||xd->mode_info_context->mbmi.mode ==SPLITMV)
+                xd->mode_info_context->mbmi.txfm_size = TX_4X4;
+            else
+                xd->mode_info_context->mbmi.txfm_size = TX_8X8;
+        }
     }
-
     tx_type = xd->mode_info_context->mbmi.txfm_size;
 
     if (xd->mode_info_context->mbmi.mb_skip_coeff)
diff --git a/vp8/encoder/encodeframe.c b/vp8/encoder/encodeframe.c
index 8feea41a6..cf32685b0 100644
--- a/vp8/encoder/encodeframe.c
+++ b/vp8/encoder/encodeframe.c
@@ -1106,8 +1106,8 @@ int vp8cx_encode_intra_macro_block(VP8_COMP *cpi, MACROBLOCK *x, TOKENEXTRA **t)
 
     /* test code: set transform size based on mode selection */
     if(cpi->common.txfm_mode == ALLOW_8X8
-        && x->e_mbd.mode_info_context->mbmi.mode != I8X8_PRED
-        && x->e_mbd.mode_info_context->mbmi.mode != B_PRED)
+        && ( x->e_mbd.mode_info_context->mbmi.mode == DC_PRED
+           || x->e_mbd.mode_info_context->mbmi.mode == TM_PRED))
     {
         x->e_mbd.mode_info_context->mbmi.txfm_size = TX_8X8;
         cpi->t8x8_count++;
diff --git a/vp8/encoder/ratectrl.c b/vp8/encoder/ratectrl.c
index 0a9f12a31..a57996612 100644
--- a/vp8/encoder/ratectrl.c
+++ b/vp8/encoder/ratectrl.c
@@ -241,9 +241,15 @@ void vp8_setup_key_frame(VP8_COMP *cpi)
     vpx_memset(cpi->common.fc.pre_mvc_hp, 0, sizeof(cpi->common.fc.pre_mvc_hp));  //initialize pre_mvc to all zero.
 #endif
 
-
     cpi->common.txfm_mode = ONLY_4X4;
 
+    if( cpi->common.Width * cpi->common.Height > 640*360
+        && vp8_ac_yquant(cpi->common.base_qindex) > 171)
+        cpi->common.txfm_mode = ALLOW_8X8;
+    else
+        cpi->common.txfm_mode = ONLY_4X4;
+
+
     //cpi->common.filter_level = 0;      // Reset every key frame.
     cpi->common.filter_level = cpi->common.base_qindex * 3 / 8 ;
 
@@ -280,7 +286,6 @@ void vp8_setup_inter_frame(VP8_COMP *cpi)
 {
 
     if(cpi->common.Width * cpi->common.Height > 640*360)
-        //||cpi->this_frame_target < 7 * cpi->common.MBs)
         cpi->common.txfm_mode = ALLOW_8X8;
     else
         cpi->common.txfm_mode = ONLY_4X4;
-- 
2.40.0