bitdepth conversion: really use num elements

The previous implementation confused bit/bytes/elements. It was using
'32' as the multiplier but that was mistakenly adopted because a 32x32
transform embedded the stride.

Change-Id: Ieeb867a332416b9a40580b5e7c9b20088e9e691a

diff --git a/vp9/encoder/x86/vp9_dct_sse2.asm b/vp9/encoder/x86/vp9_dct_sse2.asm
index e24cabba0456d97bec83437d801c4b7b409a5b52..8152dce864266fc8d3d5206674d2cd74e2ec2336 100644 (file)
--- a/vp9/encoder/x86/vp9_dct_sse2.asm
+++ b/vp9/encoder/x86/vp9_dct_sse2.asm
@@ -64,6 +64,6 @@ cglobal fwht4x4, 3, 4, 8, input, output, stride
   psllw           m1,        2
 
   STORE_TRAN_LOW 0, outputq, 0, 2, 3
-  STORE_TRAN_LOW 1, outputq, 1, 2, 3
+  STORE_TRAN_LOW 1, outputq, 8, 2, 3
 
   RET

diff --git a/vp9/encoder/x86/vp9_error_sse2.asm b/vp9/encoder/x86/vp9_error_sse2.asm
index dcedf913dbef96238d71ce99e30b8ca16e9dee58..5186d3087468211f14103e2a0a43fc74881170cd 100644 (file)
--- a/vp9/encoder/x86/vp9_error_sse2.asm
+++ b/vp9/encoder/x86/vp9_error_sse2.asm
@@ -91,8 +91,8 @@ cglobal block_error_fp, 3, 3, 6, uqc, dqc, size
 .loop:
   LOAD_TRAN_LOW 2, uqcq, 0
   LOAD_TRAN_LOW 0, dqcq, 0
-  LOAD_TRAN_LOW 3, uqcq, 1
-  LOAD_TRAN_LOW 1, dqcq, 1
+  LOAD_TRAN_LOW 3, uqcq, 8
+  LOAD_TRAN_LOW 1, dqcq, 8
   INCREMENT_ELEMENTS_TRAN_LOW uqcq, 16
   INCREMENT_ELEMENTS_TRAN_LOW dqcq, 16
   sub    sizeq, 16

diff --git a/vpx_dsp/x86/avg_ssse3_x86_64.asm b/vpx_dsp/x86/avg_ssse3_x86_64.asm
index 4b486c15bba1efb3588a0a8d7f8e5418bc543300..22e0a086cc20c2143bb559c44a2bc3a047884afd 100644 (file)
--- a/vpx_dsp/x86/avg_ssse3_x86_64.asm
+++ b/vpx_dsp/x86/avg_ssse3_x86_64.asm
@@ -117,14 +117,14 @@ cglobal hadamard_8x8, 3, 5, 11, input, stride, output
   TRANSPOSE8X8 0, 1, 2, 3, 4, 5, 6, 7, 9, 10
   HMD8_1D
 
-  STORE_TRAN_LOW 0, outputq, 0, 8, 9
-  STORE_TRAN_LOW 1, outputq, 1, 8, 9
-  STORE_TRAN_LOW 2, outputq, 2, 8, 9
-  STORE_TRAN_LOW 3, outputq, 3, 8, 9
-  STORE_TRAN_LOW 4, outputq, 4, 8, 9
-  STORE_TRAN_LOW 5, outputq, 5, 8, 9
-  STORE_TRAN_LOW 6, outputq, 6, 8, 9
-  STORE_TRAN_LOW 7, outputq, 7, 8, 9
+  STORE_TRAN_LOW 0, outputq,  0, 8, 9
+  STORE_TRAN_LOW 1, outputq,  8, 8, 9
+  STORE_TRAN_LOW 2, outputq, 16, 8, 9
+  STORE_TRAN_LOW 3, outputq, 24, 8, 9
+  STORE_TRAN_LOW 4, outputq, 32, 8, 9
+  STORE_TRAN_LOW 5, outputq, 40, 8, 9
+  STORE_TRAN_LOW 6, outputq, 48, 8, 9
+  STORE_TRAN_LOW 7, outputq, 56, 8, 9
 
   RET
 %endif

diff --git a/vpx_dsp/x86/bitdepth_conversion_sse2.asm b/vpx_dsp/x86/bitdepth_conversion_sse2.asm
index b2df5207a4c588b19552839a558c37d6ab93c135..2bcbc0ac12d0b96413e2351db6b07cfa89404edd 100644 (file)
--- a/vpx_dsp/x86/bitdepth_conversion_sse2.asm
+++ b/vpx_dsp/x86/bitdepth_conversion_sse2.asm
@@ -32,21 +32,21 @@
 %endmacro
 
 ; Load %2 + %3 into m%1.
-; %3 is the offset in elements, not bits.
+; %3 is the offset in elements, not bytes.
 ; If tran_low_t is 16 bits (low bit depth configuration) then load the value
 ; directly. If tran_low_t is 32 bits (high bit depth configuration) then pack
 ; the values down to 16 bits.
 %macro LOAD_TRAN_LOW 3
 %if CONFIG_VP9_HIGHBITDEPTH
-  mova     m%1, [%2 + %3 * 32]
-  packssdw m%1, [%2 + %3 * 32 + 16]
+  mova     m%1, [%2 + %3 * 4]
+  packssdw m%1, [%2 + %3 * 4 + 16]
 %else
-  mova     m%1, [%2 + %3 * 16]
+  mova     m%1, [%2 + %3 * 2]
 %endif
 %endmacro
 
 ; Store m%1 to %2 + %3.
-; %3 is the offset in elements, not bits.
+; %3 is the offset in elements, not bytes.
 ; If tran_low_t is 16 bits (low bit depth configuration) then store the value
 ; directly. If tran_low_t is 32 bits (high bit depth configuration) then sign
 ; extend the values first.
@@ -58,9 +58,9 @@
   pcmpgtw                   m%4, m%1
   punpcklwd                 m%5, m%4
   punpckhwd                 m%1, m%4
-  mova      [%2 + %3 * 32 +  0], m%5
-  mova      [%2 + %3 * 32 + 16], m%1
+  mova       [%2 + %3 * 4 +  0], m%5
+  mova       [%2 + %3 * 4 + 16], m%1
 %else
-  mova           [%2 + %3 * 16], m%1
+  mova            [%2 + %3 * 2], m%1
 %endif
 %endmacro

diff --git a/vpx_dsp/x86/inv_txfm_ssse3_x86_64.asm b/vpx_dsp/x86/inv_txfm_ssse3_x86_64.asm
index 77bfe86cf49d5796bc69182efc3989799ba18df0..a3e0b7fc5b19c64344ce47e05737ded478a9a3a8 100644 (file)
--- a/vpx_dsp/x86/inv_txfm_ssse3_x86_64.asm
+++ b/vpx_dsp/x86/inv_txfm_ssse3_x86_64.asm
@@ -984,14 +984,14 @@ idct32x32_135:
   mov             r7, 2
 
 idct32x32_135_transpose:
-  LOAD_TRAN_LOW 0, r3,  0
-  LOAD_TRAN_LOW 1, r3,  4
-  LOAD_TRAN_LOW 2, r3,  8
-  LOAD_TRAN_LOW 3, r3, 12
-  LOAD_TRAN_LOW 4, r3, 16
-  LOAD_TRAN_LOW 5, r3, 20
-  LOAD_TRAN_LOW 6, r3, 24
-  LOAD_TRAN_LOW 7, r3, 28
+  LOAD_TRAN_LOW 0, r3,   0
+  LOAD_TRAN_LOW 1, r3,  32
+  LOAD_TRAN_LOW 2, r3,  64
+  LOAD_TRAN_LOW 3, r3,  96
+  LOAD_TRAN_LOW 4, r3, 128
+  LOAD_TRAN_LOW 5, r3, 160
+  LOAD_TRAN_LOW 6, r3, 192
+  LOAD_TRAN_LOW 7, r3, 224
 
   TRANSPOSE8X8  0, 1, 2, 3, 4, 5, 6, 7, 9, 10
 
@@ -1422,14 +1422,14 @@ idct32x32_1024:
   mov             r7, 4
 
 idct32x32_1024_transpose:
-  LOAD_TRAN_LOW 0, r3,  0
-  LOAD_TRAN_LOW 1, r3,  4
-  LOAD_TRAN_LOW 2, r3,  8
-  LOAD_TRAN_LOW 3, r3, 12
-  LOAD_TRAN_LOW 4, r3, 16
-  LOAD_TRAN_LOW 5, r3, 20
-  LOAD_TRAN_LOW 6, r3, 24
-  LOAD_TRAN_LOW 7, r3, 28
+  LOAD_TRAN_LOW 0, r3,   0
+  LOAD_TRAN_LOW 1, r3,  32
+  LOAD_TRAN_LOW 2, r3,  64
+  LOAD_TRAN_LOW 3, r3,  96
+  LOAD_TRAN_LOW 4, r3, 128
+  LOAD_TRAN_LOW 5, r3, 160
+  LOAD_TRAN_LOW 6, r3, 192
+  LOAD_TRAN_LOW 7, r3, 224
 
   TRANSPOSE8X8  0, 1, 2, 3, 4, 5, 6, 7, 9, 10
 

diff --git a/vpx_dsp/x86/inv_wht_sse2.asm b/vpx_dsp/x86/inv_wht_sse2.asm
index a9c52dc3d68e030a404bdade3b18e0de2f006aa4..bcf1a6ef98954f05bfd52b5b20fbd9c370ab5d05 100644 (file)
--- a/vpx_dsp/x86/inv_wht_sse2.asm
+++ b/vpx_dsp/x86/inv_wht_sse2.asm
@@ -84,7 +84,7 @@ SECTION .text
 INIT_XMM sse2
 cglobal iwht4x4_16_add, 3, 3, 7, input, output, stride
   LOAD_TRAN_LOW    0, inputq, 0
-  LOAD_TRAN_LOW    1, inputq, 1
+  LOAD_TRAN_LOW    1, inputq, 8
   psraw           m0,        2
   psraw           m1,        2