Resolve -Wshorten-64-to-32 in variance.

The subtrahend is small enough to fit into uint32_t.

Change-Id: Ic4d7128aaa665eaf6b25d562610ba8942c46137f
(cherry picked from commit c0241664aac3a1805db9bd8e09e071ac326531e0)

diff --git a/aom_dsp/variance.c b/aom_dsp/variance.c
index 90617d9cf59d6183a5ec1059b409ff90f607d4f6..c93f98e3c57561164f0d0d0ad08be254dc6c71e9 100644 (file)
--- a/aom_dsp/variance.c
+++ b/aom_dsp/variance.c
@@ -148,7 +148,7 @@ static void var_filter_block2d_bil_second_pass(const uint16_t *a, uint8_t *b,
                                      uint32_t *sse) {                \
     int sum;                                                         \
     variance(a, a_stride, b, b_stride, W, H, sse, &sum);             \
-    return *sse - (((int64_t)sum * sum) / (W * H));                  \
+    return *sse - (uint32_t)(((int64_t)sum * sum) / (W * H));        \
   }
 
 #define SUBPIX_VAR(W, H)                                                \

diff --git a/aom_dsp/x86/variance_sse2.c b/aom_dsp/x86/variance_sse2.c
index 04668371bbe9363fe45eaded8c45706a62d85443..d9563aa7f18f1cd3c66a6dbf4137ec111ddb6c7f 100644 (file)
--- a/aom_dsp/x86/variance_sse2.c
+++ b/aom_dsp/x86/variance_sse2.c
@@ -249,7 +249,7 @@ unsigned int aom_variance32x32_sse2(const uint8_t *src, int src_stride,
                 aom_get16x16var_sse2, 16);
   assert(sum <= 255 * 32 * 32);
   assert(sum >= -255 * 32 * 32);
-  return *sse - (((int64_t)sum * sum) >> 10);
+  return *sse - (unsigned int)(((int64_t)sum * sum) >> 10);
 }
 
 unsigned int aom_variance32x16_sse2(const uint8_t *src, int src_stride,
@@ -260,7 +260,7 @@ unsigned int aom_variance32x16_sse2(const uint8_t *src, int src_stride,
                 aom_get16x16var_sse2, 16);
   assert(sum <= 255 * 32 * 16);
   assert(sum >= -255 * 32 * 16);
-  return *sse - (((int64_t)sum * sum) >> 9);
+  return *sse - (unsigned int)(((int64_t)sum * sum) >> 9);
 }
 
 unsigned int aom_variance16x32_sse2(const uint8_t *src, int src_stride,
@@ -271,7 +271,7 @@ unsigned int aom_variance16x32_sse2(const uint8_t *src, int src_stride,
                 aom_get16x16var_sse2, 16);
   assert(sum <= 255 * 32 * 16);
   assert(sum >= -255 * 32 * 16);
-  return *sse - (((int64_t)sum * sum) >> 9);
+  return *sse - (unsigned int)(((int64_t)sum * sum) >> 9);
 }
 
 unsigned int aom_variance64x64_sse2(const uint8_t *src, int src_stride,
@@ -282,7 +282,7 @@ unsigned int aom_variance64x64_sse2(const uint8_t *src, int src_stride,
                 aom_get16x16var_sse2, 16);
   assert(sum <= 255 * 64 * 64);
   assert(sum >= -255 * 64 * 64);
-  return *sse - (((int64_t)sum * sum) >> 12);
+  return *sse - (unsigned int)(((int64_t)sum * sum) >> 12);
 }
 
 unsigned int aom_variance64x32_sse2(const uint8_t *src, int src_stride,
@@ -293,7 +293,7 @@ unsigned int aom_variance64x32_sse2(const uint8_t *src, int src_stride,
                 aom_get16x16var_sse2, 16);
   assert(sum <= 255 * 64 * 32);
   assert(sum >= -255 * 64 * 32);
-  return *sse - (((int64_t)sum * sum) >> 11);
+  return *sse - (unsigned int)(((int64_t)sum * sum) >> 11);
 }
 
 unsigned int aom_variance32x64_sse2(const uint8_t *src, int src_stride,
@@ -304,7 +304,7 @@ unsigned int aom_variance32x64_sse2(const uint8_t *src, int src_stride,
                 aom_get16x16var_sse2, 16);
   assert(sum <= 255 * 64 * 32);
   assert(sum >= -255 * 64 * 32);
-  return *sse - (((int64_t)sum * sum) >> 11);
+  return *sse - (unsigned int)(((int64_t)sum * sum) >> 11);
 }
 
 unsigned int aom_mse8x8_sse2(const uint8_t *src, int src_stride,
@@ -381,7 +381,7 @@ DECLS(ssse3, ssse3);
       }                                                                        \
     }                                                                          \
     *sse_ptr = sse;                                                            \
-    return sse - (cast_prod(cast se * se) >> (wlog2 + hlog2));                 \
+    return sse - (unsigned int)(cast_prod(cast se * se) >> (wlog2 + hlog2));   \
   }
 
 #define FNS(opt1, opt2)                              \
@@ -452,7 +452,7 @@ DECLS(ssse3, ssse3);
       }                                                                        \
     }                                                                          \
     *sseptr = sse;                                                             \
-    return sse - (cast_prod(cast se * se) >> (wlog2 + hlog2));                 \
+    return sse - (unsigned int)(cast_prod(cast se * se) >> (wlog2 + hlog2));   \
   }
 
 #define FNS(opt1, opt2)                              \