Resolve -Wshorten-64-to-32 in highbd variance.

For 8-bit the subtrahend is small enough to fit into uint32_t.

This is the same that was done for:
c0241664a Resolve -Wshorten-64-to-32 in variance.

For 10/12-bit apply:
63a37d16f Prevent negative variance

Change-Id: Iab35e3f3f269035e17c711bd6cc01272c3137e1d

diff --git a/vpx_dsp/variance.c b/vpx_dsp/variance.c
index d1fa0560d594ca91b70d40b86fba2d4ea7e16793..4214150251ffb5b3874e6969a997810ab097c58d 100644 (file)
--- a/vpx_dsp/variance.c
+++ b/vpx_dsp/variance.c
@@ -294,7 +294,7 @@ static void highbd_12_variance(const uint8_t *a8, int a_stride,
                                               uint32_t *sse) {                 \
     int sum;                                                                   \
     highbd_8_variance(a, a_stride, b, b_stride, W, H, sse, &sum);              \
-    return *sse - (((int64_t)sum * sum) / (W * H));                            \
+    return *sse - (uint32_t)(((int64_t)sum * sum) / (W * H));                  \
   }                                                                            \
                                                                                \
   uint32_t vpx_highbd_10_variance##W##x##H##_c(const uint8_t *a, int a_stride, \

diff --git a/vpx_dsp/x86/highbd_variance_sse2.c b/vpx_dsp/x86/highbd_variance_sse2.c
index 414ae5de1ae06f8d31a8f9c32eb0283439f9bd71..a6f7c3d25dfd0d3a77f67d399f8fdd572551a509 100644 (file)
--- a/vpx_dsp/x86/highbd_variance_sse2.c
+++ b/vpx_dsp/x86/highbd_variance_sse2.c
@@ -135,7 +135,7 @@ HIGH_GET_VAR(8);
     highbd_8_variance_sse2(                                                \
         src, src_stride, ref, ref_stride, w, h, sse, &sum,                 \
         vpx_highbd_calc##block_size##x##block_size##var_sse2, block_size); \
-    return *sse - (((int64_t)sum * sum) >> shift);                         \
+    return *sse - (uint32_t)(((int64_t)sum * sum) >> shift);               \
   }                                                                        \
                                                                            \
   uint32_t vpx_highbd_10_variance##w##x##h##_sse2(                         \
@@ -293,12 +293,13 @@ DECLS(sse2);
       }                                                                        \
     }                                                                          \
     *sse_ptr = sse;                                                            \
-    return sse - ((cast se * se) >> (wlog2 + hlog2));                          \
+    return sse - (uint32_t)((cast se * se) >> (wlog2 + hlog2));                \
   }                                                                            \
                                                                                \
   uint32_t vpx_highbd_10_sub_pixel_variance##w##x##h##_##opt(                  \
       const uint8_t *src8, int src_stride, int x_offset, int y_offset,         \
       const uint8_t *dst8, int dst_stride, uint32_t *sse_ptr) {                \
+    int64_t var;                                                               \
     uint32_t sse;                                                              \
     uint16_t *src = CONVERT_TO_SHORTPTR(src8);                                 \
     uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);                                 \
@@ -328,7 +329,8 @@ DECLS(sse2);
     se = ROUND_POWER_OF_TWO(se, 2);                                            \
     sse = ROUND_POWER_OF_TWO(sse, 4);                                          \
     *sse_ptr = sse;                                                            \
-    return sse - ((cast se * se) >> (wlog2 + hlog2));                          \
+    var = (int64_t)(sse) - ((cast se * se) >> (wlog2 + hlog2));                \
+    return (var >= 0) ? (uint32_t)var : 0;                                     \
   }                                                                            \
                                                                                \
   uint32_t vpx_highbd_12_sub_pixel_variance##w##x##h##_##opt(                  \
@@ -337,6 +339,7 @@ DECLS(sse2);
     int start_row;                                                             \
     uint32_t sse;                                                              \
     int se = 0;                                                                \
+    int64_t var;                                                               \
     uint64_t long_sse = 0;                                                     \
     uint16_t *src = CONVERT_TO_SHORTPTR(src8);                                 \
     uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);                                 \
@@ -375,7 +378,8 @@ DECLS(sse2);
     se = ROUND_POWER_OF_TWO(se, 4);                                            \
     sse = (uint32_t)ROUND_POWER_OF_TWO(long_sse, 8);                           \
     *sse_ptr = sse;                                                            \
-    return sse - ((cast se * se) >> (wlog2 + hlog2));                          \
+    var = (int64_t)(sse) - ((cast se * se) >> (wlog2 + hlog2));                \
+    return (var >= 0) ? (uint32_t)var : 0;                                     \
   }
 
 #define FNS(opt)                        \
@@ -444,13 +448,14 @@ DECLS(sse2);
       }                                                                        \
     }                                                                          \
     *sse_ptr = sse;                                                            \
-    return sse - ((cast se * se) >> (wlog2 + hlog2));                          \
+    return sse - (uint32_t)((cast se * se) >> (wlog2 + hlog2));                \
   }                                                                            \
                                                                                \
   uint32_t vpx_highbd_10_sub_pixel_avg_variance##w##x##h##_##opt(              \
       const uint8_t *src8, int src_stride, int x_offset, int y_offset,         \
       const uint8_t *dst8, int dst_stride, uint32_t *sse_ptr,                  \
       const uint8_t *sec8) {                                                   \
+    int64_t var;                                                               \
     uint32_t sse;                                                              \
     uint16_t *src = CONVERT_TO_SHORTPTR(src8);                                 \
     uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);                                 \
@@ -481,7 +486,8 @@ DECLS(sse2);
     se = ROUND_POWER_OF_TWO(se, 2);                                            \
     sse = ROUND_POWER_OF_TWO(sse, 4);                                          \
     *sse_ptr = sse;                                                            \
-    return sse - ((cast se * se) >> (wlog2 + hlog2));                          \
+    var = (int64_t)(sse) - ((cast se * se) >> (wlog2 + hlog2));                \
+    return (var >= 0) ? (uint32_t)var : 0;                                     \
   }                                                                            \
                                                                                \
   uint32_t vpx_highbd_12_sub_pixel_avg_variance##w##x##h##_##opt(              \
@@ -489,6 +495,7 @@ DECLS(sse2);
       const uint8_t *dst8, int dst_stride, uint32_t *sse_ptr,                  \
       const uint8_t *sec8) {                                                   \
     int start_row;                                                             \
+    int64_t var;                                                               \
     uint32_t sse;                                                              \
     int se = 0;                                                                \
     uint64_t long_sse = 0;                                                     \
@@ -530,7 +537,8 @@ DECLS(sse2);
     se = ROUND_POWER_OF_TWO(se, 4);                                            \
     sse = (uint32_t)ROUND_POWER_OF_TWO(long_sse, 8);                           \
     *sse_ptr = sse;                                                            \
-    return sse - ((cast se * se) >> (wlog2 + hlog2));                          \
+    var = (int64_t)(sse) - ((cast se * se) >> (wlog2 + hlog2));                \
+    return (var >= 0) ? (uint32_t)var : 0;                                     \
   }
 
 #define FNS(opt1)                        \