Fix implicit integer sign change and truncation (part 2)

diff --git a/common/mc.c b/common/mc.c
index 0f384fd766c39f78705f1d1143588b4d9e868b66..d5e6d550a31c4bcdd3c96bf418553ca4fd3d346a 100644 (file)
--- a/common/mc.c
+++ b/common/mc.c
@@ -423,7 +423,7 @@ static void integral_init4h( uint16_t *sum, pixel *pix, intptr_t stride )
     int v = pix[0]+pix[1]+pix[2]+pix[3];
     for( int x = 0; x < stride-4; x++ )
     {
-        sum[x] = v + sum[x-stride];
+        sum[x] = (uint16_t)(v + sum[x-stride]);
         v += pix[x+4] - pix[x];
     }
 }
@@ -433,7 +433,7 @@ static void integral_init8h( uint16_t *sum, pixel *pix, intptr_t stride )
     int v = pix[0]+pix[1]+pix[2]+pix[3]+pix[4]+pix[5]+pix[6]+pix[7];
     for( int x = 0; x < stride-8; x++ )
     {
-        sum[x] = v + sum[x-stride];
+        sum[x] = (uint16_t)(v + sum[x-stride]);
         v += pix[x+8] - pix[x];
     }
 }
@@ -441,15 +441,15 @@ static void integral_init8h( uint16_t *sum, pixel *pix, intptr_t stride )
 static void integral_init4v( uint16_t *sum8, uint16_t *sum4, intptr_t stride )
 {
     for( int x = 0; x < stride-8; x++ )
-        sum4[x] = sum8[x+4*stride] - sum8[x];
+        sum4[x] = (uint16_t)(sum8[x+4*stride] - sum8[x]);
     for( int x = 0; x < stride-8; x++ )
-        sum8[x] = sum8[x+8*stride] + sum8[x+8*stride+4] - sum8[x] - sum8[x+4];
+        sum8[x] = (uint16_t)(sum8[x+8*stride] + sum8[x+8*stride+4] - sum8[x] - sum8[x+4]);
 }
 
 static void integral_init8v( uint16_t *sum8, intptr_t stride )
 {
     for( int x = 0; x < stride-8; x++ )
-        sum8[x] = sum8[x+8*stride] - sum8[x];
+        sum8[x] = (uint16_t)(sum8[x+8*stride] - sum8[x]);
 }
 
 void x264_frame_init_lowres( x264_t *h, x264_frame_t *frame )
@@ -550,8 +550,8 @@ static void mbtree_propagate_list( x264_t *h, uint16_t *ref_costs, int16_t (*mvs
 
         int x = mvs[i][0];
         int y = mvs[i][1];
-        unsigned mbx = (x>>5)+i;
-        unsigned mby = (y>>5)+mb_y;
+        unsigned mbx = (unsigned)((x>>5)+i);
+        unsigned mby = (unsigned)((y>>5)+mb_y);
         unsigned idx0 = mbx + mby * stride;
         unsigned idx2 = idx0 + stride;
         x &= 31;

diff --git a/common/mvpred.c b/common/mvpred.c
index 0053a634adec2564a9780fab93a8540a78a59ad6..a5f48773a997a116331c836deb09818477d9ed8c 100644 (file)
--- a/common/mvpred.c
+++ b/common/mvpred.c
@@ -470,23 +470,23 @@ int x264_mb_predict_mv_direct16x16( x264_t *h, int *b_changed )
     {
         int changed;
 
-        changed  = M32( h->mb.cache.direct_mv[0][0] ) ^ M32( h->mb.cache.mv[0][x264_scan8[0]] );
-        changed |= M32( h->mb.cache.direct_mv[1][0] ) ^ M32( h->mb.cache.mv[1][x264_scan8[0]] );
+        changed  = (int)(M32( h->mb.cache.direct_mv[0][0] ) ^ M32( h->mb.cache.mv[0][x264_scan8[0]] ));
+        changed |= (int)(M32( h->mb.cache.direct_mv[1][0] ) ^ M32( h->mb.cache.mv[1][x264_scan8[0]] ));
         changed |= h->mb.cache.direct_ref[0][0] ^ h->mb.cache.ref[0][x264_scan8[0]];
         changed |= h->mb.cache.direct_ref[1][0] ^ h->mb.cache.ref[1][x264_scan8[0]];
         if( !changed && h->mb.i_partition != D_16x16 )
         {
-            changed |= M32( h->mb.cache.direct_mv[0][3] ) ^ M32( h->mb.cache.mv[0][x264_scan8[12]] );
-            changed |= M32( h->mb.cache.direct_mv[1][3] ) ^ M32( h->mb.cache.mv[1][x264_scan8[12]] );
+            changed |= (int)(M32( h->mb.cache.direct_mv[0][3] ) ^ M32( h->mb.cache.mv[0][x264_scan8[12]] ));
+            changed |= (int)(M32( h->mb.cache.direct_mv[1][3] ) ^ M32( h->mb.cache.mv[1][x264_scan8[12]] ));
             changed |= h->mb.cache.direct_ref[0][3] ^ h->mb.cache.ref[0][x264_scan8[12]];
             changed |= h->mb.cache.direct_ref[1][3] ^ h->mb.cache.ref[1][x264_scan8[12]];
         }
         if( !changed && h->mb.i_partition == D_8x8 )
         {
-            changed |= M32( h->mb.cache.direct_mv[0][1] ) ^ M32( h->mb.cache.mv[0][x264_scan8[4]] );
-            changed |= M32( h->mb.cache.direct_mv[1][1] ) ^ M32( h->mb.cache.mv[1][x264_scan8[4]] );
-            changed |= M32( h->mb.cache.direct_mv[0][2] ) ^ M32( h->mb.cache.mv[0][x264_scan8[8]] );
-            changed |= M32( h->mb.cache.direct_mv[1][2] ) ^ M32( h->mb.cache.mv[1][x264_scan8[8]] );
+            changed |= (int)(M32( h->mb.cache.direct_mv[0][1] ) ^ M32( h->mb.cache.mv[0][x264_scan8[4]] ));
+            changed |= (int)(M32( h->mb.cache.direct_mv[1][1] ) ^ M32( h->mb.cache.mv[1][x264_scan8[4]] ));
+            changed |= (int)(M32( h->mb.cache.direct_mv[0][2] ) ^ M32( h->mb.cache.mv[0][x264_scan8[8]] ));
+            changed |= (int)(M32( h->mb.cache.direct_mv[1][2] ) ^ M32( h->mb.cache.mv[1][x264_scan8[8]] ));
             changed |= h->mb.cache.direct_ref[0][1] ^ h->mb.cache.ref[0][x264_scan8[4]];
             changed |= h->mb.cache.direct_ref[1][1] ^ h->mb.cache.ref[1][x264_scan8[4]];
             changed |= h->mb.cache.direct_ref[0][2] ^ h->mb.cache.ref[0][x264_scan8[8]];
@@ -590,8 +590,8 @@ void x264_mb_predict_mv_ref16x16( x264_t *h, int i_list, int i_ref, int16_t (*mv
         { \
             int mb_index = h->mb.i_mb_xy + dx + dy*h->mb.i_mb_stride; \
             int scale = (curpoc - refpoc) * l0->inv_ref_poc[MB_INTERLACED&field]; \
-            mvc[i][0] = (l0->mv16x16[mb_index][0]*scale + 128) >> 8; \
-            mvc[i][1] = (l0->mv16x16[mb_index][1]*scale + 128) >> 8; \
+            mvc[i][0] = x264_clip3( (l0->mv16x16[mb_index][0]*scale + 128) >> 8, INT16_MIN, INT16_MAX ); \
+            mvc[i][1] = x264_clip3( (l0->mv16x16[mb_index][1]*scale + 128) >> 8, INT16_MIN, INT16_MAX ); \
             i++; \
         }
 

diff --git a/common/osdep.h b/common/osdep.h
index 2ce349b3ddcac6146270b81cec4fe238865e3233..dfa4aaecfd5b0b0ba09064b6dd11f30df46b89ed 100644 (file)
--- a/common/osdep.h
+++ b/common/osdep.h
@@ -496,7 +496,7 @@ static ALWAYS_INLINE uintptr_t endian_fix( uintptr_t x )
 }
 static ALWAYS_INLINE uint16_t endian_fix16( uint16_t x )
 {
-    return (x<<8)|(x>>8);
+    return (uint16_t)((x<<8)|(x>>8));
 }
 #endif
 

diff --git a/common/pixel.c b/common/pixel.c
index c20c54126c2ecbe4df48d4a64ed7180858dd0b42..113df307e68bf8db3a4a05833d5b8f6084bdbf31 100644 (file)
--- a/common/pixel.c
+++ b/common/pixel.c
@@ -267,11 +267,11 @@ static NOINLINE int x264_pixel_satd_4x4( pixel *pix1, intptr_t i_pix1, pixel *pi
     sum2_t sum = 0;
     for( int i = 0; i < 4; i++, pix1 += i_pix1, pix2 += i_pix2 )
     {
-        a0 = pix1[0] - pix2[0];
-        a1 = pix1[1] - pix2[1];
+        a0 = (sum2_t)(pix1[0] - pix2[0]);
+        a1 = (sum2_t)(pix1[1] - pix2[1]);
         b0 = (a0+a1) + ((a0-a1)<<BITS_PER_SUM);
-        a2 = pix1[2] - pix2[2];
-        a3 = pix1[3] - pix2[3];
+        a2 = (sum2_t)(pix1[2] - pix2[2]);
+        a3 = (sum2_t)(pix1[3] - pix2[3]);
         b1 = (a2+a3) + ((a2-a3)<<BITS_PER_SUM);
         tmp[i][0] = b0 + b1;
         tmp[i][1] = b0 - b1;
@@ -292,10 +292,10 @@ static NOINLINE int x264_pixel_satd_8x4( pixel *pix1, intptr_t i_pix1, pixel *pi
     sum2_t sum = 0;
     for( int i = 0; i < 4; i++, pix1 += i_pix1, pix2 += i_pix2 )
     {
-        a0 = (pix1[0] - pix2[0]) + ((sum2_t)(pix1[4] - pix2[4]) << BITS_PER_SUM);
-        a1 = (pix1[1] - pix2[1]) + ((sum2_t)(pix1[5] - pix2[5]) << BITS_PER_SUM);
-        a2 = (pix1[2] - pix2[2]) + ((sum2_t)(pix1[6] - pix2[6]) << BITS_PER_SUM);
-        a3 = (pix1[3] - pix2[3]) + ((sum2_t)(pix1[7] - pix2[7]) << BITS_PER_SUM);
+        a0 = (sum2_t)(pix1[0] - pix2[0]) + ((sum2_t)(pix1[4] - pix2[4]) << BITS_PER_SUM);
+        a1 = (sum2_t)(pix1[1] - pix2[1]) + ((sum2_t)(pix1[5] - pix2[5]) << BITS_PER_SUM);
+        a2 = (sum2_t)(pix1[2] - pix2[2]) + ((sum2_t)(pix1[6] - pix2[6]) << BITS_PER_SUM);
+        a3 = (sum2_t)(pix1[3] - pix2[3]) + ((sum2_t)(pix1[7] - pix2[7]) << BITS_PER_SUM);
         HADAMARD4( tmp[i][0], tmp[i][1], tmp[i][2], tmp[i][3], a0,a1,a2,a3 );
     }
     for( int i = 0; i < 4; i++ )
@@ -336,17 +336,17 @@ static NOINLINE int sa8d_8x8( pixel *pix1, intptr_t i_pix1, pixel *pix2, intptr_
     sum2_t sum = 0;
     for( int i = 0; i < 8; i++, pix1 += i_pix1, pix2 += i_pix2 )
     {
-        a0 = pix1[0] - pix2[0];
-        a1 = pix1[1] - pix2[1];
+        a0 = (sum2_t)(pix1[0] - pix2[0]);
+        a1 = (sum2_t)(pix1[1] - pix2[1]);
         b0 = (a0+a1) + ((a0-a1)<<BITS_PER_SUM);
-        a2 = pix1[2] - pix2[2];
-        a3 = pix1[3] - pix2[3];
+        a2 = (sum2_t)(pix1[2] - pix2[2]);
+        a3 = (sum2_t)(pix1[3] - pix2[3]);
         b1 = (a2+a3) + ((a2-a3)<<BITS_PER_SUM);
-        a4 = pix1[4] - pix2[4];
-        a5 = pix1[5] - pix2[5];
+        a4 = (sum2_t)(pix1[4] - pix2[4]);
+        a5 = (sum2_t)(pix1[5] - pix2[5]);
         b2 = (a4+a5) + ((a4-a5)<<BITS_PER_SUM);
-        a6 = pix1[6] - pix2[6];
-        a7 = pix1[7] - pix2[7];
+        a6 = (sum2_t)(pix1[6] - pix2[6]);
+        a7 = (sum2_t)(pix1[7] - pix2[7]);
         b3 = (a6+a7) + ((a6-a7)<<BITS_PER_SUM);
         HADAMARD4( tmp[i][0], tmp[i][1], tmp[i][2], tmp[i][3], b0,b1,b2,b3 );
     }

diff --git a/common/quant.c b/common/quant.c
index fe8c81d75d37e46eae00b03ffe2fd5204d11dc89..6b71225ac0a0e0d84ba64811d105bfde53cfc3e7 100644 (file)
--- a/common/quant.c
+++ b/common/quant.c
@@ -47,9 +47,9 @@
 #define QUANT_ONE( coef, mf, f ) \
 { \
     if( (coef) > 0 ) \
-        (coef) = (f + (coef)) * (mf) >> 16; \
+        (coef) = ((f) + (uint32_t)(coef)) * (mf) >> 16; \
     else \
-        (coef) = - ((f - (coef)) * (mf) >> 16); \
+        (coef) = -(int32_t)(((f) + (uint32_t)(-coef)) * (mf) >> 16); \
     nz |= (coef); \
 }
 

diff --git a/common/set.c b/common/set.c
index 859a249d94d06486a2634ce4bf4739545788dfa2..c2152bd59ea8b868a4781c9b7c22b2f5403e507d 100644 (file)
--- a/common/set.c
+++ b/common/set.c
@@ -168,7 +168,8 @@ int x264_cqm_init( x264_t *h )
             for( int i = 0; i < 16; i++ )
             {
                 h->unquant4_mf[i_list][q][i] = (1ULL << (q/6 + 15 + 8)) / quant4_mf[i_list][q%6][i];
-                h->quant4_mf[i_list][q][i] = j = SHIFT(quant4_mf[i_list][q%6][i], q/6 - 1);
+                j = SHIFT(quant4_mf[i_list][q%6][i], q/6 - 1);
+                h->quant4_mf[i_list][q][i] = (uint16_t)j;
                 if( !j )
                 {
                     min_qp_err = X264_MIN( min_qp_err, q );

diff --git a/tools/checkasm.c b/tools/checkasm.c
index 8e5c3e4e88389209032cfde04672f90eac9aae54..9b59c1e18baa2a03e3ea7895a047cbc62a17f1d9 100644 (file)
--- a/tools/checkasm.c
+++ b/tools/checkasm.c
@@ -1893,7 +1893,7 @@ static int check_mc( uint32_t cpu_ref, uint32_t cpu_new )
         for( size_t size = 16; size < 512; size += 16 )
         {
             for( size_t i = 0; i < size; i++ )
-                buf1[i] = rand();
+                buf1[i] = (uint8_t)rand();
             memset( buf4-1, 0xAA, size + 2 );
             call_c( mc_c.memcpy_aligned, buf3, buf1, size );
             call_a( mc_a.memcpy_aligned, buf4, buf1, size );
@@ -2815,7 +2815,7 @@ static int check_bitstream( uint32_t cpu_ref, uint32_t cpu_new )
             int test_size = i < 10 ? i+1 : rand() & 0x3fff;
             /* Test 8 different probability distributions of zeros */
             for( int j = 0; j < test_size+32; j++ )
-                input[j] = (rand()&((1 << ((i&7)+1)) - 1)) * rand();
+                input[j] = (uint8_t)((rand()&((1 << ((i&7)+1)) - 1)) * rand());
             uint8_t *end_c = (uint8_t*)call_c1( bs_c.nal_escape, output1, input, input+test_size );
             uint8_t *end_a = (uint8_t*)call_a1( bs_a.nal_escape, output2, input, input+test_size );
             int size_c = end_c-output1;
@@ -2828,7 +2828,7 @@ static int check_bitstream( uint32_t cpu_ref, uint32_t cpu_new )
             }
         }
         for( int j = 0; j < size+32; j++ )
-            input[j] = rand();
+            input[j] = (uint8_t)rand();
         call_c2( bs_c.nal_escape, output1, input, input+size );
         call_a2( bs_a.nal_escape, output2, input, input+size );
         free(input);
@@ -3001,7 +3001,7 @@ REALIGN_STACK int main( int argc, char **argv )
         argv++;
     }
 
-    unsigned seed = ( argc > 1 ) ? strtoul(argv[1], NULL, 0) : x264_mdate();
+    unsigned seed = ( argc > 1 ) ? strtoul(argv[1], NULL, 0) : (unsigned)x264_mdate();
     fprintf( stderr, "x264: using random seed %u\n", seed );
     srand( seed );