]> granicus.if.org Git - libx264/commitdiff
Eliminate intentional array overflow in quant matrix handling
authorFiona Glaser <fiona@x264.com>
Thu, 14 Jan 2010 01:21:31 +0000 (20:21 -0500)
committerFiona Glaser <fiona@x264.com>
Thu, 14 Jan 2010 04:47:11 +0000 (23:47 -0500)
While it probably never caused problems, it was incredibly ugly and evil.

common/set.c

index a7bca7db456f3360af500663432a95d41c5599ec..f9379f08b31f0c25cd80c1407de3943d2138f927 100644 (file)
@@ -184,26 +184,30 @@ fail:
     return -1;
 }
 
+#define CQM_DELETE( n, max )\
+    for( i = 0; i < max; i++ )\
+    {\
+        for( j = 0; j < i; j++ )\
+            if( h->quant##n##_mf[i] == h->quant##n##_mf[j] )\
+                break;\
+        if( j == i )\
+        {\
+            x264_free( h->  quant##n##_mf[i] );\
+            x264_free( h->dequant##n##_mf[i] );\
+            x264_free( h->unquant##n##_mf[i] );\
+        }\
+        for( j = 0; j < i; j++ )\
+            if( h->quant##n##_bias[i] == h->quant##n##_bias[j] )\
+                break;\
+        if( j == i )\
+            x264_free( h->quant##n##_bias[i] );\
+    }
+
 void x264_cqm_delete( x264_t *h )
 {
     int i, j;
-    for( i = 0; i < 6; i++ )
-    {
-        for( j = 0; j < i; j++ )
-            if( h->quant4_mf[i] == h->quant4_mf[j] )
-                break;
-        if( j == i )
-        {
-            x264_free( h->  quant4_mf[i] );
-            x264_free( h->dequant4_mf[i] );
-            x264_free( h->unquant4_mf[i] );
-        }
-        for( j = 0; j < i; j++ )
-            if( h->quant4_bias[i] == h->quant4_bias[j] )
-                break;
-        if( j == i )
-            x264_free( h->quant4_bias[i] );
-    }
+    CQM_DELETE( 4, 4 );
+    CQM_DELETE( 8, 2 );
 }
 
 static int x264_cqm_parse_jmlist( x264_t *h, const char *buf, const char *name,