Check against some overflows and allocation failures on blur

This still doesn't catch all cases.

diff --git a/libass/ass_bitmap.c b/libass/ass_bitmap.c
index c7a9fb5c3e3853cde13cf526f0a14248f36a0638..98ed5561d990ac116fefaea382cde1629a615d57 100644 (file)
--- a/libass/ass_bitmap.c
+++ b/libass/ass_bitmap.c
@@ -41,6 +41,11 @@ int generate_tables(ASS_SynthPriv *priv, double radius)
     double volume_diff, volume_factor = 0;
     unsigned volume;
 
+    if (radius < 0)
+        return -1;
+    if (radius + 2.0 > INT_MAX / 2)
+        radius = INT_MAX / 2;
+
     if (priv->radius == radius)
         return 0;
     else
@@ -50,10 +55,13 @@ int generate_tables(ASS_SynthPriv *priv, double radius)
     priv->g_w = 2 * priv->g_r + 1;
 
     if (priv->g_r) {
-        priv->g0 = realloc(priv->g0, priv->g_w * sizeof(double));
-        priv->g = realloc(priv->g, priv->g_w * sizeof(unsigned));
-        priv->gt2 = realloc(priv->gt2, 256 * priv->g_w * sizeof(unsigned));
-        if (priv->g == NULL || priv->gt2 == NULL) {
+        priv->g0 = ass_realloc_array(priv->g0, priv->g_w, sizeof(double));
+        priv->g = ass_realloc_array(priv->g, priv->g_w, sizeof(unsigned));
+        priv->gt2 = ass_realloc_array(priv->gt2, priv->g_w, 256 * sizeof(unsigned));
+        if (!priv->g || !priv->g0 || !priv->gt2) {
+            free(priv->g0);
+            free(priv->g);
+            free(priv->gt2);
             return -1;
         }
     }
@@ -113,8 +121,10 @@ void resize_tmp(ASS_SynthPriv *priv, int w, int h)
 ASS_SynthPriv *ass_synth_init(double radius)
 {
     ASS_SynthPriv *priv = calloc(1, sizeof(ASS_SynthPriv));
-    if (priv)
-        generate_tables(priv, radius);
+    if (priv && generate_tables(priv, radius) < 0) {
+        free(priv);
+        priv = NULL;
+    }
     return priv;
 }
 

diff --git a/libass/ass_render.c b/libass/ass_render.c
index c1cf548c0eda006b5e2618fe712d466b32b06030..cde8ef4851199b4945b425b2c93de241b9e8c783 100644 (file)
--- a/libass/ass_render.c
+++ b/libass/ass_render.c
@@ -1848,8 +1848,7 @@ static void apply_blur(CombinedBitmapInfo *info, ASS_Renderer *render_priv)
     }
 
     // Apply gaussian blur
-    if (blur_radius > 0.0) {
-        generate_tables(priv_blur, blur_radius);
+    if (blur_radius > 0.0 && generate_tables(priv_blur, blur_radius) >= 0) {
         if (bm_o)
             ass_gauss_blur(bm_o->buffer, priv_blur->tmp,
                            bm_o->w, bm_o->h, bm_o->stride,