fix integer overflow in ordering function

The test case for #1906 triggers an overflow in this function, viewable with
UBSan:

  $ dot -Kirco -Tgv -o /dev/null tests/graphs/root.gv
  /builds/graphviz/graphviz/lib/label/xlabels.c:35:15: runtime error: signed
    integer overflow: -1884993080 - 1219985688 cannot be represented in type
    'int'

Gitlab: #1906

diff --git a/lib/label/xlabels.c b/lib/label/xlabels.c
index 0e4c4928a38206373b30016802653334e8de7a7b..ae4e606104d1a9bfa9b81da88eadf4283deef918 100644 (file)
--- a/lib/label/xlabels.c
+++ b/lib/label/xlabels.c
@@ -32,7 +32,13 @@ static int icompare(Dt_t * dt, void * v1, void * v2, Dtdisc_t * disc)
     (void)dt;
     (void)disc;
     int k1 = *((int *) v1), k2 = *((int *) v2);
-    return k1 - k2;
+    if (k1 < k2) {
+      return -1;
+    }
+    if (k1 > k2) {
+      return 1;
+    }
+    return 0;
 }
 
 static XLabels_t *xlnew(object_t * objs, int n_objs, xlabel_t * lbls,