From: Matthew Fernandez <matthew.fernandez@gmail.com>
Date: Sun, 19 Jun 2022 21:38:37 +0000 (-0700)
Subject: fix integer overflow in ordering function
X-Git-Tag: 5.0.0~19^2~3
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=cccb8b1d22a18031fc92d93133c7fa14ef7e1361;p=graphviz

fix integer overflow in ordering function

The test case for #1906 triggers an overflow in this function, viewable with
UBSan:

  $ dot -Kirco -Tgv -o /dev/null tests/graphs/root.gv
  /builds/graphviz/graphviz/lib/label/xlabels.c:35:15: runtime error: signed
    integer overflow: -1884993080 - 1219985688 cannot be represented in type
    'int'

Gitlab: #1906
---

diff --git a/lib/label/xlabels.c b/lib/label/xlabels.c
index 0e4c4928a..ae4e60610 100644
--- a/lib/label/xlabels.c
+++ b/lib/label/xlabels.c
@@ -32,7 +32,13 @@ static int icompare(Dt_t * dt, void * v1, void * v2, Dtdisc_t * disc)
     (void)dt;
     (void)disc;
     int k1 = *((int *) v1), k2 = *((int *) v2);
-    return k1 - k2;
+    if (k1 < k2) {
+      return -1;
+    }
+    if (k1 > k2) {
+      return 1;
+    }
+    return 0;
 }
 
 static XLabels_t *xlnew(object_t * objs, int n_objs, xlabel_t * lbls,