From: Matthew Fernandez <matthew.fernandez@gmail.com>
Date: Thu, 7 Oct 2021 02:57:42 +0000 (-0700)
Subject: portcmp: rephrase to avoid conversion to int
X-Git-Tag: 2.49.2~14^2~8
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=079fe43b981e8891ddac12f9ce7cb12d03b8cde3;p=graphviz

portcmp: rephrase to avoid conversion to int

This function was seemingly incorrectly casting the (float) result of a float
subtraction to an integer. Apart from triggering compiler warnings, this had the
effect of concluding points that were merely close together were actually equal.
---

diff --git a/lib/dotgen/dotsplines.c b/lib/dotgen/dotsplines.c
index 3abe30e87..aadb4fa43 100644
--- a/lib/dotgen/dotsplines.c
+++ b/lib/dotgen/dotsplines.c
@@ -170,15 +170,19 @@ static splineInfo sinfo = {.swapEnds = swap_ends_p,
 
 int portcmp(port p0, port p1)
 {
-    int rv;
     if (p1.defined == FALSE)
 	return (p0.defined ? 1 : 0);
     if (p0.defined == FALSE)
 	return -1;
-    rv = p0.p.x - p1.p.x;
-    if (rv == 0)
-	rv = p0.p.y - p1.p.y;
-    return rv;
+    if (p0.p.x < p1.p.x)
+	return -1;
+    if (p0.p.x > p1.p.x)
+	return 1;
+    if (p0.p.y < p1.p.y)
+	return -1;
+    if (p0.p.y > p1.p.y)
+	return 1;
+    return 0;
 }
 
 /* swap_bezier: