From: Matthew Fernandez <matthew.fernandez@gmail.com>
Date: Fri, 25 Feb 2022 16:18:17 +0000 (-0800)
Subject: cgraph: fix: handle allocation failures in 'agcanon' and friends
X-Git-Tag: 4.0.0~205^2
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=343ac83aae88692ba36fcd4e6895c3917ab258d2;p=graphviz

cgraph: fix: handle allocation failures in 'agcanon' and friends

There are a number of cgraph API functions that use an internally managed buffer
to save the caller from having to allocate space themselves. Failure to expand
this buffer was being silently ignored, resulting in messy crashes when memory
was exhausted. These failures are now checked for and `agcanon` and `agcanonStr`
return `NULL` on allocation failure. `agwrite` returns `EOF` on allocation
failure.
---

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 134c13099..9d8c1c62d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased (3.0.1)]
 
+### Fixed
+
+- `agcanon`, `agcanonStr`, and `agwrite` now return error values on memory
+  allocation failures instead of crashing or corrupting data
+
 ## [3.0.0] – 2022-02-26
 
 ### Changed
diff --git a/lib/cgraph/write.c b/lib/cgraph/write.c
index 438cd8c26..9f139e274 100644
--- a/lib/cgraph/write.c
+++ b/lib/cgraph/write.c
@@ -14,6 +14,7 @@
 #include <stdio.h>		/* need sprintf() */
 #include <ctype.h>
 #include <cgraph/cghdr.h>
+#include <cgraph/likely.h>
 #include <cgraph/strcasecmp.h>
 #include <inttypes.h>
 
@@ -168,7 +169,10 @@ static char *getoutputbuffer(const char *str)
 
     req = MAX(2 * strlen(str) + 2, BUFSIZ);
     if (req > len) {
-	rv = realloc(rv, req);
+	char *r = realloc(rv, req);
+	if (UNLIKELY(r == NULL))
+	    return NULL;
+	rv = r;
 	len = req;
     }
     return rv;
@@ -181,7 +185,10 @@ static char *getoutputbuffer(const char *str)
  */
 char *agcanonStr(char *str)
 {
-    return agstrcanon(str, getoutputbuffer(str));
+    char *buffer = getoutputbuffer(str);
+    if (UNLIKELY(buffer == NULL))
+        return NULL;
+    return agstrcanon(str, buffer);
 }
 
 /*
@@ -192,6 +199,8 @@ char *agcanonStr(char *str)
 char *agcanon(char *str, int html)
 {
     char* buf = getoutputbuffer(str);
+    if (UNLIKELY(buf == NULL))
+	return NULL;
     if (html)
 	return agcanonhtmlstr(str, buf);
     else
@@ -201,10 +210,14 @@ char *agcanon(char *str, int html)
 static int _write_canonstr(Agraph_t * g, iochan_t * ofile, char *str,
 			   int chk)
 {
-    if (chk)
+    if (chk) {
 	str = agcanonStr(str);
-    else
-	str = _agstrcanon(str, getoutputbuffer(str));
+    } else {
+	char *buffer = getoutputbuffer(str);
+	if (UNLIKELY(buffer == NULL))
+	    return EOF;
+	str = _agstrcanon(str, buffer);
+    }
     return ioput(g, ofile, str);
 }