cgraph: fix: handle allocation failures in 'agcanon' and friends

There are a number of cgraph API functions that use an internally managed buffer
to save the caller from having to allocate space themselves. Failure to expand
this buffer was being silently ignored, resulting in messy crashes when memory
was exhausted. These failures are now checked for and `agcanon` and `agcanonStr`
return `NULL` on allocation failure. `agwrite` returns `EOF` on allocation
failure.

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 134c130994c066cec3b2bef76e87062526df7fc0..9d8c1c62d784db20855ee6bbc6760a456b54d08f 100644 (file)
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased (3.0.1)]
 
+### Fixed
+
+- `agcanon`, `agcanonStr`, and `agwrite` now return error values on memory
+  allocation failures instead of crashing or corrupting data
+
 ## [3.0.0] – 2022-02-26
 
 ### Changed

diff --git a/lib/cgraph/write.c b/lib/cgraph/write.c
index 438cd8c262aa6196d3524bdba74af423dfe57ca4..9f139e2748bdf7e18e3da4b7cb32a9d188fc5774 100644 (file)
--- a/lib/cgraph/write.c
+++ b/lib/cgraph/write.c
@@ -14,6 +14,7 @@
 #include <stdio.h>             /* need sprintf() */
 #include <ctype.h>
 #include <cgraph/cghdr.h>
+#include <cgraph/likely.h>
 #include <cgraph/strcasecmp.h>
 #include <inttypes.h>
 
@@ -168,7 +169,10 @@ static char *getoutputbuffer(const char *str)
 
     req = MAX(2 * strlen(str) + 2, BUFSIZ);
     if (req > len) {
-       rv = realloc(rv, req);
+       char *r = realloc(rv, req);
+       if (UNLIKELY(r == NULL))
+           return NULL;
+       rv = r;
        len = req;
     }
     return rv;
@@ -181,7 +185,10 @@ static char *getoutputbuffer(const char *str)
  */
 char *agcanonStr(char *str)
 {
-    return agstrcanon(str, getoutputbuffer(str));
+    char *buffer = getoutputbuffer(str);
+    if (UNLIKELY(buffer == NULL))
+        return NULL;
+    return agstrcanon(str, buffer);
 }
 
 /*
@@ -192,6 +199,8 @@ char *agcanonStr(char *str)
 char *agcanon(char *str, int html)
 {
     char* buf = getoutputbuffer(str);
+    if (UNLIKELY(buf == NULL))
+       return NULL;
     if (html)
        return agcanonhtmlstr(str, buf);
     else
@@ -201,10 +210,14 @@ char *agcanon(char *str, int html)
 static int _write_canonstr(Agraph_t * g, iochan_t * ofile, char *str,
                           int chk)
 {
-    if (chk)
+    if (chk) {
        str = agcanonStr(str);
-    else
-       str = _agstrcanon(str, getoutputbuffer(str));
+    } else {
+       char *buffer = getoutputbuffer(str);
+       if (UNLIKELY(buffer == NULL))
+           return EOF;
+       str = _agstrcanon(str, buffer);
+    }
     return ioput(g, ofile, str);
 }