fdpgen: layout: fix heap-buffer-overflow by storing dimension in root graph

The layout allocates memory based on the dimension of the root graph,
but since the dimension was stored in the subgraph and the dimension
in the root graph defaulted to zero, too little memory was allocated.

An alternative solution would have been to use the dimension of the
subgraph, but this had other implications and the chosen solution is
the same as what the other two layout engines supporting the `dim`
attribute (neato and sfdp) use.

diff --git a/lib/fdpgen/layout.c b/lib/fdpgen/layout.c
index ad440b78ba5dc6a16a220d1f5d299babf34dce36..61a0364240f0908dcc7370fdcd8e288517773ca1 100644 (file)
--- a/lib/fdpgen/layout.c
+++ b/lib/fdpgen/layout.c
@@ -433,7 +433,7 @@ static graph_t *deriveGraph(graph_t * g, layout_info * infop)
 #ifdef DEBUG
     GORIG(dg) = g;
 #endif
-    GD_ndim(dg) = GD_ndim(g);
+    GD_ndim(dg) = GD_ndim(agroot(g));
 
     /* Copy attributes from g.
      */
@@ -1012,7 +1012,7 @@ mkClusters (graph_t * g, clist_t* pclist, graph_t* parent)
        if (!strncmp(agnameof(subg), "cluster", 7)) {
            agbindrec(subg, "Agraphinfo_t", sizeof(Agraphinfo_t), true);
            GD_alg(subg) = NEW(gdata);  /* freed in cleanup_subgs */
-           GD_ndim(subg) = GD_ndim(parent);
+           GD_ndim(subg) = GD_ndim(agroot(parent));
            LEVEL(subg) = LEVEL(parent) + 1;
            GPARENT(subg) = parent;
            addCluster(clist, subg);
@@ -1033,8 +1033,8 @@ static void fdp_init_graph(Agraph_t * g)
 {
     setEdgeType (g, EDGETYPE_LINE);
     GD_alg(g) = NEW(gdata);    /* freed in cleanup_graph */
-    GD_ndim(g) = late_int(g, agattr(g,AGRAPH, "dim", NULL), 2, 2);
-    Ndim = GD_ndim(g) = MIN(GD_ndim(g), MAXDIM);
+    GD_ndim(agroot(g)) = late_int(g, agattr(g,AGRAPH, "dim", NULL), 2, 2);
+    Ndim = GD_ndim(agroot(g)) = MIN(GD_ndim(agroot(g)), MAXDIM);
 
     mkClusters (g, NULL, g);
     fdp_initParams(g);