From: Teodor Sigaev <teodor@sigaev.ru>
Date: Wed, 2 Sep 2015 17:08:58 +0000 (+0300)
Subject: Allow usage of huge maintenance_work_mem for GIN build.
X-Git-Tag: REL9_6_BETA1~1417
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=30bb26b5e04cff911db960801e32e5f57045eb61;p=postgresql

Allow usage of huge maintenance_work_mem for GIN build.

Currently, in-memory posting list during GIN build process is limited 1GB
because of using repalloc. The patch replaces call of repalloc to repalloc_huge.
It increases limit of posting list from 180 millions
(1GB / sizeof(ItemPointerData)) to 4 billions limited by maxcount/count fields
in GinEntryAccumulator and subsequent calls. Check added.

Also, fix accounting of allocatedMemory during build to prevent integer
overflow with maintenance_work_mem > 4GB.

Robert Abraham <robert.abraham86@googlemail.com> with additions by me
---

diff --git a/src/backend/access/gin/ginbulk.c b/src/backend/access/gin/ginbulk.c
index e40e5be36b..5a8546998d 100644
--- a/src/backend/access/gin/ginbulk.c
+++ b/src/backend/access/gin/ginbulk.c
@@ -14,6 +14,8 @@
 
 #include "postgres.h"
 
+#include <limits.h>
+
 #include "access/gin_private.h"
 #include "utils/datum.h"
 #include "utils/memutils.h"
@@ -36,10 +38,16 @@ ginCombineData(RBNode *existing, const RBNode *newdata, void *arg)
 	 */
 	if (eo->count >= eo->maxcount)
 	{
+		if (eo->maxcount > INT_MAX)
+			ereport(ERROR,
+					(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
+					 errmsg("posting list is too long"),
+					 errhint("Reduce maintenance_work_mem")));
+
 		accum->allocatedMemory -= GetMemoryChunkSpace(eo->list);
 		eo->maxcount *= 2;
 		eo->list = (ItemPointerData *)
-			repalloc(eo->list, sizeof(ItemPointerData) * eo->maxcount);
+			repalloc_huge(eo->list, sizeof(ItemPointerData) * eo->maxcount);
 		accum->allocatedMemory += GetMemoryChunkSpace(eo->list);
 	}
 
diff --git a/src/backend/access/gin/ginfast.c b/src/backend/access/gin/ginfast.c
index c5732c3dc0..54e593d8a9 100644
--- a/src/backend/access/gin/ginfast.c
+++ b/src/backend/access/gin/ginfast.c
@@ -814,7 +814,7 @@ ginInsertCleanup(GinState *ginstate,
 		 */
 		if (GinPageGetOpaque(page)->rightlink == InvalidBlockNumber ||
 			(GinPageHasFullRow(page) &&
-			 (accum.allocatedMemory >= maintenance_work_mem * 1024L)))
+			 (accum.allocatedMemory >= (Size)maintenance_work_mem * 1024L)))
 		{
 			ItemPointerData *list;
 			uint32		nlist;
diff --git a/src/backend/access/gin/gininsert.c b/src/backend/access/gin/gininsert.c
index fc44f0205d..49e91853c3 100644
--- a/src/backend/access/gin/gininsert.c
+++ b/src/backend/access/gin/gininsert.c
@@ -281,7 +281,7 @@ ginBuildCallback(Relation index, HeapTuple htup, Datum *values,
 							   &htup->t_self);
 
 	/* If we've maxed out our available memory, dump everything to the index */
-	if (buildstate->accum.allocatedMemory >= maintenance_work_mem * 1024L)
+	if (buildstate->accum.allocatedMemory >= (Size)maintenance_work_mem * 1024L)
 	{
 		ItemPointerData *list;
 		Datum		key;
diff --git a/src/include/access/gin_private.h b/src/include/access/gin_private.h
index 5095fc1dba..acbe36a741 100644
--- a/src/include/access/gin_private.h
+++ b/src/include/access/gin_private.h
@@ -903,7 +903,7 @@ typedef struct GinEntryAccumulator
 typedef struct
 {
 	GinState   *ginstate;
-	long		allocatedMemory;
+	Size		allocatedMemory;
 	GinEntryAccumulator *entryallocator;
 	uint32		eas_used;
 	RBTree	   *tree;