1 /*-------------------------------------------------------------------------
4 * insert routines for the postgres inverted index access method.
7 * Portions Copyright (c) 1996-2015, PostgreSQL Global Development Group
8 * Portions Copyright (c) 1994, Regents of the University of California
11 * src/backend/access/gin/gininsert.c
12 *-------------------------------------------------------------------------
17 #include "access/gin_private.h"
18 #include "access/xloginsert.h"
19 #include "catalog/index.h"
20 #include "miscadmin.h"
21 #include "storage/bufmgr.h"
22 #include "storage/smgr.h"
23 #include "storage/indexfsm.h"
24 #include "utils/memutils.h"
25 #include "utils/rel.h"
32 GinStatsData buildStats;
34 MemoryContext funcCtx;
35 BuildAccumulator accum;
40 * Adds array of item pointers to tuple's posting list, or
41 * creates posting tree and tuple pointing to tree in case
42 * of not enough space. Max size of tuple is defined in
43 * GinFormTuple(). Returns a new, modified index tuple.
44 * items[] must be in sorted order with no duplicates.
47 addItemPointersToLeafTuple(GinState *ginstate,
49 ItemPointerData *items, uint32 nitem,
50 GinStatsData *buildStats)
54 GinNullCategory category;
56 ItemPointerData *newItems,
60 GinPostingList *compressedList;
62 Assert(!GinIsPostingTree(old));
64 attnum = gintuple_get_attrnum(ginstate, old);
65 key = gintuple_get_key(ginstate, old, &category);
67 /* merge the old and new posting lists */
68 oldItems = ginReadTuple(ginstate, attnum, old, &oldNPosting);
70 newItems = ginMergeItemPointers(items, nitem,
71 oldItems, oldNPosting,
74 /* Compress the posting list, and try to a build tuple with room for it */
76 compressedList = ginCompressPostingList(newItems, newNPosting, GinMaxItemSize,
81 res = GinFormTuple(ginstate, attnum, key, category,
82 (char *) compressedList,
83 SizeOfGinPostingList(compressedList),
86 pfree(compressedList);
90 /* posting list would be too big, convert to posting tree */
91 BlockNumber postingRoot;
94 * Initialize posting tree with the old tuple's posting list. It's
95 * surely small enough to fit on one posting-tree page, and should
96 * already be in order with no duplicates.
98 postingRoot = createPostingTree(ginstate->index,
103 /* Now insert the TIDs-to-be-added into the posting tree */
104 ginInsertItemPointers(ginstate->index, postingRoot,
108 /* And build a new posting-tree-only result tuple */
109 res = GinFormTuple(ginstate, attnum, key, category, NULL, 0, 0, true);
110 GinSetPostingTree(res, postingRoot);
118 * Build a fresh leaf tuple, either posting-list or posting-tree format
119 * depending on whether the given items list will fit.
120 * items[] must be in sorted order with no duplicates.
122 * This is basically the same logic as in addItemPointersToLeafTuple,
123 * but working from slightly different input.
126 buildFreshLeafTuple(GinState *ginstate,
127 OffsetNumber attnum, Datum key, GinNullCategory category,
128 ItemPointerData *items, uint32 nitem,
129 GinStatsData *buildStats)
131 IndexTuple res = NULL;
132 GinPostingList *compressedList;
134 /* try to build a posting list tuple with all the items */
135 compressedList = ginCompressPostingList(items, nitem, GinMaxItemSize, NULL);
138 res = GinFormTuple(ginstate, attnum, key, category,
139 (char *) compressedList,
140 SizeOfGinPostingList(compressedList),
142 pfree(compressedList);
146 /* posting list would be too big, build posting tree */
147 BlockNumber postingRoot;
150 * Build posting-tree-only result tuple. We do this first so as to
151 * fail quickly if the key is too big.
153 res = GinFormTuple(ginstate, attnum, key, category, NULL, 0, 0, true);
156 * Initialize a new posting tree with the TIDs.
158 postingRoot = createPostingTree(ginstate->index, items, nitem,
161 /* And save the root link in the result tuple */
162 GinSetPostingTree(res, postingRoot);
169 * Insert one or more heap TIDs associated with the given key value.
170 * This will either add a single key entry, or enlarge a pre-existing entry.
172 * During an index build, buildStats is non-null and the counters
173 * it contains should be incremented as needed.
176 ginEntryInsert(GinState *ginstate,
177 OffsetNumber attnum, Datum key, GinNullCategory category,
178 ItemPointerData *items, uint32 nitem,
179 GinStatsData *buildStats)
182 GinBtreeEntryInsertData insertdata;
183 GinBtreeStack *stack;
187 insertdata.isDelete = FALSE;
189 /* During index build, count the to-be-inserted entry */
191 buildStats->nEntries++;
193 ginPrepareEntryScan(&btree, attnum, key, category, ginstate);
195 stack = ginFindLeafPage(&btree, false);
196 page = BufferGetPage(stack->buffer);
198 if (btree.findItem(&btree, stack))
200 /* found pre-existing entry */
201 itup = (IndexTuple) PageGetItem(page, PageGetItemId(page, stack->off));
203 if (GinIsPostingTree(itup))
205 /* add entries to existing posting tree */
206 BlockNumber rootPostingTree = GinGetPostingTree(itup);
208 /* release all stack */
209 LockBuffer(stack->buffer, GIN_UNLOCK);
210 freeGinBtreeStack(stack);
212 /* insert into posting tree */
213 ginInsertItemPointers(ginstate->index, rootPostingTree,
219 /* modify an existing leaf entry */
220 itup = addItemPointersToLeafTuple(ginstate, itup,
221 items, nitem, buildStats);
223 insertdata.isDelete = TRUE;
227 /* no match, so construct a new leaf entry */
228 itup = buildFreshLeafTuple(ginstate, attnum, key, category,
229 items, nitem, buildStats);
232 /* Insert the new or modified leaf tuple */
233 insertdata.entry = itup;
234 ginInsertValue(&btree, stack, &insertdata, buildStats);
239 * Extract index entries for a single indexable item, and add them to the
240 * BuildAccumulator's state.
242 * This function is used only during initial index creation.
245 ginHeapTupleBulkInsert(GinBuildState *buildstate, OffsetNumber attnum,
246 Datum value, bool isNull,
250 GinNullCategory *categories;
252 MemoryContext oldCtx;
254 oldCtx = MemoryContextSwitchTo(buildstate->funcCtx);
255 entries = ginExtractEntries(buildstate->accum.ginstate, attnum,
257 &nentries, &categories);
258 MemoryContextSwitchTo(oldCtx);
260 ginInsertBAEntries(&buildstate->accum, heapptr, attnum,
261 entries, categories, nentries);
263 buildstate->indtuples += nentries;
265 MemoryContextReset(buildstate->funcCtx);
269 ginBuildCallback(Relation index, HeapTuple htup, Datum *values,
270 bool *isnull, bool tupleIsAlive, void *state)
272 GinBuildState *buildstate = (GinBuildState *) state;
273 MemoryContext oldCtx;
276 oldCtx = MemoryContextSwitchTo(buildstate->tmpCtx);
278 for (i = 0; i < buildstate->ginstate.origTupdesc->natts; i++)
279 ginHeapTupleBulkInsert(buildstate, (OffsetNumber) (i + 1),
280 values[i], isnull[i],
283 /* If we've maxed out our available memory, dump everything to the index */
284 if (buildstate->accum.allocatedMemory >= maintenance_work_mem * 1024L)
286 ItemPointerData *list;
288 GinNullCategory category;
292 ginBeginBAScan(&buildstate->accum);
293 while ((list = ginGetBAEntry(&buildstate->accum,
294 &attnum, &key, &category, &nlist)) != NULL)
296 /* there could be many entries, so be willing to abort here */
297 CHECK_FOR_INTERRUPTS();
298 ginEntryInsert(&buildstate->ginstate, attnum, key, category,
299 list, nlist, &buildstate->buildStats);
302 MemoryContextReset(buildstate->tmpCtx);
303 ginInitBA(&buildstate->accum);
306 MemoryContextSwitchTo(oldCtx);
310 ginbuild(PG_FUNCTION_ARGS)
312 Relation heap = (Relation) PG_GETARG_POINTER(0);
313 Relation index = (Relation) PG_GETARG_POINTER(1);
314 IndexInfo *indexInfo = (IndexInfo *) PG_GETARG_POINTER(2);
315 IndexBuildResult *result;
317 GinBuildState buildstate;
320 ItemPointerData *list;
322 GinNullCategory category;
324 MemoryContext oldCtx;
327 if (RelationGetNumberOfBlocks(index) != 0)
328 elog(ERROR, "index \"%s\" already contains data",
329 RelationGetRelationName(index));
331 initGinState(&buildstate.ginstate, index);
332 buildstate.indtuples = 0;
333 memset(&buildstate.buildStats, 0, sizeof(GinStatsData));
335 /* initialize the meta page */
336 MetaBuffer = GinNewBuffer(index);
338 /* initialize the root page */
339 RootBuffer = GinNewBuffer(index);
341 START_CRIT_SECTION();
342 GinInitMetabuffer(MetaBuffer);
343 MarkBufferDirty(MetaBuffer);
344 GinInitBuffer(RootBuffer, GIN_LEAF);
345 MarkBufferDirty(RootBuffer);
347 if (RelationNeedsWAL(index))
353 XLogRegisterBuffer(0, MetaBuffer, REGBUF_WILL_INIT);
354 XLogRegisterBuffer(1, RootBuffer, REGBUF_WILL_INIT);
356 recptr = XLogInsert(RM_GIN_ID, XLOG_GIN_CREATE_INDEX);
358 page = BufferGetPage(RootBuffer);
359 PageSetLSN(page, recptr);
361 page = BufferGetPage(MetaBuffer);
362 PageSetLSN(page, recptr);
365 UnlockReleaseBuffer(MetaBuffer);
366 UnlockReleaseBuffer(RootBuffer);
369 /* count the root as first entry page */
370 buildstate.buildStats.nEntryPages++;
373 * create a temporary memory context that is reset once for each tuple
374 * inserted into the index
376 buildstate.tmpCtx = AllocSetContextCreate(CurrentMemoryContext,
377 "Gin build temporary context",
378 ALLOCSET_DEFAULT_MINSIZE,
379 ALLOCSET_DEFAULT_INITSIZE,
380 ALLOCSET_DEFAULT_MAXSIZE);
382 buildstate.funcCtx = AllocSetContextCreate(buildstate.tmpCtx,
383 "Gin build temporary context for user-defined function",
384 ALLOCSET_DEFAULT_MINSIZE,
385 ALLOCSET_DEFAULT_INITSIZE,
386 ALLOCSET_DEFAULT_MAXSIZE);
388 buildstate.accum.ginstate = &buildstate.ginstate;
389 ginInitBA(&buildstate.accum);
392 * Do the heap scan. We disallow sync scan here because dataPlaceToPage
393 * prefers to receive tuples in TID order.
395 reltuples = IndexBuildHeapScan(heap, index, indexInfo, false,
396 ginBuildCallback, (void *) &buildstate);
398 /* dump remaining entries to the index */
399 oldCtx = MemoryContextSwitchTo(buildstate.tmpCtx);
400 ginBeginBAScan(&buildstate.accum);
401 while ((list = ginGetBAEntry(&buildstate.accum,
402 &attnum, &key, &category, &nlist)) != NULL)
404 /* there could be many entries, so be willing to abort here */
405 CHECK_FOR_INTERRUPTS();
406 ginEntryInsert(&buildstate.ginstate, attnum, key, category,
407 list, nlist, &buildstate.buildStats);
409 MemoryContextSwitchTo(oldCtx);
411 MemoryContextDelete(buildstate.tmpCtx);
414 * Update metapage stats
416 buildstate.buildStats.nTotalPages = RelationGetNumberOfBlocks(index);
417 ginUpdateStats(index, &buildstate.buildStats);
422 result = (IndexBuildResult *) palloc(sizeof(IndexBuildResult));
424 result->heap_tuples = reltuples;
425 result->index_tuples = buildstate.indtuples;
427 PG_RETURN_POINTER(result);
431 * ginbuildempty() -- build an empty gin index in the initialization fork
434 ginbuildempty(PG_FUNCTION_ARGS)
436 Relation index = (Relation) PG_GETARG_POINTER(0);
440 /* An empty GIN index has two pages. */
442 ReadBufferExtended(index, INIT_FORKNUM, P_NEW, RBM_NORMAL, NULL);
443 LockBuffer(MetaBuffer, BUFFER_LOCK_EXCLUSIVE);
445 ReadBufferExtended(index, INIT_FORKNUM, P_NEW, RBM_NORMAL, NULL);
446 LockBuffer(RootBuffer, BUFFER_LOCK_EXCLUSIVE);
448 /* Initialize and xlog metabuffer and root buffer. */
449 START_CRIT_SECTION();
450 GinInitMetabuffer(MetaBuffer);
451 MarkBufferDirty(MetaBuffer);
452 log_newpage_buffer(MetaBuffer, false);
453 GinInitBuffer(RootBuffer, GIN_LEAF);
454 MarkBufferDirty(RootBuffer);
455 log_newpage_buffer(RootBuffer, false);
458 /* Unlock and release the buffers. */
459 UnlockReleaseBuffer(MetaBuffer);
460 UnlockReleaseBuffer(RootBuffer);
466 * Insert index entries for a single indexable item during "normal"
467 * (non-fast-update) insertion
470 ginHeapTupleInsert(GinState *ginstate, OffsetNumber attnum,
471 Datum value, bool isNull,
475 GinNullCategory *categories;
479 entries = ginExtractEntries(ginstate, attnum, value, isNull,
480 &nentries, &categories);
482 for (i = 0; i < nentries; i++)
483 ginEntryInsert(ginstate, attnum, entries[i], categories[i],
488 gininsert(PG_FUNCTION_ARGS)
490 Relation index = (Relation) PG_GETARG_POINTER(0);
491 Datum *values = (Datum *) PG_GETARG_POINTER(1);
492 bool *isnull = (bool *) PG_GETARG_POINTER(2);
493 ItemPointer ht_ctid = (ItemPointer) PG_GETARG_POINTER(3);
496 Relation heapRel = (Relation) PG_GETARG_POINTER(4);
497 IndexUniqueCheck checkUnique = (IndexUniqueCheck) PG_GETARG_INT32(5);
500 MemoryContext oldCtx;
501 MemoryContext insertCtx;
504 insertCtx = AllocSetContextCreate(CurrentMemoryContext,
505 "Gin insert temporary context",
506 ALLOCSET_DEFAULT_MINSIZE,
507 ALLOCSET_DEFAULT_INITSIZE,
508 ALLOCSET_DEFAULT_MAXSIZE);
510 oldCtx = MemoryContextSwitchTo(insertCtx);
512 initGinState(&ginstate, index);
514 if (GinGetUseFastUpdate(index))
516 GinTupleCollector collector;
518 memset(&collector, 0, sizeof(GinTupleCollector));
520 for (i = 0; i < ginstate.origTupdesc->natts; i++)
521 ginHeapTupleFastCollect(&ginstate, &collector,
522 (OffsetNumber) (i + 1),
523 values[i], isnull[i],
526 ginHeapTupleFastInsert(&ginstate, &collector);
530 for (i = 0; i < ginstate.origTupdesc->natts; i++)
531 ginHeapTupleInsert(&ginstate, (OffsetNumber) (i + 1),
532 values[i], isnull[i],
536 MemoryContextSwitchTo(oldCtx);
537 MemoryContextDelete(insertCtx);
539 PG_RETURN_BOOL(false);