granicus.if.org Git - postgresql/blob - src/backend/access/gin/gininsert.c

   1 /*-------------------------------------------------------------------------
   2  *
   3  * gininsert.c
   4  *        insert routines for the postgres inverted index access method.
   5  *
   6  *
   7  * Portions Copyright (c) 1996-2013, PostgreSQL Global Development Group
   8  * Portions Copyright (c) 1994, Regents of the University of California
   9  *
  10  * IDENTIFICATION
  11  *                      src/backend/access/gin/gininsert.c
  12  *-------------------------------------------------------------------------
  13  */
  14
  15 #include "postgres.h"
  16
  17 #include "access/gin_private.h"
  18 #include "access/heapam_xlog.h"
  19 #include "catalog/index.h"
  20 #include "miscadmin.h"
  21 #include "storage/bufmgr.h"
  22 #include "storage/smgr.h"
  23 #include "storage/indexfsm.h"
  24 #include "utils/memutils.h"
  25 #include "utils/rel.h"
  26
  27
  28 typedef struct
  29 {
  30         GinState        ginstate;
  31         double          indtuples;
  32         GinStatsData buildStats;
  33         MemoryContext tmpCtx;
  34         MemoryContext funcCtx;
  35         BuildAccumulator accum;
  36 } GinBuildState;
  37
  38
  39 /*
  40  * Adds array of item pointers to tuple's posting list, or
  41  * creates posting tree and tuple pointing to tree in case
  42  * of not enough space.  Max size of tuple is defined in
  43  * GinFormTuple().      Returns a new, modified index tuple.
  44  * items[] must be in sorted order with no duplicates.
  45  */
  46 static IndexTuple
  47 addItemPointersToLeafTuple(GinState *ginstate,
  48                                                    IndexTuple old,
  49                                                    ItemPointerData *items, uint32 nitem,
  50                                                    GinStatsData *buildStats)
  51 {
  52         OffsetNumber attnum;
  53         Datum           key;
  54         GinNullCategory category;
  55         IndexTuple      res;
  56
  57         Assert(!GinIsPostingTree(old));
  58
  59         attnum = gintuple_get_attrnum(ginstate, old);
  60         key = gintuple_get_key(ginstate, old, &category);
  61
  62         /* try to build tuple with room for all the items */
  63         res = GinFormTuple(ginstate, attnum, key, category,
  64                                            NULL, nitem + GinGetNPosting(old),
  65                                            false);
  66
  67         if (res)
  68         {
  69                 /* good, small enough */
  70                 uint32          newnitem;
  71
  72                 /* fill in the posting list with union of old and new TIDs */
  73                 newnitem = ginMergeItemPointers(GinGetPosting(res),
  74                                                                                 GinGetPosting(old),
  75                                                                                 GinGetNPosting(old),
  76                                                                                 items, nitem);
  77                 /* merge might have eliminated some duplicate items */
  78                 GinShortenTuple(res, newnitem);
  79         }
  80         else
  81         {
  82                 /* posting list would be too big, convert to posting tree */
  83                 BlockNumber postingRoot;
  84
  85                 /*
  86                  * Initialize posting tree with the old tuple's posting list.  It's
  87                  * surely small enough to fit on one posting-tree page, and should
  88                  * already be in order with no duplicates.
  89                  */
  90                 postingRoot = createPostingTree(ginstate->index,
  91                                                                                 GinGetPosting(old),
  92                                                                                 GinGetNPosting(old),
  93                                                                                 buildStats);
  94
  95                 /* Now insert the TIDs-to-be-added into the posting tree */
  96                 ginInsertItemPointers(ginstate->index, postingRoot,
  97                                                           items, nitem,
  98                                                           buildStats);
  99
 100                 /* And build a new posting-tree-only result tuple */
 101                 res = GinFormTuple(ginstate, attnum, key, category, NULL, 0, true);
 102                 GinSetPostingTree(res, postingRoot);
 103         }
 104
 105         return res;
 106 }
 107
 108 /*
 109  * Build a fresh leaf tuple, either posting-list or posting-tree format
 110  * depending on whether the given items list will fit.
 111  * items[] must be in sorted order with no duplicates.
 112  *
 113  * This is basically the same logic as in addItemPointersToLeafTuple,
 114  * but working from slightly different input.
 115  */
 116 static IndexTuple
 117 buildFreshLeafTuple(GinState *ginstate,
 118                                         OffsetNumber attnum, Datum key, GinNullCategory category,
 119                                         ItemPointerData *items, uint32 nitem,
 120                                         GinStatsData *buildStats)
 121 {
 122         IndexTuple      res;
 123
 124         /* try to build a posting list tuple with all the items */
 125         res = GinFormTuple(ginstate, attnum, key, category,
 126                                            items, nitem, false);
 127
 128         if (!res)
 129         {
 130                 /* posting list would be too big, build posting tree */
 131                 BlockNumber postingRoot;
 132
 133                 /*
 134                  * Build posting-tree-only result tuple.  We do this first so as to
 135                  * fail quickly if the key is too big.
 136                  */
 137                 res = GinFormTuple(ginstate, attnum, key, category, NULL, 0, true);
 138
 139                 /*
 140                  * Initialize a new posting tree with the TIDs.
 141                  */
 142                 postingRoot = createPostingTree(ginstate->index, items, nitem,
 143                                                                                 buildStats);
 144
 145                 /* And save the root link in the result tuple */
 146                 GinSetPostingTree(res, postingRoot);
 147         }
 148
 149         return res;
 150 }
 151
 152 /*
 153  * Insert one or more heap TIDs associated with the given key value.
 154  * This will either add a single key entry, or enlarge a pre-existing entry.
 155  *
 156  * During an index build, buildStats is non-null and the counters
 157  * it contains should be incremented as needed.
 158  */
 159 void
 160 ginEntryInsert(GinState *ginstate,
 161                            OffsetNumber attnum, Datum key, GinNullCategory category,
 162                            ItemPointerData *items, uint32 nitem,
 163                            GinStatsData *buildStats)
 164 {
 165         GinBtreeData btree;
 166         GinBtreeEntryInsertData insertdata;
 167         GinBtreeStack *stack;
 168         IndexTuple      itup;
 169         Page            page;
 170
 171         insertdata.isDelete = FALSE;
 172
 173         /* During index build, count the to-be-inserted entry */
 174         if (buildStats)
 175                 buildStats->nEntries++;
 176
 177         ginPrepareEntryScan(&btree, attnum, key, category, ginstate);
 178
 179         stack = ginFindLeafPage(&btree, false);
 180         page = BufferGetPage(stack->buffer);
 181
 182         if (btree.findItem(&btree, stack))
 183         {
 184                 /* found pre-existing entry */
 185                 itup = (IndexTuple) PageGetItem(page, PageGetItemId(page, stack->off));
 186
 187                 if (GinIsPostingTree(itup))
 188                 {
 189                         /* add entries to existing posting tree */
 190                         BlockNumber rootPostingTree = GinGetPostingTree(itup);
 191
 192                         /* release all stack */
 193                         LockBuffer(stack->buffer, GIN_UNLOCK);
 194                         freeGinBtreeStack(stack);
 195
 196                         /* insert into posting tree */
 197                         ginInsertItemPointers(ginstate->index, rootPostingTree,
 198                                                                   items, nitem,
 199                                                                   buildStats);
 200                         return;
 201                 }
 202
 203                 /* modify an existing leaf entry */
 204                 itup = addItemPointersToLeafTuple(ginstate, itup,
 205                                                                                   items, nitem, buildStats);
 206
 207                 insertdata.isDelete = TRUE;
 208         }
 209         else
 210         {
 211                 /* no match, so construct a new leaf entry */
 212                 itup = buildFreshLeafTuple(ginstate, attnum, key, category,
 213                                                                    items, nitem, buildStats);
 214         }
 215
 216         /* Insert the new or modified leaf tuple */
 217         insertdata.entry = itup;
 218         ginInsertValue(&btree, stack, &insertdata, buildStats);
 219         pfree(itup);
 220 }
 221
 222 /*
 223  * Extract index entries for a single indexable item, and add them to the
 224  * BuildAccumulator's state.
 225  *
 226  * This function is used only during initial index creation.
 227  */
 228 static void
 229 ginHeapTupleBulkInsert(GinBuildState *buildstate, OffsetNumber attnum,
 230                                            Datum value, bool isNull,
 231                                            ItemPointer heapptr)
 232 {
 233         Datum      *entries;
 234         GinNullCategory *categories;
 235         int32           nentries;
 236         MemoryContext oldCtx;
 237
 238         oldCtx = MemoryContextSwitchTo(buildstate->funcCtx);
 239         entries = ginExtractEntries(buildstate->accum.ginstate, attnum,
 240                                                                 value, isNull,
 241                                                                 &nentries, &categories);
 242         MemoryContextSwitchTo(oldCtx);
 243
 244         ginInsertBAEntries(&buildstate->accum, heapptr, attnum,
 245                                            entries, categories, nentries);
 246
 247         buildstate->indtuples += nentries;
 248
 249         MemoryContextReset(buildstate->funcCtx);
 250 }
 251
 252 static void
 253 ginBuildCallback(Relation index, HeapTuple htup, Datum *values,
 254                                  bool *isnull, bool tupleIsAlive, void *state)
 255 {
 256         GinBuildState *buildstate = (GinBuildState *) state;
 257         MemoryContext oldCtx;
 258         int                     i;
 259
 260         oldCtx = MemoryContextSwitchTo(buildstate->tmpCtx);
 261
 262         for (i = 0; i < buildstate->ginstate.origTupdesc->natts; i++)
 263                 ginHeapTupleBulkInsert(buildstate, (OffsetNumber) (i + 1),
 264                                                            values[i], isnull[i],
 265                                                            &htup->t_self);
 266
 267         /* If we've maxed out our available memory, dump everything to the index */
 268         if (buildstate->accum.allocatedMemory >= maintenance_work_mem * 1024L)
 269         {
 270                 ItemPointerData *list;
 271                 Datum           key;
 272                 GinNullCategory category;
 273                 uint32          nlist;
 274                 OffsetNumber attnum;
 275
 276                 ginBeginBAScan(&buildstate->accum);
 277                 while ((list = ginGetBAEntry(&buildstate->accum,
 278                                                                   &attnum, &key, &category, &nlist)) != NULL)
 279                 {
 280                         /* there could be many entries, so be willing to abort here */
 281                         CHECK_FOR_INTERRUPTS();
 282                         ginEntryInsert(&buildstate->ginstate, attnum, key, category,
 283                                                    list, nlist, &buildstate->buildStats);
 284                 }
 285
 286                 MemoryContextReset(buildstate->tmpCtx);
 287                 ginInitBA(&buildstate->accum);
 288         }
 289
 290         MemoryContextSwitchTo(oldCtx);
 291 }
 292
 293 Datum
 294 ginbuild(PG_FUNCTION_ARGS)
 295 {
 296         Relation        heap = (Relation) PG_GETARG_POINTER(0);
 297         Relation        index = (Relation) PG_GETARG_POINTER(1);
 298         IndexInfo  *indexInfo = (IndexInfo *) PG_GETARG_POINTER(2);
 299         IndexBuildResult *result;
 300         double          reltuples;
 301         GinBuildState buildstate;
 302         Buffer          RootBuffer,
 303                                 MetaBuffer;
 304         ItemPointerData *list;
 305         Datum           key;
 306         GinNullCategory category;
 307         uint32          nlist;
 308         MemoryContext oldCtx;
 309         OffsetNumber attnum;
 310
 311         if (RelationGetNumberOfBlocks(index) != 0)
 312                 elog(ERROR, "index \"%s\" already contains data",
 313                          RelationGetRelationName(index));
 314
 315         initGinState(&buildstate.ginstate, index);
 316         buildstate.indtuples = 0;
 317         memset(&buildstate.buildStats, 0, sizeof(GinStatsData));
 318
 319         /* initialize the meta page */
 320         MetaBuffer = GinNewBuffer(index);
 321
 322         /* initialize the root page */
 323         RootBuffer = GinNewBuffer(index);
 324
 325         START_CRIT_SECTION();
 326         GinInitMetabuffer(MetaBuffer);
 327         MarkBufferDirty(MetaBuffer);
 328         GinInitBuffer(RootBuffer, GIN_LEAF);
 329         MarkBufferDirty(RootBuffer);
 330
 331         if (RelationNeedsWAL(index))
 332         {
 333                 XLogRecPtr      recptr;
 334                 XLogRecData rdata;
 335                 Page            page;
 336
 337                 rdata.buffer = InvalidBuffer;
 338                 rdata.data = (char *) &(index->rd_node);
 339                 rdata.len = sizeof(RelFileNode);
 340                 rdata.next = NULL;
 341
 342                 recptr = XLogInsert(RM_GIN_ID, XLOG_GIN_CREATE_INDEX, &rdata);
 343
 344                 page = BufferGetPage(RootBuffer);
 345                 PageSetLSN(page, recptr);
 346
 347                 page = BufferGetPage(MetaBuffer);
 348                 PageSetLSN(page, recptr);
 349         }
 350
 351         UnlockReleaseBuffer(MetaBuffer);
 352         UnlockReleaseBuffer(RootBuffer);
 353         END_CRIT_SECTION();
 354
 355         /* count the root as first entry page */
 356         buildstate.buildStats.nEntryPages++;
 357
 358         /*
 359          * create a temporary memory context that is reset once for each tuple
 360          * inserted into the index
 361          */
 362         buildstate.tmpCtx = AllocSetContextCreate(CurrentMemoryContext,
 363                                                                                           "Gin build temporary context",
 364                                                                                           ALLOCSET_DEFAULT_MINSIZE,
 365                                                                                           ALLOCSET_DEFAULT_INITSIZE,
 366                                                                                           ALLOCSET_DEFAULT_MAXSIZE);
 367
 368         buildstate.funcCtx = AllocSetContextCreate(buildstate.tmpCtx,
 369                                          "Gin build temporary context for user-defined function",
 370                                                                                            ALLOCSET_DEFAULT_MINSIZE,
 371                                                                                            ALLOCSET_DEFAULT_INITSIZE,
 372                                                                                            ALLOCSET_DEFAULT_MAXSIZE);
 373
 374         buildstate.accum.ginstate = &buildstate.ginstate;
 375         ginInitBA(&buildstate.accum);
 376
 377         /*
 378          * Do the heap scan.  We disallow sync scan here because dataPlaceToPage
 379          * prefers to receive tuples in TID order.
 380          */
 381         reltuples = IndexBuildHeapScan(heap, index, indexInfo, false,
 382                                                                    ginBuildCallback, (void *) &buildstate);
 383
 384         /* dump remaining entries to the index */
 385         oldCtx = MemoryContextSwitchTo(buildstate.tmpCtx);
 386         ginBeginBAScan(&buildstate.accum);
 387         while ((list = ginGetBAEntry(&buildstate.accum,
 388                                                                  &attnum, &key, &category, &nlist)) != NULL)
 389         {
 390                 /* there could be many entries, so be willing to abort here */
 391                 CHECK_FOR_INTERRUPTS();
 392                 ginEntryInsert(&buildstate.ginstate, attnum, key, category,
 393                                            list, nlist, &buildstate.buildStats);
 394         }
 395         MemoryContextSwitchTo(oldCtx);
 396
 397         MemoryContextDelete(buildstate.tmpCtx);
 398
 399         /*
 400          * Update metapage stats
 401          */
 402         buildstate.buildStats.nTotalPages = RelationGetNumberOfBlocks(index);
 403         ginUpdateStats(index, &buildstate.buildStats);
 404
 405         /*
 406          * Return statistics
 407          */
 408         result = (IndexBuildResult *) palloc(sizeof(IndexBuildResult));
 409
 410         result->heap_tuples = reltuples;
 411         result->index_tuples = buildstate.indtuples;
 412
 413         PG_RETURN_POINTER(result);
 414 }
 415
 416 /*
 417  *      ginbuildempty() -- build an empty gin index in the initialization fork
 418  */
 419 Datum
 420 ginbuildempty(PG_FUNCTION_ARGS)
 421 {
 422         Relation        index = (Relation) PG_GETARG_POINTER(0);
 423         Buffer          RootBuffer,
 424                                 MetaBuffer;
 425
 426         /* An empty GIN index has two pages. */
 427         MetaBuffer =
 428                 ReadBufferExtended(index, INIT_FORKNUM, P_NEW, RBM_NORMAL, NULL);
 429         LockBuffer(MetaBuffer, BUFFER_LOCK_EXCLUSIVE);
 430         RootBuffer =
 431                 ReadBufferExtended(index, INIT_FORKNUM, P_NEW, RBM_NORMAL, NULL);
 432         LockBuffer(RootBuffer, BUFFER_LOCK_EXCLUSIVE);
 433
 434         /* Initialize and xlog metabuffer and root buffer. */
 435         START_CRIT_SECTION();
 436         GinInitMetabuffer(MetaBuffer);
 437         MarkBufferDirty(MetaBuffer);
 438         log_newpage_buffer(MetaBuffer);
 439         GinInitBuffer(RootBuffer, GIN_LEAF);
 440         MarkBufferDirty(RootBuffer);
 441         log_newpage_buffer(RootBuffer);
 442         END_CRIT_SECTION();
 443
 444         /* Unlock and release the buffers. */
 445         UnlockReleaseBuffer(MetaBuffer);
 446         UnlockReleaseBuffer(RootBuffer);
 447
 448         PG_RETURN_VOID();
 449 }
 450
 451 /*
 452  * Insert index entries for a single indexable item during "normal"
 453  * (non-fast-update) insertion
 454  */
 455 static void
 456 ginHeapTupleInsert(GinState *ginstate, OffsetNumber attnum,
 457                                    Datum value, bool isNull,
 458                                    ItemPointer item)
 459 {
 460         Datum      *entries;
 461         GinNullCategory *categories;
 462         int32           i,
 463                                 nentries;
 464
 465         entries = ginExtractEntries(ginstate, attnum, value, isNull,
 466                                                                 &nentries, &categories);
 467
 468         for (i = 0; i < nentries; i++)
 469                 ginEntryInsert(ginstate, attnum, entries[i], categories[i],
 470                                            item, 1, NULL);
 471 }
 472
 473 Datum
 474 gininsert(PG_FUNCTION_ARGS)
 475 {
 476         Relation        index = (Relation) PG_GETARG_POINTER(0);
 477         Datum      *values = (Datum *) PG_GETARG_POINTER(1);
 478         bool       *isnull = (bool *) PG_GETARG_POINTER(2);
 479         ItemPointer ht_ctid = (ItemPointer) PG_GETARG_POINTER(3);
 480
 481 #ifdef NOT_USED
 482         Relation        heapRel = (Relation) PG_GETARG_POINTER(4);
 483         IndexUniqueCheck checkUnique = (IndexUniqueCheck) PG_GETARG_INT32(5);
 484 #endif
 485         GinState        ginstate;
 486         MemoryContext oldCtx;
 487         MemoryContext insertCtx;
 488         int                     i;
 489
 490         insertCtx = AllocSetContextCreate(CurrentMemoryContext,
 491                                                                           "Gin insert temporary context",
 492                                                                           ALLOCSET_DEFAULT_MINSIZE,
 493                                                                           ALLOCSET_DEFAULT_INITSIZE,
 494                                                                           ALLOCSET_DEFAULT_MAXSIZE);
 495
 496         oldCtx = MemoryContextSwitchTo(insertCtx);
 497
 498         initGinState(&ginstate, index);
 499
 500         if (GinGetUseFastUpdate(index))
 501         {
 502                 GinTupleCollector collector;
 503
 504                 memset(&collector, 0, sizeof(GinTupleCollector));
 505
 506                 for (i = 0; i < ginstate.origTupdesc->natts; i++)
 507                         ginHeapTupleFastCollect(&ginstate, &collector,
 508                                                                         (OffsetNumber) (i + 1),
 509                                                                         values[i], isnull[i],
 510                                                                         ht_ctid);
 511
 512                 ginHeapTupleFastInsert(&ginstate, &collector);
 513         }
 514         else
 515         {
 516                 for (i = 0; i < ginstate.origTupdesc->natts; i++)
 517                         ginHeapTupleInsert(&ginstate, (OffsetNumber) (i + 1),
 518                                                            values[i], isnull[i],
 519                                                            ht_ctid);
 520         }
 521
 522         MemoryContextSwitchTo(oldCtx);
 523         MemoryContextDelete(insertCtx);
 524
 525         PG_RETURN_BOOL(false);
 526 }