From 854823fa334cb826eed50da751801d0693b10173 Mon Sep 17 00:00:00 2001 From: Teodor Sigaev Date: Fri, 22 Dec 2017 13:33:16 +0300 Subject: [PATCH] Add optional compression method to SP-GiST Patch allows to have different types of column and value stored in leaf tuples of SP-GiST. The main application of feature is to transform complex column type to simple indexed type or for truncating too long value, transformation could be lossy. Simple example: polygons are converted to their bounding boxes, this opclass follows. Authors: me, Heikki Linnakangas, Alexander Korotkov, Nikita Glukhov Reviewed-By: all authors + Darafei Praliaskouski Discussions: https://www.postgresql.org/message-id/5447B3FF.2080406@sigaev.ru https://www.postgresql.org/message-id/flat/54907069.1030506@sigaev.ru#54907069.1030506@sigaev.ru --- doc/src/sgml/spgist.sgml | 92 +++++++++++++++++++------ src/backend/access/spgist/spgdoinsert.c | 37 ++++++++-- src/backend/access/spgist/spgscan.c | 6 +- src/backend/access/spgist/spgutils.c | 21 +++++- src/backend/access/spgist/spgvalidate.c | 50 +++++++++++++- src/include/access/spgist.h | 5 +- src/include/access/spgist_private.h | 8 ++- 7 files changed, 182 insertions(+), 37 deletions(-) diff --git a/doc/src/sgml/spgist.sgml b/doc/src/sgml/spgist.sgml index 139c8ed8f7..b4a8be476e 100644 --- a/doc/src/sgml/spgist.sgml +++ b/doc/src/sgml/spgist.sgml @@ -240,20 +240,22 @@ There are five user-defined methods that an index operator class for - SP-GiST must provide. All five follow the convention - of accepting two internal arguments, the first of which is a - pointer to a C struct containing input values for the support method, - while the second argument is a pointer to a C struct where output values - must be placed. Four of the methods just return void, since - all their results appear in the output struct; but + SP-GiST must provide, and one is optional. All five + mandatory methods follow the convention of accepting two internal + arguments, the first of which is a pointer to a C struct containing input + values for the support method, while the second argument is a pointer to a + C struct where output values must be placed. Four of the mandatory methods just + return void, since all their results appear in the output struct; but leaf_consistent additionally returns a boolean result. The methods must not modify any fields of their input structs. In all cases, the output struct is initialized to zeroes before calling the - user-defined method. + user-defined method. Optional sixth method compress + accepts datum to be indexed as the only argument and returns value suitable + for physical storage in leaf tuple. - The five user-defined methods are: + The five mandatory user-defined methods are: @@ -283,6 +285,7 @@ typedef struct spgConfigOut { Oid prefixType; /* Data type of inner-tuple prefixes */ Oid labelType; /* Data type of inner-tuple node labels */ + Oid leafType; /* Data type of leaf-tuple values */ bool canReturnData; /* Opclass can reconstruct original data */ bool longValuesOK; /* Opclass can cope with values > 1 page */ } spgConfigOut; @@ -305,6 +308,22 @@ typedef struct spgConfigOut class is capable of segmenting long values by repeated suffixing (see ). + + + leafType is typically the same as + attType. For the reasons of backward + compatibility, method config can + leave leafType uninitialized; that would + give the same effect as setting leafType equal + to attType. When attType + and leafType are different, then optional + method compress must be provided. + Method compress is responsible + for transformation of datums to be indexed from attType + to leafType. + Note: both consistent functions will get scankeys + unchanged, without transformation using compress. + @@ -380,10 +399,16 @@ typedef struct spgChooseOut } spgChooseOut; - datum is the original datum that was to be inserted - into the index. - leafDatum is initially the same as - datum, but can change at lower levels of the tree + datum is the original datum of + spgConfigIn.attType + type that was to be inserted into the index. + leafDatum is a value of + spgConfigOut.leafType + type which is initially an result of method + compress applied to datum + when method compress is provided, or same value as + datum otherwise. + leafDatum can change at lower levels of the tree if the choose or picksplit methods change it. When the insertion search reaches a leaf page, the current value of leafDatum is what will be stored @@ -418,7 +443,7 @@ typedef struct spgChooseOut Set levelAdd to the increment in level caused by descending through that node, or leave it as zero if the operator class does not use levels. - Set restDatum to equal datum + Set restDatum to equal leafDatum if the operator class does not modify datums from one level to the next, or otherwise set it to the modified value to be used as leafDatum at the next level. @@ -509,7 +534,9 @@ typedef struct spgPickSplitOut nTuples is the number of leaf tuples provided. - datums is an array of their datum values. + datums is an array of their datum values of + spgConfigOut.leafType + type. level is the current level that all the leaf tuples share, which will become the level of the new inner tuple. @@ -624,7 +651,8 @@ typedef struct spgInnerConsistentOut reconstructedValue is the value reconstructed for the parent tuple; it is (Datum) 0 at the root level or if the inner_consistent function did not provide a value at the - parent level. + parent level. reconstructedValue is always of + spgConfigOut.leafType type. traversalValue is a pointer to any traverse data passed down from the previous call of inner_consistent on the parent index tuple, or NULL at the root level. @@ -659,6 +687,7 @@ typedef struct spgInnerConsistentOut necessarily so, so an array is used.) If value reconstruction is needed, set reconstructedValues to an array of the values + of spgConfigOut.leafType type reconstructed for each child node to be visited; otherwise, leave reconstructedValues as NULL. If it is desired to pass down additional out-of-band information @@ -730,7 +759,8 @@ typedef struct spgLeafConsistentOut reconstructedValue is the value reconstructed for the parent tuple; it is (Datum) 0 at the root level or if the inner_consistent function did not provide a value at the - parent level. + parent level. reconstructedValue is always of + spgConfigOut.leafType type. traversalValue is a pointer to any traverse data passed down from the previous call of inner_consistent on the parent index tuple, or NULL at the root level. @@ -739,16 +769,18 @@ typedef struct spgLeafConsistentOut returnData is true if reconstructed data is required for this query; this will only be so if the config function asserted canReturnData. - leafDatum is the key value stored in the current - leaf tuple. + leafDatum is the key value of + spgConfigOut.leafType + stored in the current leaf tuple. The function must return true if the leaf tuple matches the query, or false if not. In the true case, if returnData is true then - leafValue must be set to the value originally supplied - to be indexed for this leaf tuple. Also, + leafValue must be set to the value of + spgConfigIn.attType type + originally supplied to be indexed for this leaf tuple. Also, recheck may be set to true if the match is uncertain and so the operator(s) must be re-applied to the actual heap tuple to verify the match. @@ -757,6 +789,26 @@ typedef struct spgLeafConsistentOut + + The optional user-defined method is: + + + + + Datum compress(Datum in) + + + Converts the data item into a format suitable for physical storage in + a leaf tuple of index page. It accepts + spgConfigIn.attType + value and return + spgConfigOut.leafType + value. Output value should not be toasted. + + + + + All the SP-GiST support methods are normally called in a short-lived memory context; that is, CurrentMemoryContext will be reset diff --git a/src/backend/access/spgist/spgdoinsert.c b/src/backend/access/spgist/spgdoinsert.c index a5f4c4059c..a8cb8c7bdc 100644 --- a/src/backend/access/spgist/spgdoinsert.c +++ b/src/backend/access/spgist/spgdoinsert.c @@ -1906,14 +1906,37 @@ spgdoinsert(Relation index, SpGistState *state, procinfo = index_getprocinfo(index, 1, SPGIST_CHOOSE_PROC); /* - * Since we don't use index_form_tuple in this AM, we have to make sure + * Prepare the leaf datum to insert. + * + * If an optional "compress" method is provided, then call it to form + * the leaf datum from the input datum. Otherwise store the input datum as + * is. Since we don't use index_form_tuple in this AM, we have to make sure * value to be inserted is not toasted; FormIndexDatum doesn't guarantee - * that. + * that. But we assume the "compress" method to return an untoasted value. */ - if (!isnull && state->attType.attlen == -1) - datum = PointerGetDatum(PG_DETOAST_DATUM(datum)); + if (!isnull) + { + if (OidIsValid(index_getprocid(index, 1, SPGIST_COMPRESS_PROC))) + { + FmgrInfo *compressProcinfo = NULL; + + compressProcinfo = index_getprocinfo(index, 1, SPGIST_COMPRESS_PROC); + leafDatum = FunctionCall1Coll(compressProcinfo, + index->rd_indcollation[0], + datum); + } + else + { + Assert(state->attLeafType.type == state->attType.type); - leafDatum = datum; + if (state->attType.attlen == -1) + leafDatum = PointerGetDatum(PG_DETOAST_DATUM(datum)); + else + leafDatum = datum; + } + } + else + leafDatum = (Datum) 0; /* * Compute space needed for a leaf tuple containing the given datum. @@ -1923,7 +1946,7 @@ spgdoinsert(Relation index, SpGistState *state, */ if (!isnull) leafSize = SGLTHDRSZ + sizeof(ItemIdData) + - SpGistGetTypeSize(&state->attType, leafDatum); + SpGistGetTypeSize(&state->attLeafType, leafDatum); else leafSize = SGDTSIZE + sizeof(ItemIdData); @@ -2138,7 +2161,7 @@ spgdoinsert(Relation index, SpGistState *state, { leafDatum = out.result.matchNode.restDatum; leafSize = SGLTHDRSZ + sizeof(ItemIdData) + - SpGistGetTypeSize(&state->attType, leafDatum); + SpGistGetTypeSize(&state->attLeafType, leafDatum); } /* diff --git a/src/backend/access/spgist/spgscan.c b/src/backend/access/spgist/spgscan.c index 7965b5846d..c64a174143 100644 --- a/src/backend/access/spgist/spgscan.c +++ b/src/backend/access/spgist/spgscan.c @@ -40,7 +40,7 @@ typedef struct ScanStackEntry static void freeScanStackEntry(SpGistScanOpaque so, ScanStackEntry *stackEntry) { - if (!so->state.attType.attbyval && + if (!so->state.attLeafType.attbyval && DatumGetPointer(stackEntry->reconstructedValue) != NULL) pfree(DatumGetPointer(stackEntry->reconstructedValue)); if (stackEntry->traversalValue) @@ -527,8 +527,8 @@ redirect: if (out.reconstructedValues) newEntry->reconstructedValue = datumCopy(out.reconstructedValues[i], - so->state.attType.attbyval, - so->state.attType.attlen); + so->state.attLeafType.attbyval, + so->state.attLeafType.attlen); else newEntry->reconstructedValue = (Datum) 0; diff --git a/src/backend/access/spgist/spgutils.c b/src/backend/access/spgist/spgutils.c index bd5301f383..e571f0cce0 100644 --- a/src/backend/access/spgist/spgutils.c +++ b/src/backend/access/spgist/spgutils.c @@ -125,6 +125,22 @@ spgGetCache(Relation index) /* Get the information we need about each relevant datatype */ fillTypeDesc(&cache->attType, atttype); + + if (OidIsValid(cache->config.leafType) && + cache->config.leafType != atttype) + { + if (!OidIsValid(index_getprocid(index, 1, SPGIST_COMPRESS_PROC))) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("compress method must not defined when leaf type is different from input type"))); + + fillTypeDesc(&cache->attLeafType, cache->config.leafType); + } + else + { + cache->attLeafType = cache->attType; + } + fillTypeDesc(&cache->attPrefixType, cache->config.prefixType); fillTypeDesc(&cache->attLabelType, cache->config.labelType); @@ -164,6 +180,7 @@ initSpGistState(SpGistState *state, Relation index) state->config = cache->config; state->attType = cache->attType; + state->attLeafType = cache->attLeafType; state->attPrefixType = cache->attPrefixType; state->attLabelType = cache->attLabelType; @@ -618,7 +635,7 @@ spgFormLeafTuple(SpGistState *state, ItemPointer heapPtr, /* compute space needed (note result is already maxaligned) */ size = SGLTHDRSZ; if (!isnull) - size += SpGistGetTypeSize(&state->attType, datum); + size += SpGistGetTypeSize(&state->attLeafType, datum); /* * Ensure that we can replace the tuple with a dead tuple later. This @@ -634,7 +651,7 @@ spgFormLeafTuple(SpGistState *state, ItemPointer heapPtr, tup->nextOffset = InvalidOffsetNumber; tup->heapPtr = *heapPtr; if (!isnull) - memcpyDatum(SGLTDATAPTR(tup), &state->attType, datum); + memcpyDatum(SGLTDATAPTR(tup), &state->attLeafType, datum); return tup; } diff --git a/src/backend/access/spgist/spgvalidate.c b/src/backend/access/spgist/spgvalidate.c index 157cf2a028..440b3ce917 100644 --- a/src/backend/access/spgist/spgvalidate.c +++ b/src/backend/access/spgist/spgvalidate.c @@ -22,6 +22,7 @@ #include "catalog/pg_opfamily.h" #include "catalog/pg_type.h" #include "utils/builtins.h" +#include "utils/lsyscache.h" #include "utils/regproc.h" #include "utils/syscache.h" @@ -52,6 +53,10 @@ spgvalidate(Oid opclassoid) OpFamilyOpFuncGroup *opclassgroup; int i; ListCell *lc; + spgConfigIn configIn; + spgConfigOut configOut; + Oid configOutLefttype = InvalidOid; + Oid configOutRighttype = InvalidOid; /* Fetch opclass information */ classtup = SearchSysCache1(CLAOID, ObjectIdGetDatum(opclassoid)); @@ -74,6 +79,7 @@ spgvalidate(Oid opclassoid) /* Fetch all operators and support functions of the opfamily */ oprlist = SearchSysCacheList1(AMOPSTRATEGY, ObjectIdGetDatum(opfamilyoid)); proclist = SearchSysCacheList1(AMPROCNUM, ObjectIdGetDatum(opfamilyoid)); + grouplist = identify_opfamily_groups(oprlist, proclist); /* Check individual support functions */ for (i = 0; i < proclist->n_members; i++) @@ -100,6 +106,40 @@ spgvalidate(Oid opclassoid) switch (procform->amprocnum) { case SPGIST_CONFIG_PROC: + ok = check_amproc_signature(procform->amproc, VOIDOID, true, + 2, 2, INTERNALOID, INTERNALOID); + configIn.attType = procform->amproclefttype; + memset(&configOut, 0, sizeof(configOut)); + + OidFunctionCall2(procform->amproc, + PointerGetDatum(&configIn), + PointerGetDatum(&configOut)); + + configOutLefttype = procform->amproclefttype; + configOutRighttype = procform->amprocrighttype; + + /* + * When leaf and attribute types are the same, compress function + * is not required and we set corresponding bit in functionset + * for later group consistency check. + */ + if (!OidIsValid(configOut.leafType) || + configOut.leafType == configIn.attType) + { + foreach(lc, grouplist) + { + OpFamilyOpFuncGroup *group = lfirst(lc); + + if (group->lefttype == procform->amproclefttype && + group->righttype == procform->amprocrighttype) + { + group->functionset |= + ((uint64) 1) << SPGIST_COMPRESS_PROC; + break; + } + } + } + break; case SPGIST_CHOOSE_PROC: case SPGIST_PICKSPLIT_PROC: case SPGIST_INNER_CONSISTENT_PROC: @@ -110,6 +150,15 @@ spgvalidate(Oid opclassoid) ok = check_amproc_signature(procform->amproc, BOOLOID, true, 2, 2, INTERNALOID, INTERNALOID); break; + case SPGIST_COMPRESS_PROC: + if (configOutLefttype != procform->amproclefttype || + configOutRighttype != procform->amprocrighttype) + ok = false; + else + ok = check_amproc_signature(procform->amproc, + configOut.leafType, true, + 1, 1, procform->amproclefttype); + break; default: ereport(INFO, (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), @@ -178,7 +227,6 @@ spgvalidate(Oid opclassoid) } /* Now check for inconsistent groups of operators/functions */ - grouplist = identify_opfamily_groups(oprlist, proclist); opclassgroup = NULL; foreach(lc, grouplist) { diff --git a/src/include/access/spgist.h b/src/include/access/spgist.h index d1bc396e6d..06b1d88e5a 100644 --- a/src/include/access/spgist.h +++ b/src/include/access/spgist.h @@ -30,7 +30,9 @@ #define SPGIST_PICKSPLIT_PROC 3 #define SPGIST_INNER_CONSISTENT_PROC 4 #define SPGIST_LEAF_CONSISTENT_PROC 5 -#define SPGISTNProc 5 +#define SPGIST_COMPRESS_PROC 6 +#define SPGISTNRequiredProc 5 +#define SPGISTNProc 6 /* * Argument structs for spg_config method @@ -44,6 +46,7 @@ typedef struct spgConfigOut { Oid prefixType; /* Data type of inner-tuple prefixes */ Oid labelType; /* Data type of inner-tuple node labels */ + Oid leafType; /* Data type of leaf-tuple values */ bool canReturnData; /* Opclass can reconstruct original data */ bool longValuesOK; /* Opclass can cope with values > 1 page */ } spgConfigOut; diff --git a/src/include/access/spgist_private.h b/src/include/access/spgist_private.h index 1c4b321b6c..e55de9dc54 100644 --- a/src/include/access/spgist_private.h +++ b/src/include/access/spgist_private.h @@ -119,7 +119,8 @@ typedef struct SpGistState { spgConfigOut config; /* filled in by opclass config method */ - SpGistTypeDesc attType; /* type of input data and leaf values */ + SpGistTypeDesc attType; /* type of values to be indexed/restored */ + SpGistTypeDesc attLeafType; /* type of leaf-tuple values */ SpGistTypeDesc attPrefixType; /* type of inner-tuple prefix values */ SpGistTypeDesc attLabelType; /* type of node label values */ @@ -178,7 +179,8 @@ typedef struct SpGistCache { spgConfigOut config; /* filled in by opclass config method */ - SpGistTypeDesc attType; /* type of input data and leaf values */ + SpGistTypeDesc attType; /* type of values to be indexed/restored */ + SpGistTypeDesc attLeafType; /* type of leaf-tuple values */ SpGistTypeDesc attPrefixType; /* type of inner-tuple prefix values */ SpGistTypeDesc attLabelType; /* type of node label values */ @@ -300,7 +302,7 @@ typedef SpGistLeafTupleData *SpGistLeafTuple; #define SGLTHDRSZ MAXALIGN(sizeof(SpGistLeafTupleData)) #define SGLTDATAPTR(x) (((char *) (x)) + SGLTHDRSZ) -#define SGLTDATUM(x, s) ((s)->attType.attbyval ? \ +#define SGLTDATUM(x, s) ((s)->attLeafType.attbyval ? \ *(Datum *) SGLTDATAPTR(x) : \ PointerGetDatum(SGLTDATAPTR(x))) -- 2.40.0