PG_MODULE_MAGIC;
/* GUC variables */
-double similarity_threshold = 0.3f;
+double similarity_threshold = 0.3f;
+double word_similarity_threshold = 0.6f;
void _PG_init(void);
PG_FUNCTION_INFO_V1(show_limit);
PG_FUNCTION_INFO_V1(show_trgm);
PG_FUNCTION_INFO_V1(similarity);
+PG_FUNCTION_INFO_V1(word_similarity);
PG_FUNCTION_INFO_V1(similarity_dist);
PG_FUNCTION_INFO_V1(similarity_op);
+PG_FUNCTION_INFO_V1(word_similarity_op);
+PG_FUNCTION_INFO_V1(word_similarity_commutator_op);
+PG_FUNCTION_INFO_V1(word_similarity_dist_op);
+PG_FUNCTION_INFO_V1(word_similarity_dist_commutator_op);
+
+/* Trigram with position */
+typedef struct
+{
+ trgm trg;
+ int index;
+} pos_trgm;
/*
* Module load callback
NULL,
NULL,
NULL);
+ DefineCustomRealVariable("pg_trgm.word_similarity_threshold",
+ "Sets the threshold used by the <%% operator.",
+ "Valid range is 0.0 .. 1.0.",
+ &word_similarity_threshold,
+ 0.6,
+ 0.0,
+ 1.0,
+ PGC_USERSET,
+ 0,
+ NULL,
+ NULL,
+ NULL);
}
/*
* Deprecated function.
- * Use "pg_trgm.similarity_threshold" GUC variable instead of this function
+ * Use "pg_trgm.similarity_threshold" GUC variable instead of this function.
*/
Datum
set_limit(PG_FUNCTION_ARGS)
if (nlimit < 0 || nlimit > 1.0)
ereport(ERROR,
(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
- errmsg("wrong limit, should be between 0 and 1")));
+ errmsg("wrong threshold, should be between 0 and 1")));
similarity_threshold = nlimit;
PG_RETURN_FLOAT4(similarity_threshold);
}
/*
* Deprecated function.
- * Use "pg_trgm.similarity_threshold" GUC variable instead of this function
+ * Use "pg_trgm.similarity_threshold" GUC variable instead of this function.
*/
Datum
show_limit(PG_FUNCTION_ARGS)
return tptr;
}
-TRGM *
-generate_trgm(char *str, int slen)
+/*
+ * Make array of trigrams without sorting and removing duplicate items.
+ *
+ * trg: where to return the array of trigrams.
+ * str: source string, of length slen bytes.
+ *
+ * Returns length of the generated array.
+ */
+static int
+generate_trgm_only(trgm *trg, char *str, int slen)
{
- TRGM *trg;
- char *buf;
trgm *tptr;
- int len,
- charlen,
+ char *buf;
+ int charlen,
bytelen;
char *bword,
*eword;
- /*
- * Guard against possible overflow in the palloc requests below. (We
- * don't worry about the additive constants, since palloc can detect
- * requests that are a little above MaxAllocSize --- we just need to
- * prevent integer overflow in the multiplications.)
- */
- if ((Size) (slen / 2) >= (MaxAllocSize / (sizeof(trgm) * 3)) ||
- (Size) slen >= (MaxAllocSize / pg_database_encoding_max_length()))
- ereport(ERROR,
- (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
- errmsg("out of memory")));
-
- trg = (TRGM *) palloc(TRGMHDRSIZE + sizeof(trgm) * (slen / 2 + 1) *3);
- trg->flag = ARRKEY;
- SET_VARSIZE(trg, TRGMHDRSIZE);
-
if (slen + LPADDING + RPADDING < 3 || slen == 0)
- return trg;
+ return 0;
- tptr = GETARR(trg);
+ tptr = trg;
/* Allocate a buffer for case-folded, blank-padded words */
buf = (char *) palloc(slen * pg_database_encoding_max_length() + 4);
pfree(buf);
- if ((len = tptr - GETARR(trg)) == 0)
+ return tptr - trg;
+}
+
+/*
+ * Guard against possible overflow in the palloc requests below. (We
+ * don't worry about the additive constants, since palloc can detect
+ * requests that are a little above MaxAllocSize --- we just need to
+ * prevent integer overflow in the multiplications.)
+ */
+static void
+protect_out_of_mem(int slen)
+{
+ if ((Size) (slen / 2) >= (MaxAllocSize / (sizeof(trgm) * 3)) ||
+ (Size) slen >= (MaxAllocSize / pg_database_encoding_max_length()))
+ ereport(ERROR,
+ (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
+ errmsg("out of memory")));
+}
+
+/*
+ * Make array of trigrams with sorting and removing duplicate items.
+ *
+ * str: source string, of length slen bytes.
+ *
+ * Returns the sorted array of unique trigrams.
+ */
+TRGM *
+generate_trgm(char *str, int slen)
+{
+ TRGM *trg;
+ int len;
+
+ protect_out_of_mem(slen);
+
+ trg = (TRGM *) palloc(TRGMHDRSIZE + sizeof(trgm) * (slen / 2 + 1) *3);
+ trg->flag = ARRKEY;
+
+ len = generate_trgm_only(GETARR(trg), str, slen);
+ SET_VARSIZE(trg, CALCGTSIZE(ARRKEY, len));
+
+ if (len == 0)
return trg;
/*
return trg;
}
+/*
+ * Make array of positional trigrams from two trigram arrays trg1 and trg2.
+ *
+ * trg1: trigram array of search pattern, of length len1. trg1 is required
+ * word which positions don't matter and replaced with -1.
+ * trg2: trigram array of text, of length len2. trg2 is haystack where we
+ * search and have to store its positions.
+ *
+ * Returns concatenated trigram array.
+ */
+static pos_trgm *
+make_positional_trgm(trgm *trg1, int len1, trgm *trg2, int len2)
+{
+ pos_trgm *result;
+ int i, len = len1 + len2;
+
+ result = (pos_trgm *) palloc(sizeof(pos_trgm) * len);
+
+ for (i = 0; i < len1; i++)
+ {
+ memcpy(&result[i].trg, &trg1[i], sizeof(trgm));
+ result[i].index = -1;
+ }
+
+ for (i = 0; i < len2; i++)
+ {
+ memcpy(&result[i + len1].trg, &trg2[i], sizeof(trgm));
+ result[i + len1].index = i;
+ }
+
+ return result;
+}
+
+/*
+ * Compare position trigrams: compare trigrams first and position second.
+ */
+static int
+comp_ptrgm(const void *v1, const void *v2)
+{
+ const pos_trgm *p1 = (const pos_trgm *)v1;
+ const pos_trgm *p2 = (const pos_trgm *)v2;
+ int cmp;
+
+ cmp = CMPTRGM(p1->trg, p2->trg);
+ if (cmp != 0)
+ return cmp;
+
+ if (p1->index < p2->index)
+ return -1;
+ else if (p1->index == p2->index)
+ return 0;
+ else
+ return 1;
+}
+
+/*
+ * Iterative search function which calculates maximum similarity with word in
+ * the string. But maximum similarity is calculated only if check_only == false.
+ *
+ * trg2indexes: array which stores indexes of the array "found".
+ * found: array which stores true of false values.
+ * ulen1: count of unique trigrams of array "trg1".
+ * len2: length of array "trg2" and array "trg2indexes".
+ * len: length of the array "found".
+ * check_only: if true then only check existaince of similar search pattern in
+ * text.
+ *
+ * Returns word similarity.
+ */
+static float4
+iterate_word_similarity(int *trg2indexes,
+ bool *found,
+ int ulen1,
+ int len2,
+ int len,
+ bool check_only)
+{
+ int *lastpos,
+ i,
+ ulen2 = 0,
+ count = 0,
+ upper = -1,
+ lower = -1;
+ float4 smlr_cur,
+ smlr_max = 0.0f;
+
+ /* Memorise last position of each trigram */
+ lastpos = (int *) palloc(sizeof(int) * len);
+ memset(lastpos, -1, sizeof(int) * len);
+
+ for (i = 0; i < len2; i++)
+ {
+ /* Get index of next trigram */
+ int trgindex = trg2indexes[i];
+
+ /* Update last position of this trigram */
+ if (lower >= 0 || found[trgindex])
+ {
+ if (lastpos[trgindex] < 0)
+ {
+ ulen2++;
+ if (found[trgindex])
+ count++;
+ }
+ lastpos[trgindex] = i;
+ }
+
+ /* Adjust lower bound if this trigram is present in required substing */
+ if (found[trgindex])
+ {
+ int prev_lower,
+ tmp_ulen2,
+ tmp_lower,
+ tmp_count;
+
+ upper = i;
+ if (lower == -1)
+ {
+ lower = i;
+ ulen2 = 1;
+ }
+
+ smlr_cur = CALCSML(count, ulen1, ulen2);
+
+ /* Also try to adjust upper bound for greater similarity */
+ tmp_count = count;
+ tmp_ulen2 = ulen2;
+ prev_lower = lower;
+ for (tmp_lower = lower; tmp_lower <= upper; tmp_lower++)
+ {
+ float smlr_tmp = CALCSML(tmp_count, ulen1, tmp_ulen2);
+ int tmp_trgindex;
+
+ if (smlr_tmp > smlr_cur)
+ {
+ smlr_cur = smlr_tmp;
+ ulen2 = tmp_ulen2;
+ lower = tmp_lower;
+ count = tmp_count;
+ }
+ /*
+ * if we only check that word similarity is greater than
+ * pg_trgm.word_similarity_threshold we do not need to calculate
+ * a maximum similarity.
+ */
+ if (check_only && smlr_cur >= word_similarity_threshold)
+ break;
+
+ tmp_trgindex = trg2indexes[tmp_lower];
+ if (lastpos[tmp_trgindex] == tmp_lower)
+ {
+ tmp_ulen2--;
+ if (found[tmp_trgindex])
+ tmp_count--;
+ }
+ }
+
+ smlr_max = Max(smlr_max, smlr_cur);
+ /*
+ * if we only check that word similarity is greater than
+ * pg_trgm.word_similarity_threshold we do not need to calculate a
+ * maximum similarity
+ */
+ if (check_only && smlr_max >= word_similarity_threshold)
+ break;
+
+ for (tmp_lower = prev_lower; tmp_lower < lower; tmp_lower++)
+ {
+ int tmp_trgindex;
+ tmp_trgindex = trg2indexes[tmp_lower];
+ if (lastpos[tmp_trgindex] == tmp_lower)
+ lastpos[tmp_trgindex] = -1;
+ }
+ }
+ }
+
+ pfree(lastpos);
+
+ return smlr_max;
+}
+
+/*
+ * Calculate word similarity.
+ * This function prepare two arrays: "trg2indexes" and "found". Then this arrays
+ * are used to calculate word similarity using iterate_word_similarity().
+ *
+ * "trg2indexes" is array which stores indexes of the array "found".
+ * In other words:
+ * trg2indexes[j] = i;
+ * found[i] = true (or false);
+ * If found[i] == true then there is trigram trg2[j] in array "trg1".
+ * If found[i] == false then there is not trigram trg2[j] in array "trg1".
+ *
+ * str1: search pattern string, of length slen1 bytes.
+ * str2: text in which we are looking for a word, of length slen2 bytes.
+ * check_only: if true then only check existaince of similar search pattern in
+ * text.
+ *
+ * Returns word similarity.
+ */
+static float4
+calc_word_similarity(char *str1, int slen1, char *str2, int slen2,
+ bool check_only)
+{
+ bool *found;
+ pos_trgm *ptrg;
+ trgm *trg1;
+ trgm *trg2;
+ int len1,
+ len2,
+ len,
+ i,
+ j,
+ ulen1;
+ int *trg2indexes;
+ float4 result;
+
+ protect_out_of_mem(slen1 + slen2);
+
+ /* Make positional trigrams */
+ trg1 = (trgm *) palloc(sizeof(trgm) * (slen1 / 2 + 1) * 3);
+ trg2 = (trgm *) palloc(sizeof(trgm) * (slen2 / 2 + 1) * 3);
+
+ len1 = generate_trgm_only(trg1, str1, slen1);
+ len2 = generate_trgm_only(trg2, str2, slen2);
+
+ ptrg = make_positional_trgm(trg1, len1, trg2, len2);
+ len = len1 + len2;
+ qsort(ptrg, len, sizeof(pos_trgm), comp_ptrgm);
+
+ pfree(trg1);
+ pfree(trg2);
+
+ /*
+ * Merge positional trigrams array: enumerate each trigram and find its
+ * presence in required word.
+ */
+ trg2indexes = (int *) palloc(sizeof(int) * len2);
+ found = (bool *) palloc0(sizeof(bool) * len);
+
+ ulen1 = 0;
+ j = 0;
+ for (i = 0; i < len; i++)
+ {
+ if (i > 0)
+ {
+ int cmp = CMPTRGM(ptrg[i - 1].trg, ptrg[i].trg);
+ if (cmp != 0)
+ {
+ if (found[j])
+ ulen1++;
+ j++;
+ }
+ }
+
+ if (ptrg[i].index >= 0)
+ {
+ trg2indexes[ptrg[i].index] = j;
+ }
+ else
+ {
+ found[j] = true;
+ }
+ }
+ if (found[j])
+ ulen1++;
+
+ /* Run iterative procedure to find maximum similarity with word */
+ result = iterate_word_similarity(trg2indexes, found, ulen1, len2, len,
+ check_only);
+
+ pfree(trg2indexes);
+ pfree(found);
+ pfree(ptrg);
+
+ return result;
+}
+
+
/*
* Extract the next non-wildcard part of a search string, ie, a word bounded
* by '_' or '%' meta-characters, non-word characters or string end.
bytelen;
const char *eword;
- /*
- * Guard against possible overflow in the palloc requests below. (We
- * don't worry about the additive constants, since palloc can detect
- * requests that are a little above MaxAllocSize --- we just need to
- * prevent integer overflow in the multiplications.)
- */
- if ((Size) (slen / 2) >= (MaxAllocSize / (sizeof(trgm) * 3)) ||
- (Size) slen >= (MaxAllocSize / pg_database_encoding_max_length()))
- ereport(ERROR,
- (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
- errmsg("out of memory")));
+ protect_out_of_mem(slen);
trg = (TRGM *) palloc(TRGMHDRSIZE + sizeof(trgm) * (slen / 2 + 1) *3);
trg->flag = ARRKEY;
}
float4
-cnt_sml(TRGM *trg1, TRGM *trg2)
+cnt_sml(TRGM *trg1, TRGM *trg2, bool inexact)
{
trgm *ptr1,
*ptr2;
}
}
-#ifdef DIVUNION
- return ((float4) count) / ((float4) (len1 + len2 - count));
-#else
- return ((float4) count) / ((float4) ((len1 > len2) ? len1 : len2));
-#endif
-
+ /*
+ * If inexact then len2 is equal to count, because we don't know actual
+ * length of second string in inexact search and we can assume that count
+ * is a lower bound of len2.
+ */
+ return CALCSML(count, len1, inexact ? count : len2);
}
+
/*
* Returns whether trg2 contains all trigrams in trg1.
* This relies on the trigram arrays being sorted.
trg1 = generate_trgm(VARDATA(in1), VARSIZE(in1) - VARHDRSZ);
trg2 = generate_trgm(VARDATA(in2), VARSIZE(in2) - VARHDRSZ);
- res = cnt_sml(trg1, trg2);
+ res = cnt_sml(trg1, trg2, false);
pfree(trg1);
pfree(trg2);
PG_RETURN_FLOAT4(res);
}
+Datum
+word_similarity(PG_FUNCTION_ARGS)
+{
+ text *in1 = PG_GETARG_TEXT_PP(0);
+ text *in2 = PG_GETARG_TEXT_PP(1);
+ float4 res;
+
+ res = calc_word_similarity(VARDATA_ANY(in1), VARSIZE_ANY_EXHDR(in1),
+ VARDATA_ANY(in2), VARSIZE_ANY_EXHDR(in2),
+ false);
+
+ PG_FREE_IF_COPY(in1, 0);
+ PG_FREE_IF_COPY(in2, 1);
+ PG_RETURN_FLOAT4(res);
+}
+
Datum
similarity_dist(PG_FUNCTION_ARGS)
{
PG_RETURN_BOOL(res >= similarity_threshold);
}
+
+Datum
+word_similarity_op(PG_FUNCTION_ARGS)
+{
+ text *in1 = PG_GETARG_TEXT_PP(0);
+ text *in2 = PG_GETARG_TEXT_PP(1);
+ float4 res;
+
+ res = calc_word_similarity(VARDATA_ANY(in1), VARSIZE_ANY_EXHDR(in1),
+ VARDATA_ANY(in2), VARSIZE_ANY_EXHDR(in2),
+ true);
+
+ PG_FREE_IF_COPY(in1, 0);
+ PG_FREE_IF_COPY(in2, 1);
+ PG_RETURN_BOOL(res >= word_similarity_threshold);
+}
+
+Datum
+word_similarity_commutator_op(PG_FUNCTION_ARGS)
+{
+ text *in1 = PG_GETARG_TEXT_PP(0);
+ text *in2 = PG_GETARG_TEXT_PP(1);
+ float4 res;
+
+ res = calc_word_similarity(VARDATA_ANY(in2), VARSIZE_ANY_EXHDR(in2),
+ VARDATA_ANY(in1), VARSIZE_ANY_EXHDR(in1),
+ true);
+
+ PG_FREE_IF_COPY(in1, 0);
+ PG_FREE_IF_COPY(in2, 1);
+ PG_RETURN_BOOL(res >= word_similarity_threshold);
+}
+
+Datum
+word_similarity_dist_op(PG_FUNCTION_ARGS)
+{
+ text *in1 = PG_GETARG_TEXT_PP(0);
+ text *in2 = PG_GETARG_TEXT_PP(1);
+ float4 res;
+
+ res = calc_word_similarity(VARDATA_ANY(in1), VARSIZE_ANY_EXHDR(in1),
+ VARDATA_ANY(in2), VARSIZE_ANY_EXHDR(in2),
+ false);
+
+ PG_FREE_IF_COPY(in1, 0);
+ PG_FREE_IF_COPY(in2, 1);
+ PG_RETURN_FLOAT4(1.0 - res);
+}
+
+Datum
+word_similarity_dist_commutator_op(PG_FUNCTION_ARGS)
+{
+ text *in1 = PG_GETARG_TEXT_PP(0);
+ text *in2 = PG_GETARG_TEXT_PP(1);
+ float4 res;
+
+ res = calc_word_similarity(VARDATA_ANY(in2), VARSIZE_ANY_EXHDR(in2),
+ VARDATA_ANY(in1), VARSIZE_ANY_EXHDR(in1),
+ false);
+
+ PG_FREE_IF_COPY(in1, 0);
+ PG_FREE_IF_COPY(in2, 1);
+ PG_RETURN_FLOAT4(1.0 - res);
+}
projects / postgresql / commitdiff