granicus.if.org Git - postgresql/blob - contrib/pg_trgm/trgm_op.c

   1 /*
   2  * contrib/pg_trgm/trgm_op.c
   3  */
   4 #include "postgres.h"
   5
   6 #include <ctype.h>
   7
   8 #include "trgm.h"
   9
  10 #include "catalog/pg_type.h"
  11 #include "tsearch/ts_locale.h"
  12 #include "utils/array.h"
  13
  14
  15 PG_MODULE_MAGIC;
  16
  17 float4          trgm_limit = 0.3f;
  18
  19 PG_FUNCTION_INFO_V1(set_limit);
  20 Datum           set_limit(PG_FUNCTION_ARGS);
  21
  22 PG_FUNCTION_INFO_V1(show_limit);
  23 Datum           show_limit(PG_FUNCTION_ARGS);
  24
  25 PG_FUNCTION_INFO_V1(show_trgm);
  26 Datum           show_trgm(PG_FUNCTION_ARGS);
  27
  28 PG_FUNCTION_INFO_V1(similarity);
  29 Datum           similarity(PG_FUNCTION_ARGS);
  30
  31 PG_FUNCTION_INFO_V1(similarity_dist);
  32 Datum           similarity_dist(PG_FUNCTION_ARGS);
  33
  34 PG_FUNCTION_INFO_V1(similarity_op);
  35 Datum           similarity_op(PG_FUNCTION_ARGS);
  36
  37
  38 Datum
  39 set_limit(PG_FUNCTION_ARGS)
  40 {
  41         float4          nlimit = PG_GETARG_FLOAT4(0);
  42
  43         if (nlimit < 0 || nlimit > 1.0)
  44                 elog(ERROR, "wrong limit, should be between 0 and 1");
  45         trgm_limit = nlimit;
  46         PG_RETURN_FLOAT4(trgm_limit);
  47 }
  48
  49 Datum
  50 show_limit(PG_FUNCTION_ARGS)
  51 {
  52         PG_RETURN_FLOAT4(trgm_limit);
  53 }
  54
  55 static int
  56 comp_trgm(const void *a, const void *b)
  57 {
  58         return CMPTRGM(a, b);
  59 }
  60
  61 static int
  62 unique_array(trgm *a, int len)
  63 {
  64         trgm       *curend,
  65                            *tmp;
  66
  67         curend = tmp = a;
  68         while (tmp - a < len)
  69                 if (CMPTRGM(tmp, curend))
  70                 {
  71                         curend++;
  72                         CPTRGM(curend, tmp);
  73                         tmp++;
  74                 }
  75                 else
  76                         tmp++;
  77
  78         return curend + 1 - a;
  79 }
  80
  81 #ifdef KEEPONLYALNUM
  82 #define iswordchr(c)    (t_isalpha(c) || t_isdigit(c))
  83 #else
  84 #define iswordchr(c)    (!t_isspace(c))
  85 #endif
  86
  87 /*
  88  * Finds first word in string, returns pointer to the word,
  89  * endword points to the character after word
  90  */
  91 static char *
  92 find_word(char *str, int lenstr, char **endword, int *charlen)
  93 {
  94         char       *beginword = str;
  95
  96         while (beginword - str < lenstr && !iswordchr(beginword))
  97                 beginword += pg_mblen(beginword);
  98
  99         if (beginword - str >= lenstr)
 100                 return NULL;
 101
 102         *endword = beginword;
 103         *charlen = 0;
 104         while (*endword - str < lenstr && iswordchr(*endword))
 105         {
 106                 *endword += pg_mblen(*endword);
 107                 (*charlen)++;
 108         }
 109
 110         return beginword;
 111 }
 112
 113 #ifdef USE_WIDE_UPPER_LOWER
 114 static void
 115 cnt_trigram(trgm *tptr, char *str, int bytelen)
 116 {
 117         if (bytelen == 3)
 118         {
 119                 CPTRGM(tptr, str);
 120         }
 121         else
 122         {
 123                 pg_crc32        crc;
 124
 125                 INIT_CRC32(crc);
 126                 COMP_CRC32(crc, str, bytelen);
 127                 FIN_CRC32(crc);
 128
 129                 /*
 130                  * use only 3 upper bytes from crc, hope, it's good enough hashing
 131                  */
 132                 CPTRGM(tptr, &crc);
 133         }
 134 }
 135 #endif
 136
 137 /*
 138  * Adds trigrams from words (already padded).
 139  */
 140 static trgm *
 141 make_trigrams(trgm *tptr, char *str, int bytelen, int charlen)
 142 {
 143         char       *ptr = str;
 144
 145         if (charlen < 3)
 146                 return tptr;
 147
 148 #ifdef USE_WIDE_UPPER_LOWER
 149         if (pg_database_encoding_max_length() > 1)
 150         {
 151                 int                     lenfirst = pg_mblen(str),
 152                                         lenmiddle = pg_mblen(str + lenfirst),
 153                                         lenlast = pg_mblen(str + lenfirst + lenmiddle);
 154
 155                 while ((ptr - str) + lenfirst + lenmiddle + lenlast <= bytelen)
 156                 {
 157                         cnt_trigram(tptr, ptr, lenfirst + lenmiddle + lenlast);
 158
 159                         ptr += lenfirst;
 160                         tptr++;
 161
 162                         lenfirst = lenmiddle;
 163                         lenmiddle = lenlast;
 164                         lenlast = pg_mblen(ptr + lenfirst + lenmiddle);
 165                 }
 166         }
 167         else
 168 #endif
 169         {
 170                 Assert(bytelen == charlen);
 171
 172                 while (ptr - str < bytelen - 2 /* number of trigrams = strlen - 2 */ )
 173                 {
 174                         CPTRGM(tptr, ptr);
 175                         ptr++;
 176                         tptr++;
 177                 }
 178         }
 179
 180         return tptr;
 181 }
 182
 183 TRGM *
 184 generate_trgm(char *str, int slen)
 185 {
 186         TRGM       *trg;
 187         char       *buf;
 188         trgm       *tptr;
 189         int                     len,
 190                                 charlen,
 191                                 bytelen;
 192         char       *bword,
 193                            *eword;
 194
 195         trg = (TRGM *) palloc(TRGMHDRSIZE + sizeof(trgm) * (slen / 2 + 1) *3);
 196         trg->flag = ARRKEY;
 197         SET_VARSIZE(trg, TRGMHDRSIZE);
 198
 199         if (slen + LPADDING + RPADDING < 3 || slen == 0)
 200                 return trg;
 201
 202         tptr = GETARR(trg);
 203
 204         buf = palloc(sizeof(char) * (slen + 4));
 205
 206         if (LPADDING > 0)
 207         {
 208                 *buf = ' ';
 209                 if (LPADDING > 1)
 210                         *(buf + 1) = ' ';
 211         }
 212
 213         eword = str;
 214         while ((bword = find_word(eword, slen - (eword - str), &eword, &charlen)) != NULL)
 215         {
 216 #ifdef IGNORECASE
 217                 bword = lowerstr_with_len(bword, eword - bword);
 218                 bytelen = strlen(bword);
 219 #else
 220                 bytelen = eword - bword;
 221 #endif
 222
 223                 memcpy(buf + LPADDING, bword, bytelen);
 224
 225 #ifdef IGNORECASE
 226                 pfree(bword);
 227 #endif
 228                 buf[LPADDING + bytelen] = ' ';
 229                 buf[LPADDING + bytelen + 1] = ' ';
 230
 231                 /*
 232                  * count trigrams
 233                  */
 234                 tptr = make_trigrams(tptr, buf, bytelen + LPADDING + RPADDING,
 235                                                          charlen + LPADDING + RPADDING);
 236         }
 237
 238         pfree(buf);
 239
 240         if ((len = tptr - GETARR(trg)) == 0)
 241                 return trg;
 242
 243         if (len > 0)
 244         {
 245                 qsort((void *) GETARR(trg), len, sizeof(trgm), comp_trgm);
 246                 len = unique_array(GETARR(trg), len);
 247         }
 248
 249         SET_VARSIZE(trg, CALCGTSIZE(ARRKEY, len));
 250
 251         return trg;
 252 }
 253
 254 /*
 255  * Extract the next non-wildcard part of a search string, ie, a word bounded
 256  * by '_' or '%' meta-characters, non-word characters or string end.
 257  *
 258  * str: source string, of length lenstr bytes (need not be null-terminated)
 259  * buf: where to return the substring (must be long enough)
 260  * *bytelen: receives byte length of the found substring
 261  * *charlen: receives character length of the found substring
 262  *
 263  * Returns pointer to end+1 of the found substring in the source string.
 264  * Returns NULL if no word found (in which case buf, bytelen, charlen not set)
 265  *
 266  * If the found word is bounded by non-word characters or string boundaries
 267  * then this function will include corresponding padding spaces into buf.
 268  */
 269 static const char *
 270 get_wildcard_part(const char *str, int lenstr,
 271                                   char *buf, int *bytelen, int *charlen)
 272 {
 273         const char *beginword = str;
 274         const char *endword;
 275         char       *s = buf;
 276         bool        in_wildcard_meta = false;
 277         bool        in_escape = false;
 278         int         clen;
 279
 280         /*
 281          * Find the first word character remembering whether last character was
 282          * wildcard meta-character.
 283          */
 284         while (beginword - str < lenstr)
 285         {
 286                 if (in_escape)
 287                 {
 288                         in_escape = false;
 289                         in_wildcard_meta = false;
 290                         if (iswordchr(beginword))
 291                                 break;
 292                 }
 293                 else
 294                 {
 295                         if (ISESCAPECHAR(beginword))
 296                                 in_escape = true;
 297                         else if (ISWILDCARDCHAR(beginword))
 298                                 in_wildcard_meta = true;
 299                         else if (iswordchr(beginword))
 300                                 break;
 301                         else
 302                                 in_wildcard_meta = false;
 303                 }
 304                 beginword += pg_mblen(beginword);
 305         }
 306
 307         /*
 308          * Handle string end.
 309          */
 310         if (beginword - str >= lenstr)
 311                 return NULL;
 312
 313         /*
 314          * Add left padding spaces if last character wasn't wildcard
 315          * meta-character.
 316          */
 317         *charlen = 0;
 318         if (!in_wildcard_meta)
 319         {
 320                 if (LPADDING > 0)
 321                 {
 322                         *s++ = ' ';
 323                         (*charlen)++;
 324                         if (LPADDING > 1)
 325                         {
 326                                 *s++ = ' ';
 327                                 (*charlen)++;
 328                         }
 329                 }
 330         }
 331
 332         /*
 333          * Copy data into buf until wildcard meta-character, non-word character or
 334          * string boundary.  Strip escapes during copy.
 335          */
 336         endword = beginword;
 337         in_wildcard_meta = false;
 338         in_escape = false;
 339         while (endword - str < lenstr)
 340         {
 341                 clen = pg_mblen(endword);
 342                 if (in_escape)
 343                 {
 344                         in_escape = false;
 345                         in_wildcard_meta = false;
 346                         if (iswordchr(endword))
 347                         {
 348                                 memcpy(s, endword, clen);
 349                                 (*charlen)++;
 350                                 s += clen;
 351                         }
 352                         else
 353                                 break;
 354                 }
 355                 else
 356                 {
 357                         if (ISESCAPECHAR(endword))
 358                                 in_escape = true;
 359                         else if (ISWILDCARDCHAR(endword))
 360                         {
 361                                 in_wildcard_meta = true;
 362                                 break;
 363                         }
 364                         else if (iswordchr(endword))
 365                         {
 366                                 memcpy(s, endword, clen);
 367                                 (*charlen)++;
 368                                 s += clen;
 369                         }
 370                         else
 371                         {
 372                                 in_wildcard_meta = false;
 373                                 break;
 374                         }
 375                 }
 376                 endword += clen;
 377         }
 378
 379         /*
 380          * Add right padding spaces if last character wasn't wildcard
 381          * meta-character.
 382          */
 383         if (!in_wildcard_meta)
 384         {
 385                 if (RPADDING > 0)
 386                 {
 387                         *s++ = ' ';
 388                         (*charlen)++;
 389                         if (RPADDING > 1)
 390                         {
 391                                 *s++ = ' ';
 392                                 (*charlen)++;
 393                         }
 394                 }
 395         }
 396
 397         *bytelen = s - buf;
 398         return endword;
 399 }
 400
 401 /*
 402  * Generates trigrams for wildcard search string.
 403  *
 404  * Returns array of trigrams that must occur in any string that matches the
 405  * wildcard string.  For example, given pattern "a%bcd%" the trigrams
 406  * " a", "bcd" would be extracted.
 407  */
 408 TRGM *
 409 generate_wildcard_trgm(const char *str, int slen)
 410 {
 411         TRGM       *trg;
 412         char       *buf,
 413                        *buf2;
 414         trgm       *tptr;
 415         int                     len,
 416                                 charlen,
 417                                 bytelen;
 418         const char *eword;
 419
 420         trg = (TRGM *) palloc(TRGMHDRSIZE + sizeof(trgm) * (slen / 2 + 1) * 3);
 421         trg->flag = ARRKEY;
 422         SET_VARSIZE(trg, TRGMHDRSIZE);
 423
 424         if (slen + LPADDING + RPADDING < 3 || slen == 0)
 425                 return trg;
 426
 427         tptr = GETARR(trg);
 428
 429         buf = palloc(sizeof(char) * (slen + 4));
 430
 431         /*
 432          * Extract trigrams from each substring extracted by get_wildcard_part.
 433          */
 434         eword = str;
 435         while ((eword = get_wildcard_part(eword, slen - (eword - str),
 436                                                                           buf, &bytelen, &charlen)) != NULL)
 437         {
 438 #ifdef IGNORECASE
 439                 buf2 = lowerstr_with_len(buf, bytelen);
 440                 bytelen = strlen(buf2);
 441 #else
 442                 buf2 = buf;
 443 #endif
 444
 445                 /*
 446                  * count trigrams
 447                  */
 448                 tptr = make_trigrams(tptr, buf2, bytelen, charlen);
 449 #ifdef IGNORECASE
 450                 pfree(buf2);
 451 #endif
 452         }
 453
 454         pfree(buf);
 455
 456         if ((len = tptr - GETARR(trg)) == 0)
 457                 return trg;
 458
 459         /*
 460          * Make trigrams unique.
 461          */
 462         if (len > 0)
 463         {
 464                 qsort((void *) GETARR(trg), len, sizeof(trgm), comp_trgm);
 465                 len = unique_array(GETARR(trg), len);
 466         }
 467
 468         SET_VARSIZE(trg, CALCGTSIZE(ARRKEY, len));
 469
 470         return trg;
 471 }
 472
 473 uint32
 474 trgm2int(trgm *ptr)
 475 {
 476         uint32          val = 0;
 477
 478         val |= *(((unsigned char *) ptr));
 479         val <<= 8;
 480         val |= *(((unsigned char *) ptr) + 1);
 481         val <<= 8;
 482         val |= *(((unsigned char *) ptr) + 2);
 483
 484         return val;
 485 }
 486
 487 Datum
 488 show_trgm(PG_FUNCTION_ARGS)
 489 {
 490         text       *in = PG_GETARG_TEXT_P(0);
 491         TRGM       *trg;
 492         Datum      *d;
 493         ArrayType  *a;
 494         trgm       *ptr;
 495         int                     i;
 496
 497         trg = generate_trgm(VARDATA(in), VARSIZE(in) - VARHDRSZ);
 498         d = (Datum *) palloc(sizeof(Datum) * (1 + ARRNELEM(trg)));
 499
 500         for (i = 0, ptr = GETARR(trg); i < ARRNELEM(trg); i++, ptr++)
 501         {
 502                 text       *item = (text *) palloc(VARHDRSZ + Max(12, pg_database_encoding_max_length() * 3));
 503
 504                 if (pg_database_encoding_max_length() > 1 && !ISPRINTABLETRGM(ptr))
 505                 {
 506                         snprintf(VARDATA(item), 12, "0x%06x", trgm2int(ptr));
 507                         SET_VARSIZE(item, VARHDRSZ + strlen(VARDATA(item)));
 508                 }
 509                 else
 510                 {
 511                         SET_VARSIZE(item, VARHDRSZ + 3);
 512                         CPTRGM(VARDATA(item), ptr);
 513                 }
 514                 d[i] = PointerGetDatum(item);
 515         }
 516
 517         a = construct_array(
 518                                                 d,
 519                                                 ARRNELEM(trg),
 520                                                 TEXTOID,
 521                                                 -1,
 522                                                 false,
 523                                                 'i'
 524                 );
 525
 526         for (i = 0; i < ARRNELEM(trg); i++)
 527                 pfree(DatumGetPointer(d[i]));
 528
 529         pfree(d);
 530         pfree(trg);
 531         PG_FREE_IF_COPY(in, 0);
 532
 533         PG_RETURN_POINTER(a);
 534 }
 535
 536 float4
 537 cnt_sml(TRGM *trg1, TRGM *trg2)
 538 {
 539         trgm       *ptr1,
 540                            *ptr2;
 541         int                     count = 0;
 542         int                     len1,
 543                                 len2;
 544
 545         ptr1 = GETARR(trg1);
 546         ptr2 = GETARR(trg2);
 547
 548         len1 = ARRNELEM(trg1);
 549         len2 = ARRNELEM(trg2);
 550
 551         while (ptr1 - GETARR(trg1) < len1 && ptr2 - GETARR(trg2) < len2)
 552         {
 553                 int                     res = CMPTRGM(ptr1, ptr2);
 554
 555                 if (res < 0)
 556                         ptr1++;
 557                 else if (res > 0)
 558                         ptr2++;
 559                 else
 560                 {
 561                         ptr1++;
 562                         ptr2++;
 563                         count++;
 564                 }
 565         }
 566
 567 #ifdef DIVUNION
 568         return ((((float4) count) / ((float4) (len1 + len2 - count))));
 569 #else
 570         return (((float) count) / ((float) ((len1 > len2) ? len1 : len2)));
 571 #endif
 572
 573 }
 574
 575 /*
 576  * Returns whether trg2 contains all trigrams in trg1.
 577  * This relies on the trigram arrays being sorted.
 578  */
 579 bool
 580 trgm_contained_by(TRGM *trg1, TRGM *trg2)
 581 {
 582         trgm       *ptr1,
 583                            *ptr2;
 584         int                     len1,
 585                                 len2;
 586
 587         ptr1 = GETARR(trg1);
 588         ptr2 = GETARR(trg2);
 589
 590         len1 = ARRNELEM(trg1);
 591         len2 = ARRNELEM(trg2);
 592
 593         while (ptr1 - GETARR(trg1) < len1 && ptr2 - GETARR(trg2) < len2)
 594         {
 595                 int                     res = CMPTRGM(ptr1, ptr2);
 596
 597                 if (res < 0)
 598                         return false;
 599                 else if (res > 0)
 600                         ptr2++;
 601                 else
 602                 {
 603                         ptr1++;
 604                         ptr2++;
 605                 }
 606         }
 607         if (ptr1 - GETARR(trg1) < len1)
 608                 return false;
 609         else
 610                 return true;
 611 }
 612
 613 Datum
 614 similarity(PG_FUNCTION_ARGS)
 615 {
 616         text       *in1 = PG_GETARG_TEXT_P(0);
 617         text       *in2 = PG_GETARG_TEXT_P(1);
 618         TRGM       *trg1,
 619                            *trg2;
 620         float4          res;
 621
 622         trg1 = generate_trgm(VARDATA(in1), VARSIZE(in1) - VARHDRSZ);
 623         trg2 = generate_trgm(VARDATA(in2), VARSIZE(in2) - VARHDRSZ);
 624
 625         res = cnt_sml(trg1, trg2);
 626
 627         pfree(trg1);
 628         pfree(trg2);
 629         PG_FREE_IF_COPY(in1, 0);
 630         PG_FREE_IF_COPY(in2, 1);
 631
 632         PG_RETURN_FLOAT4(res);
 633 }
 634
 635 Datum
 636 similarity_dist(PG_FUNCTION_ARGS)
 637 {
 638         float4          res = DatumGetFloat4(DirectFunctionCall2(similarity,
 639                                                                                                                  PG_GETARG_DATUM(0),
 640                                                                                                                  PG_GETARG_DATUM(1)));
 641         PG_RETURN_FLOAT4(1.0 - res);
 642 }
 643
 644 Datum
 645 similarity_op(PG_FUNCTION_ARGS)
 646 {
 647         float4          res = DatumGetFloat4(DirectFunctionCall2(similarity,
 648                                                                                                                  PG_GETARG_DATUM(0),
 649                                                                                                                  PG_GETARG_DATUM(1)));
 650
 651         PG_RETURN_BOOL(res >= trgm_limit);
 652 }