Allow multi-character source strings in contrib/unaccent.

This could be useful in languages where diacritic signs are represented as
separate characters; more generally it supports using unaccent dictionaries
for substring substitutions beyond narrowly conceived "diacritic removal".
In any case, since the rule-file parser doesn't complain about
multi-character source strings, it behooves us to do something unsurprising
with them.

diff --git a/contrib/unaccent/unaccent.c b/contrib/unaccent/unaccent.c
index 5a31f85a132a0227e2e6b7c6783a265f22e4553e..0101506b4580f328c5a549e38227eba76f412705 100644 (file)
--- a/contrib/unaccent/unaccent.c
+++ b/contrib/unaccent/unaccent.c
@@ -23,9 +23,16 @@
 PG_MODULE_MAGIC;
 
 /*
- * Unaccent dictionary uses a trie to find a character to replace. Each node of
- * the trie is an array of 256 TrieChar structs (n-th element of array
- * corresponds to byte)
+ * An unaccent dictionary uses a trie to find a string to replace.  Each node
+ * of the trie is an array of 256 TrieChar structs; the N-th element of the
+ * array corresponds to next byte value N.  That element can contain both a
+ * replacement string (to be used if the source string ends with this byte)
+ * and a link to another trie node (to be followed if there are more bytes).
+ *
+ * Note that the trie search logic pays no attention to multibyte character
+ * boundaries.  This is OK as long as both the data entered into the trie and
+ * the data we're trying to look up are validly encoded; no partial-character
+ * matches will occur.
  */
 typedef struct TrieChar
 {
@@ -36,34 +43,38 @@ typedef struct TrieChar
 
 /*
  * placeChar - put str into trie's structure, byte by byte.
+ *
+ * If node is NULL, we need to make a new node, which will be returned;
+ * otherwise the return value is the same as node.
  */
 static TrieChar *
-placeChar(TrieChar *node, unsigned char *str, int lenstr, char *replaceTo, int replacelen)
+placeChar(TrieChar *node, const unsigned char *str, int lenstr,
+                 const char *replaceTo, int replacelen)
 {
        TrieChar   *curnode;
 
        if (!node)
-       {
-               node = palloc(sizeof(TrieChar) * 256);
-               memset(node, 0, sizeof(TrieChar) * 256);
-       }
+               node = (TrieChar *) palloc0(sizeof(TrieChar) * 256);
+
+       Assert(lenstr > 0);                     /* else str[0] doesn't exist */
 
        curnode = node + *str;
 
-       if (lenstr == 1)
+       if (lenstr <= 1)
        {
                if (curnode->replaceTo)
-                       elog(WARNING, "duplicate TO argument, use first one");
+                       elog(WARNING, "duplicate source strings, first one will be used");
                else
                {
                        curnode->replacelen = replacelen;
-                       curnode->replaceTo = palloc(replacelen);
+                       curnode->replaceTo = (char *) palloc(replacelen);
                        memcpy(curnode->replaceTo, replaceTo, replacelen);
                }
        }
        else
        {
-               curnode->nextChar = placeChar(curnode->nextChar, str + 1, lenstr - 1, replaceTo, replacelen);
+               curnode->nextChar = placeChar(curnode->nextChar, str + 1, lenstr - 1,
+                                                                         replaceTo, replacelen);
        }
 
        return node;
@@ -213,23 +224,35 @@ initTrie(char *filename)
 }
 
 /*
- * findReplaceTo - find multibyte character in trie
+ * findReplaceTo - find longest possible match in trie
+ *
+ * On success, returns pointer to ending subnode, plus length of matched
+ * source string in *p_matchlen.  On failure, returns NULL.
  */
 static TrieChar *
-findReplaceTo(TrieChar *node, unsigned char *src, int srclen)
+findReplaceTo(TrieChar *node, const unsigned char *src, int srclen,
+                         int *p_matchlen)
 {
-       while (node)
+       TrieChar   *result = NULL;
+       int                     matchlen = 0;
+
+       *p_matchlen = 0;                        /* prevent uninitialized-variable warnings */
+
+       while (node && matchlen < srclen)
        {
-               node = node + *src;
-               if (srclen == 1)
-                       return node;
+               node = node + src[matchlen];
+               matchlen++;
+
+               if (node->replaceTo)
+               {
+                       result = node;
+                       *p_matchlen = matchlen;
+               }
 
-               src++;
-               srclen--;
                node = node->nextChar;
        }
 
-       return NULL;
+       return result;
 }
 
 PG_FUNCTION_INFO_V1(unaccent_init);
@@ -280,18 +303,17 @@ unaccent_lexize(PG_FUNCTION_ARGS)
        TrieChar   *rootTrie = (TrieChar *) PG_GETARG_POINTER(0);
        char       *srcchar = (char *) PG_GETARG_POINTER(1);
        int32           len = PG_GETARG_INT32(2);
-       char       *srcstart,
+       char       *srcstart = srcchar,
                           *trgchar = NULL;
-       int                     charlen;
        TSLexeme   *res = NULL;
-       TrieChar   *node;
 
-       srcstart = srcchar;
-       while (srcchar - srcstart < len)
+       while (len > 0)
        {
-               charlen = pg_mblen(srcchar);
+               TrieChar   *node;
+               int                     matchlen;
 
-               node = findReplaceTo(rootTrie, (unsigned char *) srcchar, charlen);
+               node = findReplaceTo(rootTrie, (unsigned char *) srcchar, len,
+                                                        &matchlen);
                if (node && node->replaceTo)
                {
                        if (!res)
@@ -309,13 +331,18 @@ unaccent_lexize(PG_FUNCTION_ARGS)
                        memcpy(trgchar, node->replaceTo, node->replacelen);
                        trgchar += node->replacelen;
                }
-               else if (res)
+               else
                {
-                       memcpy(trgchar, srcchar, charlen);
-                       trgchar += charlen;
+                       matchlen = pg_mblen(srcchar);
+                       if (res)
+                       {
+                               memcpy(trgchar, srcchar, matchlen);
+                               trgchar += matchlen;
+                       }
                }
 
-               srcchar += charlen;
+               srcchar += matchlen;
+               len -= matchlen;
        }
 
        if (res)

diff --git a/doc/src/sgml/unaccent.sgml b/doc/src/sgml/unaccent.sgml
index aef0031dcbcc40073046ab3c09d47ea949818e13..1382fafc5ec3859ae14f8a3e0f26d57c46bbf0de 100644 (file)
--- a/doc/src/sgml/unaccent.sgml
+++ b/doc/src/sgml/unaccent.sgml
@@ -70,6 +70,14 @@
     </para>
    </listitem>
 
+   <listitem>
+    <para>
+     Actually, each <quote>character</> can be any string not containing
+     whitespace, so <filename>unaccent</> dictionaries could be used for
+     other sorts of substring substitutions besides diacritic removal.
+    </para>
+   </listitem>
+
    <listitem>
     <para>
      As with other <productname>PostgreSQL</> text search configuration files,