Fix assorted bugs in contrib/unaccent's configuration file parsing.

Make it use t_isspace() to identify whitespace, rather than relying on
sscanf which is known to get it wrong on some platform/locale combinations.
Get rid of fixed-size buffers.  Make it actually continue to parse the file
after ignoring a line with untranslatable characters, as was obviously
intended.

The first of these issues is per gripe from J Smith, though not exactly
either of his proposed patches.

diff --git a/contrib/unaccent/unaccent.c b/contrib/unaccent/unaccent.c
index d9c2eac2e74833c666939d4b74db183dbb5863be..d22f5c7beaa875451539b14983abe2f142b619eb 100644 (file)
--- a/contrib/unaccent/unaccent.c
+++ b/contrib/unaccent/unaccent.c
@@ -91,35 +91,83 @@ initSuffixTree(char *filename)
 
        do
        {
-               char            src[4096];
-               char            trg[4096];
-               int                     srclen;
-               int                     trglen;
-               char       *line = NULL;
-
+               /*
+                * pg_do_encoding_conversion() (called by tsearch_readline()) will
+                * emit exception if it finds untranslatable characters in current
+                * locale. We just skip such lines, continuing with the next.
+                */
                skip = true;
 
                PG_TRY();
                {
-                       /*
-                        * pg_do_encoding_conversion() (called by tsearch_readline()) will
-                        * emit exception if it finds untranslatable characters in current
-                        * locale. We just skip such characters.
-                        */
+                       char       *line;
+
                        while ((line = tsearch_readline(&trst)) != NULL)
                        {
-                               if (sscanf(line, "%s\t%s\n", src, trg) != 2)
-                                       continue;
+                               /*
+                                * The format of each line must be "src trg" where src and trg
+                                * are sequences of one or more non-whitespace characters,
+                                * separated by whitespace.  Whitespace at start or end of
+                                * line is ignored.
+                                */
+                               int                     state;
+                               char       *ptr;
+                               char       *src = NULL;
+                               char       *trg = NULL;
+                               int                     ptrlen;
+                               int                     srclen = 0;
+                               int                     trglen = 0;
+
+                               state = 0;
+                               for (ptr = line; *ptr; ptr += ptrlen)
+                               {
+                                       ptrlen = pg_mblen(ptr);
+                                       /* ignore whitespace, but end src or trg */
+                                       if (t_isspace(ptr))
+                                       {
+                                               if (state == 1)
+                                                       state = 2;
+                                               else if (state == 3)
+                                                       state = 4;
+                                               continue;
+                                       }
+                                       switch (state)
+                                       {
+                                               case 0:
+                                                       /* start of src */
+                                                       src = ptr;
+                                                       srclen = ptrlen;
+                                                       state = 1;
+                                                       break;
+                                               case 1:
+                                                       /* continue src */
+                                                       srclen += ptrlen;
+                                                       break;
+                                               case 2:
+                                                       /* start of trg */
+                                                       trg = ptr;
+                                                       trglen = ptrlen;
+                                                       state = 3;
+                                                       break;
+                                               case 3:
+                                                       /* continue trg */
+                                                       trglen += ptrlen;
+                                                       break;
+                                               default:
+                                                       /* bogus line format */
+                                                       state = -1;
+                                                       break;
+                                       }
+                               }
 
-                               srclen = strlen(src);
-                               trglen = strlen(trg);
+                               if (state >= 3)
+                                       rootSuffixTree = placeChar(rootSuffixTree,
+                                                                                          (unsigned char *) src, srclen,
+                                                                                          trg, trglen);
 
-                               rootSuffixTree = placeChar(rootSuffixTree,
-                                                                                  (unsigned char *) src, srclen,
-                                                                                  trg, trglen);
-                               skip = false;
                                pfree(line);
                        }
+                       skip = false;
                }
                PG_CATCH();
                {