*
* Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group
*
+ * Ispell dictionary
+ * -----------------
+ *
+ * Rules of dictionaries are defined in two files with .affix and .dict
+ * extensions. They are used by spell checker programs Ispell and Hunspell.
+ *
+ * An .affix file declares morphological rules to get a basic form of words.
+ * The format of an .affix file has different structure for Ispell and Hunspell
+ * dictionaries. The Hunspell format is more complicated. But when an .affix
+ * file is imported and compiled, it is stored in the same structure AffixNode.
+ *
+ * A .dict file stores a list of basic forms of words with references to
+ * affix rules. The format of a .dict file has the same structure for Ispell
+ * and Hunspell dictionaries.
+ *
+ * Compilation of a dictionary
+ * ---------------------------
+ *
+ * A compiled dictionary is stored in the IspellDict structure. Compilation of
+ * a dictionary is divided into the several steps:
+ * - NIImportDictionary() - stores each word of a .dict file in the
+ * temporary Spell field.
+ * - NIImportAffixes() - stores affix rules of an .affix file in the
+ * Affix field (not temporary) if an .affix file has the Ispell format.
+ * -> NIImportOOAffixes() - stores affix rules if an .affix file has the
+ * Hunspell format. The AffixData field is initialized if AF parameter
+ * is defined.
+ * - NISortDictionary() - builds a prefix tree (Trie) from the words list
+ * and stores it in the Dictionary field. The words list is got from the
+ * Spell field. The AffixData field is initialized if AF parameter is not
+ * defined.
+ * - NISortAffixes():
+ * - builds a list of compond affixes from the affix list and stores it
+ * in the CompoundAffix.
+ * - builds prefix trees (Trie) from the affix list for prefixes and suffixes
+ * and stores them in Suffix and Prefix fields.
+ * The affix list is got from the Affix field.
+ *
+ * Memory management
+ * -----------------
+ *
+ * The IspellDict structure has the Spell field which is used only in compile
+ * time. The Spell field stores a words list. It can take a lot of memory.
+ * Therefore when a dictionary is compiled this field is cleared by
+ * NIFinishBuild().
+ *
+ * All resources which should cleared by NIFinishBuild() is initialized using
+ * tmpalloc() and tmpalloc0().
*
* IDENTIFICATION
* src/backend/tsearch/spell.c
{
return (strcmp((*(SPELL *const *) s1)->word, (*(SPELL *const *) s2)->word));
}
+
static int
cmpspellaffix(const void *s1, const void *s2)
{
- return (strncmp((*(SPELL *const *) s1)->p.flag, (*(SPELL *const *) s2)->p.flag, MAXFLAGLEN));
+ return (strcmp((*(SPELL *const *) s1)->p.flag,
+ (*(SPELL *const *) s2)->p.flag));
}
static char *
return 0;
}
+/*
+ * Compares affixes.
+ * First compares the type of an affix. Prefixes should go before affixes.
+ * If types are equal then compares replaceable string.
+ */
static int
cmpaffix(const void *s1, const void *s2)
{
(const unsigned char *) a2->repl);
}
+/*
+ * Gets an affix flag from string representation (a set of affixes).
+ *
+ * Several flags can be stored in a single string. Flags can be represented by:
+ * - 1 character (FM_CHAR).
+ * - 2 characters (FM_LONG).
+ * - numbers from 1 to 65000 (FM_NUM).
+ *
+ * Depending on the flagMode an affix string can have the following format:
+ * - FM_CHAR: ABCD
+ * Here we have 4 flags: A, B, C and D
+ * - FM_LONG: ABCDE*
+ * Here we have 3 flags: AB, CD and E*
+ * - FM_NUM: 200,205,50
+ * Here we have 3 flags: 200, 205 and 50
+ *
+ * Conf: current dictionary.
+ * sflag: string representation (a set of affixes) of an affix flag.
+ * sflagnext: returns reference to the start of a next affix flag in the sflag.
+ *
+ * Returns an integer representation of the affix flag.
+ */
+static uint16
+DecodeFlag(IspellDict *Conf, char *sflag, char **sflagnext)
+{
+ int32 s;
+ char *next;
+ unsigned char *usflag;
+
+ switch (Conf->flagMode)
+ {
+ case FM_LONG:
+ /*
+ * Hunspell docs says flag could contains only
+ * ASCII characters
+ */
+ if (!(pg_mblen(sflag) == 1 && isascii(sflag[0]) &&
+ pg_mblen(sflag + 1) == 1 && isascii(sflag[1])))
+ ereport(ERROR,
+ (errcode(ERRCODE_CONFIG_FILE_ERROR),
+ errmsg("non-ASCII affix flag \"%s\"", sflag)));
+
+ usflag = (unsigned char *)sflag;
+ s = ((int)usflag[0]) << 8 | ((int)usflag[1]);
+ if (sflagnext)
+ /* Go to start of the next flag */
+ *sflagnext = sflag + 2;
+ break;
+ case FM_NUM:
+ s = strtol(sflag, &next, 10);
+ if (s <= 0 || s > FLAGNUM_MAXSIZE)
+ ereport(ERROR,
+ (errcode(ERRCODE_CONFIG_FILE_ERROR),
+ errmsg("invalid affix flag \"%s\"", sflag)));
+
+ if (sflagnext)
+ {
+ /* Go to start of the next flag */
+ if (next)
+ {
+ bool met_comma = false;
+
+ while (*next)
+ {
+ if (!(pg_mblen(next) == 1 && isascii(*next)))
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_CONFIG_FILE_ERROR),
+ errmsg("non-ASCII affix flag \"%s\"",
+ sflag)));
+ }
+ else if (isdigit(*next))
+ {
+ if (!met_comma)
+ ereport(ERROR,
+ (errcode(ERRCODE_CONFIG_FILE_ERROR),
+ errmsg("invalid affix flag \"%s\"",
+ sflag)));
+ break;
+ }
+ else if (*next == ',')
+ {
+ if (met_comma)
+ ereport(ERROR,
+ (errcode(ERRCODE_CONFIG_FILE_ERROR),
+ errmsg("invalid affix flag \"%s\"",
+ sflag)));
+ met_comma = true;
+ }
+ else if (!isspace(*next))
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_CONFIG_FILE_ERROR),
+ errmsg("invalid character in affix flag \"%s\"", sflag)));
+ }
+
+ next++;
+ }
+
+ if (*next == '\0')
+ next = NULL;
+ }
+
+ *sflagnext = next;
+ }
+ break;
+ default:
+ if (!(pg_mblen(sflag) == 1 && isascii(*sflag)))
+ ereport(ERROR,
+ (errcode(ERRCODE_CONFIG_FILE_ERROR),
+ errmsg("non-ASCII affix flag \"%s\"", sflag)));
+
+ s = *sflag;
+ if (sflagnext)
+ /* Go to start of the next flag */
+ *sflagnext = sflag + pg_mblen(sflag);
+ }
+
+ return (uint16)s;
+}
+
+/*
+ * Checks if the affix set Conf->AffixData[affix] contains affixflag.
+ * Conf->AffixData[affix] is the string representation of an affix flags.
+ * Conf->AffixData[affix] does not contain affixflag if this flag is not used
+ * actually by the .dict file.
+ *
+ * Conf: current dictionary.
+ * affix: index of the Conf->AffixData array.
+ * affixflag: integer representation of the affix flag.
+ *
+ * Returns true if the string Conf->AffixData[affix] contains affixflag,
+ * otherwise returns false.
+ */
+static bool
+IsAffixFlagInUse(IspellDict *Conf, int affix, uint16 affixflag)
+{
+ char *flagcur;
+ char *flagnext = NULL;
+
+ if (affixflag == 0)
+ return true;
+
+ flagcur = Conf->AffixData[affix];
+
+ while (*flagcur)
+ {
+ /* Compare first affix flag in flagcur with affixflag */
+ if (DecodeFlag(Conf, flagcur, &flagnext) == affixflag)
+ return true;
+ /* Otherwise go to next flag */
+ if (flagnext)
+ flagcur = flagnext;
+ /* If we have not flags anymore then exit */
+ else
+ break;
+ }
+
+ /* Could not find affixflag */
+ return false;
+}
+
+/*
+ * Adds the new word into the temporary array Spell.
+ *
+ * Conf: current dictionary.
+ * word: new word.
+ * flag: set of affix flags. Integer representation of flag can be got by
+ * DecodeFlag().
+ */
static void
NIAddSpell(IspellDict *Conf, const char *word, const char *flag)
{
}
Conf->Spell[Conf->nspell] = (SPELL *) tmpalloc(SPELLHDRSZ + strlen(word) + 1);
strcpy(Conf->Spell[Conf->nspell]->word, word);
- strlcpy(Conf->Spell[Conf->nspell]->p.flag, flag, MAXFLAGLEN);
+ Conf->Spell[Conf->nspell]->p.flag = (*flag != '\0')
+ ? cpstrdup(Conf, flag) : VoidString;
Conf->nspell++;
}
/*
- * import dictionary
+ * Imports dictionary into the temporary array Spell.
*
- * Note caller must already have applied get_tsearch_config_filename
+ * Note caller must already have applied get_tsearch_config_filename.
+ *
+ * Conf: current dictionary.
+ * filename: path to the .dict file.
*/
void
NIImportDictionary(IspellDict *Conf, const char *filename)
{
char *s,
*pstr;
+ /* Set of affix flags */
const char *flag;
/* Extract flag from the line */
tsearch_readline_end(&trst);
}
-
+/*
+ * Searches a basic form of word in the prefix tree. This word was generated
+ * using an affix rule. This rule may not be presented in an affix set of
+ * a basic form of word.
+ *
+ * For example, we have the entry in the .dict file:
+ * meter/GMD
+ *
+ * The affix rule with the flag S:
+ * SFX S y ies [^aeiou]y
+ * is not presented here.
+ *
+ * The affix rule with the flag M:
+ * SFX M 0 's .
+ * is presented here.
+ *
+ * Conf: current dictionary.
+ * word: basic form of word.
+ * affixflag: integer representation of the affix flag, by which a basic form of
+ * word was generated.
+ * flag: compound flag used to compare with StopMiddle->compoundflag.
+ *
+ * Returns 1 if the word was found in the prefix tree, else returns 0.
+ */
static int
FindWord(IspellDict *Conf, const char *word, int affixflag, int flag)
{
{
if (flag == 0)
{
+ /*
+ * The word can be formed only with another word.
+ * And in the flag parameter there is not a sign
+ * that we search compound words.
+ */
if (StopMiddle->compoundflag & FF_COMPOUNDONLY)
return 0;
}
else if ((flag & StopMiddle->compoundflag) == 0)
return 0;
- if ((affixflag == 0) || (strchr(Conf->AffixData[StopMiddle->affix], affixflag) != NULL))
+ /*
+ * Check if this affix rule is presented in the affix set
+ * with index StopMiddle->affix.
+ */
+ if (IsAffixFlagInUse(Conf, StopMiddle->affix, affixflag))
return 1;
}
node = StopMiddle->node;
return 0;
}
+/*
+ * Adds a new affix rule to the Affix field.
+ *
+ * Conf: current dictionary.
+ * flag: integer representation of the affix flag ('\' in the below example).
+ * flagflags: set of flags from the flagval field for this affix rule. This set
+ * is listed after '/' character in the added string (repl).
+ *
+ * For example L flag in the hunspell_sample.affix:
+ * SFX \ 0 Y/L [^Y]
+ *
+ * mask: condition for search ('[^Y]' in the above example).
+ * find: stripping characters from beginning (at prefix) or end (at suffix)
+ * of the word ('0' in the above example, 0 means that there is not
+ * stripping character).
+ * repl: adding string after stripping ('Y' in the above example).
+ * type: FF_SUFFIX or FF_PREFIX.
+ */
static void
NIAddAffix(IspellDict *Conf, int flag, char flagflags, const char *mask, const char *find, const char *repl, int type)
{
Affix = Conf->Affix + Conf->naffixes;
- if (strcmp(mask, ".") == 0)
+ /* This affix rule can be applied for words with any ending */
+ if (strcmp(mask, ".") == 0 || *mask == '\0')
{
Affix->issimple = 1;
Affix->isregis = 0;
}
+ /* This affix rule will use regis to search word ending */
else if (RS_isRegis(mask))
{
Affix->issimple = 0;
Affix->isregis = 1;
- RS_compile(&(Affix->reg.regis), (type == FF_SUFFIX) ? true : false,
+ RS_compile(&(Affix->reg.regis), (type == FF_SUFFIX),
*mask ? mask : VoidString);
}
+ /* This affix rule will use regex_t to search word ending */
else
{
int masklen;
Conf->naffixes++;
}
-
/* Parsing states for parse_affentry() and friends */
#define PAE_WAIT_MASK 0
#define PAE_INMASK 1
*pmask = *pfind = *prepl = '\0';
- return (*mask && (*find || *repl)) ? true : false;
+ return (*mask && (*find || *repl));
}
+/*
+ * Sets up a correspondence for the affix parameter with the affix flag.
+ *
+ * Conf: current dictionary.
+ * s: affix flag in string.
+ * val: affix parameter.
+ */
static void
addFlagValue(IspellDict *Conf, char *s, uint32 val)
{
(errcode(ERRCODE_CONFIG_FILE_ERROR),
errmsg("multibyte flag character is not allowed")));
- Conf->flagval[*(unsigned char *) s] = (unsigned char) val;
+ Conf->flagval[DecodeFlag(Conf, s, (char **)NULL)] = (unsigned char) val;
Conf->usecompound = true;
}
/*
- * Import an affix file that follows MySpell or Hunspell format
+ * Returns a set of affix parameters which correspondence to the set of affix
+ * flags s.
+ */
+static int
+getFlagValues(IspellDict *Conf, char *s)
+{
+ uint32 flag = 0;
+ char *flagcur;
+ char *flagnext = NULL;
+
+ flagcur = s;
+ while (*flagcur)
+ {
+ flag |= Conf->flagval[DecodeFlag(Conf, flagcur, &flagnext)];
+ if (flagnext)
+ flagcur = flagnext;
+ else
+ break;
+ }
+
+ return flag;
+}
+
+/*
+ * Returns a flag set using the s parameter.
+ *
+ * If Conf->useFlagAliases is true then the s parameter is index of the
+ * Conf->AffixData array and function returns its entry.
+ * Else function returns the s parameter.
+ */
+static char *
+getFlags(IspellDict *Conf, char *s)
+{
+ if (Conf->useFlagAliases)
+ {
+ int curaffix = strtol(s, (char **)NULL, 10);
+
+ if (curaffix && curaffix <= Conf->nAffixData)
+ /*
+ * Do not substract 1 from curaffix
+ * because empty string was added in NIImportOOAffixes
+ */
+ return Conf->AffixData[curaffix];
+ else
+ return VoidString;
+ }
+ else
+ return s;
+}
+
+/*
+ * Import an affix file that follows MySpell or Hunspell format.
+ *
+ * Conf: current dictionary.
+ * filename: path to the .affix file.
*/
static void
NIImportOOAffixes(IspellDict *Conf, const char *filename)
char repl[BUFSIZ],
*prepl;
bool isSuffix = false;
- int flag = 0;
+ int naffix = 0,
+ curaffix = 0;
+ int flag = 0,
+ sflaglen = 0;
char flagflags = 0;
tsearch_readline_state trst;
char *recoded;
/* read file to find any flag */
memset(Conf->flagval, 0, sizeof(Conf->flagval));
Conf->usecompound = false;
+ Conf->useFlagAliases = false;
+ Conf->flagMode = FM_CHAR;
if (!tsearch_readline_begin(&trst, filename))
ereport(ERROR,
while (*s && t_isspace(s))
s += pg_mblen(s);
- if (*s && STRNCMP(s, "default") != 0)
- ereport(ERROR,
+ if (*s)
+ {
+ if (STRNCMP(s, "long") == 0)
+ Conf->flagMode = FM_LONG;
+ else if (STRNCMP(s, "num") == 0)
+ Conf->flagMode = FM_NUM;
+ else if (STRNCMP(s, "default") != 0)
+ ereport(ERROR,
(errcode(ERRCODE_CONFIG_FILE_ERROR),
- errmsg("Ispell dictionary supports only default flag value")));
+ errmsg("Ispell dictionary supports only default, "
+ "long and num flag value")));
+ }
}
pfree(recoded);
if (ptype)
pfree(ptype);
ptype = lowerstr_ctx(Conf, type);
+
+ /* First try to parse AF parameter (alias compression) */
+ if (STRNCMP(ptype, "af") == 0)
+ {
+ /* First line is the number of aliases */
+ if (!Conf->useFlagAliases)
+ {
+ Conf->useFlagAliases = true;
+ naffix = atoi(sflag);
+ if (naffix == 0)
+ ereport(ERROR,
+ (errcode(ERRCODE_CONFIG_FILE_ERROR),
+ errmsg("invalid number of flag vector aliases")));
+
+ /* Also reserve place for empty flag set */
+ naffix++;
+
+ Conf->AffixData = (char **) palloc0(naffix * sizeof(char *));
+ Conf->lenAffixData = Conf->nAffixData = naffix;
+
+ /* Add empty flag set into AffixData */
+ Conf->AffixData[curaffix] = VoidString;
+ curaffix++;
+ }
+ /* Other lines is aliases */
+ else
+ {
+ if (curaffix < naffix)
+ {
+ Conf->AffixData[curaffix] = cpstrdup(Conf, sflag);
+ curaffix++;
+ }
+ }
+ goto nextline;
+ }
+ /* Else try to parse prefixes and suffixes */
if (fields_read < 4 ||
(STRNCMP(ptype, "sfx") != 0 && STRNCMP(ptype, "pfx") != 0))
goto nextline;
+ sflaglen = strlen(sflag);
+ if (sflaglen == 0
+ || (sflaglen > 1 && Conf->flagMode == FM_CHAR)
+ || (sflaglen > 2 && Conf->flagMode == FM_LONG))
+ goto nextline;
+
+ /*
+ * Affix header. For example:
+ * SFX \ N 1
+ */
if (fields_read == 4)
{
- if (strlen(sflag) != 1)
- goto nextline;
- flag = *sflag;
- isSuffix = (STRNCMP(ptype, "sfx") == 0) ? true : false;
+ /* Convert the affix flag to int */
+ flag = DecodeFlag(Conf, sflag, (char **)NULL);
+
+ isSuffix = (STRNCMP(ptype, "sfx") == 0);
if (t_iseq(find, 'y') || t_iseq(find, 'Y'))
flagflags = FF_CROSSPRODUCT;
else
flagflags = 0;
}
+ /*
+ * Affix fields. For example:
+ * SFX \ 0 Y/L [^Y]
+ */
else
{
char *ptr;
int aflg = 0;
- if (strlen(sflag) != 1 || flag != *sflag || flag == 0)
+ if (flag == 0)
goto nextline;
prepl = lowerstr_ctx(Conf, repl);
/* Find position of '/' in lowercased string "prepl" */
*/
*ptr = '\0';
ptr = repl + (ptr - prepl) + 1;
- while (*ptr)
- {
- aflg |= Conf->flagval[*(unsigned char *) ptr];
- ptr++;
- }
+ aflg |= getFlagValues(Conf, getFlags(Conf, ptr));
}
pfind = lowerstr_ctx(Conf, find);
pmask = lowerstr_ctx(Conf, mask);
memset(Conf->flagval, 0, sizeof(Conf->flagval));
Conf->usecompound = false;
+ Conf->useFlagAliases = false;
+ Conf->flagMode = FM_CHAR;
while ((recoded = tsearch_readline(&trst)) != NULL)
{
NIImportOOAffixes(Conf, filename);
}
+/*
+ * Merges two affix flag sets and stores a new affix flag set into
+ * Conf->AffixData.
+ *
+ * Returns index of a new affix flag set.
+ */
static int
MergeAffix(IspellDict *Conf, int a1, int a2)
{
return Conf->nAffixData - 1;
}
+/*
+ * Returns a set of affix parameters which correspondence to the set of affix
+ * flags with the given index.
+ */
static uint32
makeCompoundFlags(IspellDict *Conf, int affix)
{
- uint32 flag = 0;
- char *str = Conf->AffixData[affix];
-
- while (str && *str)
- {
- flag |= Conf->flagval[*(unsigned char *) str];
- str++;
- }
-
- return (flag & FF_DICTFLAGMASK);
+ char *str = Conf->AffixData[affix];
+ return (getFlagValues(Conf, str) & FF_DICTFLAGMASK);
}
+/*
+ * Makes a prefix tree for the given level.
+ *
+ * Conf: current dictionary.
+ * low: lower index of the Conf->Spell array.
+ * high: upper index of the Conf->Spell array.
+ * level: current prefix tree level.
+ */
static SPNode *
mkSPNode(IspellDict *Conf, int low, int high, int level)
{
{
if (lastchar)
{
+ /* Next level of the prefix tree */
data->node = mkSPNode(Conf, lownew, i, level + 1);
lownew = i;
data++;
}
}
+ /* Next level of the prefix tree */
data->node = mkSPNode(Conf, lownew, high, level + 1);
return rs;
/* compress affixes */
- /* Count the number of different flags used in the dictionary */
-
- qsort((void *) Conf->Spell, Conf->nspell, sizeof(SPELL *), cmpspellaffix);
-
- naffix = 0;
- for (i = 0; i < Conf->nspell; i++)
+ /*
+ * If we use flag aliases then we need to use Conf->AffixData filled
+ * in the NIImportOOAffixes().
+ */
+ if (Conf->useFlagAliases)
{
- if (i == 0 || strncmp(Conf->Spell[i]->p.flag, Conf->Spell[i - 1]->p.flag, MAXFLAGLEN))
- naffix++;
+ for (i = 0; i < Conf->nspell; i++)
+ {
+ curaffix = strtol(Conf->Spell[i]->p.flag, (char **)NULL, 10);
+ if (curaffix && curaffix <= Conf->nAffixData)
+ Conf->Spell[i]->p.d.affix = curaffix;
+ else
+ /*
+ * If Conf->Spell[i]->p.flag is empty, then get empty value of
+ * Conf->AffixData (0 index).
+ */
+ Conf->Spell[i]->p.d.affix = 0;
+ Conf->Spell[i]->p.d.len = strlen(Conf->Spell[i]->word);
+ }
}
+ /* Otherwise fill Conf->AffixData here */
+ else
+ {
+ /* Count the number of different flags used in the dictionary */
+ qsort((void *) Conf->Spell, Conf->nspell, sizeof(SPELL *),
+ cmpspellaffix);
- /*
- * Fill in Conf->AffixData with the affixes that were used in the
- * dictionary. Replace textual flag-field of Conf->Spell entries with
- * indexes into Conf->AffixData array.
- */
- Conf->AffixData = (char **) palloc0(naffix * sizeof(char *));
+ naffix = 0;
+ for (i = 0; i < Conf->nspell; i++)
+ {
+ if (i == 0
+ || strcmp(Conf->Spell[i]->p.flag, Conf->Spell[i - 1]->p.flag))
+ naffix++;
+ }
- curaffix = -1;
- for (i = 0; i < Conf->nspell; i++)
- {
- if (i == 0 || strncmp(Conf->Spell[i]->p.flag, Conf->AffixData[curaffix], MAXFLAGLEN))
+ /*
+ * Fill in Conf->AffixData with the affixes that were used in the
+ * dictionary. Replace textual flag-field of Conf->Spell entries with
+ * indexes into Conf->AffixData array.
+ */
+ Conf->AffixData = (char **) palloc0(naffix * sizeof(char *));
+
+ curaffix = -1;
+ for (i = 0; i < Conf->nspell; i++)
{
- curaffix++;
- Assert(curaffix < naffix);
- Conf->AffixData[curaffix] = cpstrdup(Conf, Conf->Spell[i]->p.flag);
+ if (i == 0
+ || strcmp(Conf->Spell[i]->p.flag, Conf->AffixData[curaffix]))
+ {
+ curaffix++;
+ Assert(curaffix < naffix);
+ Conf->AffixData[curaffix] = cpstrdup(Conf,
+ Conf->Spell[i]->p.flag);
+ }
+
+ Conf->Spell[i]->p.d.affix = curaffix;
+ Conf->Spell[i]->p.d.len = strlen(Conf->Spell[i]->word);
}
- Conf->Spell[i]->p.d.affix = curaffix;
- Conf->Spell[i]->p.d.len = strlen(Conf->Spell[i]->word);
+ Conf->lenAffixData = Conf->nAffixData = naffix;
}
- Conf->lenAffixData = Conf->nAffixData = naffix;
-
+ /* Start build a prefix tree */
qsort((void *) Conf->Spell, Conf->nspell, sizeof(SPELL *), cmpspell);
Conf->Dictionary = mkSPNode(Conf, 0, Conf->nspell, 0);
}
+/*
+ * Makes a prefix tree for the given level using the repl string of an affix
+ * rule. Affixes with empty replace string do not include in the prefix tree.
+ * This affixes are included by mkVoidAffix().
+ *
+ * Conf: current dictionary.
+ * low: lower index of the Conf->Affix array.
+ * high: upper index of the Conf->Affix array.
+ * level: current prefix tree level.
+ * type: FF_SUFFIX or FF_PREFIX.
+ */
static AffixNode *
mkANode(IspellDict *Conf, int low, int high, int level, int type)
{
{
if (lastchar)
{
+ /* Next level of the prefix tree */
data->node = mkANode(Conf, lownew, i, level + 1, type);
if (naff)
{
}
}
+ /* Next level of the prefix tree */
data->node = mkANode(Conf, lownew, high, level + 1, type);
if (naff)
{
return rs;
}
+/*
+ * Makes the root void node in the prefix tree. The root void node is created
+ * for affixes which have empty replace string ("repl" field).
+ */
static void
mkVoidAffix(IspellDict *Conf, bool issuffix, int startsuffix)
{
Conf->Prefix = Affix;
}
-
+ /* Count affixes with empty replace string */
for (i = start; i < end; i++)
if (Conf->Affix[i].replen == 0)
cnt++;
+ /* There is not affixes with empty replace string */
if (cnt == 0)
return;
}
}
+/*
+ * Checks if the affixflag is used by dictionary. Conf->AffixData does not
+ * contain affixflag if this flag is not used actually by the .dict file.
+ *
+ * Conf: current dictionary.
+ * affixflag: integer representation of the affix flag.
+ *
+ * Returns true if the Conf->AffixData array contains affixflag, otherwise
+ * returns false.
+ */
static bool
-isAffixInUse(IspellDict *Conf, char flag)
+isAffixInUse(IspellDict *Conf, uint16 affixflag)
{
int i;
for (i = 0; i < Conf->nAffixData; i++)
- if (strchr(Conf->AffixData[i], flag) != NULL)
+ if (IsAffixFlagInUse(Conf, i, affixflag))
return true;
return false;
}
+/*
+ * Builds Conf->Prefix and Conf->Suffix trees from the imported affixes.
+ */
void
NISortAffixes(IspellDict *Conf)
{
if (Conf->naffixes == 0)
return;
+ /* Store compound affixes in the Conf->CompoundAffix array */
if (Conf->naffixes > 1)
qsort((void *) Conf->Affix, Conf->naffixes, sizeof(AFFIX), cmpaffix);
Conf->CompoundAffix = ptr = (CMPDAffix *) palloc(sizeof(CMPDAffix) * Conf->naffixes);
firstsuffix = i;
if ((Affix->flagflags & FF_COMPOUNDFLAG) && Affix->replen > 0 &&
- isAffixInUse(Conf, (char) Affix->flag))
+ isAffixInUse(Conf, Affix->flag))
{
if (ptr == Conf->CompoundAffix ||
ptr->issuffix != (ptr - 1)->issuffix ||
/* leave only unique and minimals suffixes */
ptr->affix = Affix->repl;
ptr->len = Affix->replen;
- ptr->issuffix = (Affix->type == FF_SUFFIX) ? true : false;
+ ptr->issuffix = (Affix->type == FF_SUFFIX);
ptr++;
}
}
ptr->affix = NULL;
Conf->CompoundAffix = (CMPDAffix *) repalloc(Conf->CompoundAffix, sizeof(CMPDAffix) * (ptr - Conf->CompoundAffix + 1));
+ /* Start build a prefix tree */
Conf->Prefix = mkANode(Conf, 0, firstsuffix, 0, FF_PREFIX);
Conf->Suffix = mkANode(Conf, firstsuffix, Conf->naffixes, 0, FF_SUFFIX);
mkVoidAffix(Conf, true, firstsuffix);
if (StopLow < StopHigh)
{
- if (level == FF_COMPOUNDBEGIN)
+ if (startpos == 0)
compoundflag = FF_COMPOUNDBEGIN;
else if (level == wordlen - 1)
compoundflag = FF_COMPOUNDLAST;
{foot,ball,klubber}
(1 row)
+-- Test ISpell dictionary with hunspell affix file with FLAG long parameter
+CREATE TEXT SEARCH DICTIONARY hunspell_long (
+ Template=ispell,
+ DictFile=hunspell_sample_long,
+ AffFile=hunspell_sample_long
+);
+SELECT ts_lexize('hunspell_long', 'skies');
+ ts_lexize
+-----------
+ {sky}
+(1 row)
+
+SELECT ts_lexize('hunspell_long', 'bookings');
+ ts_lexize
+----------------
+ {booking,book}
+(1 row)
+
+SELECT ts_lexize('hunspell_long', 'booking');
+ ts_lexize
+----------------
+ {booking,book}
+(1 row)
+
+SELECT ts_lexize('hunspell_long', 'foot');
+ ts_lexize
+-----------
+ {foot}
+(1 row)
+
+SELECT ts_lexize('hunspell_long', 'foots');
+ ts_lexize
+-----------
+ {foot}
+(1 row)
+
+SELECT ts_lexize('hunspell_long', 'rebookings');
+ ts_lexize
+----------------
+ {booking,book}
+(1 row)
+
+SELECT ts_lexize('hunspell_long', 'rebooking');
+ ts_lexize
+----------------
+ {booking,book}
+(1 row)
+
+SELECT ts_lexize('hunspell_long', 'rebook');
+ ts_lexize
+-----------
+
+(1 row)
+
+SELECT ts_lexize('hunspell_long', 'unbookings');
+ ts_lexize
+-----------
+ {book}
+(1 row)
+
+SELECT ts_lexize('hunspell_long', 'unbooking');
+ ts_lexize
+-----------
+ {book}
+(1 row)
+
+SELECT ts_lexize('hunspell_long', 'unbook');
+ ts_lexize
+-----------
+ {book}
+(1 row)
+
+SELECT ts_lexize('hunspell_long', 'footklubber');
+ ts_lexize
+----------------
+ {foot,klubber}
+(1 row)
+
+SELECT ts_lexize('hunspell_long', 'footballklubber');
+ ts_lexize
+------------------------------------------------------
+ {footballklubber,foot,ball,klubber,football,klubber}
+(1 row)
+
+SELECT ts_lexize('hunspell_long', 'ballyklubber');
+ ts_lexize
+----------------
+ {ball,klubber}
+(1 row)
+
+SELECT ts_lexize('hunspell_long', 'footballyklubber');
+ ts_lexize
+---------------------
+ {foot,ball,klubber}
+(1 row)
+
+-- Test ISpell dictionary with hunspell affix file with FLAG num parameter
+CREATE TEXT SEARCH DICTIONARY hunspell_num (
+ Template=ispell,
+ DictFile=hunspell_sample_num,
+ AffFile=hunspell_sample_num
+);
+SELECT ts_lexize('hunspell_num', 'skies');
+ ts_lexize
+-----------
+ {sky}
+(1 row)
+
+SELECT ts_lexize('hunspell_num', 'bookings');
+ ts_lexize
+----------------
+ {booking,book}
+(1 row)
+
+SELECT ts_lexize('hunspell_num', 'booking');
+ ts_lexize
+----------------
+ {booking,book}
+(1 row)
+
+SELECT ts_lexize('hunspell_num', 'foot');
+ ts_lexize
+-----------
+ {foot}
+(1 row)
+
+SELECT ts_lexize('hunspell_num', 'foots');
+ ts_lexize
+-----------
+ {foot}
+(1 row)
+
+SELECT ts_lexize('hunspell_num', 'rebookings');
+ ts_lexize
+----------------
+ {booking,book}
+(1 row)
+
+SELECT ts_lexize('hunspell_num', 'rebooking');
+ ts_lexize
+----------------
+ {booking,book}
+(1 row)
+
+SELECT ts_lexize('hunspell_num', 'rebook');
+ ts_lexize
+-----------
+
+(1 row)
+
+SELECT ts_lexize('hunspell_num', 'unbookings');
+ ts_lexize
+-----------
+ {book}
+(1 row)
+
+SELECT ts_lexize('hunspell_num', 'unbooking');
+ ts_lexize
+-----------
+ {book}
+(1 row)
+
+SELECT ts_lexize('hunspell_num', 'unbook');
+ ts_lexize
+-----------
+ {book}
+(1 row)
+
+SELECT ts_lexize('hunspell_num', 'footklubber');
+ ts_lexize
+----------------
+ {foot,klubber}
+(1 row)
+
+SELECT ts_lexize('hunspell_num', 'footballklubber');
+ ts_lexize
+------------------------------------------------------
+ {footballklubber,foot,ball,klubber,football,klubber}
+(1 row)
+
+SELECT ts_lexize('hunspell_num', 'ballyklubber');
+ ts_lexize
+----------------
+ {ball,klubber}
+(1 row)
+
+SELECT ts_lexize('hunspell_num', 'footballyklubber');
+ ts_lexize
+---------------------
+ {foot,ball,klubber}
+(1 row)
+
-- Synonim dictionary
CREATE TEXT SEARCH DICTIONARY synonym (
Template=synonym,
'foot':B & 'ball':B & 'klubber':B & ( 'booking':A | 'book':A ) & 'sky'
(1 row)
+-- Test ispell dictionary with hunspell affix with FLAG long in configuration
+ALTER TEXT SEARCH CONFIGURATION hunspell_tst ALTER MAPPING
+ REPLACE hunspell WITH hunspell_long;
+SELECT to_tsvector('hunspell_tst', 'Booking the skies after rebookings for footballklubber from a foot');
+ to_tsvector
+----------------------------------------------------------------------------------------------------
+ 'ball':7 'book':1,5 'booking':1,5 'foot':7,10 'football':7 'footballklubber':7 'klubber':7 'sky':3
+(1 row)
+
+SELECT to_tsquery('hunspell_tst', 'footballklubber');
+ to_tsquery
+------------------------------------------------------------------------------
+ ( 'footballklubber' | 'foot' & 'ball' & 'klubber' ) | 'football' & 'klubber'
+(1 row)
+
+SELECT to_tsquery('hunspell_tst', 'footballyklubber:b & rebookings:A & sky');
+ to_tsquery
+------------------------------------------------------------------------
+ 'foot':B & 'ball':B & 'klubber':B & ( 'booking':A | 'book':A ) & 'sky'
+(1 row)
+
+-- Test ispell dictionary with hunspell affix with FLAG num in configuration
+ALTER TEXT SEARCH CONFIGURATION hunspell_tst ALTER MAPPING
+ REPLACE hunspell_long WITH hunspell_num;
+SELECT to_tsvector('hunspell_tst', 'Booking the skies after rebookings for footballklubber from a foot');
+ to_tsvector
+----------------------------------------------------------------------------------------------------
+ 'ball':7 'book':1,5 'booking':1,5 'foot':7,10 'football':7 'footballklubber':7 'klubber':7 'sky':3
+(1 row)
+
+SELECT to_tsquery('hunspell_tst', 'footballklubber');
+ to_tsquery
+------------------------------------------------------------------------------
+ ( 'footballklubber' | 'foot' & 'ball' & 'klubber' ) | 'football' & 'klubber'
+(1 row)
+
+SELECT to_tsquery('hunspell_tst', 'footballyklubber:b & rebookings:A & sky');
+ to_tsquery
+------------------------------------------------------------------------
+ 'foot':B & 'ball':B & 'klubber':B & ( 'booking':A | 'book':A ) & 'sky'
+(1 row)
+
-- Test synonym dictionary in configuration
CREATE TEXT SEARCH CONFIGURATION synonym_tst (
COPY=english
projects / postgresql / commitdiff