]> granicus.if.org Git - postgresql/commitdiff
1 Eliminate duplicate field HLWORD->skip
authorTeodor Sigaev <teodor@sigaev.ru>
Mon, 28 Jun 2004 16:19:09 +0000 (16:19 +0000)
committerTeodor Sigaev <teodor@sigaev.ru>
Mon, 28 Jun 2004 16:19:09 +0000 (16:19 +0000)
2 Rework support for html tags in parser
3 add HighlightAll to headline function for generating highlighted
  whole text with saved html tags

contrib/tsearch2/expected/tsearch2.out
contrib/tsearch2/sql/tsearch2.sql
contrib/tsearch2/ts_cfg.c
contrib/tsearch2/ts_cfg.h
contrib/tsearch2/wordparser/parser.l
contrib/tsearch2/wparser_def.c

index fb836c087a1b2dad2ffdde7ae1a9ea0e71c229f7..93fc11dad14939752dc8abd00d74d1023c856fe3 100644 (file)
@@ -458,20 +458,20 @@ select * from parse('default', '345 qwe@efd.r \' http://www.com/ http://aew.werc
     12 |  
      1 | asdf
     12 |  
-    13 |  
+    13 | <fr>
      1 | qwer
     12 |  
      1 | jf
     12 |  
      1 | sdjk
-    13 |  
+    13 | <we hjwer <werrwe>
     12 |  
      3 | ewr1
     12 | >
     12 |  
      3 | ewri2
     12 |  
-    13 |  
+    13 | <a href="qwe<qwe>">
     12 | 
 
     19 | /usr/local/fff
@@ -515,7 +515,7 @@ select * from parse('default', '345 qwe@efd.r \' http://www.com/ http://aew.werc
     22 | 234
     12 |  
 
-    13 |  
+    13 | <i <b>
     12 |  
      1 | wow
     12 |   
@@ -2130,6 +2130,35 @@ A thousand years to trace
 The granite features of this cliff
 (1 row)
 
+select headline('
+<html>
+<!-- some comment -->
+<body>
+Sea view wow <u>foo bar</u> <i>qq</i>
+<a href="http://www.google.com/foo.bar.html" target="_blank">YES &nbsp;</a>
+ff-bg
+<script>
+       document.write(15);
+</script>
+</body>
+</html>', 
+to_tsquery('sea&foo'), 'HighlightAll=true');
+                                                                                                              headline                                                                                                               
+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+<html>
+<!-- some comment -->
+<body>
+<b>Sea</b> view wow <u><b>foo</b> bar</u> <i>qq</i>
+<a href="http://www.google.com/foo.bar.html" target="_blank">YES &nbsp;</a>
+ ff-bg
+<script>
+       document.write(15);
+</script>
+</body>
+</html>
+(1 row)
+
 --check debug
 select * from ts_debug('Tsearch module for PostgreSQL 7.3.3');
  ts_name | tok_type | description |   token    | dict_name |   tsvector   
index 231ddaebe5e957c1befbd94707952c2085ea9d44..0a980608f7c2631e6d7109bb258d29fdc29d2bdf 100644 (file)
@@ -253,6 +253,20 @@ The sculpture of these granite seams,
 Upon a woman s face. E.  J.  Pratt  (1882 1964)
 ', to_tsquery('sea'));
 
+
+select headline('
+<html>
+<!-- some comment -->
+<body>
+Sea view wow <u>foo bar</u> <i>qq</i>
+<a href="http://www.google.com/foo.bar.html" target="_blank">YES &nbsp;</a>
+ff-bg
+<script>
+       document.write(15);
+</script>
+</body>
+</html>', 
+to_tsquery('sea&foo'), 'HighlightAll=true');
 --check debug
 select * from ts_debug('Tsearch module for PostgreSQL 7.3.3');
 
index efd79a1e32ffa8755a26f2aa22a81f969ba9b612..4e0a0bb90436e1728148ee6db7c5b433bd66bbb0 100644 (file)
@@ -510,7 +510,7 @@ genhl(HLPRSTEXT * prs)
                        ptr = ((char *) out) + dist;
                }
 
-               if (wrd->in && !wrd->skip && !wrd->repeated)
+               if (wrd->in && !wrd->repeated)
                {
                        if (wrd->replace)
                        {
@@ -532,7 +532,7 @@ genhl(HLPRSTEXT * prs)
                                        ptr += prs->stopsellen;
                                }
                        }
-               }
+               } else
 
                if (!wrd->repeated)
                        pfree(wrd->word);
index 9bf65144b208591b167b44869a6a5e1463ab0b2d..e000233178d1e556029ba7f8a46e336644765124 100644 (file)
@@ -46,13 +46,13 @@ typedef struct
 
 typedef struct
 {
-       uint16          len;
-       uint8           selected:1,
+       uint32          selected:1,
                                in:1,
-                               skip:1,
                                replace:1,
-                               repeated:1;
-       uint8           type;
+                               repeated:1,
+                               unused:4,
+                               type:8,
+                               len:16;
        char       *word;
        ITEM       *item;
 }      HLWORD;
index e80f5fea903061699972dcb56dd8fe8ff07afdb7..8c46edf7b8b25d660980ac17fd3839efa4c1b204 100644 (file)
 
 char *token = NULL;  /* pointer to token */
 int tokenlen;
-char *s     = NULL;  /* to return WHOLE hyphenated-word */
+static char *s     = NULL;  /* to return WHOLE hyphenated-word */
 
 YY_BUFFER_STATE buf = NULL; /* buffer to parse; it need for parse from string */
 
+typedef struct {
+       int tlen;
+       int clen;
+       char *str;
+} TagStorage;
+
+static TagStorage ts={0,0,NULL};
+
+static void
+addTag() {
+       while( ts.clen+tsearch2_yyleng+1 > ts.tlen ) {
+               ts.tlen*=2;
+               ts.str=realloc(ts.str,ts.tlen);
+               if (!ts.str)
+                       ereport(ERROR,
+                                       (errcode(ERRCODE_OUT_OF_MEMORY),
+                                        errmsg("out of memory")));
+        }
+        memcpy(ts.str+ts.clen,tsearch2_yytext,tsearch2_yyleng);
+        ts.clen+=tsearch2_yyleng;
+       ts.str[ts.clen]='\0';
+}
+
+static void
+startTag() {
+       if ( ts.str==NULL ) {
+               ts.tlen=tsearch2_yyleng+1;
+               ts.str=malloc(ts.tlen);
+               if (!ts.str)
+                       ereport(ERROR,
+                                (errcode(ERRCODE_OUT_OF_MEMORY),
+                                 errmsg("out of memory")));
+       }
+       ts.clen=0;
+       ts.str[0]='\0';
+       addTag();
+}
+
 %}
 
 %option 8bit
@@ -46,47 +84,46 @@ URI         [-_[:alnum:]/%,\.;=&?#]+
 
 %%
 
-"<"[Ss][Cc][Rr][Ii][Pp][Tt] { BEGIN INSCRIPT; }
+"<"[Ss][Cc][Rr][Ii][Pp][Tt] { BEGIN INSCRIPT; startTag(); }
 
 <INSCRIPT>"</"[Ss][Cc][Rr][Ii][Pp][Tt]">" {
        BEGIN INITIAL; 
-       *tsearch2_yytext=' '; *(tsearch2_yytext+1) = '\0'; 
-       token = tsearch2_yytext;
-       tokenlen = tsearch2_yyleng;
-       return SPACE;
+       addTag();
+       token = ts.str;
+       tokenlen = ts.clen;
+       return TAG;
 }
 
-"<!--" { BEGIN INCOMMENT; }
+"<!--" { BEGIN INCOMMENT; startTag(); }
 
 <INCOMMENT>"-->"       { 
        BEGIN INITIAL;
-       *tsearch2_yytext=' '; *(tsearch2_yytext+1) = '\0'; 
-       token = tsearch2_yytext;
-       tokenlen = tsearch2_yyleng;
-       return SPACE;
+       addTag();
+       token = ts.str;
+       tokenlen = ts.clen;
+       return TAG;
 }
 
 
-"<"[\![:alpha:]]       { BEGIN INTAG; }
+"<"[\![:alpha:]]       { BEGIN INTAG; startTag(); }
 
-"</"[[:alpha:]]        { BEGIN INTAG; }
+"</"[[:alpha:]]        { BEGIN INTAG; startTag(); }
 
-<INTAG>"\""    { BEGIN QINTAG; }
+<INTAG>"\""    { BEGIN QINTAG; addTag(); }
 
-<QINTAG>"\\\"" ;
+<QINTAG>"\\\"" { addTag(); }
 
-<QINTAG>"\""   { BEGIN INTAG; }
+<QINTAG>"\""   { BEGIN INTAG; addTag(); }
 
 <INTAG>">"     { 
        BEGIN INITIAL;
-       token = tsearch2_yytext;
-       *tsearch2_yytext=' '; 
-       token = tsearch2_yytext;
-       tokenlen = 1;
+       addTag();
+       token = ts.str;
+       tokenlen = ts.clen;
        return TAG;
 }
 
-<QINTAG,INTAG,INCOMMENT,INSCRIPT>.|\n  ;
+<QINTAG,INTAG,INCOMMENT,INSCRIPT>.|\n { addTag(); }    
 
 \&(quot|amp|nbsp|lt|gt)\;   {
        token = tsearch2_yytext;
@@ -295,3 +332,4 @@ void tsearch2_start_parse_str(char* str, int limit) {
        tsearch2_yy_switch_to_buffer( buf );
        BEGIN INITIAL;
 }
+
index a3d61126282c0f07998693d7c490879e2b70d275..035e5f2495d62ab6ff47fa0275092d096c91fed1 100644 (file)
@@ -78,6 +78,7 @@ prsd_end(PG_FUNCTION_ARGS)
 
 #define IDIGNORE(x) ( (x)==13 || (x)==14 || (x)==12 || (x)==23 )
 #define HLIDIGNORE(x) ( (x)==5 || (x)==13 || (x)==15 || (x)==16 || (x)==17 )
+#define HTMLHLIDIGNORE(x) ( (x)==5 || (x)==15 || (x)==16 || (x)==17 )
 #define NONWORDTOKEN(x) ( (x)==12 || HLIDIGNORE(x) )
 #define NOENDTOKEN(x)  ( NONWORDTOKEN(x) || (x)==7 || (x)==8 || (x)==20 || (x)==21 || (x)==22 || IDIGNORE(x) )
 
@@ -196,6 +197,7 @@ prsd_headline(PG_FUNCTION_ARGS)
                                curlen;
 
        int                     i;
+       int                     highlight=0;
 
        /* config */
        prs->startsel = NULL;
@@ -220,6 +222,15 @@ prsd_headline(PG_FUNCTION_ARGS)
                                prs->startsel = pstrdup(mptr->value);
                        else if (pg_strcasecmp(mptr->key, "StopSel") == 0)
                                prs->stopsel = pstrdup(mptr->value);
+                       else if (pg_strcasecmp(mptr->key, "HighlightAll") == 0)
+                               highlight = (
+                                       pg_strcasecmp(mptr->value, "1")==0 || 
+                                       pg_strcasecmp(mptr->value, "on")==0 || 
+                                       pg_strcasecmp(mptr->value, "true")==0 || 
+                                       pg_strcasecmp(mptr->value, "t")==0 || 
+                                       pg_strcasecmp(mptr->value, "y")==0 || 
+                                       pg_strcasecmp(mptr->value, "yes")==0 ) ?
+                               1 : 0;
 
                        pfree(mptr->key);
                        pfree(mptr->value);
@@ -228,124 +239,133 @@ prsd_headline(PG_FUNCTION_ARGS)
                }
                pfree(map);
 
-               if (min_words >= max_words)
-                       ereport(ERROR,
+               if (highlight==0) {
+                       if (min_words >= max_words)
+                               ereport(ERROR,
                                        (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
                                         errmsg("MinWords should be less than MaxWords")));
-               if (min_words <= 0)
-                       ereport(ERROR,
+                       if (min_words <= 0)
+                               ereport(ERROR,
                                        (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
                                         errmsg("MinWords should be positive")));
-               if (shortword < 0)
-                       ereport(ERROR,
+                       if (shortword < 0)
+                               ereport(ERROR,
                                        (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
                                         errmsg("ShortWord should be >= 0")));
-       }
-
-       while (hlCover(prs, query, &p, &q))
-       {
-               /* find cover len in words */
-               curlen = 0;
-               poslen = 0;
-               for (i = p; i <= q && curlen < max_words; i++)
-               {
-                       if (!NONWORDTOKEN(prs->words[i].type))
-                               curlen++;
-                       if (prs->words[i].item && !prs->words[i].repeated)
-                               poslen++;
-                       pose = i;
                }
+       }
 
-               if (poslen < bestlen && !(NOENDTOKEN(prs->words[beste].type) || prs->words[beste].len <= shortword))
+       if (highlight==0) {
+               while (hlCover(prs, query, &p, &q))
                {
-                       /* best already finded, so try one more cover */
-                       p++;
-                       continue;
-               }
-
-               posb=p;
-               if (curlen < max_words)
-               {                                               /* find good end */
-                       for (i = i - 1; i < prs->curwords && curlen < max_words; i++)
+                       /* find cover len in words */
+                       curlen = 0;
+                       poslen = 0;
+                       for (i = p; i <= q && curlen < max_words; i++)
                        {
-                               if (i != q)
+                               if (!NONWORDTOKEN(prs->words[i].type))
+                                       curlen++;
+                               if (prs->words[i].item && !prs->words[i].repeated)
+                                       poslen++;
+                               pose = i;
+                       }
+       
+                       if (poslen < bestlen && !(NOENDTOKEN(prs->words[beste].type) || prs->words[beste].len <= shortword))
+                       {
+                               /* best already finded, so try one more cover */
+                               p++;
+                               continue;
+                       }
+       
+                       posb=p;
+                       if (curlen < max_words)
+                       {                                               /* find good end */
+                               for (i = i - 1; i < prs->curwords && curlen < max_words; i++)
                                {
-                                       if (!NONWORDTOKEN(prs->words[i].type))
-                                               curlen++;
-                                       if (prs->words[i].item && !prs->words[i].repeated)
-                                               poslen++;
+                                       if (i != q)
+                                       {
+                                               if (!NONWORDTOKEN(prs->words[i].type))
+                                                       curlen++;
+                                               if (prs->words[i].item && !prs->words[i].repeated)
+                                                       poslen++;
+                                       }
+                                       pose = i;
+                                       if (NOENDTOKEN(prs->words[i].type) || prs->words[i].len <= shortword)
+                                               continue;
+                                       if (curlen >= min_words)
+                                               break;
+                               }
+                               if ( curlen < min_words && i>=prs->curwords ) { /* got end of text and our cover is shoter than min_words */
+                                       for(i=p; i>= 0; i--) {
+                                               if (!NONWORDTOKEN(prs->words[i].type))
+                                                       curlen++;
+                                               if (prs->words[i].item && !prs->words[i].repeated)
+                                                       poslen++;
+                                               if (NOENDTOKEN(prs->words[i].type) || prs->words[i].len <= shortword)
+                                                       continue;
+                                               if (curlen >= min_words)
+                                                       break;
+                                       }
+                                       posb=(i>=0) ? i : 0;
                                }
-                               pose = i;
-                               if (NOENDTOKEN(prs->words[i].type) || prs->words[i].len <= shortword)
-                                       continue;
-                               if (curlen >= min_words)
-                                       break;
                        }
-                       if ( curlen < min_words && i>=prs->curwords ) { /* got end of text and our cover is shoter than min_words */
-                               for(i=p; i>= 0; i--) {
+                       else
+                       {                                               /* shorter cover :((( */
+                               for (; curlen > min_words; i--)
+                               {
                                        if (!NONWORDTOKEN(prs->words[i].type))
-                                               curlen++;
+                                               curlen--;
                                        if (prs->words[i].item && !prs->words[i].repeated)
-                                               poslen++;
+                                               poslen--;
+                                       pose = i;
                                        if (NOENDTOKEN(prs->words[i].type) || prs->words[i].len <= shortword)
                                                continue;
-                                       if (curlen >= min_words)
-                                               break;
+                                       break;
                                }
-                               posb=(i>=0) ? i : 0;
                        }
-               }
-               else
-               {                                               /* shorter cover :((( */
-                       for (; curlen > min_words; i--)
+       
+                       if (bestlen < 0 || (poslen > bestlen && !(NOENDTOKEN(prs->words[pose].type) || prs->words[pose].len <= shortword)) ||
+                               (bestlen >= 0 && !(NOENDTOKEN(prs->words[pose].type) || prs->words[pose].len <= shortword) &&
+                                (NOENDTOKEN(prs->words[beste].type) || prs->words[beste].len <= shortword)))
                        {
-                               if (!NONWORDTOKEN(prs->words[i].type))
-                                       curlen--;
-                               if (prs->words[i].item && !prs->words[i].repeated)
-                                       poslen--;
-                               pose = i;
-                               if (NOENDTOKEN(prs->words[i].type) || prs->words[i].len <= shortword)
-                                       continue;
-                               break;
+                               bestb = posb;
+                               beste = pose;
+                               bestlen = poslen;
                        }
+       
+                       p++;
                }
 
-               if (bestlen < 0 || (poslen > bestlen && !(NOENDTOKEN(prs->words[pose].type) || prs->words[pose].len <= shortword)) ||
-                       (bestlen >= 0 && !(NOENDTOKEN(prs->words[pose].type) || prs->words[pose].len <= shortword) &&
-                        (NOENDTOKEN(prs->words[beste].type) || prs->words[beste].len <= shortword)))
+               if (bestlen < 0)
                {
-                       bestb = posb;
+                       curlen = 0;
+                       for (i = 0; i < prs->curwords && curlen < min_words; i++)
+                       {
+                               if (!NONWORDTOKEN(prs->words[i].type))
+                                       curlen++;
+                               pose = i;
+                       }
+                       bestb = 0;
                        beste = pose;
-                       bestlen = poslen;
                }
-
-               p++;
-       }
-
-       if (bestlen < 0)
-       {
-               curlen = 0;
-               poslen = 0;
-               for (i = 0; i < prs->curwords && curlen < min_words; i++)
-               {
-                       if (!NONWORDTOKEN(prs->words[i].type))
-                               curlen++;
-                       pose = i;
-               }
-               bestb = 0;
-               beste = pose;
+       } else {
+               bestb=0;
+               beste=prs->curwords-1;
        }
 
        for (i = bestb; i <= beste; i++)
        {
                if (prs->words[i].item)
                        prs->words[i].selected = 1;
-               if (prs->words[i].repeated)
-                       prs->words[i].skip = 1;
-               if (HLIDIGNORE(prs->words[i].type))
-                       prs->words[i].replace = 1;
+               if ( highlight==0 ) { 
+                       if (HLIDIGNORE(prs->words[i].type))
+                               prs->words[i].replace = 1;
+               } else {
+                       if (HTMLHLIDIGNORE(prs->words[i].type))
+                               prs->words[i].replace = 1;
+               }
 
-               prs->words[i].in = 1;
+               prs->words[i].in = (prs->words[i].repeated) ? 0 : 1;
        }
 
        if (!prs->startsel)