From: David Cramer <david@thingbag.net>
Date: Mon, 2 Jan 2012 15:46:43 +0000 (+0000)
Subject: Webhelp: fixed bug where words like key, day, and nucleus, were not found due to... 
X-Git-Tag: release/1.79.1~6^2~598
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=729dfded757d296ab409c352f9f0a93c282b5f6d;p=docbook-dsssl

Webhelp: fixed bug where words like key, day, and nucleus, were not found due to differences in the way the client stemmer and indexer stemmed words
---

diff --git a/xsl/webhelp/template/content/search/nwSearchFnt.js b/xsl/webhelp/template/content/search/nwSearchFnt.js
index c00a03056..ae740df67 100644
--- a/xsl/webhelp/template/content/search/nwSearchFnt.js
+++ b/xsl/webhelp/template/content/search/nwSearchFnt.js
@@ -128,11 +128,21 @@ function Effectuer_recherche(expressionInput) {
     var fileAndWordList = new Array();
     var txt_wordsnotfound = "";
 
-
-    /* expressionInput, search input is lower cased, plus replacement of special chars
-    * nqu: expressionInput, la recherche est lower cased, plus remplacement des char speciaux
-    * */
-    searchFor = expressionInput.toLowerCase().replace(/<\//g, "_st_").replace(/\$_/g, "_di_").replace(/\.|%2C|%3B|%21|%3A|@|\/|\*/g, " ").replace(/(%20)+/g, " ").replace(/_st_/g, "</").replace(/_di_/g, "%24_");
+
+    // --------------------------------------
+    // Begin Thu's patch 
+    /*nqu: expressionInput, la recherche est lower cased, plus remplacement des char speciaux*/
+    //The original replacement expression is: 
+    //searchFor = expressionInput.toLowerCase().replace(/<\//g, "_st_").replace(/\$_/g, "_di_").replace(/\.|%2C|%3B|%21|%3A|@|\/|\*/g, " ").replace(/(%20)+/g, " ").replace(/_st_/g, "</").replace(/_di_/g, "%24_");
+    //The above expression was error prone because it did not deal with words that have a . as part of the word correctly, for example, document.txt
+    
+    //Do not automatically replace a . with a space
+    searchFor = expressionInput.toLowerCase().replace(/<\//g, "_st_").replace(/\$_/g, "_di_").replace(/%2C|%3B|%21|%3A|@|\/|\*/g, " ").replace(/(%20)+/g, " ").replace(/_st_/g, "</").replace(/_di_/g, "%24_");
+    
+    //If it ends with a period, replace it with a space
+    searchFor = searchFor.replace(/[.]$/,"");
+    // End Thu's Patch
+    // ------------------------------------------
 
     searchFor = searchFor.replace(/  +/g, " ");
     searchFor = searchFor.replace(/ $/, "").replace(/^ /, "");
@@ -169,40 +179,28 @@ function Effectuer_recherche(expressionInput) {
      * Compare with the indexed words (in the w[] array), and push words that are in it to tempTab.
      */
     var tempTab = new Array();
-	
-	var splitedValues = expressionInput.split(" ");
-	finalWordsList = finalWordsList.concat(splitedValues);
-            finalArray = finalWordsList;
-	finalArray = removeDuplicate(finalArray);
-    // OXYGEN PATCH START.
-    var wordsArray = '';
-    // OXYGEN PATCH END.
-    for (var t in finalWordsList) {	
-    // OXYGEN PATCH START.
-		if (doStem){
-    // OXYGEN PATCH END.
-	        if (w[finalWordsList[t].toString()] == undefined) {
-	            txt_wordsnotfound += finalWordsList[t] + " ";
-	        } else {
-	            tempTab.push(finalWordsList[t]);
+	
+    // ---------------------------------------
+    // Thu's patch
+    //Do not use associative array in for loop, for example:
+    //for(var t in finalWordsList)
+    //it causes errors when finalWordList contains 
+    //stemmed words such as: kei from the stemmed word: key
+    for(var t=0;t<finalWordsList.length;++t){
+        var aWord=finalWordsList[t];
+        //w is a Map like Object, use the current word in finalWordList as the key
+        if(w[aWord] == undefined){
+            txt_wordsnotfound += aWord + " ";
 	        }
-    // OXYGEN PATCH START.
-    	} else {
-    		var searchedValue = finalWordsList[t].toString();
-    		if (wordsStartsWith(searchedValue) != undefined){
-    			wordsArray+=wordsStartsWith(searchedValue);
+        else{
+            tempTab.push(aWord);
     		}
     	}
-    // OXYGEN PATCH END.
-    }
-    // OXYGEN PATCH START.
-    wordsArray = wordsArray.substr(0, wordsArray.length - 1);    
-	if (!doStem){		
-		finalWordsList = wordsArray.split(",");
-	} else {
     	finalWordsList = tempTab;		
-	}
-    // OXYGEN PATCH END.
+    //Check all the inputs to see if the root words are in the finalWordsList, if not add them there
+    var inputs = expressionInput.split(' ');
+    // Thu's Patch 
+    // -------------------------------------------
 
     //-------------------------OXYGEN PATCH START-----------------------
     txt_wordsnotfound = expressionInput;
@@ -380,17 +378,31 @@ function wordsStartsWith(searchedValue){
 
 function tokenize(wordsList){
     var stemmedWordsList = new Array(); // Array with the words to look for after removing spaces
-    var cleanwordsList = new Array(); // Array with the words to look for
-    for(var j in wordsList){
+    var cleanwordsList = new Array(); // Array with the words to look for
+    // -------------------------------------------------
+    // Thu's patch
+    for(var j=0;j<wordsList.length;++j){
         var word = wordsList[j];
+        var originalWord=word;
         if(typeof stemmer != "undefined" ){
+            var stemmedWord=stemmer(word);
+            if(w[stemmedWord]!=undefined){
             stemQueryMap[stemmer(word)] = word;
+            }
+            else{
+                stemQueryMap[originalWord]=originalWord;
+            }
         } else {
+            if(w[word]!=undefined){
             stemQueryMap[word] = word;
         }
+            else{
+                stemQueryMap[originalWord]=originalWord;
+            }
+        }
     } 
      //stemmedWordsList is the stemmed list of words separated by spaces.
-    for (var t in wordsList) {
+    for (var t=0;t<wordsList.length;++t) {
         wordsList[t] = wordsList[t].replace(/(%22)|^-/g, "");
         if (wordsList[t] != "%20") {
             scriptLetterTab.add(wordsList[t].charAt(0));
@@ -402,8 +414,15 @@ function tokenize(wordsList){
         //Do the stemming using Porter's stemming algorithm
         for (var i = 0; i < cleanwordsList.length; i++) {			
             var stemWord = stemmer(cleanwordsList[i]);			
+            if(w[stemWord]!=undefined){
             stemmedWordsList.push(stemWord);
         }
+            else{
+                stemmedWordsList.push(cleanwordsList[i]);               
+            }
+        }
+    // End Thu's patch
+    // -------------------------------------------
     } else {
         stemmedWordsList = cleanwordsList;
     }
diff --git a/xsl/webhelp/template/content/search/stemmers/en_stemmer.js b/xsl/webhelp/template/content/search/stemmers/en_stemmer.js
index f58012f2b..439b1696f 100644
--- a/xsl/webhelp/template/content/search/stemmers/en_stemmer.js
+++ b/xsl/webhelp/template/content/search/stemmers/en_stemmer.js
@@ -105,7 +105,7 @@ var stemmer = (function(){
 		}
 
 		// Step 1c
-		re = /^(.+?)y$/;
+		re = /^(....+?)y$/;
 		if (re.test(w)) {
 			var fp = re.exec(w);
 			stem = fp[1];