From 41638a573b235380306772e2a522ef491add800d Mon Sep 17 00:00:00 2001
From: Kasun Gajasinghe <kasunbg@gmail.com>
Date: Sun, 8 Aug 2010 16:43:13 +0000
Subject: [PATCH] =?utf8?q?Added=20client-side=20support=20for=20cjk=20sear?=
 =?utf8?q?ching.=20By=20default,=20for=20cjk,=202-gram=20tokenizing=20is?=
 =?utf8?q?=20used=20in=20both=20client=20run=20time=20and=20indexer=20buil?=
 =?utf8?q?d=20time.=20i.e.=20=20=E3=82=AF=E3=81=AB=E6=8E=A5=E7=B6=9A=20wil?=
 =?utf8?q?l=20be=20tokenized=20to=20"=E3=82=AF=E3=81=AB",=20"=E3=81=AB?=
 =?utf8?q?=E6=8E=A5",=20"=E6=8E=A5=E7=B6=9A".=20Better=20tokenizing=20sugg?=
 =?utf8?q?estions=20are=20IKAnalzer=20and=20Paoding=20Analyzer.=20These=20?=
 =?utf8?q?are=20dictionary=20based,=20so=20that=20results=20are=20more=20a?=
 =?utf8?q?ccurate.?=
MIME-Version: 1.0
Content-Type: text/plain; charset=utf8
Content-Transfer-Encoding: 8bit

---
 xsl/webhelp/indexer/lib/nw-cms.jar            | Bin 27877 -> 27958 bytes
 .../com/nexwave/nquindexer/IndexerTask.java   |   8 +-
 .../com/nexwave/nquindexer/WriteJSFiles.java  |   2 +
 .../template/content/search/nwSearchFnt.js    | 252 +++++++++++-------
 4 files changed, 166 insertions(+), 96 deletions(-)

diff --git a/xsl/webhelp/indexer/lib/nw-cms.jar b/xsl/webhelp/indexer/lib/nw-cms.jar
index 886f2eb30dad8d58b9b94770b062d19b7fa2404b..95bedc392491336af3144534d43976e8a7132723 100755
GIT binary patch
delta 6392
zcmZWuWmr^Ew}t_vn?dPj7(j_3rMo+%LztnZ1Ob5~sgl9~(w##$BAp^Bf>P2TNDZZc
z#8tfad!Bpm`ElO0-nI7H`}{g<y?bL8vuO^KSXToJn*ihQlbVN@K+J{p_78|f-9kUp
zEqsUk1tdA9;4S)LG9Y7|@sKW<1pn>x-69H8>=rjLCI4bQJ18HM{Es0eiNNTfBEi5&
zrpLhGLXtsgk;@La$TKWr;3?%-&w<277biEn5F&!FUg0x>CP{Zx`@wb@HvB5CVJ?X_
z)J|@M#%3&)U7T)-BJ5I=3)^z(A}Se^Y*Nf7vI~>%PdW~McN|FEkCixn#xdMP<aD)r
zelwmU0|Ul7uTQ+kGjU&l&j|S12$b#p0Gu`oY35#kMj5W&mwuNu01?y5VS!*^%wYU5
z8lREx16qwyhNHK{|1yRsXXDVDNP2vkkf`IsvIa$<X=L>Wv=L1QXYW_|*`0#Up=EQT
zj=cy|G{c=GI9h5Ts1p>7A(OPMVr9}Zca=Jr^8sx@6U^B=?te<icS#Y<(Oun6hDcoc
zfHtQIhIAjadkC`u*Ua5a?KQO5>l~55=B%~-84uGH9pAN;+%gmIVNP41aVRi^cIOl7
zR=RU@C@&#7Sg*$iVio8X#J99^R1gjG!k5y7$zyOZeKhIr;>n9|2J>0OCljGkrJN)P
z%i);=cEngR2U_2IVfI94HPjA*sY)-_6F)GA@oWi-y{8BI;&<K(j^z<9ThM$T60=sG
zCY0&Ip?Gl8Bi9OhkWYSh*smx?!@f;@;33TpoM>o<c}<_@t-v=vd)FiZ-W4yth21T_
z6+%@li<JoT85_|xLZ`?_$F3Zd6|1w(_|=C!S!b;HY!p>;rA>D#7Yk?Ubc*YYd9#(#
z%}f&{F18AQ#?O}$l}{ySb?ocYcu^ckPGTp^`V%Xd3ph?p^I|nJdwUO7KXgz2K3ja@
zs6F7$`^2gJcVo<hQU{4Cr*9J#XSP9?wJ++Q_h}9awqK<S=a%^Nd=)#x<guNBkOh)x
zYyA3*Vl8ml-g@zdCHi**y5VDo&H|BpzQNPC1m@ngz_wP=C|1CfmFkE&eRUkf?jkq)
z353nQF*vI#b*NY$F*j-FPp_64tN=APQX;n0*jJy$V<nt1H05~`o-yg<Rw|WUYmzYL
z2k6(=Ijve-M<Mp?f9`B&uHrc|PE7VK){A7vnD3BV_?*5cUl>Ki+zWuUHi`o|vfu2r
zHFqFDe$(qmmAefMTitGED2;jd3A6k?$1jsgAN<%YG&Wv%C3zt!w0nG#^pgbEV^jP%
z;f@L!K5P;E1qu-H3)}WtV_%0kO?Y^m8%3L5&nZ<KRm?i7QBT1_He_%GKb&W7#)o@b
zD_G#O0(XBVcPLwaSNY0=aE?({G9w8xqxmcVwRctP9ZS#_S?1s>YUEqN9Xhb7`25-N
zqHrYxgTzDDAKHS2*~X@^Er`B&eT#IpMx6r6TII%!Fp~{tWARs};h<mW^N2nPrh$I>
z{B-hB3Nr__UxFJJ`BW^GB#w&jt-m5s4E6%F;FkE;&aWN4zc%^$d`?9@{uR9FmTwsa
zoVd#Cm^sU+X(~^#z{)x6Xf%q%r0L_gQ7ML5T*mheiq`cQZhW26!U!b0VkDc&sQk$G
z^_N-vDnZq2vpY0xA<$y280v6ADYX}D<-8y3v1miWib15kbYMU1#lkK-Y!57}avRDg
zhBY+iRAMC0)Mil2oNXkwn~F_z^`2}%vG)t#gyL0g)C%^hzE56gMlJCQDnesARqc*#
z3E}Z}SW5Ot&8MsecJm^maH*f8uj<uzBq74;G<gFsXU-y(wIEMhujDM6HE;Y;XTuEK
zlP$SK3h<`SrH<s|RwcbRPVAe6M^O0y`n*`_`z|IlBK6t3qN`~k$~eu-#`yC9c;SwY
z{t&Ik_NW=+g<upn;q#Z@hR?tGy@*z2{G752BQ$yY!pM%^j4Qn~LrDMP?pK(0It1)T
zo2~01vCmA9T-jvMz2jLP%CN4#+xGzH+2Q7HX&N5nw42SokDum^-9ydB_<nuRfg91c
znGZi)O*8{Nn?t{yJ;r(N@iO!|8n}b+ZiwbwDDQH*r)(k=H=9>s0PD?~W5%t0C4u-{
zzARnnm}ZsV18rd{FjSz5sU>6hO}6>ga584ot<Xk>gnz9i6c^3>GBadiq*i(na6+yY
zk%_G%xJgx3zr1*uX4nvoL%?a1S|D1M`x=TvYD);x9Wvn8`V>RM9$W8c0{l7W9r`{N
zl=Xc>jL0dUo_&?FK-B9Nxs_a%QTv!cuJQ;cF!*7+KtV}EgEEJDcSdb=U9gw&$*HHl
zpRbRI<yTtU4n8@PY81BtYB)bKWvH+ehO!J8nEb3AS$>{fLQR`#wy^l|e4jjAtFKhn
z?CnINfH9)fDz;QRk_ev=08iJKM|~9Ne~%Udvp*K%M2%_ILWRR<HqPl+_D5ARSw{o=
zou-{u%wG+rCKjckSynXatHXB)(i6(eaT^ssPHZRg@VQi?U!+a+MnCgu@(nkiYfbOz
zOPMR~mA91Bth>33^hhEQ%=>KX_1sWLFP4k5{zQ>`SExL3u*{<ns2eIx2LE!=oQ?G!
zTWlpJ{1GMcV@KS)fB*3hFT0`sEMpyWPmzO>RWlY^o`}>c4yr)t+n45OV;uLjxkJIg
zccIN|0yqcDoRU&Nm7B^A^HK;lw;#m)ekSW)^u<WaojN-iiML0OH5K&muk3b2>bM4%
zTU>-PBIR(^_@(5+fj10>^b0<2r*4;DhWE$CvdWsODl0pt6h;b|J~}HSdX5LbIA~pE
zAG$qxwUD0Msu5!5SNrMe83o@UVnk1)=xK1#1D?~|^l+~^I)<eCuq9Q8sboXn0Q}ca
z52>nS*sT$7eiU7W&cWnDE|VYRy_l*tmw*QM!)D85`jXfT3jjhsf2&_C(i^YCR+Zkh
zJh`k75OkxH;RjO;`vfDi_Fy~e9p0v$e#OEGbzHAqqT-)!DZf>#UMri<=yO6fO=ZQI
zU!-WOAk}NvrQyFibn)>yaL|=U135Lq9PJrQDO893gIu^@JlKa1<<Vc`>b0ezGh$d~
z2#I$8R|fh-Yk=JcPnpxVPeo=QIu;3Y7pE)TZ$7WHLb-QpHe0d<28pwF6vfjwLtXAc
z$q}EAxZ{HSk}nBZDE){6WDnD1Ew>ea!lT5$YPL`f`#K4HRA-8SNqtRBt9*!)2&sgO
z^j?H7KS<!g`R%Wy+c5jh?XZYV{w$vTD-{TO6I_q1<N>lLl<J=3%~>{Y=)W6^^o|lK
zYd`T%p$=w4<4Q6-nadF%zL2k{TyRUJrLp^xnifUSIq^=?IhAH|%-aN3mq}bdptNYR
z0p{SNb`jhP0ax&7^-Py$cB-Dg7l+^Fy9nOR=bEy`Y@@Kk!4TFRSb8GFvsNuAu(%iV
zOKh$E&u=RSws;UjDz+9jrpm<oI54X|xE<CZ7X!W=V)=4n%c^`Kf*CFveO_?#Yo?`9
z(feYb{fxS20e?dXON{m**1e;*#9=(^&qx*gKOE59fv&%Ona8^RizJM1{ZV9|;CeM}
z<c~MN9RF1K<+W`pV~=fWH*;5RO83Z{OU%^iwIf_W;`zcg&%@^*uj(b9OJ0j-NhJbL
zzVnaRq0Wh>kgGR}c<vDphOEm%2b$PyU98%V2c4BQH`E>>83XgQ(wGXwp98#WY`$gI
zsdUdTrz686_Nq?36-sOu&mlIpSUQ_kAG-6`3L8>oh@*96`vvZ1-@A&_`X$CIgYzh^
zD=`}ecqDf|z&#~C`A#Z8R$qkeRl|!Ldz~xxAo{RyVkaEk^pqU2sw8Wnd(B(H>-~9P
z{#e<U35(Q{z+$I{p?T#eIUcfbgS`0$hddroQatM9XN~Wg4!58wV@^g`VfbjW$xqc#
zmlK#}62*4?VC0m<`dV|S(;Gv`oE335Yu$+iU?uIH^dzzvu_4BM4(y2`N?1HF5ifTu
zeM)SCNJJ(cKO|l^R55&^sMID`ZL???KGNM-9*DW5_<BQ$0*i85hPd%=E;nC7C;~e@
zb&2Rk`4^32+*xP`eS<68`)0<RuI390QT&TkZxRtVYxUjjd-pjOO^i)0jzpWPX->#Y
z0AK2=0`$^?6v(ZjKk&j}yes7ug<<GEy=&0D_&8Y!Xia67;T$8BG$jY?qP4JZZB{#+
zwEKtZ2A;)|0)$(qdzJoaX2_VTQ4Ke)Wx_`rIEk+kDr~wb`28a-LQM%0g0l4kM8rb5
z^U-svQpUS{Sf5?{qJQBy|J=xh=#MIs0><@B++ju|UOHIEoq50ew(@f4fi>}#vY4l+
zhVRdZb0xN<aoa1t`_UO&anjPye;`_Lu(c+S8+n(v!s~eM0e&>kRKc8EU4^Pl$Ncn?
zF@z`jNDZ>sScu5!>}1}+=1o(SgHUP;>nd)DRfd9j?CGk)w+m`$+9=u-LZ58{z>-q(
zUfJcrd(2RtP0Mf+{6zzAax)IUg9?;h3pF={K5;pyfACeNDhIP|K@FC3Hl#5%la)Pr
zGFmsRfb5!y9mn_*_X2+-n3_6He<Q<>qIhcJw~Q*wYRNGeAIng_tnYDzOfWTxD?>=w
zI(|Vr%RwSGuF4|$&^~A}6O@w<fB;8=zyN|k1VNiHj<f{MHJa*-ZS9cC%5WK1yiM^f
zbPYI};pFz#bEsgY6mH!!7MYserwUQ^&$D0vZ+z^97jm%1c8xaNB)+mhCF#STjd_3P
z(KOO7d+P=FQU7S1EzS|~!=`VPgUgFP1-DFPVY0W}e7ws|YTgSih*Adlz6-TW>5uvF
zs4D$35oOyK69}jaNz1n0@UGMcsdxRtKEZCsgyR&sMj%I$TIrg+jZ6oWIQSgP?mL<~
zu{bA?UHC~0Tr^SU&cv>O_e(5v5>5m3`jt-VYM!s*eJGf~l_oD0msj;r?qiSGF(ofL
zSkqmUR%QupqS8u39&<cw!H^ux5HDPq-?LX5)6N_kc@;IAY#sGY8Gmeuxfd2ul4vcN
zv}LOAcztqg{Xdf`9>(A4lANNI@OI1`V-WaffS|G<;=l9e8!#isGA;&25>kf;1^`r#
zz&DHbqA&>wS(Cb=XsTjp#2Z|5^B5r(yh6oE1Xenkrc-S1{<qz(n)+{P9C^+`E5F@>
z8jM}srX4F;yC2B!_M!}TeqVi&>0Y`xca)(RPYbHaz%Jda&-pF(Yr%W`Ja}CEz5mMU
zc?sr14B+T@U!RtJl7LfKh_#?=5+Lg%$5Anwp{1fXNZ6RwOCVfhT=(|?`w<Y(rYGoo
zW`2h<78_;idb;D8A0mi((9~de_<4})ikoDcWw(U9A7l~SgQ4y0C%Ra$+)i6H_}j3}
zx*J!>wF_5R(1WR3y07=8QJ`YK%Ixx>VW34n#BiZ{8A-dU8gU2>On;OEtb~C4U+_8j
z9Wz%;506O}u6nnfw?(~FI0G^29V&t2QGPMiz-!CPx~S?J9T;~9YSo2+@e$f5WT(x^
z{V}fqOT8Kd4u?SM;zU=dy~H60LrJpsTd({VV&)OPg3_xSY6XvSBlt{blaiD6!}r*=
z#=Yu5TJx0^8+&AquJeE-C)w6BeSMqR?zWAg_JY@WEW@YRbMw?0ylMp5@WE#r$D8<H
zcHwlx%91dDA=^#m(cUIQba<#c*)wUWCH7K$^N@*G!fC?o^4V7Nhk}*v))h^?fG2$V
zTes_+tr4ph^I_c{XO`VzMP{~O{IK7bM*S3r0eA02td#q+kIw;`Q2@c8k@y)zSLK@7
zQzAj2t*1?P18TN10$!u%G)T+mkxe?tI0=45t{-RA+JqK6)-{=860OzC8w8A&Nt*&+
zGQcM0;!BpQC2*@^>K~cpCI+9%q>iyh39(6INS?acjk*&qkj2ByB9S7KC)qBV@Cy3@
z{^QA-dz5K&`M|bcB2!$UTfz8;kI-J+cW&O*%Ko_puj(Imgn5HT3GKip?Ya%`ZDygq
zY;T%K8-Iw!i!*lcOiTob!d-LZdP>j@aSmwSJ@e<=R|hNN3`>J=;+Sk+IikX!F`HMu
zk=kgTg=Hm)KfuC~nexOO-<gS)!;zUhw2OOew6VY3fdWu*chU3=<RC&eDia`Ca`KS!
z(r|Emraz}XZQ%U^38wAV8BQ=XmwxVK+#!2)Ce;2$mj}AL@D6$OTR%Hyvxq(+X{qGt
zB|+zxL%K_2*=j9phR)stpUV*IbGAEW13>|;PcL6D#W0E<!+VYtuzkBTlY%M<t}E8~
zD%u4YmjEL)y`4S+zR2F9{!w{uh^|WLc!)@@IU?UJ^D!yi#jEhnRrisRB`9_Nvag8u
zXU74r*xK*}FD|1}1seDzF&`VTC<JLd$GVox*`#FinJbGOdgIJ^N*QY~;%?fpkZV|h
zb&)MEp4pPq+#JtO@We81Pa;r}vn)aB0gL#SC*Vg?^k`V0LpQIp!$|j8dATV{tK9i}
zdef*v^q#>%>;t{u7GG=Y0=qw7n-0RqQktkTS3f|*1dAnjOQbq;mei7wr=s`4<IY{o
zo#`VM+K6qs0v9Om-Pa#aRquXnBDUK0OXKk_W(oub*liSL{8QeF&ugkjmURM$Zmq=u
z+Cb6WDG7^dyyCskj688CiK>bNR)~cTg-fh<2q@0DdtT=e&&+hv)7aF&;lr`wY6Um_
z0Q~J%LrReC91)ePghRjy!n?o7E3LM-kl)!~Wcx=0{g7@zNnr-N#D?Ru*`}_h;ihYz
zwNWC5WJu{k8P3!O_3OY3edtYyI^(&+6%gk}**;Ysa7E-%`MfBH#4w5F$V<k^AcZ#)
z#ZHdF&Lk3_V6_A1cBv7HPv6%I0Ta&Qrsdc}M4Rr9)p1Z(?{G(*;w#DY-PwA?GJz2*
zAyg^!L{JEq<%}m`0DI4PlGkJ|n0i&{j4#}Fw;=i{Zwj(iz3w<R=yA)MKel%BF)*R0
z*o&s89o#m~h48;z_T%-S){^QL=US_^QIv|-)iJ1m=7Y(L)k?~C=u_||U$r-wJV5gr
zpK#R{iwni)^nX+>d}ZVk?f6>JiTqt;WU65F^yndL6<jSQlJ(1*EF|$`#PyJg0eSVZ
z?H<vWQ*1=-6Y8C4e#FG?!Q&8CGr$Q*XElv@iRQ05{tm&T4Y|7JbJ&o<!^f#6wCFR&
zss1Iy#>XN+%7BSeKtO^PaJ;9%C04pdEM2%3{wd2S20<p9y18`|{fSt3S<(vA0V||{
z%F~XVj)(uGly`QDa<!~dB|72(Gc}=-h{{~_7PBYUq^YTMV4+c*keRP>3Sji%J+(sB
zkuzbn{B&{?_9fd5-6h|uwe%$)MI!jj8_zPdvz*{Y5^jA)@zdyCs6t1QOjo-9Zb)4?
zt%vZ@KI_r`0G-CVa0T1ZQ&zrdPt2(CdHU7xF|85JLGd>I3BH;PQm?{hezxf;4(evX
z#%fmMt(x}|R9j#2%|6@7^8?%P7@CPkluzqt2Dld06_U`dw*ET!lNY=9HrKa~yxBg@
zFvK~BsCWc9MATX>P5lgzf5;YOVXrP8_VZEWGydmt#y|0ciHR9%m&JEob5nh%O-hM(
z>`fFEKGl3X*}5#F5%N{`J*HIjEg3&O%JyLR+7=z5VUaV{H`4D<k;DL`D#=KFvhp7o
zkNDjmn;+}7LgE>{RkhiJg;%a<;huJmPjMbCdOnpd&tJ)SGFg^G%+cZEqY6Ev7}(tI
z>_UUVS4k6_Lm9);5HZ_pt<mZv*vm3}&T=X5#wCFekM$;)3Dd0iovWg;*EgX5)gnl(
zGz|De)x%)@n756>Zy`p|twArkgMDidNN)WLok}JB3&qIJ<K2dw)$0F+m^B*7ZkZYL
zFnp{(8adJm{!iylgUkF4Am`!#^lxH>=&i4UkiW$-1Rb&j!T#U&zg#j0)mz`(LGu<-
z4kouP%YP(tB&nm?zYIq@5H$so97=-}r=aH5)xZQ%Vf_{AZ`&WK3>BX1y+5wpT?`E9
zt$hFQRT%P_;darkBjaryVkbHfF7yw@%kzJ0{HISM{l7KTks(h1KC<o4a!(jC*qIu6
z;PkH|nKK<oM)(g!gH(3Dce{zLGaaT4GRT?#)~orGpe^|);VzP2lI;H-os~S7Sr3GP
zL5%(9I^cgDU0fQ;>>_eo!_<WiG$H%PWkR;ff{`c}qFegiAG%8ImQGazBi&s8(KN1f
eAbX8lT3h4Zf9SwJG`+<w-Qo(llMDa5+kXLpPPk|Q

delta 6326
zcmZ8lWmr^Ew`PVMx@!pO?o>j$yStkq1`z2%IwS|AyGyzoX`}^2!a+&_r34WKF5<o4
z^L+RGIPY5TTJJvlJSX<rtL+nDcNKu6rHYJ#iS+jXIiaWEup(nTfN0z!jL<&9KFeQ#
z69D)h(GNfk-*CZzy8<x(*XMb}YJlh??gGUBVlyMK7(nn~NbqRy>Hu+(kbJ0+kXYe-
zgp}~tP6V(#a*!YK?nUpByF-&$C#i4)6+a-t4%t{9V2+QioGc}(2IQ-Zuzs1!`ThrI
zcXM(~yHo3?qW%YiGW~#$25FpD?5!Csmwl#dpVmIzNJ6T<1qw9hs5b%J`gU&b@9%!~
z|GIm52RGad*g`?#95P+^1ou1{(gLkx7T|Bm(`dqCPLydZ3Ac2`LrfxOPp*s!B*(U=
z=yP{Y&*_6+jTy>a(I;8joY<pTTA%2nSrTm>iigOCdc>V@5%h5ly_CD+-tHSi++bV0
zcqe!T2&d)0(gyXQJtsx5xMz-ix=kA^h&}Np2%#I|@xSN-hoE|3oVbdhWbKI1|1pU8
z3_5`Yisa75JjXau6yc0~-Whve;D1R*a5t%VKN{bkfq#XLb;H7Q&oq!j|7Q&c=Y$rm
zVto$tes|T>om}Awa}1EQvj~+MI~UlMxrYigIx3P#k&X>@-@NSD(n%vv;}2Z$rF;`>
zG-HbzuCjT4wxvw3aao90L#{?Q;1vyJM}7`7N2}-@_yg^s`{L7uG=mQ<k8z9@ox-Uv
z;<Rc7`_AV<g%d;-hdb%>xxR=K?8h51*wn=U2oB}U27Kz{JG6!a0O9T*9S?aBPfMBe
z52IXHt!_LCrccE4^JY=v{oCXQ%G^6pdGJqU0J0dn3jOocE8Qiha-U-4;24|Bd6O_2
zrb@QcO{?yd*Q0?MbZU-G4TMRDf{gkjTHO~dDPlGP-(p={s=F06#~3j5U$5tU*zVm(
zsVwq2*3zj~me?rQsLr;L`&C(z^!sR|H-JC&;D`QeNuesjzJ<|W6nxXqULhKxDoq&%
zOl<nfQ8#K7#r_4%`EQJW#U2bU`NaKz>DV~CyLS;O2Uk$Gkpj<!lQI;y30f`H+foS@
zRO(d>)yjxK63&$=oQaGYT|LQllSA3n!y+u4C@H@=t6K<`!f+-^`R6FrDQcAhTSVTH
ze`7qY=aOp<62Bqw(P%;yykt|hvFs72Orqq#rxI#4#m`L$h#>FMt!`uvq_v!e`Dv&f
zG8MmRG~{v<NjrIap;9cnS~PD`_}%l9)^suy)bV-}X_s}EnJG7Pimi;*b{woeuP396
zwp?ifwD!tT@%kX)Qc`wQUYf6z^{fr!q%xFD`$UIV<%E$+E^~S4E>+psD=MEVnQhKa
zfL?~M2kfqxx#FZy0Ff_%{MKrNnMr!RwFnc-3^+qGckq~WC^gD;I@Z|iF6^vxvo%%P
zlv~z?^MiS+o#ty;$uBzAOXgygde<jXIJ|czsX94j!n&{iK)&Q9Yy{C+%VSl5>Y7Ts
zv%HwKv!R^7*yOrp^|zZyj0q^2I-+B%W1Q5-t_}PzVygFk^!BVtbrWyGzGYI1>{z6d
z<4qt3m?W0zMyU;UvruZ=GjXpKae%wjC>Yi>zH8}pGj=3VkeHUIB0@UM!_W~DgyQ4P
z(d6RqiRIZr5(CX~je{~dRJ!t8qObBWUzvBIBn^}hilZH;BS63oprMK%pRiH~CKwfx
z+CX8*phhtT(SKGgua^j$#9jE%1Fv13Z<osF+G6TJv%Wh*^^#)=Z&*}6ueyz*)NGvp
zZpLWJWFJwtJRQ@lcF4){RL%K`qOKQvQB;Y9_Tmn|n847!O`JB5$Ys@*IU^G!d*$-^
zX-p-WymWPbrlI3`*w2ruU<nly0Ud~g&}NOiZtw+R8^Q#zqWlqtsi~buIk7FIQk}FA
zGYJZe-k4H0?}u|Jk}eV+&wCq1gQgTj1+@j~tR!}WLI}1?{Z+#Dw+bT;p~%|fkk$l3
z_n}ro3l;7WVMs<pxJX6xg%R)D%VQ=6w24rvMQL_F&NofIroo7~00YNevoG;EOzUBJ
zgcea?#-pJ+paqP3i^Q)CMbQf1dWO}Zd(pQDyN%<PYS>|&FUO70ciXqi=~WYRQJ7ez
zrxLjX9YLws40~lM&CDmIA(|um(O^OaD2w76cZ5RX#r&P>0WnYK0G)`g|3|RU_0DOj
zLsI1%t#E1F^~?w;qcjL2hg9uN_+wN71kX&J8)%6L9EV|dklj@E3u*(%lX>{ROG*jp
zCdvz8q6^iIG3>}{&!|teIGNRHR)LtRv|>|F*dZ0Qv;`y*61l`9Ics2UXI#F|!5AEN
z#ttg})<(KIRVb~powDi_E(<fBPuITgm>I&)1GD3gHG)Hg_$E`9htvEb(kUv_Wt;No
zkjo>_4q&T)dc4$5-2<{Q3Hc}q^h;Pirp1lEZVI~en%ZOVO>yYkb6)+i<{CQ|lNy<I
zLbsR^pb1#n6=cvA`Nh!~QNF-LRxoceC@E2nw(sul;f^KT>LZ8}%XOiW6WJ4!S$faI
zJjhF8C&LoHbJ>txNSUJPP}wLq=Wc+Sq`Re5x(%~taNDexw0<!~vF2i6u(IR!VcDxK
zyW6{JDuI!#-B8F`Zna1E%r$b=h;3Le^2?12H{J^3;fas6ZU^*84}njaLBG6-^3x<%
zni@L`j-`N1Jz4Dzn#q^`!47(BA3MVHRQ>sii;O|N42PsmtYe5uSHXj91$1TJUNJQ~
zFf8z^vhJeJ+7MpP&&0Hiq-qxv6Bn%BzPP0gXL;2G|8t{j&YF#9U7a||rnY)7*|fZ8
z1-tH<zu2$6e##yjF{!y28EK)tt>qh75?);7xVFEO9X6@K{v?GvACXSFKlM|$w<TZQ
zbc{8*cfpw$KS1A>Xr%tPaN8js-!6v!*CE*X=5laM5=YxqkitwI#qAp!&9$L=mDpjg
zJWDtGo3!k(B-3S{Y?nkPY?UvQe;C}Vmx~tRel{1Q(CkXC7tAG5U2Pl7r60_^3Yr2e
z3qvEqfL8{_+exe0$(s{+s&Q9;Y}1gZL{mqcxUXsyj9GGsA-R%Ft*<u4huPe5OKf4|
zzRxc*?4E6|doYr)m&ZwaG>-}dRJC1TZ}?_?I}{)|<&xOxSaMcZ&9pX1(TDE1)*vd-
zvkHw$(1{C9mxwj&G}#NhcQaKRA<=|IwLi5)7rnXiO)7MrkgzOe_9;>mt)qG|IX-Wv
zAm%c7nG<*&!+Wck%HjPgEx6x!VwnJzQ036{w(OI6^QBMcL{xLUa8)nD?G0%N!`znF
zv-JjsGS{ER4on@oizNJ`9|#0?NkViT4UHCcOU+Md<W;?e)u>7Q8PZoc{Zo{}%jvFM
zG2xy<UZ;+%Fiw`*FY>er)WnhOdhhBy!AN>C1hDS;+L->y_oo_xJySHtZDWD3qf?8(
zo>`jXcExKezA^Og7qXXE4H7gBir371ir1Wgh+kRfW8aQrwGVBw7hkdr{vz7L^_<5>
zTn}gugtj~3QvvF|@y!4pr%x~d^`aO|fcijuaX`HwMj60k^+_6f4@tNd#Yq6VwGI-x
zH9s<KI6ImyaI-2LffvIw)WQvme(4gJu_)HtJGGlPzo~zex4OyXH|s(hVXVd>P$>zy
zi^tePK1r*~%4p##ZWPs0#BRRP+`lY9Ll<@Ksrq66(a>-{@6{^VG;09W`in#gip<3W
zBR&C76o))f4DSO!es68GJ(|$C(AFmxxY83L=roieOY}j6Gr_nsG-3WQb4BaEc4p+(
z7eDuLczGSMFzl?X(GKbkscoiCb{eWa(Jg)-V#^GxK!Ua%svGGHFW*`%f11gexVAtV
z!-VW6Wzb=y)_Tim9CsGk92X}eBzPEc2Y*MHQL)pNmFJ@P%dyDlHPNVg1yJE)4=!*s
zi(`^CkR=Z6z@JcsW8s~{1d)prtp-`qa2*<6hqm?y5jQg+#6XM4%gge!gK^lkr6Y`#
z<K1nnw6U0-x9)I%SsH$1GM>{baa*RYGsf%DY8XnjhK#WRz5Fb`AaDP~rv6OH9QoOm
z1cj4{<{J-5-YAdX&hB4+@$O^SZZ?E|T;wys>}lj}|7qs|F<-QVsab9Dbwr$5sqf>r
zG?4P&>eEKH(oOLE&Z98;h4BL%W}ssg%u&@sw{zB=_)eh|yC>W+2j3#BtZ0F*qa|J=
zZ2<~H`;;gXSoz-a&8tMMhhI5zj_mH8m+BywDtizKx)gn0PhM?`x3ax&QdsN9K~>Ah
z$PP$%y;mSwm&`5%_5$`)toVcFC};aJ-~E&r9uCVNjboCrEfsF8vgIIC0-x)=v7#Ij
zYPaD9frxN7s+3pf<dE5SDH4Xu)W}TW`n@IJD^-k2)v4-9X4$J(V~C@kG@aO&Bvq^1
z-=~$IqMVOv&rOi)UhHbGT5wp8czuhCw_f0_1+*j45+dHM!agU3BF);to?{{`eU{mr
zh!D2v6@|v3Nt#^XsQ4nE_jG{#>E{YNa_mSTaFn{%Z{e!x3Gax~;s8u`D0$eiRI0~3
ztSCckM#TD(SHHiJ#aaeu<5&ul;CTU;^4Ai66?;<Eo_X5_?+yGTdQGRg7seUGqM}WS
zF)#N{u2HUrVJKG{L<l$$LYD}U06O^wItgSRIHpdhI4P$5JiBG@n@xY0kz`*?kA5-E
z&#&FP=;Da{z0Q{eqtol+fW6s*#GBd~6C~S^bXkuZ8|lK9?t3HtvVXlLTlavHyUthL
zcub(XT#F9-X(w7F?!eO!7s3_a(}S)O&)}77>~*PK*jELO#T*%KZnKc;t{X85=y}oB
zyBF_&4ihHX#_qs_^26tLU*43;r!_J8^6MVFT_4c5rz1<REku1&7#SI<w+~w7(WVW)
z##b_M^vgJ>7!$DTbx(uhuG@%=(cwoIp^G~dI$+QZXbUj2nI&auUvoyEMG;^MIAP5b
zuRgU56~-Dc?a7j!?B((`u&W)Wn0;NDnRn=OWB~FsL%E{;zilZ7(%&6q92k-2g@uGf
zM+)ZwQa7s-reZyAF%i^!{|sU_DIm_@%x@+hkRo&>Bz(9vhX5>)*pVcp-%dn8Oiaos
zbwS=_G!);<+Eu<3JP?7zBnY01uu^f&JqpR8CiiU<Yi9i5(u=rtUoxrDHx-IoQfUCw
zEi8kV0{U;_Vd4E3{w}oO*{t4%9F)qv)&1_=<DmVszJ0N7|E+Y@S)>l?`?&^MBFa=H
zwq#1m6a(HWI#|qWpy~(+2UDZE!C$hB9vl@;h4+hXoWoN{&{igL%==aJB73=>T66=o
zqhRgOE&Jxg8T!YeZa=^FFmzczyiM?sWRK>%8>qn`x?m3}t==geD(!Luh<1K_0J-Oy
z6^~Z;6iQNH6<(ls%0^OPa7uw%u!R3kjvQ~xyXRUQ50*AgcIA~Vf_HupPxYRGy>B>2
zQ9sCd#86Lsbfs^yC|G@D<>{L-eet3%V9dTMgekDyZ*lX=$;Qk9t}!{Ok?2V6vCg&Q
zrLhTTdo_*#owralPaoMqK2E_ElBBRKo0`}-*fyYR%Gvq5cBt9D#>P`?409j*Ug$lU
zJ1#99fW_JRPr=a@3B$wR9D;mWc!Z2oiN&W9FLuxOD3*_`!5mQ%0{;BgdkXJCrtF#8
zCvIQfSL}+R@L99tQ?@ZTmm7}5+a>B3uHTlq1O_G5)NJ9}E#lS?8Uteu-9*0S#@X-w
zO!GGxZF-yDBrec4t4z}XnIN5V7yLXeCCpTT3KKT}ELx`dk~p`-<jE%LkB*`{{%<eR
zsA{@X+ce>c04jRo7eQp2nCnND4Y}m#0Ud~4Od9jFJ+|eW{DJGq-<vYexX$@ojrMPV
zhwRT<9Yc!!Um@mGkT=AP&)HrjG?JIti#ju?-<jF6+cF76Wo1N-WR8CjRv*Lr2xW8w
zcf%5DbNPdk*V-?g^w6Kp*cyzX{?RJ*hrR{6lEq(=a*B;uqbyvW3R7#AjK9m3l3nZ7
zYwX4$yu%=SVKyZGeJx=8J_-8pA&Zu78&lC|vwx$WHn1GOJL_J3o|pDC1PwEP@s#L~
z-WXuL3gP%HWX8C*YMoltOn>PV%dDJ1#tH_F0)pMO6ra6C_*3`q;KN27D376Hs1Zfm
zsDDhPp7!lp&RXoPhuPixf!TguUg`u}*1=PH(%z*W+~g*HMzlW*iT;fBwKcME8}F(l
z@p)s$^KFfQZApoZXLm!AB4akesIz>`_6B?EVI{-)DZXb>9MeJkvU89U2xWz6SQKmn
z<;?O_MB(n3m_s@!Ibl;cwb9I0(7Ke09rZv;fx`0Ll+wHDiPH48albNUH=>XHGAzo~
z^^_GMbWqT5u89n=p34iVO+(ftL&?3B-6HFmg}R!ecSJ09sg(LV>^jIa75iGhTR08B
zkOUl>0$$X2i70!_R)6ZO3O+>W+<btUnr8T~{c$g`|4qMgXS^>o^QN6R_sjb#8UAuH
z#tMmk%g^dO42K>j>?up3(omtS&fND4A~L_c0Ne!WpTXv4pCvtew@Kc<&k8xarGSSL
z+!7mG>_G(BD9Z2d%G$qWA3RNOx1jaZXUJ^Ba*x_8{oc{kr<x0uP_7kvhN{v5YsK6o
z=rS36yEm~M!`4Q147u)FWG>4{e(S1{dco_4OAHZn4A^PzB-Hs$0Uapiwec7JcF;mK
zqvc;wn!_lz3%P-qTI~y&-dvnIL}^S4{LK9pyRt+Y8+5J1_F1|t@lSKeSX->cfxJWr
zYRB^J0#7vH+MtXzY0h;rbwn#2#t01Ek$rfbbL>2F)-v8qHIaM>MjpAIEVNX)`RWLs
zLrO4Zs}uY|Du_;Z)g3?D?bM#_Fu0KTH0B)9kB^p881KVrklLlvq?*t@qP2gHk$a3N
zYL1C2@)t~;uagqzO+pw6@<x~YdD2W836g8?<Hw1}=g^xFSZDiWck$T3A_o{mmEzNZ
zv)1}%vBJ|O{t(<6h7|hcTApI21(o=5bwRmw<65^w5~zS`u^@3{;!&;}z5te{VosbN
z4&I@or>i(`ys2Yj5r!?=^c^=PYjMKmJ{w%fk2q<I6c&<NNI-rD`^0`Pt;8xy<%haP
zLUsDw!=EA+^(zLeUriXyS`rP10fegf;`Mbh4a4hAgApe*K!mgVEsXDz5%NqYrf3cZ
zRawjmNn<Ba*35q5jbBW~Wd|vaxwIf<93;VqcwYI7R%uPshbyHTD(9A?YR;PEmpqYX
zyGz0rk<}b-6j#yrB3G8fs*-<r2vbR}yfMrp`@1pkfz(m;HkRoy4)XJ|KY(Q}trr|q
zl>ia?V|V&v(I0H^UxX<Yi|cFtsByDv;x-X8ZtvCh<$Ej&W7d+O<sMcF)n%%j?v5^>
zm1^g<5~+&@aO?D41AzB-3sd9*g&y=@9U9@<J>gLXL(P%9r-Y>!OMJs4^WRq#ilK@f
z91HL8l-LZ%Qr=?1UUp|@Jz4uXv8FETDCkEJNXfb>vNER+K3i>Y!fB2AUOq;apDk@`
zSeilH6*JCVFqVGmNuAUaAEhduzZ7$kz{rZu8!zcKuMOQk|C5YQFD3q&$skrKeLuRW
zN=`*HYbj89hiCK0qA)j#t}te=&w+sXmnMVgG>g%%z*`p5XMJz;XWQDO2F(_85GGe5
z7p*!^d%~}CjNPJLZyhiq7y@~$WJ~XX*DC4+_o)B1<Tr;1OaUI(cr_s!;G^Lta)|P1
zn2LY>7gCT+`xn}fTEln@bt^aj3qe&|@gEsBvv3gdLu(seZ~w17ePI9mZvf8e@ULxc
z;UMzpXFA9{BCR7Be&xXUzxKaeYe%I=zt&Oh5pNug9;=Q2bhhDEPRjo>oWMX7LiiH^
z1zeSo6n^D|{p4?1lB39R_x+*GE=`7n#P-;2|92{VHAwxKEa6P^sJCzi120$~DRDNU
z|LFfy*5Lhry(q_HGK_-|e&>w+I0lXj82Cg0KI=jXmvZ?xtn~x+L-2v3fM>Xn)Bc@O
z$4g(@dK4riRM10R^<PuMgU`6|K8^%%1p{xz9%N*2Ay=x$JE12ie4-6RLc)2Fvi_Is
zs{|bCD*ULce$c_B9$Z>DmNW_c#`WLCFd<-|x$-0ZO_>DV2l+?4JkUg{k92`5`G4r?
T2YSl%kp{W3plRCwz1sf(t-)<{

diff --git a/xsl/webhelp/indexer/src/com/nexwave/nquindexer/IndexerTask.java b/xsl/webhelp/indexer/src/com/nexwave/nquindexer/IndexerTask.java
index 2a347d3f5..cfde68342 100755
--- a/xsl/webhelp/indexer/src/com/nexwave/nquindexer/IndexerTask.java
+++ b/xsl/webhelp/indexer/src/com/nexwave/nquindexer/IndexerTask.java
@@ -51,7 +51,7 @@ public class IndexerTask{
 
 	// ANT parameters
 	private String htmldir=null;
-    private String indexerLanguage="en";
+    public static String indexerLanguage="en";
 
     //supported languages: add new additions to this. don't include country codes to the end such as en_US or en_UK,
     // as stemmers doesn't find a difference between them.
@@ -90,7 +90,7 @@ public class IndexerTask{
             int i=0;
             for (;i<supportedLanguages.length;i++) {
                 if(indexerLanguage.equals(supportedLanguages[i])){
-                    this.indexerLanguage = supportedLanguages[i];
+                    IndexerTask.indexerLanguage = supportedLanguages[i];
                     break;
                 }
             }
@@ -100,10 +100,10 @@ public class IndexerTask{
                 System.out.println("The given language, \""+indexerLanguage+"\", does not have extensive support for " +
                         "searching or language code is specified in a bad format. Check documentation for details. " +
                         "Language now defaults to english.");
-                this.indexerLanguage = "en";
+                IndexerTask.indexerLanguage = "en";
             } 
         } else {
-            this.indexerLanguage = "en";
+            IndexerTask.indexerLanguage = "en";
         } 
     }
 
diff --git a/xsl/webhelp/indexer/src/com/nexwave/nquindexer/WriteJSFiles.java b/xsl/webhelp/indexer/src/com/nexwave/nquindexer/WriteJSFiles.java
index 5c6c45166..985d2080d 100755
--- a/xsl/webhelp/indexer/src/com/nexwave/nquindexer/WriteJSFiles.java
+++ b/xsl/webhelp/indexer/src/com/nexwave/nquindexer/WriteJSFiles.java
@@ -182,6 +182,8 @@ public class WriteJSFiles {
                       The value is the numbers of the files in which the word exists.
                       Example: w["key"]="file1,file2,file3";*/
                     int count = 0;
+                    if(i==1)
+                        out.write("var indexerLanguage=\""+IndexerTask.indexerLanguage+"\";\n");
                     out.write("//Auto generated index for searching.\n");
                     while (keyIt.hasNext()) {        //&& (tempLetter == tstr.charAt(0)) 
                         out.write("w[\"" + tstr + "\"]" + "=\"" + indexMap.get(tstr) + "\";\n");
diff --git a/xsl/webhelp/template/content/search/nwSearchFnt.js b/xsl/webhelp/template/content/search/nwSearchFnt.js
index 18206946c..fe01240ef 100755
--- a/xsl/webhelp/template/content/search/nwSearchFnt.js
+++ b/xsl/webhelp/template/content/search/nwSearchFnt.js
@@ -8,11 +8,9 @@
  */
 
 //string initialization
-
-
-htmlfileList = "htmlFileList.js";
-htmlfileinfoList = "htmlFileInfoList.js";
-
+var htmlfileList = "htmlFileList.js";
+var htmlfileinfoList = "htmlFileInfoList.js";
+var useCJKTokenizing = false;
 
 /* Cette fonction verifie la validite de la recherche entrre par l utilisateur */
 function Verifie(ditaSearch_Form) {
@@ -56,13 +54,12 @@ function Effectuer_recherche(expressionInput) {
     //DisplayWaitingMessage();
 
     /*data initialisation*/
-    searchFor = "";       // expression en lowercase et sans les caracteres speciaux
+    searchFor = "";       // expression en lowercase et sans les caracte    res speciaux
     //w = new Object();  // hashtable, key=word, value = list of the index of the html files
     scriptLetterTab = new scriptfirstchar(); // Array containing the first letter of each word to look for
     var scriptsarray = new Array(); // Array with the name of the scripts to load
     var wordsList = new Array(); // Array with the words to look for
-    var cleanwordsList = new Array(); // Array with the words to look for
-    var stemmedWordsList = new Array(); // Array with the words to look for after removing spaces
+    var finalWordsList = new Array(); // Array with the words to look for after removing spaces
     var listNumerosDesFicStr = "";
     var ou_recherche = true;
     var linkTab = new Array();
@@ -80,32 +77,18 @@ function Effectuer_recherche(expressionInput) {
     wordsList = searchFor.split(" ");
     wordsList.sort();
 
-    for(var j in wordsList){
-        var word = wordsList[j];
-        if(typeof stemmer != "undefined" ){
-            stemQueryMap[stemmer(word)] = word;
-        } else {
-            stemQueryMap[word] = word;
-        }
-    }
-
-     //stemmedWordsList is the stemmed list of words separated by spaces.
-    for (t in wordsList) {
-        wordsList[t] = wordsList[t].replace(/(%22)|^-/g, "")
-        if (wordsList[t] != "%20") {
-            scriptLetterTab.add(wordsList[t].charAt(0));
-            cleanwordsList.push(wordsList[t]);
-        }
-    }
-
-    if(typeof stemmer != "undefined" ){
-        //Do the stemming using Porter's stemming algorithm
-        for (var i = 0; i < cleanwordsList.length; i++) {
-            var stemWord = stemmer(cleanwordsList[i]);
-            stemmedWordsList.push(stemWord);
-        }
+    //set the tokenizing method
+    if(typeof indexerLanguage != "undefined" && (indexerLanguage=="cn" || indexerLanguage=="ja" ||indexerLanguage=="ko")){
+        useCJKTokenizing=true;
     } else {
-        stemmedWordsList = cleanwordsList;
+        useCJKTokenizing=false;
+    }
+    //If Lucene CJKTokenizer was used as the indexer, then useCJKTokenizing will be true. Else, do normal tokenizing.
+    // 2-gram tokenizinghappens in CJKTokenizing,  
+    if(useCJKTokenizing){
+        finalWordsList = cjkTokenize(wordsList);
+    } else { 
+        finalWordsList = tokenize(wordsList);
     }
 
     //load the scripts with the indices: the following lines do not work on the server. To be corrected
@@ -117,32 +100,29 @@ function Effectuer_recherche(expressionInput) {
      * Compare with the indexed words (in the w[] array), and push words that are in it to tempTab.
      */
     var tempTab = new Array();
-    for (t in stemmedWordsList) {
-        if (w[stemmedWordsList[t].toString()] == undefined) {
-            txt_wordsnotfound += stemmedWordsList[t] + " ";
+    for (t in finalWordsList) {
+        if (w[finalWordsList[t].toString()] == undefined) {
+            txt_wordsnotfound += finalWordsList[t] + " ";
         } else {
-            tempTab.push(stemmedWordsList[t]);
+            tempTab.push(finalWordsList[t]);
         }
     }
-    stemmedWordsList = tempTab;
+    finalWordsList = tempTab;
 
-    if (stemmedWordsList.length) {
+    if (finalWordsList.length) {
 
-        // recherche 'et' et 'ou' en une fois
-        fileAndWordList = SortResults(stemmedWordsList);
+        //search 'and' and 'or' one time
+        fileAndWordList = SortResults(finalWordsList);
 
         cpt = fileAndWordList.length;
-        for (i = cpt - 1; i >= 0; i--) {
+        for (var i = cpt - 1; i >= 0; i--) {
             if (fileAndWordList[i] != undefined) {
-
                 linkTab.push("<p>" + txt_results_for + " " + "<span class=\"searchExpression\">" + fileAndWordList[i][0].motslisteDisplay + "</span>" + "</p>");
 
                 linkTab.push("<ul class='searchresult'>");
                 for (t in fileAndWordList[i]) {
                     //DEBUG: alert(": "+ fileAndWordList[i][t].filenb+" " +fileAndWordList[i][t].motsliste);
                     //linkTab.push("<li><a href=\"../"+fl[fileAndWordList[i][t].filenb]+"\">"+fl[fileAndWordList[i][t].filenb]+"</a></li>");
-
-
                     tempInfo = fil[fileAndWordList[i][t].filenb];
                     pos1 = tempInfo.indexOf("@@@");
                     pos2 = tempInfo.lastIndexOf("@@@");
@@ -151,24 +131,21 @@ function Effectuer_recherche(expressionInput) {
                     tempShortdesc = tempInfo.substring(pos2 + 3, tempInfo.length);
 
                     //file:///home/kasun/docbook/WEBHELP/webhelp-draft-output-format-idea/src/main/resources/web/webhelp/installation.html
-                   var linkString = "<li><a href=" + tempPath + ">" + tempTitle + "</a>";
-                   // var linkString = "<li><a href=\"installation.html\">" + tempTitle + "</a>";
+                    var linkString = "<li><a href=" + tempPath + ">" + tempTitle + "</a>";
+                    // var linkString = "<li><a href=\"installation.html\">" + tempTitle + "</a>";
                     if ((tempShortdesc != "null")) {
                         linkString += "\n<div class=\"shortdesclink\">" + tempShortdesc + "</div>";
                     }
                     linkString += "</li>";
-
                     linkTab.push(linkString);
-
                 }
                 linkTab.push("</ul>");
             }
         }
     }
 
-    var results="";
-    if (linkTab.length > 0) {
-
+    var results = "";
+    if (linkTab.length > 0) { 
         /*writeln ("<p>" + txt_results_for + " " + "<span class=\"searchExpression\">"  + cleanwordsList + "</span>" + "<br/>"+"</p>");*/
         results = "<p>";
         //write("<ul class='searchresult'>");
@@ -176,47 +153,133 @@ function Effectuer_recherche(expressionInput) {
             results += linkTab[t].toString();
         }
         results += "</p>";
-    } else{
-         results = "<p>"+"Your search returned no results for "+ "<span class=\"searchExpression\">" + txt_wordsnotfound + "</span>" +"</p>";
+    } else {
+        results = "<p>" + "Your search returned no results for " + "<span class=\"searchExpression\">" + txt_wordsnotfound + "</span>" + "</p>";
     }
     //alert(results);
+    document.getElementById('searchResults').innerHTML = results; 
+}
+
+function tokenize(wordsList){
+    var stemmedWordsList = new Array(); // Array with the words to look for after removing spaces
+    var cleanwordsList = new Array(); // Array with the words to look for
+    for(var j in wordsList){
+        var word = wordsList[j];
+        if(typeof stemmer != "undefined" ){
+            stemQueryMap[stemmer(word)] = word;
+        } else {
+            stemQueryMap[word] = word;
+        }
+    } 
+     //stemmedWordsList is the stemmed list of words separated by spaces.
+    for (t in wordsList) {
+        wordsList[t] = wordsList[t].replace(/(%22)|^-/g, "")
+        if (wordsList[t] != "%20") {
+            scriptLetterTab.add(wordsList[t].charAt(0));
+            cleanwordsList.push(wordsList[t]);
+        }
+    }
+
+    if(typeof stemmer != "undefined" ){
+        //Do the stemming using Porter's stemming algorithm
+        for (var i = 0; i < cleanwordsList.length; i++) {
+            var stemWord = stemmer(cleanwordsList[i]);
+            stemmedWordsList.push(stemWord);
+        }
+    } else {
+        stemmedWordsList = cleanwordsList;
+    }
+    return stemmedWordsList;
+}
+
+function cjkTokenize(wordsList){
+    var allTokens= new Array();
+    var notCJKTokens= new Array();
+    var j=0;
+    for(j=0;j<wordsList.length;j++){
+        var word = wordsList[j];
+        if(getAvgAsciiValue(word) < 127){
+            notCJKTokens.push(word);
+        } else { 
+            var tokenizer = new CJKTokenizer(word);
+            var tokensTmp = tokenizer.getAllTokens();
+            allTokens = allTokens.concat(tokensTmp);
+        }
+    }
+    allTokens = allTokens.concat(tokenize(notCJKTokens));
+    return allTokens;
+}
+
+//A simple way to determine whether the query is in english or not.
+function getAvgAsciiValue(word){
+    var tmp = 0;
+    var num = word.length < 5 ? word.length:5;
+    for(i=0;i<num;i++){
+        if(i==5) break;
+        tmp += word.charCodeAt(i);
+    }
+    return tmp/num;
+}
 
-    document.getElementById('searchResults').innerHTML = results;
-
-    /* Display results * /
-     with (parent.frames['searchresults'].document) {
-     writeln("<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.0 Transitional//EN\">\n<html><head>");
-     writeln("<meta http-equiv=\"content-type\" content=\"text/html; charset=utf-8\">");
-     //writeln("<link href=\"css/commonltr.css\" type=\"text/css\" rel=\"stylesheet\">" );
-     //writeln("<link rel=\"stylesheet\" type=\"text/css\" href=\"css/search.css\">") ;
-     writeln("<style>body{\
-     font-family: verdana, sans-serif;\
-     font-size: .7em;\
-     background: #f3f3f3; }\
-     .searchExpression{ font-weight: bold;}</style>") ;
-     writeln("<title>"+txt_filesfound+"</title></head>");
-     writeln("<body onload = \"self.focus()\">");
-     //writeln("<h2>" + txt_search_result + " " + "<i>" + wordsList + "</i>" + "</h2>");
-
-     // If no results, display a message
-     if ( txt_wordsnotfound != "" ) {writeln("<p>"+"Your search returned no results for "+ "<span class=\"searchExpression\">" + txt_wordsnotfound + "</span>" +"</p>")}
-
-     // If results: display them
-     if (linkTab.length > 0  ) {
-
-     /*writeln ("<p>" + txt_results_for + " " + "<span class=\"searchExpression\">"  + cleanwordsList + "</span>" + "<br/>"+"</p>");* /
-     write("<p>");
-     //write("<ul class='searchresult'>");
-     for (t in linkTab) {
-     writeln(linkTab[t].toString())
-     }
-     writeln("</p>");
-     }
-
-     writeln ("</body></html>");
-     close() ;
-
-     }   */
+//CJKTokenizer
+function CJKTokenizer(input){
+    this.input = input;
+    this.offset=-1;
+    this.tokens = new Array(); 
+    this.incrementToken = incrementToken;
+    this.tokenize = tokenize;
+    this.getAllTokens = getAllTokens;
+    this.unique = unique;
+
+    function incrementToken(){
+		if(this.input.length - 2 <= this.offset){
+		//	console.log("false "+offset);
+			return false;
+		}
+		else {
+			this.offset+=1;
+			return true;
+		}
+	}
+
+	function tokenize(){
+		//document.getElementById("content").innerHTML += x.substring(offset,offset+2)+"<br>";
+		return this.input.substring(this.offset,this.offset+2);
+	}
+
+	function getAllTokens(){
+		while(this.incrementToken()){
+			var tmp = this.tokenize();
+			this.tokens.push(tmp);
+		}
+		var sortedTokens = this.unique(this.tokens);
+
+        return sortedTokens;    
+//		document.getElementById("content").innerHTML += tokens+" ";
+//		document.getElementById("content").innerHTML += "<br>dada"+sortedTokens+" ";
+//		console.log(tokens.length+"dsdsds");
+		/*for(i=0;i<tokens.length;i++){
+			console.log(tokens[i]);
+			var ss = tokens[i] == sortedTokens[i];
+
+//			document.getElementById("content").innerHTML += "<br>dada"+un[i]+"- "+stems[i]+"&nbsp;&nbsp;&nbsp;"+ ss;
+			document.getElementById("content").innerHTML += "<br>"+sortedTokens[i];
+		}*/
+	}
+
+	function unique(a)
+	{
+	   var r = new Array();
+	   o:for(var i = 0, n = a.length; i < n; i++)
+	   {
+	      for(var x = 0, y = r.length; x < y; x++)
+	      {
+		 if(r[x]==a[i]) continue o;
+	      }
+	      r[r.length] = a[i];
+	   }
+	   return r;
+	} 
 }
 
 
@@ -316,9 +379,10 @@ function onScriptLoadedFunc(e) {
     }
 }
 
+/*
 function onLoadComplete() {
     alert("loaded !!");
-}
+} */
 
 /* End of scriptloader functions */
 
@@ -413,7 +477,11 @@ function SortResults(mots) {
 
         var tempDisplay = new Array();
         for (var x in tab) {
-            tempDisplay.push(stemQueryMap[tab[x]]); //get the original word from the stem word.
+            if(stemQueryMap[tab[x]] != undefined){
+                tempDisplay.push(stemQueryMap[tab[x]]); //get the original word from the stem word.
+            } else {
+                tempDisplay.push(tab[x]); //no stem is available. (probably a CJK language)
+            }
         }
         var tempDispString = tempDisplay.join(", ");
 
-- 
2.40.0