/******************************************************************************
** This file is an amalgamation of many separate C source files from SQLite
-** version 3.6.20. By combining all the individual C code files into this
+** version 3.6.21. By combining all the individual C code files into this
** single large file, the entire code can be compiled as a one translation
** unit. This allows many compilers to do optimizations that would not be
** possible if the files were compiled separately. Performance improvements
*************************************************************************
**
** This file defines various limits of what SQLite can process.
-**
-** @(#) $Id$
*/
/*
** may be executed.
*/
#ifndef SQLITE_MAX_TRIGGER_DEPTH
-#if defined(SQLITE_SMALL_STACK)
-# define SQLITE_MAX_TRIGGER_DEPTH 10
-#else
# define SQLITE_MAX_TRIGGER_DEPTH 1000
#endif
-#endif
/************** End of sqliteLimit.h *****************************************/
/************** Continuing where we left off in sqliteInt.h ******************/
**
** Requirements: [H10011] [H10014]
*/
-#define SQLITE_VERSION "3.6.20"
-#define SQLITE_VERSION_NUMBER 3006020
-#define SQLITE_SOURCE_ID "2009-11-04 13:30:02 eb7a544fe49d1626bacecfe53ddc03fe082e3243"
+#define SQLITE_VERSION "3.6.21"
+#define SQLITE_VERSION_NUMBER 3006021
+#define SQLITE_SOURCE_ID "2009-12-07 16:39:13 1ed88e9d01e9eda5cbc622e7614277f29bcc551c"
/*
** CAPI3REF: Run-Time Library Version Numbers {H10020} <S60100>
** The sqlite3_initialize() routine initializes the
** SQLite library. The sqlite3_shutdown() routine
** deallocates any resources that were allocated by sqlite3_initialize().
-** This routines are designed to aid in process initialization and
+** These routines are designed to aid in process initialization and
** shutdown on embedded systems. Workstation applications using
** SQLite normally do not need to invoke either of these routines.
**
** For the purposes of this routine, an [INSERT] is considered to
** be successful even if it is subsequently rolled back.
**
+** This function is accessible to SQL statements via the
+** [last_insert_rowid() SQL function].
+**
** Requirements:
** [H12221] [H12223]
**
** However, the number returned does not include changes
** caused by subtriggers since those have their own context.
**
-** See also the [sqlite3_total_changes()] interface and the
-** [count_changes pragma].
+** See also the [sqlite3_total_changes()] interface, the
+** [count_changes pragma], and the [changes() SQL function].
**
** Requirements:
** [H12241] [H12243]
** completed (when the statement handle is passed to [sqlite3_reset()] or
** [sqlite3_finalize()]).
**
-** See also the [sqlite3_changes()] interface and the
-** [count_changes pragma].
+** See also the [sqlite3_changes()] interface, the
+** [count_changes pragma], and the [total_changes() SQL function].
**
** Requirements:
** [H12261] [H12263]
** In the "v2" interfaces, the prepared statement
** that is returned (the [sqlite3_stmt] object) contains a copy of the
** original SQL text. This causes the [sqlite3_step()] interface to
-** behave a differently in three ways:
+** behave differently in three ways:
**
** <ol>
** <li>
** {H12606} Extension loading must be enabled using
** [sqlite3_enable_load_extension()] prior to calling this API,
** otherwise an error will be returned.
+**
+** See also the [load_extension() SQL function].
*/
SQLITE_API int sqlite3_load_extension(
sqlite3 *db, /* Load the extension into this database connection */
*************************************************************************
** This is the header file for the generic hash-table implemenation
** used in SQLite.
-**
-** $Id$
*/
#ifndef _SQLITE_HASH_H_
#define _SQLITE_HASH_H_
#define ROUNDDOWN8(x) ((x)&~7)
/*
-** Assert that the pointer X is aligned to an 8-byte boundary.
+** Assert that the pointer X is aligned to an 8-byte boundary. This
+** macro is used only within assert() to verify that the code gets
+** all alignment restrictions correct.
+**
+** Except, if SQLITE_4_BYTE_ALIGNED_MALLOC is defined, then the
+** underlying malloc() implemention might return us 4-byte aligned
+** pointers. In that case, only verify 4-byte alignment.
*/
-#define EIGHT_BYTE_ALIGNMENT(X) ((((char*)(X) - (char*)0)&7)==0)
+#ifdef SQLITE_4_BYTE_ALIGNED_MALLOC
+# define EIGHT_BYTE_ALIGNMENT(X) ((((char*)(X) - (char*)0)&3)==0)
+#else
+# define EIGHT_BYTE_ALIGNMENT(X) ((((char*)(X) - (char*)0)&7)==0)
+#endif
/*
** This header file defines the interface that the sqlite B-Tree file
** subsystem. See comments in the source code for a detailed description
** of what each interface routine does.
-**
-** @(#) $Id$
*/
#ifndef _BTREE_H_
#define _BTREE_H_
BtCursor *pCursor /* Space to write cursor structure */
);
SQLITE_PRIVATE int sqlite3BtreeCursorSize(void);
+SQLITE_PRIVATE void sqlite3BtreeCursorZero(BtCursor*);
SQLITE_PRIVATE int sqlite3BtreeCloseCursor(BtCursor*);
SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked(
** This header defines the interface to the virtual database engine
** or VDBE. The VDBE implements an abstract machine that runs a
** simple program to access and modify the underlying database.
-**
-** $Id$
*/
#ifndef _SQLITE_VDBE_H_
#define _SQLITE_VDBE_H_
struct VdbeOp {
u8 opcode; /* What operation to perform */
signed char p4type; /* One of the P4_xxx constants for p4 */
- u8 opflags; /* Not currently used */
+ u8 opflags; /* Mask of the OPFLG_* flags in opcodes.h */
u8 p5; /* Fifth parameter is an unsigned character */
int p1; /* First operand */
int p2; /* Second parameter (often the jump destination) */
#define OPFLG_IN1 0x0004 /* in1: P1 is an input */
#define OPFLG_IN2 0x0008 /* in2: P2 is an input */
#define OPFLG_IN3 0x0010 /* in3: P3 is an input */
-#define OPFLG_OUT3 0x0020 /* out3: P3 is an output */
+#define OPFLG_OUT2 0x0020 /* out2: P2 is an output */
+#define OPFLG_OUT3 0x0040 /* out3: P3 is an output */
#define OPFLG_INITIALIZER {\
-/* 0 */ 0x00, 0x01, 0x01, 0x04, 0x04, 0x10, 0x00, 0x02,\
-/* 8 */ 0x02, 0x02, 0x02, 0x02, 0x00, 0x00, 0x04, 0x04,\
-/* 16 */ 0x00, 0x00, 0x00, 0x04, 0x04, 0x05, 0x04, 0x00,\
+/* 0 */ 0x00, 0x01, 0x05, 0x04, 0x04, 0x10, 0x00, 0x02,\
+/* 8 */ 0x02, 0x02, 0x02, 0x02, 0x00, 0x00, 0x24, 0x24,\
+/* 16 */ 0x00, 0x00, 0x00, 0x24, 0x04, 0x05, 0x04, 0x00,\
/* 24 */ 0x00, 0x01, 0x05, 0x05, 0x00, 0x00, 0x00, 0x02,\
/* 32 */ 0x00, 0x00, 0x00, 0x02, 0x10, 0x00, 0x00, 0x00,\
/* 40 */ 0x00, 0x00, 0x00, 0x11, 0x11, 0x11, 0x11, 0x08,\
/* 48 */ 0x11, 0x11, 0x11, 0x11, 0x02, 0x02, 0x00, 0x00,\
/* 56 */ 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x01, 0x01,\
-/* 64 */ 0x01, 0x01, 0x01, 0x08, 0x2c, 0x2c, 0x00, 0x02,\
-/* 72 */ 0x11, 0x05, 0x05, 0x15, 0x15, 0x15, 0x15, 0x15,\
-/* 80 */ 0x15, 0x11, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c,\
-/* 88 */ 0x2c, 0x2c, 0x2c, 0x2c, 0x02, 0x04, 0x02, 0x00,\
+/* 64 */ 0x01, 0x01, 0x01, 0x08, 0x4c, 0x4c, 0x00, 0x02,\
+/* 72 */ 0x01, 0x05, 0x05, 0x15, 0x15, 0x15, 0x15, 0x15,\
+/* 80 */ 0x15, 0x01, 0x4c, 0x4c, 0x4c, 0x4c, 0x4c, 0x4c,\
+/* 88 */ 0x4c, 0x4c, 0x4c, 0x4c, 0x02, 0x24, 0x02, 0x00,\
/* 96 */ 0x02, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\
-/* 104 */ 0x08, 0x21, 0x15, 0x01, 0x02, 0x00, 0x01, 0x08,\
+/* 104 */ 0x0c, 0x45, 0x15, 0x01, 0x02, 0x00, 0x01, 0x08,\
/* 112 */ 0x05, 0x05, 0x05, 0x00, 0x00, 0x00, 0x01, 0x00,\
/* 120 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01,\
/* 128 */ 0x00, 0x00, 0x02, 0x02, 0x00, 0x00, 0x00, 0x00,\
SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe*,int,int,int);
SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int);
SQLITE_PRIVATE int sqlite3VdbeAddOp4(Vdbe*,int,int,int,int,const char *zP4,int);
+SQLITE_PRIVATE int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int);
SQLITE_PRIVATE int sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp);
SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe*, int addr, int P1);
SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe*, int addr, int P2);
SQLITE_PRIVATE void sqlite3VdbeProgramDelete(sqlite3 *, SubProgram *, int);
SQLITE_PRIVATE sqlite3_value *sqlite3VdbeGetValue(Vdbe*, int, u8);
SQLITE_PRIVATE void sqlite3VdbeSetVarmask(Vdbe*, int);
+#ifndef SQLITE_OMIT_TRACE
+SQLITE_PRIVATE char *sqlite3VdbeExpandSql(Vdbe*, const char*);
+#endif
SQLITE_PRIVATE UnpackedRecord *sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,char*,int);
SQLITE_PRIVATE void sqlite3VdbeDeleteUnpackedRecord(UnpackedRecord*);
** This header file defines the interface that the sqlite page cache
** subsystem. The page cache subsystem reads and writes a file a page
** at a time and provides a journal for rollback.
-**
-** @(#) $Id$
*/
#ifndef _PAGER_H_
*************************************************************************
** This header file defines the interface that the sqlite page cache
** subsystem.
-**
-** @(#) $Id$
*/
#ifndef _PCACHE_H_
**
** This header file is #include-ed by sqliteInt.h and thus ends up
** being included by every source file.
-**
-** $Id$
*/
#ifndef _SQLITE_OS_H_
#define _SQLITE_OS_H_
** NOTE: source files should *not* #include this header file directly.
** Source files should #include the sqliteInt.h file and let that file
** include this one indirectly.
-**
-** $Id$
*/
#define SQLITE_FUNC_NEEDCOLL 0x08 /* sqlite3GetFuncCollSeq() might be called */
#define SQLITE_FUNC_PRIVATE 0x10 /* Allowed for internal use only */
#define SQLITE_FUNC_COUNT 0x20 /* Built-in count(*) aggregate */
+#define SQLITE_FUNC_COALESCE 0x40 /* Built-in coalesce() or ifnull() function */
/*
** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are
** the option is available (at compile-time).
*/
#if SQLITE_MAX_VARIABLE_NUMBER<=32767
-typedef i64 ynVar;
+typedef i16 ynVar;
#else
typedef int ynVar;
#endif
** The Parse.pTriggerPrg list never contains two entries with the same
** values for both pTrigger and orconf.
**
-** The TriggerPrg.oldmask variable is set to a mask of old.* columns
+** The TriggerPrg.aColmask[0] variable is set to a mask of old.* columns
** accessed (or set to 0 for triggers fired as a result of INSERT
-** statements).
+** statements). Similarly, the TriggerPrg.aColmask[1] variable is set to
+** a mask of new.* columns used by the program.
*/
struct TriggerPrg {
Trigger *pTrigger; /* Trigger this program was coded from */
int orconf; /* Default ON CONFLICT policy */
SubProgram *pProgram; /* Program implementing pTrigger/orconf */
- u32 oldmask; /* Mask of old.* columns accessed */
+ u32 aColmask[2]; /* Masks of old.*, new.* columns accessed */
TriggerPrg *pNext; /* Next entry in Parse.pTriggerPrg list */
};
Parse *pToplevel; /* Parse structure for main program (or NULL) */
Table *pTriggerTab; /* Table triggers are being coded for */
u32 oldmask; /* Mask of old.* columns referenced */
+ u32 newmask; /* Mask of new.* columns referenced */
u8 eTriggerOp; /* TK_UPDATE, TK_INSERT or TK_DELETE */
u8 eOrconf; /* Default ON CONFLICT policy for trigger steps */
u8 disableTriggers; /* True to disable triggers */
SQLITE_PRIVATE int sqlite3IsNaN(double);
SQLITE_PRIVATE void sqlite3VXPrintf(StrAccum*, int, const char*, va_list);
+#ifndef SQLITE_OMIT_TRACE
+SQLITE_PRIVATE void sqlite3XPrintf(StrAccum*, const char*, ...);
+#endif
SQLITE_PRIVATE char *sqlite3MPrintf(sqlite3*,const char*, ...);
SQLITE_PRIVATE char *sqlite3VMPrintf(sqlite3*,const char*, va_list);
SQLITE_PRIVATE char *sqlite3MAppendf(sqlite3*,char*,const char*,...);
SQLITE_PRIVATE Expr *sqlite3ExprAnd(sqlite3*,Expr*, Expr*);
SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse*,ExprList*, Token*);
SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse*, Expr*);
-SQLITE_PRIVATE void sqlite3ExprClear(sqlite3*, Expr*);
SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3*, Expr*);
SQLITE_PRIVATE ExprList *sqlite3ExprListAppend(Parse*,ExprList*,Expr*);
SQLITE_PRIVATE void sqlite3ExprListSetName(Parse*,ExprList*,Token*,int);
SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr*);
SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr*);
SQLITE_PRIVATE int sqlite3ExprIsInteger(Expr*, int*);
+SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr*);
+SQLITE_PRIVATE void sqlite3ExprCodeIsNullJump(Vdbe*, const Expr*, int, int);
+SQLITE_PRIVATE int sqlite3ExprNeedsNoAffinityChange(const Expr*, char);
SQLITE_PRIVATE int sqlite3IsRowid(const char*);
SQLITE_PRIVATE void sqlite3GenerateRowDelete(Parse*, Table*, int, int, int, Trigger *, int);
SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int*);
SQLITE_PRIVATE TriggerStep *sqlite3TriggerDeleteStep(sqlite3*,Token*, Expr*);
SQLITE_PRIVATE void sqlite3DeleteTrigger(sqlite3*, Trigger*);
SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTrigger(sqlite3*,int,const char*);
-SQLITE_PRIVATE u32 sqlite3TriggerOldmask(Parse*,Trigger*,ExprList*,Table*,int);
+SQLITE_PRIVATE u32 sqlite3TriggerColmask(Parse*,Trigger*,ExprList*,int,int,Table*,int);
# define sqlite3ParseToplevel(p) ((p)->pToplevel ? (p)->pToplevel : (p))
#else
# define sqlite3TriggersExist(B,C,D,E,F) 0
# define sqlite3CodeRowTriggerDirect(A,B,C,D,E,F)
# define sqlite3TriggerList(X, Y) 0
# define sqlite3ParseToplevel(p) p
-# define sqlite3TriggerOldmask(A,B,C,D,E) 0
+# define sqlite3TriggerColmask(A,B,C,D,E,F,G) 0
#endif
SQLITE_PRIVATE int sqlite3JoinType(Parse*, Token*, Token*, Token*);
SQLITE_PRIVATE int sqlite3ValueFromExpr(sqlite3 *, Expr *, u8, u8, sqlite3_value **);
SQLITE_PRIVATE void sqlite3ValueApplyAffinity(sqlite3_value *, u8, u8);
#ifndef SQLITE_AMALGAMATION
+SQLITE_PRIVATE const unsigned char sqlite3OpcodeProperty[];
SQLITE_PRIVATE const unsigned char sqlite3UpperToLower[];
SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[];
SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config;
SQLITE_PRIVATE int sqlite3VtabBegin(sqlite3 *, VTable *);
SQLITE_PRIVATE FuncDef *sqlite3VtabOverloadFunction(sqlite3 *,FuncDef*, int nArg, Expr*);
SQLITE_PRIVATE void sqlite3InvalidFunction(sqlite3_context*,int,sqlite3_value**);
+SQLITE_PRIVATE int sqlite3VdbeParameterIndex(Vdbe*, const char*, int);
SQLITE_PRIVATE int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *);
SQLITE_PRIVATE int sqlite3Reprepare(Vdbe*);
SQLITE_PRIVATE void sqlite3ExprListCheckLength(Parse*, ExprList*, const char*);
** This file contains definitions of global variables and contants.
*/
-
/* An array to map all upper-case characters into their corresponding
** lower-case character.
**
** isalnum() 0x06
** isxdigit() 0x08
** toupper() 0x20
+** SQLite identifier character 0x40
**
** Bit 0x20 is set if the mapped character requires translation to upper
** case. i.e. if the character is a lower-case ASCII character.
** Standard function tolower() is implemented using the sqlite3UpperToLower[]
** array. tolower() is used more often than toupper() by SQLite.
**
+** Bit 0x40 is set if the character non-alphanumeric and can be used in an
+** SQLite identifier. Identifiers are alphanumerics, "_", "$", and any
+** non-ASCII UTF character. Hence the test for whether or not a character is
+** part of an identifier is 0x46.
+**
** SQLite's versions are identical to the standard versions assuming a
** locale of "C". They are implemented as macros in sqliteInt.h.
*/
0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, /* 08..0f ........ */
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 10..17 ........ */
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 18..1f ........ */
- 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 20..27 !"#$%&' */
+ 0x01, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, /* 20..27 !"#$%&' */
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 28..2f ()*+,-./ */
0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, /* 30..37 01234567 */
0x0c, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 38..3f 89:;<=>? */
0x00, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x02, /* 40..47 @ABCDEFG */
0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 48..4f HIJKLMNO */
0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 50..57 PQRSTUVW */
- 0x02, 0x02, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, /* 58..5f XYZ[\]^_ */
+ 0x02, 0x02, 0x02, 0x00, 0x00, 0x00, 0x00, 0x40, /* 58..5f XYZ[\]^_ */
0x00, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x22, /* 60..67 `abcdefg */
0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 68..6f hijklmno */
0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 70..77 pqrstuvw */
0x22, 0x22, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, /* 78..7f xyz{|}~. */
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 80..87 ........ */
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 88..8f ........ */
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 90..97 ........ */
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 98..9f ........ */
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* a0..a7 ........ */
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* a8..af ........ */
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* b0..b7 ........ */
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* b8..bf ........ */
-
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* c0..c7 ........ */
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* c8..cf ........ */
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* d0..d7 ........ */
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* d8..df ........ */
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* e0..e7 ........ */
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* e8..ef ........ */
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* f0..f7 ........ */
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 /* f8..ff ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 80..87 ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 88..8f ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 90..97 ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 98..9f ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* a0..a7 ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* a8..af ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* b0..b7 ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* b8..bf ........ */
+
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* c0..c7 ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* c8..cf ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* d0..d7 ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* d8..df ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* e0..e7 ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* e8..ef ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* f0..f7 ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40 /* f8..ff ........ */
};
#endif
*/
SQLITE_PRIVATE int sqlite3PendingByte = 0x40000000;
+/*
+** Properties of opcodes. The OPFLG_INITIALIZER macro is
+** created by mkopcodeh.awk during compilation. Data is obtained
+** from the comments following the "case OP_xxxx:" statements in
+** the vdbe.c file.
+*/
+SQLITE_PRIVATE const unsigned char sqlite3OpcodeProperty[] = OPFLG_INITIALIZER;
+
/************** End of global.c **********************************************/
/************** Begin file status.c ******************************************/
/*
**
** This module implements the sqlite3_status() interface and related
** functionality.
-**
-** $Id$
*/
/*
** sqlite3RegisterDateTimeFunctions() found at the bottom of the file.
** All other code has file scope.
**
-** $Id$
-**
** SQLite processes all times and dates as Julian Day numbers. The
** dates and times are stored as the number of days since noon
** in Greenwich on November 24, 4714 B.C. according to the Gregorian
**
** This file contains OS interface code that is common to all
** architectures.
-**
-** $Id$
*/
#define _SQLITE_OS_C_ 1
#undef _SQLITE_OS_C_
int nPathOut,
char *zPathOut
){
+ zPathOut[0] = 0;
return pVfs->xFullPathname(pVfs, zPath, nPathOut, zPathOut);
}
#ifndef SQLITE_OMIT_LOAD_EXTENSION
**
*************************************************************************
**
-** $Id$
-*/
-
-/*
** This file contains code to support the concept of "benign"
** malloc failures (when the xMalloc() or xRealloc() method of the
** sqlite3_mem_methods structure fails to allocate a block of memory
** here always fail. SQLite will not operate with these drivers. These
** are merely placeholders. Real drivers must be substituted using
** sqlite3_config() before SQLite will operate.
-**
-** $Id$
*/
/*
**
** This file contains implementations of the low-level memory allocation
** routines specified in the sqlite3_mem_methods object.
-**
-** $Id$
*/
/*
**
** This file contains implementations of the low-level memory allocation
** routines specified in the sqlite3_mem_methods object.
-**
-** $Id$
*/
/*
**
** This version of the memory allocation subsystem is included
** in the build only if SQLITE_ENABLE_MEMSYS3 is defined.
-**
-** $Id$
*/
/*
** This file contains the C functions that implement mutexes.
**
** This file contains code that is common across all mutex implementations.
-
-**
-** $Id$
*/
#if defined(SQLITE_DEBUG) && !defined(SQLITE_MUTEX_OMIT)
** If compiled with SQLITE_DEBUG, then additional logic is inserted
** that does error checking on mutexes to make sure they are being
** called correctly.
-**
-** $Id$
*/
**
*************************************************************************
** This file contains the C functions that implement mutexes for OS/2
-**
-** $Id$
*/
/*
sqlite3_free( p );
}
+#ifdef SQLITE_DEBUG
+/*
+** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routine are
+** intended for use inside assert() statements.
+*/
+static int os2MutexHeld(sqlite3_mutex *p){
+ TID tid;
+ PID pid;
+ ULONG ulCount;
+ PTIB ptib;
+ if( p!=0 ) {
+ DosQueryMutexSem(p->mutex, &pid, &tid, &ulCount);
+ } else {
+ DosGetInfoBlocks(&ptib, NULL);
+ tid = ptib->tib_ptib2->tib2_ultid;
+ }
+ return p==0 || (p->nRef!=0 && p->owner==tid);
+}
+static int os2MutexNotheld(sqlite3_mutex *p){
+ TID tid;
+ PID pid;
+ ULONG ulCount;
+ PTIB ptib;
+ if( p!= 0 ) {
+ DosQueryMutexSem(p->mutex, &pid, &tid, &ulCount);
+ } else {
+ DosGetInfoBlocks(&ptib, NULL);
+ tid = ptib->tib_ptib2->tib2_ultid;
+ }
+ return p==0 || p->nRef==0 || p->owner!=tid;
+}
+#endif
+
/*
** The sqlite3_mutex_enter() and sqlite3_mutex_try() routines attempt
** to enter a mutex. If another thread is already within the mutex,
DosReleaseMutexSem(p->mutex);
}
-#ifdef SQLITE_DEBUG
-/*
-** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routine are
-** intended for use inside assert() statements.
-*/
-static int os2MutexHeld(sqlite3_mutex *p){
- TID tid;
- PID pid;
- ULONG ulCount;
- PTIB ptib;
- if( p!=0 ) {
- DosQueryMutexSem(p->mutex, &pid, &tid, &ulCount);
- } else {
- DosGetInfoBlocks(&ptib, NULL);
- tid = ptib->tib_ptib2->tib2_ultid;
- }
- return p==0 || (p->nRef!=0 && p->owner==tid);
-}
-static int os2MutexNotheld(sqlite3_mutex *p){
- TID tid;
- PID pid;
- ULONG ulCount;
- PTIB ptib;
- if( p!= 0 ) {
- DosQueryMutexSem(p->mutex, &pid, &tid, &ulCount);
- } else {
- DosGetInfoBlocks(&ptib, NULL);
- tid = ptib->tib_ptib2->tib2_ultid;
- }
- return p==0 || p->nRef==0 || p->owner!=tid;
-}
-#endif
-
SQLITE_PRIVATE sqlite3_mutex_methods *sqlite3DefaultMutex(void){
static sqlite3_mutex_methods sMutex = {
os2MutexInit,
**
*************************************************************************
** This file contains the C functions that implement mutexes for pthreads
-**
-** $Id$
*/
/*
**
*************************************************************************
** This file contains the C functions that implement mutexes for win32
-**
-** $Id$
*/
/*
*************************************************************************
**
** Memory allocation functions used throughout sqlite.
-**
-** $Id$
*/
/*
** an historical reference. Most of the "enhancements" have been backed
** out so that the functionality is now the same as standard printf().
**
-** $Id$
-**
**************************************************************************
**
** The following modules is an enhanced replacement for the "printf" subroutines
case etSQLESCAPE:
case etSQLESCAPE2:
case etSQLESCAPE3: {
- int i, j, n, isnull;
+ int i, j, k, n, isnull;
int needQuote;
char ch;
char q = ((xtype==etSQLESCAPE3)?'"':'\''); /* Quote character */
char *escarg = va_arg(ap,char*);
isnull = escarg==0;
if( isnull ) escarg = (xtype==etSQLESCAPE2 ? "NULL" : "(NULL)");
- for(i=n=0; (ch=escarg[i])!=0; i++){
+ k = precision;
+ for(i=n=0; (ch=escarg[i])!=0 && k!=0; i++, k--){
if( ch==q ) n++;
}
needQuote = !isnull && xtype==etSQLESCAPE2;
}
j = 0;
if( needQuote ) bufpt[j++] = q;
- for(i=0; (ch=escarg[i])!=0; i++){
- bufpt[j++] = ch;
+ k = i;
+ for(i=0; i<k; i++){
+ bufpt[j++] = ch = escarg[i];
if( ch==q ) bufpt[j++] = ch;
}
if( needQuote ) bufpt[j++] = q;
bufpt[j] = 0;
length = j;
- /* The precision is ignored on %q and %Q */
- /* if( precision>=0 && precision<length ) length = precision; */
+ /* The precision in %q and %Q means how many input characters to
+ ** consume, not the length of the output...
+ ** if( precision>=0 && precision<length ) length = precision; */
break;
}
case etTOKEN: {
}
#endif
+#ifndef SQLITE_OMIT_TRACE
+/*
+** variable-argument wrapper around sqlite3VXPrintf().
+*/
+SQLITE_PRIVATE void sqlite3XPrintf(StrAccum *p, const char *zFormat, ...){
+ va_list ap;
+ va_start(ap,zFormat);
+ sqlite3VXPrintf(p, 1, zFormat, ap);
+ va_end(ap);
+}
+#endif
+
/************** End of printf.c **********************************************/
/************** Begin file random.c ******************************************/
/*
**
** Random numbers are used by some of the database backends in order
** to generate random integer keys for tables or random filenames.
-**
-** $Id$
*/
** This file contains routines used to translate between UTF-8,
** UTF-16, UTF-16BE, and UTF-16LE.
**
-** $Id$
-**
** Notes on UTF-8:
**
** Byte-0 Byte-1 Byte-2 Byte-3 Value
** source code file "vdbe.c". When that file became too big (over
** 6000 lines long) it was split up into several smaller files and
** this header information was factored out.
-**
-** $Id$
*/
#ifndef _VDBEINT_H_
#define _VDBEINT_H_
SQLITE_PRIVATE void sqlite3VdbeMemReleaseExternal(Mem *p);
SQLITE_PRIVATE int sqlite3VdbeMemFinalize(Mem*, FuncDef*);
SQLITE_PRIVATE const char *sqlite3OpcodeName(int);
-SQLITE_PRIVATE int sqlite3VdbeOpcodeHasProperty(int, int);
SQLITE_PRIVATE int sqlite3VdbeMemGrow(Mem *pMem, int n, int preserve);
SQLITE_PRIVATE int sqlite3VdbeCloseStatement(Vdbe *, int);
SQLITE_PRIVATE void sqlite3VdbeFrameDelete(VdbeFrame*);
*************************************************************************
** This is the implementation of generic hash-tables
** used in SQLite.
-**
-** $Id$
*/
/* Turn bulk memory into a hash table object by initializing the
******************************************************************************
**
** This file contains code that is specific to OS/2.
-**
-** $Id$
*/
**
** This file should be #included by the os_*.c files only. It is not a
** general purpose header file.
-**
-** $Id$
*/
#ifndef _OS_COMMON_H_
#define _OS_COMMON_H_
**
** This file contains inline asm code for retrieving "high-performance"
** counters for x86 class CPUs.
-**
-** $Id$
*/
#ifndef _HWTIME_H_
#define _HWTIME_H_
**
** This file should be #included by the os_*.c files only. It is not a
** general purpose header file.
-**
-** $Id$
*/
#ifndef _OS_COMMON_H_
#define _OS_COMMON_H_
**
** This file contains inline asm code for retrieving "high-performance"
** counters for x86 class CPUs.
-**
-** $Id$
*/
#ifndef _HWTIME_H_
#define _HWTIME_H_
#endif
unixLeaveMutex();
- OSTRACE4("TEST WR-LOCK %d %d %d\n", pFile->h, rc, reserved);
+ OSTRACE4("TEST WR-LOCK %d %d %d (unix)\n", pFile->h, rc, reserved);
*pResOut = reserved;
return rc;
int tErrno;
assert( pFile );
- OSTRACE7("LOCK %d %s was %s(%s,%d) pid=%d\n", pFile->h,
+ OSTRACE7("LOCK %d %s was %s(%s,%d) pid=%d (unix)\n", pFile->h,
locktypeName(locktype), locktypeName(pFile->locktype),
locktypeName(pLock->locktype), pLock->cnt , getpid());
** unixEnterMutex() hasn't been called yet.
*/
if( pFile->locktype>=locktype ){
- OSTRACE3("LOCK %d %s ok (already held)\n", pFile->h,
+ OSTRACE3("LOCK %d %s ok (already held) (unix)\n", pFile->h,
locktypeName(locktype));
return SQLITE_OK;
}
end_lock:
unixLeaveMutex();
- OSTRACE4("LOCK %d %s %s\n", pFile->h, locktypeName(locktype),
+ OSTRACE4("LOCK %d %s %s (unix)\n", pFile->h, locktypeName(locktype),
rc==SQLITE_OK ? "ok" : "failed");
return rc;
}
int tErrno; /* Error code from system call errors */
assert( pFile );
- OSTRACE7("UNLOCK %d %d was %d(%d,%d) pid=%d\n", pFile->h, locktype,
+ OSTRACE7("UNLOCK %d %d was %d(%d,%d) pid=%d (unix)\n", pFile->h, locktype,
pFile->locktype, pFile->pLock->locktype, pFile->pLock->cnt, getpid());
assert( locktype<=SHARED_LOCK );
const char *zLockFile = (const char*)pFile->lockingContext;
reserved = access(zLockFile, 0)==0;
}
- OSTRACE4("TEST WR-LOCK %d %d %d\n", pFile->h, rc, reserved);
+ OSTRACE4("TEST WR-LOCK %d %d %d (dotlock)\n", pFile->h, rc, reserved);
*pResOut = reserved;
return rc;
}
char *zLockFile = (char *)pFile->lockingContext;
assert( pFile );
- OSTRACE5("UNLOCK %d %d was %d pid=%d\n", pFile->h, locktype,
+ OSTRACE5("UNLOCK %d %d was %d pid=%d (dotlock)\n", pFile->h, locktype,
pFile->locktype, getpid());
assert( locktype<=SHARED_LOCK );
}
}
}
- OSTRACE4("TEST WR-LOCK %d %d %d\n", pFile->h, rc, reserved);
+ OSTRACE4("TEST WR-LOCK %d %d %d (flock)\n", pFile->h, rc, reserved);
#ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS
if( (rc & SQLITE_IOERR) == SQLITE_IOERR ){
/* got it, set the type and return ok */
pFile->locktype = locktype;
}
- OSTRACE4("LOCK %d %s %s\n", pFile->h, locktypeName(locktype),
+ OSTRACE4("LOCK %d %s %s (flock)\n", pFile->h, locktypeName(locktype),
rc==SQLITE_OK ? "ok" : "failed");
#ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS
if( (rc & SQLITE_IOERR) == SQLITE_IOERR ){
unixFile *pFile = (unixFile*)id;
assert( pFile );
- OSTRACE5("UNLOCK %d %d was %d pid=%d\n", pFile->h, locktype,
+ OSTRACE5("UNLOCK %d %d was %d pid=%d (flock)\n", pFile->h, locktype,
pFile->locktype, getpid());
assert( locktype<=SHARED_LOCK );
sem_post(pSem);
}
}
- OSTRACE4("TEST WR-LOCK %d %d %d\n", pFile->h, rc, reserved);
+ OSTRACE4("TEST WR-LOCK %d %d %d (sem)\n", pFile->h, rc, reserved);
*pResOut = reserved;
return rc;
assert( pFile );
assert( pSem );
- OSTRACE5("UNLOCK %d %d was %d pid=%d\n", pFile->h, locktype,
+ OSTRACE5("UNLOCK %d %d was %d pid=%d (sem)\n", pFile->h, locktype,
pFile->locktype, getpid());
assert( locktype<=SHARED_LOCK );
}
}
- OSTRACE4("TEST WR-LOCK %d %d %d\n", pFile->h, rc, reserved);
+ OSTRACE4("TEST WR-LOCK %d %d %d (afp)\n", pFile->h, rc, reserved);
*pResOut = reserved;
return rc;
afpLockingContext *context = (afpLockingContext *) pFile->lockingContext;
assert( pFile );
- OSTRACE5("LOCK %d %s was %s pid=%d\n", pFile->h,
+ OSTRACE5("LOCK %d %s was %s pid=%d (afp)\n", pFile->h,
locktypeName(locktype), locktypeName(pFile->locktype), getpid());
/* If there is already a lock of this type or more restrictive on the
** unixEnterMutex() hasn't been called yet.
*/
if( pFile->locktype>=locktype ){
- OSTRACE3("LOCK %d %s ok (already held)\n", pFile->h,
+ OSTRACE3("LOCK %d %s ok (already held) (afp)\n", pFile->h,
locktypeName(locktype));
return SQLITE_OK;
}
afp_end_lock:
unixLeaveMutex();
- OSTRACE4("LOCK %d %s %s\n", pFile->h, locktypeName(locktype),
+ OSTRACE4("LOCK %d %s %s (afp)\n", pFile->h, locktypeName(locktype),
rc==SQLITE_OK ? "ok" : "failed");
return rc;
}
afpLockingContext *pCtx = (afpLockingContext *) pFile->lockingContext;
assert( pFile );
- OSTRACE5("UNLOCK %d %d was %d pid=%d\n", pFile->h, locktype,
+ OSTRACE5("UNLOCK %d %d was %d pid=%d (afp)\n", pFile->h, locktype,
pFile->locktype, getpid());
assert( locktype<=SHARED_LOCK );
((unixFile*)id)->lastErrno = errno;
return SQLITE_IOERR_TRUNCATE;
}else{
+#ifndef NDEBUG
+ /* If we are doing a normal write to a database file (as opposed to
+ ** doing a hot-journal rollback or a write to some file other than a
+ ** normal database file) and we truncate the file to zero length,
+ ** that effectively updates the change counter. This might happen
+ ** when restoring a database using the backup API from a zero-length
+ ** source.
+ */
+ if( ((unixFile*)id)->inNormalWrite && nByte==0 ){
+ ((unixFile*)id)->transCntrChng = 1;
+ }
+#endif
+
return SQLITE_OK;
}
}
**
** This file should be #included by the os_*.c files only. It is not a
** general purpose header file.
-**
-** $Id$
*/
#ifndef _OS_COMMON_H_
#define _OS_COMMON_H_
**
** This file contains inline asm code for retrieving "high-performance"
** counters for x86 class CPUs.
-**
-** $Id$
*/
#ifndef _HWTIME_H_
#define _HWTIME_H_
** Bitvec object is the number of pages in the database file at the
** start of a transaction, and is thus usually less than a few thousand,
** but can be as large as 2 billion for a really big database.
-**
-** @(#) $Id$
*/
/* Size of the Bitvec structure in bytes. */
**
*************************************************************************
** This file implements that page cache.
-**
-** @(#) $Id$
*/
/*
** of the SQLITE_CONFIG_PAGECACHE and sqlite3_release_memory() features.
** If the default page cache implementation is overriden, then neither of
** these two features are available.
-**
-** @(#) $Id$
*/
pPage->iKey = iNew;
pPage->pNext = pCache->apHash[h];
pCache->apHash[h] = pPage;
-
- /* The xRekey() interface is only used to move pages earlier in the
- ** database file (in order to move all free pages to the end of the
- ** file where they can be truncated off.) Hence, it is not possible
- ** for the new page number to be greater than the largest previously
- ** fetched page. But we retain the following test in case xRekey()
- ** begins to be used in different ways in the future.
- */
- if( NEVER(iNew>pCache->iMaxKey) ){
+ if( iNew>pCache->iMaxKey ){
pCache->iMaxKey = iNew;
}
**
** There is an added cost of O(N) when switching between TEST and
** SMALLEST primitives.
-**
-** $Id$
*/
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
-**
-** @(#) $Id$
*/
#ifndef SQLITE_OMIT_DISKIO
/* Check that the values read from the page-size and sector-size fields
** are within range. To be 'in range', both values need to be a power
- ** of two greater than or equal to 512, and not greater than their
+ ** of two greater than or equal to 512 or 32, and not greater than their
** respective compile time maximum limits.
*/
- if( iPageSize<512 || iSectorSize<512
+ if( iPageSize<512 || iSectorSize<32
|| iPageSize>SQLITE_MAX_PAGE_SIZE || iSectorSize>MAX_SECTOR_SIZE
|| ((iPageSize-1)&iPageSize)!=0 || ((iSectorSize-1)&iSectorSize)!=0
){
pPager->changeCountDone = 0;
pPager->state = PAGER_UNLOCK;
+ pPager->dbModified = 0;
}
}
PgHdr *pPg; /* An existing page in the cache */
Pgno pgno; /* The page number of a page in journal */
u32 cksum; /* Checksum used for sanity checking */
- u8 *aData; /* Temporary storage for the page */
+ char *aData; /* Temporary storage for the page */
sqlite3_file *jfd; /* The file descriptor for the journal file */
assert( (isMainJrnl&~1)==0 ); /* isMainJrnl is 0 or 1 */
assert( isMainJrnl || pDone ); /* pDone always used on sub-journals */
assert( isSavepnt || pDone==0 ); /* pDone never used on non-savepoint */
- aData = (u8*)pPager->pTmpSpace;
+ aData = pPager->pTmpSpace;
assert( aData ); /* Temp storage must have already been allocated */
/* Read the page number and page data from the journal or sub-journal
jfd = isMainJrnl ? pPager->jfd : pPager->sjfd;
rc = read32bits(jfd, *pOffset, &pgno);
if( rc!=SQLITE_OK ) return rc;
- rc = sqlite3OsRead(jfd, aData, pPager->pageSize, (*pOffset)+4);
+ rc = sqlite3OsRead(jfd, (u8*)aData, pPager->pageSize, (*pOffset)+4);
if( rc!=SQLITE_OK ) return rc;
*pOffset += pPager->pageSize + 4 + isMainJrnl*4;
if( isMainJrnl ){
rc = read32bits(jfd, (*pOffset)-4, &cksum);
if( rc ) return rc;
- if( !isSavepnt && pager_cksum(pPager, aData)!=cksum ){
+ if( !isSavepnt && pager_cksum(pPager, (u8*)aData)!=cksum ){
return SQLITE_DONE;
}
}
pPg = pager_lookup(pPager, pgno);
assert( pPg || !MEMDB );
PAGERTRACE(("PLAYBACK %d page %d hash(%08x) %s\n",
- PAGERID(pPager), pgno, pager_datahash(pPager->pageSize, aData),
- (isMainJrnl?"main-journal":"sub-journal")
+ PAGERID(pPager), pgno, pager_datahash(pPager->pageSize, (u8*)aData),
+ (isMainJrnl?"main-journal":"sub-journal")
));
if( (pPager->state>=PAGER_EXCLUSIVE)
&& (pPg==0 || 0==(pPg->flags&PGHDR_NEED_SYNC))
&& !isUnsync
){
i64 ofst = (pgno-1)*(i64)pPager->pageSize;
- rc = sqlite3OsWrite(pPager->fd, aData, pPager->pageSize, ofst);
+ rc = sqlite3OsWrite(pPager->fd, (u8*)aData, pPager->pageSize, ofst);
if( pgno>pPager->dbFileSize ){
pPager->dbFileSize = pgno;
}
if( pPager->pBackup ){
CODEC1(pPager, aData, pgno, 3, rc=SQLITE_NOMEM);
- sqlite3BackupUpdate(pPager->pBackup, pgno, aData);
- CODEC1(pPager, aData, pgno, 0, rc=SQLITE_NOMEM);
+ sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)aData);
+ CODEC2(pPager, aData, pgno, 7, rc=SQLITE_NOMEM, aData);
}
}else if( !isMainJrnl && pPg==0 ){
/* If this is a rollback of a savepoint and data was not written to
*/
void *pData;
pData = pPg->pData;
- memcpy(pData, aData, pPager->pageSize);
+ memcpy(pData, (u8*)aData, pPager->pageSize);
pPager->xReiniter(pPg);
if( isMainJrnl && (!isSavepnt || *pOffset<=pPager->journalHdr) ){
/* If the contents of this page were just restored from the main
** For temporary files the effective sector size is always 512 bytes.
**
** Otherwise, for non-temporary files, the effective sector size is
-** the value returned by the xSectorSize() method rounded up to 512 if
-** it is less than 512, or rounded down to MAX_SECTOR_SIZE if it
+** the value returned by the xSectorSize() method rounded up to 32 if
+** it is less than 32, or rounded down to MAX_SECTOR_SIZE if it
** is greater than MAX_SECTOR_SIZE.
*/
static void setSectorSize(Pager *pPager){
*/
pPager->sectorSize = sqlite3OsSectorSize(pPager->fd);
}
- if( pPager->sectorSize<512 ){
+ if( pPager->sectorSize<32 ){
pPager->sectorSize = 512;
}
if( pPager->sectorSize>MAX_SECTOR_SIZE ){
assert( PAGER_RESERVED==RESERVED_LOCK );
assert( PAGER_EXCLUSIVE==EXCLUSIVE_LOCK );
- /* If the file is currently unlocked then the size must be unknown */
+ /* If the file is currently unlocked then the size must be unknown. It
+ ** must not have been modified at this point.
+ */
assert( pPager->state>=PAGER_SHARED || pPager->dbSizeValid==0 );
+ assert( pPager->state>=PAGER_SHARED || pPager->dbModified==0 );
/* Check that this is either a no-op (because the requested lock is
** already held, or one of the transistions that the busy-handler
** any such pages to the file.
**
** Also, do not write out any page that has the PGHDR_DONT_WRITE flag
- ** set (set by sqlite3PagerDontWrite()).
+ ** set (set by sqlite3PagerDontWrite()). Note that if compiled with
+ ** SQLITE_SECURE_DELETE the PGHDR_DONT_WRITE bit is never set and so
+ ** the second test is always true.
*/
if( pgno<=pPager->dbSize && 0==(pList->flags&PGHDR_DONT_WRITE) ){
i64 offset = (pgno-1)*(i64)pPager->pageSize; /* Offset to write */
goto pager_acquire_err;
}
- if( nMax<(int)pgno || MEMDB || noContent ){
+ if( MEMDB || nMax<(int)pgno || noContent ){
if( pgno>pPager->mxPgno ){
rc = SQLITE_FULL;
goto pager_acquire_err;
}
#endif
+#ifndef SQLITE_SECURE_DELETE
/*
** A call to this routine tells the pager that it is not necessary to
** write the information on page pPg back to the disk, even though
#endif
}
}
+#endif /* !defined(SQLITE_SECURE_DELETE) */
/*
** This routine is called to increment the value of the database file
#endif
assert( pPager->state>=PAGER_RESERVED );
- if( !pPager->changeCountDone && ALWAYS(pPager->dbSize>0) ){
+ if( !pPager->changeCountDone && pPager->dbSize>0 ){
PgHdr *pPgHdr; /* Reference to page 1 */
u32 change_counter; /* Initial value of change-counter field */
void *pCodec
){
if( pPager->xCodecFree ) pPager->xCodecFree(pPager->pCodec);
- pPager->xCodec = xCodec;
+ pPager->xCodec = pPager->memDb ? 0 : xCodec;
pPager->xCodecSizeChng = xCodecSizeChng;
pPager->xCodecFree = xCodecFree;
pPager->pCodec = pCodec;
assert( pPg->nRef>0 );
+ /* In order to be able to rollback, an in-memory database must journal
+ ** the page we are moving from.
+ */
+ if( MEMDB ){
+ rc = sqlite3PagerWrite(pPg);
+ if( rc ) return rc;
+ }
+
/* If the page being moved is dirty and has not been saved by the latest
** savepoint, then save the current contents of the page into the
** sub-journal now. This is required to handle the following scenario:
** one or more savepoint bitvecs. This is the reason this function
** may return SQLITE_NOMEM.
*/
- if( pPg->flags&PGHDR_DIRTY
+ if( pPg->flags&PGHDR_DIRTY
&& subjRequiresPage(pPg)
&& SQLITE_OK!=(rc = subjournalPage(pPg))
){
assert( !pPgOld || pPgOld->nRef==1 );
if( pPgOld ){
pPg->flags |= (pPgOld->flags&PGHDR_NEED_SYNC);
- sqlite3PcacheDrop(pPgOld);
+ if( MEMDB ){
+ /* Do not discard pages from an in-memory database since we might
+ ** need to rollback later. Just move the page out of the way. */
+ assert( pPager->dbSizeValid );
+ sqlite3PcacheMove(pPgOld, pPager->dbSize+1);
+ }else{
+ sqlite3PcacheDrop(pPgOld);
+ }
}
origPgno = pPg->pgno;
/*
** For an in-memory database, make sure the original page continues
- ** to exist, in case the transaction needs to roll back. We allocate
- ** the page now, instead of at rollback, because we can better deal
- ** with an out-of-memory error now. Ticket #3761.
+ ** to exist, in case the transaction needs to roll back. Use pPgOld
+ ** as the original page since it has already been allocated.
*/
if( MEMDB ){
- DbPage *pNew;
- rc = sqlite3PagerAcquire(pPager, origPgno, &pNew, 1);
- if( rc!=SQLITE_OK ){
- sqlite3PcacheMove(pPg, origPgno);
- return rc;
- }
- sqlite3PagerUnref(pNew);
+ sqlite3PcacheMove(pPgOld, origPgno);
+ sqlite3PagerUnref(pPgOld);
}
return SQLITE_OK;
**
*************************************************************************
**
-** $Id$
-**
** This file contains code used to implement mutexes on Btree objects.
** This code really belongs in btree.c. But btree.c is getting too
** big and we want to break it down some. This packaged seemed like
** May you share freely, never taking more than you give.
**
*************************************************************************
-** $Id$
-**
** This file implements a external (disk-based) database using BTrees.
** For a detailed discussion of BTrees, refer to
**
** May you share freely, never taking more than you give.
**
*************************************************************************
-** $Id$
-**
** This file implements a external (disk-based) database using BTrees.
** See the header comment on "btreeInt.h" for additional information.
** Including a description of file format and an overview of operation.
int top; /* First byte of cell content area */
int gap; /* First byte of gap between cell pointers and cell content */
int rc; /* Integer return code */
+ int usableSize; /* Usable size of the page */
assert( sqlite3PagerIswriteable(pPage->pDbPage) );
assert( pPage->pBt );
assert( nByte>=0 ); /* Minimum cell size is 4 */
assert( pPage->nFree>=nByte );
assert( pPage->nOverflow==0 );
- assert( nByte<pPage->pBt->usableSize-8 );
+ usableSize = pPage->pBt->usableSize;
+ assert( nByte < usableSize-8 );
nFrag = data[hdr+7];
assert( pPage->cellOffset == hdr + 12 - 4*pPage->leaf );
*/
int pc, addr;
for(addr=hdr+1; (pc = get2byte(&data[addr]))>0; addr=pc){
- int size = get2byte(&data[pc+2]); /* Size of free slot */
+ int size; /* Size of the free slot */
+ if( pc>usableSize-4 || pc<addr+4 ){
+ return SQLITE_CORRUPT_BKPT;
+ }
+ size = get2byte(&data[pc+2]);
if( size>=nByte ){
int x = size - nByte;
testcase( x==4 );
** fragmented bytes within the page. */
memcpy(&data[addr], &data[pc], 2);
data[hdr+7] = (u8)(nFrag + x);
+ }else if( size+pc > usableSize ){
+ return SQLITE_CORRUPT_BKPT;
}else{
/* The slot remains on the free-list. Reduce its size to account
** for the portion used by the new allocation. */
assert( sqlite3PagerGetData(pPage->pDbPage) == data );
assert( sqlite3PagerIswriteable(pPage->pDbPage) );
assert( sqlite3_mutex_held(pBt->mutex) );
- /*memset(&data[hdr], 0, pBt->usableSize - hdr);*/
+#ifdef SQLITE_SECURE_DELETE
+ memset(&data[hdr], 0, pBt->usableSize - hdr);
+#endif
data[hdr] = (char)flags;
first = hdr + 8 + 4*((flags&PTF_LEAF)==0 ?1:0);
memset(&data[hdr+1], 0, 4);
*/
static void releasePage(MemPage *pPage){
if( pPage ){
- assert( pPage->nOverflow==0 || sqlite3PagerPageRefcount(pPage->pDbPage)>1 );
assert( pPage->aData );
assert( pPage->pBt );
assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage );
int nPage;
assert( sqlite3_mutex_held(pBt->mutex) );
- /* The database size has already been measured and cached, so failure
- ** is impossible here. If the original size measurement failed, then
- ** processing aborts before entering this routine. */
rc = sqlite3PagerPagecount(pBt->pPager, &nPage);
- if( NEVER(rc!=SQLITE_OK) || nPage>0 ){
+ if( rc!=SQLITE_OK || nPage>0 ){
return rc;
}
pP1 = pBt->pPage1;
** root page of a b-tree. If it is not, then the cursor acquired
** will not work correctly.
**
-** It is assumed that the sqlite3BtreeCursorSize() bytes of memory
-** pointed to by pCur have been zeroed by the caller.
+** It is assumed that the sqlite3BtreeCursorZero() has been called
+** on pCur to initialize the memory space prior to invoking this routine.
*/
static int btreeCursor(
Btree *p, /* The btree */
** this routine.
*/
SQLITE_PRIVATE int sqlite3BtreeCursorSize(void){
- return sizeof(BtCursor);
+ return ROUND8(sizeof(BtCursor));
+}
+
+/*
+** Initialize memory that will be converted into a BtCursor object.
+**
+** The simple approach here would be to memset() the entire object
+** to zero. But it turns out that the apPage[] and aiIdx[] arrays
+** do not need to be zeroed and they are large, so we can save a lot
+** of run-time by skipping the initialization of those elements.
+*/
+SQLITE_PRIVATE void sqlite3BtreeCursorZero(BtCursor *p){
+ memset(p, 0, offsetof(BtCursor, iPage));
}
/*
assert( i>=0 && i<=pPage->nCell+pPage->nOverflow );
assert( pPage->nCell<=MX_CELL(pPage->pBt) && MX_CELL(pPage->pBt)<=5460 );
assert( pPage->nOverflow<=ArraySize(pPage->aOvfl) );
- assert( sz==cellSizePtr(pPage, pCell) );
assert( sqlite3_mutex_held(pPage->pBt->mutex) );
+ /* The cell should normally be sized correctly. However, when moving a
+ ** malformed cell from a leaf page to an interior page, if the cell size
+ ** wanted to be less than 4 but got rounded up to 4 on the leaf, then size
+ ** might be less than 8 (leaf-size + pointer) on the interior node. Hence
+ ** the term after the || in the following assert(). */
+ assert( sz==cellSizePtr(pPage, pCell) || (sz==8 && iChild>0) );
if( pPage->nOverflow || sz+2>pPage->nFree ){
if( pTemp ){
memcpy(pTemp+nSkip, pCell+nSkip, sz-nSkip);
u8 * const aTo = pTo->aData;
int const iFromHdr = pFrom->hdrOffset;
int const iToHdr = ((pTo->pgno==1) ? 100 : 0);
- TESTONLY(int rc;)
+ int rc;
int iData;
memcpy(&aTo[iToHdr], &aFrom[iFromHdr], pFrom->cellOffset + 2*pFrom->nCell);
/* Reinitialize page pTo so that the contents of the MemPage structure
- ** match the new data. The initialization of pTo "cannot" fail, as the
- ** data copied from pFrom is known to be valid. */
+ ** match the new data. The initialization of pTo can actually fail under
+ ** fairly obscure circumstances, even though it is a copy of initialized
+ ** page pFrom.
+ */
pTo->isInit = 0;
- TESTONLY(rc = ) btreeInitPage(pTo);
- assert( rc==SQLITE_OK );
+ rc = btreeInitPage(pTo);
+ if( rc!=SQLITE_OK ){
+ *pRC = rc;
+ return;
+ }
/* If this is an auto-vacuum database, update the pointer-map entries
** for any b-tree or overflow pages that pTo now contains the pointers to.
*/
static int clearDatabasePage(
BtShared *pBt, /* The BTree that contains the table */
- Pgno pgno, /* Page number to clear */
- int freePageFlag, /* Deallocate page if true */
- int *pnChange
+ Pgno pgno, /* Page number to clear */
+ int freePageFlag, /* Deallocate page if true */
+ int *pnChange /* Add number of Cells freed to this counter */
){
MemPage *pPage;
int rc;
*************************************************************************
** This file contains the implementation of the sqlite3_backup_XXX()
** API functions and the related features.
-**
-** $Id$
*/
/* Macro to find the minimum of two numeric values.
** stores a single value in the VDBE. Mem is an opaque structure visible
** only within the VDBE. Interface routines refer to a Mem using the
** name sqlite_value
-**
-** $Id$
*/
/*
int f1, f2;
int combined_flags;
- /* Interchange pMem1 and pMem2 if the collating sequence specifies
- ** DESC order.
- */
f1 = pMem1->flags;
f2 = pMem2->flags;
combined_flags = f1|f2;
}
op = pExpr->op;
if( op==TK_REGISTER ){
- op = pExpr->op2;
+ op = pExpr->op2; /* This only happens with SQLITE_ENABLE_STAT2 */
}
if( op==TK_STRING || op==TK_FLOAT || op==TK_INTEGER ){
** a VDBE (or an "sqlite3_stmt" as it is known to the outside world.) Prior
** to version 2.8.7, all this code was combined into the vdbe.c source file.
** But that file was getting too big so this subroutines were split out.
-**
-** $Id$
*/
return addr;
}
+/*
+** Add an opcode that includes the p4 value as an integer.
+*/
+SQLITE_PRIVATE int sqlite3VdbeAddOp4Int(
+ Vdbe *p, /* Add the opcode to this VM */
+ int op, /* The new opcode */
+ int p1, /* The P1 operand */
+ int p2, /* The P2 operand */
+ int p3, /* The P3 operand */
+ int p4 /* The P4 operand as an integer */
+){
+ int addr = sqlite3VdbeAddOp3(p, op, p1, p2, p3);
+ sqlite3VdbeChangeP4(p, addr, SQLITE_INT_TO_PTR(p4), P4_INT32);
+ return addr;
+}
+
/*
** Create a new symbolic label for an instruction that has yet to be
** coded. The symbolic label is really just a negative number. The
** Variable *pMaxFuncArgs is set to the maximum value of any P2 argument
** to an OP_Function, OP_AggStep or OP_VFilter opcode. This is used by
** sqlite3VdbeMakeReady() to size the Vdbe.apArg[] array.
+**
+** The Op.opflags field is set on all opcodes.
*/
static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){
int i;
for(pOp=p->aOp, i=p->nOp-1; i>=0; i--, pOp++){
u8 opcode = pOp->opcode;
+ pOp->opflags = sqlite3OpcodeProperty[opcode];
if( opcode==OP_Function || opcode==OP_AggStep ){
if( pOp->p5>nMaxArgs ) nMaxArgs = pOp->p5;
-#ifndef SQLITE_OMIT_VIRTUALTABLE
- }else if( opcode==OP_VUpdate ){
- if( pOp->p2>nMaxArgs ) nMaxArgs = pOp->p2;
-#endif
}else if( opcode==OP_Transaction && pOp->p2!=0 ){
p->readOnly = 0;
#ifndef SQLITE_OMIT_VIRTUALTABLE
+ }else if( opcode==OP_VUpdate ){
+ if( pOp->p2>nMaxArgs ) nMaxArgs = pOp->p2;
}else if( opcode==OP_VFilter ){
int n;
assert( p->nOp - i >= 3 );
#endif
}
- if( sqlite3VdbeOpcodeHasProperty(opcode, OPFLG_JUMP) && pOp->p2<0 ){
+ if( (pOp->opflags & OPFLG_JUMP)!=0 && pOp->p2<0 ){
assert( -1-pOp->p2<p->nLabel );
pOp->p2 = aLabel[-1-pOp->p2];
}
VdbeOp *pOut = &p->aOp[i+addr];
pOut->opcode = pIn->opcode;
pOut->p1 = pIn->p1;
- if( p2<0 && sqlite3VdbeOpcodeHasProperty(pOut->opcode, OPFLG_JUMP) ){
+ if( p2<0 && (sqlite3OpcodeProperty[pOut->opcode] & OPFLG_JUMP)!=0 ){
pOut->p2 = addr + ADDR(p2);
}else{
pOut->p2 = p2;
pKeyInfo = pPKey2->pKeyInfo;
mem1.enc = pKeyInfo->enc;
mem1.db = pKeyInfo->db;
- mem1.flags = 0;
- mem1.u.i = 0; /* not needed, here to silence compiler warning */
- mem1.zMalloc = 0;
+ /* mem1.flags = 0; // Will be initialized by sqlite3VdbeSerialGet() */
+ VVA_ONLY( mem1.zMalloc = 0; ) /* Only needed by assert() statements */
+
+ /* Compilers may complain that mem1.u.i is potentially uninitialized.
+ ** We could initialize it, as shown here, to silence those complaints.
+ ** But in fact, mem1.u.i will never actually be used initialized, and doing
+ ** the unnecessary initialization has a measurable negative performance
+ ** impact, since this routine is a very high runner. And so, we choose
+ ** to ignore the compiler warnings and leave this variable uninitialized.
+ */
+ /* mem1.u.i = 0; // not needed, here to silence compiler warning */
idx1 = getVarint32(aKey1, szHdr1);
d1 = szHdr1;
rc = sqlite3MemCompare(&mem1, &pPKey2->aMem[i],
i<nField ? pKeyInfo->aColl[i] : 0);
if( rc!=0 ){
- break;
+ assert( mem1.zMalloc==0 ); /* See comment below */
+
+ /* Invert the result if we are using DESC sort order. */
+ if( pKeyInfo->aSortOrder && i<nField && pKeyInfo->aSortOrder[i] ){
+ rc = -rc;
+ }
+
+ /* If the PREFIX_SEARCH flag is set and all fields except the final
+ ** rowid field were equal, then clear the PREFIX_SEARCH flag and set
+ ** pPKey2->rowid to the value of the rowid field in (pKey1, nKey1).
+ ** This is used by the OP_IsUnique opcode.
+ */
+ if( (pPKey2->flags & UNPACKED_PREFIX_SEARCH) && i==(pPKey2->nField-1) ){
+ assert( idx1==szHdr1 && rc );
+ assert( mem1.flags & MEM_Int );
+ pPKey2->flags &= ~UNPACKED_PREFIX_SEARCH;
+ pPKey2->rowid = mem1.u.i;
+ }
+
+ return rc;
}
i++;
}
- /* No memory allocation is ever used on mem1. */
- if( NEVER(mem1.zMalloc) ) sqlite3VdbeMemRelease(&mem1);
-
- /* If the PREFIX_SEARCH flag is set and all fields except the final
- ** rowid field were equal, then clear the PREFIX_SEARCH flag and set
- ** pPKey2->rowid to the value of the rowid field in (pKey1, nKey1).
- ** This is used by the OP_IsUnique opcode.
+ /* No memory allocation is ever used on mem1. Prove this using
+ ** the following assert(). If the assert() fails, it indicates a
+ ** memory leak and a need to call sqlite3VdbeMemRelease(&mem1).
*/
- if( (pPKey2->flags & UNPACKED_PREFIX_SEARCH) && i==(pPKey2->nField-1) ){
- assert( idx1==szHdr1 && rc );
- assert( mem1.flags & MEM_Int );
- pPKey2->flags &= ~UNPACKED_PREFIX_SEARCH;
- pPKey2->rowid = mem1.u.i;
- }
+ assert( mem1.zMalloc==0 );
- if( rc==0 ){
- /* rc==0 here means that one of the keys ran out of fields and
- ** all the fields up to that point were equal. If the UNPACKED_INCRKEY
- ** flag is set, then break the tie by treating key2 as larger.
- ** If the UPACKED_PREFIX_MATCH flag is set, then keys with common prefixes
- ** are considered to be equal. Otherwise, the longer key is the
- ** larger. As it happens, the pPKey2 will always be the longer
- ** if there is a difference.
- */
- if( pPKey2->flags & UNPACKED_INCRKEY ){
- rc = -1;
- }else if( pPKey2->flags & UNPACKED_PREFIX_MATCH ){
- /* Leave rc==0 */
- }else if( idx1<szHdr1 ){
- rc = 1;
- }
- }else if( pKeyInfo->aSortOrder && i<pKeyInfo->nField
- && pKeyInfo->aSortOrder[i] ){
- rc = -rc;
+ /* rc==0 here means that one of the keys ran out of fields and
+ ** all the fields up to that point were equal. If the UNPACKED_INCRKEY
+ ** flag is set, then break the tie by treating key2 as larger.
+ ** If the UPACKED_PREFIX_MATCH flag is set, then keys with common prefixes
+ ** are considered to be equal. Otherwise, the longer key is the
+ ** larger. As it happens, the pPKey2 will always be the longer
+ ** if there is a difference.
+ */
+ assert( rc==0 );
+ if( pPKey2->flags & UNPACKED_INCRKEY ){
+ rc = -1;
+ }else if( pPKey2->flags & UNPACKED_PREFIX_MATCH ){
+ /* Leave rc==0 */
+ }else if( idx1<szHdr1 ){
+ rc = 1;
}
-
return rc;
}
**
** This file contains code use to implement APIs that are part of the
** VDBE.
-**
-** $Id$
*/
#ifndef SQLITE_OMIT_DEPRECATED
** with that name. If there is no variable with the given name,
** return 0.
*/
-SQLITE_API int sqlite3_bind_parameter_index(sqlite3_stmt *pStmt, const char *zName){
- Vdbe *p = (Vdbe*)pStmt;
+SQLITE_PRIVATE int sqlite3VdbeParameterIndex(Vdbe *p, const char *zName, int nName){
int i;
if( p==0 ){
return 0;
if( zName ){
for(i=0; i<p->nVar; i++){
const char *z = p->azVar[i];
- if( z && strcmp(z,zName)==0 ){
+ if( z && memcmp(z,zName,nName)==0 && z[nName]==0 ){
return i+1;
}
}
}
return 0;
}
+SQLITE_API int sqlite3_bind_parameter_index(sqlite3_stmt *pStmt, const char *zName){
+ return sqlite3VdbeParameterIndex((Vdbe*)pStmt, zName, sqlite3Strlen30(zName));
+}
/*
** Transfer all bindings from the first statement over to the second.
}
/************** End of vdbeapi.c *********************************************/
+/************** Begin file vdbetrace.c ***************************************/
+/*
+** 2009 November 25
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This file contains code used to insert the values of host parameters
+** (aka "wildcards") into the SQL text output by sqlite3_trace().
+*/
+
+#ifndef SQLITE_OMIT_TRACE
+
+/*
+** zSql is a zero-terminated string of UTF-8 SQL text. Return the number of
+** bytes in this text up to but excluding the first character in
+** a host parameter. If the text contains no host parameters, return
+** the total number of bytes in the text.
+*/
+static int findNextHostParameter(const char *zSql, int *pnToken){
+ int tokenType;
+ int nTotal = 0;
+ int n;
+
+ *pnToken = 0;
+ while( zSql[0] ){
+ n = sqlite3GetToken((u8*)zSql, &tokenType);
+ assert( n>0 && tokenType!=TK_ILLEGAL );
+ if( tokenType==TK_VARIABLE ){
+ *pnToken = n;
+ break;
+ }
+ nTotal += n;
+ zSql += n;
+ }
+ return nTotal;
+}
+
+/*
+** Return a pointer to a string in memory obtained form sqlite3DbMalloc() which
+** holds a copy of zRawSql but with host parameters expanded to their
+** current bindings.
+**
+** The calling function is responsible for making sure the memory returned
+** is eventually freed.
+**
+** ALGORITHM: Scan the input string looking for host parameters in any of
+** these forms: ?, ?N, $A, @A, :A. Take care to avoid text within
+** string literals, quoted identifier names, and comments. For text forms,
+** the host parameter index is found by scanning the perpared
+** statement for the corresponding OP_Variable opcode. Once the host
+** parameter index is known, locate the value in p->aVar[]. Then render
+** the value as a literal in place of the host parameter name.
+*/
+SQLITE_PRIVATE char *sqlite3VdbeExpandSql(
+ Vdbe *p, /* The prepared statement being evaluated */
+ const char *zRawSql /* Raw text of the SQL statement */
+){
+ sqlite3 *db; /* The database connection */
+ int idx = 0; /* Index of a host parameter */
+ int nextIndex = 1; /* Index of next ? host parameter */
+ int n; /* Length of a token prefix */
+ int nToken; /* Length of the parameter token */
+ int i; /* Loop counter */
+ Mem *pVar; /* Value of a host parameter */
+ StrAccum out; /* Accumulate the output here */
+ char zBase[100]; /* Initial working space */
+
+ db = p->db;
+ sqlite3StrAccumInit(&out, zBase, sizeof(zBase),
+ db->aLimit[SQLITE_LIMIT_LENGTH]);
+ out.db = db;
+ while( zRawSql[0] ){
+ n = findNextHostParameter(zRawSql, &nToken);
+ assert( n>0 );
+ sqlite3StrAccumAppend(&out, zRawSql, n);
+ zRawSql += n;
+ assert( zRawSql[0] || nToken==0 );
+ if( nToken==0 ) break;
+ if( zRawSql[0]=='?' ){
+ if( nToken>1 ){
+ assert( sqlite3Isdigit(zRawSql[1]) );
+ sqlite3GetInt32(&zRawSql[1], &idx);
+ }else{
+ idx = nextIndex;
+ }
+ }else{
+ assert( zRawSql[0]==':' || zRawSql[0]=='$' || zRawSql[0]=='@' );
+ testcase( zRawSql[0]==':' );
+ testcase( zRawSql[0]=='$' );
+ testcase( zRawSql[0]=='@' );
+ idx = sqlite3VdbeParameterIndex(p, zRawSql, nToken);
+ assert( idx>0 );
+ }
+ zRawSql += nToken;
+ nextIndex = idx + 1;
+ assert( idx>0 && idx<=p->nVar );
+ pVar = &p->aVar[idx-1];
+ if( pVar->flags & MEM_Null ){
+ sqlite3StrAccumAppend(&out, "NULL", 4);
+ }else if( pVar->flags & MEM_Int ){
+ sqlite3XPrintf(&out, "%lld", pVar->u.i);
+ }else if( pVar->flags & MEM_Real ){
+ sqlite3XPrintf(&out, "%!.15g", pVar->r);
+ }else if( pVar->flags & MEM_Str ){
+#ifndef SQLITE_OMIT_UTF16
+ u8 enc = ENC(db);
+ if( enc!=SQLITE_UTF8 ){
+ Mem utf8;
+ memset(&utf8, 0, sizeof(utf8));
+ utf8.db = db;
+ sqlite3VdbeMemSetStr(&utf8, pVar->z, pVar->n, enc, SQLITE_STATIC);
+ sqlite3VdbeChangeEncoding(&utf8, SQLITE_UTF8);
+ sqlite3XPrintf(&out, "'%.*q'", utf8.n, utf8.z);
+ sqlite3VdbeMemRelease(&utf8);
+ }else
+#endif
+ {
+ sqlite3XPrintf(&out, "'%.*q'", pVar->n, pVar->z);
+ }
+ }else if( pVar->flags & MEM_Zero ){
+ sqlite3XPrintf(&out, "zeroblob(%d)", pVar->u.nZero);
+ }else{
+ assert( pVar->flags & MEM_Blob );
+ sqlite3StrAccumAppend(&out, "x'", 2);
+ for(i=0; i<pVar->n; i++){
+ sqlite3XPrintf(&out, "%02x", pVar->z[i]&0xff);
+ }
+ sqlite3StrAccumAppend(&out, "'", 1);
+ }
+ }
+ return sqlite3StrAccumFinish(&out);
+}
+
+#endif /* #ifndef SQLITE_OMIT_TRACE */
+
+/************** End of vdbetrace.c *******************************************/
/************** Begin file vdbe.c ********************************************/
/*
** 2001 September 15
** of the code in this file is, therefore, important. See other comments
** in this file for details. If in doubt, do not deviate from existing
** commenting and indentation practices when changing or adding code.
-**
-** $Id$
*/
/*
}
}
-/*
-** Properties of opcodes. The OPFLG_INITIALIZER macro is
-** created by mkopcodeh.awk during compilation. Data is obtained
-** from the comments following the "case OP_xxxx:" statements in
-** this file.
-*/
-static const unsigned char opcodeProperty[] = OPFLG_INITIALIZER;
-
-/*
-** Return true if an opcode has any of the OPFLG_xxx properties
-** specified by mask.
-*/
-SQLITE_PRIVATE int sqlite3VdbeOpcodeHasProperty(int opcode, int mask){
- assert( opcode>0 && opcode<(int)sizeof(opcodeProperty) );
- return (opcodeProperty[opcode]&mask)!=0;
-}
-
/*
** Allocate VdbeCursor number iCur. Return a pointer to it. Return NULL
** if we run out of memory.
int nByte;
VdbeCursor *pCx = 0;
nByte =
- sizeof(VdbeCursor) +
+ ROUND8(sizeof(VdbeCursor)) +
(isBtreeCursor?sqlite3BtreeCursorSize():0) +
2*nField*sizeof(u32);
}
if( SQLITE_OK==sqlite3VdbeMemGrow(pMem, nByte, 0) ){
p->apCsr[iCur] = pCx = (VdbeCursor*)pMem->z;
- memset(pMem->z, 0, nByte);
+ memset(pCx, 0, sizeof(VdbeCursor));
pCx->iDb = iDb;
pCx->nField = nField;
if( nField ){
- pCx->aType = (u32 *)&pMem->z[sizeof(VdbeCursor)];
+ pCx->aType = (u32 *)&pMem->z[ROUND8(sizeof(VdbeCursor))];
}
if( isBtreeCursor ){
pCx->pCursor = (BtCursor*)
- &pMem->z[sizeof(VdbeCursor)+2*nField*sizeof(u32)];
+ &pMem->z[ROUND8(sizeof(VdbeCursor))+2*nField*sizeof(u32)];
+ sqlite3BtreeCursorZero(pCx->pCursor);
}
}
return pCx;
**
** This file contains inline asm code for retrieving "high-performance"
** counters for x86 class CPUs.
-**
-** $Id$
*/
#ifndef _HWTIME_H_
#define _HWTIME_H_
Vdbe *p /* The VDBE */
){
int pc; /* The program counter */
+ Op *aOp = p->aOp; /* Copy of p->aOp */
Op *pOp; /* Current operation */
int rc = SQLITE_OK; /* Value to return */
sqlite3 *db = p->db; /* The database */
+ u8 resetSchemaOnFault = 0; /* Reset schema after an error if true */
u8 encoding = ENC(db); /* The database encoding */
+#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
+ int checkProgress; /* True if progress callbacks are enabled */
+ int nProgressOps = 0; /* Opcodes executed since progress callback. */
+#endif
+ Mem *aMem = p->aMem; /* Copy of p->aMem */
Mem *pIn1 = 0; /* 1st input operand */
Mem *pIn2 = 0; /* 2nd input operand */
Mem *pIn3 = 0; /* 3rd input operand */
Mem *pOut = 0; /* Output operand */
- u8 opProperty;
int iCompare = 0; /* Result of last OP_Compare operation */
int *aPermute = 0; /* Permutation of columns for OP_Compare */
#ifdef VDBE_PROFILE
u64 start; /* CPU clock count at start of opcode */
int origPc; /* Program counter at start of opcode */
-#endif
-#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
- int nProgressOps = 0; /* Opcodes executed since progress callback. */
#endif
/********************************************************************
** Automatically generated code
Mem *pReg; /* PseudoTable input register */
} am;
struct OP_Affinity_stack_vars {
- char *zAffinity; /* The affinity to be applied */
- Mem *pData0; /* First register to which to apply affinity */
- Mem *pLast; /* Last register to which to apply affinity */
- Mem *pRec; /* Current register */
+ const char *zAffinity; /* The affinity to be applied */
+ char cAff; /* A single character of affinity */
} an;
struct OP_MakeRecord_stack_vars {
u8 *zNewRecord; /* A buffer to hold the data for the new record */
VdbeCursor *pC;
int res;
UnpackedRecord *pIdxKey;
+ UnpackedRecord r;
char aTempRec[ROUND8(sizeof(UnpackedRecord)) + sizeof(Mem)*3 + 7];
} bb;
struct OP_IsUnique_stack_vars {
VdbeCursor *pCx;
BtCursor *pCrsr;
u16 nField;
- Mem *aMem;
+ Mem *aMx;
UnpackedRecord r; /* B-Tree index search key */
i64 R; /* Rowid stored in register P3 */
} bc;
char *z; /* Text of the error report */
Mem *pnErr; /* Register keeping track of errors remaining */
} bv;
- struct OP_RowSetAdd_stack_vars {
- Mem *pIdx;
- Mem *pVal;
- } bw;
struct OP_RowSetRead_stack_vars {
- Mem *pIdx;
i64 val;
- } bx;
+ } bw;
struct OP_RowSetTest_stack_vars {
int iSet;
int exists;
- } by;
+ } bx;
struct OP_Program_stack_vars {
int nMem; /* Number of memory registers for sub-program */
int nByte; /* Bytes of runtime space required for sub-program */
VdbeFrame *pFrame; /* New vdbe frame to execute in */
SubProgram *pProgram; /* Sub-program to execute */
void *t; /* Token identifying trigger */
- } bz;
+ } by;
struct OP_Param_stack_vars {
VdbeFrame *pFrame;
Mem *pIn;
- } ca;
+ } bz;
struct OP_MemMax_stack_vars {
Mem *pIn1;
VdbeFrame *pFrame;
- } cb;
+ } ca;
struct OP_AggStep_stack_vars {
int n;
int i;
Mem *pRec;
sqlite3_context ctx;
sqlite3_value **apVal;
- } cc;
+ } cb;
struct OP_AggFinal_stack_vars {
Mem *pMem;
- } cd;
+ } cc;
struct OP_IncrVacuum_stack_vars {
Btree *pBt;
- } ce;
+ } cd;
struct OP_VBegin_stack_vars {
VTable *pVTab;
- } cf;
+ } ce;
struct OP_VOpen_stack_vars {
VdbeCursor *pCur;
sqlite3_vtab_cursor *pVtabCursor;
sqlite3_vtab *pVtab;
sqlite3_module *pModule;
- } cg;
+ } cf;
struct OP_VFilter_stack_vars {
int nArg;
int iQuery;
int res;
int i;
Mem **apArg;
- } ch;
+ } cg;
struct OP_VColumn_stack_vars {
sqlite3_vtab *pVtab;
const sqlite3_module *pModule;
Mem *pDest;
sqlite3_context sContext;
- } ci;
+ } ch;
struct OP_VNext_stack_vars {
sqlite3_vtab *pVtab;
const sqlite3_module *pModule;
int res;
VdbeCursor *pCur;
- } cj;
+ } ci;
struct OP_VRename_stack_vars {
sqlite3_vtab *pVtab;
Mem *pName;
- } ck;
+ } cj;
struct OP_VUpdate_stack_vars {
sqlite3_vtab *pVtab;
sqlite3_module *pModule;
sqlite_int64 rowid;
Mem **apArg;
Mem *pX;
- } cl;
+ } ck;
struct OP_Pagecount_stack_vars {
int p1;
int nPage;
Pager *pPager;
- } cm;
+ } cl;
struct OP_Trace_stack_vars {
char *zTrace;
- } cn;
+ } cm;
} u;
/* End automatically generated code
********************************************************************/
db->busyHandler.nBusy = 0;
CHECK_FOR_INTERRUPT;
sqlite3VdbeIOTraceSql(p);
+#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
+ checkProgress = db->xProgress!=0;
+#endif
#ifdef SQLITE_DEBUG
sqlite3BeginBenignMalloc();
if( p->pc==0
printf("VDBE Program Listing:\n");
sqlite3VdbePrintSql(p);
for(i=0; i<p->nOp; i++){
- sqlite3VdbePrintOp(stdout, i, &p->aOp[i]);
+ sqlite3VdbePrintOp(stdout, i, &aOp[i]);
}
}
if( fileExists(db, "vdbe_trace") ){
origPc = pc;
start = sqlite3Hwtime();
#endif
- pOp = &p->aOp[pc];
+ pOp = &aOp[pc];
/* Only allow tracing if SQLITE_DEBUG is defined.
*/
** If the progress callback returns non-zero, exit the virtual machine with
** a return code SQLITE_ABORT.
*/
- if( db->xProgress ){
+ if( checkProgress ){
if( db->nProgressOps==nProgressOps ){
int prc;
if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
}
#endif
- /* Do common setup processing for any opcode that is marked
- ** with the "out2-prerelease" tag. Such opcodes have a single
- ** output which is specified by the P2 parameter. The P2 register
- ** is initialized to a NULL.
+ /* On any opcode with the "out2-prerelase" tag, free any
+ ** external allocations out of mem[p2] and set mem[p2] to be
+ ** an undefined integer. Opcodes will either fill in the integer
+ ** value or convert mem[p2] to a different type.
*/
- opProperty = opcodeProperty[pOp->opcode];
- if( (opProperty & OPFLG_OUT2_PRERELEASE)!=0 ){
+ assert( pOp->opflags==sqlite3OpcodeProperty[pOp->opcode] );
+ if( pOp->opflags & OPFLG_OUT2_PRERELEASE ){
assert( pOp->p2>0 );
assert( pOp->p2<=p->nMem );
- pOut = &p->aMem[pOp->p2];
+ pOut = &aMem[pOp->p2];
sqlite3VdbeMemReleaseExternal(pOut);
- pOut->flags = MEM_Null;
- pOut->n = 0;
- }else
-
- /* Do common setup for opcodes marked with one of the following
- ** combinations of properties.
- **
- ** in1
- ** in1 in2
- ** in1 in2 out3
- ** in1 in3
- **
- ** Variables pIn1, pIn2, and pIn3 are made to point to appropriate
- ** registers for inputs. Variable pOut points to the output register.
- */
- if( (opProperty & OPFLG_IN1)!=0 ){
+ pOut->flags = MEM_Int;
+ }
+
+ /* Sanity checking on other operands */
+#ifdef SQLITE_DEBUG
+ if( (pOp->opflags & OPFLG_IN1)!=0 ){
assert( pOp->p1>0 );
assert( pOp->p1<=p->nMem );
- pIn1 = &p->aMem[pOp->p1];
- REGISTER_TRACE(pOp->p1, pIn1);
- if( (opProperty & OPFLG_IN2)!=0 ){
- assert( pOp->p2>0 );
- assert( pOp->p2<=p->nMem );
- pIn2 = &p->aMem[pOp->p2];
- REGISTER_TRACE(pOp->p2, pIn2);
- /* As currently implemented, in2 implies out3. There is no reason
- ** why this has to be, it just worked out that way. */
- assert( (opProperty & OPFLG_OUT3)!=0 );
- assert( pOp->p3>0 );
- assert( pOp->p3<=p->nMem );
- pOut = &p->aMem[pOp->p3];
- }else if( (opProperty & OPFLG_IN3)!=0 ){
- assert( pOp->p3>0 );
- assert( pOp->p3<=p->nMem );
- pIn3 = &p->aMem[pOp->p3];
- REGISTER_TRACE(pOp->p3, pIn3);
- }
- }else if( (opProperty & OPFLG_IN2)!=0 ){
+ REGISTER_TRACE(pOp->p1, &aMem[pOp->p1]);
+ }
+ if( (pOp->opflags & OPFLG_IN2)!=0 ){
assert( pOp->p2>0 );
assert( pOp->p2<=p->nMem );
- pIn2 = &p->aMem[pOp->p2];
- REGISTER_TRACE(pOp->p2, pIn2);
- }else if( (opProperty & OPFLG_IN3)!=0 ){
+ REGISTER_TRACE(pOp->p2, &aMem[pOp->p2]);
+ }
+ if( (pOp->opflags & OPFLG_IN3)!=0 ){
assert( pOp->p3>0 );
assert( pOp->p3<=p->nMem );
- pIn3 = &p->aMem[pOp->p3];
- REGISTER_TRACE(pOp->p3, pIn3);
+ REGISTER_TRACE(pOp->p3, &aMem[pOp->p3]);
}
-
+ if( (pOp->opflags & OPFLG_OUT2)!=0 ){
+ assert( pOp->p2>0 );
+ assert( pOp->p2<=p->nMem );
+ }
+ if( (pOp->opflags & OPFLG_OUT3)!=0 ){
+ assert( pOp->p3>0 );
+ assert( pOp->p3<=p->nMem );
+ }
+#endif
+
switch( pOp->opcode ){
/*****************************************************************************
** Write the current address onto register P1
** and then jump to address P2.
*/
-case OP_Gosub: { /* jump */
- assert( pOp->p1>0 );
- assert( pOp->p1<=p->nMem );
- pIn1 = &p->aMem[pOp->p1];
+case OP_Gosub: { /* jump, in1 */
+ pIn1 = &aMem[pOp->p1];
assert( (pIn1->flags & MEM_Dyn)==0 );
pIn1->flags = MEM_Int;
pIn1->u.i = pc;
** Jump to the next instruction after the address in register P1.
*/
case OP_Return: { /* in1 */
+ pIn1 = &aMem[pOp->p1];
assert( pIn1->flags & MEM_Int );
pc = (int)pIn1->u.i;
break;
#if 0 /* local variables moved into u.aa */
int pcDest;
#endif /* local variables moved into u.aa */
+ pIn1 = &aMem[pOp->p1];
assert( (pIn1->flags & MEM_Dyn)==0 );
pIn1->flags = MEM_Int;
u.aa.pcDest = (int)pIn1->u.i;
** value in register P3 is not NULL, then this routine is a no-op.
*/
case OP_HaltIfNull: { /* in3 */
+ pIn3 = &aMem[pOp->p3];
if( (pIn3->flags & MEM_Null)==0 ) break;
/* Fall through into OP_Halt */
}
** as the p2 of the calling OP_Program. */
pc = p->aOp[pc].p2-1;
}
+ aOp = p->aOp;
+ aMem = p->aMem;
break;
}
** The 32-bit integer value P1 is written into register P2.
*/
case OP_Integer: { /* out2-prerelease */
- pOut->flags = MEM_Int;
pOut->u.i = pOp->p1;
break;
}
*/
case OP_Int64: { /* out2-prerelease */
assert( pOp->p4.pI64!=0 );
- pOut->flags = MEM_Int;
pOut->u.i = *pOp->p4.pI64;
break;
}
** Write a NULL into register P2.
*/
case OP_Null: { /* out2-prerelease */
+ pOut->flags = MEM_Null;
break;
}
if( sqlite3VdbeMemTooBig(u.ab.pVar) ){
goto too_big;
}
- pOut = &p->aMem[u.ab.p2++];
+ pOut = &aMem[u.ab.p2++];
sqlite3VdbeMemReleaseExternal(pOut);
pOut->flags = MEM_Null;
sqlite3VdbeMemShallowCopy(pOut, u.ab.pVar, MEM_Static);
assert( u.ac.n>0 && u.ac.p1>0 && u.ac.p2>0 );
assert( u.ac.p1+u.ac.n<=u.ac.p2 || u.ac.p2+u.ac.n<=u.ac.p1 );
- pIn1 = &p->aMem[u.ac.p1];
- pOut = &p->aMem[u.ac.p2];
+ pIn1 = &aMem[u.ac.p1];
+ pOut = &aMem[u.ac.p2];
while( u.ac.n-- ){
- assert( pOut<=&p->aMem[p->nMem] );
- assert( pIn1<=&p->aMem[p->nMem] );
+ assert( pOut<=&aMem[p->nMem] );
+ assert( pIn1<=&aMem[p->nMem] );
u.ac.zMalloc = pOut->zMalloc;
pOut->zMalloc = 0;
sqlite3VdbeMemMove(pOut, pIn1);
** This instruction makes a deep copy of the value. A duplicate
** is made of any string or blob constant. See also OP_SCopy.
*/
-case OP_Copy: { /* in1 */
- assert( pOp->p2>0 );
- assert( pOp->p2<=p->nMem );
- pOut = &p->aMem[pOp->p2];
+case OP_Copy: { /* in1, out2 */
+ pIn1 = &aMem[pOp->p1];
+ pOut = &aMem[pOp->p2];
assert( pOut!=pIn1 );
sqlite3VdbeMemShallowCopy(pOut, pIn1, MEM_Ephem);
Deephemeralize(pOut);
** during the lifetime of the copy. Use OP_Copy to make a complete
** copy.
*/
-case OP_SCopy: { /* in1 */
- REGISTER_TRACE(pOp->p1, pIn1);
- assert( pOp->p2>0 );
- assert( pOp->p2<=p->nMem );
- pOut = &p->aMem[pOp->p2];
+case OP_SCopy: { /* in1, out2 */
+ pIn1 = &aMem[pOp->p1];
+ pOut = &aMem[pOp->p2];
assert( pOut!=pIn1 );
sqlite3VdbeMemShallowCopy(pOut, pIn1, MEM_Ephem);
REGISTER_TRACE(pOp->p2, pOut);
** and have an assigned type. The results are de-ephemeralized as
** as side effect.
*/
- u.ad.pMem = p->pResultSet = &p->aMem[pOp->p1];
+ u.ad.pMem = p->pResultSet = &aMem[pOp->p1];
for(u.ad.i=0; u.ad.i<pOp->p2; u.ad.i++){
sqlite3VdbeMemNulTerminate(&u.ad.pMem[u.ad.i]);
sqlite3VdbeMemStoreType(&u.ad.pMem[u.ad.i]);
i64 nByte;
#endif /* local variables moved into u.ae */
+ pIn1 = &aMem[pOp->p1];
+ pIn2 = &aMem[pOp->p2];
+ pOut = &aMem[pOp->p3];
assert( pIn1!=pOut );
if( (pIn1->flags | pIn2->flags) & MEM_Null ){
sqlite3VdbeMemSetNull(pOut);
double rB; /* Real value of right operand */
#endif /* local variables moved into u.af */
+ pIn1 = &aMem[pOp->p1];
applyNumericAffinity(pIn1);
+ pIn2 = &aMem[pOp->p2];
applyNumericAffinity(pIn2);
+ pOut = &aMem[pOp->p3];
u.af.flags = pIn1->flags | pIn2->flags;
if( (u.af.flags & MEM_Null)!=0 ) goto arithmetic_result_is_null;
if( (pIn1->flags & pIn2->flags & MEM_Int)==MEM_Int ){
assert( u.ag.n==0 || (pOp->p2>0 && pOp->p2+u.ag.n<=p->nMem+1) );
assert( pOp->p3<pOp->p2 || pOp->p3>=pOp->p2+u.ag.n );
- u.ag.pArg = &p->aMem[pOp->p2];
+ u.ag.pArg = &aMem[pOp->p2];
for(u.ag.i=0; u.ag.i<u.ag.n; u.ag.i++, u.ag.pArg++){
u.ag.apVal[u.ag.i] = u.ag.pArg;
sqlite3VdbeMemStoreType(u.ag.pArg);
}
assert( pOp->p3>0 && pOp->p3<=p->nMem );
- pOut = &p->aMem[pOp->p3];
+ pOut = &aMem[pOp->p3];
u.ag.ctx.s.flags = MEM_Null;
u.ag.ctx.s.db = db;
u.ag.ctx.s.xDel = 0;
u.ag.ctx.isError = 0;
if( u.ag.ctx.pFunc->flags & SQLITE_FUNC_NEEDCOLL ){
- assert( pOp>p->aOp );
+ assert( pOp>aOp );
assert( pOp[-1].p4type==P4_COLLSEQ );
assert( pOp[-1].opcode==OP_CollSeq );
u.ag.ctx.pColl = pOp[-1].p4.pColl;
i64 b;
#endif /* local variables moved into u.ah */
+ pIn1 = &aMem[pOp->p1];
+ pIn2 = &aMem[pOp->p2];
+ pOut = &aMem[pOp->p3];
if( (pIn1->flags | pIn2->flags) & MEM_Null ){
sqlite3VdbeMemSetNull(pOut);
break;
** To force any register to be an integer, just add 0.
*/
case OP_AddImm: { /* in1 */
+ pIn1 = &aMem[pOp->p1];
sqlite3VdbeMemIntegerify(pIn1);
pIn1->u.i += pOp->p2;
break;
** raise an SQLITE_MISMATCH exception.
*/
case OP_MustBeInt: { /* jump, in1 */
+ pIn1 = &aMem[pOp->p1];
applyAffinity(pIn1, SQLITE_AFF_NUMERIC, encoding);
if( (pIn1->flags & MEM_Int)==0 ){
if( pOp->p2==0 ){
** to have only a real value.
*/
case OP_RealAffinity: { /* in1 */
+ pIn1 = &aMem[pOp->p1];
if( pIn1->flags & MEM_Int ){
sqlite3VdbeMemRealify(pIn1);
}
** A NULL value is not changed by this routine. It remains NULL.
*/
case OP_ToText: { /* same as TK_TO_TEXT, in1 */
+ pIn1 = &aMem[pOp->p1];
if( pIn1->flags & MEM_Null ) break;
assert( MEM_Str==(MEM_Blob>>3) );
pIn1->flags |= (pIn1->flags&MEM_Blob)>>3;
** A NULL value is not changed by this routine. It remains NULL.
*/
case OP_ToBlob: { /* same as TK_TO_BLOB, in1 */
+ pIn1 = &aMem[pOp->p1];
if( pIn1->flags & MEM_Null ) break;
if( (pIn1->flags & MEM_Blob)==0 ){
applyAffinity(pIn1, SQLITE_AFF_TEXT, encoding);
** A NULL value is not changed by this routine. It remains NULL.
*/
case OP_ToNumeric: { /* same as TK_TO_NUMERIC, in1 */
+ pIn1 = &aMem[pOp->p1];
if( (pIn1->flags & (MEM_Null|MEM_Int|MEM_Real))==0 ){
sqlite3VdbeMemNumerify(pIn1);
}
** A NULL value is not changed by this routine. It remains NULL.
*/
case OP_ToInt: { /* same as TK_TO_INT, in1 */
+ pIn1 = &aMem[pOp->p1];
if( (pIn1->flags & MEM_Null)==0 ){
sqlite3VdbeMemIntegerify(pIn1);
}
** A NULL value is not changed by this routine. It remains NULL.
*/
case OP_ToReal: { /* same as TK_TO_REAL, in1 */
+ pIn1 = &aMem[pOp->p1];
if( (pIn1->flags & MEM_Null)==0 ){
sqlite3VdbeMemRealify(pIn1);
}
char affinity; /* Affinity to use for comparison */
#endif /* local variables moved into u.ai */
+ pIn1 = &aMem[pOp->p1];
+ pIn3 = &aMem[pOp->p3];
if( (pIn1->flags | pIn3->flags)&MEM_Null ){
/* One or both operands are NULL */
if( pOp->p5 & SQLITE_NULLEQ ){
** The jump is taken if the SQLITE_JUMPIFNULL bit is set.
*/
if( pOp->p5 & SQLITE_STOREP2 ){
- pOut = &p->aMem[pOp->p2];
+ pOut = &aMem[pOp->p2];
MemSetTypeFlag(pOut, MEM_Null);
REGISTER_TRACE(pOp->p2, pOut);
}else if( pOp->p5 & SQLITE_JUMPIFNULL ){
}
if( pOp->p5 & SQLITE_STOREP2 ){
- pOut = &p->aMem[pOp->p2];
+ pOut = &aMem[pOp->p2];
MemSetTypeFlag(pOut, MEM_Int);
pOut->u.i = u.ai.res;
REGISTER_TRACE(pOp->p2, pOut);
#endif /* SQLITE_DEBUG */
for(u.aj.i=0; u.aj.i<u.aj.n; u.aj.i++){
u.aj.idx = aPermute ? aPermute[u.aj.i] : u.aj.i;
- REGISTER_TRACE(u.aj.p1+u.aj.idx, &p->aMem[u.aj.p1+u.aj.idx]);
- REGISTER_TRACE(u.aj.p2+u.aj.idx, &p->aMem[u.aj.p2+u.aj.idx]);
+ REGISTER_TRACE(u.aj.p1+u.aj.idx, &aMem[u.aj.p1+u.aj.idx]);
+ REGISTER_TRACE(u.aj.p2+u.aj.idx, &aMem[u.aj.p2+u.aj.idx]);
assert( u.aj.i<u.aj.pKeyInfo->nField );
u.aj.pColl = u.aj.pKeyInfo->aColl[u.aj.i];
u.aj.bRev = u.aj.pKeyInfo->aSortOrder[u.aj.i];
- iCompare = sqlite3MemCompare(&p->aMem[u.aj.p1+u.aj.idx], &p->aMem[u.aj.p2+u.aj.idx], u.aj.pColl);
+ iCompare = sqlite3MemCompare(&aMem[u.aj.p1+u.aj.idx], &aMem[u.aj.p2+u.aj.idx], u.aj.pColl);
if( iCompare ){
if( u.aj.bRev ) iCompare = -iCompare;
break;
int v2; /* Right operand: 0==FALSE, 1==TRUE, 2==UNKNOWN or NULL */
#endif /* local variables moved into u.ak */
+ pIn1 = &aMem[pOp->p1];
if( pIn1->flags & MEM_Null ){
u.ak.v1 = 2;
}else{
u.ak.v1 = sqlite3VdbeIntValue(pIn1)!=0;
}
+ pIn2 = &aMem[pOp->p2];
if( pIn2->flags & MEM_Null ){
u.ak.v2 = 2;
}else{
static const unsigned char or_logic[] = { 0, 1, 2, 1, 1, 1, 2, 1, 2 };
u.ak.v1 = or_logic[u.ak.v1*3+u.ak.v2];
}
+ pOut = &aMem[pOp->p3];
if( u.ak.v1==2 ){
MemSetTypeFlag(pOut, MEM_Null);
}else{
** boolean complement in register P2. If the value in register P1 is
** NULL, then a NULL is stored in P2.
*/
-case OP_Not: { /* same as TK_NOT, in1 */
- pOut = &p->aMem[pOp->p2];
+case OP_Not: { /* same as TK_NOT, in1, out2 */
+ pIn1 = &aMem[pOp->p1];
+ pOut = &aMem[pOp->p2];
if( pIn1->flags & MEM_Null ){
sqlite3VdbeMemSetNull(pOut);
}else{
** ones-complement of the P1 value into register P2. If P1 holds
** a NULL then store a NULL in P2.
*/
-case OP_BitNot: { /* same as TK_BITNOT, in1 */
- pOut = &p->aMem[pOp->p2];
+case OP_BitNot: { /* same as TK_BITNOT, in1, out2 */
+ pIn1 = &aMem[pOp->p1];
+ pOut = &aMem[pOp->p2];
if( pIn1->flags & MEM_Null ){
sqlite3VdbeMemSetNull(pOut);
}else{
#if 0 /* local variables moved into u.al */
int c;
#endif /* local variables moved into u.al */
+ pIn1 = &aMem[pOp->p1];
if( pIn1->flags & MEM_Null ){
u.al.c = pOp->p3;
}else{
** Jump to P2 if the value in register P1 is NULL.
*/
case OP_IsNull: { /* same as TK_ISNULL, jump, in1 */
+ pIn1 = &aMem[pOp->p1];
if( (pIn1->flags & MEM_Null)!=0 ){
pc = pOp->p2 - 1;
}
** Jump to P2 if the value in register P1 is not NULL.
*/
case OP_NotNull: { /* same as TK_NOTNULL, jump, in1 */
+ pIn1 = &aMem[pOp->p1];
if( (pIn1->flags & MEM_Null)==0 ){
pc = pOp->p2 - 1;
}
memset(&u.am.sMem, 0, sizeof(u.am.sMem));
assert( u.am.p1<p->nCursor );
assert( pOp->p3>0 && pOp->p3<=p->nMem );
- u.am.pDest = &p->aMem[pOp->p3];
+ u.am.pDest = &aMem[pOp->p3];
MemSetTypeFlag(u.am.pDest, MEM_Null);
u.am.zRec = 0;
assert( rc==SQLITE_OK ); /* DataSize() cannot fail */
}
}else if( u.am.pC->pseudoTableReg>0 ){
- u.am.pReg = &p->aMem[u.am.pC->pseudoTableReg];
+ u.am.pReg = &aMem[u.am.pC->pseudoTableReg];
assert( u.am.pReg->flags & MEM_Blob );
u.am.payloadSize = u.am.pReg->n;
u.am.zRec = u.am.pReg->z;
*/
case OP_Affinity: {
#if 0 /* local variables moved into u.an */
- char *zAffinity; /* The affinity to be applied */
- Mem *pData0; /* First register to which to apply affinity */
- Mem *pLast; /* Last register to which to apply affinity */
- Mem *pRec; /* Current register */
+ const char *zAffinity; /* The affinity to be applied */
+ char cAff; /* A single character of affinity */
#endif /* local variables moved into u.an */
u.an.zAffinity = pOp->p4.z;
- u.an.pData0 = &p->aMem[pOp->p1];
- u.an.pLast = &u.an.pData0[pOp->p2-1];
- for(u.an.pRec=u.an.pData0; u.an.pRec<=u.an.pLast; u.an.pRec++){
- ExpandBlob(u.an.pRec);
- applyAffinity(u.an.pRec, u.an.zAffinity[u.an.pRec-u.an.pData0], encoding);
+ assert( u.an.zAffinity!=0 );
+ assert( u.an.zAffinity[pOp->p2]==0 );
+ pIn1 = &aMem[pOp->p1];
+ while( (u.an.cAff = *(u.an.zAffinity++))!=0 ){
+ assert( pIn1 <= &p->aMem[p->nMem] );
+ ExpandBlob(pIn1);
+ applyAffinity(pIn1, u.an.cAff, encoding);
+ pIn1++;
}
break;
}
u.ao.nField = pOp->p1;
u.ao.zAffinity = pOp->p4.z;
assert( u.ao.nField>0 && pOp->p2>0 && pOp->p2+u.ao.nField<=p->nMem+1 );
- u.ao.pData0 = &p->aMem[u.ao.nField];
+ u.ao.pData0 = &aMem[u.ao.nField];
u.ao.nField = pOp->p2;
u.ao.pLast = &u.ao.pData0[u.ao.nField-1];
u.ao.file_format = p->minWriteFileFormat;
** sqlite3VdbeMemGrow() could clobber the value before it is used).
*/
assert( pOp->p3<pOp->p1 || pOp->p3>=pOp->p1+pOp->p2 );
- pOut = &p->aMem[pOp->p3];
+ pOut = &aMem[pOp->p3];
if( sqlite3VdbeMemGrow(pOut, (int)u.ao.nByte, 0) ){
goto no_mem;
}
}else{
u.ap.nEntry = 0;
}
- pOut->flags = MEM_Int;
pOut->u.i = u.ap.nEntry;
break;
}
sqlite3BtreeGetMeta(db->aDb[u.at.iDb].pBt, u.at.iCookie, (u32 *)&u.at.iMeta);
pOut->u.i = u.at.iMeta;
- MemSetTypeFlag(pOut, MEM_Int);
break;
}
assert( (p->btreeMask & (1<<pOp->p1))!=0 );
u.au.pDb = &db->aDb[pOp->p1];
assert( u.au.pDb->pBt!=0 );
+ pIn3 = &aMem[pOp->p3];
sqlite3VdbeMemIntegerify(pIn3);
/* See note about index shifting on OP_ReadCookie */
rc = sqlite3BtreeUpdateMeta(u.au.pDb->pBt, pOp->p2, (int)pIn3->u.i);
if( pOp->p5 ){
assert( u.aw.p2>0 );
assert( u.aw.p2<=p->nMem );
- pIn2 = &p->aMem[u.aw.p2];
+ pIn2 = &aMem[u.aw.p2];
sqlite3VdbeMemIntegerify(pIn2);
u.aw.p2 = (int)pIn2->u.i;
/* The u.aw.p2 value always comes from a prior OP_CreateTable opcode and
u.az.pC = p->apCsr[pOp->p1];
assert( u.az.pC!=0 );
assert( u.az.pC->pseudoTableReg==0 );
+ assert( OP_SeekLe == OP_SeekLt+1 );
+ assert( OP_SeekGe == OP_SeekLt+2 );
+ assert( OP_SeekGt == OP_SeekLt+3 );
if( u.az.pC->pCursor!=0 ){
u.az.oc = pOp->opcode;
u.az.pC->nullRow = 0;
/* The input value in P3 might be of any type: integer, real, string,
** blob, or NULL. But it needs to be an integer before we can do
** the seek, so covert it. */
+ pIn3 = &aMem[pOp->p3];
applyNumericAffinity(pIn3);
u.az.iKey = sqlite3VdbeIntValue(pIn3);
u.az.pC->rowidIsValid = 0;
** integer. */
u.az.res = 1;
if( pIn3->r<0 ){
- if( u.az.oc==OP_SeekGt || u.az.oc==OP_SeekGe ){
+ if( u.az.oc>=OP_SeekGe ){ assert( u.az.oc==OP_SeekGe || u.az.oc==OP_SeekGt );
rc = sqlite3BtreeFirst(u.az.pC->pCursor, &u.az.res);
if( rc!=SQLITE_OK ) goto abort_due_to_error;
}
}else{
- if( u.az.oc==OP_SeekLt || u.az.oc==OP_SeekLe ){
+ if( u.az.oc<=OP_SeekLe ){ assert( u.az.oc==OP_SeekLt || u.az.oc==OP_SeekLe );
rc = sqlite3BtreeLast(u.az.pC->pCursor, &u.az.res);
if( rc!=SQLITE_OK ) goto abort_due_to_error;
}
assert( u.az.nField>0 );
u.az.r.pKeyInfo = u.az.pC->pKeyInfo;
u.az.r.nField = (u16)u.az.nField;
- if( u.az.oc==OP_SeekGt || u.az.oc==OP_SeekLe ){
- u.az.r.flags = UNPACKED_INCRKEY;
- }else{
- u.az.r.flags = 0;
- }
- u.az.r.aMem = &p->aMem[pOp->p3];
+
+ /* The next line of code computes as follows, only faster:
+ ** if( u.az.oc==OP_SeekGt || u.az.oc==OP_SeekLe ){
+ ** u.az.r.flags = UNPACKED_INCRKEY;
+ ** }else{
+ ** u.az.r.flags = 0;
+ ** }
+ */
+ u.az.r.flags = (u16)(UNPACKED_INCRKEY * (1 & (u.az.oc - OP_SeekLt)));
+ assert( u.az.oc!=OP_SeekGt || u.az.r.flags==UNPACKED_INCRKEY );
+ assert( u.az.oc!=OP_SeekLe || u.az.r.flags==UNPACKED_INCRKEY );
+ assert( u.az.oc!=OP_SeekGe || u.az.r.flags==0 );
+ assert( u.az.oc!=OP_SeekLt || u.az.r.flags==0 );
+
+ u.az.r.aMem = &aMem[pOp->p3];
+ ExpandBlob(u.az.r.aMem);
rc = sqlite3BtreeMovetoUnpacked(u.az.pC->pCursor, &u.az.r, 0, 0, &u.az.res);
if( rc!=SQLITE_OK ){
goto abort_due_to_error;
#ifdef SQLITE_TEST
sqlite3_search_count++;
#endif
- if( u.az.oc==OP_SeekGe || u.az.oc==OP_SeekGt ){
+ if( u.az.oc>=OP_SeekGe ){ assert( u.az.oc==OP_SeekGe || u.az.oc==OP_SeekGt );
if( u.az.res<0 || (u.az.res==0 && u.az.oc==OP_SeekGt) ){
rc = sqlite3BtreeNext(u.az.pC->pCursor, &u.az.res);
if( rc!=SQLITE_OK ) goto abort_due_to_error;
if( ALWAYS(u.ba.pC->pCursor!=0) ){
assert( u.ba.pC->isTable );
u.ba.pC->nullRow = 0;
+ pIn2 = &aMem[pOp->p2];
u.ba.pC->movetoTarget = sqlite3VdbeIntValue(pIn2);
u.ba.pC->rowidIsValid = 0;
u.ba.pC->deferredMoveto = 1;
}
-/* Opcode: Found P1 P2 P3 * *
+/* Opcode: Found P1 P2 P3 P4 *
**
-** Register P3 holds a blob constructed by MakeRecord. P1 is an index.
-** If an entry that matches the value in register p3 exists in P1 then
-** jump to P2. If the P3 value does not match any entry in P1
-** then fall thru. The P1 cursor is left pointing at the matching entry
-** if it exists.
-**
-** This instruction is used to implement the IN operator where the
-** left-hand side is a SELECT statement. P1 may be a true index, or it
-** may be a temporary index that holds the results of the SELECT
-** statement. This instruction is also used to implement the
-** DISTINCT keyword in SELECT statements.
-**
-** This instruction checks if index P1 contains a record for which
-** the first N serialized values exactly match the N serialized values
-** in the record in register P3, where N is the total number of values in
-** the P3 record (the P3 record is a prefix of the P1 record).
+** If P4==0 then register P3 holds a blob constructed by MakeRecord. If
+** P4>0 then register P3 is the first of P4 registers that form an unpacked
+** record.
**
-** See also: NotFound, IsUnique, NotExists
+** Cursor P1 is on an index btree. If the record identified by P3 and P4
+** is a prefix of any entry in P1 then a jump is made to P2 and
+** P1 is left pointing at the matching entry.
*/
-/* Opcode: NotFound P1 P2 P3 * *
+/* Opcode: NotFound P1 P2 P3 P4 *
**
-** Register P3 holds a blob constructed by MakeRecord. P1 is
-** an index. If no entry exists in P1 that matches the blob then jump
-** to P2. If an entry does existing, fall through. The cursor is left
-** pointing to the entry that matches.
+** If P4==0 then register P3 holds a blob constructed by MakeRecord. If
+** P4>0 then register P3 is the first of P4 registers that form an unpacked
+** record.
+**
+** Cursor P1 is on an index btree. If the record identified by P3 and P4
+** is not the prefix of any entry in P1 then a jump is made to P2. If P1
+** does contain an entry whose prefix matches the P3/P4 record then control
+** falls through to the next instruction and P1 is left pointing at the
+** matching entry.
**
** See also: Found, NotExists, IsUnique
*/
VdbeCursor *pC;
int res;
UnpackedRecord *pIdxKey;
+ UnpackedRecord r;
char aTempRec[ROUND8(sizeof(UnpackedRecord)) + sizeof(Mem)*3 + 7];
#endif /* local variables moved into u.bb */
u.bb.alreadyExists = 0;
assert( pOp->p1>=0 && pOp->p1<p->nCursor );
+ assert( pOp->p4type==P4_INT32 );
u.bb.pC = p->apCsr[pOp->p1];
assert( u.bb.pC!=0 );
+ pIn3 = &aMem[pOp->p3];
if( ALWAYS(u.bb.pC->pCursor!=0) ){
assert( u.bb.pC->isTable==0 );
- assert( pIn3->flags & MEM_Blob );
- ExpandBlob(pIn3);
- u.bb.pIdxKey = sqlite3VdbeRecordUnpack(u.bb.pC->pKeyInfo, pIn3->n, pIn3->z,
- u.bb.aTempRec, sizeof(u.bb.aTempRec));
- if( u.bb.pIdxKey==0 ){
- goto no_mem;
- }
- if( pOp->opcode==OP_Found ){
+ if( pOp->p4.i>0 ){
+ u.bb.r.pKeyInfo = u.bb.pC->pKeyInfo;
+ u.bb.r.nField = (u16)pOp->p4.i;
+ u.bb.r.aMem = pIn3;
+ u.bb.r.flags = UNPACKED_PREFIX_MATCH;
+ u.bb.pIdxKey = &u.bb.r;
+ }else{
+ assert( pIn3->flags & MEM_Blob );
+ ExpandBlob(pIn3);
+ u.bb.pIdxKey = sqlite3VdbeRecordUnpack(u.bb.pC->pKeyInfo, pIn3->n, pIn3->z,
+ u.bb.aTempRec, sizeof(u.bb.aTempRec));
+ if( u.bb.pIdxKey==0 ){
+ goto no_mem;
+ }
u.bb.pIdxKey->flags |= UNPACKED_PREFIX_MATCH;
}
rc = sqlite3BtreeMovetoUnpacked(u.bb.pC->pCursor, u.bb.pIdxKey, 0, 0, &u.bb.res);
- sqlite3VdbeDeleteUnpackedRecord(u.bb.pIdxKey);
+ if( pOp->p4.i==0 ){
+ sqlite3VdbeDeleteUnpackedRecord(u.bb.pIdxKey);
+ }
if( rc!=SQLITE_OK ){
break;
}
/* Opcode: IsUnique P1 P2 P3 P4 *
**
-** Cursor P1 is open on an index. So it has no data and its key consists
-** of a record generated by OP_MakeRecord where the last field is the
-** rowid of the entry that the index refers to.
+** Cursor P1 is open on an index b-tree - that is to say, a btree which
+** no data and where the key are records generated by OP_MakeRecord with
+** the list field being the integer ROWID of the entry that the index
+** entry refers to.
**
** The P3 register contains an integer record number. Call this record
** number R. Register P4 is the first in a set of N contiguous registers
VdbeCursor *pCx;
BtCursor *pCrsr;
u16 nField;
- Mem *aMem;
+ Mem *aMx;
UnpackedRecord r; /* B-Tree index search key */
i64 R; /* Rowid stored in register P3 */
#endif /* local variables moved into u.bc */
- u.bc.aMem = &p->aMem[pOp->p4.i];
+ pIn3 = &aMem[pOp->p3];
+ u.bc.aMx = &aMem[pOp->p4.i];
/* Assert that the values of parameters P1 and P4 are in range. */
assert( pOp->p4type==P4_INT32 );
assert( pOp->p4.i>0 && pOp->p4.i<=p->nMem );
/* If any of the values are NULL, take the jump. */
u.bc.nField = u.bc.pCx->pKeyInfo->nField;
for(u.bc.ii=0; u.bc.ii<u.bc.nField; u.bc.ii++){
- if( u.bc.aMem[u.bc.ii].flags & MEM_Null ){
+ if( u.bc.aMx[u.bc.ii].flags & MEM_Null ){
pc = pOp->p2 - 1;
u.bc.pCrsr = 0;
break;
}
}
- assert( (u.bc.aMem[u.bc.nField].flags & MEM_Null)==0 );
+ assert( (u.bc.aMx[u.bc.nField].flags & MEM_Null)==0 );
if( u.bc.pCrsr!=0 ){
/* Populate the index search key. */
u.bc.r.pKeyInfo = u.bc.pCx->pKeyInfo;
u.bc.r.nField = u.bc.nField + 1;
u.bc.r.flags = UNPACKED_PREFIX_SEARCH;
- u.bc.r.aMem = u.bc.aMem;
+ u.bc.r.aMem = u.bc.aMx;
/* Extract the value of u.bc.R from register P3. */
sqlite3VdbeMemIntegerify(pIn3);
u64 iKey;
#endif /* local variables moved into u.bd */
+ pIn3 = &aMem[pOp->p3];
assert( pIn3->flags & MEM_Int );
assert( pOp->p1>=0 && pOp->p1<p->nCursor );
u.bd.pC = p->apCsr[pOp->p1];
assert( pOp->p1>=0 && pOp->p1<p->nCursor );
assert( p->apCsr[pOp->p1]!=0 );
pOut->u.i = p->apCsr[pOp->p1]->seqCount++;
- MemSetTypeFlag(pOut, MEM_Int);
break;
}
}else{
/* Assert that P3 is a valid memory cell. */
assert( pOp->p3<=p->nMem );
- u.be.pMem = &p->aMem[pOp->p3];
+ u.be.pMem = &aMem[pOp->p3];
}
REGISTER_TRACE(pOp->p3, u.be.pMem);
u.be.pC->deferredMoveto = 0;
u.be.pC->cacheStatus = CACHE_STALE;
}
- MemSetTypeFlag(pOut, MEM_Int);
pOut->u.i = u.be.v;
break;
}
int op; /* Opcode for update hook: SQLITE_UPDATE or SQLITE_INSERT */
#endif /* local variables moved into u.bf */
- u.bf.pData = &p->aMem[pOp->p2];
+ u.bf.pData = &aMem[pOp->p2];
assert( pOp->p1>=0 && pOp->p1<p->nCursor );
u.bf.pC = p->apCsr[pOp->p1];
assert( u.bf.pC!=0 );
REGISTER_TRACE(pOp->p2, u.bf.pData);
if( pOp->opcode==OP_Insert ){
- u.bf.pKey = &p->aMem[pOp->p3];
+ u.bf.pKey = &aMem[pOp->p3];
assert( u.bf.pKey->flags & MEM_Int );
REGISTER_TRACE(pOp->p3, u.bf.pKey);
u.bf.iKey = u.bf.pKey->u.i;
i64 n64;
#endif /* local variables moved into u.bh */
- pOut = &p->aMem[pOp->p2];
+ pOut = &aMem[pOp->p2];
/* Note that RowKey and RowData are really exactly the same instruction */
assert( pOp->p1>=0 && pOp->p1<p->nCursor );
assert( u.bi.pC!=0 );
assert( u.bi.pC->pseudoTableReg==0 );
if( u.bi.pC->nullRow ){
- /* Do nothing so that reg[P2] remains NULL */
+ pOut->flags = MEM_Null;
break;
}else if( u.bi.pC->deferredMoveto ){
u.bi.v = u.bi.pC->movetoTarget;
}
}
pOut->u.i = u.bi.v;
- MemSetTypeFlag(pOut, MEM_Int);
break;
}
assert( pOp->p1>=0 && pOp->p1<p->nCursor );
u.bn.pC = p->apCsr[pOp->p1];
assert( u.bn.pC!=0 );
+ pIn2 = &aMem[pOp->p2];
assert( pIn2->flags & MEM_Blob );
u.bn.pCrsr = u.bn.pC->pCursor;
if( ALWAYS(u.bn.pCrsr!=0) ){
u.bo.r.pKeyInfo = u.bo.pC->pKeyInfo;
u.bo.r.nField = (u16)pOp->p3;
u.bo.r.flags = 0;
- u.bo.r.aMem = &p->aMem[pOp->p2];
+ u.bo.r.aMem = &aMem[pOp->p2];
rc = sqlite3BtreeMovetoUnpacked(u.bo.pCrsr, &u.bo.r, 0, 0, &u.bo.res);
if( rc==SQLITE_OK && u.bo.res==0 ){
rc = sqlite3BtreeDelete(u.bo.pCrsr);
u.bp.pC = p->apCsr[pOp->p1];
assert( u.bp.pC!=0 );
u.bp.pCrsr = u.bp.pC->pCursor;
+ pOut->flags = MEM_Null;
if( ALWAYS(u.bp.pCrsr!=0) ){
rc = sqlite3VdbeCursorMoveto(u.bp.pC);
if( NEVER(rc) ) goto abort_due_to_error;
if( rc!=SQLITE_OK ){
goto abort_due_to_error;
}
- MemSetTypeFlag(pOut, MEM_Int);
pOut->u.i = u.bp.rowid;
+ pOut->flags = MEM_Int;
}
}
break;
** If P5 is non-zero then the key value is increased by an epsilon prior
** to the comparison. This makes the opcode work like IdxLE.
*/
-case OP_IdxLT: /* jump, in3 */
-case OP_IdxGE: { /* jump, in3 */
+case OP_IdxLT: /* jump */
+case OP_IdxGE: { /* jump */
#if 0 /* local variables moved into u.bq */
VdbeCursor *pC;
int res;
}else{
u.bq.r.flags = UNPACKED_IGNORE_ROWID;
}
- u.bq.r.aMem = &p->aMem[pOp->p3];
+ u.bq.r.aMem = &aMem[pOp->p3];
rc = sqlite3VdbeIdxKeyCompare(u.bq.pC, &u.bq.r, &u.bq.res);
if( pOp->opcode==OP_IdxLT ){
u.bq.res = -u.bq.res;
#else
u.br.iCnt = db->activeVdbeCnt;
#endif
+ pOut->flags = MEM_Null;
if( u.br.iCnt>1 ){
rc = SQLITE_LOCKED;
p->errorAction = OE_Abort;
assert( u.br.iCnt==1 );
assert( (p->btreeMask & (1<<u.br.iDb))!=0 );
rc = sqlite3BtreeDropTable(db->aDb[u.br.iDb].pBt, pOp->p1, &u.br.iMoved);
- MemSetTypeFlag(pOut, MEM_Int);
+ pOut->flags = MEM_Int;
pOut->u.i = u.br.iMoved;
#ifndef SQLITE_OMIT_AUTOVACUUM
if( rc==SQLITE_OK && u.br.iMoved!=0 ){
sqlite3RootPageMoved(&db->aDb[u.br.iDb], u.br.iMoved, pOp->p1);
+ resetSchemaOnFault = 1;
}
#endif
}
if( pOp->p3 ){
p->nChange += u.bs.nChange;
if( pOp->p3>0 ){
- p->aMem[pOp->p3].u.i += u.bs.nChange;
+ aMem[pOp->p3].u.i += u.bs.nChange;
}
}
break;
}
rc = sqlite3BtreeCreateTable(u.bt.pDb->pBt, &u.bt.pgno, u.bt.flags);
pOut->u.i = u.bt.pgno;
- MemSetTypeFlag(pOut, MEM_Int);
break;
}
u.bv.aRoot = sqlite3DbMallocRaw(db, sizeof(int)*(u.bv.nRoot+1) );
if( u.bv.aRoot==0 ) goto no_mem;
assert( pOp->p3>0 && pOp->p3<=p->nMem );
- u.bv.pnErr = &p->aMem[pOp->p3];
+ u.bv.pnErr = &aMem[pOp->p3];
assert( (u.bv.pnErr->flags & MEM_Int)!=0 );
assert( (u.bv.pnErr->flags & (MEM_Str|MEM_Blob))==0 );
- pIn1 = &p->aMem[pOp->p1];
+ pIn1 = &aMem[pOp->p1];
for(u.bv.j=0; u.bv.j<u.bv.nRoot; u.bv.j++){
u.bv.aRoot[u.bv.j] = (int)sqlite3VdbeIntValue(&pIn1[u.bv.j]);
}
**
** An assertion fails if P2 is not an integer.
*/
-case OP_RowSetAdd: { /* in2 */
-#if 0 /* local variables moved into u.bw */
- Mem *pIdx;
- Mem *pVal;
-#endif /* local variables moved into u.bw */
- assert( pOp->p1>0 && pOp->p1<=p->nMem );
- u.bw.pIdx = &p->aMem[pOp->p1];
- assert( pOp->p2>0 && pOp->p2<=p->nMem );
- u.bw.pVal = &p->aMem[pOp->p2];
- assert( (u.bw.pVal->flags & MEM_Int)!=0 );
- if( (u.bw.pIdx->flags & MEM_RowSet)==0 ){
- sqlite3VdbeMemSetRowSet(u.bw.pIdx);
- if( (u.bw.pIdx->flags & MEM_RowSet)==0 ) goto no_mem;
- }
- sqlite3RowSetInsert(u.bw.pIdx->u.pRowSet, u.bw.pVal->u.i);
+case OP_RowSetAdd: { /* in1, in2 */
+ pIn1 = &aMem[pOp->p1];
+ pIn2 = &aMem[pOp->p2];
+ assert( (pIn2->flags & MEM_Int)!=0 );
+ if( (pIn1->flags & MEM_RowSet)==0 ){
+ sqlite3VdbeMemSetRowSet(pIn1);
+ if( (pIn1->flags & MEM_RowSet)==0 ) goto no_mem;
+ }
+ sqlite3RowSetInsert(pIn1->u.pRowSet, pIn2->u.i);
break;
}
** register P3. Or, if boolean index P1 is initially empty, leave P3
** unchanged and jump to instruction P2.
*/
-case OP_RowSetRead: { /* jump, out3 */
-#if 0 /* local variables moved into u.bx */
- Mem *pIdx;
+case OP_RowSetRead: { /* jump, in1, out3 */
+#if 0 /* local variables moved into u.bw */
i64 val;
-#endif /* local variables moved into u.bx */
- assert( pOp->p1>0 && pOp->p1<=p->nMem );
+#endif /* local variables moved into u.bw */
CHECK_FOR_INTERRUPT;
- u.bx.pIdx = &p->aMem[pOp->p1];
- pOut = &p->aMem[pOp->p3];
- if( (u.bx.pIdx->flags & MEM_RowSet)==0
- || sqlite3RowSetNext(u.bx.pIdx->u.pRowSet, &u.bx.val)==0
+ pIn1 = &aMem[pOp->p1];
+ if( (pIn1->flags & MEM_RowSet)==0
+ || sqlite3RowSetNext(pIn1->u.pRowSet, &u.bw.val)==0
){
/* The boolean index is empty */
- sqlite3VdbeMemSetNull(u.bx.pIdx);
+ sqlite3VdbeMemSetNull(pIn1);
pc = pOp->p2 - 1;
}else{
/* A value was pulled from the index */
- assert( pOp->p3>0 && pOp->p3<=p->nMem );
- sqlite3VdbeMemSetInt64(pOut, u.bx.val);
+ sqlite3VdbeMemSetInt64(&aMem[pOp->p3], u.bw.val);
}
break;
}
** inserted as part of some other set).
*/
case OP_RowSetTest: { /* jump, in1, in3 */
-#if 0 /* local variables moved into u.by */
+#if 0 /* local variables moved into u.bx */
int iSet;
int exists;
-#endif /* local variables moved into u.by */
+#endif /* local variables moved into u.bx */
- u.by.iSet = pOp->p4.i;
+ pIn1 = &aMem[pOp->p1];
+ pIn3 = &aMem[pOp->p3];
+ u.bx.iSet = pOp->p4.i;
assert( pIn3->flags&MEM_Int );
/* If there is anything other than a rowset object in memory cell P1,
}
assert( pOp->p4type==P4_INT32 );
- assert( u.by.iSet==-1 || u.by.iSet>=0 );
- if( u.by.iSet ){
- u.by.exists = sqlite3RowSetTest(pIn1->u.pRowSet,
- (u8)(u.by.iSet>=0 ? u.by.iSet & 0xf : 0xff),
+ assert( u.bx.iSet==-1 || u.bx.iSet>=0 );
+ if( u.bx.iSet ){
+ u.bx.exists = sqlite3RowSetTest(pIn1->u.pRowSet,
+ (u8)(u.bx.iSet>=0 ? u.bx.iSet & 0xf : 0xff),
pIn3->u.i);
- if( u.by.exists ){
+ if( u.bx.exists ){
pc = pOp->p2 - 1;
break;
}
}
- if( u.by.iSet>=0 ){
+ if( u.bx.iSet>=0 ){
sqlite3RowSetInsert(pIn1->u.pRowSet, pIn3->u.i);
}
break;
** P4 is a pointer to the VM containing the trigger program.
*/
case OP_Program: { /* jump */
-#if 0 /* local variables moved into u.bz */
+#if 0 /* local variables moved into u.by */
int nMem; /* Number of memory registers for sub-program */
int nByte; /* Bytes of runtime space required for sub-program */
Mem *pRt; /* Register to allocate runtime space */
VdbeFrame *pFrame; /* New vdbe frame to execute in */
SubProgram *pProgram; /* Sub-program to execute */
void *t; /* Token identifying trigger */
-#endif /* local variables moved into u.bz */
+#endif /* local variables moved into u.by */
- u.bz.pProgram = pOp->p4.pProgram;
- u.bz.pRt = &p->aMem[pOp->p3];
- assert( u.bz.pProgram->nOp>0 );
+ u.by.pProgram = pOp->p4.pProgram;
+ u.by.pRt = &aMem[pOp->p3];
+ assert( u.by.pProgram->nOp>0 );
/* If the p5 flag is clear, then recursive invocation of triggers is
** disabled for backwards compatibility (p5 is set if this sub-program
** single trigger all have the same value for the SubProgram.token
** variable. */
if( pOp->p5 ){
- u.bz.t = u.bz.pProgram->token;
- for(u.bz.pFrame=p->pFrame; u.bz.pFrame && u.bz.pFrame->token!=u.bz.t; u.bz.pFrame=u.bz.pFrame->pParent);
- if( u.bz.pFrame ) break;
+ u.by.t = u.by.pProgram->token;
+ for(u.by.pFrame=p->pFrame; u.by.pFrame && u.by.pFrame->token!=u.by.t; u.by.pFrame=u.by.pFrame->pParent);
+ if( u.by.pFrame ) break;
}
if( p->nFrame>=db->aLimit[SQLITE_LIMIT_TRIGGER_DEPTH] ){
break;
}
- /* Register u.bz.pRt is used to store the memory required to save the state
+ /* Register u.by.pRt is used to store the memory required to save the state
** of the current program, and the memory required at runtime to execute
- ** the trigger program. If this trigger has been fired before, then u.bz.pRt
+ ** the trigger program. If this trigger has been fired before, then u.by.pRt
** is already allocated. Otherwise, it must be initialized. */
- if( (u.bz.pRt->flags&MEM_Frame)==0 ){
+ if( (u.by.pRt->flags&MEM_Frame)==0 ){
/* SubProgram.nMem is set to the number of memory cells used by the
** program stored in SubProgram.aOp. As well as these, one memory
** cell is required for each cursor used by the program. Set local
- ** variable u.bz.nMem (and later, VdbeFrame.nChildMem) to this value.
+ ** variable u.by.nMem (and later, VdbeFrame.nChildMem) to this value.
*/
- u.bz.nMem = u.bz.pProgram->nMem + u.bz.pProgram->nCsr;
- u.bz.nByte = ROUND8(sizeof(VdbeFrame))
- + u.bz.nMem * sizeof(Mem)
- + u.bz.pProgram->nCsr * sizeof(VdbeCursor *);
- u.bz.pFrame = sqlite3DbMallocZero(db, u.bz.nByte);
- if( !u.bz.pFrame ){
+ u.by.nMem = u.by.pProgram->nMem + u.by.pProgram->nCsr;
+ u.by.nByte = ROUND8(sizeof(VdbeFrame))
+ + u.by.nMem * sizeof(Mem)
+ + u.by.pProgram->nCsr * sizeof(VdbeCursor *);
+ u.by.pFrame = sqlite3DbMallocZero(db, u.by.nByte);
+ if( !u.by.pFrame ){
goto no_mem;
}
- sqlite3VdbeMemRelease(u.bz.pRt);
- u.bz.pRt->flags = MEM_Frame;
- u.bz.pRt->u.pFrame = u.bz.pFrame;
-
- u.bz.pFrame->v = p;
- u.bz.pFrame->nChildMem = u.bz.nMem;
- u.bz.pFrame->nChildCsr = u.bz.pProgram->nCsr;
- u.bz.pFrame->pc = pc;
- u.bz.pFrame->aMem = p->aMem;
- u.bz.pFrame->nMem = p->nMem;
- u.bz.pFrame->apCsr = p->apCsr;
- u.bz.pFrame->nCursor = p->nCursor;
- u.bz.pFrame->aOp = p->aOp;
- u.bz.pFrame->nOp = p->nOp;
- u.bz.pFrame->token = u.bz.pProgram->token;
-
- u.bz.pEnd = &VdbeFrameMem(u.bz.pFrame)[u.bz.pFrame->nChildMem];
- for(u.bz.pMem=VdbeFrameMem(u.bz.pFrame); u.bz.pMem!=u.bz.pEnd; u.bz.pMem++){
- u.bz.pMem->flags = MEM_Null;
- u.bz.pMem->db = db;
+ sqlite3VdbeMemRelease(u.by.pRt);
+ u.by.pRt->flags = MEM_Frame;
+ u.by.pRt->u.pFrame = u.by.pFrame;
+
+ u.by.pFrame->v = p;
+ u.by.pFrame->nChildMem = u.by.nMem;
+ u.by.pFrame->nChildCsr = u.by.pProgram->nCsr;
+ u.by.pFrame->pc = pc;
+ u.by.pFrame->aMem = p->aMem;
+ u.by.pFrame->nMem = p->nMem;
+ u.by.pFrame->apCsr = p->apCsr;
+ u.by.pFrame->nCursor = p->nCursor;
+ u.by.pFrame->aOp = p->aOp;
+ u.by.pFrame->nOp = p->nOp;
+ u.by.pFrame->token = u.by.pProgram->token;
+
+ u.by.pEnd = &VdbeFrameMem(u.by.pFrame)[u.by.pFrame->nChildMem];
+ for(u.by.pMem=VdbeFrameMem(u.by.pFrame); u.by.pMem!=u.by.pEnd; u.by.pMem++){
+ u.by.pMem->flags = MEM_Null;
+ u.by.pMem->db = db;
}
}else{
- u.bz.pFrame = u.bz.pRt->u.pFrame;
- assert( u.bz.pProgram->nMem+u.bz.pProgram->nCsr==u.bz.pFrame->nChildMem );
- assert( u.bz.pProgram->nCsr==u.bz.pFrame->nChildCsr );
- assert( pc==u.bz.pFrame->pc );
+ u.by.pFrame = u.by.pRt->u.pFrame;
+ assert( u.by.pProgram->nMem+u.by.pProgram->nCsr==u.by.pFrame->nChildMem );
+ assert( u.by.pProgram->nCsr==u.by.pFrame->nChildCsr );
+ assert( pc==u.by.pFrame->pc );
}
p->nFrame++;
- u.bz.pFrame->pParent = p->pFrame;
- u.bz.pFrame->lastRowid = db->lastRowid;
- u.bz.pFrame->nChange = p->nChange;
+ u.by.pFrame->pParent = p->pFrame;
+ u.by.pFrame->lastRowid = db->lastRowid;
+ u.by.pFrame->nChange = p->nChange;
p->nChange = 0;
- p->pFrame = u.bz.pFrame;
- p->aMem = &VdbeFrameMem(u.bz.pFrame)[-1];
- p->nMem = u.bz.pFrame->nChildMem;
- p->nCursor = (u16)u.bz.pFrame->nChildCsr;
- p->apCsr = (VdbeCursor **)&p->aMem[p->nMem+1];
- p->aOp = u.bz.pProgram->aOp;
- p->nOp = u.bz.pProgram->nOp;
+ p->pFrame = u.by.pFrame;
+ p->aMem = aMem = &VdbeFrameMem(u.by.pFrame)[-1];
+ p->nMem = u.by.pFrame->nChildMem;
+ p->nCursor = (u16)u.by.pFrame->nChildCsr;
+ p->apCsr = (VdbeCursor **)&aMem[p->nMem+1];
+ p->aOp = aOp = u.by.pProgram->aOp;
+ p->nOp = u.by.pProgram->nOp;
pc = -1;
break;
** calling OP_Program instruction.
*/
case OP_Param: { /* out2-prerelease */
-#if 0 /* local variables moved into u.ca */
+#if 0 /* local variables moved into u.bz */
VdbeFrame *pFrame;
Mem *pIn;
-#endif /* local variables moved into u.ca */
- u.ca.pFrame = p->pFrame;
- u.ca.pIn = &u.ca.pFrame->aMem[pOp->p1 + u.ca.pFrame->aOp[u.ca.pFrame->pc].p1];
- sqlite3VdbeMemShallowCopy(pOut, u.ca.pIn, MEM_Ephem);
+#endif /* local variables moved into u.bz */
+ u.bz.pFrame = p->pFrame;
+ u.bz.pIn = &u.bz.pFrame->aMem[pOp->p1 + u.bz.pFrame->aOp[u.bz.pFrame->pc].p1];
+ sqlite3VdbeMemShallowCopy(pOut, u.bz.pIn, MEM_Ephem);
break;
}
** an integer.
*/
case OP_MemMax: { /* in2 */
-#if 0 /* local variables moved into u.cb */
+#if 0 /* local variables moved into u.ca */
Mem *pIn1;
VdbeFrame *pFrame;
-#endif /* local variables moved into u.cb */
+#endif /* local variables moved into u.ca */
if( p->pFrame ){
- for(u.cb.pFrame=p->pFrame; u.cb.pFrame->pParent; u.cb.pFrame=u.cb.pFrame->pParent);
- u.cb.pIn1 = &u.cb.pFrame->aMem[pOp->p1];
+ for(u.ca.pFrame=p->pFrame; u.ca.pFrame->pParent; u.ca.pFrame=u.ca.pFrame->pParent);
+ u.ca.pIn1 = &u.ca.pFrame->aMem[pOp->p1];
}else{
- u.cb.pIn1 = &p->aMem[pOp->p1];
+ u.ca.pIn1 = &aMem[pOp->p1];
}
- sqlite3VdbeMemIntegerify(u.cb.pIn1);
+ sqlite3VdbeMemIntegerify(u.ca.pIn1);
+ pIn2 = &aMem[pOp->p2];
sqlite3VdbeMemIntegerify(pIn2);
- if( u.c
b.pIn1->u.i<pIn2->u.i){
- u.cb.pIn1->u.i = pIn2->u.i;
+ if( u.c
a.pIn1->u.i<pIn2->u.i){
+ u.ca.pIn1->u.i = pIn2->u.i;
}
break;
}
** not contain an integer. An assertion fault will result if you try.
*/
case OP_IfPos: { /* jump, in1 */
+ pIn1 = &aMem[pOp->p1];
assert( pIn1->flags&MEM_Int );
if( pIn1->u.i>0 ){
pc = pOp->p2 - 1;
** not contain an integer. An assertion fault will result if you try.
*/
case OP_IfNeg: { /* jump, in1 */
+ pIn1 = &aMem[pOp->p1];
assert( pIn1->flags&MEM_Int );
if( pIn1->u.i<0 ){
pc = pOp->p2 - 1;
break;
}
-/* Opcode: IfZero P1 P2 * * *
+/* Opcode: IfZero P1 P2 P3 * *
**
-** If the value of register P1 is exactly 0, jump to P2.
+** The register P1 must contain an integer. Add literal P3 to the
+** value in register P1. If the result is exactly 0, jump to P2.
**
** It is illegal to use this instruction on a register that does
** not contain an integer. An assertion fault will result if you try.
*/
case OP_IfZero: { /* jump, in1 */
+ pIn1 = &aMem[pOp->p1];
assert( pIn1->flags&MEM_Int );
+ pIn1->u.i += pOp->p3;
if( pIn1->u.i==0 ){
pc = pOp->p2 - 1;
}
** successors.
*/
case OP_AggStep: {
-#if 0 /* local variables moved into u.cc */
+#if 0 /* local variables moved into u.cb */
int n;
int i;
Mem *pMem;
Mem *pRec;
sqlite3_context ctx;
sqlite3_value **apVal;
-#endif /* local variables moved into u.cc */
+#endif /* local variables moved into u.cb */
- u.cc.n = pOp->p5;
- assert( u.cc.n>=0 );
- u.cc.pRec = &p->aMem[pOp->p2];
- u.cc.apVal = p->apArg;
- assert( u.cc.apVal || u.cc.n==0 );
- for(u.cc.i=0; u.cc.i<u.cc.n; u.cc.i++, u.cc.pRec++){
- u.cc.apVal[u.cc.i] = u.cc.pRec;
- sqlite3VdbeMemStoreType(u.cc.pRec);
+ u.cb.n = pOp->p5;
+ assert( u.cb.n>=0 );
+ u.cb.pRec = &aMem[pOp->p2];
+ u.cb.apVal = p->apArg;
+ assert( u.cb.apVal || u.cb.n==0 );
+ for(u.cb.i=0; u.cb.i<u.cb.n; u.cb.i++, u.cb.pRec++){
+ u.cb.apVal[u.cb.i] = u.cb.pRec;
+ sqlite3VdbeMemStoreType(u.cb.pRec);
}
- u.cc.ctx.pFunc = pOp->p4.pFunc;
+ u.cb.ctx.pFunc = pOp->p4.pFunc;
assert( pOp->p3>0 && pOp->p3<=p->nMem );
- u.cc.ctx.pMem = u.cc.pMem = &p->aMem[pOp->p3];
- u.cc.pMem->n++;
- u.cc.ctx.s.flags = MEM_Null;
- u.cc.ctx.s.z = 0;
- u.cc.ctx.s.zMalloc = 0;
- u.cc.ctx.s.xDel = 0;
- u.cc.ctx.s.db = db;
- u.cc.ctx.isError = 0;
- u.cc.ctx.pColl = 0;
- if( u.cc.ctx.pFunc->flags & SQLITE_FUNC_NEEDCOLL ){
+ u.cb.ctx.pMem = u.cb.pMem = &aMem[pOp->p3];
+ u.cb.pMem->n++;
+ u.cb.ctx.s.flags = MEM_Null;
+ u.cb.ctx.s.z = 0;
+ u.cb.ctx.s.zMalloc = 0;
+ u.cb.ctx.s.xDel = 0;
+ u.cb.ctx.s.db = db;
+ u.cb.ctx.isError = 0;
+ u.cb.ctx.pColl = 0;
+ if( u.cb.ctx.pFunc->flags & SQLITE_FUNC_NEEDCOLL ){
assert( pOp>p->aOp );
assert( pOp[-1].p4type==P4_COLLSEQ );
assert( pOp[-1].opcode==OP_CollSeq );
- u.cc.ctx.pColl = pOp[-1].p4.pColl;
+ u.cb.ctx.pColl = pOp[-1].p4.pColl;
}
- (u.cc.ctx.pFunc->xStep)(&u.cc.ctx, u.cc.n, u.cc.apVal);
- if( u.cc.ctx.isError ){
- sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&u.cc.ctx.s));
- rc = u.cc.ctx.isError;
+ (u.cb.ctx.pFunc->xStep)(&u.cb.ctx, u.cb.n, u.cb.apVal);
+ if( u.cb.ctx.isError ){
+ sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&u.cb.ctx.s));
+ rc = u.cb.ctx.isError;
}
- sqlite3VdbeMemRelease(&u.cc.ctx.s);
+ sqlite3VdbeMemRelease(&u.cb.ctx.s);
break;
}
** the step function was not previously called.
*/
case OP_AggFinal: {
-#if 0 /* local variables moved into u.cd */
+#if 0 /* local variables moved into u.cc */
Mem *pMem;
-#endif /* local variables moved into u.cd */
+#endif /* local variables moved into u.cc */
assert( pOp->p1>0 && pOp->p1<=p->nMem );
- u.cd.pMem = &p->aMem[pOp->p1];
- assert( (u.cd.pMem->flags & ~(MEM_Null|MEM_Agg))==0 );
- rc = sqlite3VdbeMemFinalize(u.cd.pMem, pOp->p4.pFunc);
+ u.cc.pMem = &aMem[pOp->p1];
+ assert( (u.cc.pMem->flags & ~(MEM_Null|MEM_Agg))==0 );
+ rc = sqlite3VdbeMemFinalize(u.cc.pMem, pOp->p4.pFunc);
if( rc ){
- sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(u.cd.pMem));
+ sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(u.cc.pMem));
}
- sqlite3VdbeChangeEncoding(u.cd.pMem, encoding);
- UPDATE_MAX_BLOBSIZE(u.cd.pMem);
- if( sqlite3VdbeMemTooBig(u.cd.pMem) ){
+ sqlite3VdbeChangeEncoding(u.cc.pMem, encoding);
+ UPDATE_MAX_BLOBSIZE(u.cc.pMem);
+ if( sqlite3VdbeMemTooBig(u.cc.pMem) ){
goto too_big;
}
break;
** P2. Otherwise, fall through to the next instruction.
*/
case OP_IncrVacuum: { /* jump */
-#if 0 /* local variables moved into u.ce */
+#if 0 /* local variables moved into u.cd */
Btree *pBt;
-#endif /* local variables moved into u.ce */
+#endif /* local variables moved into u.cd */
assert( pOp->p1>=0 && pOp->p1<db->nDb );
assert( (p->btreeMask & (1<<pOp->p1))!=0 );
- u.ce.pBt = db->aDb[pOp->p1].pBt;
- rc = sqlite3BtreeIncrVacuum(u.ce.pBt);
+ u.cd.pBt = db->aDb[pOp->p1].pBt;
+ rc = sqlite3BtreeIncrVacuum(u.cd.pBt);
if( rc==SQLITE_DONE ){
pc = pOp->p2 - 1;
rc = SQLITE_OK;
** code will be set to SQLITE_LOCKED.
*/
case OP_VBegin: {
-#if 0 /* local variables moved into u.cf */
+#if 0 /* local variables moved into u.ce */
VTable *pVTab;
-#endif /* local variables moved into u.cf */
- u.cf.pVTab = pOp->p4.pVtab;
- rc = sqlite3VtabBegin(db, u.cf.pVTab);
- if( u.cf.pVTab ){
+#endif /* local variables moved into u.ce */
+ u.ce.pVTab = pOp->p4.pVtab;
+ rc = sqlite3VtabBegin(db, u.ce.pVTab);
+ if( u.ce.pVTab ){
sqlite3DbFree(db, p->zErrMsg);
- p->zErrMsg = u.cf.pVTab->pVtab->zErrMsg;
- u.cf.pVTab->pVtab->zErrMsg = 0;
+ p->zErrMsg = u.ce.pVTab->pVtab->zErrMsg;
+ u.ce.pVTab->pVtab->zErrMsg = 0;
}
break;
}
** table and stores that cursor in P1.
*/
case OP_VOpen: {
-#if 0 /* local variables moved into u.cg */
+#if 0 /* local variables moved into u.cf */
VdbeCursor *pCur;
sqlite3_vtab_cursor *pVtabCursor;
sqlite3_vtab *pVtab;
sqlite3_module *pModule;
-#endif /* local variables moved into u.cg */
+#endif /* local variables moved into u.cf */
- u.cg.pCur = 0;
- u.cg.pVtabCursor = 0;
- u.cg.pVtab = pOp->p4.pVtab->pVtab;
- u.cg.pModule = (sqlite3_module *)u.cg.pVtab->pModule;
- assert(u.cg.pVtab && u.cg.pModule);
+ u.cf.pCur = 0;
+ u.cf.pVtabCursor = 0;
+ u.cf.pVtab = pOp->p4.pVtab->pVtab;
+ u.cf.pModule = (sqlite3_module *)u.cf.pVtab->pModule;
+ assert(u.cf.pVtab && u.cf.pModule);
if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
- rc = u.cg.pModule->xOpen(u.cg.pVtab, &u.cg.pVtabCursor);
+ rc = u.cf.pModule->xOpen(u.cf.pVtab, &u.cf.pVtabCursor);
sqlite3DbFree(db, p->zErrMsg);
- p->zErrMsg = u.cg.pVtab->zErrMsg;
- u.cg.pVtab->zErrMsg = 0;
+ p->zErrMsg = u.cf.pVtab->zErrMsg;
+ u.cf.pVtab->zErrMsg = 0;
if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse;
if( SQLITE_OK==rc ){
/* Initialize sqlite3_vtab_cursor base class */
- u.cg.pVtabCursor->pVtab = u.cg.pVtab;
+ u.cf.pVtabCursor->pVtab = u.cf.pVtab;
/* Initialise vdbe cursor object */
- u.cg.pCur = allocateCursor(p, pOp->p1, 0, -1, 0);
- if( u.cg.pCur ){
- u.cg.pCur->pVtabCursor = u.cg.pVtabCursor;
- u.cg.pCur->pModule = u.cg.pVtabCursor->pVtab->pModule;
+ u.cf.pCur = allocateCursor(p, pOp->p1, 0, -1, 0);
+ if( u.cf.pCur ){
+ u.cf.pCur->pVtabCursor = u.cf.pVtabCursor;
+ u.cf.pCur->pModule = u.cf.pVtabCursor->pVtab->pModule;
}else{
db->mallocFailed = 1;
- u.cg.pModule->xClose(u.cg.pVtabCursor);
+ u.cf.pModule->xClose(u.cf.pVtabCursor);
}
}
break;
** A jump is made to P2 if the result set after filtering would be empty.
*/
case OP_VFilter: { /* jump */
-#if 0 /* local variables moved into u.ch */
+#if 0 /* local variables moved into u.cg */
int nArg;
int iQuery;
const sqlite3_module *pModule;
int res;
int i;
Mem **apArg;
-#endif /* local variables moved into u.ch */
+#endif /* local variables moved into u.cg */
- u.ch.pQuery = &p->aMem[pOp->p3];
- u.ch.pArgc = &u.ch.pQuery[1];
- u.ch.pCur = p->apCsr[pOp->p1];
- REGISTER_TRACE(pOp->p3, u.ch.pQuery);
- assert( u.ch.pCur->pVtabCursor );
- u.ch.pVtabCursor = u.ch.pCur->pVtabCursor;
- u.ch.pVtab = u.ch.pVtabCursor->pVtab;
- u.ch.pModule = u.ch.pVtab->pModule;
+ u.cg.pQuery = &aMem[pOp->p3];
+ u.cg.pArgc = &u.cg.pQuery[1];
+ u.cg.pCur = p->apCsr[pOp->p1];
+ REGISTER_TRACE(pOp->p3, u.cg.pQuery);
+ assert( u.cg.pCur->pVtabCursor );
+ u.cg.pVtabCursor = u.cg.pCur->pVtabCursor;
+ u.cg.pVtab = u.cg.pVtabCursor->pVtab;
+ u.cg.pModule = u.cg.pVtab->pModule;
/* Grab the index number and argc parameters */
- assert( (u.ch.pQuery->flags&MEM_Int)!=0 && u.ch.pArgc->flags==MEM_Int );
- u.ch.nArg = (int)u.ch.pArgc->u.i;
- u.ch.iQuery = (int)u.ch.pQuery->u.i;
+ assert( (u.cg.pQuery->flags&MEM_Int)!=0 && u.cg.pArgc->flags==MEM_Int );
+ u.cg.nArg = (int)u.cg.pArgc->u.i;
+ u.cg.iQuery = (int)u.cg.pQuery->u.i;
/* Invoke the xFilter method */
{
- u.ch.res = 0;
- u.ch.apArg = p->apArg;
- for(u.ch.i = 0; u.ch.i<u.ch.nArg; u.ch.i++){
- u.ch.apArg[u.ch.i] = &u.ch.pArgc[u.ch.i+1];
- sqlite3VdbeMemStoreType(u.ch.apArg[u.ch.i]);
+ u.cg.res = 0;
+ u.cg.apArg = p->apArg;
+ for(u.cg.i = 0; u.cg.i<u.cg.nArg; u.cg.i++){
+ u.cg.apArg[u.cg.i] = &u.cg.pArgc[u.cg.i+1];
+ sqlite3VdbeMemStoreType(u.cg.apArg[u.cg.i]);
}
if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
p->inVtabMethod = 1;
- rc = u.ch.pModule->xFilter(u.ch.pVtabCursor, u.ch.iQuery, pOp->p4.z, u.ch.nArg, u.ch.apArg);
+ rc = u.cg.pModule->xFilter(u.cg.pVtabCursor, u.cg.iQuery, pOp->p4.z, u.cg.nArg, u.cg.apArg);
p->inVtabMethod = 0;
sqlite3DbFree(db, p->zErrMsg);
- p->zErrMsg = u.ch.pVtab->zErrMsg;
- u.ch.pVtab->zErrMsg = 0;
+ p->zErrMsg = u.cg.pVtab->zErrMsg;
+ u.cg.pVtab->zErrMsg = 0;
if( rc==SQLITE_OK ){
- u.ch.res = u.ch.pModule->xEof(u.ch.pVtabCursor);
+ u.cg.res = u.cg.pModule->xEof(u.cg.pVtabCursor);
}
if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse;
- if( u.ch.res ){
+ if( u.cg.res ){
pc = pOp->p2 - 1;
}
}
- u.ch.pCur->nullRow = 0;
+ u.cg.pCur->nullRow = 0;
break;
}
** P1 cursor is pointing to into register P3.
*/
case OP_VColumn: {
-#if 0 /* local variables moved into u.ci */
+#if 0 /* local variables moved into u.ch */
sqlite3_vtab *pVtab;
const sqlite3_module *pModule;
Mem *pDest;
sqlite3_context sContext;
-#endif /* local variables moved into u.ci */
+#endif /* local variables moved into u.ch */
VdbeCursor *pCur = p->apCsr[pOp->p1];
assert( pCur->pVtabCursor );
assert( pOp->p3>0 && pOp->p3<=p->nMem );
- u.ci.pDest = &p->aMem[pOp->p3];
+ u.ch.pDest = &aMem[pOp->p3];
if( pCur->nullRow ){
- sqlite3VdbeMemSetNull(u.ci.pDest);
+ sqlite3VdbeMemSetNull(u.ch.pDest);
break;
}
- u.ci.pVtab = pCur->pVtabCursor->pVtab;
- u.ci.pModule = u.ci.pVtab->pModule;
- assert( u.ci.pModule->xColumn );
- memset(&u.ci.sContext, 0, sizeof(u.ci.sContext));
+ u.ch.pVtab = pCur->pVtabCursor->pVtab;
+ u.ch.pModule = u.ch.pVtab->pModule;
+ assert( u.ch.pModule->xColumn );
+ memset(&u.ch.sContext, 0, sizeof(u.ch.sContext));
/* The output cell may already have a buffer allocated. Move
- ** the current contents to u.ci.sContext.s so in case the user-function
+ ** the current contents to u.ch.sContext.s so in case the user-function
** can use the already allocated buffer instead of allocating a
** new one.
*/
- sqlite3VdbeMemMove(&u.ci.sContext.s, u.ci.pDest);
- MemSetTypeFlag(&u.ci.sContext.s, MEM_Null);
+ sqlite3VdbeMemMove(&u.ch.sContext.s, u.ch.pDest);
+ MemSetTypeFlag(&u.ch.sContext.s, MEM_Null);
if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
- rc = u.ci.pModule->xColumn(pCur->pVtabCursor, &u.ci.sContext, pOp->p2);
+ rc = u.ch.pModule->xColumn(pCur->pVtabCursor, &u.ch.sContext, pOp->p2);
sqlite3DbFree(db, p->zErrMsg);
- p->zErrMsg = u.ci.pVtab->zErrMsg;
- u.ci.pVtab->zErrMsg = 0;
- if( u.ci.sContext.isError ){
- rc = u.ci.sContext.isError;
+ p->zErrMsg = u.ch.pVtab->zErrMsg;
+ u.ch.pVtab->zErrMsg = 0;
+ if( u.ch.sContext.isError ){
+ rc = u.ch.sContext.isError;
}
/* Copy the result of the function to the P3 register. We
** do this regardless of whether or not an error occurred to ensure any
- ** dynamic allocation in u.ci.sContext.s (a Mem struct) is released.
+ ** dynamic allocation in u.ch.sContext.s (a Mem struct) is released.
*/
- sqlite3VdbeChangeEncoding(&u.ci.sContext.s, encoding);
- REGISTER_TRACE(pOp->p3, u.ci.pDest);
- sqlite3VdbeMemMove(u.ci.pDest, &u.ci.sContext.s);
- UPDATE_MAX_BLOBSIZE(u.ci.pDest);
+ sqlite3VdbeChangeEncoding(&u.ch.sContext.s, encoding);
+ sqlite3VdbeMemMove(u.ch.pDest, &u.ch.sContext.s);
+ REGISTER_TRACE(pOp->p3, u.ch.pDest);
+ UPDATE_MAX_BLOBSIZE(u.ch.pDest);
if( sqlite3SafetyOn(db) ){
goto abort_due_to_misuse;
}
- if( sqlite3VdbeMemTooBig(u.ci.pDest) ){
+ if( sqlite3VdbeMemTooBig(u.ch.pDest) ){
goto too_big;
}
break;
** the end of its result set, then fall through to the next instruction.
*/
case OP_VNext: { /* jump */
-#if 0 /* local variables moved into u.cj */
+#if 0 /* local variables moved into u.ci */
sqlite3_vtab *pVtab;
const sqlite3_module *pModule;
int res;
VdbeCursor *pCur;
-#endif /* local variables moved into u.cj */
+#endif /* local variables moved into u.ci */
- u.cj.res = 0;
- u.cj.pCur = p->apCsr[pOp->p1];
- assert( u.cj.pCur->pVtabCursor );
- if( u.cj.pCur->nullRow ){
+ u.ci.res = 0;
+ u.ci.pCur = p->apCsr[pOp->p1];
+ assert( u.ci.pCur->pVtabCursor );
+ if( u.ci.pCur->nullRow ){
break;
}
- u.cj.pVtab = u.cj.pCur->pVtabCursor->pVtab;
- u.cj.pModule = u.cj.pVtab->pModule;
- assert( u.cj.pModule->xNext );
+ u.ci.pVtab = u.ci.pCur->pVtabCursor->pVtab;
+ u.ci.pModule = u.ci.pVtab->pModule;
+ assert( u.ci.pModule->xNext );
/* Invoke the xNext() method of the module. There is no way for the
** underlying implementation to return an error if one occurs during
*/
if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
p->inVtabMethod = 1;
- rc = u.cj.pModule->xNext(u.cj.pCur->pVtabCursor);
+ rc = u.ci.pModule->xNext(u.ci.pCur->pVtabCursor);
p->inVtabMethod = 0;
sqlite3DbFree(db, p->zErrMsg);
- p->zErrMsg = u.cj.pVtab->zErrMsg;
- u.cj.pVtab->zErrMsg = 0;
+ p->zErrMsg = u.ci.pVtab->zErrMsg;
+ u.ci.pVtab->zErrMsg = 0;
if( rc==SQLITE_OK ){
- u.cj.res = u.cj.pModule->xEof(u.cj.pCur->pVtabCursor);
+ u.ci.res = u.ci.pModule->xEof(u.ci.pCur->pVtabCursor);
}
if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse;
- if( !u.cj.res ){
+ if( !u.ci.res ){
/* If there is data, jump to P2 */
pc = pOp->p2 - 1;
}
** in register P1 is passed as the zName argument to the xRename method.
*/
case OP_VRename: {
-#if 0 /* local variables moved into u.ck */
+#if 0 /* local variables moved into u.cj */
sqlite3_vtab *pVtab;
Mem *pName;
-#endif /* local variables moved into u.ck */
+#endif /* local variables moved into u.cj */
- u.ck.pVtab = pOp->p4.pVtab->pVtab;
- u.ck.pName = &p->aMem[pOp->p1];
- assert( u.ck.pVtab->pModule->xRename );
- REGISTER_TRACE(pOp->p1, u.ck.pName);
- assert( u.ck.pName->flags & MEM_Str );
+ u.cj.pVtab = pOp->p4.pVtab->pVtab;
+ u.cj.pName = &aMem[pOp->p1];
+ assert( u.cj.pVtab->pModule->xRename );
+ REGISTER_TRACE(pOp->p1, u.cj.pName);
+ assert( u.cj.pName->flags & MEM_Str );
if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
- rc = u.ck.pVtab->pModule->xRename(u.ck.pVtab, u.ck.pName->z);
+ rc = u.cj.pVtab->pModule->xRename(u.cj.pVtab, u.cj.pName->z);
sqlite3DbFree(db, p->zErrMsg);
- p->zErrMsg = u.ck.pVtab->zErrMsg;
- u.ck.pVtab->zErrMsg = 0;
+ p->zErrMsg = u.cj.pVtab->zErrMsg;
+ u.cj.pVtab->zErrMsg = 0;
if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse;
break;
** is set to the value of the rowid for the row just inserted.
*/
case OP_VUpdate: {
-#if 0 /* local variables moved into u.cl */
+#if 0 /* local variables moved into u.ck */
sqlite3_vtab *pVtab;
sqlite3_module *pModule;
int nArg;
sqlite_int64 rowid;
Mem **apArg;
Mem *pX;
-#endif /* local variables moved into u.cl */
+#endif /* local variables moved into u.ck */
- u.cl.pVtab = pOp->p4.pVtab->pVtab;
- u.cl.pModule = (sqlite3_module *)u.cl.pVtab->pModule;
- u.cl.nArg = pOp->p2;
+ u.ck.pVtab = pOp->p4.pVtab->pVtab;
+ u.ck.pModule = (sqlite3_module *)u.ck.pVtab->pModule;
+ u.ck.nArg = pOp->p2;
assert( pOp->p4type==P4_VTAB );
- if( ALWAYS(u.cl.pModule->xUpdate) ){
- u.cl.apArg = p->apArg;
- u.cl.pX = &p->aMem[pOp->p3];
- for(u.cl.i=0; u.cl.i<u.cl.nArg; u.cl.i++){
- sqlite3VdbeMemStoreType(u.cl.pX);
- u.cl.apArg[u.cl.i] = u.cl.pX;
- u.cl.pX++;
+ if( ALWAYS(u.ck.pModule->xUpdate) ){
+ u.ck.apArg = p->apArg;
+ u.ck.pX = &aMem[pOp->p3];
+ for(u.ck.i=0; u.ck.i<u.ck.nArg; u.ck.i++){
+ sqlite3VdbeMemStoreType(u.ck.pX);
+ u.ck.apArg[u.ck.i] = u.ck.pX;
+ u.ck.pX++;
}
if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
- rc = u.cl.pModule->xUpdate(u.cl.pVtab, u.cl.nArg, u.cl.apArg, &u.cl.rowid);
+ rc = u.ck.pModule->xUpdate(u.ck.pVtab, u.ck.nArg, u.ck.apArg, &u.ck.rowid);
sqlite3DbFree(db, p->zErrMsg);
- p->zErrMsg = u.cl.pVtab->zErrMsg;
- u.cl.pVtab->zErrMsg = 0;
+ p->zErrMsg = u.ck.pVtab->zErrMsg;
+ u.ck.pVtab->zErrMsg = 0;
if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse;
if( rc==SQLITE_OK && pOp->p1 ){
- assert( u.cl.nArg>1 && u.cl.apArg[0] && (u.cl.apArg[0]->flags&MEM_Null) );
- db->lastRowid = u.cl.rowid;
+ assert( u.ck.nArg>1 && u.ck.apArg[0] && (u.ck.apArg[0]->flags&MEM_Null) );
+ db->lastRowid = u.ck.rowid;
}
p->nChange++;
}
** Write the current number of pages in database P1 to memory cell P2.
*/
case OP_Pagecount: { /* out2-prerelease */
-#if 0 /* local variables moved into u.cm */
+#if 0 /* local variables moved into u.cl */
int p1;
int nPage;
Pager *pPager;
-#endif /* local variables moved into u.cm */
+#endif /* local variables moved into u.cl */
- u.cm.p1 = pOp->p1;
- u.cm.pPager = sqlite3BtreePager(db->aDb[u.cm.p1].pBt);
- rc = sqlite3PagerPagecount(u.cm.pPager, &u.cm.nPage);
+ u.cl.p1 = pOp->p1;
+ u.cl.pPager = sqlite3BtreePager(db->aDb[u.cl.p1].pBt);
+ rc = sqlite3PagerPagecount(u.cl.pPager, &u.cl.nPage);
/* OP_Pagecount is always called from within a read transaction. The
** page count has already been successfully read and cached. So the
** sqlite3PagerPagecount() call above cannot fail. */
if( ALWAYS(rc==SQLITE_OK) ){
- pOut->flags = MEM_Int;
- pOut->u.i = u.cm.nPage;
+ pOut->u.i = u.cl.nPage;
}
break;
}
** the UTF-8 string contained in P4 is emitted on the trace callback.
*/
case OP_Trace: {
-#if 0 /* local variables moved into u.cn */
+#if 0 /* local variables moved into u.cm */
char *zTrace;
-#endif /* local variables moved into u.cn */
+#endif /* local variables moved into u.cm */
- u.cn.zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql);
- if( u.cn.zTrace ){
+ u.cm.zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql);
+ if( u.cm.zTrace ){
if( db->xTrace ){
- db->xTrace(db->pTraceArg, u.cn.zTrace);
+ char *z = sqlite3VdbeExpandSql(p, u.cm.zTrace);
+ db->xTrace(db->pTraceArg, z);
+ sqlite3DbFree(db, z);
}
#ifdef SQLITE_DEBUG
if( (db->flags & SQLITE_SqlTrace)!=0 ){
- sqlite3DebugPrintf("SQL-trace: %s\n", u.cn.zTrace);
+ sqlite3DebugPrintf("SQL-trace: %s\n", u.cm.zTrace);
}
#endif /* SQLITE_DEBUG */
}
pOp->cnt++;
#if 0
fprintf(stdout, "%10llu ", elapsed);
- sqlite3VdbePrintOp(stdout, origPc, &p->aOp[origPc]);
+ sqlite3VdbePrintOp(stdout, origPc, &aOp[origPc]);
#endif
}
#endif
#ifdef SQLITE_DEBUG
if( p->trace ){
if( rc!=0 ) fprintf(p->trace,"rc=%d\n",rc);
- if( opProperty & OPFLG_OUT2_PRERELEASE ){
- registerTrace(p->trace, pOp->p2, pOut);
+ if( pOp->opflags & (OPFLG_OUT2_PRERELEASE|OPFLG_OUT2) ){
+ registerTrace(p->trace, pOp->p2, &aMem[pOp->p2]);
}
- if( opProperty & OPFLG_OUT3 ){
- registerTrace(p->trace, pOp->p3, pOut);
+ if( pOp->opflags & OPFLG_OUT3 ){
+ registerTrace(p->trace, pOp->p3, &aMem[pOp->p3]);
}
}
#endif /* SQLITE_DEBUG */
sqlite3VdbeHalt(p);
if( rc==SQLITE_IOERR_NOMEM ) db->mallocFailed = 1;
rc = SQLITE_ERROR;
+ if( resetSchemaOnFault ) sqlite3ResetInternalSchema(db, 0);
/* This is the only way out of this procedure. We have to
** release the mutexes on btrees that were acquired at the
*************************************************************************
**
** This file contains code used to implement incremental BLOB I/O.
-**
-** $Id$
*/
**
*************************************************************************
**
-** @(#) $Id$
-*/
-
-#ifdef SQLITE_ENABLE_ATOMIC_WRITE
-
-/*
** This file implements a special kind of sqlite3_file object used
** by SQLite to create journal files if the atomic-write optimization
** is enabled.
** buffer, or
** 2) The sqlite3JournalCreate() function is called.
*/
-
+#ifdef SQLITE_ENABLE_ATOMIC_WRITE
/*
** This file contains code use to implement an in-memory rollback journal.
** The in-memory rollback journal is used to journal transactions for
** ":memory:" databases and when the journal_mode=MEMORY pragma is used.
-**
-** @(#) $Id$
*/
/* Forward references to internal structures */
*************************************************************************
** This file contains routines used for walking the parser tree for
** an SQL statement.
-**
-** $Id$
*/
** This file contains routines used for walking the parser tree and
** resolve all identifiers by associating them with a particular
** table and column.
-**
-** $Id$
*/
/*
pDup->pColl = pExpr->pColl;
pDup->flags |= EP_ExpCollate;
}
- sqlite3ExprClear(db, pExpr);
+
+ /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This
+ ** prevents ExprDelete() from deleting the Expr structure itself,
+ ** allowing it to be repopulated by the memcpy() on the following line.
+ */
+ ExprSetProperty(pExpr, EP_Static);
+ sqlite3ExprDelete(db, pExpr);
memcpy(pExpr, pDup, sizeof(*pExpr));
sqlite3DbFree(db, pDup);
}
testcase( iCol==31 );
testcase( iCol==32 );
pParse->oldmask |= (iCol>=32 ? 0xffffffff : (((u32)1)<<iCol));
+ }else{
+ testcase( iCol==31 );
+ testcase( iCol==32 );
+ pParse->newmask |= (iCol>=32 ? 0xffffffff : (((u32)1)<<iCol));
}
pExpr->iColumn = (i16)iCol;
pExpr->pTab = pTab;
}
/*
-** Clear an expression structure without deleting the structure itself.
-** Substructure is deleted.
+** Recursively delete an expression tree.
*/
-SQLITE_PRIVATE void sqlite3ExprClear(sqlite3 *db, Expr *p){
- assert( p!=0 );
+SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3 *db, Expr *p){
+ if( p==0 ) return;
if( !ExprHasAnyProperty(p, EP_TokenOnly) ){
sqlite3ExprDelete(db, p->pLeft);
sqlite3ExprDelete(db, p->pRight);
sqlite3ExprListDelete(db, p->x.pList);
}
}
-}
-
-/*
-** Recursively delete an expression tree.
-*/
-SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3 *db, Expr *p){
- if( p==0 ) return;
- sqlite3ExprClear(db, p);
if( !ExprHasProperty(p, EP_Static) ){
sqlite3DbFree(db, p);
}
return rc;
}
+/*
+** Return FALSE if there is no chance that the expression can be NULL.
+**
+** If the expression might be NULL or if the expression is too complex
+** to tell return TRUE.
+**
+** This routine is used as an optimization, to skip OP_IsNull opcodes
+** when we know that a value cannot be NULL. Hence, a false positive
+** (returning TRUE when in fact the expression can never be NULL) might
+** be a small performance hit but is otherwise harmless. On the other
+** hand, a false negative (returning FALSE when the result could be NULL)
+** will likely result in an incorrect answer. So when in doubt, return
+** TRUE.
+*/
+SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr *p){
+ u8 op;
+ while( p->op==TK_UPLUS || p->op==TK_UMINUS ){ p = p->pLeft; }
+ op = p->op;
+ if( op==TK_REGISTER ) op = p->op2;
+ switch( op ){
+ case TK_INTEGER:
+ case TK_STRING:
+ case TK_FLOAT:
+ case TK_BLOB:
+ return 0;
+ default:
+ return 1;
+ }
+}
+
+/*
+** Generate an OP_IsNull instruction that tests register iReg and jumps
+** to location iDest if the value in iReg is NULL. The value in iReg
+** was computed by pExpr. If we can look at pExpr at compile-time and
+** determine that it can never generate a NULL, then the OP_IsNull operation
+** can be omitted.
+*/
+SQLITE_PRIVATE void sqlite3ExprCodeIsNullJump(
+ Vdbe *v, /* The VDBE under construction */
+ const Expr *pExpr, /* Only generate OP_IsNull if this expr can be NULL */
+ int iReg, /* Test the value in this register for NULL */
+ int iDest /* Jump here if the value is null */
+){
+ if( sqlite3ExprCanBeNull(pExpr) ){
+ sqlite3VdbeAddOp2(v, OP_IsNull, iReg, iDest);
+ }
+}
+
+/*
+** Return TRUE if the given expression is a constant which would be
+** unchanged by OP_Affinity with the affinity given in the second
+** argument.
+**
+** This routine is used to determine if the OP_Affinity operation
+** can be omitted. When in doubt return FALSE. A false negative
+** is harmless. A false positive, however, can result in the wrong
+** answer.
+*/
+SQLITE_PRIVATE int sqlite3ExprNeedsNoAffinityChange(const Expr *p, char aff){
+ u8 op;
+ if( aff==SQLITE_AFF_NONE ) return 1;
+ while( p->op==TK_UPLUS || p->op==TK_UMINUS ){ p = p->pLeft; }
+ op = p->op;
+ if( op==TK_REGISTER ) op = p->op2;
+ switch( op ){
+ case TK_INTEGER: {
+ return aff==SQLITE_AFF_INTEGER || aff==SQLITE_AFF_NUMERIC;
+ }
+ case TK_FLOAT: {
+ return aff==SQLITE_AFF_REAL || aff==SQLITE_AFF_NUMERIC;
+ }
+ case TK_STRING: {
+ return aff==SQLITE_AFF_TEXT;
+ }
+ case TK_BLOB: {
+ return 1;
+ }
+ case TK_COLUMN: {
+ assert( p->iTable>=0 ); /* p cannot be part of a CHECK constraint */
+ return p->iColumn<0
+ && (aff==SQLITE_AFF_INTEGER || aff==SQLITE_AFF_NUMERIC);
+ }
+ default: {
+ return 0;
+ }
+ }
+}
+
/*
** Return TRUE if the given string is a row-id column name.
*/
** When the b-tree is being used for membership tests, the calling function
** needs to know whether or not the structure contains an SQL NULL
** value in order to correctly evaluate expressions like "X IN (Y, Z)".
-** If there is a chance that the b-tree might contain a NULL value at
+** If there is any chance that the (...) might contain a NULL value at
** runtime, then a register is allocated and the register number written
-** to *prNotFound. If there is no chance that the b-tree contains a
+** to *prNotFound. If there is no chance that the (...) contains a
** NULL value, then *prNotFound is left unchanged.
**
** If a register is allocated and its location stored in *prNotFound, then
-** its initial value is NULL. If the b-tree does not remain constant
-** for the duration of the query (i.e. the SELECT that generates the b-tree
+** its initial value is NULL. If the (...) does not remain constant
+** for the duration of the query (i.e. the SELECT within the (...)
** is a correlated subquery) then the value of the allocated register is
-** reset to NULL each time the b-tree is repopulated. This allows the
+** reset to NULL each time the subquery is rerun. This allows the
** caller to use vdbe code equivalent to the following:
**
** if( register==NULL ){
affinity = sqlite3ExprAffinity(pLeft);
/* Whether this is an 'x IN(SELECT...)' or an 'x IN(<exprlist>)'
- ** expression it is handled the same way. A virtual table is
+ ** expression it is handled the same way. An ephemeral table is
** filled with single-field index keys representing the results
** from the SELECT or the <exprlist>.
**
}
#endif /* SQLITE_OMIT_SUBQUERY */
+#ifndef SQLITE_OMIT_SUBQUERY
+/*
+** Generate code for an IN expression.
+**
+** x IN (SELECT ...)
+** x IN (value, value, ...)
+**
+** The left-hand side (LHS) is a scalar expression. The right-hand side (RHS)
+** is an array of zero or more values. The expression is true if the LHS is
+** contained within the RHS. The value of the expression is unknown (NULL)
+** if the LHS is NULL or if the LHS is not contained within the RHS and the
+** RHS contains one or more NULL values.
+**
+** This routine generates code will jump to destIfFalse if the LHS is not
+** contained within the RHS. If due to NULLs we cannot determine if the LHS
+** is contained in the RHS then jump to destIfNull. If the LHS is contained
+** within the RHS then fall through.
+*/
+static void sqlite3ExprCodeIN(
+ Parse *pParse, /* Parsing and code generating context */
+ Expr *pExpr, /* The IN expression */
+ int destIfFalse, /* Jump here if LHS is not contained in the RHS */
+ int destIfNull /* Jump here if the results are unknown due to NULLs */
+){
+ int rRhsHasNull = 0; /* Register that is true if RHS contains NULL values */
+ char affinity; /* Comparison affinity to use */
+ int eType; /* Type of the RHS */
+ int r1; /* Temporary use register */
+ Vdbe *v; /* Statement under construction */
+
+ /* Compute the RHS. After this step, the table with cursor
+ ** pExpr->iTable will contains the values that make up the RHS.
+ */
+ v = pParse->pVdbe;
+ assert( v!=0 ); /* OOM detected prior to this routine */
+ VdbeNoopComment((v, "begin IN expr"));
+ eType = sqlite3FindInIndex(pParse, pExpr, &rRhsHasNull);
+
+ /* Figure out the affinity to use to create a key from the results
+ ** of the expression. affinityStr stores a static string suitable for
+ ** P4 of OP_MakeRecord.
+ */
+ affinity = comparisonAffinity(pExpr);
+
+ /* Code the LHS, the <expr> from "<expr> IN (...)".
+ */
+ sqlite3ExprCachePush(pParse);
+ r1 = sqlite3GetTempReg(pParse);
+ sqlite3ExprCode(pParse, pExpr->pLeft, r1);
+ sqlite3VdbeAddOp2(v, OP_IsNull, r1, destIfNull);
+
+
+ if( eType==IN_INDEX_ROWID ){
+ /* In this case, the RHS is the ROWID of table b-tree
+ */
+ sqlite3VdbeAddOp2(v, OP_MustBeInt, r1, destIfFalse);
+ sqlite3VdbeAddOp3(v, OP_NotExists, pExpr->iTable, destIfFalse, r1);
+ }else{
+ /* In this case, the RHS is an index b-tree.
+ */
+ sqlite3VdbeAddOp4(v, OP_Affinity, r1, 1, 0, &affinity, 1);
+
+ /* If the set membership test fails, then the result of the
+ ** "x IN (...)" expression must be either 0 or NULL. If the set
+ ** contains no NULL values, then the result is 0. If the set
+ ** contains one or more NULL values, then the result of the
+ ** expression is also NULL.
+ */
+ if( rRhsHasNull==0 || destIfFalse==destIfNull ){
+ /* This branch runs if it is known at compile time that the RHS
+ ** cannot contain NULL values. This happens as the result
+ ** of a "NOT NULL" constraint in the database schema.
+ **
+ ** Also run this branch if NULL is equivalent to FALSE
+ ** for this particular IN operator.
+ */
+ sqlite3VdbeAddOp4Int(v, OP_NotFound, pExpr->iTable, destIfFalse, r1, 1);
+
+ }else{
+ /* In this branch, the RHS of the IN might contain a NULL and
+ ** the presence of a NULL on the RHS makes a difference in the
+ ** outcome.
+ */
+ int j1, j2, j3;
+
+ /* First check to see if the LHS is contained in the RHS. If so,
+ ** then the presence of NULLs in the RHS does not matter, so jump
+ ** over all of the code that follows.
+ */
+ j1 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, r1, 1);
+
+ /* Here we begin generating code that runs if the LHS is not
+ ** contained within the RHS. Generate additional code that
+ ** tests the RHS for NULLs. If the RHS contains a NULL then
+ ** jump to destIfNull. If there are no NULLs in the RHS then
+ ** jump to destIfFalse.
+ */
+ j2 = sqlite3VdbeAddOp1(v, OP_NotNull, rRhsHasNull);
+ j3 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, rRhsHasNull, 1);
+ sqlite3VdbeAddOp2(v, OP_Integer, -1, rRhsHasNull);
+ sqlite3VdbeJumpHere(v, j3);
+ sqlite3VdbeAddOp2(v, OP_AddImm, rRhsHasNull, 1);
+ sqlite3VdbeJumpHere(v, j2);
+
+ /* Jump to the appropriate target depending on whether or not
+ ** the RHS contains a NULL
+ */
+ sqlite3VdbeAddOp2(v, OP_If, rRhsHasNull, destIfNull);
+ sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse);
+
+ /* The OP_Found at the top of this branch jumps here when true,
+ ** causing the overall IN expression evaluation to fall through.
+ */
+ sqlite3VdbeJumpHere(v, j1);
+ }
+ }
+ sqlite3ReleaseTempReg(pParse, r1);
+ sqlite3ExprCachePop(pParse, 1);
+ VdbeComment((v, "end IN expr"));
+}
+#endif /* SQLITE_OMIT_SUBQUERY */
+
/*
** Duplicate an 8-byte value
*/
sqlite3ErrorMsg(pParse, "unknown function: %.*s()", nId, zId);
break;
}
+
+ /* Attempt a direct implementation of the built-in COALESCE() and
+ ** IFNULL() functions. This avoids unnecessary evalation of
+ ** arguments past the first non-NULL argument.
+ */
+ if( pDef->flags & SQLITE_FUNC_COALESCE ){
+ int endCoalesce = sqlite3VdbeMakeLabel(v);
+ assert( nFarg>=2 );
+ sqlite3ExprCode(pParse, pFarg->a[0].pExpr, target);
+ for(i=1; i<nFarg; i++){
+ sqlite3VdbeAddOp2(v, OP_NotNull, target, endCoalesce);
+ sqlite3ExprCacheRemove(pParse, target);
+ sqlite3ExprCachePush(pParse);
+ sqlite3ExprCode(pParse, pFarg->a[i].pExpr, target);
+ sqlite3ExprCachePop(pParse, 1);
+ }
+ sqlite3VdbeResolveLabel(v, endCoalesce);
+ break;
+ }
+
+
if( pFarg ){
r1 = sqlite3GetTempRange(pParse, nFarg);
sqlite3ExprCachePush(pParse); /* Ticket 2ea2425d34be */
break;
}
case TK_IN: {
- int rNotFound = 0;
- int rMayHaveNull = 0;
- int j2, j3, j4, j5;
- char affinity;
- int eType;
-
- VdbeNoopComment((v, "begin IN expr r%d", target));
- eType = sqlite3FindInIndex(pParse, pExpr, &rMayHaveNull);
- if( rMayHaveNull ){
- rNotFound = ++pParse->nMem;
- }
-
- /* Figure out the affinity to use to create a key from the results
- ** of the expression. affinityStr stores a static string suitable for
- ** P4 of OP_MakeRecord.
- */
- affinity = comparisonAffinity(pExpr);
-
-
- /* Code the <expr> from "<expr> IN (...)". The temporary table
- ** pExpr->iTable contains the values that make up the (...) set.
- */
- sqlite3ExprCachePush(pParse);
- sqlite3ExprCode(pParse, pExpr->pLeft, target);
- j2 = sqlite3VdbeAddOp1(v, OP_IsNull, target);
- if( eType==IN_INDEX_ROWID ){
- j3 = sqlite3VdbeAddOp1(v, OP_MustBeInt, target);
- j4 = sqlite3VdbeAddOp3(v, OP_NotExists, pExpr->iTable, 0, target);
- sqlite3VdbeAddOp2(v, OP_Integer, 1, target);
- j5 = sqlite3VdbeAddOp0(v, OP_Goto);
- sqlite3VdbeJumpHere(v, j3);
- sqlite3VdbeJumpHere(v, j4);
- sqlite3VdbeAddOp2(v, OP_Integer, 0, target);
- }else{
- r2 = regFree2 = sqlite3GetTempReg(pParse);
-
- /* Create a record and test for set membership. If the set contains
- ** the value, then jump to the end of the test code. The target
- ** register still contains the true (1) value written to it earlier.
- */
- sqlite3VdbeAddOp4(v, OP_MakeRecord, target, 1, r2, &affinity, 1);
- sqlite3VdbeAddOp2(v, OP_Integer, 1, target);
- j5 = sqlite3VdbeAddOp3(v, OP_Found, pExpr->iTable, 0, r2);
-
- /* If the set membership test fails, then the result of the
- ** "x IN (...)" expression must be either 0 or NULL. If the set
- ** contains no NULL values, then the result is 0. If the set
- ** contains one or more NULL values, then the result of the
- ** expression is also NULL.
- */
- if( rNotFound==0 ){
- /* This branch runs if it is known at compile time (now) that
- ** the set contains no NULL values. This happens as the result
- ** of a "NOT NULL" constraint in the database schema. No need
- ** to test the data structure at runtime in this case.
- */
- sqlite3VdbeAddOp2(v, OP_Integer, 0, target);
- }else{
- /* This block populates the rNotFound register with either NULL
- ** or 0 (an integer value). If the data structure contains one
- ** or more NULLs, then set rNotFound to NULL. Otherwise, set it
- ** to 0. If register rMayHaveNull is already set to some value
- ** other than NULL, then the test has already been run and
- ** rNotFound is already populated.
- */
- static const char nullRecord[] = { 0x02, 0x00 };
- j3 = sqlite3VdbeAddOp1(v, OP_NotNull, rMayHaveNull);
- sqlite3VdbeAddOp2(v, OP_Null, 0, rNotFound);
- sqlite3VdbeAddOp4(v, OP_Blob, 2, rMayHaveNull, 0,
- nullRecord, P4_STATIC);
- j4 = sqlite3VdbeAddOp3(v, OP_Found, pExpr->iTable, 0, rMayHaveNull);
- sqlite3VdbeAddOp2(v, OP_Integer, 0, rNotFound);
- sqlite3VdbeJumpHere(v, j4);
- sqlite3VdbeJumpHere(v, j3);
-
- /* Copy the value of register rNotFound (which is either NULL or 0)
- ** into the target register. This will be the result of the
- ** expression.
- */
- sqlite3VdbeAddOp2(v, OP_Copy, rNotFound, target);
- }
- }
- sqlite3VdbeJumpHere(v, j2);
- sqlite3VdbeJumpHere(v, j5);
- sqlite3ExprCachePop(pParse, 1);
- VdbeComment((v, "end IN expr r%d", target));
+ int destIfFalse = sqlite3VdbeMakeLabel(v);
+ int destIfNull = sqlite3VdbeMakeLabel(v);
+ sqlite3VdbeAddOp2(v, OP_Null, 0, target);
+ sqlite3ExprCodeIN(pParse, pExpr, destIfFalse, destIfNull);
+ sqlite3VdbeAddOp2(v, OP_Integer, 1, target);
+ sqlite3VdbeResolveLabel(v, destIfFalse);
+ sqlite3VdbeAddOp2(v, OP_AddImm, target, 0);
+ sqlite3VdbeResolveLabel(v, destIfNull);
break;
}
-#endif
+#endif /* SQLITE_OMIT_SUBQUERY */
+
+
/*
** x BETWEEN y AND z
**
return n;
}
+/*
+** Generate code for a BETWEEN operator.
+**
+** x BETWEEN y AND z
+**
+** The above is equivalent to
+**
+** x>=y AND x<=z
+**
+** Code it as such, taking care to do the common subexpression
+** elementation of x.
+*/
+static void exprCodeBetween(
+ Parse *pParse, /* Parsing and code generating context */
+ Expr *pExpr, /* The BETWEEN expression */
+ int dest, /* Jump here if the jump is taken */
+ int jumpIfTrue, /* Take the jump if the BETWEEN is true */
+ int jumpIfNull /* Take the jump if the BETWEEN is NULL */
+){
+ Expr exprAnd; /* The AND operator in x>=y AND x<=z */
+ Expr compLeft; /* The x>=y term */
+ Expr compRight; /* The x<=z term */
+ Expr exprX; /* The x subexpression */
+ int regFree1 = 0; /* Temporary use register */
+
+ assert( !ExprHasProperty(pExpr, EP_xIsSelect) );
+ exprX = *pExpr->pLeft;
+ exprAnd.op = TK_AND;
+ exprAnd.pLeft = &compLeft;
+ exprAnd.pRight = &compRight;
+ compLeft.op = TK_GE;
+ compLeft.pLeft = &exprX;
+ compLeft.pRight = pExpr->x.pList->a[0].pExpr;
+ compRight.op = TK_LE;
+ compRight.pLeft = &exprX;
+ compRight.pRight = pExpr->x.pList->a[1].pExpr;
+ exprX.iTable = sqlite3ExprCodeTemp(pParse, &exprX, ®Free1);
+ exprX.op = TK_REGISTER;
+ if( jumpIfTrue ){
+ sqlite3ExprIfTrue(pParse, &exprAnd, dest, jumpIfNull);
+ }else{
+ sqlite3ExprIfFalse(pParse, &exprAnd, dest, jumpIfNull);
+ }
+ sqlite3ReleaseTempReg(pParse, regFree1);
+
+ /* Ensure adequate test coverage */
+ testcase( jumpIfTrue==0 && jumpIfNull==0 && regFree1==0 );
+ testcase( jumpIfTrue==0 && jumpIfNull==0 && regFree1!=0 );
+ testcase( jumpIfTrue==0 && jumpIfNull!=0 && regFree1==0 );
+ testcase( jumpIfTrue==0 && jumpIfNull!=0 && regFree1!=0 );
+ testcase( jumpIfTrue!=0 && jumpIfNull==0 && regFree1==0 );
+ testcase( jumpIfTrue!=0 && jumpIfNull==0 && regFree1!=0 );
+ testcase( jumpIfTrue!=0 && jumpIfNull!=0 && regFree1==0 );
+ testcase( jumpIfTrue!=0 && jumpIfNull!=0 && regFree1!=0 );
+}
+
/*
** Generate code for a boolean expression such that a jump is made
** to the label "dest" if the expression is true but execution
break;
}
case TK_BETWEEN: {
- /* x BETWEEN y AND z
- **
- ** Is equivalent to
- **
- ** x>=y AND x<=z
- **
- ** Code it as such, taking care to do the common subexpression
- ** elementation of x.
- */
- Expr exprAnd;
- Expr compLeft;
- Expr compRight;
- Expr exprX;
-
- assert( !ExprHasProperty(pExpr, EP_xIsSelect) );
- exprX = *pExpr->pLeft;
- exprAnd.op = TK_AND;
- exprAnd.pLeft = &compLeft;
- exprAnd.pRight = &compRight;
- compLeft.op = TK_GE;
- compLeft.pLeft = &exprX;
- compLeft.pRight = pExpr->x.pList->a[0].pExpr;
- compRight.op = TK_LE;
- compRight.pLeft = &exprX;
- compRight.pRight = pExpr->x.pList->a[1].pExpr;
- exprX.iTable = sqlite3ExprCodeTemp(pParse, &exprX, ®Free1);
- testcase( regFree1==0 );
- exprX.op = TK_REGISTER;
testcase( jumpIfNull==0 );
- sqlite3ExprIfTrue(pParse, &exprAnd, dest, jumpIfNull);
+ exprCodeBetween(pParse, pExpr, dest, 1, jumpIfNull);
+ break;
+ }
+ case TK_IN: {
+ int destIfFalse = sqlite3VdbeMakeLabel(v);
+ int destIfNull = jumpIfNull ? dest : destIfFalse;
+ sqlite3ExprCodeIN(pParse, pExpr, destIfFalse, destIfNull);
+ sqlite3VdbeAddOp2(v, OP_Goto, 0, dest);
+ sqlite3VdbeResolveLabel(v, destIfFalse);
break;
}
default: {
break;
}
case TK_NOT: {
+ testcase( jumpIfNull==0 );
sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull);
break;
}
break;
}
case TK_BETWEEN: {
- /* x BETWEEN y AND z
- **
- ** Is equivalent to
- **
- ** x>=y AND x<=z
- **
- ** Code it as such, taking care to do the common subexpression
- ** elementation of x.
- */
- Expr exprAnd;
- Expr compLeft;
- Expr compRight;
- Expr exprX;
-
- assert( !ExprHasProperty(pExpr, EP_xIsSelect) );
- exprX = *pExpr->pLeft;
- exprAnd.op = TK_AND;
- exprAnd.pLeft = &compLeft;
- exprAnd.pRight = &compRight;
- compLeft.op = TK_GE;
- compLeft.pLeft = &exprX;
- compLeft.pRight = pExpr->x.pList->a[0].pExpr;
- compRight.op = TK_LE;
- compRight.pLeft = &exprX;
- compRight.pRight = pExpr->x.pList->a[1].pExpr;
- exprX.iTable = sqlite3ExprCodeTemp(pParse, &exprX, ®Free1);
- testcase( regFree1==0 );
- exprX.op = TK_REGISTER;
testcase( jumpIfNull==0 );
- sqlite3ExprIfFalse(pParse, &exprAnd, dest, jumpIfNull);
+ exprCodeBetween(pParse, pExpr, dest, 0, jumpIfNull);
+ break;
+ }
+ case TK_IN: {
+ if( jumpIfNull ){
+ sqlite3ExprCodeIN(pParse, pExpr, dest, dest);
+ }else{
+ int destIfNull = sqlite3VdbeMakeLabel(v);
+ sqlite3ExprCodeIN(pParse, pExpr, dest, destIfNull);
+ sqlite3VdbeResolveLabel(v, destIfNull);
+ }
break;
}
default: {
*************************************************************************
** This file contains C code routines that used to generate VDBE code
** that implements the ALTER TABLE command.
-**
-** $Id$
*/
/*
**
*************************************************************************
** This file contains code associated with the ANALYZE command.
-**
-** @(#) $Id$
*/
#ifndef SQLITE_OMIT_ANALYZE
**
*************************************************************************
** This file contains code used to implement the ATTACH and DETACH commands.
-**
-** $Id$
*/
#ifndef SQLITE_OMIT_ATTACH
** API. This facility is an optional feature of the library. Embedded
** systems that do not need this facility may omit it by recompiling
** the library with -DSQLITE_OMIT_AUTHORIZATION=1
-**
-** $Id$
*/
/*
** BEGIN TRANSACTION
** COMMIT
** ROLLBACK
-**
-** $Id$
*/
/*
**
** This file contains functions used to access the internal hash tables
** of user defined functions and collation sequences.
-**
-** $Id$
*/
*************************************************************************
** This file contains C code routines that are called by the parser
** in order to generate code for DELETE FROM statements.
-**
-** $Id$
*/
/*
/* TODO: Could use temporary registers here. Also could attempt to
** avoid copying the contents of the rowid register. */
- mask = sqlite3TriggerOldmask(pParse, pTrigger, 0, pTab, onconf);
+ mask = sqlite3TriggerColmask(
+ pParse, pTrigger, 0, 0, TRIGGER_BEFORE|TRIGGER_AFTER, pTab, onconf
+ );
mask |= sqlite3FkOldmask(pParse, pTab);
iOld = pParse->nMem+1;
pParse->nMem += (1 + pTab->nCol);
return regBase;
}
-
/************** End of delete.c **********************************************/
/************** Begin file func.c ********************************************/
/*
** If x is a blob, then we count bytes.
**
** If p1 is negative, then we begin abs(p1) from the end of x[].
+**
+** If p2 is negative, return the p2 characters preceeding p1.
*/
static void substrFunc(
sqlite3_context *context,
return;
}
p0type = sqlite3_value_type(argv[0]);
+ p1 = sqlite3_value_int(argv[1]);
if( p0type==SQLITE_BLOB ){
len = sqlite3_value_bytes(argv[0]);
z = sqlite3_value_blob(argv[0]);
z = sqlite3_value_text(argv[0]);
if( z==0 ) return;
len = 0;
- for(z2=z; *z2; len++){
- SQLITE_SKIP_UTF8(z2);
+ if( p1<0 ){
+ for(z2=z; *z2; len++){
+ SQLITE_SKIP_UTF8(z2);
+ }
}
}
- p1 = sqlite3_value_int(argv[1]);
if( argc==3 ){
p2 = sqlite3_value_int(argv[2]);
if( p2<0 ){
}
}
assert( p1>=0 && p2>=0 );
- if( p1+p2>len ){
- p2 = len-p1;
- if( p2<0 ) p2 = 0;
- }
if( p0type!=SQLITE_BLOB ){
while( *z && p1 ){
SQLITE_SKIP_UTF8(z);
}
sqlite3_result_text(context, (char*)z, (int)(z2-z), SQLITE_TRANSIENT);
}else{
+ if( p1+p2>len ){
+ p2 = len-p1;
+ if( p2<0 ) p2 = 0;
+ }
sqlite3_result_blob(context, (char*)&z[p1], (int)p2, SQLITE_TRANSIENT);
}
}
}
}
+
+#if 0 /* This function is never used. */
+/*
+** The COALESCE() and IFNULL() functions used to be implemented as shown
+** here. But now they are implemented as VDBE code so that unused arguments
+** do not have to be computed. This legacy implementation is retained as
+** comment.
+*/
/*
** Implementation of the IFNULL(), NVL(), and COALESCE() functions.
** All three do the same thing. They return the first non-NULL
}
}
}
+#endif /* NOT USED */
+#define ifnullFunc versionFunc /* Substitute function - never called */
/*
** Implementation of random(). Return a random integer.
FUNCTION(upper, 1, 0, 0, upperFunc ),
FUNCTION(lower, 1, 0, 0, lowerFunc ),
FUNCTION(coalesce, 1, 0, 0, 0 ),
- FUNCTION(coalesce, -1, 0, 0, ifnullFunc ),
FUNCTION(coalesce, 0, 0, 0, 0 ),
+/* FUNCTION(coalesce, -1, 0, 0, ifnullFunc ), */
+ {-1,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"coalesce",0},
FUNCTION(hex, 1, 0, 0, hexFunc ),
- FUNCTION(ifnull, 2, 0, 1, ifnullFunc ),
+/* FUNCTION(ifnull, 2, 0, 0, ifnullFunc ), */
+ {2,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"ifnull",0},
FUNCTION(random, 0, 0, 0, randomFunc ),
FUNCTION(randomblob, 1, 0, 0, randomBlob ),
FUNCTION(nullif, 2, 0, 1, nullifFunc ),
sqlite3VdbeAddOp3(v, OP_MakeRecord, regTemp, nCol, regRec);
sqlite3VdbeChangeP4(v, -1, sqlite3IndexAffinityStr(v, pIdx), 0);
- sqlite3VdbeAddOp3(v, OP_Found, iCur, iOk, regRec);
+ sqlite3VdbeAddOp4Int(v, OP_Found, iCur, iOk, regRec, 0);
sqlite3ReleaseTempReg(pParse, regRec);
sqlite3ReleaseTempRange(pParse, regTemp, nCol);
*************************************************************************
** This file contains C code routines that are called by the parser
** to handle INSERT statements in SQLite.
-**
-** $Id$
*/
/*
** implement the programmer interface to the library. Routines in
** other files are for internal use by SQLite and should not be
** accessed by users of the library.
-**
-** $Id$
*/
*************************************************************************
** This file contains code used to dynamically load extensions into
** the SQLite library.
-**
-** $Id$
*/
#ifndef SQLITE_CORE
** an SQLite instance. Shared libraries that intend to be loaded
** as extensions by SQLite should #include this file instead of
** sqlite3.h.
-**
-** @(#) $Id$
*/
#ifndef _SQLITE3EXT_H_
#define _SQLITE3EXT_H_
**
*************************************************************************
** This file contains code used to implement the PRAGMA command.
-**
-** $Id$
*/
/* Ignore this whole file if pragmas are disabled
sqlite3VdbeAddOp2(v, OP_AddImm, 2, 1); /* increment entry count */
for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){
int jmp2;
+ int r1;
static const VdbeOpList idxErr[] = {
{ OP_AddImm, 1, -1, 0},
{ OP_String8, 0, 3, 0}, /* 1 */
{ OP_IfPos, 1, 0, 0}, /* 9 */
{ OP_Halt, 0, 0, 0},
};
- sqlite3GenerateIndexKey(pParse, pIdx, 1, 3, 1);
- jmp2 = sqlite3VdbeAddOp3(v, OP_Found, j+2, 0, 3);
+ r1 = sqlite3GenerateIndexKey(pParse, pIdx, 1, 3, 0);
+ jmp2 = sqlite3VdbeAddOp4Int(v, OP_Found, j+2, 0, r1, pIdx->nColumn+1);
addr = sqlite3VdbeAddOpList(v, ArraySize(idxErr), idxErr);
sqlite3VdbeChangeP4(v, addr+1, "rowid ", P4_STATIC);
sqlite3VdbeChangeP4(v, addr+3, " missing from index ", P4_STATIC);
** This file contains the implementation of the sqlite3_prepare()
** interface, and routines that contribute to loading the database schema
** from disk.
-**
-** $Id$
*/
/*
*************************************************************************
** This file contains C code routines that are called by the parser
** to handle SELECT statements in SQLite.
-**
-** $Id$
*/
v = pParse->pVdbe;
r1 = sqlite3GetTempReg(pParse);
+ sqlite3VdbeAddOp4Int(v, OP_Found, iTab, addrRepeat, iMem, N);
sqlite3VdbeAddOp3(v, OP_MakeRecord, iMem, N, r1);
- sqlite3VdbeAddOp3(v, OP_Found, iTab, addrRepeat, r1);
sqlite3VdbeAddOp2(v, OP_IdxInsert, iTab, r1);
sqlite3ReleaseTempReg(pParse, r1);
}
if( p->iLimit ){
assert( pOrderBy==0 ); /* If there is an ORDER BY, the call to
** pushOntoSorter() would have cleared p->iLimit */
- sqlite3VdbeAddOp2(v, OP_AddImm, p->iLimit, -1);
- sqlite3VdbeAddOp2(v, OP_IfZero, p->iLimit, iBreak);
+ sqlite3VdbeAddOp3(v, OP_IfZero, p->iLimit, iBreak, -1);
}
}
Vdbe *v = 0;
int iLimit = 0;
int iOffset;
- int addr1;
+ int addr1, n;
if( p->iLimit ) return;
/*
p->iLimit = iLimit = ++pParse->nMem;
v = sqlite3GetVdbe(pParse);
if( NEVER(v==0) ) return; /* VDBE should have already been allocated */
- sqlite3ExprCode(pParse, p->pLimit, iLimit);
- sqlite3VdbeAddOp1(v, OP_MustBeInt, iLimit);
- VdbeComment((v, "LIMIT counter"));
- sqlite3VdbeAddOp2(v, OP_IfZero, iLimit, iBreak);
+ if( sqlite3ExprIsInteger(p->pLimit, &n) ){
+ sqlite3VdbeAddOp2(v, OP_Integer, n, iLimit);
+ VdbeComment((v, "LIMIT counter"));
+ if( n==0 ){
+ sqlite3VdbeAddOp2(v, OP_Goto, 0, iBreak);
+ }
+ }else{
+ sqlite3ExprCode(pParse, p->pLimit, iLimit);
+ sqlite3VdbeAddOp1(v, OP_MustBeInt, iLimit);
+ VdbeComment((v, "LIMIT counter"));
+ sqlite3VdbeAddOp2(v, OP_IfZero, iLimit, iBreak);
+ }
if( p->pOffset ){
p->iOffset = iOffset = ++pParse->nMem;
pParse->nMem++; /* Allocate an extra register for limit+offset */
sqlite3VdbeAddOp2(v, OP_Rewind, tab1, iBreak);
r1 = sqlite3GetTempReg(pParse);
iStart = sqlite3VdbeAddOp2(v, OP_RowKey, tab1, r1);
- sqlite3VdbeAddOp3(v, OP_NotFound, tab2, iCont, r1);
+ sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0);
sqlite3ReleaseTempReg(pParse, r1);
selectInnerLoop(pParse, p, p->pEList, tab1, p->pEList->nExpr,
0, -1, &dest, iCont, iBreak);
/* Jump to the end of the loop if the LIMIT is reached.
*/
if( p->iLimit ){
- sqlite3VdbeAddOp2(v, OP_AddImm, p->iLimit, -1);
- sqlite3VdbeAddOp2(v, OP_IfZero, p->iLimit, iBreak);
+ sqlite3VdbeAddOp3(v, OP_IfZero, p->iLimit, iBreak, -1);
}
/* Generate the subroutine return
**
** These routines are in a separate files so that they will not be linked
** if they are not used.
-**
-** $Id$
*/
#ifndef SQLITE_OMIT_GET_TABLE
** May you share freely, never taking more than you give.
**
*************************************************************************
-**
-**
-** $Id$
+** This file contains the implementation for TRIGGERs
*/
#ifndef SQLITE_OMIT_TRIGGER
pProgram->nRef = 1;
pPrg->pTrigger = pTrigger;
pPrg->orconf = orconf;
- pPrg->oldmask = 0xffffffff;
+ pPrg->aColmask[0] = 0xffffffff;
+ pPrg->aColmask[1] = 0xffffffff;
/* Allocate and populate a new Parse context to use for coding the
** trigger sub-program. */
pProgram->nMem = pSubParse->nMem;
pProgram->nCsr = pSubParse->nTab;
pProgram->token = (void *)pTrigger;
- pPrg->oldmask = pSubParse->oldmask;
+ pPrg->aColmask[0] = pSubParse->oldmask;
+ pPrg->aColmask[1] = pSubParse->newmask;
sqlite3VdbeDelete(v);
}
}
/*
-** Triggers fired by UPDATE or DELETE statements may access values stored
-** in the old.* pseudo-table. This function returns a 32-bit bitmask
-** indicating which columns of the old.* table actually are used by
-** triggers. This information may be used by the caller to avoid having
-** to load the entire old.* record into memory when executing an UPDATE
-** or DELETE command.
+** Triggers may access values stored in the old.* or new.* pseudo-table.
+** This function returns a 32-bit bitmask indicating which columns of the
+** old.* or new.* tables actually are used by triggers. This information
+** may be used by the caller, for example, to avoid having to load the entire
+** old.* record into memory when executing an UPDATE or DELETE command.
**
** Bit 0 of the returned mask is set if the left-most column of the
-** table may be accessed using an old.<col> reference. Bit 1 is set if
+** table may be accessed using an [old|new].<col> reference. Bit 1 is set if
** the second leftmost column value is required, and so on. If there
** are more than 32 columns in the table, and at least one of the columns
** with an index greater than 32 may be accessed, 0xffffffff is returned.
**
-** It is not possible to determine if the old.rowid column is accessed
-** by triggers. The caller must always assume that it is.
+** It is not possible to determine if the old.rowid or new.rowid column is
+** accessed by triggers. The caller must always assume that it is.
+**
+** Parameter isNew must be either 1 or 0. If it is 0, then the mask returned
+** applies to the old.* table. If 1, the new.* table.
**
-** There is no equivalent function for new.* references.
+** Parameter tr_tm must be a mask with one or both of the TRIGGER_BEFORE
+** and TRIGGER_AFTER bits set. Values accessed by BEFORE triggers are only
+** included in the returned mask if the TRIGGER_BEFORE bit is set in the
+** tr_tm parameter. Similarly, values accessed by AFTER triggers are only
+** included in the returned mask if the TRIGGER_AFTER bit is set in tr_tm.
*/
-SQLITE_PRIVATE u32 sqlite3TriggerOldmask(
+SQLITE_PRIVATE u32 sqlite3TriggerColmask(
Parse *pParse, /* Parse context */
Trigger *pTrigger, /* List of triggers on table pTab */
ExprList *pChanges, /* Changes list for any UPDATE OF triggers */
+ int isNew, /* 1 for new.* ref mask, 0 for old.* ref mask */
+ int tr_tm, /* Mask of TRIGGER_BEFORE|TRIGGER_AFTER */
Table *pTab, /* The table to code triggers from */
int orconf /* Default ON CONFLICT policy for trigger steps */
){
u32 mask = 0;
Trigger *p;
+ assert( isNew==1 || isNew==0 );
for(p=pTrigger; p; p=p->pNext){
- if( p->op==op && checkColumnOverlap(p->pColumns,pChanges) ){
+ if( p->op==op && (tr_tm&p->tr_tm)
+ && checkColumnOverlap(p->pColumns,pChanges)
+ ){
TriggerPrg *pPrg;
pPrg = getRowTrigger(pParse, p, pTab, orconf);
if( pPrg ){
- mask |= pPrg->oldmask;
+ mask |= pPrg->aColmask[isNew];
}
}
}
*************************************************************************
** This file contains C code routines that are called by the parser
** to handle UPDATE statements.
-**
-** $Id$
*/
#ifndef SQLITE_OMIT_VIRTUALTABLE
AuthContext sContext; /* The authorization context */
NameContext sNC; /* The name-context to resolve expressions in */
int iDb; /* Database containing the table being updated */
- int j1; /* Addresses of jump instructions */
int okOnePass; /* True for one-pass algorithm without the FIFO */
int hasFK; /* True if foreign key processing is required */
#ifndef SQLITE_OMIT_TRIGGER
- int isView; /* Trying to update a view */
- Trigger *pTrigger; /* List of triggers on pTab, if required */
+ int isView; /* True when updating a view (INSTEAD OF trigger) */
+ Trigger *pTrigger; /* List of triggers on pTab, if required */
+ int tmask; /* Mask of TRIGGER_BEFORE|TRIGGER_AFTER */
#endif
+ int newmask; /* Mask of NEW.* columns accessed by BEFORE triggers */
/* Register Allocations */
int regRowCount = 0; /* A count of rows changed */
** updated is a view.
*/
#ifndef SQLITE_OMIT_TRIGGER
- pTrigger = sqlite3TriggersExist(pParse, pTab, TK_UPDATE, pChanges, 0);
+ pTrigger = sqlite3TriggersExist(pParse, pTab, TK_UPDATE, pChanges, &tmask);
isView = pTab->pSelect!=0;
+ assert( pTrigger || tmask==0 );
#else
# define pTrigger 0
# define isView 0
+# define tmask 0
#endif
#ifdef SQLITE_OMIT_VIEW
# undef isView
if( sqlite3ViewGetColumnNames(pParse, pTab) ){
goto update_cleanup;
}
- if( sqlite3IsReadOnly(pParse, pTab, (pTrigger?1:0)) ){
+ if( sqlite3IsReadOnly(pParse, pTab, tmask) ){
goto update_cleanup;
}
aXRef = sqlite3DbMallocRaw(db, sizeof(int) * pTab->nCol );
** with the required old.* column data. */
if( hasFK || pTrigger ){
u32 oldmask = (hasFK ? sqlite3FkOldmask(pParse, pTab) : 0);
- oldmask |= sqlite3TriggerOldmask(pParse, pTrigger, pChanges, pTab, onError);
+ oldmask |= sqlite3TriggerColmask(pParse,
+ pTrigger, pChanges, 0, TRIGGER_BEFORE|TRIGGER_AFTER, pTab, onError
+ );
for(i=0; i<pTab->nCol; i++){
if( aXRef[i]<0 || oldmask==0xffffffff || (oldmask & (1<<i)) ){
sqlite3VdbeAddOp3(v, OP_Column, iCur, i, regOld+i);
/* Populate the array of registers beginning at regNew with the new
** row data. This array is used to check constaints, create the new
** table and index records, and as the values for any new.* references
- ** made by triggers. */
+ ** made by triggers.
+ **
+ ** If there are one or more BEFORE triggers, then do not populate the
+ ** registers associated with columns that are (a) not modified by
+ ** this UPDATE statement and (b) not accessed by new.* references. The
+ ** values for registers not modified by the UPDATE must be reloaded from
+ ** the database after the BEFORE triggers are fired anyway (as the trigger
+ ** may have modified them). So not loading those that are not going to
+ ** be used eliminates some redundant opcodes.
+ */
+ newmask = sqlite3TriggerColmask(
+ pParse, pTrigger, pChanges, 1, TRIGGER_BEFORE, pTab, onError
+ );
for(i=0; i<pTab->nCol; i++){
if( i==pTab->iPKey ){
sqlite3VdbeAddOp2(v, OP_Null, 0, regNew+i);
}else{
j = aXRef[i];
- if( j<0 ){
+ if( j>=0 ){
+ sqlite3ExprCode(pParse, pChanges->a[j].pExpr, regNew+i);
+ }else if( 0==(tmask&TRIGGER_BEFORE) || i>31 || (newmask&(1<<i)) ){
+ /* This branch loads the value of a column that will not be changed
+ ** into a register. This is done if there are no BEFORE triggers, or
+ ** if there are one or more BEFORE triggers that use this value via
+ ** a new.* reference in a trigger program.
+ */
+ testcase( i==31 );
+ testcase( i==32 );
sqlite3VdbeAddOp3(v, OP_Column, iCur, i, regNew+i);
sqlite3ColumnDefault(v, pTab, i, regNew+i);
- }else{
- sqlite3ExprCode(pParse, pChanges->a[j].pExpr, regNew+i);
}
}
}
/* Fire any BEFORE UPDATE triggers. This happens before constraints are
- ** verified. One could argue that this is wrong. */
- if( pTrigger ){
+ ** verified. One could argue that this is wrong.
+ */
+ if( tmask&TRIGGER_BEFORE ){
sqlite3VdbeAddOp2(v, OP_Affinity, regNew, pTab->nCol);
sqlite3TableAffinityStr(v, pTab);
sqlite3CodeRowTrigger(pParse, pTrigger, TK_UPDATE, pChanges,
** case, jump to the next row. No updates or AFTER triggers are
** required. This behaviour - what happens when the row being updated
** is deleted or renamed by a BEFORE trigger - is left undefined in the
- ** documentation. */
+ ** documentation.
+ */
sqlite3VdbeAddOp3(v, OP_NotExists, iCur, addr, regOldRowid);
+
+ /* If it did not delete it, the row-trigger may still have modified
+ ** some of the columns of the row being updated. Load the values for
+ ** all columns not modified by the update statement into their
+ ** registers in case this has happened.
+ */
+ for(i=0; i<pTab->nCol; i++){
+ if( aXRef[i]<0 && i!=pTab->iPKey ){
+ sqlite3VdbeAddOp3(v, OP_Column, iCur, i, regNew+i);
+ sqlite3ColumnDefault(v, pTab, i, regNew+i);
+ }
+ }
}
if( !isView ){
+ int j1; /* Address of jump instruction */
/* Do constraint checks. */
sqlite3GenerateConstraintChecks(pParse, pTab, iCur, regNewRowid,
**
** Most of the code in this file may be omitted by defining the
** SQLITE_OMIT_VACUUM macro.
-**
-** $Id$
*/
#if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH)
**
*************************************************************************
** This file contains code used to help implement virtual tables.
-**
-** $Id$
*/
#ifndef SQLITE_OMIT_VIRTUALTABLE
** rows. Indices are selected and used to speed the search when doing
** so is applicable. Because this module is responsible for selecting
** indices, you might also think of this module as the "query optimizer".
-**
-** $Id$
*/
/*
** Code an OP_Affinity opcode to apply the column affinity string zAff
** to the n registers starting at base.
**
-** Buffer zAff was allocated using sqlite3DbMalloc(). It is the
-** responsibility of this function to arrange for it to be eventually
-** freed using sqlite3DbFree().
+** As an optimization, SQLITE_AFF_NONE entries (which are no-ops) at the
+** beginning and end of zAff are ignored. If all entries in zAff are
+** SQLITE_AFF_NONE, then no code gets generated.
+**
+** This routine makes its own copy of zAff so that the caller is free
+** to modify zAff after this routine returns.
*/
static void codeApplyAffinity(Parse *pParse, int base, int n, char *zAff){
Vdbe *v = pParse->pVdbe;
+ if( zAff==0 ){
+ assert( pParse->db->mallocFailed );
+ return;
+ }
assert( v!=0 );
- sqlite3VdbeAddOp2(v, OP_Affinity, base, n);
- sqlite3VdbeChangeP4(v, -1, zAff, P4_DYNAMIC);
- sqlite3ExprCacheAffinityChange(pParse, base, n);
+
+ /* Adjust base and n to skip over SQLITE_AFF_NONE entries at the beginning
+ ** and end of the affinity string.
+ */
+ while( n>0 && zAff[0]==SQLITE_AFF_NONE ){
+ n--;
+ base++;
+ zAff++;
+ }
+ while( n>1 && zAff[n-1]==SQLITE_AFF_NONE ){
+ n--;
+ }
+
+ /* Code the OP_Affinity opcode if there is anything left to do. */
+ if( n>0 ){
+ sqlite3VdbeAddOp2(v, OP_Affinity, base, n);
+ sqlite3VdbeChangeP4(v, -1, zAff, n);
+ sqlite3ExprCacheAffinityChange(pParse, base, n);
+ }
}
/*
** Generate code that will evaluate all == and IN constraints for an
-** index. The values for all constraints are left on the stack.
+** index.
**
** For example, consider table t1(a,b,c,d,e,f) with index i1(a,b,c).
** Suppose the WHERE clause is this: a==5 AND b IN (1,2,3) AND c>5 AND c<10
**
** In the example above nEq==2. But this subroutine works for any value
** of nEq including 0. If nEq==0, this routine is nearly a no-op.
-** The only thing it does is allocate the pLevel->iMem memory cell.
+** The only thing it does is allocate the pLevel->iMem memory cell and
+** compute the affinity string.
**
** This routine always allocates at least one memory cell and returns
** the index of that memory cell. The code that
testcase( pTerm->eOperator & WO_ISNULL );
testcase( pTerm->eOperator & WO_IN );
if( (pTerm->eOperator & (WO_ISNULL|WO_IN))==0 ){
- sqlite3VdbeAddOp2(v, OP_IsNull, regBase+j, pLevel->addrBrk);
- if( zAff
- && sqlite3CompareAffinity(pTerm->pExpr->pRight, zAff[j])==SQLITE_AFF_NONE
- ){
- zAff[j] = SQLITE_AFF_NONE;
+ Expr *pRight = pTerm->pExpr->pRight;
+ sqlite3ExprCodeIsNullJump(v, pRight, regBase+j, pLevel->addrBrk);
+ if( zAff ){
+ if( sqlite3CompareAffinity(pRight, zAff[j])==SQLITE_AFF_NONE ){
+ zAff[j] = SQLITE_AFF_NONE;
+ }
+ if( sqlite3ExprNeedsNoAffinityChange(pRight, zAff[j]) ){
+ zAff[j] = SQLITE_AFF_NONE;
+ }
}
}
}
const struct sqlite3_index_constraint *aConstraint =
pVtabIdx->aConstraint;
+ sqlite3ExprCachePush(pParse);
iReg = sqlite3GetTempRange(pParse, nConstraint+2);
for(j=1; j<=nConstraint; j++){
for(k=0; k<nConstraint; k++){
pLevel->p1 = iCur;
pLevel->p2 = sqlite3VdbeCurrentAddr(v);
sqlite3ReleaseTempRange(pParse, iReg, nConstraint+2);
+ sqlite3ExprCachePop(pParse, 1);
}else
#endif /* SQLITE_OMIT_VIRTUALTABLE */
if( pRangeStart ){
Expr *pRight = pRangeStart->pExpr->pRight;
sqlite3ExprCode(pParse, pRight, regBase+nEq);
- sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt);
- if( zAff
- && sqlite3CompareAffinity(pRight, zAff[nConstraint])==SQLITE_AFF_NONE
- ){
- /* Since the comparison is to be performed with no conversions applied
- ** to the operands, set the affinity to apply to pRight to
- ** SQLITE_AFF_NONE. */
- zAff[nConstraint] = SQLITE_AFF_NONE;
- }
+ sqlite3ExprCodeIsNullJump(v, pRight, regBase+nEq, addrNxt);
+ if( zAff ){
+ if( sqlite3CompareAffinity(pRight, zAff[nConstraint])==SQLITE_AFF_NONE){
+ /* Since the comparison is to be performed with no conversions
+ ** applied to the operands, set the affinity to apply to pRight to
+ ** SQLITE_AFF_NONE. */
+ zAff[nConstraint] = SQLITE_AFF_NONE;
+ }
+ if( sqlite3ExprNeedsNoAffinityChange(pRight, zAff[nConstraint]) ){
+ zAff[nConstraint] = SQLITE_AFF_NONE;
+ }
+ }
nConstraint++;
}else if( isMinQuery ){
sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq);
testcase( op==OP_SeekGe );
testcase( op==OP_SeekLe );
testcase( op==OP_SeekLt );
- sqlite3VdbeAddOp4(v, op, iIdxCur, addrNxt, regBase,
- SQLITE_INT_TO_PTR(nConstraint), P4_INT32);
+ sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint);
/* Load the value for the inequality constraint at the end of the
** range (if any).
Expr *pRight = pRangeEnd->pExpr->pRight;
sqlite3ExprCacheRemove(pParse, regBase+nEq);
sqlite3ExprCode(pParse, pRight, regBase+nEq);
- sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt);
- zAff = sqlite3DbStrDup(pParse->db, zAff);
- if( zAff
- && sqlite3CompareAffinity(pRight, zAff[nConstraint])==SQLITE_AFF_NONE
- ){
- /* Since the comparison is to be performed with no conversions applied
- ** to the operands, set the affinity to apply to pRight to
- ** SQLITE_AFF_NONE. */
- zAff[nConstraint] = SQLITE_AFF_NONE;
- }
+ sqlite3ExprCodeIsNullJump(v, pRight, regBase+nEq, addrNxt);
+ if( zAff ){
+ if( sqlite3CompareAffinity(pRight, zAff[nConstraint])==SQLITE_AFF_NONE){
+ /* Since the comparison is to be performed with no conversions
+ ** applied to the operands, set the affinity to apply to pRight to
+ ** SQLITE_AFF_NONE. */
+ zAff[nConstraint] = SQLITE_AFF_NONE;
+ }
+ if( sqlite3ExprNeedsNoAffinityChange(pRight, zAff[nConstraint]) ){
+ zAff[nConstraint] = SQLITE_AFF_NONE;
+ }
+ }
codeApplyAffinity(pParse, regBase, nEq+1, zAff);
nConstraint++;
}
+ sqlite3DbFree(pParse->db, zAff);
/* Top of the loop body */
pLevel->p2 = sqlite3VdbeCurrentAddr(v);
testcase( op==OP_IdxGE );
testcase( op==OP_IdxLT );
if( op!=OP_Noop ){
- sqlite3VdbeAddOp4(v, op, iIdxCur, addrNxt, regBase,
- SQLITE_INT_TO_PTR(nConstraint), P4_INT32);
+ sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint);
sqlite3VdbeChangeP5(v, endEq!=bRev ?1:0);
}
int r;
r = sqlite3ExprCodeGetColumn(pParse, pTabItem->pTab, -1, iCur,
regRowid, 0);
- sqlite3VdbeAddOp4(v, OP_RowSetTest, regRowset,
- sqlite3VdbeCurrentAddr(v)+2,
- r, SQLITE_INT_TO_PTR(iSet), P4_INT32);
+ sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset,
+ sqlite3VdbeCurrentAddr(v)+2, r, iSet);
}
sqlite3VdbeAddOp2(v, OP_Gosub, regReturn, iLoopBody);
Bitmask b = pTabItem->colUsed;
int n = 0;
for(; b; b=b>>1, n++){}
- sqlite3VdbeChangeP4(v, sqlite3VdbeCurrentAddr(v)-1, SQLITE_INT_TO_PTR(n), P4_INT32);
+ sqlite3VdbeChangeP4(v, sqlite3VdbeCurrentAddr(v)-1,
+ SQLITE_INT_TO_PTR(n), P4_INT32);
assert( n<=pTab->nCol );
}
}else{
if( pLevel->iLeftJoin ){
int addr;
addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin);
- sqlite3VdbeAddOp1(v, OP_NullRow, pTabList->a[i].iCursor);
+ assert( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0
+ || (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 );
+ if( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0 ){
+ sqlite3VdbeAddOp1(v, OP_NullRow, pTabList->a[i].iCursor);
+ }
if( pLevel->iIdxCur>=0 ){
sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iIdxCur);
}
int k, j, last;
VdbeOp *pOp;
Index *pIdx = pLevel->plan.u.pIdx;
- int useIndexOnly = pLevel->plan.wsFlags & WHERE_IDX_ONLY;
assert( pIdx!=0 );
pOp = sqlite3VdbeGetOp(v, pWInfo->iTop);
break;
}
}
- assert(!useIndexOnly || j<pIdx->nColumn);
+ assert( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0
+ || j<pIdx->nColumn );
}else if( pOp->opcode==OP_Rowid ){
pOp->p1 = pLevel->iIdxCur;
pOp->opcode = OP_IdxRowid;
- }else if( pOp->opcode==OP_NullRow && useIndexOnly ){
- pOp->opcode = OP_Noop;
}
}
}
pOut->zEnd = &pPostOp->z[pPostOp->n];
}
+ /* A routine to convert a binary TK_IS or TK_ISNOT expression into a
+ ** unary TK_ISNULL or TK_NOTNULL expression. */
+ static void binaryToUnaryIfNull(Parse *pParse, Expr *pY, Expr *pA, int op){
+ sqlite3 *db = pParse->db;
+ if( db->mallocFailed==0 && pY->op==TK_NULL ){
+ pA->op = (u8)op;
+ sqlite3ExprDelete(db, pA->pRight);
+ pA->pRight = 0;
+ }
+ }
+
/* Construct an expression node for a unary prefix operator
*/
static void spanUnaryPrefix(
case 216: /* expr ::= expr IS expr */
{
spanBinaryExpr(&yygotominor.yy346,pParse,TK_IS,&yymsp[-2].minor.yy346,&yymsp[0].minor.yy346);
- if( pParse->db->mallocFailed==0 && yymsp[0].minor.yy346.pExpr->op==TK_NULL ){
- yygotominor.yy346.pExpr->op = TK_ISNULL;
- }
+ binaryToUnaryIfNull(pParse, yymsp[0].minor.yy346.pExpr, yygotominor.yy346.pExpr, TK_ISNULL);
}
break;
case 217: /* expr ::= expr IS NOT expr */
{
spanBinaryExpr(&yygotominor.yy346,pParse,TK_ISNOT,&yymsp[-3].minor.yy346,&yymsp[0].minor.yy346);
- if( pParse->db->mallocFailed==0 && yymsp[0].minor.yy346.pExpr->op==TK_NULL ){
- yygotominor.yy346.pExpr->op = TK_NOTNULL;
- }
+ binaryToUnaryIfNull(pParse, yymsp[0].minor.yy346.pExpr, yygotominor.yy346.pExpr, TK_NOTNULL);
}
break;
case 218: /* expr ::= NOT expr */
** This file contains C code that splits an SQL input string up into
** individual tokens and sends those tokens one-by-one over to the
** parser for analysis.
-**
-** $Id$
*/
/*
** But the feature is undocumented.
*/
#ifdef SQLITE_ASCII
-SQLITE_PRIVATE const char sqlite3IsAsciiIdChar[] = {
-/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
- 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
- 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
- 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
-};
-#define IdChar(C) (((c=C)&0x80)!=0 || (c>0x1f && sqlite3IsAsciiIdChar[c-0x20]))
+#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0)
#endif
#ifdef SQLITE_EBCDIC
SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[] = {
** This code used to be part of the tokenizer.c source file. But by
** separating it out, the code will be automatically omitted from
** static links that do not use it.
-**
-** $Id$
*/
#ifndef SQLITE_OMIT_COMPLETE
*/
#ifndef SQLITE_AMALGAMATION
#ifdef SQLITE_ASCII
-SQLITE_PRIVATE const char sqlite3IsAsciiIdChar[];
-#define IdChar(C) (((c=C)&0x80)!=0 || (c>0x1f && sqlite3IsAsciiIdChar[c-0x20]))
+#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0)
#endif
#ifdef SQLITE_EBCDIC
SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[];
break;
}
default: {
- int c;
+#ifdef SQLITE_EBCDIC
+ unsigned char c;
+#endif
if( IdChar((u8)*zSql) ){
/* Keywords and unquoted identifiers */
int nId;
**
** This file contains the implementation of the sqlite3_unlock_notify()
** API method and its associated functionality.
-**
-** $Id$
*/
/* Omit this entire file if SQLITE_ENABLE_UNLOCK_NOTIFY is not defined. */
# define SQLITE_CORE 1
#endif
-
-/************** Include fts3_expr.h in the middle of fts3.c ******************/
-/************** Begin file fts3_expr.h ***************************************/
+/************** Include fts3Int.h in the middle of fts3.c ********************/
+/************** Begin file fts3Int.h *****************************************/
/*
-** 2008 Nov 28
+** 2009 Nov 12
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
*/
-/************** Include fts3_tokenizer.h in the middle of fts3_expr.h ********/
+#ifndef _FTSINT_H
+#define _FTSINT_H
+
+#if !defined(NDEBUG) && !defined(SQLITE_DEBUG)
+# define NDEBUG 1
+#endif
+
+/************** Include fts3_tokenizer.h in the middle of fts3Int.h **********/
/************** Begin file fts3_tokenizer.h **********************************/
/*
** 2006 July 10
#endif /* _FTS3_TOKENIZER_H_ */
/************** End of fts3_tokenizer.h **************************************/
-/************** Continuing where we left off in fts3_expr.h ******************/
-
-/*
-** The following describes the syntax supported by the fts3 MATCH
-** operator in a similar format to that used by the lemon parser
-** generator. This module does not use actually lemon, it uses a
-** custom parser.
-**
-** query ::= andexpr (OR andexpr)*.
-**
-** andexpr ::= notexpr (AND? notexpr)*.
-**
-** notexpr ::= nearexpr (NOT nearexpr|-TOKEN)*.
-** notexpr ::= LP query RP.
-**
-** nearexpr ::= phrase (NEAR distance_opt nearexpr)*.
-**
-** distance_opt ::= .
-** distance_opt ::= / INTEGER.
-**
-** phrase ::= TOKEN.
-** phrase ::= COLUMN:TOKEN.
-** phrase ::= "TOKEN TOKEN TOKEN...".
-*/
-
-typedef struct Fts3Expr Fts3Expr;
-typedef struct Fts3Phrase Fts3Phrase;
-
-/*
-** A "phrase" is a sequence of one or more tokens that must match in
-** sequence. A single token is the base case and the most common case.
-** For a sequence of tokens contained in "...", nToken will be the number
-** of tokens in the string.
-*/
-struct Fts3Phrase {
- int nToken; /* Number of tokens in the phrase */
- int iColumn; /* Index of column this phrase must match */
- int isNot; /* Phrase prefixed by unary not (-) operator */
- struct PhraseToken {
- char *z; /* Text of the token */
- int n; /* Number of bytes in buffer pointed to by z */
- int isPrefix; /* True if token ends in with a "*" character */
- } aToken[1]; /* One entry for each token in the phrase */
-};
-
-/*
-** A tree of these objects forms the RHS of a MATCH operator.
-*/
-struct Fts3Expr {
- int eType; /* One of the FTSQUERY_XXX values defined below */
- int nNear; /* Valid if eType==FTSQUERY_NEAR */
- Fts3Expr *pParent; /* pParent->pLeft==this or pParent->pRight==this */
- Fts3Expr *pLeft; /* Left operand */
- Fts3Expr *pRight; /* Right operand */
- Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */
-};
-
-SQLITE_PRIVATE int sqlite3Fts3ExprParse(sqlite3_tokenizer *, char **, int, int,
- const char *, int, Fts3Expr **);
-SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *);
-
-/*
-** Candidate values for Fts3Query.eType. Note that the order of the first
-** four values is in order of precedence when parsing expressions. For
-** example, the following:
-**
-** "a OR b AND c NOT d NEAR e"
-**
-** is equivalent to:
-**
-** "a OR (b AND (c NOT (d NEAR e)))"
-*/
-#define FTSQUERY_NEAR 1
-#define FTSQUERY_NOT 2
-#define FTSQUERY_AND 3
-#define FTSQUERY_OR 4
-#define FTSQUERY_PHRASE 5
-
-#ifdef SQLITE_TEST
-SQLITE_PRIVATE void sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
-#endif
-
-/************** End of fts3_expr.h *******************************************/
-/************** Continuing where we left off in fts3.c ***********************/
-/************** Include fts3_hash.h in the middle of fts3.c ******************/
+/************** Continuing where we left off in fts3Int.h ********************/
+/************** Include fts3_hash.h in the middle of fts3Int.h ***************/
/************** Begin file fts3_hash.h ***************************************/
/*
** 2001 September 22
#define _FTS3_HASH_H_
/* Forward declarations of structures. */
-typedef struct fts3Hash fts3Hash;
-typedef struct fts3HashElem fts3HashElem;
+typedef struct Fts3Hash Fts3Hash;
+typedef struct Fts3HashElem Fts3HashElem;
/* A complete hash table is an instance of the following structure.
** The internals of this structure are intended to be opaque -- client
** accessing this structure are really macros, so we can't really make
** this structure opaque.
*/
-struct fts3Hash {
+struct Fts3Hash {
char keyClass; /* HASH_INT, _POINTER, _STRING, _BINARY */
char copyKey; /* True if copy of key made on insert */
int count; /* Number of entries in this table */
- fts3HashElem *first; /* The first element of the array */
+ Fts3HashElem *first; /* The first element of the array */
int htsize; /* Number of buckets in the hash table */
struct _fts3ht { /* the hash table */
int count; /* Number of entries with this hash */
- fts3HashElem *chain; /* Pointer to first entry with this hash */
+ Fts3HashElem *chain; /* Pointer to first entry with this hash */
} *ht;
};
** Again, this structure is intended to be opaque, but it can't really
** be opaque because it is used by macros.
*/
-struct fts3HashElem {
- fts3HashElem *next, *prev; /* Next and previous elements in the table */
+struct Fts3HashElem {
+ Fts3HashElem *next, *prev; /* Next and previous elements in the table */
void *data; /* Data associated with this element */
void *pKey; int nKey; /* Key associated with this element */
};
/*
** Access routines. To delete, insert a NULL pointer.
*/
-SQLITE_PRIVATE void sqlite3Fts3HashInit(fts3Hash*, int keytype, int copyKey);
-SQLITE_PRIVATE void *sqlite3Fts3HashInsert(fts3Hash*, const void *pKey, int nKey, void *pData);
-SQLITE_PRIVATE void *sqlite3Fts3HashFind(const fts3Hash*, const void *pKey, int nKey);
-SQLITE_PRIVATE void sqlite3Fts3HashClear(fts3Hash*);
+SQLITE_PRIVATE void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey);
+SQLITE_PRIVATE void *sqlite3Fts3HashInsert(Fts3Hash*, const void *pKey, int nKey, void *pData);
+SQLITE_PRIVATE void *sqlite3Fts3HashFind(const Fts3Hash*, const void *pKey, int nKey);
+SQLITE_PRIVATE void sqlite3Fts3HashClear(Fts3Hash*);
/*
** Shorthand for the functions above
** Macros for looping over all elements of a hash table. The idiom is
** like this:
**
-** fts3Hash h;
-** fts3HashElem *p;
+** Fts3Hash h;
+** Fts3HashElem *p;
** ...
** for(p=fts3HashFirst(&h); p; p=fts3HashNext(p)){
** SomeStructure *pData = fts3HashData(p);
#endif /* _FTS3_HASH_H_ */
/************** End of fts3_hash.h *******************************************/
+/************** Continuing where we left off in fts3Int.h ********************/
+
+/*
+** This constant controls how often segments are merged. Once there are
+** FTS3_MERGE_COUNT segments of level N, they are merged into a single
+** segment of level N+1.
+*/
+#define FTS3_MERGE_COUNT 16
+
+/*
+** This is the maximum amount of data (in bytes) to store in the
+** Fts3Table.pendingTerms hash table. Normally, the hash table is
+** populated as documents are inserted/updated/deleted in a transaction
+** and used to create a new segment when the transaction is committed.
+** However if this limit is reached midway through a transaction, a new
+** segment is created and the hash table cleared immediately.
+*/
+#define FTS3_MAX_PENDING_DATA (1*1024*1024)
+
+/*
+** Macro to return the number of elements in an array. SQLite has a
+** similar macro called ArraySize(). Use a different name to avoid
+** a collision when building an amalgamation with built-in FTS3.
+*/
+#define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0])))
+
+/*
+** Maximum length of a varint encoded integer. The varint format is different
+** from that used by SQLite, so the maximum length is 10, not 9.
+*/
+#define FTS3_VARINT_MAX 10
+
+/*
+** This section provides definitions to allow the
+** FTS3 extension to be compiled outside of the
+** amalgamation.
+*/
+#ifndef SQLITE_AMALGAMATION
+/*
+** Macros indicating that conditional expressions are always true or
+** false.
+*/
+# define ALWAYS(x) (x)
+# define NEVER(X) (x)
+/*
+** Internal types used by SQLite.
+*/
+typedef unsigned char u8; /* 1-byte (or larger) unsigned integer */
+typedef short int i16; /* 2-byte (or larger) signed integer */
+/*
+** Macro used to suppress compiler warnings for unused parameters.
+*/
+#define UNUSED_PARAMETER(x) (void)(x)
+#endif
+
+typedef struct Fts3Table Fts3Table;
+typedef struct Fts3Cursor Fts3Cursor;
+typedef struct Fts3Expr Fts3Expr;
+typedef struct Fts3Phrase Fts3Phrase;
+typedef struct Fts3SegReader Fts3SegReader;
+typedef struct Fts3SegFilter Fts3SegFilter;
+
+/*
+** A connection to a fulltext index is an instance of the following
+** structure. The xCreate and xConnect methods create an instance
+** of this structure and xDestroy and xDisconnect free that instance.
+** All other methods receive a pointer to the structure as one of their
+** arguments.
+*/
+struct Fts3Table {
+ sqlite3_vtab base; /* Base class used by SQLite core */
+ sqlite3 *db; /* The database connection */
+ const char *zDb; /* logical database name */
+ const char *zName; /* virtual table name */
+ int nColumn; /* number of named columns in virtual table */
+ char **azColumn; /* column names. malloced */
+ sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */
+
+ /* Precompiled statements used by the implementation. Each of these
+ ** statements is run and reset within a single virtual table API call.
+ */
+ sqlite3_stmt *aStmt[18];
+
+ /* Pointer to string containing the SQL:
+ **
+ ** "SELECT block FROM %_segments WHERE blockid BETWEEN ? AND ?
+ ** ORDER BY blockid"
+ */
+ char *zSelectLeaves;
+ int nLeavesStmt; /* Valid statements in aLeavesStmt */
+ int nLeavesTotal; /* Total number of prepared leaves stmts */
+ int nLeavesAlloc; /* Allocated size of aLeavesStmt */
+ sqlite3_stmt **aLeavesStmt; /* Array of prepared zSelectLeaves stmts */
+
+ int nNodeSize; /* Soft limit for node size */
+
+ /* The following hash table is used to buffer pending index updates during
+ ** transactions. Variable nPendingData estimates the memory size of the
+ ** pending data, including hash table overhead, but not malloc overhead.
+ ** When nPendingData exceeds FTS3_MAX_PENDING_DATA, the buffer is flushed
+ ** automatically. Variable iPrevDocid is the docid of the most recently
+ ** inserted record.
+ */
+ int nPendingData;
+ sqlite_int64 iPrevDocid;
+ Fts3Hash pendingTerms;
+};
+
+/*
+** When the core wants to read from the virtual table, it creates a
+** virtual table cursor (an instance of the following structure) using
+** the xOpen method. Cursors are destroyed using the xClose method.
+*/
+struct Fts3Cursor {
+ sqlite3_vtab_cursor base; /* Base class used by SQLite core */
+ i16 eSearch; /* Search strategy (see below) */
+ u8 isEof; /* True if at End Of Results */
+ u8 isRequireSeek; /* True if must seek pStmt to %_content row */
+ sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */
+ Fts3Expr *pExpr; /* Parsed MATCH query string */
+ sqlite3_int64 iPrevId; /* Previous id read from aDoclist */
+ char *pNextId; /* Pointer into the body of aDoclist */
+ char *aDoclist; /* List of docids for full-text queries */
+ int nDoclist; /* Size of buffer at aDoclist */
+};
+
+/*
+** The Fts3Cursor.eSearch member is always set to one of the following.
+** Actualy, Fts3Cursor.eSearch can be greater than or equal to
+** FTS3_FULLTEXT_SEARCH. If so, then Fts3Cursor.eSearch - 2 is the index
+** of the column to be searched. For example, in
+**
+** CREATE VIRTUAL TABLE ex1 USING fts3(a,b,c,d);
+** SELECT docid FROM ex1 WHERE b MATCH 'one two three';
+**
+** Because the LHS of the MATCH operator is 2nd column "b",
+** Fts3Cursor.eSearch will be set to FTS3_FULLTEXT_SEARCH+1. (+0 for a,
+** +1 for b, +2 for c, +3 for d.) If the LHS of MATCH were "ex1"
+** indicating that all columns should be searched,
+** then eSearch would be set to FTS3_FULLTEXT_SEARCH+4.
+*/
+#define FTS3_FULLSCAN_SEARCH 0 /* Linear scan of %_content table */
+#define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */
+#define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */
+
+/*
+** A "phrase" is a sequence of one or more tokens that must match in
+** sequence. A single token is the base case and the most common case.
+** For a sequence of tokens contained in "...", nToken will be the number
+** of tokens in the string.
+*/
+struct Fts3Phrase {
+ int nToken; /* Number of tokens in the phrase */
+ int iColumn; /* Index of column this phrase must match */
+ int isNot; /* Phrase prefixed by unary not (-) operator */
+ struct PhraseToken {
+ char *z; /* Text of the token */
+ int n; /* Number of bytes in buffer pointed to by z */
+ int isPrefix; /* True if token ends in with a "*" character */
+ } aToken[1]; /* One entry for each token in the phrase */
+};
+
+/*
+** A tree of these objects forms the RHS of a MATCH operator.
+*/
+struct Fts3Expr {
+ int eType; /* One of the FTSQUERY_XXX values defined below */
+ int nNear; /* Valid if eType==FTSQUERY_NEAR */
+ Fts3Expr *pParent; /* pParent->pLeft==this or pParent->pRight==this */
+ Fts3Expr *pLeft; /* Left operand */
+ Fts3Expr *pRight; /* Right operand */
+ Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */
+};
+
+/*
+** Candidate values for Fts3Query.eType. Note that the order of the first
+** four values is in order of precedence when parsing expressions. For
+** example, the following:
+**
+** "a OR b AND c NOT d NEAR e"
+**
+** is equivalent to:
+**
+** "a OR (b AND (c NOT (d NEAR e)))"
+*/
+#define FTSQUERY_NEAR 1
+#define FTSQUERY_NOT 2
+#define FTSQUERY_AND 3
+#define FTSQUERY_OR 4
+#define FTSQUERY_PHRASE 5
+
+
+/* fts3_init.c */
+SQLITE_PRIVATE int sqlite3Fts3DeleteVtab(int, sqlite3_vtab *);
+SQLITE_PRIVATE int sqlite3Fts3InitVtab(int, sqlite3*, void*, int, const char*const*,
+ sqlite3_vtab **, char **);
+
+/* fts3_write.c */
+SQLITE_PRIVATE int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*);
+SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *);
+SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *);
+SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *);
+SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(Fts3Table *,int, sqlite3_int64,
+ sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**);
+SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3Table *, Fts3SegReader *);
+SQLITE_PRIVATE int sqlite3Fts3SegReaderIterate(
+ Fts3Table *, Fts3SegReader **, int, Fts3SegFilter *,
+ int (*)(Fts3Table *, void *, char *, int, char *, int), void *
+);
+SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char const**, int*);
+SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table*, sqlite3_stmt **);
+
+/* Flags allowed as part of the 4th argument to SegmentReaderIterate() */
+#define FTS3_SEGMENT_REQUIRE_POS 0x00000001
+#define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002
+#define FTS3_SEGMENT_COLUMN_FILTER 0x00000004
+#define FTS3_SEGMENT_PREFIX 0x00000008
+
+/* Type passed as 4th argument to SegmentReaderIterate() */
+struct Fts3SegFilter {
+ const char *zTerm;
+ int nTerm;
+ int iCol;
+ int flags;
+};
+
+/* fts3.c */
+SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *, sqlite3_int64);
+SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);
+SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *, int *);
+SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64);
+SQLITE_PRIVATE void sqlite3Fts3Dequote(char *);
+
+/* fts3_tokenizer.c */
+SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *, int *);
+SQLITE_PRIVATE int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *);
+SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(Fts3Hash *pHash,
+ const char *, sqlite3_tokenizer **, const char **, char **
+);
+
+/* fts3_snippet.c */
+SQLITE_PRIVATE void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*);
+SQLITE_PRIVATE void sqlite3Fts3Snippet(sqlite3_context*, Fts3Cursor*,
+ const char *, const char *, const char *
+);
+
+/* fts3_expr.c */
+SQLITE_PRIVATE int sqlite3Fts3ExprParse(sqlite3_tokenizer *,
+ char **, int, int, const char *, int, Fts3Expr **
+);
+SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *);
+#ifdef SQLITE_TEST
+SQLITE_PRIVATE void sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
+#endif
+
+#endif /* _FTSINT_H */
+
+/************** End of fts3Int.h *********************************************/
/************** Continuing where we left off in fts3.c ***********************/
+
+
#ifndef SQLITE_CORE
SQLITE_EXTENSION_INIT1
#endif
+
/* TODO(shess) MAN, this thing needs some refactoring. At minimum, it
** would be nice to order the file better, perhaps something along the
** lines of:
# define FTSTRACE(A)
#endif
-/* It is not safe to call isspace(), tolower(), or isalnum() on
-** hi-bit-set characters. This is the same solution used in the
-** tokenizer.
-*/
-/* TODO(shess) The snippet-generation code should be using the
-** tokenizer-generated tokens rather than doing its own local
-** tokenization.
-*/
-/* TODO(shess) Is __isascii() a portable version of (c&0x80)==0? */
-static int safe_isspace(char c){
- return (c&0x80)==0 ? isspace(c) : 0;
-}
-static int safe_tolower(char c){
- return (c&0x80)==0 ? tolower(c) : c;
-}
-static int safe_isalnum(char c){
- return (c&0x80)==0 ? isalnum(c) : 0;
-}
-
typedef enum DocListType {
DL_DOCIDS, /* docids only */
DL_POSITIONS, /* docids + positions */
POS_BASE
};
-/* MERGE_COUNT controls how often we merge segments (see comment at
-** top of file).
-*/
-#define MERGE_COUNT 16
-
/* utility functions */
/* CLEAR() and SCRAMBLE() abstract memset() on a pointer to a single
# define SCRAMBLE(b)
#endif
-/* We may need up to VARINT_MAX bytes to store an encoded 64-bit integer. */
-#define VARINT_MAX 10
-
-/* Write a 64-bit variable-length integer to memory starting at p[0].
- * The length of data written will be between 1 and VARINT_MAX bytes.
- * The number of bytes written is returned. */
-static int fts3PutVarint(char *p, sqlite_int64 v){
+/*
+** Write a 64-bit variable-length integer to memory starting at p[0].
+** The length of data written will be between 1 and FTS3_VARINT_MAX bytes.
+** The number of bytes written is returned.
+*/
+SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){
unsigned char *q = (unsigned char *) p;
sqlite_uint64 vu = v;
do{
vu >>= 7;
}while( vu!=0 );
q[-1] &= 0x7f; /* turn off high bit in final byte */
- assert( q - (unsigned char *)p <= VARINT_MAX );
+ assert( q - (unsigned char *)p <= FTS3_VARINT_MAX );
return (int) (q - (unsigned char *)p);
}
-/* Read a 64-bit variable-length integer from memory starting at p[0].
- * Return the number of bytes read, or 0 on error.
- * The value is stored in *v. */
-static int fts3GetVarint(const char *p, sqlite_int64 *v){
+/*
+** Read a 64-bit variable-length integer from memory starting at p[0].
+** Return the number of bytes read, or 0 on error.
+** The value is stored in *v.
+*/
+SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){
const unsigned char *q = (const unsigned char *) p;
sqlite_uint64 x = 0, y = 1;
while( (*q & 0x80) == 0x80 ){
x += y * (*q++ & 0x7f);
y <<= 7;
- if( q - (unsigned char *)p >= VARINT_MAX ){ /* bad data */
+ if( q - (unsigned char *)p >= FTS3_VARINT_MAX ){ /* bad data */
assert( 0 );
return 0;
}
return (int) (q - (unsigned char *)p);
}
-static int fts3GetVarint32(const char *p, int *pi){
+/*
+** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to a
+** 32-bit integer before it is returned.
+*/
+SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *p, int *pi){
sqlite_int64 i;
- int ret = fts3GetVarint(p, &i);
+ int ret = sqlite3Fts3GetVarint(p, &i);
*pi = (int) i;
assert( *pi==i );
return ret;
}
-/*******************************************************************/
-/* DataBuffer is used to collect data into a buffer in piecemeal
-** fashion. It implements the usual distinction between amount of
-** data currently stored (nData) and buffer capacity (nCapacity).
-**
-** dataBufferInit - create a buffer with given initial capacity.
-** dataBufferReset - forget buffer's data, retaining capacity.
-** dataBufferDestroy - free buffer's data.
-** dataBufferSwap - swap contents of two buffers.
-** dataBufferExpand - expand capacity without adding data.
-** dataBufferAppend - append data.
-** dataBufferAppend2 - append two pieces of data at once.
-** dataBufferReplace - replace buffer's data.
-*/
-typedef struct DataBuffer {
- char *pData; /* Pointer to malloc'ed buffer. */
- int nCapacity; /* Size of pData buffer. */
- int nData; /* End of data loaded into pData. */
-} DataBuffer;
-
-static void dataBufferInit(DataBuffer *pBuffer, int nCapacity){
- assert( nCapacity>=0 );
- pBuffer->nData = 0;
- pBuffer->nCapacity = nCapacity;
- pBuffer->pData = nCapacity==0 ? NULL : sqlite3_malloc(nCapacity);
-}
-static void dataBufferReset(DataBuffer *pBuffer){
- pBuffer->nData = 0;
-}
-static void dataBufferDestroy(DataBuffer *pBuffer){
- if( pBuffer->pData!=NULL ) sqlite3_free(pBuffer->pData);
- SCRAMBLE(pBuffer);
-}
-static void dataBufferSwap(DataBuffer *pBuffer1, DataBuffer *pBuffer2){
- DataBuffer tmp = *pBuffer1;
- *pBuffer1 = *pBuffer2;
- *pBuffer2 = tmp;
-}
-static void dataBufferExpand(DataBuffer *pBuffer, int nAddCapacity){
- assert( nAddCapacity>0 );
- /* TODO(shess) Consider expanding more aggressively. Note that the
- ** underlying malloc implementation may take care of such things for
- ** us already.
- */
- if( pBuffer->nData+nAddCapacity>pBuffer->nCapacity ){
- pBuffer->nCapacity = pBuffer->nData+nAddCapacity;
- pBuffer->pData = sqlite3_realloc(pBuffer->pData, pBuffer->nCapacity);
- }
-}
-static void dataBufferAppend(DataBuffer *pBuffer,
- const char *pSource, int nSource){
- assert( nSource>0 && pSource!=NULL );
- dataBufferExpand(pBuffer, nSource);
- memcpy(pBuffer->pData+pBuffer->nData, pSource, nSource);
- pBuffer->nData += nSource;
-}
-static void dataBufferAppend2(DataBuffer *pBuffer,
- const char *pSource1, int nSource1,
- const char *pSource2, int nSource2){
- assert( nSource1>0 && pSource1!=NULL );
- assert( nSource2>0 && pSource2!=NULL );
- dataBufferExpand(pBuffer, nSource1+nSource2);
- memcpy(pBuffer->pData+pBuffer->nData, pSource1, nSource1);
- memcpy(pBuffer->pData+pBuffer->nData+nSource1, pSource2, nSource2);
- pBuffer->nData += nSource1+nSource2;
-}
-static void dataBufferReplace(DataBuffer *pBuffer,
- const char *pSource, int nSource){
- dataBufferReset(pBuffer);
- dataBufferAppend(pBuffer, pSource, nSource);
-}
-
-/* StringBuffer is a null-terminated version of DataBuffer. */
-typedef struct StringBuffer {
- DataBuffer b; /* Includes null terminator. */
-} StringBuffer;
-
-static void initStringBuffer(StringBuffer *sb){
- dataBufferInit(&sb->b, 100);
- dataBufferReplace(&sb->b, "", 1);
-}
-static int stringBufferLength(StringBuffer *sb){
- return sb->b.nData-1;
-}
-static char *stringBufferData(StringBuffer *sb){
- return sb->b.pData;
-}
-static void stringBufferDestroy(StringBuffer *sb){
- dataBufferDestroy(&sb->b);
-}
-
-static void nappend(StringBuffer *sb, const char *zFrom, int nFrom){
- assert( sb->b.nData>0 );
- if( nFrom>0 ){
- sb->b.nData--;
- dataBufferAppend2(&sb->b, zFrom, nFrom, "", 1);
- }
-}
-static void append(StringBuffer *sb, const char *zFrom){
- nappend(sb, zFrom, strlen(zFrom));
-}
-
-/* Append a list of strings separated by commas. */
-static void appendList(StringBuffer *sb, int nString, char **azString){
- int i;
- for(i=0; i<nString; ++i){
- if( i>0 ) append(sb, ", ");
- append(sb, azString[i]);
- }
-}
-
-static int endsInWhiteSpace(StringBuffer *p){
- return stringBufferLength(p)>0 &&
- safe_isspace(stringBufferData(p)[stringBufferLength(p)-1]);
-}
-
-/* If the StringBuffer ends in something other than white space, add a
-** single space character to the end.
+/*
+** Return the number of bytes required to store the value passed as the
+** first argument in varint form.
*/
-static void appendWhiteSpace(StringBuffer *p){
- if( stringBufferLength(p)==0 ) return;
- if( !endsInWhiteSpace(p) ) append(p, " ");
+SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64 v){
+ int i = 0;
+ do{
+ i++;
+ v >>= 7;
+ }while( v!=0 );
+ return i;
}
-/* Remove white space from the end of the StringBuffer */
-static void trimWhiteSpace(StringBuffer *p){
- while( endsInWhiteSpace(p) ){
- p->b.pData[--p->b.nData-1] = '\0';
- }
-}
-
-/*******************************************************************/
-/* DLReader is used to read document elements from a doclist. The
-** current docid is cached, so dlrDocid() is fast. DLReader does not
-** own the doclist buffer.
-**
-** dlrAtEnd - true if there's no more data to read.
-** dlrDocid - docid of current document.
-** dlrDocData - doclist data for current document (including docid).
-** dlrDocDataBytes - length of same.
-** dlrAllDataBytes - length of all remaining data.
-** dlrPosData - position data for current document.
-** dlrPosDataLen - length of pos data for current document (incl POS_END).
-** dlrStep - step to current document.
-** dlrInit - initial for doclist of given type against given data.
-** dlrDestroy - clean up.
-**
-** Expected usage is something like:
-**
-** DLReader reader;
-** dlrInit(&reader, pData, nData);
-** while( !dlrAtEnd(&reader) ){
-** // calls to dlrDocid() and kin.
-** dlrStep(&reader);
-** }
-** dlrDestroy(&reader);
+/*
+** Convert an SQL-style quoted string into a normal string by removing
+** the quote characters. The conversion is done in-place. If the
+** input does not begin with a quote character, then this routine
+** is a no-op.
+**
+** Examples:
+**
+** "abc" becomes abc
+** 'xyz' becomes xyz
+** [pqr] becomes pqr
+** `mno` becomes mno
*/
-typedef struct DLReader {
- DocListType iType;
- const char *pData;
- int nData;
+SQLITE_PRIVATE void sqlite3Fts3Dequote(char *z){
+ int quote;
+ int i, j;
- sqlite_int64 iDocid;
- int nElement;
-} DLReader;
-
-static int dlrAtEnd(DLReader *pReader){
- assert( pReader->nData>=0 );
- return pReader->nData==0;
-}
-static sqlite_int64 dlrDocid(DLReader *pReader){
- assert( !dlrAtEnd(pReader) );
- return pReader->iDocid;
-}
-static const char *dlrDocData(DLReader *pReader){
- assert( !dlrAtEnd(pReader) );
- return pReader->pData;
-}
-static int dlrDocDataBytes(DLReader *pReader){
- assert( !dlrAtEnd(pReader) );
- return pReader->nElement;
-}
-static int dlrAllDataBytes(DLReader *pReader){
- assert( !dlrAtEnd(pReader) );
- return pReader->nData;
-}
-/* TODO(shess) Consider adding a field to track iDocid varint length
-** to make these two functions faster. This might matter (a tiny bit)
-** for queries.
-*/
-static const char *dlrPosData(DLReader *pReader){
- sqlite_int64 iDummy;
- int n = fts3GetVarint(pReader->pData, &iDummy);
- assert( !dlrAtEnd(pReader) );
- return pReader->pData+n;
-}
-static int dlrPosDataLen(DLReader *pReader){
- sqlite_int64 iDummy;
- int n = fts3GetVarint(pReader->pData, &iDummy);
- assert( !dlrAtEnd(pReader) );
- return pReader->nElement-n;
-}
-static void dlrStep(DLReader *pReader){
- assert( !dlrAtEnd(pReader) );
-
- /* Skip past current doclist element. */
- assert( pReader->nElement<=pReader->nData );
- pReader->pData += pReader->nElement;
- pReader->nData -= pReader->nElement;
-
- /* If there is more data, read the next doclist element. */
- if( pReader->nData!=0 ){
- sqlite_int64 iDocidDelta;
- int iDummy, n = fts3GetVarint(pReader->pData, &iDocidDelta);
- pReader->iDocid += iDocidDelta;
- if( pReader->iType>=DL_POSITIONS ){
- assert( n<pReader->nData );
- while( 1 ){
- n += fts3GetVarint32(pReader->pData+n, &iDummy);
- assert( n<=pReader->nData );
- if( iDummy==POS_END ) break;
- if( iDummy==POS_COLUMN ){
- n += fts3GetVarint32(pReader->pData+n, &iDummy);
- assert( n<pReader->nData );
- }else if( pReader->iType==DL_POSITIONS_OFFSETS ){
- n += fts3GetVarint32(pReader->pData+n, &iDummy);
- n += fts3GetVarint32(pReader->pData+n, &iDummy);
- assert( n<pReader->nData );
- }
+ quote = z[0];
+ switch( quote ){
+ case '\'': break;
+ case '"': break;
+ case '`': break; /* For MySQL compatibility */
+ case '[': quote = ']'; break; /* For MS SqlServer compatibility */
+ default: return;
+ }
+ for(i=1, j=0; z[i]; i++){
+ if( z[i]==quote ){
+ if( z[i+1]==quote ){
+ z[j++] = (char)quote;
+ i++;
+ }else{
+ z[j++] = 0;
+ break;
}
+ }else{
+ z[j++] = z[i];
}
- pReader->nElement = n;
- assert( pReader->nElement<=pReader->nData );
}
}
-static void dlrInit(DLReader *pReader, DocListType iType,
- const char *pData, int nData){
- assert( pData!=NULL && nData!=0 );
- pReader->iType = iType;
- pReader->pData = pData;
- pReader->nData = nData;
- pReader->nElement = 0;
- pReader->iDocid = 0;
- /* Load the first element's data. There must be a first element. */
- dlrStep(pReader);
-}
-static void dlrDestroy(DLReader *pReader){
- SCRAMBLE(pReader);
+static void fts3GetDeltaVarint(char **pp, sqlite3_int64 *pVal){
+ sqlite3_int64 iVal;
+ *pp += sqlite3Fts3GetVarint(*pp, &iVal);
+ *pVal += iVal;
}
-#ifndef NDEBUG
-/* Verify that the doclist can be validly decoded. Also returns the
-** last docid found because it is convenient in other assertions for
-** DLWriter.
-*/
-static void docListValidate(DocListType iType, const char *pData, int nData,
- sqlite_int64 *pLastDocid){
- sqlite_int64 iPrevDocid = 0;
- assert( nData>0 );
- assert( pData!=0 );
- assert( pData+nData>pData );
- while( nData!=0 ){
- sqlite_int64 iDocidDelta;
- int n = fts3GetVarint(pData, &iDocidDelta);
- iPrevDocid += iDocidDelta;
- if( iType>DL_DOCIDS ){
- int iDummy;
- while( 1 ){
- n += fts3GetVarint32(pData+n, &iDummy);
- if( iDummy==POS_END ) break;
- if( iDummy==POS_COLUMN ){
- n += fts3GetVarint32(pData+n, &iDummy);
- }else if( iType>DL_POSITIONS ){
- n += fts3GetVarint32(pData+n, &iDummy);
- n += fts3GetVarint32(pData+n, &iDummy);
- }
- assert( n<=nData );
- }
- }
- assert( n<=nData );
- pData += n;
- nData -= n;
+static void fts3GetDeltaVarint2(char **pp, char *pEnd, sqlite3_int64 *pVal){
+ if( *pp>=pEnd ){
+ *pp = 0;
+ }else{
+ fts3GetDeltaVarint(pp, pVal);
}
- if( pLastDocid ) *pLastDocid = iPrevDocid;
}
-#define ASSERT_VALID_DOCLIST(i, p, n, o) docListValidate(i, p, n, o)
-#else
-#define ASSERT_VALID_DOCLIST(i, p, n, o) assert( 1 )
-#endif
-/*******************************************************************/
-/* DLWriter is used to write doclist data to a DataBuffer. DLWriter
-** always appends to the buffer and does not own it.
-**
-** dlwInit - initialize to write a given type doclistto a buffer.
-** dlwDestroy - clear the writer's memory. Does not free buffer.
-** dlwAppend - append raw doclist data to buffer.
-** dlwCopy - copy next doclist from reader to writer.
-** dlwAdd - construct doclist element and append to buffer.
-** Only apply dlwAdd() to DL_DOCIDS doclists (else use PLWriter).
+/*
+** The xDisconnect() virtual table method.
*/
-typedef struct DLWriter {
- DocListType iType;
- DataBuffer *b;
- sqlite_int64 iPrevDocid;
-#ifndef NDEBUG
- int has_iPrevDocid;
-#endif
-} DLWriter;
+static int fts3DisconnectMethod(sqlite3_vtab *pVtab){
+ Fts3Table *p = (Fts3Table *)pVtab;
+ int i;
-static void dlwInit(DLWriter *pWriter, DocListType iType, DataBuffer *b){
- pWriter->b = b;
- pWriter->iType = iType;
- pWriter->iPrevDocid = 0;
-#ifndef NDEBUG
- pWriter->has_iPrevDocid = 0;
-#endif
-}
-static void dlwDestroy(DLWriter *pWriter){
- SCRAMBLE(pWriter);
+ assert( p->nPendingData==0 );
+
+ /* Free any prepared statements held */
+ for(i=0; i<SizeofArray(p->aStmt); i++){
+ sqlite3_finalize(p->aStmt[i]);
+ }
+ for(i=0; i<p->nLeavesStmt; i++){
+ sqlite3_finalize(p->aLeavesStmt[i]);
+ }
+ sqlite3_free(p->zSelectLeaves);
+ sqlite3_free(p->aLeavesStmt);
+
+ /* Invoke the tokenizer destructor to free the tokenizer. */
+ p->pTokenizer->pModule->xDestroy(p->pTokenizer);
+
+ sqlite3_free(p);
+ return SQLITE_OK;
}
-/* iFirstDocid is the first docid in the doclist in pData. It is
-** needed because pData may point within a larger doclist, in which
-** case the first item would be delta-encoded.
-**
-** iLastDocid is the final docid in the doclist in pData. It is
-** needed to create the new iPrevDocid for future delta-encoding. The
-** code could decode the passed doclist to recreate iLastDocid, but
-** the only current user (docListMerge) already has decoded this
-** information.
-*/
-/* TODO(shess) This has become just a helper for docListMerge.
-** Consider a refactor to make this cleaner.
-*/
-static void dlwAppend(DLWriter *pWriter,
- const char *pData, int nData,
- sqlite_int64 iFirstDocid, sqlite_int64 iLastDocid){
- sqlite_int64 iDocid = 0;
- char c[VARINT_MAX];
- int nFirstOld, nFirstNew; /* Old and new varint len of first docid. */
-#ifndef NDEBUG
- sqlite_int64 iLastDocidDelta;
-#endif
- /* Recode the initial docid as delta from iPrevDocid. */
- nFirstOld = fts3GetVarint(pData, &iDocid);
- assert( nFirstOld<nData || (nFirstOld==nData && pWriter->iType==DL_DOCIDS) );
- nFirstNew = fts3PutVarint(c, iFirstDocid-pWriter->iPrevDocid);
+/*
+** The xDestroy() virtual table method.
+*/
+static int fts3DestroyMethod(sqlite3_vtab *pVtab){
+ int rc; /* Return code */
+ Fts3Table *p = (Fts3Table *)pVtab;
- /* Verify that the incoming doclist is valid AND that it ends with
- ** the expected docid. This is essential because we'll trust this
- ** docid in future delta-encoding.
- */
- ASSERT_VALID_DOCLIST(pWriter->iType, pData, nData, &iLastDocidDelta);
- assert( iLastDocid==iFirstDocid-iDocid+iLastDocidDelta );
+ /* Create a script to drop the underlying three storage tables. */
+ char *zSql = sqlite3_mprintf(
+ "DROP TABLE IF EXISTS %Q.'%q_content';"
+ "DROP TABLE IF EXISTS %Q.'%q_segments';"
+ "DROP TABLE IF EXISTS %Q.'%q_segdir';",
+ p->zDb, p->zName, p->zDb, p->zName, p->zDb, p->zName
+ );
- /* Append recoded initial docid and everything else. Rest of docids
- ** should have been delta-encoded from previous initial docid.
+ /* If malloc has failed, set rc to SQLITE_NOMEM. Otherwise, try to
+ ** execute the SQL script created above.
*/
- if( nFirstOld<nData ){
- dataBufferAppend2(pWriter->b, c, nFirstNew,
- pData+nFirstOld, nData-nFirstOld);
+ if( zSql ){
+ rc = sqlite3_exec(p->db, zSql, 0, 0, 0);
+ sqlite3_free(zSql);
}else{
- dataBufferAppend(pWriter->b, c, nFirstNew);
+ rc = SQLITE_NOMEM;
}
- pWriter->iPrevDocid = iLastDocid;
-}
-static void dlwCopy(DLWriter *pWriter, DLReader *pReader){
- dlwAppend(pWriter, dlrDocData(pReader), dlrDocDataBytes(pReader),
- dlrDocid(pReader), dlrDocid(pReader));
-}
-static void dlwAdd(DLWriter *pWriter, sqlite_int64 iDocid){
- char c[VARINT_MAX];
- int n = fts3PutVarint(c, iDocid-pWriter->iPrevDocid);
- /* Docids must ascend. */
- assert( !pWriter->has_iPrevDocid || iDocid>pWriter->iPrevDocid );
- assert( pWriter->iType==DL_DOCIDS );
-
- dataBufferAppend(pWriter->b, c, n);
- pWriter->iPrevDocid = iDocid;
-#ifndef NDEBUG
- pWriter->has_iPrevDocid = 1;
-#endif
+ /* If everything has worked, invoke fts3DisconnectMethod() to free the
+ ** memory associated with the Fts3Table structure and return SQLITE_OK.
+ ** Otherwise, return an SQLite error code.
+ */
+ return (rc==SQLITE_OK ? fts3DisconnectMethod(pVtab) : rc);
}
-/*******************************************************************/
-/* PLReader is used to read data from a document's position list. As
-** the caller steps through the list, data is cached so that varints
-** only need to be decoded once.
-**
-** plrInit, plrDestroy - create/destroy a reader.
-** plrColumn, plrPosition, plrStartOffset, plrEndOffset - accessors
-** plrAtEnd - at end of stream, only call plrDestroy once true.
-** plrStep - step to the next element.
-*/
-typedef struct PLReader {
- /* These refer to the next position's data. nData will reach 0 when
- ** reading the last position, so plrStep() signals EOF by setting
- ** pData to NULL.
- */
- const char *pData;
- int nData;
- DocListType iType;
- int iColumn; /* the last column read */
- int iPosition; /* the last position read */
- int iStartOffset; /* the last start offset read */
- int iEndOffset; /* the last end offset read */
-} PLReader;
+/*
+** Invoke sqlite3_declare_vtab() to declare the schema for the FTS3 table
+** passed as the first argument. This is done as part of the xConnect()
+** and xCreate() methods.
+*/
+static int fts3DeclareVtab(Fts3Table *p){
+ int i; /* Iterator variable */
+ int rc; /* Return code */
+ char *zSql; /* SQL statement passed to declare_vtab() */
+ char *zCols; /* List of user defined columns */
-static int plrAtEnd(PLReader *pReader){
- return pReader->pData==NULL;
-}
-static int plrColumn(PLReader *pReader){
- assert( !plrAtEnd(pReader) );
- return pReader->iColumn;
-}
-static int plrPosition(PLReader *pReader){
- assert( !plrAtEnd(pReader) );
- return pReader->iPosition;
-}
-static int plrStartOffset(PLReader *pReader){
- assert( !plrAtEnd(pReader) );
- return pReader->iStartOffset;
-}
-static int plrEndOffset(PLReader *pReader){
- assert( !plrAtEnd(pReader) );
- return pReader->iEndOffset;
-}
-static void plrStep(PLReader *pReader){
- int i, n;
+ /* Create a list of user columns for the virtual table */
+ zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]);
+ for(i=1; zCols && i<p->nColumn; i++){
+ zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]);
+ }
- assert( !plrAtEnd(pReader) );
+ /* Create the whole "CREATE TABLE" statement to pass to SQLite */
+ zSql = sqlite3_mprintf(
+ "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN)", zCols, p->zName
+ );
- if( pReader->nData==0 ){
- pReader->pData = NULL;
- return;
+ if( !zCols || !zSql ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3_declare_vtab(p->db, zSql);
}
- n = fts3GetVarint32(pReader->pData, &i);
- if( i==POS_COLUMN ){
- n += fts3GetVarint32(pReader->pData+n, &pReader->iColumn);
- pReader->iPosition = 0;
- pReader->iStartOffset = 0;
- n += fts3GetVarint32(pReader->pData+n, &i);
- }
- /* Should never see adjacent column changes. */
- assert( i!=POS_COLUMN );
+ sqlite3_free(zSql);
+ sqlite3_free(zCols);
+ return rc;
+}
- if( i==POS_END ){
- pReader->nData = 0;
- pReader->pData = NULL;
- return;
- }
+/*
+** Create the backing store tables (%_content, %_segments and %_segdir)
+** required by the FTS3 table passed as the only argument. This is done
+** as part of the vtab xCreate() method.
+*/
+static int fts3CreateTables(Fts3Table *p){
+ int rc; /* Return code */
+ int i; /* Iterator variable */
+ char *zContentCols; /* Columns of %_content table */
+ char *zSql; /* SQL script to create required tables */
+
+ /* Create a list of user columns for the content table */
+ zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY");
+ for(i=0; zContentCols && i<p->nColumn; i++){
+ char *z = p->azColumn[i];
+ zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z);
+ }
+
+ /* Create the whole SQL script */
+ zSql = sqlite3_mprintf(
+ "CREATE TABLE %Q.'%q_content'(%s);"
+ "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);"
+ "CREATE TABLE %Q.'%q_segdir'("
+ "level INTEGER,"
+ "idx INTEGER,"
+ "start_block INTEGER,"
+ "leaves_end_block INTEGER,"
+ "end_block INTEGER,"
+ "root BLOB,"
+ "PRIMARY KEY(level, idx)"
+ ");",
+ p->zDb, p->zName, zContentCols, p->zDb, p->zName, p->zDb, p->zName
+ );
- pReader->iPosition += i-POS_BASE;
- if( pReader->iType==DL_POSITIONS_OFFSETS ){
- n += fts3GetVarint32(pReader->pData+n, &i);
- pReader->iStartOffset += i;
- n += fts3GetVarint32(pReader->pData+n, &i);
- pReader->iEndOffset = pReader->iStartOffset+i;
+ /* Unless a malloc() failure has occurred, execute the SQL script to
+ ** create the tables used to store data for this FTS3 virtual table.
+ */
+ if( zContentCols==0 || zSql==0 ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3_exec(p->db, zSql, 0, 0, 0);
}
- assert( n<=pReader->nData );
- pReader->pData += n;
- pReader->nData -= n;
-}
-static void plrInit(PLReader *pReader, DLReader *pDLReader){
- pReader->pData = dlrPosData(pDLReader);
- pReader->nData = dlrPosDataLen(pDLReader);
- pReader->iType = pDLReader->iType;
- pReader->iColumn = 0;
- pReader->iPosition = 0;
- pReader->iStartOffset = 0;
- pReader->iEndOffset = 0;
- plrStep(pReader);
-}
-static void plrDestroy(PLReader *pReader){
- SCRAMBLE(pReader);
+ sqlite3_free(zSql);
+ sqlite3_free(zContentCols);
+ return rc;
}
-/*******************************************************************/
-/* PLWriter is used in constructing a document's position list. As a
-** convenience, if iType is DL_DOCIDS, PLWriter becomes a no-op.
-** PLWriter writes to the associated DLWriter's buffer.
-**
-** plwInit - init for writing a document's poslist.
-** plwDestroy - clear a writer.
-** plwAdd - append position and offset information.
-** plwCopy - copy next position's data from reader to writer.
-** plwTerminate - add any necessary doclist terminator.
-**
-** Calling plwAdd() after plwTerminate() may result in a corrupt
-** doclist.
-*/
-/* TODO(shess) Until we've written the second item, we can cache the
-** first item's information. Then we'd have three states:
+/*
+** This function is the implementation of both the xConnect and xCreate
+** methods of the FTS3 virtual table.
**
-** - initialized with docid, no positions.
-** - docid and one position.
-** - docid and multiple positions.
+** The argv[] array contains the following:
**
-** Only the last state needs to actually write to dlw->b, which would
-** be an improvement in the DLCollector case.
-*/
-typedef struct PLWriter {
- DLWriter *dlw;
+** argv[0] -> module name
+** argv[1] -> database name
+** argv[2] -> table name
+** argv[...] -> "column name" and other module argument fields.
+*/
+int fts3InitVtab(
+ int isCreate, /* True for xCreate, false for xConnect */
+ sqlite3 *db, /* The SQLite database connection */
+ void *pAux, /* Hash table containing tokenizers */
+ int argc, /* Number of elements in argv array */
+ const char * const *argv, /* xCreate/xConnect argument array */
+ sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */
+ char **pzErr /* Write any error message here */
+){
+ Fts3Hash *pHash = (Fts3Hash *)pAux;
+ Fts3Table *p; /* Pointer to allocated vtab */
+ int rc; /* Return code */
+ int i; /* Iterator variable */
+ int nByte; /* Size of allocation used for *p */
+ int iCol;
+ int nString = 0;
+ int nCol = 0;
+ char *zCsr;
+ int nDb;
+ int nName;
- int iColumn; /* the last column written */
- int iPos; /* the last position written */
- int iOffset; /* the last start offset written */
-} PLWriter;
+ const char *zTokenizer = 0; /* Name of tokenizer to use */
+ sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */
-/* TODO(shess) In the case where the parent is reading these values
-** from a PLReader, we could optimize to a copy if that PLReader has
-** the same type as pWriter.
-*/
-static void plwAdd(PLWriter *pWriter, int iColumn, int iPos,
- int iStartOffset, int iEndOffset){
- /* Worst-case space for POS_COLUMN, iColumn, iPosDelta,
- ** iStartOffsetDelta, and iEndOffsetDelta.
- */
- char c[5*VARINT_MAX];
- int n = 0;
+#ifdef SQLITE_TEST
+ const char *zTestParam = 0;
+ if( strncmp(argv[argc-1], "test:", 5)==0 ){
+ zTestParam = argv[argc-1];
+ argc--;
+ }
+#endif
+
+ nDb = (int)strlen(argv[1]) + 1;
+ nName = (int)strlen(argv[2]) + 1;
+ for(i=3; i<argc; i++){
+ char const *z = argv[i];
+ rc = sqlite3Fts3InitTokenizer(pHash, z, &pTokenizer, &zTokenizer, pzErr);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+ if( z!=zTokenizer ){
+ nString += (int)(strlen(z) + 1);
+ }
+ }
+ nCol = argc - 3 - (zTokenizer!=0);
+ if( zTokenizer==0 ){
+ rc = sqlite3Fts3InitTokenizer(pHash, 0, &pTokenizer, 0, pzErr);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+ assert( pTokenizer );
+ }
- /* Ban plwAdd() after plwTerminate(). */
- assert( pWriter->iPos!=-1 );
+ if( nCol==0 ){
+ nCol = 1;
+ }
- if( pWriter->dlw->iType==DL_DOCIDS ) return;
+ /* Allocate and populate the Fts3Table structure. */
+ nByte = sizeof(Fts3Table) + /* Fts3Table */
+ nCol * sizeof(char *) + /* azColumn */
+ nName + /* zName */
+ nDb + /* zDb */
+ nString; /* Space for azColumn strings */
+ p = (Fts3Table*)sqlite3_malloc(nByte);
+ if( p==0 ){
+ rc = SQLITE_NOMEM;
+ goto fts3_init_out;
+ }
+ memset(p, 0, nByte);
- if( iColumn!=pWriter->iColumn ){
- n += fts3PutVarint(c+n, POS_COLUMN);
- n += fts3PutVarint(c+n, iColumn);
- pWriter->iColumn = iColumn;
- pWriter->iPos = 0;
- pWriter->iOffset = 0;
+ p->db = db;
+ p->nColumn = nCol;
+ p->nPendingData = 0;
+ p->azColumn = (char **)&p[1];
+ p->pTokenizer = pTokenizer;
+ p->nNodeSize = 1000;
+ zCsr = (char *)&p->azColumn[nCol];
+
+ fts3HashInit(&p->pendingTerms, FTS3_HASH_STRING, 1);
+
+ /* Fill in the zName and zDb fields of the vtab structure. */
+ p->zName = zCsr;
+ memcpy(zCsr, argv[2], nName);
+ zCsr += nName;
+ p->zDb = zCsr;
+ memcpy(zCsr, argv[1], nDb);
+ zCsr += nDb;
+
+ /* Fill in the azColumn array */
+ iCol = 0;
+ for(i=3; i<argc; i++){
+ if( argv[i]!=zTokenizer ){
+ char *z;
+ int n;
+ z = (char *)sqlite3Fts3NextToken(argv[i], &n);
+ memcpy(zCsr, z, n);
+ zCsr[n] = '\0';
+ sqlite3Fts3Dequote(zCsr);
+ p->azColumn[iCol++] = zCsr;
+ zCsr += n+1;
+ assert( zCsr <= &((char *)p)[nByte] );
+ }
}
- assert( iPos>=pWriter->iPos );
- n += fts3PutVarint(c+n, POS_BASE+(iPos-pWriter->iPos));
- pWriter->iPos = iPos;
- if( pWriter->dlw->iType==DL_POSITIONS_OFFSETS ){
- assert( iStartOffset>=pWriter->iOffset );
- n += fts3PutVarint(c+n, iStartOffset-pWriter->iOffset);
- pWriter->iOffset = iStartOffset;
- assert( iEndOffset>=iStartOffset );
- n += fts3PutVarint(c+n, iEndOffset-iStartOffset);
+ if( iCol==0 ){
+ assert( nCol==1 );
+ p->azColumn[0] = "content";
}
- dataBufferAppend(pWriter->dlw->b, c, n);
-}
-static void plwCopy(PLWriter *pWriter, PLReader *pReader){
- plwAdd(pWriter, plrColumn(pReader), plrPosition(pReader),
- plrStartOffset(pReader), plrEndOffset(pReader));
-}
-static void plwInit(PLWriter *pWriter, DLWriter *dlw, sqlite_int64 iDocid){
- char c[VARINT_MAX];
- int n;
- pWriter->dlw = dlw;
+ /* If this is an xCreate call, create the underlying tables in the
+ ** database. TODO: For xConnect(), it could verify that said tables exist.
+ */
+ if( isCreate ){
+ rc = fts3CreateTables(p);
+ if( rc!=SQLITE_OK ) goto fts3_init_out;
+ }
- /* Docids must ascend. */
- assert( !pWriter->dlw->has_iPrevDocid || iDocid>pWriter->dlw->iPrevDocid );
- n = fts3PutVarint(c, iDocid-pWriter->dlw->iPrevDocid);
- dataBufferAppend(pWriter->dlw->b, c, n);
- pWriter->dlw->iPrevDocid = iDocid;
-#ifndef NDEBUG
- pWriter->dlw->has_iPrevDocid = 1;
-#endif
+ rc = fts3DeclareVtab(p);
+ if( rc!=SQLITE_OK ) goto fts3_init_out;
- pWriter->iColumn = 0;
- pWriter->iPos = 0;
- pWriter->iOffset = 0;
-}
-/* TODO(shess) Should plwDestroy() also terminate the doclist? But
-** then plwDestroy() would no longer be just a destructor, it would
-** also be doing work, which isn't consistent with the overall idiom.
-** Another option would be for plwAdd() to always append any necessary
-** terminator, so that the output is always correct. But that would
-** add incremental work to the common case with the only benefit being
-** API elegance. Punt for now.
-*/
-static void plwTerminate(PLWriter *pWriter){
- if( pWriter->dlw->iType>DL_DOCIDS ){
- char c[VARINT_MAX];
- int n = fts3PutVarint(c, POS_END);
- dataBufferAppend(pWriter->dlw->b, c, n);
+#ifdef SQLITE_TEST
+ if( zTestParam ){
+ p->nNodeSize = atoi(&zTestParam[5]);
}
-#ifndef NDEBUG
- /* Mark as terminated for assert in plwAdd(). */
- pWriter->iPos = -1;
-#endif
-}
-static void plwDestroy(PLWriter *pWriter){
- SCRAMBLE(pWriter);
-}
-
-/*******************************************************************/
-/* DLCollector wraps PLWriter and DLWriter to provide a
-** dynamically-allocated doclist area to use during tokenization.
-**
-** dlcNew - malloc up and initialize a collector.
-** dlcDelete - destroy a collector and all contained items.
-** dlcAddPos - append position and offset information.
-** dlcAddDoclist - add the collected doclist to the given buffer.
-** dlcNext - terminate the current document and open another.
-*/
-typedef struct DLCollector {
- DataBuffer b;
- DLWriter dlw;
- PLWriter plw;
-} DLCollector;
-
-/* TODO(shess) This could also be done by calling plwTerminate() and
-** dataBufferAppend(). I tried that, expecting nominal performance
-** differences, but it seemed to pretty reliably be worth 1% to code
-** it this way. I suspect it is the incremental malloc overhead (some
-** percentage of the plwTerminate() calls will cause a realloc), so
-** this might be worth revisiting if the DataBuffer implementation
-** changes.
-*/
-static void dlcAddDoclist(DLCollector *pCollector, DataBuffer *b){
- if( pCollector->dlw.iType>DL_DOCIDS ){
- char c[VARINT_MAX];
- int n = fts3PutVarint(c, POS_END);
- dataBufferAppend2(b, pCollector->b.pData, pCollector->b.nData, c, n);
- }else{
- dataBufferAppend(b, pCollector->b.pData, pCollector->b.nData);
+#endif
+ *ppVTab = &p->base;
+
+fts3_init_out:
+ assert( p || (pTokenizer && rc!=SQLITE_OK) );
+ if( rc!=SQLITE_OK ){
+ if( p ){
+ fts3DisconnectMethod((sqlite3_vtab *)p);
+ }else{
+ pTokenizer->pModule->xDestroy(pTokenizer);
+ }
}
-}
-static void dlcNext(DLCollector *pCollector, sqlite_int64 iDocid){
- plwTerminate(&pCollector->plw);
- plwDestroy(&pCollector->plw);
- plwInit(&pCollector->plw, &pCollector->dlw, iDocid);
-}
-static void dlcAddPos(DLCollector *pCollector, int iColumn, int iPos,
- int iStartOffset, int iEndOffset){
- plwAdd(&pCollector->plw, iColumn, iPos, iStartOffset, iEndOffset);
+ return rc;
}
-static DLCollector *dlcNew(sqlite_int64 iDocid, DocListType iType){
- DLCollector *pCollector = sqlite3_malloc(sizeof(DLCollector));
- dataBufferInit(&pCollector->b, 0);
- dlwInit(&pCollector->dlw, iType, &pCollector->b);
- plwInit(&pCollector->plw, &pCollector->dlw, iDocid);
- return pCollector;
-}
-static void dlcDelete(DLCollector *pCollector){
- plwDestroy(&pCollector->plw);
- dlwDestroy(&pCollector->dlw);
- dataBufferDestroy(&pCollector->b);
- SCRAMBLE(pCollector);
- sqlite3_free(pCollector);
+/*
+** The xConnect() and xCreate() methods for the virtual table. All the
+** work is done in function fts3InitVtab().
+*/
+static int fts3ConnectMethod(
+ sqlite3 *db, /* Database connection */
+ void *pAux, /* Pointer to tokenizer hash table */
+ int argc, /* Number of elements in argv array */
+ const char * const *argv, /* xCreate/xConnect argument array */
+ sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
+ char **pzErr /* OUT: sqlite3_malloc'd error message */
+){
+ return fts3InitVtab(0, db, pAux, argc, argv, ppVtab, pzErr);
+}
+static int fts3CreateMethod(
+ sqlite3 *db, /* Database connection */
+ void *pAux, /* Pointer to tokenizer hash table */
+ int argc, /* Number of elements in argv array */
+ const char * const *argv, /* xCreate/xConnect argument array */
+ sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
+ char **pzErr /* OUT: sqlite3_malloc'd error message */
+){
+ return fts3InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr);
}
-
-/* Copy the doclist data of iType in pData/nData into *out, trimming
-** unnecessary data as we go. Only columns matching iColumn are
-** copied, all columns copied if iColumn is -1. Elements with no
-** matching columns are dropped. The output is an iOutType doclist.
-*/
-/* NOTE(shess) This code is only valid after all doclists are merged.
-** If this is run before merges, then doclist items which represent
-** deletion will be trimmed, and will thus not effect a deletion
-** during the merge.
+/*
+** Implementation of the xBestIndex method for FTS3 tables. There
+** are three possible strategies, in order of preference:
+**
+** 1. Direct lookup by rowid or docid.
+** 2. Full-text search using a MATCH operator on a non-docid column.
+** 3. Linear scan of %_content table.
*/
-static void docListTrim(DocListType iType, const char *pData, int nData,
- int iColumn, DocListType iOutType, DataBuffer *out){
- DLReader dlReader;
- DLWriter dlWriter;
-
- assert( iOutType<=iType );
-
- dlrInit(&dlReader, iType, pData, nData);
- dlwInit(&dlWriter, iOutType, out);
-
- while( !dlrAtEnd(&dlReader) ){
- PLReader plReader;
- PLWriter plWriter;
- int match = 0;
+static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
+ Fts3Table *p = (Fts3Table *)pVTab;
+ int i; /* Iterator variable */
+ int iCons = -1; /* Index of constraint to use */
- plrInit(&plReader, &dlReader);
+ /* By default use a full table scan. This is an expensive option,
+ ** so search through the constraints to see if a more efficient
+ ** strategy is possible.
+ */
+ pInfo->idxNum = FTS3_FULLSCAN_SEARCH;
+ pInfo->estimatedCost = 500000;
+ for(i=0; i<pInfo->nConstraint; i++){
+ struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i];
+ if( pCons->usable==0 ) continue;
- while( !plrAtEnd(&plReader) ){
- if( iColumn==-1 || plrColumn(&plReader)==iColumn ){
- if( !match ){
- plwInit(&plWriter, &dlWriter, dlrDocid(&dlReader));
- match = 1;
- }
- plwAdd(&plWriter, plrColumn(&plReader), plrPosition(&plReader),
- plrStartOffset(&plReader), plrEndOffset(&plReader));
- }
- plrStep(&plReader);
- }
- if( match ){
- plwTerminate(&plWriter);
- plwDestroy(&plWriter);
+ /* A direct lookup on the rowid or docid column. This is the best
+ ** strategy in all cases. Assign a cost of 1.0 and return early.
+ */
+ if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ
+ && (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1 )
+ ){
+ pInfo->idxNum = FTS3_DOCID_SEARCH;
+ pInfo->estimatedCost = 1.0;
+ iCons = i;
+ break;
}
- plrDestroy(&plReader);
- dlrStep(&dlReader);
+ /* A MATCH constraint. Use a full-text search.
+ **
+ ** If there is more than one MATCH constraint available, use the first
+ ** one encountered. If there is both a MATCH constraint and a direct
+ ** rowid/docid lookup, prefer the rowid/docid strategy.
+ */
+ if( iCons<0
+ && pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH
+ && pCons->iColumn>=0 && pCons->iColumn<=p->nColumn
+ ){
+ pInfo->idxNum = FTS3_FULLTEXT_SEARCH + pCons->iColumn;
+ pInfo->estimatedCost = 2.0;
+ iCons = i;
+ }
}
- dlwDestroy(&dlWriter);
- dlrDestroy(&dlReader);
+
+ if( iCons>=0 ){
+ pInfo->aConstraintUsage[iCons].argvIndex = 1;
+ pInfo->aConstraintUsage[iCons].omit = 1;
+ }
+ return SQLITE_OK;
}
-/* Used by docListMerge() to keep doclists in the ascending order by
-** docid, then ascending order by age (so the newest comes first).
+/*
+** Implementation of xOpen method.
*/
-typedef struct OrderedDLReader {
- DLReader *pReader;
+static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
+ sqlite3_vtab_cursor *pCsr; /* Allocated cursor */
- /* TODO(shess) If we assume that docListMerge pReaders is ordered by
- ** age (which we do), then we could use pReader comparisons to break
- ** ties.
- */
- int idx;
-} OrderedDLReader;
+ UNUSED_PARAMETER(pVTab);
-/* Order eof to end, then by docid asc, idx desc. */
-static int orderedDLReaderCmp(OrderedDLReader *r1, OrderedDLReader *r2){
- if( dlrAtEnd(r1->pReader) ){
- if( dlrAtEnd(r2->pReader) ) return 0; /* Both atEnd(). */
- return 1; /* Only r1 atEnd(). */
+ /* Allocate a buffer large enough for an Fts3Cursor structure. If the
+ ** allocation succeeds, zero it and return SQLITE_OK. Otherwise,
+ ** if the allocation fails, return SQLITE_NOMEM.
+ */
+ *ppCsr = pCsr = (sqlite3_vtab_cursor *)sqlite3_malloc(sizeof(Fts3Cursor));
+ if( !pCsr ){
+ return SQLITE_NOMEM;
}
- if( dlrAtEnd(r2->pReader) ) return -1; /* Only r2 atEnd(). */
+ memset(pCsr, 0, sizeof(Fts3Cursor));
+ return SQLITE_OK;
+}
- if( dlrDocid(r1->pReader)<dlrDocid(r2->pReader) ) return -1;
- if( dlrDocid(r1->pReader)>dlrDocid(r2->pReader) ) return 1;
+/****************************************************************/
+/****************************************************************/
+/****************************************************************/
+/****************************************************************/
- /* Descending on idx. */
- return r2->idx-r1->idx;
-}
-/* Bubble p[0] to appropriate place in p[1..n-1]. Assumes that
-** p[1..n-1] is already sorted.
-*/
-/* TODO(shess) Is this frequent enough to warrant a binary search?
-** Before implementing that, instrument the code to check. In most
-** current usage, I expect that p[0] will be less than p[1] a very
-** high proportion of the time.
+/*
+** Close the cursor. For additional information see the documentation
+** on the xClose method of the virtual table interface.
*/
-static void orderedDLReaderReorder(OrderedDLReader *p, int n){
- while( n>1 && orderedDLReaderCmp(p, p+1)>0 ){
- OrderedDLReader tmp = p[0];
- p[0] = p[1];
- p[1] = tmp;
- n--;
- p++;
+static int fulltextClose(sqlite3_vtab_cursor *pCursor){
+ Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
+ sqlite3_finalize(pCsr->pStmt);
+ sqlite3Fts3ExprFree(pCsr->pExpr);
+ sqlite3_free(pCsr->aDoclist);
+ sqlite3_free(pCsr);
+ return SQLITE_OK;
+}
+
+static int fts3CursorSeek(Fts3Cursor *pCsr){
+ if( pCsr->isRequireSeek ){
+ pCsr->isRequireSeek = 0;
+ sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId);
+ if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){
+ return SQLITE_OK;
+ }else{
+ int rc;
+ pCsr->isEof = 1;
+ if( SQLITE_OK==(rc = sqlite3_reset(pCsr->pStmt)) ){
+ rc = SQLITE_ERROR;
+ }
+ return rc;
+ }
+ }else{
+ return SQLITE_OK;
}
}
-/* Given an array of doclist readers, merge their doclist elements
-** into out in sorted order (by docid), dropping elements from older
-** readers when there is a duplicate docid. pReaders is assumed to be
-** ordered by age, oldest first.
-*/
-/* TODO(shess) nReaders must be <= MERGE_COUNT. This should probably
-** be fixed.
-*/
-static void docListMerge(DataBuffer *out,
- DLReader *pReaders, int nReaders){
- OrderedDLReader readers[MERGE_COUNT];
- DLWriter writer;
- int i, n;
- const char *pStart = 0;
- int nStart = 0;
- sqlite_int64 iFirstDocid = 0, iLastDocid = 0;
+static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){
+ int rc = SQLITE_OK; /* Return code */
+ Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
- assert( nReaders>0 );
- if( nReaders==1 ){
- dataBufferAppend(out, dlrDocData(pReaders), dlrAllDataBytes(pReaders));
- return;
+ if( pCsr->aDoclist==0 ){
+ if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){
+ pCsr->isEof = 1;
+ rc = sqlite3_reset(pCsr->pStmt);
+ }
+ }else if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){
+ pCsr->isEof = 1;
+ }else{
+ sqlite3_reset(pCsr->pStmt);
+ fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId);
+ pCsr->isRequireSeek = 1;
}
+ return rc;
+}
- assert( nReaders<=MERGE_COUNT );
- n = 0;
- for(i=0; i<nReaders; i++){
- assert( pReaders[i].iType==pReaders[0].iType );
- readers[i].pReader = pReaders+i;
- readers[i].idx = i;
- n += dlrAllDataBytes(&pReaders[i]);
- }
- /* Conservatively size output to sum of inputs. Output should end
- ** up strictly smaller than input.
- */
- dataBufferExpand(out, n);
- /* Get the readers into sorted order. */
- while( i-->0 ){
- orderedDLReaderReorder(readers+i, nReaders-i);
- }
+/*
+** The buffer pointed to by argument zNode (size nNode bytes) contains the
+** root node of a b-tree segment. The segment is guaranteed to be at least
+** one level high (i.e. the root node is not also a leaf). If successful,
+** this function locates the leaf node of the segment that may contain the
+** term specified by arguments zTerm and nTerm and writes its block number
+** to *piLeaf.
+**
+** It is possible that the returned leaf node does not contain the specified
+** term. However, if the segment does contain said term, it is stored on
+** the identified leaf node. Because this function only inspects interior
+** segment nodes (and never loads leaf nodes into memory), it is not possible
+** to be sure.
+**
+** If an error occurs, an error code other than SQLITE_OK is returned.
+*/
+static int fts3SelectLeaf(
+ Fts3Table *p, /* Virtual table handle */
+ const char *zTerm, /* Term to select leaves for */
+ int nTerm, /* Size of term zTerm in bytes */
+ const char *zNode, /* Buffer containing segment interior node */
+ int nNode, /* Size of buffer at zNode */
+ sqlite3_int64 *piLeaf /* Selected leaf node */
+){
+ int rc = SQLITE_OK; /* Return code */
+ const char *zCsr = zNode; /* Cursor to iterate through node */
+ const char *zEnd = &zCsr[nNode];/* End of interior node buffer */
+ char *zBuffer = 0; /* Buffer to load terms into */
+ int nAlloc = 0; /* Size of allocated buffer */
- dlwInit(&writer, pReaders[0].iType, out);
- while( !dlrAtEnd(readers[0].pReader) ){
- sqlite_int64 iDocid = dlrDocid(readers[0].pReader);
+ while( 1 ){
+ int isFirstTerm = 1; /* True when processing first term on page */
+ int iHeight; /* Height of this node in tree */
+ sqlite3_int64 iChild; /* Block id of child node to descend to */
+ int nBlock; /* Size of child node in bytes */
- /* If this is a continuation of the current buffer to copy, extend
- ** that buffer. memcpy() seems to be more efficient if it has a
- ** lots of data to copy.
- */
- if( dlrDocData(readers[0].pReader)==pStart+nStart ){
- nStart += dlrDocDataBytes(readers[0].pReader);
- }else{
- if( pStart!=0 ){
- dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid);
+ zCsr += sqlite3Fts3GetVarint32(zCsr, &iHeight);
+ zCsr += sqlite3Fts3GetVarint(zCsr, &iChild);
+
+ while( zCsr<zEnd ){
+ int cmp; /* memcmp() result */
+ int nSuffix; /* Size of term suffix */
+ int nPrefix = 0; /* Size of term prefix */
+ int nBuffer; /* Total term size */
+
+ /* Load the next term on the node into zBuffer */
+ if( !isFirstTerm ){
+ zCsr += sqlite3Fts3GetVarint32(zCsr, &nPrefix);
+ }
+ isFirstTerm = 0;
+ zCsr += sqlite3Fts3GetVarint32(zCsr, &nSuffix);
+ if( nPrefix+nSuffix>nAlloc ){
+ char *zNew;
+ nAlloc = (nPrefix+nSuffix) * 2;
+ zNew = (char *)sqlite3_realloc(zBuffer, nAlloc);
+ if( !zNew ){
+ sqlite3_free(zBuffer);
+ return SQLITE_NOMEM;
+ }
+ zBuffer = zNew;
}
- pStart = dlrDocData(readers[0].pReader);
- nStart = dlrDocDataBytes(readers[0].pReader);
- iFirstDocid = iDocid;
- }
- iLastDocid = iDocid;
- dlrStep(readers[0].pReader);
+ memcpy(&zBuffer[nPrefix], zCsr, nSuffix);
+ nBuffer = nPrefix + nSuffix;
+ zCsr += nSuffix;
+
+ /* Compare the term we are searching for with the term just loaded from
+ ** the interior node. If the specified term is greater than or equal
+ ** to the term from the interior node, then all terms on the sub-tree
+ ** headed by node iChild are smaller than zTerm. No need to search
+ ** iChild.
+ **
+ ** If the interior node term is larger than the specified term, then
+ ** the tree headed by iChild may contain the specified term.
+ */
+ cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer));
+ if( cmp<0 || (cmp==0 && nBuffer>nTerm) ) break;
+ iChild++;
+ };
- /* Drop all of the older elements with the same docid. */
- for(i=1; i<nReaders &&
- !dlrAtEnd(readers[i].pReader) &&
- dlrDocid(readers[i].pReader)==iDocid; i++){
- dlrStep(readers[i].pReader);
+ /* If (iHeight==1), the children of this interior node are leaves. The
+ ** specified term may be present on leaf node iChild.
+ */
+ if( iHeight==1 ){
+ *piLeaf = iChild;
+ break;
}
- /* Get the readers back into order. */
- while( i-->0 ){
- orderedDLReaderReorder(readers+i, nReaders-i);
- }
+ /* Descend to interior node iChild. */
+ rc = sqlite3Fts3ReadBlock(p, iChild, &zCsr, &nBlock);
+ if( rc!=SQLITE_OK ) break;
+ zEnd = &zCsr[nBlock];
}
-
- /* Copy over any remaining elements. */
- if( nStart>0 ) dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid);
- dlwDestroy(&writer);
+ sqlite3_free(zBuffer);
+ return rc;
}
-/* Helper function for posListUnion(). Compares the current position
-** between left and right, returning as standard C idiom of <0 if
-** left<right, >0 if left>right, and 0 if left==right. "End" always
-** compares greater.
+/*
+** This function is used to create delta-encoded serialized lists of FTS3
+** varints. Each call to this function appends a single varint to a list.
*/
-static int posListCmp(PLReader *pLeft, PLReader *pRight){
- assert( pLeft->iType==pRight->iType );
- if( pLeft->iType==DL_DOCIDS ) return 0;
-
- if( plrAtEnd(pLeft) ) return plrAtEnd(pRight) ? 0 : 1;
- if( plrAtEnd(pRight) ) return -1;
-
- if( plrColumn(pLeft)<plrColumn(pRight) ) return -1;
- if( plrColumn(pLeft)>plrColumn(pRight) ) return 1;
-
- if( plrPosition(pLeft)<plrPosition(pRight) ) return -1;
- if( plrPosition(pLeft)>plrPosition(pRight) ) return 1;
- if( pLeft->iType==DL_POSITIONS ) return 0;
+static void fts3PutDeltaVarint(
+ char **pp, /* IN/OUT: Output pointer */
+ sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */
+ sqlite3_int64 iVal /* Write this value to the list */
+){
+ assert( iVal-*piPrev > 0 || (*piPrev==0 && iVal==0) );
+ *pp += sqlite3Fts3PutVarint(*pp, iVal-*piPrev);
+ *piPrev = iVal;
+}
- if( plrStartOffset(pLeft)<plrStartOffset(pRight) ) return -1;
- if( plrStartOffset(pLeft)>plrStartOffset(pRight) ) return 1;
+static void fts3PoslistCopy(char **pp, char **ppPoslist){
+ char *pEnd = *ppPoslist;
+ char c = 0;
+ while( *pEnd | c ) c = *pEnd++ & 0x80;
+ pEnd++;
+ if( pp ){
+ int n = (int)(pEnd - *ppPoslist);
+ char *p = *pp;
+ memcpy(p, *ppPoslist, n);
+ p += n;
+ *pp = p;
+ }
+ *ppPoslist = pEnd;
+}
- if( plrEndOffset(pLeft)<plrEndOffset(pRight) ) return -1;
- if( plrEndOffset(pLeft)>plrEndOffset(pRight) ) return 1;
+static void fts3ColumnlistCopy(char **pp, char **ppPoslist){
+ char *pEnd = *ppPoslist;
+ char c = 0;
- return 0;
+ /* A column-list is terminated by either a 0x01 or 0x00. */
+ while( 0xFE & (*pEnd | c) ) c = *pEnd++ & 0x80;
+ if( pp ){
+ int n = (int)(pEnd - *ppPoslist);
+ char *p = *pp;
+ memcpy(p, *ppPoslist, n);
+ p += n;
+ *pp = p;
+ }
+ *ppPoslist = pEnd;
}
-/* Write the union of position lists in pLeft and pRight to pOut.
-** "Union" in this case meaning "All unique position tuples". Should
-** work with any doclist type, though both inputs and the output
-** should be the same type.
+/*
+** Value used to signify the end of an offset-list. This is safe because
+** it is not possible to have a document with 2^31 terms.
*/
-static void posListUnion(DLReader *pLeft, DLReader *pRight, DLWriter *pOut){
- PLReader left, right;
- PLWriter writer;
+#define OFFSET_LIST_END 0x7fffffff
- assert( dlrDocid(pLeft)==dlrDocid(pRight) );
- assert( pLeft->iType==pRight->iType );
- assert( pLeft->iType==pOut->iType );
+/*
+** This function is used to help parse offset-lists. When this function is
+** called, *pp may point to the start of the next varint in the offset-list
+** being parsed, or it may point to 1 byte past the end of the offset-list
+** (in which case **pp will be 0x00 or 0x01).
+**
+** If *pp points past the end of the current offset list, set *pi to
+** OFFSET_LIST_END and return. Otherwise, read the next varint from *pp,
+** increment the current value of *pi by the value read, and set *pp to
+** point to the next value before returning.
+*/
+static void fts3ReadNextPos(
+ char **pp, /* IN/OUT: Pointer into offset-list buffer */
+ sqlite3_int64 *pi /* IN/OUT: Value read from offset-list */
+){
+ if( **pp&0xFE ){
+ fts3GetDeltaVarint(pp, pi);
+ *pi -= 2;
+ }else{
+ *pi = OFFSET_LIST_END;
+ }
+}
- plrInit(&left, pLeft);
- plrInit(&right, pRight);
- plwInit(&writer, pOut, dlrDocid(pLeft));
+/*
+** If parameter iCol is not 0, write an 0x01 byte followed by the value of
+** iCol encoded as a varint to *pp.
+**
+** Set *pp to point to the byte just after the last byte written before
+** returning (do not modify it if iCol==0). Return the total number of bytes
+** written (0 if iCol==0).
+*/
+static int fts3PutColNumber(char **pp, int iCol){
+ int n = 0; /* Number of bytes written */
+ if( iCol ){
+ char *p = *pp; /* Output pointer */
+ n = 1 + sqlite3Fts3PutVarint(&p[1], iCol);
+ *p = 0x01;
+ *pp = &p[n];
+ }
+ return n;
+}
- while( !plrAtEnd(&left) || !plrAtEnd(&right) ){
- int c = posListCmp(&left, &right);
- if( c<0 ){
- plwCopy(&writer, &left);
- plrStep(&left);
- }else if( c>0 ){
- plwCopy(&writer, &right);
- plrStep(&right);
+/*
+**
+*/
+static void fts3PoslistMerge(
+ char **pp, /* Output buffer */
+ char **pp1, /* Left input list */
+ char **pp2 /* Right input list */
+){
+ char *p = *pp;
+ char *p1 = *pp1;
+ char *p2 = *pp2;
+
+ while( *p1 || *p2 ){
+ int iCol1;
+ int iCol2;
+
+ if( *p1==0x01 ) sqlite3Fts3GetVarint32(&p1[1], &iCol1);
+ else if( *p1==0x00 ) iCol1 = OFFSET_LIST_END;
+ else iCol1 = 0;
+
+ if( *p2==0x01 ) sqlite3Fts3GetVarint32(&p2[1], &iCol2);
+ else if( *p2==0x00 ) iCol2 = OFFSET_LIST_END;
+ else iCol2 = 0;
+
+ if( iCol1==iCol2 ){
+ sqlite3_int64 i1 = 0;
+ sqlite3_int64 i2 = 0;
+ sqlite3_int64 iPrev = 0;
+ int n = fts3PutColNumber(&p, iCol1);
+ p1 += n;
+ p2 += n;
+
+ /* At this point, both p1 and p2 point to the start of offset-lists.
+ ** An offset-list is a list of non-negative delta-encoded varints, each
+ ** incremented by 2 before being stored. Each list is terminated by a 0
+ ** or 1 value (0x00 or 0x01). The following block merges the two lists
+ ** and writes the results to buffer p. p is left pointing to the byte
+ ** after the list written. No terminator (0x00 or 0x01) is written to
+ ** the output.
+ */
+ fts3GetDeltaVarint(&p1, &i1);
+ fts3GetDeltaVarint(&p2, &i2);
+ do {
+ fts3PutDeltaVarint(&p, &iPrev, (i1<i2) ? i1 : i2);
+ iPrev -= 2;
+ if( i1==i2 ){
+ fts3ReadNextPos(&p1, &i1);
+ fts3ReadNextPos(&p2, &i2);
+ }else if( i1<i2 ){
+ fts3ReadNextPos(&p1, &i1);
+ }else{
+ fts3ReadNextPos(&p2, &i2);
+ }
+ }while( i1!=OFFSET_LIST_END || i2!=OFFSET_LIST_END );
+ }else if( iCol1<iCol2 ){
+ p1 += fts3PutColNumber(&p, iCol1);
+ fts3ColumnlistCopy(&p, &p1);
}else{
- plwCopy(&writer, &left);
- plrStep(&left);
- plrStep(&right);
+ p2 += fts3PutColNumber(&p, iCol2);
+ fts3ColumnlistCopy(&p, &p2);
}
}
- plwTerminate(&writer);
- plwDestroy(&writer);
- plrDestroy(&left);
- plrDestroy(&right);
+ *p++ = '\0';
+ *pp = p;
+ *pp1 = p1 + 1;
+ *pp2 = p2 + 1;
}
-/* Write the union of doclists in pLeft and pRight to pOut. For
-** docids in common between the inputs, the union of the position
-** lists is written. Inputs and outputs are always type DL_DEFAULT.
+/*
+** nToken==1 searches for adjacent positions.
*/
-static void docListUnion(
- const char *pLeft, int nLeft,
- const char *pRight, int nRight,
- DataBuffer *pOut /* Write the combined doclist here */
+static int fts3PoslistPhraseMerge(
+ char **pp, /* Output buffer */
+ int nToken, /* Maximum difference in token positions */
+ int isSaveLeft, /* Save the left position */
+ char **pp1, /* Left input list */
+ char **pp2 /* Right input list */
){
- DLReader left, right;
- DLWriter writer;
+ char *p = (pp ? *pp : 0);
+ char *p1 = *pp1;
+ char *p2 = *pp2;
- if( nLeft==0 ){
- if( nRight!=0) dataBufferAppend(pOut, pRight, nRight);
- return;
- }
- if( nRight==0 ){
- dataBufferAppend(pOut, pLeft, nLeft);
- return;
+ int iCol1 = 0;
+ int iCol2 = 0;
+ assert( *p1!=0 && *p2!=0 );
+ if( *p1==0x01 ){
+ p1++;
+ p1 += sqlite3Fts3GetVarint32(p1, &iCol1);
}
-
- dlrInit(&left, DL_DEFAULT, pLeft, nLeft);
- dlrInit(&right, DL_DEFAULT, pRight, nRight);
- dlwInit(&writer, DL_DEFAULT, pOut);
-
- while( !dlrAtEnd(&left) || !dlrAtEnd(&right) ){
- if( dlrAtEnd(&right) ){
- dlwCopy(&writer, &left);
- dlrStep(&left);
- }else if( dlrAtEnd(&left) ){
- dlwCopy(&writer, &right);
- dlrStep(&right);
- }else if( dlrDocid(&left)<dlrDocid(&right) ){
- dlwCopy(&writer, &left);
- dlrStep(&left);
- }else if( dlrDocid(&left)>dlrDocid(&right) ){
- dlwCopy(&writer, &right);
- dlrStep(&right);
- }else{
- posListUnion(&left, &right, &writer);
- dlrStep(&left);
- dlrStep(&right);
- }
+ if( *p2==0x01 ){
+ p2++;
+ p2 += sqlite3Fts3GetVarint32(p2, &iCol2);
}
- dlrDestroy(&left);
- dlrDestroy(&right);
- dlwDestroy(&writer);
-}
-
-/*
-** This function is used as part of the implementation of phrase and
-** NEAR matching.
-**
-** pLeft and pRight are DLReaders positioned to the same docid in
-** lists of type DL_POSITION. This function writes an entry to the
-** DLWriter pOut for each position in pRight that is less than
-** (nNear+1) greater (but not equal to or smaller) than a position
-** in pLeft. For example, if nNear is 0, and the positions contained
-** by pLeft and pRight are:
-**
-** pLeft: 5 10 15 20
-** pRight: 6 9 17 21
-**
-** then the docid is added to pOut. If pOut is of type DL_POSITIONS,
-** then a positionids "6" and "21" are also added to pOut.
-**
-** If boolean argument isSaveLeft is true, then positionids are copied
-** from pLeft instead of pRight. In the example above, the positions "5"
-** and "20" would be added instead of "6" and "21".
-*/
-static void posListPhraseMerge(
- DLReader *pLeft,
- DLReader *pRight,
- int nNear,
- int isSaveLeft,
- DLWriter *pOut
-){
- PLReader left, right;
- PLWriter writer;
- int match = 0;
+ while( 1 ){
+ if( iCol1==iCol2 ){
+ char *pSave = p;
+ sqlite3_int64 iPrev = 0;
+ sqlite3_int64 iPos1 = 0;
+ sqlite3_int64 iPos2 = 0;
- assert( dlrDocid(pLeft)==dlrDocid(pRight) );
- assert( pOut->iType!=DL_POSITIONS_OFFSETS );
+ if( pp && iCol1 ){
+ *p++ = 0x01;
+ p += sqlite3Fts3PutVarint(p, iCol1);
+ }
- plrInit(&left, pLeft);
- plrInit(&right, pRight);
+ assert( *p1!=0x00 && *p2!=0x00 && *p1!=0x01 && *p2!=0x01 );
+ fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2;
+ fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2;
- while( !plrAtEnd(&left) && !plrAtEnd(&right) ){
- if( plrColumn(&left)<plrColumn(&right) ){
- plrStep(&left);
- }else if( plrColumn(&left)>plrColumn(&right) ){
- plrStep(&right);
- }else if( plrPosition(&left)>=plrPosition(&right) ){
- plrStep(&right);
- }else{
- if( (plrPosition(&right)-plrPosition(&left))<=(nNear+1) ){
- if( !match ){
- plwInit(&writer, pOut, dlrDocid(pLeft));
- match = 1;
+ while( 1 ){
+ if( iPos2>iPos1 && iPos2<=iPos1+nToken ){
+ sqlite3_int64 iSave;
+ if( !pp ){
+ fts3PoslistCopy(0, &p2);
+ fts3PoslistCopy(0, &p1);
+ *pp1 = p1;
+ *pp2 = p2;
+ return 1;
+ }
+ iSave = isSaveLeft ? iPos1 : iPos2;
+ fts3PutDeltaVarint(&p, &iPrev, iSave+2); iPrev -= 2;
+ pSave = 0;
}
- if( !isSaveLeft ){
- plwAdd(&writer, plrColumn(&right), plrPosition(&right), 0, 0);
+ if( (!isSaveLeft && iPos2<=(iPos1+nToken)) || iPos2<=iPos1 ){
+ if( (*p2&0xFE)==0 ) break;
+ fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2;
}else{
- plwAdd(&writer, plrColumn(&left), plrPosition(&left), 0, 0);
+ if( (*p1&0xFE)==0 ) break;
+ fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2;
}
- plrStep(&right);
- }else{
- plrStep(&left);
+
}
+ if( pSave && pp ){
+ p = pSave;
+ }
+
+ fts3ColumnlistCopy(0, &p1);
+ fts3ColumnlistCopy(0, &p2);
+ assert( (*p1&0xFE)==0 && (*p2&0xFE)==0 );
+ if( 0==*p1 || 0==*p2 ) break;
+
+ p1++;
+ p1 += sqlite3Fts3GetVarint32(p1, &iCol1);
+ p2++;
+ p2 += sqlite3Fts3GetVarint32(p2, &iCol2);
}
- }
- if( match ){
- plwTerminate(&writer);
- plwDestroy(&writer);
+ /* Advance pointer p1 or p2 (whichever corresponds to the smaller of
+ ** iCol1 and iCol2) so that it points to either the 0x00 that marks the
+ ** end of the position list, or the 0x01 that precedes the next
+ ** column-number in the position list.
+ */
+ else if( iCol1<iCol2 ){
+ fts3ColumnlistCopy(0, &p1);
+ if( 0==*p1 ) break;
+ p1++;
+ p1 += sqlite3Fts3GetVarint32(p1, &iCol1);
+ }else{
+ fts3ColumnlistCopy(0, &p2);
+ if( 0==*p2 ) break;
+ p2++;
+ p2 += sqlite3Fts3GetVarint32(p2, &iCol2);
+ }
}
- plrDestroy(&left);
- plrDestroy(&right);
+ fts3PoslistCopy(0, &p2);
+ fts3PoslistCopy(0, &p1);
+ *pp1 = p1;
+ *pp2 = p2;
+ if( !pp || *pp==p ){
+ return 0;
+ }
+ *p++ = 0x00;
+ *pp = p;
+ return 1;
}
/*
-** Compare the values pointed to by the PLReaders passed as arguments.
-** Return -1 if the value pointed to by pLeft is considered less than
-** the value pointed to by pRight, +1 if it is considered greater
-** than it, or 0 if it is equal. i.e.
-**
-** (*pLeft - *pRight)
-**
-** A PLReader that is in the EOF condition is considered greater than
-** any other. If neither argument is in EOF state, the return value of
-** plrColumn() is used. If the plrColumn() values are equal, the
-** comparison is on the basis of plrPosition().
+** Merge two position-lists as required by the NEAR operator.
*/
-static int plrCompare(PLReader *pLeft, PLReader *pRight){
- assert(!plrAtEnd(pLeft) || !plrAtEnd(pRight));
+static int fts3PoslistNearMerge(
+ char **pp, /* Output buffer */
+ char *aTmp, /* Temporary buffer space */
+ int nRight, /* Maximum difference in token positions */
+ int nLeft, /* Maximum difference in token positions */
+ char **pp1, /* IN/OUT: Left input list */
+ char **pp2 /* IN/OUT: Right input list */
+){
+ char *p1 = *pp1;
+ char *p2 = *pp2;
+
+ if( !pp ){
+ if( fts3PoslistPhraseMerge(0, nRight, 0, pp1, pp2) ) return 1;
+ *pp1 = p1;
+ *pp2 = p2;
+ return fts3PoslistPhraseMerge(0, nLeft, 0, pp2, pp1);
+ }else{
+ char *pTmp1 = aTmp;
+ char *pTmp2;
+ char *aTmp2;
+ int res = 1;
+
+ fts3PoslistPhraseMerge(&pTmp1, nRight, 0, pp1, pp2);
+ aTmp2 = pTmp2 = pTmp1;
+ *pp1 = p1;
+ *pp2 = p2;
+ fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, pp2, pp1);
+ if( pTmp1!=aTmp && pTmp2!=aTmp2 ){
+ fts3PoslistMerge(pp, &aTmp, &aTmp2);
+ }else if( pTmp1!=aTmp ){
+ fts3PoslistCopy(pp, &aTmp);
+ }else if( pTmp2!=aTmp2 ){
+ fts3PoslistCopy(pp, &aTmp2);
+ }else{
+ res = 0;
+ }
- if( plrAtEnd(pRight) || plrAtEnd(pLeft) ){
- return (plrAtEnd(pRight) ? -1 : 1);
- }
- if( plrColumn(pLeft)!=plrColumn(pRight) ){
- return ((plrColumn(pLeft)<plrColumn(pRight)) ? -1 : 1);
+ return res;
}
- if( plrPosition(pLeft)!=plrPosition(pRight) ){
- return ((plrPosition(pLeft)<plrPosition(pRight)) ? -1 : 1);
- }
- return 0;
}
-/* We have two doclists with positions: pLeft and pRight. Depending
-** on the value of the nNear parameter, perform either a phrase
-** intersection (if nNear==0) or a NEAR intersection (if nNear>0)
-** and write the results into pOut.
-**
-** A phrase intersection means that two documents only match
-** if pLeft.iPos+1==pRight.iPos.
-**
-** A NEAR intersection means that two documents only match if
-** (abs(pLeft.iPos-pRight.iPos)<nNear).
-**
-** If a NEAR intersection is requested, then the nPhrase argument should
-** be passed the number of tokens in the two operands to the NEAR operator
-** combined. For example:
-**
-** Query syntax nPhrase
-** ------------------------------------
-** "A B C" NEAR "D E" 5
-** A NEAR B 2
+/*
+** Values that may be used as the first parameter to fts3DoclistMerge().
+*/
+#define MERGE_NOT 2 /* D + D -> D */
+#define MERGE_AND 3 /* D + D -> D */
+#define MERGE_OR 4 /* D + D -> D */
+#define MERGE_POS_OR 5 /* P + P -> P */
+#define MERGE_PHRASE 6 /* P + P -> D */
+#define MERGE_POS_PHRASE 7 /* P + P -> P */
+#define MERGE_NEAR 8 /* P + P -> D */
+#define MERGE_POS_NEAR 9 /* P + P -> P */
+
+/*
+** Merge the two doclists passed in buffer a1 (size n1 bytes) and a2
+** (size n2 bytes). The output is written to pre-allocated buffer aBuffer,
+** which is guaranteed to be large enough to hold the results. The number
+** of bytes written to aBuffer is stored in *pnBuffer before returning.
**
-** iType controls the type of data written to pOut. If iType is
-** DL_POSITIONS, the positions are those from pRight.
+** If successful, SQLITE_OK is returned. Otherwise, if a malloc error
+** occurs while allocating a temporary buffer as part of the merge operation,
+** SQLITE_NOMEM is returned.
*/
-static void docListPhraseMerge(
- const char *pLeft, int nLeft,
- const char *pRight, int nRight,
- int nNear, /* 0 for a phrase merge, non-zero for a NEAR merge */
- int nPhrase, /* Number of tokens in left+right operands to NEAR */
- DocListType iType, /* Type of doclist to write to pOut */
- DataBuffer *pOut /* Write the combined doclist here */
+static int fts3DoclistMerge(
+ int mergetype, /* One of the MERGE_XXX constants */
+ int nParam1, /* Used by MERGE_NEAR and MERGE_POS_NEAR */
+ int nParam2, /* Used by MERGE_NEAR and MERGE_POS_NEAR */
+ char *aBuffer, /* Pre-allocated output buffer */
+ int *pnBuffer, /* OUT: Bytes written to aBuffer */
+ char *a1, /* Buffer containing first doclist */
+ int n1, /* Size of buffer a1 */
+ char *a2, /* Buffer containing second doclist */
+ int n2 /* Size of buffer a2 */
){
- DLReader left, right;
- DLWriter writer;
+ sqlite3_int64 i1 = 0;
+ sqlite3_int64 i2 = 0;
+ sqlite3_int64 iPrev = 0;
+
+ char *p = aBuffer;
+ char *p1 = a1;
+ char *p2 = a2;
+ char *pEnd1 = &a1[n1];
+ char *pEnd2 = &a2[n2];
+
+ assert( mergetype==MERGE_OR || mergetype==MERGE_POS_OR
+ || mergetype==MERGE_AND || mergetype==MERGE_NOT
+ || mergetype==MERGE_PHRASE || mergetype==MERGE_POS_PHRASE
+ || mergetype==MERGE_NEAR || mergetype==MERGE_POS_NEAR
+ );
- if( nLeft==0 || nRight==0 ) return;
+ if( !aBuffer ){
+ return SQLITE_NOMEM;
+ }
+ if( n1==0 && n2==0 ){
+ *pnBuffer = 0;
+ return SQLITE_OK;
+ }
- assert( iType!=DL_POSITIONS_OFFSETS );
+ /* Read the first docid from each doclist */
+ fts3GetDeltaVarint2(&p1, pEnd1, &i1);
+ fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+
+ switch( mergetype ){
+ case MERGE_OR:
+ case MERGE_POS_OR:
+ while( p1 || p2 ){
+ if( p2 && p1 && i1==i2 ){
+ fts3PutDeltaVarint(&p, &iPrev, i1);
+ if( mergetype==MERGE_POS_OR ) fts3PoslistMerge(&p, &p1, &p2);
+ fts3GetDeltaVarint2(&p1, pEnd1, &i1);
+ fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+ }else if( !p2 || (p1 && i1<i2) ){
+ fts3PutDeltaVarint(&p, &iPrev, i1);
+ if( mergetype==MERGE_POS_OR ) fts3PoslistCopy(&p, &p1);
+ fts3GetDeltaVarint2(&p1, pEnd1, &i1);
+ }else{
+ fts3PutDeltaVarint(&p, &iPrev, i2);
+ if( mergetype==MERGE_POS_OR ) fts3PoslistCopy(&p, &p2);
+ fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+ }
+ }
+ break;
- dlrInit(&left, DL_POSITIONS, pLeft, nLeft);
- dlrInit(&right, DL_POSITIONS, pRight, nRight);
- dlwInit(&writer, iType, pOut);
+ case MERGE_AND:
+ while( p1 && p2 ){
+ if( i1==i2 ){
+ fts3PutDeltaVarint(&p, &iPrev, i1);
+ fts3GetDeltaVarint2(&p1, pEnd1, &i1);
+ fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+ }else if( i1<i2 ){
+ fts3GetDeltaVarint2(&p1, pEnd1, &i1);
+ }else{
+ fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+ }
+ }
+ break;
- while( !dlrAtEnd(&left) && !dlrAtEnd(&right) ){
- if( dlrDocid(&left)<dlrDocid(&right) ){
- dlrStep(&left);
- }else if( dlrDocid(&right)<dlrDocid(&left) ){
- dlrStep(&right);
- }else{
- if( nNear==0 ){
- posListPhraseMerge(&left, &right, 0, 0, &writer);
- }else{
- /* This case occurs when two terms (simple terms or phrases) are
- * connected by a NEAR operator, span (nNear+1). i.e.
- *
- * '"terrible company" NEAR widget'
- */
- DataBuffer one = {0, 0, 0};
- DataBuffer two = {0, 0, 0};
-
- DLWriter dlwriter2;
- DLReader dr1 = {0, 0, 0, 0, 0};
- DLReader dr2 = {0, 0, 0, 0, 0};
-
- dlwInit(&dlwriter2, iType, &one);
- posListPhraseMerge(&right, &left, nNear-3+nPhrase, 1, &dlwriter2);
- dlwInit(&dlwriter2, iType, &two);
- posListPhraseMerge(&left, &right, nNear-1, 0, &dlwriter2);
-
- if( one.nData) dlrInit(&dr1, iType, one.pData, one.nData);
- if( two.nData) dlrInit(&dr2, iType, two.pData, two.nData);
-
- if( !dlrAtEnd(&dr1) || !dlrAtEnd(&dr2) ){
- PLReader pr1 = {0};
- PLReader pr2 = {0};
-
- PLWriter plwriter;
- plwInit(&plwriter, &writer, dlrDocid(dlrAtEnd(&dr1)?&dr2:&dr1));
-
- if( one.nData ) plrInit(&pr1, &dr1);
- if( two.nData ) plrInit(&pr2, &dr2);
- while( !plrAtEnd(&pr1) || !plrAtEnd(&pr2) ){
- int iCompare = plrCompare(&pr1, &pr2);
- switch( iCompare ){
- case -1:
- plwCopy(&plwriter, &pr1);
- plrStep(&pr1);
- break;
- case 1:
- plwCopy(&plwriter, &pr2);
- plrStep(&pr2);
- break;
- case 0:
- plwCopy(&plwriter, &pr1);
- plrStep(&pr1);
- plrStep(&pr2);
- break;
- }
+ case MERGE_NOT:
+ while( p1 ){
+ if( p2 && i1==i2 ){
+ fts3GetDeltaVarint2(&p1, pEnd1, &i1);
+ fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+ }else if( !p2 || i1<i2 ){
+ fts3PutDeltaVarint(&p, &iPrev, i1);
+ fts3GetDeltaVarint2(&p1, pEnd1, &i1);
+ }else{
+ fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+ }
+ }
+ break;
+
+ case MERGE_POS_PHRASE:
+ case MERGE_PHRASE: {
+ char **ppPos = (mergetype==MERGE_PHRASE ? 0 : &p);
+ while( p1 && p2 ){
+ if( i1==i2 ){
+ char *pSave = p;
+ sqlite3_int64 iPrevSave = iPrev;
+ fts3PutDeltaVarint(&p, &iPrev, i1);
+ if( 0==fts3PoslistPhraseMerge(ppPos, 1, 0, &p1, &p2) ){
+ p = pSave;
+ iPrev = iPrevSave;
}
- plwTerminate(&plwriter);
+ fts3GetDeltaVarint2(&p1, pEnd1, &i1);
+ fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+ }else if( i1<i2 ){
+ fts3PoslistCopy(0, &p1);
+ fts3GetDeltaVarint2(&p1, pEnd1, &i1);
+ }else{
+ fts3PoslistCopy(0, &p2);
+ fts3GetDeltaVarint2(&p2, pEnd2, &i2);
}
- dataBufferDestroy(&one);
- dataBufferDestroy(&two);
}
- dlrStep(&left);
- dlrStep(&right);
+ break;
}
- }
- dlrDestroy(&left);
- dlrDestroy(&right);
- dlwDestroy(&writer);
-}
-
-/* We have two DL_DOCIDS doclists: pLeft and pRight.
-** Write the intersection of these two doclists into pOut as a
-** DL_DOCIDS doclist.
-*/
-static void docListAndMerge(
- const char *pLeft, int nLeft,
- const char *pRight, int nRight,
- DataBuffer *pOut /* Write the combined doclist here */
-){
- DLReader left, right;
- DLWriter writer;
+ default: assert( mergetype==MERGE_POS_NEAR || mergetype==MERGE_NEAR ); {
+ char *aTmp = 0;
+ char **ppPos = 0;
+ if( mergetype==MERGE_POS_NEAR ){
+ ppPos = &p;
+ aTmp = sqlite3_malloc(2*(n1+n2));
+ if( !aTmp ){
+ return SQLITE_NOMEM;
+ }
+ }
- if( nLeft==0 || nRight==0 ) return;
+ while( p1 && p2 ){
+ if( i1==i2 ){
+ char *pSave = p;
+ sqlite3_int64 iPrevSave = iPrev;
+ fts3PutDeltaVarint(&p, &iPrev, i1);
- dlrInit(&left, DL_DOCIDS, pLeft, nLeft);
- dlrInit(&right, DL_DOCIDS, pRight, nRight);
- dlwInit(&writer, DL_DOCIDS, pOut);
+ if( !fts3PoslistNearMerge(ppPos, aTmp, nParam1, nParam2, &p1, &p2) ){
+ iPrev = iPrevSave;
+ p = pSave;
+ }
- while( !dlrAtEnd(&left) && !dlrAtEnd(&right) ){
- if( dlrDocid(&left)<dlrDocid(&right) ){
- dlrStep(&left);
- }else if( dlrDocid(&right)<dlrDocid(&left) ){
- dlrStep(&right);
- }else{
- dlwAdd(&writer, dlrDocid(&left));
- dlrStep(&left);
- dlrStep(&right);
+ fts3GetDeltaVarint2(&p1, pEnd1, &i1);
+ fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+ }else if( i1<i2 ){
+ fts3PoslistCopy(0, &p1);
+ fts3GetDeltaVarint2(&p1, pEnd1, &i1);
+ }else{
+ fts3PoslistCopy(0, &p2);
+ fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+ }
+ }
+ sqlite3_free(aTmp);
+ break;
}
}
- dlrDestroy(&left);
- dlrDestroy(&right);
- dlwDestroy(&writer);
+ *pnBuffer = (int)(p-aBuffer);
+ return SQLITE_OK;
}
-/* We have two DL_DOCIDS doclists: pLeft and pRight.
-** Write the union of these two doclists into pOut as a
-** DL_DOCIDS doclist.
+/*
+** A pointer to an instance of this structure is used as the context
+** argument to sqlite3Fts3SegReaderIterate()
+*/
+typedef struct TermSelect TermSelect;
+struct TermSelect {
+ int isReqPos;
+ char *aOutput; /* Malloc'd output buffer */
+ int nOutput; /* Size of output in bytes */
+};
+
+/*
+** This function is used as the sqlite3Fts3SegReaderIterate() callback when
+** querying the full-text index for a doclist associated with a term or
+** term-prefix.
*/
-static void docListOrMerge(
- const char *pLeft, int nLeft,
- const char *pRight, int nRight,
- DataBuffer *pOut /* Write the combined doclist here */
+static int fts3TermSelectCb(
+ Fts3Table *p, /* Virtual table object */
+ void *pContext, /* Pointer to TermSelect structure */
+ char *zTerm,
+ int nTerm,
+ char *aDoclist,
+ int nDoclist
){
- DLReader left, right;
- DLWriter writer;
+ TermSelect *pTS = (TermSelect *)pContext;
+ int nNew = pTS->nOutput + nDoclist;
+ char *aNew = sqlite3_malloc(nNew);
- if( nLeft==0 ){
- if( nRight!=0 ) dataBufferAppend(pOut, pRight, nRight);
- return;
- }
- if( nRight==0 ){
- dataBufferAppend(pOut, pLeft, nLeft);
- return;
+ UNUSED_PARAMETER(p);
+ UNUSED_PARAMETER(zTerm);
+ UNUSED_PARAMETER(nTerm);
+
+ if( !aNew ){
+ return SQLITE_NOMEM;
}
- dlrInit(&left, DL_DOCIDS, pLeft, nLeft);
- dlrInit(&right, DL_DOCIDS, pRight, nRight);
- dlwInit(&writer, DL_DOCIDS, pOut);
-
- while( !dlrAtEnd(&left) || !dlrAtEnd(&right) ){
- if( dlrAtEnd(&right) ){
- dlwAdd(&writer, dlrDocid(&left));
- dlrStep(&left);
- }else if( dlrAtEnd(&left) ){
- dlwAdd(&writer, dlrDocid(&right));
- dlrStep(&right);
- }else if( dlrDocid(&left)<dlrDocid(&right) ){
- dlwAdd(&writer, dlrDocid(&left));
- dlrStep(&left);
- }else if( dlrDocid(&right)<dlrDocid(&left) ){
- dlwAdd(&writer, dlrDocid(&right));
- dlrStep(&right);
- }else{
- dlwAdd(&writer, dlrDocid(&left));
- dlrStep(&left);
- dlrStep(&right);
- }
+ if( pTS->nOutput==0 ){
+ /* If this is the first term selected, copy the doclist to the output
+ ** buffer using memcpy(). TODO: Add a way to transfer control of the
+ ** aDoclist buffer from the caller so as to avoid the memcpy().
+ */
+ memcpy(aNew, aDoclist, nDoclist);
+ }else{
+ /* The output buffer is not empty. Merge doclist aDoclist with the
+ ** existing output. This can only happen with prefix-searches (as
+ ** searches for exact terms return exactly one doclist).
+ */
+ int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR);
+ fts3DoclistMerge(mergetype, 0, 0,
+ aNew, &nNew, pTS->aOutput, pTS->nOutput, aDoclist, nDoclist
+ );
}
- dlrDestroy(&left);
- dlrDestroy(&right);
- dlwDestroy(&writer);
+ sqlite3_free(pTS->aOutput);
+ pTS->aOutput = aNew;
+ pTS->nOutput = nNew;
+
+ return SQLITE_OK;
}
-/* We have two DL_DOCIDS doclists: pLeft and pRight.
-** Write into pOut as DL_DOCIDS doclist containing all documents that
-** occur in pLeft but not in pRight.
+/*
+** This function retreives the doclist for the specified term (or term
+** prefix) from the database.
+**
+** The returned doclist may be in one of two formats, depending on the
+** value of parameter isReqPos. If isReqPos is zero, then the doclist is
+** a sorted list of delta-compressed docids. If isReqPos is non-zero,
+** then the returned list is in the same format as is stored in the
+** database without the found length specifier at the start of on-disk
+** doclists.
*/
-static void docListExceptMerge(
- const char *pLeft, int nLeft,
- const char *pRight, int nRight,
- DataBuffer *pOut /* Write the combined doclist here */
+static int fts3TermSelect(
+ Fts3Table *p, /* Virtual table handle */
+ int iColumn, /* Column to query (or -ve for all columns) */
+ const char *zTerm, /* Term to query for */
+ int nTerm, /* Size of zTerm in bytes */
+ int isPrefix, /* True for a prefix search */
+ int isReqPos, /* True to include position lists in output */
+ int *pnOut, /* OUT: Size of buffer at *ppOut */
+ char **ppOut /* OUT: Malloced result buffer */
){
- DLReader left, right;
- DLWriter writer;
+ int i;
+ TermSelect tsc;
+ Fts3SegFilter filter; /* Segment term filter configuration */
+ Fts3SegReader **apSegment = 0; /* Array of segments to read data from */
+ int nSegment = 0; /* Size of apSegment array */
+ int nAlloc = 0; /* Allocated size of segment array */
+ int rc; /* Return code */
+ sqlite3_stmt *pStmt; /* SQL statement to scan %_segdir table */
+ int iAge = 0; /* Used to assign ages to segments */
+
+ /* Loop through the entire %_segdir table. For each segment, create a
+ ** Fts3SegReader to iterate through the subset of the segment leaves
+ ** that may contain a term that matches zTerm/nTerm. For non-prefix
+ ** searches, this is always a single leaf. For prefix searches, this
+ ** may be a contiguous block of leaves.
+ **
+ ** The code in this loop does not actually load any leaves into memory
+ ** (unless the root node happens to be a leaf). It simply examines the
+ ** b-tree structure to determine which leaves need to be inspected.
+ */
+ rc = sqlite3Fts3AllSegdirs(p, &pStmt);
+ while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
+ Fts3SegReader *pNew = 0;
+ int nRoot = sqlite3_column_bytes(pStmt, 4);
+ char const *zRoot = sqlite3_column_blob(pStmt, 4);
+ if( sqlite3_column_int64(pStmt, 1)==0 ){
+ /* The entire segment is stored on the root node (which must be a
+ ** leaf). Do not bother inspecting any data in this case, just
+ ** create a Fts3SegReader to scan the single leaf.
+ */
+ rc = sqlite3Fts3SegReaderNew(p, iAge, 0, 0, 0, zRoot, nRoot, &pNew);
+ }else{
+ int rc2; /* Return value of sqlite3Fts3ReadBlock() */
+ sqlite3_int64 i1; /* Blockid of leaf that may contain zTerm */
+ rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1);
+ if( rc==SQLITE_OK ){
+ sqlite3_int64 i2 = sqlite3_column_int64(pStmt, 2);
+ rc = sqlite3Fts3SegReaderNew(p, iAge, i1, i2, 0, 0, 0, &pNew);
+ }
- if( nLeft==0 ) return;
- if( nRight==0 ){
- dataBufferAppend(pOut, pLeft, nLeft);
- return;
+ /* The following call to ReadBlock() serves to reset the SQL statement
+ ** used to retrieve blocks of data from the %_segments table. If it is
+ ** not reset here, then it may remain classified as an active statement
+ ** by SQLite, which may lead to "DROP TABLE" or "DETACH" commands
+ ** failing.
+ */
+ rc2 = sqlite3Fts3ReadBlock(p, 0, 0, 0);
+ if( rc==SQLITE_OK ){
+ rc = rc2;
+ }
+ }
+ iAge++;
+
+ /* If a new Fts3SegReader was allocated, add it to the apSegment array. */
+ assert( pNew!=0 || rc!=SQLITE_OK );
+ if( pNew ){
+ if( nSegment==nAlloc ){
+ Fts3SegReader **pArray;
+ nAlloc += 16;
+ pArray = (Fts3SegReader **)sqlite3_realloc(
+ apSegment, nAlloc*sizeof(Fts3SegReader *)
+ );
+ if( !pArray ){
+ sqlite3Fts3SegReaderFree(p, pNew);
+ rc = SQLITE_NOMEM;
+ goto finished;
+ }
+ apSegment = pArray;
+ }
+ apSegment[nSegment++] = pNew;
+ }
+ }
+ if( rc!=SQLITE_DONE ){
+ assert( rc!=SQLITE_OK );
+ goto finished;
}
- dlrInit(&left, DL_DOCIDS, pLeft, nLeft);
- dlrInit(&right, DL_DOCIDS, pRight, nRight);
- dlwInit(&writer, DL_DOCIDS, pOut);
+ memset(&tsc, 0, sizeof(TermSelect));
+ tsc.isReqPos = isReqPos;
- while( !dlrAtEnd(&left) ){
- while( !dlrAtEnd(&right) && dlrDocid(&right)<dlrDocid(&left) ){
- dlrStep(&right);
- }
- if( dlrAtEnd(&right) || dlrDocid(&left)<dlrDocid(&right) ){
- dlwAdd(&writer, dlrDocid(&left));
- }
- dlrStep(&left);
+ filter.flags = FTS3_SEGMENT_IGNORE_EMPTY
+ | (isPrefix ? FTS3_SEGMENT_PREFIX : 0)
+ | (isReqPos ? FTS3_SEGMENT_REQUIRE_POS : 0)
+ | (iColumn<p->nColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0);
+ filter.iCol = iColumn;
+ filter.zTerm = zTerm;
+ filter.nTerm = nTerm;
+
+ rc = sqlite3Fts3SegReaderIterate(p, apSegment, nSegment, &filter,
+ fts3TermSelectCb, (void *)&tsc
+ );
+
+ if( rc==SQLITE_OK ){
+ *ppOut = tsc.aOutput;
+ *pnOut = tsc.nOutput;
+ }else{
+ sqlite3_free(tsc.aOutput);
}
- dlrDestroy(&left);
- dlrDestroy(&right);
- dlwDestroy(&writer);
+finished:
+ sqlite3_reset(pStmt);
+ for(i=0; i<nSegment; i++){
+ sqlite3Fts3SegReaderFree(p, apSegment[i]);
+ }
+ sqlite3_free(apSegment);
+ return rc;
}
-static char *string_dup_n(const char *s, int n){
- char *str = sqlite3_malloc(n + 1);
- memcpy(str, s, n);
- str[n] = '\0';
- return str;
-}
-/* Duplicate a string; the caller must free() the returned string.
- * (We don't use strdup() since it is not part of the standard C library and
- * may not be available everywhere.) */
-static char *string_dup(const char *s){
- return string_dup_n(s, strlen(s));
-}
+/*
+** Return a DocList corresponding to the phrase *pPhrase.
+*/
+static int fts3PhraseSelect(
+ Fts3Table *p, /* Virtual table handle */
+ Fts3Phrase *pPhrase, /* Phrase to return a doclist for */
+ int isReqPos, /* True if output should contain positions */
+ char **paOut, /* OUT: Pointer to malloc'd result buffer */
+ int *pnOut /* OUT: Size of buffer at *paOut */
+){
+ char *pOut = 0;
+ int nOut = 0;
+ int rc = SQLITE_OK;
+ int ii;
+ int iCol = pPhrase->iColumn;
+ int isTermPos = (pPhrase->nToken>1 || isReqPos);
-/* Format a string, replacing each occurrence of the % character with
- * zDb.zName. This may be more convenient than sqlite_mprintf()
- * when one string is used repeatedly in a format string.
- * The caller must free() the returned string. */
-static char *string_format(const char *zFormat,
- const char *zDb, const char *zName){
- const char *p;
- size_t len = 0;
- size_t nDb = strlen(zDb);
- size_t nName = strlen(zName);
- size_t nFullTableName = nDb+1+nName;
- char *result;
- char *r;
+ assert( p->nPendingData==0 );
- /* first compute length needed */
- for(p = zFormat ; *p ; ++p){
- len += (*p=='%' ? nFullTableName : 1);
- }
- len += 1; /* for null terminator */
+ for(ii=0; ii<pPhrase->nToken; ii++){
+ struct PhraseToken *pTok = &pPhrase->aToken[ii];
+ char *z = pTok->z; /* Next token of the phrase */
+ int n = pTok->n; /* Size of z in bytes */
+ int isPrefix = pTok->isPrefix;/* True if token is a prefix */
+ char *pList; /* Pointer to token doclist */
+ int nList; /* Size of buffer at pList */
- r = result = sqlite3_malloc(len);
- for(p = zFormat; *p; ++p){
- if( *p=='%' ){
- memcpy(r, zDb, nDb);
- r += nDb;
- *r++ = '.';
- memcpy(r, zName, nName);
- r += nName;
- } else {
- *r++ = *p;
+ rc = fts3TermSelect(p, iCol, z, n, isPrefix, isTermPos, &nList, &pList);
+ if( rc!=SQLITE_OK ) break;
+
+ if( ii==0 ){
+ pOut = pList;
+ nOut = nList;
+ }else{
+ /* Merge the new term list and the current output. If this is the
+ ** last term in the phrase, and positions are not required in the
+ ** output of this function, the positions can be dropped as part
+ ** of this merge. Either way, the result of this merge will be
+ ** smaller than nList bytes. The code in fts3DoclistMerge() is written
+ ** so that it is safe to use pList as the output as well as an input
+ ** in this case.
+ */
+ int mergetype = MERGE_POS_PHRASE;
+ if( ii==pPhrase->nToken-1 && !isReqPos ){
+ mergetype = MERGE_PHRASE;
+ }
+ fts3DoclistMerge(mergetype, 0, 0, pList, &nOut, pOut, nOut, pList, nList);
+ sqlite3_free(pOut);
+ pOut = pList;
}
}
- *r++ = '\0';
- assert( r == result + len );
- return result;
-}
-
-static int sql_exec(sqlite3 *db, const char *zDb, const char *zName,
- const char *zFormat){
- char *zCommand = string_format(zFormat, zDb, zName);
- int rc;
- FTSTRACE(("FTS3 sql: %s\n", zCommand));
- rc = sqlite3_exec(db, zCommand, NULL, 0, NULL);
- sqlite3_free(zCommand);
- return rc;
-}
-static int sql_prepare(sqlite3 *db, const char *zDb, const char *zName,
- sqlite3_stmt **ppStmt, const char *zFormat){
- char *zCommand = string_format(zFormat, zDb, zName);
- int rc;
- FTSTRACE(("FTS3 prepare: %s\n", zCommand));
- rc = sqlite3_prepare_v2(db, zCommand, -1, ppStmt, NULL);
- sqlite3_free(zCommand);
+ if( rc==SQLITE_OK ){
+ *paOut = pOut;
+ *pnOut = nOut;
+ }else{
+ sqlite3_free(pOut);
+ }
return rc;
}
-/* end utility functions */
-
-/* Forward reference */
-typedef struct fulltext_vtab fulltext_vtab;
-
/*
-** An instance of the following structure keeps track of generated
-** matching-word offset information and snippets.
-*/
-typedef struct Snippet {
- int nMatch; /* Total number of matches */
- int nAlloc; /* Space allocated for aMatch[] */
- struct snippetMatch { /* One entry for each matching term */
- char snStatus; /* Status flag for use while constructing snippets */
- short int iCol; /* The column that contains the match */
- short int iTerm; /* The index in Query.pTerms[] of the matching term */
- int iToken; /* The index of the matching document token */
- short int nByte; /* Number of bytes in the term */
- int iStart; /* The offset to the first character of the term */
- } *aMatch; /* Points to space obtained from malloc */
- char *zOffset; /* Text rendering of aMatch[] */
- int nOffset; /* strlen(zOffset) */
- char *zSnippet; /* Snippet text */
- int nSnippet; /* strlen(zSnippet) */
-} Snippet;
-
-
-typedef enum QueryType {
- QUERY_GENERIC, /* table scan */
- QUERY_DOCID, /* lookup by docid */
- QUERY_FULLTEXT /* QUERY_FULLTEXT + [i] is a full-text search for column i*/
-} QueryType;
-
-typedef enum fulltext_statement {
- CONTENT_INSERT_STMT,
- CONTENT_SELECT_STMT,
- CONTENT_UPDATE_STMT,
- CONTENT_DELETE_STMT,
- CONTENT_EXISTS_STMT,
-
- BLOCK_INSERT_STMT,
- BLOCK_SELECT_STMT,
- BLOCK_DELETE_STMT,
- BLOCK_DELETE_ALL_STMT,
-
- SEGDIR_MAX_INDEX_STMT,
- SEGDIR_SET_STMT,
- SEGDIR_SELECT_LEVEL_STMT,
- SEGDIR_SPAN_STMT,
- SEGDIR_DELETE_STMT,
- SEGDIR_SELECT_SEGMENT_STMT,
- SEGDIR_SELECT_ALL_STMT,
- SEGDIR_DELETE_ALL_STMT,
- SEGDIR_COUNT_STMT,
-
- MAX_STMT /* Always at end! */
-} fulltext_statement;
-
-/* These must exactly match the enum above. */
-/* TODO(shess): Is there some risk that a statement will be used in two
-** cursors at once, e.g. if a query joins a virtual table to itself?
-** If so perhaps we should move some of these to the cursor object.
-*/
-static const char *const fulltext_zStatement[MAX_STMT] = {
- /* CONTENT_INSERT */ NULL, /* generated in contentInsertStatement() */
- /* CONTENT_SELECT */ NULL, /* generated in contentSelectStatement() */
- /* CONTENT_UPDATE */ NULL, /* generated in contentUpdateStatement() */
- /* CONTENT_DELETE */ "delete from %_content where docid = ?",
- /* CONTENT_EXISTS */ "select docid from %_content limit 1",
-
- /* BLOCK_INSERT */
- "insert into %_segments (blockid, block) values (null, ?)",
- /* BLOCK_SELECT */ "select block from %_segments where blockid = ?",
- /* BLOCK_DELETE */ "delete from %_segments where blockid between ? and ?",
- /* BLOCK_DELETE_ALL */ "delete from %_segments",
-
- /* SEGDIR_MAX_INDEX */ "select max(idx) from %_segdir where level = ?",
- /* SEGDIR_SET */ "insert into %_segdir values (?, ?, ?, ?, ?, ?)",
- /* SEGDIR_SELECT_LEVEL */
- "select start_block, leaves_end_block, root from %_segdir "
- " where level = ? order by idx",
- /* SEGDIR_SPAN */
- "select min(start_block), max(end_block) from %_segdir "
- " where level = ? and start_block <> 0",
- /* SEGDIR_DELETE */ "delete from %_segdir where level = ?",
-
- /* NOTE(shess): The first three results of the following two
- ** statements must match.
- */
- /* SEGDIR_SELECT_SEGMENT */
- "select start_block, leaves_end_block, root from %_segdir "
- " where level = ? and idx = ?",
- /* SEGDIR_SELECT_ALL */
- "select start_block, leaves_end_block, root from %_segdir "
- " order by level desc, idx asc",
- /* SEGDIR_DELETE_ALL */ "delete from %_segdir",
- /* SEGDIR_COUNT */ "select count(*), ifnull(max(level),0) from %_segdir",
-};
-
-/*
-** A connection to a fulltext index is an instance of the following
-** structure. The xCreate and xConnect methods create an instance
-** of this structure and xDestroy and xDisconnect free that instance.
-** All other methods receive a pointer to the structure as one of their
-** arguments.
+** Evaluate the full-text expression pExpr against fts3 table pTab. Store
+** the resulting doclist in *paOut and *pnOut.
*/
-struct fulltext_vtab {
- sqlite3_vtab base; /* Base class used by SQLite core */
- sqlite3 *db; /* The database connection */
- const char *zDb; /* logical database name */
- const char *zName; /* virtual table name */
- int nColumn; /* number of columns in virtual table */
- char **azColumn; /* column names. malloced */
- char **azContentColumn; /* column names in content table; malloced */
- sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */
-
- /* Precompiled statements which we keep as long as the table is
- ** open.
- */
- sqlite3_stmt *pFulltextStatements[MAX_STMT];
-
- /* Precompiled statements used for segment merges. We run a
- ** separate select across the leaf level of each tree being merged.
- */
- sqlite3_stmt *pLeafSelectStmts[MERGE_COUNT];
- /* The statement used to prepare pLeafSelectStmts. */
-#define LEAF_SELECT \
- "select block from %_segments where blockid between ? and ? order by blockid"
-
- /* These buffer pending index updates during transactions.
- ** nPendingData estimates the memory size of the pending data. It
- ** doesn't include the hash-bucket overhead, nor any malloc
- ** overhead. When nPendingData exceeds kPendingThreshold, the
- ** buffer is flushed even before the transaction closes.
- ** pendingTerms stores the data, and is only valid when nPendingData
- ** is >=0 (nPendingData<0 means pendingTerms has not been
- ** initialized). iPrevDocid is the last docid written, used to make
- ** certain we're inserting in sorted order.
- */
- int nPendingData;
-#define kPendingThreshold (1*1024*1024)
- sqlite_int64 iPrevDocid;
- fts3Hash pendingTerms;
-};
-
-/*
-** When the core wants to do a query, it create a cursor using a
-** call to xOpen. This structure is an instance of a cursor. It
-** is destroyed by xClose.
-*/
-typedef struct fulltext_cursor {
- sqlite3_vtab_cursor base; /* Base class used by SQLite core */
- QueryType iCursorType; /* Copy of sqlite3_index_info.idxNum */
- sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */
- int eof; /* True if at End Of Results */
- Fts3Expr *pExpr; /* Parsed MATCH query string */
- Snippet snippet; /* Cached snippet for the current row */
- int iColumn; /* Column being searched */
- DataBuffer result; /* Doclist results from fulltextQuery */
- DLReader reader; /* Result reader if result not empty */
-} fulltext_cursor;
-
-static fulltext_vtab *cursor_vtab(fulltext_cursor *c){
- return (fulltext_vtab *) c->base.pVtab;
-}
-
-static const sqlite3_module fts3Module; /* forward declaration */
+static int evalFts3Expr(
+ Fts3Table *p, /* Virtual table handle */
+ Fts3Expr *pExpr, /* Parsed fts3 expression */
+ char **paOut, /* OUT: Pointer to malloc'd result buffer */
+ int *pnOut /* OUT: Size of buffer at *paOut */
+){
+ int rc = SQLITE_OK; /* Return code */
-/* Return a dynamically generated statement of the form
- * insert into %_content (docid, ...) values (?, ...)
- */
-static const char *contentInsertStatement(fulltext_vtab *v){
- StringBuffer sb;
- int i;
+ /* Zero the output parameters. */
+ *paOut = 0;
+ *pnOut = 0;
- initStringBuffer(&sb);
- append(&sb, "insert into %_content (docid, ");
- appendList(&sb, v->nColumn, v->azContentColumn);
- append(&sb, ") values (?");
- for(i=0; i<v->nColumn; ++i)
- append(&sb, ", ?");
- append(&sb, ")");
- return stringBufferData(&sb);
-}
+ if( pExpr ){
+ if( pExpr->eType==FTSQUERY_PHRASE ){
+ int isReqPos = (pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR);
+ rc = fts3PhraseSelect(p, pExpr->pPhrase, isReqPos, paOut, pnOut);
+ }else{
+ char *aLeft;
+ char *aRight;
+ int nLeft;
+ int nRight;
-/* Return a dynamically generated statement of the form
- * select <content columns> from %_content where docid = ?
- */
-static const char *contentSelectStatement(fulltext_vtab *v){
- StringBuffer sb;
- initStringBuffer(&sb);
- append(&sb, "SELECT ");
- appendList(&sb, v->nColumn, v->azContentColumn);
- append(&sb, " FROM %_content WHERE docid = ?");
- return stringBufferData(&sb);
-}
+ if( SQLITE_OK==(rc = evalFts3Expr(p, pExpr->pRight, &aRight, &nRight))
+ && SQLITE_OK==(rc = evalFts3Expr(p, pExpr->pLeft, &aLeft, &nLeft))
+ ){
+ assert( pExpr->eType==FTSQUERY_NEAR || pExpr->eType==FTSQUERY_OR
+ || pExpr->eType==FTSQUERY_AND || pExpr->eType==FTSQUERY_NOT
+ );
+ switch( pExpr->eType ){
+ case FTSQUERY_NEAR: {
+ Fts3Expr *pLeft;
+ Fts3Expr *pRight;
+ int mergetype = MERGE_NEAR;
+ int nParam1;
+ int nParam2;
+ char *aBuffer;
+
+ if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){
+ mergetype = MERGE_POS_NEAR;
+ }
+ pLeft = pExpr->pLeft;
+ while( pLeft->eType==FTSQUERY_NEAR ){
+ pLeft=pLeft->pRight;
+ }
+ pRight = pExpr->pRight;
+ assert( pRight->eType==FTSQUERY_PHRASE );
+ assert( pLeft->eType==FTSQUERY_PHRASE );
-/* Return a dynamically generated statement of the form
- * update %_content set [col_0] = ?, [col_1] = ?, ...
- * where docid = ?
- */
-static const char *contentUpdateStatement(fulltext_vtab *v){
- StringBuffer sb;
- int i;
+ nParam1 = pExpr->nNear+1;
+ nParam2 = nParam1+pLeft->pPhrase->nToken+pRight->pPhrase->nToken-2;
+ aBuffer = sqlite3_malloc(nLeft+nRight+1);
+ rc = fts3DoclistMerge(mergetype, nParam1, nParam2, aBuffer,
+ pnOut, aLeft, nLeft, aRight, nRight
+ );
+ if( rc!=SQLITE_OK ){
+ sqlite3_free(aBuffer);
+ }else{
+ *paOut = aBuffer;
+ }
+ sqlite3_free(aLeft);
+ break;
+ }
- initStringBuffer(&sb);
- append(&sb, "update %_content set ");
- for(i=0; i<v->nColumn; ++i) {
- if( i>0 ){
- append(&sb, ", ");
- }
- append(&sb, v->azContentColumn[i]);
- append(&sb, " = ?");
- }
- append(&sb, " where docid = ?");
- return stringBufferData(&sb);
-}
+ case FTSQUERY_OR: {
+ /* Allocate a buffer for the output. The maximum size is the
+ ** sum of the sizes of the two input buffers. The +1 term is
+ ** so that a buffer of zero bytes is never allocated - this can
+ ** cause fts3DoclistMerge() to incorrectly return SQLITE_NOMEM.
+ */
+ char *aBuffer = sqlite3_malloc(nRight+nLeft+1);
+ rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut,
+ aLeft, nLeft, aRight, nRight
+ );
+ *paOut = aBuffer;
+ sqlite3_free(aLeft);
+ break;
+ }
-/* Puts a freshly-prepared statement determined by iStmt in *ppStmt.
-** If the indicated statement has never been prepared, it is prepared
-** and cached, otherwise the cached version is reset.
-*/
-static int sql_get_statement(fulltext_vtab *v, fulltext_statement iStmt,
- sqlite3_stmt **ppStmt){
- assert( iStmt<MAX_STMT );
- if( v->pFulltextStatements[iStmt]==NULL ){
- const char *zStmt;
- int rc;
- switch( iStmt ){
- case CONTENT_INSERT_STMT:
- zStmt = contentInsertStatement(v); break;
- case CONTENT_SELECT_STMT:
- zStmt = contentSelectStatement(v); break;
- case CONTENT_UPDATE_STMT:
- zStmt = contentUpdateStatement(v); break;
- default:
- zStmt = fulltext_zStatement[iStmt];
+ default: {
+ assert( FTSQUERY_NOT==MERGE_NOT && FTSQUERY_AND==MERGE_AND );
+ fts3DoclistMerge(pExpr->eType, 0, 0, aLeft, pnOut,
+ aLeft, nLeft, aRight, nRight
+ );
+ *paOut = aLeft;
+ break;
+ }
+ }
+ }
+ sqlite3_free(aRight);
}
- rc = sql_prepare(v->db, v->zDb, v->zName, &v->pFulltextStatements[iStmt],
- zStmt);
- if( zStmt != fulltext_zStatement[iStmt]) sqlite3_free((void *) zStmt);
- if( rc!=SQLITE_OK ) return rc;
- } else {
- int rc = sqlite3_reset(v->pFulltextStatements[iStmt]);
- if( rc!=SQLITE_OK ) return rc;
}
- *ppStmt = v->pFulltextStatements[iStmt];
- return SQLITE_OK;
-}
-
-/* Like sqlite3_step(), but convert SQLITE_DONE to SQLITE_OK and
-** SQLITE_ROW to SQLITE_ERROR. Useful for statements like UPDATE,
-** where we expect no results.
-*/
-static int sql_single_step(sqlite3_stmt *s){
- int rc = sqlite3_step(s);
- return (rc==SQLITE_DONE) ? SQLITE_OK : rc;
+ return rc;
}
-/* Like sql_get_statement(), but for special replicated LEAF_SELECT
-** statements. idx -1 is a special case for an uncached version of
-** the statement (used in the optimize implementation).
+/*
+** This is the xFilter interface for the virtual table. See
+** the virtual table xFilter method documentation for additional
+** information.
+**
+** If idxNum==FTS3_FULLSCAN_SEARCH then do a full table scan against
+** the %_content table.
+**
+** If idxNum==FTS3_DOCID_SEARCH then do a docid lookup for a single entry
+** in the %_content table.
+**
+** If idxNum>=FTS3_FULLTEXT_SEARCH then use the full text index. The
+** column on the left-hand side of the MATCH operator is column
+** number idxNum-FTS3_FULLTEXT_SEARCH, 0 indexed. argv[0] is the right-hand
+** side of the MATCH operator.
*/
-/* TODO(shess) Write version for generic statements and then share
-** that between the cached-statement functions.
+/* TODO(shess) Upgrade the cursor initialization and destruction to
+** account for fts3FilterMethod() being called multiple times on the
+** same cursor. The current solution is very fragile. Apply fix to
+** fts3 as appropriate.
*/
-static int sql_get_leaf_statement(fulltext_vtab *v, int idx,
- sqlite3_stmt **ppStmt){
- assert( idx>=-1 && idx<MERGE_COUNT );
- if( idx==-1 ){
- return sql_prepare(v->db, v->zDb, v->zName, ppStmt, LEAF_SELECT);
- }else if( v->pLeafSelectStmts[idx]==NULL ){
- int rc = sql_prepare(v->db, v->zDb, v->zName, &v->pLeafSelectStmts[idx],
- LEAF_SELECT);
- if( rc!=SQLITE_OK ) return rc;
- }else{
- int rc = sqlite3_reset(v->pLeafSelectStmts[idx]);
- if( rc!=SQLITE_OK ) return rc;
- }
+static int fts3FilterMethod(
+ sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */
+ int idxNum, /* Strategy index */
+ const char *idxStr, /* Unused */
+ int nVal, /* Number of elements in apVal */
+ sqlite3_value **apVal /* Arguments for the indexing scheme */
+){
+ const char *azSql[] = {
+ "SELECT * FROM %Q.'%q_content' WHERE docid = ?", /* non-full-table-scan */
+ "SELECT * FROM %Q.'%q_content'", /* full-table-scan */
+ };
+ int rc; /* Return code */
+ char *zSql; /* SQL statement used to access %_content */
+ Fts3Table *p = (Fts3Table *)pCursor->pVtab;
+ Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
- *ppStmt = v->pLeafSelectStmts[idx];
- return SQLITE_OK;
-}
+ UNUSED_PARAMETER(idxStr);
+ UNUSED_PARAMETER(nVal);
-/* insert into %_content (docid, ...) values ([docid], [pValues])
-** If the docid contains SQL NULL, then a unique docid will be
-** generated.
-*/
-static int content_insert(fulltext_vtab *v, sqlite3_value *docid,
- sqlite3_value **pValues){
- sqlite3_stmt *s;
- int i;
- int rc = sql_get_statement(v, CONTENT_INSERT_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
+ assert( idxNum>=0 && idxNum<=(FTS3_FULLTEXT_SEARCH+p->nColumn) );
+ assert( nVal==0 || nVal==1 );
+ assert( (nVal==0)==(idxNum==FTS3_FULLSCAN_SEARCH) );
- rc = sqlite3_bind_value(s, 1, docid);
- if( rc!=SQLITE_OK ) return rc;
+ /* In case the cursor has been used before, clear it now. */
+ sqlite3_finalize(pCsr->pStmt);
+ sqlite3_free(pCsr->aDoclist);
+ memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
- for(i=0; i<v->nColumn; ++i){
- rc = sqlite3_bind_value(s, 2+i, pValues[i]);
- if( rc!=SQLITE_OK ) return rc;
+ /* Compile a SELECT statement for this cursor. For a full-table-scan, the
+ ** statement loops through all rows of the %_content table. For a
+ ** full-text query or docid lookup, the statement retrieves a single
+ ** row by docid.
+ */
+ zSql = sqlite3_mprintf(azSql[idxNum==FTS3_FULLSCAN_SEARCH], p->zDb, p->zName);
+ if( !zSql ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
+ sqlite3_free(zSql);
}
+ if( rc!=SQLITE_OK ) return rc;
+ pCsr->eSearch = (i16)idxNum;
- return sql_single_step(s);
-}
+ if( idxNum==FTS3_DOCID_SEARCH ){
+ rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
+ }else if( idxNum!=FTS3_FULLSCAN_SEARCH ){
+ int iCol = idxNum-FTS3_FULLTEXT_SEARCH;
+ const char *zQuery = (const char *)sqlite3_value_text(apVal[0]);
-/* update %_content set col0 = pValues[0], col1 = pValues[1], ...
- * where docid = [iDocid] */
-static int content_update(fulltext_vtab *v, sqlite3_value **pValues,
- sqlite_int64 iDocid){
- sqlite3_stmt *s;
- int i;
- int rc = sql_get_statement(v, CONTENT_UPDATE_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
+ if( zQuery==0 && sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
+ return SQLITE_NOMEM;
+ }
+ rc = sqlite3Fts3PendingTermsFlush(p);
+ if( rc!=SQLITE_OK ) return rc;
- for(i=0; i<v->nColumn; ++i){
- rc = sqlite3_bind_value(s, 1+i, pValues[i]);
+ rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->nColumn,
+ iCol, zQuery, -1, &pCsr->pExpr
+ );
if( rc!=SQLITE_OK ) return rc;
+
+ rc = evalFts3Expr(p, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist);
+ pCsr->pNextId = pCsr->aDoclist;
+ pCsr->iPrevId = 0;
}
- rc = sqlite3_bind_int64(s, 1+v->nColumn, iDocid);
if( rc!=SQLITE_OK ) return rc;
-
- return sql_single_step(s);
+ return fts3NextMethod(pCursor);
}
-static void freeStringArray(int nString, const char **pString){
- int i;
+/*
+** This is the xEof method of the virtual table. SQLite calls this
+** routine to find out if it has reached the end of a result set.
+*/
+static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){
+ return ((Fts3Cursor *)pCursor)->isEof;
+}
- for (i=0 ; i < nString ; ++i) {
- if( pString[i]!=NULL ) sqlite3_free((void *) pString[i]);
+/*
+** This is the xRowid method. The SQLite core calls this routine to
+** retrieve the rowid for the current row of the result set. fts3
+** exposes %_content.docid as the rowid for the virtual table. The
+** rowid should be written to *pRowid.
+*/
+static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
+ Fts3Cursor *pCsr = (Fts3Cursor *) pCursor;
+ if( pCsr->aDoclist ){
+ *pRowid = pCsr->iPrevId;
+ }else{
+ *pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
}
- sqlite3_free((void *) pString);
+ return SQLITE_OK;
}
-/* select * from %_content where docid = [iDocid]
- * The caller must delete the returned array and all strings in it.
- * null fields will be NULL in the returned array.
- *
- * TODO: Perhaps we should return pointer/length strings here for consistency
- * with other code which uses pointer/length. */
-static int content_select(fulltext_vtab *v, sqlite_int64 iDocid,
- const char ***pValues){
- sqlite3_stmt *s;
- const char **values;
- int i;
- int rc;
-
- *pValues = NULL;
-
- rc = sql_get_statement(v, CONTENT_SELECT_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int64(s, 1, iDocid);
- if( rc!=SQLITE_OK ) return rc;
+/*
+** This is the xColumn method, called by SQLite to request a value from
+** the row that the supplied cursor currently points to.
+*/
+static int fts3ColumnMethod(
+ sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
+ sqlite3_context *pContext, /* Context for sqlite3_result_xxx() calls */
+ int iCol /* Index of column to read value from */
+){
+ int rc; /* Return Code */
+ Fts3Cursor *pCsr = (Fts3Cursor *) pCursor;
+ Fts3Table *p = (Fts3Table *)pCursor->pVtab;
- rc = sqlite3_step(s);
- if( rc!=SQLITE_ROW ) return rc;
+ /* The column value supplied by SQLite must be in range. */
+ assert( iCol>=0 && iCol<=p->nColumn+1 );
- values = (const char **) sqlite3_malloc(v->nColumn * sizeof(const char *));
- for(i=0; i<v->nColumn; ++i){
- if( sqlite3_column_type(s, i)==SQLITE_NULL ){
- values[i] = NULL;
+ rc = fts3CursorSeek(pCsr);
+ if( rc==SQLITE_OK ){
+ if( iCol==p->nColumn+1 ){
+ /* This call is a request for the "docid" column. Since "docid" is an
+ ** alias for "rowid", use the xRowid() method to obtain the value.
+ */
+ sqlite3_int64 iRowid;
+ rc = fts3RowidMethod(pCursor, &iRowid);
+ sqlite3_result_int64(pContext, iRowid);
+ }else if( iCol==p->nColumn ){
+ /* The extra column whose name is the same as the table.
+ ** Return a blob which is a pointer to the cursor.
+ */
+ sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT);
}else{
- values[i] = string_dup((char*)sqlite3_column_text(s, i));
+ sqlite3_result_value(pContext, sqlite3_column_value(pCsr->pStmt, iCol+1));
}
}
-
- /* We expect only one row. We must execute another sqlite3_step()
- * to complete the iteration; otherwise the table will remain locked. */
- rc = sqlite3_step(s);
- if( rc==SQLITE_DONE ){
- *pValues = values;
- return SQLITE_OK;
- }
-
- freeStringArray(v->nColumn, values);
return rc;
}
-/* delete from %_content where docid = [iDocid ] */
-static int content_delete(fulltext_vtab *v, sqlite_int64 iDocid){
- sqlite3_stmt *s;
- int rc = sql_get_statement(v, CONTENT_DELETE_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int64(s, 1, iDocid);
- if( rc!=SQLITE_OK ) return rc;
-
- return sql_single_step(s);
+/*
+** This function is the implementation of the xUpdate callback used by
+** FTS3 virtual tables. It is invoked by SQLite each time a row is to be
+** inserted, updated or deleted.
+*/
+static int fts3UpdateMethod(
+ sqlite3_vtab *pVtab, /* Virtual table handle */
+ int nArg, /* Size of argument array */
+ sqlite3_value **apVal, /* Array of arguments */
+ sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */
+){
+ return sqlite3Fts3UpdateMethod(pVtab, nArg, apVal, pRowid);
}
-/* Returns SQLITE_ROW if any rows exist in %_content, SQLITE_DONE if
-** no rows exist, and any error in case of failure.
+/*
+** Implementation of xSync() method. Flush the contents of the pending-terms
+** hash-table to the database.
*/
-static int content_exists(fulltext_vtab *v){
- sqlite3_stmt *s;
- int rc = sql_get_statement(v, CONTENT_EXISTS_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_step(s);
- if( rc!=SQLITE_ROW ) return rc;
-
- /* We expect only one row. We must execute another sqlite3_step()
- * to complete the iteration; otherwise the table will remain locked. */
- rc = sqlite3_step(s);
- if( rc==SQLITE_DONE ) return SQLITE_ROW;
- if( rc==SQLITE_ROW ) return SQLITE_ERROR;
- return rc;
+static int fts3SyncMethod(sqlite3_vtab *pVtab){
+ return sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab);
}
-/* insert into %_segments values ([pData])
-** returns assigned blockid in *piBlockid
+/*
+** Implementation of xBegin() method. This is a no-op.
*/
-static int block_insert(fulltext_vtab *v, const char *pData, int nData,
- sqlite_int64 *piBlockid){
- sqlite3_stmt *s;
- int rc = sql_get_statement(v, BLOCK_INSERT_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_blob(s, 1, pData, nData, SQLITE_STATIC);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_step(s);
- if( rc==SQLITE_ROW ) return SQLITE_ERROR;
- if( rc!=SQLITE_DONE ) return rc;
-
- /* blockid column is an alias for rowid. */
- *piBlockid = sqlite3_last_insert_rowid(v->db);
+static int fts3BeginMethod(sqlite3_vtab *pVtab){
+ UNUSED_PARAMETER(pVtab);
+ assert( ((Fts3Table *)pVtab)->nPendingData==0 );
return SQLITE_OK;
}
-/* delete from %_segments
-** where blockid between [iStartBlockid] and [iEndBlockid]
-**
-** Deletes the range of blocks, inclusive, used to delete the blocks
-** which form a segment.
+/*
+** Implementation of xCommit() method. This is a no-op. The contents of
+** the pending-terms hash-table have already been flushed into the database
+** by fts3SyncMethod().
*/
-static int block_delete(fulltext_vtab *v,
- sqlite_int64 iStartBlockid, sqlite_int64 iEndBlockid){
- sqlite3_stmt *s;
- int rc = sql_get_statement(v, BLOCK_DELETE_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int64(s, 1, iStartBlockid);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int64(s, 2, iEndBlockid);
- if( rc!=SQLITE_OK ) return rc;
-
- return sql_single_step(s);
+static int fts3CommitMethod(sqlite3_vtab *pVtab){
+ UNUSED_PARAMETER(pVtab);
+ assert( ((Fts3Table *)pVtab)->nPendingData==0 );
+ return SQLITE_OK;
}
-/* Returns SQLITE_ROW with *pidx set to the maximum segment idx found
-** at iLevel. Returns SQLITE_DONE if there are no segments at
-** iLevel. Otherwise returns an error.
+/*
+** Implementation of xRollback(). Discard the contents of the pending-terms
+** hash-table. Any changes made to the database are reverted by SQLite.
*/
-static int segdir_max_index(fulltext_vtab *v, int iLevel, int *pidx){
- sqlite3_stmt *s;
- int rc = sql_get_statement(v, SEGDIR_MAX_INDEX_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int(s, 1, iLevel);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_step(s);
- /* Should always get at least one row due to how max() works. */
- if( rc==SQLITE_DONE ) return SQLITE_DONE;
- if( rc!=SQLITE_ROW ) return rc;
+static int fts3RollbackMethod(sqlite3_vtab *pVtab){
+ sqlite3Fts3PendingTermsClear((Fts3Table *)pVtab);
+ return SQLITE_OK;
+}
- /* NULL means that there were no inputs to max(). */
- if( SQLITE_NULL==sqlite3_column_type(s, 0) ){
- rc = sqlite3_step(s);
- if( rc==SQLITE_ROW ) return SQLITE_ERROR;
- return rc;
+/*
+** Helper function used by the implementation of the overloaded snippet(),
+** offsets() and optimize() SQL functions.
+**
+** If the value passed as the third argument is a blob of size
+** sizeof(Fts3Cursor*), then the blob contents are copied to the
+** output variable *ppCsr and SQLITE_OK is returned. Otherwise, an error
+** message is written to context pContext and SQLITE_ERROR returned. The
+** string passed via zFunc is used as part of the error message.
+*/
+static int fts3FunctionArg(
+ sqlite3_context *pContext, /* SQL function call context */
+ const char *zFunc, /* Function name */
+ sqlite3_value *pVal, /* argv[0] passed to function */
+ Fts3Cursor **ppCsr /* OUT: Store cursor handle here */
+){
+ Fts3Cursor *pRet;
+ if( sqlite3_value_type(pVal)!=SQLITE_BLOB
+ || sqlite3_value_bytes(pVal)!=sizeof(Fts3Cursor *)
+ ){
+ char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc);
+ sqlite3_result_error(pContext, zErr, -1);
+ sqlite3_free(zErr);
+ return SQLITE_ERROR;
}
-
- *pidx = sqlite3_column_int(s, 0);
-
- /* We expect only one row. We must execute another sqlite3_step()
- * to complete the iteration; otherwise the table will remain locked. */
- rc = sqlite3_step(s);
- if( rc==SQLITE_ROW ) return SQLITE_ERROR;
- if( rc!=SQLITE_DONE ) return rc;
- return SQLITE_ROW;
+ memcpy(&pRet, sqlite3_value_blob(pVal), sizeof(Fts3Cursor *));
+ *ppCsr = pRet;
+ return SQLITE_OK;
}
-/* insert into %_segdir values (
-** [iLevel], [idx],
-** [iStartBlockid], [iLeavesEndBlockid], [iEndBlockid],
-** [pRootData]
-** )
+/*
+** Implementation of the snippet() function for FTS3
*/
-static int segdir_set(fulltext_vtab *v, int iLevel, int idx,
- sqlite_int64 iStartBlockid,
- sqlite_int64 iLeavesEndBlockid,
- sqlite_int64 iEndBlockid,
- const char *pRootData, int nRootData){
- sqlite3_stmt *s;
- int rc = sql_get_statement(v, SEGDIR_SET_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int(s, 1, iLevel);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int(s, 2, idx);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int64(s, 3, iStartBlockid);
- if( rc!=SQLITE_OK ) return rc;
+static void fts3SnippetFunc(
+ sqlite3_context *pContext, /* SQLite function call context */
+ int nVal, /* Size of apVal[] array */
+ sqlite3_value **apVal /* Array of arguments */
+){
+ Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */
+ const char *zStart = "<b>";
+ const char *zEnd = "</b>";
+ const char *zEllipsis = "<b>...</b>";
- rc = sqlite3_bind_int64(s, 4, iLeavesEndBlockid);
- if( rc!=SQLITE_OK ) return rc;
+ /* There must be at least one argument passed to this function (otherwise
+ ** the non-overloaded version would have been called instead of this one).
+ */
+ assert( nVal>=1 );
- rc = sqlite3_bind_int64(s, 5, iEndBlockid);
- if( rc!=SQLITE_OK ) return rc;
+ if( nVal>4 ){
+ sqlite3_result_error(pContext,
+ "wrong number of arguments to function snippet()", -1);
+ return;
+ }
+ if( fts3FunctionArg(pContext, "snippet", apVal[0], &pCsr) ) return;
- rc = sqlite3_bind_blob(s, 6, pRootData, nRootData, SQLITE_STATIC);
- if( rc!=SQLITE_OK ) return rc;
+ switch( nVal ){
+ case 4: zEllipsis = (const char*)sqlite3_value_text(apVal[3]);
+ case 3: zEnd = (const char*)sqlite3_value_text(apVal[2]);
+ case 2: zStart = (const char*)sqlite3_value_text(apVal[1]);
+ }
- return sql_single_step(s);
+ sqlite3Fts3Snippet(pContext, pCsr, zStart, zEnd, zEllipsis);
}
-/* Queries %_segdir for the block span of the segments in level
-** iLevel. Returns SQLITE_DONE if there are no blocks for iLevel,
-** SQLITE_ROW if there are blocks, else an error.
+/*
+** Implementation of the offsets() function for FTS3
*/
-static int segdir_span(fulltext_vtab *v, int iLevel,
- sqlite_int64 *piStartBlockid,
- sqlite_int64 *piEndBlockid){
- sqlite3_stmt *s;
- int rc = sql_get_statement(v, SEGDIR_SPAN_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int(s, 1, iLevel);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_step(s);
- if( rc==SQLITE_DONE ) return SQLITE_DONE; /* Should never happen */
- if( rc!=SQLITE_ROW ) return rc;
-
- /* This happens if all segments at this level are entirely inline. */
- if( SQLITE_NULL==sqlite3_column_type(s, 0) ){
- /* We expect only one row. We must execute another sqlite3_step()
- * to complete the iteration; otherwise the table will remain locked. */
- int rc2 = sqlite3_step(s);
- if( rc2==SQLITE_ROW ) return SQLITE_ERROR;
- return rc2;
- }
+static void fts3OffsetsFunc(
+ sqlite3_context *pContext, /* SQLite function call context */
+ int nVal, /* Size of argument array */
+ sqlite3_value **apVal /* Array of arguments */
+){
+ Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */
- *piStartBlockid = sqlite3_column_int64(s, 0);
- *piEndBlockid = sqlite3_column_int64(s, 1);
+ UNUSED_PARAMETER(nVal);
- /* We expect only one row. We must execute another sqlite3_step()
- * to complete the iteration; otherwise the table will remain locked. */
- rc = sqlite3_step(s);
- if( rc==SQLITE_ROW ) return SQLITE_ERROR;
- if( rc!=SQLITE_DONE ) return rc;
- return SQLITE_ROW;
+ assert( nVal==1 );
+ if( fts3FunctionArg(pContext, "offsets", apVal[0], &pCsr) ) return;
+ assert( pCsr );
+ sqlite3Fts3Offsets(pContext, pCsr);
}
-/* Delete the segment blocks and segment directory records for all
-** segments at iLevel.
+/*
+** Implementation of the special optimize() function for FTS3. This
+** function merges all segments in the database to a single segment.
+** Example usage is:
+**
+** SELECT optimize(t) FROM t LIMIT 1;
+**
+** where 't' is the name of an FTS3 table.
*/
-static int segdir_delete(fulltext_vtab *v, int iLevel){
- sqlite3_stmt *s;
- sqlite_int64 iStartBlockid, iEndBlockid;
- int rc = segdir_span(v, iLevel, &iStartBlockid, &iEndBlockid);
- if( rc!=SQLITE_ROW && rc!=SQLITE_DONE ) return rc;
+static void fts3OptimizeFunc(
+ sqlite3_context *pContext, /* SQLite function call context */
+ int nVal, /* Size of argument array */
+ sqlite3_value **apVal /* Array of arguments */
+){
+ int rc; /* Return code */
+ Fts3Table *p; /* Virtual table handle */
+ Fts3Cursor *pCursor; /* Cursor handle passed through apVal[0] */
- if( rc==SQLITE_ROW ){
- rc = block_delete(v, iStartBlockid, iEndBlockid);
- if( rc!=SQLITE_OK ) return rc;
- }
+ UNUSED_PARAMETER(nVal);
- /* Delete the segment directory itself. */
- rc = sql_get_statement(v, SEGDIR_DELETE_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
+ assert( nVal==1 );
+ if( fts3FunctionArg(pContext, "optimize", apVal[0], &pCursor) ) return;
+ p = (Fts3Table *)pCursor->base.pVtab;
+ assert( p );
- rc = sqlite3_bind_int64(s, 1, iLevel);
- if( rc!=SQLITE_OK ) return rc;
+ rc = sqlite3Fts3Optimize(p);
- return sql_single_step(s);
+ switch( rc ){
+ case SQLITE_OK:
+ sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC);
+ break;
+ case SQLITE_DONE:
+ sqlite3_result_text(pContext, "Index already optimal", -1, SQLITE_STATIC);
+ break;
+ default:
+ sqlite3_result_error_code(pContext, rc);
+ break;
+ }
}
-/* Delete entire fts index, SQLITE_OK on success, relevant error on
-** failure.
+/*
+** This routine implements the xFindFunction method for the FTS3
+** virtual table.
*/
-static int segdir_delete_all(fulltext_vtab *v){
- sqlite3_stmt *s;
- int rc = sql_get_statement(v, SEGDIR_DELETE_ALL_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
+static int fts3FindFunctionMethod(
+ sqlite3_vtab *pVtab, /* Virtual table handle */
+ int nArg, /* Number of SQL function arguments */
+ const char *zName, /* Name of SQL function */
+ void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */
+ void **ppArg /* Unused */
+){
+ struct Overloaded {
+ const char *zName;
+ void (*xFunc)(sqlite3_context*,int,sqlite3_value**);
+ } aOverload[] = {
+ { "snippet", fts3SnippetFunc },
+ { "offsets", fts3OffsetsFunc },
+ { "optimize", fts3OptimizeFunc },
+ };
+ int i; /* Iterator variable */
- rc = sql_single_step(s);
- if( rc!=SQLITE_OK ) return rc;
+ UNUSED_PARAMETER(pVtab);
+ UNUSED_PARAMETER(nArg);
+ UNUSED_PARAMETER(ppArg);
- rc = sql_get_statement(v, BLOCK_DELETE_ALL_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
+ for(i=0; i<SizeofArray(aOverload); i++){
+ if( strcmp(zName, aOverload[i].zName)==0 ){
+ *pxFunc = aOverload[i].xFunc;
+ return 1;
+ }
+ }
- return sql_single_step(s);
+ /* No function of the specified name was found. Return 0. */
+ return 0;
}
-/* Returns SQLITE_OK with *pnSegments set to the number of entries in
-** %_segdir and *piMaxLevel set to the highest level which has a
-** segment. Otherwise returns the SQLite error which caused failure.
+/*
+** Implementation of FTS3 xRename method. Rename an fts3 table.
*/
-static int segdir_count(fulltext_vtab *v, int *pnSegments, int *piMaxLevel){
- sqlite3_stmt *s;
- int rc = sql_get_statement(v, SEGDIR_COUNT_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_step(s);
- /* TODO(shess): This case should not be possible? Should stronger
- ** measures be taken if it happens?
- */
- if( rc==SQLITE_DONE ){
- *pnSegments = 0;
- *piMaxLevel = 0;
- return SQLITE_OK;
+static int fts3RenameMethod(
+ sqlite3_vtab *pVtab, /* Virtual table handle */
+ const char *zName /* New name of table */
+){
+ Fts3Table *p = (Fts3Table *)pVtab;
+ int rc = SQLITE_NOMEM; /* Return Code */
+ char *zSql; /* SQL script to run to rename tables */
+
+ zSql = sqlite3_mprintf(
+ "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';"
+ "ALTER TABLE %Q.'%q_segments' RENAME TO '%q_segments';"
+ "ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';"
+ , p->zDb, p->zName, zName
+ , p->zDb, p->zName, zName
+ , p->zDb, p->zName, zName
+ );
+ if( zSql ){
+ rc = sqlite3_exec(p->db, zSql, 0, 0, 0);
+ sqlite3_free(zSql);
}
- if( rc!=SQLITE_ROW ) return rc;
+ return rc;
+}
- *pnSegments = sqlite3_column_int(s, 0);
- *piMaxLevel = sqlite3_column_int(s, 1);
+static const sqlite3_module fts3Module = {
+ /* iVersion */ 0,
+ /* xCreate */ fts3CreateMethod,
+ /* xConnect */ fts3ConnectMethod,
+ /* xBestIndex */ fts3BestIndexMethod,
+ /* xDisconnect */ fts3DisconnectMethod,
+ /* xDestroy */ fts3DestroyMethod,
+ /* xOpen */ fts3OpenMethod,
+ /* xClose */ fulltextClose,
+ /* xFilter */ fts3FilterMethod,
+ /* xNext */ fts3NextMethod,
+ /* xEof */ fts3EofMethod,
+ /* xColumn */ fts3ColumnMethod,
+ /* xRowid */ fts3RowidMethod,
+ /* xUpdate */ fts3UpdateMethod,
+ /* xBegin */ fts3BeginMethod,
+ /* xSync */ fts3SyncMethod,
+ /* xCommit */ fts3CommitMethod,
+ /* xRollback */ fts3RollbackMethod,
+ /* xFindFunction */ fts3FindFunctionMethod,
+ /* xRename */ fts3RenameMethod,
+};
- /* We expect only one row. We must execute another sqlite3_step()
- * to complete the iteration; otherwise the table will remain locked. */
- rc = sqlite3_step(s);
- if( rc==SQLITE_DONE ) return SQLITE_OK;
- if( rc==SQLITE_ROW ) return SQLITE_ERROR;
- return rc;
+/*
+** This function is registered as the module destructor (called when an
+** FTS3 enabled database connection is closed). It frees the memory
+** allocated for the tokenizer hash table.
+*/
+static void hashDestroy(void *p){
+ Fts3Hash *pHash = (Fts3Hash *)p;
+ sqlite3Fts3HashClear(pHash);
+ sqlite3_free(pHash);
}
-/* TODO(shess) clearPendingTerms() is far down the file because
-** writeZeroSegment() is far down the file because LeafWriter is far
-** down the file. Consider refactoring the code to move the non-vtab
-** code above the vtab code so that we don't need this forward
-** reference.
+/*
+** The fts3 built-in tokenizers - "simple" and "porter" - are implemented
+** in files fts3_tokenizer1.c and fts3_porter.c respectively. The following
+** two forward declarations are for functions declared in these files
+** used to retrieve the respective implementations.
+**
+** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed
+** to by the argument to point a the "simple" tokenizer implementation.
+** Function ...PorterTokenizerModule() sets *pModule to point to the
+** porter tokenizer/stemmer implementation.
*/
-static int clearPendingTerms(fulltext_vtab *v);
+SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
+SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule);
+SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule);
/*
-** Free the memory used to contain a fulltext_vtab structure.
+** Initialise the fts3 extension. If this extension is built as part
+** of the sqlite library, then this function is called directly by
+** SQLite. If fts3 is built as a dynamically loadable extension, this
+** function is called by the sqlite3_extension_init() entry point.
*/
-static void fulltext_vtab_destroy(fulltext_vtab *v){
- int iStmt, i;
+SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){
+ int rc = SQLITE_OK;
+ Fts3Hash *pHash = 0;
+ const sqlite3_tokenizer_module *pSimple = 0;
+ const sqlite3_tokenizer_module *pPorter = 0;
+ const sqlite3_tokenizer_module *pIcu = 0;
- FTSTRACE(("FTS3 Destroy %p\n", v));
- for( iStmt=0; iStmt<MAX_STMT; iStmt++ ){
- if( v->pFulltextStatements[iStmt]!=NULL ){
- sqlite3_finalize(v->pFulltextStatements[iStmt]);
- v->pFulltextStatements[iStmt] = NULL;
- }
+ sqlite3Fts3SimpleTokenizerModule(&pSimple);
+ sqlite3Fts3PorterTokenizerModule(&pPorter);
+#ifdef SQLITE_ENABLE_ICU
+ sqlite3Fts3IcuTokenizerModule(&pIcu);
+#endif
+
+ /* Allocate and initialise the hash-table used to store tokenizers. */
+ pHash = sqlite3_malloc(sizeof(Fts3Hash));
+ if( !pHash ){
+ rc = SQLITE_NOMEM;
+ }else{
+ sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1);
}
- for( i=0; i<MERGE_COUNT; i++ ){
- if( v->pLeafSelectStmts[i]!=NULL ){
- sqlite3_finalize(v->pLeafSelectStmts[i]);
- v->pLeafSelectStmts[i] = NULL;
+ /* Load the built-in tokenizers into the hash table */
+ if( rc==SQLITE_OK ){
+ if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple)
+ || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter)
+ || (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu))
+ ){
+ rc = SQLITE_NOMEM;
}
}
- if( v->pTokenizer!=NULL ){
- v->pTokenizer->pModule->xDestroy(v->pTokenizer);
- v->pTokenizer = NULL;
- }
+#ifdef SQLITE_TEST
+ sqlite3Fts3ExprInitTestInterface(db);
+#endif
- clearPendingTerms(v);
+ /* Create the virtual table wrapper around the hash-table and overload
+ ** the two scalar functions. If this is successful, register the
+ ** module with sqlite.
+ */
+ if( SQLITE_OK==rc
+ && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer"))
+ && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1))
+ && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1))
+ && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1))
+ ){
+ return sqlite3_create_module_v2(
+ db, "fts3", &fts3Module, (void *)pHash, hashDestroy
+ );
+ }
- sqlite3_free(v->azColumn);
- for(i = 0; i < v->nColumn; ++i) {
- sqlite3_free(v->azContentColumn[i]);
+ /* An error has occurred. Delete the hash table and return the error code. */
+ assert( rc!=SQLITE_OK );
+ if( pHash ){
+ sqlite3Fts3HashClear(pHash);
+ sqlite3_free(pHash);
}
- sqlite3_free(v->azContentColumn);
- sqlite3_free(v);
+ return rc;
+}
+
+#if !SQLITE_CORE
+SQLITE_API int sqlite3_extension_init(
+ sqlite3 *db,
+ char **pzErrMsg,
+ const sqlite3_api_routines *pApi
+){
+ SQLITE_EXTENSION_INIT2(pApi)
+ return sqlite3Fts3Init(db);
}
+#endif
+
+#endif
+/************** End of fts3.c ************************************************/
+/************** Begin file fts3_expr.c ***************************************/
/*
-** Token types for parsing the arguments to xConnect or xCreate.
+** 2008 Nov 28
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This module contains code that implements a parser for fts3 query strings
+** (the right-hand argument to the MATCH operator). Because the supported
+** syntax is relatively simple, the whole tokenizer/parser system is
+** hand-coded.
*/
-#define TOKEN_EOF 0 /* End of file */
-#define TOKEN_SPACE 1 /* Any kind of whitespace */
-#define TOKEN_ID 2 /* An identifier */
-#define TOKEN_STRING 3 /* A string literal */
-#define TOKEN_PUNCT 4 /* A single punctuation character */
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
/*
-** If X is a character that can be used in an identifier then
-** ftsIdChar(X) will be true. Otherwise it is false.
+** By default, this module parses the legacy syntax that has been
+** traditionally used by fts3. Or, if SQLITE_ENABLE_FTS3_PARENTHESIS
+** is defined, then it uses the new syntax. The differences between
+** the new and the old syntaxes are:
**
-** For ASCII, any character with the high-order bit set is
-** allowed in an identifier. For 7-bit characters,
-** isFtsIdChar[X] must be 1.
+** a) The new syntax supports parenthesis. The old does not.
**
-** Ticket #1066. the SQL standard does not allow '$' in the
-** middle of identfiers. But many SQL implementations do.
-** SQLite will allow '$' in identifiers for compatibility.
-** But the feature is undocumented.
+** b) The new syntax supports the AND and NOT operators. The old does not.
+**
+** c) The old syntax supports the "-" token qualifier. This is not
+** supported by the new syntax (it is replaced by the NOT operator).
+**
+** d) When using the old syntax, the OR operator has a greater precedence
+** than an implicit AND. When using the new, both implicity and explicit
+** AND operators have a higher precedence than OR.
+**
+** If compiled with SQLITE_TEST defined, then this module exports the
+** symbol "int sqlite3_fts3_enable_parentheses". Setting this variable
+** to zero causes the module to use the old syntax. If it is set to
+** non-zero the new syntax is activated. This is so both syntaxes can
+** be tested using a single build of testfixture.
+**
+** The following describes the syntax supported by the fts3 MATCH
+** operator in a similar format to that used by the lemon parser
+** generator. This module does not use actually lemon, it uses a
+** custom parser.
+**
+** query ::= andexpr (OR andexpr)*.
+**
+** andexpr ::= notexpr (AND? notexpr)*.
+**
+** notexpr ::= nearexpr (NOT nearexpr|-TOKEN)*.
+** notexpr ::= LP query RP.
+**
+** nearexpr ::= phrase (NEAR distance_opt nearexpr)*.
+**
+** distance_opt ::= .
+** distance_opt ::= / INTEGER.
+**
+** phrase ::= TOKEN.
+** phrase ::= COLUMN:TOKEN.
+** phrase ::= "TOKEN TOKEN TOKEN...".
*/
-static const char isFtsIdChar[] = {
-/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
- 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
- 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
- 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
-};
-#define ftsIdChar(C) (((c=C)&0x80)!=0 || (c>0x1f && isFtsIdChar[c-0x20]))
+#ifdef SQLITE_TEST
+SQLITE_API int sqlite3_fts3_enable_parentheses = 0;
+#else
+# ifdef SQLITE_ENABLE_FTS3_PARENTHESIS
+# define sqlite3_fts3_enable_parentheses 1
+# else
+# define sqlite3_fts3_enable_parentheses 0
+# endif
+#endif
/*
-** Return the length of the token that begins at z[0].
-** Store the token type in *tokenType before returning.
+** Default span for NEAR operators.
*/
-static int ftsGetToken(const char *z, int *tokenType){
- int i, c;
- switch( *z ){
- case 0: {
- *tokenType = TOKEN_EOF;
- return 0;
- }
- case ' ': case '\t': case '\n': case '\f': case '\r': {
- for(i=1; safe_isspace(z[i]); i++){}
- *tokenType = TOKEN_SPACE;
- return i;
- }
- case '`':
- case '\'':
- case '"': {
- int delim = z[0];
- for(i=1; (c=z[i])!=0; i++){
- if( c==delim ){
- if( z[i+1]==delim ){
- i++;
- }else{
- break;
- }
- }
- }
- *tokenType = TOKEN_STRING;
- return i + (c!=0);
- }
- case '[': {
- for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){}
- *tokenType = TOKEN_ID;
- return i;
- }
- default: {
- if( !ftsIdChar(*z) ){
- break;
- }
- for(i=1; ftsIdChar(z[i]); i++){}
- *tokenType = TOKEN_ID;
- return i;
- }
- }
- *tokenType = TOKEN_PUNCT;
- return 1;
-}
+#define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10
-/*
-** A token extracted from a string is an instance of the following
-** structure.
-*/
-typedef struct FtsToken {
- const char *z; /* Pointer to token text. Not '\000' terminated */
- short int n; /* Length of the token text in bytes. */
-} FtsToken;
+
+typedef struct ParseContext ParseContext;
+struct ParseContext {
+ sqlite3_tokenizer *pTokenizer; /* Tokenizer module */
+ const char **azCol; /* Array of column names for fts3 table */
+ int nCol; /* Number of entries in azCol[] */
+ int iDefaultCol; /* Default column to query */
+ sqlite3_context *pCtx; /* Write error message here */
+ int nNest; /* Number of nested brackets */
+};
/*
-** Given a input string (which is really one of the argv[] parameters
-** passed into xConnect or xCreate) split the string up into tokens.
-** Return an array of pointers to '\000' terminated strings, one string
-** for each non-whitespace token.
-**
-** The returned array is terminated by a single NULL pointer.
+** This function is equivalent to the standard isspace() function.
**
-** Space to hold the returned array is obtained from a single
-** malloc and should be freed by passing the return value to free().
-** The individual strings within the token list are all a part of
-** the single memory allocation and will all be freed at once.
+** The standard isspace() can be awkward to use safely, because although it
+** is defined to accept an argument of type int, its behaviour when passed
+** an integer that falls outside of the range of the unsigned char type
+** is undefined (and sometimes, "undefined" means segfault). This wrapper
+** is defined to accept an argument of type char, and always returns 0 for
+** any values that fall outside of the range of the unsigned char type (i.e.
+** negative values).
*/
-static char **tokenizeString(const char *z, int *pnToken){
- int nToken = 0;
- FtsToken *aToken = sqlite3_malloc( strlen(z) * sizeof(aToken[0]) );
- int n = 1;
- int e, i;
- int totalSize = 0;
- char **azToken;
- char *zCopy;
- while( n>0 ){
- n = ftsGetToken(z, &e);
- if( e!=TOKEN_SPACE ){
- aToken[nToken].z = z;
- aToken[nToken].n = n;
- nToken++;
- totalSize += n+1;
- }
- z += n;
- }
- azToken = (char**)sqlite3_malloc( nToken*sizeof(char*) + totalSize );
- zCopy = (char*)&azToken[nToken];
- nToken--;
- for(i=0; i<nToken; i++){
- azToken[i] = zCopy;
- n = aToken[i].n;
- memcpy(zCopy, aToken[i].z, n);
- zCopy[n] = 0;
- zCopy += n+1;
- }
- azToken[nToken] = 0;
- sqlite3_free(aToken);
- *pnToken = nToken;
- return azToken;
+static int fts3isspace(char c){
+ return (c&0x80)==0 ? isspace(c) : 0;
}
/*
-** Convert an SQL-style quoted string into a normal string by removing
-** the quote characters. The conversion is done in-place. If the
-** input does not begin with a quote character, then this routine
-** is a no-op.
-**
-** Examples:
+** Extract the next token from buffer z (length n) using the tokenizer
+** and other information (column names etc.) in pParse. Create an Fts3Expr
+** structure of type FTSQUERY_PHRASE containing a phrase consisting of this
+** single token and set *ppExpr to point to it. If the end of the buffer is
+** reached before a token is found, set *ppExpr to zero. It is the
+** responsibility of the caller to eventually deallocate the allocated
+** Fts3Expr structure (if any) by passing it to sqlite3_free().
**
-** "abc" becomes abc
-** 'xyz' becomes xyz
-** [pqr] becomes pqr
-** `mno` becomes mno
+** Return SQLITE_OK if successful, or SQLITE_NOMEM if a memory allocation
+** fails.
*/
-static void dequoteString(char *z){
- int quote;
- int i, j;
- if( z==0 ) return;
- quote = z[0];
- switch( quote ){
- case '\'': break;
- case '"': break;
- case '`': break; /* For MySQL compatibility */
- case '[': quote = ']'; break; /* For MS SqlServer compatibility */
- default: return;
- }
- for(i=1, j=0; z[i]; i++){
- if( z[i]==quote ){
- if( z[i+1]==quote ){
- z[j++] = quote;
- i++;
+static int getNextToken(
+ ParseContext *pParse, /* fts3 query parse context */
+ int iCol, /* Value for Fts3Phrase.iColumn */
+ const char *z, int n, /* Input string */
+ Fts3Expr **ppExpr, /* OUT: expression */
+ int *pnConsumed /* OUT: Number of bytes consumed */
+){
+ sqlite3_tokenizer *pTokenizer = pParse->pTokenizer;
+ sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
+ int rc;
+ sqlite3_tokenizer_cursor *pCursor;
+ Fts3Expr *pRet = 0;
+ int nConsumed = 0;
+
+ rc = pModule->xOpen(pTokenizer, z, n, &pCursor);
+ if( rc==SQLITE_OK ){
+ const char *zToken;
+ int nToken, iStart, iEnd, iPosition;
+ int nByte; /* total space to allocate */
+
+ pCursor->pTokenizer = pTokenizer;
+ rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition);
+
+ if( rc==SQLITE_OK ){
+ nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken;
+ pRet = (Fts3Expr *)sqlite3_malloc(nByte);
+ if( !pRet ){
+ rc = SQLITE_NOMEM;
}else{
- z[j++] = 0;
- break;
- }
- }else{
- z[j++] = z[i];
- }
- }
-}
+ memset(pRet, 0, nByte);
+ pRet->eType = FTSQUERY_PHRASE;
+ pRet->pPhrase = (Fts3Phrase *)&pRet[1];
+ pRet->pPhrase->nToken = 1;
+ pRet->pPhrase->iColumn = iCol;
+ pRet->pPhrase->aToken[0].n = nToken;
+ pRet->pPhrase->aToken[0].z = (char *)&pRet->pPhrase[1];
+ memcpy(pRet->pPhrase->aToken[0].z, zToken, nToken);
-/*
-** The input azIn is a NULL-terminated list of tokens. Remove the first
-** token and all punctuation tokens. Remove the quotes from
-** around string literal tokens.
-**
-** Example:
-**
-** input: tokenize chinese ( 'simplifed' , 'mixed' )
-** output: chinese simplifed mixed
-**
-** Another example:
-**
-** input: delimiters ( '[' , ']' , '...' )
-** output: [ ] ...
-*/
-static void tokenListToIdList(char **azIn){
- int i, j;
- if( azIn ){
- for(i=0, j=-1; azIn[i]; i++){
- if( safe_isalnum(azIn[i][0]) || azIn[i][1] ){
- dequoteString(azIn[i]);
- if( j>=0 ){
- azIn[j] = azIn[i];
+ if( iEnd<n && z[iEnd]=='*' ){
+ pRet->pPhrase->aToken[0].isPrefix = 1;
+ iEnd++;
+ }
+ if( !sqlite3_fts3_enable_parentheses && iStart>0 && z[iStart-1]=='-' ){
+ pRet->pPhrase->isNot = 1;
}
- j++;
}
+ nConsumed = iEnd;
}
- azIn[j] = 0;
+
+ pModule->xClose(pCursor);
}
+
+ *pnConsumed = nConsumed;
+ *ppExpr = pRet;
+ return rc;
}
/*
-** Find the first alphanumeric token in the string zIn. Null-terminate
-** this token. Remove any quotation marks. And return a pointer to
-** the result.
-*/
-static char *firstToken(char *zIn, char **pzTail){
- int n, ttype;
- while(1){
- n = ftsGetToken(zIn, &ttype);
- if( ttype==TOKEN_SPACE ){
- zIn += n;
- }else if( ttype==TOKEN_EOF ){
- *pzTail = zIn;
- return 0;
- }else{
- zIn[n] = 0;
- *pzTail = &zIn[1];
- dequoteString(zIn);
- return zIn;
- }
- }
- /*NOTREACHED*/
-}
-
-/* Return true if...
-**
-** * s begins with the string t, ignoring case
-** * s is longer than t
-** * The first character of s beyond t is not a alphanumeric
-**
-** Ignore leading space in *s.
-**
-** To put it another way, return true if the first token of
-** s[] is t[].
+** Enlarge a memory allocation. If an out-of-memory allocation occurs,
+** then free the old allocation.
*/
-static int startsWith(const char *s, const char *t){
- while( safe_isspace(*s) ){ s++; }
- while( *t ){
- if( safe_tolower(*s++)!=safe_tolower(*t++) ) return 0;
+void *fts3ReallocOrFree(void *pOrig, int nNew){
+ void *pRet = sqlite3_realloc(pOrig, nNew);
+ if( !pRet ){
+ sqlite3_free(pOrig);
}
- return *s!='_' && !safe_isalnum(*s);
+ return pRet;
}
/*
-** An instance of this structure defines the "spec" of a
-** full text index. This structure is populated by parseSpec
-** and use by fulltextConnect and fulltextCreate.
-*/
-typedef struct TableSpec {
- const char *zDb; /* Logical database name */
- const char *zName; /* Name of the full-text index */
- int nColumn; /* Number of columns to be indexed */
- char **azColumn; /* Original names of columns to be indexed */
- char **azContentColumn; /* Column names for %_content */
- char **azTokenizer; /* Name of tokenizer and its arguments */
-} TableSpec;
-
-/*
-** Reclaim all of the memory used by a TableSpec
+** Buffer zInput, length nInput, contains the contents of a quoted string
+** that appeared as part of an fts3 query expression. Neither quote character
+** is included in the buffer. This function attempts to tokenize the entire
+** input buffer and create an Fts3Expr structure of type FTSQUERY_PHRASE
+** containing the results.
+**
+** If successful, SQLITE_OK is returned and *ppExpr set to point at the
+** allocated Fts3Expr structure. Otherwise, either SQLITE_NOMEM (out of memory
+** error) or SQLITE_ERROR (tokenization error) is returned and *ppExpr set
+** to 0.
*/
-static void clearTableSpec(TableSpec *p) {
- sqlite3_free(p->azColumn);
- sqlite3_free(p->azContentColumn);
- sqlite3_free(p->azTokenizer);
-}
-
-/* Parse a CREATE VIRTUAL TABLE statement, which looks like this:
- *
- * CREATE VIRTUAL TABLE email
- * USING fts3(subject, body, tokenize mytokenizer(myarg))
- *
- * We return parsed information in a TableSpec structure.
- *
- */
-static int parseSpec(TableSpec *pSpec, int argc, const char *const*argv,
- char**pzErr){
- int i, n;
- char *z, *zDummy;
- char **azArg;
- const char *zTokenizer = 0; /* argv[] entry describing the tokenizer */
+static int getNextString(
+ ParseContext *pParse, /* fts3 query parse context */
+ const char *zInput, int nInput, /* Input string */
+ Fts3Expr **ppExpr /* OUT: expression */
+){
+ sqlite3_tokenizer *pTokenizer = pParse->pTokenizer;
+ sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
+ int rc;
+ Fts3Expr *p = 0;
+ sqlite3_tokenizer_cursor *pCursor = 0;
+ char *zTemp = 0;
+ int nTemp = 0;
- assert( argc>=3 );
- /* Current interface:
- ** argv[0] - module name
- ** argv[1] - database name
- ** argv[2] - table name
- ** argv[3..] - columns, optionally followed by tokenizer specification
- ** and snippet delimiters specification.
- */
+ rc = pModule->xOpen(pTokenizer, zInput, nInput, &pCursor);
+ if( rc==SQLITE_OK ){
+ int ii;
+ pCursor->pTokenizer = pTokenizer;
+ for(ii=0; rc==SQLITE_OK; ii++){
+ const char *zToken;
+ int nToken, iBegin, iEnd, iPos;
+ rc = pModule->xNext(pCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos);
+ if( rc==SQLITE_OK ){
+ int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
+ p = fts3ReallocOrFree(p, nByte+ii*sizeof(struct PhraseToken));
+ zTemp = fts3ReallocOrFree(zTemp, nTemp + nToken);
+ if( !p || !zTemp ){
+ goto no_mem;
+ }
+ if( ii==0 ){
+ memset(p, 0, nByte);
+ p->pPhrase = (Fts3Phrase *)&p[1];
+ }
+ p->pPhrase = (Fts3Phrase *)&p[1];
+ p->pPhrase->nToken = ii+1;
+ p->pPhrase->aToken[ii].n = nToken;
+ memcpy(&zTemp[nTemp], zToken, nToken);
+ nTemp += nToken;
+ if( iEnd<nInput && zInput[iEnd]=='*' ){
+ p->pPhrase->aToken[ii].isPrefix = 1;
+ }else{
+ p->pPhrase->aToken[ii].isPrefix = 0;
+ }
+ }
+ }
- /* Make a copy of the complete argv[][] array in a single allocation.
- ** The argv[][] array is read-only and transient. We can write to the
- ** copy in order to modify things and the copy is persistent.
- */
- CLEAR(pSpec);
- for(i=n=0; i<argc; i++){
- n += strlen(argv[i]) + 1;
- }
- azArg = sqlite3_malloc( sizeof(char*)*argc + n );
- if( azArg==0 ){
- return SQLITE_NOMEM;
+ pModule->xClose(pCursor);
+ pCursor = 0;
}
- z = (char*)&azArg[argc];
- for(i=0; i<argc; i++){
- azArg[i] = z;
- strcpy(z, argv[i]);
- z += strlen(z)+1;
- }
-
- /* Identify the column names and the tokenizer and delimiter arguments
- ** in the argv[][] array.
- */
- pSpec->zDb = azArg[1];
- pSpec->zName = azArg[2];
- pSpec->nColumn = 0;
- pSpec->azColumn = azArg;
- zTokenizer = "tokenize simple";
- for(i=3; i<argc; ++i){
- if( startsWith(azArg[i],"tokenize") ){
- zTokenizer = azArg[i];
+
+ if( rc==SQLITE_DONE ){
+ int jj;
+ char *zNew = NULL;
+ int nNew = 0;
+ int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
+ nByte += (p?(p->pPhrase->nToken-1):0) * sizeof(struct PhraseToken);
+ p = fts3ReallocOrFree(p, nByte + nTemp);
+ if( !p ){
+ goto no_mem;
+ }
+ if( zTemp ){
+ zNew = &(((char *)p)[nByte]);
+ memcpy(zNew, zTemp, nTemp);
}else{
- z = azArg[pSpec->nColumn] = firstToken(azArg[i], &zDummy);
- pSpec->nColumn++;
+ memset(p, 0, nByte+nTemp);
}
- }
- if( pSpec->nColumn==0 ){
- azArg[0] = "content";
- pSpec->nColumn = 1;
- }
-
- /*
- ** Construct the list of content column names.
- **
- ** Each content column name will be of the form cNNAAAA
- ** where NN is the column number and AAAA is the sanitized
- ** column name. "sanitized" means that special characters are
- ** converted to "_". The cNN prefix guarantees that all column
- ** names are unique.
- **
- ** The AAAA suffix is not strictly necessary. It is included
- ** for the convenience of people who might examine the generated
- ** %_content table and wonder what the columns are used for.
- */
- pSpec->azContentColumn = sqlite3_malloc( pSpec->nColumn * sizeof(char *) );
- if( pSpec->azContentColumn==0 ){
- clearTableSpec(pSpec);
- return SQLITE_NOMEM;
- }
- for(i=0; i<pSpec->nColumn; i++){
- char *p;
- pSpec->azContentColumn[i] = sqlite3_mprintf("c%d%s", i, azArg[i]);
- for (p = pSpec->azContentColumn[i]; *p ; ++p) {
- if( !safe_isalnum(*p) ) *p = '_';
+ p->pPhrase = (Fts3Phrase *)&p[1];
+ for(jj=0; jj<p->pPhrase->nToken; jj++){
+ p->pPhrase->aToken[jj].z = &zNew[nNew];
+ nNew += p->pPhrase->aToken[jj].n;
}
+ sqlite3_free(zTemp);
+ p->eType = FTSQUERY_PHRASE;
+ p->pPhrase->iColumn = pParse->iDefaultCol;
+ rc = SQLITE_OK;
}
- /*
- ** Parse the tokenizer specification string.
- */
- pSpec->azTokenizer = tokenizeString(zTokenizer, &n);
- tokenListToIdList(pSpec->azTokenizer);
+ *ppExpr = p;
+ return rc;
+no_mem:
- return SQLITE_OK;
+ if( pCursor ){
+ pModule->xClose(pCursor);
+ }
+ sqlite3_free(zTemp);
+ sqlite3_free(p);
+ *ppExpr = 0;
+ return SQLITE_NOMEM;
}
/*
-** Generate a CREATE TABLE statement that describes the schema of
-** the virtual table. Return a pointer to this schema string.
+** Function getNextNode(), which is called by fts3ExprParse(), may itself
+** call fts3ExprParse(). So this forward declaration is required.
+*/
+static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *);
+
+/*
+** The output variable *ppExpr is populated with an allocated Fts3Expr
+** structure, or set to 0 if the end of the input buffer is reached.
**
-** Space is obtained from sqlite3_mprintf() and should be freed
-** using sqlite3_free().
+** Returns an SQLite error code. SQLITE_OK if everything works, SQLITE_NOMEM
+** if a malloc failure occurs, or SQLITE_ERROR if a parse error is encountered.
+** If SQLITE_ERROR is returned, pContext is populated with an error message.
*/
-static char *fulltextSchema(
- int nColumn, /* Number of columns */
- const char *const* azColumn, /* List of columns */
- const char *zTableName /* Name of the table */
-){
- int i;
- char *zSchema, *zNext;
- const char *zSep = "(";
- zSchema = sqlite3_mprintf("CREATE TABLE x");
- for(i=0; i<nColumn; i++){
- zNext = sqlite3_mprintf("%s%s%Q", zSchema, zSep, azColumn[i]);
- sqlite3_free(zSchema);
- zSchema = zNext;
- zSep = ",";
- }
- zNext = sqlite3_mprintf("%s,%Q HIDDEN", zSchema, zTableName);
- sqlite3_free(zSchema);
- zSchema = zNext;
- zNext = sqlite3_mprintf("%s,docid HIDDEN)", zSchema);
- sqlite3_free(zSchema);
- return zNext;
-}
-
-/*
-** Build a new sqlite3_vtab structure that will describe the
-** fulltext index defined by spec.
-*/
-static int constructVtab(
- sqlite3 *db, /* The SQLite database connection */
- fts3Hash *pHash, /* Hash table containing tokenizers */
- TableSpec *spec, /* Parsed spec information from parseSpec() */
- sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */
- char **pzErr /* Write any error message here */
+static int getNextNode(
+ ParseContext *pParse, /* fts3 query parse context */
+ const char *z, int n, /* Input string */
+ Fts3Expr **ppExpr, /* OUT: expression */
+ int *pnConsumed /* OUT: Number of bytes consumed */
){
+ static const struct Fts3Keyword {
+ char *z; /* Keyword text */
+ unsigned char n; /* Length of the keyword */
+ unsigned char parenOnly; /* Only valid in paren mode */
+ unsigned char eType; /* Keyword code */
+ } aKeyword[] = {
+ { "OR" , 2, 0, FTSQUERY_OR },
+ { "AND", 3, 1, FTSQUERY_AND },
+ { "NOT", 3, 1, FTSQUERY_NOT },
+ { "NEAR", 4, 0, FTSQUERY_NEAR }
+ };
+ int ii;
+ int iCol;
+ int iColLen;
int rc;
- int n;
- fulltext_vtab *v = 0;
- const sqlite3_tokenizer_module *m = NULL;
- char *schema;
-
- char const *zTok; /* Name of tokenizer to use for this fts table */
- int nTok; /* Length of zTok, including nul terminator */
-
- v = (fulltext_vtab *) sqlite3_malloc(sizeof(fulltext_vtab));
- if( v==0 ) return SQLITE_NOMEM;
- CLEAR(v);
- /* sqlite will initialize v->base */
- v->db = db;
- v->zDb = spec->zDb; /* Freed when azColumn is freed */
- v->zName = spec->zName; /* Freed when azColumn is freed */
- v->nColumn = spec->nColumn;
- v->azContentColumn = spec->azContentColumn;
- spec->azContentColumn = 0;
- v->azColumn = spec->azColumn;
- spec->azColumn = 0;
-
- if( spec->azTokenizer==0 ){
- return SQLITE_NOMEM;
- }
+ Fts3Expr *pRet = 0;
- zTok = spec->azTokenizer[0];
- if( !zTok ){
- zTok = "simple";
- }
- nTok = strlen(zTok)+1;
+ const char *zInput = z;
+ int nInput = n;
- m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zTok, nTok);
- if( !m ){
- *pzErr = sqlite3_mprintf("unknown tokenizer: %s", spec->azTokenizer[0]);
- rc = SQLITE_ERROR;
- goto err;
+ /* Skip over any whitespace before checking for a keyword, an open or
+ ** close bracket, or a quoted string.
+ */
+ while( nInput>0 && fts3isspace(*zInput) ){
+ nInput--;
+ zInput++;
}
-
- for(n=0; spec->azTokenizer[n]; n++){}
- if( n ){
- rc = m->xCreate(n-1, (const char*const*)&spec->azTokenizer[1],
- &v->pTokenizer);
- }else{
- rc = m->xCreate(0, 0, &v->pTokenizer);
+ if( nInput==0 ){
+ return SQLITE_DONE;
}
- if( rc!=SQLITE_OK ) goto err;
- v->pTokenizer->pModule = m;
-
- /* TODO: verify the existence of backing tables foo_content, foo_term */
-
- schema = fulltextSchema(v->nColumn, (const char*const*)v->azColumn,
- spec->zName);
- rc = sqlite3_declare_vtab(db, schema);
- sqlite3_free(schema);
- if( rc!=SQLITE_OK ) goto err;
-
- memset(v->pFulltextStatements, 0, sizeof(v->pFulltextStatements));
-
- /* Indicate that the buffer is not live. */
- v->nPendingData = -1;
-
- *ppVTab = &v->base;
- FTSTRACE(("FTS3 Connect %p\n", v));
-
- return rc;
-
-err:
- fulltext_vtab_destroy(v);
- return rc;
-}
-static int fulltextConnect(
- sqlite3 *db,
- void *pAux,
- int argc, const char *const*argv,
- sqlite3_vtab **ppVTab,
- char **pzErr
-){
- TableSpec spec;
- int rc = parseSpec(&spec, argc, argv, pzErr);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = constructVtab(db, (fts3Hash *)pAux, &spec, ppVTab, pzErr);
- clearTableSpec(&spec);
- return rc;
-}
+ /* See if we are dealing with a keyword. */
+ for(ii=0; ii<(int)(sizeof(aKeyword)/sizeof(struct Fts3Keyword)); ii++){
+ const struct Fts3Keyword *pKey = &aKeyword[ii];
-/* The %_content table holds the text of each document, with
-** the docid column exposed as the SQLite rowid for the table.
-*/
-/* TODO(shess) This comment needs elaboration to match the updated
-** code. Work it into the top-of-file comment at that time.
-*/
-static int fulltextCreate(sqlite3 *db, void *pAux,
- int argc, const char * const *argv,
- sqlite3_vtab **ppVTab, char **pzErr){
- int rc;
- TableSpec spec;
- StringBuffer schema;
- FTSTRACE(("FTS3 Create\n"));
+ if( (pKey->parenOnly & ~sqlite3_fts3_enable_parentheses)!=0 ){
+ continue;
+ }
- rc = parseSpec(&spec, argc, argv, pzErr);
- if( rc!=SQLITE_OK ) return rc;
+ if( nInput>=pKey->n && 0==memcmp(zInput, pKey->z, pKey->n) ){
+ int nNear = SQLITE_FTS3_DEFAULT_NEAR_PARAM;
+ int nKey = pKey->n;
+ char cNext;
- initStringBuffer(&schema);
- append(&schema, "CREATE TABLE %_content(");
- append(&schema, " docid INTEGER PRIMARY KEY,");
- appendList(&schema, spec.nColumn, spec.azContentColumn);
- append(&schema, ")");
- rc = sql_exec(db, spec.zDb, spec.zName, stringBufferData(&schema));
- stringBufferDestroy(&schema);
- if( rc!=SQLITE_OK ) goto out;
-
- rc = sql_exec(db, spec.zDb, spec.zName,
- "create table %_segments("
- " blockid INTEGER PRIMARY KEY,"
- " block blob"
- ");"
- );
- if( rc!=SQLITE_OK ) goto out;
-
- rc = sql_exec(db, spec.zDb, spec.zName,
- "create table %_segdir("
- " level integer,"
- " idx integer,"
- " start_block integer,"
- " leaves_end_block integer,"
- " end_block integer,"
- " root blob,"
- " primary key(level, idx)"
- ");");
- if( rc!=SQLITE_OK ) goto out;
-
- rc = constructVtab(db, (fts3Hash *)pAux, &spec, ppVTab, pzErr);
-
-out:
- clearTableSpec(&spec);
- return rc;
-}
+ /* If this is a "NEAR" keyword, check for an explicit nearness. */
+ if( pKey->eType==FTSQUERY_NEAR ){
+ assert( nKey==4 );
+ if( zInput[4]=='/' && zInput[5]>='0' && zInput[5]<='9' ){
+ nNear = 0;
+ for(nKey=5; zInput[nKey]>='0' && zInput[nKey]<='9'; nKey++){
+ nNear = nNear * 10 + (zInput[nKey] - '0');
+ }
+ }
+ }
-/* Decide how to handle an SQL query. */
-static int fulltextBestIndex(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
- fulltext_vtab *v = (fulltext_vtab *)pVTab;
- int i;
- FTSTRACE(("FTS3 BestIndex\n"));
-
- for(i=0; i<pInfo->nConstraint; ++i){
- const struct sqlite3_index_constraint *pConstraint;
- pConstraint = &pInfo->aConstraint[i];
- if( pConstraint->usable ) {
- if( (pConstraint->iColumn==-1 || pConstraint->iColumn==v->nColumn+1) &&
- pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){
- pInfo->idxNum = QUERY_DOCID; /* lookup by docid */
- FTSTRACE(("FTS3 QUERY_DOCID\n"));
- } else if( pConstraint->iColumn>=0 && pConstraint->iColumn<=v->nColumn &&
- pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH ){
- /* full-text search */
- pInfo->idxNum = QUERY_FULLTEXT + pConstraint->iColumn;
- FTSTRACE(("FTS3 QUERY_FULLTEXT %d\n", pConstraint->iColumn));
- } else continue;
-
- pInfo->aConstraintUsage[i].argvIndex = 1;
- pInfo->aConstraintUsage[i].omit = 1;
-
- /* An arbitrary value for now.
- * TODO: Perhaps docid matches should be considered cheaper than
- * full-text searches. */
- pInfo->estimatedCost = 1.0;
+ /* At this point this is probably a keyword. But for that to be true,
+ ** the next byte must contain either whitespace, an open or close
+ ** parenthesis, a quote character, or EOF.
+ */
+ cNext = zInput[nKey];
+ if( fts3isspace(cNext)
+ || cNext=='"' || cNext=='(' || cNext==')' || cNext==0
+ ){
+ pRet = (Fts3Expr *)sqlite3_malloc(sizeof(Fts3Expr));
+ if( !pRet ){
+ return SQLITE_NOMEM;
+ }
+ memset(pRet, 0, sizeof(Fts3Expr));
+ pRet->eType = pKey->eType;
+ pRet->nNear = nNear;
+ *ppExpr = pRet;
+ *pnConsumed = (int)((zInput - z) + nKey);
+ return SQLITE_OK;
+ }
- return SQLITE_OK;
+ /* Turns out that wasn't a keyword after all. This happens if the
+ ** user has supplied a token such as "ORacle". Continue.
+ */
}
}
- pInfo->idxNum = QUERY_GENERIC;
- return SQLITE_OK;
-}
-
-static int fulltextDisconnect(sqlite3_vtab *pVTab){
- FTSTRACE(("FTS3 Disconnect %p\n", pVTab));
- fulltext_vtab_destroy((fulltext_vtab *)pVTab);
- return SQLITE_OK;
-}
-
-static int fulltextDestroy(sqlite3_vtab *pVTab){
- fulltext_vtab *v = (fulltext_vtab *)pVTab;
- int rc;
- FTSTRACE(("FTS3 Destroy %p\n", pVTab));
- rc = sql_exec(v->db, v->zDb, v->zName,
- "drop table if exists %_content;"
- "drop table if exists %_segments;"
- "drop table if exists %_segdir;"
- );
- if( rc!=SQLITE_OK ) return rc;
+ /* Check for an open bracket. */
+ if( sqlite3_fts3_enable_parentheses ){
+ if( *zInput=='(' ){
+ int nConsumed;
+ int rc;
+ pParse->nNest++;
+ rc = fts3ExprParse(pParse, &zInput[1], nInput-1, ppExpr, &nConsumed);
+ if( rc==SQLITE_OK && !*ppExpr ){
+ rc = SQLITE_DONE;
+ }
+ *pnConsumed = (int)((zInput - z) + 1 + nConsumed);
+ return rc;
+ }
+
+ /* Check for a close bracket. */
+ if( *zInput==')' ){
+ pParse->nNest--;
+ *pnConsumed = (int)((zInput - z) + 1);
+ return SQLITE_DONE;
+ }
+ }
- fulltext_vtab_destroy((fulltext_vtab *)pVTab);
- return SQLITE_OK;
-}
+ /* See if we are dealing with a quoted phrase. If this is the case, then
+ ** search for the closing quote and pass the whole string to getNextString()
+ ** for processing. This is easy to do, as fts3 has no syntax for escaping
+ ** a quote character embedded in a string.
+ */
+ if( *zInput=='"' ){
+ for(ii=1; ii<nInput && zInput[ii]!='"'; ii++);
+ *pnConsumed = (int)((zInput - z) + ii + 1);
+ if( ii==nInput ){
+ return SQLITE_ERROR;
+ }
+ return getNextString(pParse, &zInput[1], ii-1, ppExpr);
+ }
-static int fulltextOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
- fulltext_cursor *c;
- c = (fulltext_cursor *) sqlite3_malloc(sizeof(fulltext_cursor));
- if( c ){
- memset(c, 0, sizeof(fulltext_cursor));
- /* sqlite will initialize c->base */
- *ppCursor = &c->base;
- FTSTRACE(("FTS3 Open %p: %p\n", pVTab, c));
- return SQLITE_OK;
- }else{
- return SQLITE_NOMEM;
+ /* If control flows to this point, this must be a regular token, or
+ ** the end of the input. Read a regular token using the sqlite3_tokenizer
+ ** interface. Before doing so, figure out if there is an explicit
+ ** column specifier for the token.
+ **
+ ** TODO: Strangely, it is not possible to associate a column specifier
+ ** with a quoted phrase, only with a single token. Not sure if this was
+ ** an implementation artifact or an intentional decision when fts3 was
+ ** first implemented. Whichever it was, this module duplicates the
+ ** limitation.
+ */
+ iCol = pParse->iDefaultCol;
+ iColLen = 0;
+ for(ii=0; ii<pParse->nCol; ii++){
+ const char *zStr = pParse->azCol[ii];
+ int nStr = (int)strlen(zStr);
+ if( nInput>nStr && zInput[nStr]==':'
+ && sqlite3_strnicmp(zStr, zInput, nStr)==0
+ ){
+ iCol = ii;
+ iColLen = (int)((zInput - z) + nStr + 1);
+ break;
+ }
}
+ rc = getNextToken(pParse, iCol, &z[iColLen], n-iColLen, ppExpr, pnConsumed);
+ *pnConsumed += iColLen;
+ return rc;
}
-/* Free all of the dynamically allocated memory held by the
-** Snippet
+/*
+** The argument is an Fts3Expr structure for a binary operator (any type
+** except an FTSQUERY_PHRASE). Return an integer value representing the
+** precedence of the operator. Lower values have a higher precedence (i.e.
+** group more tightly). For example, in the C language, the == operator
+** groups more tightly than ||, and would therefore have a higher precedence.
+**
+** When using the new fts3 query syntax (when SQLITE_ENABLE_FTS3_PARENTHESIS
+** is defined), the order of the operators in precedence from highest to
+** lowest is:
+**
+** NEAR
+** NOT
+** AND (including implicit ANDs)
+** OR
+**
+** Note that when using the old query syntax, the OR operator has a higher
+** precedence than the AND operator.
*/
-static void snippetClear(Snippet *p){
- sqlite3_free(p->aMatch);
- sqlite3_free(p->zOffset);
- sqlite3_free(p->zSnippet);
- CLEAR(p);
+static int opPrecedence(Fts3Expr *p){
+ assert( p->eType!=FTSQUERY_PHRASE );
+ if( sqlite3_fts3_enable_parentheses ){
+ return p->eType;
+ }else if( p->eType==FTSQUERY_NEAR ){
+ return 1;
+ }else if( p->eType==FTSQUERY_OR ){
+ return 2;
+ }
+ assert( p->eType==FTSQUERY_AND );
+ return 3;
}
/*
-** Append a single entry to the p->aMatch[] log.
+** Argument ppHead contains a pointer to the current head of a query
+** expression tree being parsed. pPrev is the expression node most recently
+** inserted into the tree. This function adds pNew, which is always a binary
+** operator node, into the expression tree based on the relative precedence
+** of pNew and the existing nodes of the tree. This may result in the head
+** of the tree changing, in which case *ppHead is set to the new root node.
*/
-static void snippetAppendMatch(
- Snippet *p, /* Append the entry to this snippet */
- int iCol, int iTerm, /* The column and query term */
- int iToken, /* Matching token in document */
- int iStart, int nByte /* Offset and size of the match */
+static void insertBinaryOperator(
+ Fts3Expr **ppHead, /* Pointer to the root node of a tree */
+ Fts3Expr *pPrev, /* Node most recently inserted into the tree */
+ Fts3Expr *pNew /* New binary node to insert into expression tree */
){
- int i;
- struct snippetMatch *pMatch;
- if( p->nMatch+1>=p->nAlloc ){
- p->nAlloc = p->nAlloc*2 + 10;
- p->aMatch = sqlite3_realloc(p->aMatch, p->nAlloc*sizeof(p->aMatch[0]) );
- if( p->aMatch==0 ){
- p->nMatch = 0;
- p->nAlloc = 0;
- return;
- }
+ Fts3Expr *pSplit = pPrev;
+ while( pSplit->pParent && opPrecedence(pSplit->pParent)<=opPrecedence(pNew) ){
+ pSplit = pSplit->pParent;
}
- i = p->nMatch++;
- pMatch = &p->aMatch[i];
- pMatch->iCol = iCol;
- pMatch->iTerm = iTerm;
- pMatch->iToken = iToken;
- pMatch->iStart = iStart;
- pMatch->nByte = nByte;
-}
-/*
-** Sizing information for the circular buffer used in snippetOffsetsOfColumn()
-*/
-#define FTS3_ROTOR_SZ (32)
-#define FTS3_ROTOR_MASK (FTS3_ROTOR_SZ-1)
+ if( pSplit->pParent ){
+ assert( pSplit->pParent->pRight==pSplit );
+ pSplit->pParent->pRight = pNew;
+ pNew->pParent = pSplit->pParent;
+ }else{
+ *ppHead = pNew;
+ }
+ pNew->pLeft = pSplit;
+ pSplit->pParent = pNew;
+}
/*
-** Function to iterate through the tokens of a compiled expression.
+** Parse the fts3 query expression found in buffer z, length n. This function
+** returns either when the end of the buffer is reached or an unmatched
+** closing bracket - ')' - is encountered.
**
-** Except, skip all tokens on the right-hand side of a NOT operator.
-** This function is used to find tokens as part of snippet and offset
-** generation and we do nt want snippets and offsets to report matches
-** for tokens on the RHS of a NOT.
+** If successful, SQLITE_OK is returned, *ppExpr is set to point to the
+** parsed form of the expression and *pnConsumed is set to the number of
+** bytes read from buffer z. Otherwise, *ppExpr is set to 0 and SQLITE_NOMEM
+** (out of memory error) or SQLITE_ERROR (parse error) is returned.
*/
-static int fts3NextExprToken(Fts3Expr **ppExpr, int *piToken){
- Fts3Expr *p = *ppExpr;
- int iToken = *piToken;
- if( iToken<0 ){
- /* In this case the expression p is the root of an expression tree.
- ** Move to the first token in the expression tree.
- */
- while( p->pLeft ){
- p = p->pLeft;
- }
- iToken = 0;
- }else{
- assert(p && p->eType==FTSQUERY_PHRASE );
- if( iToken<(p->pPhrase->nToken-1) ){
- iToken++;
- }else{
- iToken = 0;
- while( p->pParent && p->pParent->pLeft!=p ){
- assert( p->pParent->pRight==p );
- p = p->pParent;
- }
- p = p->pParent;
- if( p ){
- assert( p->pRight!=0 );
- p = p->pRight;
- while( p->pLeft ){
- p = p->pLeft;
- }
- }
- }
- }
-
- *ppExpr = p;
- *piToken = iToken;
- return p?1:0;
-}
-
-/*
-** Return TRUE if the expression node pExpr is located beneath the
-** RHS of a NOT operator.
-*/
-static int fts3ExprBeneathNot(Fts3Expr *p){
- Fts3Expr *pParent;
- while( p ){
- pParent = p->pParent;
- if( pParent && pParent->eType==FTSQUERY_NOT && pParent->pRight==p ){
- return 1;
- }
- p = pParent;
- }
- return 0;
-}
-
-/*
-** Add entries to pSnippet->aMatch[] for every match that occurs against
-** document zDoc[0..nDoc-1] which is stored in column iColumn.
-*/
-static void snippetOffsetsOfColumn(
- fulltext_cursor *pCur, /* The fulltest search cursor */
- Snippet *pSnippet, /* The Snippet object to be filled in */
- int iColumn, /* Index of fulltext table column */
- const char *zDoc, /* Text of the fulltext table column */
- int nDoc /* Length of zDoc in bytes */
+static int fts3ExprParse(
+ ParseContext *pParse, /* fts3 query parse context */
+ const char *z, int n, /* Text of MATCH query */
+ Fts3Expr **ppExpr, /* OUT: Parsed query structure */
+ int *pnConsumed /* OUT: Number of bytes consumed */
){
- const sqlite3_tokenizer_module *pTModule; /* The tokenizer module */
- sqlite3_tokenizer *pTokenizer; /* The specific tokenizer */
- sqlite3_tokenizer_cursor *pTCursor; /* Tokenizer cursor */
- fulltext_vtab *pVtab; /* The full text index */
- int nColumn; /* Number of columns in the index */
- int i, j; /* Loop counters */
- int rc; /* Return code */
- unsigned int match, prevMatch; /* Phrase search bitmasks */
- const char *zToken; /* Next token from the tokenizer */
- int nToken; /* Size of zToken */
- int iBegin, iEnd, iPos; /* Offsets of beginning and end */
-
- /* The following variables keep a circular buffer of the last
- ** few tokens */
- unsigned int iRotor = 0; /* Index of current token */
- int iRotorBegin[FTS3_ROTOR_SZ]; /* Beginning offset of token */
- int iRotorLen[FTS3_ROTOR_SZ]; /* Length of token */
-
- pVtab = cursor_vtab(pCur);
- nColumn = pVtab->nColumn;
- pTokenizer = pVtab->pTokenizer;
- pTModule = pTokenizer->pModule;
- rc = pTModule->xOpen(pTokenizer, zDoc, nDoc, &pTCursor);
- if( rc ) return;
- pTCursor->pTokenizer = pTokenizer;
+ Fts3Expr *pRet = 0;
+ Fts3Expr *pPrev = 0;
+ Fts3Expr *pNotBranch = 0; /* Only used in legacy parse mode */
+ int nIn = n;
+ const char *zIn = z;
+ int rc = SQLITE_OK;
+ int isRequirePhrase = 1;
- prevMatch = 0;
- while( !pTModule->xNext(pTCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos) ){
- Fts3Expr *pIter = pCur->pExpr;
- int iIter = -1;
- iRotorBegin[iRotor&FTS3_ROTOR_MASK] = iBegin;
- iRotorLen[iRotor&FTS3_ROTOR_MASK] = iEnd-iBegin;
- match = 0;
- for(i=0; i<(FTS3_ROTOR_SZ-1) && fts3NextExprToken(&pIter, &iIter); i++){
- int nPhrase; /* Number of tokens in current phrase */
- struct PhraseToken *pToken; /* Current token */
- int iCol; /* Column index */
+ while( rc==SQLITE_OK ){
+ Fts3Expr *p = 0;
+ int nByte = 0;
+ rc = getNextNode(pParse, zIn, nIn, &p, &nByte);
+ if( rc==SQLITE_OK ){
+ int isPhrase;
- if( fts3ExprBeneathNot(pIter) ) continue;
- nPhrase = pIter->pPhrase->nToken;
- pToken = &pIter->pPhrase->aToken[iIter];
- iCol = pIter->pPhrase->iColumn;
- if( iCol>=0 && iCol<nColumn && iCol!=iColumn ) continue;
- if( pToken->n>nToken ) continue;
- if( !pToken->isPrefix && pToken->n<nToken ) continue;
- assert( pToken->n<=nToken );
- if( memcmp(pToken->z, zToken, pToken->n) ) continue;
- if( iIter>0 && (prevMatch & (1<<i))==0 ) continue;
- match |= 1<<i;
- if( i==(FTS3_ROTOR_SZ-2) || nPhrase==iIter+1 ){
- for(j=nPhrase-1; j>=0; j--){
- int k = (iRotor-j) & FTS3_ROTOR_MASK;
- snippetAppendMatch(pSnippet, iColumn, i-j, iPos-j,
- iRotorBegin[k], iRotorLen[k]);
+ if( !sqlite3_fts3_enable_parentheses
+ && p->eType==FTSQUERY_PHRASE && p->pPhrase->isNot
+ ){
+ /* Create an implicit NOT operator. */
+ Fts3Expr *pNot = sqlite3_malloc(sizeof(Fts3Expr));
+ if( !pNot ){
+ sqlite3Fts3ExprFree(p);
+ rc = SQLITE_NOMEM;
+ goto exprparse_out;
}
- }
- }
- prevMatch = match<<1;
- iRotor++;
- }
- pTModule->xClose(pTCursor);
-}
-
-/*
-** Remove entries from the pSnippet structure to account for the NEAR
-** operator. When this is called, pSnippet contains the list of token
-** offsets produced by treating all NEAR operators as AND operators.
-** This function removes any entries that should not be present after
-** accounting for the NEAR restriction. For example, if the queried
-** document is:
-**
-** "A B C D E A"
-**
-** and the query is:
-**
-** A NEAR/0 E
-**
-** then when this function is called the Snippet contains token offsets
-** 0, 4 and 5. This function removes the "0" entry (because the first A
-** is not near enough to an E).
-**
-** When this function is called, the value pointed to by parameter piLeft is
-** the integer id of the left-most token in the expression tree headed by
-** pExpr. This function increments *piLeft by the total number of tokens
-** in the expression tree headed by pExpr.
-**
-** Return 1 if any trimming occurs. Return 0 if no trimming is required.
-*/
-static int trimSnippetOffsets(
- Fts3Expr *pExpr, /* The search expression */
- Snippet *pSnippet, /* The set of snippet offsets to be trimmed */
- int *piLeft /* Index of left-most token in pExpr */
-){
- if( pExpr ){
- if( trimSnippetOffsets(pExpr->pLeft, pSnippet, piLeft) ){
- return 1;
- }
+ memset(pNot, 0, sizeof(Fts3Expr));
+ pNot->eType = FTSQUERY_NOT;
+ pNot->pRight = p;
+ if( pNotBranch ){
+ pNot->pLeft = pNotBranch;
+ }
+ pNotBranch = pNot;
+ p = pPrev;
+ }else{
+ int eType = p->eType;
+ assert( eType!=FTSQUERY_PHRASE || !p->pPhrase->isNot );
+ isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft);
- switch( pExpr->eType ){
- case FTSQUERY_PHRASE:
- *piLeft += pExpr->pPhrase->nToken;
- break;
- case FTSQUERY_NEAR: {
- /* The right-hand-side of a NEAR operator is always a phrase. The
- ** left-hand-side is either a phrase or an expression tree that is
- ** itself headed by a NEAR operator. The following initializations
- ** set local variable iLeft to the token number of the left-most
- ** token in the right-hand phrase, and iRight to the right most
- ** token in the same phrase. For example, if we had:
- **
- ** <col> MATCH '"abc def" NEAR/2 "ghi jkl"'
- **
- ** then iLeft will be set to 2 (token number of ghi) and nToken will
- ** be set to 4.
+ /* The isRequirePhrase variable is set to true if a phrase or
+ ** an expression contained in parenthesis is required. If a
+ ** binary operator (AND, OR, NOT or NEAR) is encounted when
+ ** isRequirePhrase is set, this is a syntax error.
*/
- Fts3Expr *pLeft = pExpr->pLeft;
- Fts3Expr *pRight = pExpr->pRight;
- int iLeft = *piLeft;
- int nNear = pExpr->nNear;
- int nToken = pRight->pPhrase->nToken;
- int jj, ii;
- if( pLeft->eType==FTSQUERY_NEAR ){
- pLeft = pLeft->pRight;
+ if( !isPhrase && isRequirePhrase ){
+ sqlite3Fts3ExprFree(p);
+ rc = SQLITE_ERROR;
+ goto exprparse_out;
}
- assert( pRight->eType==FTSQUERY_PHRASE );
- assert( pLeft->eType==FTSQUERY_PHRASE );
- nToken += pLeft->pPhrase->nToken;
-
- for(ii=0; ii<pSnippet->nMatch; ii++){
- struct snippetMatch *p = &pSnippet->aMatch[ii];
- if( p->iTerm==iLeft ){
- int isOk = 0;
- /* Snippet ii is an occurence of query term iLeft in the document.
- ** It occurs at position (p->iToken) of the document. We now
- ** search for an instance of token (iLeft-1) somewhere in the
- ** range (p->iToken - nNear)...(p->iToken + nNear + nToken) within
- ** the set of snippetMatch structures. If one is found, proceed.
- ** If one cannot be found, then remove snippets ii..(ii+N-1)
- ** from the matching snippets, where N is the number of tokens
- ** in phrase pRight->pPhrase.
- */
- for(jj=0; isOk==0 && jj<pSnippet->nMatch; jj++){
- struct snippetMatch *p2 = &pSnippet->aMatch[jj];
- if( p2->iTerm==(iLeft-1) ){
- if( p2->iToken>=(p->iToken-nNear-1)
- && p2->iToken<(p->iToken+nNear+nToken)
- ){
- isOk = 1;
- }
- }
- }
- if( !isOk ){
- int kk;
- for(kk=0; kk<pRight->pPhrase->nToken; kk++){
- pSnippet->aMatch[kk+ii].iTerm = -2;
- }
- return 1;
- }
- }
- if( p->iTerm==(iLeft-1) ){
- int isOk = 0;
- for(jj=0; isOk==0 && jj<pSnippet->nMatch; jj++){
- struct snippetMatch *p2 = &pSnippet->aMatch[jj];
- if( p2->iTerm==iLeft ){
- if( p2->iToken<=(p->iToken+nNear+1)
- && p2->iToken>(p->iToken-nNear-nToken)
- ){
- isOk = 1;
- }
- }
- }
- if( !isOk ){
- int kk;
- for(kk=0; kk<pLeft->pPhrase->nToken; kk++){
- pSnippet->aMatch[ii-kk].iTerm = -2;
- }
- return 1;
- }
+
+ if( isPhrase && !isRequirePhrase ){
+ /* Insert an implicit AND operator. */
+ Fts3Expr *pAnd;
+ assert( pRet && pPrev );
+ pAnd = sqlite3_malloc(sizeof(Fts3Expr));
+ if( !pAnd ){
+ sqlite3Fts3ExprFree(p);
+ rc = SQLITE_NOMEM;
+ goto exprparse_out;
}
+ memset(pAnd, 0, sizeof(Fts3Expr));
+ pAnd->eType = FTSQUERY_AND;
+ insertBinaryOperator(&pRet, pPrev, pAnd);
+ pPrev = pAnd;
}
- break;
- }
- }
-
- if( trimSnippetOffsets(pExpr->pRight, pSnippet, piLeft) ){
- return 1;
- }
- }
- return 0;
-}
-
-/*
-** Compute all offsets for the current row of the query.
-** If the offsets have already been computed, this routine is a no-op.
-*/
-static void snippetAllOffsets(fulltext_cursor *p){
- int nColumn;
- int iColumn, i;
- int iFirst, iLast;
- int iTerm = 0;
- fulltext_vtab *pFts = cursor_vtab(p);
-
- if( p->snippet.nMatch || p->pExpr==0 ){
- return;
- }
- nColumn = pFts->nColumn;
- iColumn = (p->iCursorType - QUERY_FULLTEXT);
- if( iColumn<0 || iColumn>=nColumn ){
- /* Look for matches over all columns of the full-text index */
- iFirst = 0;
- iLast = nColumn-1;
- }else{
- /* Look for matches in the iColumn-th column of the index only */
- iFirst = iColumn;
- iLast = iColumn;
- }
- for(i=iFirst; i<=iLast; i++){
- const char *zDoc;
- int nDoc;
- zDoc = (const char*)sqlite3_column_text(p->pStmt, i+1);
- nDoc = sqlite3_column_bytes(p->pStmt, i+1);
- snippetOffsetsOfColumn(p, &p->snippet, i, zDoc, nDoc);
- }
-
- while( trimSnippetOffsets(p->pExpr, &p->snippet, &iTerm) ){
- iTerm = 0;
- }
-}
-
-/*
-** Convert the information in the aMatch[] array of the snippet
-** into the string zOffset[0..nOffset-1]. This string is used as
-** the return of the SQL offsets() function.
-*/
-static void snippetOffsetText(Snippet *p){
- int i;
- int cnt = 0;
- StringBuffer sb;
- char zBuf[200];
- if( p->zOffset ) return;
- initStringBuffer(&sb);
- for(i=0; i<p->nMatch; i++){
- struct snippetMatch *pMatch = &p->aMatch[i];
- if( pMatch->iTerm>=0 ){
- /* If snippetMatch.iTerm is less than 0, then the match was
- ** discarded as part of processing the NEAR operator (see the
- ** trimSnippetOffsetsForNear() function for details). Ignore
- ** it in this case
- */
- zBuf[0] = ' ';
- sqlite3_snprintf(sizeof(zBuf)-1, &zBuf[cnt>0], "%d %d %d %d",
- pMatch->iCol, pMatch->iTerm, pMatch->iStart, pMatch->nByte);
- append(&sb, zBuf);
- cnt++;
- }
- }
- p->zOffset = stringBufferData(&sb);
- p->nOffset = stringBufferLength(&sb);
-}
-
-/*
-** zDoc[0..nDoc-1] is phrase of text. aMatch[0..nMatch-1] are a set
-** of matching words some of which might be in zDoc. zDoc is column
-** number iCol.
-**
-** iBreak is suggested spot in zDoc where we could begin or end an
-** excerpt. Return a value similar to iBreak but possibly adjusted
-** to be a little left or right so that the break point is better.
-*/
-static int wordBoundary(
- int iBreak, /* The suggested break point */
- const char *zDoc, /* Document text */
- int nDoc, /* Number of bytes in zDoc[] */
- struct snippetMatch *aMatch, /* Matching words */
- int nMatch, /* Number of entries in aMatch[] */
- int iCol /* The column number for zDoc[] */
-){
- int i;
- if( iBreak<=10 ){
- return 0;
- }
- if( iBreak>=nDoc-10 ){
- return nDoc;
- }
- for(i=0; i<nMatch && aMatch[i].iCol<iCol; i++){}
- while( i<nMatch && aMatch[i].iStart+aMatch[i].nByte<iBreak ){ i++; }
- if( i<nMatch ){
- if( aMatch[i].iStart<iBreak+10 ){
- return aMatch[i].iStart;
- }
- if( i>0 && aMatch[i-1].iStart+aMatch[i-1].nByte>=iBreak ){
- return aMatch[i-1].iStart;
- }
- }
- for(i=1; i<=10; i++){
- if( safe_isspace(zDoc[iBreak-i]) ){
- return iBreak - i + 1;
- }
- if( safe_isspace(zDoc[iBreak+i]) ){
- return iBreak + i + 1;
- }
- }
- return iBreak;
-}
-
-
-/*
-** Allowed values for Snippet.aMatch[].snStatus
-*/
-#define SNIPPET_IGNORE 0 /* It is ok to omit this match from the snippet */
-#define SNIPPET_DESIRED 1 /* We want to include this match in the snippet */
-
-/*
-** Generate the text of a snippet.
-*/
-static void snippetText(
- fulltext_cursor *pCursor, /* The cursor we need the snippet for */
- const char *zStartMark, /* Markup to appear before each match */
- const char *zEndMark, /* Markup to appear after each match */
- const char *zEllipsis /* Ellipsis mark */
-){
- int i, j;
- struct snippetMatch *aMatch;
- int nMatch;
- int nDesired;
- StringBuffer sb;
- int tailCol;
- int tailOffset;
- int iCol;
- int nDoc;
- const char *zDoc;
- int iStart, iEnd;
- int tailEllipsis = 0;
- int iMatch;
+ /* This test catches attempts to make either operand of a NEAR
+ ** operator something other than a phrase. For example, either of
+ ** the following:
+ **
+ ** (bracketed expression) NEAR phrase
+ ** phrase NEAR (bracketed expression)
+ **
+ ** Return an error in either case.
+ */
+ if( pPrev && (
+ (eType==FTSQUERY_NEAR && !isPhrase && pPrev->eType!=FTSQUERY_PHRASE)
+ || (eType!=FTSQUERY_PHRASE && isPhrase && pPrev->eType==FTSQUERY_NEAR)
+ )){
+ sqlite3Fts3ExprFree(p);
+ rc = SQLITE_ERROR;
+ goto exprparse_out;
+ }
-
- sqlite3_free(pCursor->snippet.zSnippet);
- pCursor->snippet.zSnippet = 0;
- aMatch = pCursor->snippet.aMatch;
- nMatch = pCursor->snippet.nMatch;
- initStringBuffer(&sb);
-
- for(i=0; i<nMatch; i++){
- aMatch[i].snStatus = SNIPPET_IGNORE;
- }
- nDesired = 0;
- for(i=0; i<FTS3_ROTOR_SZ; i++){
- for(j=0; j<nMatch; j++){
- if( aMatch[j].iTerm==i ){
- aMatch[j].snStatus = SNIPPET_DESIRED;
- nDesired++;
- break;
- }
- }
- }
-
- iMatch = 0;
- tailCol = -1;
- tailOffset = 0;
- for(i=0; i<nMatch && nDesired>0; i++){
- if( aMatch[i].snStatus!=SNIPPET_DESIRED ) continue;
- nDesired--;
- iCol = aMatch[i].iCol;
- zDoc = (const char*)sqlite3_column_text(pCursor->pStmt, iCol+1);
- nDoc = sqlite3_column_bytes(pCursor->pStmt, iCol+1);
- iStart = aMatch[i].iStart - 40;
- iStart = wordBoundary(iStart, zDoc, nDoc, aMatch, nMatch, iCol);
- if( iStart<=10 ){
- iStart = 0;
- }
- if( iCol==tailCol && iStart<=tailOffset+20 ){
- iStart = tailOffset;
- }
- if( (iCol!=tailCol && tailCol>=0) || iStart!=tailOffset ){
- trimWhiteSpace(&sb);
- appendWhiteSpace(&sb);
- append(&sb, zEllipsis);
- appendWhiteSpace(&sb);
- }
- iEnd = aMatch[i].iStart + aMatch[i].nByte + 40;
- iEnd = wordBoundary(iEnd, zDoc, nDoc, aMatch, nMatch, iCol);
- if( iEnd>=nDoc-10 ){
- iEnd = nDoc;
- tailEllipsis = 0;
- }else{
- tailEllipsis = 1;
- }
- while( iMatch<nMatch && aMatch[iMatch].iCol<iCol ){ iMatch++; }
- while( iStart<iEnd ){
- while( iMatch<nMatch && aMatch[iMatch].iStart<iStart
- && aMatch[iMatch].iCol<=iCol ){
- iMatch++;
- }
- if( iMatch<nMatch && aMatch[iMatch].iStart<iEnd
- && aMatch[iMatch].iCol==iCol ){
- nappend(&sb, &zDoc[iStart], aMatch[iMatch].iStart - iStart);
- iStart = aMatch[iMatch].iStart;
- append(&sb, zStartMark);
- nappend(&sb, &zDoc[iStart], aMatch[iMatch].nByte);
- append(&sb, zEndMark);
- iStart += aMatch[iMatch].nByte;
- for(j=iMatch+1; j<nMatch; j++){
- if( aMatch[j].iTerm==aMatch[iMatch].iTerm
- && aMatch[j].snStatus==SNIPPET_DESIRED ){
- nDesired--;
- aMatch[j].snStatus = SNIPPET_IGNORE;
+ if( isPhrase ){
+ if( pRet ){
+ assert( pPrev && pPrev->pLeft && pPrev->pRight==0 );
+ pPrev->pRight = p;
+ p->pParent = pPrev;
+ }else{
+ pRet = p;
}
+ }else{
+ insertBinaryOperator(&pRet, pPrev, p);
}
- }else{
- nappend(&sb, &zDoc[iStart], iEnd - iStart);
- iStart = iEnd;
+ isRequirePhrase = !isPhrase;
}
+ assert( nByte>0 );
}
- tailCol = iCol;
- tailOffset = iEnd;
- }
- trimWhiteSpace(&sb);
- if( tailEllipsis ){
- appendWhiteSpace(&sb);
- append(&sb, zEllipsis);
- }
- pCursor->snippet.zSnippet = stringBufferData(&sb);
- pCursor->snippet.nSnippet = stringBufferLength(&sb);
-}
-
-
-/*
-** Close the cursor. For additional information see the documentation
-** on the xClose method of the virtual table interface.
-*/
-static int fulltextClose(sqlite3_vtab_cursor *pCursor){
- fulltext_cursor *c = (fulltext_cursor *) pCursor;
- FTSTRACE(("FTS3 Close %p\n", c));
- sqlite3_finalize(c->pStmt);
- sqlite3Fts3ExprFree(c->pExpr);
- snippetClear(&c->snippet);
- if( c->result.nData!=0 ){
- dlrDestroy(&c->reader);
- }
- dataBufferDestroy(&c->result);
- sqlite3_free(c);
- return SQLITE_OK;
-}
-
-static int fulltextNext(sqlite3_vtab_cursor *pCursor){
- fulltext_cursor *c = (fulltext_cursor *) pCursor;
- int rc;
-
- FTSTRACE(("FTS3 Next %p\n", pCursor));
- snippetClear(&c->snippet);
- if( c->iCursorType < QUERY_FULLTEXT ){
- /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */
- rc = sqlite3_step(c->pStmt);
- switch( rc ){
- case SQLITE_ROW:
- c->eof = 0;
- return SQLITE_OK;
- case SQLITE_DONE:
- c->eof = 1;
- return SQLITE_OK;
- default:
- c->eof = 1;
- return rc;
- }
- } else { /* full-text query */
- rc = sqlite3_reset(c->pStmt);
- if( rc!=SQLITE_OK ) return rc;
-
- if( c->result.nData==0 || dlrAtEnd(&c->reader) ){
- c->eof = 1;
- return SQLITE_OK;
- }
- rc = sqlite3_bind_int64(c->pStmt, 1, dlrDocid(&c->reader));
- dlrStep(&c->reader);
- if( rc!=SQLITE_OK ) return rc;
- /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */
- rc = sqlite3_step(c->pStmt);
- if( rc==SQLITE_ROW ){ /* the case we expect */
- c->eof = 0;
- return SQLITE_OK;
- }
- /* an error occurred; abort */
- return rc==SQLITE_DONE ? SQLITE_ERROR : rc;
+ assert( rc!=SQLITE_OK || (nByte>0 && nByte<=nIn) );
+ nIn -= nByte;
+ zIn += nByte;
+ pPrev = p;
}
-}
-
-/* TODO(shess) If we pushed LeafReader to the top of the file, or to
-** another file, term_select() could be pushed above
-** docListOfTerm().
-*/
-static int termSelect(fulltext_vtab *v, int iColumn,
- const char *pTerm, int nTerm, int isPrefix,
- DocListType iType, DataBuffer *out);
-
-/*
-** Return a DocList corresponding to the phrase *pPhrase.
-**
-** The resulting DL_DOCIDS doclist is stored in pResult, which is
-** overwritten.
-*/
-static int docListOfPhrase(
- fulltext_vtab *pTab, /* The full text index */
- Fts3Phrase *pPhrase, /* Phrase to return a doclist corresponding to */
- DocListType eListType, /* Either DL_DOCIDS or DL_POSITIONS */
- DataBuffer *pResult /* Write the result here */
-){
- int ii;
- int rc = SQLITE_OK;
- int iCol = pPhrase->iColumn;
- DocListType eType = eListType;
- assert( eType==DL_POSITIONS || eType==DL_DOCIDS );
- if( pPhrase->nToken>1 ){
- eType = DL_POSITIONS;
+ if( rc==SQLITE_DONE && pRet && isRequirePhrase ){
+ rc = SQLITE_ERROR;
}
- /* This code should never be called with buffered updates. */
- assert( pTab->nPendingData<0 );
-
- for(ii=0; rc==SQLITE_OK && ii<pPhrase->nToken; ii++){
- DataBuffer tmp;
- struct PhraseToken *p = &pPhrase->aToken[ii];
- rc = termSelect(pTab, iCol, p->z, p->n, p->isPrefix, eType, &tmp);
- if( rc==SQLITE_OK ){
- if( ii==0 ){
- *pResult = tmp;
+ if( rc==SQLITE_DONE ){
+ rc = SQLITE_OK;
+ if( !sqlite3_fts3_enable_parentheses && pNotBranch ){
+ if( !pRet ){
+ rc = SQLITE_ERROR;
}else{
- DataBuffer res = *pResult;
- dataBufferInit(pResult, 0);
- if( ii==(pPhrase->nToken-1) ){
- eType = eListType;
- }
- docListPhraseMerge(
- res.pData, res.nData, tmp.pData, tmp.nData, 0, 0, eType, pResult
- );
- dataBufferDestroy(&res);
- dataBufferDestroy(&tmp);
- }
- }
- }
-
- return rc;
-}
-
-/*
-** Evaluate the full-text expression pExpr against fts3 table pTab. Write
-** the results into pRes.
-*/
-static int evalFts3Expr(
- fulltext_vtab *pTab, /* Fts3 Virtual table object */
- Fts3Expr *pExpr, /* Parsed fts3 expression */
- DataBuffer *pRes /* OUT: Write results of the expression here */
-){
- int rc = SQLITE_OK;
-
- /* Initialize the output buffer. If this is an empty query (pExpr==0),
- ** this is all that needs to be done. Empty queries produce empty
- ** result sets.
- */
- dataBufferInit(pRes, 0);
-
- if( pExpr ){
- if( pExpr->eType==FTSQUERY_PHRASE ){
- DocListType eType = DL_DOCIDS;
- if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){
- eType = DL_POSITIONS;
- }
- rc = docListOfPhrase(pTab, pExpr->pPhrase, eType, pRes);
- }else{
- DataBuffer lhs;
- DataBuffer rhs;
-
- dataBufferInit(&rhs, 0);
- if( SQLITE_OK==(rc = evalFts3Expr(pTab, pExpr->pLeft, &lhs))
- && SQLITE_OK==(rc = evalFts3Expr(pTab, pExpr->pRight, &rhs))
- ){
- switch( pExpr->eType ){
- case FTSQUERY_NEAR: {
- int nToken;
- Fts3Expr *pLeft;
- DocListType eType = DL_DOCIDS;
- if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){
- eType = DL_POSITIONS;
- }
- pLeft = pExpr->pLeft;
- while( pLeft->eType==FTSQUERY_NEAR ){
- pLeft=pLeft->pRight;
- }
- assert( pExpr->pRight->eType==FTSQUERY_PHRASE );
- assert( pLeft->eType==FTSQUERY_PHRASE );
- nToken = pLeft->pPhrase->nToken + pExpr->pRight->pPhrase->nToken;
- docListPhraseMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData,
- pExpr->nNear+1, nToken, eType, pRes
- );
- break;
- }
- case FTSQUERY_NOT: {
- docListExceptMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData,pRes);
- break;
- }
- case FTSQUERY_AND: {
- docListAndMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData, pRes);
- break;
- }
- case FTSQUERY_OR: {
- docListOrMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData, pRes);
- break;
- }
+ Fts3Expr *pIter = pNotBranch;
+ while( pIter->pLeft ){
+ pIter = pIter->pLeft;
}
+ pIter->pLeft = pRet;
+ pRet = pNotBranch;
}
- dataBufferDestroy(&lhs);
- dataBufferDestroy(&rhs);
}
}
+ *pnConsumed = n - nIn;
- return rc;
-}
-
-/* TODO(shess) Refactor the code to remove this forward decl. */
-static int flushPendingTerms(fulltext_vtab *v);
-
-/* Perform a full-text query using the search expression in
-** zInput[0..nInput-1]. Return a list of matching documents
-** in pResult.
-**
-** Queries must match column iColumn. Or if iColumn>=nColumn
-** they are allowed to match against any column.
-*/
-static int fulltextQuery(
- fulltext_vtab *v, /* The full text index */
- int iColumn, /* Match against this column by default */
- const char *zInput, /* The query string */
- int nInput, /* Number of bytes in zInput[] */
- DataBuffer *pResult, /* Write the result doclist here */
- Fts3Expr **ppExpr /* Put parsed query string here */
-){
- int rc;
-
- /* TODO(shess) Instead of flushing pendingTerms, we could query for
- ** the relevant term and merge the doclist into what we receive from
- ** the database. Wait and see if this is a common issue, first.
- **
- ** A good reason not to flush is to not generate update-related
- ** error codes from here.
- */
-
- /* Flush any buffered updates before executing the query. */
- rc = flushPendingTerms(v);
- if( rc!=SQLITE_OK ){
- return rc;
- }
-
- /* Parse the query passed to the MATCH operator. */
- rc = sqlite3Fts3ExprParse(v->pTokenizer,
- v->azColumn, v->nColumn, iColumn, zInput, nInput, ppExpr
- );
+exprparse_out:
if( rc!=SQLITE_OK ){
- assert( 0==(*ppExpr) );
- return rc;
- }
-
- return evalFts3Expr(v, *ppExpr, pResult);
-}
-
-/*
-** This is the xFilter interface for the virtual table. See
-** the virtual table xFilter method documentation for additional
-** information.
-**
-** If idxNum==QUERY_GENERIC then do a full table scan against
-** the %_content table.
-**
-** If idxNum==QUERY_DOCID then do a docid lookup for a single entry
-** in the %_content table.
-**
-** If idxNum>=QUERY_FULLTEXT then use the full text index. The
-** column on the left-hand side of the MATCH operator is column
-** number idxNum-QUERY_FULLTEXT, 0 indexed. argv[0] is the right-hand
-** side of the MATCH operator.
-*/
-/* TODO(shess) Upgrade the cursor initialization and destruction to
-** account for fulltextFilter() being called multiple times on the
-** same cursor. The current solution is very fragile. Apply fix to
-** fts3 as appropriate.
-*/
-static int fulltextFilter(
- sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */
- int idxNum, const char *idxStr, /* Which indexing scheme to use */
- int argc, sqlite3_value **argv /* Arguments for the indexing scheme */
-){
- fulltext_cursor *c = (fulltext_cursor *) pCursor;
- fulltext_vtab *v = cursor_vtab(c);
- int rc;
-
- FTSTRACE(("FTS3 Filter %p\n",pCursor));
-
- /* If the cursor has a statement that was not prepared according to
- ** idxNum, clear it. I believe all calls to fulltextFilter with a
- ** given cursor will have the same idxNum , but in this case it's
- ** easy to be safe.
- */
- if( c->pStmt && c->iCursorType!=idxNum ){
- sqlite3_finalize(c->pStmt);
- c->pStmt = NULL;
- }
-
- /* Get a fresh statement appropriate to idxNum. */
- /* TODO(shess): Add a prepared-statement cache in the vt structure.
- ** The cache must handle multiple open cursors. Easier to cache the
- ** statement variants at the vt to reduce malloc/realloc/free here.
- ** Or we could have a StringBuffer variant which allowed stack
- ** construction for small values.
- */
- if( !c->pStmt ){
- StringBuffer sb;
- initStringBuffer(&sb);
- append(&sb, "SELECT docid, ");
- appendList(&sb, v->nColumn, v->azContentColumn);
- append(&sb, " FROM %_content");
- if( idxNum!=QUERY_GENERIC ) append(&sb, " WHERE docid = ?");
- rc = sql_prepare(v->db, v->zDb, v->zName, &c->pStmt,
- stringBufferData(&sb));
- stringBufferDestroy(&sb);
- if( rc!=SQLITE_OK ) return rc;
- c->iCursorType = idxNum;
- }else{
- sqlite3_reset(c->pStmt);
- assert( c->iCursorType==idxNum );
- }
-
- switch( idxNum ){
- case QUERY_GENERIC:
- break;
-
- case QUERY_DOCID:
- rc = sqlite3_bind_int64(c->pStmt, 1, sqlite3_value_int64(argv[0]));
- if( rc!=SQLITE_OK ) return rc;
- break;
-
- default: /* full-text search */
- {
- int iCol = idxNum-QUERY_FULLTEXT;
- const char *zQuery = (const char *)sqlite3_value_text(argv[0]);
- assert( idxNum<=QUERY_FULLTEXT+v->nColumn);
- assert( argc==1 );
- if( c->result.nData!=0 ){
- /* This case happens if the same cursor is used repeatedly. */
- dlrDestroy(&c->reader);
- dataBufferReset(&c->result);
- }else{
- dataBufferInit(&c->result, 0);
- }
- rc = fulltextQuery(v, iCol, zQuery, -1, &c->result, &c->pExpr);
- if( rc!=SQLITE_OK ) return rc;
- if( c->result.nData!=0 ){
- dlrInit(&c->reader, DL_DOCIDS, c->result.pData, c->result.nData);
- }
- break;
- }
- }
-
- return fulltextNext(pCursor);
-}
-
-/* This is the xEof method of the virtual table. The SQLite core
-** calls this routine to find out if it has reached the end of
-** a query's results set.
-*/
-static int fulltextEof(sqlite3_vtab_cursor *pCursor){
- fulltext_cursor *c = (fulltext_cursor *) pCursor;
- return c->eof;
-}
-
-/* This is the xColumn method of the virtual table. The SQLite
-** core calls this method during a query when it needs the value
-** of a column from the virtual table. This method needs to use
-** one of the sqlite3_result_*() routines to store the requested
-** value back in the pContext.
-*/
-static int fulltextColumn(sqlite3_vtab_cursor *pCursor,
- sqlite3_context *pContext, int idxCol){
- fulltext_cursor *c = (fulltext_cursor *) pCursor;
- fulltext_vtab *v = cursor_vtab(c);
-
- if( idxCol<v->nColumn ){
- sqlite3_value *pVal = sqlite3_column_value(c->pStmt, idxCol+1);
- sqlite3_result_value(pContext, pVal);
- }else if( idxCol==v->nColumn ){
- /* The extra column whose name is the same as the table.
- ** Return a blob which is a pointer to the cursor
- */
- sqlite3_result_blob(pContext, &c, sizeof(c), SQLITE_TRANSIENT);
- }else if( idxCol==v->nColumn+1 ){
- /* The docid column, which is an alias for rowid. */
- sqlite3_value *pVal = sqlite3_column_value(c->pStmt, 0);
- sqlite3_result_value(pContext, pVal);
- }
- return SQLITE_OK;
-}
-
-/* This is the xRowid method. The SQLite core calls this routine to
-** retrieve the rowid for the current row of the result set. fts3
-** exposes %_content.docid as the rowid for the virtual table. The
-** rowid should be written to *pRowid.
-*/
-static int fulltextRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
- fulltext_cursor *c = (fulltext_cursor *) pCursor;
-
- *pRowid = sqlite3_column_int64(c->pStmt, 0);
- return SQLITE_OK;
-}
-
-/* Add all terms in [zText] to pendingTerms table. If [iColumn] > 0,
-** we also store positions and offsets in the hash table using that
-** column number.
-*/
-static int buildTerms(fulltext_vtab *v, sqlite_int64 iDocid,
- const char *zText, int iColumn){
- sqlite3_tokenizer *pTokenizer = v->pTokenizer;
- sqlite3_tokenizer_cursor *pCursor;
- const char *pToken;
- int nTokenBytes;
- int iStartOffset, iEndOffset, iPosition;
- int rc;
-
- rc = pTokenizer->pModule->xOpen(pTokenizer, zText, -1, &pCursor);
- if( rc!=SQLITE_OK ) return rc;
-
- pCursor->pTokenizer = pTokenizer;
- while( SQLITE_OK==(rc=pTokenizer->pModule->xNext(pCursor,
- &pToken, &nTokenBytes,
- &iStartOffset, &iEndOffset,
- &iPosition)) ){
- DLCollector *p;
- int nData; /* Size of doclist before our update. */
-
- /* Positions can't be negative; we use -1 as a terminator
- * internally. Token can't be NULL or empty. */
- if( iPosition<0 || pToken == NULL || nTokenBytes == 0 ){
- rc = SQLITE_ERROR;
- break;
- }
-
- p = fts3HashFind(&v->pendingTerms, pToken, nTokenBytes);
- if( p==NULL ){
- nData = 0;
- p = dlcNew(iDocid, DL_DEFAULT);
- fts3HashInsert(&v->pendingTerms, pToken, nTokenBytes, p);
-
- /* Overhead for our hash table entry, the key, and the value. */
- v->nPendingData += sizeof(struct fts3HashElem)+sizeof(*p)+nTokenBytes;
- }else{
- nData = p->b.nData;
- if( p->dlw.iPrevDocid!=iDocid ) dlcNext(p, iDocid);
- }
- if( iColumn>=0 ){
- dlcAddPos(p, iColumn, iPosition, iStartOffset, iEndOffset);
- }
-
- /* Accumulate data added by dlcNew or dlcNext, and dlcAddPos. */
- v->nPendingData += p->b.nData-nData;
+ sqlite3Fts3ExprFree(pRet);
+ sqlite3Fts3ExprFree(pNotBranch);
+ pRet = 0;
}
-
- /* TODO(shess) Check return? Should this be able to cause errors at
- ** this point? Actually, same question about sqlite3_finalize(),
- ** though one could argue that failure there means that the data is
- ** not durable. *ponder*
- */
- pTokenizer->pModule->xClose(pCursor);
- if( SQLITE_DONE == rc ) return SQLITE_OK;
+ *ppExpr = pRet;
return rc;
}
-/* Add doclists for all terms in [pValues] to pendingTerms table. */
-static int insertTerms(fulltext_vtab *v, sqlite_int64 iDocid,
- sqlite3_value **pValues){
- int i;
- for(i = 0; i < v->nColumn ; ++i){
- char *zText = (char*)sqlite3_value_text(pValues[i]);
- int rc = buildTerms(v, iDocid, zText, i);
- if( rc!=SQLITE_OK ) return rc;
- }
- return SQLITE_OK;
-}
-
-/* Add empty doclists for all terms in the given row's content to
-** pendingTerms.
-*/
-static int deleteTerms(fulltext_vtab *v, sqlite_int64 iDocid){
- const char **pValues;
- int i, rc;
-
- /* TODO(shess) Should we allow such tables at all? */
- if( DL_DEFAULT==DL_DOCIDS ) return SQLITE_ERROR;
-
- rc = content_select(v, iDocid, &pValues);
- if( rc!=SQLITE_OK ) return rc;
-
- for(i = 0 ; i < v->nColumn; ++i) {
- rc = buildTerms(v, iDocid, pValues[i], -1);
- if( rc!=SQLITE_OK ) break;
- }
-
- freeStringArray(v->nColumn, pValues);
- return SQLITE_OK;
-}
-
-/* TODO(shess) Refactor the code to remove this forward decl. */
-static int initPendingTerms(fulltext_vtab *v, sqlite_int64 iDocid);
-
-/* Insert a row into the %_content table; set *piDocid to be the ID of the
-** new row. Add doclists for terms to pendingTerms.
-*/
-static int index_insert(fulltext_vtab *v, sqlite3_value *pRequestDocid,
- sqlite3_value **pValues, sqlite_int64 *piDocid){
- int rc;
-
- rc = content_insert(v, pRequestDocid, pValues); /* execute an SQL INSERT */
- if( rc!=SQLITE_OK ) return rc;
-
- /* docid column is an alias for rowid. */
- *piDocid = sqlite3_last_insert_rowid(v->db);
- rc = initPendingTerms(v, *piDocid);
- if( rc!=SQLITE_OK ) return rc;
-
- return insertTerms(v, *piDocid, pValues);
-}
-
-/* Delete a row from the %_content table; add empty doclists for terms
-** to pendingTerms.
-*/
-static int index_delete(fulltext_vtab *v, sqlite_int64 iRow){
- int rc = initPendingTerms(v, iRow);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = deleteTerms(v, iRow);
- if( rc!=SQLITE_OK ) return rc;
-
- return content_delete(v, iRow); /* execute an SQL DELETE */
-}
-
-/* Update a row in the %_content table; add delete doclists to
-** pendingTerms for old terms not in the new data, add insert doclists
-** to pendingTerms for terms in the new data.
-*/
-static int index_update(fulltext_vtab *v, sqlite_int64 iRow,
- sqlite3_value **pValues){
- int rc = initPendingTerms(v, iRow);
- if( rc!=SQLITE_OK ) return rc;
-
- /* Generate an empty doclist for each term that previously appeared in this
- * row. */
- rc = deleteTerms(v, iRow);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = content_update(v, pValues, iRow); /* execute an SQL UPDATE */
- if( rc!=SQLITE_OK ) return rc;
-
- /* Now add positions for terms which appear in the updated row. */
- return insertTerms(v, iRow, pValues);
-}
-
-/*******************************************************************/
-/* InteriorWriter is used to collect terms and block references into
-** interior nodes in %_segments. See commentary at top of file for
-** format.
-*/
-
-/* How large interior nodes can grow. */
-#define INTERIOR_MAX 2048
-
-/* Minimum number of terms per interior node (except the root). This
-** prevents large terms from making the tree too skinny - must be >0
-** so that the tree always makes progress. Note that the min tree
-** fanout will be INTERIOR_MIN_TERMS+1.
-*/
-#define INTERIOR_MIN_TERMS 7
-#if INTERIOR_MIN_TERMS<1
-# error INTERIOR_MIN_TERMS must be greater than 0.
-#endif
-
-/* ROOT_MAX controls how much data is stored inline in the segment
-** directory.
-*/
-/* TODO(shess) Push ROOT_MAX down to whoever is writing things. It's
-** only here so that interiorWriterRootInfo() and leafWriterRootInfo()
-** can both see it, but if the caller passed it in, we wouldn't even
-** need a define.
-*/
-#define ROOT_MAX 1024
-#if ROOT_MAX<VARINT_MAX*2
-# error ROOT_MAX must have enough space for a header.
-#endif
-
-/* InteriorBlock stores a linked-list of interior blocks while a lower
-** layer is being constructed.
-*/
-typedef struct InteriorBlock {
- DataBuffer term; /* Leftmost term in block's subtree. */
- DataBuffer data; /* Accumulated data for the block. */
- struct InteriorBlock *next;
-} InteriorBlock;
-
-static InteriorBlock *interiorBlockNew(int iHeight, sqlite_int64 iChildBlock,
- const char *pTerm, int nTerm){
- InteriorBlock *block = sqlite3_malloc(sizeof(InteriorBlock));
- char c[VARINT_MAX+VARINT_MAX];
- int n;
-
- if( block ){
- memset(block, 0, sizeof(*block));
- dataBufferInit(&block->term, 0);
- dataBufferReplace(&block->term, pTerm, nTerm);
-
- n = fts3PutVarint(c, iHeight);
- n += fts3PutVarint(c+n, iChildBlock);
- dataBufferInit(&block->data, INTERIOR_MAX);
- dataBufferReplace(&block->data, c, n);
- }
- return block;
-}
-
-#ifndef NDEBUG
-/* Verify that the data is readable as an interior node. */
-static void interiorBlockValidate(InteriorBlock *pBlock){
- const char *pData = pBlock->data.pData;
- int nData = pBlock->data.nData;
- int n, iDummy;
- sqlite_int64 iBlockid;
-
- assert( nData>0 );
- assert( pData!=0 );
- assert( pData+nData>pData );
-
- /* Must lead with height of node as a varint(n), n>0 */
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>0 );
- assert( n<nData );
- pData += n;
- nData -= n;
-
- /* Must contain iBlockid. */
- n = fts3GetVarint(pData, &iBlockid);
- assert( n>0 );
- assert( n<=nData );
- pData += n;
- nData -= n;
-
- /* Zero or more terms of positive length */
- if( nData!=0 ){
- /* First term is not delta-encoded. */
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>0 );
- assert( n+iDummy>0);
- assert( n+iDummy<=nData );
- pData += n+iDummy;
- nData -= n+iDummy;
-
- /* Following terms delta-encoded. */
- while( nData!=0 ){
- /* Length of shared prefix. */
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>=0 );
- assert( n<nData );
- pData += n;
- nData -= n;
-
- /* Length and data of distinct suffix. */
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>0 );
- assert( n+iDummy>0);
- assert( n+iDummy<=nData );
- pData += n+iDummy;
- nData -= n+iDummy;
- }
- }
-}
-#define ASSERT_VALID_INTERIOR_BLOCK(x) interiorBlockValidate(x)
-#else
-#define ASSERT_VALID_INTERIOR_BLOCK(x) assert( 1 )
-#endif
-
-typedef struct InteriorWriter {
- int iHeight; /* from 0 at leaves. */
- InteriorBlock *first, *last;
- struct InteriorWriter *parentWriter;
-
- DataBuffer term; /* Last term written to block "last". */
- sqlite_int64 iOpeningChildBlock; /* First child block in block "last". */
-#ifndef NDEBUG
- sqlite_int64 iLastChildBlock; /* for consistency checks. */
-#endif
-} InteriorWriter;
-
-/* Initialize an interior node where pTerm[nTerm] marks the leftmost
-** term in the tree. iChildBlock is the leftmost child block at the
-** next level down the tree.
-*/
-static void interiorWriterInit(int iHeight, const char *pTerm, int nTerm,
- sqlite_int64 iChildBlock,
- InteriorWriter *pWriter){
- InteriorBlock *block;
- assert( iHeight>0 );
- CLEAR(pWriter);
-
- pWriter->iHeight = iHeight;
- pWriter->iOpeningChildBlock = iChildBlock;
-#ifndef NDEBUG
- pWriter->iLastChildBlock = iChildBlock;
-#endif
- block = interiorBlockNew(iHeight, iChildBlock, pTerm, nTerm);
- pWriter->last = pWriter->first = block;
- ASSERT_VALID_INTERIOR_BLOCK(pWriter->last);
- dataBufferInit(&pWriter->term, 0);
-}
-
-/* Append the child node rooted at iChildBlock to the interior node,
-** with pTerm[nTerm] as the leftmost term in iChildBlock's subtree.
-*/
-static void interiorWriterAppend(InteriorWriter *pWriter,
- const char *pTerm, int nTerm,
- sqlite_int64 iChildBlock){
- char c[VARINT_MAX+VARINT_MAX];
- int n, nPrefix = 0;
-
- ASSERT_VALID_INTERIOR_BLOCK(pWriter->last);
-
- /* The first term written into an interior node is actually
- ** associated with the second child added (the first child was added
- ** in interiorWriterInit, or in the if clause at the bottom of this
- ** function). That term gets encoded straight up, with nPrefix left
- ** at 0.
- */
- if( pWriter->term.nData==0 ){
- n = fts3PutVarint(c, nTerm);
- }else{
- while( nPrefix<pWriter->term.nData &&
- pTerm[nPrefix]==pWriter->term.pData[nPrefix] ){
- nPrefix++;
- }
-
- n = fts3PutVarint(c, nPrefix);
- n += fts3PutVarint(c+n, nTerm-nPrefix);
- }
-
-#ifndef NDEBUG
- pWriter->iLastChildBlock++;
-#endif
- assert( pWriter->iLastChildBlock==iChildBlock );
-
- /* Overflow to a new block if the new term makes the current block
- ** too big, and the current block already has enough terms.
- */
- if( pWriter->last->data.nData+n+nTerm-nPrefix>INTERIOR_MAX &&
- iChildBlock-pWriter->iOpeningChildBlock>INTERIOR_MIN_TERMS ){
- pWriter->last->next = interiorBlockNew(pWriter->iHeight, iChildBlock,
- pTerm, nTerm);
- pWriter->last = pWriter->last->next;
- pWriter->iOpeningChildBlock = iChildBlock;
- dataBufferReset(&pWriter->term);
- }else{
- dataBufferAppend2(&pWriter->last->data, c, n,
- pTerm+nPrefix, nTerm-nPrefix);
- dataBufferReplace(&pWriter->term, pTerm, nTerm);
- }
- ASSERT_VALID_INTERIOR_BLOCK(pWriter->last);
-}
-
-/* Free the space used by pWriter, including the linked-list of
-** InteriorBlocks, and parentWriter, if present.
-*/
-static int interiorWriterDestroy(InteriorWriter *pWriter){
- InteriorBlock *block = pWriter->first;
-
- while( block!=NULL ){
- InteriorBlock *b = block;
- block = block->next;
- dataBufferDestroy(&b->term);
- dataBufferDestroy(&b->data);
- sqlite3_free(b);
- }
- if( pWriter->parentWriter!=NULL ){
- interiorWriterDestroy(pWriter->parentWriter);
- sqlite3_free(pWriter->parentWriter);
- }
- dataBufferDestroy(&pWriter->term);
- SCRAMBLE(pWriter);
- return SQLITE_OK;
-}
-
-/* If pWriter can fit entirely in ROOT_MAX, return it as the root info
-** directly, leaving *piEndBlockid unchanged. Otherwise, flush
-** pWriter to %_segments, building a new layer of interior nodes, and
-** recursively ask for their root into.
-*/
-static int interiorWriterRootInfo(fulltext_vtab *v, InteriorWriter *pWriter,
- char **ppRootInfo, int *pnRootInfo,
- sqlite_int64 *piEndBlockid){
- InteriorBlock *block = pWriter->first;
- sqlite_int64 iBlockid = 0;
- int rc;
-
- /* If we can fit the segment inline */
- if( block==pWriter->last && block->data.nData<ROOT_MAX ){
- *ppRootInfo = block->data.pData;
- *pnRootInfo = block->data.nData;
- return SQLITE_OK;
- }
-
- /* Flush the first block to %_segments, and create a new level of
- ** interior node.
- */
- ASSERT_VALID_INTERIOR_BLOCK(block);
- rc = block_insert(v, block->data.pData, block->data.nData, &iBlockid);
- if( rc!=SQLITE_OK ) return rc;
- *piEndBlockid = iBlockid;
-
- pWriter->parentWriter = sqlite3_malloc(sizeof(*pWriter->parentWriter));
- interiorWriterInit(pWriter->iHeight+1,
- block->term.pData, block->term.nData,
- iBlockid, pWriter->parentWriter);
-
- /* Flush additional blocks and append to the higher interior
- ** node.
- */
- for(block=block->next; block!=NULL; block=block->next){
- ASSERT_VALID_INTERIOR_BLOCK(block);
- rc = block_insert(v, block->data.pData, block->data.nData, &iBlockid);
- if( rc!=SQLITE_OK ) return rc;
- *piEndBlockid = iBlockid;
-
- interiorWriterAppend(pWriter->parentWriter,
- block->term.pData, block->term.nData, iBlockid);
- }
-
- /* Parent node gets the chance to be the root. */
- return interiorWriterRootInfo(v, pWriter->parentWriter,
- ppRootInfo, pnRootInfo, piEndBlockid);
-}
-
-/****************************************************************/
-/* InteriorReader is used to read off the data from an interior node
-** (see comment at top of file for the format).
-*/
-typedef struct InteriorReader {
- const char *pData;
- int nData;
-
- DataBuffer term; /* previous term, for decoding term delta. */
-
- sqlite_int64 iBlockid;
-} InteriorReader;
-
-static void interiorReaderDestroy(InteriorReader *pReader){
- dataBufferDestroy(&pReader->term);
- SCRAMBLE(pReader);
-}
-
-/* TODO(shess) The assertions are great, but what if we're in NDEBUG
-** and the blob is empty or otherwise contains suspect data?
-*/
-static void interiorReaderInit(const char *pData, int nData,
- InteriorReader *pReader){
- int n, nTerm;
-
- /* Require at least the leading flag byte */
- assert( nData>0 );
- assert( pData[0]!='\0' );
-
- CLEAR(pReader);
-
- /* Decode the base blockid, and set the cursor to the first term. */
- n = fts3GetVarint(pData+1, &pReader->iBlockid);
- assert( 1+n<=nData );
- pReader->pData = pData+1+n;
- pReader->nData = nData-(1+n);
-
- /* A single-child interior node (such as when a leaf node was too
- ** large for the segment directory) won't have any terms.
- ** Otherwise, decode the first term.
- */
- if( pReader->nData==0 ){
- dataBufferInit(&pReader->term, 0);
- }else{
- n = fts3GetVarint32(pReader->pData, &nTerm);
- dataBufferInit(&pReader->term, nTerm);
- dataBufferReplace(&pReader->term, pReader->pData+n, nTerm);
- assert( n+nTerm<=pReader->nData );
- pReader->pData += n+nTerm;
- pReader->nData -= n+nTerm;
- }
-}
-
-static int interiorReaderAtEnd(InteriorReader *pReader){
- return pReader->term.nData==0;
-}
-
-static sqlite_int64 interiorReaderCurrentBlockid(InteriorReader *pReader){
- return pReader->iBlockid;
-}
-
-static int interiorReaderTermBytes(InteriorReader *pReader){
- assert( !interiorReaderAtEnd(pReader) );
- return pReader->term.nData;
-}
-static const char *interiorReaderTerm(InteriorReader *pReader){
- assert( !interiorReaderAtEnd(pReader) );
- return pReader->term.pData;
-}
-
-/* Step forward to the next term in the node. */
-static void interiorReaderStep(InteriorReader *pReader){
- assert( !interiorReaderAtEnd(pReader) );
-
- /* If the last term has been read, signal eof, else construct the
- ** next term.
- */
- if( pReader->nData==0 ){
- dataBufferReset(&pReader->term);
- }else{
- int n, nPrefix, nSuffix;
-
- n = fts3GetVarint32(pReader->pData, &nPrefix);
- n += fts3GetVarint32(pReader->pData+n, &nSuffix);
-
- /* Truncate the current term and append suffix data. */
- pReader->term.nData = nPrefix;
- dataBufferAppend(&pReader->term, pReader->pData+n, nSuffix);
-
- assert( n+nSuffix<=pReader->nData );
- pReader->pData += n+nSuffix;
- pReader->nData -= n+nSuffix;
- }
- pReader->iBlockid++;
-}
-
-/* Compare the current term to pTerm[nTerm], returning strcmp-style
-** results. If isPrefix, equality means equal through nTerm bytes.
-*/
-static int interiorReaderTermCmp(InteriorReader *pReader,
- const char *pTerm, int nTerm, int isPrefix){
- const char *pReaderTerm = interiorReaderTerm(pReader);
- int nReaderTerm = interiorReaderTermBytes(pReader);
- int c, n = nReaderTerm<nTerm ? nReaderTerm : nTerm;
-
- if( n==0 ){
- if( nReaderTerm>0 ) return -1;
- if( nTerm>0 ) return 1;
- return 0;
- }
-
- c = memcmp(pReaderTerm, pTerm, n);
- if( c!=0 ) return c;
- if( isPrefix && n==nTerm ) return 0;
- return nReaderTerm - nTerm;
-}
-
-/****************************************************************/
-/* LeafWriter is used to collect terms and associated doclist data
-** into leaf blocks in %_segments (see top of file for format info).
-** Expected usage is:
-**
-** LeafWriter writer;
-** leafWriterInit(0, 0, &writer);
-** while( sorted_terms_left_to_process ){
-** // data is doclist data for that term.
-** rc = leafWriterStep(v, &writer, pTerm, nTerm, pData, nData);
-** if( rc!=SQLITE_OK ) goto err;
-** }
-** rc = leafWriterFinalize(v, &writer);
-**err:
-** leafWriterDestroy(&writer);
-** return rc;
-**
-** leafWriterStep() may write a collected leaf out to %_segments.
-** leafWriterFinalize() finishes writing any buffered data and stores
-** a root node in %_segdir. leafWriterDestroy() frees all buffers and
-** InteriorWriters allocated as part of writing this segment.
-**
-** TODO(shess) Document leafWriterStepMerge().
-*/
-
-/* Put terms with data this big in their own block. */
-#define STANDALONE_MIN 1024
-
-/* Keep leaf blocks below this size. */
-#define LEAF_MAX 2048
-
-typedef struct LeafWriter {
- int iLevel;
- int idx;
- sqlite_int64 iStartBlockid; /* needed to create the root info */
- sqlite_int64 iEndBlockid; /* when we're done writing. */
-
- DataBuffer term; /* previous encoded term */
- DataBuffer data; /* encoding buffer */
-
- /* bytes of first term in the current node which distinguishes that
- ** term from the last term of the previous node.
- */
- int nTermDistinct;
-
- InteriorWriter parentWriter; /* if we overflow */
- int has_parent;
-} LeafWriter;
-
-static void leafWriterInit(int iLevel, int idx, LeafWriter *pWriter){
- CLEAR(pWriter);
- pWriter->iLevel = iLevel;
- pWriter->idx = idx;
-
- dataBufferInit(&pWriter->term, 32);
-
- /* Start out with a reasonably sized block, though it can grow. */
- dataBufferInit(&pWriter->data, LEAF_MAX);
-}
-
-#ifndef NDEBUG
-/* Verify that the data is readable as a leaf node. */
-static void leafNodeValidate(const char *pData, int nData){
- int n, iDummy;
-
- if( nData==0 ) return;
- assert( nData>0 );
- assert( pData!=0 );
- assert( pData+nData>pData );
-
- /* Must lead with a varint(0) */
- n = fts3GetVarint32(pData, &iDummy);
- assert( iDummy==0 );
- assert( n>0 );
- assert( n<nData );
- pData += n;
- nData -= n;
-
- /* Leading term length and data must fit in buffer. */
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>0 );
- assert( n+iDummy>0 );
- assert( n+iDummy<nData );
- pData += n+iDummy;
- nData -= n+iDummy;
-
- /* Leading term's doclist length and data must fit. */
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>0 );
- assert( n+iDummy>0 );
- assert( n+iDummy<=nData );
- ASSERT_VALID_DOCLIST(DL_DEFAULT, pData+n, iDummy, NULL);
- pData += n+iDummy;
- nData -= n+iDummy;
-
- /* Verify that trailing terms and doclists also are readable. */
- while( nData!=0 ){
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>=0 );
- assert( n<nData );
- pData += n;
- nData -= n;
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>0 );
- assert( n+iDummy>0 );
- assert( n+iDummy<nData );
- pData += n+iDummy;
- nData -= n+iDummy;
-
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>0 );
- assert( n+iDummy>0 );
- assert( n+iDummy<=nData );
- ASSERT_VALID_DOCLIST(DL_DEFAULT, pData+n, iDummy, NULL);
- pData += n+iDummy;
- nData -= n+iDummy;
- }
-}
-#define ASSERT_VALID_LEAF_NODE(p, n) leafNodeValidate(p, n)
-#else
-#define ASSERT_VALID_LEAF_NODE(p, n) assert( 1 )
-#endif
-
-/* Flush the current leaf node to %_segments, and adding the resulting
-** blockid and the starting term to the interior node which will
-** contain it.
-*/
-static int leafWriterInternalFlush(fulltext_vtab *v, LeafWriter *pWriter,
- int iData, int nData){
- sqlite_int64 iBlockid = 0;
- const char *pStartingTerm;
- int nStartingTerm, rc, n;
-
- /* Must have the leading varint(0) flag, plus at least some
- ** valid-looking data.
- */
- assert( nData>2 );
- assert( iData>=0 );
- assert( iData+nData<=pWriter->data.nData );
- ASSERT_VALID_LEAF_NODE(pWriter->data.pData+iData, nData);
-
- rc = block_insert(v, pWriter->data.pData+iData, nData, &iBlockid);
- if( rc!=SQLITE_OK ) return rc;
- assert( iBlockid!=0 );
-
- /* Reconstruct the first term in the leaf for purposes of building
- ** the interior node.
- */
- n = fts3GetVarint32(pWriter->data.pData+iData+1, &nStartingTerm);
- pStartingTerm = pWriter->data.pData+iData+1+n;
- assert( pWriter->data.nData>iData+1+n+nStartingTerm );
- assert( pWriter->nTermDistinct>0 );
- assert( pWriter->nTermDistinct<=nStartingTerm );
- nStartingTerm = pWriter->nTermDistinct;
-
- if( pWriter->has_parent ){
- interiorWriterAppend(&pWriter->parentWriter,
- pStartingTerm, nStartingTerm, iBlockid);
- }else{
- interiorWriterInit(1, pStartingTerm, nStartingTerm, iBlockid,
- &pWriter->parentWriter);
- pWriter->has_parent = 1;
- }
-
- /* Track the span of this segment's leaf nodes. */
- if( pWriter->iEndBlockid==0 ){
- pWriter->iEndBlockid = pWriter->iStartBlockid = iBlockid;
- }else{
- pWriter->iEndBlockid++;
- assert( iBlockid==pWriter->iEndBlockid );
- }
-
- return SQLITE_OK;
-}
-static int leafWriterFlush(fulltext_vtab *v, LeafWriter *pWriter){
- int rc = leafWriterInternalFlush(v, pWriter, 0, pWriter->data.nData);
- if( rc!=SQLITE_OK ) return rc;
-
- /* Re-initialize the output buffer. */
- dataBufferReset(&pWriter->data);
-
- return SQLITE_OK;
-}
-
-/* Fetch the root info for the segment. If the entire leaf fits
-** within ROOT_MAX, then it will be returned directly, otherwise it
-** will be flushed and the root info will be returned from the
-** interior node. *piEndBlockid is set to the blockid of the last
-** interior or leaf node written to disk (0 if none are written at
-** all).
-*/
-static int leafWriterRootInfo(fulltext_vtab *v, LeafWriter *pWriter,
- char **ppRootInfo, int *pnRootInfo,
- sqlite_int64 *piEndBlockid){
- /* we can fit the segment entirely inline */
- if( !pWriter->has_parent && pWriter->data.nData<ROOT_MAX ){
- *ppRootInfo = pWriter->data.pData;
- *pnRootInfo = pWriter->data.nData;
- *piEndBlockid = 0;
+/*
+** Parameters z and n contain a pointer to and length of a buffer containing
+** an fts3 query expression, respectively. This function attempts to parse the
+** query expression and create a tree of Fts3Expr structures representing the
+** parsed expression. If successful, *ppExpr is set to point to the head
+** of the parsed expression tree and SQLITE_OK is returned. If an error
+** occurs, either SQLITE_NOMEM (out-of-memory error) or SQLITE_ERROR (parse
+** error) is returned and *ppExpr is set to 0.
+**
+** If parameter n is a negative number, then z is assumed to point to a
+** nul-terminated string and the length is determined using strlen().
+**
+** The first parameter, pTokenizer, is passed the fts3 tokenizer module to
+** use to normalize query tokens while parsing the expression. The azCol[]
+** array, which is assumed to contain nCol entries, should contain the names
+** of each column in the target fts3 table, in order from left to right.
+** Column names must be nul-terminated strings.
+**
+** The iDefaultCol parameter should be passed the index of the table column
+** that appears on the left-hand-side of the MATCH operator (the default
+** column to match against for tokens for which a column name is not explicitly
+** specified as part of the query string), or -1 if tokens may by default
+** match any table column.
+*/
+SQLITE_PRIVATE int sqlite3Fts3ExprParse(
+ sqlite3_tokenizer *pTokenizer, /* Tokenizer module */
+ char **azCol, /* Array of column names for fts3 table */
+ int nCol, /* Number of entries in azCol[] */
+ int iDefaultCol, /* Default column to query */
+ const char *z, int n, /* Text of MATCH query */
+ Fts3Expr **ppExpr /* OUT: Parsed query structure */
+){
+ int nParsed;
+ int rc;
+ ParseContext sParse;
+ sParse.pTokenizer = pTokenizer;
+ sParse.azCol = (const char **)azCol;
+ sParse.nCol = nCol;
+ sParse.iDefaultCol = iDefaultCol;
+ sParse.nNest = 0;
+ if( z==0 ){
+ *ppExpr = 0;
return SQLITE_OK;
}
-
- /* Flush remaining leaf data. */
- if( pWriter->data.nData>0 ){
- int rc = leafWriterFlush(v, pWriter);
- if( rc!=SQLITE_OK ) return rc;
+ if( n<0 ){
+ n = (int)strlen(z);
}
+ rc = fts3ExprParse(&sParse, z, n, ppExpr, &nParsed);
- /* We must have flushed a leaf at some point. */
- assert( pWriter->has_parent );
-
- /* Tenatively set the end leaf blockid as the end blockid. If the
- ** interior node can be returned inline, this will be the final
- ** blockid, otherwise it will be overwritten by
- ** interiorWriterRootInfo().
- */
- *piEndBlockid = pWriter->iEndBlockid;
+ /* Check for mismatched parenthesis */
+ if( rc==SQLITE_OK && sParse.nNest ){
+ rc = SQLITE_ERROR;
+ sqlite3Fts3ExprFree(*ppExpr);
+ *ppExpr = 0;
+ }
- return interiorWriterRootInfo(v, &pWriter->parentWriter,
- ppRootInfo, pnRootInfo, piEndBlockid);
+ return rc;
}
-/* Collect the rootInfo data and store it into the segment directory.
-** This has the effect of flushing the segment's leaf data to
-** %_segments, and also flushing any interior nodes to %_segments.
+/*
+** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse().
*/
-static int leafWriterFinalize(fulltext_vtab *v, LeafWriter *pWriter){
- sqlite_int64 iEndBlockid;
- char *pRootInfo;
- int rc, nRootInfo;
-
- rc = leafWriterRootInfo(v, pWriter, &pRootInfo, &nRootInfo, &iEndBlockid);
- if( rc!=SQLITE_OK ) return rc;
+SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *p){
+ if( p ){
+ sqlite3Fts3ExprFree(p->pLeft);
+ sqlite3Fts3ExprFree(p->pRight);
+ sqlite3_free(p);
+ }
+}
- /* Don't bother storing an entirely empty segment. */
- if( iEndBlockid==0 && nRootInfo==0 ) return SQLITE_OK;
+/****************************************************************************
+*****************************************************************************
+** Everything after this point is just test code.
+*/
- return segdir_set(v, pWriter->iLevel, pWriter->idx,
- pWriter->iStartBlockid, pWriter->iEndBlockid,
- iEndBlockid, pRootInfo, nRootInfo);
-}
+#ifdef SQLITE_TEST
-static void leafWriterDestroy(LeafWriter *pWriter){
- if( pWriter->has_parent ) interiorWriterDestroy(&pWriter->parentWriter);
- dataBufferDestroy(&pWriter->term);
- dataBufferDestroy(&pWriter->data);
-}
-/* Encode a term into the leafWriter, delta-encoding as appropriate.
-** Returns the length of the new term which distinguishes it from the
-** previous term, which can be used to set nTermDistinct when a node
-** boundary is crossed.
+/*
+** Function to query the hash-table of tokenizers (see README.tokenizers).
*/
-static int leafWriterEncodeTerm(LeafWriter *pWriter,
- const char *pTerm, int nTerm){
- char c[VARINT_MAX+VARINT_MAX];
- int n, nPrefix = 0;
+static int queryTestTokenizer(
+ sqlite3 *db,
+ const char *zName,
+ const sqlite3_tokenizer_module **pp
+){
+ int rc;
+ sqlite3_stmt *pStmt;
+ const char zSql[] = "SELECT fts3_tokenizer(?)";
- assert( nTerm>0 );
- while( nPrefix<pWriter->term.nData &&
- pTerm[nPrefix]==pWriter->term.pData[nPrefix] ){
- nPrefix++;
- /* Failing this implies that the terms weren't in order. */
- assert( nPrefix<nTerm );
+ *pp = 0;
+ rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+ if( rc!=SQLITE_OK ){
+ return rc;
}
- if( pWriter->data.nData==0 ){
- /* Encode the node header and leading term as:
- ** varint(0)
- ** varint(nTerm)
- ** char pTerm[nTerm]
- */
- n = fts3PutVarint(c, '\0');
- n += fts3PutVarint(c+n, nTerm);
- dataBufferAppend2(&pWriter->data, c, n, pTerm, nTerm);
- }else{
- /* Delta-encode the term as:
- ** varint(nPrefix)
- ** varint(nSuffix)
- ** char pTermSuffix[nSuffix]
- */
- n = fts3PutVarint(c, nPrefix);
- n += fts3PutVarint(c+n, nTerm-nPrefix);
- dataBufferAppend2(&pWriter->data, c, n, pTerm+nPrefix, nTerm-nPrefix);
+ sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
+ if( SQLITE_ROW==sqlite3_step(pStmt) ){
+ if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
+ memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
+ }
}
- dataBufferReplace(&pWriter->term, pTerm, nTerm);
- return nPrefix+1;
+ return sqlite3_finalize(pStmt);
}
-/* Used to avoid a memmove when a large amount of doclist data is in
-** the buffer. This constructs a node and term header before
-** iDoclistData and flushes the resulting complete node using
-** leafWriterInternalFlush().
+/*
+** This function is part of the test interface for the query parser. It
+** writes a text representation of the query expression pExpr into the
+** buffer pointed to by argument zBuf. It is assumed that zBuf is large
+** enough to store the required text representation.
*/
-static int leafWriterInlineFlush(fulltext_vtab *v, LeafWriter *pWriter,
- const char *pTerm, int nTerm,
- int iDoclistData){
- char c[VARINT_MAX+VARINT_MAX];
- int iData, n = fts3PutVarint(c, 0);
- n += fts3PutVarint(c+n, nTerm);
+static void exprToString(Fts3Expr *pExpr, char *zBuf){
+ switch( pExpr->eType ){
+ case FTSQUERY_PHRASE: {
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
+ int i;
+ zBuf += sprintf(zBuf, "PHRASE %d %d", pPhrase->iColumn, pPhrase->isNot);
+ for(i=0; i<pPhrase->nToken; i++){
+ zBuf += sprintf(zBuf," %.*s",pPhrase->aToken[i].n,pPhrase->aToken[i].z);
+ zBuf += sprintf(zBuf,"%s", (pPhrase->aToken[i].isPrefix?"+":""));
+ }
+ return;
+ }
- /* There should always be room for the header. Even if pTerm shared
- ** a substantial prefix with the previous term, the entire prefix
- ** could be constructed from earlier data in the doclist, so there
- ** should be room.
- */
- assert( iDoclistData>=n+nTerm );
+ case FTSQUERY_NEAR:
+ zBuf += sprintf(zBuf, "NEAR/%d ", pExpr->nNear);
+ break;
+ case FTSQUERY_NOT:
+ zBuf += sprintf(zBuf, "NOT ");
+ break;
+ case FTSQUERY_AND:
+ zBuf += sprintf(zBuf, "AND ");
+ break;
+ case FTSQUERY_OR:
+ zBuf += sprintf(zBuf, "OR ");
+ break;
+ }
- iData = iDoclistData-(n+nTerm);
- memcpy(pWriter->data.pData+iData, c, n);
- memcpy(pWriter->data.pData+iData+n, pTerm, nTerm);
+ zBuf += sprintf(zBuf, "{");
+ exprToString(pExpr->pLeft, zBuf);
+ zBuf += strlen(zBuf);
+ zBuf += sprintf(zBuf, "} ");
- return leafWriterInternalFlush(v, pWriter, iData, pWriter->data.nData-iData);
+ zBuf += sprintf(zBuf, "{");
+ exprToString(pExpr->pRight, zBuf);
+ zBuf += strlen(zBuf);
+ zBuf += sprintf(zBuf, "}");
}
-/* Push pTerm[nTerm] along with the doclist data to the leaf layer of
-** %_segments.
+/*
+** This is the implementation of a scalar SQL function used to test the
+** expression parser. It should be called as follows:
+**
+** fts3_exprtest(<tokenizer>, <expr>, <column 1>, ...);
+**
+** The first argument, <tokenizer>, is the name of the fts3 tokenizer used
+** to parse the query expression (see README.tokenizers). The second argument
+** is the query expression to parse. Each subsequent argument is the name
+** of a column of the fts3 table that the query expression may refer to.
+** For example:
+**
+** SELECT fts3_exprtest('simple', 'Bill col2:Bloggs', 'col1', 'col2');
*/
-static int leafWriterStepMerge(fulltext_vtab *v, LeafWriter *pWriter,
- const char *pTerm, int nTerm,
- DLReader *pReaders, int nReaders){
- char c[VARINT_MAX+VARINT_MAX];
- int iTermData = pWriter->data.nData, iDoclistData;
- int i, nData, n, nActualData, nActual, rc, nTermDistinct;
-
- ASSERT_VALID_LEAF_NODE(pWriter->data.pData, pWriter->data.nData);
- nTermDistinct = leafWriterEncodeTerm(pWriter, pTerm, nTerm);
-
- /* Remember nTermDistinct if opening a new node. */
- if( iTermData==0 ) pWriter->nTermDistinct = nTermDistinct;
-
- iDoclistData = pWriter->data.nData;
+static void fts3ExprTest(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+ sqlite3_tokenizer_module const *pModule = 0;
+ sqlite3_tokenizer *pTokenizer = 0;
+ int rc;
+ char **azCol = 0;
+ const char *zExpr;
+ int nExpr;
+ int nCol;
+ int ii;
+ Fts3Expr *pExpr;
+ sqlite3 *db = sqlite3_context_db_handle(context);
- /* Estimate the length of the merged doclist so we can leave space
- ** to encode it.
- */
- for(i=0, nData=0; i<nReaders; i++){
- nData += dlrAllDataBytes(&pReaders[i]);
+ if( argc<3 ){
+ sqlite3_result_error(context,
+ "Usage: fts3_exprtest(tokenizer, expr, col1, ...", -1
+ );
+ return;
}
- n = fts3PutVarint(c, nData);
- dataBufferAppend(&pWriter->data, c, n);
-
- docListMerge(&pWriter->data, pReaders, nReaders);
- ASSERT_VALID_DOCLIST(DL_DEFAULT,
- pWriter->data.pData+iDoclistData+n,
- pWriter->data.nData-iDoclistData-n, NULL);
-
- /* The actual amount of doclist data at this point could be smaller
- ** than the length we encoded. Additionally, the space required to
- ** encode this length could be smaller. For small doclists, this is
- ** not a big deal, we can just use memmove() to adjust things.
- */
- nActualData = pWriter->data.nData-(iDoclistData+n);
- nActual = fts3PutVarint(c, nActualData);
- assert( nActualData<=nData );
- assert( nActual<=n );
- /* If the new doclist is big enough for force a standalone leaf
- ** node, we can immediately flush it inline without doing the
- ** memmove().
- */
- /* TODO(shess) This test matches leafWriterStep(), which does this
- ** test before it knows the cost to varint-encode the term and
- ** doclist lengths. At some point, change to
- ** pWriter->data.nData-iTermData>STANDALONE_MIN.
- */
- if( nTerm+nActualData>STANDALONE_MIN ){
- /* Push leaf node from before this term. */
- if( iTermData>0 ){
- rc = leafWriterInternalFlush(v, pWriter, 0, iTermData);
- if( rc!=SQLITE_OK ) return rc;
-
- pWriter->nTermDistinct = nTermDistinct;
- }
-
- /* Fix the encoded doclist length. */
- iDoclistData += n - nActual;
- memcpy(pWriter->data.pData+iDoclistData, c, nActual);
-
- /* Push the standalone leaf node. */
- rc = leafWriterInlineFlush(v, pWriter, pTerm, nTerm, iDoclistData);
- if( rc!=SQLITE_OK ) return rc;
-
- /* Leave the node empty. */
- dataBufferReset(&pWriter->data);
-
- return rc;
+ rc = queryTestTokenizer(db,
+ (const char *)sqlite3_value_text(argv[0]), &pModule);
+ if( rc==SQLITE_NOMEM ){
+ sqlite3_result_error_nomem(context);
+ goto exprtest_out;
+ }else if( !pModule ){
+ sqlite3_result_error(context, "No such tokenizer module", -1);
+ goto exprtest_out;
}
- /* At this point, we know that the doclist was small, so do the
- ** memmove if indicated.
- */
- if( nActual<n ){
- memmove(pWriter->data.pData+iDoclistData+nActual,
- pWriter->data.pData+iDoclistData+n,
- pWriter->data.nData-(iDoclistData+n));
- pWriter->data.nData -= n-nActual;
+ rc = pModule->xCreate(0, 0, &pTokenizer);
+ assert( rc==SQLITE_NOMEM || rc==SQLITE_OK );
+ if( rc==SQLITE_NOMEM ){
+ sqlite3_result_error_nomem(context);
+ goto exprtest_out;
}
+ pTokenizer->pModule = pModule;
- /* Replace written length with actual length. */
- memcpy(pWriter->data.pData+iDoclistData, c, nActual);
-
- /* If the node is too large, break things up. */
- /* TODO(shess) This test matches leafWriterStep(), which does this
- ** test before it knows the cost to varint-encode the term and
- ** doclist lengths. At some point, change to
- ** pWriter->data.nData>LEAF_MAX.
- */
- if( iTermData+nTerm+nActualData>LEAF_MAX ){
- /* Flush out the leading data as a node */
- rc = leafWriterInternalFlush(v, pWriter, 0, iTermData);
- if( rc!=SQLITE_OK ) return rc;
-
- pWriter->nTermDistinct = nTermDistinct;
-
- /* Rebuild header using the current term */
- n = fts3PutVarint(pWriter->data.pData, 0);
- n += fts3PutVarint(pWriter->data.pData+n, nTerm);
- memcpy(pWriter->data.pData+n, pTerm, nTerm);
- n += nTerm;
-
- /* There should always be room, because the previous encoding
- ** included all data necessary to construct the term.
- */
- assert( n<iDoclistData );
- /* So long as STANDALONE_MIN is half or less of LEAF_MAX, the
- ** following memcpy() is safe (as opposed to needing a memmove).
- */
- assert( 2*STANDALONE_MIN<=LEAF_MAX );
- assert( n+pWriter->data.nData-iDoclistData<iDoclistData );
- memcpy(pWriter->data.pData+n,
- pWriter->data.pData+iDoclistData,
- pWriter->data.nData-iDoclistData);
- pWriter->data.nData -= iDoclistData-n;
+ zExpr = (const char *)sqlite3_value_text(argv[1]);
+ nExpr = sqlite3_value_bytes(argv[1]);
+ nCol = argc-2;
+ azCol = (char **)sqlite3_malloc(nCol*sizeof(char *));
+ if( !azCol ){
+ sqlite3_result_error_nomem(context);
+ goto exprtest_out;
+ }
+ for(ii=0; ii<nCol; ii++){
+ azCol[ii] = (char *)sqlite3_value_text(argv[ii+2]);
}
- ASSERT_VALID_LEAF_NODE(pWriter->data.pData, pWriter->data.nData);
-
- return SQLITE_OK;
-}
-
-/* Push pTerm[nTerm] along with the doclist data to the leaf layer of
-** %_segments.
-*/
-/* TODO(shess) Revise writeZeroSegment() so that doclists are
-** constructed directly in pWriter->data.
-*/
-static int leafWriterStep(fulltext_vtab *v, LeafWriter *pWriter,
- const char *pTerm, int nTerm,
- const char *pData, int nData){
- int rc;
- DLReader reader;
-
- dlrInit(&reader, DL_DEFAULT, pData, nData);
- rc = leafWriterStepMerge(v, pWriter, pTerm, nTerm, &reader, 1);
- dlrDestroy(&reader);
-
- return rc;
-}
-
-
-/****************************************************************/
-/* LeafReader is used to iterate over an individual leaf node. */
-typedef struct LeafReader {
- DataBuffer term; /* copy of current term. */
-
- const char *pData; /* data for current term. */
- int nData;
-} LeafReader;
-
-static void leafReaderDestroy(LeafReader *pReader){
- dataBufferDestroy(&pReader->term);
- SCRAMBLE(pReader);
-}
-
-static int leafReaderAtEnd(LeafReader *pReader){
- return pReader->nData<=0;
-}
-
-/* Access the current term. */
-static int leafReaderTermBytes(LeafReader *pReader){
- return pReader->term.nData;
-}
-static const char *leafReaderTerm(LeafReader *pReader){
- assert( pReader->term.nData>0 );
- return pReader->term.pData;
-}
-
-/* Access the doclist data for the current term. */
-static int leafReaderDataBytes(LeafReader *pReader){
- int nData;
- assert( pReader->term.nData>0 );
- fts3GetVarint32(pReader->pData, &nData);
- return nData;
-}
-static const char *leafReaderData(LeafReader *pReader){
- int n, nData;
- assert( pReader->term.nData>0 );
- n = fts3GetVarint32(pReader->pData, &nData);
- return pReader->pData+n;
-}
-
-static void leafReaderInit(const char *pData, int nData,
- LeafReader *pReader){
- int nTerm, n;
-
- assert( nData>0 );
- assert( pData[0]=='\0' );
-
- CLEAR(pReader);
-
- /* Read the first term, skipping the header byte. */
- n = fts3GetVarint32(pData+1, &nTerm);
- dataBufferInit(&pReader->term, nTerm);
- dataBufferReplace(&pReader->term, pData+1+n, nTerm);
-
- /* Position after the first term. */
- assert( 1+n+nTerm<nData );
- pReader->pData = pData+1+n+nTerm;
- pReader->nData = nData-1-n-nTerm;
-}
-
-/* Step the reader forward to the next term. */
-static void leafReaderStep(LeafReader *pReader){
- int n, nData, nPrefix, nSuffix;
- assert( !leafReaderAtEnd(pReader) );
-
- /* Skip previous entry's data block. */
- n = fts3GetVarint32(pReader->pData, &nData);
- assert( n+nData<=pReader->nData );
- pReader->pData += n+nData;
- pReader->nData -= n+nData;
- if( !leafReaderAtEnd(pReader) ){
- /* Construct the new term using a prefix from the old term plus a
- ** suffix from the leaf data.
- */
- n = fts3GetVarint32(pReader->pData, &nPrefix);
- n += fts3GetVarint32(pReader->pData+n, &nSuffix);
- assert( n+nSuffix<pReader->nData );
- pReader->term.nData = nPrefix;
- dataBufferAppend(&pReader->term, pReader->pData+n, nSuffix);
+ rc = sqlite3Fts3ExprParse(
+ pTokenizer, azCol, nCol, nCol, zExpr, nExpr, &pExpr
+ );
+ if( rc==SQLITE_NOMEM ){
+ sqlite3_result_error_nomem(context);
+ goto exprtest_out;
+ }else if( rc==SQLITE_OK ){
+ char zBuf[4096];
+ exprToString(pExpr, zBuf);
+ sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT);
+ sqlite3Fts3ExprFree(pExpr);
+ }else{
+ sqlite3_result_error(context, "Error parsing expression", -1);
+ }
- pReader->pData += n+nSuffix;
- pReader->nData -= n+nSuffix;
+exprtest_out:
+ if( pModule && pTokenizer ){
+ rc = pModule->xDestroy(pTokenizer);
}
+ sqlite3_free(azCol);
}
-/* strcmp-style comparison of pReader's current term against pTerm.
-** If isPrefix, equality means equal through nTerm bytes.
+/*
+** Register the query expression parser test function fts3_exprtest()
+** with database connection db.
*/
-static int leafReaderTermCmp(LeafReader *pReader,
- const char *pTerm, int nTerm, int isPrefix){
- int c, n = pReader->term.nData<nTerm ? pReader->term.nData : nTerm;
- if( n==0 ){
- if( pReader->term.nData>0 ) return -1;
- if(nTerm>0 ) return 1;
- return 0;
- }
-
- c = memcmp(pReader->term.pData, pTerm, n);
- if( c!=0 ) return c;
- if( isPrefix && n==nTerm ) return 0;
- return pReader->term.nData - nTerm;
+SQLITE_PRIVATE void sqlite3Fts3ExprInitTestInterface(sqlite3* db){
+ sqlite3_create_function(
+ db, "fts3_exprtest", -1, SQLITE_UTF8, 0, fts3ExprTest, 0, 0
+ );
}
+#endif
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
-/****************************************************************/
-/* LeavesReader wraps LeafReader to allow iterating over the entire
-** leaf layer of the tree.
+/************** End of fts3_expr.c *******************************************/
+/************** Begin file fts3_hash.c ***************************************/
+/*
+** 2001 September 22
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This is the implementation of generic hash-tables used in SQLite.
+** We've modified it slightly to serve as a standalone hash table
+** implementation for the full-text indexing module.
*/
-typedef struct LeavesReader {
- int idx; /* Index within the segment. */
-
- sqlite3_stmt *pStmt; /* Statement we're streaming leaves from. */
- int eof; /* we've seen SQLITE_DONE from pStmt. */
- LeafReader leafReader; /* reader for the current leaf. */
- DataBuffer rootData; /* root data for inline. */
-} LeavesReader;
-
-/* Access the current term. */
-static int leavesReaderTermBytes(LeavesReader *pReader){
- assert( !pReader->eof );
- return leafReaderTermBytes(&pReader->leafReader);
-}
-static const char *leavesReaderTerm(LeavesReader *pReader){
- assert( !pReader->eof );
- return leafReaderTerm(&pReader->leafReader);
-}
+/*
+** The code in this file is only compiled if:
+**
+** * The FTS3 module is being built as an extension
+** (in which case SQLITE_CORE is not defined), or
+**
+** * The FTS3 module is being built into the core of
+** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
+*/
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
-/* Access the doclist data for the current term. */
-static int leavesReaderDataBytes(LeavesReader *pReader){
- assert( !pReader->eof );
- return leafReaderDataBytes(&pReader->leafReader);
-}
-static const char *leavesReaderData(LeavesReader *pReader){
- assert( !pReader->eof );
- return leafReaderData(&pReader->leafReader);
-}
-static int leavesReaderAtEnd(LeavesReader *pReader){
- return pReader->eof;
-}
-/* loadSegmentLeaves() may not read all the way to SQLITE_DONE, thus
-** leaving the statement handle open, which locks the table.
-*/
-/* TODO(shess) This "solution" is not satisfactory. Really, there
-** should be check-in function for all statement handles which
-** arranges to call sqlite3_reset(). This most likely will require
-** modification to control flow all over the place, though, so for now
-** just punt.
-**
-** Note the the current system assumes that segment merges will run to
-** completion, which is why this particular probably hasn't arisen in
-** this case. Probably a brittle assumption.
+/*
+** Malloc and Free functions
*/
-static int leavesReaderReset(LeavesReader *pReader){
- return sqlite3_reset(pReader->pStmt);
-}
-
-static void leavesReaderDestroy(LeavesReader *pReader){
- /* If idx is -1, that means we're using a non-cached statement
- ** handle in the optimize() case, so we need to release it.
- */
- if( pReader->pStmt!=NULL && pReader->idx==-1 ){
- sqlite3_finalize(pReader->pStmt);
+static void *fts3HashMalloc(int n){
+ void *p = sqlite3_malloc(n);
+ if( p ){
+ memset(p, 0, n);
}
- leafReaderDestroy(&pReader->leafReader);
- dataBufferDestroy(&pReader->rootData);
- SCRAMBLE(pReader);
+ return p;
+}
+static void fts3HashFree(void *p){
+ sqlite3_free(p);
}
-/* Initialize pReader with the given root data (if iStartBlockid==0
-** the leaf data was entirely contained in the root), or from the
-** stream of blocks between iStartBlockid and iEndBlockid, inclusive.
+/* Turn bulk memory into a hash table object by initializing the
+** fields of the Hash structure.
+**
+** "pNew" is a pointer to the hash table that is to be initialized.
+** keyClass is one of the constants
+** FTS3_HASH_BINARY or FTS3_HASH_STRING. The value of keyClass
+** determines what kind of key the hash table will use. "copyKey" is
+** true if the hash table should make its own private copy of keys and
+** false if it should just use the supplied pointer.
*/
-static int leavesReaderInit(fulltext_vtab *v,
- int idx,
- sqlite_int64 iStartBlockid,
- sqlite_int64 iEndBlockid,
- const char *pRootData, int nRootData,
- LeavesReader *pReader){
- CLEAR(pReader);
- pReader->idx = idx;
-
- dataBufferInit(&pReader->rootData, 0);
- if( iStartBlockid==0 ){
- /* Entire leaf level fit in root data. */
- dataBufferReplace(&pReader->rootData, pRootData, nRootData);
- leafReaderInit(pReader->rootData.pData, pReader->rootData.nData,
- &pReader->leafReader);
- }else{
- sqlite3_stmt *s;
- int rc = sql_get_leaf_statement(v, idx, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int64(s, 1, iStartBlockid);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int64(s, 2, iEndBlockid);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_step(s);
- if( rc==SQLITE_DONE ){
- pReader->eof = 1;
- return SQLITE_OK;
- }
- if( rc!=SQLITE_ROW ) return rc;
-
- pReader->pStmt = s;
- leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0),
- sqlite3_column_bytes(pReader->pStmt, 0),
- &pReader->leafReader);
- }
- return SQLITE_OK;
+SQLITE_PRIVATE void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey){
+ assert( pNew!=0 );
+ assert( keyClass>=FTS3_HASH_STRING && keyClass<=FTS3_HASH_BINARY );
+ pNew->keyClass = keyClass;
+ pNew->copyKey = copyKey;
+ pNew->first = 0;
+ pNew->count = 0;
+ pNew->htsize = 0;
+ pNew->ht = 0;
}
-/* Step the current leaf forward to the next term. If we reach the
-** end of the current leaf, step forward to the next leaf block.
+/* Remove all entries from a hash table. Reclaim all memory.
+** Call this routine to delete a hash table or to reset a hash table
+** to the empty state.
*/
-static int leavesReaderStep(fulltext_vtab *v, LeavesReader *pReader){
- assert( !leavesReaderAtEnd(pReader) );
- leafReaderStep(&pReader->leafReader);
+SQLITE_PRIVATE void sqlite3Fts3HashClear(Fts3Hash *pH){
+ Fts3HashElem *elem; /* For looping over all elements of the table */
- if( leafReaderAtEnd(&pReader->leafReader) ){
- int rc;
- if( pReader->rootData.pData ){
- pReader->eof = 1;
- return SQLITE_OK;
- }
- rc = sqlite3_step(pReader->pStmt);
- if( rc!=SQLITE_ROW ){
- pReader->eof = 1;
- return rc==SQLITE_DONE ? SQLITE_OK : rc;
+ assert( pH!=0 );
+ elem = pH->first;
+ pH->first = 0;
+ fts3HashFree(pH->ht);
+ pH->ht = 0;
+ pH->htsize = 0;
+ while( elem ){
+ Fts3HashElem *next_elem = elem->next;
+ if( pH->copyKey && elem->pKey ){
+ fts3HashFree(elem->pKey);
}
- leafReaderDestroy(&pReader->leafReader);
- leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0),
- sqlite3_column_bytes(pReader->pStmt, 0),
- &pReader->leafReader);
+ fts3HashFree(elem);
+ elem = next_elem;
}
- return SQLITE_OK;
+ pH->count = 0;
}
-/* Order LeavesReaders by their term, ignoring idx. Readers at eof
-** always sort to the end.
+/*
+** Hash and comparison functions when the mode is FTS3_HASH_STRING
*/
-static int leavesReaderTermCmp(LeavesReader *lr1, LeavesReader *lr2){
- if( leavesReaderAtEnd(lr1) ){
- if( leavesReaderAtEnd(lr2) ) return 0;
- return 1;
+static int fts3StrHash(const void *pKey, int nKey){
+ const char *z = (const char *)pKey;
+ int h = 0;
+ if( nKey<=0 ) nKey = (int) strlen(z);
+ while( nKey > 0 ){
+ h = (h<<3) ^ h ^ *z++;
+ nKey--;
}
- if( leavesReaderAtEnd(lr2) ) return -1;
-
- return leafReaderTermCmp(&lr1->leafReader,
- leavesReaderTerm(lr2), leavesReaderTermBytes(lr2),
- 0);
+ return h & 0x7fffffff;
}
-
-/* Similar to leavesReaderTermCmp(), with additional ordering by idx
-** so that older segments sort before newer segments.
-*/
-static int leavesReaderCmp(LeavesReader *lr1, LeavesReader *lr2){
- int c = leavesReaderTermCmp(lr1, lr2);
- if( c!=0 ) return c;
- return lr1->idx-lr2->idx;
+static int fts3StrCompare(const void *pKey1, int n1, const void *pKey2, int n2){
+ if( n1!=n2 ) return 1;
+ return strncmp((const char*)pKey1,(const char*)pKey2,n1);
}
-/* Assume that pLr[1]..pLr[nLr] are sorted. Bubble pLr[0] into its
-** sorted position.
+/*
+** Hash and comparison functions when the mode is FTS3_HASH_BINARY
*/
-static void leavesReaderReorder(LeavesReader *pLr, int nLr){
- while( nLr>1 && leavesReaderCmp(pLr, pLr+1)>0 ){
- LeavesReader tmp = pLr[0];
- pLr[0] = pLr[1];
- pLr[1] = tmp;
- nLr--;
- pLr++;
+static int fts3BinHash(const void *pKey, int nKey){
+ int h = 0;
+ const char *z = (const char *)pKey;
+ while( nKey-- > 0 ){
+ h = (h<<3) ^ h ^ *(z++);
}
+ return h & 0x7fffffff;
}
-
-/* Initializes pReaders with the segments from level iLevel, returning
-** the number of segments in *piReaders. Leaves pReaders in sorted
-** order.
-*/
-static int leavesReadersInit(fulltext_vtab *v, int iLevel,
- LeavesReader *pReaders, int *piReaders){
- sqlite3_stmt *s;
- int i, rc = sql_get_statement(v, SEGDIR_SELECT_LEVEL_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int(s, 1, iLevel);
- if( rc!=SQLITE_OK ) return rc;
-
- i = 0;
- while( (rc = sqlite3_step(s))==SQLITE_ROW ){
- sqlite_int64 iStart = sqlite3_column_int64(s, 0);
- sqlite_int64 iEnd = sqlite3_column_int64(s, 1);
- const char *pRootData = sqlite3_column_blob(s, 2);
- int nRootData = sqlite3_column_bytes(s, 2);
-
- assert( i<MERGE_COUNT );
- rc = leavesReaderInit(v, i, iStart, iEnd, pRootData, nRootData,
- &pReaders[i]);
- if( rc!=SQLITE_OK ) break;
-
- i++;
- }
- if( rc!=SQLITE_DONE ){
- while( i-->0 ){
- leavesReaderDestroy(&pReaders[i]);
- }
- return rc;
- }
-
- *piReaders = i;
-
- /* Leave our results sorted by term, then age. */
- while( i-- ){
- leavesReaderReorder(pReaders+i, *piReaders-i);
- }
- return SQLITE_OK;
+static int fts3BinCompare(const void *pKey1, int n1, const void *pKey2, int n2){
+ if( n1!=n2 ) return 1;
+ return memcmp(pKey1,pKey2,n1);
}
-/* Merge doclists from pReaders[nReaders] into a single doclist, which
-** is written to pWriter. Assumes pReaders is ordered oldest to
-** newest.
+/*
+** Return a pointer to the appropriate hash function given the key class.
+**
+** The C syntax in this function definition may be unfamilar to some
+** programmers, so we provide the following additional explanation:
+**
+** The name of the function is "ftsHashFunction". The function takes a
+** single parameter "keyClass". The return value of ftsHashFunction()
+** is a pointer to another function. Specifically, the return value
+** of ftsHashFunction() is a pointer to a function that takes two parameters
+** with types "const void*" and "int" and returns an "int".
*/
-/* TODO(shess) Consider putting this inline in segmentMerge(). */
-static int leavesReadersMerge(fulltext_vtab *v,
- LeavesReader *pReaders, int nReaders,
- LeafWriter *pWriter){
- DLReader dlReaders[MERGE_COUNT];
- const char *pTerm = leavesReaderTerm(pReaders);
- int i, nTerm = leavesReaderTermBytes(pReaders);
-
- assert( nReaders<=MERGE_COUNT );
-
- for(i=0; i<nReaders; i++){
- dlrInit(&dlReaders[i], DL_DEFAULT,
- leavesReaderData(pReaders+i),
- leavesReaderDataBytes(pReaders+i));
+static int (*ftsHashFunction(int keyClass))(const void*,int){
+ if( keyClass==FTS3_HASH_STRING ){
+ return &fts3StrHash;
+ }else{
+ assert( keyClass==FTS3_HASH_BINARY );
+ return &fts3BinHash;
}
-
- return leafWriterStepMerge(v, pWriter, pTerm, nTerm, dlReaders, nReaders);
}
-/* Forward ref due to mutual recursion with segdirNextIndex(). */
-static int segmentMerge(fulltext_vtab *v, int iLevel);
-
-/* Put the next available index at iLevel into *pidx. If iLevel
-** already has MERGE_COUNT segments, they are merged to a higher
-** level to make room.
+/*
+** Return a pointer to the appropriate hash function given the key class.
+**
+** For help in interpreted the obscure C code in the function definition,
+** see the header comment on the previous function.
*/
-static int segdirNextIndex(fulltext_vtab *v, int iLevel, int *pidx){
- int rc = segdir_max_index(v, iLevel, pidx);
- if( rc==SQLITE_DONE ){ /* No segments at iLevel. */
- *pidx = 0;
- }else if( rc==SQLITE_ROW ){
- if( *pidx==(MERGE_COUNT-1) ){
- rc = segmentMerge(v, iLevel);
- if( rc!=SQLITE_OK ) return rc;
- *pidx = 0;
- }else{
- (*pidx)++;
- }
+static int (*ftsCompareFunction(int keyClass))(const void*,int,const void*,int){
+ if( keyClass==FTS3_HASH_STRING ){
+ return &fts3StrCompare;
}else{
- return rc;
+ assert( keyClass==FTS3_HASH_BINARY );
+ return &fts3BinCompare;
}
- return SQLITE_OK;
}
-/* Merge MERGE_COUNT segments at iLevel into a new segment at
-** iLevel+1. If iLevel+1 is already full of segments, those will be
-** merged to make room.
+/* Link an element into the hash table
*/
-static int segmentMerge(fulltext_vtab *v, int iLevel){
- LeafWriter writer;
- LeavesReader lrs[MERGE_COUNT];
- int i, rc, idx = 0;
-
- /* Determine the next available segment index at the next level,
- ** merging as necessary.
- */
- rc = segdirNextIndex(v, iLevel+1, &idx);
- if( rc!=SQLITE_OK ) return rc;
-
- /* TODO(shess) This assumes that we'll always see exactly
- ** MERGE_COUNT segments to merge at a given level. That will be
- ** broken if we allow the developer to request preemptive or
- ** deferred merging.
- */
- memset(&lrs, '\0', sizeof(lrs));
- rc = leavesReadersInit(v, iLevel, lrs, &i);
- if( rc!=SQLITE_OK ) return rc;
- assert( i==MERGE_COUNT );
-
- leafWriterInit(iLevel+1, idx, &writer);
-
- /* Since leavesReaderReorder() pushes readers at eof to the end,
- ** when the first reader is empty, all will be empty.
- */
- while( !leavesReaderAtEnd(lrs) ){
- /* Figure out how many readers share their next term. */
- for(i=1; i<MERGE_COUNT && !leavesReaderAtEnd(lrs+i); i++){
- if( 0!=leavesReaderTermCmp(lrs, lrs+i) ) break;
- }
-
- rc = leavesReadersMerge(v, lrs, i, &writer);
- if( rc!=SQLITE_OK ) goto err;
-
- /* Step forward those that were merged. */
- while( i-->0 ){
- rc = leavesReaderStep(v, lrs+i);
- if( rc!=SQLITE_OK ) goto err;
-
- /* Reorder by term, then by age. */
- leavesReaderReorder(lrs+i, MERGE_COUNT-i);
- }
- }
-
- for(i=0; i<MERGE_COUNT; i++){
- leavesReaderDestroy(&lrs[i]);
- }
-
- rc = leafWriterFinalize(v, &writer);
- leafWriterDestroy(&writer);
- if( rc!=SQLITE_OK ) return rc;
-
- /* Delete the merged segment data. */
- return segdir_delete(v, iLevel);
-
- err:
- for(i=0; i<MERGE_COUNT; i++){
- leavesReaderDestroy(&lrs[i]);
+static void fts3HashInsertElement(
+ Fts3Hash *pH, /* The complete hash table */
+ struct _fts3ht *pEntry, /* The entry into which pNew is inserted */
+ Fts3HashElem *pNew /* The element to be inserted */
+){
+ Fts3HashElem *pHead; /* First element already in pEntry */
+ pHead = pEntry->chain;
+ if( pHead ){
+ pNew->next = pHead;
+ pNew->prev = pHead->prev;
+ if( pHead->prev ){ pHead->prev->next = pNew; }
+ else { pH->first = pNew; }
+ pHead->prev = pNew;
+ }else{
+ pNew->next = pH->first;
+ if( pH->first ){ pH->first->prev = pNew; }
+ pNew->prev = 0;
+ pH->first = pNew;
}
- leafWriterDestroy(&writer);
- return rc;
-}
-
-/* Accumulate the union of *acc and *pData into *acc. */
-static void docListAccumulateUnion(DataBuffer *acc,
- const char *pData, int nData) {
- DataBuffer tmp = *acc;
- dataBufferInit(acc, tmp.nData+nData);
- docListUnion(tmp.pData, tmp.nData, pData, nData, acc);
- dataBufferDestroy(&tmp);
+ pEntry->count++;
+ pEntry->chain = pNew;
}
-/* TODO(shess) It might be interesting to explore different merge
-** strategies, here. For instance, since this is a sorted merge, we
-** could easily merge many doclists in parallel. With some
-** comprehension of the storage format, we could merge all of the
-** doclists within a leaf node directly from the leaf node's storage.
-** It may be worthwhile to merge smaller doclists before larger
-** doclists, since they can be traversed more quickly - but the
-** results may have less overlap, making them more expensive in a
-** different way.
-*/
-/* Scan pReader for pTerm/nTerm, and merge the term's doclist over
-** *out (any doclists with duplicate docids overwrite those in *out).
-** Internal function for loadSegmentLeaf().
+/* Resize the hash table so that it cantains "new_size" buckets.
+** "new_size" must be a power of 2. The hash table might fail
+** to resize if sqliteMalloc() fails.
+**
+** Return non-zero if a memory allocation error occurs.
*/
-static int loadSegmentLeavesInt(fulltext_vtab *v, LeavesReader *pReader,
- const char *pTerm, int nTerm, int isPrefix,
- DataBuffer *out){
- /* doclist data is accumulated into pBuffers similar to how one does
- ** increment in binary arithmetic. If index 0 is empty, the data is
- ** stored there. If there is data there, it is merged and the
- ** results carried into position 1, with further merge-and-carry
- ** until an empty position is found.
- */
- DataBuffer *pBuffers = NULL;
- int nBuffers = 0, nMaxBuffers = 0, rc;
-
- assert( nTerm>0 );
-
- for(rc=SQLITE_OK; rc==SQLITE_OK && !leavesReaderAtEnd(pReader);
- rc=leavesReaderStep(v, pReader)){
- /* TODO(shess) Really want leavesReaderTermCmp(), but that name is
- ** already taken to compare the terms of two LeavesReaders. Think
- ** on a better name. [Meanwhile, break encapsulation rather than
- ** use a confusing name.]
- */
- int c = leafReaderTermCmp(&pReader->leafReader, pTerm, nTerm, isPrefix);
- if( c>0 ) break; /* Past any possible matches. */
- if( c==0 ){
- const char *pData = leavesReaderData(pReader);
- int iBuffer, nData = leavesReaderDataBytes(pReader);
-
- /* Find the first empty buffer. */
- for(iBuffer=0; iBuffer<nBuffers; ++iBuffer){
- if( 0==pBuffers[iBuffer].nData ) break;
- }
-
- /* Out of buffers, add an empty one. */
- if( iBuffer==nBuffers ){
- if( nBuffers==nMaxBuffers ){
- DataBuffer *p;
- nMaxBuffers += 20;
-
- /* Manual realloc so we can handle NULL appropriately. */
- p = sqlite3_malloc(nMaxBuffers*sizeof(*pBuffers));
- if( p==NULL ){
- rc = SQLITE_NOMEM;
- break;
- }
-
- if( nBuffers>0 ){
- assert(pBuffers!=NULL);
- memcpy(p, pBuffers, nBuffers*sizeof(*pBuffers));
- sqlite3_free(pBuffers);
- }
- pBuffers = p;
- }
- dataBufferInit(&(pBuffers[nBuffers]), 0);
- nBuffers++;
- }
-
- /* At this point, must have an empty at iBuffer. */
- assert(iBuffer<nBuffers && pBuffers[iBuffer].nData==0);
-
- /* If empty was first buffer, no need for merge logic. */
- if( iBuffer==0 ){
- dataBufferReplace(&(pBuffers[0]), pData, nData);
- }else{
- /* pAcc is the empty buffer the merged data will end up in. */
- DataBuffer *pAcc = &(pBuffers[iBuffer]);
- DataBuffer *p = &(pBuffers[0]);
+static int fts3Rehash(Fts3Hash *pH, int new_size){
+ struct _fts3ht *new_ht; /* The new hash table */
+ Fts3HashElem *elem, *next_elem; /* For looping over existing elements */
+ int (*xHash)(const void*,int); /* The hash function */
- /* Handle position 0 specially to avoid need to prime pAcc
- ** with pData/nData.
- */
- dataBufferSwap(p, pAcc);
- docListAccumulateUnion(pAcc, pData, nData);
+ assert( (new_size & (new_size-1))==0 );
+ new_ht = (struct _fts3ht *)fts3HashMalloc( new_size*sizeof(struct _fts3ht) );
+ if( new_ht==0 ) return 1;
+ fts3HashFree(pH->ht);
+ pH->ht = new_ht;
+ pH->htsize = new_size;
+ xHash = ftsHashFunction(pH->keyClass);
+ for(elem=pH->first, pH->first=0; elem; elem = next_elem){
+ int h = (*xHash)(elem->pKey, elem->nKey) & (new_size-1);
+ next_elem = elem->next;
+ fts3HashInsertElement(pH, &new_ht[h], elem);
+ }
+ return 0;
+}
- /* Accumulate remaining doclists into pAcc. */
- for(++p; p<pAcc; ++p){
- docListAccumulateUnion(pAcc, p->pData, p->nData);
+/* This function (for internal use only) locates an element in an
+** hash table that matches the given key. The hash for this key has
+** already been computed and is passed as the 4th parameter.
+*/
+static Fts3HashElem *fts3FindElementByHash(
+ const Fts3Hash *pH, /* The pH to be searched */
+ const void *pKey, /* The key we are searching for */
+ int nKey,
+ int h /* The hash for this key. */
+){
+ Fts3HashElem *elem; /* Used to loop thru the element list */
+ int count; /* Number of elements left to test */
+ int (*xCompare)(const void*,int,const void*,int); /* comparison function */
- /* dataBufferReset() could allow a large doclist to blow up
- ** our memory requirements.
- */
- if( p->nCapacity<1024 ){
- dataBufferReset(p);
- }else{
- dataBufferDestroy(p);
- dataBufferInit(p, 0);
- }
- }
+ if( pH->ht ){
+ struct _fts3ht *pEntry = &pH->ht[h];
+ elem = pEntry->chain;
+ count = pEntry->count;
+ xCompare = ftsCompareFunction(pH->keyClass);
+ while( count-- && elem ){
+ if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){
+ return elem;
}
+ elem = elem->next;
}
}
+ return 0;
+}
- /* Union all the doclists together into *out. */
- /* TODO(shess) What if *out is big? Sigh. */
- if( rc==SQLITE_OK && nBuffers>0 ){
- int iBuffer;
- for(iBuffer=0; iBuffer<nBuffers; ++iBuffer){
- if( pBuffers[iBuffer].nData>0 ){
- if( out->nData==0 ){
- dataBufferSwap(out, &(pBuffers[iBuffer]));
- }else{
- docListAccumulateUnion(out, pBuffers[iBuffer].pData,
- pBuffers[iBuffer].nData);
- }
- }
- }
+/* Remove a single entry from the hash table given a pointer to that
+** element and a hash on the element's key.
+*/
+static void fts3RemoveElementByHash(
+ Fts3Hash *pH, /* The pH containing "elem" */
+ Fts3HashElem* elem, /* The element to be removed from the pH */
+ int h /* Hash value for the element */
+){
+ struct _fts3ht *pEntry;
+ if( elem->prev ){
+ elem->prev->next = elem->next;
+ }else{
+ pH->first = elem->next;
}
-
- while( nBuffers-- ){
- dataBufferDestroy(&(pBuffers[nBuffers]));
+ if( elem->next ){
+ elem->next->prev = elem->prev;
+ }
+ pEntry = &pH->ht[h];
+ if( pEntry->chain==elem ){
+ pEntry->chain = elem->next;
+ }
+ pEntry->count--;
+ if( pEntry->count<=0 ){
+ pEntry->chain = 0;
+ }
+ if( pH->copyKey && elem->pKey ){
+ fts3HashFree(elem->pKey);
+ }
+ fts3HashFree( elem );
+ pH->count--;
+ if( pH->count<=0 ){
+ assert( pH->first==0 );
+ assert( pH->count==0 );
+ fts3HashClear(pH);
}
- if( pBuffers!=NULL ) sqlite3_free(pBuffers);
-
- return rc;
-}
-
-/* Call loadSegmentLeavesInt() with pData/nData as input. */
-static int loadSegmentLeaf(fulltext_vtab *v, const char *pData, int nData,
- const char *pTerm, int nTerm, int isPrefix,
- DataBuffer *out){
- LeavesReader reader;
- int rc;
-
- assert( nData>1 );
- assert( *pData=='\0' );
- rc = leavesReaderInit(v, 0, 0, 0, pData, nData, &reader);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = loadSegmentLeavesInt(v, &reader, pTerm, nTerm, isPrefix, out);
- leavesReaderReset(&reader);
- leavesReaderDestroy(&reader);
- return rc;
}
-/* Call loadSegmentLeavesInt() with the leaf nodes from iStartLeaf to
-** iEndLeaf (inclusive) as input, and merge the resulting doclist into
-** out.
+/* Attempt to locate an element of the hash table pH with a key
+** that matches pKey,nKey. Return the data for this element if it is
+** found, or NULL if there is no match.
*/
-static int loadSegmentLeaves(fulltext_vtab *v,
- sqlite_int64 iStartLeaf, sqlite_int64 iEndLeaf,
- const char *pTerm, int nTerm, int isPrefix,
- DataBuffer *out){
- int rc;
- LeavesReader reader;
-
- assert( iStartLeaf<=iEndLeaf );
- rc = leavesReaderInit(v, 0, iStartLeaf, iEndLeaf, NULL, 0, &reader);
- if( rc!=SQLITE_OK ) return rc;
+SQLITE_PRIVATE void *sqlite3Fts3HashFind(const Fts3Hash *pH, const void *pKey, int nKey){
+ int h; /* A hash on key */
+ Fts3HashElem *elem; /* The element that matches key */
+ int (*xHash)(const void*,int); /* The hash function */
- rc = loadSegmentLeavesInt(v, &reader, pTerm, nTerm, isPrefix, out);
- leavesReaderReset(&reader);
- leavesReaderDestroy(&reader);
- return rc;
+ if( pH==0 || pH->ht==0 ) return 0;
+ xHash = ftsHashFunction(pH->keyClass);
+ assert( xHash!=0 );
+ h = (*xHash)(pKey,nKey);
+ assert( (pH->htsize & (pH->htsize-1))==0 );
+ elem = fts3FindElementByHash(pH,pKey,nKey, h & (pH->htsize-1));
+ return elem ? elem->data : 0;
}
-/* Taking pData/nData as an interior node, find the sequence of child
-** nodes which could include pTerm/nTerm/isPrefix. Note that the
-** interior node terms logically come between the blocks, so there is
-** one more blockid than there are terms (that block contains terms >=
-** the last interior-node term).
-*/
-/* TODO(shess) The calling code may already know that the end child is
-** not worth calculating, because the end may be in a later sibling
-** node. Consider whether breaking symmetry is worthwhile. I suspect
-** it is not worthwhile.
+/* Insert an element into the hash table pH. The key is pKey,nKey
+** and the data is "data".
+**
+** If no element exists with a matching key, then a new
+** element is created. A copy of the key is made if the copyKey
+** flag is set. NULL is returned.
+**
+** If another element already exists with the same key, then the
+** new data replaces the old data and the old data is returned.
+** The key is not copied in this instance. If a malloc fails, then
+** the new data is returned and the hash table is unchanged.
+**
+** If the "data" parameter to this function is NULL, then the
+** element corresponding to "key" is removed from the hash table.
*/
-static void getChildrenContaining(const char *pData, int nData,
- const char *pTerm, int nTerm, int isPrefix,
- sqlite_int64 *piStartChild,
- sqlite_int64 *piEndChild){
- InteriorReader reader;
-
- assert( nData>1 );
- assert( *pData!='\0' );
- interiorReaderInit(pData, nData, &reader);
+SQLITE_PRIVATE void *sqlite3Fts3HashInsert(
+ Fts3Hash *pH, /* The hash table to insert into */
+ const void *pKey, /* The key */
+ int nKey, /* Number of bytes in the key */
+ void *data /* The data */
+){
+ int hraw; /* Raw hash value of the key */
+ int h; /* the hash of the key modulo hash table size */
+ Fts3HashElem *elem; /* Used to loop thru the element list */
+ Fts3HashElem *new_elem; /* New element added to the pH */
+ int (*xHash)(const void*,int); /* The hash function */
- /* Scan for the first child which could contain pTerm/nTerm. */
- while( !interiorReaderAtEnd(&reader) ){
- if( interiorReaderTermCmp(&reader, pTerm, nTerm, 0)>0 ) break;
- interiorReaderStep(&reader);
+ assert( pH!=0 );
+ xHash = ftsHashFunction(pH->keyClass);
+ assert( xHash!=0 );
+ hraw = (*xHash)(pKey, nKey);
+ assert( (pH->htsize & (pH->htsize-1))==0 );
+ h = hraw & (pH->htsize-1);
+ elem = fts3FindElementByHash(pH,pKey,nKey,h);
+ if( elem ){
+ void *old_data = elem->data;
+ if( data==0 ){
+ fts3RemoveElementByHash(pH,elem,h);
+ }else{
+ elem->data = data;
+ }
+ return old_data;
}
- *piStartChild = interiorReaderCurrentBlockid(&reader);
-
- /* Keep scanning to find a term greater than our term, using prefix
- ** comparison if indicated. If isPrefix is false, this will be the
- ** same blockid as the starting block.
- */
- while( !interiorReaderAtEnd(&reader) ){
- if( interiorReaderTermCmp(&reader, pTerm, nTerm, isPrefix)>0 ) break;
- interiorReaderStep(&reader);
+ if( data==0 ) return 0;
+ if( (pH->htsize==0 && fts3Rehash(pH,8))
+ || (pH->count>=pH->htsize && fts3Rehash(pH, pH->htsize*2))
+ ){
+ pH->count = 0;
+ return data;
}
- *piEndChild = interiorReaderCurrentBlockid(&reader);
-
- interiorReaderDestroy(&reader);
-
- /* Children must ascend, and if !prefix, both must be the same. */
- assert( *piEndChild>=*piStartChild );
- assert( isPrefix || *piStartChild==*piEndChild );
+ assert( pH->htsize>0 );
+ new_elem = (Fts3HashElem*)fts3HashMalloc( sizeof(Fts3HashElem) );
+ if( new_elem==0 ) return data;
+ if( pH->copyKey && pKey!=0 ){
+ new_elem->pKey = fts3HashMalloc( nKey );
+ if( new_elem->pKey==0 ){
+ fts3HashFree(new_elem);
+ return data;
+ }
+ memcpy((void*)new_elem->pKey, pKey, nKey);
+ }else{
+ new_elem->pKey = (void*)pKey;
+ }
+ new_elem->nKey = nKey;
+ pH->count++;
+ assert( pH->htsize>0 );
+ assert( (pH->htsize & (pH->htsize-1))==0 );
+ h = hraw & (pH->htsize-1);
+ fts3HashInsertElement(pH, &pH->ht[h], new_elem);
+ new_elem->data = data;
+ return 0;
}
-/* Read block at iBlockid and pass it with other params to
-** getChildrenContaining().
-*/
-static int loadAndGetChildrenContaining(
- fulltext_vtab *v,
- sqlite_int64 iBlockid,
- const char *pTerm, int nTerm, int isPrefix,
- sqlite_int64 *piStartChild, sqlite_int64 *piEndChild
-){
- sqlite3_stmt *s = NULL;
- int rc;
-
- assert( iBlockid!=0 );
- assert( pTerm!=NULL );
- assert( nTerm!=0 ); /* TODO(shess) Why not allow this? */
- assert( piStartChild!=NULL );
- assert( piEndChild!=NULL );
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
- rc = sql_get_statement(v, BLOCK_SELECT_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
+/************** End of fts3_hash.c *******************************************/
+/************** Begin file fts3_porter.c *************************************/
+/*
+** 2006 September 30
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+*************************************************************************
+** Implementation of the full-text-search tokenizer that implements
+** a Porter stemmer.
+*/
- rc = sqlite3_bind_int64(s, 1, iBlockid);
- if( rc!=SQLITE_OK ) return rc;
+/*
+** The code in this file is only compiled if:
+**
+** * The FTS3 module is being built as an extension
+** (in which case SQLITE_CORE is not defined), or
+**
+** * The FTS3 module is being built into the core of
+** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
+*/
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
- rc = sqlite3_step(s);
- if( rc==SQLITE_DONE ) return SQLITE_ERROR;
- if( rc!=SQLITE_ROW ) return rc;
- getChildrenContaining(sqlite3_column_blob(s, 0), sqlite3_column_bytes(s, 0),
- pTerm, nTerm, isPrefix, piStartChild, piEndChild);
- /* We expect only one row. We must execute another sqlite3_step()
- * to complete the iteration; otherwise the table will remain
- * locked. */
- rc = sqlite3_step(s);
- if( rc==SQLITE_ROW ) return SQLITE_ERROR;
- if( rc!=SQLITE_DONE ) return rc;
- return SQLITE_OK;
-}
+/*
+** Class derived from sqlite3_tokenizer
+*/
+typedef struct porter_tokenizer {
+ sqlite3_tokenizer base; /* Base class */
+} porter_tokenizer;
-/* Traverse the tree represented by pData[nData] looking for
-** pTerm[nTerm], placing its doclist into *out. This is internal to
-** loadSegment() to make error-handling cleaner.
+/*
+** Class derived from sqlit3_tokenizer_cursor
*/
-static int loadSegmentInt(fulltext_vtab *v, const char *pData, int nData,
- sqlite_int64 iLeavesEnd,
- const char *pTerm, int nTerm, int isPrefix,
- DataBuffer *out){
- /* Special case where root is a leaf. */
- if( *pData=='\0' ){
- return loadSegmentLeaf(v, pData, nData, pTerm, nTerm, isPrefix, out);
- }else{
- int rc;
- sqlite_int64 iStartChild, iEndChild;
+typedef struct porter_tokenizer_cursor {
+ sqlite3_tokenizer_cursor base;
+ const char *zInput; /* input we are tokenizing */
+ int nInput; /* size of the input */
+ int iOffset; /* current position in zInput */
+ int iToken; /* index of next token to be returned */
+ char *zToken; /* storage for current token */
+ int nAllocated; /* space allocated to zToken buffer */
+} porter_tokenizer_cursor;
- /* Process pData as an interior node, then loop down the tree
- ** until we find the set of leaf nodes to scan for the term.
- */
- getChildrenContaining(pData, nData, pTerm, nTerm, isPrefix,
- &iStartChild, &iEndChild);
- while( iStartChild>iLeavesEnd ){
- sqlite_int64 iNextStart, iNextEnd;
- rc = loadAndGetChildrenContaining(v, iStartChild, pTerm, nTerm, isPrefix,
- &iNextStart, &iNextEnd);
- if( rc!=SQLITE_OK ) return rc;
- /* If we've branched, follow the end branch, too. */
- if( iStartChild!=iEndChild ){
- sqlite_int64 iDummy;
- rc = loadAndGetChildrenContaining(v, iEndChild, pTerm, nTerm, isPrefix,
- &iDummy, &iNextEnd);
- if( rc!=SQLITE_OK ) return rc;
- }
+/*
+** Create a new tokenizer instance.
+*/
+static int porterCreate(
+ int argc, const char * const *argv,
+ sqlite3_tokenizer **ppTokenizer
+){
+ porter_tokenizer *t;
- assert( iNextStart<=iNextEnd );
- iStartChild = iNextStart;
- iEndChild = iNextEnd;
- }
- assert( iStartChild<=iLeavesEnd );
- assert( iEndChild<=iLeavesEnd );
+ UNUSED_PARAMETER(argc);
+ UNUSED_PARAMETER(argv);
- /* Scan through the leaf segments for doclists. */
- return loadSegmentLeaves(v, iStartChild, iEndChild,
- pTerm, nTerm, isPrefix, out);
- }
+ t = (porter_tokenizer *) sqlite3_malloc(sizeof(*t));
+ if( t==NULL ) return SQLITE_NOMEM;
+ memset(t, 0, sizeof(*t));
+ *ppTokenizer = &t->base;
+ return SQLITE_OK;
}
-/* Call loadSegmentInt() to collect the doclist for pTerm/nTerm, then
-** merge its doclist over *out (any duplicate doclists read from the
-** segment rooted at pData will overwrite those in *out).
-*/
-/* TODO(shess) Consider changing this to determine the depth of the
-** leaves using either the first characters of interior nodes (when
-** ==1, we're one level above the leaves), or the first character of
-** the root (which will describe the height of the tree directly).
-** Either feels somewhat tricky to me.
-*/
-/* TODO(shess) The current merge is likely to be slow for large
-** doclists (though it should process from newest/smallest to
-** oldest/largest, so it may not be that bad). It might be useful to
-** modify things to allow for N-way merging. This could either be
-** within a segment, with pairwise merges across segments, or across
-** all segments at once.
+/*
+** Destroy a tokenizer
*/
-static int loadSegment(fulltext_vtab *v, const char *pData, int nData,
- sqlite_int64 iLeavesEnd,
- const char *pTerm, int nTerm, int isPrefix,
- DataBuffer *out){
- DataBuffer result;
- int rc;
-
- assert( nData>1 );
-
- /* This code should never be called with buffered updates. */
- assert( v->nPendingData<0 );
-
- dataBufferInit(&result, 0);
- rc = loadSegmentInt(v, pData, nData, iLeavesEnd,
- pTerm, nTerm, isPrefix, &result);
- if( rc==SQLITE_OK && result.nData>0 ){
- if( out->nData==0 ){
- DataBuffer tmp = *out;
- *out = result;
- result = tmp;
- }else{
- DataBuffer merged;
- DLReader readers[2];
-
- dlrInit(&readers[0], DL_DEFAULT, out->pData, out->nData);
- dlrInit(&readers[1], DL_DEFAULT, result.pData, result.nData);
- dataBufferInit(&merged, out->nData+result.nData);
- docListMerge(&merged, readers, 2);
- dataBufferDestroy(out);
- *out = merged;
- dlrDestroy(&readers[0]);
- dlrDestroy(&readers[1]);
- }
- }
- dataBufferDestroy(&result);
- return rc;
+static int porterDestroy(sqlite3_tokenizer *pTokenizer){
+ sqlite3_free(pTokenizer);
+ return SQLITE_OK;
}
-/* Scan the database and merge together the posting lists for the term
-** into *out.
+/*
+** Prepare to begin tokenizing a particular string. The input
+** string to be tokenized is zInput[0..nInput-1]. A cursor
+** used to incrementally tokenize this string is returned in
+** *ppCursor.
*/
-static int termSelect(
- fulltext_vtab *v,
- int iColumn,
- const char *pTerm, int nTerm, /* Term to query for */
- int isPrefix, /* True for a prefix search */
- DocListType iType,
- DataBuffer *out /* Write results here */
+static int porterOpen(
+ sqlite3_tokenizer *pTokenizer, /* The tokenizer */
+ const char *zInput, int nInput, /* String to be tokenized */
+ sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
){
- DataBuffer doclist;
- sqlite3_stmt *s;
- int rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
+ porter_tokenizer_cursor *c;
- /* This code should never be called with buffered updates. */
- assert( v->nPendingData<0 );
+ UNUSED_PARAMETER(pTokenizer);
- dataBufferInit(&doclist, 0);
- dataBufferInit(out, 0);
+ c = (porter_tokenizer_cursor *) sqlite3_malloc(sizeof(*c));
+ if( c==NULL ) return SQLITE_NOMEM;
- /* Traverse the segments from oldest to newest so that newer doclist
- ** elements for given docids overwrite older elements.
- */
- while( (rc = sqlite3_step(s))==SQLITE_ROW ){
- const char *pData = sqlite3_column_blob(s, 2);
- const int nData = sqlite3_column_bytes(s, 2);
- const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1);
- rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, isPrefix,
- &doclist);
- if( rc!=SQLITE_OK ) goto err;
- }
- if( rc==SQLITE_DONE ){
- if( doclist.nData!=0 ){
- /* TODO(shess) The old term_select_all() code applied the column
- ** restrict as we merged segments, leading to smaller buffers.
- ** This is probably worthwhile to bring back, once the new storage
- ** system is checked in.
- */
- if( iColumn==v->nColumn) iColumn = -1;
- docListTrim(DL_DEFAULT, doclist.pData, doclist.nData,
- iColumn, iType, out);
- }
- rc = SQLITE_OK;
+ c->zInput = zInput;
+ if( zInput==0 ){
+ c->nInput = 0;
+ }else if( nInput<0 ){
+ c->nInput = (int)strlen(zInput);
+ }else{
+ c->nInput = nInput;
}
+ c->iOffset = 0; /* start tokenizing at the beginning */
+ c->iToken = 0;
+ c->zToken = NULL; /* no space allocated, yet. */
+ c->nAllocated = 0;
- err:
- dataBufferDestroy(&doclist);
- return rc;
+ *ppCursor = &c->base;
+ return SQLITE_OK;
}
-/****************************************************************/
-/* Used to hold hashtable data for sorting. */
-typedef struct TermData {
- const char *pTerm;
- int nTerm;
- DLCollector *pCollector;
-} TermData;
-
-/* Orders TermData elements in strcmp fashion ( <0 for less-than, 0
-** for equal, >0 for greater-than).
+/*
+** Close a tokenization cursor previously opened by a call to
+** porterOpen() above.
*/
-static int termDataCmp(const void *av, const void *bv){
- const TermData *a = (const TermData *)av;
- const TermData *b = (const TermData *)bv;
- int n = a->nTerm<b->nTerm ? a->nTerm : b->nTerm;
- int c = memcmp(a->pTerm, b->pTerm, n);
- if( c!=0 ) return c;
- return a->nTerm-b->nTerm;
+static int porterClose(sqlite3_tokenizer_cursor *pCursor){
+ porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor;
+ sqlite3_free(c->zToken);
+ sqlite3_free(c);
+ return SQLITE_OK;
}
-
-/* Order pTerms data by term, then write a new level 0 segment using
-** LeafWriter.
+/*
+** Vowel or consonant
*/
-static int writeZeroSegment(fulltext_vtab *v, fts3Hash *pTerms){
- fts3HashElem *e;
- int idx, rc, i, n;
- TermData *pData;
- LeafWriter writer;
- DataBuffer dl;
-
- /* Determine the next index at level 0, merging as necessary. */
- rc = segdirNextIndex(v, 0, &idx);
- if( rc!=SQLITE_OK ) return rc;
-
- n = fts3HashCount(pTerms);
- pData = sqlite3_malloc(n*sizeof(TermData));
-
- for(i = 0, e = fts3HashFirst(pTerms); e; i++, e = fts3HashNext(e)){
- assert( i<n );
- pData[i].pTerm = fts3HashKey(e);
- pData[i].nTerm = fts3HashKeysize(e);
- pData[i].pCollector = fts3HashData(e);
- }
- assert( i==n );
-
- /* TODO(shess) Should we allow user-defined collation sequences,
- ** here? I think we only need that once we support prefix searches.
- */
- if( n>1 ) qsort(pData, n, sizeof(*pData), termDataCmp);
-
- /* TODO(shess) Refactor so that we can write directly to the segment
- ** DataBuffer, as happens for segment merges.
- */
- leafWriterInit(0, idx, &writer);
- dataBufferInit(&dl, 0);
- for(i=0; i<n; i++){
- dataBufferReset(&dl);
- dlcAddDoclist(pData[i].pCollector, &dl);
- rc = leafWriterStep(v, &writer,
- pData[i].pTerm, pData[i].nTerm, dl.pData, dl.nData);
- if( rc!=SQLITE_OK ) goto err;
- }
- rc = leafWriterFinalize(v, &writer);
+static const char cType[] = {
+ 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0,
+ 1, 1, 1, 2, 1
+};
- err:
- dataBufferDestroy(&dl);
- sqlite3_free(pData);
- leafWriterDestroy(&writer);
- return rc;
+/*
+** isConsonant() and isVowel() determine if their first character in
+** the string they point to is a consonant or a vowel, according
+** to Porter ruls.
+**
+** A consonate is any letter other than 'a', 'e', 'i', 'o', or 'u'.
+** 'Y' is a consonant unless it follows another consonant,
+** in which case it is a vowel.
+**
+** In these routine, the letters are in reverse order. So the 'y' rule
+** is that 'y' is a consonant unless it is followed by another
+** consonent.
+*/
+static int isVowel(const char*);
+static int isConsonant(const char *z){
+ int j;
+ char x = *z;
+ if( x==0 ) return 0;
+ assert( x>='a' && x<='z' );
+ j = cType[x-'a'];
+ if( j<2 ) return j;
+ return z[1]==0 || isVowel(z + 1);
}
-
-/* If pendingTerms has data, free it. */
-static int clearPendingTerms(fulltext_vtab *v){
- if( v->nPendingData>=0 ){
- fts3HashElem *e;
- for(e=fts3HashFirst(&v->pendingTerms); e; e=fts3HashNext(e)){
- dlcDelete(fts3HashData(e));
- }
- fts3HashClear(&v->pendingTerms);
- v->nPendingData = -1;
- }
- return SQLITE_OK;
+static int isVowel(const char *z){
+ int j;
+ char x = *z;
+ if( x==0 ) return 0;
+ assert( x>='a' && x<='z' );
+ j = cType[x-'a'];
+ if( j<2 ) return 1-j;
+ return isConsonant(z + 1);
}
-/* If pendingTerms has data, flush it to a level-zero segment, and
-** free it.
+/*
+** Let any sequence of one or more vowels be represented by V and let
+** C be sequence of one or more consonants. Then every word can be
+** represented as:
+**
+** [C] (VC){m} [V]
+**
+** In prose: A word is an optional consonant followed by zero or
+** vowel-consonant pairs followed by an optional vowel. "m" is the
+** number of vowel consonant pairs. This routine computes the value
+** of m for the first i bytes of a word.
+**
+** Return true if the m-value for z is 1 or more. In other words,
+** return true if z contains at least one vowel that is followed
+** by a consonant.
+**
+** In this routine z[] is in reverse order. So we are really looking
+** for an instance of of a consonant followed by a vowel.
*/
-static int flushPendingTerms(fulltext_vtab *v){
- if( v->nPendingData>=0 ){
- int rc = writeZeroSegment(v, &v->pendingTerms);
- if( rc==SQLITE_OK ) clearPendingTerms(v);
- return rc;
- }
- return SQLITE_OK;
+static int m_gt_0(const char *z){
+ while( isVowel(z) ){ z++; }
+ if( *z==0 ) return 0;
+ while( isConsonant(z) ){ z++; }
+ return *z!=0;
}
-/* If pendingTerms is "too big", or docid is out of order, flush it.
-** Regardless, be certain that pendingTerms is initialized for use.
+/* Like mgt0 above except we are looking for a value of m which is
+** exactly 1
*/
-static int initPendingTerms(fulltext_vtab *v, sqlite_int64 iDocid){
- /* TODO(shess) Explore whether partially flushing the buffer on
- ** forced-flush would provide better performance. I suspect that if
- ** we ordered the doclists by size and flushed the largest until the
- ** buffer was half empty, that would let the less frequent terms
- ** generate longer doclists.
- */
- if( iDocid<=v->iPrevDocid || v->nPendingData>kPendingThreshold ){
- int rc = flushPendingTerms(v);
- if( rc!=SQLITE_OK ) return rc;
- }
- if( v->nPendingData<0 ){
- fts3HashInit(&v->pendingTerms, FTS3_HASH_STRING, 1);
- v->nPendingData = 0;
- }
- v->iPrevDocid = iDocid;
- return SQLITE_OK;
+static int m_eq_1(const char *z){
+ while( isVowel(z) ){ z++; }
+ if( *z==0 ) return 0;
+ while( isConsonant(z) ){ z++; }
+ if( *z==0 ) return 0;
+ while( isVowel(z) ){ z++; }
+ if( *z==0 ) return 1;
+ while( isConsonant(z) ){ z++; }
+ return *z==0;
}
-/* This function implements the xUpdate callback; it is the top-level entry
- * point for inserting, deleting or updating a row in a full-text table. */
-static int fulltextUpdate(sqlite3_vtab *pVtab, int nArg, sqlite3_value **ppArg,
- sqlite_int64 *pRowid){
- fulltext_vtab *v = (fulltext_vtab *) pVtab;
- int rc;
-
- FTSTRACE(("FTS3 Update %p\n", pVtab));
-
- if( nArg<2 ){
- rc = index_delete(v, sqlite3_value_int64(ppArg[0]));
- if( rc==SQLITE_OK ){
- /* If we just deleted the last row in the table, clear out the
- ** index data.
- */
- rc = content_exists(v);
- if( rc==SQLITE_ROW ){
- rc = SQLITE_OK;
- }else if( rc==SQLITE_DONE ){
- /* Clear the pending terms so we don't flush a useless level-0
- ** segment when the transaction closes.
- */
- rc = clearPendingTerms(v);
- if( rc==SQLITE_OK ){
- rc = segdir_delete_all(v);
- }
- }
- }
- } else if( sqlite3_value_type(ppArg[0]) != SQLITE_NULL ){
- /* An update:
- * ppArg[0] = old rowid
- * ppArg[1] = new rowid
- * ppArg[2..2+v->nColumn-1] = values
- * ppArg[2+v->nColumn] = value for magic column (we ignore this)
- * ppArg[2+v->nColumn+1] = value for docid
- */
- sqlite_int64 rowid = sqlite3_value_int64(ppArg[0]);
- if( sqlite3_value_type(ppArg[1]) != SQLITE_INTEGER ||
- sqlite3_value_int64(ppArg[1]) != rowid ){
- rc = SQLITE_ERROR; /* we don't allow changing the rowid */
- }else if( sqlite3_value_type(ppArg[2+v->nColumn+1]) != SQLITE_INTEGER ||
- sqlite3_value_int64(ppArg[2+v->nColumn+1]) != rowid ){
- rc = SQLITE_ERROR; /* we don't allow changing the docid */
- }else{
- assert( nArg==2+v->nColumn+2);
- rc = index_update(v, rowid, &ppArg[2]);
- }
- } else {
- /* An insert:
- * ppArg[1] = requested rowid
- * ppArg[2..2+v->nColumn-1] = values
- * ppArg[2+v->nColumn] = value for magic column (we ignore this)
- * ppArg[2+v->nColumn+1] = value for docid
- */
- sqlite3_value *pRequestDocid = ppArg[2+v->nColumn+1];
- assert( nArg==2+v->nColumn+2);
- if( SQLITE_NULL != sqlite3_value_type(pRequestDocid) &&
- SQLITE_NULL != sqlite3_value_type(ppArg[1]) ){
- /* TODO(shess) Consider allowing this to work if the values are
- ** identical. I'm inclined to discourage that usage, though,
- ** given that both rowid and docid are special columns. Better
- ** would be to define one or the other as the default winner,
- ** but should it be fts3-centric (docid) or SQLite-centric
- ** (rowid)?
- */
- rc = SQLITE_ERROR;
- }else{
- if( SQLITE_NULL == sqlite3_value_type(pRequestDocid) ){
- pRequestDocid = ppArg[1];
- }
- rc = index_insert(v, pRequestDocid, &ppArg[2], pRowid);
- }
- }
-
- return rc;
+/* Like mgt0 above except we are looking for a value of m>1 instead
+** or m>0
+*/
+static int m_gt_1(const char *z){
+ while( isVowel(z) ){ z++; }
+ if( *z==0 ) return 0;
+ while( isConsonant(z) ){ z++; }
+ if( *z==0 ) return 0;
+ while( isVowel(z) ){ z++; }
+ if( *z==0 ) return 0;
+ while( isConsonant(z) ){ z++; }
+ return *z!=0;
}
-static int fulltextSync(sqlite3_vtab *pVtab){
- FTSTRACE(("FTS3 xSync()\n"));
- return flushPendingTerms((fulltext_vtab *)pVtab);
+/*
+** Return TRUE if there is a vowel anywhere within z[0..n-1]
+*/
+static int hasVowel(const char *z){
+ while( isConsonant(z) ){ z++; }
+ return *z!=0;
}
-static int fulltextBegin(sqlite3_vtab *pVtab){
- fulltext_vtab *v = (fulltext_vtab *) pVtab;
- FTSTRACE(("FTS3 xBegin()\n"));
-
- /* Any buffered updates should have been cleared by the previous
- ** transaction.
- */
- assert( v->nPendingData<0 );
- return clearPendingTerms(v);
+/*
+** Return TRUE if the word ends in a double consonant.
+**
+** The text is reversed here. So we are really looking at
+** the first two characters of z[].
+*/
+static int doubleConsonant(const char *z){
+ return isConsonant(z) && z[0]==z[1];
}
-static int fulltextCommit(sqlite3_vtab *pVtab){
- fulltext_vtab *v = (fulltext_vtab *) pVtab;
- FTSTRACE(("FTS3 xCommit()\n"));
-
- /* Buffered updates should have been cleared by fulltextSync(). */
- assert( v->nPendingData<0 );
- return clearPendingTerms(v);
+/*
+** Return TRUE if the word ends with three letters which
+** are consonant-vowel-consonent and where the final consonant
+** is not 'w', 'x', or 'y'.
+**
+** The word is reversed here. So we are really checking the
+** first three letters and the first one cannot be in [wxy].
+*/
+static int star_oh(const char *z){
+ return
+ isConsonant(z) &&
+ z[0]!='w' && z[0]!='x' && z[0]!='y' &&
+ isVowel(z+1) &&
+ isConsonant(z+2);
}
-static int fulltextRollback(sqlite3_vtab *pVtab){
- FTSTRACE(("FTS3 xRollback()\n"));
- return clearPendingTerms((fulltext_vtab *)pVtab);
+/*
+** If the word ends with zFrom and xCond() is true for the stem
+** of the word that preceeds the zFrom ending, then change the
+** ending to zTo.
+**
+** The input word *pz and zFrom are both in reverse order. zTo
+** is in normal order.
+**
+** Return TRUE if zFrom matches. Return FALSE if zFrom does not
+** match. Not that TRUE is returned even if xCond() fails and
+** no substitution occurs.
+*/
+static int stem(
+ char **pz, /* The word being stemmed (Reversed) */
+ const char *zFrom, /* If the ending matches this... (Reversed) */
+ const char *zTo, /* ... change the ending to this (not reversed) */
+ int (*xCond)(const char*) /* Condition that must be true */
+){
+ char *z = *pz;
+ while( *zFrom && *zFrom==*z ){ z++; zFrom++; }
+ if( *zFrom!=0 ) return 0;
+ if( xCond && !xCond(z) ) return 1;
+ while( *zTo ){
+ *(--z) = *(zTo++);
+ }
+ *pz = z;
+ return 1;
}
/*
-** Implementation of the snippet() function for FTS3
+** This is the fallback stemmer used when the porter stemmer is
+** inappropriate. The input word is copied into the output with
+** US-ASCII case folding. If the input word is too long (more
+** than 20 bytes if it contains no digits or more than 6 bytes if
+** it contains digits) then word is truncated to 20 or 6 bytes
+** by taking 10 or 3 bytes from the beginning and end.
*/
-static void snippetFunc(
- sqlite3_context *pContext,
- int argc,
- sqlite3_value **argv
-){
- fulltext_cursor *pCursor;
- if( argc<1 ) return;
- if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ||
- sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
- sqlite3_result_error(pContext, "illegal first argument to html_snippet",-1);
- }else{
- const char *zStart = "<b>";
- const char *zEnd = "</b>";
- const char *zEllipsis = "<b>...</b>";
- memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
- if( argc>=2 ){
- zStart = (const char*)sqlite3_value_text(argv[1]);
- if( argc>=3 ){
- zEnd = (const char*)sqlite3_value_text(argv[2]);
- if( argc>=4 ){
- zEllipsis = (const char*)sqlite3_value_text(argv[3]);
- }
- }
+static void copy_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
+ int i, mx, j;
+ int hasDigit = 0;
+ for(i=0; i<nIn; i++){
+ char c = zIn[i];
+ if( c>='A' && c<='Z' ){
+ zOut[i] = c - 'A' + 'a';
+ }else{
+ if( c>='0' && c<='9' ) hasDigit = 1;
+ zOut[i] = c;
}
- snippetAllOffsets(pCursor);
- snippetText(pCursor, zStart, zEnd, zEllipsis);
- sqlite3_result_text(pContext, pCursor->snippet.zSnippet,
- pCursor->snippet.nSnippet, SQLITE_STATIC);
}
-}
-
-/*
-** Implementation of the offsets() function for FTS3
-*/
-static void snippetOffsetsFunc(
- sqlite3_context *pContext,
- int argc,
- sqlite3_value **argv
-){
- fulltext_cursor *pCursor;
- if( argc<1 ) return;
- if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ||
- sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
- sqlite3_result_error(pContext, "illegal first argument to offsets",-1);
- }else{
- memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
- snippetAllOffsets(pCursor);
- snippetOffsetText(&pCursor->snippet);
- sqlite3_result_text(pContext,
- pCursor->snippet.zOffset, pCursor->snippet.nOffset,
- SQLITE_STATIC);
+ mx = hasDigit ? 3 : 10;
+ if( nIn>mx*2 ){
+ for(j=mx, i=nIn-mx; i<nIn; i++, j++){
+ zOut[j] = zOut[i];
+ }
+ i = j;
}
+ zOut[i] = 0;
+ *pnOut = i;
}
-/* OptLeavesReader is nearly identical to LeavesReader, except that
-** where LeavesReader is geared towards the merging of complete
-** segment levels (with exactly MERGE_COUNT segments), OptLeavesReader
-** is geared towards implementation of the optimize() function, and
-** can merge all segments simultaneously. This version may be
-** somewhat less efficient than LeavesReader because it merges into an
-** accumulator rather than doing an N-way merge, but since segment
-** size grows exponentially (so segment count logrithmically) this is
-** probably not an immediate problem.
-*/
-/* TODO(shess): Prove that assertion, or extend the merge code to
-** merge tree fashion (like the prefix-searching code does).
-*/
-/* TODO(shess): OptLeavesReader and LeavesReader could probably be
-** merged with little or no loss of performance for LeavesReader. The
-** merged code would need to handle >MERGE_COUNT segments, and would
-** also need to be able to optionally optimize away deletes.
-*/
-typedef struct OptLeavesReader {
- /* Segment number, to order readers by age. */
- int segment;
- LeavesReader reader;
-} OptLeavesReader;
-static int optLeavesReaderAtEnd(OptLeavesReader *pReader){
- return leavesReaderAtEnd(&pReader->reader);
-}
-static int optLeavesReaderTermBytes(OptLeavesReader *pReader){
- return leavesReaderTermBytes(&pReader->reader);
-}
-static const char *optLeavesReaderData(OptLeavesReader *pReader){
- return leavesReaderData(&pReader->reader);
-}
-static int optLeavesReaderDataBytes(OptLeavesReader *pReader){
- return leavesReaderDataBytes(&pReader->reader);
-}
-static const char *optLeavesReaderTerm(OptLeavesReader *pReader){
- return leavesReaderTerm(&pReader->reader);
-}
-static int optLeavesReaderStep(fulltext_vtab *v, OptLeavesReader *pReader){
- return leavesReaderStep(v, &pReader->reader);
-}
-static int optLeavesReaderTermCmp(OptLeavesReader *lr1, OptLeavesReader *lr2){
- return leavesReaderTermCmp(&lr1->reader, &lr2->reader);
-}
-/* Order by term ascending, segment ascending (oldest to newest), with
-** exhausted readers to the end.
-*/
-static int optLeavesReaderCmp(OptLeavesReader *lr1, OptLeavesReader *lr2){
- int c = optLeavesReaderTermCmp(lr1, lr2);
- if( c!=0 ) return c;
- return lr1->segment-lr2->segment;
-}
-/* Bubble pLr[0] to appropriate place in pLr[1..nLr-1]. Assumes that
-** pLr[1..nLr-1] is already sorted.
+/*
+** Stem the input word zIn[0..nIn-1]. Store the output in zOut.
+** zOut is at least big enough to hold nIn bytes. Write the actual
+** size of the output word (exclusive of the '\0' terminator) into *pnOut.
+**
+** Any upper-case characters in the US-ASCII character set ([A-Z])
+** are converted to lower case. Upper-case UTF characters are
+** unchanged.
+**
+** Words that are longer than about 20 bytes are stemmed by retaining
+** a few bytes from the beginning and the end of the word. If the
+** word contains digits, 3 bytes are taken from the beginning and
+** 3 bytes from the end. For long words without digits, 10 bytes
+** are taken from each end. US-ASCII case folding still applies.
+**
+** If the input word contains not digits but does characters not
+** in [a-zA-Z] then no stemming is attempted and this routine just
+** copies the input into the input into the output with US-ASCII
+** case folding.
+**
+** Stemming never increases the length of the word. So there is
+** no chance of overflowing the zOut buffer.
*/
-static void optLeavesReaderReorder(OptLeavesReader *pLr, int nLr){
- while( nLr>1 && optLeavesReaderCmp(pLr, pLr+1)>0 ){
- OptLeavesReader tmp = pLr[0];
- pLr[0] = pLr[1];
- pLr[1] = tmp;
- nLr--;
- pLr++;
+static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
+ int i, j;
+ char zReverse[28];
+ char *z, *z2;
+ if( nIn<3 || nIn>=sizeof(zReverse)-7 ){
+ /* The word is too big or too small for the porter stemmer.
+ ** Fallback to the copy stemmer */
+ copy_stemmer(zIn, nIn, zOut, pnOut);
+ return;
}
-}
-
-/* optimize() helper function. Put the readers in order and iterate
-** through them, merging doclists for matching terms into pWriter.
-** Returns SQLITE_OK on success, or the SQLite error code which
-** prevented success.
-*/
-static int optimizeInternal(fulltext_vtab *v,
- OptLeavesReader *readers, int nReaders,
- LeafWriter *pWriter){
- int i, rc = SQLITE_OK;
- DataBuffer doclist, merged, tmp;
-
- /* Order the readers. */
- i = nReaders;
- while( i-- > 0 ){
- optLeavesReaderReorder(&readers[i], nReaders-i);
+ for(i=0, j=sizeof(zReverse)-6; i<nIn; i++, j--){
+ char c = zIn[i];
+ if( c>='A' && c<='Z' ){
+ zReverse[j] = c + 'a' - 'A';
+ }else if( c>='a' && c<='z' ){
+ zReverse[j] = c;
+ }else{
+ /* The use of a character not in [a-zA-Z] means that we fallback
+ ** to the copy stemmer */
+ copy_stemmer(zIn, nIn, zOut, pnOut);
+ return;
+ }
}
+ memset(&zReverse[sizeof(zReverse)-5], 0, 5);
+ z = &zReverse[j+1];
- dataBufferInit(&doclist, LEAF_MAX);
- dataBufferInit(&merged, LEAF_MAX);
-
- /* Exhausted readers bubble to the end, so when the first reader is
- ** at eof, all are at eof.
- */
- while( !optLeavesReaderAtEnd(&readers[0]) ){
- /* Figure out how many readers share the next term. */
- for(i=1; i<nReaders && !optLeavesReaderAtEnd(&readers[i]); i++){
- if( 0!=optLeavesReaderTermCmp(&readers[0], &readers[i]) ) break;
+ /* Step 1a */
+ if( z[0]=='s' ){
+ if(
+ !stem(&z, "sess", "ss", 0) &&
+ !stem(&z, "sei", "i", 0) &&
+ !stem(&z, "ss", "ss", 0)
+ ){
+ z++;
}
+ }
- /* Special-case for no merge. */
- if( i==1 ){
- /* Trim deletions from the doclist. */
- dataBufferReset(&merged);
- docListTrim(DL_DEFAULT,
- optLeavesReaderData(&readers[0]),
- optLeavesReaderDataBytes(&readers[0]),
- -1, DL_DEFAULT, &merged);
- }else{
- DLReader dlReaders[MERGE_COUNT];
- int iReader, nReaders;
-
- /* Prime the pipeline with the first reader's doclist. After
- ** one pass index 0 will reference the accumulated doclist.
- */
- dlrInit(&dlReaders[0], DL_DEFAULT,
- optLeavesReaderData(&readers[0]),
- optLeavesReaderDataBytes(&readers[0]));
- iReader = 1;
-
- assert( iReader<i ); /* Must execute the loop at least once. */
- while( iReader<i ){
- /* Merge 16 inputs per pass. */
- for( nReaders=1; iReader<i && nReaders<MERGE_COUNT;
- iReader++, nReaders++ ){
- dlrInit(&dlReaders[nReaders], DL_DEFAULT,
- optLeavesReaderData(&readers[iReader]),
- optLeavesReaderDataBytes(&readers[iReader]));
- }
-
- /* Merge doclists and swap result into accumulator. */
- dataBufferReset(&merged);
- docListMerge(&merged, dlReaders, nReaders);
- tmp = merged;
- merged = doclist;
- doclist = tmp;
+ /* Step 1b */
+ z2 = z;
+ if( stem(&z, "dee", "ee", m_gt_0) ){
+ /* Do nothing. The work was all in the test */
+ }else if(
+ (stem(&z, "gni", "", hasVowel) || stem(&z, "de", "", hasVowel))
+ && z!=z2
+ ){
+ if( stem(&z, "ta", "ate", 0) ||
+ stem(&z, "lb", "ble", 0) ||
+ stem(&z, "zi", "ize", 0) ){
+ /* Do nothing. The work was all in the test */
+ }else if( doubleConsonant(z) && (*z!='l' && *z!='s' && *z!='z') ){
+ z++;
+ }else if( m_eq_1(z) && star_oh(z) ){
+ *(--z) = 'e';
+ }
+ }
- while( nReaders-- > 0 ){
- dlrDestroy(&dlReaders[nReaders]);
- }
+ /* Step 1c */
+ if( z[0]=='y' && hasVowel(z+1) ){
+ z[0] = 'i';
+ }
- /* Accumulated doclist to reader 0 for next pass. */
- dlrInit(&dlReaders[0], DL_DEFAULT, doclist.pData, doclist.nData);
- }
+ /* Step 2 */
+ switch( z[1] ){
+ case 'a':
+ stem(&z, "lanoita", "ate", m_gt_0) ||
+ stem(&z, "lanoit", "tion", m_gt_0);
+ break;
+ case 'c':
+ stem(&z, "icne", "ence", m_gt_0) ||
+ stem(&z, "icna", "ance", m_gt_0);
+ break;
+ case 'e':
+ stem(&z, "rezi", "ize", m_gt_0);
+ break;
+ case 'g':
+ stem(&z, "igol", "log", m_gt_0);
+ break;
+ case 'l':
+ stem(&z, "ilb", "ble", m_gt_0) ||
+ stem(&z, "illa", "al", m_gt_0) ||
+ stem(&z, "iltne", "ent", m_gt_0) ||
+ stem(&z, "ile", "e", m_gt_0) ||
+ stem(&z, "ilsuo", "ous", m_gt_0);
+ break;
+ case 'o':
+ stem(&z, "noitazi", "ize", m_gt_0) ||
+ stem(&z, "noita", "ate", m_gt_0) ||
+ stem(&z, "rota", "ate", m_gt_0);
+ break;
+ case 's':
+ stem(&z, "msila", "al", m_gt_0) ||
+ stem(&z, "ssenevi", "ive", m_gt_0) ||
+ stem(&z, "ssenluf", "ful", m_gt_0) ||
+ stem(&z, "ssensuo", "ous", m_gt_0);
+ break;
+ case 't':
+ stem(&z, "itila", "al", m_gt_0) ||
+ stem(&z, "itivi", "ive", m_gt_0) ||
+ stem(&z, "itilib", "ble", m_gt_0);
+ break;
+ }
- /* Destroy reader that was left in the pipeline. */
- dlrDestroy(&dlReaders[0]);
+ /* Step 3 */
+ switch( z[0] ){
+ case 'e':
+ stem(&z, "etaci", "ic", m_gt_0) ||
+ stem(&z, "evita", "", m_gt_0) ||
+ stem(&z, "ezila", "al", m_gt_0);
+ break;
+ case 'i':
+ stem(&z, "itici", "ic", m_gt_0);
+ break;
+ case 'l':
+ stem(&z, "laci", "ic", m_gt_0) ||
+ stem(&z, "luf", "", m_gt_0);
+ break;
+ case 's':
+ stem(&z, "ssen", "", m_gt_0);
+ break;
+ }
- /* Trim deletions from the doclist. */
- dataBufferReset(&merged);
- docListTrim(DL_DEFAULT, doclist.pData, doclist.nData,
- -1, DL_DEFAULT, &merged);
- }
+ /* Step 4 */
+ switch( z[1] ){
+ case 'a':
+ if( z[0]=='l' && m_gt_1(z+2) ){
+ z += 2;
+ }
+ break;
+ case 'c':
+ if( z[0]=='e' && z[2]=='n' && (z[3]=='a' || z[3]=='e') && m_gt_1(z+4) ){
+ z += 4;
+ }
+ break;
+ case 'e':
+ if( z[0]=='r' && m_gt_1(z+2) ){
+ z += 2;
+ }
+ break;
+ case 'i':
+ if( z[0]=='c' && m_gt_1(z+2) ){
+ z += 2;
+ }
+ break;
+ case 'l':
+ if( z[0]=='e' && z[2]=='b' && (z[3]=='a' || z[3]=='i') && m_gt_1(z+4) ){
+ z += 4;
+ }
+ break;
+ case 'n':
+ if( z[0]=='t' ){
+ if( z[2]=='a' ){
+ if( m_gt_1(z+3) ){
+ z += 3;
+ }
+ }else if( z[2]=='e' ){
+ stem(&z, "tneme", "", m_gt_1) ||
+ stem(&z, "tnem", "", m_gt_1) ||
+ stem(&z, "tne", "", m_gt_1);
+ }
+ }
+ break;
+ case 'o':
+ if( z[0]=='u' ){
+ if( m_gt_1(z+2) ){
+ z += 2;
+ }
+ }else if( z[3]=='s' || z[3]=='t' ){
+ stem(&z, "noi", "", m_gt_1);
+ }
+ break;
+ case 's':
+ if( z[0]=='m' && z[2]=='i' && m_gt_1(z+3) ){
+ z += 3;
+ }
+ break;
+ case 't':
+ stem(&z, "eta", "", m_gt_1) ||
+ stem(&z, "iti", "", m_gt_1);
+ break;
+ case 'u':
+ if( z[0]=='s' && z[2]=='o' && m_gt_1(z+3) ){
+ z += 3;
+ }
+ break;
+ case 'v':
+ case 'z':
+ if( z[0]=='e' && z[2]=='i' && m_gt_1(z+3) ){
+ z += 3;
+ }
+ break;
+ }
- /* Only pass doclists with hits (skip if all hits deleted). */
- if( merged.nData>0 ){
- rc = leafWriterStep(v, pWriter,
- optLeavesReaderTerm(&readers[0]),
- optLeavesReaderTermBytes(&readers[0]),
- merged.pData, merged.nData);
- if( rc!=SQLITE_OK ) goto err;
+ /* Step 5a */
+ if( z[0]=='e' ){
+ if( m_gt_1(z+1) ){
+ z++;
+ }else if( m_eq_1(z+1) && !star_oh(z+1) ){
+ z++;
}
+ }
- /* Step merged readers to next term and reorder. */
- while( i-- > 0 ){
- rc = optLeavesReaderStep(v, &readers[i]);
- if( rc!=SQLITE_OK ) goto err;
-
- optLeavesReaderReorder(&readers[i], nReaders-i);
- }
+ /* Step 5b */
+ if( m_gt_1(z) && z[0]=='l' && z[1]=='l' ){
+ z++;
}
- err:
- dataBufferDestroy(&doclist);
- dataBufferDestroy(&merged);
- return rc;
+ /* z[] is now the stemmed word in reverse order. Flip it back
+ ** around into forward order and return.
+ */
+ *pnOut = i = (int)strlen(z);
+ zOut[i] = 0;
+ while( *z ){
+ zOut[--i] = *(z++);
+ }
}
-/* Implement optimize() function for FTS3. optimize(t) merges all
-** segments in the fts index into a single segment. 't' is the magic
-** table-named column.
+/*
+** Characters that can be part of a token. We assume any character
+** whose value is greater than 0x80 (any UTF character) can be
+** part of a token. In other words, delimiters all must have
+** values of 0x7f or lower.
*/
-static void optimizeFunc(sqlite3_context *pContext,
- int argc, sqlite3_value **argv){
- fulltext_cursor *pCursor;
- if( argc>1 ){
- sqlite3_result_error(pContext, "excess arguments to optimize()",-1);
- }else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ||
- sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
- sqlite3_result_error(pContext, "illegal first argument to optimize",-1);
- }else{
- fulltext_vtab *v;
- int i, rc, iMaxLevel;
- OptLeavesReader *readers;
- int nReaders;
- LeafWriter writer;
- sqlite3_stmt *s;
-
- memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
- v = cursor_vtab(pCursor);
-
- /* Flush any buffered updates before optimizing. */
- rc = flushPendingTerms(v);
- if( rc!=SQLITE_OK ) goto err;
-
- rc = segdir_count(v, &nReaders, &iMaxLevel);
- if( rc!=SQLITE_OK ) goto err;
- if( nReaders==0 || nReaders==1 ){
- sqlite3_result_text(pContext, "Index already optimal", -1,
- SQLITE_STATIC);
- return;
- }
-
- rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s);
- if( rc!=SQLITE_OK ) goto err;
-
- readers = sqlite3_malloc(nReaders*sizeof(readers[0]));
- if( readers==NULL ) goto err;
-
- /* Note that there will already be a segment at this position
- ** until we call segdir_delete() on iMaxLevel.
- */
- leafWriterInit(iMaxLevel, 0, &writer);
-
- i = 0;
- while( (rc = sqlite3_step(s))==SQLITE_ROW ){
- sqlite_int64 iStart = sqlite3_column_int64(s, 0);
- sqlite_int64 iEnd = sqlite3_column_int64(s, 1);
- const char *pRootData = sqlite3_column_blob(s, 2);
- int nRootData = sqlite3_column_bytes(s, 2);
+static const char porterIdChar[] = {
+/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
+ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
+ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
+};
+#define isDelim(C) (((ch=C)&0x80)==0 && (ch<0x30 || !porterIdChar[ch-0x30]))
- assert( i<nReaders );
- rc = leavesReaderInit(v, -1, iStart, iEnd, pRootData, nRootData,
- &readers[i].reader);
- if( rc!=SQLITE_OK ) break;
+/*
+** Extract the next token from a tokenization cursor. The cursor must
+** have been opened by a prior call to porterOpen().
+*/
+static int porterNext(
+ sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by porterOpen */
+ const char **pzToken, /* OUT: *pzToken is the token text */
+ int *pnBytes, /* OUT: Number of bytes in token */
+ int *piStartOffset, /* OUT: Starting offset of token */
+ int *piEndOffset, /* OUT: Ending offset of token */
+ int *piPosition /* OUT: Position integer of token */
+){
+ porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor;
+ const char *z = c->zInput;
- readers[i].segment = i;
- i++;
- }
+ while( c->iOffset<c->nInput ){
+ int iStartOffset, ch;
- /* If we managed to successfully read them all, optimize them. */
- if( rc==SQLITE_DONE ){
- assert( i==nReaders );
- rc = optimizeInternal(v, readers, nReaders, &writer);
+ /* Scan past delimiter characters */
+ while( c->iOffset<c->nInput && isDelim(z[c->iOffset]) ){
+ c->iOffset++;
}
- while( i-- > 0 ){
- leavesReaderDestroy(&readers[i].reader);
+ /* Count non-delimiter characters. */
+ iStartOffset = c->iOffset;
+ while( c->iOffset<c->nInput && !isDelim(z[c->iOffset]) ){
+ c->iOffset++;
}
- sqlite3_free(readers);
- /* If we've successfully gotten to here, delete the old segments
- ** and flush the interior structure of the new segment.
- */
- if( rc==SQLITE_OK ){
- for( i=0; i<=iMaxLevel; i++ ){
- rc = segdir_delete(v, i);
- if( rc!=SQLITE_OK ) break;
+ if( c->iOffset>iStartOffset ){
+ int n = c->iOffset-iStartOffset;
+ if( n>c->nAllocated ){
+ c->nAllocated = n+20;
+ c->zToken = sqlite3_realloc(c->zToken, c->nAllocated);
+ if( c->zToken==NULL ) return SQLITE_NOMEM;
}
-
- if( rc==SQLITE_OK ) rc = leafWriterFinalize(v, &writer);
- }
-
- leafWriterDestroy(&writer);
-
- if( rc!=SQLITE_OK ) goto err;
-
- sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC);
- return;
-
- /* TODO(shess): Error-handling needs to be improved along the
- ** lines of the dump_ functions.
- */
- err:
- {
- char buf[512];
- sqlite3_snprintf(sizeof(buf), buf, "Error in optimize: %s",
- sqlite3_errmsg(sqlite3_context_db_handle(pContext)));
- sqlite3_result_error(pContext, buf, -1);
+ porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes);
+ *pzToken = c->zToken;
+ *piStartOffset = iStartOffset;
+ *piEndOffset = c->iOffset;
+ *piPosition = c->iToken++;
+ return SQLITE_OK;
}
}
+ return SQLITE_DONE;
}
-#ifdef SQLITE_TEST
-/* Generate an error of the form "<prefix>: <msg>". If msg is NULL,
-** pull the error from the context's db handle.
-*/
-static void generateError(sqlite3_context *pContext,
- const char *prefix, const char *msg){
- char buf[512];
- if( msg==NULL ) msg = sqlite3_errmsg(sqlite3_context_db_handle(pContext));
- sqlite3_snprintf(sizeof(buf), buf, "%s: %s", prefix, msg);
- sqlite3_result_error(pContext, buf, -1);
-}
-
-/* Helper function to collect the set of terms in the segment into
-** pTerms. The segment is defined by the leaf nodes between
-** iStartBlockid and iEndBlockid, inclusive, or by the contents of
-** pRootData if iStartBlockid is 0 (in which case the entire segment
-** fit in a leaf).
-*/
-static int collectSegmentTerms(fulltext_vtab *v, sqlite3_stmt *s,
- fts3Hash *pTerms){
- const sqlite_int64 iStartBlockid = sqlite3_column_int64(s, 0);
- const sqlite_int64 iEndBlockid = sqlite3_column_int64(s, 1);
- const char *pRootData = sqlite3_column_blob(s, 2);
- const int nRootData = sqlite3_column_bytes(s, 2);
- LeavesReader reader;
- int rc = leavesReaderInit(v, 0, iStartBlockid, iEndBlockid,
- pRootData, nRootData, &reader);
- if( rc!=SQLITE_OK ) return rc;
-
- while( rc==SQLITE_OK && !leavesReaderAtEnd(&reader) ){
- const char *pTerm = leavesReaderTerm(&reader);
- const int nTerm = leavesReaderTermBytes(&reader);
- void *oldValue = sqlite3Fts3HashFind(pTerms, pTerm, nTerm);
- void *newValue = (void *)((char *)oldValue+1);
-
- /* From the comment before sqlite3Fts3HashInsert in fts3_hash.c,
- ** the data value passed is returned in case of malloc failure.
- */
- if( newValue==sqlite3Fts3HashInsert(pTerms, pTerm, nTerm, newValue) ){
- rc = SQLITE_NOMEM;
- }else{
- rc = leavesReaderStep(v, &reader);
- }
- }
+/*
+** The set of routines that implement the porter-stemmer tokenizer
+*/
+static const sqlite3_tokenizer_module porterTokenizerModule = {
+ 0,
+ porterCreate,
+ porterDestroy,
+ porterOpen,
+ porterClose,
+ porterNext,
+};
- leavesReaderDestroy(&reader);
- return rc;
+/*
+** Allocate a new porter tokenizer. Return a pointer to the new
+** tokenizer in *ppModule
+*/
+SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule(
+ sqlite3_tokenizer_module const**ppModule
+){
+ *ppModule = &porterTokenizerModule;
}
-/* Helper function to build the result string for dump_terms(). */
-static int generateTermsResult(sqlite3_context *pContext, fts3Hash *pTerms){
- int iTerm, nTerms, nResultBytes, iByte;
- char *result;
- TermData *pData;
- fts3HashElem *e;
-
- /* Iterate pTerms to generate an array of terms in pData for
- ** sorting.
- */
- nTerms = fts3HashCount(pTerms);
- assert( nTerms>0 );
- pData = sqlite3_malloc(nTerms*sizeof(TermData));
- if( pData==NULL ) return SQLITE_NOMEM;
-
- nResultBytes = 0;
- for(iTerm = 0, e = fts3HashFirst(pTerms); e; iTerm++, e = fts3HashNext(e)){
- nResultBytes += fts3HashKeysize(e)+1; /* Term plus trailing space */
- assert( iTerm<nTerms );
- pData[iTerm].pTerm = fts3HashKey(e);
- pData[iTerm].nTerm = fts3HashKeysize(e);
- pData[iTerm].pCollector = fts3HashData(e); /* unused */
- }
- assert( iTerm==nTerms );
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
- assert( nResultBytes>0 ); /* nTerms>0, nResultsBytes must be, too. */
- result = sqlite3_malloc(nResultBytes);
- if( result==NULL ){
- sqlite3_free(pData);
- return SQLITE_NOMEM;
- }
+/************** End of fts3_porter.c *****************************************/
+/************** Begin file fts3_tokenizer.c **********************************/
+/*
+** 2007 June 22
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This is part of an SQLite module implementing full-text search.
+** This particular file implements the generic tokenizer interface.
+*/
- if( nTerms>1 ) qsort(pData, nTerms, sizeof(*pData), termDataCmp);
+/*
+** The code in this file is only compiled if:
+**
+** * The FTS3 module is being built as an extension
+** (in which case SQLITE_CORE is not defined), or
+**
+** * The FTS3 module is being built into the core of
+** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
+*/
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
- /* Read the terms in order to build the result. */
- iByte = 0;
- for(iTerm=0; iTerm<nTerms; ++iTerm){
- memcpy(result+iByte, pData[iTerm].pTerm, pData[iTerm].nTerm);
- iByte += pData[iTerm].nTerm;
- result[iByte++] = ' ';
- }
- assert( iByte==nResultBytes );
- assert( result[nResultBytes-1]==' ' );
- result[nResultBytes-1] = '\0';
+#ifndef SQLITE_CORE
+ SQLITE_EXTENSION_INIT1
+#endif
- /* Passes away ownership of result. */
- sqlite3_result_text(pContext, result, nResultBytes-1, sqlite3_free);
- sqlite3_free(pData);
- return SQLITE_OK;
-}
-/* Implements dump_terms() for use in inspecting the fts3 index from
-** tests. TEXT result containing the ordered list of terms joined by
-** spaces. dump_terms(t, level, idx) dumps the terms for the segment
-** specified by level, idx (in %_segdir), while dump_terms(t) dumps
-** all terms in the index. In both cases t is the fts table's magic
-** table-named column.
+/*
+** Implementation of the SQL scalar function for accessing the underlying
+** hash table. This function may be called as follows:
+**
+** SELECT <function-name>(<key-name>);
+** SELECT <function-name>(<key-name>, <pointer>);
+**
+** where <function-name> is the name passed as the second argument
+** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer').
+**
+** If the <pointer> argument is specified, it must be a blob value
+** containing a pointer to be stored as the hash data corresponding
+** to the string <key-name>. If <pointer> is not specified, then
+** the string <key-name> must already exist in the has table. Otherwise,
+** an error is returned.
+**
+** Whether or not the <pointer> argument is specified, the value returned
+** is a blob containing the pointer stored as the hash data corresponding
+** to string <key-name> (after the hash-table is updated, if applicable).
*/
-static void dumpTermsFunc(
- sqlite3_context *pContext,
- int argc, sqlite3_value **argv
+static void scalarFunc(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
){
- fulltext_cursor *pCursor;
- if( argc!=3 && argc!=1 ){
- generateError(pContext, "dump_terms", "incorrect arguments");
- }else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ||
- sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
- generateError(pContext, "dump_terms", "illegal first argument");
- }else{
- fulltext_vtab *v;
- fts3Hash terms;
- sqlite3_stmt *s = NULL;
- int rc;
+ Fts3Hash *pHash;
+ void *pPtr = 0;
+ const unsigned char *zName;
+ int nName;
- memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
- v = cursor_vtab(pCursor);
+ assert( argc==1 || argc==2 );
- /* If passed only the cursor column, get all segments. Otherwise
- ** get the segment described by the following two arguments.
- */
- if( argc==1 ){
- rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s);
- }else{
- rc = sql_get_statement(v, SEGDIR_SELECT_SEGMENT_STMT, &s);
- if( rc==SQLITE_OK ){
- rc = sqlite3_bind_int(s, 1, sqlite3_value_int(argv[1]));
- if( rc==SQLITE_OK ){
- rc = sqlite3_bind_int(s, 2, sqlite3_value_int(argv[2]));
- }
- }
- }
+ pHash = (Fts3Hash *)sqlite3_user_data(context);
- if( rc!=SQLITE_OK ){
- generateError(pContext, "dump_terms", NULL);
+ zName = sqlite3_value_text(argv[0]);
+ nName = sqlite3_value_bytes(argv[0])+1;
+
+ if( argc==2 ){
+ void *pOld;
+ int n = sqlite3_value_bytes(argv[1]);
+ if( n!=sizeof(pPtr) ){
+ sqlite3_result_error(context, "argument type mismatch", -1);
return;
}
-
- /* Collect the terms for each segment. */
- sqlite3Fts3HashInit(&terms, FTS3_HASH_STRING, 1);
- while( (rc = sqlite3_step(s))==SQLITE_ROW ){
- rc = collectSegmentTerms(v, s, &terms);
- if( rc!=SQLITE_OK ) break;
+ pPtr = *(void **)sqlite3_value_blob(argv[1]);
+ pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr);
+ if( pOld==pPtr ){
+ sqlite3_result_error(context, "out of memory", -1);
+ return;
}
-
- if( rc!=SQLITE_DONE ){
- sqlite3_reset(s);
- generateError(pContext, "dump_terms", NULL);
- }else{
- const int nTerms = fts3HashCount(&terms);
- if( nTerms>0 ){
- rc = generateTermsResult(pContext, &terms);
- if( rc==SQLITE_NOMEM ){
- generateError(pContext, "dump_terms", "out of memory");
- }else{
- assert( rc==SQLITE_OK );
- }
- }else if( argc==3 ){
- /* The specific segment asked for could not be found. */
- generateError(pContext, "dump_terms", "segment not found");
- }else{
- /* No segments found. */
- /* TODO(shess): It should be impossible to reach this. This
- ** case can only happen for an empty table, in which case
- ** SQLite has no rows to call this function on.
- */
- sqlite3_result_null(pContext);
- }
+ }else{
+ pPtr = sqlite3Fts3HashFind(pHash, zName, nName);
+ if( !pPtr ){
+ char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
+ sqlite3_result_error(context, zErr, -1);
+ sqlite3_free(zErr);
+ return;
}
- sqlite3Fts3HashClear(&terms);
}
-}
-
-/* Expand the DL_DEFAULT doclist in pData into a text result in
-** pContext.
-*/
-static void createDoclistResult(sqlite3_context *pContext,
- const char *pData, int nData){
- DataBuffer dump;
- DLReader dlReader;
- assert( pData!=NULL && nData>0 );
-
- dataBufferInit(&dump, 0);
- dlrInit(&dlReader, DL_DEFAULT, pData, nData);
- for( ; !dlrAtEnd(&dlReader); dlrStep(&dlReader) ){
- char buf[256];
- PLReader plReader;
+ sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
+}
- plrInit(&plReader, &dlReader);
- if( DL_DEFAULT==DL_DOCIDS || plrAtEnd(&plReader) ){
- sqlite3_snprintf(sizeof(buf), buf, "[%lld] ", dlrDocid(&dlReader));
- dataBufferAppend(&dump, buf, strlen(buf));
- }else{
- int iColumn = plrColumn(&plReader);
-
- sqlite3_snprintf(sizeof(buf), buf, "[%lld %d[",
- dlrDocid(&dlReader), iColumn);
- dataBufferAppend(&dump, buf, strlen(buf));
-
- for( ; !plrAtEnd(&plReader); plrStep(&plReader) ){
- if( plrColumn(&plReader)!=iColumn ){
- iColumn = plrColumn(&plReader);
- sqlite3_snprintf(sizeof(buf), buf, "] %d[", iColumn);
- assert( dump.nData>0 );
- dump.nData--; /* Overwrite trailing space. */
- assert( dump.pData[dump.nData]==' ');
- dataBufferAppend(&dump, buf, strlen(buf));
- }
- if( DL_DEFAULT==DL_POSITIONS_OFFSETS ){
- sqlite3_snprintf(sizeof(buf), buf, "%d,%d,%d ",
- plrPosition(&plReader),
- plrStartOffset(&plReader), plrEndOffset(&plReader));
- }else if( DL_DEFAULT==DL_POSITIONS ){
- sqlite3_snprintf(sizeof(buf), buf, "%d ", plrPosition(&plReader));
- }else{
- assert( NULL=="Unhandled DL_DEFAULT value");
- }
- dataBufferAppend(&dump, buf, strlen(buf));
+static int fts3IsIdChar(char c){
+ static const char isFtsIdChar[] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */
+ 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
+ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
+ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
+ };
+ return (c&0x80 || isFtsIdChar[(int)(c)]);
+}
+
+SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *zStr, int *pn){
+ const char *z1;
+ const char *z2 = 0;
+
+ /* Find the start of the next token. */
+ z1 = zStr;
+ while( z2==0 ){
+ switch( *z1 ){
+ case '\0': return 0; /* No more tokens here */
+ case '\'':
+ case '"':
+ case '`': {
+ z2 = &z1[1];
+ while( *z2 && (z2[0]!=*z1 || z2[1]==*z1) ) z2++;
+ if( *z2 ) z2++;
+ break;
}
- plrDestroy(&plReader);
+ case '[':
+ z2 = &z1[1];
+ while( *z2 && z2[0]!=']' ) z2++;
+ if( *z2 ) z2++;
+ break;
- assert( dump.nData>0 );
- dump.nData--; /* Overwrite trailing space. */
- assert( dump.pData[dump.nData]==' ');
- dataBufferAppend(&dump, "]] ", 3);
+ default:
+ if( fts3IsIdChar(*z1) ){
+ z2 = &z1[1];
+ while( fts3IsIdChar(*z2) ) z2++;
+ }else{
+ z1++;
+ }
}
}
- dlrDestroy(&dlReader);
-
- assert( dump.nData>0 );
- dump.nData--; /* Overwrite trailing space. */
- assert( dump.pData[dump.nData]==' ');
- dump.pData[dump.nData] = '\0';
- assert( dump.nData>0 );
- /* Passes ownership of dump's buffer to pContext. */
- sqlite3_result_text(pContext, dump.pData, dump.nData, sqlite3_free);
- dump.pData = NULL;
- dump.nData = dump.nCapacity = 0;
+ *pn = (int)(z2-z1);
+ return z1;
}
-/* Implements dump_doclist() for use in inspecting the fts3 index from
-** tests. TEXT result containing a string representation of the
-** doclist for the indicated term. dump_doclist(t, term, level, idx)
-** dumps the doclist for term from the segment specified by level, idx
-** (in %_segdir), while dump_doclist(t, term) dumps the logical
-** doclist for the term across all segments. The per-segment doclist
-** can contain deletions, while the full-index doclist will not
-** (deletions are omitted).
-**
-** Result formats differ with the setting of DL_DEFAULTS. Examples:
-**
-** DL_DOCIDS: [1] [3] [7]
-** DL_POSITIONS: [1 0[0 4] 1[17]] [3 1[5]]
-** DL_POSITIONS_OFFSETS: [1 0[0,0,3 4,23,26] 1[17,102,105]] [3 1[5,20,23]]
-**
-** In each case the number after the outer '[' is the docid. In the
-** latter two cases, the number before the inner '[' is the column
-** associated with the values within. For DL_POSITIONS the numbers
-** within are the positions, for DL_POSITIONS_OFFSETS they are the
-** position, the start offset, and the end offset.
-*/
-static void dumpDoclistFunc(
- sqlite3_context *pContext,
- int argc, sqlite3_value **argv
+SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(
+ Fts3Hash *pHash, /* Tokenizer hash table */
+ const char *zArg, /* Possible tokenizer specification */
+ sqlite3_tokenizer **ppTok, /* OUT: Tokenizer (if applicable) */
+ const char **pzTokenizer, /* OUT: Set to zArg if is tokenizer */
+ char **pzErr /* OUT: Set to malloced error message */
){
- fulltext_cursor *pCursor;
- if( argc!=2 && argc!=4 ){
- generateError(pContext, "dump_doclist", "incorrect arguments");
- }else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ||
- sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
- generateError(pContext, "dump_doclist", "illegal first argument");
- }else if( sqlite3_value_text(argv[1])==NULL ||
- sqlite3_value_text(argv[1])[0]=='\0' ){
- generateError(pContext, "dump_doclist", "empty second argument");
- }else{
- const char *pTerm = (const char *)sqlite3_value_text(argv[1]);
- const int nTerm = strlen(pTerm);
- fulltext_vtab *v;
- int rc;
- DataBuffer doclist;
-
- memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
- v = cursor_vtab(pCursor);
-
- dataBufferInit(&doclist, 0);
-
- /* termSelect() yields the same logical doclist that queries are
- ** run against.
- */
- if( argc==2 ){
- rc = termSelect(v, v->nColumn, pTerm, nTerm, 0, DL_DEFAULT, &doclist);
- }else{
- sqlite3_stmt *s = NULL;
-
- /* Get our specific segment's information. */
- rc = sql_get_statement(v, SEGDIR_SELECT_SEGMENT_STMT, &s);
- if( rc==SQLITE_OK ){
- rc = sqlite3_bind_int(s, 1, sqlite3_value_int(argv[2]));
- if( rc==SQLITE_OK ){
- rc = sqlite3_bind_int(s, 2, sqlite3_value_int(argv[3]));
- }
- }
+ int rc;
+ char *z = (char *)zArg;
+ int n;
+ char *zCopy;
+ char *zEnd; /* Pointer to nul-term of zCopy */
+ sqlite3_tokenizer_module *m;
- if( rc==SQLITE_OK ){
- rc = sqlite3_step(s);
+ if( !z ){
+ zCopy = sqlite3_mprintf("simple");
+ }else{
+ if( sqlite3_strnicmp(z, "tokenize", 8) || fts3IsIdChar(z[8])){
+ return SQLITE_OK;
+ }
+ zCopy = sqlite3_mprintf("%s", &z[8]);
+ *pzTokenizer = zArg;
+ }
+ if( !zCopy ){
+ return SQLITE_NOMEM;
+ }
- if( rc==SQLITE_DONE ){
- dataBufferDestroy(&doclist);
- generateError(pContext, "dump_doclist", "segment not found");
- return;
- }
+ zEnd = &zCopy[strlen(zCopy)];
- /* Found a segment, load it into doclist. */
- if( rc==SQLITE_ROW ){
- const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1);
- const char *pData = sqlite3_column_blob(s, 2);
- const int nData = sqlite3_column_bytes(s, 2);
+ z = (char *)sqlite3Fts3NextToken(zCopy, &n);
+ z[n] = '\0';
+ sqlite3Fts3Dequote(z);
- /* loadSegment() is used by termSelect() to load each
- ** segment's data.
- */
- rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, 0,
- &doclist);
- if( rc==SQLITE_OK ){
- rc = sqlite3_step(s);
-
- /* Should not have more than one matching segment. */
- if( rc!=SQLITE_DONE ){
- sqlite3_reset(s);
- dataBufferDestroy(&doclist);
- generateError(pContext, "dump_doclist", "invalid segdir");
- return;
- }
- rc = SQLITE_OK;
- }
- }
+ m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, z, (int)strlen(z)+1);
+ if( !m ){
+ *pzErr = sqlite3_mprintf("unknown tokenizer: %s", z);
+ rc = SQLITE_ERROR;
+ }else{
+ char const **aArg = 0;
+ int iArg = 0;
+ z = &z[n+1];
+ while( z<zEnd && (NULL!=(z = (char *)sqlite3Fts3NextToken(z, &n))) ){
+ int nNew = sizeof(char *)*(iArg+1);
+ char const **aNew = (const char **)sqlite3_realloc((void *)aArg, nNew);
+ if( !aNew ){
+ sqlite3_free(zCopy);
+ sqlite3_free((void *)aArg);
+ return SQLITE_NOMEM;
}
-
- sqlite3_reset(s);
+ aArg = aNew;
+ aArg[iArg++] = z;
+ z[n] = '\0';
+ sqlite3Fts3Dequote(z);
+ z = &z[n+1];
}
-
- if( rc==SQLITE_OK ){
- if( doclist.nData>0 ){
- createDoclistResult(pContext, doclist.pData, doclist.nData);
- }else{
- /* TODO(shess): This can happen if the term is not present, or
- ** if all instances of the term have been deleted and this is
- ** an all-index dump. It may be interesting to distinguish
- ** these cases.
- */
- sqlite3_result_text(pContext, "", 0, SQLITE_STATIC);
- }
- }else if( rc==SQLITE_NOMEM ){
- /* Handle out-of-memory cases specially because if they are
- ** generated in fts3 code they may not be reflected in the db
- ** handle.
- */
- /* TODO(shess): Handle this more comprehensively.
- ** sqlite3ErrStr() has what I need, but is internal.
- */
- generateError(pContext, "dump_doclist", "out of memory");
+ rc = m->xCreate(iArg, aArg, ppTok);
+ assert( rc!=SQLITE_OK || *ppTok );
+ if( rc!=SQLITE_OK ){
+ *pzErr = sqlite3_mprintf("unknown tokenizer");
}else{
- generateError(pContext, "dump_doclist", NULL);
+ (*ppTok)->pModule = m;
}
-
- dataBufferDestroy(&doclist);
+ sqlite3_free((void *)aArg);
}
+
+ sqlite3_free(zCopy);
+ return rc;
}
-#endif
-/*
-** This routine implements the xFindFunction method for the FTS3
-** virtual table.
-*/
-static int fulltextFindFunction(
- sqlite3_vtab *pVtab,
- int nArg,
- const char *zName,
- void (**pxFunc)(sqlite3_context*,int,sqlite3_value**),
- void **ppArg
-){
- if( strcmp(zName,"snippet")==0 ){
- *pxFunc = snippetFunc;
- return 1;
- }else if( strcmp(zName,"offsets")==0 ){
- *pxFunc = snippetOffsetsFunc;
- return 1;
- }else if( strcmp(zName,"optimize")==0 ){
- *pxFunc = optimizeFunc;
- return 1;
+
#ifdef SQLITE_TEST
- /* NOTE(shess): These functions are present only for testing
- ** purposes. No particular effort is made to optimize their
- ** execution or how they build their results.
- */
- }else if( strcmp(zName,"dump_terms")==0 ){
- /* fprintf(stderr, "Found dump_terms\n"); */
- *pxFunc = dumpTermsFunc;
- return 1;
- }else if( strcmp(zName,"dump_doclist")==0 ){
- /* fprintf(stderr, "Found dump_doclist\n"); */
- *pxFunc = dumpDoclistFunc;
- return 1;
-#endif
- }
- return 0;
-}
+
/*
-** Rename an fts3 table.
+** Implementation of a special SQL scalar function for testing tokenizers
+** designed to be used in concert with the Tcl testing framework. This
+** function must be called with two arguments:
+**
+** SELECT <function-name>(<key-name>, <input-string>);
+** SELECT <function-name>(<key-name>, <pointer>);
+**
+** where <function-name> is the name passed as the second argument
+** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer')
+** concatenated with the string '_test' (e.g. 'fts3_tokenizer_test').
+**
+** The return value is a string that may be interpreted as a Tcl
+** list. For each token in the <input-string>, three elements are
+** added to the returned list. The first is the token position, the
+** second is the token text (folded, stemmed, etc.) and the third is the
+** substring of <input-string> associated with the token. For example,
+** using the built-in "simple" tokenizer:
+**
+** SELECT fts_tokenizer_test('simple', 'I don't see how');
+**
+** will return the string:
+**
+** "{0 i I 1 dont don't 2 see see 3 how how}"
+**
*/
-static int fulltextRename(
- sqlite3_vtab *pVtab,
- const char *zName
+static void testFunc(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
){
- fulltext_vtab *p = (fulltext_vtab *)pVtab;
- int rc = SQLITE_NOMEM;
- char *zSql = sqlite3_mprintf(
- "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';"
- "ALTER TABLE %Q.'%q_segments' RENAME TO '%q_segments';"
- "ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';"
- , p->zDb, p->zName, zName
- , p->zDb, p->zName, zName
- , p->zDb, p->zName, zName
- );
- if( zSql ){
- rc = sqlite3_exec(p->db, zSql, 0, 0, 0);
- sqlite3_free(zSql);
- }
- return rc;
-}
+ Fts3Hash *pHash;
+ sqlite3_tokenizer_module *p;
+ sqlite3_tokenizer *pTokenizer = 0;
+ sqlite3_tokenizer_cursor *pCsr = 0;
-static const sqlite3_module fts3Module = {
- /* iVersion */ 0,
- /* xCreate */ fulltextCreate,
- /* xConnect */ fulltextConnect,
- /* xBestIndex */ fulltextBestIndex,
- /* xDisconnect */ fulltextDisconnect,
- /* xDestroy */ fulltextDestroy,
- /* xOpen */ fulltextOpen,
- /* xClose */ fulltextClose,
- /* xFilter */ fulltextFilter,
- /* xNext */ fulltextNext,
- /* xEof */ fulltextEof,
- /* xColumn */ fulltextColumn,
- /* xRowid */ fulltextRowid,
- /* xUpdate */ fulltextUpdate,
- /* xBegin */ fulltextBegin,
- /* xSync */ fulltextSync,
- /* xCommit */ fulltextCommit,
- /* xRollback */ fulltextRollback,
- /* xFindFunction */ fulltextFindFunction,
- /* xRename */ fulltextRename,
-};
+ const char *zErr = 0;
-static void hashDestroy(void *p){
- fts3Hash *pHash = (fts3Hash *)p;
- sqlite3Fts3HashClear(pHash);
- sqlite3_free(pHash);
-}
+ const char *zName;
+ int nName;
+ const char *zInput;
+ int nInput;
-/*
-** The fts3 built-in tokenizers - "simple" and "porter" - are implemented
-** in files fts3_tokenizer1.c and fts3_porter.c respectively. The following
-** two forward declarations are for functions declared in these files
-** used to retrieve the respective implementations.
-**
-** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed
-** to by the argument to point a the "simple" tokenizer implementation.
-** Function ...PorterTokenizerModule() sets *pModule to point to the
-** porter tokenizer/stemmer implementation.
-*/
-SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
-SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule);
-SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule);
+ const char *zArg = 0;
-SQLITE_PRIVATE int sqlite3Fts3InitHashTable(sqlite3 *, fts3Hash *, const char *);
+ const char *zToken;
+ int nToken;
+ int iStart;
+ int iEnd;
+ int iPos;
-/*
-** Initialise the fts3 extension. If this extension is built as part
-** of the sqlite library, then this function is called directly by
-** SQLite. If fts3 is built as a dynamically loadable extension, this
-** function is called by the sqlite3_extension_init() entry point.
-*/
-SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){
- int rc = SQLITE_OK;
- fts3Hash *pHash = 0;
- const sqlite3_tokenizer_module *pSimple = 0;
- const sqlite3_tokenizer_module *pPorter = 0;
- const sqlite3_tokenizer_module *pIcu = 0;
+ Tcl_Obj *pRet;
- sqlite3Fts3SimpleTokenizerModule(&pSimple);
- sqlite3Fts3PorterTokenizerModule(&pPorter);
-#ifdef SQLITE_ENABLE_ICU
- sqlite3Fts3IcuTokenizerModule(&pIcu);
-#endif
+ assert( argc==2 || argc==3 );
- /* Allocate and initialise the hash-table used to store tokenizers. */
- pHash = sqlite3_malloc(sizeof(fts3Hash));
- if( !pHash ){
- rc = SQLITE_NOMEM;
- }else{
- sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1);
+ nName = sqlite3_value_bytes(argv[0]);
+ zName = (const char *)sqlite3_value_text(argv[0]);
+ nInput = sqlite3_value_bytes(argv[argc-1]);
+ zInput = (const char *)sqlite3_value_text(argv[argc-1]);
+
+ if( argc==3 ){
+ zArg = (const char *)sqlite3_value_text(argv[1]);
}
- /* Load the built-in tokenizers into the hash table */
- if( rc==SQLITE_OK ){
- if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple)
- || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter)
- || (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu))
- ){
- rc = SQLITE_NOMEM;
- }
+ pHash = (Fts3Hash *)sqlite3_user_data(context);
+ p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1);
+
+ if( !p ){
+ char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
+ sqlite3_result_error(context, zErr, -1);
+ sqlite3_free(zErr);
+ return;
}
-#ifdef SQLITE_TEST
- sqlite3Fts3ExprInitTestInterface(db);
-#endif
+ pRet = Tcl_NewObj();
+ Tcl_IncrRefCount(pRet);
- /* Create the virtual table wrapper around the hash-table and overload
- ** the two scalar functions. If this is successful, register the
- ** module with sqlite.
- */
- if( SQLITE_OK==rc
- && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer"))
- && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1))
- && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", -1))
- && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", -1))
-#ifdef SQLITE_TEST
- && SQLITE_OK==(rc = sqlite3_overload_function(db, "dump_terms", -1))
- && SQLITE_OK==(rc = sqlite3_overload_function(db, "dump_doclist", -1))
-#endif
- ){
- return sqlite3_create_module_v2(
- db, "fts3", &fts3Module, (void *)pHash, hashDestroy
- );
+ if( SQLITE_OK!=p->xCreate(zArg ? 1 : 0, &zArg, &pTokenizer) ){
+ zErr = "error in xCreate()";
+ goto finish;
+ }
+ pTokenizer->pModule = p;
+ if( SQLITE_OK!=p->xOpen(pTokenizer, zInput, nInput, &pCsr) ){
+ zErr = "error in xOpen()";
+ goto finish;
}
+ pCsr->pTokenizer = pTokenizer;
- /* An error has occurred. Delete the hash table and return the error code. */
- assert( rc!=SQLITE_OK );
- if( pHash ){
- sqlite3Fts3HashClear(pHash);
- sqlite3_free(pHash);
+ while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){
+ Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos));
+ Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
+ zToken = &zInput[iStart];
+ nToken = iEnd-iStart;
+ Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
}
- return rc;
+
+ if( SQLITE_OK!=p->xClose(pCsr) ){
+ zErr = "error in xClose()";
+ goto finish;
+ }
+ if( SQLITE_OK!=p->xDestroy(pTokenizer) ){
+ zErr = "error in xDestroy()";
+ goto finish;
+ }
+
+finish:
+ if( zErr ){
+ sqlite3_result_error(context, zErr, -1);
+ }else{
+ sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT);
+ }
+ Tcl_DecrRefCount(pRet);
}
-#if !SQLITE_CORE
-SQLITE_API int sqlite3_extension_init(
+static
+int registerTokenizer(
sqlite3 *db,
- char **pzErrMsg,
- const sqlite3_api_routines *pApi
+ char *zName,
+ const sqlite3_tokenizer_module *p
){
- SQLITE_EXTENSION_INIT2(pApi)
- return sqlite3Fts3Init(db);
-}
-#endif
+ int rc;
+ sqlite3_stmt *pStmt;
+ const char zSql[] = "SELECT fts3_tokenizer(?, ?)";
-#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
+ rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
-/************** End of fts3.c ************************************************/
-/************** Begin file fts3_expr.c ***************************************/
-/*
-** 2008 Nov 28
-**
-** The author disclaims copyright to this source code. In place of
-** a legal notice, here is a blessing:
-**
-** May you do good and not evil.
-** May you find forgiveness for yourself and forgive others.
-** May you share freely, never taking more than you give.
-**
-******************************************************************************
-**
-** This module contains code that implements a parser for fts3 query strings
-** (the right-hand argument to the MATCH operator). Because the supported
-** syntax is relatively simple, the whole tokenizer/parser system is
-** hand-coded. The public interface to this module is declared in source
-** code file "fts3_expr.h".
-*/
-#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+ sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
+ sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);
+ sqlite3_step(pStmt);
-/*
-** By default, this module parses the legacy syntax that has been
-** traditionally used by fts3. Or, if SQLITE_ENABLE_FTS3_PARENTHESIS
-** is defined, then it uses the new syntax. The differences between
-** the new and the old syntaxes are:
-**
-** a) The new syntax supports parenthesis. The old does not.
-**
-** b) The new syntax supports the AND and NOT operators. The old does not.
-**
-** c) The old syntax supports the "-" token qualifier. This is not
-** supported by the new syntax (it is replaced by the NOT operator).
-**
-** d) When using the old syntax, the OR operator has a greater precedence
-** than an implicit AND. When using the new, both implicity and explicit
-** AND operators have a higher precedence than OR.
-**
-** If compiled with SQLITE_TEST defined, then this module exports the
-** symbol "int sqlite3_fts3_enable_parentheses". Setting this variable
-** to zero causes the module to use the old syntax. If it is set to
-** non-zero the new syntax is activated. This is so both syntaxes can
-** be tested using a single build of testfixture.
-*/
-#ifdef SQLITE_TEST
-SQLITE_API int sqlite3_fts3_enable_parentheses = 0;
-#else
-# ifdef SQLITE_ENABLE_FTS3_PARENTHESIS
-# define sqlite3_fts3_enable_parentheses 1
-# else
-# define sqlite3_fts3_enable_parentheses 0
-# endif
-#endif
+ return sqlite3_finalize(pStmt);
+}
-/*
-** Default span for NEAR operators.
-*/
-#define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10
+static
+int queryTokenizer(
+ sqlite3 *db,
+ char *zName,
+ const sqlite3_tokenizer_module **pp
+){
+ int rc;
+ sqlite3_stmt *pStmt;
+ const char zSql[] = "SELECT fts3_tokenizer(?)";
+ *pp = 0;
+ rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
-typedef struct ParseContext ParseContext;
-struct ParseContext {
- sqlite3_tokenizer *pTokenizer; /* Tokenizer module */
- const char **azCol; /* Array of column names for fts3 table */
- int nCol; /* Number of entries in azCol[] */
- int iDefaultCol; /* Default column to query */
- sqlite3_context *pCtx; /* Write error message here */
- int nNest; /* Number of nested brackets */
-};
+ sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
+ if( SQLITE_ROW==sqlite3_step(pStmt) ){
+ if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
+ memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
+ }
+ }
-/*
-** This function is equivalent to the standard isspace() function.
-**
-** The standard isspace() can be awkward to use safely, because although it
-** is defined to accept an argument of type int, its behaviour when passed
-** an integer that falls outside of the range of the unsigned char type
-** is undefined (and sometimes, "undefined" means segfault). This wrapper
-** is defined to accept an argument of type char, and always returns 0 for
-** any values that fall outside of the range of the unsigned char type (i.e.
-** negative values).
-*/
-static int fts3isspace(char c){
- return (c&0x80)==0 ? isspace(c) : 0;
+ return sqlite3_finalize(pStmt);
}
+SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
+
/*
-** Extract the next token from buffer z (length n) using the tokenizer
-** and other information (column names etc.) in pParse. Create an Fts3Expr
-** structure of type FTSQUERY_PHRASE containing a phrase consisting of this
-** single token and set *ppExpr to point to it. If the end of the buffer is
-** reached before a token is found, set *ppExpr to zero. It is the
-** responsibility of the caller to eventually deallocate the allocated
-** Fts3Expr structure (if any) by passing it to sqlite3_free().
+** Implementation of the scalar function fts3_tokenizer_internal_test().
+** This function is used for testing only, it is not included in the
+** build unless SQLITE_TEST is defined.
+**
+** The purpose of this is to test that the fts3_tokenizer() function
+** can be used as designed by the C-code in the queryTokenizer and
+** registerTokenizer() functions above. These two functions are repeated
+** in the README.tokenizer file as an example, so it is important to
+** test them.
+**
+** To run the tests, evaluate the fts3_tokenizer_internal_test() scalar
+** function with no arguments. An assert() will fail if a problem is
+** detected. i.e.:
+**
+** SELECT fts3_tokenizer_internal_test();
**
-** Return SQLITE_OK if successful, or SQLITE_NOMEM if a memory allocation
-** fails.
*/
-static int getNextToken(
- ParseContext *pParse, /* fts3 query parse context */
- int iCol, /* Value for Fts3Phrase.iColumn */
- const char *z, int n, /* Input string */
- Fts3Expr **ppExpr, /* OUT: expression */
- int *pnConsumed /* OUT: Number of bytes consumed */
+static void intTestFunc(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
){
- sqlite3_tokenizer *pTokenizer = pParse->pTokenizer;
- sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
int rc;
- sqlite3_tokenizer_cursor *pCursor;
- Fts3Expr *pRet = 0;
- int nConsumed = 0;
-
- rc = pModule->xOpen(pTokenizer, z, n, &pCursor);
- if( rc==SQLITE_OK ){
- const char *zToken;
- int nToken, iStart, iEnd, iPosition;
- int nByte; /* total space to allocate */
+ const sqlite3_tokenizer_module *p1;
+ const sqlite3_tokenizer_module *p2;
+ sqlite3 *db = (sqlite3 *)sqlite3_user_data(context);
- pCursor->pTokenizer = pTokenizer;
- rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition);
+ UNUSED_PARAMETER(argc);
+ UNUSED_PARAMETER(argv);
- if( rc==SQLITE_OK ){
- nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken;
- pRet = (Fts3Expr *)sqlite3_malloc(nByte);
- if( !pRet ){
- rc = SQLITE_NOMEM;
- }else{
- memset(pRet, 0, nByte);
- pRet->eType = FTSQUERY_PHRASE;
- pRet->pPhrase = (Fts3Phrase *)&pRet[1];
- pRet->pPhrase->nToken = 1;
- pRet->pPhrase->iColumn = iCol;
- pRet->pPhrase->aToken[0].n = nToken;
- pRet->pPhrase->aToken[0].z = (char *)&pRet->pPhrase[1];
- memcpy(pRet->pPhrase->aToken[0].z, zToken, nToken);
+ /* Test the query function */
+ sqlite3Fts3SimpleTokenizerModule(&p1);
+ rc = queryTokenizer(db, "simple", &p2);
+ assert( rc==SQLITE_OK );
+ assert( p1==p2 );
+ rc = queryTokenizer(db, "nosuchtokenizer", &p2);
+ assert( rc==SQLITE_ERROR );
+ assert( p2==0 );
+ assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") );
- if( iEnd<n && z[iEnd]=='*' ){
- pRet->pPhrase->aToken[0].isPrefix = 1;
- iEnd++;
- }
- if( !sqlite3_fts3_enable_parentheses && iStart>0 && z[iStart-1]=='-' ){
- pRet->pPhrase->isNot = 1;
- }
- }
- nConsumed = iEnd;
- }
+ /* Test the storage function */
+ rc = registerTokenizer(db, "nosuchtokenizer", p1);
+ assert( rc==SQLITE_OK );
+ rc = queryTokenizer(db, "nosuchtokenizer", &p2);
+ assert( rc==SQLITE_OK );
+ assert( p2==p1 );
- pModule->xClose(pCursor);
- }
-
- *pnConsumed = nConsumed;
- *ppExpr = pRet;
- return rc;
+ sqlite3_result_text(context, "ok", -1, SQLITE_STATIC);
}
+#endif
/*
-** Enlarge a memory allocation. If an out-of-memory allocation occurs,
-** then free the old allocation.
-*/
-void *fts3ReallocOrFree(void *pOrig, int nNew){
- void *pRet = sqlite3_realloc(pOrig, nNew);
- if( !pRet ){
- sqlite3_free(pOrig);
- }
- return pRet;
-}
-
-/*
-** Buffer zInput, length nInput, contains the contents of a quoted string
-** that appeared as part of an fts3 query expression. Neither quote character
-** is included in the buffer. This function attempts to tokenize the entire
-** input buffer and create an Fts3Expr structure of type FTSQUERY_PHRASE
-** containing the results.
+** Set up SQL objects in database db used to access the contents of
+** the hash table pointed to by argument pHash. The hash table must
+** been initialised to use string keys, and to take a private copy
+** of the key when a value is inserted. i.e. by a call similar to:
**
-** If successful, SQLITE_OK is returned and *ppExpr set to point at the
-** allocated Fts3Expr structure. Otherwise, either SQLITE_NOMEM (out of memory
-** error) or SQLITE_ERROR (tokenization error) is returned and *ppExpr set
-** to 0.
+** sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1);
+**
+** This function adds a scalar function (see header comment above
+** scalarFunc() in this file for details) and, if ENABLE_TABLE is
+** defined at compilation time, a temporary virtual table (see header
+** comment above struct HashTableVtab) to the database schema. Both
+** provide read/write access to the contents of *pHash.
+**
+** The third argument to this function, zName, is used as the name
+** of both the scalar and, if created, the virtual table.
*/
-static int getNextString(
- ParseContext *pParse, /* fts3 query parse context */
- const char *zInput, int nInput, /* Input string */
- Fts3Expr **ppExpr /* OUT: expression */
+SQLITE_PRIVATE int sqlite3Fts3InitHashTable(
+ sqlite3 *db,
+ Fts3Hash *pHash,
+ const char *zName
){
- sqlite3_tokenizer *pTokenizer = pParse->pTokenizer;
- sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
- int rc;
- Fts3Expr *p = 0;
- sqlite3_tokenizer_cursor *pCursor = 0;
- char *zTemp = 0;
- int nTemp = 0;
-
- rc = pModule->xOpen(pTokenizer, zInput, nInput, &pCursor);
- if( rc==SQLITE_OK ){
- int ii;
- pCursor->pTokenizer = pTokenizer;
- for(ii=0; rc==SQLITE_OK; ii++){
- const char *zToken;
- int nToken, iBegin, iEnd, iPos;
- rc = pModule->xNext(pCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos);
- if( rc==SQLITE_OK ){
- int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
- p = fts3ReallocOrFree(p, nByte+ii*sizeof(struct PhraseToken));
- zTemp = fts3ReallocOrFree(zTemp, nTemp + nToken);
- if( !p || !zTemp ){
- goto no_mem;
- }
- if( ii==0 ){
- memset(p, 0, nByte);
- p->pPhrase = (Fts3Phrase *)&p[1];
- }
- p->pPhrase = (Fts3Phrase *)&p[1];
- p->pPhrase->nToken = ii+1;
- p->pPhrase->aToken[ii].n = nToken;
- memcpy(&zTemp[nTemp], zToken, nToken);
- nTemp += nToken;
- if( iEnd<nInput && zInput[iEnd]=='*' ){
- p->pPhrase->aToken[ii].isPrefix = 1;
- }else{
- p->pPhrase->aToken[ii].isPrefix = 0;
- }
- }
- }
+ int rc = SQLITE_OK;
+ void *p = (void *)pHash;
+ const int any = SQLITE_ANY;
+ char *zTest = 0;
+ char *zTest2 = 0;
- pModule->xClose(pCursor);
- pCursor = 0;
+#ifdef SQLITE_TEST
+ void *pdb = (void *)db;
+ zTest = sqlite3_mprintf("%s_test", zName);
+ zTest2 = sqlite3_mprintf("%s_internal_test", zName);
+ if( !zTest || !zTest2 ){
+ rc = SQLITE_NOMEM;
}
+#endif
- if( rc==SQLITE_DONE ){
- int jj;
- char *zNew;
- int nNew = 0;
- int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
- nByte += (p?(p->pPhrase->nToken-1):0) * sizeof(struct PhraseToken);
- p = fts3ReallocOrFree(p, nByte + nTemp);
- if( !p ){
- goto no_mem;
- }
- if( zTemp ){
- zNew = &(((char *)p)[nByte]);
- memcpy(zNew, zTemp, nTemp);
- }else{
- memset(p, 0, nByte+nTemp);
- }
- p->pPhrase = (Fts3Phrase *)&p[1];
- for(jj=0; jj<p->pPhrase->nToken; jj++){
- p->pPhrase->aToken[jj].z = &zNew[nNew];
- nNew += p->pPhrase->aToken[jj].n;
- }
- sqlite3_free(zTemp);
- p->eType = FTSQUERY_PHRASE;
- p->pPhrase->iColumn = pParse->iDefaultCol;
- rc = SQLITE_OK;
- }
+ if( SQLITE_OK!=rc
+ || SQLITE_OK!=(rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0))
+ || SQLITE_OK!=(rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0))
+#ifdef SQLITE_TEST
+ || SQLITE_OK!=(rc = sqlite3_create_function(db, zTest, 2, any, p, testFunc, 0, 0))
+ || SQLITE_OK!=(rc = sqlite3_create_function(db, zTest, 3, any, p, testFunc, 0, 0))
+ || SQLITE_OK!=(rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0))
+#endif
+ );
- *ppExpr = p;
+ sqlite3_free(zTest);
+ sqlite3_free(zTest2);
return rc;
-no_mem:
-
- if( pCursor ){
- pModule->xClose(pCursor);
- }
- sqlite3_free(zTemp);
- sqlite3_free(p);
- *ppExpr = 0;
- return SQLITE_NOMEM;
}
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
+
+/************** End of fts3_tokenizer.c **************************************/
+/************** Begin file fts3_tokenizer1.c *********************************/
/*
-** Function getNextNode(), which is called by fts3ExprParse(), may itself
-** call fts3ExprParse(). So this forward declaration is required.
+** 2006 Oct 10
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** Implementation of the "simple" full-text-search tokenizer.
*/
-static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *);
/*
-** The output variable *ppExpr is populated with an allocated Fts3Expr
-** structure, or set to 0 if the end of the input buffer is reached.
+** The code in this file is only compiled if:
**
-** Returns an SQLite error code. SQLITE_OK if everything works, SQLITE_NOMEM
-** if a malloc failure occurs, or SQLITE_ERROR if a parse error is encountered.
-** If SQLITE_ERROR is returned, pContext is populated with an error message.
+** * The FTS3 module is being built as an extension
+** (in which case SQLITE_CORE is not defined), or
+**
+** * The FTS3 module is being built into the core of
+** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
*/
-static int getNextNode(
- ParseContext *pParse, /* fts3 query parse context */
- const char *z, int n, /* Input string */
- Fts3Expr **ppExpr, /* OUT: expression */
- int *pnConsumed /* OUT: Number of bytes consumed */
-){
- static const struct Fts3Keyword {
- char z[4]; /* Keyword text */
- unsigned char n; /* Length of the keyword */
- unsigned char parenOnly; /* Only valid in paren mode */
- unsigned char eType; /* Keyword code */
- } aKeyword[] = {
- { "OR" , 2, 0, FTSQUERY_OR },
- { "AND", 3, 1, FTSQUERY_AND },
- { "NOT", 3, 1, FTSQUERY_NOT },
- { "NEAR", 4, 0, FTSQUERY_NEAR }
- };
- int ii;
- int iCol;
- int iColLen;
- int rc;
- Fts3Expr *pRet = 0;
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
- const char *zInput = z;
- int nInput = n;
- /* Skip over any whitespace before checking for a keyword, an open or
- ** close bracket, or a quoted string.
- */
- while( nInput>0 && fts3isspace(*zInput) ){
- nInput--;
- zInput++;
- }
- if( nInput==0 ){
- return SQLITE_DONE;
- }
- /* See if we are dealing with a keyword. */
- for(ii=0; ii<(int)(sizeof(aKeyword)/sizeof(struct Fts3Keyword)); ii++){
- const struct Fts3Keyword *pKey = &aKeyword[ii];
- if( (pKey->parenOnly & ~sqlite3_fts3_enable_parentheses)!=0 ){
- continue;
- }
+typedef struct simple_tokenizer {
+ sqlite3_tokenizer base;
+ char delim[128]; /* flag ASCII delimiters */
+} simple_tokenizer;
- if( nInput>=pKey->n && 0==memcmp(zInput, pKey->z, pKey->n) ){
- int nNear = SQLITE_FTS3_DEFAULT_NEAR_PARAM;
- int nKey = pKey->n;
- char cNext;
+typedef struct simple_tokenizer_cursor {
+ sqlite3_tokenizer_cursor base;
+ const char *pInput; /* input we are tokenizing */
+ int nBytes; /* size of the input */
+ int iOffset; /* current position in pInput */
+ int iToken; /* index of next token to be returned */
+ char *pToken; /* storage for current token */
+ int nTokenAllocated; /* space allocated to zToken buffer */
+} simple_tokenizer_cursor;
- /* If this is a "NEAR" keyword, check for an explicit nearness. */
- if( pKey->eType==FTSQUERY_NEAR ){
- assert( nKey==4 );
- if( zInput[4]=='/' && zInput[5]>='0' && zInput[5]<='9' ){
- nNear = 0;
- for(nKey=5; zInput[nKey]>='0' && zInput[nKey]<='9'; nKey++){
- nNear = nNear * 10 + (zInput[nKey] - '0');
- }
- }
- }
- /* At this point this is probably a keyword. But for that to be true,
- ** the next byte must contain either whitespace, an open or close
- ** parenthesis, a quote character, or EOF.
- */
- cNext = zInput[nKey];
- if( fts3isspace(cNext)
- || cNext=='"' || cNext=='(' || cNext==')' || cNext==0
- ){
- pRet = (Fts3Expr *)sqlite3_malloc(sizeof(Fts3Expr));
- memset(pRet, 0, sizeof(Fts3Expr));
- pRet->eType = pKey->eType;
- pRet->nNear = nNear;
- *ppExpr = pRet;
- *pnConsumed = (zInput - z) + nKey;
- return SQLITE_OK;
- }
+static int simpleDelim(simple_tokenizer *t, unsigned char c){
+ return c<0x80 && t->delim[c];
+}
- /* Turns out that wasn't a keyword after all. This happens if the
- ** user has supplied a token such as "ORacle". Continue.
- */
- }
- }
+/*
+** Create a new tokenizer instance.
+*/
+static int simpleCreate(
+ int argc, const char * const *argv,
+ sqlite3_tokenizer **ppTokenizer
+){
+ simple_tokenizer *t;
- /* Check for an open bracket. */
- if( sqlite3_fts3_enable_parentheses ){
- if( *zInput=='(' ){
- int nConsumed;
- int rc;
- pParse->nNest++;
- rc = fts3ExprParse(pParse, &zInput[1], nInput-1, ppExpr, &nConsumed);
- if( rc==SQLITE_OK && !*ppExpr ){
- rc = SQLITE_DONE;
- }
- *pnConsumed = (zInput - z) + 1 + nConsumed;
- return rc;
- }
-
- /* Check for a close bracket. */
- if( *zInput==')' ){
- pParse->nNest--;
- *pnConsumed = (zInput - z) + 1;
- return SQLITE_DONE;
- }
- }
+ t = (simple_tokenizer *) sqlite3_malloc(sizeof(*t));
+ if( t==NULL ) return SQLITE_NOMEM;
+ memset(t, 0, sizeof(*t));
- /* See if we are dealing with a quoted phrase. If this is the case, then
- ** search for the closing quote and pass the whole string to getNextString()
- ** for processing. This is easy to do, as fts3 has no syntax for escaping
- ** a quote character embedded in a string.
+ /* TODO(shess) Delimiters need to remain the same from run to run,
+ ** else we need to reindex. One solution would be a meta-table to
+ ** track such information in the database, then we'd only want this
+ ** information on the initial create.
*/
- if( *zInput=='"' ){
- for(ii=1; ii<nInput && zInput[ii]!='"'; ii++);
- *pnConsumed = (zInput - z) + ii + 1;
- if( ii==nInput ){
- return SQLITE_ERROR;
+ if( argc>1 ){
+ int i, n = (int)strlen(argv[1]);
+ for(i=0; i<n; i++){
+ unsigned char ch = argv[1][i];
+ /* We explicitly don't support UTF-8 delimiters for now. */
+ if( ch>=0x80 ){
+ sqlite3_free(t);
+ return SQLITE_ERROR;
+ }
+ t->delim[ch] = 1;
}
- return getNextString(pParse, &zInput[1], ii-1, ppExpr);
- }
-
-
- /* If control flows to this point, this must be a regular token, or
- ** the end of the input. Read a regular token using the sqlite3_tokenizer
- ** interface. Before doing so, figure out if there is an explicit
- ** column specifier for the token.
- **
- ** TODO: Strangely, it is not possible to associate a column specifier
- ** with a quoted phrase, only with a single token. Not sure if this was
- ** an implementation artifact or an intentional decision when fts3 was
- ** first implemented. Whichever it was, this module duplicates the
- ** limitation.
- */
- iCol = pParse->iDefaultCol;
- iColLen = 0;
- for(ii=0; ii<pParse->nCol; ii++){
- const char *zStr = pParse->azCol[ii];
- int nStr = strlen(zStr);
- if( nInput>nStr && zInput[nStr]==':'
- && sqlite3_strnicmp(zStr, zInput, nStr)==0
- ){
- iCol = ii;
- iColLen = ((zInput - z) + nStr + 1);
- break;
+ } else {
+ /* Mark non-alphanumeric ASCII characters as delimiters */
+ int i;
+ for(i=1; i<0x80; i++){
+ t->delim[i] = !isalnum(i) ? -1 : 0;
}
}
- rc = getNextToken(pParse, iCol, &z[iColLen], n-iColLen, ppExpr, pnConsumed);
- *pnConsumed += iColLen;
- return rc;
+
+ *ppTokenizer = &t->base;
+ return SQLITE_OK;
}
/*
-** The argument is an Fts3Expr structure for a binary operator (any type
-** except an FTSQUERY_PHRASE). Return an integer value representing the
-** precedence of the operator. Lower values have a higher precedence (i.e.
-** group more tightly). For example, in the C language, the == operator
-** groups more tightly than ||, and would therefore have a higher precedence.
-**
-** When using the new fts3 query syntax (when SQLITE_ENABLE_FTS3_PARENTHESIS
-** is defined), the order of the operators in precedence from highest to
-** lowest is:
-**
-** NEAR
-** NOT
-** AND (including implicit ANDs)
-** OR
-**
-** Note that when using the old query syntax, the OR operator has a higher
-** precedence than the AND operator.
+** Destroy a tokenizer
*/
-static int opPrecedence(Fts3Expr *p){
- assert( p->eType!=FTSQUERY_PHRASE );
- if( sqlite3_fts3_enable_parentheses ){
- return p->eType;
- }else if( p->eType==FTSQUERY_NEAR ){
- return 1;
- }else if( p->eType==FTSQUERY_OR ){
- return 2;
- }
- assert( p->eType==FTSQUERY_AND );
- return 3;
+static int simpleDestroy(sqlite3_tokenizer *pTokenizer){
+ sqlite3_free(pTokenizer);
+ return SQLITE_OK;
}
/*
-** Argument ppHead contains a pointer to the current head of a query
-** expression tree being parsed. pPrev is the expression node most recently
-** inserted into the tree. This function adds pNew, which is always a binary
-** operator node, into the expression tree based on the relative precedence
-** of pNew and the existing nodes of the tree. This may result in the head
-** of the tree changing, in which case *ppHead is set to the new root node.
+** Prepare to begin tokenizing a particular string. The input
+** string to be tokenized is pInput[0..nBytes-1]. A cursor
+** used to incrementally tokenize this string is returned in
+** *ppCursor.
*/
-static void insertBinaryOperator(
- Fts3Expr **ppHead, /* Pointer to the root node of a tree */
- Fts3Expr *pPrev, /* Node most recently inserted into the tree */
- Fts3Expr *pNew /* New binary node to insert into expression tree */
+static int simpleOpen(
+ sqlite3_tokenizer *pTokenizer, /* The tokenizer */
+ const char *pInput, int nBytes, /* String to be tokenized */
+ sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
){
- Fts3Expr *pSplit = pPrev;
- while( pSplit->pParent && opPrecedence(pSplit->pParent)<=opPrecedence(pNew) ){
- pSplit = pSplit->pParent;
- }
+ simple_tokenizer_cursor *c;
- if( pSplit->pParent ){
- assert( pSplit->pParent->pRight==pSplit );
- pSplit->pParent->pRight = pNew;
- pNew->pParent = pSplit->pParent;
+ UNUSED_PARAMETER(pTokenizer);
+
+ c = (simple_tokenizer_cursor *) sqlite3_malloc(sizeof(*c));
+ if( c==NULL ) return SQLITE_NOMEM;
+
+ c->pInput = pInput;
+ if( pInput==0 ){
+ c->nBytes = 0;
+ }else if( nBytes<0 ){
+ c->nBytes = (int)strlen(pInput);
}else{
- *ppHead = pNew;
+ c->nBytes = nBytes;
}
- pNew->pLeft = pSplit;
- pSplit->pParent = pNew;
+ c->iOffset = 0; /* start tokenizing at the beginning */
+ c->iToken = 0;
+ c->pToken = NULL; /* no space allocated, yet. */
+ c->nTokenAllocated = 0;
+
+ *ppCursor = &c->base;
+ return SQLITE_OK;
}
/*
-** Parse the fts3 query expression found in buffer z, length n. This function
-** returns either when the end of the buffer is reached or an unmatched
-** closing bracket - ')' - is encountered.
-**
-** If successful, SQLITE_OK is returned, *ppExpr is set to point to the
-** parsed form of the expression and *pnConsumed is set to the number of
-** bytes read from buffer z. Otherwise, *ppExpr is set to 0 and SQLITE_NOMEM
-** (out of memory error) or SQLITE_ERROR (parse error) is returned.
+** Close a tokenization cursor previously opened by a call to
+** simpleOpen() above.
*/
-static int fts3ExprParse(
- ParseContext *pParse, /* fts3 query parse context */
- const char *z, int n, /* Text of MATCH query */
- Fts3Expr **ppExpr, /* OUT: Parsed query structure */
- int *pnConsumed /* OUT: Number of bytes consumed */
-){
- Fts3Expr *pRet = 0;
- Fts3Expr *pPrev = 0;
- Fts3Expr *pNotBranch = 0; /* Only used in legacy parse mode */
- int nIn = n;
- const char *zIn = z;
- int rc = SQLITE_OK;
- int isRequirePhrase = 1;
-
- while( rc==SQLITE_OK ){
- Fts3Expr *p = 0;
- int nByte = 0;
- rc = getNextNode(pParse, zIn, nIn, &p, &nByte);
- if( rc==SQLITE_OK ){
- int isPhrase;
-
- if( !sqlite3_fts3_enable_parentheses
- && p->eType==FTSQUERY_PHRASE && p->pPhrase->isNot
- ){
- /* Create an implicit NOT operator. */
- Fts3Expr *pNot = sqlite3_malloc(sizeof(Fts3Expr));
- if( !pNot ){
- sqlite3Fts3ExprFree(p);
- rc = SQLITE_NOMEM;
- goto exprparse_out;
- }
- memset(pNot, 0, sizeof(Fts3Expr));
- pNot->eType = FTSQUERY_NOT;
- pNot->pRight = p;
- if( pNotBranch ){
- pNot->pLeft = pNotBranch;
- }
- pNotBranch = pNot;
- p = pPrev;
- }else{
- int eType = p->eType;
- assert( eType!=FTSQUERY_PHRASE || !p->pPhrase->isNot );
- isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft);
-
- /* The isRequirePhrase variable is set to true if a phrase or
- ** an expression contained in parenthesis is required. If a
- ** binary operator (AND, OR, NOT or NEAR) is encounted when
- ** isRequirePhrase is set, this is a syntax error.
- */
- if( !isPhrase && isRequirePhrase ){
- sqlite3Fts3ExprFree(p);
- rc = SQLITE_ERROR;
- goto exprparse_out;
- }
-
- if( isPhrase && !isRequirePhrase ){
- /* Insert an implicit AND operator. */
- Fts3Expr *pAnd;
- assert( pRet && pPrev );
- pAnd = sqlite3_malloc(sizeof(Fts3Expr));
- if( !pAnd ){
- sqlite3Fts3ExprFree(p);
- rc = SQLITE_NOMEM;
- goto exprparse_out;
- }
- memset(pAnd, 0, sizeof(Fts3Expr));
- pAnd->eType = FTSQUERY_AND;
- insertBinaryOperator(&pRet, pPrev, pAnd);
- pPrev = pAnd;
- }
+static int simpleClose(sqlite3_tokenizer_cursor *pCursor){
+ simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
+ sqlite3_free(c->pToken);
+ sqlite3_free(c);
+ return SQLITE_OK;
+}
- /* This test catches attempts to make either operand of a NEAR
- ** operator something other than a phrase. For example, either of
- ** the following:
- **
- ** (bracketed expression) NEAR phrase
- ** phrase NEAR (bracketed expression)
- **
- ** Return an error in either case.
- */
- if( pPrev && (
- (eType==FTSQUERY_NEAR && !isPhrase && pPrev->eType!=FTSQUERY_PHRASE)
- || (eType!=FTSQUERY_PHRASE && isPhrase && pPrev->eType==FTSQUERY_NEAR)
- )){
- sqlite3Fts3ExprFree(p);
- rc = SQLITE_ERROR;
- goto exprparse_out;
- }
-
- if( isPhrase ){
- if( pRet ){
- assert( pPrev && pPrev->pLeft && pPrev->pRight==0 );
- pPrev->pRight = p;
- p->pParent = pPrev;
- }else{
- pRet = p;
- }
- }else{
- insertBinaryOperator(&pRet, pPrev, p);
- }
- isRequirePhrase = !isPhrase;
- }
- assert( nByte>0 );
+/*
+** Extract the next token from a tokenization cursor. The cursor must
+** have been opened by a prior call to simpleOpen().
+*/
+static int simpleNext(
+ sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */
+ const char **ppToken, /* OUT: *ppToken is the token text */
+ int *pnBytes, /* OUT: Number of bytes in token */
+ int *piStartOffset, /* OUT: Starting offset of token */
+ int *piEndOffset, /* OUT: Ending offset of token */
+ int *piPosition /* OUT: Position integer of token */
+){
+ simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
+ simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer;
+ unsigned char *p = (unsigned char *)c->pInput;
+
+ while( c->iOffset<c->nBytes ){
+ int iStartOffset;
+
+ /* Scan past delimiter characters */
+ while( c->iOffset<c->nBytes && simpleDelim(t, p[c->iOffset]) ){
+ c->iOffset++;
}
- assert( rc!=SQLITE_OK || (nByte>0 && nByte<=nIn) );
- nIn -= nByte;
- zIn += nByte;
- pPrev = p;
- }
- if( rc==SQLITE_DONE && pRet && isRequirePhrase ){
- rc = SQLITE_ERROR;
- }
+ /* Count non-delimiter characters. */
+ iStartOffset = c->iOffset;
+ while( c->iOffset<c->nBytes && !simpleDelim(t, p[c->iOffset]) ){
+ c->iOffset++;
+ }
- if( rc==SQLITE_DONE ){
- rc = SQLITE_OK;
- if( !sqlite3_fts3_enable_parentheses && pNotBranch ){
- if( !pRet ){
- rc = SQLITE_ERROR;
- }else{
- Fts3Expr *pIter = pNotBranch;
- while( pIter->pLeft ){
- pIter = pIter->pLeft;
- }
- pIter->pLeft = pRet;
- pRet = pNotBranch;
+ if( c->iOffset>iStartOffset ){
+ int i, n = c->iOffset-iStartOffset;
+ if( n>c->nTokenAllocated ){
+ c->nTokenAllocated = n+20;
+ c->pToken = sqlite3_realloc(c->pToken, c->nTokenAllocated);
+ if( c->pToken==NULL ) return SQLITE_NOMEM;
+ }
+ for(i=0; i<n; i++){
+ /* TODO(shess) This needs expansion to handle UTF-8
+ ** case-insensitivity.
+ */
+ unsigned char ch = p[iStartOffset+i];
+ c->pToken[i] = (char)(ch<0x80 ? tolower(ch) : ch);
}
+ *ppToken = c->pToken;
+ *pnBytes = n;
+ *piStartOffset = iStartOffset;
+ *piEndOffset = c->iOffset;
+ *piPosition = c->iToken++;
+
+ return SQLITE_OK;
}
}
- *pnConsumed = n - nIn;
+ return SQLITE_DONE;
+}
-exprparse_out:
- if( rc!=SQLITE_OK ){
- sqlite3Fts3ExprFree(pRet);
- sqlite3Fts3ExprFree(pNotBranch);
- pRet = 0;
- }
- *ppExpr = pRet;
- return rc;
+/*
+** The set of routines that implement the simple tokenizer
+*/
+static const sqlite3_tokenizer_module simpleTokenizerModule = {
+ 0,
+ simpleCreate,
+ simpleDestroy,
+ simpleOpen,
+ simpleClose,
+ simpleNext,
+};
+
+/*
+** Allocate a new simple tokenizer. Return a pointer to the new
+** tokenizer in *ppModule
+*/
+SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(
+ sqlite3_tokenizer_module const**ppModule
+){
+ *ppModule = &simpleTokenizerModule;
}
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
+
+/************** End of fts3_tokenizer1.c *************************************/
+/************** Begin file fts3_write.c **************************************/
/*
-** Parameters z and n contain a pointer to and length of a buffer containing
-** an fts3 query expression, respectively. This function attempts to parse the
-** query expression and create a tree of Fts3Expr structures representing the
-** parsed expression. If successful, *ppExpr is set to point to the head
-** of the parsed expression tree and SQLITE_OK is returned. If an error
-** occurs, either SQLITE_NOMEM (out-of-memory error) or SQLITE_ERROR (parse
-** error) is returned and *ppExpr is set to 0.
+** 2009 Oct 23
**
-** If parameter n is a negative number, then z is assumed to point to a
-** nul-terminated string and the length is determined using strlen().
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
**
-** The first parameter, pTokenizer, is passed the fts3 tokenizer module to
-** use to normalize query tokens while parsing the expression. The azCol[]
-** array, which is assumed to contain nCol entries, should contain the names
-** of each column in the target fts3 table, in order from left to right.
-** Column names must be nul-terminated strings.
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
**
-** The iDefaultCol parameter should be passed the index of the table column
-** that appears on the left-hand-side of the MATCH operator (the default
-** column to match against for tokens for which a column name is not explicitly
-** specified as part of the query string), or -1 if tokens may by default
-** match any table column.
+******************************************************************************
+**
+** This file is part of the SQLite FTS3 extension module. Specifically,
+** this file contains code to insert, update and delete rows from FTS3
+** tables. It also contains code to merge FTS3 b-tree segments. Some
+** of the sub-routines used to merge segments are also used by the query
+** code in fts3.c.
*/
-SQLITE_PRIVATE int sqlite3Fts3ExprParse(
- sqlite3_tokenizer *pTokenizer, /* Tokenizer module */
- char **azCol, /* Array of column names for fts3 table */
- int nCol, /* Number of entries in azCol[] */
- int iDefaultCol, /* Default column to query */
- const char *z, int n, /* Text of MATCH query */
- Fts3Expr **ppExpr /* OUT: Parsed query structure */
-){
- int nParsed;
- int rc;
- ParseContext sParse;
- sParse.pTokenizer = pTokenizer;
- sParse.azCol = (const char **)azCol;
- sParse.nCol = nCol;
- sParse.iDefaultCol = iDefaultCol;
- sParse.nNest = 0;
- if( z==0 ){
- *ppExpr = 0;
- return SQLITE_OK;
- }
- if( n<0 ){
- n = strlen(z);
- }
- rc = fts3ExprParse(&sParse, z, n, ppExpr, &nParsed);
- /* Check for mismatched parenthesis */
- if( rc==SQLITE_OK && sParse.nNest ){
- rc = SQLITE_ERROR;
- sqlite3Fts3ExprFree(*ppExpr);
- *ppExpr = 0;
- }
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
- return rc;
-}
+
+typedef struct PendingList PendingList;
+typedef struct SegmentNode SegmentNode;
+typedef struct SegmentWriter SegmentWriter;
/*
-** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse().
+** Data structure used while accumulating terms in the pending-terms hash
+** table. The hash table entry maps from term (a string) to a malloced
+** instance of this structure.
*/
-SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *p){
- if( p ){
- sqlite3Fts3ExprFree(p->pLeft);
- sqlite3Fts3ExprFree(p->pRight);
- sqlite3_free(p);
- }
-}
+struct PendingList {
+ int nData;
+ char *aData;
+ int nSpace;
+ sqlite3_int64 iLastDocid;
+ sqlite3_int64 iLastCol;
+ sqlite3_int64 iLastPos;
+};
-/****************************************************************************
-*****************************************************************************
-** Everything after this point is just test code.
-*/
+/*
+** An instance of this structure is used to iterate through the terms on
+** a contiguous set of segment b-tree leaf nodes. Although the details of
+** this structure are only manipulated by code in this file, opaque handles
+** of type Fts3SegReader* are also used by code in fts3.c to iterate through
+** terms when querying the full-text index. See functions:
+**
+** sqlite3Fts3SegReaderNew()
+** sqlite3Fts3SegReaderFree()
+** sqlite3Fts3SegReaderIterate()
+*/
+struct Fts3SegReader {
+ int iIdx; /* Index within level */
+ sqlite3_int64 iStartBlock;
+ sqlite3_int64 iEndBlock;
+ sqlite3_stmt *pStmt; /* SQL Statement to access leaf nodes */
+ char *aNode; /* Pointer to node data (or NULL) */
+ int nNode; /* Size of buffer at aNode (or 0) */
+ int nTermAlloc; /* Allocated size of zTerm buffer */
+
+ /* Variables set by fts3SegReaderNext(). These may be read directly
+ ** by the caller. They are valid from the time SegmentReaderNew() returns
+ ** until SegmentReaderNext() returns something other than SQLITE_OK
+ ** (i.e. SQLITE_DONE).
+ */
+ int nTerm; /* Number of bytes in current term */
+ char *zTerm; /* Pointer to current term */
+ char *aDoclist; /* Pointer to doclist of current entry */
+ int nDoclist; /* Size of doclist in current entry */
+
+ /* The following variables are used to iterate through the current doclist */
+ char *pOffsetList;
+ sqlite3_int64 iDocid;
+};
-#ifdef SQLITE_TEST
+/*
+** An instance of this structure is used to create a segment b-tree in the
+** database. The internal details of this type are only accessed by the
+** following functions:
+**
+** fts3SegWriterAdd()
+** fts3SegWriterFlush()
+** fts3SegWriterFree()
+*/
+struct SegmentWriter {
+ SegmentNode *pTree; /* Pointer to interior tree structure */
+ sqlite3_int64 iFirst; /* First slot in %_segments written */
+ sqlite3_int64 iFree; /* Next free slot in %_segments */
+ char *zTerm; /* Pointer to previous term buffer */
+ int nTerm; /* Number of bytes in zTerm */
+ int nMalloc; /* Size of malloc'd buffer at zMalloc */
+ char *zMalloc; /* Malloc'd space (possibly) used for zTerm */
+ int nSize; /* Size of allocation at aData */
+ int nData; /* Bytes of data in aData */
+ char *aData; /* Pointer to block from malloc() */
+};
+/*
+** Type SegmentNode is used by the following three functions to create
+** the interior part of the segment b+-tree structures (everything except
+** the leaf nodes). These functions and type are only ever used by code
+** within the fts3SegWriterXXX() family of functions described above.
+**
+** fts3NodeAddTerm()
+** fts3NodeWrite()
+** fts3NodeFree()
+*/
+struct SegmentNode {
+ SegmentNode *pParent; /* Parent node (or NULL for root node) */
+ SegmentNode *pRight; /* Pointer to right-sibling */
+ SegmentNode *pLeftmost; /* Pointer to left-most node of this depth */
+ int nEntry; /* Number of terms written to node so far */
+ char *zTerm; /* Pointer to previous term buffer */
+ int nTerm; /* Number of bytes in zTerm */
+ int nMalloc; /* Size of malloc'd buffer at zMalloc */
+ char *zMalloc; /* Malloc'd space (possibly) used for zTerm */
+ int nData; /* Bytes of valid data so far */
+ char *aData; /* Node data */
+};
/*
-** Function to query the hash-table of tokenizers (see README.tokenizers).
-*/
-static int queryTestTokenizer(
- sqlite3 *db,
- const char *zName,
- const sqlite3_tokenizer_module **pp
+** Valid values for the second argument to fts3SqlStmt().
+*/
+#define SQL_DELETE_CONTENT 0
+#define SQL_IS_EMPTY 1
+#define SQL_DELETE_ALL_CONTENT 2
+#define SQL_DELETE_ALL_SEGMENTS 3
+#define SQL_DELETE_ALL_SEGDIR 4
+#define SQL_SELECT_CONTENT_BY_ROWID 5
+#define SQL_NEXT_SEGMENT_INDEX 6
+#define SQL_INSERT_SEGMENTS 7
+#define SQL_NEXT_SEGMENTS_ID 8
+#define SQL_INSERT_SEGDIR 9
+#define SQL_SELECT_LEVEL 10
+#define SQL_SELECT_ALL_LEVEL 11
+#define SQL_SELECT_LEVEL_COUNT 12
+#define SQL_SELECT_SEGDIR_COUNT_MAX 13
+#define SQL_DELETE_SEGDIR_BY_LEVEL 14
+#define SQL_DELETE_SEGMENTS_RANGE 15
+#define SQL_CONTENT_INSERT 16
+#define SQL_GET_BLOCK 17
+
+/*
+** This function is used to obtain an SQLite prepared statement handle
+** for the statement identified by the second argument. If successful,
+** *pp is set to the requested statement handle and SQLITE_OK returned.
+** Otherwise, an SQLite error code is returned and *pp is set to 0.
+**
+** If argument apVal is not NULL, then it must point to an array with
+** at least as many entries as the requested statement has bound
+** parameters. The values are bound to the statements parameters before
+** returning.
+*/
+static int fts3SqlStmt(
+ Fts3Table *p, /* Virtual table handle */
+ int eStmt, /* One of the SQL_XXX constants above */
+ sqlite3_stmt **pp, /* OUT: Statement handle */
+ sqlite3_value **apVal /* Values to bind to statement */
){
- int rc;
+ const char *azSql[] = {
+/* 0 */ "DELETE FROM %Q.'%q_content' WHERE rowid = ?",
+/* 1 */ "SELECT NOT EXISTS(SELECT docid FROM %Q.'%q_content' WHERE rowid!=?)",
+/* 2 */ "DELETE FROM %Q.'%q_content'",
+/* 3 */ "DELETE FROM %Q.'%q_segments'",
+/* 4 */ "DELETE FROM %Q.'%q_segdir'",
+/* 5 */ "SELECT * FROM %Q.'%q_content' WHERE rowid=?",
+/* 6 */ "SELECT coalesce(max(idx)+1, 0) FROM %Q.'%q_segdir' WHERE level=?",
+/* 7 */ "INSERT INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)",
+/* 8 */ "SELECT coalesce(max(blockid)+1, 1) FROM %Q.'%q_segments'",
+/* 9 */ "INSERT INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)",
+
+ /* Return segments in order from oldest to newest.*/
+/* 10 */ "SELECT idx, start_block, leaves_end_block, end_block, root "
+ "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC",
+/* 11 */ "SELECT idx, start_block, leaves_end_block, end_block, root "
+ "FROM %Q.'%q_segdir' ORDER BY level DESC, idx ASC",
+
+/* 12 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?",
+/* 13 */ "SELECT count(*), max(level) FROM %Q.'%q_segdir'",
+
+/* 14 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?",
+/* 15 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?",
+/* 16 */ "INSERT INTO %Q.'%q_content' VALUES(%z)",
+/* 17 */ "SELECT block FROM %Q.'%q_segments' WHERE blockid = ?",
+ };
+ int rc = SQLITE_OK;
sqlite3_stmt *pStmt;
- const char zSql[] = "SELECT fts3_tokenizer(?)";
- *pp = 0;
- rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
- if( rc!=SQLITE_OK ){
- return rc;
+ assert( SizeofArray(azSql)==SizeofArray(p->aStmt) );
+ assert( eStmt<SizeofArray(azSql) && eStmt>=0 );
+
+ pStmt = p->aStmt[eStmt];
+ if( !pStmt ){
+ char *zSql;
+ if( eStmt==SQL_CONTENT_INSERT ){
+ int i; /* Iterator variable */
+ char *zVarlist; /* The "?, ?, ..." string */
+ zVarlist = (char *)sqlite3_malloc(2*p->nColumn+2);
+ if( !zVarlist ){
+ *pp = 0;
+ return SQLITE_NOMEM;
+ }
+ zVarlist[0] = '?';
+ zVarlist[p->nColumn*2+1] = '\0';
+ for(i=1; i<=p->nColumn; i++){
+ zVarlist[i*2-1] = ',';
+ zVarlist[i*2] = '?';
+ }
+ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, zVarlist);
+ }else{
+ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName);
+ }
+ if( !zSql ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, NULL);
+ sqlite3_free(zSql);
+ assert( rc==SQLITE_OK || pStmt==0 );
+ p->aStmt[eStmt] = pStmt;
+ }
}
-
- sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
- if( SQLITE_ROW==sqlite3_step(pStmt) ){
- if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
- memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
+ if( apVal ){
+ int i;
+ int nParam = sqlite3_bind_parameter_count(pStmt);
+ for(i=0; rc==SQLITE_OK && i<nParam; i++){
+ rc = sqlite3_bind_value(pStmt, i+1, apVal[i]);
}
}
-
- return sqlite3_finalize(pStmt);
+ *pp = pStmt;
+ return rc;
}
/*
-** This function is part of the test interface for the query parser. It
-** writes a text representation of the query expression pExpr into the
-** buffer pointed to by argument zBuf. It is assumed that zBuf is large
-** enough to store the required text representation.
+** Similar to fts3SqlStmt(). Except, after binding the parameters in
+** array apVal[] to the SQL statement identified by eStmt, the statement
+** is executed.
+**
+** Returns SQLITE_OK if the statement is successfully executed, or an
+** SQLite error code otherwise.
*/
-static void exprToString(Fts3Expr *pExpr, char *zBuf){
- switch( pExpr->eType ){
- case FTSQUERY_PHRASE: {
- Fts3Phrase *pPhrase = pExpr->pPhrase;
- int i;
- zBuf += sprintf(zBuf, "PHRASE %d %d", pPhrase->iColumn, pPhrase->isNot);
- for(i=0; i<pPhrase->nToken; i++){
- zBuf += sprintf(zBuf," %.*s",pPhrase->aToken[i].n,pPhrase->aToken[i].z);
- zBuf += sprintf(zBuf,"%s", (pPhrase->aToken[i].isPrefix?"+":""));
- }
- return;
- }
-
- case FTSQUERY_NEAR:
- zBuf += sprintf(zBuf, "NEAR/%d ", pExpr->nNear);
- break;
- case FTSQUERY_NOT:
- zBuf += sprintf(zBuf, "NOT ");
- break;
- case FTSQUERY_AND:
- zBuf += sprintf(zBuf, "AND ");
- break;
- case FTSQUERY_OR:
- zBuf += sprintf(zBuf, "OR ");
- break;
+static int fts3SqlExec(Fts3Table *p, int eStmt, sqlite3_value **apVal){
+ sqlite3_stmt *pStmt;
+ int rc = fts3SqlStmt(p, eStmt, &pStmt, apVal);
+ if( rc==SQLITE_OK ){
+ sqlite3_step(pStmt);
+ rc = sqlite3_reset(pStmt);
}
+ return rc;
+}
- zBuf += sprintf(zBuf, "{");
- exprToString(pExpr->pLeft, zBuf);
- zBuf += strlen(zBuf);
- zBuf += sprintf(zBuf, "} ");
- zBuf += sprintf(zBuf, "{");
- exprToString(pExpr->pRight, zBuf);
- zBuf += strlen(zBuf);
- zBuf += sprintf(zBuf, "}");
+/*
+** Read a single block from the %_segments table. If the specified block
+** does not exist, return SQLITE_CORRUPT. If some other error (malloc, IO
+** etc.) occurs, return the appropriate SQLite error code.
+**
+** Otherwise, if successful, set *pzBlock to point to a buffer containing
+** the block read from the database, and *pnBlock to the size of the read
+** block in bytes.
+**
+** WARNING: The returned buffer is only valid until the next call to
+** sqlite3Fts3ReadBlock().
+*/
+SQLITE_PRIVATE int sqlite3Fts3ReadBlock(
+ Fts3Table *p,
+ sqlite3_int64 iBlock,
+ char const **pzBlock,
+ int *pnBlock
+){
+ sqlite3_stmt *pStmt;
+ int rc = fts3SqlStmt(p, SQL_GET_BLOCK, &pStmt, 0);
+ if( rc!=SQLITE_OK ) return rc;
+ sqlite3_reset(pStmt);
+
+ if( pzBlock ){
+ sqlite3_bind_int64(pStmt, 1, iBlock);
+ rc = sqlite3_step(pStmt);
+ if( rc!=SQLITE_ROW ){
+ return SQLITE_CORRUPT;
+ }
+
+ *pnBlock = sqlite3_column_bytes(pStmt, 0);
+ *pzBlock = (char *)sqlite3_column_blob(pStmt, 0);
+ if( !*pzBlock ){
+ return SQLITE_NOMEM;
+ }
+ }
+ return SQLITE_OK;
}
/*
-** This is the implementation of a scalar SQL function used to test the
-** expression parser. It should be called as follows:
+** Set *ppStmt to a statement handle that may be used to iterate through
+** all rows in the %_segdir table, from oldest to newest. If successful,
+** return SQLITE_OK. If an error occurs while preparing the statement,
+** return an SQLite error code.
**
-** fts3_exprtest(<tokenizer>, <expr>, <column 1>, ...);
+** There is only ever one instance of this SQL statement compiled for
+** each FTS3 table.
**
-** The first argument, <tokenizer>, is the name of the fts3 tokenizer used
-** to parse the query expression (see README.tokenizers). The second argument
-** is the query expression to parse. Each subsequent argument is the name
-** of a column of the fts3 table that the query expression may refer to.
-** For example:
+** The statement returns the following columns from the %_segdir table:
**
-** SELECT fts3_exprtest('simple', 'Bill col2:Bloggs', 'col1', 'col2');
+** 0: idx
+** 1: start_block
+** 2: leaves_end_block
+** 3: end_block
+** 4: root
*/
-static void fts3ExprTest(
- sqlite3_context *context,
- int argc,
- sqlite3_value **argv
+SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table *p, sqlite3_stmt **ppStmt){
+ return fts3SqlStmt(p, SQL_SELECT_ALL_LEVEL, ppStmt, 0);
+}
+
+
+/*
+** Append a single varint to a PendingList buffer. SQLITE_OK is returned
+** if successful, or an SQLite error code otherwise.
+**
+** This function also serves to allocate the PendingList structure itself.
+** For example, to create a new PendingList structure containing two
+** varints:
+**
+** PendingList *p = 0;
+** fts3PendingListAppendVarint(&p, 1);
+** fts3PendingListAppendVarint(&p, 2);
+*/
+static int fts3PendingListAppendVarint(
+ PendingList **pp, /* IN/OUT: Pointer to PendingList struct */
+ sqlite3_int64 i /* Value to append to data */
){
- sqlite3_tokenizer_module const *pModule = 0;
- sqlite3_tokenizer *pTokenizer = 0;
- int rc;
- char **azCol = 0;
- const char *zExpr;
- int nExpr;
- int nCol;
- int ii;
- Fts3Expr *pExpr;
- sqlite3 *db = sqlite3_context_db_handle(context);
+ PendingList *p = *pp;
- if( argc<3 ){
- sqlite3_result_error(context,
- "Usage: fts3_exprtest(tokenizer, expr, col1, ...", -1
- );
- return;
+ /* Allocate or grow the PendingList as required. */
+ if( !p ){
+ p = sqlite3_malloc(sizeof(*p) + 100);
+ if( !p ){
+ return SQLITE_NOMEM;
+ }
+ p->nSpace = 100;
+ p->aData = (char *)&p[1];
+ p->nData = 0;
}
-
- rc = queryTestTokenizer(db,
- (const char *)sqlite3_value_text(argv[0]), &pModule);
- if( rc==SQLITE_NOMEM ){
- sqlite3_result_error_nomem(context);
- goto exprtest_out;
- }else if( !pModule ){
- sqlite3_result_error(context, "No such tokenizer module", -1);
- goto exprtest_out;
+ else if( p->nData+FTS3_VARINT_MAX+1>p->nSpace ){
+ int nNew = p->nSpace * 2;
+ p = sqlite3_realloc(p, sizeof(*p) + nNew);
+ if( !p ){
+ sqlite3_free(*pp);
+ *pp = 0;
+ return SQLITE_NOMEM;
+ }
+ p->nSpace = nNew;
+ p->aData = (char *)&p[1];
}
- rc = pModule->xCreate(0, 0, &pTokenizer);
- assert( rc==SQLITE_NOMEM || rc==SQLITE_OK );
- if( rc==SQLITE_NOMEM ){
- sqlite3_result_error_nomem(context);
- goto exprtest_out;
- }
- pTokenizer->pModule = pModule;
+ /* Append the new serialized varint to the end of the list. */
+ p->nData += sqlite3Fts3PutVarint(&p->aData[p->nData], i);
+ p->aData[p->nData] = '\0';
+ *pp = p;
+ return SQLITE_OK;
+}
- zExpr = (const char *)sqlite3_value_text(argv[1]);
- nExpr = sqlite3_value_bytes(argv[1]);
- nCol = argc-2;
- azCol = (char **)sqlite3_malloc(nCol*sizeof(char *));
- if( !azCol ){
- sqlite3_result_error_nomem(context);
- goto exprtest_out;
+/*
+** Add a docid/column/position entry to a PendingList structure. Non-zero
+** is returned if the structure is sqlite3_realloced as part of adding
+** the entry. Otherwise, zero.
+**
+** If an OOM error occurs, *pRc is set to SQLITE_NOMEM before returning.
+** Zero is always returned in this case. Otherwise, if no OOM error occurs,
+** it is set to SQLITE_OK.
+*/
+static int fts3PendingListAppend(
+ PendingList **pp, /* IN/OUT: PendingList structure */
+ sqlite3_int64 iDocid, /* Docid for entry to add */
+ sqlite3_int64 iCol, /* Column for entry to add */
+ sqlite3_int64 iPos, /* Position of term for entry to add */
+ int *pRc /* OUT: Return code */
+){
+ PendingList *p = *pp;
+ int rc = SQLITE_OK;
+
+ assert( !p || p->iLastDocid<=iDocid );
+
+ if( !p || p->iLastDocid!=iDocid ){
+ sqlite3_int64 iDelta = iDocid - (p ? p->iLastDocid : 0);
+ if( p ){
+ assert( p->nData<p->nSpace );
+ assert( p->aData[p->nData]==0 );
+ p->nData++;
+ }
+ if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iDelta)) ){
+ goto pendinglistappend_out;
+ }
+ p->iLastCol = -1;
+ p->iLastPos = 0;
+ p->iLastDocid = iDocid;
}
- for(ii=0; ii<nCol; ii++){
- azCol[ii] = (char *)sqlite3_value_text(argv[ii+2]);
+ if( iCol>0 && p->iLastCol!=iCol ){
+ if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, 1))
+ || SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iCol))
+ ){
+ goto pendinglistappend_out;
+ }
+ p->iLastCol = iCol;
+ p->iLastPos = 0;
}
-
- rc = sqlite3Fts3ExprParse(
- pTokenizer, azCol, nCol, nCol, zExpr, nExpr, &pExpr
- );
- if( rc==SQLITE_NOMEM ){
- sqlite3_result_error_nomem(context);
- goto exprtest_out;
- }else if( rc==SQLITE_OK ){
- char zBuf[4096];
- exprToString(pExpr, zBuf);
- sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT);
- sqlite3Fts3ExprFree(pExpr);
- }else{
- sqlite3_result_error(context, "Error parsing expression", -1);
+ if( iCol>=0 ){
+ assert( iPos>p->iLastPos || (iPos==0 && p->iLastPos==0) );
+ rc = fts3PendingListAppendVarint(&p, 2+iPos-p->iLastPos);
+ if( rc==SQLITE_OK ){
+ p->iLastPos = iPos;
+ }
}
-exprtest_out:
- if( pModule && pTokenizer ){
- rc = pModule->xDestroy(pTokenizer);
+ pendinglistappend_out:
+ *pRc = rc;
+ if( p!=*pp ){
+ *pp = p;
+ return 1;
}
- sqlite3_free(azCol);
+ return 0;
}
/*
-** Register the query expression parser test function fts3_exprtest()
-** with database connection db.
+** Tokenize the nul-terminated string zText and add all tokens to the
+** pending-terms hash-table. The docid used is that currently stored in
+** p->iPrevDocid, and the column is specified by argument iCol.
+**
+** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code.
*/
-SQLITE_PRIVATE void sqlite3Fts3ExprInitTestInterface(sqlite3* db){
- sqlite3_create_function(
- db, "fts3_exprtest", -1, SQLITE_UTF8, 0, fts3ExprTest, 0, 0
- );
-}
+static int fts3PendingTermsAdd(Fts3Table *p, const char *zText, int iCol){
+ int rc;
+ int iStart;
+ int iEnd;
+ int iPos;
-#endif
-#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
+ char const *zToken;
+ int nToken;
-/************** End of fts3_expr.c *******************************************/
-/************** Begin file fts3_hash.c ***************************************/
-/*
-** 2001 September 22
-**
-** The author disclaims copyright to this source code. In place of
-** a legal notice, here is a blessing:
-**
-** May you do good and not evil.
-** May you find forgiveness for yourself and forgive others.
-** May you share freely, never taking more than you give.
-**
-*************************************************************************
-** This is the implementation of generic hash-tables used in SQLite.
-** We've modified it slightly to serve as a standalone hash table
-** implementation for the full-text indexing module.
-*/
+ sqlite3_tokenizer *pTokenizer = p->pTokenizer;
+ sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
+ sqlite3_tokenizer_cursor *pCsr;
+ int (*xNext)(sqlite3_tokenizer_cursor *pCursor,
+ const char**,int*,int*,int*,int*);
-/*
-** The code in this file is only compiled if:
-**
-** * The FTS3 module is being built as an extension
-** (in which case SQLITE_CORE is not defined), or
-**
-** * The FTS3 module is being built into the core of
-** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
-*/
-#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+ assert( pTokenizer && pModule );
+
+ rc = pModule->xOpen(pTokenizer, zText, -1, &pCsr);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+ pCsr->pTokenizer = pTokenizer;
+ xNext = pModule->xNext;
+ while( SQLITE_OK==rc
+ && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos))
+ ){
+ PendingList *pList;
+ /* Positions cannot be negative; we use -1 as a terminator internally.
+ ** Tokens must have a non-zero length.
+ */
+ if( iPos<0 || !zToken || nToken<=0 ){
+ rc = SQLITE_ERROR;
+ break;
+ }
-/*
-** Malloc and Free functions
+ pList = (PendingList *)fts3HashFind(&p->pendingTerms, zToken, nToken);
+ if( pList ){
+ p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem));
+ }
+ if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){
+ if( pList==fts3HashInsert(&p->pendingTerms, zToken, nToken, pList) ){
+ /* Malloc failed while inserting the new entry. This can only
+ ** happen if there was no previous entry for this token.
+ */
+ assert( 0==fts3HashFind(&p->pendingTerms, zToken, nToken) );
+ sqlite3_free(pList);
+ rc = SQLITE_NOMEM;
+ }
+ }
+ if( rc==SQLITE_OK ){
+ p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem));
+ }
+ }
+
+ pModule->xClose(pCsr);
+ return (rc==SQLITE_DONE ? SQLITE_OK : rc);
+}
+
+/*
+** Calling this function indicates that subsequent calls to
+** fts3PendingTermsAdd() are to add term/position-list pairs for the
+** contents of the document with docid iDocid.
*/
-static void *fts3HashMalloc(int n){
- void *p = sqlite3_malloc(n);
- if( p ){
- memset(p, 0, n);
+static int fts3PendingTermsDocid(Fts3Table *p, sqlite_int64 iDocid){
+ /* TODO(shess) Explore whether partially flushing the buffer on
+ ** forced-flush would provide better performance. I suspect that if
+ ** we ordered the doclists by size and flushed the largest until the
+ ** buffer was half empty, that would let the less frequent terms
+ ** generate longer doclists.
+ */
+ if( iDocid<=p->iPrevDocid || p->nPendingData>FTS3_MAX_PENDING_DATA ){
+ int rc = sqlite3Fts3PendingTermsFlush(p);
+ if( rc!=SQLITE_OK ) return rc;
}
- return p;
+ p->iPrevDocid = iDocid;
+ return SQLITE_OK;
}
-static void fts3HashFree(void *p){
- sqlite3_free(p);
+
+SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *p){
+ Fts3HashElem *pElem;
+ for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){
+ sqlite3_free(fts3HashData(pElem));
+ }
+ fts3HashClear(&p->pendingTerms);
+ p->nPendingData = 0;
}
-/* Turn bulk memory into a hash table object by initializing the
-** fields of the Hash structure.
+/*
+** This function is called by the xUpdate() method as part of an INSERT
+** operation. It adds entries for each term in the new record to the
+** pendingTerms hash table.
**
-** "pNew" is a pointer to the hash table that is to be initialized.
-** keyClass is one of the constants
-** FTS3_HASH_BINARY or FTS3_HASH_STRING. The value of keyClass
-** determines what kind of key the hash table will use. "copyKey" is
-** true if the hash table should make its own private copy of keys and
-** false if it should just use the supplied pointer.
+** Argument apVal is the same as the similarly named argument passed to
+** fts3InsertData(). Parameter iDocid is the docid of the new row.
*/
-SQLITE_PRIVATE void sqlite3Fts3HashInit(fts3Hash *pNew, int keyClass, int copyKey){
- assert( pNew!=0 );
- assert( keyClass>=FTS3_HASH_STRING && keyClass<=FTS3_HASH_BINARY );
- pNew->keyClass = keyClass;
- pNew->copyKey = copyKey;
- pNew->first = 0;
- pNew->count = 0;
- pNew->htsize = 0;
- pNew->ht = 0;
+static int fts3InsertTerms(Fts3Table *p, sqlite3_value **apVal){
+ int i; /* Iterator variable */
+ for(i=2; i<p->nColumn+2; i++){
+ const char *zText = (const char *)sqlite3_value_text(apVal[i]);
+ if( zText ){
+ int rc = fts3PendingTermsAdd(p, zText, i-2);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+ }
+ }
+ return SQLITE_OK;
}
-/* Remove all entries from a hash table. Reclaim all memory.
-** Call this routine to delete a hash table or to reset a hash table
-** to the empty state.
-*/
-SQLITE_PRIVATE void sqlite3Fts3HashClear(fts3Hash *pH){
- fts3HashElem *elem; /* For looping over all elements of the table */
+/*
+** This function is called by the xUpdate() method for an INSERT operation.
+** The apVal parameter is passed a copy of the apVal argument passed by
+** SQLite to the xUpdate() method. i.e:
+**
+** apVal[0] Not used for INSERT.
+** apVal[1] rowid
+** apVal[2] Left-most user-defined column
+** ...
+** apVal[p->nColumn+1] Right-most user-defined column
+** apVal[p->nColumn+2] Hidden column with same name as table
+** apVal[p->nColumn+3] Hidden "docid" column (alias for rowid)
+*/
+static int fts3InsertData(
+ Fts3Table *p, /* Full-text table */
+ sqlite3_value **apVal, /* Array of values to insert */
+ sqlite3_int64 *piDocid /* OUT: Docid for row just inserted */
+){
+ int rc; /* Return code */
+ sqlite3_stmt *pContentInsert; /* INSERT INTO %_content VALUES(...) */
- assert( pH!=0 );
- elem = pH->first;
- pH->first = 0;
- fts3HashFree(pH->ht);
- pH->ht = 0;
- pH->htsize = 0;
- while( elem ){
- fts3HashElem *next_elem = elem->next;
- if( pH->copyKey && elem->pKey ){
- fts3HashFree(elem->pKey);
- }
- fts3HashFree(elem);
- elem = next_elem;
+ /* Locate the statement handle used to insert data into the %_content
+ ** table. The SQL for this statement is:
+ **
+ ** INSERT INTO %_content VALUES(?, ?, ?, ...)
+ **
+ ** The statement features N '?' variables, where N is the number of user
+ ** defined columns in the FTS3 table, plus one for the docid field.
+ */
+ rc = fts3SqlStmt(p, SQL_CONTENT_INSERT, &pContentInsert, &apVal[1]);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+
+ /* There is a quirk here. The users INSERT statement may have specified
+ ** a value for the "rowid" field, for the "docid" field, or for both.
+ ** Which is a problem, since "rowid" and "docid" are aliases for the
+ ** same value. For example:
+ **
+ ** INSERT INTO fts3tbl(rowid, docid) VALUES(1, 2);
+ **
+ ** In FTS3, this is an error. It is an error to specify non-NULL values
+ ** for both docid and some other rowid alias.
+ */
+ if( SQLITE_NULL!=sqlite3_value_type(apVal[3+p->nColumn]) ){
+ if( SQLITE_NULL==sqlite3_value_type(apVal[0])
+ && SQLITE_NULL!=sqlite3_value_type(apVal[1])
+ ){
+ /* A rowid/docid conflict. */
+ return SQLITE_ERROR;
+ }
+ rc = sqlite3_bind_value(pContentInsert, 1, apVal[3+p->nColumn]);
+ if( rc!=SQLITE_OK ) return rc;
}
- pH->count = 0;
+
+ /* Execute the statement to insert the record. Set *piDocid to the
+ ** new docid value.
+ */
+ sqlite3_step(pContentInsert);
+ rc = sqlite3_reset(pContentInsert);
+
+ *piDocid = sqlite3_last_insert_rowid(p->db);
+ return rc;
}
+
+
/*
-** Hash and comparison functions when the mode is FTS3_HASH_STRING
+** Remove all data from the FTS3 table. Clear the hash table containing
+** pending terms.
*/
-static int fts3StrHash(const void *pKey, int nKey){
- const char *z = (const char *)pKey;
- int h = 0;
- if( nKey<=0 ) nKey = (int) strlen(z);
- while( nKey > 0 ){
- h = (h<<3) ^ h ^ *z++;
- nKey--;
+static int fts3DeleteAll(Fts3Table *p){
+ int rc; /* Return code */
+
+ /* Discard the contents of the pending-terms hash table. */
+ sqlite3Fts3PendingTermsClear(p);
+
+ /* Delete everything from the %_content, %_segments and %_segdir tables. */
+ rc = fts3SqlExec(p, SQL_DELETE_ALL_CONTENT, 0);
+ if( rc==SQLITE_OK ){
+ rc = fts3SqlExec(p, SQL_DELETE_ALL_SEGMENTS, 0);
}
- return h & 0x7fffffff;
-}
-static int fts3StrCompare(const void *pKey1, int n1, const void *pKey2, int n2){
- if( n1!=n2 ) return 1;
- return strncmp((const char*)pKey1,(const char*)pKey2,n1);
+ if( rc==SQLITE_OK ){
+ rc = fts3SqlExec(p, SQL_DELETE_ALL_SEGDIR, 0);
+ }
+ return rc;
}
/*
-** Hash and comparison functions when the mode is FTS3_HASH_BINARY
+** The first element in the apVal[] array is assumed to contain the docid
+** (an integer) of a row about to be deleted. Remove all terms from the
+** full-text index.
*/
-static int fts3BinHash(const void *pKey, int nKey){
- int h = 0;
- const char *z = (const char *)pKey;
- while( nKey-- > 0 ){
- h = (h<<3) ^ h ^ *(z++);
+static int fts3DeleteTerms(Fts3Table *p, sqlite3_value **apVal){
+ int rc;
+ sqlite3_stmt *pSelect;
+
+ rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, apVal);
+ if( rc==SQLITE_OK ){
+ if( SQLITE_ROW==sqlite3_step(pSelect) ){
+ int i;
+ for(i=1; i<=p->nColumn; i++){
+ const char *zText = (const char *)sqlite3_column_text(pSelect, i);
+ rc = fts3PendingTermsAdd(p, zText, -1);
+ if( rc!=SQLITE_OK ){
+ sqlite3_reset(pSelect);
+ return rc;
+ }
+ }
+ }
+ rc = sqlite3_reset(pSelect);
+ }else{
+ sqlite3_reset(pSelect);
}
- return h & 0x7fffffff;
-}
-static int fts3BinCompare(const void *pKey1, int n1, const void *pKey2, int n2){
- if( n1!=n2 ) return 1;
- return memcmp(pKey1,pKey2,n1);
+ return rc;
}
/*
-** Return a pointer to the appropriate hash function given the key class.
+** Forward declaration to account for the circular dependency between
+** functions fts3SegmentMerge() and fts3AllocateSegdirIdx().
+*/
+static int fts3SegmentMerge(Fts3Table *, int);
+
+/*
+** This function allocates a new level iLevel index in the segdir table.
+** Usually, indexes are allocated within a level sequentially starting
+** with 0, so the allocated index is one greater than the value returned
+** by:
**
-** The C syntax in this function definition may be unfamilar to some
-** programmers, so we provide the following additional explanation:
+** SELECT max(idx) FROM %_segdir WHERE level = :iLevel
**
-** The name of the function is "ftsHashFunction". The function takes a
-** single parameter "keyClass". The return value of ftsHashFunction()
-** is a pointer to another function. Specifically, the return value
-** of ftsHashFunction() is a pointer to a function that takes two parameters
-** with types "const void*" and "int" and returns an "int".
+** However, if there are already FTS3_MERGE_COUNT indexes at the requested
+** level, they are merged into a single level (iLevel+1) segment and the
+** allocated index is 0.
+**
+** If successful, *piIdx is set to the allocated index slot and SQLITE_OK
+** returned. Otherwise, an SQLite error code is returned.
*/
-static int (*ftsHashFunction(int keyClass))(const void*,int){
- if( keyClass==FTS3_HASH_STRING ){
- return &fts3StrHash;
- }else{
- assert( keyClass==FTS3_HASH_BINARY );
- return &fts3BinHash;
+static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){
+ int rc; /* Return Code */
+ sqlite3_stmt *pNextIdx; /* Query for next idx at level iLevel */
+ int iNext = 0; /* Result of query pNextIdx */
+
+ /* Set variable iNext to the next available segdir index at level iLevel. */
+ rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_int(pNextIdx, 1, iLevel);
+ if( SQLITE_ROW==sqlite3_step(pNextIdx) ){
+ iNext = sqlite3_column_int(pNextIdx, 0);
+ }
+ rc = sqlite3_reset(pNextIdx);
+ }
+
+ if( rc==SQLITE_OK ){
+ /* If iNext is FTS3_MERGE_COUNT, indicating that level iLevel is already
+ ** full, merge all segments in level iLevel into a single iLevel+1
+ ** segment and allocate (newly freed) index 0 at level iLevel. Otherwise,
+ ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext.
+ */
+ if( iNext>=FTS3_MERGE_COUNT ){
+ rc = fts3SegmentMerge(p, iLevel);
+ *piIdx = 0;
+ }else{
+ *piIdx = iNext;
+ }
}
+
+ return rc;
}
/*
-** Return a pointer to the appropriate hash function given the key class.
-**
-** For help in interpreted the obscure C code in the function definition,
-** see the header comment on the previous function.
+** Move the iterator passed as the first argument to the next term in the
+** segment. If successful, SQLITE_OK is returned. If there is no next term,
+** SQLITE_DONE. Otherwise, an SQLite error code.
*/
-static int (*ftsCompareFunction(int keyClass))(const void*,int,const void*,int){
- if( keyClass==FTS3_HASH_STRING ){
- return &fts3StrCompare;
+static int fts3SegReaderNext(Fts3SegReader *pReader){
+ char *pNext; /* Cursor variable */
+ int nPrefix; /* Number of bytes in term prefix */
+ int nSuffix; /* Number of bytes in term suffix */
+
+ if( !pReader->aDoclist ){
+ pNext = pReader->aNode;
}else{
- assert( keyClass==FTS3_HASH_BINARY );
- return &fts3BinCompare;
+ pNext = &pReader->aDoclist[pReader->nDoclist];
+ }
+
+ if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){
+ int rc;
+ if( !pReader->pStmt ){
+ pReader->aNode = 0;
+ return SQLITE_OK;
+ }
+ rc = sqlite3_step(pReader->pStmt);
+ if( rc!=SQLITE_ROW ){
+ pReader->aNode = 0;
+ return (rc==SQLITE_DONE ? SQLITE_OK : rc);
+ }
+ pReader->nNode = sqlite3_column_bytes(pReader->pStmt, 0);
+ pReader->aNode = (char *)sqlite3_column_blob(pReader->pStmt, 0);
+ pNext = pReader->aNode;
+ }
+
+ pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix);
+ pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix);
+
+ if( nPrefix+nSuffix>pReader->nTermAlloc ){
+ int nNew = (nPrefix+nSuffix)*2;
+ char *zNew = sqlite3_realloc(pReader->zTerm, nNew);
+ if( !zNew ){
+ return SQLITE_NOMEM;
+ }
+ pReader->zTerm = zNew;
+ pReader->nTermAlloc = nNew;
}
+ memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix);
+ pReader->nTerm = nPrefix+nSuffix;
+ pNext += nSuffix;
+ pNext += sqlite3Fts3GetVarint32(pNext, &pReader->nDoclist);
+ assert( pNext<&pReader->aNode[pReader->nNode] );
+ pReader->aDoclist = pNext;
+ pReader->pOffsetList = 0;
+ return SQLITE_OK;
}
-/* Link an element into the hash table
+/*
+** Set the SegReader to point to the first docid in the doclist associated
+** with the current term.
*/
-static void fts3HashInsertElement(
- fts3Hash *pH, /* The complete hash table */
- struct _fts3ht *pEntry, /* The entry into which pNew is inserted */
- fts3HashElem *pNew /* The element to be inserted */
-){
- fts3HashElem *pHead; /* First element already in pEntry */
- pHead = pEntry->chain;
- if( pHead ){
- pNew->next = pHead;
- pNew->prev = pHead->prev;
- if( pHead->prev ){ pHead->prev->next = pNew; }
- else { pH->first = pNew; }
- pHead->prev = pNew;
- }else{
- pNew->next = pH->first;
- if( pH->first ){ pH->first->prev = pNew; }
- pNew->prev = 0;
- pH->first = pNew;
- }
- pEntry->count++;
- pEntry->chain = pNew;
+static void fts3SegReaderFirstDocid(Fts3SegReader *pReader){
+ int n;
+ assert( pReader->aDoclist );
+ assert( !pReader->pOffsetList );
+ n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid);
+ pReader->pOffsetList = &pReader->aDoclist[n];
}
+/*
+** Advance the SegReader to point to the next docid in the doclist
+** associated with the current term.
+**
+** If arguments ppOffsetList and pnOffsetList are not NULL, then
+** *ppOffsetList is set to point to the first column-offset list
+** in the doclist entry (i.e. immediately past the docid varint).
+** *pnOffsetList is set to the length of the set of column-offset
+** lists, not including the nul-terminator byte. For example:
+*/
+static void fts3SegReaderNextDocid(
+ Fts3SegReader *pReader,
+ char **ppOffsetList,
+ int *pnOffsetList
+){
+ char *p = pReader->pOffsetList;
+ char c = 0;
-/* Resize the hash table so that it cantains "new_size" buckets.
-** "new_size" must be a power of 2. The hash table might fail
-** to resize if sqliteMalloc() fails.
-*/
-static void fts3Rehash(fts3Hash *pH, int new_size){
- struct _fts3ht *new_ht; /* The new hash table */
- fts3HashElem *elem, *next_elem; /* For looping over existing elements */
- int (*xHash)(const void*,int); /* The hash function */
+ /* Pointer p currently points at the first byte of an offset list. The
+ ** following two lines advance it to point one byte past the end of
+ ** the same offset list.
+ */
+ while( *p | c ) c = *p++ & 0x80;
+ p++;
- assert( (new_size & (new_size-1))==0 );
- new_ht = (struct _fts3ht *)fts3HashMalloc( new_size*sizeof(struct _fts3ht) );
- if( new_ht==0 ) return;
- fts3HashFree(pH->ht);
- pH->ht = new_ht;
- pH->htsize = new_size;
- xHash = ftsHashFunction(pH->keyClass);
- for(elem=pH->first, pH->first=0; elem; elem = next_elem){
- int h = (*xHash)(elem->pKey, elem->nKey) & (new_size-1);
- next_elem = elem->next;
- fts3HashInsertElement(pH, &new_ht[h], elem);
+ /* If required, populate the output variables with a pointer to and the
+ ** size of the previous offset-list.
+ */
+ if( ppOffsetList ){
+ *ppOffsetList = pReader->pOffsetList;
+ *pnOffsetList = (int)(p - pReader->pOffsetList - 1);
+ }
+
+ /* If there are no more entries in the doclist, set pOffsetList to
+ ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and
+ ** Fts3SegReader.pOffsetList to point to the next offset list before
+ ** returning.
+ */
+ if( p>=&pReader->aDoclist[pReader->nDoclist] ){
+ pReader->pOffsetList = 0;
+ }else{
+ sqlite3_int64 iDelta;
+ pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta);
+ pReader->iDocid += iDelta;
}
}
-/* This function (for internal use only) locates an element in an
-** hash table that matches the given key. The hash for this key has
-** already been computed and is passed as the 4th parameter.
+/*
+** Free all allocations associated with the iterator passed as the
+** second argument.
*/
-static fts3HashElem *fts3FindElementByHash(
- const fts3Hash *pH, /* The pH to be searched */
- const void *pKey, /* The key we are searching for */
- int nKey,
- int h /* The hash for this key. */
-){
- fts3HashElem *elem; /* Used to loop thru the element list */
- int count; /* Number of elements left to test */
- int (*xCompare)(const void*,int,const void*,int); /* comparison function */
-
- if( pH->ht ){
- struct _fts3ht *pEntry = &pH->ht[h];
- elem = pEntry->chain;
- count = pEntry->count;
- xCompare = ftsCompareFunction(pH->keyClass);
- while( count-- && elem ){
- if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){
- return elem;
- }
- elem = elem->next;
+SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3Table *p, Fts3SegReader *pReader){
+ if( pReader ){
+ if( pReader->pStmt ){
+ /* Move the leaf-range SELECT statement to the aLeavesStmt[] array,
+ ** so that it can be reused when required by another query.
+ */
+ assert( p->nLeavesStmt<p->nLeavesTotal );
+ sqlite3_reset(pReader->pStmt);
+ p->aLeavesStmt[p->nLeavesStmt++] = pReader->pStmt;
}
+ sqlite3_free(pReader->zTerm);
+ sqlite3_free(pReader);
}
- return 0;
}
-/* Remove a single entry from the hash table given a pointer to that
-** element and a hash on the element's key.
+/*
+** Allocate a new SegReader object.
*/
-static void fts3RemoveElementByHash(
- fts3Hash *pH, /* The pH containing "elem" */
- fts3HashElem* elem, /* The element to be removed from the pH */
- int h /* Hash value for the element */
+SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(
+ Fts3Table *p, /* Virtual table handle */
+ int iAge, /* Segment "age". */
+ sqlite3_int64 iStartLeaf, /* First leaf to traverse */
+ sqlite3_int64 iEndLeaf, /* Final leaf to traverse */
+ sqlite3_int64 iEndBlock, /* Final block of segment */
+ const char *zRoot, /* Buffer containing root node */
+ int nRoot, /* Size of buffer containing root node */
+ Fts3SegReader **ppReader /* OUT: Allocated Fts3SegReader */
){
- struct _fts3ht *pEntry;
- if( elem->prev ){
- elem->prev->next = elem->next;
- }else{
- pH->first = elem->next;
- }
- if( elem->next ){
- elem->next->prev = elem->prev;
- }
- pEntry = &pH->ht[h];
- if( pEntry->chain==elem ){
- pEntry->chain = elem->next;
+ int rc = SQLITE_OK; /* Return code */
+ Fts3SegReader *pReader; /* Newly allocated SegReader object */
+ int nExtra = 0; /* Bytes to allocate segment root node */
+
+ if( iStartLeaf==0 ){
+ nExtra = nRoot;
}
- pEntry->count--;
- if( pEntry->count<=0 ){
- pEntry->chain = 0;
+
+ pReader = (Fts3SegReader *)sqlite3_malloc(sizeof(Fts3SegReader) + nExtra);
+ if( !pReader ){
+ return SQLITE_NOMEM;
}
- if( pH->copyKey && elem->pKey ){
- fts3HashFree(elem->pKey);
+ memset(pReader, 0, sizeof(Fts3SegReader));
+ pReader->iStartBlock = iStartLeaf;
+ pReader->iIdx = iAge;
+ pReader->iEndBlock = iEndBlock;
+
+ if( nExtra ){
+ /* The entire segment is stored in the root node. */
+ pReader->aNode = (char *)&pReader[1];
+ pReader->nNode = nRoot;
+ memcpy(pReader->aNode, zRoot, nRoot);
+ }else{
+ /* If the text of the SQL statement to iterate through a contiguous
+ ** set of entries in the %_segments table has not yet been composed,
+ ** compose it now.
+ */
+ if( !p->zSelectLeaves ){
+ p->zSelectLeaves = sqlite3_mprintf(
+ "SELECT block FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ? "
+ "ORDER BY blockid", p->zDb, p->zName
+ );
+ if( !p->zSelectLeaves ){
+ rc = SQLITE_NOMEM;
+ goto finished;
+ }
+ }
+
+ /* If there are no free statements in the aLeavesStmt[] array, prepare
+ ** a new statement now. Otherwise, reuse a prepared statement from
+ ** aLeavesStmt[].
+ */
+ if( p->nLeavesStmt==0 ){
+ if( p->nLeavesTotal==p->nLeavesAlloc ){
+ int nNew = p->nLeavesAlloc + 16;
+ sqlite3_stmt **aNew = (sqlite3_stmt **)sqlite3_realloc(
+ p->aLeavesStmt, nNew*sizeof(sqlite3_stmt *)
+ );
+ if( !aNew ){
+ rc = SQLITE_NOMEM;
+ goto finished;
+ }
+ p->nLeavesAlloc = nNew;
+ p->aLeavesStmt = aNew;
+ }
+ rc = sqlite3_prepare_v2(p->db, p->zSelectLeaves, -1, &pReader->pStmt, 0);
+ if( rc!=SQLITE_OK ){
+ goto finished;
+ }
+ p->nLeavesTotal++;
+ }else{
+ pReader->pStmt = p->aLeavesStmt[--p->nLeavesStmt];
+ }
+
+ /* Bind the start and end leaf blockids to the prepared SQL statement. */
+ sqlite3_bind_int64(pReader->pStmt, 1, iStartLeaf);
+ sqlite3_bind_int64(pReader->pStmt, 2, iEndLeaf);
}
- fts3HashFree( elem );
- pH->count--;
- if( pH->count<=0 ){
- assert( pH->first==0 );
- assert( pH->count==0 );
- fts3HashClear(pH);
+ rc = fts3SegReaderNext(pReader);
+
+ finished:
+ if( rc==SQLITE_OK ){
+ *ppReader = pReader;
+ }else{
+ sqlite3Fts3SegReaderFree(p, pReader);
}
+ return rc;
}
-/* Attempt to locate an element of the hash table pH with a key
-** that matches pKey,nKey. Return the data for this element if it is
-** found, or NULL if there is no match.
-*/
-SQLITE_PRIVATE void *sqlite3Fts3HashFind(const fts3Hash *pH, const void *pKey, int nKey){
- int h; /* A hash on key */
- fts3HashElem *elem; /* The element that matches key */
- int (*xHash)(const void*,int); /* The hash function */
-
- if( pH==0 || pH->ht==0 ) return 0;
- xHash = ftsHashFunction(pH->keyClass);
- assert( xHash!=0 );
- h = (*xHash)(pKey,nKey);
- assert( (pH->htsize & (pH->htsize-1))==0 );
- elem = fts3FindElementByHash(pH,pKey,nKey, h & (pH->htsize-1));
- return elem ? elem->data : 0;
-}
-/* Insert an element into the hash table pH. The key is pKey,nKey
-** and the data is "data".
+/*
+** The second argument to this function is expected to be a statement of
+** the form:
**
-** If no element exists with a matching key, then a new
-** element is created. A copy of the key is made if the copyKey
-** flag is set. NULL is returned.
+** SELECT
+** idx, -- col 0
+** start_block, -- col 1
+** leaves_end_block, -- col 2
+** end_block, -- col 3
+** root -- col 4
+** FROM %_segdir ...
**
-** If another element already exists with the same key, then the
-** new data replaces the old data and the old data is returned.
-** The key is not copied in this instance. If a malloc fails, then
-** the new data is returned and the hash table is unchanged.
+** This function allocates and initializes a Fts3SegReader structure to
+** iterate through the terms stored in the segment identified by the
+** current row that pStmt is pointing to.
**
-** If the "data" parameter to this function is NULL, then the
-** element corresponding to "key" is removed from the hash table.
+** If successful, the Fts3SegReader is left pointing to the first term
+** in the segment and SQLITE_OK is returned. Otherwise, an SQLite error
+** code is returned.
*/
-SQLITE_PRIVATE void *sqlite3Fts3HashInsert(
- fts3Hash *pH, /* The hash table to insert into */
- const void *pKey, /* The key */
- int nKey, /* Number of bytes in the key */
- void *data /* The data */
+static int fts3SegReaderNew(
+ Fts3Table *p, /* Virtual table handle */
+ sqlite3_stmt *pStmt, /* See above */
+ int iAge, /* Segment "age". */
+ Fts3SegReader **ppReader /* OUT: Allocated Fts3SegReader */
){
- int hraw; /* Raw hash value of the key */
- int h; /* the hash of the key modulo hash table size */
- fts3HashElem *elem; /* Used to loop thru the element list */
- fts3HashElem *new_elem; /* New element added to the pH */
- int (*xHash)(const void*,int); /* The hash function */
+ return sqlite3Fts3SegReaderNew(p, iAge,
+ sqlite3_column_int64(pStmt, 1),
+ sqlite3_column_int64(pStmt, 2),
+ sqlite3_column_int64(pStmt, 3),
+ sqlite3_column_blob(pStmt, 4),
+ sqlite3_column_bytes(pStmt, 4),
+ ppReader
+ );
+}
- assert( pH!=0 );
- xHash = ftsHashFunction(pH->keyClass);
- assert( xHash!=0 );
- hraw = (*xHash)(pKey, nKey);
- assert( (pH->htsize & (pH->htsize-1))==0 );
- h = hraw & (pH->htsize-1);
- elem = fts3FindElementByHash(pH,pKey,nKey,h);
- if( elem ){
- void *old_data = elem->data;
- if( data==0 ){
- fts3RemoveElementByHash(pH,elem,h);
+/*
+** Compare the entries pointed to by two Fts3SegReader structures.
+** Comparison is as follows:
+**
+** 1) EOF is greater than not EOF.
+**
+** 2) The current terms (if any) are compared with memcmp(). If one
+** term is a prefix of another, the longer term is considered the
+** larger.
+**
+** 3) By segment age. An older segment is considered larger.
+*/
+static int fts3SegReaderCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){
+ int rc;
+ if( pLhs->aNode && pRhs->aNode ){
+ int rc2 = pLhs->nTerm - pRhs->nTerm;
+ if( rc2<0 ){
+ rc = memcmp(pLhs->zTerm, pRhs->zTerm, pLhs->nTerm);
}else{
- elem->data = data;
- }
- return old_data;
- }
- if( data==0 ) return 0;
- if( pH->htsize==0 ){
- fts3Rehash(pH,8);
- if( pH->htsize==0 ){
- pH->count = 0;
- return data;
+ rc = memcmp(pLhs->zTerm, pRhs->zTerm, pRhs->nTerm);
}
- }
- new_elem = (fts3HashElem*)fts3HashMalloc( sizeof(fts3HashElem) );
- if( new_elem==0 ) return data;
- if( pH->copyKey && pKey!=0 ){
- new_elem->pKey = fts3HashMalloc( nKey );
- if( new_elem->pKey==0 ){
- fts3HashFree(new_elem);
- return data;
+ if( rc==0 ){
+ rc = rc2;
}
- memcpy((void*)new_elem->pKey, pKey, nKey);
}else{
- new_elem->pKey = (void*)pKey;
+ rc = (pLhs->aNode==0) - (pRhs->aNode==0);
}
- new_elem->nKey = nKey;
- pH->count++;
- if( pH->count > pH->htsize ){
- fts3Rehash(pH,pH->htsize*2);
+ if( rc==0 ){
+ rc = pRhs->iIdx - pLhs->iIdx;
}
- assert( pH->htsize>0 );
- assert( (pH->htsize & (pH->htsize-1))==0 );
- h = hraw & (pH->htsize-1);
- fts3HashInsertElement(pH, &pH->ht[h], new_elem);
- new_elem->data = data;
- return 0;
+ assert( rc!=0 );
+ return rc;
}
-#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
-
-/************** End of fts3_hash.c *******************************************/
-/************** Begin file fts3_porter.c *************************************/
/*
-** 2006 September 30
+** A different comparison function for SegReader structures. In this
+** version, it is assumed that each SegReader points to an entry in
+** a doclist for identical terms. Comparison is made as follows:
**
-** The author disclaims copyright to this source code. In place of
-** a legal notice, here is a blessing:
+** 1) EOF (end of doclist in this case) is greater than not EOF.
**
-** May you do good and not evil.
-** May you find forgiveness for yourself and forgive others.
-** May you share freely, never taking more than you give.
+** 2) By current docid.
**
-*************************************************************************
-** Implementation of the full-text-search tokenizer that implements
-** a Porter stemmer.
+** 3) By segment age. An older segment is considered larger.
*/
+static int fts3SegReaderDoclistCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){
+ int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0);
+ if( rc==0 ){
+ if( pLhs->iDocid==pRhs->iDocid ){
+ rc = pRhs->iIdx - pLhs->iIdx;
+ }else{
+ rc = (pLhs->iDocid > pRhs->iDocid) ? 1 : -1;
+ }
+ }
+ assert( pLhs->aNode && pRhs->aNode );
+ return rc;
+}
/*
-** The code in this file is only compiled if:
-**
-** * The FTS3 module is being built as an extension
-** (in which case SQLITE_CORE is not defined), or
+** Compare the term that the Fts3SegReader object passed as the first argument
+** points to with the term specified by arguments zTerm and nTerm.
**
-** * The FTS3 module is being built into the core of
-** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
-*/
-#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
-
-
-
-
-/*
-** Class derived from sqlite3_tokenizer
+** If the pSeg iterator is already at EOF, return 0. Otherwise, return
+** -ve if the pSeg term is less than zTerm/nTerm, 0 if the two terms are
+** equal, or +ve if the pSeg term is greater than zTerm/nTerm.
*/
-typedef struct porter_tokenizer {
- sqlite3_tokenizer base; /* Base class */
-} porter_tokenizer;
+static int fts3SegReaderTermCmp(
+ Fts3SegReader *pSeg, /* Segment reader object */
+ const char *zTerm, /* Term to compare to */
+ int nTerm /* Size of term zTerm in bytes */
+){
+ int res = 0;
+ if( pSeg->aNode ){
+ if( pSeg->nTerm>nTerm ){
+ res = memcmp(pSeg->zTerm, zTerm, nTerm);
+ }else{
+ res = memcmp(pSeg->zTerm, zTerm, pSeg->nTerm);
+ }
+ if( res==0 ){
+ res = pSeg->nTerm-nTerm;
+ }
+ }
+ return res;
+}
/*
-** Class derived from sqlit3_tokenizer_cursor
+** Argument apSegment is an array of nSegment elements. It is known that
+** the final (nSegment-nSuspect) members are already in sorted order
+** (according to the comparison function provided). This function shuffles
+** the array around until all entries are in sorted order.
*/
-typedef struct porter_tokenizer_cursor {
- sqlite3_tokenizer_cursor base;
- const char *zInput; /* input we are tokenizing */
- int nInput; /* size of the input */
- int iOffset; /* current position in zInput */
- int iToken; /* index of next token to be returned */
- char *zToken; /* storage for current token */
- int nAllocated; /* space allocated to zToken buffer */
-} porter_tokenizer_cursor;
+static void fts3SegReaderSort(
+ Fts3SegReader **apSegment, /* Array to sort entries of */
+ int nSegment, /* Size of apSegment array */
+ int nSuspect, /* Unsorted entry count */
+ int (*xCmp)(Fts3SegReader *, Fts3SegReader *) /* Comparison function */
+){
+ int i; /* Iterator variable */
+ assert( nSuspect<=nSegment );
-/* Forward declaration */
-static const sqlite3_tokenizer_module porterTokenizerModule;
+ if( nSuspect==nSegment ) nSuspect--;
+ for(i=nSuspect-1; i>=0; i--){
+ int j;
+ for(j=i; j<(nSegment-1); j++){
+ Fts3SegReader *pTmp;
+ if( xCmp(apSegment[j], apSegment[j+1])<0 ) break;
+ pTmp = apSegment[j+1];
+ apSegment[j+1] = apSegment[j];
+ apSegment[j] = pTmp;
+ }
+ }
+#ifndef NDEBUG
+ /* Check that the list really is sorted now. */
+ for(i=0; i<(nSuspect-1); i++){
+ assert( xCmp(apSegment[i], apSegment[i+1])<0 );
+ }
+#endif
+}
-/*
-** Create a new tokenizer instance.
+/*
+** Insert a record into the %_segments table.
*/
-static int porterCreate(
- int argc, const char * const *argv,
- sqlite3_tokenizer **ppTokenizer
+static int fts3WriteSegment(
+ Fts3Table *p, /* Virtual table handle */
+ sqlite3_int64 iBlock, /* Block id for new block */
+ char *z, /* Pointer to buffer containing block data */
+ int n /* Size of buffer z in bytes */
){
- porter_tokenizer *t;
- t = (porter_tokenizer *) sqlite3_malloc(sizeof(*t));
- if( t==NULL ) return SQLITE_NOMEM;
- memset(t, 0, sizeof(*t));
- *ppTokenizer = &t->base;
- return SQLITE_OK;
+ sqlite3_stmt *pStmt;
+ int rc = fts3SqlStmt(p, SQL_INSERT_SEGMENTS, &pStmt, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_int64(pStmt, 1, iBlock);
+ rc = sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC);
+ if( rc==SQLITE_OK ){
+ sqlite3_step(pStmt);
+ rc = sqlite3_reset(pStmt);
+ }
+ }
+ return rc;
+}
+
+/*
+** Insert a record into the %_segdir table.
+*/
+static int fts3WriteSegdir(
+ Fts3Table *p, /* Virtual table handle */
+ int iLevel, /* Value for "level" field */
+ int iIdx, /* Value for "idx" field */
+ sqlite3_int64 iStartBlock, /* Value for "start_block" field */
+ sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */
+ sqlite3_int64 iEndBlock, /* Value for "end_block" field */
+ char *zRoot, /* Blob value for "root" field */
+ int nRoot /* Number of bytes in buffer zRoot */
+){
+ sqlite3_stmt *pStmt;
+ int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_int(pStmt, 1, iLevel);
+ sqlite3_bind_int(pStmt, 2, iIdx);
+ sqlite3_bind_int64(pStmt, 3, iStartBlock);
+ sqlite3_bind_int64(pStmt, 4, iLeafEndBlock);
+ sqlite3_bind_int64(pStmt, 5, iEndBlock);
+ rc = sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC);
+ if( rc==SQLITE_OK ){
+ sqlite3_step(pStmt);
+ rc = sqlite3_reset(pStmt);
+ }
+ }
+ return rc;
}
/*
-** Destroy a tokenizer
+** Return the size of the common prefix (if any) shared by zPrev and
+** zNext, in bytes. For example,
+**
+** fts3PrefixCompress("abc", 3, "abcdef", 6) // returns 3
+** fts3PrefixCompress("abX", 3, "abcdef", 6) // returns 2
+** fts3PrefixCompress("abX", 3, "Xbcdef", 6) // returns 0
*/
-static int porterDestroy(sqlite3_tokenizer *pTokenizer){
- sqlite3_free(pTokenizer);
- return SQLITE_OK;
+static int fts3PrefixCompress(
+ const char *zPrev, /* Buffer containing previous term */
+ int nPrev, /* Size of buffer zPrev in bytes */
+ const char *zNext, /* Buffer containing next term */
+ int nNext /* Size of buffer zNext in bytes */
+){
+ int n;
+ UNUSED_PARAMETER(nNext);
+ for(n=0; n<nPrev && zPrev[n]==zNext[n]; n++);
+ return n;
}
/*
-** Prepare to begin tokenizing a particular string. The input
-** string to be tokenized is zInput[0..nInput-1]. A cursor
-** used to incrementally tokenize this string is returned in
-** *ppCursor.
+** Add term zTerm to the SegmentNode. It is guaranteed that zTerm is larger
+** (according to memcmp) than the previous term.
*/
-static int porterOpen(
- sqlite3_tokenizer *pTokenizer, /* The tokenizer */
- const char *zInput, int nInput, /* String to be tokenized */
- sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
+static int fts3NodeAddTerm(
+ Fts3Table *p, /* Virtual table handle */
+ SegmentNode **ppTree, /* IN/OUT: SegmentNode handle */
+ int isCopyTerm, /* True if zTerm/nTerm is transient */
+ const char *zTerm, /* Pointer to buffer containing term */
+ int nTerm /* Size of term in bytes */
){
- porter_tokenizer_cursor *c;
+ SegmentNode *pTree = *ppTree;
+ int rc;
+ SegmentNode *pNew;
+
+ /* First try to append the term to the current node. Return early if
+ ** this is possible.
+ */
+ if( pTree ){
+ int nData = pTree->nData; /* Current size of node in bytes */
+ int nReq = nData; /* Required space after adding zTerm */
+ int nPrefix; /* Number of bytes of prefix compression */
+ int nSuffix; /* Suffix length */
+
+ nPrefix = fts3PrefixCompress(pTree->zTerm, pTree->nTerm, zTerm, nTerm);
+ nSuffix = nTerm-nPrefix;
+
+ nReq += sqlite3Fts3VarintLen(nPrefix)+sqlite3Fts3VarintLen(nSuffix)+nSuffix;
+ if( nReq<=p->nNodeSize || !pTree->zTerm ){
+
+ if( nReq>p->nNodeSize ){
+ /* An unusual case: this is the first term to be added to the node
+ ** and the static node buffer (p->nNodeSize bytes) is not large
+ ** enough. Use a separately malloced buffer instead This wastes
+ ** p->nNodeSize bytes, but since this scenario only comes about when
+ ** the database contain two terms that share a prefix of almost 2KB,
+ ** this is not expected to be a serious problem.
+ */
+ assert( pTree->aData==(char *)&pTree[1] );
+ pTree->aData = (char *)sqlite3_malloc(nReq);
+ if( !pTree->aData ){
+ return SQLITE_NOMEM;
+ }
+ }
- c = (porter_tokenizer_cursor *) sqlite3_malloc(sizeof(*c));
- if( c==NULL ) return SQLITE_NOMEM;
+ if( pTree->zTerm ){
+ /* There is no prefix-length field for first term in a node */
+ nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nPrefix);
+ }
- c->zInput = zInput;
- if( zInput==0 ){
- c->nInput = 0;
- }else if( nInput<0 ){
- c->nInput = (int)strlen(zInput);
+ nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nSuffix);
+ memcpy(&pTree->aData[nData], &zTerm[nPrefix], nSuffix);
+ pTree->nData = nData + nSuffix;
+ pTree->nEntry++;
+
+ if( isCopyTerm ){
+ if( pTree->nMalloc<nTerm ){
+ char *zNew = sqlite3_realloc(pTree->zMalloc, nTerm*2);
+ if( !zNew ){
+ return SQLITE_NOMEM;
+ }
+ pTree->nMalloc = nTerm*2;
+ pTree->zMalloc = zNew;
+ }
+ pTree->zTerm = pTree->zMalloc;
+ memcpy(pTree->zTerm, zTerm, nTerm);
+ pTree->nTerm = nTerm;
+ }else{
+ pTree->zTerm = (char *)zTerm;
+ pTree->nTerm = nTerm;
+ }
+ return SQLITE_OK;
+ }
+ }
+
+ /* If control flows to here, it was not possible to append zTerm to the
+ ** current node. Create a new node (a right-sibling of the current node).
+ ** If this is the first node in the tree, the term is added to it.
+ **
+ ** Otherwise, the term is not added to the new node, it is left empty for
+ ** now. Instead, the term is inserted into the parent of pTree. If pTree
+ ** has no parent, one is created here.
+ */
+ pNew = (SegmentNode *)sqlite3_malloc(sizeof(SegmentNode) + p->nNodeSize);
+ if( !pNew ){
+ return SQLITE_NOMEM;
+ }
+ memset(pNew, 0, sizeof(SegmentNode));
+ pNew->nData = 1 + FTS3_VARINT_MAX;
+ pNew->aData = (char *)&pNew[1];
+
+ if( pTree ){
+ SegmentNode *pParent = pTree->pParent;
+ rc = fts3NodeAddTerm(p, &pParent, isCopyTerm, zTerm, nTerm);
+ if( pTree->pParent==0 ){
+ pTree->pParent = pParent;
+ }
+ pTree->pRight = pNew;
+ pNew->pLeftmost = pTree->pLeftmost;
+ pNew->pParent = pParent;
+ pNew->zMalloc = pTree->zMalloc;
+ pNew->nMalloc = pTree->nMalloc;
+ pTree->zMalloc = 0;
}else{
- c->nInput = nInput;
+ pNew->pLeftmost = pNew;
+ rc = fts3NodeAddTerm(p, &pNew, isCopyTerm, zTerm, nTerm);
}
- c->iOffset = 0; /* start tokenizing at the beginning */
- c->iToken = 0;
- c->zToken = NULL; /* no space allocated, yet. */
- c->nAllocated = 0;
- *ppCursor = &c->base;
- return SQLITE_OK;
+ *ppTree = pNew;
+ return rc;
}
/*
-** Close a tokenization cursor previously opened by a call to
-** porterOpen() above.
+** Helper function for fts3NodeWrite().
*/
-static int porterClose(sqlite3_tokenizer_cursor *pCursor){
- porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor;
- sqlite3_free(c->zToken);
- sqlite3_free(c);
- return SQLITE_OK;
+static int fts3TreeFinishNode(
+ SegmentNode *pTree,
+ int iHeight,
+ sqlite3_int64 iLeftChild
+){
+ int nStart;
+ assert( iHeight>=1 && iHeight<128 );
+ nStart = FTS3_VARINT_MAX - sqlite3Fts3VarintLen(iLeftChild);
+ pTree->aData[nStart] = (char)iHeight;
+ sqlite3Fts3PutVarint(&pTree->aData[nStart+1], iLeftChild);
+ return nStart;
}
-/*
-** Vowel or consonant
-*/
-static const char cType[] = {
- 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0,
- 1, 1, 1, 2, 1
-};
/*
-** isConsonant() and isVowel() determine if their first character in
-** the string they point to is a consonant or a vowel, according
-** to Porter ruls.
+** Write the buffer for the segment node pTree and all of its peers to the
+** database. Then call this function recursively to write the parent of
+** pTree and its peers to the database.
**
-** A consonate is any letter other than 'a', 'e', 'i', 'o', or 'u'.
-** 'Y' is a consonant unless it follows another consonant,
-** in which case it is a vowel.
+** Except, if pTree is a root node, do not write it to the database. Instead,
+** set output variables *paRoot and *pnRoot to contain the root node.
**
-** In these routine, the letters are in reverse order. So the 'y' rule
-** is that 'y' is a consonant unless it is followed by another
-** consonent.
+** If successful, SQLITE_OK is returned and output variable *piLast is
+** set to the largest blockid written to the database (or zero if no
+** blocks were written to the db). Otherwise, an SQLite error code is
+** returned.
*/
-static int isVowel(const char*);
-static int isConsonant(const char *z){
- int j;
- char x = *z;
- if( x==0 ) return 0;
- assert( x>='a' && x<='z' );
- j = cType[x-'a'];
- if( j<2 ) return j;
- return z[1]==0 || isVowel(z + 1);
+static int fts3NodeWrite(
+ Fts3Table *p, /* Virtual table handle */
+ SegmentNode *pTree, /* SegmentNode handle */
+ int iHeight, /* Height of this node in tree */
+ sqlite3_int64 iLeaf, /* Block id of first leaf node */
+ sqlite3_int64 iFree, /* Block id of next free slot in %_segments */
+ sqlite3_int64 *piLast, /* OUT: Block id of last entry written */
+ char **paRoot, /* OUT: Data for root node */
+ int *pnRoot /* OUT: Size of root node in bytes */
+){
+ int rc = SQLITE_OK;
+
+ if( !pTree->pParent ){
+ /* Root node of the tree. */
+ int nStart = fts3TreeFinishNode(pTree, iHeight, iLeaf);
+ *piLast = iFree-1;
+ *pnRoot = pTree->nData - nStart;
+ *paRoot = &pTree->aData[nStart];
+ }else{
+ SegmentNode *pIter;
+ sqlite3_int64 iNextFree = iFree;
+ sqlite3_int64 iNextLeaf = iLeaf;
+ for(pIter=pTree->pLeftmost; pIter && rc==SQLITE_OK; pIter=pIter->pRight){
+ int nStart = fts3TreeFinishNode(pIter, iHeight, iNextLeaf);
+ int nWrite = pIter->nData - nStart;
+
+ rc = fts3WriteSegment(p, iNextFree, &pIter->aData[nStart], nWrite);
+ iNextFree++;
+ iNextLeaf += (pIter->nEntry+1);
+ }
+ if( rc==SQLITE_OK ){
+ assert( iNextLeaf==iFree );
+ rc = fts3NodeWrite(
+ p, pTree->pParent, iHeight+1, iFree, iNextFree, piLast, paRoot, pnRoot
+ );
+ }
+ }
+
+ return rc;
}
-static int isVowel(const char *z){
- int j;
- char x = *z;
- if( x==0 ) return 0;
- assert( x>='a' && x<='z' );
- j = cType[x-'a'];
- if( j<2 ) return 1-j;
- return isConsonant(z + 1);
+
+/*
+** Free all memory allocations associated with the tree pTree.
+*/
+static void fts3NodeFree(SegmentNode *pTree){
+ if( pTree ){
+ SegmentNode *p = pTree->pLeftmost;
+ fts3NodeFree(p->pParent);
+ while( p ){
+ SegmentNode *pRight = p->pRight;
+ if( p->aData!=(char *)&p[1] ){
+ sqlite3_free(p->aData);
+ }
+ assert( pRight==0 || p->zMalloc==0 );
+ sqlite3_free(p->zMalloc);
+ sqlite3_free(p);
+ p = pRight;
+ }
+ }
}
/*
-** Let any sequence of one or more vowels be represented by V and let
-** C be sequence of one or more consonants. Then every word can be
-** represented as:
-**
-** [C] (VC){m} [V]
-**
-** In prose: A word is an optional consonant followed by zero or
-** vowel-consonant pairs followed by an optional vowel. "m" is the
-** number of vowel consonant pairs. This routine computes the value
-** of m for the first i bytes of a word.
-**
-** Return true if the m-value for z is 1 or more. In other words,
-** return true if z contains at least one vowel that is followed
-** by a consonant.
+** Add a term to the segment being constructed by the SegmentWriter object
+** *ppWriter. When adding the first term to a segment, *ppWriter should
+** be passed NULL. This function will allocate a new SegmentWriter object
+** and return it via the input/output variable *ppWriter in this case.
**
-** In this routine z[] is in reverse order. So we are really looking
-** for an instance of of a consonant followed by a vowel.
+** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code.
*/
-static int m_gt_0(const char *z){
- while( isVowel(z) ){ z++; }
- if( *z==0 ) return 0;
- while( isConsonant(z) ){ z++; }
- return *z!=0;
+static int fts3SegWriterAdd(
+ Fts3Table *p, /* Virtual table handle */
+ SegmentWriter **ppWriter, /* IN/OUT: SegmentWriter handle */
+ int isCopyTerm, /* True if buffer zTerm must be copied */
+ const char *zTerm, /* Pointer to buffer containing term */
+ int nTerm, /* Size of term in bytes */
+ const char *aDoclist, /* Pointer to buffer containing doclist */
+ int nDoclist /* Size of doclist in bytes */
+){
+ int nPrefix; /* Size of term prefix in bytes */
+ int nSuffix; /* Size of term suffix in bytes */
+ int nReq; /* Number of bytes required on leaf page */
+ int nData;
+ SegmentWriter *pWriter = *ppWriter;
+
+ if( !pWriter ){
+ int rc;
+ sqlite3_stmt *pStmt;
+
+ /* Allocate the SegmentWriter structure */
+ pWriter = (SegmentWriter *)sqlite3_malloc(sizeof(SegmentWriter));
+ if( !pWriter ) return SQLITE_NOMEM;
+ memset(pWriter, 0, sizeof(SegmentWriter));
+ *ppWriter = pWriter;
+
+ /* Allocate a buffer in which to accumulate data */
+ pWriter->aData = (char *)sqlite3_malloc(p->nNodeSize);
+ if( !pWriter->aData ) return SQLITE_NOMEM;
+ pWriter->nSize = p->nNodeSize;
+
+ /* Find the next free blockid in the %_segments table */
+ rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pStmt, 0);
+ if( rc!=SQLITE_OK ) return rc;
+ if( SQLITE_ROW==sqlite3_step(pStmt) ){
+ pWriter->iFree = sqlite3_column_int64(pStmt, 0);
+ pWriter->iFirst = pWriter->iFree;
+ }
+ rc = sqlite3_reset(pStmt);
+ if( rc!=SQLITE_OK ) return rc;
+ }
+ nData = pWriter->nData;
+
+ nPrefix = fts3PrefixCompress(pWriter->zTerm, pWriter->nTerm, zTerm, nTerm);
+ nSuffix = nTerm-nPrefix;
+
+ /* Figure out how many bytes are required by this new entry */
+ nReq = sqlite3Fts3VarintLen(nPrefix) + /* varint containing prefix size */
+ sqlite3Fts3VarintLen(nSuffix) + /* varint containing suffix size */
+ nSuffix + /* Term suffix */
+ sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */
+ nDoclist; /* Doclist data */
+
+ if( nData>0 && nData+nReq>p->nNodeSize ){
+ int rc;
+
+ /* The current leaf node is full. Write it out to the database. */
+ rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData);
+ if( rc!=SQLITE_OK ) return rc;
+
+ /* Add the current term to the interior node tree. The term added to
+ ** the interior tree must:
+ **
+ ** a) be greater than the largest term on the leaf node just written
+ ** to the database (still available in pWriter->zTerm), and
+ **
+ ** b) be less than or equal to the term about to be added to the new
+ ** leaf node (zTerm/nTerm).
+ **
+ ** In other words, it must be the prefix of zTerm 1 byte longer than
+ ** the common prefix (if any) of zTerm and pWriter->zTerm.
+ */
+ assert( nPrefix<nTerm );
+ rc = fts3NodeAddTerm(p, &pWriter->pTree, isCopyTerm, zTerm, nPrefix+1);
+ if( rc!=SQLITE_OK ) return rc;
+
+ nData = 0;
+ pWriter->nTerm = 0;
+
+ nPrefix = 0;
+ nSuffix = nTerm;
+ nReq = 1 + /* varint containing prefix size */
+ sqlite3Fts3VarintLen(nTerm) + /* varint containing suffix size */
+ nTerm + /* Term suffix */
+ sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */
+ nDoclist; /* Doclist data */
+ }
+
+ /* If the buffer currently allocated is too small for this entry, realloc
+ ** the buffer to make it large enough.
+ */
+ if( nReq>pWriter->nSize ){
+ char *aNew = sqlite3_realloc(pWriter->aData, nReq);
+ if( !aNew ) return SQLITE_NOMEM;
+ pWriter->aData = aNew;
+ pWriter->nSize = nReq;
+ }
+ assert( nData+nReq<=pWriter->nSize );
+
+ /* Append the prefix-compressed term and doclist to the buffer. */
+ nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nPrefix);
+ nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nSuffix);
+ memcpy(&pWriter->aData[nData], &zTerm[nPrefix], nSuffix);
+ nData += nSuffix;
+ nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nDoclist);
+ memcpy(&pWriter->aData[nData], aDoclist, nDoclist);
+ pWriter->nData = nData + nDoclist;
+
+ /* Save the current term so that it can be used to prefix-compress the next.
+ ** If the isCopyTerm parameter is true, then the buffer pointed to by
+ ** zTerm is transient, so take a copy of the term data. Otherwise, just
+ ** store a copy of the pointer.
+ */
+ if( isCopyTerm ){
+ if( nTerm>pWriter->nMalloc ){
+ char *zNew = sqlite3_realloc(pWriter->zMalloc, nTerm*2);
+ if( !zNew ){
+ return SQLITE_NOMEM;
+ }
+ pWriter->nMalloc = nTerm*2;
+ pWriter->zMalloc = zNew;
+ pWriter->zTerm = zNew;
+ }
+ assert( pWriter->zTerm==pWriter->zMalloc );
+ memcpy(pWriter->zTerm, zTerm, nTerm);
+ }else{
+ pWriter->zTerm = (char *)zTerm;
+ }
+ pWriter->nTerm = nTerm;
+
+ return SQLITE_OK;
}
-/* Like mgt0 above except we are looking for a value of m which is
-** exactly 1
+/*
+** Flush all data associated with the SegmentWriter object pWriter to the
+** database. This function must be called after all terms have been added
+** to the segment using fts3SegWriterAdd(). If successful, SQLITE_OK is
+** returned. Otherwise, an SQLite error code.
*/
-static int m_eq_1(const char *z){
- while( isVowel(z) ){ z++; }
- if( *z==0 ) return 0;
- while( isConsonant(z) ){ z++; }
- if( *z==0 ) return 0;
- while( isVowel(z) ){ z++; }
- if( *z==0 ) return 1;
- while( isConsonant(z) ){ z++; }
- return *z==0;
+static int fts3SegWriterFlush(
+ Fts3Table *p, /* Virtual table handle */
+ SegmentWriter *pWriter, /* SegmentWriter to flush to the db */
+ int iLevel, /* Value for 'level' column of %_segdir */
+ int iIdx /* Value for 'idx' column of %_segdir */
+){
+ int rc; /* Return code */
+ if( pWriter->pTree ){
+ sqlite3_int64 iLast = 0; /* Largest block id written to database */
+ sqlite3_int64 iLastLeaf; /* Largest leaf block id written to db */
+ char *zRoot = NULL; /* Pointer to buffer containing root node */
+ int nRoot = 0; /* Size of buffer zRoot */
+
+ iLastLeaf = pWriter->iFree;
+ rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, pWriter->nData);
+ if( rc==SQLITE_OK ){
+ rc = fts3NodeWrite(p, pWriter->pTree, 1,
+ pWriter->iFirst, pWriter->iFree, &iLast, &zRoot, &nRoot);
+ }
+ if( rc==SQLITE_OK ){
+ rc = fts3WriteSegdir(
+ p, iLevel, iIdx, pWriter->iFirst, iLastLeaf, iLast, zRoot, nRoot);
+ }
+ }else{
+ /* The entire tree fits on the root node. Write it to the segdir table. */
+ rc = fts3WriteSegdir(
+ p, iLevel, iIdx, 0, 0, 0, pWriter->aData, pWriter->nData);
+ }
+ return rc;
}
-/* Like mgt0 above except we are looking for a value of m>1 instead
-** or m>0
+/*
+** Release all memory held by the SegmentWriter object passed as the
+** first argument.
*/
-static int m_gt_1(const char *z){
- while( isVowel(z) ){ z++; }
- if( *z==0 ) return 0;
- while( isConsonant(z) ){ z++; }
- if( *z==0 ) return 0;
- while( isVowel(z) ){ z++; }
- if( *z==0 ) return 0;
- while( isConsonant(z) ){ z++; }
- return *z!=0;
+static void fts3SegWriterFree(SegmentWriter *pWriter){
+ if( pWriter ){
+ sqlite3_free(pWriter->aData);
+ sqlite3_free(pWriter->zMalloc);
+ fts3NodeFree(pWriter->pTree);
+ sqlite3_free(pWriter);
+ }
}
/*
-** Return TRUE if there is a vowel anywhere within z[0..n-1]
+** The first value in the apVal[] array is assumed to contain an integer.
+** This function tests if there exist any documents with docid values that
+** are different from that integer. i.e. if deleting the document with docid
+** apVal[0] would mean the FTS3 table were empty.
+**
+** If successful, *pisEmpty is set to true if the table is empty except for
+** document apVal[0], or false otherwise, and SQLITE_OK is returned. If an
+** error occurs, an SQLite error code is returned.
*/
-static int hasVowel(const char *z){
- while( isConsonant(z) ){ z++; }
- return *z!=0;
+static int fts3IsEmpty(Fts3Table *p, sqlite3_value **apVal, int *pisEmpty){
+ sqlite3_stmt *pStmt;
+ int rc;
+ rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, apVal);
+ if( rc==SQLITE_OK ){
+ if( SQLITE_ROW==sqlite3_step(pStmt) ){
+ *pisEmpty = sqlite3_column_int(pStmt, 0);
+ }
+ rc = sqlite3_reset(pStmt);
+ }
+ return rc;
}
/*
-** Return TRUE if the word ends in a double consonant.
+** Set *pnSegment to the number of segments of level iLevel in the database.
**
-** The text is reversed here. So we are really looking at
-** the first two characters of z[].
+** Return SQLITE_OK if successful, or an SQLite error code if not.
*/
-static int doubleConsonant(const char *z){
- return isConsonant(z) && z[0]==z[1] && isConsonant(z+1);
+static int fts3SegmentCount(Fts3Table *p, int iLevel, int *pnSegment){
+ sqlite3_stmt *pStmt;
+ int rc;
+
+ assert( iLevel>=0 );
+ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_COUNT, &pStmt, 0);
+ if( rc!=SQLITE_OK ) return rc;
+ sqlite3_bind_int(pStmt, 1, iLevel);
+ if( SQLITE_ROW==sqlite3_step(pStmt) ){
+ *pnSegment = sqlite3_column_int(pStmt, 0);
+ }
+ return sqlite3_reset(pStmt);
}
/*
-** Return TRUE if the word ends with three letters which
-** are consonant-vowel-consonent and where the final consonant
-** is not 'w', 'x', or 'y'.
+** Set *pnSegment to the total number of segments in the database. Set
+** *pnMax to the largest segment level in the database (segment levels
+** are stored in the 'level' column of the %_segdir table).
**
-** The word is reversed here. So we are really checking the
-** first three letters and the first one cannot be in [wxy].
+** Return SQLITE_OK if successful, or an SQLite error code if not.
*/
-static int star_oh(const char *z){
- return
- z[0]!=0 && isConsonant(z) &&
- z[0]!='w' && z[0]!='x' && z[0]!='y' &&
- z[1]!=0 && isVowel(z+1) &&
- z[2]!=0 && isConsonant(z+2);
+static int fts3SegmentCountMax(Fts3Table *p, int *pnSegment, int *pnMax){
+ sqlite3_stmt *pStmt;
+ int rc;
+
+ rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_COUNT_MAX, &pStmt, 0);
+ if( rc!=SQLITE_OK ) return rc;
+ if( SQLITE_ROW==sqlite3_step(pStmt) ){
+ *pnSegment = sqlite3_column_int(pStmt, 0);
+ *pnMax = sqlite3_column_int(pStmt, 1);
+ }
+ return sqlite3_reset(pStmt);
}
/*
-** If the word ends with zFrom and xCond() is true for the stem
-** of the word that preceeds the zFrom ending, then change the
-** ending to zTo.
+** This function is used after merging multiple segments into a single large
+** segment to delete the old, now redundant, segment b-trees. Specifically,
+** it:
+**
+** 1) Deletes all %_segments entries for the segments associated with
+** each of the SegReader objects in the array passed as the third
+** argument, and
**
-** The input word *pz and zFrom are both in reverse order. zTo
-** is in normal order.
+** 2) deletes all %_segdir entries with level iLevel, or all %_segdir
+** entries regardless of level if (iLevel<0).
**
-** Return TRUE if zFrom matches. Return FALSE if zFrom does not
-** match. Not that TRUE is returned even if xCond() fails and
-** no substitution occurs.
+** SQLITE_OK is returned if successful, otherwise an SQLite error code.
*/
-static int stem(
- char **pz, /* The word being stemmed (Reversed) */
- const char *zFrom, /* If the ending matches this... (Reversed) */
- const char *zTo, /* ... change the ending to this (not reversed) */
- int (*xCond)(const char*) /* Condition that must be true */
+static int fts3DeleteSegdir(
+ Fts3Table *p, /* Virtual table handle */
+ int iLevel, /* Level of %_segdir entries to delete */
+ Fts3SegReader **apSegment, /* Array of SegReader objects */
+ int nReader /* Size of array apSegment */
){
- char *z = *pz;
- while( *zFrom && *zFrom==*z ){ z++; zFrom++; }
- if( *zFrom!=0 ) return 0;
- if( xCond && !xCond(z) ) return 1;
- while( *zTo ){
- *(--z) = *(zTo++);
+ int rc; /* Return Code */
+ int i; /* Iterator variable */
+ sqlite3_stmt *pDelete; /* SQL statement to delete rows */
+
+ rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDelete, 0);
+ for(i=0; rc==SQLITE_OK && i<nReader; i++){
+ Fts3SegReader *pSegment = apSegment[i];
+ if( pSegment->iStartBlock ){
+ sqlite3_bind_int64(pDelete, 1, pSegment->iStartBlock);
+ sqlite3_bind_int64(pDelete, 2, pSegment->iEndBlock);
+ sqlite3_step(pDelete);
+ rc = sqlite3_reset(pDelete);
+ }
}
- *pz = z;
- return 1;
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+
+ if( iLevel>=0 ){
+ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_BY_LEVEL, &pDelete, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_int(pDelete, 1, iLevel);
+ sqlite3_step(pDelete);
+ rc = sqlite3_reset(pDelete);
+ }
+ }else{
+ rc = fts3SqlExec(p, SQL_DELETE_ALL_SEGDIR, 0);
+ }
+
+ return rc;
}
/*
-** This is the fallback stemmer used when the porter stemmer is
-** inappropriate. The input word is copied into the output with
-** US-ASCII case folding. If the input word is too long (more
-** than 20 bytes if it contains no digits or more than 6 bytes if
-** it contains digits) then word is truncated to 20 or 6 bytes
-** by taking 10 or 3 bytes from the beginning and end.
+** When this function is called, buffer *ppList (size *pnList bytes) contains
+** a position list that may (or may not) feature multiple columns. This
+** function adjusts the pointer *ppList and the length *pnList so that they
+** identify the subset of the position list that corresponds to column iCol.
+**
+** If there are no entries in the input position list for column iCol, then
+** *pnList is set to zero before returning.
*/
-static void copy_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
- int i, mx, j;
- int hasDigit = 0;
- for(i=0; i<nIn; i++){
- int c = zIn[i];
- if( c>='A' && c<='Z' ){
- zOut[i] = c - 'A' + 'a';
- }else{
- if( c>='0' && c<='9' ) hasDigit = 1;
- zOut[i] = c;
+static void fts3ColumnFilter(
+ int iCol, /* Column to filter on */
+ char **ppList, /* IN/OUT: Pointer to position list */
+ int *pnList /* IN/OUT: Size of buffer *ppList in bytes */
+){
+ char *pList = *ppList;
+ int nList = *pnList;
+ char *pEnd = &pList[nList];
+ int iCurrent = 0;
+ char *p = pList;
+
+ assert( iCol>=0 );
+ while( 1 ){
+ char c = 0;
+ while( p<pEnd && (c | *p)&0xFE ) c = *p++ & 0x80;
+
+ if( iCol==iCurrent ){
+ nList = (int)(p - pList);
+ break;
}
- }
- mx = hasDigit ? 3 : 10;
- if( nIn>mx*2 ){
- for(j=mx, i=nIn-mx; i<nIn; i++, j++){
- zOut[j] = zOut[i];
+
+ nList -= (int)(p - pList);
+ pList = p;
+ if( nList==0 ){
+ break;
}
- i = j;
+ p = &pList[1];
+ p += sqlite3Fts3GetVarint32(p, &iCurrent);
}
- zOut[i] = 0;
- *pnOut = i;
-}
+ *ppList = pList;
+ *pnList = nList;
+}
/*
-** Stem the input word zIn[0..nIn-1]. Store the output in zOut.
-** zOut is at least big enough to hold nIn bytes. Write the actual
-** size of the output word (exclusive of the '\0' terminator) into *pnOut.
-**
-** Any upper-case characters in the US-ASCII character set ([A-Z])
-** are converted to lower case. Upper-case UTF characters are
-** unchanged.
-**
-** Words that are longer than about 20 bytes are stemmed by retaining
-** a few bytes from the beginning and the end of the word. If the
-** word contains digits, 3 bytes are taken from the beginning and
-** 3 bytes from the end. For long words without digits, 10 bytes
-** are taken from each end. US-ASCII case folding still applies.
-**
-** If the input word contains not digits but does characters not
-** in [a-zA-Z] then no stemming is attempted and this routine just
-** copies the input into the input into the output with US-ASCII
-** case folding.
-**
-** Stemming never increases the length of the word. So there is
-** no chance of overflowing the zOut buffer.
+** sqlite3Fts3SegReaderIterate() callback used when merging multiple
+** segments to create a single, larger segment.
*/
-static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
- int i, j, c;
- char zReverse[28];
- char *z, *z2;
- if( nIn<3 || nIn>=sizeof(zReverse)-7 ){
- /* The word is too big or too small for the porter stemmer.
- ** Fallback to the copy stemmer */
- copy_stemmer(zIn, nIn, zOut, pnOut);
- return;
- }
- for(i=0, j=sizeof(zReverse)-6; i<nIn; i++, j--){
- c = zIn[i];
- if( c>='A' && c<='Z' ){
- zReverse[j] = c + 'a' - 'A';
- }else if( c>='a' && c<='z' ){
- zReverse[j] = c;
- }else{
- /* The use of a character not in [a-zA-Z] means that we fallback
- ** to the copy stemmer */
- copy_stemmer(zIn, nIn, zOut, pnOut);
- return;
+static int fts3MergeCallback(
+ Fts3Table *p,
+ void *pContext,
+ char *zTerm,
+ int nTerm,
+ char *aDoclist,
+ int nDoclist
+){
+ SegmentWriter **ppW = (SegmentWriter **)pContext;
+ return fts3SegWriterAdd(p, ppW, 1, zTerm, nTerm, aDoclist, nDoclist);
+}
+
+/*
+** This function is used to iterate through a contiguous set of terms
+** stored in the full-text index. It merges data contained in one or
+** more segments to support this.
+**
+** The second argument is passed an array of pointers to SegReader objects
+** allocated with sqlite3Fts3SegReaderNew(). This function merges the range
+** of terms selected by each SegReader. If a single term is present in
+** more than one segment, the associated doclists are merged. For each
+** term and (possibly merged) doclist in the merged range, the callback
+** function xFunc is invoked with its arguments set as follows.
+**
+** arg 0: Copy of 'p' parameter passed to this function
+** arg 1: Copy of 'pContext' parameter passed to this function
+** arg 2: Pointer to buffer containing term
+** arg 3: Size of arg 2 buffer in bytes
+** arg 4: Pointer to buffer containing doclist
+** arg 5: Size of arg 2 buffer in bytes
+**
+** The 4th argument to this function is a pointer to a structure of type
+** Fts3SegFilter, defined in fts3Int.h. The contents of this structure
+** further restrict the range of terms that callbacks are made for and
+** modify the behaviour of this function. See comments above structure
+** definition for details.
+*/
+SQLITE_PRIVATE int sqlite3Fts3SegReaderIterate(
+ Fts3Table *p, /* Virtual table handle */
+ Fts3SegReader **apSegment, /* Array of Fts3SegReader objects */
+ int nSegment, /* Size of apSegment array */
+ Fts3SegFilter *pFilter, /* Restrictions on range of iteration */
+ int (*xFunc)(Fts3Table *, void *, char *, int, char *, int), /* Callback */
+ void *pContext /* Callback context (2nd argument) */
+){
+ int i; /* Iterator variable */
+ char *aBuffer = 0; /* Buffer to merge doclists in */
+ int nAlloc = 0; /* Allocated size of aBuffer buffer */
+ int rc = SQLITE_OK; /* Return code */
+
+ int isIgnoreEmpty = (pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY);
+ int isRequirePos = (pFilter->flags & FTS3_SEGMENT_REQUIRE_POS);
+ int isColFilter = (pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER);
+ int isPrefix = (pFilter->flags & FTS3_SEGMENT_PREFIX);
+
+ /* If there are zero segments, this function is a no-op. This scenario
+ ** comes about only when reading from an empty database.
+ */
+ if( nSegment==0 ) goto finished;
+
+ /* If the Fts3SegFilter defines a specific term (or term prefix) to search
+ ** for, then advance each segment iterator until it points to a term of
+ ** equal or greater value than the specified term. This prevents many
+ ** unnecessary merge/sort operations for the case where single segment
+ ** b-tree leaf nodes contain more than one term.
+ */
+ if( pFilter->zTerm ){
+ int nTerm = pFilter->nTerm;
+ const char *zTerm = pFilter->zTerm;
+ for(i=0; i<nSegment; i++){
+ Fts3SegReader *pSeg = apSegment[i];
+ while( fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ){
+ rc = fts3SegReaderNext(pSeg);
+ if( rc!=SQLITE_OK ) goto finished;
+ }
}
}
- memset(&zReverse[sizeof(zReverse)-5], 0, 5);
- z = &zReverse[j+1];
+ fts3SegReaderSort(apSegment, nSegment, nSegment, fts3SegReaderCmp);
+ while( apSegment[0]->aNode ){
+ int nTerm = apSegment[0]->nTerm;
+ char *zTerm = apSegment[0]->zTerm;
+ int nMerge = 1;
- /* Step 1a */
- if( z[0]=='s' ){
- if(
- !stem(&z, "sess", "ss", 0) &&
- !stem(&z, "sei", "i", 0) &&
- !stem(&z, "ss", "ss", 0)
+ /* If this is a prefix-search, and if the term that apSegment[0] points
+ ** to does not share a suffix with pFilter->zTerm/nTerm, then all
+ ** required callbacks have been made. In this case exit early.
+ **
+ ** Similarly, if this is a search for an exact match, and the first term
+ ** of segment apSegment[0] is not a match, exit early.
+ */
+ if( pFilter->zTerm ){
+ if( nTerm<pFilter->nTerm
+ || (!isPrefix && nTerm>pFilter->nTerm)
+ || memcmp(zTerm, pFilter->zTerm, pFilter->nTerm)
){
- z++;
+ goto finished;
+ }
}
- }
- /* Step 1b */
- z2 = z;
- if( stem(&z, "dee", "ee", m_gt_0) ){
- /* Do nothing. The work was all in the test */
- }else if(
- (stem(&z, "gni", "", hasVowel) || stem(&z, "de", "", hasVowel))
- && z!=z2
- ){
- if( stem(&z, "ta", "ate", 0) ||
- stem(&z, "lb", "ble", 0) ||
- stem(&z, "zi", "ize", 0) ){
- /* Do nothing. The work was all in the test */
- }else if( doubleConsonant(z) && (*z!='l' && *z!='s' && *z!='z') ){
- z++;
- }else if( m_eq_1(z) && star_oh(z) ){
- *(--z) = 'e';
- }
- }
+ while( nMerge<nSegment
+ && apSegment[nMerge]->aNode
+ && apSegment[nMerge]->nTerm==nTerm
+ && 0==memcmp(zTerm, apSegment[nMerge]->zTerm, nTerm)
+ ){
+ nMerge++;
+ }
- /* Step 1c */
- if( z[0]=='y' && hasVowel(z+1) ){
- z[0] = 'i';
- }
+ if( nMerge==1 && !isIgnoreEmpty && !isColFilter && isRequirePos ){
+ Fts3SegReader *p0 = apSegment[0];
+ rc = xFunc(p, pContext, zTerm, nTerm, p0->aDoclist, p0->nDoclist);
+ if( rc!=SQLITE_OK ) goto finished;
+ }else{
+ int nDoclist = 0; /* Size of doclist */
+ sqlite3_int64 iPrev = 0; /* Previous docid stored in doclist */
- /* Step 2 */
- switch( z[1] ){
- case 'a':
- stem(&z, "lanoita", "ate", m_gt_0) ||
- stem(&z, "lanoit", "tion", m_gt_0);
- break;
- case 'c':
- stem(&z, "icne", "ence", m_gt_0) ||
- stem(&z, "icna", "ance", m_gt_0);
- break;
- case 'e':
- stem(&z, "rezi", "ize", m_gt_0);
- break;
- case 'g':
- stem(&z, "igol", "log", m_gt_0);
- break;
- case 'l':
- stem(&z, "ilb", "ble", m_gt_0) ||
- stem(&z, "illa", "al", m_gt_0) ||
- stem(&z, "iltne", "ent", m_gt_0) ||
- stem(&z, "ile", "e", m_gt_0) ||
- stem(&z, "ilsuo", "ous", m_gt_0);
- break;
- case 'o':
- stem(&z, "noitazi", "ize", m_gt_0) ||
- stem(&z, "noita", "ate", m_gt_0) ||
- stem(&z, "rota", "ate", m_gt_0);
- break;
- case 's':
- stem(&z, "msila", "al", m_gt_0) ||
- stem(&z, "ssenevi", "ive", m_gt_0) ||
- stem(&z, "ssenluf", "ful", m_gt_0) ||
- stem(&z, "ssensuo", "ous", m_gt_0);
- break;
- case 't':
- stem(&z, "itila", "al", m_gt_0) ||
- stem(&z, "itivi", "ive", m_gt_0) ||
- stem(&z, "itilib", "ble", m_gt_0);
- break;
- }
+ /* The current term of the first nMerge entries in the array
+ ** of Fts3SegReader objects is the same. The doclists must be merged
+ ** and a single term added to the new segment.
+ */
+ for(i=0; i<nMerge; i++){
+ fts3SegReaderFirstDocid(apSegment[i]);
+ }
+ fts3SegReaderSort(apSegment, nMerge, nMerge, fts3SegReaderDoclistCmp);
+ while( apSegment[0]->pOffsetList ){
+ int j; /* Number of segments that share a docid */
+ char *pList;
+ int nList;
+ int nByte;
+ sqlite3_int64 iDocid = apSegment[0]->iDocid;
+ fts3SegReaderNextDocid(apSegment[0], &pList, &nList);
+ j = 1;
+ while( j<nMerge
+ && apSegment[j]->pOffsetList
+ && apSegment[j]->iDocid==iDocid
+ ){
+ fts3SegReaderNextDocid(apSegment[j], 0, 0);
+ j++;
+ }
- /* Step 3 */
- switch( z[0] ){
- case 'e':
- stem(&z, "etaci", "ic", m_gt_0) ||
- stem(&z, "evita", "", m_gt_0) ||
- stem(&z, "ezila", "al", m_gt_0);
- break;
- case 'i':
- stem(&z, "itici", "ic", m_gt_0);
- break;
- case 'l':
- stem(&z, "laci", "ic", m_gt_0) ||
- stem(&z, "luf", "", m_gt_0);
- break;
- case 's':
- stem(&z, "ssen", "", m_gt_0);
- break;
- }
+ if( isColFilter ){
+ fts3ColumnFilter(pFilter->iCol, &pList, &nList);
+ }
- /* Step 4 */
- switch( z[1] ){
- case 'a':
- if( z[0]=='l' && m_gt_1(z+2) ){
- z += 2;
- }
- break;
- case 'c':
- if( z[0]=='e' && z[2]=='n' && (z[3]=='a' || z[3]=='e') && m_gt_1(z+4) ){
- z += 4;
- }
- break;
- case 'e':
- if( z[0]=='r' && m_gt_1(z+2) ){
- z += 2;
- }
- break;
- case 'i':
- if( z[0]=='c' && m_gt_1(z+2) ){
- z += 2;
- }
- break;
- case 'l':
- if( z[0]=='e' && z[2]=='b' && (z[3]=='a' || z[3]=='i') && m_gt_1(z+4) ){
- z += 4;
- }
- break;
- case 'n':
- if( z[0]=='t' ){
- if( z[2]=='a' ){
- if( m_gt_1(z+3) ){
- z += 3;
- }
- }else if( z[2]=='e' ){
- stem(&z, "tneme", "", m_gt_1) ||
- stem(&z, "tnem", "", m_gt_1) ||
- stem(&z, "tne", "", m_gt_1);
- }
- }
- break;
- case 'o':
- if( z[0]=='u' ){
- if( m_gt_1(z+2) ){
- z += 2;
- }
- }else if( z[3]=='s' || z[3]=='t' ){
- stem(&z, "noi", "", m_gt_1);
- }
- break;
- case 's':
- if( z[0]=='m' && z[2]=='i' && m_gt_1(z+3) ){
- z += 3;
- }
- break;
- case 't':
- stem(&z, "eta", "", m_gt_1) ||
- stem(&z, "iti", "", m_gt_1);
- break;
- case 'u':
- if( z[0]=='s' && z[2]=='o' && m_gt_1(z+3) ){
- z += 3;
- }
- break;
- case 'v':
- case 'z':
- if( z[0]=='e' && z[2]=='i' && m_gt_1(z+3) ){
- z += 3;
- }
- break;
- }
+ if( !isIgnoreEmpty || nList>0 ){
+ nByte = sqlite3Fts3VarintLen(iDocid-iPrev) + (isRequirePos?nList+1:0);
+ if( nDoclist+nByte>nAlloc ){
+ char *aNew;
+ nAlloc = nDoclist+nByte*2;
+ aNew = sqlite3_realloc(aBuffer, nAlloc);
+ if( !aNew ){
+ rc = SQLITE_NOMEM;
+ goto finished;
+ }
+ aBuffer = aNew;
+ }
+ nDoclist += sqlite3Fts3PutVarint(&aBuffer[nDoclist], iDocid-iPrev);
+ iPrev = iDocid;
+ if( isRequirePos ){
+ memcpy(&aBuffer[nDoclist], pList, nList);
+ nDoclist += nList;
+ aBuffer[nDoclist++] = '\0';
+ }
+ }
- /* Step 5a */
- if( z[0]=='e' ){
- if( m_gt_1(z+1) ){
- z++;
- }else if( m_eq_1(z+1) && !star_oh(z+1) ){
- z++;
+ fts3SegReaderSort(apSegment, nMerge, j, fts3SegReaderDoclistCmp);
+ }
+
+ if( nDoclist>0 ){
+ rc = xFunc(p, pContext, zTerm, nTerm, aBuffer, nDoclist);
+ if( rc!=SQLITE_OK ) goto finished;
+ }
}
- }
- /* Step 5b */
- if( m_gt_1(z) && z[0]=='l' && z[1]=='l' ){
- z++;
- }
+ /* If there is a term specified to filter on, and this is not a prefix
+ ** search, return now. The callback that corresponds to the required
+ ** term (if such a term exists in the index) has already been made.
+ */
+ if( pFilter->zTerm && !isPrefix ){
+ goto finished;
+ }
- /* z[] is now the stemmed word in reverse order. Flip it back
- ** around into forward order and return.
- */
- *pnOut = i = strlen(z);
- zOut[i] = 0;
- while( *z ){
- zOut[--i] = *(z++);
+ for(i=0; i<nMerge; i++){
+ rc = fts3SegReaderNext(apSegment[i]);
+ if( rc!=SQLITE_OK ) goto finished;
+ }
+ fts3SegReaderSort(apSegment, nSegment, nMerge, fts3SegReaderCmp);
}
-}
-/*
-** Characters that can be part of a token. We assume any character
-** whose value is greater than 0x80 (any UTF character) can be
-** part of a token. In other words, delimiters all must have
-** values of 0x7f or lower.
-*/
-static const char porterIdChar[] = {
-/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
- 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
- 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
-};
-#define isDelim(C) (((ch=C)&0x80)==0 && (ch<0x30 || !porterIdChar[ch-0x30]))
+ finished:
+ sqlite3_free(aBuffer);
+ return rc;
+}
/*
-** Extract the next token from a tokenization cursor. The cursor must
-** have been opened by a prior call to porterOpen().
+** Merge all level iLevel segments in the database into a single
+** iLevel+1 segment. Or, if iLevel<0, merge all segments into a
+** single segment with a level equal to the numerically largest level
+** currently present in the database.
+**
+** If this function is called with iLevel<0, but there is only one
+** segment in the database, SQLITE_DONE is returned immediately.
+** Otherwise, if successful, SQLITE_OK is returned. If an error occurs,
+** an SQLite error code is returned.
*/
-static int porterNext(
- sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by porterOpen */
- const char **pzToken, /* OUT: *pzToken is the token text */
- int *pnBytes, /* OUT: Number of bytes in token */
- int *piStartOffset, /* OUT: Starting offset of token */
- int *piEndOffset, /* OUT: Ending offset of token */
- int *piPosition /* OUT: Position integer of token */
-){
- porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor;
- const char *z = c->zInput;
-
- while( c->iOffset<c->nInput ){
- int iStartOffset, ch;
-
- /* Scan past delimiter characters */
- while( c->iOffset<c->nInput && isDelim(z[c->iOffset]) ){
- c->iOffset++;
+static int fts3SegmentMerge(Fts3Table *p, int iLevel){
+ int i; /* Iterator variable */
+ int rc; /* Return code */
+ int iIdx; /* Index of new segment */
+ int iNewLevel; /* Level to create new segment at */
+ sqlite3_stmt *pStmt;
+ SegmentWriter *pWriter = 0;
+ int nSegment = 0; /* Number of segments being merged */
+ Fts3SegReader **apSegment = 0; /* Array of Segment iterators */
+ Fts3SegFilter filter; /* Segment term filter condition */
+
+ if( iLevel<0 ){
+ /* This call is to merge all segments in the database to a single
+ ** segment. The level of the new segment is equal to the the numerically
+ ** greatest segment level currently present in the database. The index
+ ** of the new segment is always 0.
+ */
+ iIdx = 0;
+ rc = fts3SegmentCountMax(p, &nSegment, &iNewLevel);
+ if( nSegment==1 ){
+ return SQLITE_DONE;
}
+ }else{
+ /* This call is to merge all segments at level iLevel. Find the next
+ ** available segment index at level iLevel+1. The call to
+ ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to
+ ** a single iLevel+2 segment if necessary.
+ */
+ iNewLevel = iLevel+1;
+ rc = fts3AllocateSegdirIdx(p, iNewLevel, &iIdx);
+ if( rc!=SQLITE_OK ) return rc;
+ rc = fts3SegmentCount(p, iLevel, &nSegment);
+ }
+ if( rc!=SQLITE_OK ) return rc;
+ assert( nSegment>0 );
+ assert( iNewLevel>=0 );
- /* Count non-delimiter characters. */
- iStartOffset = c->iOffset;
- while( c->iOffset<c->nInput && !isDelim(z[c->iOffset]) ){
- c->iOffset++;
+ /* Allocate space for an array of pointers to segment iterators. */
+ apSegment = (Fts3SegReader**)sqlite3_malloc(sizeof(Fts3SegReader *)*nSegment);
+ if( !apSegment ){
+ return SQLITE_NOMEM;
+ }
+ memset(apSegment, 0, sizeof(Fts3SegReader *)*nSegment);
+
+ /* Allocate a Fts3SegReader structure for each segment being merged. A
+ ** Fts3SegReader stores the state data required to iterate through all
+ ** entries on all leaves of a single segment.
+ */
+ assert( SQL_SELECT_LEVEL+1==SQL_SELECT_ALL_LEVEL);
+ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL+(iLevel<0), &pStmt, 0);
+ if( rc!=SQLITE_OK ) goto finished;
+ sqlite3_bind_int(pStmt, 1, iLevel);
+ for(i=0; SQLITE_ROW==(sqlite3_step(pStmt)); i++){
+ rc = fts3SegReaderNew(p, pStmt, i, &apSegment[i]);
+ if( rc!=SQLITE_OK ){
+ goto finished;
}
+ }
+ rc = sqlite3_reset(pStmt);
+ pStmt = 0;
+ if( rc!=SQLITE_OK ) goto finished;
- if( c->iOffset>iStartOffset ){
- int n = c->iOffset-iStartOffset;
- if( n>c->nAllocated ){
- c->nAllocated = n+20;
- c->zToken = sqlite3_realloc(c->zToken, c->nAllocated);
- if( c->zToken==NULL ) return SQLITE_NOMEM;
- }
- porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes);
- *pzToken = c->zToken;
- *piStartOffset = iStartOffset;
- *piEndOffset = c->iOffset;
- *piPosition = c->iToken++;
- return SQLITE_OK;
+ memset(&filter, 0, sizeof(Fts3SegFilter));
+ filter.flags = FTS3_SEGMENT_REQUIRE_POS;
+ filter.flags |= (iLevel<0 ? FTS3_SEGMENT_IGNORE_EMPTY : 0);
+ rc = sqlite3Fts3SegReaderIterate(p, apSegment, nSegment,
+ &filter, fts3MergeCallback, (void *)&pWriter
+ );
+ if( rc!=SQLITE_OK ) goto finished;
+
+ rc = fts3DeleteSegdir(p, iLevel, apSegment, nSegment);
+ if( rc==SQLITE_OK ){
+ rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx);
+ }
+
+ finished:
+ fts3SegWriterFree(pWriter);
+ if( apSegment ){
+ for(i=0; i<nSegment; i++){
+ sqlite3Fts3SegReaderFree(p, apSegment[i]);
}
+ sqlite3_free(apSegment);
}
- return SQLITE_DONE;
+ sqlite3_reset(pStmt);
+ return rc;
}
/*
-** The set of routines that implement the porter-stemmer tokenizer
+** This is a comparison function used as a qsort() callback when sorting
+** an array of pending terms by term. This occurs as part of flushing
+** the contents of the pending-terms hash table to the database.
*/
-static const sqlite3_tokenizer_module porterTokenizerModule = {
- 0,
- porterCreate,
- porterDestroy,
- porterOpen,
- porterClose,
- porterNext,
-};
+static int qsortCompare(const void *lhs, const void *rhs){
+ char *z1 = fts3HashKey(*(Fts3HashElem **)lhs);
+ char *z2 = fts3HashKey(*(Fts3HashElem **)rhs);
+ int n1 = fts3HashKeysize(*(Fts3HashElem **)lhs);
+ int n2 = fts3HashKeysize(*(Fts3HashElem **)rhs);
-/*
-** Allocate a new porter tokenizer. Return a pointer to the new
-** tokenizer in *ppModule
-*/
-SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule(
- sqlite3_tokenizer_module const**ppModule
-){
- *ppModule = &porterTokenizerModule;
+ int n = (n1<n2 ? n1 : n2);
+ int c = memcmp(z1, z2, n);
+ if( c==0 ){
+ c = n1 - n2;
+ }
+ return c;
}
-#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
-/************** End of fts3_porter.c *****************************************/
-/************** Begin file fts3_tokenizer.c **********************************/
-/*
-** 2007 June 22
-**
-** The author disclaims copyright to this source code. In place of
-** a legal notice, here is a blessing:
-**
-** May you do good and not evil.
-** May you find forgiveness for yourself and forgive others.
-** May you share freely, never taking more than you give.
-**
-******************************************************************************
-**
-** This is part of an SQLite module implementing full-text search.
-** This particular file implements the generic tokenizer interface.
+/*
+** Flush the contents of pendingTerms to a level 0 segment.
*/
+SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
+ Fts3HashElem *pElem;
+ int idx, rc, i;
+ Fts3HashElem **apElem; /* Array of pointers to hash elements */
+ int nElem; /* Number of terms in new segment */
+ SegmentWriter *pWriter = 0; /* Used to write the segment */
-/*
-** The code in this file is only compiled if:
-**
-** * The FTS3 module is being built as an extension
-** (in which case SQLITE_CORE is not defined), or
-**
-** * The FTS3 module is being built into the core of
-** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
-*/
-#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+ /* Find the number of terms that will make up the new segment. If there
+ ** are no terms, return early (do not bother to write an empty segment).
+ */
+ nElem = fts3HashCount(&p->pendingTerms);
+ if( nElem==0 ){
+ assert( p->nPendingData==0 );
+ return SQLITE_OK;
+ }
-#ifndef SQLITE_CORE
- SQLITE_EXTENSION_INIT1
-#endif
+ /* Determine the next index at level 0, merging as necessary. */
+ rc = fts3AllocateSegdirIdx(p, 0, &idx);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+ apElem = sqlite3_malloc(nElem*sizeof(Fts3HashElem *));
+ if( !apElem ){
+ return SQLITE_NOMEM;
+ }
-/*
-** Implementation of the SQL scalar function for accessing the underlying
-** hash table. This function may be called as follows:
-**
-** SELECT <function-name>(<key-name>);
-** SELECT <function-name>(<key-name>, <pointer>);
-**
-** where <function-name> is the name passed as the second argument
-** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer').
-**
-** If the <pointer> argument is specified, it must be a blob value
-** containing a pointer to be stored as the hash data corresponding
-** to the string <key-name>. If <pointer> is not specified, then
-** the string <key-name> must already exist in the has table. Otherwise,
-** an error is returned.
-**
-** Whether or not the <pointer> argument is specified, the value returned
-** is a blob containing the pointer stored as the hash data corresponding
-** to string <key-name> (after the hash-table is updated, if applicable).
-*/
-static void scalarFunc(
- sqlite3_context *context,
- int argc,
- sqlite3_value **argv
-){
- fts3Hash *pHash;
- void *pPtr = 0;
- const unsigned char *zName;
- int nName;
+ i = 0;
+ for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){
+ apElem[i++] = pElem;
+ }
+ assert( i==nElem );
- assert( argc==1 || argc==2 );
+ /* TODO(shess) Should we allow user-defined collation sequences,
+ ** here? I think we only need that once we support prefix searches.
+ ** Also, should we be using qsort()?
+ */
+ if( nElem>1 ){
+ qsort(apElem, nElem, sizeof(Fts3HashElem *), qsortCompare);
+ }
- pHash = (fts3Hash *)sqlite3_user_data(context);
- zName = sqlite3_value_text(argv[0]);
- nName = sqlite3_value_bytes(argv[0])+1;
+ /* Write the segment tree into the database. */
+ for(i=0; rc==SQLITE_OK && i<nElem; i++){
+ const char *z = fts3HashKey(apElem[i]);
+ int n = fts3HashKeysize(apElem[i]);
+ PendingList *pList = fts3HashData(apElem[i]);
+ rc = fts3SegWriterAdd(p, &pWriter, 0, z, n, pList->aData, pList->nData+1);
+ }
+ if( rc==SQLITE_OK ){
+ rc = fts3SegWriterFlush(p, pWriter, 0, idx);
+ }
- if( argc==2 ){
- void *pOld;
- int n = sqlite3_value_bytes(argv[1]);
- if( n!=sizeof(pPtr) ){
- sqlite3_result_error(context, "argument type mismatch", -1);
- return;
+ /* Free all allocated resources before returning */
+ fts3SegWriterFree(pWriter);
+ sqlite3_free(apElem);
+ sqlite3Fts3PendingTermsClear(p);
+ return rc;
+}
+
+/*
+** This function does the work for the xUpdate method of FTS3 virtual
+** tables.
+*/
+SQLITE_PRIVATE int sqlite3Fts3UpdateMethod(
+ sqlite3_vtab *pVtab, /* FTS3 vtab object */
+ int nArg, /* Size of argument array */
+ sqlite3_value **apVal, /* Array of arguments */
+ sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */
+){
+ Fts3Table *p = (Fts3Table *)pVtab;
+ int rc = SQLITE_OK; /* Return Code */
+ int isRemove = 0; /* True for an UPDATE or DELETE */
+ sqlite3_int64 iRemove = 0; /* Rowid removed by UPDATE or DELETE */
+
+ /* If this is a DELETE or UPDATE operation, remove the old record. */
+ if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
+ int isEmpty;
+ rc = fts3IsEmpty(p, apVal, &isEmpty);
+ if( rc==SQLITE_OK ){
+ if( isEmpty ){
+ /* Deleting this row means the whole table is empty. In this case
+ ** delete the contents of all three tables and throw away any
+ ** data in the pendingTerms hash table.
+ */
+ rc = fts3DeleteAll(p);
+ }else{
+ isRemove = 1;
+ iRemove = sqlite3_value_int64(apVal[0]);
+ rc = fts3PendingTermsDocid(p, iRemove);
+ if( rc==SQLITE_OK ){
+ rc = fts3DeleteTerms(p, apVal);
+ if( rc==SQLITE_OK ){
+ rc = fts3SqlExec(p, SQL_DELETE_CONTENT, apVal);
+ }
+ }
+ }
}
- pPtr = *(void **)sqlite3_value_blob(argv[1]);
- pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr);
- if( pOld==pPtr ){
- sqlite3_result_error(context, "out of memory", -1);
- return;
+ }
+
+ /* If this is an INSERT or UPDATE operation, insert the new record. */
+ if( nArg>1 && rc==SQLITE_OK ){
+ rc = fts3InsertData(p, apVal, pRowid);
+ if( rc==SQLITE_OK && (!isRemove || *pRowid!=iRemove) ){
+ rc = fts3PendingTermsDocid(p, *pRowid);
}
- }else{
- pPtr = sqlite3Fts3HashFind(pHash, zName, nName);
- if( !pPtr ){
- char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
- sqlite3_result_error(context, zErr, -1);
- sqlite3_free(zErr);
- return;
+ if( rc==SQLITE_OK ){
+ rc = fts3InsertTerms(p, apVal);
}
}
- sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
+ return rc;
}
-#ifdef SQLITE_TEST
+/*
+** Flush any data in the pending-terms hash table to disk. If successful,
+** merge all segments in the database (including the new segment, if
+** there was any data to flush) into a single segment.
+*/
+SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *p){
+ int rc;
+ rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0);
+ if( rc==SQLITE_OK ){
+ rc = sqlite3Fts3PendingTermsFlush(p);
+ if( rc==SQLITE_OK ){
+ rc = fts3SegmentMerge(p, -1);
+ }
+ if( rc==SQLITE_OK ){
+ rc = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
+ }else{
+ sqlite3_exec(p->db, "ROLLBACK TO fts3 ; RELEASE fts3", 0, 0, 0);
+ }
+ }
+ return rc;
+}
+#endif
+/************** End of fts3_write.c ******************************************/
+/************** Begin file fts3_snippet.c ************************************/
/*
-** Implementation of a special SQL scalar function for testing tokenizers
-** designed to be used in concert with the Tcl testing framework. This
-** function must be called with two arguments:
-**
-** SELECT <function-name>(<key-name>, <input-string>);
-** SELECT <function-name>(<key-name>, <pointer>);
-**
-** where <function-name> is the name passed as the second argument
-** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer')
-** concatenated with the string '_test' (e.g. 'fts3_tokenizer_test').
+** 2009 Oct 23
**
-** The return value is a string that may be interpreted as a Tcl
-** list. For each token in the <input-string>, three elements are
-** added to the returned list. The first is the token position, the
-** second is the token text (folded, stemmed, etc.) and the third is the
-** substring of <input-string> associated with the token. For example,
-** using the built-in "simple" tokenizer:
-**
-** SELECT fts_tokenizer_test('simple', 'I don't see how');
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
**
-** will return the string:
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
**
-** "{0 i I 1 dont don't 2 see see 3 how how}"
-**
+******************************************************************************
*/
-static void testFunc(
- sqlite3_context *context,
- int argc,
- sqlite3_value **argv
-){
- fts3Hash *pHash;
- sqlite3_tokenizer_module *p;
- sqlite3_tokenizer *pTokenizer = 0;
- sqlite3_tokenizer_cursor *pCsr = 0;
-
- const char *zErr = 0;
- const char *zName;
- int nName;
- const char *zInput;
- int nInput;
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
- const char *zArg = 0;
- const char *zToken;
- int nToken;
- int iStart;
- int iEnd;
- int iPos;
+typedef struct Snippet Snippet;
- Tcl_Obj *pRet;
+/*
+** An instance of the following structure keeps track of generated
+** matching-word offset information and snippets.
+*/
+struct Snippet {
+ int nMatch; /* Total number of matches */
+ int nAlloc; /* Space allocated for aMatch[] */
+ struct snippetMatch { /* One entry for each matching term */
+ char snStatus; /* Status flag for use while constructing snippets */
+ short int nByte; /* Number of bytes in the term */
+ short int iCol; /* The column that contains the match */
+ short int iTerm; /* The index in Query.pTerms[] of the matching term */
+ int iToken; /* The index of the matching document token */
+ int iStart; /* The offset to the first character of the term */
+ } *aMatch; /* Points to space obtained from malloc */
+ char *zOffset; /* Text rendering of aMatch[] */
+ int nOffset; /* strlen(zOffset) */
+ char *zSnippet; /* Snippet text */
+ int nSnippet; /* strlen(zSnippet) */
+};
- assert( argc==2 || argc==3 );
- nName = sqlite3_value_bytes(argv[0]);
- zName = (const char *)sqlite3_value_text(argv[0]);
- nInput = sqlite3_value_bytes(argv[argc-1]);
- zInput = (const char *)sqlite3_value_text(argv[argc-1]);
+/* It is not safe to call isspace(), tolower(), or isalnum() on
+** hi-bit-set characters. This is the same solution used in the
+** tokenizer.
+*/
+static int fts3snippetIsspace(char c){
+ return (c&0x80)==0 ? isspace(c) : 0;
+}
- if( argc==3 ){
- zArg = (const char *)sqlite3_value_text(argv[1]);
- }
- pHash = (fts3Hash *)sqlite3_user_data(context);
- p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1);
+/*
+** A StringBuffer object holds a zero-terminated string that grows
+** arbitrarily by appending. Space to hold the string is obtained
+** from sqlite3_malloc(). After any memory allocation failure,
+** StringBuffer.z is set to NULL and no further allocation is attempted.
+*/
+typedef struct StringBuffer {
+ char *z; /* Text of the string. Space from malloc. */
+ int nUsed; /* Number bytes of z[] used, not counting \000 terminator */
+ int nAlloc; /* Bytes allocated for z[] */
+} StringBuffer;
- if( !p ){
- char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
- sqlite3_result_error(context, zErr, -1);
- sqlite3_free(zErr);
- return;
- }
- pRet = Tcl_NewObj();
- Tcl_IncrRefCount(pRet);
+/*
+** Initialize a new StringBuffer.
+*/
+static void fts3SnippetSbInit(StringBuffer *p){
+ p->nAlloc = 100;
+ p->nUsed = 0;
+ p->z = sqlite3_malloc( p->nAlloc );
+}
- if( SQLITE_OK!=p->xCreate(zArg ? 1 : 0, &zArg, &pTokenizer) ){
- zErr = "error in xCreate()";
- goto finish;
- }
- pTokenizer->pModule = p;
- if( SQLITE_OK!=p->xOpen(pTokenizer, zInput, nInput, &pCsr) ){
- zErr = "error in xOpen()";
- goto finish;
- }
- pCsr->pTokenizer = pTokenizer;
+/*
+** Append text to the string buffer.
+*/
+static void fts3SnippetAppend(StringBuffer *p, const char *zNew, int nNew){
+ if( p->z==0 ) return;
+ if( nNew<0 ) nNew = (int)strlen(zNew);
+ if( p->nUsed + nNew >= p->nAlloc ){
+ int nAlloc;
+ char *zNew;
- while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){
- Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos));
- Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
- zToken = &zInput[iStart];
- nToken = iEnd-iStart;
- Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
+ nAlloc = p->nUsed + nNew + p->nAlloc;
+ zNew = sqlite3_realloc(p->z, nAlloc);
+ if( zNew==0 ){
+ sqlite3_free(p->z);
+ p->z = 0;
+ return;
+ }
+ p->z = zNew;
+ p->nAlloc = nAlloc;
}
+ memcpy(&p->z[p->nUsed], zNew, nNew);
+ p->nUsed += nNew;
+ p->z[p->nUsed] = 0;
+}
- if( SQLITE_OK!=p->xClose(pCsr) ){
- zErr = "error in xClose()";
- goto finish;
- }
- if( SQLITE_OK!=p->xDestroy(pTokenizer) ){
- zErr = "error in xDestroy()";
- goto finish;
+/* If the StringBuffer ends in something other than white space, add a
+** single space character to the end.
+*/
+static void fts3SnippetAppendWhiteSpace(StringBuffer *p){
+ if( p->z && p->nUsed && !fts3snippetIsspace(p->z[p->nUsed-1]) ){
+ fts3SnippetAppend(p, " ", 1);
}
+}
-finish:
- if( zErr ){
- sqlite3_result_error(context, zErr, -1);
- }else{
- sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT);
+/* Remove white space from the end of the StringBuffer */
+static void fts3SnippetTrimWhiteSpace(StringBuffer *p){
+ if( p->z ){
+ while( p->nUsed && fts3snippetIsspace(p->z[p->nUsed-1]) ){
+ p->nUsed--;
+ }
+ p->z[p->nUsed] = 0;
}
- Tcl_DecrRefCount(pRet);
}
-static
-int registerTokenizer(
- sqlite3 *db,
- char *zName,
- const sqlite3_tokenizer_module *p
-){
- int rc;
- sqlite3_stmt *pStmt;
- const char zSql[] = "SELECT fts3_tokenizer(?, ?)";
-
- rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
- if( rc!=SQLITE_OK ){
- return rc;
+/*
+** Release all memory associated with the Snippet structure passed as
+** an argument.
+*/
+static void fts3SnippetFree(Snippet *p){
+ if( p ){
+ sqlite3_free(p->aMatch);
+ sqlite3_free(p->zOffset);
+ sqlite3_free(p->zSnippet);
+ sqlite3_free(p);
}
-
- sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
- sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);
- sqlite3_step(pStmt);
-
- return sqlite3_finalize(pStmt);
}
-static
-int queryTokenizer(
- sqlite3 *db,
- char *zName,
- const sqlite3_tokenizer_module **pp
+/*
+** Append a single entry to the p->aMatch[] log.
+*/
+static int snippetAppendMatch(
+ Snippet *p, /* Append the entry to this snippet */
+ int iCol, int iTerm, /* The column and query term */
+ int iToken, /* Matching token in document */
+ int iStart, int nByte /* Offset and size of the match */
){
- int rc;
- sqlite3_stmt *pStmt;
- const char zSql[] = "SELECT fts3_tokenizer(?)";
-
- *pp = 0;
- rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
- if( rc!=SQLITE_OK ){
- return rc;
- }
-
- sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
- if( SQLITE_ROW==sqlite3_step(pStmt) ){
- if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
- memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
+ int i;
+ struct snippetMatch *pMatch;
+ if( p->nMatch+1>=p->nAlloc ){
+ struct snippetMatch *pNew;
+ p->nAlloc = p->nAlloc*2 + 10;
+ pNew = sqlite3_realloc(p->aMatch, p->nAlloc*sizeof(p->aMatch[0]) );
+ if( pNew==0 ){
+ p->aMatch = 0;
+ p->nMatch = 0;
+ p->nAlloc = 0;
+ return SQLITE_NOMEM;
}
+ p->aMatch = pNew;
}
-
- return sqlite3_finalize(pStmt);
+ i = p->nMatch++;
+ pMatch = &p->aMatch[i];
+ pMatch->iCol = (short)iCol;
+ pMatch->iTerm = (short)iTerm;
+ pMatch->iToken = iToken;
+ pMatch->iStart = iStart;
+ pMatch->nByte = (short)nByte;
+ return SQLITE_OK;
}
-SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
+/*
+** Sizing information for the circular buffer used in snippetOffsetsOfColumn()
+*/
+#define FTS3_ROTOR_SZ (32)
+#define FTS3_ROTOR_MASK (FTS3_ROTOR_SZ-1)
/*
-** Implementation of the scalar function fts3_tokenizer_internal_test().
-** This function is used for testing only, it is not included in the
-** build unless SQLITE_TEST is defined.
-**
-** The purpose of this is to test that the fts3_tokenizer() function
-** can be used as designed by the C-code in the queryTokenizer and
-** registerTokenizer() functions above. These two functions are repeated
-** in the README.tokenizer file as an example, so it is important to
-** test them.
-**
-** To run the tests, evaluate the fts3_tokenizer_internal_test() scalar
-** function with no arguments. An assert() will fail if a problem is
-** detected. i.e.:
-**
-** SELECT fts3_tokenizer_internal_test();
+** Function to iterate through the tokens of a compiled expression.
**
+** Except, skip all tokens on the right-hand side of a NOT operator.
+** This function is used to find tokens as part of snippet and offset
+** generation and we do nt want snippets and offsets to report matches
+** for tokens on the RHS of a NOT.
*/
-static void intTestFunc(
- sqlite3_context *context,
- int argc,
- sqlite3_value **argv
-){
- int rc;
- const sqlite3_tokenizer_module *p1;
- const sqlite3_tokenizer_module *p2;
- sqlite3 *db = (sqlite3 *)sqlite3_user_data(context);
-
- /* Test the query function */
- sqlite3Fts3SimpleTokenizerModule(&p1);
- rc = queryTokenizer(db, "simple", &p2);
- assert( rc==SQLITE_OK );
- assert( p1==p2 );
- rc = queryTokenizer(db, "nosuchtokenizer", &p2);
- assert( rc==SQLITE_ERROR );
- assert( p2==0 );
- assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") );
-
- /* Test the storage function */
- rc = registerTokenizer(db, "nosuchtokenizer", p1);
- assert( rc==SQLITE_OK );
- rc = queryTokenizer(db, "nosuchtokenizer", &p2);
- assert( rc==SQLITE_OK );
- assert( p2==p1 );
+static int fts3NextExprToken(Fts3Expr **ppExpr, int *piToken){
+ Fts3Expr *p = *ppExpr;
+ int iToken = *piToken;
+ if( iToken<0 ){
+ /* In this case the expression p is the root of an expression tree.
+ ** Move to the first token in the expression tree.
+ */
+ while( p->pLeft ){
+ p = p->pLeft;
+ }
+ iToken = 0;
+ }else{
+ assert(p && p->eType==FTSQUERY_PHRASE );
+ if( iToken<(p->pPhrase->nToken-1) ){
+ iToken++;
+ }else{
+ iToken = 0;
+ while( p->pParent && p->pParent->pLeft!=p ){
+ assert( p->pParent->pRight==p );
+ p = p->pParent;
+ }
+ p = p->pParent;
+ if( p ){
+ assert( p->pRight!=0 );
+ p = p->pRight;
+ while( p->pLeft ){
+ p = p->pLeft;
+ }
+ }
+ }
+ }
- sqlite3_result_text(context, "ok", -1, SQLITE_STATIC);
+ *ppExpr = p;
+ *piToken = iToken;
+ return p?1:0;
}
-#endif
+/*
+** Return TRUE if the expression node pExpr is located beneath the
+** RHS of a NOT operator.
+*/
+static int fts3ExprBeneathNot(Fts3Expr *p){
+ Fts3Expr *pParent;
+ while( p ){
+ pParent = p->pParent;
+ if( pParent && pParent->eType==FTSQUERY_NOT && pParent->pRight==p ){
+ return 1;
+ }
+ p = pParent;
+ }
+ return 0;
+}
/*
-** Set up SQL objects in database db used to access the contents of
-** the hash table pointed to by argument pHash. The hash table must
-** been initialised to use string keys, and to take a private copy
-** of the key when a value is inserted. i.e. by a call similar to:
-**
-** sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1);
-**
-** This function adds a scalar function (see header comment above
-** scalarFunc() in this file for details) and, if ENABLE_TABLE is
-** defined at compilation time, a temporary virtual table (see header
-** comment above struct HashTableVtab) to the database schema. Both
-** provide read/write access to the contents of *pHash.
-**
-** The third argument to this function, zName, is used as the name
-** of both the scalar and, if created, the virtual table.
+** Add entries to pSnippet->aMatch[] for every match that occurs against
+** document zDoc[0..nDoc-1] which is stored in column iColumn.
*/
-SQLITE_PRIVATE int sqlite3Fts3InitHashTable(
- sqlite3 *db,
- fts3Hash *pHash,
- const char *zName
+static int snippetOffsetsOfColumn(
+ Fts3Cursor *pCur, /* The fulltest search cursor */
+ Snippet *pSnippet, /* The Snippet object to be filled in */
+ int iColumn, /* Index of fulltext table column */
+ const char *zDoc, /* Text of the fulltext table column */
+ int nDoc /* Length of zDoc in bytes */
){
- int rc = SQLITE_OK;
- void *p = (void *)pHash;
- const int any = SQLITE_ANY;
- char *zTest = 0;
- char *zTest2 = 0;
+ const sqlite3_tokenizer_module *pTModule; /* The tokenizer module */
+ sqlite3_tokenizer *pTokenizer; /* The specific tokenizer */
+ sqlite3_tokenizer_cursor *pTCursor; /* Tokenizer cursor */
+ Fts3Table *pVtab; /* The full text index */
+ int nColumn; /* Number of columns in the index */
+ int i, j; /* Loop counters */
+ int rc; /* Return code */
+ unsigned int match, prevMatch; /* Phrase search bitmasks */
+ const char *zToken; /* Next token from the tokenizer */
+ int nToken; /* Size of zToken */
+ int iBegin, iEnd, iPos; /* Offsets of beginning and end */
-#ifdef SQLITE_TEST
- void *pdb = (void *)db;
- zTest = sqlite3_mprintf("%s_test", zName);
- zTest2 = sqlite3_mprintf("%s_internal_test", zName);
- if( !zTest || !zTest2 ){
- rc = SQLITE_NOMEM;
- }
-#endif
+ /* The following variables keep a circular buffer of the last
+ ** few tokens */
+ unsigned int iRotor = 0; /* Index of current token */
+ int iRotorBegin[FTS3_ROTOR_SZ]; /* Beginning offset of token */
+ int iRotorLen[FTS3_ROTOR_SZ]; /* Length of token */
+
+ pVtab = (Fts3Table *)pCur->base.pVtab;
+ nColumn = pVtab->nColumn;
+ pTokenizer = pVtab->pTokenizer;
+ pTModule = pTokenizer->pModule;
+ rc = pTModule->xOpen(pTokenizer, zDoc, nDoc, &pTCursor);
+ if( rc ) return rc;
+ pTCursor->pTokenizer = pTokenizer;
- if( rc!=SQLITE_OK
- || (rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0))
- || (rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0))
-#ifdef SQLITE_TEST
- || (rc = sqlite3_create_function(db, zTest, 2, any, p, testFunc, 0, 0))
- || (rc = sqlite3_create_function(db, zTest, 3, any, p, testFunc, 0, 0))
- || (rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0))
-#endif
- );
+ prevMatch = 0;
+ while( (rc = pTModule->xNext(pTCursor, &zToken, &nToken,
+ &iBegin, &iEnd, &iPos))==SQLITE_OK ){
+ Fts3Expr *pIter = pCur->pExpr;
+ int iIter = -1;
+ iRotorBegin[iRotor&FTS3_ROTOR_MASK] = iBegin;
+ iRotorLen[iRotor&FTS3_ROTOR_MASK] = iEnd-iBegin;
+ match = 0;
+ for(i=0; i<(FTS3_ROTOR_SZ-1) && fts3NextExprToken(&pIter, &iIter); i++){
+ int nPhrase; /* Number of tokens in current phrase */
+ struct PhraseToken *pToken; /* Current token */
+ int iCol; /* Column index */
- sqlite3_free(zTest);
- sqlite3_free(zTest2);
- return rc;
+ if( fts3ExprBeneathNot(pIter) ) continue;
+ nPhrase = pIter->pPhrase->nToken;
+ pToken = &pIter->pPhrase->aToken[iIter];
+ iCol = pIter->pPhrase->iColumn;
+ if( iCol>=0 && iCol<nColumn && iCol!=iColumn ) continue;
+ if( pToken->n>nToken ) continue;
+ if( !pToken->isPrefix && pToken->n<nToken ) continue;
+ assert( pToken->n<=nToken );
+ if( memcmp(pToken->z, zToken, pToken->n) ) continue;
+ if( iIter>0 && (prevMatch & (1<<i))==0 ) continue;
+ match |= 1<<i;
+ if( i==(FTS3_ROTOR_SZ-2) || nPhrase==iIter+1 ){
+ for(j=nPhrase-1; j>=0; j--){
+ int k = (iRotor-j) & FTS3_ROTOR_MASK;
+ rc = snippetAppendMatch(pSnippet, iColumn, i-j, iPos-j,
+ iRotorBegin[k], iRotorLen[k]);
+ if( rc ) goto end_offsets_of_column;
+ }
+ }
+ }
+ prevMatch = match<<1;
+ iRotor++;
+ }
+end_offsets_of_column:
+ pTModule->xClose(pTCursor);
+ return rc==SQLITE_DONE ? SQLITE_OK : rc;
}
-#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
-
-/************** End of fts3_tokenizer.c **************************************/
-/************** Begin file fts3_tokenizer1.c *********************************/
/*
-** 2006 Oct 10
-**
-** The author disclaims copyright to this source code. In place of
-** a legal notice, here is a blessing:
+** Remove entries from the pSnippet structure to account for the NEAR
+** operator. When this is called, pSnippet contains the list of token
+** offsets produced by treating all NEAR operators as AND operators.
+** This function removes any entries that should not be present after
+** accounting for the NEAR restriction. For example, if the queried
+** document is:
**
-** May you do good and not evil.
-** May you find forgiveness for yourself and forgive others.
-** May you share freely, never taking more than you give.
+** "A B C D E A"
**
-******************************************************************************
+** and the query is:
+**
+** A NEAR/0 E
**
-** Implementation of the "simple" full-text-search tokenizer.
-*/
-
-/*
-** The code in this file is only compiled if:
+** then when this function is called the Snippet contains token offsets
+** 0, 4 and 5. This function removes the "0" entry (because the first A
+** is not near enough to an E).
**
-** * The FTS3 module is being built as an extension
-** (in which case SQLITE_CORE is not defined), or
+** When this function is called, the value pointed to by parameter piLeft is
+** the integer id of the left-most token in the expression tree headed by
+** pExpr. This function increments *piLeft by the total number of tokens
+** in the expression tree headed by pExpr.
**
-** * The FTS3 module is being built into the core of
-** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
+** Return 1 if any trimming occurs. Return 0 if no trimming is required.
*/
-#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
-
-
-
-
-typedef struct simple_tokenizer {
- sqlite3_tokenizer base;
- char delim[128]; /* flag ASCII delimiters */
-} simple_tokenizer;
-
-typedef struct simple_tokenizer_cursor {
- sqlite3_tokenizer_cursor base;
- const char *pInput; /* input we are tokenizing */
- int nBytes; /* size of the input */
- int iOffset; /* current position in pInput */
- int iToken; /* index of next token to be returned */
- char *pToken; /* storage for current token */
- int nTokenAllocated; /* space allocated to zToken buffer */
-} simple_tokenizer_cursor;
+static int trimSnippetOffsets(
+ Fts3Expr *pExpr, /* The search expression */
+ Snippet *pSnippet, /* The set of snippet offsets to be trimmed */
+ int *piLeft /* Index of left-most token in pExpr */
+){
+ if( pExpr ){
+ if( trimSnippetOffsets(pExpr->pLeft, pSnippet, piLeft) ){
+ return 1;
+ }
+ switch( pExpr->eType ){
+ case FTSQUERY_PHRASE:
+ *piLeft += pExpr->pPhrase->nToken;
+ break;
+ case FTSQUERY_NEAR: {
+ /* The right-hand-side of a NEAR operator is always a phrase. The
+ ** left-hand-side is either a phrase or an expression tree that is
+ ** itself headed by a NEAR operator. The following initializations
+ ** set local variable iLeft to the token number of the left-most
+ ** token in the right-hand phrase, and iRight to the right most
+ ** token in the same phrase. For example, if we had:
+ **
+ ** <col> MATCH '"abc def" NEAR/2 "ghi jkl"'
+ **
+ ** then iLeft will be set to 2 (token number of ghi) and nToken will
+ ** be set to 4.
+ */
+ Fts3Expr *pLeft = pExpr->pLeft;
+ Fts3Expr *pRight = pExpr->pRight;
+ int iLeft = *piLeft;
+ int nNear = pExpr->nNear;
+ int nToken = pRight->pPhrase->nToken;
+ int jj, ii;
+ if( pLeft->eType==FTSQUERY_NEAR ){
+ pLeft = pLeft->pRight;
+ }
+ assert( pRight->eType==FTSQUERY_PHRASE );
+ assert( pLeft->eType==FTSQUERY_PHRASE );
+ nToken += pLeft->pPhrase->nToken;
-/* Forward declaration */
-static const sqlite3_tokenizer_module simpleTokenizerModule;
+ for(ii=0; ii<pSnippet->nMatch; ii++){
+ struct snippetMatch *p = &pSnippet->aMatch[ii];
+ if( p->iTerm==iLeft ){
+ int isOk = 0;
+ /* Snippet ii is an occurence of query term iLeft in the document.
+ ** It occurs at position (p->iToken) of the document. We now
+ ** search for an instance of token (iLeft-1) somewhere in the
+ ** range (p->iToken - nNear)...(p->iToken + nNear + nToken) within
+ ** the set of snippetMatch structures. If one is found, proceed.
+ ** If one cannot be found, then remove snippets ii..(ii+N-1)
+ ** from the matching snippets, where N is the number of tokens
+ ** in phrase pRight->pPhrase.
+ */
+ for(jj=0; isOk==0 && jj<pSnippet->nMatch; jj++){
+ struct snippetMatch *p2 = &pSnippet->aMatch[jj];
+ if( p2->iTerm==(iLeft-1) ){
+ if( p2->iToken>=(p->iToken-nNear-1)
+ && p2->iToken<(p->iToken+nNear+nToken)
+ ){
+ isOk = 1;
+ }
+ }
+ }
+ if( !isOk ){
+ int kk;
+ for(kk=0; kk<pRight->pPhrase->nToken; kk++){
+ pSnippet->aMatch[kk+ii].iTerm = -2;
+ }
+ return 1;
+ }
+ }
+ if( p->iTerm==(iLeft-1) ){
+ int isOk = 0;
+ for(jj=0; isOk==0 && jj<pSnippet->nMatch; jj++){
+ struct snippetMatch *p2 = &pSnippet->aMatch[jj];
+ if( p2->iTerm==iLeft ){
+ if( p2->iToken<=(p->iToken+nNear+1)
+ && p2->iToken>(p->iToken-nNear-nToken)
+ ){
+ isOk = 1;
+ }
+ }
+ }
+ if( !isOk ){
+ int kk;
+ for(kk=0; kk<pLeft->pPhrase->nToken; kk++){
+ pSnippet->aMatch[ii-kk].iTerm = -2;
+ }
+ return 1;
+ }
+ }
+ }
+ break;
+ }
+ }
-static int simpleDelim(simple_tokenizer *t, unsigned char c){
- return c<0x80 && t->delim[c];
+ if( trimSnippetOffsets(pExpr->pRight, pSnippet, piLeft) ){
+ return 1;
+ }
+ }
+ return 0;
}
/*
-** Create a new tokenizer instance.
+** Compute all offsets for the current row of the query.
+** If the offsets have already been computed, this routine is a no-op.
*/
-static int simpleCreate(
- int argc, const char * const *argv,
- sqlite3_tokenizer **ppTokenizer
-){
- simple_tokenizer *t;
+static int snippetAllOffsets(Fts3Cursor *pCsr, Snippet **ppSnippet){
+ Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; /* The FTS3 virtual table */
+ int nColumn; /* Number of columns. Docid does count */
+ int iColumn; /* Index of of a column */
+ int i; /* Loop index */
+ int iFirst; /* First column to search */
+ int iLast; /* Last coumn to search */
+ int iTerm = 0;
+ Snippet *pSnippet;
+ int rc = SQLITE_OK;
- t = (simple_tokenizer *) sqlite3_malloc(sizeof(*t));
- if( t==NULL ) return SQLITE_NOMEM;
- memset(t, 0, sizeof(*t));
+ if( pCsr->pExpr==0 ){
+ return SQLITE_OK;
+ }
- /* TODO(shess) Delimiters need to remain the same from run to run,
- ** else we need to reindex. One solution would be a meta-table to
- ** track such information in the database, then we'd only want this
- ** information on the initial create.
- */
- if( argc>1 ){
- int i, n = strlen(argv[1]);
- for(i=0; i<n; i++){
- unsigned char ch = argv[1][i];
- /* We explicitly don't support UTF-8 delimiters for now. */
- if( ch>=0x80 ){
- sqlite3_free(t);
- return SQLITE_ERROR;
- }
- t->delim[ch] = 1;
- }
- } else {
- /* Mark non-alphanumeric ASCII characters as delimiters */
- int i;
- for(i=1; i<0x80; i++){
- t->delim[i] = !isalnum(i);
+ pSnippet = (Snippet *)sqlite3_malloc(sizeof(Snippet));
+ *ppSnippet = pSnippet;
+ if( !pSnippet ){
+ return SQLITE_NOMEM;
+ }
+ memset(pSnippet, 0, sizeof(Snippet));
+
+ nColumn = p->nColumn;
+ iColumn = (pCsr->eSearch - 2);
+ if( iColumn<0 || iColumn>=nColumn ){
+ /* Look for matches over all columns of the full-text index */
+ iFirst = 0;
+ iLast = nColumn-1;
+ }else{
+ /* Look for matches in the iColumn-th column of the index only */
+ iFirst = iColumn;
+ iLast = iColumn;
+ }
+ for(i=iFirst; rc==SQLITE_OK && i<=iLast; i++){
+ const char *zDoc;
+ int nDoc;
+ zDoc = (const char*)sqlite3_column_text(pCsr->pStmt, i+1);
+ nDoc = sqlite3_column_bytes(pCsr->pStmt, i+1);
+ if( zDoc==0 && sqlite3_column_type(pCsr->pStmt, i+1)!=SQLITE_NULL ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = snippetOffsetsOfColumn(pCsr, pSnippet, i, zDoc, nDoc);
}
}
- *ppTokenizer = &t->base;
- return SQLITE_OK;
+ while( trimSnippetOffsets(pCsr->pExpr, pSnippet, &iTerm) ){
+ iTerm = 0;
+ }
+
+ return rc;
}
/*
-** Destroy a tokenizer
+** Convert the information in the aMatch[] array of the snippet
+** into the string zOffset[0..nOffset-1]. This string is used as
+** the return of the SQL offsets() function.
*/
-static int simpleDestroy(sqlite3_tokenizer *pTokenizer){
- sqlite3_free(pTokenizer);
- return SQLITE_OK;
+static void snippetOffsetText(Snippet *p){
+ int i;
+ int cnt = 0;
+ StringBuffer sb;
+ char zBuf[200];
+ if( p->zOffset ) return;
+ fts3SnippetSbInit(&sb);
+ for(i=0; i<p->nMatch; i++){
+ struct snippetMatch *pMatch = &p->aMatch[i];
+ if( pMatch->iTerm>=0 ){
+ /* If snippetMatch.iTerm is less than 0, then the match was
+ ** discarded as part of processing the NEAR operator (see the
+ ** trimSnippetOffsetsForNear() function for details). Ignore
+ ** it in this case
+ */
+ zBuf[0] = ' ';
+ sqlite3_snprintf(sizeof(zBuf)-1, &zBuf[cnt>0], "%d %d %d %d",
+ pMatch->iCol, pMatch->iTerm, pMatch->iStart, pMatch->nByte);
+ fts3SnippetAppend(&sb, zBuf, -1);
+ cnt++;
+ }
+ }
+ p->zOffset = sb.z;
+ p->nOffset = sb.z ? sb.nUsed : 0;
}
/*
-** Prepare to begin tokenizing a particular string. The input
-** string to be tokenized is pInput[0..nBytes-1]. A cursor
-** used to incrementally tokenize this string is returned in
-** *ppCursor.
+** zDoc[0..nDoc-1] is phrase of text. aMatch[0..nMatch-1] are a set
+** of matching words some of which might be in zDoc. zDoc is column
+** number iCol.
+**
+** iBreak is suggested spot in zDoc where we could begin or end an
+** excerpt. Return a value similar to iBreak but possibly adjusted
+** to be a little left or right so that the break point is better.
*/
-static int simpleOpen(
- sqlite3_tokenizer *pTokenizer, /* The tokenizer */
- const char *pInput, int nBytes, /* String to be tokenized */
- sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
+static int wordBoundary(
+ int iBreak, /* The suggested break point */
+ const char *zDoc, /* Document text */
+ int nDoc, /* Number of bytes in zDoc[] */
+ struct snippetMatch *aMatch, /* Matching words */
+ int nMatch, /* Number of entries in aMatch[] */
+ int iCol /* The column number for zDoc[] */
){
- simple_tokenizer_cursor *c;
-
- c = (simple_tokenizer_cursor *) sqlite3_malloc(sizeof(*c));
- if( c==NULL ) return SQLITE_NOMEM;
-
- c->pInput = pInput;
- if( pInput==0 ){
- c->nBytes = 0;
- }else if( nBytes<0 ){
- c->nBytes = (int)strlen(pInput);
- }else{
- c->nBytes = nBytes;
+ int i;
+ if( iBreak<=10 ){
+ return 0;
}
- c->iOffset = 0; /* start tokenizing at the beginning */
- c->iToken = 0;
- c->pToken = NULL; /* no space allocated, yet. */
- c->nTokenAllocated = 0;
-
- *ppCursor = &c->base;
- return SQLITE_OK;
+ if( iBreak>=nDoc-10 ){
+ return nDoc;
+ }
+ for(i=0; ALWAYS(i<nMatch) && aMatch[i].iCol<iCol; i++){}
+ while( i<nMatch && aMatch[i].iStart+aMatch[i].nByte<iBreak ){ i++; }
+ if( i<nMatch ){
+ if( aMatch[i].iStart<iBreak+10 ){
+ return aMatch[i].iStart;
+ }
+ if( i>0 && aMatch[i-1].iStart+aMatch[i-1].nByte>=iBreak ){
+ return aMatch[i-1].iStart;
+ }
+ }
+ for(i=1; i<=10; i++){
+ if( fts3snippetIsspace(zDoc[iBreak-i]) ){
+ return iBreak - i + 1;
+ }
+ if( fts3snippetIsspace(zDoc[iBreak+i]) ){
+ return iBreak + i + 1;
+ }
+ }
+ return iBreak;
}
+
+
/*
-** Close a tokenization cursor previously opened by a call to
-** simpleOpen() above.
+** Allowed values for Snippet.aMatch[].snStatus
*/
-static int simpleClose(sqlite3_tokenizer_cursor *pCursor){
- simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
- sqlite3_free(c->pToken);
- sqlite3_free(c);
- return SQLITE_OK;
-}
+#define SNIPPET_IGNORE 0 /* It is ok to omit this match from the snippet */
+#define SNIPPET_DESIRED 1 /* We want to include this match in the snippet */
/*
-** Extract the next token from a tokenization cursor. The cursor must
-** have been opened by a prior call to simpleOpen().
+** Generate the text of a snippet.
*/
-static int simpleNext(
- sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */
- const char **ppToken, /* OUT: *ppToken is the token text */
- int *pnBytes, /* OUT: Number of bytes in token */
- int *piStartOffset, /* OUT: Starting offset of token */
- int *piEndOffset, /* OUT: Ending offset of token */
- int *piPosition /* OUT: Position integer of token */
+static void snippetText(
+ Fts3Cursor *pCursor, /* The cursor we need the snippet for */
+ Snippet *pSnippet,
+ const char *zStartMark, /* Markup to appear before each match */
+ const char *zEndMark, /* Markup to appear after each match */
+ const char *zEllipsis /* Ellipsis mark */
){
- simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
- simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer;
- unsigned char *p = (unsigned char *)c->pInput;
+ int i, j;
+ struct snippetMatch *aMatch;
+ int nMatch;
+ int nDesired;
+ StringBuffer sb;
+ int tailCol;
+ int tailOffset;
+ int iCol;
+ int nDoc;
+ const char *zDoc;
+ int iStart, iEnd;
+ int tailEllipsis = 0;
+ int iMatch;
+
- while( c->iOffset<c->nBytes ){
- int iStartOffset;
+ sqlite3_free(pSnippet->zSnippet);
+ pSnippet->zSnippet = 0;
+ aMatch = pSnippet->aMatch;
+ nMatch = pSnippet->nMatch;
+ fts3SnippetSbInit(&sb);
- /* Scan past delimiter characters */
- while( c->iOffset<c->nBytes && simpleDelim(t, p[c->iOffset]) ){
- c->iOffset++;
+ for(i=0; i<nMatch; i++){
+ aMatch[i].snStatus = SNIPPET_IGNORE;
+ }
+ nDesired = 0;
+ for(i=0; i<FTS3_ROTOR_SZ; i++){
+ for(j=0; j<nMatch; j++){
+ if( aMatch[j].iTerm==i ){
+ aMatch[j].snStatus = SNIPPET_DESIRED;
+ nDesired++;
+ break;
+ }
}
+ }
- /* Count non-delimiter characters. */
- iStartOffset = c->iOffset;
- while( c->iOffset<c->nBytes && !simpleDelim(t, p[c->iOffset]) ){
- c->iOffset++;
+ iMatch = 0;
+ tailCol = -1;
+ tailOffset = 0;
+ for(i=0; i<nMatch && nDesired>0; i++){
+ if( aMatch[i].snStatus!=SNIPPET_DESIRED ) continue;
+ nDesired--;
+ iCol = aMatch[i].iCol;
+ zDoc = (const char*)sqlite3_column_text(pCursor->pStmt, iCol+1);
+ nDoc = sqlite3_column_bytes(pCursor->pStmt, iCol+1);
+ iStart = aMatch[i].iStart - 40;
+ iStart = wordBoundary(iStart, zDoc, nDoc, aMatch, nMatch, iCol);
+ if( iStart<=10 ){
+ iStart = 0;
}
-
- if( c->iOffset>iStartOffset ){
- int i, n = c->iOffset-iStartOffset;
- if( n>c->nTokenAllocated ){
- c->nTokenAllocated = n+20;
- c->pToken = sqlite3_realloc(c->pToken, c->nTokenAllocated);
- if( c->pToken==NULL ) return SQLITE_NOMEM;
+ if( iCol==tailCol && iStart<=tailOffset+20 ){
+ iStart = tailOffset;
+ }
+ if( (iCol!=tailCol && tailCol>=0) || iStart!=tailOffset ){
+ fts3SnippetTrimWhiteSpace(&sb);
+ fts3SnippetAppendWhiteSpace(&sb);
+ fts3SnippetAppend(&sb, zEllipsis, -1);
+ fts3SnippetAppendWhiteSpace(&sb);
+ }
+ iEnd = aMatch[i].iStart + aMatch[i].nByte + 40;
+ iEnd = wordBoundary(iEnd, zDoc, nDoc, aMatch, nMatch, iCol);
+ if( iEnd>=nDoc-10 ){
+ iEnd = nDoc;
+ tailEllipsis = 0;
+ }else{
+ tailEllipsis = 1;
+ }
+ while( iMatch<nMatch && aMatch[iMatch].iCol<iCol ){ iMatch++; }
+ while( iStart<iEnd ){
+ while( iMatch<nMatch && aMatch[iMatch].iStart<iStart
+ && aMatch[iMatch].iCol<=iCol ){
+ iMatch++;
}
- for(i=0; i<n; i++){
- /* TODO(shess) This needs expansion to handle UTF-8
- ** case-insensitivity.
- */
- unsigned char ch = p[iStartOffset+i];
- c->pToken[i] = ch<0x80 ? tolower(ch) : ch;
+ if( iMatch<nMatch && aMatch[iMatch].iStart<iEnd
+ && aMatch[iMatch].iCol==iCol ){
+ fts3SnippetAppend(&sb, &zDoc[iStart], aMatch[iMatch].iStart - iStart);
+ iStart = aMatch[iMatch].iStart;
+ fts3SnippetAppend(&sb, zStartMark, -1);
+ fts3SnippetAppend(&sb, &zDoc[iStart], aMatch[iMatch].nByte);
+ fts3SnippetAppend(&sb, zEndMark, -1);
+ iStart += aMatch[iMatch].nByte;
+ for(j=iMatch+1; j<nMatch; j++){
+ if( aMatch[j].iTerm==aMatch[iMatch].iTerm
+ && aMatch[j].snStatus==SNIPPET_DESIRED ){
+ nDesired--;
+ aMatch[j].snStatus = SNIPPET_IGNORE;
+ }
+ }
+ }else{
+ fts3SnippetAppend(&sb, &zDoc[iStart], iEnd - iStart);
+ iStart = iEnd;
}
- *ppToken = c->pToken;
- *pnBytes = n;
- *piStartOffset = iStartOffset;
- *piEndOffset = c->iOffset;
- *piPosition = c->iToken++;
-
- return SQLITE_OK;
}
+ tailCol = iCol;
+ tailOffset = iEnd;
}
- return SQLITE_DONE;
+ fts3SnippetTrimWhiteSpace(&sb);
+ if( tailEllipsis ){
+ fts3SnippetAppendWhiteSpace(&sb);
+ fts3SnippetAppend(&sb, zEllipsis, -1);
+ }
+ pSnippet->zSnippet = sb.z;
+ pSnippet->nSnippet = sb.z ? sb.nUsed : 0;
}
-/*
-** The set of routines that implement the simple tokenizer
-*/
-static const sqlite3_tokenizer_module simpleTokenizerModule = {
- 0,
- simpleCreate,
- simpleDestroy,
- simpleOpen,
- simpleClose,
- simpleNext,
-};
+SQLITE_PRIVATE void sqlite3Fts3Offsets(
+ sqlite3_context *pCtx, /* SQLite function call context */
+ Fts3Cursor *pCsr /* Cursor object */
+){
+ Snippet *p; /* Snippet structure */
+ int rc = snippetAllOffsets(pCsr, &p);
+ if( rc==SQLITE_OK ){
+ snippetOffsetText(p);
+ if( p->zOffset ){
+ sqlite3_result_text(pCtx, p->zOffset, p->nOffset, SQLITE_TRANSIENT);
+ }else{
+ sqlite3_result_error_nomem(pCtx);
+ }
+ }else{
+ sqlite3_result_error_nomem(pCtx);
+ }
+ fts3SnippetFree(p);
+}
-/*
-** Allocate a new simple tokenizer. Return a pointer to the new
-** tokenizer in *ppModule
-*/
-SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(
- sqlite3_tokenizer_module const**ppModule
+SQLITE_PRIVATE void sqlite3Fts3Snippet(
+ sqlite3_context *pCtx, /* SQLite function call context */
+ Fts3Cursor *pCsr, /* Cursor object */
+ const char *zStart, /* Snippet start text - "<b>" */
+ const char *zEnd, /* Snippet end text - "</b>" */
+ const char *zEllipsis /* Snippet ellipsis text - "<b>...</b>" */
){
- *ppModule = &simpleTokenizerModule;
+ Snippet *p; /* Snippet structure */
+ int rc = snippetAllOffsets(pCsr, &p);
+ if( rc==SQLITE_OK ){
+ snippetText(pCsr, p, zStart, zEnd, zEllipsis);
+ if( p->zSnippet ){
+ sqlite3_result_text(pCtx, p->zSnippet, p->nSnippet, SQLITE_TRANSIENT);
+ }else{
+ sqlite3_result_error_nomem(pCtx);
+ }
+ }else{
+ sqlite3_result_error_nomem(pCtx);
+ }
+ fts3SnippetFree(p);
}
-#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
+#endif
-/************** End of fts3_tokenizer1.c *************************************/
+/************** End of fts3_snippet.c ****************************************/
/************** Begin file rtree.c *******************************************/
/*
** 2001 September 15
*************************************************************************
** This file contains code for implementations of the r-tree and r*-tree
** algorithms packaged as an SQLite virtual table module.
-**
-** $Id$
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RTREE)
projects / php / commitdiff