From: Ilia Alshanetsky Date: Sat, 11 Dec 2010 14:57:34 +0000 (+0000) Subject: Upgraded bundled Sqlite3 to version 3.7.4. X-Git-Tag: php-5.3.6RC1~248 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=9e48381b283720d4536f8a364e87408c3d54004c;p=php Upgraded bundled Sqlite3 to version 3.7.4. --- diff --git a/NEWS b/NEWS index 2b1eacbbaf..aa4c876e0e 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,8 @@ PHP NEWS ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| ?? ??? 20??, PHP 5.3.5 +- Upgraded bundled Sqlite3 to version 3.7.4. (Ilia) + - Zend Engine: . Indirect reference to $this fails to resolve if direct $this is never used in method. (Scott) diff --git a/ext/sqlite3/CREDITS b/ext/sqlite3/CREDITS index 6b2d3f383e..2bea9b88e2 100644 --- a/ext/sqlite3/CREDITS +++ b/ext/sqlite3/CREDITS @@ -1,2 +1,2 @@ SQLite3 -Scott MacVicar +Scott MacVicar, Ilia Alshanetsky diff --git a/ext/sqlite3/libsqlite/sqlite3.c b/ext/sqlite3/libsqlite/sqlite3.c index 511c15d417..8f7f3b358e 100644 --- a/ext/sqlite3/libsqlite/sqlite3.c +++ b/ext/sqlite3/libsqlite/sqlite3.c @@ -4,11 +4,11 @@ /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.7.3. By combining all the individual C code files into this +** version 3.7.4. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a one translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements -** of 5% are more are commonly seen when SQLite is compiled as a single +** of 5% or more are commonly seen when SQLite is compiled as a single ** translation unit. ** ** This file is all you need to compile SQLite. To use SQLite in other @@ -654,9 +654,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.7.3" -#define SQLITE_VERSION_NUMBER 3007003 -#define SQLITE_SOURCE_ID "2010-10-08 02:34:02 2677848087c9c090efb17c1893e77d6136a9111d" +#define SQLITE_VERSION "3.7.4" +#define SQLITE_VERSION_NUMBER 3007004 +#define SQLITE_SOURCE_ID "2010-12-07 20:14:09 a586a4deeb25330037a49df295b36aaf624d0f45" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -1089,6 +1089,18 @@ SQLITE_API int sqlite3_exec( ** equal SQLITE_SYNC_NORMAL, that means to use normal fsync() semantics. ** If the lower four bits equal SQLITE_SYNC_FULL, that means ** to use Mac OS X style fullsync instead of fsync(). +** +** Do not confuse the SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL flags +** with the [PRAGMA synchronous]=NORMAL and [PRAGMA synchronous]=FULL +** settings. The [synchronous pragma] determines when calls to the +** xSync VFS method occur and applies uniformly across all platforms. +** The SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL flags determine how +** energetic or rigorous or forceful the sync operations are and +** only make a difference on Mac OSX for the default SQLite code. +** (Third-party VFS implementations might also make the distinction +** between SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL, but among the +** operating systems natively supported by SQLite, only Mac OSX +** cares about the difference.) */ #define SQLITE_SYNC_NORMAL 0x00002 #define SQLITE_SYNC_FULL 0x00003 @@ -1257,6 +1269,8 @@ struct sqlite3_io_methods { #define SQLITE_LAST_ERRNO 4 #define SQLITE_FCNTL_SIZE_HINT 5 #define SQLITE_FCNTL_CHUNK_SIZE 6 +#define SQLITE_FCNTL_FILE_POINTER 7 + /* ** CAPI3REF: Mutex Handle @@ -3180,6 +3194,20 @@ SQLITE_API int sqlite3_prepare16_v2( */ SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt); +/* +** CAPI3REF: Determine If An SQL Statement Writes The Database +** +** ^The sqlite3_stmt_readonly(X) interface returns true (non-zero) if +** the [prepared statement] X is [SELECT] statement and false (zero) if +** X is an [INSERT], [UPDATE], [DELETE], CREATE, DROP, [ANALYZE], +** [ALTER], or [REINDEX] statement. +** If X is a NULL pointer or any other kind of statement, including but +** not limited to [ATTACH], [DETACH], [COMMIT], [ROLLBACK], [RELEASE], +** [SAVEPOINT], [PRAGMA], or [VACUUM] the result of sqlite3_stmt_readonly(X) is +** undefined. +*/ +SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt); + /* ** CAPI3REF: Dynamically Typed Value Object ** KEYWORDS: {protected sqlite3_value} {unprotected sqlite3_value} @@ -3279,7 +3307,10 @@ typedef struct sqlite3_context sqlite3_context; ** ** ^The fifth argument to sqlite3_bind_blob(), sqlite3_bind_text(), and ** sqlite3_bind_text16() is a destructor used to dispose of the BLOB or -** string after SQLite has finished with it. ^If the fifth argument is +** string after SQLite has finished with it. ^The destructor is called +** to dispose of the BLOB or string even if the call to sqlite3_bind_blob(), +** sqlite3_bind_text(), or sqlite3_bind_text16() fails. +** ^If the fifth argument is ** the special value [SQLITE_STATIC], then SQLite assumes that the ** information is in static, unmanaged space and does not need to be freed. ** ^If the fifth argument has the value [SQLITE_TRANSIENT], then @@ -3919,12 +3950,15 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** SQL function or aggregate, pass NULL poiners for all three function ** callbacks. ** -** ^If the tenth parameter to sqlite3_create_function_v2() is not NULL, -** then it is invoked when the function is deleted, either by being -** overloaded or when the database connection closes. -** ^When the destructure callback of the tenth parameter is invoked, it -** is passed a single argument which is a copy of the pointer which was -** the fifth parameter to sqlite3_create_function_v2(). +** ^(If the tenth parameter to sqlite3_create_function_v2() is not NULL, +** then it is destructor for the application data pointer. +** The destructor is invoked when the function is deleted, either by being +** overloaded or when the database connection closes.)^ +** ^The destructor is also invoked if the call to +** sqlite3_create_function_v2() fails. +** ^When the destructor callback of the tenth parameter is invoked, it +** is passed a single argument which is a copy of the application data +** pointer which was the fifth parameter to sqlite3_create_function_v2(). ** ** ^It is permitted to register multiple implementations of the same ** functions with the same name but with either differing numbers of @@ -4388,6 +4422,15 @@ SQLITE_API void sqlite3_result_zeroblob(sqlite3_context*, int n); ** calls to the collation creation functions or when the ** [database connection] is closed using [sqlite3_close()]. ** +** ^The xDestroy callback is not called if the +** sqlite3_create_collation_v2() function fails. Applications that invoke +** sqlite3_create_collation_v2() with a non-NULL xDestroy argument should +** check the return code and dispose of the application data pointer +** themselves rather than expecting SQLite to deal with it for them. +** This is different from every other SQLite interface. The inconsistency +** is unfortunate but cannot be changed without breaking backwards +** compatibility. +** ** See also: [sqlite3_collation_needed()] and [sqlite3_collation_needed16()]. */ SQLITE_API int sqlite3_create_collation( @@ -5142,7 +5185,9 @@ struct sqlite3_index_info { ** ^The sqlite3_create_module_v2() interface has a fifth parameter which ** is a pointer to a destructor for the pClientData. ^SQLite will ** invoke the destructor function (if it is not NULL) when SQLite -** no longer needs the pClientData pointer. ^The sqlite3_create_module() +** no longer needs the pClientData pointer. ^The destructor will also +** be invoked if the call to sqlite3_create_module_v2() fails. +** ^The sqlite3_create_module() ** interface is equivalent to sqlite3_create_module_v2() with a NULL ** destructor. */ @@ -5325,6 +5370,30 @@ SQLITE_API int sqlite3_blob_open( sqlite3_blob **ppBlob ); +/* +** CAPI3REF: Move a BLOB Handle to a New Row +** +** ^This function is used to move an existing blob handle so that it points +** to a different row of the same database table. ^The new row is identified +** by the rowid value passed as the second argument. Only the row can be +** changed. ^The database, table and column on which the blob handle is open +** remain the same. Moving an existing blob handle to a new row can be +** faster than closing the existing handle and opening a new one. +** +** ^(The new row must meet the same criteria as for [sqlite3_blob_open()] - +** it must exist and there must be either a blob or text value stored in +** the nominated column.)^ ^If the new row is not present in the table, or if +** it does not contain a blob or text value, or if another error occurs, an +** SQLite error code is returned and the blob handle is considered aborted. +** ^All subsequent calls to [sqlite3_blob_read()], [sqlite3_blob_write()] or +** [sqlite3_blob_reopen()] on an aborted blob handle immediately return +** SQLITE_ABORT. ^Calling [sqlite3_blob_bytes()] on an aborted blob handle +** always returns zero. +** +** ^This function sets the database handle error code and message. +*/ +SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64); + /* ** CAPI3REF: Close A BLOB Handle ** @@ -5732,7 +5801,7 @@ SQLITE_API sqlite3_mutex *sqlite3_db_mutex(sqlite3*); ** ^The [sqlite3_file_control()] interface makes a direct call to the ** xFileControl method for the [sqlite3_io_methods] object associated ** with a particular database identified by the second argument. ^The -** name of the database "main" for the main database or "temp" for the +** name of the database is "main" for the main database or "temp" for the ** TEMP database, or the name that appears after the AS keyword for ** databases that are added using the [ATTACH] SQL command. ** ^A NULL pointer can be used in place of "main" to refer to the @@ -5742,6 +5811,12 @@ SQLITE_API sqlite3_mutex *sqlite3_db_mutex(sqlite3*); ** the xFileControl method. ^The return value of the xFileControl ** method becomes the return value of this routine. ** +** ^The SQLITE_FCNTL_FILE_POINTER value for the op parameter causes +** a pointer to the underlying [sqlite3_file] object to be written into +** the space pointed to by the 4th parameter. ^The SQLITE_FCNTL_FILE_POINTER +** case is a short-circuit path which does not actually invoke the +** underlying sqlite3_io_methods.xFileControl method. +** ** ^If the second parameter (zDbName) does not match the name of any ** open database file, then SQLITE_ERROR is returned. ^This error ** code is not remembered and will not be recalled by [sqlite3_errcode()] @@ -7411,7 +7486,7 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( SQLITE_PRIVATE int sqlite3BtreeClose(Btree*); SQLITE_PRIVATE int sqlite3BtreeSetCacheSize(Btree*,int); -SQLITE_PRIVATE int sqlite3BtreeSetSafetyLevel(Btree*,int,int); +SQLITE_PRIVATE int sqlite3BtreeSetSafetyLevel(Btree*,int,int,int); SQLITE_PRIVATE int sqlite3BtreeSyncDisabled(Btree*); SQLITE_PRIVATE int sqlite3BtreeSetPageSize(Btree *p, int nPagesize, int nReserve, int eFix); SQLITE_PRIVATE int sqlite3BtreeGetPageSize(Btree*); @@ -7895,12 +7970,12 @@ typedef struct VdbeOpList VdbeOpList; #define OP_VRename 132 #define OP_VUpdate 133 #define OP_Pagecount 134 -#define OP_Trace 135 -#define OP_Noop 136 -#define OP_Explain 137 +#define OP_MaxPgcnt 135 +#define OP_Trace 136 +#define OP_Noop 137 +#define OP_Explain 138 /* The following opcode values are never used */ -#define OP_NotUsed_138 138 #define OP_NotUsed_139 139 #define OP_NotUsed_140 140 @@ -7933,7 +8008,7 @@ typedef struct VdbeOpList VdbeOpList; /* 104 */ 0x00, 0x0c, 0x45, 0x15, 0x01, 0x02, 0x00, 0x01,\ /* 112 */ 0x08, 0x05, 0x05, 0x05, 0x00, 0x00, 0x00, 0x02,\ /* 120 */ 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ -/* 128 */ 0x01, 0x00, 0x02, 0x01, 0x00, 0x00, 0x02, 0x00,\ +/* 128 */ 0x01, 0x00, 0x02, 0x01, 0x00, 0x00, 0x02, 0x02,\ /* 136 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04,\ /* 144 */ 0x04, 0x04,} @@ -8118,7 +8193,7 @@ SQLITE_PRIVATE void sqlite3PagerSetBusyhandler(Pager*, int(*)(void *), void *); SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager*, u32*, int); SQLITE_PRIVATE int sqlite3PagerMaxPageCount(Pager*, int); SQLITE_PRIVATE void sqlite3PagerSetCachesize(Pager*, int); -SQLITE_PRIVATE void sqlite3PagerSetSafetyLevel(Pager*,int,int); +SQLITE_PRIVATE void sqlite3PagerSetSafetyLevel(Pager*,int,int,int); SQLITE_PRIVATE int sqlite3PagerLockingMode(Pager *, int); SQLITE_PRIVATE int sqlite3PagerSetJournalMode(Pager *, int); SQLITE_PRIVATE int sqlite3PagerGetJournalMode(Pager*); @@ -8983,13 +9058,14 @@ struct sqlite3 { #define SQLITE_ReadUncommitted 0x0080000 /* For shared-cache mode */ #define SQLITE_LegacyFileFmt 0x00100000 /* Create new databases in format 1 */ #define SQLITE_FullFSync 0x00200000 /* Use full fsync on the backend */ -#define SQLITE_LoadExtension 0x00400000 /* Enable load_extension */ +#define SQLITE_CkptFullFSync 0x00400000 /* Use full fsync for checkpoint */ #define SQLITE_RecoveryMode 0x00800000 /* Ignore schema errors */ #define SQLITE_ReverseOrder 0x01000000 /* Reverse unordered SELECTs */ #define SQLITE_RecTriggers 0x02000000 /* Enable recursive triggers */ #define SQLITE_ForeignKeys 0x04000000 /* Enforce foreign key constraints */ #define SQLITE_AutoIndex 0x08000000 /* Enable automatic indexes */ #define SQLITE_PreferBuiltin 0x10000000 /* Preference to built-in funcs */ +#define SQLITE_LoadExtension 0x20000000 /* Enable load_extension */ /* ** Bits of the sqlite3.flags field that are used by the @@ -9002,6 +9078,7 @@ struct sqlite3 { #define SQLITE_IndexSearch 0x08 /* Disable indexes for searching */ #define SQLITE_IndexCover 0x10 /* Disable index covering table */ #define SQLITE_GroupByOrder 0x20 /* Disable GROUPBY cover of ORDERBY */ +#define SQLITE_FactorOutConst 0x40 /* Disable factoring out constants */ #define SQLITE_OptMask 0xff /* Mask of all disablable opts */ /* @@ -9890,6 +9967,9 @@ struct SrcList { u8 isPopulated; /* Temporary table associated with SELECT is populated */ u8 jointype; /* Type of join between this able and the previous */ u8 notIndexed; /* True if there is a NOT INDEXED clause */ +#ifndef SQLITE_OMIT_EXPLAIN + u8 iSelectId; /* If pSelect!=0, the id of the sub-select in EQP */ +#endif int iCursor; /* The VDBE cursor number used to access this table */ Expr *pOn; /* The ON clause of a join */ IdList *pUsing; /* The USING clause of a join */ @@ -9928,6 +10008,7 @@ struct SrcList { struct WherePlan { u32 wsFlags; /* WHERE_* flags that describe the strategy */ u32 nEq; /* Number of == constraints */ + double nRow; /* Estimated number of rows (for EQP) */ union { Index *pIdx; /* Index when WHERE_INDEXED is true */ struct WhereTerm *pTerm; /* WHERE clause term for OR-search */ @@ -10012,6 +10093,7 @@ struct WhereInfo { int nLevel; /* Number of nested loop */ struct WhereClause *pWC; /* Decomposition of the WHERE clause */ double savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */ + double nRowOut; /* Estimated number of output rows */ WhereLevel a[1]; /* Information about each nest loop in WHERE */ }; @@ -10087,6 +10169,7 @@ struct Select { Expr *pOffset; /* OFFSET expression. NULL means not used. */ int iLimit, iOffset; /* Memory registers holding LIMIT & OFFSET counters */ int addrOpenEphm[3]; /* OP_OpenEphem opcodes related to this select */ + double nSelectRow; /* Estimated number of result rows */ }; /* @@ -10282,6 +10365,11 @@ struct Parse { int nHeight; /* Expression tree height of current sub-select */ Table *pZombieTab; /* List of Table objects to delete after code gen */ TriggerPrg *pTriggerPrg; /* Linked list of coded triggers */ + +#ifndef SQLITE_OMIT_EXPLAIN + int iSelectId; + int iNextSelectId; +#endif }; #ifdef SQLITE_OMIT_VIRTUALTABLE @@ -10897,6 +10985,7 @@ SQLITE_PRIVATE int sqlite3FixExprList(DbFixer*, ExprList*); SQLITE_PRIVATE int sqlite3FixTriggerStep(DbFixer*, TriggerStep*); SQLITE_PRIVATE int sqlite3AtoF(const char *z, double*, int, u8); SQLITE_PRIVATE int sqlite3GetInt32(const char *, int*); +SQLITE_PRIVATE int sqlite3Atoi(const char*); SQLITE_PRIVATE int sqlite3Utf16ByteLen(const void *pData, int nChar); SQLITE_PRIVATE int sqlite3Utf8CharLen(const char *pData, int nByte); SQLITE_PRIVATE int sqlite3Utf8Read(const u8*, const u8**); @@ -11975,26 +12064,34 @@ typedef struct VdbeCursor VdbeCursor; ** restoring the state of the VM to as it was before the sub-program ** began executing. ** -** Frames are stored in a linked list headed at Vdbe.pParent. Vdbe.pParent -** is the parent of the current frame, or zero if the current frame -** is the main Vdbe program. +** The memory for a VdbeFrame object is allocated and managed by a memory +** cell in the parent (calling) frame. When the memory cell is deleted or +** overwritten, the VdbeFrame object is not freed immediately. Instead, it +** is linked into the Vdbe.pDelFrame list. The contents of the Vdbe.pDelFrame +** list is deleted when the VM is reset in VdbeHalt(). The reason for doing +** this instead of deleting the VdbeFrame immediately is to avoid recursive +** calls to sqlite3VdbeMemRelease() when the memory cells belonging to the +** child frame are released. +** +** The currently executing frame is stored in Vdbe.pFrame. Vdbe.pFrame is +** set to NULL if the currently executing frame is the main program. */ typedef struct VdbeFrame VdbeFrame; struct VdbeFrame { Vdbe *v; /* VM this frame belongs to */ - int pc; /* Program Counter */ - Op *aOp; /* Program instructions */ + int pc; /* Program Counter in parent (calling) frame */ + Op *aOp; /* Program instructions for parent frame */ int nOp; /* Size of aOp array */ - Mem *aMem; /* Array of memory cells */ + Mem *aMem; /* Array of memory cells for parent frame */ int nMem; /* Number of entries in aMem */ - VdbeCursor **apCsr; /* Element of Vdbe cursors */ + VdbeCursor **apCsr; /* Array of Vdbe cursors for parent frame */ u16 nCursor; /* Number of entries in apCsr */ void *token; /* Copy of SubProgram.token */ int nChildMem; /* Number of memory cells for child frame */ int nChildCsr; /* Number of cursors for child frame */ i64 lastRowid; /* Last insert rowid (sqlite3.lastRowid) */ int nChange; /* Statement changes (Vdbe.nChanges) */ - VdbeFrame *pParent; /* Parent of this frame */ + VdbeFrame *pParent; /* Parent of this frame, or NULL if parent is main */ }; #define VdbeFrameMem(p) ((Mem *)&((u8 *)p)[ROUND8(sizeof(VdbeFrame))]) @@ -12211,6 +12308,7 @@ struct Vdbe { FILE *trace; /* Write an execution trace here, if not NULL */ #endif VdbeFrame *pFrame; /* Parent frame */ + VdbeFrame *pDelFrame; /* List of frame objects to free on VM reset */ int nFrame; /* Number of frames in pFrame list */ u32 expmask; /* Binding to these vars invalidates VM */ SubProgram *pProgram; /* Linked list of all sub-programs used by VM */ @@ -20399,6 +20497,16 @@ SQLITE_PRIVATE int sqlite3GetInt32(const char *zNum, int *pValue){ return 1; } +/* +** Return a 32-bit integer value extracted from a string. If the +** string is not an integer, just return 0. +*/ +SQLITE_PRIVATE int sqlite3Atoi(const char *z){ + int x = 0; + if( z ) sqlite3GetInt32(z, &x); + return x; +} + /* ** The variable-length integer encoding is as follows: ** @@ -21328,10 +21436,10 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* 132 */ "VRename", /* 133 */ "VUpdate", /* 134 */ "Pagecount", - /* 135 */ "Trace", - /* 136 */ "Noop", - /* 137 */ "Explain", - /* 138 */ "NotUsed_138", + /* 135 */ "MaxPgcnt", + /* 136 */ "Trace", + /* 137 */ "Noop", + /* 138 */ "Explain", /* 139 */ "NotUsed_139", /* 140 */ "NotUsed_140", /* 141 */ "ToText", @@ -27317,9 +27425,24 @@ static int findCreateFileMode( int nDb; /* Number of valid bytes in zDb */ struct stat sStat; /* Output of stat() on database file */ - nDb = sqlite3Strlen30(zPath) - ((flags & SQLITE_OPEN_WAL) ? 4 : 8); + /* zPath is a path to a WAL or journal file. The following block derives + ** the path to the associated database file from zPath. This block handles + ** the following naming conventions: + ** + ** "-journal" + ** "-wal" + ** "-journal-NNNN" + ** "-wal-NNNN" + ** + ** where NNNN is a 4 digit decimal number. The NNNN naming schemes are + ** used by the test_multiplex.c module. + */ + nDb = sqlite3Strlen30(zPath) - 1; + while( nDb>0 && zPath[nDb]!='l' ) nDb--; + nDb -= ((flags & SQLITE_OPEN_WAL) ? 3 : 7); memcpy(zDb, zPath, nDb); zDb[nDb] = '\0'; + if( 0==stat(zDb, &sStat) ){ *pMode = sStat.st_mode & 0777; }else{ @@ -27734,7 +27857,7 @@ static void *unixDlOpen(sqlite3_vfs *NotUsed, const char *zFilename){ ** error message. */ static void unixDlError(sqlite3_vfs *NotUsed, int nBuf, char *zBufOut){ - char *zErr; + const char *zErr; UNUSED_PARAMETER(NotUsed); unixEnterMutex(); zErr = dlerror(); @@ -27871,7 +27994,7 @@ static int unixCurrentTimeInt64(sqlite3_vfs *NotUsed, sqlite3_int64 *piNow){ #if defined(NO_GETTOD) time_t t; time(&t); - *piNow = ((sqlite3_int64)i)*1000 + unixEpoch; + *piNow = ((sqlite3_int64)t)*1000 + unixEpoch; #elif OS_VXWORKS struct timespec sNow; clock_gettime(CLOCK_REALTIME, &sNow); @@ -28328,27 +28451,27 @@ static int proxyBreakConchLock(unixFile *pFile, uuid_t myHostID){ pathLen = strlcpy(tPath, cPath, MAXPATHLEN); if( pathLen>MAXPATHLEN || pathLen<6 || (strlcpy(&tPath[pathLen-5], "break", 6) != 5) ){ - sprintf(errmsg, "path error (len %d)", (int)pathLen); + sqlite3_snprintf(sizeof(errmsg),errmsg,"path error (len %d)",(int)pathLen); goto end_breaklock; } /* read the conch content */ readLen = pread(conchFile->h, buf, PROXY_MAXCONCHLEN, 0); if( readLenexclusiveMode ); + assert( !pPager->exclusiveMode || pPager->eLock==eLock ); assert( eLock==NO_LOCK || eLock==SHARED_LOCK ); assert( eLock!=NO_LOCK || pagerUseWal(pPager)==0 ); if( isOpen(pPager->fd) ){ @@ -35873,7 +36006,7 @@ static int zeroJournalHdr(Pager *pPager, int doTruncate){ rc = sqlite3OsWrite(pPager->jfd, zeroHdr, sizeof(zeroHdr), 0); } if( rc==SQLITE_OK && !pPager->noSync ){ - rc = sqlite3OsSync(pPager->jfd, SQLITE_SYNC_DATAONLY|pPager->sync_flags); + rc = sqlite3OsSync(pPager->jfd, SQLITE_SYNC_DATAONLY|pPager->syncFlags); } /* At this point the transaction is committed but the write lock @@ -37325,7 +37458,7 @@ end_playback: if( rc==SQLITE_OK && !pPager->noSync && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) ){ - rc = sqlite3OsSync(pPager->fd, pPager->sync_flags); + rc = sqlite3OsSync(pPager->fd, pPager->syncFlags); } if( rc==SQLITE_OK ){ rc = pager_end_transaction(pPager, zMaster[0]!='\0'); @@ -37499,13 +37632,13 @@ static int pagerWalFrames( PgHdr *pList, /* List of frames to log */ Pgno nTruncate, /* Database size after this commit */ int isCommit, /* True if this is a commit */ - int sync_flags /* Flags to pass to OsSync() (or 0) */ + int syncFlags /* Flags to pass to OsSync() (or 0) */ ){ int rc; /* Return code */ assert( pPager->pWal ); rc = sqlite3WalFrames(pPager->pWal, - pPager->pageSize, pList, nTruncate, isCommit, sync_flags + pPager->pageSize, pList, nTruncate, isCommit, syncFlags ); if( rc==SQLITE_OK && pPager->pBackup ){ PgHdr *p; @@ -37835,14 +37968,49 @@ SQLITE_PRIVATE void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ ** assurance that the journal will not be corrupted to the ** point of causing damage to the database during rollback. ** +** The above is for a rollback-journal mode. For WAL mode, OFF continues +** to mean that no syncs ever occur. NORMAL means that the WAL is synced +** prior to the start of checkpoint and that the database file is synced +** at the conclusion of the checkpoint if the entire content of the WAL +** was written back into the database. But no sync operations occur for +** an ordinary commit in NORMAL mode with WAL. FULL means that the WAL +** file is synced following each commit operation, in addition to the +** syncs associated with NORMAL. +** +** Do not confuse synchronous=FULL with SQLITE_SYNC_FULL. The +** SQLITE_SYNC_FULL macro means to use the MacOSX-style full-fsync +** using fcntl(F_FULLFSYNC). SQLITE_SYNC_NORMAL means to do an +** ordinary fsync() call. There is no difference between SQLITE_SYNC_FULL +** and SQLITE_SYNC_NORMAL on platforms other than MacOSX. But the +** synchronous=FULL versus synchronous=NORMAL setting determines when +** the xSync primitive is called and is relevant to all platforms. +** ** Numeric values associated with these states are OFF==1, NORMAL=2, ** and FULL=3. */ #ifndef SQLITE_OMIT_PAGER_PRAGMAS -SQLITE_PRIVATE void sqlite3PagerSetSafetyLevel(Pager *pPager, int level, int bFullFsync){ +SQLITE_PRIVATE void sqlite3PagerSetSafetyLevel( + Pager *pPager, /* The pager to set safety level for */ + int level, /* PRAGMA synchronous. 1=OFF, 2=NORMAL, 3=FULL */ + int bFullFsync, /* PRAGMA fullfsync */ + int bCkptFullFsync /* PRAGMA checkpoint_fullfsync */ +){ + assert( level>=1 && level<=3 ); pPager->noSync = (level==1 || pPager->tempFile) ?1:0; pPager->fullSync = (level==3 && !pPager->tempFile) ?1:0; - pPager->sync_flags = (bFullFsync?SQLITE_SYNC_FULL:SQLITE_SYNC_NORMAL); + if( pPager->noSync ){ + pPager->syncFlags = 0; + pPager->ckptSyncFlags = 0; + }else if( bFullFsync ){ + pPager->syncFlags = SQLITE_SYNC_FULL; + pPager->ckptSyncFlags = SQLITE_SYNC_FULL; + }else if( bCkptFullFsync ){ + pPager->syncFlags = SQLITE_SYNC_NORMAL; + pPager->ckptSyncFlags = SQLITE_SYNC_FULL; + }else{ + pPager->syncFlags = SQLITE_SYNC_NORMAL; + pPager->ckptSyncFlags = SQLITE_SYNC_NORMAL; + } } #endif @@ -38021,9 +38189,8 @@ SQLITE_PRIVATE int sqlite3PagerMaxPageCount(Pager *pPager, int mxPage){ if( mxPage>0 ){ pPager->mxPgno = mxPage; } - if( pPager->eState!=PAGER_OPEN && pPager->mxPgnodbSize ){ - pPager->mxPgno = pPager->dbSize; - } + assert( pPager->eState!=PAGER_OPEN ); /* Called only by OP_MaxPgcnt */ + assert( pPager->mxPgno>=pPager->dbSize ); /* OP_MaxPgcnt enforces this */ return pPager->mxPgno; } @@ -38228,10 +38395,7 @@ SQLITE_PRIVATE int sqlite3PagerClose(Pager *pPager){ /* pPager->errCode = 0; */ pPager->exclusiveMode = 0; #ifndef SQLITE_OMIT_WAL - sqlite3WalClose(pPager->pWal, - (pPager->noSync ? 0 : pPager->sync_flags), - pPager->pageSize, pTmp - ); + sqlite3WalClose(pPager->pWal, pPager->ckptSyncFlags, pPager->pageSize, pTmp); pPager->pWal = 0; #endif pager_reset(pPager); @@ -38397,7 +38561,7 @@ static int syncJournal(Pager *pPager, int newHdr){ if( pPager->fullSync && 0==(iDc&SQLITE_IOCAP_SEQUENTIAL) ){ PAGERTRACE(("SYNC journal of %d\n", PAGERID(pPager))); IOTRACE(("JSYNC %p\n", pPager)) - rc = sqlite3OsSync(pPager->jfd, pPager->sync_flags); + rc = sqlite3OsSync(pPager->jfd, pPager->syncFlags); if( rc!=SQLITE_OK ) return rc; } IOTRACE(("JHDR %p %lld\n", pPager, pPager->journalHdr)); @@ -38409,8 +38573,8 @@ static int syncJournal(Pager *pPager, int newHdr){ if( 0==(iDc&SQLITE_IOCAP_SEQUENTIAL) ){ PAGERTRACE(("SYNC journal of %d\n", PAGERID(pPager))); IOTRACE(("JSYNC %p\n", pPager)) - rc = sqlite3OsSync(pPager->jfd, pPager->sync_flags| - (pPager->sync_flags==SQLITE_SYNC_FULL?SQLITE_SYNC_DATAONLY:0) + rc = sqlite3OsSync(pPager->jfd, pPager->syncFlags| + (pPager->syncFlags==SQLITE_SYNC_FULL?SQLITE_SYNC_DATAONLY:0) ); if( rc!=SQLITE_OK ) return rc; } @@ -39000,7 +39164,8 @@ SQLITE_PRIVATE int sqlite3PagerOpen( assert( useJournal || pPager->tempFile ); pPager->noSync = pPager->tempFile; pPager->fullSync = pPager->noSync ?0:1; - pPager->sync_flags = SQLITE_SYNC_NORMAL; + pPager->syncFlags = pPager->noSync ? 0 : SQLITE_SYNC_NORMAL; + pPager->ckptSyncFlags = pPager->syncFlags; /* pPager->pFirst = 0; */ /* pPager->pFirstSynced = 0; */ /* pPager->pLast = 0; */ @@ -39100,7 +39265,7 @@ static int hasHotJournal(Pager *pPager, int *pExists){ sqlite3BeginBenignMalloc(); if( pagerLockDb(pPager, RESERVED_LOCK)==SQLITE_OK ){ sqlite3OsDelete(pVfs, pPager->zJournal, 0); - pagerUnlockDb(pPager, SHARED_LOCK); + if( !pPager->exclusiveMode ) pagerUnlockDb(pPager, SHARED_LOCK); } sqlite3EndBenignMalloc(); }else{ @@ -40123,10 +40288,10 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ } /* -** Sync the pager file to disk. This is a no-op for in-memory files +** Sync the database file to disk. This is a no-op for in-memory databases ** or pages with the Pager.noSync flag set. ** -** If successful, or called on a pager for which it is a no-op, this +** If successful, or if called on a pager for which it is a no-op, this ** function returns SQLITE_OK. Otherwise, an IO error code is returned. */ SQLITE_PRIVATE int sqlite3PagerSync(Pager *pPager){ @@ -40135,7 +40300,7 @@ SQLITE_PRIVATE int sqlite3PagerSync(Pager *pPager){ if( pPager->noSync ){ rc = SQLITE_OK; }else{ - rc = sqlite3OsSync(pPager->fd, pPager->sync_flags); + rc = sqlite3OsSync(pPager->fd, pPager->syncFlags); } return rc; } @@ -40224,7 +40389,7 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne( PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache); if( pList ){ rc = pagerWalFrames(pPager, pList, pPager->dbSize, 1, - (pPager->fullSync ? pPager->sync_flags : 0) + (pPager->fullSync ? pPager->syncFlags : 0) ); } if( rc==SQLITE_OK ){ @@ -40355,7 +40520,7 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne( /* Finally, sync the database file. */ if( !pPager->noSync && !noSync ){ - rc = sqlite3OsSync(pPager->fd, pPager->sync_flags); + rc = sqlite3OsSync(pPager->fd, pPager->syncFlags); } IOTRACE(("DBSYNC %p\n", pPager)) } @@ -40926,7 +41091,8 @@ SQLITE_PRIVATE int sqlite3PagerLockingMode(Pager *pPager, int eMode){ || eMode==PAGER_LOCKINGMODE_EXCLUSIVE ); assert( PAGER_LOCKINGMODE_QUERY<0 ); assert( PAGER_LOCKINGMODE_NORMAL>=0 && PAGER_LOCKINGMODE_EXCLUSIVE>=0 ); - if( eMode>=0 && !pPager->tempFile ){ + assert( pPager->exclusiveMode || 0==sqlite3WalHeapMemory(pPager->pWal) ); + if( eMode>=0 && !pPager->tempFile && !sqlite3WalHeapMemory(pPager->pWal) ){ pPager->exclusiveMode = (u8)eMode; } return (int)pPager->exclusiveMode; @@ -41095,10 +41261,8 @@ SQLITE_PRIVATE int sqlite3PagerCheckpoint(Pager *pPager){ int rc = SQLITE_OK; if( pPager->pWal ){ u8 *zBuf = (u8 *)pPager->pTmpSpace; - rc = sqlite3WalCheckpoint(pPager->pWal, - (pPager->noSync ? 0 : pPager->sync_flags), - pPager->pageSize, zBuf - ); + rc = sqlite3WalCheckpoint(pPager->pWal, pPager->ckptSyncFlags, + pPager->pageSize, zBuf); } return rc; } @@ -41113,9 +41277,61 @@ SQLITE_PRIVATE int sqlite3PagerWalCallback(Pager *pPager){ */ SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager){ const sqlite3_io_methods *pMethods = pPager->fd->pMethods; - return pMethods->iVersion>=2 && pMethods->xShmMap!=0; + return pPager->exclusiveMode || (pMethods->iVersion>=2 && pMethods->xShmMap); +} + +/* +** Attempt to take an exclusive lock on the database file. If a PENDING lock +** is obtained instead, immediately release it. +*/ +static int pagerExclusiveLock(Pager *pPager){ + int rc; /* Return code */ + + assert( pPager->eLock==SHARED_LOCK || pPager->eLock==EXCLUSIVE_LOCK ); + rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); + if( rc!=SQLITE_OK ){ + /* If the attempt to grab the pending lock failed, release the + ** exclusive lock that may have been obtained instead. */ + pagerUnlockDb(pPager, SHARED_LOCK); + } + + return rc; } +/* +** Call sqlite3WalOpen() to open the WAL handle. If the pager is in +** exclusive-locking mode when this function is called, take an EXCLUSIVE +** lock on the database file and use heap-memory to store the wal-index +** in. Otherwise, use the normal shared-memory. +*/ +static int pagerOpenWal(Pager *pPager){ + int rc = SQLITE_OK; + + assert( pPager->pWal==0 && pPager->tempFile==0 ); + assert( pPager->eLock==SHARED_LOCK || pPager->eLock==EXCLUSIVE_LOCK || pPager->noReadlock); + + /* If the pager is already in exclusive-mode, the WAL module will use + ** heap-memory for the wal-index instead of the VFS shared-memory + ** implementation. Take the exclusive lock now, before opening the WAL + ** file, to make sure this is safe. + */ + if( pPager->exclusiveMode ){ + rc = pagerExclusiveLock(pPager); + } + + /* Open the connection to the log file. If this operation fails, + ** (e.g. due to malloc() failure), return an error code. + */ + if( rc==SQLITE_OK ){ + rc = sqlite3WalOpen(pPager->pVfs, + pPager->fd, pPager->zWal, pPager->exclusiveMode, &pPager->pWal + ); + } + + return rc; +} + + /* ** The caller must be holding a SHARED lock on the database file to call ** this function. @@ -41149,11 +41365,7 @@ SQLITE_PRIVATE int sqlite3PagerOpenWal( /* Close any rollback journal previously open */ sqlite3OsClose(pPager->jfd); - /* Open the connection to the log file. If this operation fails, - ** (e.g. due to malloc() failure), unlock the database file and - ** return an error code. - */ - rc = sqlite3WalOpen(pPager->pVfs, pPager->fd, pPager->zWal, &pPager->pWal); + rc = pagerOpenWal(pPager); if( rc==SQLITE_OK ){ pPager->journalMode = PAGER_JOURNALMODE_WAL; pPager->eState = PAGER_OPEN; @@ -41192,8 +41404,7 @@ SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager){ ); } if( rc==SQLITE_OK && logexists ){ - rc = sqlite3WalOpen(pPager->pVfs, pPager->fd, - pPager->zWal, &pPager->pWal); + rc = pagerOpenWal(pPager); } } @@ -41201,17 +41412,11 @@ SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager){ ** the database file, the log and log-summary files will be deleted. */ if( rc==SQLITE_OK && pPager->pWal ){ - rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); + rc = pagerExclusiveLock(pPager); if( rc==SQLITE_OK ){ - rc = sqlite3WalClose(pPager->pWal, - (pPager->noSync ? 0 : pPager->sync_flags), - pPager->pageSize, (u8*)pPager->pTmpSpace - ); + rc = sqlite3WalClose(pPager->pWal, pPager->ckptSyncFlags, + pPager->pageSize, (u8*)pPager->pTmpSpace); pPager->pWal = 0; - }else{ - /* If we cannot get an EXCLUSIVE lock, downgrade the PENDING lock - ** that we did get back to SHARED. */ - pagerUnlockDb(pPager, SQLITE_LOCK_SHARED); } } return rc; @@ -41667,6 +41872,13 @@ struct Wal { #endif }; +/* +** Candidate values for Wal.exclusiveMode. +*/ +#define WAL_NORMAL_MODE 0 +#define WAL_EXCLUSIVE_MODE 1 +#define WAL_HEAPMEMORY_MODE 2 + /* ** Each page of the wal-index mapping contains a hash-table made up of ** an array of HASHTABLE_NSLOT elements of the following type. @@ -41753,9 +41965,14 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){ /* Request a pointer to the required page from the VFS */ if( pWal->apWiData[iPage]==0 ){ - rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, - pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] - ); + if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){ + pWal->apWiData[iPage] = (u32 volatile *)sqlite3MallocZero(WALINDEX_PGSZ); + if( !pWal->apWiData[iPage] ) rc = SQLITE_NOMEM; + }else{ + rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, + pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] + ); + } } *ppPage = pWal->apWiData[iPage]; @@ -41838,6 +42055,12 @@ static void walChecksumBytes( aOut[1] = s2; } +static void walShmBarrier(Wal *pWal){ + if( pWal->exclusiveMode!=WAL_HEAPMEMORY_MODE ){ + sqlite3OsShmBarrier(pWal->pDbFd); + } +} + /* ** Write the header information in pWal->hdr into the wal-index. ** @@ -41852,7 +42075,7 @@ static void walIndexWriteHdr(Wal *pWal){ pWal->hdr.iVersion = WALINDEX_MAX_VERSION; walChecksumBytes(1, (u8*)&pWal->hdr, nCksum, 0, pWal->hdr.aCksum); memcpy((void *)&aHdr[1], (void *)&pWal->hdr, sizeof(WalIndexHdr)); - sqlite3OsShmBarrier(pWal->pDbFd); + walShmBarrier(pWal); memcpy((void *)&aHdr[0], (void *)&pWal->hdr, sizeof(WalIndexHdr)); } @@ -42424,7 +42647,15 @@ recovery_error: ** Close an open wal-index. */ static void walIndexClose(Wal *pWal, int isDelete){ - sqlite3OsShmUnmap(pWal->pDbFd, isDelete); + if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){ + int i; + for(i=0; inWiData; i++){ + sqlite3_free((void *)pWal->apWiData[i]); + pWal->apWiData[i] = 0; + } + }else{ + sqlite3OsShmUnmap(pWal->pDbFd, isDelete); + } } /* @@ -42446,6 +42677,7 @@ SQLITE_PRIVATE int sqlite3WalOpen( sqlite3_vfs *pVfs, /* vfs module to open wal and wal-index */ sqlite3_file *pDbFd, /* The open database file */ const char *zWalName, /* Name of the WAL file */ + int bNoShm, /* True to run in heap-memory mode */ Wal **ppWal /* OUT: Allocated Wal handle */ ){ int rc; /* Return Code */ @@ -42479,6 +42711,7 @@ SQLITE_PRIVATE int sqlite3WalOpen( pRet->pDbFd = pDbFd; pRet->readLock = -1; pRet->zWalName = zWalName; + pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE); /* Open file handle on the write-ahead log file. */ flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL); @@ -42912,7 +43145,9 @@ SQLITE_PRIVATE int sqlite3WalClose( */ rc = sqlite3OsLock(pWal->pDbFd, SQLITE_LOCK_EXCLUSIVE); if( rc==SQLITE_OK ){ - pWal->exclusiveMode = 1; + if( pWal->exclusiveMode==WAL_NORMAL_MODE ){ + pWal->exclusiveMode = WAL_EXCLUSIVE_MODE; + } rc = sqlite3WalCheckpoint(pWal, sync_flags, nBuf, zBuf); if( rc==SQLITE_OK ){ isDelete = 1; @@ -42968,7 +43203,7 @@ static int walIndexTryHdr(Wal *pWal, int *pChanged){ */ aHdr = walIndexHdr(pWal); memcpy(&h1, (void *)&aHdr[0], sizeof(h1)); - sqlite3OsShmBarrier(pWal->pDbFd); + walShmBarrier(pWal); memcpy(&h2, (void *)&aHdr[1], sizeof(h2)); if( memcmp(&h1, &h2, sizeof(h1))!=0 ){ @@ -43169,7 +43404,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ ** and can be safely ignored. */ rc = walLockShared(pWal, WAL_READ_LOCK(0)); - sqlite3OsShmBarrier(pWal->pDbFd); + walShmBarrier(pWal); if( rc==SQLITE_OK ){ if( memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){ /* It is not safe to allow the reader to continue here if frames @@ -43263,7 +43498,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ ** log-wrap (either of which would require an exclusive lock on ** WAL_READ_LOCK(mxI)) has not occurred since the snapshot was valid. */ - sqlite3OsShmBarrier(pWal->pDbFd); + walShmBarrier(pWal); if( pInfo->aReadMark[mxI]!=mxReadMark || memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){ @@ -43906,13 +44141,14 @@ SQLITE_PRIVATE int sqlite3WalCallback(Wal *pWal){ ** on the main database file before invoking this operation. ** ** If op is negative, then do a dry-run of the op==1 case but do -** not actually change anything. The pager uses this to see if it +** not actually change anything. The pager uses this to see if it ** should acquire the database exclusive lock prior to invoking ** the op==1 case. */ SQLITE_PRIVATE int sqlite3WalExclusiveMode(Wal *pWal, int op){ int rc; assert( pWal->writeLock==0 ); + assert( pWal->exclusiveMode!=WAL_HEAPMEMORY_MODE || op==-1 ); /* pWal->readLock is usually set, but might be -1 if there was a ** prior error while attempting to acquire are read-lock. This cannot @@ -43946,6 +44182,15 @@ SQLITE_PRIVATE int sqlite3WalExclusiveMode(Wal *pWal, int op){ return rc; } +/* +** Return true if the argument is non-NULL and the WAL module is using +** heap-memory for the wal-index. Otherwise, if the argument is NULL or the +** WAL module is using shared-memory, return false. +*/ +SQLITE_PRIVATE int sqlite3WalHeapMemory(Wal *pWal){ + return (pWal && pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ); +} + #endif /* #ifndef SQLITE_OMIT_WAL */ /************** End of wal.c *************************************************/ @@ -46669,17 +46914,13 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( const int isTempDb = zFilename==0 || zFilename[0]==0; /* Set the variable isMemdb to true for an in-memory database, or - ** false for a file-based database. This symbol is only required if - ** either of the shared-data or autovacuum features are compiled - ** into the library. + ** false for a file-based database. */ -#if !defined(SQLITE_OMIT_SHARED_CACHE) || !defined(SQLITE_OMIT_AUTOVACUUM) - #ifdef SQLITE_OMIT_MEMORYDB - const int isMemdb = 0; - #else - const int isMemdb = (zFilename && strcmp(zFilename, ":memory:")==0) - || (isTempDb && sqlite3TempInMemory(db)); - #endif +#ifdef SQLITE_OMIT_MEMORYDB + const int isMemdb = 0; +#else + const int isMemdb = (zFilename && strcmp(zFilename, ":memory:")==0) + || (isTempDb && sqlite3TempInMemory(db)); #endif assert( db!=0 ); @@ -47063,11 +47304,17 @@ SQLITE_PRIVATE int sqlite3BtreeSetCacheSize(Btree *p, int mxPage){ ** probability of damage to near zero but with a write performance reduction. */ #ifndef SQLITE_OMIT_PAGER_PRAGMAS -SQLITE_PRIVATE int sqlite3BtreeSetSafetyLevel(Btree *p, int level, int fullSync){ +SQLITE_PRIVATE int sqlite3BtreeSetSafetyLevel( + Btree *p, /* The btree to set the safety level on */ + int level, /* PRAGMA synchronous. 1=OFF, 2=NORMAL, 3=FULL */ + int fullSync, /* PRAGMA fullfsync. */ + int ckptFullSync /* PRAGMA checkpoint_fullfync */ +){ BtShared *pBt = p->pBt; assert( sqlite3_mutex_held(p->db->mutex) ); + assert( level>=1 && level<=3 ); sqlite3BtreeEnter(p); - sqlite3PagerSetSafetyLevel(pBt->pPager, level, fullSync); + sqlite3PagerSetSafetyLevel(pBt->pPager, level, fullSync, ckptFullSync); sqlite3BtreeLeave(p); return SQLITE_OK; } @@ -48125,8 +48372,8 @@ static void btreeEndTransaction(Btree *p){ ** are no active cursors, it also releases the read lock. */ SQLITE_PRIVATE int sqlite3BtreeCommitPhaseTwo(Btree *p){ - BtShared *pBt = p->pBt; + if( p->inTrans==TRANS_NONE ) return SQLITE_OK; sqlite3BtreeEnter(p); btreeIntegrity(p); @@ -48135,6 +48382,7 @@ SQLITE_PRIVATE int sqlite3BtreeCommitPhaseTwo(Btree *p){ */ if( p->inTrans==TRANS_WRITE ){ int rc; + BtShared *pBt = p->pBt; assert( pBt->inTransaction==TRANS_WRITE ); assert( pBt->nTransaction>0 ); rc = sqlite3PagerCommitPhaseTwo(pBt->pPager); @@ -53053,8 +53301,7 @@ SQLITE_PRIVATE int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void SQLITE_PRIVATE void sqlite3BtreeCacheOverflow(BtCursor *pCur){ assert( cursorHoldsMutex(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); - assert(!pCur->isIncrblobHandle); - assert(!pCur->aOverflow); + invalidateOverflowCache(pCur); pCur->isIncrblobHandle = 1; } #endif @@ -54242,7 +54489,9 @@ SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem *pMem){ */ SQLITE_PRIVATE void sqlite3VdbeMemSetNull(Mem *pMem){ if( pMem->flags & MEM_Frame ){ - sqlite3VdbeFrameDelete(pMem->u.pFrame); + VdbeFrame *pFrame = pMem->u.pFrame; + pFrame->pParent = pFrame->v->pDelFrame; + pFrame->v->pDelFrame = pFrame; } if( pMem->flags & MEM_RowSet ){ sqlite3RowSetClear(pMem->u.pRowSet); @@ -56087,12 +56336,10 @@ SQLITE_PRIVATE int sqlite3VdbeList( pMem->type = SQLITE_INTEGER; pMem++; - if( p->explain==1 ){ - pMem->flags = MEM_Int; - pMem->u.i = pOp->p3; /* P3 */ - pMem->type = SQLITE_INTEGER; - pMem++; - } + pMem->flags = MEM_Int; + pMem->u.i = pOp->p3; /* P3 */ + pMem->type = SQLITE_INTEGER; + pMem++; if( sqlite3VdbeMemGrow(pMem, 32, 0) ){ /* P4 */ assert( p->db->mallocFailed ); @@ -56137,7 +56384,7 @@ SQLITE_PRIVATE int sqlite3VdbeList( } } - p->nResColumn = 8 - 5*(p->explain-1); + p->nResColumn = 8 - 4*(p->explain-1); p->rc = SQLITE_OK; rc = SQLITE_ROW; } @@ -56444,6 +56691,11 @@ static void closeAllCursors(Vdbe *p){ if( p->aMem ){ releaseMemArray(&p->aMem[1], p->nMem); } + while( p->pDelFrame ){ + VdbeFrame *pDel = p->pDelFrame; + p->pDelFrame = pDel->pParent; + sqlite3VdbeFrameDelete(pDel); + } } /* @@ -59084,6 +59336,8 @@ static int bindText( rc = sqlite3ApiExit(p->db, rc); } sqlite3_mutex_leave(p->db->mutex); + }else if( xDel!=SQLITE_STATIC && xDel!=SQLITE_TRANSIENT ){ + xDel((void*)zData); } return rc; } @@ -59326,6 +59580,14 @@ SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt *pStmt){ return pStmt ? ((Vdbe*)pStmt)->db : 0; } +/* +** Return true if the prepared statement is guaranteed to not modify the +** database. +*/ +SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt){ + return pStmt ? ((Vdbe*)pStmt)->readOnly : 1; +} + /* ** Return a pointer to the next prepared statement after pStmt associated ** with database connection pDb. If pStmt is NULL, return the first @@ -59810,13 +60072,13 @@ static void applyAffinity( ** into a numeric representation. Use either INTEGER or REAL whichever ** is appropriate. But only do the conversion if it is possible without ** loss of information and return the revised type of the argument. -** -** This is an EXPERIMENTAL api and is subject to change or removal. */ SQLITE_API int sqlite3_value_numeric_type(sqlite3_value *pVal){ Mem *pMem = (Mem*)pVal; - applyNumericAffinity(pMem); - sqlite3VdbeMemStoreType(pMem); + if( pMem->type==SQLITE_TEXT ){ + applyNumericAffinity(pMem); + sqlite3VdbeMemStoreType(pMem); + } return pMem->type; } @@ -61517,7 +61779,6 @@ case OP_AddImm: { /* in1 */ */ case OP_MustBeInt: { /* jump, in1 */ pIn1 = &aMem[pOp->p1]; - memAboutToChange(p, pIn1); applyAffinity(pIn1, SQLITE_AFF_NUMERIC, encoding); if( (pIn1->flags & MEM_Int)==0 ){ if( pOp->p2==0 ){ @@ -65903,6 +66164,32 @@ case OP_Pagecount: { /* out2-prerelease */ } #endif + +#ifndef SQLITE_OMIT_PAGER_PRAGMAS +/* Opcode: MaxPgcnt P1 P2 P3 * * +** +** Try to set the maximum page count for database P1 to the value in P3. +** Do not let the maximum page count fall below the current page count and +** do not change the maximum page count value if P3==0. +** +** Store the maximum page count after the change in register P2. +*/ +case OP_MaxPgcnt: { /* out2-prerelease */ + unsigned int newMax; + Btree *pBt; + + pBt = db->aDb[pOp->p1].pBt; + newMax = 0; + if( pOp->p3 ){ + newMax = sqlite3BtreeLastPage(pBt); + if( newMax < (unsigned)pOp->p3 ) newMax = (unsigned)pOp->p3; + } + pOut->u.i = sqlite3BtreeMaxPageCount(pBt, newMax); + break; +} +#endif + + #ifndef SQLITE_OMIT_TRACE /* Opcode: Trace * * * P4 * ** @@ -66077,11 +66364,82 @@ struct Incrblob { int flags; /* Copy of "flags" passed to sqlite3_blob_open() */ int nByte; /* Size of open blob, in bytes */ int iOffset; /* Byte offset of blob in cursor data */ + int iCol; /* Table column this handle is open on */ BtCursor *pCsr; /* Cursor pointing at blob row */ sqlite3_stmt *pStmt; /* Statement holding cursor open */ sqlite3 *db; /* The associated database */ }; + +/* +** This function is used by both blob_open() and blob_reopen(). It seeks +** the b-tree cursor associated with blob handle p to point to row iRow. +** If successful, SQLITE_OK is returned and subsequent calls to +** sqlite3_blob_read() or sqlite3_blob_write() access the specified row. +** +** If an error occurs, or if the specified row does not exist or does not +** contain a value of type TEXT or BLOB in the column nominated when the +** blob handle was opened, then an error code is returned and *pzErr may +** be set to point to a buffer containing an error message. It is the +** responsibility of the caller to free the error message buffer using +** sqlite3DbFree(). +** +** If an error does occur, then the b-tree cursor is closed. All subsequent +** calls to sqlite3_blob_read(), blob_write() or blob_reopen() will +** immediately return SQLITE_ABORT. +*/ +static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){ + int rc; /* Error code */ + char *zErr = 0; /* Error message */ + Vdbe *v = (Vdbe *)p->pStmt; + + /* Set the value of the SQL statements only variable to integer iRow. + ** This is done directly instead of using sqlite3_bind_int64() to avoid + ** triggering asserts related to mutexes. + */ + assert( v->aVar[0].flags&MEM_Int ); + v->aVar[0].u.i = iRow; + + rc = sqlite3_step(p->pStmt); + if( rc==SQLITE_ROW ){ + u32 type = v->apCsr[0]->aType[p->iCol]; + if( type<12 ){ + zErr = sqlite3MPrintf(p->db, "cannot open value of type %s", + type==0?"null": type==7?"real": "integer" + ); + rc = SQLITE_ERROR; + sqlite3_finalize(p->pStmt); + p->pStmt = 0; + }else{ + p->iOffset = v->apCsr[0]->aOffset[p->iCol]; + p->nByte = sqlite3VdbeSerialTypeLen(type); + p->pCsr = v->apCsr[0]->pCursor; + sqlite3BtreeEnterCursor(p->pCsr); + sqlite3BtreeCacheOverflow(p->pCsr); + sqlite3BtreeLeaveCursor(p->pCsr); + } + } + + if( rc==SQLITE_ROW ){ + rc = SQLITE_OK; + }else if( p->pStmt ){ + rc = sqlite3_finalize(p->pStmt); + p->pStmt = 0; + if( rc==SQLITE_OK ){ + zErr = sqlite3MPrintf(p->db, "no such rowid: %lld", iRow); + rc = SQLITE_ERROR; + }else{ + zErr = sqlite3MPrintf(p->db, "%s", sqlite3_errmsg(p->db)); + } + } + + assert( rc!=SQLITE_OK || zErr==0 ); + assert( rc!=SQLITE_ROW && rc!=SQLITE_DONE ); + + *pzErr = zErr; + return rc; +} + /* ** Open a blob handle. */ @@ -66122,29 +66480,35 @@ SQLITE_API int sqlite3_blob_open( {OP_OpenWrite, 0, 0, 0}, /* 4: Open cursor 0 for read/write */ {OP_Variable, 1, 1, 1}, /* 5: Push the rowid to the stack */ - {OP_NotExists, 0, 9, 1}, /* 6: Seek the cursor */ + {OP_NotExists, 0, 10, 1}, /* 6: Seek the cursor */ {OP_Column, 0, 0, 1}, /* 7 */ {OP_ResultRow, 1, 0, 0}, /* 8 */ - {OP_Close, 0, 0, 0}, /* 9 */ - {OP_Halt, 0, 0, 0}, /* 10 */ + {OP_Goto, 0, 5, 0}, /* 9 */ + {OP_Close, 0, 0, 0}, /* 10 */ + {OP_Halt, 0, 0, 0}, /* 11 */ }; - Vdbe *v = 0; int rc = SQLITE_OK; char *zErr = 0; Table *pTab; - Parse *pParse; + Parse *pParse = 0; + Incrblob *pBlob = 0; + flags = !!flags; /* flags = (flags ? 1 : 0); */ *ppBlob = 0; + sqlite3_mutex_enter(db->mutex); + + pBlob = (Incrblob *)sqlite3DbMallocZero(db, sizeof(Incrblob)); + if( !pBlob ) goto blob_open_out; pParse = sqlite3StackAllocRaw(db, sizeof(*pParse)); - if( pParse==0 ){ - rc = SQLITE_NOMEM; - goto blob_open_out; - } + if( !pParse ) goto blob_open_out; + do { memset(pParse, 0, sizeof(Parse)); pParse->db = db; + sqlite3DbFree(db, zErr); + zErr = 0; sqlite3BtreeEnterAll(db); pTab = sqlite3LocateTable(pParse, 0, zTable, zDb); @@ -66170,7 +66534,7 @@ SQLITE_API int sqlite3_blob_open( } /* Now search pTab for the exact column. */ - for(iCol=0; iCol < pTab->nCol; iCol++) { + for(iCol=0; iColnCol; iCol++) { if( sqlite3StrICmp(pTab->aCol[iCol].zName, zColumn)==0 ){ break; } @@ -66224,11 +66588,14 @@ SQLITE_API int sqlite3_blob_open( } } - v = sqlite3VdbeCreate(db); - if( v ){ + pBlob->pStmt = (sqlite3_stmt *)sqlite3VdbeCreate(db); + assert( pBlob->pStmt || db->mallocFailed ); + if( pBlob->pStmt ){ + Vdbe *v = (Vdbe *)pBlob->pStmt; int iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + sqlite3VdbeAddOpList(v, sizeof(openBlob)/sizeof(VdbeOpList), openBlob); - flags = !!flags; /* flags = (flags ? 1 : 0); */ + /* Configure the OP_Transaction */ sqlite3VdbeChangeP1(v, 0, iDb); @@ -66271,65 +66638,25 @@ SQLITE_API int sqlite3_blob_open( } } + pBlob->flags = flags; + pBlob->iCol = iCol; + pBlob->db = db; sqlite3BtreeLeaveAll(db); if( db->mallocFailed ){ goto blob_open_out; } - - sqlite3_bind_int64((sqlite3_stmt *)v, 1, iRow); - rc = sqlite3_step((sqlite3_stmt *)v); - if( rc!=SQLITE_ROW ){ - nAttempt++; - rc = sqlite3_finalize((sqlite3_stmt *)v); - sqlite3DbFree(db, zErr); - zErr = sqlite3MPrintf(db, sqlite3_errmsg(db)); - v = 0; - } - } while( nAttempt<5 && rc==SQLITE_SCHEMA ); - - if( rc==SQLITE_ROW ){ - /* The row-record has been opened successfully. Check that the - ** column in question contains text or a blob. If it contains - ** text, it is up to the caller to get the encoding right. - */ - Incrblob *pBlob; - u32 type = v->apCsr[0]->aType[iCol]; - - if( type<12 ){ - sqlite3DbFree(db, zErr); - zErr = sqlite3MPrintf(db, "cannot open value of type %s", - type==0?"null": type==7?"real": "integer" - ); - rc = SQLITE_ERROR; - goto blob_open_out; - } - pBlob = (Incrblob *)sqlite3DbMallocZero(db, sizeof(Incrblob)); - if( db->mallocFailed ){ - sqlite3DbFree(db, pBlob); - goto blob_open_out; - } - pBlob->flags = flags; - pBlob->pCsr = v->apCsr[0]->pCursor; - sqlite3BtreeEnterCursor(pBlob->pCsr); - sqlite3BtreeCacheOverflow(pBlob->pCsr); - sqlite3BtreeLeaveCursor(pBlob->pCsr); - pBlob->pStmt = (sqlite3_stmt *)v; - pBlob->iOffset = v->apCsr[0]->aOffset[iCol]; - pBlob->nByte = sqlite3VdbeSerialTypeLen(type); - pBlob->db = db; - *ppBlob = (sqlite3_blob *)pBlob; - rc = SQLITE_OK; - }else if( rc==SQLITE_OK ){ - sqlite3DbFree(db, zErr); - zErr = sqlite3MPrintf(db, "no such rowid: %lld", iRow); - rc = SQLITE_ERROR; - } + sqlite3_bind_int64(pBlob->pStmt, 1, iRow); + rc = blobSeekToRow(pBlob, iRow, &zErr); + } while( (++nAttempt)<5 && rc==SQLITE_SCHEMA ); blob_open_out: - if( v && (rc!=SQLITE_OK || db->mallocFailed) ){ - sqlite3VdbeFinalize(v); + if( rc==SQLITE_OK && db->mallocFailed==0 ){ + *ppBlob = (sqlite3_blob *)pBlob; + }else{ + if( pBlob && pBlob->pStmt ) sqlite3VdbeFinalize((Vdbe *)pBlob->pStmt); + sqlite3DbFree(db, pBlob); } - sqlite3Error(db, rc, zErr); + sqlite3Error(db, rc, (zErr ? "%s" : 0), zErr); sqlite3DbFree(db, zErr); sqlite3StackFree(db, pParse); rc = sqlite3ApiExit(db, rc); @@ -66382,7 +66709,7 @@ static int blobReadWrite( /* Request is out of range. Return a transient error. */ rc = SQLITE_ERROR; sqlite3Error(db, SQLITE_ERROR, 0); - } else if( v==0 ){ + }else if( v==0 ){ /* If there is no statement handle, then the blob-handle has ** already been invalidated. Return SQLITE_ABORT in this case. */ @@ -66430,7 +66757,47 @@ SQLITE_API int sqlite3_blob_write(sqlite3_blob *pBlob, const void *z, int n, int */ SQLITE_API int sqlite3_blob_bytes(sqlite3_blob *pBlob){ Incrblob *p = (Incrblob *)pBlob; - return p ? p->nByte : 0; + return (p && p->pStmt) ? p->nByte : 0; +} + +/* +** Move an existing blob handle to point to a different row of the same +** database table. +** +** If an error occurs, or if the specified row does not exist or does not +** contain a blob or text value, then an error code is returned and the +** database handle error code and message set. If this happens, then all +** subsequent calls to sqlite3_blob_xxx() functions (except blob_close()) +** immediately return SQLITE_ABORT. +*/ +SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){ + int rc; + Incrblob *p = (Incrblob *)pBlob; + sqlite3 *db; + + if( p==0 ) return SQLITE_MISUSE_BKPT; + db = p->db; + sqlite3_mutex_enter(db->mutex); + + if( p->pStmt==0 ){ + /* If there is no statement handle, then the blob-handle has + ** already been invalidated. Return SQLITE_ABORT in this case. + */ + rc = SQLITE_ABORT; + }else{ + char *zErr; + rc = blobSeekToRow(p, iRow, &zErr); + if( rc!=SQLITE_OK ){ + sqlite3Error(db, rc, (zErr ? "%s" : 0), zErr); + sqlite3DbFree(db, zErr); + } + assert( rc!=SQLITE_SCHEMA ); + } + + rc = sqlite3ApiExit(db, rc); + assert( rc==SQLITE_OK || p->pStmt==0 ); + sqlite3_mutex_leave(db->mutex); + return rc; } #endif /* #ifndef SQLITE_OMIT_INCRBLOB */ @@ -68763,6 +69130,9 @@ SQLITE_PRIVATE Expr *sqlite3PExpr( ){ Expr *p = sqlite3ExprAlloc(pParse->db, op, pToken, 1); sqlite3ExprAttachSubtrees(pParse->db, p, pLeft, pRight); + if( p ) { + sqlite3ExprCheckHeight(pParse, p->nHeight); + } return p; } @@ -69876,6 +70246,16 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( assert( testAddr>0 || pParse->db->mallocFailed ); } +#ifndef SQLITE_OMIT_EXPLAIN + if( pParse->explain==2 ){ + char *zMsg = sqlite3MPrintf( + pParse->db, "EXECUTE %s%s SUBQUERY %d", testAddr?"":"CORRELATED ", + pExpr->op==TK_IN?"LIST":"SCALAR", pParse->iNextSelectId + ); + sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC); + } +#endif + switch( pExpr->op ){ case TK_IN: { char affinity; /* Affinity of the LHS of the IN */ @@ -69921,6 +70301,7 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( sqlite3SelectDestInit(&dest, SRT_Set, pExpr->iTable); dest.affinity = (u8)affinity; assert( (pExpr->iTable&0x0000FFFF)==pExpr->iTable ); + pExpr->x.pSelect->iLimit = 0; if( sqlite3Select(pParse, pExpr->x.pSelect, &dest) ){ return 0; } @@ -70021,6 +70402,7 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( sqlite3ExprDelete(pParse->db, pSel->pLimit); pSel->pLimit = sqlite3PExpr(pParse, TK_INTEGER, 0, 0, &sqlite3IntTokens[1]); + pSel->iLimit = 0; if( sqlite3Select(pParse, pSel, &dest) ){ return 0; } @@ -71304,9 +71686,22 @@ static int evalConstExpr(Walker *pWalker, Expr *pExpr){ ** Preevaluate constant subexpressions within pExpr and store the ** results in registers. Modify pExpr so that the constant subexpresions ** are TK_REGISTER opcodes that refer to the precomputed values. +** +** This routine is a no-op if the jump to the cookie-check code has +** already occur. Since the cookie-check jump is generated prior to +** any other serious processing, this check ensures that there is no +** way to accidently bypass the constant initializations. +** +** This routine is also a no-op if the SQLITE_FactorOutConst optimization +** is disabled via the sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS) +** interface. This allows test logic to verify that the same answer is +** obtained for queries regardless of whether or not constants are +** precomputed into registers or if they are inserted in-line. */ SQLITE_PRIVATE void sqlite3ExprCodeConstants(Parse *pParse, Expr *pExpr){ Walker w; + if( pParse->cookieGoto ) return; + if( (pParse->db->flags & SQLITE_FactorOutConst)!=0 ) return; w.xExprCallback = evalConstExpr; w.xSelectCallback = 0; w.pParse = pParse; @@ -84062,6 +84457,27 @@ struct sqlite3_api_routines { sqlite3_stmt *(*next_stmt)(sqlite3*,sqlite3_stmt*); const char *(*sql)(sqlite3_stmt*); int (*status)(int,int*,int*,int); + int (*backup_finish)(sqlite3_backup*); + sqlite3_backup *(*backup_init)(sqlite3*,const char*,sqlite3*,const char*); + int (*backup_pagecount)(sqlite3_backup*); + int (*backup_remaining)(sqlite3_backup*); + int (*backup_step)(sqlite3_backup*,int); + const char *(*compileoption_get)(int); + int (*compileoption_used)(const char*); + int (*create_function_v2)(sqlite3*,const char*,int,int,void*,void (*xFunc)(sqlite3_context*,int,sqlite3_value**),void (*xStep)(sqlite3_context*,int,sqlite3_value**),void (*xFinal)(sqlite3_context*),void(*xDestroy)(void*)); + int (*db_config)(sqlite3*,int,...); + sqlite3_mutex *(*db_mutex)(sqlite3*); + int (*db_status)(sqlite3*,int,int*,int*,int); + int (*extended_errcode)(sqlite3*); + void (*log)(int,const char*,...); + sqlite3_int64 (*soft_heap_limit64)(sqlite3_int64); + const char *(*sourceid)(void); + int (*stmt_status)(sqlite3_stmt*,int,int); + int (*strnicmp)(const char*,const char*,int); + int (*unlock_notify)(sqlite3*,void(*)(void**,int),void*); + int (*wal_autocheckpoint)(sqlite3*,int); + int (*wal_checkpoint)(sqlite3*,const char*); + void *(*wal_hook)(sqlite3*,int(*)(void*,sqlite3*,const char*,int),void*); }; /* @@ -84241,6 +84657,27 @@ struct sqlite3_api_routines { #define sqlite3_next_stmt sqlite3_api->next_stmt #define sqlite3_sql sqlite3_api->sql #define sqlite3_status sqlite3_api->status +#define sqlite3_backup_finish sqlite3_api->backup_finish +#define sqlite3_backup_init sqlite3_api->backup_init +#define sqlite3_backup_pagecount sqlite3_api->backup_pagecount +#define sqlite3_backup_remaining sqlite3_api->backup_remaining +#define sqlite3_backup_step sqlite3_api->backup_step +#define sqlite3_compileoption_get sqlite3_api->compileoption_get +#define sqlite3_compileoption_used sqlite3_api->compileoption_used +#define sqlite3_create_function_v2 sqlite3_api->create_function_v2 +#define sqlite3_db_config sqlite3_api->db_config +#define sqlite3_db_mutex sqlite3_api->db_mutex +#define sqlite3_db_status sqlite3_api->db_status +#define sqlite3_extended_errcode sqlite3_api->extended_errcode +#define sqlite3_log sqlite3_api->log +#define sqlite3_soft_heap_limit64 sqlite3_api->soft_heap_limit64 +#define sqlite3_sourceid sqlite3_api->sourceid +#define sqlite3_stmt_status sqlite3_api->stmt_status +#define sqlite3_strnicmp sqlite3_api->strnicmp +#define sqlite3_unlock_notify sqlite3_api->unlock_notify +#define sqlite3_wal_autocheckpoint sqlite3_api->wal_autocheckpoint +#define sqlite3_wal_checkpoint sqlite3_api->wal_checkpoint +#define sqlite3_wal_hook sqlite3_api->wal_hook #endif /* SQLITE_CORE */ #define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api = 0; @@ -84558,6 +84995,46 @@ static const sqlite3_api_routines sqlite3Apis = { sqlite3_next_stmt, sqlite3_sql, sqlite3_status, + + /* + ** Added for 3.7.4 + */ + sqlite3_backup_finish, + sqlite3_backup_init, + sqlite3_backup_pagecount, + sqlite3_backup_remaining, + sqlite3_backup_step, +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS + sqlite3_compileoption_get, + sqlite3_compileoption_used, +#else + 0, + 0, +#endif + sqlite3_create_function_v2, + sqlite3_db_config, + sqlite3_db_mutex, + sqlite3_db_status, + sqlite3_extended_errcode, + sqlite3_log, + sqlite3_soft_heap_limit64, + sqlite3_sourceid, + sqlite3_stmt_status, + sqlite3_strnicmp, +#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY + sqlite3_unlock_notify, +#else + 0, +#endif +#ifndef SQLITE_OMIT_WAL + sqlite3_wal_autocheckpoint, + sqlite3_wal_checkpoint, + sqlite3_wal_hook, +#else + 0, + 0, + 0, +#endif }; /* @@ -84873,7 +85350,7 @@ static u8 getSafetyLevel(const char *z){ static const u8 iValue[] = {1, 0, 0, 0, 1, 1, 2}; int i, n; if( sqlite3Isdigit(*z) ){ - return (u8)atoi(z); + return (u8)sqlite3Atoi(z); } n = sqlite3Strlen30(z); for(i=0; i=0&&i<=2)?i:0); } #endif /* ifndef SQLITE_OMIT_AUTOVACUUM */ @@ -85010,6 +85487,7 @@ static int flagPragma(Parse *pParse, const char *zLeft, const char *zRight){ { "empty_result_callbacks", SQLITE_NullCallback }, { "legacy_file_format", SQLITE_LegacyFileFmt }, { "fullfsync", SQLITE_FullFSync }, + { "checkpoint_fullfsync", SQLITE_CkptFullFSync }, { "reverse_unordered_selects", SQLITE_ReverseOrder }, #ifndef SQLITE_OMIT_AUTOMATIC_INDEX { "automatic_index", SQLITE_AutoIndex }, @@ -85221,7 +85699,7 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeChangeP1(v, addr+1, iDb); sqlite3VdbeChangeP1(v, addr+6, SQLITE_DEFAULT_CACHE_SIZE); }else{ - int size = atoi(zRight); + int size = sqlite3Atoi(zRight); if( size<0 ) size = -size; sqlite3BeginWriteOperation(pParse, 0, iDb); sqlite3VdbeAddOp2(v, OP_Integer, size, 1); @@ -85250,35 +85728,13 @@ SQLITE_PRIVATE void sqlite3Pragma( /* Malloc may fail when setting the page-size, as there is an internal ** buffer that the pager module resizes using sqlite3_realloc(). */ - db->nextPagesize = atoi(zRight); + db->nextPagesize = sqlite3Atoi(zRight); if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize, -1, 0) ){ db->mallocFailed = 1; } } }else - /* - ** PRAGMA [database.]max_page_count - ** PRAGMA [database.]max_page_count=N - ** - ** The first form reports the current setting for the - ** maximum number of pages in the database file. The - ** second form attempts to change this setting. Both - ** forms return the current setting. - */ - if( sqlite3StrICmp(zLeft,"max_page_count")==0 ){ - Btree *pBt = pDb->pBt; - int newMax = 0; - assert( pBt!=0 ); - if( zRight ){ - newMax = atoi(zRight); - } - if( ALWAYS(pBt) ){ - newMax = sqlite3BtreeMaxPageCount(pBt, newMax); - } - returnSingleInt(pParse, "max_page_count", newMax); - }else - /* ** PRAGMA [database.]secure_delete ** PRAGMA [database.]secure_delete=ON/OFF @@ -85305,19 +85761,33 @@ SQLITE_PRIVATE void sqlite3Pragma( }else /* + ** PRAGMA [database.]max_page_count + ** PRAGMA [database.]max_page_count=N + ** + ** The first form reports the current setting for the + ** maximum number of pages in the database file. The + ** second form attempts to change this setting. Both + ** forms return the current setting. + ** ** PRAGMA [database.]page_count ** ** Return the number of pages in the specified database. */ - if( sqlite3StrICmp(zLeft,"page_count")==0 ){ + if( sqlite3StrICmp(zLeft,"page_count")==0 + || sqlite3StrICmp(zLeft,"max_page_count")==0 + ){ int iReg; if( sqlite3ReadSchema(pParse) ) goto pragma_out; sqlite3CodeVerifySchema(pParse, iDb); iReg = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Pagecount, iDb, iReg); + if( zLeft[0]=='p' ){ + sqlite3VdbeAddOp2(v, OP_Pagecount, iDb, iReg); + }else{ + sqlite3VdbeAddOp3(v, OP_MaxPgcnt, iDb, iReg, sqlite3Atoi(zRight)); + } sqlite3VdbeAddOp2(v, OP_ResultRow, iReg, 1); sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "page_count", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLeft, SQLITE_TRANSIENT); }else /* @@ -85539,7 +86009,7 @@ SQLITE_PRIVATE void sqlite3Pragma( if( !zRight ){ returnSingleInt(pParse, "cache_size", pDb->pSchema->cache_size); }else{ - int size = atoi(zRight); + int size = sqlite3Atoi(zRight); if( size<0 ) size = -size; pDb->pSchema->cache_size = size; sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size); @@ -85932,7 +86402,7 @@ SQLITE_PRIVATE void sqlite3Pragma( /* Set the maximum error count */ mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; if( zRight ){ - mxErr = atoi(zRight); + sqlite3GetInt32(zRight, &mxErr); if( mxErr<=0 ){ mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; } @@ -86189,7 +86659,7 @@ SQLITE_PRIVATE void sqlite3Pragma( }; int addr = sqlite3VdbeAddOpList(v, ArraySize(setCookie), setCookie); sqlite3VdbeChangeP1(v, addr, iDb); - sqlite3VdbeChangeP1(v, addr+1, atoi(zRight)); + sqlite3VdbeChangeP1(v, addr+1, sqlite3Atoi(zRight)); sqlite3VdbeChangeP1(v, addr+2, iDb); sqlite3VdbeChangeP2(v, addr+2, iCookie); }else{ @@ -86250,8 +86720,7 @@ SQLITE_PRIVATE void sqlite3Pragma( */ if( sqlite3StrICmp(zLeft, "wal_autocheckpoint")==0 ){ if( zRight ){ - int nAuto = atoi(zRight); - sqlite3_wal_autocheckpoint(db, nAuto); + sqlite3_wal_autocheckpoint(db, sqlite3Atoi(zRight)); } returnSingleInt(pParse, "wal_autocheckpoint", db->xWalCallback==sqlite3WalDefaultHook ? @@ -86341,7 +86810,8 @@ SQLITE_PRIVATE void sqlite3Pragma( #ifndef SQLITE_OMIT_PAGER_PRAGMAS if( db->autoCommit ){ sqlite3BtreeSetSafetyLevel(pDb->pBt, pDb->safety_level, - (db->flags&SQLITE_FullFSync)!=0); + (db->flags&SQLITE_FullFSync)!=0, + (db->flags&SQLITE_CkptFullFSync)!=0); } #endif pragma_out: @@ -86433,7 +86903,7 @@ SQLITE_PRIVATE int sqlite3InitCallback(void *pInit, int argc, char **argv, char assert( db->init.busy ); db->init.iDb = iDb; - db->init.newTnum = atoi(argv[1]); + db->init.newTnum = sqlite3Atoi(argv[1]); db->init.orphanTrigger = 0; TESTONLY(rcp = ) sqlite3_prepare(db, argv[2], -1, &pStmt, 0); rc = db->errCode; @@ -86982,13 +87452,13 @@ static int sqlite3Prepare( if( rc==SQLITE_OK && pParse->pVdbe && pParse->explain ){ static const char * const azColName[] = { "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment", - "order", "from", "detail" + "selectid", "order", "from", "detail" }; int iFirst, mx; if( pParse->explain==2 ){ - sqlite3VdbeSetNumCols(pParse->pVdbe, 3); + sqlite3VdbeSetNumCols(pParse->pVdbe, 4); iFirst = 8; - mx = 11; + mx = 12; }else{ sqlite3VdbeSetNumCols(pParse->pVdbe, 8); iFirst = 0; @@ -87987,6 +88457,88 @@ static KeyInfo *keyInfoFromExprList(Parse *pParse, ExprList *pList){ return pInfo; } +#ifndef SQLITE_OMIT_COMPOUND_SELECT +/* +** Name of the connection operator, used for error messages. +*/ +static const char *selectOpName(int id){ + char *z; + switch( id ){ + case TK_ALL: z = "UNION ALL"; break; + case TK_INTERSECT: z = "INTERSECT"; break; + case TK_EXCEPT: z = "EXCEPT"; break; + default: z = "UNION"; break; + } + return z; +} +#endif /* SQLITE_OMIT_COMPOUND_SELECT */ + +#ifndef SQLITE_OMIT_EXPLAIN +/* +** Unless an "EXPLAIN QUERY PLAN" command is being processed, this function +** is a no-op. Otherwise, it adds a single row of output to the EQP result, +** where the caption is of the form: +** +** "USE TEMP B-TREE FOR xxx" +** +** where xxx is one of "DISTINCT", "ORDER BY" or "GROUP BY". Exactly which +** is determined by the zUsage argument. +*/ +static void explainTempTable(Parse *pParse, const char *zUsage){ + if( pParse->explain==2 ){ + Vdbe *v = pParse->pVdbe; + char *zMsg = sqlite3MPrintf(pParse->db, "USE TEMP B-TREE FOR %s", zUsage); + sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC); + } +} + +/* +** Unless an "EXPLAIN QUERY PLAN" command is being processed, this function +** is a no-op. Otherwise, it adds a single row of output to the EQP result, +** where the caption is of one of the two forms: +** +** "COMPOSITE SUBQUERIES iSub1 and iSub2 (op)" +** "COMPOSITE SUBQUERIES iSub1 and iSub2 USING TEMP B-TREE (op)" +** +** where iSub1 and iSub2 are the integers passed as the corresponding +** function parameters, and op is the text representation of the parameter +** of the same name. The parameter "op" must be one of TK_UNION, TK_EXCEPT, +** TK_INTERSECT or TK_ALL. The first form is used if argument bUseTmp is +** false, or the second form if it is true. +*/ +static void explainComposite( + Parse *pParse, /* Parse context */ + int op, /* One of TK_UNION, TK_EXCEPT etc. */ + int iSub1, /* Subquery id 1 */ + int iSub2, /* Subquery id 2 */ + int bUseTmp /* True if a temp table was used */ +){ + assert( op==TK_UNION || op==TK_EXCEPT || op==TK_INTERSECT || op==TK_ALL ); + if( pParse->explain==2 ){ + Vdbe *v = pParse->pVdbe; + char *zMsg = sqlite3MPrintf( + pParse->db, "COMPOUND SUBQUERIES %d AND %d %s(%s)", iSub1, iSub2, + bUseTmp?"USING TEMP B-TREE ":"", selectOpName(op) + ); + sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC); + } +} + +/* +** Assign expression b to lvalue a. A second, no-op, version of this macro +** is provided when SQLITE_OMIT_EXPLAIN is defined. This allows the code +** in sqlite3Select() to assign values to structure member variables that +** only exist if SQLITE_OMIT_EXPLAIN is not defined without polluting the +** code with #ifndef directives. +*/ +# define explainSetInteger(a, b) a = b + +#else +/* No-op versions of the explainXXX() functions and macros. */ +# define explainTempTable(y,z) +# define explainComposite(v,w,x,y,z) +# define explainSetInteger(y,z) +#endif /* ** If the inner loop was generated using a non-null pOrderBy argument, @@ -88334,22 +88886,6 @@ static void generateColumnNames( generateColumnTypes(pParse, pTabList, pEList); } -#ifndef SQLITE_OMIT_COMPOUND_SELECT -/* -** Name of the connection operator, used for error messages. -*/ -static const char *selectOpName(int id){ - char *z; - switch( id ){ - case TK_ALL: z = "UNION ALL"; break; - case TK_INTERSECT: z = "INTERSECT"; break; - case TK_EXCEPT: z = "EXCEPT"; break; - default: z = "UNION"; break; - } - return z; -} -#endif /* SQLITE_OMIT_COMPOUND_SELECT */ - /* ** Given a an expression list (which is really the list of expressions ** that form the result set of a SELECT statement) compute appropriate @@ -88512,6 +89048,7 @@ SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ assert( db->lookaside.bEnabled==0 ); pTab->nRef = 1; pTab->zName = 0; + pTab->nRowEst = 1000000; selectColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); selectAddColumnTypeAndCollation(pParse, pTab->nCol, pTab->aCol, pSelect); pTab->iPKey = -1; @@ -88582,6 +89119,8 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ VdbeComment((v, "LIMIT counter")); if( n==0 ){ sqlite3VdbeAddOp2(v, OP_Goto, 0, iBreak); + }else{ + if( p->nSelectRow > (double)n ) p->nSelectRow = (double)n; } }else{ sqlite3ExprCode(pParse, p->pLimit, iLimit); @@ -88682,6 +89221,10 @@ static int multiSelect( SelectDest dest; /* Alternative data destination */ Select *pDelete = 0; /* Chain of simple selects to delete */ sqlite3 *db; /* Database connection */ +#ifndef SQLITE_OMIT_EXPLAIN + int iSub1; /* EQP id of left-hand query */ + int iSub2; /* EQP id of right-hand query */ +#endif /* Make sure there is no ORDER BY or LIMIT clause on prior SELECTs. Only ** the last (right-most) SELECT in the series may have an ORDER BY or LIMIT. @@ -88739,9 +89282,11 @@ static int multiSelect( switch( p->op ){ case TK_ALL: { int addr = 0; + int nLimit; assert( !pPrior->pLimit ); pPrior->pLimit = p->pLimit; pPrior->pOffset = p->pOffset; + explainSetInteger(iSub1, pParse->iNextSelectId); rc = sqlite3Select(pParse, pPrior, &dest); p->pLimit = 0; p->pOffset = 0; @@ -88755,10 +89300,18 @@ static int multiSelect( addr = sqlite3VdbeAddOp1(v, OP_IfZero, p->iLimit); VdbeComment((v, "Jump ahead if LIMIT reached")); } + explainSetInteger(iSub2, pParse->iNextSelectId); rc = sqlite3Select(pParse, p, &dest); testcase( rc!=SQLITE_OK ); pDelete = p->pPrior; p->pPrior = pPrior; + p->nSelectRow += pPrior->nSelectRow; + if( pPrior->pLimit + && sqlite3ExprIsInteger(pPrior->pLimit, &nLimit) + && p->nSelectRow > (double)nLimit + ){ + p->nSelectRow = (double)nLimit; + } if( addr ){ sqlite3VdbeJumpHere(v, addr); } @@ -88802,6 +89355,7 @@ static int multiSelect( */ assert( !pPrior->pOrderBy ); sqlite3SelectDestInit(&uniondest, priorOp, unionTab); + explainSetInteger(iSub1, pParse->iNextSelectId); rc = sqlite3Select(pParse, pPrior, &uniondest); if( rc ){ goto multi_select_end; @@ -88821,6 +89375,7 @@ static int multiSelect( pOffset = p->pOffset; p->pOffset = 0; uniondest.eDest = op; + explainSetInteger(iSub2, pParse->iNextSelectId); rc = sqlite3Select(pParse, p, &uniondest); testcase( rc!=SQLITE_OK ); /* Query flattening in sqlite3Select() might refill p->pOrderBy. @@ -88829,6 +89384,7 @@ static int multiSelect( pDelete = p->pPrior; p->pPrior = pPrior; p->pOrderBy = 0; + if( p->op==TK_UNION ) p->nSelectRow += pPrior->nSelectRow; sqlite3ExprDelete(db, p->pLimit); p->pLimit = pLimit; p->pOffset = pOffset; @@ -88886,6 +89442,7 @@ static int multiSelect( /* Code the SELECTs to our left into temporary table "tab1". */ sqlite3SelectDestInit(&intersectdest, SRT_Union, tab1); + explainSetInteger(iSub1, pParse->iNextSelectId); rc = sqlite3Select(pParse, pPrior, &intersectdest); if( rc ){ goto multi_select_end; @@ -88902,10 +89459,12 @@ static int multiSelect( pOffset = p->pOffset; p->pOffset = 0; intersectdest.iParm = tab2; + explainSetInteger(iSub2, pParse->iNextSelectId); rc = sqlite3Select(pParse, p, &intersectdest); testcase( rc!=SQLITE_OK ); pDelete = p->pPrior; p->pPrior = pPrior; + if( p->nSelectRow>pPrior->nSelectRow ) p->nSelectRow = pPrior->nSelectRow; sqlite3ExprDelete(db, p->pLimit); p->pLimit = pLimit; p->pOffset = pOffset; @@ -88938,6 +89497,8 @@ static int multiSelect( } } + explainComposite(pParse, p->op, iSub1, iSub2, p->op!=TK_ALL); + /* Compute collating sequences used by ** temporary tables needed to implement the compound select. ** Attach the KeyInfo structure to all temporary tables. @@ -89281,6 +89842,10 @@ static int multiSelectOrderBy( ExprList *pOrderBy; /* The ORDER BY clause */ int nOrderBy; /* Number of terms in the ORDER BY clause */ int *aPermute; /* Mapping from ORDER BY terms to result set columns */ +#ifndef SQLITE_OMIT_EXPLAIN + int iSub1; /* EQP id of left-hand query */ + int iSub2; /* EQP id of right-hand query */ +#endif assert( p->pOrderBy!=0 ); assert( pKeyDup==0 ); /* "Managed" code needs this. Ticket #3382. */ @@ -89434,6 +89999,7 @@ static int multiSelectOrderBy( */ VdbeNoopComment((v, "Begin coroutine for left SELECT")); pPrior->iLimit = regLimitA; + explainSetInteger(iSub1, pParse->iNextSelectId); sqlite3Select(pParse, pPrior, &destA); sqlite3VdbeAddOp2(v, OP_Integer, 1, regEofA); sqlite3VdbeAddOp1(v, OP_Yield, regAddrA); @@ -89448,6 +90014,7 @@ static int multiSelectOrderBy( savedOffset = p->iOffset; p->iLimit = regLimitB; p->iOffset = 0; + explainSetInteger(iSub2, pParse->iNextSelectId); sqlite3Select(pParse, p, &destB); p->iLimit = savedLimit; p->iOffset = savedOffset; @@ -89484,6 +90051,7 @@ static int multiSelectOrderBy( sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); sqlite3VdbeAddOp1(v, OP_Yield, regAddrB); sqlite3VdbeAddOp2(v, OP_Goto, 0, addrEofA); + p->nSelectRow += pPrior->nSelectRow; } /* Generate a subroutine to run when the results from select B @@ -89491,6 +90059,7 @@ static int multiSelectOrderBy( */ if( op==TK_INTERSECT ){ addrEofB = addrEofA; + if( p->nSelectRow > pPrior->nSelectRow ) p->nSelectRow = pPrior->nSelectRow; }else{ VdbeNoopComment((v, "eof-B subroutine")); addrEofB = sqlite3VdbeAddOp2(v, OP_If, regEofA, labelEnd); @@ -89578,6 +90147,7 @@ static int multiSelectOrderBy( /*** TBD: Insert subroutine calls to close cursors on incomplete **** subqueries ****/ + explainComposite(pParse, p->op, iSub1, iSub2, 0); return SQLITE_OK; } #endif @@ -90311,6 +90881,7 @@ static int selectExpander(Walker *pWalker, Select *p){ while( pSel->pPrior ){ pSel = pSel->pPrior; } selectColumnsFromExprList(pParse, pSel->pEList, &pTab->nCol, &pTab->aCol); pTab->iPKey = -1; + pTab->nRowEst = 1000000; pTab->tabFlags |= TF_Ephemeral; #endif }else{ @@ -90804,6 +91375,11 @@ SQLITE_PRIVATE int sqlite3Select( int iEnd; /* Address of the end of the query */ sqlite3 *db; /* The database connection */ +#ifndef SQLITE_OMIT_EXPLAIN + int iRestoreSelectId = pParse->iSelectId; + pParse->iSelectId = pParse->iNextSelectId++; +#endif + db = pParse->db; if( p==0 || db->mallocFailed || pParse->nErr ){ return 1; @@ -90875,8 +91451,10 @@ SQLITE_PRIVATE int sqlite3Select( }else{ sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); assert( pItem->isPopulated==0 ); + explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); sqlite3Select(pParse, pSub, &dest); pItem->isPopulated = 1; + pItem->pTab->nRowEst = (unsigned)pSub->nSelectRow; } if( /*pParse->nErr ||*/ db->mallocFailed ){ goto select_end; @@ -90910,10 +91488,12 @@ SQLITE_PRIVATE int sqlite3Select( mxSelect = db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT]; if( mxSelect && cnt>mxSelect ){ sqlite3ErrorMsg(pParse, "too many terms in compound SELECT"); - return 1; + goto select_end; } } - return multiSelect(pParse, p, pDest); + rc = multiSelect(pParse, p, pDest); + explainSetInteger(pParse->iSelectId, iRestoreSelectId); + return rc; } #endif @@ -90925,7 +91505,6 @@ SQLITE_PRIVATE int sqlite3Select( p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0); pGroupBy = p->pGroupBy; p->selFlags &= ~SF_Distinct; - isDistinct = 0; } /* If there is both a GROUP BY and an ORDER BY clause and they are @@ -90968,11 +91547,12 @@ SQLITE_PRIVATE int sqlite3Select( /* Set the limiter. */ iEnd = sqlite3VdbeMakeLabel(v); + p->nSelectRow = (double)LARGEST_INT64; computeLimitRegisters(pParse, p, iEnd); /* Open a virtual index to use for the distinct set. */ - if( isDistinct ){ + if( p->selFlags & SF_Distinct ){ KeyInfo *pKeyInfo; assert( isAgg || pGroupBy ); distinct = pParse->nTab++; @@ -90991,6 +91571,7 @@ SQLITE_PRIVATE int sqlite3Select( */ pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pOrderBy, 0); if( pWInfo==0 ) goto select_end; + if( pWInfo->nRowOut < p->nSelectRow ) p->nSelectRow = pWInfo->nRowOut; /* If sorting index that was created by a prior OP_OpenEphemeral ** instruction ended up not being needed, then change the OP_OpenEphemeral @@ -91035,6 +91616,9 @@ SQLITE_PRIVATE int sqlite3Select( for(k=pGroupBy->nExpr, pItem=pGroupBy->a; k>0; k--, pItem++){ pItem->iAlias = 0; } + if( p->nSelectRow>(double)100 ) p->nSelectRow = (double)100; + }else{ + p->nSelectRow = (double)1; } @@ -91131,6 +91715,9 @@ SQLITE_PRIVATE int sqlite3Select( int nCol; int nGroupBy; + explainTempTable(pParse, + isDistinct && !(p->selFlags&SF_Distinct)?"DISTINCT":"GROUP BY"); + groupBySort = 1; nGroupBy = pGroupBy->nExpr; nCol = nGroupBy + 1; @@ -91392,10 +91979,15 @@ SQLITE_PRIVATE int sqlite3Select( } /* endif aggregate query */ + if( distinct>=0 ){ + explainTempTable(pParse, "DISTINCT"); + } + /* If there is an ORDER BY clause, then we need to sort the results ** and send them to the callback one by one. */ if( pOrderBy ){ + explainTempTable(pParse, "ORDER BY"); generateSortTail(pParse, p, v, pEList->nExpr, pDest); } @@ -91412,6 +92004,7 @@ SQLITE_PRIVATE int sqlite3Select( ** successful coding of the SELECT. */ select_end: + explainSetInteger(pParse->iSelectId, iRestoreSelectId); /* Identify column names if results of the SELECT are to be output. */ @@ -94501,7 +95094,7 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ } db->pVTab = 0; }else{ - sqlite3Error(db, SQLITE_ERROR, zErr); + sqlite3Error(db, SQLITE_ERROR, (zErr ? "%s" : 0), zErr); sqlite3DbFree(db, zErr); rc = SQLITE_ERROR; } @@ -94963,7 +95556,6 @@ struct WhereMaskSet { struct WhereCost { WherePlan plan; /* The lookup strategy */ double rCost; /* Overall cost of pursuing this search strategy */ - double nRow; /* Estimated number of output rows */ Bitmask used; /* Bitmask of cursors used by this plan */ }; @@ -95006,10 +95598,11 @@ struct WhereCost { #define WHERE_COLUMN_IN 0x00040000 /* x IN (...) */ #define WHERE_COLUMN_NULL 0x00080000 /* x IS NULL */ #define WHERE_INDEXED 0x000f0000 /* Anything that uses an index */ -#define WHERE_NOT_FULLSCAN 0x000f3000 /* Does not do a full table scan */ +#define WHERE_NOT_FULLSCAN 0x100f3000 /* Does not do a full table scan */ #define WHERE_IN_ABLE 0x000f1000 /* Able to support an IN operator */ #define WHERE_TOP_LIMIT 0x00100000 /* xEXPR or x>=EXPR constraint */ +#define WHERE_BOTH_LIMIT 0x00300000 /* Both x>EXPR and xa[pWC->nTerm]; /* End of pWC->a[] */ WhereTerm *pTerm; /* A single term of the WHERE clause */ - /* No OR-clause optimization allowed if the NOT INDEXED clause is used */ - if( pSrc->notIndexed ){ + /* No OR-clause optimization allowed if the INDEXED BY or NOT INDEXED clauses + ** are used */ + if( pSrc->notIndexed || pSrc->pIndex!=0 ){ return; } @@ -96391,7 +96985,7 @@ static void bestOrClauseIndex( continue; } rTotal += sTermCost.rCost; - nRow += sTermCost.nRow; + nRow += sTermCost.plan.nRow; used |= sTermCost.used; if( rTotal>=pCost->rCost ) break; } @@ -96410,8 +97004,8 @@ static void bestOrClauseIndex( WHERETRACE(("... multi-index OR cost=%.9g nrow=%.9g\n", rTotal, nRow)); if( rTotalrCost ){ pCost->rCost = rTotal; - pCost->nRow = nRow; pCost->used = used; + pCost->plan.nRow = nRow; pCost->plan.wsFlags = flags; pCost->plan.u.pTerm = pTerm; } @@ -96495,7 +97089,7 @@ static void bestAutomaticIndex( WHERETRACE(("auto-index reduces cost from %.2f to %.2f\n", pCost->rCost, costTempIdx)); pCost->rCost = costTempIdx; - pCost->nRow = logN + 1; + pCost->plan.nRow = logN + 1; pCost->plan.wsFlags = WHERE_TEMP_INDEX; pCost->used = pTerm->prereqRight; break; @@ -97568,11 +98162,11 @@ static void bestBtreeIndex( ** index and its cost in the pCost structure. */ if( (!pIdx || wsFlags) - && (costrCost || (cost<=pCost->rCost && nRownRow)) + && (costrCost || (cost<=pCost->rCost && nRowplan.nRow)) ){ pCost->rCost = cost; - pCost->nRow = nRow; pCost->used = used; + pCost->plan.nRow = nRow; pCost->plan.wsFlags = (wsFlags&wsFlagMask); pCost->plan.nEq = nEq; pCost->plan.u.pIdx = pIdx; @@ -97901,6 +98495,161 @@ static int codeAllEqualityTerms( return regBase; } +#ifndef SQLITE_OMIT_EXPLAIN +/* +** This routine is a helper for explainIndexRange() below +** +** pStr holds the text of an expression that we are building up one term +** at a time. This routine adds a new term to the end of the expression. +** Terms are separated by AND so add the "AND" text for second and subsequent +** terms only. +*/ +static void explainAppendTerm( + StrAccum *pStr, /* The text expression being built */ + int iTerm, /* Index of this term. First is zero */ + const char *zColumn, /* Name of the column */ + const char *zOp /* Name of the operator */ +){ + if( iTerm ) sqlite3StrAccumAppend(pStr, " AND ", 5); + sqlite3StrAccumAppend(pStr, zColumn, -1); + sqlite3StrAccumAppend(pStr, zOp, 1); + sqlite3StrAccumAppend(pStr, "?", 1); +} + +/* +** Argument pLevel describes a strategy for scanning table pTab. This +** function returns a pointer to a string buffer containing a description +** of the subset of table rows scanned by the strategy in the form of an +** SQL expression. Or, if all rows are scanned, NULL is returned. +** +** For example, if the query: +** +** SELECT * FROM t1 WHERE a=1 AND b>2; +** +** is run and there is an index on (a, b), then this function returns a +** string similar to: +** +** "a=? AND b>?" +** +** The returned pointer points to memory obtained from sqlite3DbMalloc(). +** It is the responsibility of the caller to free the buffer when it is +** no longer required. +*/ +static char *explainIndexRange(sqlite3 *db, WhereLevel *pLevel, Table *pTab){ + WherePlan *pPlan = &pLevel->plan; + Index *pIndex = pPlan->u.pIdx; + int nEq = pPlan->nEq; + int i, j; + Column *aCol = pTab->aCol; + int *aiColumn = pIndex->aiColumn; + StrAccum txt; + + if( nEq==0 && (pPlan->wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ){ + return 0; + } + sqlite3StrAccumInit(&txt, 0, 0, SQLITE_MAX_LENGTH); + txt.db = db; + sqlite3StrAccumAppend(&txt, " (", 2); + for(i=0; i"); + } + if( pPlan->wsFlags&WHERE_TOP_LIMIT ){ + explainAppendTerm(&txt, i, aCol[aiColumn[j]].zName, "<"); + } + sqlite3StrAccumAppend(&txt, ")", 1); + return sqlite3StrAccumFinish(&txt); +} + +/* +** This function is a no-op unless currently processing an EXPLAIN QUERY PLAN +** command. If the query being compiled is an EXPLAIN QUERY PLAN, a single +** record is added to the output to describe the table scan strategy in +** pLevel. +*/ +static void explainOneScan( + Parse *pParse, /* Parse context */ + SrcList *pTabList, /* Table list this loop refers to */ + WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */ + int iLevel, /* Value for "level" column of output */ + int iFrom, /* Value for "from" column of output */ + u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ +){ + if( pParse->explain==2 ){ + u32 flags = pLevel->plan.wsFlags; + struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom]; + Vdbe *v = pParse->pVdbe; /* VM being constructed */ + sqlite3 *db = pParse->db; /* Database handle */ + char *zMsg; /* Text to add to EQP output */ + sqlite3_int64 nRow; /* Expected number of rows visited by scan */ + int iId = pParse->iSelectId; /* Select id (left-most output column) */ + int isSearch; /* True for a SEARCH. False for SCAN. */ + + if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_ONETABLE_ONLY) ) return; + + isSearch = (pLevel->plan.nEq>0) + || (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 + || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX)); + + zMsg = sqlite3MPrintf(db, "%s", isSearch?"SEARCH":"SCAN"); + if( pItem->pSelect ){ + zMsg = sqlite3MAppendf(db, zMsg, "%s SUBQUERY %d", zMsg,pItem->iSelectId); + }else{ + zMsg = sqlite3MAppendf(db, zMsg, "%s TABLE %s", zMsg, pItem->zName); + } + + if( pItem->zAlias ){ + zMsg = sqlite3MAppendf(db, zMsg, "%s AS %s", zMsg, pItem->zAlias); + } + if( (flags & WHERE_INDEXED)!=0 ){ + char *zWhere = explainIndexRange(db, pLevel, pItem->pTab); + zMsg = sqlite3MAppendf(db, zMsg, "%s USING %s%sINDEX%s%s%s", zMsg, + ((flags & WHERE_TEMP_INDEX)?"AUTOMATIC ":""), + ((flags & WHERE_IDX_ONLY)?"COVERING ":""), + ((flags & WHERE_TEMP_INDEX)?"":" "), + ((flags & WHERE_TEMP_INDEX)?"": pLevel->plan.u.pIdx->zName), + zWhere + ); + sqlite3DbFree(db, zWhere); + }else if( flags & (WHERE_ROWID_EQ|WHERE_ROWID_RANGE) ){ + zMsg = sqlite3MAppendf(db, zMsg, "%s USING INTEGER PRIMARY KEY", zMsg); + + if( flags&WHERE_ROWID_EQ ){ + zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid=?)", zMsg); + }else if( (flags&WHERE_BOTH_LIMIT)==WHERE_BOTH_LIMIT ){ + zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid>? AND rowid?)", zMsg); + }else if( flags&WHERE_TOP_LIMIT ){ + zMsg = sqlite3MAppendf(db, zMsg, "%s (rowidplan.u.pVtabIdx; + zMsg = sqlite3MAppendf(db, zMsg, "%s VIRTUAL TABLE INDEX %d:%s", zMsg, + pVtabIdx->idxNum, pVtabIdx->idxStr); + } +#endif + if( wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX) ){ + testcase( wctrlFlags & WHERE_ORDERBY_MIN ); + nRow = 1; + }else{ + nRow = (sqlite3_int64)pLevel->plan.nRow; + } + zMsg = sqlite3MAppendf(db, zMsg, "%s (~%lld rows)", zMsg, nRow); + sqlite3VdbeAddOp4(v, OP_Explain, iId, iLevel, iFrom, zMsg, P4_DYNAMIC); + } +} +#else +# define explainOneScan(u,v,w,x,y,z) +#endif /* SQLITE_OMIT_EXPLAIN */ + + /* ** Generate code for the start of the iLevel-th loop in the WHERE clause ** implementation described by pWInfo. @@ -98308,7 +99057,7 @@ static Bitmask codeOneLoopStart( r1 = sqlite3GetTempReg(pParse); testcase( pLevel->plan.wsFlags & WHERE_BTM_LIMIT ); testcase( pLevel->plan.wsFlags & WHERE_TOP_LIMIT ); - if( pLevel->plan.wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT) ){ + if( (pLevel->plan.wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 ){ sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, nEq, r1); sqlite3VdbeAddOp2(v, OP_IsNull, r1, addrCont); } @@ -98442,6 +99191,9 @@ static Bitmask codeOneLoopStart( WHERE_OMIT_OPEN | WHERE_OMIT_CLOSE | WHERE_FORCE_TABLE | WHERE_ONETABLE_ONLY); if( pSubWInfo ){ + explainOneScan( + pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0 + ); if( (wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ int iSet = ((ii==pOrWc->nTerm-1)?-1:ii); int r; @@ -98837,6 +99589,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( memset(&bestPlan, 0, sizeof(bestPlan)); bestPlan.rCost = SQLITE_BIG_DBL; + WHERETRACE(("*** Begin search for loop %d ***\n", i)); /* Loop through the remaining entries in the FROM clause to find the ** next nested loop. The loop tests all FROM clause entries @@ -98901,6 +99654,8 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( pOrderBy = ((i==0 && ppOrderBy )?*ppOrderBy:0); if( pTabItem->pIndex==0 ) nUnconstrained++; + WHERETRACE(("=== trying table %d with isOptimal=%d ===\n", + j, isOptimal)); assert( pTabItem->pTab ); #ifndef SQLITE_OMIT_VIRTUALTABLE if( IsVirtual(pTabItem->pTab) ){ @@ -98951,10 +99706,12 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( && (nUnconstrained==0 || pTabItem->pIndex==0 /* (3) */ || NEVER((sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0)) && (bestJ<0 || sCost.rCost=0 ); assert( notReady & getMask(pMaskSet, pTabList->a[bestJ].iCursor) ); - WHERETRACE(("*** Optimizer selects table %d for loop %d\n", bestJ, - pLevel-pWInfo->a)); + WHERETRACE(("*** Optimizer selects table %d for loop %d" + " with cost=%g and nRow=%g\n", + bestJ, pLevel-pWInfo->a, bestPlan.rCost, bestPlan.plan.nRow)); if( (bestPlan.plan.wsFlags & WHERE_ORDERBY)!=0 ){ *ppOrderBy = 0; } @@ -98979,7 +99737,9 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( } notReady &= ~getMask(pMaskSet, pTabList->a[bestJ].iCursor); pLevel->iFrom = (u8)bestJ; - if( bestPlan.nRow>=(double)1 ) pParse->nQueryLoop *= bestPlan.nRow; + if( bestPlan.plan.nRow>=(double)1 ){ + pParse->nQueryLoop *= bestPlan.plan.nRow; + } /* Check that if the table scanned by this loop iteration had an ** INDEXED BY clause attached to it, that the named index is being @@ -99027,44 +99787,15 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( */ sqlite3CodeVerifySchema(pParse, -1); /* Insert the cookie verifier Goto */ notReady = ~(Bitmask)0; + pWInfo->nRowOut = (double)1; for(i=0, pLevel=pWInfo->a; iexplain==2 ){ - char *zMsg; - struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom]; - zMsg = sqlite3MPrintf(db, "TABLE %s", pItem->zName); - if( pItem->zAlias ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s AS %s", zMsg, pItem->zAlias); - } - if( (pLevel->plan.wsFlags & WHERE_TEMP_INDEX)!=0 ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s WITH AUTOMATIC INDEX", zMsg); - }else if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s WITH INDEX %s", - zMsg, pLevel->plan.u.pIdx->zName); - }else if( pLevel->plan.wsFlags & WHERE_MULTI_OR ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s VIA MULTI-INDEX UNION", zMsg); - }else if( pLevel->plan.wsFlags & (WHERE_ROWID_EQ|WHERE_ROWID_RANGE) ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s USING PRIMARY KEY", zMsg); - } -#ifndef SQLITE_OMIT_VIRTUALTABLE - else if( (pLevel->plan.wsFlags & WHERE_VIRTUALTABLE)!=0 ){ - sqlite3_index_info *pVtabIdx = pLevel->plan.u.pVtabIdx; - zMsg = sqlite3MAppendf(db, zMsg, "%s VIRTUAL TABLE INDEX %d:%s", zMsg, - pVtabIdx->idxNum, pVtabIdx->idxStr); - } -#endif - if( pLevel->plan.wsFlags & WHERE_ORDERBY ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s ORDER BY", zMsg); - } - sqlite3VdbeAddOp4(v, OP_Explain, i, pLevel->iFrom, 0, zMsg, P4_DYNAMIC); - } -#endif /* SQLITE_OMIT_EXPLAIN */ pTabItem = &pTabList->a[pLevel->iFrom]; pTab = pTabItem->pTab; pLevel->iTabCur = pTabItem->iCursor; + pWInfo->nRowOut *= pLevel->plan.nRow; iDb = sqlite3SchemaToIndex(db, pTab->pSchema); if( (pTab->tabFlags & TF_Ephemeral)!=0 || pTab->pSelect ){ /* Do nothing */ @@ -99120,8 +99851,10 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( */ notReady = ~(Bitmask)0; for(i=0; ia[i]; + explainOneScan(pParse, pTabList, pLevel, i, pLevel->iFrom, wctrlFlags); notReady = codeOneLoopStart(pWInfo, i, wctrlFlags, notReady); - pWInfo->iContinue = pWInfo->a[i].addrCont; + pWInfo->iContinue = pLevel->addrCont; } #ifdef SQLITE_TEST /* For testing and debugging use only */ @@ -105523,13 +106256,12 @@ static int createCollation( } pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 1); - if( pColl ){ - pColl->xCmp = xCompare; - pColl->pUser = pCtx; - pColl->xDel = xDel; - pColl->enc = (u8)(enc2 | (enc & SQLITE_UTF16_ALIGNED)); - pColl->type = collType; - } + if( pColl==0 ) return SQLITE_NOMEM; + pColl->xCmp = xCompare; + pColl->pUser = pCtx; + pColl->xDel = xDel; + pColl->enc = (u8)(enc2 | (enc & SQLITE_UTF16_ALIGNED)); + pColl->type = collType; sqlite3Error(db, SQLITE_OK, 0); return SQLITE_OK; } @@ -106278,7 +107010,10 @@ SQLITE_API int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, vo assert( pPager!=0 ); fd = sqlite3PagerFile(pPager); assert( fd!=0 ); - if( fd->pMethods ){ + if( op==SQLITE_FCNTL_FILE_POINTER ){ + *(sqlite3_file**)pArg = fd; + rc = SQLITE_OK; + }else if( fd->pMethods ){ rc = sqlite3OsFileControl(fd, op, pArg); } sqlite3BtreeLeave(pBtree); @@ -107497,8 +108232,14 @@ SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem(const Fts3Hash *, const voi ** Macros indicating that conditional expressions are always true or ** false. */ +#ifdef SQLITE_COVERAGE_TEST +# define ALWAYS(x) (1) +# define NEVER(X) (0) +#else # define ALWAYS(x) (x) # define NEVER(X) (x) +#endif + /* ** Internal types used by SQLite. */ @@ -107516,8 +108257,12 @@ typedef struct Fts3Table Fts3Table; typedef struct Fts3Cursor Fts3Cursor; typedef struct Fts3Expr Fts3Expr; typedef struct Fts3Phrase Fts3Phrase; -typedef struct Fts3SegReader Fts3SegReader; +typedef struct Fts3PhraseToken Fts3PhraseToken; + typedef struct Fts3SegFilter Fts3SegFilter; +typedef struct Fts3DeferredToken Fts3DeferredToken; +typedef struct Fts3SegReader Fts3SegReader; +typedef struct Fts3SegReaderArray Fts3SegReaderArray; /* ** A connection to a fulltext index is an instance of the following @@ -107538,22 +108283,14 @@ struct Fts3Table { /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ - sqlite3_stmt *aStmt[25]; - - /* Pointer to string containing the SQL: - ** - ** "SELECT block FROM %_segments WHERE blockid BETWEEN ? AND ? - ** ORDER BY blockid" - */ - char *zSelectLeaves; - int nLeavesStmt; /* Valid statements in aLeavesStmt */ - int nLeavesTotal; /* Total number of prepared leaves stmts */ - int nLeavesAlloc; /* Allocated size of aLeavesStmt */ - sqlite3_stmt **aLeavesStmt; /* Array of prepared zSelectLeaves stmts */ + sqlite3_stmt *aStmt[24]; int nNodeSize; /* Soft limit for node size */ - u8 bHasContent; /* True if %_content table exists */ + u8 bHasStat; /* True if %_stat table exists */ u8 bHasDocsize; /* True if %_docsize table exists */ + int nPgsz; /* Page size for host database */ + char *zSegmentsTbl; /* Name of %_segments table */ + sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ /* The following hash table is used to buffer pending index updates during ** transactions. Variable nPendingData estimates the memory size of the @@ -107580,14 +108317,25 @@ struct Fts3Cursor { u8 isRequireSeek; /* True if must seek pStmt to %_content row */ sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ Fts3Expr *pExpr; /* Parsed MATCH query string */ + int nPhrase; /* Number of matchable phrases in query */ + Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */ sqlite3_int64 iPrevId; /* Previous id read from aDoclist */ char *pNextId; /* Pointer into the body of aDoclist */ char *aDoclist; /* List of docids for full-text queries */ int nDoclist; /* Size of buffer at aDoclist */ + int eEvalmode; /* An FTS3_EVAL_XX constant */ + int nRowAvg; /* Average size of database rows, in pages */ + int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ u32 *aMatchinfo; /* Information about most recent match */ + int nMatchinfo; /* Number of elements in aMatchinfo[] */ + char *zMatchinfo; /* Matchinfo specification */ }; +#define FTS3_EVAL_FILTER 0 +#define FTS3_EVAL_NEXT 1 +#define FTS3_EVAL_MATCHINFO 2 + /* ** The Fts3Cursor.eSearch member is always set to one of the following. ** Actualy, Fts3Cursor.eSearch can be greater than or equal to @@ -107610,18 +108358,30 @@ struct Fts3Cursor { /* ** A "phrase" is a sequence of one or more tokens that must match in ** sequence. A single token is the base case and the most common case. -** For a sequence of tokens contained in "...", nToken will be the number -** of tokens in the string. -*/ +** For a sequence of tokens contained in double-quotes (i.e. "one two three") +** nToken will be the number of tokens in the string. +** +** The nDocMatch and nMatch variables contain data that may be used by the +** matchinfo() function. They are populated when the full-text index is +** queried for hits on the phrase. If one or more tokens in the phrase +** are deferred, the nDocMatch and nMatch variables are populated based +** on the assumption that the +*/ +struct Fts3PhraseToken { + char *z; /* Text of the token */ + int n; /* Number of bytes in buffer z */ + int isPrefix; /* True if token ends with a "*" character */ + int bFulltext; /* True if full-text index was used */ + Fts3SegReaderArray *pArray; /* Segment-reader for this token */ + Fts3DeferredToken *pDeferred; /* Deferred token object for this token */ +}; + struct Fts3Phrase { + /* Variables populated by fts3_expr.c when parsing a MATCH expression */ int nToken; /* Number of tokens in the phrase */ int iColumn; /* Index of column this phrase must match */ int isNot; /* Phrase prefixed by unary not (-) operator */ - struct PhraseToken { - char *z; /* Text of the token */ - int n; /* Number of bytes in buffer pointed to by z */ - int isPrefix; /* True if token ends in with a "*" character */ - } aToken[1]; /* One entry for each token in the phrase */ + Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */ }; /* @@ -107671,29 +108431,34 @@ struct Fts3Expr { #define FTSQUERY_PHRASE 5 -/* fts3_init.c */ -SQLITE_PRIVATE int sqlite3Fts3DeleteVtab(int, sqlite3_vtab *); -SQLITE_PRIVATE int sqlite3Fts3InitVtab(int, sqlite3*, void*, int, const char*const*, - sqlite3_vtab **, char **); - /* fts3_write.c */ SQLITE_PRIVATE int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*); SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *); SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *); SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *); -SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(Fts3Table *,int, sqlite3_int64, +SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(int, sqlite3_int64, sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); SQLITE_PRIVATE int sqlite3Fts3SegReaderPending(Fts3Table*,const char*,int,int,Fts3SegReader**); -SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3Table *, Fts3SegReader *); +SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *); SQLITE_PRIVATE int sqlite3Fts3SegReaderIterate( Fts3Table *, Fts3SegReader **, int, Fts3SegFilter *, int (*)(Fts3Table *, void *, char *, int, char *, int), void * ); -SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char const**, int*); +SQLITE_PRIVATE int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *); SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table*, sqlite3_stmt **); -SQLITE_PRIVATE int sqlite3Fts3MatchinfoDocsizeLocal(Fts3Cursor*, u32*); -SQLITE_PRIVATE int sqlite3Fts3MatchinfoDocsizeGlobal(Fts3Cursor*, u32*); SQLITE_PRIVATE int sqlite3Fts3ReadLock(Fts3Table *); +SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*); + +SQLITE_PRIVATE int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **); +SQLITE_PRIVATE int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **); + +SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); +SQLITE_PRIVATE int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); +SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); +SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); +SQLITE_PRIVATE char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *); + +SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *); /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ #define FTS3_SEGMENT_REQUIRE_POS 0x00000001 @@ -107717,22 +108482,24 @@ SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64); SQLITE_PRIVATE void sqlite3Fts3Dequote(char *); SQLITE_PRIVATE char *sqlite3Fts3FindPositions(Fts3Expr *, sqlite3_int64, int); -SQLITE_PRIVATE int sqlite3Fts3ExprLoadDoclist(Fts3Table *, Fts3Expr *); +SQLITE_PRIVATE int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *); +SQLITE_PRIVATE int sqlite3Fts3ExprLoadFtDoclist(Fts3Cursor *, Fts3Expr *, char **, int *); SQLITE_PRIVATE int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int); /* fts3_tokenizer.c */ SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *, int *); SQLITE_PRIVATE int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *); -SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, - const char *, sqlite3_tokenizer **, const char **, char ** +SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *, + sqlite3_tokenizer **, char ** ); +SQLITE_PRIVATE int sqlite3Fts3IsIdChar(char); /* fts3_snippet.c */ SQLITE_PRIVATE void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*); SQLITE_PRIVATE void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *, const char *, const char *, int, int ); -SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *); +SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *); /* fts3_expr.c */ SQLITE_PRIVATE int sqlite3Fts3ExprParse(sqlite3_tokenizer *, @@ -107881,16 +108648,13 @@ static int fts3DisconnectMethod(sqlite3_vtab *pVtab){ int i; assert( p->nPendingData==0 ); + assert( p->pSegments==0 ); /* Free any prepared statements held */ for(i=0; iaStmt); i++){ sqlite3_finalize(p->aStmt[i]); } - for(i=0; inLeavesStmt; i++){ - sqlite3_finalize(p->aLeavesStmt[i]); - } - sqlite3_free(p->zSelectLeaves); - sqlite3_free(p->aLeavesStmt); + sqlite3_free(p->zSegmentsTbl); /* Invoke the tokenizer destructor to free the tokenizer. */ p->pTokenizer->pModule->xDestroy(p->pTokenizer); @@ -107901,7 +108665,7 @@ static int fts3DisconnectMethod(sqlite3_vtab *pVtab){ /* ** Construct one or more SQL statements from the format string given -** and then evaluate those statements. The success code is writting +** and then evaluate those statements. The success code is written ** into *pRc. ** ** If *pRc is initially non-zero then this routine is a no-op. @@ -107953,33 +108717,38 @@ static int fts3DestroyMethod(sqlite3_vtab *pVtab){ ** Invoke sqlite3_declare_vtab() to declare the schema for the FTS3 table ** passed as the first argument. This is done as part of the xConnect() ** and xCreate() methods. +** +** If *pRc is non-zero when this function is called, it is a no-op. +** Otherwise, if an error occurs, an SQLite error code is stored in *pRc +** before returning. */ -static int fts3DeclareVtab(Fts3Table *p){ - int i; /* Iterator variable */ - int rc; /* Return code */ - char *zSql; /* SQL statement passed to declare_vtab() */ - char *zCols; /* List of user defined columns */ +static void fts3DeclareVtab(int *pRc, Fts3Table *p){ + if( *pRc==SQLITE_OK ){ + int i; /* Iterator variable */ + int rc; /* Return code */ + char *zSql; /* SQL statement passed to declare_vtab() */ + char *zCols; /* List of user defined columns */ - /* Create a list of user columns for the virtual table */ - zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]); - for(i=1; zCols && inColumn; i++){ - zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]); - } + /* Create a list of user columns for the virtual table */ + zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]); + for(i=1; zCols && inColumn; i++){ + zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]); + } - /* Create the whole "CREATE TABLE" statement to pass to SQLite */ - zSql = sqlite3_mprintf( - "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN)", zCols, p->zName - ); + /* Create the whole "CREATE TABLE" statement to pass to SQLite */ + zSql = sqlite3_mprintf( + "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN)", zCols, p->zName + ); + if( !zCols || !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_declare_vtab(p->db, zSql); + } - if( !zCols || !zSql ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_declare_vtab(p->db, zSql); + sqlite3_free(zSql); + sqlite3_free(zCols); + *pRc = rc; } - - sqlite3_free(zSql); - sqlite3_free(zCols); - return rc; } /* @@ -107998,21 +108767,19 @@ static int fts3CreateTables(Fts3Table *p){ sqlite3 *db = p->db; /* The database connection */ /* Create a list of user columns for the content table */ - if( p->bHasContent ){ - zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY"); - for(i=0; zContentCols && inColumn; i++){ - char *z = p->azColumn[i]; - zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); - } - if( zContentCols==0 ) rc = SQLITE_NOMEM; - - /* Create the content table */ - fts3DbExec(&rc, db, - "CREATE TABLE %Q.'%q_content'(%s)", - p->zDb, p->zName, zContentCols - ); - sqlite3_free(zContentCols); + zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY"); + for(i=0; zContentCols && inColumn; i++){ + char *z = p->azColumn[i]; + zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); } + if( zContentCols==0 ) rc = SQLITE_NOMEM; + + /* Create the content table */ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_content'(%s)", + p->zDb, p->zName, zContentCols + ); + sqlite3_free(zContentCols); /* Create other tables */ fts3DbExec(&rc, db, "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);", @@ -108035,6 +108802,8 @@ static int fts3CreateTables(Fts3Table *p){ "CREATE TABLE %Q.'%q_docsize'(docid INTEGER PRIMARY KEY, size BLOB);", p->zDb, p->zName ); + } + if( p->bHasStat ){ fts3DbExec(&rc, db, "CREATE TABLE %Q.'%q_stat'(id INTEGER PRIMARY KEY, value BLOB);", p->zDb, p->zName @@ -108044,39 +108813,63 @@ static int fts3CreateTables(Fts3Table *p){ } /* -** An sqlite3_exec() callback for fts3TableExists. +** Store the current database page-size in bytes in p->nPgsz. +** +** If *pRc is non-zero when this function is called, it is a no-op. +** Otherwise, if an error occurs, an SQLite error code is stored in *pRc +** before returning. */ -static int fts3TableExistsCallback(void *pArg, int n, char **pp1, char **pp2){ - UNUSED_PARAMETER(n); - UNUSED_PARAMETER(pp1); - UNUSED_PARAMETER(pp2); - *(int*)pArg = 1; - return 1; +static void fts3DatabasePageSize(int *pRc, Fts3Table *p){ + if( *pRc==SQLITE_OK ){ + int rc; /* Return code */ + char *zSql; /* SQL text "PRAGMA %Q.page_size" */ + sqlite3_stmt *pStmt; /* Compiled "PRAGMA %Q.page_size" statement */ + + zSql = sqlite3_mprintf("PRAGMA %Q.page_size", p->zDb); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_step(pStmt); + p->nPgsz = sqlite3_column_int(pStmt, 0); + rc = sqlite3_finalize(pStmt); + } + } + assert( p->nPgsz>0 || rc!=SQLITE_OK ); + sqlite3_free(zSql); + *pRc = rc; + } } /* -** Determine if a table currently exists in the database. +** "Special" FTS4 arguments are column specifications of the following form: +** +** = +** +** There may not be whitespace surrounding the "=" character. The +** term may be quoted, but the may not. */ -static void fts3TableExists( - int *pRc, /* Success code */ - sqlite3 *db, /* The database connection to test */ - const char *zDb, /* ATTACHed database within the connection */ - const char *zName, /* Name of the FTS3 table */ - const char *zSuffix, /* Shadow table extension */ - u8 *pResult /* Write results here */ +static int fts3IsSpecialColumn( + const char *z, + int *pnKey, + char **pzValue ){ - int rc = SQLITE_OK; - int res = 0; - char *zSql; - if( *pRc ) return; - zSql = sqlite3_mprintf( - "SELECT 1 FROM %Q.sqlite_master WHERE name='%q%s'", - zDb, zName, zSuffix - ); - rc = sqlite3_exec(db, zSql, fts3TableExistsCallback, &res, 0); - sqlite3_free(zSql); - *pResult = (u8)(res & 0xff); - if( rc!=SQLITE_ABORT ) *pRc = rc; + char *zValue; + const char *zCsr = z; + + while( *zCsr!='=' ){ + if( *zCsr=='\0' ) return 0; + zCsr++; + } + + *pnKey = (int)(zCsr-z); + zValue = sqlite3_mprintf("%s", &zCsr[1]); + if( zValue ){ + sqlite3Fts3Dequote(zValue); + } + *pzValue = zValue; + return 1; } /* @@ -108100,8 +108893,8 @@ static int fts3InitVtab( char **pzErr /* Write any error message here */ ){ Fts3Hash *pHash = (Fts3Hash *)pAux; - Fts3Table *p; /* Pointer to allocated vtab */ - int rc; /* Return code */ + Fts3Table *p = 0; /* Pointer to allocated vtab */ + int rc = SQLITE_OK; /* Return code */ int i; /* Iterator variable */ int nByte; /* Size of allocation used for *p */ int iCol; /* Column index */ @@ -108110,35 +108903,90 @@ static int fts3InitVtab( char *zCsr; /* Space for holding column names */ int nDb; /* Bytes required to hold database name */ int nName; /* Bytes required to hold table name */ - - const char *zTokenizer = 0; /* Name of tokenizer to use */ + int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */ + int bNoDocsize = 0; /* True to omit %_docsize table */ + const char **aCol; /* Array of column names */ sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ + assert( strlen(argv[0])==4 ); + assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) + || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4) + ); + nDb = (int)strlen(argv[1]) + 1; nName = (int)strlen(argv[2]) + 1; - for(i=3; i8 + && 0==sqlite3_strnicmp(z, "tokenize", 8) + && 0==sqlite3Fts3IsIdChar(z[8]) + ){ + rc = sqlite3Fts3InitTokenizer(pHash, &z[9], &pTokenizer, pzErr); } - if( z!=zTokenizer ){ - nString += (int)(strlen(z) + 1); + + /* Check if it is an FTS4 special argument. */ + else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){ + if( !zVal ){ + rc = SQLITE_NOMEM; + goto fts3_init_out; + } + if( nKey==9 && 0==sqlite3_strnicmp(z, "matchinfo", 9) ){ + if( strlen(zVal)==4 && 0==sqlite3_strnicmp(zVal, "fts3", 4) ){ + bNoDocsize = 1; + }else{ + *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal); + rc = SQLITE_ERROR; + } + }else{ + *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z); + rc = SQLITE_ERROR; + } + sqlite3_free(zVal); } - } - nCol = argc - 3 - (zTokenizer!=0); - if( zTokenizer==0 ){ - rc = sqlite3Fts3InitTokenizer(pHash, 0, &pTokenizer, 0, pzErr); - if( rc!=SQLITE_OK ){ - return rc; + + /* Otherwise, the argument is a column name. */ + else { + nString += (int)(strlen(z) + 1); + aCol[nCol++] = z; } - assert( pTokenizer ); } + if( rc!=SQLITE_OK ) goto fts3_init_out; if( nCol==0 ){ + assert( nString==0 ); + aCol[0] = "content"; + nString = 8; nCol = 1; } + if( pTokenizer==0 ){ + rc = sqlite3Fts3InitTokenizer(pHash, "simple", &pTokenizer, pzErr); + if( rc!=SQLITE_OK ) goto fts3_init_out; + } + assert( pTokenizer ); + + /* Allocate and populate the Fts3Table structure. */ nByte = sizeof(Fts3Table) + /* Fts3Table */ nCol * sizeof(char *) + /* azColumn */ @@ -108151,7 +108999,6 @@ static int fts3InitVtab( goto fts3_init_out; } memset(p, 0, nByte); - p->db = db; p->nColumn = nCol; p->nPendingData = 0; @@ -108159,11 +109006,12 @@ static int fts3InitVtab( p->pTokenizer = pTokenizer; p->nNodeSize = 1000; p->nMaxPendingData = FTS3_MAX_PENDING_DATA; - zCsr = (char *)&p->azColumn[nCol]; - + p->bHasDocsize = (isFts4 && bNoDocsize==0); + p->bHasStat = isFts4; fts3HashInit(&p->pendingTerms, FTS3_HASH_STRING, 1); /* Fill in the zName and zDb fields of the vtab structure. */ + zCsr = (char *)&p->azColumn[nCol]; p->zName = zCsr; memcpy(zCsr, argv[2], nName); zCsr += nName; @@ -108172,52 +109020,45 @@ static int fts3InitVtab( zCsr += nDb; /* Fill in the azColumn array */ - iCol = 0; - for(i=3; iazColumn[iCol++] = zCsr; - zCsr += n+1; - assert( zCsr <= &((char *)p)[nByte] ); - } - } - if( iCol==0 ){ - assert( nCol==1 ); - p->azColumn[0] = "content"; + for(iCol=0; iColazColumn[iCol] = zCsr; + zCsr += n+1; + assert( zCsr <= &((char *)p)[nByte] ); } /* If this is an xCreate call, create the underlying tables in the ** database. TODO: For xConnect(), it could verify that said tables exist. */ if( isCreate ){ - p->bHasContent = 1; - p->bHasDocsize = argv[0][3]=='4'; rc = fts3CreateTables(p); - }else{ - rc = SQLITE_OK; - fts3TableExists(&rc, db, argv[1], argv[2], "_content", &p->bHasContent); - fts3TableExists(&rc, db, argv[1], argv[2], "_docsize", &p->bHasDocsize); } - if( rc!=SQLITE_OK ) goto fts3_init_out; - rc = fts3DeclareVtab(p); - if( rc!=SQLITE_OK ) goto fts3_init_out; + /* Figure out the page-size for the database. This is required in order to + ** estimate the cost of loading large doclists from the database (see + ** function sqlite3Fts3SegReaderCost() for details). + */ + fts3DatabasePageSize(&rc, p); - *ppVTab = &p->base; + /* Declare the table schema to SQLite. */ + fts3DeclareVtab(&rc, p); fts3_init_out: - assert( p || (pTokenizer && rc!=SQLITE_OK) ); + + sqlite3_free((void *)aCol); if( rc!=SQLITE_OK ){ if( p ){ fts3DisconnectMethod((sqlite3_vtab *)p); - }else{ + }else if( pTokenizer ){ pTokenizer->pModule->xDestroy(pTokenizer); } + }else{ + *ppVTab = &p->base; } return rc; } @@ -108329,10 +109170,12 @@ static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ ** Close the cursor. For additional information see the documentation ** on the xClose method of the virtual table interface. */ -static int fulltextClose(sqlite3_vtab_cursor *pCursor){ +static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); sqlite3_finalize(pCsr->pStmt); sqlite3Fts3ExprFree(pCsr->pExpr); + sqlite3Fts3FreeDeferredTokens(pCsr); sqlite3_free(pCsr->aDoclist); sqlite3_free(pCsr->aMatchinfo); sqlite3_free(pCsr); @@ -108371,50 +109214,137 @@ static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ } /* -** Advance the cursor to the next row in the %_content table that -** matches the search criteria. For a MATCH search, this will be -** the next row that matches. For a full-table scan, this will be -** simply the next row in the %_content table. For a docid lookup, -** this routine simply sets the EOF flag. +** This function is used to process a single interior node when searching +** a b-tree for a term or term prefix. The node data is passed to this +** function via the zNode/nNode parameters. The term to search for is +** passed in zTerm/nTerm. ** -** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned -** even if we reach end-of-file. The fts3EofMethod() will be called -** subsequently to determine whether or not an EOF was hit. +** If piFirst is not NULL, then this function sets *piFirst to the blockid +** of the child node that heads the sub-tree that may contain the term. +** +** If piLast is not NULL, then *piLast is set to the right-most child node +** that heads a sub-tree that may contain a term for which zTerm/nTerm is +** a prefix. +** +** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK. */ -static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ +static int fts3ScanInteriorNode( + const char *zTerm, /* Term to select leaves for */ + int nTerm, /* Size of term zTerm in bytes */ + const char *zNode, /* Buffer containing segment interior node */ + int nNode, /* Size of buffer at zNode */ + sqlite3_int64 *piFirst, /* OUT: Selected child node */ + sqlite3_int64 *piLast /* OUT: Selected child node */ +){ int rc = SQLITE_OK; /* Return code */ - Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; + const char *zCsr = zNode; /* Cursor to iterate through node */ + const char *zEnd = &zCsr[nNode];/* End of interior node buffer */ + char *zBuffer = 0; /* Buffer to load terms into */ + int nAlloc = 0; /* Size of allocated buffer */ + int isFirstTerm = 1; /* True when processing first term on page */ + sqlite3_int64 iChild; /* Block id of child node to descend to */ - if( pCsr->aDoclist==0 ){ - if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ - pCsr->isEof = 1; - rc = sqlite3_reset(pCsr->pStmt); - } - }else if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){ - pCsr->isEof = 1; - }else{ - sqlite3_reset(pCsr->pStmt); - fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId); - pCsr->isRequireSeek = 1; - pCsr->isMatchinfoNeeded = 1; + /* Skip over the 'height' varint that occurs at the start of every + ** interior node. Then load the blockid of the left-child of the b-tree + ** node into variable iChild. + ** + ** Even if the data structure on disk is corrupted, this (reading two + ** varints from the buffer) does not risk an overread. If zNode is a + ** root node, then the buffer comes from a SELECT statement. SQLite does + ** not make this guarantee explicitly, but in practice there are always + ** either more than 20 bytes of allocated space following the nNode bytes of + ** contents, or two zero bytes. Or, if the node is read from the %_segments + ** table, then there are always 20 bytes of zeroed padding following the + ** nNode bytes of content (see sqlite3Fts3ReadBlock() for details). + */ + zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); + zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); + if( zCsr>zEnd ){ + return SQLITE_CORRUPT; } + + while( zCsrzEnd ){ + rc = SQLITE_CORRUPT; + goto finish_scan; + } + if( nPrefix+nSuffix>nAlloc ){ + char *zNew; + nAlloc = (nPrefix+nSuffix) * 2; + zNew = (char *)sqlite3_realloc(zBuffer, nAlloc); + if( !zNew ){ + rc = SQLITE_NOMEM; + goto finish_scan; + } + zBuffer = zNew; + } + memcpy(&zBuffer[nPrefix], zCsr, nSuffix); + nBuffer = nPrefix + nSuffix; + zCsr += nSuffix; + + /* Compare the term we are searching for with the term just loaded from + ** the interior node. If the specified term is greater than or equal + ** to the term from the interior node, then all terms on the sub-tree + ** headed by node iChild are smaller than zTerm. No need to search + ** iChild. + ** + ** If the interior node term is larger than the specified term, then + ** the tree headed by iChild may contain the specified term. + */ + cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer)); + if( piFirst && (cmp<0 || (cmp==0 && nBuffer>nTerm)) ){ + *piFirst = iChild; + piFirst = 0; + } + + if( piLast && cmp<0 ){ + *piLast = iChild; + piLast = 0; + } + + iChild++; + }; + + if( piFirst ) *piFirst = iChild; + if( piLast ) *piLast = iChild; + + finish_scan: + sqlite3_free(zBuffer); return rc; } /* -** The buffer pointed to by argument zNode (size nNode bytes) contains the -** root node of a b-tree segment. The segment is guaranteed to be at least -** one level high (i.e. the root node is not also a leaf). If successful, -** this function locates the leaf node of the segment that may contain the -** term specified by arguments zTerm and nTerm and writes its block number -** to *piLeaf. +** The buffer pointed to by argument zNode (size nNode bytes) contains an +** interior node of a b-tree segment. The zTerm buffer (size nTerm bytes) +** contains a term. This function searches the sub-tree headed by the zNode +** node for the range of leaf nodes that may contain the specified term +** or terms for which the specified term is a prefix. ** -** It is possible that the returned leaf node does not contain the specified -** term. However, if the segment does contain said term, it is stored on -** the identified leaf node. Because this function only inspects interior -** segment nodes (and never loads leaf nodes into memory), it is not possible -** to be sure. +** If piLeaf is not NULL, then *piLeaf is set to the blockid of the +** left-most leaf node in the tree that may contain the specified term. +** If piLeaf2 is not NULL, then *piLeaf2 is set to the blockid of the +** right-most leaf node that may contain a term for which the specified +** term is a prefix. +** +** It is possible that the range of returned leaf nodes does not contain +** the specified term or any terms for which it is a prefix. However, if the +** segment does contain any such terms, they are stored within the identified +** range. Because this function only inspects interior segment nodes (and +** never loads leaf nodes into memory), it is not possible to be sure. ** ** If an error occurs, an error code other than SQLITE_OK is returned. */ @@ -108424,77 +109354,41 @@ static int fts3SelectLeaf( int nTerm, /* Size of term zTerm in bytes */ const char *zNode, /* Buffer containing segment interior node */ int nNode, /* Size of buffer at zNode */ - sqlite3_int64 *piLeaf /* Selected leaf node */ + sqlite3_int64 *piLeaf, /* Selected leaf node */ + sqlite3_int64 *piLeaf2 /* Selected leaf node */ ){ - int rc = SQLITE_OK; /* Return code */ - const char *zCsr = zNode; /* Cursor to iterate through node */ - const char *zEnd = &zCsr[nNode];/* End of interior node buffer */ - char *zBuffer = 0; /* Buffer to load terms into */ - int nAlloc = 0; /* Size of allocated buffer */ + int rc; /* Return code */ + int iHeight; /* Height of this node in tree */ - while( 1 ){ - int isFirstTerm = 1; /* True when processing first term on page */ - int iHeight; /* Height of this node in tree */ - sqlite3_int64 iChild; /* Block id of child node to descend to */ - int nBlock; /* Size of child node in bytes */ + assert( piLeaf || piLeaf2 ); - zCsr += sqlite3Fts3GetVarint32(zCsr, &iHeight); - zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); - - while( zCsrnAlloc ){ - char *zNew; - nAlloc = (nPrefix+nSuffix) * 2; - zNew = (char *)sqlite3_realloc(zBuffer, nAlloc); - if( !zNew ){ - sqlite3_free(zBuffer); - return SQLITE_NOMEM; - } - zBuffer = zNew; - } - memcpy(&zBuffer[nPrefix], zCsr, nSuffix); - nBuffer = nPrefix + nSuffix; - zCsr += nSuffix; - - /* Compare the term we are searching for with the term just loaded from - ** the interior node. If the specified term is greater than or equal - ** to the term from the interior node, then all terms on the sub-tree - ** headed by node iChild are smaller than zTerm. No need to search - ** iChild. - ** - ** If the interior node term is larger than the specified term, then - ** the tree headed by iChild may contain the specified term. - */ - cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer)); - if( cmp<0 || (cmp==0 && nBuffer>nTerm) ) break; - iChild++; - }; + sqlite3Fts3GetVarint32(zNode, &iHeight); + rc = fts3ScanInteriorNode(zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2); + assert( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) ); - /* If (iHeight==1), the children of this interior node are leaves. The - ** specified term may be present on leaf node iChild. - */ - if( iHeight==1 ){ - *piLeaf = iChild; - break; + if( rc==SQLITE_OK && iHeight>1 ){ + char *zBlob = 0; /* Blob read from %_segments table */ + int nBlob; /* Size of zBlob in bytes */ + + if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){ + rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob); + if( rc==SQLITE_OK ){ + rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0); + } + sqlite3_free(zBlob); + piLeaf = 0; + zBlob = 0; } - /* Descend to interior node iChild. */ - rc = sqlite3Fts3ReadBlock(p, iChild, &zCsr, &nBlock); - if( rc!=SQLITE_OK ) break; - zEnd = &zCsr[nBlock]; + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3ReadBlock(p, piLeaf ? *piLeaf : *piLeaf2, &zBlob, &nBlob); + } + if( rc==SQLITE_OK ){ + rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2); + } + sqlite3_free(zBlob); } - sqlite3_free(zBuffer); + return rc; } @@ -108726,20 +109620,44 @@ static void fts3PoslistMerge( /* ** nToken==1 searches for adjacent positions. +** +** This function is used to merge two position lists into one. When it is +** called, *pp1 and *pp2 must both point to position lists. A position-list is +** the part of a doclist that follows each document id. For example, if a row +** contains: +** +** 'a b c'|'x y z'|'a b b a' +** +** Then the position list for this row for token 'b' would consist of: +** +** 0x02 0x01 0x02 0x03 0x03 0x00 +** +** When this function returns, both *pp1 and *pp2 are left pointing to the +** byte following the 0x00 terminator of their respective position lists. +** +** If isSaveLeft is 0, an entry is added to the output position list for +** each position in *pp2 for which there exists one or more positions in +** *pp1 so that (pos(*pp2)>pos(*pp1) && pos(*pp2)-pos(*pp1)<=nToken). i.e. +** when the *pp1 token appears before the *pp2 token, but not more than nToken +** slots before it. */ static int fts3PoslistPhraseMerge( - char **pp, /* Output buffer */ + char **pp, /* IN/OUT: Preallocated output buffer */ int nToken, /* Maximum difference in token positions */ int isSaveLeft, /* Save the left position */ - char **pp1, /* Left input list */ - char **pp2 /* Right input list */ + int isExact, /* If *pp1 is exactly nTokens before *pp2 */ + char **pp1, /* IN/OUT: Left input list */ + char **pp2 /* IN/OUT: Right input list */ ){ char *p = (pp ? *pp : 0); char *p1 = *pp1; char *p2 = *pp2; - int iCol1 = 0; int iCol2 = 0; + + /* Never set both isSaveLeft and isExact for the same invocation. */ + assert( isSaveLeft==0 || isExact==0 ); + assert( *p1!=0 && *p2!=0 ); if( *p1==POS_COLUMN ){ p1++; @@ -108768,7 +109686,9 @@ static int fts3PoslistPhraseMerge( fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; while( 1 ){ - if( iPos2>iPos1 && iPos2<=iPos1+nToken ){ + if( iPos2==iPos1+nToken + || (isExact==0 && iPos2>iPos1 && iPos2<=iPos1+nToken) + ){ sqlite3_int64 iSave; if( !pp ){ fts3PoslistCopy(0, &p2); @@ -108851,21 +109771,21 @@ static int fts3PoslistNearMerge( char *p2 = *pp2; if( !pp ){ - if( fts3PoslistPhraseMerge(0, nRight, 0, pp1, pp2) ) return 1; + if( fts3PoslistPhraseMerge(0, nRight, 0, 0, pp1, pp2) ) return 1; *pp1 = p1; *pp2 = p2; - return fts3PoslistPhraseMerge(0, nLeft, 0, pp2, pp1); + return fts3PoslistPhraseMerge(0, nLeft, 0, 0, pp2, pp1); }else{ char *pTmp1 = aTmp; char *pTmp2; char *aTmp2; int res = 1; - fts3PoslistPhraseMerge(&pTmp1, nRight, 0, pp1, pp2); + fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2); aTmp2 = pTmp2 = pTmp1; *pp1 = p1; *pp2 = p2; - fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, pp2, pp1); + fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1); if( pTmp1!=aTmp && pTmp2!=aTmp2 ){ fts3PoslistMerge(pp, &aTmp, &aTmp2); }else if( pTmp1!=aTmp ){ @@ -108911,7 +109831,8 @@ static int fts3DoclistMerge( char *a1, /* Buffer containing first doclist */ int n1, /* Size of buffer a1 */ char *a2, /* Buffer containing second doclist */ - int n2 /* Size of buffer a2 */ + int n2, /* Size of buffer a2 */ + int *pnDoc /* OUT: Number of docids in output */ ){ sqlite3_int64 i1 = 0; sqlite3_int64 i2 = 0; @@ -108922,6 +109843,7 @@ static int fts3DoclistMerge( char *p2 = a2; char *pEnd1 = &a1[n1]; char *pEnd2 = &a2[n2]; + int nDoc = 0; assert( mergetype==MERGE_OR || mergetype==MERGE_POS_OR || mergetype==MERGE_AND || mergetype==MERGE_NOT @@ -108965,6 +109887,7 @@ static int fts3DoclistMerge( fts3PutDeltaVarint(&p, &iPrev, i1); fts3GetDeltaVarint2(&p1, pEnd1, &i1); fts3GetDeltaVarint2(&p2, pEnd2, &i2); + nDoc++; }else if( i1aaOutput[i]; nOut = pTS->anOutput[i]; - pTS->aaOutput[0] = 0; + pTS->aaOutput[i] = 0; }else{ int nNew = nOut + pTS->anOutput[i]; char *aNew = sqlite3_malloc(nNew); @@ -109097,7 +110023,7 @@ static int fts3TermSelectMerge(TermSelect *pTS){ return SQLITE_NOMEM; } fts3DoclistMerge(mergetype, 0, 0, - aNew, &nNew, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut + aNew, &nNew, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, 0 ); sqlite3_free(pTS->aaOutput[i]); sqlite3_free(aOut); @@ -109168,8 +110094,8 @@ static int fts3TermSelectCb( } return SQLITE_NOMEM; } - fts3DoclistMerge(mergetype, 0, 0, - aNew, &nNew, pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge + fts3DoclistMerge(mergetype, 0, 0, aNew, &nNew, + pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge, 0 ); if( iOut>0 ) sqlite3_free(aMerge); @@ -109187,43 +110113,106 @@ static int fts3TermSelectCb( return SQLITE_OK; } +static int fts3DeferredTermSelect( + Fts3DeferredToken *pToken, /* Phrase token */ + int isTermPos, /* True to include positions */ + int *pnOut, /* OUT: Size of list */ + char **ppOut /* OUT: Body of list */ +){ + char *aSource; + int nSource; + + aSource = sqlite3Fts3DeferredDoclist(pToken, &nSource); + if( !aSource ){ + *pnOut = 0; + *ppOut = 0; + }else if( isTermPos ){ + *ppOut = sqlite3_malloc(nSource); + if( !*ppOut ) return SQLITE_NOMEM; + memcpy(*ppOut, aSource, nSource); + *pnOut = nSource; + }else{ + sqlite3_int64 docid; + *pnOut = sqlite3Fts3GetVarint(aSource, &docid); + *ppOut = sqlite3_malloc(*pnOut); + if( !*ppOut ) return SQLITE_NOMEM; + sqlite3Fts3PutVarint(*ppOut, docid); + } + + return SQLITE_OK; +} + /* -** This function retreives the doclist for the specified term (or term -** prefix) from the database. -** -** The returned doclist may be in one of two formats, depending on the -** value of parameter isReqPos. If isReqPos is zero, then the doclist is -** a sorted list of delta-compressed docids (a bare doclist). If isReqPos -** is non-zero, then the returned list is in the same format as is stored -** in the database without the found length specifier at the start of on-disk -** doclists. +** An Fts3SegReaderArray is used to store an array of Fts3SegReader objects. +** Elements are added to the array using fts3SegReaderArrayAdd(). */ -static int fts3TermSelect( - Fts3Table *p, /* Virtual table handle */ - int iColumn, /* Column to query (or -ve for all columns) */ +struct Fts3SegReaderArray { + int nSegment; /* Number of valid entries in apSegment[] */ + int nAlloc; /* Allocated size of apSegment[] */ + int nCost; /* The cost of executing SegReaderIterate() */ + Fts3SegReader *apSegment[1]; /* Array of seg-reader objects */ +}; + + +/* +** Free an Fts3SegReaderArray object. Also free all seg-readers in the +** array (using sqlite3Fts3SegReaderFree()). +*/ +static void fts3SegReaderArrayFree(Fts3SegReaderArray *pArray){ + if( pArray ){ + int i; + for(i=0; inSegment; i++){ + sqlite3Fts3SegReaderFree(pArray->apSegment[i]); + } + sqlite3_free(pArray); + } +} + +static int fts3SegReaderArrayAdd( + Fts3SegReaderArray **ppArray, + Fts3SegReader *pNew +){ + Fts3SegReaderArray *pArray = *ppArray; + + if( !pArray || pArray->nAlloc==pArray->nSegment ){ + int nNew = (pArray ? pArray->nAlloc+16 : 16); + pArray = (Fts3SegReaderArray *)sqlite3_realloc(pArray, + sizeof(Fts3SegReaderArray) + (nNew-1) * sizeof(Fts3SegReader*) + ); + if( !pArray ){ + sqlite3Fts3SegReaderFree(pNew); + return SQLITE_NOMEM; + } + if( nNew==16 ){ + pArray->nSegment = 0; + pArray->nCost = 0; + } + pArray->nAlloc = nNew; + *ppArray = pArray; + } + + pArray->apSegment[pArray->nSegment++] = pNew; + return SQLITE_OK; +} + +static int fts3TermSegReaderArray( + Fts3Cursor *pCsr, /* Virtual table cursor handle */ const char *zTerm, /* Term to query for */ int nTerm, /* Size of zTerm in bytes */ int isPrefix, /* True for a prefix search */ - int isReqPos, /* True to include position lists in output */ - int *pnOut, /* OUT: Size of buffer at *ppOut */ - char **ppOut /* OUT: Malloced result buffer */ + Fts3SegReaderArray **ppArray /* OUT: Allocated seg-reader array */ ){ - int i; - TermSelect tsc; - Fts3SegFilter filter; /* Segment term filter configuration */ - Fts3SegReader **apSegment; /* Array of segments to read data from */ - int nSegment = 0; /* Size of apSegment array */ - int nAlloc = 16; /* Allocated size of segment array */ + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; int rc; /* Return code */ + Fts3SegReaderArray *pArray = 0; /* Array object to build */ + Fts3SegReader *pReader = 0; /* Seg-reader to add to pArray */ sqlite3_stmt *pStmt = 0; /* SQL statement to scan %_segdir table */ int iAge = 0; /* Used to assign ages to segments */ - apSegment = (Fts3SegReader **)sqlite3_malloc(sizeof(Fts3SegReader*)*nAlloc); - if( !apSegment ) return SQLITE_NOMEM; - rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &apSegment[0]); - if( rc!=SQLITE_OK ) goto finished; - if( apSegment[0] ){ - nSegment = 1; + /* Allocate a seg-reader to scan the pending terms, if any. */ + rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &pReader); + if( rc==SQLITE_OK && pReader ) { + rc = fts3SegReaderArrayAdd(&pArray, pReader); } /* Loop through the entire %_segdir table. For each segment, create a @@ -109231,12 +110220,10 @@ static int fts3TermSelect( ** that may contain a term that matches zTerm/nTerm. For non-prefix ** searches, this is always a single leaf. For prefix searches, this ** may be a contiguous block of leaves. - ** - ** The code in this loop does not actually load any leaves into memory - ** (unless the root node happens to be a leaf). It simply examines the - ** b-tree structure to determine which leaves need to be inspected. */ - rc = sqlite3Fts3AllSegdirs(p, &pStmt); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3AllSegdirs(p, &pStmt); + } while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ Fts3SegReader *pNew = 0; int nRoot = sqlite3_column_bytes(pStmt, 4); @@ -109246,66 +110233,79 @@ static int fts3TermSelect( ** leaf). Do not bother inspecting any data in this case, just ** create a Fts3SegReader to scan the single leaf. */ - rc = sqlite3Fts3SegReaderNew(p, iAge, 0, 0, 0, zRoot, nRoot, &pNew); + rc = sqlite3Fts3SegReaderNew(iAge, 0, 0, 0, zRoot, nRoot, &pNew); }else{ - int rc2; /* Return value of sqlite3Fts3ReadBlock() */ - sqlite3_int64 i1; /* Blockid of leaf that may contain zTerm */ - rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1); + sqlite3_int64 i1; /* First leaf that may contain zTerm */ + sqlite3_int64 i2; /* Final leaf that may contain zTerm */ + rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1, (isPrefix?&i2:0)); + if( isPrefix==0 ) i2 = i1; if( rc==SQLITE_OK ){ - sqlite3_int64 i2 = sqlite3_column_int64(pStmt, 2); - rc = sqlite3Fts3SegReaderNew(p, iAge, i1, i2, 0, 0, 0, &pNew); + rc = sqlite3Fts3SegReaderNew(iAge, i1, i2, 0, 0, 0, &pNew); } + } + assert( (pNew==0)==(rc!=SQLITE_OK) ); - /* The following call to ReadBlock() serves to reset the SQL statement - ** used to retrieve blocks of data from the %_segments table. If it is - ** not reset here, then it may remain classified as an active statement - ** by SQLite, which may lead to "DROP TABLE" or "DETACH" commands - ** failing. - */ - rc2 = sqlite3Fts3ReadBlock(p, 0, 0, 0); - if( rc==SQLITE_OK ){ - rc = rc2; - } + /* If a new Fts3SegReader was allocated, add it to the array. */ + if( rc==SQLITE_OK ){ + rc = fts3SegReaderArrayAdd(&pArray, pNew); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3SegReaderCost(pCsr, pNew, &pArray->nCost); } iAge++; + } - /* If a new Fts3SegReader was allocated, add it to the apSegment array. */ - assert( pNew!=0 || rc!=SQLITE_OK ); - if( pNew ){ - if( nSegment==nAlloc ){ - Fts3SegReader **pArray; - nAlloc += 16; - pArray = (Fts3SegReader **)sqlite3_realloc( - apSegment, nAlloc*sizeof(Fts3SegReader *) - ); - if( !pArray ){ - sqlite3Fts3SegReaderFree(p, pNew); - rc = SQLITE_NOMEM; - goto finished; - } - apSegment = pArray; - } - apSegment[nSegment++] = pNew; - } + if( rc==SQLITE_DONE ){ + rc = sqlite3_reset(pStmt); + }else{ + sqlite3_reset(pStmt); } - if( rc!=SQLITE_DONE ){ - assert( rc!=SQLITE_OK ); - goto finished; + if( rc!=SQLITE_OK ){ + fts3SegReaderArrayFree(pArray); + pArray = 0; } + *ppArray = pArray; + return rc; +} + +/* +** This function retreives the doclist for the specified term (or term +** prefix) from the database. +** +** The returned doclist may be in one of two formats, depending on the +** value of parameter isReqPos. If isReqPos is zero, then the doclist is +** a sorted list of delta-compressed docids (a bare doclist). If isReqPos +** is non-zero, then the returned list is in the same format as is stored +** in the database without the found length specifier at the start of on-disk +** doclists. +*/ +static int fts3TermSelect( + Fts3Table *p, /* Virtual table handle */ + Fts3PhraseToken *pTok, /* Token to query for */ + int iColumn, /* Column to query (or -ve for all columns) */ + int isReqPos, /* True to include position lists in output */ + int *pnOut, /* OUT: Size of buffer at *ppOut */ + char **ppOut /* OUT: Malloced result buffer */ +){ + int rc; /* Return code */ + Fts3SegReaderArray *pArray; /* Seg-reader array for this term */ + TermSelect tsc; /* Context object for fts3TermSelectCb() */ + Fts3SegFilter filter; /* Segment term filter configuration */ + pArray = pTok->pArray; memset(&tsc, 0, sizeof(TermSelect)); tsc.isReqPos = isReqPos; filter.flags = FTS3_SEGMENT_IGNORE_EMPTY - | (isPrefix ? FTS3_SEGMENT_PREFIX : 0) + | (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0) | (isReqPos ? FTS3_SEGMENT_REQUIRE_POS : 0) | (iColumnnColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0); filter.iCol = iColumn; - filter.zTerm = zTerm; - filter.nTerm = nTerm; + filter.zTerm = pTok->z; + filter.nTerm = pTok->n; - rc = sqlite3Fts3SegReaderIterate(p, apSegment, nSegment, &filter, - fts3TermSelectCb, (void *)&tsc + rc = sqlite3Fts3SegReaderIterate(p, pArray->apSegment, pArray->nSegment, + &filter, fts3TermSelectCb, (void *)&tsc ); if( rc==SQLITE_OK ){ rc = fts3TermSelectMerge(&tsc); @@ -109315,26 +110315,112 @@ static int fts3TermSelect( *ppOut = tsc.aaOutput[0]; *pnOut = tsc.anOutput[0]; }else{ + int i; for(i=0; ipArray = 0; + return rc; +} + +/* +** This function counts the total number of docids in the doclist stored +** in buffer aList[], size nList bytes. +** +** If the isPoslist argument is true, then it is assumed that the doclist +** contains a position-list following each docid. Otherwise, it is assumed +** that the doclist is simply a list of docids stored as delta encoded +** varints. +*/ +static int fts3DoclistCountDocids(int isPoslist, char *aList, int nList){ + int nDoc = 0; /* Return value */ + if( aList ){ + char *aEnd = &aList[nList]; /* Pointer to one byte after EOF */ + char *p = aList; /* Cursor */ + if( !isPoslist ){ + /* The number of docids in the list is the same as the number of + ** varints. In FTS3 a varint consists of a single byte with the 0x80 + ** bit cleared and zero or more bytes with the 0x80 bit set. So to + ** count the varints in the buffer, just count the number of bytes + ** with the 0x80 bit clear. */ + while( ppLeft); + if( rc==SQLITE_OK ){ + rc = fts3DeferExpression(pCsr, pExpr->pRight); + } + if( pExpr->eType==FTSQUERY_PHRASE ){ + int iCol = pExpr->pPhrase->iColumn; + int i; + for(i=0; rc==SQLITE_OK && ipPhrase->nToken; i++){ + Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i]; + if( pToken->pDeferred==0 ){ + rc = sqlite3Fts3DeferToken(pCsr, pToken, iCol); + } + } + } } - sqlite3_free(apSegment); return rc; } +/* +** This function removes the position information from a doclist. When +** called, buffer aList (size *pnList bytes) contains a doclist that includes +** position information. This function removes the position information so +** that aList contains only docids, and adjusts *pnList to reflect the new +** (possibly reduced) size of the doclist. +*/ +static void fts3DoclistStripPositions( + char *aList, /* IN/OUT: Buffer containing doclist */ + int *pnList /* IN/OUT: Size of doclist in bytes */ +){ + if( aList ){ + char *aEnd = &aList[*pnList]; /* Pointer to one byte after EOF */ + char *p = aList; /* Input cursor */ + char *pOut = aList; /* Output cursor */ + + while( piColumn; int isTermPos = (pPhrase->nToken>1 || isReqPos); + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + int isFirst = 1; + int iPrevTok = 0; + int nDoc = 0; + + /* If this is an xFilter() evaluation, create a segment-reader for each + ** phrase token. Or, if this is an xNext() or snippet/offsets/matchinfo + ** evaluation, only create segment-readers if there are no Fts3DeferredToken + ** objects attached to the phrase-tokens. + */ for(ii=0; iinToken; ii++){ - struct PhraseToken *pTok = &pPhrase->aToken[ii]; - char *z = pTok->z; /* Next token of the phrase */ - int n = pTok->n; /* Size of z in bytes */ - int isPrefix = pTok->isPrefix;/* True if token is a prefix */ - char *pList; /* Pointer to token doclist */ - int nList; /* Size of buffer at pList */ - - rc = fts3TermSelect(p, iCol, z, n, isPrefix, isTermPos, &nList, &pList); + Fts3PhraseToken *pTok = &pPhrase->aToken[ii]; + if( pTok->pArray==0 ){ + if( (pCsr->eEvalmode==FTS3_EVAL_FILTER) + || (pCsr->eEvalmode==FTS3_EVAL_NEXT && pCsr->pDeferred==0) + || (pCsr->eEvalmode==FTS3_EVAL_MATCHINFO && pTok->bFulltext) + ){ + rc = fts3TermSegReaderArray( + pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pArray + ); + if( rc!=SQLITE_OK ) return rc; + } + } + } + + for(ii=0; iinToken; ii++){ + Fts3PhraseToken *pTok; /* Token to find doclist for */ + int iTok = 0; /* The token being queried this iteration */ + char *pList = 0; /* Pointer to token doclist */ + int nList = 0; /* Size of buffer at pList */ + + /* Select a token to process. If this is an xFilter() call, then tokens + ** are processed in order from least to most costly. Otherwise, tokens + ** are processed in the order in which they occur in the phrase. + */ + if( pCsr->eEvalmode==FTS3_EVAL_MATCHINFO ){ + assert( isReqPos ); + iTok = ii; + pTok = &pPhrase->aToken[iTok]; + if( pTok->bFulltext==0 ) continue; + }else if( pCsr->eEvalmode==FTS3_EVAL_NEXT || isReqPos ){ + iTok = ii; + pTok = &pPhrase->aToken[iTok]; + }else{ + int nMinCost = 0x7FFFFFFF; + int jj; + + /* Find the remaining token with the lowest cost. */ + for(jj=0; jjnToken; jj++){ + Fts3SegReaderArray *pArray = pPhrase->aToken[jj].pArray; + if( pArray && pArray->nCostnCost; + } + } + pTok = &pPhrase->aToken[iTok]; + + /* This branch is taken if it is determined that loading the doclist + ** for the next token would require more IO than loading all documents + ** currently identified by doclist pOut/nOut. No further doclists will + ** be loaded from the full-text index for this phrase. + */ + if( nMinCost>nDoc && ii>0 ){ + rc = fts3DeferExpression(pCsr, pCsr->pExpr); + break; + } + } + + if( pCsr->eEvalmode==FTS3_EVAL_NEXT && pTok->pDeferred ){ + rc = fts3DeferredTermSelect(pTok->pDeferred, isTermPos, &nList, &pList); + }else{ + if( pTok->pArray ){ + rc = fts3TermSelect(p, pTok, iCol, isTermPos, &nList, &pList); + } + pTok->bFulltext = 1; + } + assert( rc!=SQLITE_OK || pCsr->eEvalmode || pTok->pArray==0 ); if( rc!=SQLITE_OK ) break; - if( ii==0 ){ + if( isFirst ){ pOut = pList; nOut = nList; + if( pCsr->eEvalmode==FTS3_EVAL_FILTER && pPhrase->nToken>1 ){ + nDoc = fts3DoclistCountDocids(1, pOut, nOut); + } + isFirst = 0; + iPrevTok = iTok; }else{ - /* Merge the new term list and the current output. If this is the - ** last term in the phrase, and positions are not required in the - ** output of this function, the positions can be dropped as part - ** of this merge. Either way, the result of this merge will be - ** smaller than nList bytes. The code in fts3DoclistMerge() is written - ** so that it is safe to use pList as the output as well as an input - ** in this case. + /* Merge the new term list and the current output. */ + char *aLeft, *aRight; + int nLeft, nRight; + int nDist; + int mt; + + /* If this is the final token of the phrase, and positions were not + ** requested by the caller, use MERGE_PHRASE instead of POS_PHRASE. + ** This drops the position information from the output list. */ - int mergetype = MERGE_POS_PHRASE; - if( ii==pPhrase->nToken-1 && !isReqPos ){ - mergetype = MERGE_PHRASE; - } - fts3DoclistMerge(mergetype, 0, 0, pList, &nOut, pOut, nOut, pList, nList); - sqlite3_free(pOut); - pOut = pList; + mt = MERGE_POS_PHRASE; + if( ii==pPhrase->nToken-1 && !isReqPos ) mt = MERGE_PHRASE; + + assert( iPrevTok!=iTok ); + if( iPrevToknToken ){ + assert( pCsr->eEvalmode==FTS3_EVAL_FILTER && isReqPos==0 ); + fts3DoclistStripPositions(pOut, &nOut); + } *paOut = pOut; *pnOut = nOut; }else{ @@ -109390,6 +110571,14 @@ static int fts3PhraseSelect( return rc; } +/* +** This function merges two doclists according to the requirements of a +** NEAR operator. +** +** Both input doclists must include position information. The output doclist +** includes position information if the first argument to this function +** is MERGE_POS_NEAR, or does not if it is MERGE_NEAR. +*/ static int fts3NearMerge( int mergetype, /* MERGE_POS_NEAR or MERGE_NEAR */ int nNear, /* Parameter to NEAR operator */ @@ -109402,8 +110591,8 @@ static int fts3NearMerge( char **paOut, /* OUT: Results of merge (malloced) */ int *pnOut /* OUT: Sized of output buffer */ ){ - char *aOut; - int rc; + char *aOut; /* Buffer to write output doclist to */ + int rc; /* Return code */ assert( mergetype==MERGE_POS_NEAR || MERGE_NEAR ); @@ -109412,7 +110601,7 @@ static int fts3NearMerge( rc = SQLITE_NOMEM; }else{ rc = fts3DoclistMerge(mergetype, nNear+nTokenRight, nNear+nTokenLeft, - aOut, pnOut, aLeft, nLeft, aRight, nRight + aOut, pnOut, aLeft, nLeft, aRight, nRight, 0 ); if( rc!=SQLITE_OK ){ sqlite3_free(aOut); @@ -109424,8 +110613,23 @@ static int fts3NearMerge( return rc; } +/* +** This function is used as part of the processing for the snippet() and +** offsets() functions. +** +** Both pLeft and pRight are expression nodes of type FTSQUERY_PHRASE. Both +** have their respective doclists (including position information) loaded +** in Fts3Expr.aDoclist/nDoclist. This function removes all entries from +** each doclist that are not within nNear tokens of a corresponding entry +** in the other doclist. +*/ SQLITE_PRIVATE int sqlite3Fts3ExprNearTrim(Fts3Expr *pLeft, Fts3Expr *pRight, int nNear){ - int rc; + int rc; /* Return code */ + + assert( pLeft->eType==FTSQUERY_PHRASE ); + assert( pRight->eType==FTSQUERY_PHRASE ); + assert( pLeft->isLoaded && pRight->isLoaded ); + if( pLeft->aDoclist==0 || pRight->aDoclist==0 ){ sqlite3_free(pLeft->aDoclist); sqlite3_free(pRight->aDoclist); @@ -109433,8 +110637,8 @@ SQLITE_PRIVATE int sqlite3Fts3ExprNearTrim(Fts3Expr *pLeft, Fts3Expr *pRight, in pLeft->aDoclist = 0; rc = SQLITE_OK; }else{ - char *aOut; - int nOut; + char *aOut; /* Buffer in which to assemble new doclist */ + int nOut; /* Size of buffer aOut in bytes */ rc = fts3NearMerge(MERGE_POS_NEAR, nNear, pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist, @@ -109458,14 +110662,156 @@ SQLITE_PRIVATE int sqlite3Fts3ExprNearTrim(Fts3Expr *pLeft, Fts3Expr *pRight, in return rc; } + /* -** Evaluate the full-text expression pExpr against fts3 table pTab. Store -** the resulting doclist in *paOut and *pnOut. This routine mallocs for -** the space needed to store the output. The caller is responsible for -** freeing the space when it has finished. +** Allocate an Fts3SegReaderArray for each token in the expression pExpr. +** The allocated objects are stored in the Fts3PhraseToken.pArray member +** variables of each token structure. */ -static int evalFts3Expr( - Fts3Table *p, /* Virtual table handle */ +static int fts3ExprAllocateSegReaders( + Fts3Cursor *pCsr, /* FTS3 table */ + Fts3Expr *pExpr, /* Expression to create seg-readers for */ + int *pnExpr /* OUT: Number of AND'd expressions */ +){ + int rc = SQLITE_OK; /* Return code */ + + assert( pCsr->eEvalmode==FTS3_EVAL_FILTER ); + if( pnExpr && pExpr->eType!=FTSQUERY_AND ){ + (*pnExpr)++; + pnExpr = 0; + } + + if( pExpr->eType==FTSQUERY_PHRASE ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + int ii; + + for(ii=0; rc==SQLITE_OK && iinToken; ii++){ + Fts3PhraseToken *pTok = &pPhrase->aToken[ii]; + if( pTok->pArray==0 ){ + rc = fts3TermSegReaderArray( + pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pArray + ); + } + } + }else{ + rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pLeft, pnExpr); + if( rc==SQLITE_OK ){ + rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pRight, pnExpr); + } + } + return rc; +} + +/* +** Free the Fts3SegReaderArray objects associated with each token in the +** expression pExpr. In other words, this function frees the resources +** allocated by fts3ExprAllocateSegReaders(). +*/ +static void fts3ExprFreeSegReaders(Fts3Expr *pExpr){ + if( pExpr ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + if( pPhrase ){ + int kk; + for(kk=0; kknToken; kk++){ + fts3SegReaderArrayFree(pPhrase->aToken[kk].pArray); + pPhrase->aToken[kk].pArray = 0; + } + } + fts3ExprFreeSegReaders(pExpr->pLeft); + fts3ExprFreeSegReaders(pExpr->pRight); + } +} + +/* +** Return the sum of the costs of all tokens in the expression pExpr. This +** function must be called after Fts3SegReaderArrays have been allocated +** for all tokens using fts3ExprAllocateSegReaders(). +*/ +static int fts3ExprCost(Fts3Expr *pExpr){ + int nCost; /* Return value */ + if( pExpr->eType==FTSQUERY_PHRASE ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + int ii; + nCost = 0; + for(ii=0; iinToken; ii++){ + Fts3SegReaderArray *pArray = pPhrase->aToken[ii].pArray; + if( pArray ){ + nCost += pPhrase->aToken[ii].pArray->nCost; + } + } + }else{ + nCost = fts3ExprCost(pExpr->pLeft) + fts3ExprCost(pExpr->pRight); + } + return nCost; +} + +/* +** The following is a helper function (and type) for fts3EvalExpr(). It +** must be called after Fts3SegReaders have been allocated for every token +** in the expression. See the context it is called from in fts3EvalExpr() +** for further explanation. +*/ +typedef struct ExprAndCost ExprAndCost; +struct ExprAndCost { + Fts3Expr *pExpr; + int nCost; +}; +static void fts3ExprAssignCosts( + Fts3Expr *pExpr, /* Expression to create seg-readers for */ + ExprAndCost **ppExprCost /* OUT: Write to *ppExprCost */ +){ + if( pExpr->eType==FTSQUERY_AND ){ + fts3ExprAssignCosts(pExpr->pLeft, ppExprCost); + fts3ExprAssignCosts(pExpr->pRight, ppExprCost); + }else{ + (*ppExprCost)->pExpr = pExpr; + (*ppExprCost)->nCost = fts3ExprCost(pExpr); + (*ppExprCost)++; + } +} + +/* +** Evaluate the full-text expression pExpr against FTS3 table pTab. Store +** the resulting doclist in *paOut and *pnOut. This routine mallocs for +** the space needed to store the output. The caller is responsible for +** freeing the space when it has finished. +** +** This function is called in two distinct contexts: +** +** * From within the virtual table xFilter() method. In this case, the +** output doclist contains entries for all rows in the table, based on +** data read from the full-text index. +** +** In this case, if the query expression contains one or more tokens that +** are very common, then the returned doclist may contain a superset of +** the documents that actually match the expression. +** +** * From within the virtual table xNext() method. This call is only made +** if the call from within xFilter() found that there were very common +** tokens in the query expression and did return a superset of the +** matching documents. In this case the returned doclist contains only +** entries that correspond to the current row of the table. Instead of +** reading the data for each token from the full-text index, the data is +** already available in-memory in the Fts3PhraseToken.pDeferred structures. +** See fts3EvalDeferred() for how it gets there. +** +** In the first case above, Fts3Cursor.doDeferred==0. In the second (if it is +** required) Fts3Cursor.doDeferred==1. +** +** If the SQLite invokes the snippet(), offsets() or matchinfo() function +** as part of a SELECT on an FTS3 table, this function is called on each +** individual phrase expression in the query. If there were very common tokens +** found in the xFilter() call, then this function is called once for phrase +** for each row visited, and the returned doclist contains entries for the +** current row only. Otherwise, if there were no very common tokens, then this +** function is called once only for each phrase in the query and the returned +** doclist contains entries for all rows of the table. +** +** Fts3Cursor.doDeferred==1 when this function is called on phrases as a +** result of a snippet(), offsets() or matchinfo() invocation. +*/ +static int fts3EvalExpr( + Fts3Cursor *p, /* Virtual table cursor handle */ Fts3Expr *pExpr, /* Parsed fts3 expression */ char **paOut, /* OUT: Pointer to malloc'd result buffer */ int *pnOut, /* OUT: Size of buffer at *paOut */ @@ -109478,33 +110824,102 @@ static int evalFts3Expr( *pnOut = 0; if( pExpr ){ - assert( pExpr->eType==FTSQUERY_PHRASE - || pExpr->eType==FTSQUERY_NEAR - || isReqPos==0 + assert( pExpr->eType==FTSQUERY_NEAR || pExpr->eType==FTSQUERY_OR + || pExpr->eType==FTSQUERY_AND || pExpr->eType==FTSQUERY_NOT + || pExpr->eType==FTSQUERY_PHRASE ); + assert( pExpr->eType==FTSQUERY_PHRASE || isReqPos==0 ); + if( pExpr->eType==FTSQUERY_PHRASE ){ - rc = fts3PhraseSelect(p, pExpr->pPhrase, + rc = fts3PhraseSelect(p, pExpr->pPhrase, isReqPos || (pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR), paOut, pnOut ); + fts3ExprFreeSegReaders(pExpr); + }else if( p->eEvalmode==FTS3_EVAL_FILTER && pExpr->eType==FTSQUERY_AND ){ + ExprAndCost *aExpr = 0; /* Array of AND'd expressions and costs */ + int nExpr = 0; /* Size of aExpr[] */ + char *aRet = 0; /* Doclist to return to caller */ + int nRet = 0; /* Length of aRet[] in bytes */ + int nDoc = 0x7FFFFFFF; + + assert( !isReqPos ); + + rc = fts3ExprAllocateSegReaders(p, pExpr, &nExpr); + if( rc==SQLITE_OK ){ + assert( nExpr>1 ); + aExpr = sqlite3_malloc(sizeof(ExprAndCost) * nExpr); + if( !aExpr ) rc = SQLITE_NOMEM; + } + if( rc==SQLITE_OK ){ + int ii; /* Used to iterate through expressions */ + + fts3ExprAssignCosts(pExpr, &aExpr); + aExpr -= nExpr; + for(ii=0; iipExpr && (pBest==0 || pCand->nCostnCost) ){ + pBest = pCand; + } + } + + if( pBest->nCost>nDoc ){ + rc = fts3DeferExpression(p, p->pExpr); + break; + }else{ + rc = fts3EvalExpr(p, pBest->pExpr, &aNew, &nNew, 0); + if( rc!=SQLITE_OK ) break; + pBest->pExpr = 0; + if( ii==0 ){ + aRet = aNew; + nRet = nNew; + nDoc = fts3DoclistCountDocids(0, aRet, nRet); + }else{ + fts3DoclistMerge( + MERGE_AND, 0, 0, aRet, &nRet, aRet, nRet, aNew, nNew, &nDoc + ); + sqlite3_free(aNew); + } + } + } + } + + if( rc==SQLITE_OK ){ + *paOut = aRet; + *pnOut = nRet; + }else{ + assert( *paOut==0 ); + sqlite3_free(aRet); + } + sqlite3_free(aExpr); + fts3ExprFreeSegReaders(pExpr); + }else{ char *aLeft; char *aRight; int nLeft; int nRight; - if( 0==(rc = evalFts3Expr(p, pExpr->pRight, &aRight, &nRight, isReqPos)) - && 0==(rc = evalFts3Expr(p, pExpr->pLeft, &aLeft, &nLeft, isReqPos)) + assert( pExpr->eType==FTSQUERY_NEAR + || pExpr->eType==FTSQUERY_OR + || pExpr->eType==FTSQUERY_NOT + || (pExpr->eType==FTSQUERY_AND && p->eEvalmode==FTS3_EVAL_NEXT) + ); + + if( 0==(rc = fts3EvalExpr(p, pExpr->pRight, &aRight, &nRight, isReqPos)) + && 0==(rc = fts3EvalExpr(p, pExpr->pLeft, &aLeft, &nLeft, isReqPos)) ){ - assert( pExpr->eType==FTSQUERY_NEAR || pExpr->eType==FTSQUERY_OR - || pExpr->eType==FTSQUERY_AND || pExpr->eType==FTSQUERY_NOT - ); switch( pExpr->eType ){ case FTSQUERY_NEAR: { Fts3Expr *pLeft; Fts3Expr *pRight; - int mergetype = isReqPos ? MERGE_POS_NEAR : MERGE_NEAR; - + int mergetype = MERGE_NEAR; if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){ mergetype = MERGE_POS_NEAR; } @@ -109533,7 +110948,7 @@ static int evalFts3Expr( */ char *aBuffer = sqlite3_malloc(nRight+nLeft+1); rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut, - aLeft, nLeft, aRight, nRight + aLeft, nLeft, aRight, nRight, 0 ); *paOut = aBuffer; sqlite3_free(aLeft); @@ -109543,7 +110958,7 @@ static int evalFts3Expr( default: { assert( FTSQUERY_NOT==MERGE_NOT && FTSQUERY_AND==MERGE_AND ); fts3DoclistMerge(pExpr->eType, 0, 0, aLeft, pnOut, - aLeft, nLeft, aRight, nRight + aLeft, nLeft, aRight, nRight, 0 ); *paOut = aLeft; break; @@ -109554,6 +110969,89 @@ static int evalFts3Expr( } } + assert( rc==SQLITE_OK || *paOut==0 ); + return rc; +} + +/* +** This function is called from within xNext() for each row visited by +** an FTS3 query. If evaluating the FTS3 query expression within xFilter() +** was able to determine the exact set of matching rows, this function sets +** *pbRes to true and returns SQLITE_IO immediately. +** +** Otherwise, if evaluating the query expression within xFilter() returned a +** superset of the matching documents instead of an exact set (this happens +** when the query includes very common tokens and it is deemed too expensive to +** load their doclists from disk), this function tests if the current row +** really does match the FTS3 query. +** +** If an error occurs, an SQLite error code is returned. Otherwise, SQLITE_OK +** is returned and *pbRes is set to true if the current row matches the +** FTS3 query (and should be included in the results returned to SQLite), or +** false otherwise. +*/ +static int fts3EvalDeferred( + Fts3Cursor *pCsr, /* FTS3 cursor pointing at row to test */ + int *pbRes /* OUT: Set to true if row is a match */ +){ + int rc = SQLITE_OK; + if( pCsr->pDeferred==0 ){ + *pbRes = 1; + }else{ + rc = fts3CursorSeek(0, pCsr); + if( rc==SQLITE_OK ){ + sqlite3Fts3FreeDeferredDoclists(pCsr); + rc = sqlite3Fts3CacheDeferredDoclists(pCsr); + } + if( rc==SQLITE_OK ){ + char *a = 0; + int n = 0; + rc = fts3EvalExpr(pCsr, pCsr->pExpr, &a, &n, 0); + assert( n>=0 ); + *pbRes = (n>0); + sqlite3_free(a); + } + } + return rc; +} + +/* +** Advance the cursor to the next row in the %_content table that +** matches the search criteria. For a MATCH search, this will be +** the next row that matches. For a full-table scan, this will be +** simply the next row in the %_content table. For a docid lookup, +** this routine simply sets the EOF flag. +** +** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned +** even if we reach end-of-file. The fts3EofMethod() will be called +** subsequently to determine whether or not an EOF was hit. +*/ +static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ + int res; + int rc = SQLITE_OK; /* Return code */ + Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; + + pCsr->eEvalmode = FTS3_EVAL_NEXT; + do { + if( pCsr->aDoclist==0 ){ + if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ + pCsr->isEof = 1; + rc = sqlite3_reset(pCsr->pStmt); + break; + } + pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); + }else{ + if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){ + pCsr->isEof = 1; + break; + } + sqlite3_reset(pCsr->pStmt); + fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId); + pCsr->isRequireSeek = 1; + pCsr->isMatchinfoNeeded = 1; + } + }while( SQLITE_OK==(rc = fts3EvalDeferred(pCsr, &res)) && res==0 ); + return rc; } @@ -109573,11 +111071,6 @@ static int evalFts3Expr( ** number idxNum-FTS3_FULLTEXT_SEARCH, 0 indexed. argv[0] is the right-hand ** side of the MATCH operator. */ -/* TODO(shess) Upgrade the cursor initialization and destruction to -** account for fts3FilterMethod() being called multiple times on the -** same cursor. The current solution is very fragile. Apply fix to -** fts3 as appropriate. -*/ static int fts3FilterMethod( sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ int idxNum, /* Strategy index */ @@ -109600,6 +111093,7 @@ static int fts3FilterMethod( assert( idxNum>=0 && idxNum<=(FTS3_FULLTEXT_SEARCH+p->nColumn) ); assert( nVal==0 || nVal==1 ); assert( (nVal==0)==(idxNum==FTS3_FULLSCAN_SEARCH) ); + assert( p->pSegments==0 ); /* In case the cursor has been used before, clear it now. */ sqlite3_finalize(pCsr->pStmt); @@ -109607,24 +111101,7 @@ static int fts3FilterMethod( sqlite3Fts3ExprFree(pCsr->pExpr); memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); - /* Compile a SELECT statement for this cursor. For a full-table-scan, the - ** statement loops through all rows of the %_content table. For a - ** full-text query or docid lookup, the statement retrieves a single - ** row by docid. - */ - zSql = sqlite3_mprintf(azSql[idxNum==FTS3_FULLSCAN_SEARCH], p->zDb, p->zName); - if( !zSql ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); - sqlite3_free(zSql); - } - if( rc!=SQLITE_OK ) return rc; - pCsr->eSearch = (i16)idxNum; - - if( idxNum==FTS3_DOCID_SEARCH ){ - rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); - }else if( idxNum!=FTS3_FULLSCAN_SEARCH ){ + if( idxNum!=FTS3_DOCID_SEARCH && idxNum!=FTS3_FULLSCAN_SEARCH ){ int iCol = idxNum-FTS3_FULLTEXT_SEARCH; const char *zQuery = (const char *)sqlite3_value_text(apVal[0]); @@ -109646,11 +111123,30 @@ static int fts3FilterMethod( rc = sqlite3Fts3ReadLock(p); if( rc!=SQLITE_OK ) return rc; - rc = evalFts3Expr(p, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist, 0); + rc = fts3EvalExpr(pCsr, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist, 0); + sqlite3Fts3SegmentsClose(p); + if( rc!=SQLITE_OK ) return rc; pCsr->pNextId = pCsr->aDoclist; pCsr->iPrevId = 0; } + /* Compile a SELECT statement for this cursor. For a full-table-scan, the + ** statement loops through all rows of the %_content table. For a + ** full-text query or docid lookup, the statement retrieves a single + ** row by docid. + */ + zSql = sqlite3_mprintf(azSql[idxNum==FTS3_FULLSCAN_SEARCH], p->zDb, p->zName); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + } + if( rc==SQLITE_OK && idxNum==FTS3_DOCID_SEARCH ){ + rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); + } + pCsr->eSearch = (i16)idxNum; + if( rc!=SQLITE_OK ) return rc; return fts3NextMethod(pCursor); } @@ -109674,6 +111170,11 @@ static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ if( pCsr->aDoclist ){ *pRowid = pCsr->iPrevId; }else{ + /* This branch runs if the query is implemented using a full-table scan + ** (not using the full-text index). In this case grab the rowid from the + ** SELECT statement. + */ + assert( pCsr->isRequireSeek==0 ); *pRowid = sqlite3_column_int64(pCsr->pStmt, 0); } return SQLITE_OK; @@ -109736,7 +111237,9 @@ static int fts3UpdateMethod( ** hash-table to the database. */ static int fts3SyncMethod(sqlite3_vtab *pVtab){ - return sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab); + int rc = sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab); + sqlite3Fts3SegmentsClose((Fts3Table *)pVtab); + return rc; } /* @@ -109774,8 +111277,27 @@ static int fts3RollbackMethod(sqlite3_vtab *pVtab){ ** This is used by the matchinfo(), snippet() and offsets() auxillary ** functions. */ -SQLITE_PRIVATE int sqlite3Fts3ExprLoadDoclist(Fts3Table *pTab, Fts3Expr *pExpr){ - return evalFts3Expr(pTab, pExpr, &pExpr->aDoclist, &pExpr->nDoclist, 1); +SQLITE_PRIVATE int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *pCsr, Fts3Expr *pExpr){ + int rc; + assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase ); + assert( pCsr->eEvalmode==FTS3_EVAL_NEXT ); + rc = fts3EvalExpr(pCsr, pExpr, &pExpr->aDoclist, &pExpr->nDoclist, 1); + return rc; +} + +SQLITE_PRIVATE int sqlite3Fts3ExprLoadFtDoclist( + Fts3Cursor *pCsr, + Fts3Expr *pExpr, + char **paDoclist, + int *pnDoclist +){ + int rc; + assert( pCsr->eEvalmode==FTS3_EVAL_NEXT ); + assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase ); + pCsr->eEvalmode = FTS3_EVAL_MATCHINFO; + rc = fts3EvalExpr(pCsr, pExpr, paDoclist, pnDoclist, 1); + pCsr->eEvalmode = FTS3_EVAL_NEXT; + return rc; } /* @@ -109841,7 +111363,7 @@ static int fts3FunctionArg( sqlite3_context *pContext, /* SQL function call context */ const char *zFunc, /* Function name */ sqlite3_value *pVal, /* argv[0] passed to function */ - Fts3Cursor **ppCsr /* OUT: Store cursor handle here */ + Fts3Cursor **ppCsr /* OUT: Store cursor handle here */ ){ Fts3Cursor *pRet; if( sqlite3_value_type(pVal)!=SQLITE_BLOB @@ -109967,15 +111489,13 @@ static void fts3MatchinfoFunc( sqlite3_value **apVal /* Array of arguments */ ){ Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ - - if( nVal!=1 ){ - sqlite3_result_error(pContext, - "wrong number of arguments to function matchinfo()", -1); - return; - } - + assert( nVal==1 || nVal==2 ); if( SQLITE_OK==fts3FunctionArg(pContext, "matchinfo", apVal[0], &pCsr) ){ - sqlite3Fts3Matchinfo(pContext, pCsr); + const char *zArg = 0; + if( nVal>1 ){ + zArg = (const char *)sqlite3_value_text(apVal[1]); + } + sqlite3Fts3Matchinfo(pContext, pCsr, zArg); } } @@ -110036,12 +111556,13 @@ static int fts3RenameMethod( "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';", p->zDb, p->zName, zName ); - if( rc==SQLITE_ERROR ) rc = SQLITE_OK; if( p->bHasDocsize ){ fts3DbExec(&rc, db, "ALTER TABLE %Q.'%q_docsize' RENAME TO '%q_docsize';", p->zDb, p->zName, zName ); + } + if( p->bHasStat ){ fts3DbExec(&rc, db, "ALTER TABLE %Q.'%q_stat' RENAME TO '%q_stat';", p->zDb, p->zName, zName @@ -110066,7 +111587,7 @@ static const sqlite3_module fts3Module = { /* xDisconnect */ fts3DisconnectMethod, /* xDestroy */ fts3DestroyMethod, /* xOpen */ fts3OpenMethod, - /* xClose */ fulltextClose, + /* xClose */ fts3CloseMethod, /* xFilter */ fts3FilterMethod, /* xNext */ fts3NextMethod, /* xEof */ fts3EofMethod, @@ -110093,19 +111614,20 @@ static void hashDestroy(void *p){ } /* -** The fts3 built-in tokenizers - "simple" and "porter" - are implemented -** in files fts3_tokenizer1.c and fts3_porter.c respectively. The following -** two forward declarations are for functions declared in these files -** used to retrieve the respective implementations. +** The fts3 built-in tokenizers - "simple", "porter" and "icu"- are +** implemented in files fts3_tokenizer1.c, fts3_porter.c and fts3_icu.c +** respectively. The following three forward declarations are for functions +** declared in these files used to retrieve the respective implementations. ** ** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed ** to by the argument to point to the "simple" tokenizer implementation. -** Function ...PorterTokenizerModule() sets *pModule to point to the -** porter tokenizer/stemmer implementation. +** And so on. */ SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule); +#ifdef SQLITE_ENABLE_ICU SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule); +#endif /* ** Initialise the fts3 extension. If this extension is built as part @@ -110161,7 +111683,8 @@ SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer")) && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1)) && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1)) - && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", -1)) + && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 1)) + && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 2)) && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1)) ){ rc = sqlite3_create_module_v2( @@ -110304,6 +111827,18 @@ static int fts3isspace(char c){ return c==' ' || c=='\t' || c=='\n' || c=='\r' || c=='\v' || c=='\f'; } +/* +** Allocate nByte bytes of memory using sqlite3_malloc(). If successful, +** zero the memory before returning a pointer to it. If unsuccessful, +** return NULL. +*/ +static void *fts3MallocZero(int nByte){ + void *pRet = sqlite3_malloc(nByte); + if( pRet ) memset(pRet, 0, nByte); + return pRet; +} + + /* ** Extract the next token from buffer z (length n) using the tokenizer ** and other information (column names etc.) in pParse. Create an Fts3Expr @@ -110341,11 +111876,10 @@ static int getNextToken( if( rc==SQLITE_OK ){ nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken; - pRet = (Fts3Expr *)sqlite3_malloc(nByte); + pRet = (Fts3Expr *)fts3MallocZero(nByte); if( !pRet ){ rc = SQLITE_NOMEM; }else{ - memset(pRet, 0, nByte); pRet->eType = FTSQUERY_PHRASE; pRet->pPhrase = (Fts3Phrase *)&pRet[1]; pRet->pPhrase->nToken = 1; @@ -110421,7 +111955,7 @@ static int getNextString( rc = pModule->xNext(pCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos); if( rc==SQLITE_OK ){ int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase); - p = fts3ReallocOrFree(p, nByte+ii*sizeof(struct PhraseToken)); + p = fts3ReallocOrFree(p, nByte+ii*sizeof(Fts3PhraseToken)); zTemp = fts3ReallocOrFree(zTemp, nTemp + nToken); if( !p || !zTemp ){ goto no_mem; @@ -110431,6 +111965,7 @@ static int getNextString( p->pPhrase = (Fts3Phrase *)&p[1]; } p->pPhrase = (Fts3Phrase *)&p[1]; + memset(&p->pPhrase->aToken[ii], 0, sizeof(Fts3PhraseToken)); p->pPhrase->nToken = ii+1; p->pPhrase->aToken[ii].n = nToken; memcpy(&zTemp[nTemp], zToken, nToken); @@ -110452,7 +111987,7 @@ static int getNextString( char *zNew = NULL; int nNew = 0; int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase); - nByte += (p?(p->pPhrase->nToken-1):0) * sizeof(struct PhraseToken); + nByte += (p?(p->pPhrase->nToken-1):0) * sizeof(Fts3PhraseToken); p = fts3ReallocOrFree(p, nByte + nTemp); if( !p ){ goto no_mem; @@ -110570,11 +112105,10 @@ static int getNextNode( if( fts3isspace(cNext) || cNext=='"' || cNext=='(' || cNext==')' || cNext==0 ){ - pRet = (Fts3Expr *)sqlite3_malloc(sizeof(Fts3Expr)); + pRet = (Fts3Expr *)fts3MallocZero(sizeof(Fts3Expr)); if( !pRet ){ return SQLITE_NOMEM; } - memset(pRet, 0, sizeof(Fts3Expr)); pRet->eType = pKey->eType; pRet->nNear = nNear; *ppExpr = pRet; @@ -110592,7 +112126,6 @@ static int getNextNode( if( sqlite3_fts3_enable_parentheses ){ if( *zInput=='(' ){ int nConsumed; - int rc; pParse->nNest++; rc = fts3ExprParse(pParse, &zInput[1], nInput-1, ppExpr, &nConsumed); if( rc==SQLITE_OK && !*ppExpr ){ @@ -110750,13 +112283,12 @@ static int fts3ExprParse( && p->eType==FTSQUERY_PHRASE && p->pPhrase->isNot ){ /* Create an implicit NOT operator. */ - Fts3Expr *pNot = sqlite3_malloc(sizeof(Fts3Expr)); + Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr)); if( !pNot ){ sqlite3Fts3ExprFree(p); rc = SQLITE_NOMEM; goto exprparse_out; } - memset(pNot, 0, sizeof(Fts3Expr)); pNot->eType = FTSQUERY_NOT; pNot->pRight = p; if( pNotBranch ){ @@ -110784,13 +112316,12 @@ static int fts3ExprParse( /* Insert an implicit AND operator. */ Fts3Expr *pAnd; assert( pRet && pPrev ); - pAnd = sqlite3_malloc(sizeof(Fts3Expr)); + pAnd = fts3MallocZero(sizeof(Fts3Expr)); if( !pAnd ){ sqlite3Fts3ExprFree(p); rc = SQLITE_NOMEM; goto exprparse_out; } - memset(pAnd, 0, sizeof(Fts3Expr)); pAnd->eType = FTSQUERY_AND; insertBinaryOperator(&pRet, pPrev, pAnd); pPrev = pAnd; @@ -110974,47 +112505,53 @@ static int queryTestTokenizer( } /* -** This function is part of the test interface for the query parser. It -** writes a text representation of the query expression pExpr into the -** buffer pointed to by argument zBuf. It is assumed that zBuf is large -** enough to store the required text representation. +** Return a pointer to a buffer containing a text representation of the +** expression passed as the first argument. The buffer is obtained from +** sqlite3_malloc(). It is the responsibility of the caller to use +** sqlite3_free() to release the memory. If an OOM condition is encountered, +** NULL is returned. +** +** If the second argument is not NULL, then its contents are prepended to +** the returned expression text and then freed using sqlite3_free(). */ -static void exprToString(Fts3Expr *pExpr, char *zBuf){ +static char *exprToString(Fts3Expr *pExpr, char *zBuf){ switch( pExpr->eType ){ case FTSQUERY_PHRASE: { Fts3Phrase *pPhrase = pExpr->pPhrase; int i; - zBuf += sprintf(zBuf, "PHRASE %d %d", pPhrase->iColumn, pPhrase->isNot); - for(i=0; inToken; i++){ - zBuf += sprintf(zBuf," %.*s",pPhrase->aToken[i].n,pPhrase->aToken[i].z); - zBuf += sprintf(zBuf,"%s", (pPhrase->aToken[i].isPrefix?"+":"")); + zBuf = sqlite3_mprintf( + "%zPHRASE %d %d", zBuf, pPhrase->iColumn, pPhrase->isNot); + for(i=0; zBuf && inToken; i++){ + zBuf = sqlite3_mprintf("%z %.*s%s", zBuf, + pPhrase->aToken[i].n, pPhrase->aToken[i].z, + (pPhrase->aToken[i].isPrefix?"+":"") + ); } - return; + return zBuf; } case FTSQUERY_NEAR: - zBuf += sprintf(zBuf, "NEAR/%d ", pExpr->nNear); + zBuf = sqlite3_mprintf("%zNEAR/%d ", zBuf, pExpr->nNear); break; case FTSQUERY_NOT: - zBuf += sprintf(zBuf, "NOT "); + zBuf = sqlite3_mprintf("%zNOT ", zBuf); break; case FTSQUERY_AND: - zBuf += sprintf(zBuf, "AND "); + zBuf = sqlite3_mprintf("%zAND ", zBuf); break; case FTSQUERY_OR: - zBuf += sprintf(zBuf, "OR "); + zBuf = sqlite3_mprintf("%zOR ", zBuf); break; } - zBuf += sprintf(zBuf, "{"); - exprToString(pExpr->pLeft, zBuf); - zBuf += strlen(zBuf); - zBuf += sprintf(zBuf, "} "); + if( zBuf ) zBuf = sqlite3_mprintf("%z{", zBuf); + if( zBuf ) zBuf = exprToString(pExpr->pLeft, zBuf); + if( zBuf ) zBuf = sqlite3_mprintf("%z} {", zBuf); + + if( zBuf ) zBuf = exprToString(pExpr->pRight, zBuf); + if( zBuf ) zBuf = sqlite3_mprintf("%z}", zBuf); - zBuf += sprintf(zBuf, "{"); - exprToString(pExpr->pRight, zBuf); - zBuf += strlen(zBuf); - zBuf += sprintf(zBuf, "}"); + return zBuf; } /* @@ -111045,6 +112582,7 @@ static void fts3ExprTest( int nCol; int ii; Fts3Expr *pExpr; + char *zBuf = 0; sqlite3 *db = sqlite3_context_db_handle(context); if( argc<3 ){ @@ -111087,18 +112625,17 @@ static void fts3ExprTest( rc = sqlite3Fts3ExprParse( pTokenizer, azCol, nCol, nCol, zExpr, nExpr, &pExpr ); - if( rc==SQLITE_NOMEM ){ + if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){ + sqlite3_result_error(context, "Error parsing expression", -1); + }else if( rc==SQLITE_NOMEM || !(zBuf = exprToString(pExpr, 0)) ){ sqlite3_result_error_nomem(context); - goto exprtest_out; - }else if( rc==SQLITE_OK ){ - char zBuf[4096]; - exprToString(pExpr, zBuf); - sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT); - sqlite3Fts3ExprFree(pExpr); }else{ - sqlite3_result_error(context, "Error parsing expression", -1); + sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT); + sqlite3_free(zBuf); } + sqlite3Fts3ExprFree(pExpr); + exprtest_out: if( pModule && pTokenizer ){ rc = pModule->xDestroy(pTokenizer); @@ -111841,7 +113378,7 @@ static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){ int i, j; char zReverse[28]; char *z, *z2; - if( nIn<3 || nIn>=sizeof(zReverse)-7 ){ + if( nIn<3 || nIn>=(int)sizeof(zReverse)-7 ){ /* The word is too big or too small for the porter stemmer. ** Fallback to the copy stemmer */ copy_stemmer(zIn, nIn, zOut, pnOut); @@ -112240,7 +113777,7 @@ static void scalarFunc( sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT); } -static int fts3IsIdChar(char c){ +SQLITE_PRIVATE int sqlite3Fts3IsIdChar(char c){ static const char isFtsIdChar[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */ @@ -112278,9 +113815,9 @@ SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *zStr, int *pn){ break; default: - if( fts3IsIdChar(*z1) ){ + if( sqlite3Fts3IsIdChar(*z1) ){ z2 = &z1[1]; - while( fts3IsIdChar(*z2) ) z2++; + while( sqlite3Fts3IsIdChar(*z2) ) z2++; }else{ z1++; } @@ -112293,9 +113830,8 @@ SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *zStr, int *pn){ SQLITE_PRIVATE int sqlite3Fts3InitTokenizer( Fts3Hash *pHash, /* Tokenizer hash table */ - const char *zArg, /* Possible tokenizer specification */ + const char *zArg, /* Tokenizer name */ sqlite3_tokenizer **ppTok, /* OUT: Tokenizer (if applicable) */ - const char **pzTokenizer, /* OUT: Set to zArg if is tokenizer */ char **pzErr /* OUT: Set to malloced error message */ ){ int rc; @@ -112305,26 +113841,15 @@ SQLITE_PRIVATE int sqlite3Fts3InitTokenizer( char *zEnd; /* Pointer to nul-term of zCopy */ sqlite3_tokenizer_module *m; - if( !z ){ - zCopy = sqlite3_mprintf("simple"); - }else{ - if( sqlite3_strnicmp(z, "tokenize", 8) || fts3IsIdChar(z[8])){ - return SQLITE_OK; - } - zCopy = sqlite3_mprintf("%s", &z[8]); - *pzTokenizer = zArg; - } - if( !zCopy ){ - return SQLITE_NOMEM; - } - + zCopy = sqlite3_mprintf("%s", zArg); + if( !zCopy ) return SQLITE_NOMEM; zEnd = &zCopy[strlen(zCopy)]; z = (char *)sqlite3Fts3NextToken(zCopy, &n); z[n] = '\0'; sqlite3Fts3Dequote(z); - m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, z, (int)strlen(z)+1); + m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash,z,(int)strlen(z)+1); if( !m ){ *pzErr = sqlite3_mprintf("unknown tokenizer: %s", z); rc = SQLITE_ERROR; @@ -112618,15 +114143,23 @@ SQLITE_PRIVATE int sqlite3Fts3InitHashTable( } #endif - if( SQLITE_OK!=rc - || SQLITE_OK!=(rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0)) - || SQLITE_OK!=(rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0)) + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0); + } + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0); + } #ifdef SQLITE_TEST - || SQLITE_OK!=(rc = sqlite3_create_function(db, zTest, 2, any, p, testFunc, 0, 0)) - || SQLITE_OK!=(rc = sqlite3_create_function(db, zTest, 3, any, p, testFunc, 0, 0)) - || SQLITE_OK!=(rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0)) + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zTest, 2, any, p, testFunc, 0, 0); + } + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zTest, 3, any, p, testFunc, 0, 0); + } + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0); + } #endif - ); #ifdef SQLITE_TEST sqlite3_free(zTest); @@ -112893,6 +114426,18 @@ SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule( #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) +/* +** When full-text index nodes are loaded from disk, the buffer that they +** are loaded into has the following number of bytes of padding at the end +** of it. i.e. if a full-text index node is 900 bytes in size, then a buffer +** of 920 bytes is allocated for it. +** +** This means that if we have a pointer into a buffer containing node data, +** it is always safe to read up to two varints from it without risking an +** overread, even if the node data is corrupted. +*/ +#define FTS3_NODE_PADDING (FTS3_VARINT_MAX*2) + typedef struct PendingList PendingList; typedef struct SegmentNode SegmentNode; typedef struct SegmentWriter SegmentWriter; @@ -112911,6 +114456,17 @@ struct PendingList { sqlite3_int64 iLastPos; }; + +/* +** Each cursor has a (possibly empty) linked list of the following objects. +*/ +struct Fts3DeferredToken { + Fts3PhraseToken *pToken; /* Pointer to corresponding expr token */ + int iCol; /* Column token must occur in */ + Fts3DeferredToken *pNext; /* Next in list of deferred tokens */ + PendingList *pList; /* Doclist is assembled here */ +}; + /* ** An instance of this structure is used to iterate through the terms on ** a contiguous set of segment b-tree leaf nodes. Although the details of @@ -112920,6 +114476,7 @@ struct PendingList { ** ** sqlite3Fts3SegReaderNew() ** sqlite3Fts3SegReaderFree() +** sqlite3Fts3SegReaderCost() ** sqlite3Fts3SegReaderIterate() ** ** Methods used to manipulate Fts3SegReader structures: @@ -112930,12 +114487,14 @@ struct PendingList { */ struct Fts3SegReader { int iIdx; /* Index within level, or 0x7FFFFFFF for PT */ - sqlite3_int64 iStartBlock; - sqlite3_int64 iEndBlock; - sqlite3_stmt *pStmt; /* SQL Statement to access leaf nodes */ + + sqlite3_int64 iStartBlock; /* Rowid of first leaf block to traverse */ + sqlite3_int64 iLeafEndBlock; /* Rowid of final leaf block to traverse */ + sqlite3_int64 iEndBlock; /* Rowid of final block in segment (or 0) */ + sqlite3_int64 iCurrentBlock; /* Current leaf block (or 0) */ + char *aNode; /* Pointer to node data (or NULL) */ int nNode; /* Size of buffer at aNode (or 0) */ - int nTermAlloc; /* Allocated size of zTerm buffer */ Fts3HashElem **ppNextElem; /* Variables set by fts3SegReaderNext(). These may be read directly @@ -112945,6 +114504,7 @@ struct Fts3SegReader { */ int nTerm; /* Number of bytes in current term */ char *zTerm; /* Pointer to current term */ + int nTermAlloc; /* Allocated size of zTerm buffer */ char *aDoclist; /* Pointer to doclist of current entry */ int nDoclist; /* Size of doclist in current entry */ @@ -112954,6 +114514,7 @@ struct Fts3SegReader { }; #define fts3SegReaderIsPending(p) ((p)->ppNextElem!=0) +#define fts3SegReaderIsRootOnly(p) ((p)->aNode==(char *)&(p)[1]) /* ** An instance of this structure is used to create a segment b-tree in the @@ -113022,12 +114583,11 @@ struct SegmentNode { #define SQL_DELETE_SEGDIR_BY_LEVEL 16 #define SQL_DELETE_SEGMENTS_RANGE 17 #define SQL_CONTENT_INSERT 18 -#define SQL_GET_BLOCK 19 -#define SQL_DELETE_DOCSIZE 20 -#define SQL_REPLACE_DOCSIZE 21 -#define SQL_SELECT_DOCSIZE 22 -#define SQL_SELECT_DOCTOTAL 23 -#define SQL_REPLACE_DOCTOTAL 24 +#define SQL_DELETE_DOCSIZE 19 +#define SQL_REPLACE_DOCSIZE 20 +#define SQL_SELECT_DOCSIZE 21 +#define SQL_SELECT_DOCTOTAL 22 +#define SQL_REPLACE_DOCTOTAL 23 /* ** This function is used to obtain an SQLite prepared statement handle @@ -113072,12 +114632,11 @@ static int fts3SqlStmt( /* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", /* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", /* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%z)", -/* 19 */ "SELECT block FROM %Q.'%q_segments' WHERE blockid = ?", -/* 20 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?", -/* 21 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", -/* 22 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", -/* 23 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0", -/* 24 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)", +/* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?", +/* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", +/* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", +/* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0", +/* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)", }; int rc = SQLITE_OK; sqlite3_stmt *pStmt; @@ -113126,6 +114685,51 @@ static int fts3SqlStmt( return rc; } +static int fts3SelectDocsize( + Fts3Table *pTab, /* FTS3 table handle */ + int eStmt, /* Either SQL_SELECT_DOCSIZE or DOCTOTAL */ + sqlite3_int64 iDocid, /* Docid to bind for SQL_SELECT_DOCSIZE */ + sqlite3_stmt **ppStmt /* OUT: Statement handle */ +){ + sqlite3_stmt *pStmt = 0; /* Statement requested from fts3SqlStmt() */ + int rc; /* Return code */ + + assert( eStmt==SQL_SELECT_DOCSIZE || eStmt==SQL_SELECT_DOCTOTAL ); + + rc = fts3SqlStmt(pTab, eStmt, &pStmt, 0); + if( rc==SQLITE_OK ){ + if( eStmt==SQL_SELECT_DOCSIZE ){ + sqlite3_bind_int64(pStmt, 1, iDocid); + } + rc = sqlite3_step(pStmt); + if( rc!=SQLITE_ROW ){ + rc = sqlite3_reset(pStmt); + if( rc==SQLITE_OK ) rc = SQLITE_CORRUPT; + pStmt = 0; + }else{ + rc = SQLITE_OK; + } + } + + *ppStmt = pStmt; + return rc; +} + +SQLITE_PRIVATE int sqlite3Fts3SelectDoctotal( + Fts3Table *pTab, /* Fts3 table handle */ + sqlite3_stmt **ppStmt /* OUT: Statement handle */ +){ + return fts3SelectDocsize(pTab, SQL_SELECT_DOCTOTAL, 0, ppStmt); +} + +SQLITE_PRIVATE int sqlite3Fts3SelectDocsize( + Fts3Table *pTab, /* Fts3 table handle */ + sqlite3_int64 iDocid, /* Docid to read size data for */ + sqlite3_stmt **ppStmt /* OUT: Statement handle */ +){ + return fts3SelectDocsize(pTab, SQL_SELECT_DOCSIZE, iDocid, ppStmt); +} + /* ** Similar to fts3SqlStmt(). Except, after binding the parameters in ** array apVal[] to the SQL statement identified by eStmt, the statement @@ -113152,45 +114756,6 @@ static void fts3SqlExec( } -/* -** Read a single block from the %_segments table. If the specified block -** does not exist, return SQLITE_CORRUPT. If some other error (malloc, IO -** etc.) occurs, return the appropriate SQLite error code. -** -** Otherwise, if successful, set *pzBlock to point to a buffer containing -** the block read from the database, and *pnBlock to the size of the read -** block in bytes. -** -** WARNING: The returned buffer is only valid until the next call to -** sqlite3Fts3ReadBlock(). -*/ -SQLITE_PRIVATE int sqlite3Fts3ReadBlock( - Fts3Table *p, - sqlite3_int64 iBlock, - char const **pzBlock, - int *pnBlock -){ - sqlite3_stmt *pStmt; - int rc = fts3SqlStmt(p, SQL_GET_BLOCK, &pStmt, 0); - if( rc!=SQLITE_OK ) return rc; - sqlite3_reset(pStmt); - - if( pzBlock ){ - sqlite3_bind_int64(pStmt, 1, iBlock); - rc = sqlite3_step(pStmt); - if( rc!=SQLITE_ROW ){ - return (rc==SQLITE_DONE ? SQLITE_CORRUPT : rc); - } - - *pnBlock = sqlite3_column_bytes(pStmt, 0); - *pzBlock = (char *)sqlite3_column_blob(pStmt, 0); - if( sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){ - return SQLITE_CORRUPT; - } - } - return SQLITE_OK; -} - /* ** This function ensures that the caller has obtained a shared-cache ** table-lock on the %_content table. This is required before reading @@ -113359,10 +114924,10 @@ static int fts3PendingListAppend( ** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. */ static int fts3PendingTermsAdd( - Fts3Table *p, /* FTS table into which text will be inserted */ - const char *zText, /* Text of document to be inseted */ - int iCol, /* Column number into which text is inserted */ - u32 *pnWord /* OUT: Number of tokens inserted */ + Fts3Table *p, /* Table into which text will be inserted */ + const char *zText, /* Text of document to be inserted */ + int iCol, /* Column into which text is being inserted */ + u32 *pnWord /* OUT: Number of tokens inserted */ ){ int rc; int iStart; @@ -113447,6 +115012,9 @@ static int fts3PendingTermsDocid(Fts3Table *p, sqlite_int64 iDocid){ return SQLITE_OK; } +/* +** Discard the contents of the pending-terms hash table. +*/ SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *p){ Fts3HashElem *pElem; for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){ @@ -113474,6 +115042,7 @@ static int fts3InsertTerms(Fts3Table *p, sqlite3_value **apVal, u32 *aSz){ return rc; } } + aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]); } return SQLITE_OK; } @@ -113561,6 +115130,8 @@ static int fts3DeleteAll(Fts3Table *p){ fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0); if( p->bHasDocsize ){ fts3SqlExec(&rc, p, SQL_DELETE_ALL_DOCSIZE, 0); + } + if( p->bHasStat ){ fts3SqlExec(&rc, p, SQL_DELETE_ALL_STAT, 0); } return rc; @@ -113571,7 +115142,7 @@ static int fts3DeleteAll(Fts3Table *p){ ** (an integer) of a row about to be deleted. Remove all terms from the ** full-text index. */ -static void fts3DeleteTerms( +static void fts3DeleteTerms( int *pRC, /* Result code */ Fts3Table *p, /* The FTS table to delete from */ sqlite3_value **apVal, /* apVal[] contains the docid to be deleted */ @@ -113593,6 +115164,7 @@ static void fts3DeleteTerms( *pRC = rc; return; } + aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i); } } rc = sqlite3_reset(pSelect); @@ -113655,12 +115227,93 @@ static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){ return rc; } +/* +** The %_segments table is declared as follows: +** +** CREATE TABLE %_segments(blockid INTEGER PRIMARY KEY, block BLOB) +** +** This function reads data from a single row of the %_segments table. The +** specific row is identified by the iBlockid parameter. If paBlob is not +** NULL, then a buffer is allocated using sqlite3_malloc() and populated +** with the contents of the blob stored in the "block" column of the +** identified table row is. Whether or not paBlob is NULL, *pnBlob is set +** to the size of the blob in bytes before returning. +** +** If an error occurs, or the table does not contain the specified row, +** an SQLite error code is returned. Otherwise, SQLITE_OK is returned. If +** paBlob is non-NULL, then it is the responsibility of the caller to +** eventually free the returned buffer. +** +** This function may leave an open sqlite3_blob* handle in the +** Fts3Table.pSegments variable. This handle is reused by subsequent calls +** to this function. The handle may be closed by calling the +** sqlite3Fts3SegmentsClose() function. Reusing a blob handle is a handy +** performance improvement, but the blob handle should always be closed +** before control is returned to the user (to prevent a lock being held +** on the database file for longer than necessary). Thus, any virtual table +** method (xFilter etc.) that may directly or indirectly call this function +** must call sqlite3Fts3SegmentsClose() before returning. +*/ +SQLITE_PRIVATE int sqlite3Fts3ReadBlock( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iBlockid, /* Access the row with blockid=$iBlockid */ + char **paBlob, /* OUT: Blob data in malloc'd buffer */ + int *pnBlob /* OUT: Size of blob data */ +){ + int rc; /* Return code */ + + /* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */ + assert( pnBlob); + + if( p->pSegments ){ + rc = sqlite3_blob_reopen(p->pSegments, iBlockid); + }else{ + if( 0==p->zSegmentsTbl ){ + p->zSegmentsTbl = sqlite3_mprintf("%s_segments", p->zName); + if( 0==p->zSegmentsTbl ) return SQLITE_NOMEM; + } + rc = sqlite3_blob_open( + p->db, p->zDb, p->zSegmentsTbl, "block", iBlockid, 0, &p->pSegments + ); + } + + if( rc==SQLITE_OK ){ + int nByte = sqlite3_blob_bytes(p->pSegments); + if( paBlob ){ + char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING); + if( !aByte ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0); + memset(&aByte[nByte], 0, FTS3_NODE_PADDING); + if( rc!=SQLITE_OK ){ + sqlite3_free(aByte); + aByte = 0; + } + } + *paBlob = aByte; + } + *pnBlob = nByte; + } + + return rc; +} + +/* +** Close the blob handle at p->pSegments, if it is open. See comments above +** the sqlite3Fts3ReadBlock() function for details. +*/ +SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *p){ + sqlite3_blob_close(p->pSegments); + p->pSegments = 0; +} + /* ** Move the iterator passed as the first argument to the next term in the ** segment. If successful, SQLITE_OK is returned. If there is no next term, ** SQLITE_DONE. Otherwise, an SQLite error code. */ -static int fts3SegReaderNext(Fts3SegReader *pReader){ +static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ char *pNext; /* Cursor variable */ int nPrefix; /* Number of bytes in term prefix */ int nSuffix; /* Number of bytes in term suffix */ @@ -113672,7 +115325,8 @@ static int fts3SegReaderNext(Fts3SegReader *pReader){ } if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){ - int rc; + int rc; /* Return code from Fts3ReadBlock() */ + if( fts3SegReaderIsPending(pReader) ){ Fts3HashElem *pElem = *(pReader->ppNextElem); if( pElem==0 ){ @@ -113688,22 +115342,36 @@ static int fts3SegReaderNext(Fts3SegReader *pReader){ } return SQLITE_OK; } - if( !pReader->pStmt ){ - pReader->aNode = 0; - return SQLITE_OK; + + if( !fts3SegReaderIsRootOnly(pReader) ){ + sqlite3_free(pReader->aNode); } - rc = sqlite3_step(pReader->pStmt); - if( rc!=SQLITE_ROW ){ - pReader->aNode = 0; - return (rc==SQLITE_DONE ? SQLITE_OK : rc); + pReader->aNode = 0; + + /* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf + ** blocks have already been traversed. */ + assert( pReader->iCurrentBlock<=pReader->iLeafEndBlock ); + if( pReader->iCurrentBlock>=pReader->iLeafEndBlock ){ + return SQLITE_OK; } - pReader->nNode = sqlite3_column_bytes(pReader->pStmt, 0); - pReader->aNode = (char *)sqlite3_column_blob(pReader->pStmt, 0); + + rc = sqlite3Fts3ReadBlock( + p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode + ); + if( rc!=SQLITE_OK ) return rc; pNext = pReader->aNode; } + /* Because of the FTS3_NODE_PADDING bytes of padding, the following is + ** safe (no risk of overread) even if the node data is corrupted. + */ pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix); pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix); + if( nPrefix<0 || nSuffix<=0 + || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] + ){ + return SQLITE_CORRUPT; + } if( nPrefix+nSuffix>pReader->nTermAlloc ){ int nNew = (nPrefix+nSuffix)*2; @@ -113718,9 +115386,18 @@ static int fts3SegReaderNext(Fts3SegReader *pReader){ pReader->nTerm = nPrefix+nSuffix; pNext += nSuffix; pNext += sqlite3Fts3GetVarint32(pNext, &pReader->nDoclist); - assert( pNext<&pReader->aNode[pReader->nNode] ); pReader->aDoclist = pNext; pReader->pOffsetList = 0; + + /* Check that the doclist does not appear to extend past the end of the + ** b-tree node. And that the final byte of the doclist is 0x00. If either + ** of these statements is untrue, then the data structure is corrupt. + */ + if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] + || pReader->aDoclist[pReader->nDoclist-1] + ){ + return SQLITE_CORRUPT; + } return SQLITE_OK; } @@ -113784,31 +115461,105 @@ static void fts3SegReaderNextDocid( } /* -** Free all allocations associated with the iterator passed as the -** second argument. +** This function is called to estimate the amount of data that will be +** loaded from the disk If SegReaderIterate() is called on this seg-reader, +** in units of average document size. +** +** This can be used as follows: If the caller has a small doclist that +** contains references to N documents, and is considering merging it with +** a large doclist (size X "average documents"), it may opt not to load +** the large doclist if X>N. */ -SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3Table *p, Fts3SegReader *pReader){ - if( pReader ){ - if( pReader->pStmt ){ - /* Move the leaf-range SELECT statement to the aLeavesStmt[] array, - ** so that it can be reused when required by another query. +SQLITE_PRIVATE int sqlite3Fts3SegReaderCost( + Fts3Cursor *pCsr, /* FTS3 cursor handle */ + Fts3SegReader *pReader, /* Segment-reader handle */ + int *pnCost /* IN/OUT: Number of bytes read */ +){ + Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; + int rc = SQLITE_OK; /* Return code */ + int nCost = 0; /* Cost in bytes to return */ + int pgsz = p->nPgsz; /* Database page size */ + + /* If this seg-reader is reading the pending-terms table, or if all data + ** for the segment is stored on the root page of the b-tree, then the cost + ** is zero. In this case all required data is already in main memory. + */ + if( p->bHasStat + && !fts3SegReaderIsPending(pReader) + && !fts3SegReaderIsRootOnly(pReader) + ){ + int nBlob = 0; + sqlite3_int64 iBlock; + + if( pCsr->nRowAvg==0 ){ + /* The average document size, which is required to calculate the cost + ** of each doclist, has not yet been determined. Read the required + ** data from the %_stat table to calculate it. + ** + ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 + ** varints, where nCol is the number of columns in the FTS3 table. + ** The first varint is the number of documents currently stored in + ** the table. The following nCol varints contain the total amount of + ** data stored in all rows of each column of the table, from left + ** to right. */ - assert( p->nLeavesStmtnLeavesTotal ); - sqlite3_reset(pReader->pStmt); - p->aLeavesStmt[p->nLeavesStmt++] = pReader->pStmt; + sqlite3_stmt *pStmt; + rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0); + if( rc ) return rc; + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + sqlite3_int64 nDoc = 0; + sqlite3_int64 nByte = 0; + const char *a = sqlite3_column_blob(pStmt, 0); + if( a ){ + const char *pEnd = &a[sqlite3_column_bytes(pStmt, 0)]; + a += sqlite3Fts3GetVarint(a, &nDoc); + while( anRowAvg = (int)(((nByte / nDoc) + pgsz - 1) / pgsz); + } + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK || pCsr->nRowAvg==0 ) return rc; + } + + /* Assume that a blob flows over onto overflow pages if it is larger + ** than (pgsz-35) bytes in size (the file-format documentation + ** confirms this). + */ + for(iBlock=pReader->iStartBlock; iBlock<=pReader->iLeafEndBlock; iBlock++){ + rc = sqlite3Fts3ReadBlock(p, iBlock, 0, &nBlob); + if( rc!=SQLITE_OK ) break; + if( (nBlob+35)>pgsz ){ + int nOvfl = (nBlob + 34)/pgsz; + nCost += ((nOvfl + pCsr->nRowAvg - 1)/pCsr->nRowAvg); + } } - if( !fts3SegReaderIsPending(pReader) ){ - sqlite3_free(pReader->zTerm); + } + + *pnCost += nCost; + return rc; +} + +/* +** Free all allocations associated with the iterator passed as the +** second argument. +*/ +SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){ + if( pReader && !fts3SegReaderIsPending(pReader) ){ + sqlite3_free(pReader->zTerm); + if( !fts3SegReaderIsRootOnly(pReader) ){ + sqlite3_free(pReader->aNode); } - sqlite3_free(pReader); } + sqlite3_free(pReader); } /* ** Allocate a new SegReader object. */ SQLITE_PRIVATE int sqlite3Fts3SegReaderNew( - Fts3Table *p, /* Virtual table handle */ int iAge, /* Segment "age". */ sqlite3_int64 iStartLeaf, /* First leaf to traverse */ sqlite3_int64 iEndLeaf, /* Final leaf to traverse */ @@ -113821,8 +115572,9 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderNew( Fts3SegReader *pReader; /* Newly allocated SegReader object */ int nExtra = 0; /* Bytes to allocate segment root node */ + assert( iStartLeaf<=iEndLeaf ); if( iStartLeaf==0 ){ - nExtra = nRoot; + nExtra = nRoot + FTS3_NODE_PADDING; } pReader = (Fts3SegReader *)sqlite3_malloc(sizeof(Fts3SegReader) + nExtra); @@ -113830,8 +115582,9 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderNew( return SQLITE_NOMEM; } memset(pReader, 0, sizeof(Fts3SegReader)); - pReader->iStartBlock = iStartLeaf; pReader->iIdx = iAge; + pReader->iStartBlock = iStartLeaf; + pReader->iLeafEndBlock = iEndLeaf; pReader->iEndBlock = iEndBlock; if( nExtra ){ @@ -113839,59 +115592,15 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderNew( pReader->aNode = (char *)&pReader[1]; pReader->nNode = nRoot; memcpy(pReader->aNode, zRoot, nRoot); + memset(&pReader->aNode[nRoot], 0, FTS3_NODE_PADDING); }else{ - /* If the text of the SQL statement to iterate through a contiguous - ** set of entries in the %_segments table has not yet been composed, - ** compose it now. - */ - if( !p->zSelectLeaves ){ - p->zSelectLeaves = sqlite3_mprintf( - "SELECT block FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ? " - "ORDER BY blockid", p->zDb, p->zName - ); - if( !p->zSelectLeaves ){ - rc = SQLITE_NOMEM; - goto finished; - } - } - - /* If there are no free statements in the aLeavesStmt[] array, prepare - ** a new statement now. Otherwise, reuse a prepared statement from - ** aLeavesStmt[]. - */ - if( p->nLeavesStmt==0 ){ - if( p->nLeavesTotal==p->nLeavesAlloc ){ - int nNew = p->nLeavesAlloc + 16; - sqlite3_stmt **aNew = (sqlite3_stmt **)sqlite3_realloc( - p->aLeavesStmt, nNew*sizeof(sqlite3_stmt *) - ); - if( !aNew ){ - rc = SQLITE_NOMEM; - goto finished; - } - p->nLeavesAlloc = nNew; - p->aLeavesStmt = aNew; - } - rc = sqlite3_prepare_v2(p->db, p->zSelectLeaves, -1, &pReader->pStmt, 0); - if( rc!=SQLITE_OK ){ - goto finished; - } - p->nLeavesTotal++; - }else{ - pReader->pStmt = p->aLeavesStmt[--p->nLeavesStmt]; - } - - /* Bind the start and end leaf blockids to the prepared SQL statement. */ - sqlite3_bind_int64(pReader->pStmt, 1, iStartLeaf); - sqlite3_bind_int64(pReader->pStmt, 2, iEndLeaf); + pReader->iCurrentBlock = iStartLeaf-1; } - rc = fts3SegReaderNext(pReader); - finished: if( rc==SQLITE_OK ){ *ppReader = pReader; }else{ - sqlite3Fts3SegReaderFree(p, pReader); + sqlite3Fts3SegReaderFree(pReader); } return rc; } @@ -113982,7 +115691,6 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( pReader->iIdx = 0x7FFFFFFF; pReader->ppNextElem = (Fts3HashElem **)&pReader[1]; memcpy(pReader->ppNextElem, aElem, nElem*sizeof(Fts3HashElem *)); - fts3SegReaderNext(pReader); } } @@ -114015,12 +115723,11 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( ** code is returned. */ static int fts3SegReaderNew( - Fts3Table *p, /* Virtual table handle */ sqlite3_stmt *pStmt, /* See above */ int iAge, /* Segment "age". */ Fts3SegReader **ppReader /* OUT: Allocated Fts3SegReader */ ){ - return sqlite3Fts3SegReaderNew(p, iAge, + return sqlite3Fts3SegReaderNew(iAge, sqlite3_column_int64(pStmt, 1), sqlite3_column_int64(pStmt, 2), sqlite3_column_int64(pStmt, 3), @@ -114224,7 +115931,7 @@ static int fts3PrefixCompress( ** (according to memcmp) than the previous term. */ static int fts3NodeAddTerm( - Fts3Table *p, /* Virtual table handle */ + Fts3Table *p, /* Virtual table handle */ SegmentNode **ppTree, /* IN/OUT: SegmentNode handle */ int isCopyTerm, /* True if zTerm/nTerm is transient */ const char *zTerm, /* Pointer to buffer containing term */ @@ -114854,15 +116561,14 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderIterate( ** unnecessary merge/sort operations for the case where single segment ** b-tree leaf nodes contain more than one term. */ - if( pFilter->zTerm ){ + for(i=0; inTerm; const char *zTerm = pFilter->zTerm; - for(i=0; ibase.pVtab; - rc = fts3SqlStmt(p, SQL_SELECT_DOCSIZE, &pStmt, 0); - if( rc ){ - return rc; - } - sqlite3_bind_int64(pStmt, 1, pCur->iPrevId); - if( sqlite3_step(pStmt)==SQLITE_ROW ){ - nBlob = sqlite3_column_bytes(pStmt, 0); - pBlob = (const char*)sqlite3_column_blob(pStmt, 0); - for(i=j=0; inColumn && jbase.pVtab; - rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0); - if( rc ){ - return rc; - } - if( sqlite3_step(pStmt)==SQLITE_ROW ){ - nBlob = sqlite3_column_bytes(pStmt, 0); - pBlob = (const char*)sqlite3_column_blob(pStmt, 0); - j = sqlite3Fts3GetVarint(pBlob, &x); - a[0] = nDoc = (u32)(x & 0xffffffff); - for(i=0; inColumn && jiPrevDocid. The sizes are encoded as @@ -115286,16 +116923,26 @@ static void fts3InsertDocsize( } /* -** Update the 0 record of the %_stat table so that it holds a blob -** which contains the document count followed by the cumulative -** document sizes for all columns. +** Record 0 of the %_stat table contains a blob consisting of N varints, +** where N is the number of user defined columns in the fts3 table plus +** two. If nCol is the number of user defined columns, then values of the +** varints are set as follows: +** +** Varint 0: Total number of rows in the table. +** +** Varint 1..nCol: For each column, the total number of tokens stored in +** the column for all rows of the table. +** +** Varint 1+nCol: The total size, in bytes, of all text values in all +** columns of all rows of the table. +** */ static void fts3UpdateDocTotals( - int *pRC, /* The result code */ - Fts3Table *p, /* Table being updated */ - u32 *aSzIns, /* Size increases */ - u32 *aSzDel, /* Size decreases */ - int nChng /* Change in the number of documents */ + int *pRC, /* The result code */ + Fts3Table *p, /* Table being updated */ + u32 *aSzIns, /* Size increases */ + u32 *aSzDel, /* Size decreases */ + int nChng /* Change in the number of documents */ ){ char *pBlob; /* Storage for BLOB written into %_stat */ int nBlob; /* Size of BLOB written into %_stat */ @@ -115304,13 +116951,15 @@ static void fts3UpdateDocTotals( int i; /* Loop counter */ int rc; /* Result code from subfunctions */ + const int nStat = p->nColumn+2; + if( *pRC ) return; - a = sqlite3_malloc( (sizeof(u32)+10)*(p->nColumn+1) ); + a = sqlite3_malloc( (sizeof(u32)+10)*nStat ); if( a==0 ){ *pRC = SQLITE_NOMEM; return; } - pBlob = (char*)&a[p->nColumn+1]; + pBlob = (char*)&a[nStat]; rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0); if( rc ){ sqlite3_free(a); @@ -115318,11 +116967,11 @@ static void fts3UpdateDocTotals( return; } if( sqlite3_step(pStmt)==SQLITE_ROW ){ - fts3DecodeIntArray(p->nColumn+1, a, + fts3DecodeIntArray(nStat, a, sqlite3_column_blob(pStmt, 0), sqlite3_column_bytes(pStmt, 0)); }else{ - memset(a, 0, sizeof(u32)*(p->nColumn+1) ); + memset(a, 0, sizeof(u32)*(nStat) ); } sqlite3_reset(pStmt); if( nChng<0 && a[0]<(u32)(-nChng) ){ @@ -115330,7 +116979,7 @@ static void fts3UpdateDocTotals( }else{ a[0] += nChng; } - for(i=0; inColumn; i++){ + for(i=0; inColumn+1; i++){ u32 x = a[i+1]; if( x+aSzIns[i] < aSzDel[i] ){ x = 0; @@ -115339,7 +116988,7 @@ static void fts3UpdateDocTotals( } a[i+1] = x; } - fts3EncodeIntArray(p->nColumn+1, a, pBlob, &nBlob); + fts3EncodeIntArray(nStat, a, pBlob, &nBlob); rc = fts3SqlStmt(p, SQL_REPLACE_DOCTOTAL, &pStmt, 0); if( rc ){ sqlite3_free(a); @@ -115386,9 +117035,159 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ rc = SQLITE_ERROR; } + sqlite3Fts3SegmentsClose(p); return rc; } +/* +** Return the deferred doclist associated with deferred token pDeferred. +** This function assumes that sqlite3Fts3CacheDeferredDoclists() has already +** been called to allocate and populate the doclist. +*/ +SQLITE_PRIVATE char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *pDeferred, int *pnByte){ + if( pDeferred->pList ){ + *pnByte = pDeferred->pList->nData; + return pDeferred->pList->aData; + } + *pnByte = 0; + return 0; +} + +/* +** Helper fucntion for FreeDeferredDoclists(). This function removes all +** references to deferred doclists from within the tree of Fts3Expr +** structures headed by +*/ +static void fts3DeferredDoclistClear(Fts3Expr *pExpr){ + if( pExpr ){ + fts3DeferredDoclistClear(pExpr->pLeft); + fts3DeferredDoclistClear(pExpr->pRight); + if( pExpr->isLoaded ){ + sqlite3_free(pExpr->aDoclist); + pExpr->isLoaded = 0; + pExpr->aDoclist = 0; + pExpr->nDoclist = 0; + pExpr->pCurrent = 0; + pExpr->iCurrent = 0; + } + } +} + +/* +** Delete all cached deferred doclists. Deferred doclists are cached +** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function. +*/ +SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){ + Fts3DeferredToken *pDef; + for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){ + sqlite3_free(pDef->pList); + pDef->pList = 0; + } + if( pCsr->pDeferred ){ + fts3DeferredDoclistClear(pCsr->pExpr); + } +} + +/* +** Free all entries in the pCsr->pDeffered list. Entries are added to +** this list using sqlite3Fts3DeferToken(). +*/ +SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){ + Fts3DeferredToken *pDef; + Fts3DeferredToken *pNext; + for(pDef=pCsr->pDeferred; pDef; pDef=pNext){ + pNext = pDef->pNext; + sqlite3_free(pDef->pList); + sqlite3_free(pDef); + } + pCsr->pDeferred = 0; +} + +/* +** Generate deferred-doclists for all tokens in the pCsr->pDeferred list +** based on the row that pCsr currently points to. +** +** A deferred-doclist is like any other doclist with position information +** included, except that it only contains entries for a single row of the +** table, not for all rows. +*/ +SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){ + int rc = SQLITE_OK; /* Return code */ + if( pCsr->pDeferred ){ + int i; /* Used to iterate through table columns */ + sqlite3_int64 iDocid; /* Docid of the row pCsr points to */ + Fts3DeferredToken *pDef; /* Used to iterate through deferred tokens */ + + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + sqlite3_tokenizer *pT = p->pTokenizer; + sqlite3_tokenizer_module const *pModule = pT->pModule; + + assert( pCsr->isRequireSeek==0 ); + iDocid = sqlite3_column_int64(pCsr->pStmt, 0); + + for(i=0; inColumn && rc==SQLITE_OK; i++){ + const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1); + sqlite3_tokenizer_cursor *pTC = 0; + + rc = pModule->xOpen(pT, zText, -1, &pTC); + while( rc==SQLITE_OK ){ + char const *zToken; /* Buffer containing token */ + int nToken; /* Number of bytes in token */ + int iDum1, iDum2; /* Dummy variables */ + int iPos; /* Position of token in zText */ + + pTC->pTokenizer = pT; + rc = pModule->xNext(pTC, &zToken, &nToken, &iDum1, &iDum2, &iPos); + for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ + Fts3PhraseToken *pPT = pDef->pToken; + if( (pDef->iCol>=p->nColumn || pDef->iCol==i) + && (pPT->n==nToken || (pPT->isPrefix && pPT->nz, pPT->n)) + ){ + fts3PendingListAppend(&pDef->pList, iDocid, i, iPos, &rc); + } + } + } + if( pTC ) pModule->xClose(pTC); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + + for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ + if( pDef->pList ){ + rc = fts3PendingListAppendVarint(&pDef->pList, 0); + } + } + } + + return rc; +} + +/* +** Add an entry for token pToken to the pCsr->pDeferred list. +*/ +SQLITE_PRIVATE int sqlite3Fts3DeferToken( + Fts3Cursor *pCsr, /* Fts3 table cursor */ + Fts3PhraseToken *pToken, /* Token to defer */ + int iCol /* Column that token must appear in (or -1) */ +){ + Fts3DeferredToken *pDeferred; + pDeferred = sqlite3_malloc(sizeof(*pDeferred)); + if( !pDeferred ){ + return SQLITE_NOMEM; + } + memset(pDeferred, 0, sizeof(*pDeferred)); + pDeferred->pToken = pToken; + pDeferred->pNext = pCsr->pDeferred; + pDeferred->iCol = iCol; + pCsr->pDeferred = pDeferred; + + assert( pToken->pDeferred==0 ); + pToken->pDeferred = pDeferred; + + return SQLITE_OK; +} + + /* ** This function does the work for the xUpdate method of FTS3 virtual ** tables. @@ -115407,16 +117206,17 @@ SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( u32 *aSzDel; /* Sizes of deleted documents */ int nChng = 0; /* Net change in number of documents */ + assert( p->pSegments==0 ); /* Allocate space to hold the change in document sizes */ - aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*p->nColumn*2 ); + aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 ); if( aSzIns==0 ) return SQLITE_NOMEM; - aSzDel = &aSzIns[p->nColumn]; - memset(aSzIns, 0, sizeof(aSzIns[0])*p->nColumn*2); + aSzDel = &aSzIns[p->nColumn+1]; + memset(aSzIns, 0, sizeof(aSzIns[0])*(p->nColumn+1)*2); /* If this is a DELETE or UPDATE operation, remove the old record. */ if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ - int isEmpty; + int isEmpty = 0; rc = fts3IsEmpty(p, apVal, &isEmpty); if( rc==SQLITE_OK ){ if( isEmpty ){ @@ -115433,8 +117233,8 @@ SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, apVal); if( p->bHasDocsize ){ fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, apVal); - nChng--; } + nChng--; } } }else if( sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL ){ @@ -115452,16 +117252,17 @@ SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( rc = fts3InsertTerms(p, apVal, aSzIns); } if( p->bHasDocsize ){ - nChng++; fts3InsertDocsize(&rc, p, aSzIns); } + nChng++; } - if( p->bHasDocsize ){ + if( p->bHasStat ){ fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng); } sqlite3_free(aSzIns); + sqlite3Fts3SegmentsClose(p); return rc; } @@ -115485,6 +117286,7 @@ SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *p){ sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); } } + sqlite3Fts3SegmentsClose(p); return rc; } @@ -115508,6 +117310,22 @@ SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *p){ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) +/* +** Characters that may appear in the second argument to matchinfo(). +*/ +#define FTS3_MATCHINFO_NPHRASE 'p' /* 1 value */ +#define FTS3_MATCHINFO_NCOL 'c' /* 1 value */ +#define FTS3_MATCHINFO_NDOC 'n' /* 1 value */ +#define FTS3_MATCHINFO_AVGLENGTH 'a' /* nCol values */ +#define FTS3_MATCHINFO_LENGTH 'l' /* nCol values */ +#define FTS3_MATCHINFO_LCS 's' /* nCol values */ +#define FTS3_MATCHINFO_HITS 'x' /* 3*nCol*nPhrase values */ + +/* +** The default value for the second argument to matchinfo(). +*/ +#define FTS3_MATCHINFO_DEFAULT "pcx" + /* ** Used as an fts3ExprIterate() context when loading phrase doclists to @@ -115515,7 +117333,7 @@ SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *p){ */ typedef struct LoadDoclistCtx LoadDoclistCtx; struct LoadDoclistCtx { - Fts3Table *pTab; /* FTS3 Table */ + Fts3Cursor *pCsr; /* FTS3 Cursor */ int nPhrase; /* Number of phrases seen so far */ int nToken; /* Number of tokens seen so far */ }; @@ -115561,6 +117379,8 @@ typedef struct MatchInfo MatchInfo; struct MatchInfo { Fts3Cursor *pCursor; /* FTS3 Cursor */ int nCol; /* Number of columns in table */ + int nPhrase; /* Number of matchable phrases in query */ + sqlite3_int64 nDoc; /* Number of docs in database */ u32 *aMatchinfo; /* Pre-allocated buffer */ }; @@ -115709,7 +117529,7 @@ static int fts3ExprLoadDoclistsCb1(Fts3Expr *pExpr, int iPhrase, void *ctx){ p->nToken += pExpr->pPhrase->nToken; if( pExpr->isLoaded==0 ){ - rc = sqlite3Fts3ExprLoadDoclist(p->pTab, pExpr); + rc = sqlite3Fts3ExprLoadDoclist(p->pCsr, pExpr); pExpr->isLoaded = 1; if( rc==SQLITE_OK ){ rc = fts3ExprNearTrim(pExpr); @@ -115752,7 +117572,7 @@ static int fts3ExprLoadDoclists( ){ int rc; /* Return Code */ LoadDoclistCtx sCtx = {0,0,0}; /* Context for fts3ExprIterate() */ - sCtx.pTab = (Fts3Table *)pCsr->base.pVtab; + sCtx.pCsr = pCsr; rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb1, (void *)&sCtx); if( rc==SQLITE_OK ){ (void)fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb2, 0); @@ -115762,6 +117582,18 @@ static int fts3ExprLoadDoclists( return rc; } +static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ + (*(int *)ctx)++; + UNUSED_PARAMETER(pExpr); + UNUSED_PARAMETER(iPhrase); + return SQLITE_OK; +} +static int fts3ExprPhraseCount(Fts3Expr *pExpr){ + int nPhrase = 0; + (void)fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase); + return nPhrase; +} + /* ** Advance the position list iterator specified by the first two ** arguments so that it points to the first element with a value greater @@ -116274,38 +118106,87 @@ static void fts3LoadColumnlistCounts(char **pp, u32 *aOut, int isGlobal){ /* ** fts3ExprIterate() callback used to collect the "global" matchinfo stats -** for a single query. The "global" stats are those elements of the matchinfo -** array that are constant for all rows returned by the current query. +** for a single query. +** +** fts3ExprIterate() callback to load the 'global' elements of a +** FTS3_MATCHINFO_HITS matchinfo array. The global stats are those elements +** of the matchinfo array that are constant for all rows returned by the +** current query. +** +** Argument pCtx is actually a pointer to a struct of type MatchInfo. This +** function populates Matchinfo.aMatchinfo[] as follows: +** +** for(iCol=0; iColpCursor; + char *pIter; char *pEnd; - const int iStart = 2 + (iPhrase * p->nCol * 3) + 1; + char *pFree = 0; + u32 *aOut = &p->aMatchinfo[3*iPhrase*p->nCol]; assert( pExpr->isLoaded ); + assert( pExpr->eType==FTSQUERY_PHRASE ); + + if( pCsr->pDeferred ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + int ii; + for(ii=0; iinToken; ii++){ + if( pPhrase->aToken[ii].bFulltext ) break; + } + if( iinToken ){ + int nFree = 0; + int rc = sqlite3Fts3ExprLoadFtDoclist(pCsr, pExpr, &pFree, &nFree); + if( rc!=SQLITE_OK ) return rc; + pIter = pFree; + pEnd = &pFree[nFree]; + }else{ + int iCol; /* Column index */ + for(iCol=0; iColnCol; iCol++){ + aOut[iCol*3 + 1] = (u32)p->nDoc; + aOut[iCol*3 + 2] = (u32)p->nDoc; + } + return SQLITE_OK; + } + }else{ + pIter = pExpr->aDoclist; + pEnd = &pExpr->aDoclist[pExpr->nDoclist]; + } /* Fill in the global hit count matrix row for this phrase. */ - pCsr = pExpr->aDoclist; - pEnd = &pExpr->aDoclist[pExpr->nDoclist]; - while( pCsraMatchinfo[iStart], 1); + while( pIteraDoclist ){ char *pCsr; - int iStart = 2 + (iPhrase * p->nCol * 3); + int iStart = iPhrase * p->nCol * 3; int i; for(i=0; inCol; i++) p->aMatchinfo[iStart+i*3] = 0; @@ -116328,67 +118209,399 @@ static int fts3ExprLocalMatchinfoCb( return SQLITE_OK; } +static int fts3MatchinfoCheck( + Fts3Table *pTab, + char cArg, + char **pzErr +){ + if( (cArg==FTS3_MATCHINFO_NPHRASE) + || (cArg==FTS3_MATCHINFO_NCOL) + || (cArg==FTS3_MATCHINFO_NDOC && pTab->bHasStat) + || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bHasStat) + || (cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize) + || (cArg==FTS3_MATCHINFO_LCS) + || (cArg==FTS3_MATCHINFO_HITS) + ){ + return SQLITE_OK; + } + *pzErr = sqlite3_mprintf("unrecognized matchinfo request: %c", cArg); + return SQLITE_ERROR; +} + +static int fts3MatchinfoSize(MatchInfo *pInfo, char cArg){ + int nVal; /* Number of integers output by cArg */ + + switch( cArg ){ + case FTS3_MATCHINFO_NDOC: + case FTS3_MATCHINFO_NPHRASE: + case FTS3_MATCHINFO_NCOL: + nVal = 1; + break; + + case FTS3_MATCHINFO_AVGLENGTH: + case FTS3_MATCHINFO_LENGTH: + case FTS3_MATCHINFO_LCS: + nVal = pInfo->nCol; + break; + + default: + assert( cArg==FTS3_MATCHINFO_HITS ); + nVal = pInfo->nCol * pInfo->nPhrase * 3; + break; + } + + return nVal; +} + +static int fts3MatchinfoSelectDoctotal( + Fts3Table *pTab, + sqlite3_stmt **ppStmt, + sqlite3_int64 *pnDoc, + const char **paLen +){ + sqlite3_stmt *pStmt; + const char *a; + sqlite3_int64 nDoc; + + if( !*ppStmt ){ + int rc = sqlite3Fts3SelectDoctotal(pTab, ppStmt); + if( rc!=SQLITE_OK ) return rc; + } + pStmt = *ppStmt; + + a = sqlite3_column_blob(pStmt, 0); + a += sqlite3Fts3GetVarint(a, &nDoc); + *pnDoc = (u32)nDoc; + + if( paLen ) *paLen = a; + return SQLITE_OK; +} + +/* +** An instance of the following structure is used to store state while +** iterating through a multi-column position-list corresponding to the +** hits for a single phrase on a single row in order to calculate the +** values for a matchinfo() FTS3_MATCHINFO_LCS request. +*/ +typedef struct LcsIterator LcsIterator; +struct LcsIterator { + Fts3Expr *pExpr; /* Pointer to phrase expression */ + char *pRead; /* Cursor used to iterate through aDoclist */ + int iPosOffset; /* Tokens count up to end of this phrase */ + int iCol; /* Current column number */ + int iPos; /* Current position */ +}; + +/* +** If LcsIterator.iCol is set to the following value, the iterator has +** finished iterating through all offsets for all columns. +*/ +#define LCS_ITERATOR_FINISHED 0x7FFFFFFF; + +static int fts3MatchinfoLcsCb( + Fts3Expr *pExpr, /* Phrase expression node */ + int iPhrase, /* Phrase number (numbered from zero) */ + void *pCtx /* Pointer to MatchInfo structure */ +){ + LcsIterator *aIter = (LcsIterator *)pCtx; + aIter[iPhrase].pExpr = pExpr; + return SQLITE_OK; +} + +/* +** Advance the iterator passed as an argument to the next position. Return +** 1 if the iterator is at EOF or if it now points to the start of the +** position list for the next column. +*/ +static int fts3LcsIteratorAdvance(LcsIterator *pIter){ + char *pRead = pIter->pRead; + sqlite3_int64 iRead; + int rc = 0; + + pRead += sqlite3Fts3GetVarint(pRead, &iRead); + if( iRead==0 ){ + pIter->iCol = LCS_ITERATOR_FINISHED; + rc = 1; + }else{ + if( iRead==1 ){ + pRead += sqlite3Fts3GetVarint(pRead, &iRead); + pIter->iCol = (int)iRead; + pIter->iPos = pIter->iPosOffset; + pRead += sqlite3Fts3GetVarint(pRead, &iRead); + rc = 1; + } + pIter->iPos += (int)(iRead-2); + } + + pIter->pRead = pRead; + return rc; +} + +/* +** This function implements the FTS3_MATCHINFO_LCS matchinfo() flag. +** +** If the call is successful, the longest-common-substring lengths for each +** column are written into the first nCol elements of the pInfo->aMatchinfo[] +** array before returning. SQLITE_OK is returned in this case. +** +** Otherwise, if an error occurs, an SQLite error code is returned and the +** data written to the first nCol elements of pInfo->aMatchinfo[] is +** undefined. +*/ +static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){ + LcsIterator *aIter; + int i; + int iCol; + int nToken = 0; + + /* Allocate and populate the array of LcsIterator objects. The array + ** contains one element for each matchable phrase in the query. + **/ + aIter = sqlite3_malloc(sizeof(LcsIterator) * pCsr->nPhrase); + if( !aIter ) return SQLITE_NOMEM; + memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase); + (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter); + for(i=0; inPhrase; i++){ + LcsIterator *pIter = &aIter[i]; + nToken -= pIter->pExpr->pPhrase->nToken; + pIter->iPosOffset = nToken; + pIter->pRead = sqlite3Fts3FindPositions(pIter->pExpr, pCsr->iPrevId, -1); + if( pIter->pRead ){ + pIter->iPos = pIter->iPosOffset; + fts3LcsIteratorAdvance(&aIter[i]); + }else{ + pIter->iCol = LCS_ITERATOR_FINISHED; + } + } + + for(iCol=0; iColnCol; iCol++){ + int nLcs = 0; /* LCS value for this column */ + int nLive = 0; /* Number of iterators in aIter not at EOF */ + + /* Loop through the iterators in aIter[]. Set nLive to the number of + ** iterators that point to a position-list corresponding to column iCol. + */ + for(i=0; inPhrase; i++){ + assert( aIter[i].iCol>=iCol ); + if( aIter[i].iCol==iCol ) nLive++; + } + + /* The following loop runs until all iterators in aIter[] have finished + ** iterating through positions in column iCol. Exactly one of the + ** iterators is advanced each time the body of the loop is run. + */ + while( nLive>0 ){ + LcsIterator *pAdv = 0; /* The iterator to advance by one position */ + int nThisLcs = 0; /* LCS for the current iterator positions */ + + for(i=0; inPhrase; i++){ + LcsIterator *pIter = &aIter[i]; + if( iCol!=pIter->iCol ){ + /* This iterator is already at EOF for this column. */ + nThisLcs = 0; + }else{ + if( pAdv==0 || pIter->iPosiPos ){ + pAdv = pIter; + } + if( nThisLcs==0 || pIter->iPos==pIter[-1].iPos ){ + nThisLcs++; + }else{ + nThisLcs = 1; + } + if( nThisLcs>nLcs ) nLcs = nThisLcs; + } + } + if( fts3LcsIteratorAdvance(pAdv) ) nLive--; + } + + pInfo->aMatchinfo[iCol] = nLcs; + } + + sqlite3_free(aIter); + return SQLITE_OK; +} + +/* +** Populate the buffer pInfo->aMatchinfo[] with an array of integers to +** be returned by the matchinfo() function. Argument zArg contains the +** format string passed as the second argument to matchinfo (or the +** default value "pcx" if no second argument was specified). The format +** string has already been validated and the pInfo->aMatchinfo[] array +** is guaranteed to be large enough for the output. +** +** If bGlobal is true, then populate all fields of the matchinfo() output. +** If it is false, then assume that those fields that do not change between +** rows (i.e. FTS3_MATCHINFO_NPHRASE, NCOL, NDOC, AVGLENGTH and part of HITS) +** have already been populated. +** +** Return SQLITE_OK if successful, or an SQLite error code if an error +** occurs. If a value other than SQLITE_OK is returned, the state the +** pInfo->aMatchinfo[] buffer is left in is undefined. +*/ +static int fts3MatchinfoValues( + Fts3Cursor *pCsr, /* FTS3 cursor object */ + int bGlobal, /* True to grab the global stats */ + MatchInfo *pInfo, /* Matchinfo context object */ + const char *zArg /* Matchinfo format string */ +){ + int rc = SQLITE_OK; + int i; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + sqlite3_stmt *pSelect = 0; + + for(i=0; rc==SQLITE_OK && zArg[i]; i++){ + + switch( zArg[i] ){ + case FTS3_MATCHINFO_NPHRASE: + if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nPhrase; + break; + + case FTS3_MATCHINFO_NCOL: + if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nCol; + break; + + case FTS3_MATCHINFO_NDOC: + if( bGlobal ){ + sqlite3_int64 nDoc; + rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, 0); + pInfo->aMatchinfo[0] = (u32)nDoc; + } + break; + + case FTS3_MATCHINFO_AVGLENGTH: + if( bGlobal ){ + sqlite3_int64 nDoc; /* Number of rows in table */ + const char *a; /* Aggregate column length array */ + + rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, &a); + if( rc==SQLITE_OK ){ + int iCol; + for(iCol=0; iColnCol; iCol++){ + sqlite3_int64 nToken; + a += sqlite3Fts3GetVarint(a, &nToken); + pInfo->aMatchinfo[iCol] = (u32)(((u32)(nToken&0xffffffff)+nDoc/2)/nDoc); + } + } + } + break; + + case FTS3_MATCHINFO_LENGTH: { + sqlite3_stmt *pSelectDocsize = 0; + rc = sqlite3Fts3SelectDocsize(pTab, pCsr->iPrevId, &pSelectDocsize); + if( rc==SQLITE_OK ){ + int iCol; + const char *a = sqlite3_column_blob(pSelectDocsize, 0); + for(iCol=0; iColnCol; iCol++){ + sqlite3_int64 nToken; + a += sqlite3Fts3GetVarint(a, &nToken); + pInfo->aMatchinfo[iCol] = (u32)nToken; + } + } + sqlite3_reset(pSelectDocsize); + break; + } + + case FTS3_MATCHINFO_LCS: + rc = fts3ExprLoadDoclists(pCsr, 0, 0); + if( rc==SQLITE_OK ){ + rc = fts3MatchinfoLcs(pCsr, pInfo); + } + break; + + default: { + Fts3Expr *pExpr; + assert( zArg[i]==FTS3_MATCHINFO_HITS ); + pExpr = pCsr->pExpr; + rc = fts3ExprLoadDoclists(pCsr, 0, 0); + if( rc!=SQLITE_OK ) break; + if( bGlobal ){ + if( pCsr->pDeferred ){ + rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc, 0); + if( rc!=SQLITE_OK ) break; + } + rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo); + if( rc!=SQLITE_OK ) break; + } + (void)fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo); + break; + } + } + + pInfo->aMatchinfo += fts3MatchinfoSize(pInfo, zArg[i]); + } + + sqlite3_reset(pSelect); + return rc; +} + + /* ** Populate pCsr->aMatchinfo[] with data for the current row. The ** 'matchinfo' data is an array of 32-bit unsigned integers (C type u32). */ -static int fts3GetMatchinfo(Fts3Cursor *pCsr){ +static int fts3GetMatchinfo( + Fts3Cursor *pCsr, /* FTS3 Cursor object */ + const char *zArg /* Second argument to matchinfo() function */ +){ MatchInfo sInfo; Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; int rc = SQLITE_OK; + int bGlobal = 0; /* Collect 'global' stats as well as local */ + memset(&sInfo, 0, sizeof(MatchInfo)); sInfo.pCursor = pCsr; sInfo.nCol = pTab->nColumn; + /* If there is cached matchinfo() data, but the format string for the + ** cache does not match the format string for this request, discard + ** the cached data. */ + if( pCsr->zMatchinfo && strcmp(pCsr->zMatchinfo, zArg) ){ + assert( pCsr->aMatchinfo ); + sqlite3_free(pCsr->aMatchinfo); + pCsr->zMatchinfo = 0; + pCsr->aMatchinfo = 0; + } + + /* If Fts3Cursor.aMatchinfo[] is NULL, then this is the first time the + ** matchinfo function has been called for this query. In this case + ** allocate the array used to accumulate the matchinfo data and + ** initialize those elements that are constant for every row. + */ if( pCsr->aMatchinfo==0 ){ - /* If Fts3Cursor.aMatchinfo[] is NULL, then this is the first time the - ** matchinfo function has been called for this query. In this case - ** allocate the array used to accumulate the matchinfo data and - ** initialize those elements that are constant for every row. - */ - int nPhrase; /* Number of phrases */ - int nMatchinfo; /* Number of u32 elements in match-info */ + int nMatchinfo = 0; /* Number of u32 elements in match-info */ + int nArg; /* Bytes in zArg */ + int i; /* Used to iterate through zArg */ - /* Load doclists for each phrase in the query. */ - rc = fts3ExprLoadDoclists(pCsr, &nPhrase, 0); - if( rc!=SQLITE_OK ){ - return rc; - } - nMatchinfo = 2 + 3*sInfo.nCol*nPhrase; - if( pTab->bHasDocsize ){ - nMatchinfo += 1 + 2*pTab->nColumn; - } + /* Determine the number of phrases in the query */ + pCsr->nPhrase = fts3ExprPhraseCount(pCsr->pExpr); + sInfo.nPhrase = pCsr->nPhrase; - sInfo.aMatchinfo = (u32 *)sqlite3_malloc(sizeof(u32)*nMatchinfo); - if( !sInfo.aMatchinfo ){ - return SQLITE_NOMEM; + /* Determine the number of integers in the buffer returned by this call. */ + for(i=0; zArg[i]; i++){ + nMatchinfo += fts3MatchinfoSize(&sInfo, zArg[i]); } - memset(sInfo.aMatchinfo, 0, sizeof(u32)*nMatchinfo); + /* Allocate space for Fts3Cursor.aMatchinfo[] and Fts3Cursor.zMatchinfo. */ + nArg = (int)strlen(zArg); + pCsr->aMatchinfo = (u32 *)sqlite3_malloc(sizeof(u32)*nMatchinfo + nArg + 1); + if( !pCsr->aMatchinfo ) return SQLITE_NOMEM; - /* First element of match-info is the number of phrases in the query */ - sInfo.aMatchinfo[0] = nPhrase; - sInfo.aMatchinfo[1] = sInfo.nCol; - (void)fts3ExprIterate(pCsr->pExpr, fts3ExprGlobalMatchinfoCb,(void*)&sInfo); - if( pTab->bHasDocsize ){ - int ofst = 2 + 3*sInfo.aMatchinfo[0]*sInfo.aMatchinfo[1]; - rc = sqlite3Fts3MatchinfoDocsizeGlobal(pCsr, &sInfo.aMatchinfo[ofst]); - } - pCsr->aMatchinfo = sInfo.aMatchinfo; + pCsr->zMatchinfo = (char *)&pCsr->aMatchinfo[nMatchinfo]; + pCsr->nMatchinfo = nMatchinfo; + memcpy(pCsr->zMatchinfo, zArg, nArg+1); + memset(pCsr->aMatchinfo, 0, sizeof(u32)*nMatchinfo); pCsr->isMatchinfoNeeded = 1; + bGlobal = 1; } sInfo.aMatchinfo = pCsr->aMatchinfo; - if( rc==SQLITE_OK && pCsr->isMatchinfoNeeded ){ - (void)fts3ExprIterate(pCsr->pExpr, fts3ExprLocalMatchinfoCb, (void*)&sInfo); - if( pTab->bHasDocsize ){ - int ofst = 2 + 3*sInfo.aMatchinfo[0]*sInfo.aMatchinfo[1]; - rc = sqlite3Fts3MatchinfoDocsizeLocal(pCsr, &sInfo.aMatchinfo[ofst]); - } + sInfo.nPhrase = pCsr->nPhrase; + if( pCsr->isMatchinfoNeeded ){ + rc = fts3MatchinfoValues(pCsr, bGlobal, &sInfo, zArg); pCsr->isMatchinfoNeeded = 0; } - return SQLITE_OK; + return rc; } /* @@ -116449,7 +118662,7 @@ SQLITE_PRIVATE void sqlite3Fts3Snippet( ** columns of the FTS3 table. Otherwise, only column iCol is considered. */ for(iRead=0; iReadnColumn; iRead++){ - SnippetFragment sF; + SnippetFragment sF = {0, 0, 0, 0}; int iS; if( iCol>=0 && iRead!=iCol ) continue; @@ -116483,6 +118696,7 @@ SQLITE_PRIVATE void sqlite3Fts3Snippet( } snippet_out: + sqlite3Fts3SegmentsClose(pTab); if( rc!=SQLITE_OK ){ sqlite3_result_error_code(pCtx, rc); sqlite3_free(res.z); @@ -116662,6 +118876,7 @@ SQLITE_PRIVATE void sqlite3Fts3Offsets( offsets_out: sqlite3_free(sCtx.aTerm); assert( rc!=SQLITE_DONE ); + sqlite3Fts3SegmentsClose(pTab); if( rc!=SQLITE_OK ){ sqlite3_result_error_code(pCtx, rc); sqlite3_free(res.z); @@ -116674,21 +118889,43 @@ SQLITE_PRIVATE void sqlite3Fts3Offsets( /* ** Implementation of matchinfo() function. */ -SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *pContext, Fts3Cursor *pCsr){ +SQLITE_PRIVATE void sqlite3Fts3Matchinfo( + sqlite3_context *pContext, /* Function call context */ + Fts3Cursor *pCsr, /* FTS3 table cursor */ + const char *zArg /* Second arg to matchinfo() function */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; int rc; + int i; + const char *zFormat; + + if( zArg ){ + for(i=0; zArg[i]; i++){ + char *zErr = 0; + if( fts3MatchinfoCheck(pTab, zArg[i], &zErr) ){ + sqlite3_result_error(pContext, zErr, -1); + sqlite3_free(zErr); + return; + } + } + zFormat = zArg; + }else{ + zFormat = FTS3_MATCHINFO_DEFAULT; + } + if( !pCsr->pExpr ){ sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC); return; } - rc = fts3GetMatchinfo(pCsr); + + /* Retrieve matchinfo() data. */ + rc = fts3GetMatchinfo(pCsr, zFormat); + sqlite3Fts3SegmentsClose(pTab); + if( rc!=SQLITE_OK ){ sqlite3_result_error_code(pContext, rc); }else{ - Fts3Table *pTab = (Fts3Table*)pCsr->base.pVtab; - int n = sizeof(u32)*(2+pCsr->aMatchinfo[0]*pCsr->aMatchinfo[1]*3); - if( pTab->bHasDocsize ){ - n += sizeof(u32)*(1 + 2*pTab->nColumn); - } + int n = pCsr->nMatchinfo * sizeof(u32); sqlite3_result_blob(pContext, pCsr->aMatchinfo, n, SQLITE_TRANSIENT); } } @@ -116808,6 +119045,12 @@ typedef unsigned char u8; typedef unsigned int u32; #endif +/* The following macro is used to suppress compiler warnings. +*/ +#ifndef UNUSED_PARAMETER +# define UNUSED_PARAMETER(x) (void)(x) +#endif + typedef struct Rtree Rtree; typedef struct RtreeCursor RtreeCursor; typedef struct RtreeNode RtreeNode; @@ -117582,6 +119825,7 @@ static int testRtreeCell(Rtree *pRtree, RtreeCursor *pCursor, int *pbEof){ RtreeCell cell; int ii; int bRes = 0; + int rc = SQLITE_OK; nodeGetCell(pRtree, pCursor->pNode, pCursor->iCell, &cell); for(ii=0; bRes==0 && iinConstraint; ii++){ @@ -117607,12 +119851,8 @@ static int testRtreeCell(Rtree *pRtree, RtreeCursor *pCursor, int *pbEof){ break; default: { - int rc; assert( p->op==RTREE_MATCH ); rc = testRtreeGeom(pRtree, p, &cell, &bRes); - if( rc!=SQLITE_OK ){ - return rc; - } bRes = !bRes; break; } @@ -117620,7 +119860,7 @@ static int testRtreeCell(Rtree *pRtree, RtreeCursor *pCursor, int *pbEof){ } *pbEof = bRes; - return SQLITE_OK; + return rc; } /* @@ -117703,14 +119943,13 @@ static int descendToCell( rc = testRtreeCell(pRtree, pCursor, &isEof); } if( rc!=SQLITE_OK || isEof || iHeight==0 ){ - *pEof = isEof; - return rc; + goto descend_to_cell_out; } iRowid = nodeGetRowid(pRtree, pCursor->pNode, pCursor->iCell); rc = nodeAcquire(pRtree, iRowid, pCursor->pNode, &pChild); if( rc!=SQLITE_OK ){ - return rc; + goto descend_to_cell_out; } nodeRelease(pRtree, pCursor->pNode); @@ -117720,7 +119959,7 @@ static int descendToCell( pCursor->iCell = ii; rc = descendToCell(pRtree, pCursor, iHeight-1, &isEof); if( rc!=SQLITE_OK ){ - return rc; + goto descend_to_cell_out; } } @@ -117732,8 +119971,9 @@ static int descendToCell( pCursor->iCell = iSavedCell; } +descend_to_cell_out: *pEof = isEof; - return SQLITE_OK; + return rc; } /* @@ -117889,7 +120129,7 @@ static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){ /* Check that the blob is roughly the right size. */ nBlob = sqlite3_value_bytes(pValue); - if( nBlobmagic!=RTREE_GEOMETRY_MAGIC - || nBlob!=(sizeof(RtreeMatchArg) + (p->nParam-1)*sizeof(double)) + || nBlob!=(int)(sizeof(RtreeMatchArg) + (p->nParam-1)*sizeof(double)) ){ sqlite3_free(pGeom); return SQLITE_ERROR; @@ -118050,6 +120290,7 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ int iIdx = 0; char zIdxStr[RTREE_MAX_DIMENSIONS*8+1]; memset(zIdxStr, 0, sizeof(zIdxStr)); + UNUSED_PARAMETER(tab); assert( pIdxInfo->idxStr==0 ); for(ii=0; iinConstraint; ii++){ @@ -118223,6 +120464,7 @@ static float cellOverlap( if( ii!=iExclude ) #else assert( iExclude==-1 ); + UNUSED_PARAMETER(iExclude); #endif { int jj; @@ -119344,16 +121586,6 @@ static int newRowid(Rtree *pRtree, i64 *piRowid){ return rc; } -#ifndef NDEBUG -static int hashIsEmpty(Rtree *pRtree){ - int ii; - for(ii=0; iiaHash[ii] ); - } - return 1; -} -#endif - /* ** The xUpdate method for rtree module virtual tables. */ @@ -119819,6 +122051,7 @@ static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ Rtree tree; int ii; + UNUSED_PARAMETER(nArg); memset(&node, 0, sizeof(RtreeNode)); memset(&tree, 0, sizeof(Rtree)); tree.nDim = sqlite3_value_int(apArg[0]); @@ -119852,6 +122085,7 @@ static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ } static void rtreedepth(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ + UNUSED_PARAMETER(nArg); if( sqlite3_value_type(apArg[0])!=SQLITE_BLOB || sqlite3_value_bytes(apArg[0])<2 ){ @@ -119873,7 +122107,6 @@ SQLITE_PRIVATE int sqlite3RtreeInit(sqlite3 *db){ rc = sqlite3_create_function(db, "rtreenode", 2, utf8, 0, rtreenode, 0, 0); if( rc==SQLITE_OK ){ - int utf8 = SQLITE_UTF8; rc = sqlite3_create_function(db, "rtreedepth", 1, utf8, 0,rtreedepth, 0, 0); } if( rc==SQLITE_OK ){ diff --git a/ext/sqlite3/libsqlite/sqlite3.h b/ext/sqlite3/libsqlite/sqlite3.h index d78ae39b88..47ef2562a3 100644 --- a/ext/sqlite3/libsqlite/sqlite3.h +++ b/ext/sqlite3/libsqlite/sqlite3.h @@ -107,9 +107,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.7.3" -#define SQLITE_VERSION_NUMBER 3007003 -#define SQLITE_SOURCE_ID "2010-10-08 02:34:02 2677848087c9c090efb17c1893e77d6136a9111d" +#define SQLITE_VERSION "3.7.4" +#define SQLITE_VERSION_NUMBER 3007004 +#define SQLITE_SOURCE_ID "2010-12-07 20:14:09 a586a4deeb25330037a49df295b36aaf624d0f45" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -542,6 +542,18 @@ SQLITE_API int sqlite3_exec( ** equal SQLITE_SYNC_NORMAL, that means to use normal fsync() semantics. ** If the lower four bits equal SQLITE_SYNC_FULL, that means ** to use Mac OS X style fullsync instead of fsync(). +** +** Do not confuse the SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL flags +** with the [PRAGMA synchronous]=NORMAL and [PRAGMA synchronous]=FULL +** settings. The [synchronous pragma] determines when calls to the +** xSync VFS method occur and applies uniformly across all platforms. +** The SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL flags determine how +** energetic or rigorous or forceful the sync operations are and +** only make a difference on Mac OSX for the default SQLite code. +** (Third-party VFS implementations might also make the distinction +** between SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL, but among the +** operating systems natively supported by SQLite, only Mac OSX +** cares about the difference.) */ #define SQLITE_SYNC_NORMAL 0x00002 #define SQLITE_SYNC_FULL 0x00003 @@ -710,6 +722,8 @@ struct sqlite3_io_methods { #define SQLITE_LAST_ERRNO 4 #define SQLITE_FCNTL_SIZE_HINT 5 #define SQLITE_FCNTL_CHUNK_SIZE 6 +#define SQLITE_FCNTL_FILE_POINTER 7 + /* ** CAPI3REF: Mutex Handle @@ -2633,6 +2647,20 @@ SQLITE_API int sqlite3_prepare16_v2( */ SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt); +/* +** CAPI3REF: Determine If An SQL Statement Writes The Database +** +** ^The sqlite3_stmt_readonly(X) interface returns true (non-zero) if +** the [prepared statement] X is [SELECT] statement and false (zero) if +** X is an [INSERT], [UPDATE], [DELETE], CREATE, DROP, [ANALYZE], +** [ALTER], or [REINDEX] statement. +** If X is a NULL pointer or any other kind of statement, including but +** not limited to [ATTACH], [DETACH], [COMMIT], [ROLLBACK], [RELEASE], +** [SAVEPOINT], [PRAGMA], or [VACUUM] the result of sqlite3_stmt_readonly(X) is +** undefined. +*/ +SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt); + /* ** CAPI3REF: Dynamically Typed Value Object ** KEYWORDS: {protected sqlite3_value} {unprotected sqlite3_value} @@ -2732,7 +2760,10 @@ typedef struct sqlite3_context sqlite3_context; ** ** ^The fifth argument to sqlite3_bind_blob(), sqlite3_bind_text(), and ** sqlite3_bind_text16() is a destructor used to dispose of the BLOB or -** string after SQLite has finished with it. ^If the fifth argument is +** string after SQLite has finished with it. ^The destructor is called +** to dispose of the BLOB or string even if the call to sqlite3_bind_blob(), +** sqlite3_bind_text(), or sqlite3_bind_text16() fails. +** ^If the fifth argument is ** the special value [SQLITE_STATIC], then SQLite assumes that the ** information is in static, unmanaged space and does not need to be freed. ** ^If the fifth argument has the value [SQLITE_TRANSIENT], then @@ -3372,12 +3403,15 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** SQL function or aggregate, pass NULL poiners for all three function ** callbacks. ** -** ^If the tenth parameter to sqlite3_create_function_v2() is not NULL, -** then it is invoked when the function is deleted, either by being -** overloaded or when the database connection closes. -** ^When the destructure callback of the tenth parameter is invoked, it -** is passed a single argument which is a copy of the pointer which was -** the fifth parameter to sqlite3_create_function_v2(). +** ^(If the tenth parameter to sqlite3_create_function_v2() is not NULL, +** then it is destructor for the application data pointer. +** The destructor is invoked when the function is deleted, either by being +** overloaded or when the database connection closes.)^ +** ^The destructor is also invoked if the call to +** sqlite3_create_function_v2() fails. +** ^When the destructor callback of the tenth parameter is invoked, it +** is passed a single argument which is a copy of the application data +** pointer which was the fifth parameter to sqlite3_create_function_v2(). ** ** ^It is permitted to register multiple implementations of the same ** functions with the same name but with either differing numbers of @@ -3841,6 +3875,15 @@ SQLITE_API void sqlite3_result_zeroblob(sqlite3_context*, int n); ** calls to the collation creation functions or when the ** [database connection] is closed using [sqlite3_close()]. ** +** ^The xDestroy callback is not called if the +** sqlite3_create_collation_v2() function fails. Applications that invoke +** sqlite3_create_collation_v2() with a non-NULL xDestroy argument should +** check the return code and dispose of the application data pointer +** themselves rather than expecting SQLite to deal with it for them. +** This is different from every other SQLite interface. The inconsistency +** is unfortunate but cannot be changed without breaking backwards +** compatibility. +** ** See also: [sqlite3_collation_needed()] and [sqlite3_collation_needed16()]. */ SQLITE_API int sqlite3_create_collation( @@ -4595,7 +4638,9 @@ struct sqlite3_index_info { ** ^The sqlite3_create_module_v2() interface has a fifth parameter which ** is a pointer to a destructor for the pClientData. ^SQLite will ** invoke the destructor function (if it is not NULL) when SQLite -** no longer needs the pClientData pointer. ^The sqlite3_create_module() +** no longer needs the pClientData pointer. ^The destructor will also +** be invoked if the call to sqlite3_create_module_v2() fails. +** ^The sqlite3_create_module() ** interface is equivalent to sqlite3_create_module_v2() with a NULL ** destructor. */ @@ -4778,6 +4823,30 @@ SQLITE_API int sqlite3_blob_open( sqlite3_blob **ppBlob ); +/* +** CAPI3REF: Move a BLOB Handle to a New Row +** +** ^This function is used to move an existing blob handle so that it points +** to a different row of the same database table. ^The new row is identified +** by the rowid value passed as the second argument. Only the row can be +** changed. ^The database, table and column on which the blob handle is open +** remain the same. Moving an existing blob handle to a new row can be +** faster than closing the existing handle and opening a new one. +** +** ^(The new row must meet the same criteria as for [sqlite3_blob_open()] - +** it must exist and there must be either a blob or text value stored in +** the nominated column.)^ ^If the new row is not present in the table, or if +** it does not contain a blob or text value, or if another error occurs, an +** SQLite error code is returned and the blob handle is considered aborted. +** ^All subsequent calls to [sqlite3_blob_read()], [sqlite3_blob_write()] or +** [sqlite3_blob_reopen()] on an aborted blob handle immediately return +** SQLITE_ABORT. ^Calling [sqlite3_blob_bytes()] on an aborted blob handle +** always returns zero. +** +** ^This function sets the database handle error code and message. +*/ +SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64); + /* ** CAPI3REF: Close A BLOB Handle ** @@ -5185,7 +5254,7 @@ SQLITE_API sqlite3_mutex *sqlite3_db_mutex(sqlite3*); ** ^The [sqlite3_file_control()] interface makes a direct call to the ** xFileControl method for the [sqlite3_io_methods] object associated ** with a particular database identified by the second argument. ^The -** name of the database "main" for the main database or "temp" for the +** name of the database is "main" for the main database or "temp" for the ** TEMP database, or the name that appears after the AS keyword for ** databases that are added using the [ATTACH] SQL command. ** ^A NULL pointer can be used in place of "main" to refer to the @@ -5195,6 +5264,12 @@ SQLITE_API sqlite3_mutex *sqlite3_db_mutex(sqlite3*); ** the xFileControl method. ^The return value of the xFileControl ** method becomes the return value of this routine. ** +** ^The SQLITE_FCNTL_FILE_POINTER value for the op parameter causes +** a pointer to the underlying [sqlite3_file] object to be written into +** the space pointed to by the 4th parameter. ^The SQLITE_FCNTL_FILE_POINTER +** case is a short-circuit path which does not actually invoke the +** underlying sqlite3_io_methods.xFileControl method. +** ** ^If the second parameter (zDbName) does not match the name of any ** open database file, then SQLITE_ERROR is returned. ^This error ** code is not remembered and will not be recalled by [sqlite3_errcode()] diff --git a/ext/sqlite3/libsqlite/sqlite3ext.h b/ext/sqlite3/libsqlite/sqlite3ext.h index 0d37bbe01e..e45e691fd6 100644 --- a/ext/sqlite3/libsqlite/sqlite3ext.h +++ b/ext/sqlite3/libsqlite/sqlite3ext.h @@ -191,6 +191,27 @@ struct sqlite3_api_routines { sqlite3_stmt *(*next_stmt)(sqlite3*,sqlite3_stmt*); const char *(*sql)(sqlite3_stmt*); int (*status)(int,int*,int*,int); + int (*backup_finish)(sqlite3_backup*); + sqlite3_backup *(*backup_init)(sqlite3*,const char*,sqlite3*,const char*); + int (*backup_pagecount)(sqlite3_backup*); + int (*backup_remaining)(sqlite3_backup*); + int (*backup_step)(sqlite3_backup*,int); + const char *(*compileoption_get)(int); + int (*compileoption_used)(const char*); + int (*create_function_v2)(sqlite3*,const char*,int,int,void*,void (*xFunc)(sqlite3_context*,int,sqlite3_value**),void (*xStep)(sqlite3_context*,int,sqlite3_value**),void (*xFinal)(sqlite3_context*),void(*xDestroy)(void*)); + int (*db_config)(sqlite3*,int,...); + sqlite3_mutex *(*db_mutex)(sqlite3*); + int (*db_status)(sqlite3*,int,int*,int*,int); + int (*extended_errcode)(sqlite3*); + void (*log)(int,const char*,...); + sqlite3_int64 (*soft_heap_limit64)(sqlite3_int64); + const char *(*sourceid)(void); + int (*stmt_status)(sqlite3_stmt*,int,int); + int (*strnicmp)(const char*,const char*,int); + int (*unlock_notify)(sqlite3*,void(*)(void**,int),void*); + int (*wal_autocheckpoint)(sqlite3*,int); + int (*wal_checkpoint)(sqlite3*,const char*); + void *(*wal_hook)(sqlite3*,int(*)(void*,sqlite3*,const char*,int),void*); }; /* @@ -370,6 +391,27 @@ struct sqlite3_api_routines { #define sqlite3_next_stmt sqlite3_api->next_stmt #define sqlite3_sql sqlite3_api->sql #define sqlite3_status sqlite3_api->status +#define sqlite3_backup_finish sqlite3_api->backup_finish +#define sqlite3_backup_init sqlite3_api->backup_init +#define sqlite3_backup_pagecount sqlite3_api->backup_pagecount +#define sqlite3_backup_remaining sqlite3_api->backup_remaining +#define sqlite3_backup_step sqlite3_api->backup_step +#define sqlite3_compileoption_get sqlite3_api->compileoption_get +#define sqlite3_compileoption_used sqlite3_api->compileoption_used +#define sqlite3_create_function_v2 sqlite3_api->create_function_v2 +#define sqlite3_db_config sqlite3_api->db_config +#define sqlite3_db_mutex sqlite3_api->db_mutex +#define sqlite3_db_status sqlite3_api->db_status +#define sqlite3_extended_errcode sqlite3_api->extended_errcode +#define sqlite3_log sqlite3_api->log +#define sqlite3_soft_heap_limit64 sqlite3_api->soft_heap_limit64 +#define sqlite3_sourceid sqlite3_api->sourceid +#define sqlite3_stmt_status sqlite3_api->stmt_status +#define sqlite3_strnicmp sqlite3_api->strnicmp +#define sqlite3_unlock_notify sqlite3_api->unlock_notify +#define sqlite3_wal_autocheckpoint sqlite3_api->wal_autocheckpoint +#define sqlite3_wal_checkpoint sqlite3_api->wal_checkpoint +#define sqlite3_wal_hook sqlite3_api->wal_hook #endif /* SQLITE_CORE */ #define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api = 0;