*/
int sqlite3Fts5IndexInit(sqlite3*);
-int sqlite3Fts5IndexSetCookie(Fts5Index*, int);
+int sqlite3Fts5IndexIncrCookie(Fts5Index*);
/*
** Return the total number of entries read from the %_data table by
int sqlite3Fts5IndexReinit(Fts5Index *p);
int sqlite3Fts5IndexOptimize(Fts5Index *p);
int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge);
-int sqlite3Fts5IndexReset(Fts5Index *p);
+int sqlite3Fts5IndexNewTrans(Fts5Index *p);
int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
+void sqlite3Fts5IndexCloseReader(Fts5Index*);
+
/*
** End of interface to code in fts5_index.c.
**************************************************************************/
**
** Low level access to the FTS index stored in the database file. The
** routines in this file file implement all read and write access to the
-** %_data table. Other parts of the system access this functionality via
-** the interface defined in fts5Int.h.
+** %_data and %_idx tables. Other parts of the system access this
+** functionality via the interface defined in fts5Int.h.
*/
/*
** Overview:
**
-** The %_data table contains all the FTS indexes for an FTS5 virtual table.
-** As well as the main term index, there may be up to 31 prefix indexes.
-** The format is similar to FTS3/4, except that:
+** The %_data table contains the FTS index for an FTS5 virtual table.
+** All entries, for terms and prefixes, are stored in a single data
+** structure. The format is similar to FTS3/4, but differs in the
+** following ways:
**
** * all segment b-tree leaf data is stored in fixed size page records
** (e.g. 1000 bytes). A single doclist may span multiple pages. Care is
**
** * extra fields in the "structure record" record the state of ongoing
** incremental merge operations.
-**
*/
-#define FTS5_OPT_WORK_UNIT 1000 /* Number of leaf pages per optimize step */
-#define FTS5_WORK_UNIT 64 /* Number of leaf pages in unit of work */
-
-#define FTS5_MIN_DLIDX_SIZE 4 /* Add dlidx if this many empty pages */
-
-#define FTS5_MAIN_PREFIX '0'
-
-#if FTS5_MAX_PREFIX_INDEXES > 31
-# error "FTS5_MAX_PREFIX_INDEXES is too large"
-#endif
-
/*
-** Details:
+** Contents of %_data table:
**
** The %_data table managed by this module,
**
** CREATE TABLE %_data(id INTEGER PRIMARY KEY, block BLOB);
**
-** , contains the following 5 types of records. See the comments surrounding
+** , contains the following 4 types of records. See the comments surrounding
** the FTS5_*_ROWID macros below for a description of how %_data rowids are
-** assigned to each fo them.
+** assigned to each of them.
**
** 1. Structure Records:
**
** varint : size of first term
** blob: first term data
**
-** 5. Segment doclist indexes:
+** 4. Segment doclist indexes:
**
** Doclist indexes are themselves b-trees, however they usually consist of
** a single leaf record only. The format of each doclist index leaf page
**
*/
+
+#define FTS5_OPT_WORK_UNIT 1000 /* Number of leaf pages per optimize step */
+#define FTS5_WORK_UNIT 64 /* Number of leaf pages in unit of work */
+#define FTS5_MIN_DLIDX_SIZE 4 /* Add dlidx if this many empty pages */
+
+/* All entries for regular terms in the FTS index are prefixed with '0'.
+** Entries for the first prefix index are prefixed with '1'. And so on,
+** up to ('0'+31). */
+#define FTS5_MAIN_PREFIX '0'
+
+#if FTS5_MAX_PREFIX_INDEXES > 31
+# error "FTS5_MAX_PREFIX_INDEXES is too large"
+#endif
+
+
/*
** Rowids for the averages and structure records in the %_data table.
*/
sqlite3_stmt *pIdxSelect;
int nRead; /* Total number of blocks read */
- sqlite3_stmt *pDataVersion;
+ /* In-memory cache of the 'structure' record */
+ sqlite3_stmt *pDataVersion; /* PRAGMA <db>.data_version */
i64 iStructVersion; /* data_version when pStruct read */
Fts5Structure *pStruct; /* Current db structure (or NULL) */
};
/*
** Close the read-only blob handle, if it is open.
*/
-static void fts5CloseReader(Fts5Index *p){
+void sqlite3Fts5IndexCloseReader(Fts5Index *p){
if( p->pReader ){
sqlite3_blob *pReader = p->pReader;
p->pReader = 0;
assert( p->pReader==0 );
p->pReader = pBlob;
if( rc!=SQLITE_OK ){
- fts5CloseReader(p);
+ sqlite3Fts5IndexCloseReader(p);
}
if( rc==SQLITE_ABORT ) rc = SQLITE_OK;
}
if( p->pDataVersion==0 ){
p->rc = fts5IndexPrepareStmt(p, &p->pDataVersion,
sqlite3_mprintf("PRAGMA %Q.data_version", p->pConfig->zDb)
- );
+ );
if( p->rc ) return 0;
}
return iVersion;
}
+/*
+** If there is currently no cache of the index structure in memory, load
+** one from the database.
+*/
+static void fts5StructureCache(Fts5Index *p){
+ if( p->pStruct==0 ){
+ p->iStructVersion = fts5IndexDataVersion(p);
+ if( p->rc==SQLITE_OK ){
+ p->pStruct = fts5StructureReadUncached(p);
+ }
+ }
+}
+
/*
** Read, deserialize and return the structure record.
**
** is called, it is a no-op.
*/
static Fts5Structure *fts5StructureRead(Fts5Index *p){
-
- if( p->pStruct==0 ){
- p->iStructVersion = fts5IndexDataVersion(p);
- if( p->rc==SQLITE_OK ){
- p->pStruct = fts5StructureReadUncached(p);
- }
- }
+ fts5StructureCache(p);
#if 0
else{
return rc;
}
-typedef struct Fts5FlushCtx Fts5FlushCtx;
-struct Fts5FlushCtx {
- Fts5Index *pIdx;
- Fts5SegWriter writer;
-};
-
/*
** Buffer aBuf[] contains a list of varints, all small enough to fit
** in a 32-bit integer. Return the size of the largest prefix of this
int sqlite3Fts5IndexSync(Fts5Index *p, int bCommit){
assert( p->rc==SQLITE_OK );
fts5IndexFlush(p);
- if( bCommit ) fts5CloseReader(p);
+ if( bCommit ) sqlite3Fts5IndexCloseReader(p);
return fts5IndexReturn(p);
}
** records must be invalidated.
*/
int sqlite3Fts5IndexRollback(Fts5Index *p){
- fts5CloseReader(p);
+ sqlite3Fts5IndexCloseReader(p);
fts5IndexDiscardData(p);
fts5StructureInvalidate(p);
+ p->pConfig->iCookie = -1;
/* assert( p->rc==SQLITE_OK ); */
return SQLITE_OK;
}
if( p->rc ){
sqlite3Fts5IterClose(&pRet->base);
pRet = 0;
- fts5CloseReader(p);
+ sqlite3Fts5IndexCloseReader(p);
}
*ppIter = &pRet->base;
Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
Fts5Index *pIndex = pIter->pIndex;
fts5MultiIterFree(pIter);
- fts5CloseReader(pIndex);
+ sqlite3Fts5IndexCloseReader(pIndex);
}
}
/*
** Return the total number of blocks this module has read from the %_data
-** table since it was created.
+** table (since it was created by sqlite3Fts5IndexOpen).
*/
int sqlite3Fts5IndexReads(Fts5Index *p){
return p->nRead;
}
/*
-** Set the 32-bit cookie value stored at the start of all structure
-** records to the value passed as the second argument.
-**
-** Return SQLITE_OK if successful, or an SQLite error code if an error
-** occurs.
+** Increment the value of the configuration cookie stored as the first
+** 32-bits of the structure record in the database. This is done after
+** modifying the contents of the %_config table.
*/
-int sqlite3Fts5IndexSetCookie(Fts5Index *p, int iNew){
- int rc; /* Return code */
- Fts5Config *pConfig = p->pConfig; /* Configuration object */
- u8 aCookie[4]; /* Binary representation of iNew */
- sqlite3_blob *pBlob = 0;
-
- assert( p->rc==SQLITE_OK );
- sqlite3Fts5Put32(aCookie, iNew);
-
- rc = sqlite3_blob_open(pConfig->db, pConfig->zDb, p->zDataTbl,
- "block", FTS5_STRUCTURE_ROWID, 1, &pBlob
- );
- if( rc==SQLITE_OK ){
- sqlite3_blob_write(pBlob, aCookie, 4, 0);
- rc = sqlite3_blob_close(pBlob);
- }
-
- return rc;
-}
-
-int sqlite3Fts5IndexLoadConfig(Fts5Index *p){
+int sqlite3Fts5IndexIncrCookie(Fts5Index *p){
Fts5Structure *pStruct;
pStruct = fts5StructureRead(p);
+ p->pConfig->iCookie++;
+ fts5StructureWrite(p, pStruct);
fts5StructureRelease(pStruct);
return fts5IndexReturn(p);
}
+/*
+** Ensure the contents of the %_config table have been loaded into memory.
+*/
+int sqlite3Fts5IndexLoadConfig(Fts5Index *p){
+ fts5StructureCache(p);
+ return fts5IndexReturn(p);
+}
+
+int sqlite3Fts5IndexNewTrans(Fts5Index *p){
+ assert( p->pStruct==0 || p->iStructVersion!=0 );
+ if( p->pConfig->iCookie<0 || fts5IndexDataVersion(p)!=p->iStructVersion ){
+ fts5StructureInvalidate(p);
+ }
+ return fts5IndexReturn(p);
+}
+
+
/*************************************************************************
**************************************************************************
return rc;
}
-
-int sqlite3Fts5IndexReset(Fts5Index *p){
- assert( p->pStruct==0 || p->iStructVersion!=0 );
- if( fts5IndexDataVersion(p)!=p->iStructVersion ){
- fts5StructureInvalidate(p);
- }
- return fts5IndexReturn(p);
-}
for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){
if( pCsr->base.pVtab==(sqlite3_vtab*)pTab ) return SQLITE_OK;
}
- return sqlite3Fts5StorageReset(pTab->pStorage);
+ return sqlite3Fts5IndexNewTrans(pTab->pIndex);
}
/*
int rc; /* Return code */
rc = fts5NewTransaction(pTab);
+ if( rc==SQLITE_OK ){
+ rc = sqlite3Fts5IndexLoadConfig(pTab->pIndex);
+ }
if( rc==SQLITE_OK ){
nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int);
pCsr = (Fts5Cursor*)sqlite3_malloc(nByte);
rc = SQLITE_NOMEM;
}
}
+ if( rc!=SQLITE_OK ){
+ sqlite3Fts5IndexCloseReader(pTab->pIndex);
+ }
*ppCsr = (sqlite3_vtab_cursor*)pCsr;
return rc;
}
*pp = pCsr->pNext;
sqlite3_free(pCsr);
+ sqlite3Fts5IndexCloseReader(pTab->pIndex);
}
return SQLITE_OK;
}
** Implementation of xBegin() method.
*/
static int fts5BeginMethod(sqlite3_vtab *pVtab){
- fts5CheckTransactionState((Fts5Table*)pVtab, FTS5_BEGIN, 0);
- fts5NewTransaction((Fts5Table*)pVtab);
- return SQLITE_OK;
+ Fts5Table *pTab = (Fts5Table*)pVtab;
+ int rc;
+ rc = fts5NewTransaction(pTab);
+ if( rc!=SQLITE_OK ){
+ sqlite3Fts5IndexCloseReader(pTab->pIndex);
+ }
+#ifdef SQLITE_DEBUG
+ if( rc==SQLITE_OK ) fts5CheckTransactionState(pTab, FTS5_BEGIN, 0);
+#endif
+ return rc;
}
/*
return sqlite3Fts5IndexMerge(p->pIndex, nMerge);
}
-int sqlite3Fts5StorageReset(Fts5Storage *p){
- return sqlite3Fts5IndexReset(p->pIndex);
-}
-
/*
** Allocate a new rowid. This is used for "external content" tables when
** a NULL value is inserted into the rowid column. The new rowid is allocated
rc = sqlite3_reset(pReplace);
}
if( rc==SQLITE_OK && pVal ){
- int iNew = p->pConfig->iCookie + 1;
- rc = sqlite3Fts5IndexSetCookie(p->pIndex, iNew);
- if( rc==SQLITE_OK ){
- p->pConfig->iCookie = iNew;
- }
+ rc = sqlite3Fts5IndexIncrCookie(p->pIndex);
}
return rc;
}
do_faultsim_test 4 -faults oom-* -body {
db eval {SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'}
} -test {
- faultsim_test_result {0 {0 {} 3}}
+ faultsim_test_result {0 {0 {} 4}}
}
#-------------------------------------------------------------------------
--- /dev/null
+# 2016 March 26
+#
+# The author disclaims copyright to this source code. In place of
+# a legal notice, here is a blessing:
+#
+# May you do good and not evil.
+# May you find forgiveness for yourself and forgive others.
+# May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+# This file is focused on OOM errors.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+source $testdir/malloc_common.tcl
+set testprefix fts5faultC
+return_if_no_fts5
+
+#--------------------------------------------------------------------------
+# Test that if an OOM error occurs while trying to set a configuration
+# option, the in-memory and on-disk configurations are not left in an
+# inconsistent state.
+#
+
+
+
+proc posrowid {cmd} { $cmd xRowid }
+proc negrowid {cmd} { expr -1 * [$cmd xRowid] }
+
+sqlite3_fts5_create_function db posrowid posrowid
+sqlite3_fts5_create_function db negrowid negrowid
+
+do_execsql_test 1.0.0 {
+ CREATE VIRTUAL TABLE t1 USING fts5(x);
+ INSERT INTO t1 VALUES('a b c');
+ INSERT INTO t1 VALUES('d a d');
+ INSERT INTO t1 VALUES('c b a');
+}
+do_execsql_test 1.0.1 {
+ INSERT INTO t1(t1, rank) VALUES('rank', 'posrowid()');
+ SELECT rowid FROM t1('a') ORDER BY rank;
+} {1 2 3}
+do_execsql_test 1.0.2 {
+ INSERT INTO t1(t1, rank) VALUES('rank', 'negrowid()');
+ SELECT rowid FROM t1('a') ORDER BY rank;
+} {3 2 1}
+
+faultsim_save_and_close
+do_faultsim_test 1.1 -faults oom-* -prep {
+ faultsim_restore_and_reopen
+ sqlite3_fts5_create_function db posrowid posrowid
+ sqlite3_fts5_create_function db negrowid negrowid
+ execsql { SELECT * FROM t1('*reads') }
+} -body {
+ execsql { INSERT INTO t1(t1, rank) VALUES('rank', 'posrowid()') }
+} -test {
+
+ faultsim_test_result [list 0 {}]
+ sqlite3 db2 test.db
+ set ex [db2 one { SELECT v FROM t1_config WHERE k='rank' }]
+ switch -- $ex {
+ "posrowid()" { set ex {1 2 3} }
+ "negrowid()" { set ex {3 2 1} }
+ default { error 1 }
+ }
+
+ set res [db eval { SELECT rowid FROM t1('a') ORDER BY rank }]
+ if {$res != $ex} {
+ error "2: expected {$ex} got {$res}"
+ }
+ db2 close
+}
+
+
+
+
+finish_test
+
do_execsql_test 14.4 {
SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'
-} {0 {} 3}
+} {0 {} 4}
#-------------------------------------------------------------------------
reset_db
-C More\schanges\sto\sthe\sshellN.test\sscripts\sto\sget\sthem\sworking\son\sall\svariations\nof\sWindows.
-D 2016-03-26T15:36:36.768
+C Fix\sa\sproblem\swith\sOOM\shandling\swhen\ssetting\san\sfts5\sconfiguration\soption.
+D 2016-03-26T20:11:04.128
F Makefile.in f53429fb2f313c099283659d0df6f20f932c861f
F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
F Makefile.msc df0bf9ff7f8b3f4dd9fb4cc43f92fe58f6ec5c66
F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
F ext/fts5/fts5.h ff9c2782e8ed890b0de2f697a8d63971939e70c7
-F ext/fts5/fts5Int.h 3677076aecbf645a7f2a019115c6a4ec3272dd78
+F ext/fts5/fts5Int.h dde3c47b9f0e1603ab17cb6603b9409e426ce21d
F ext/fts5/fts5_aux.c daa57fb45216491814520bbb587e97bf81ced458
F ext/fts5/fts5_buffer.c 4c1502d4c956cd092c89ce4480867f9d8bf325cd
F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857
F ext/fts5/fts5_expr.c 5ca4bafe29aa3d27683c90e836192e4aefd20a3f
F ext/fts5/fts5_hash.c f3a7217c86eb8f272871be5f6aa1b6798960a337
-F ext/fts5/fts5_index.c fdd82bb421a5d1e64d004acb43f4dd9970c8d2b3
-F ext/fts5/fts5_main.c b4a0fc5bf17f2f1f056ee76cdd7d2af08b360f55
-F ext/fts5/fts5_storage.c 3309c6a8e34b974513016fd1ef47c83f5898f94c
+F ext/fts5/fts5_index.c 9019595c2dd17e1e109e0cfa7fc8bc449770f03e
+F ext/fts5/fts5_main.c 1e1e6e2d6df6b224fe9b2c75bcbe265c73b8712d
+F ext/fts5/fts5_storage.c e0aa8509e01eb22ae8e198c1de9c3200755c0d94
F ext/fts5/fts5_tcl.c f8731e0508299bd43f1a2eff7dbeaac870768966
F ext/fts5/fts5_test_mi.c 783b86697ebf773c18fc109992426c0173a055bc
F ext/fts5/fts5_test_tok.c db08af63673c3a7d39f053b36fd6e065017706be
F ext/fts5/test/fts5fault1.test e09040d3e17b8c0837101e8c79c8a874c4376fb7
F ext/fts5/test/fts5fault2.test d8c6c7f916ccbdfc10b2c69530e9dd3bc8313232
F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3
-F ext/fts5/test/fts5fault4.test dcbe3043c5611edd350191ea03a8daa190f0de5a
+F ext/fts5/test/fts5fault4.test 532b6dacb963016cbf7003196bd87fb366540277
F ext/fts5/test/fts5fault5.test 10c13a783de3f42a21e3e53e123b62ed0c3a1618
F ext/fts5/test/fts5fault6.test 9682664d679643ac6736e90c225526cc84073cda
F ext/fts5/test/fts5fault7.test cb14ea3c1f42394f06f2284abc58eecee6ff8080
F ext/fts5/test/fts5fault9.test e10e395428a9ea0596ebe752ff7123d16ab78e08
F ext/fts5/test/fts5faultA.test fa5d59c0ff62b7125cd14eee38ded1c46e15a7ea
F ext/fts5/test/fts5faultB.test 92ae906284062bf081b6c854afa54dcb1aa9ef88
+F ext/fts5/test/fts5faultC.test 10da76c6b69df05ff9095c7e56dc3a4b8d0e8f20
F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741
F ext/fts5/test/fts5fuzz1.test bece4695fc169b61ab236ada7931c6e4942cbef9
F ext/fts5/test/fts5hash.test 06f9309ccb4d5050a131594e9e47d0b21456837d
F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b
F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17
F ext/fts5/test/fts5rowid.test 16908a99d6efc9ba21081b4f2b86b3fc699839a6
-F ext/fts5/test/fts5simple.test cd23d4072ea095d652c9b6db12284cc642e49c98
+F ext/fts5/test/fts5simple.test f157c8b068f5e68473ad86bfe220b07a84e0209f
F ext/fts5/test/fts5simple2.test 98377ae1ff7749a42c21fe1a139c1ed312522c46
F ext/fts5/test/fts5simple3.test 8e71733b3d1b0e695011d02c68ebc5ca40b6124e
F ext/fts5/test/fts5synonym.test 6475d189c2e20d60795808f83e36bf9318708d48
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3bd499d3bdf4e80f83513966c2ee9dd11e67cbd1
-R 0f980ceb534899c3d74e7aadbdc990e8
-U drh
-Z 6df6f858842cee4837ac5cccff369bba
+P 8213c2f58167243411d29cc58e303b4be656f756
+R be57f13787d59ee7ecd47b81dbe52717
+T *branch * fts5
+T *sym-fts5 *
+T -sym-trunk *
+U dan
+Z 9cae65c93dd0ffab0ada61227cd993a5
-8213c2f58167243411d29cc58e303b4be656f756
\ No newline at end of file
+53b80a6d054a1c87311b3dc1c2bcfcc1b676b05a
\ No newline at end of file
projects / thirdparty / sqlite.git / commitdiff
