From: dan Date: Wed, 7 Jan 2015 17:11:11 +0000 (+0000) Subject: Add the 'rebuild' and 'delete-all' commands. X-Git-Tag: version-3.8.11~114^2~110 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=96a32bb51ce15ee759eafc010086f1ccdcc2436d;p=thirdparty%2Fsqlite.git Add the 'rebuild' and 'delete-all' commands. FossilOrigin-Name: 0cb2fed525778d96237b5b0943047665e1f636d1 --- diff --git a/ext/fts5/fts5.c b/ext/fts5/fts5.c index 3ddd23997e..c52b80f0ea 100644 --- a/ext/fts5/fts5.c +++ b/ext/fts5/fts5.c @@ -1027,6 +1027,15 @@ static int fts5SeekCursor(Fts5Cursor *pCsr){ return rc; } +static void fts5SetVtabError(Fts5Table *p, const char *zFormat, ...){ + int rc; + va_list ap; /* ... printf arguments */ + va_start(ap, zFormat); + assert( p->base.zErrMsg==0 ); + p->base.zErrMsg = sqlite3_vmprintf(zFormat, ap); + va_end(ap); +} + /* ** This function is called to handle an FTS INSERT command. In other words, ** an INSERT statement of the form: @@ -1047,11 +1056,31 @@ static int fts5SpecialInsert( sqlite3_value *pCmd, /* Value inserted into special column */ sqlite3_value *pVal /* Value inserted into rowid column */ ){ + Fts5Config *pConfig = pTab->pConfig; const char *z = (const char*)sqlite3_value_text(pCmd); int rc = SQLITE_OK; int bError = 0; - if( 0==sqlite3_stricmp("integrity-check", z) ){ + if( 0==sqlite3_stricmp("delete-all", z) ){ + if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ + fts5SetVtabError(pTab, + "'delete-all' may only be used with a " + "contentless or external content fts5 table" + ); + rc = SQLITE_ERROR; + }else{ + rc = sqlite3Fts5StorageDeleteAll(pTab->pStorage); + } + }else if( 0==sqlite3_stricmp("rebuild", z) ){ + if( pConfig->eContent==FTS5_CONTENT_NONE ){ + fts5SetVtabError(pTab, + "'rebuild' may not be used with a contentless fts5 table" + ); + rc = SQLITE_ERROR; + }else{ + rc = sqlite3Fts5StorageRebuild(pTab->pStorage); + } + }else if( 0==sqlite3_stricmp("integrity-check", z) ){ rc = sqlite3Fts5StorageIntegrity(pTab->pStorage); }else{ rc = sqlite3Fts5ConfigSetValue(pTab->pConfig, z, pVal, &bError); diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index 0908cc1248..1b9160f05b 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -93,30 +93,9 @@ typedef void (*fts5_extension_function)( ** xRowid: ** Returns the rowid of the current row. ** -** xPoslist: -** Iterate through phrase instances in the current row. If the iPhrase -** argument is 0 or greater, then only instances of phrase iPhrase are -** visited. If it is less than 0, instances of all phrases are visited. -** -** At EOF, -1 is returned and output variable iPos set to -1. -** -** -** sqlite3_int64 iPos; -** int iPhrase; -** int ii = 0; -** -** while( (iPhrase = pFts->xPoslist(pFts, -1, &ii, &iPos) >= 0 ){ -** int iCol = FTS5_POS2COLUMN(iPos); -** int iOff = FTS5_POS2OFFSET(iPos); -** // An instance of phrase iPhrase at offset iOff of column iCol. -** } -** -** -** ** xTokenize: ** Tokenize text using the tokenizer belonging to the FTS5 table. ** -** ** xQueryPhrase(pFts5, iPhrase, pUserData, xCallback): ** This API function is used to query the FTS table for phrase iPhrase ** of the current query. Specifically, a query equivalent to: diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index e6ff083b5b..893d743cda 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -340,6 +340,8 @@ int sqlite3Fts5IndexSetCookie(Fts5Index*, int); */ int sqlite3Fts5IndexReads(Fts5Index *p); +int sqlite3Fts5IndexReinit(Fts5Index *p); + /* ** End of interface to code in fts5_index.c. **************************************************************************/ @@ -421,6 +423,9 @@ int sqlite3Fts5StorageConfigValue(Fts5Storage *p, const char*, sqlite3_value*); int sqlite3Fts5StorageSpecialDelete(Fts5Storage *p, i64 iDel, sqlite3_value**); +int sqlite3Fts5StorageDeleteAll(Fts5Storage *p); +int sqlite3Fts5StorageRebuild(Fts5Storage *p); + /* ** End of interface to code in fts5_storage.c. **************************************************************************/ diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index e447169dc8..8552a35792 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -3899,6 +3899,26 @@ int sqlite3Fts5IndexRollback(Fts5Index *p){ return SQLITE_OK; } +/* +** The %_data table is completely empty when this function is called. This +** function populates it with the initial structure objects for each index, +** and the initial version of the "averages" record (a zero-byte blob). +*/ +int sqlite3Fts5IndexReinit(Fts5Index *p){ + int i; + Fts5Structure s; + + memset(&s, 0, sizeof(Fts5Structure)); + for(i=0; ipConfig->nPrefix+1; i++){ + fts5StructureWrite(p, i, &s); + } + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3Fts5IndexSetAverages(p, (const u8*)"", 0); + } + + return fts5IndexReturn(p); +} + /* ** Open a new Fts5Index handle. If the bCreate argument is true, create ** and initialize the underlying %_data table. @@ -3927,20 +3947,11 @@ int sqlite3Fts5IndexOpen( if( p->zDataTbl==0 ){ rc = SQLITE_NOMEM; }else if( bCreate ){ - int i; - Fts5Structure s; rc = sqlite3Fts5CreateTable( pConfig, "data", "id INTEGER PRIMARY KEY, block BLOB", 0, pzErr ); if( rc==SQLITE_OK ){ - memset(&s, 0, sizeof(Fts5Structure)); - for(i=0; inPrefix+1; i++){ - fts5StructureWrite(p, i, &s); - } - rc = p->rc; - } - if( rc==SQLITE_OK ){ - rc = sqlite3Fts5IndexSetAverages(p, (const u8*)"", 0); + rc = sqlite3Fts5IndexReinit(p); } } diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index 177de8185b..8431f6dd4c 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -125,24 +125,39 @@ static int fts5StorageGetStmt( return rc; } -/* -** Drop the shadow table with the postfix zPost (e.g. "content"). Return -** SQLITE_OK if successful or an SQLite error code otherwise. -*/ -int sqlite3Fts5DropTable(Fts5Config *pConfig, const char *zPost){ + +static int fts5ExecPrintf( + sqlite3 *db, + char **pzErr, + const char *zFormat, + ... +){ int rc; - char *zSql = sqlite3_mprintf("DROP TABLE IF EXISTS %Q.'%q_%q'", - pConfig->zDb, pConfig->zName, zPost - ); + va_list ap; /* ... printf arguments */ + va_start(ap, zFormat); + char *zSql = sqlite3_vmprintf(zFormat, ap); + if( zSql==0 ){ rc = SQLITE_NOMEM; }else{ - rc = sqlite3_exec(pConfig->db, zSql, 0, 0, 0); + rc = sqlite3_exec(db, zSql, 0, 0, pzErr); sqlite3_free(zSql); } + + va_end(ap); return rc; } +/* +** Drop the shadow table with the postfix zPost (e.g. "content"). Return +** SQLITE_OK if successful or an SQLite error code otherwise. +*/ +int sqlite3Fts5DropTable(Fts5Config *pConfig, const char *zPost){ + return fts5ExecPrintf(pConfig->db, 0, "DROP TABLE IF EXISTS %Q.'%q_%q'", + pConfig->zDb, pConfig->zName, zPost + ); +} + /* ** Create the shadow table named zPost, with definition zDefn. Return ** SQLITE_OK if successful, or an SQLite error code otherwise. @@ -155,25 +170,19 @@ int sqlite3Fts5CreateTable( char **pzErr /* OUT: Error message */ ){ int rc; - char *zSql = sqlite3_mprintf("CREATE TABLE %Q.'%q_%q'(%s)%s", - pConfig->zDb, pConfig->zName, zPost, zDefn, - (bWithout ? " WITHOUT ROWID" :"") + char *zErr = 0; + + rc = fts5ExecPrintf(pConfig->db, &zErr, "CREATE TABLE %Q.'%q_%q'(%s)%s", + pConfig->zDb, pConfig->zName, zPost, zDefn, bWithout?" WITHOUT ROWID":"" ); - if( zSql==0 ){ - rc = SQLITE_NOMEM; - }else{ - char *zErr = 0; - assert( *pzErr==0 ); - rc = sqlite3_exec(pConfig->db, zSql, 0, 0, &zErr); - if( zErr ){ - *pzErr = sqlite3_mprintf( - "fts5: error creating shadow table %q_%s: %s", - pConfig->zName, zPost, zErr - ); - sqlite3_free(zErr); - } - sqlite3_free(zSql); + if( zErr ){ + *pzErr = sqlite3_mprintf( + "fts5: error creating shadow table %q_%s: %s", + pConfig->zName, zPost, zErr + ); + sqlite3_free(zErr); } + return rc; } @@ -462,7 +471,7 @@ int sqlite3Fts5StorageSpecialDelete( int rc; sqlite3_stmt *pDel; - assert( p->pConfig->eContent!=FTS5_CONTENT_NORMAL ); + assert( pConfig->eContent!=FTS5_CONTENT_NORMAL ); rc = fts5StorageLoadTotals(p, 1); /* Delete the index records */ @@ -502,7 +511,78 @@ int sqlite3Fts5StorageSpecialDelete( } return rc; +} +/* +** Delete all entries in the FTS5 index. +*/ +int sqlite3Fts5StorageDeleteAll(Fts5Storage *p){ + Fts5Config *pConfig = p->pConfig; + int rc; + + /* Delete the contents of the %_data and %_docsize tables. */ + rc = fts5ExecPrintf(pConfig->db, 0, + "DELETE FROM %Q.'%q_data';" + "DELETE FROM %Q.'%q_docsize';", + pConfig->zDb, pConfig->zName, + pConfig->zDb, pConfig->zName + ); + + /* Reinitialize the %_data table. This call creates the initial structure + ** and averages records. */ + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexReinit(p->pIndex); + } + return rc; +} + +int sqlite3Fts5StorageRebuild(Fts5Storage *p){ + Fts5Buffer buf = {0,0,0}; + Fts5Config *pConfig = p->pConfig; + sqlite3_stmt *pScan = 0; + Fts5InsertCtx ctx; + int rc; + + memset(&ctx, 0, sizeof(Fts5InsertCtx)); + ctx.pStorage = p; + rc = sqlite3Fts5StorageDeleteAll(p); + if( rc==SQLITE_OK ){ + rc = fts5StorageLoadTotals(p, 1); + } + + if( rc==SQLITE_OK ){ + rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN_ASC, &pScan, 0); + } + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pScan) ){ + i64 iRowid = sqlite3_column_int64(pScan, 0); + + sqlite3Fts5BufferZero(&buf); + rc = sqlite3Fts5IndexBeginWrite(p->pIndex, iRowid); + for(ctx.iCol=0; rc==SQLITE_OK && ctx.iColnCol; ctx.iCol++){ + ctx.szCol = 0; + rc = sqlite3Fts5Tokenize(pConfig, + (const char*)sqlite3_column_text(pScan, ctx.iCol+1), + sqlite3_column_bytes(pScan, ctx.iCol+1), + (void*)&ctx, + fts5StorageInsertCallback + ); + sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol); + p->aTotalSize[ctx.iCol] += (i64)ctx.szCol; + } + p->nTotalRow++; + + if( rc==SQLITE_OK ){ + rc = fts5StorageInsertDocsize(p, iRowid, &buf); + } + } + sqlite3_free(buf.p); + + /* Write the averages record */ + if( rc==SQLITE_OK ){ + rc = fts5StorageSaveTotals(p); + } + return rc; } /* diff --git a/ext/fts5/test/fts5content.test b/ext/fts5/test/fts5content.test index 934af7b3b2..948a4308b6 100644 --- a/ext/fts5/test/fts5content.test +++ b/ext/fts5/test/fts5content.test @@ -144,6 +144,47 @@ do_execsql_test 2.7 { WHERE fts_idx MATCH 't AND x'; } {{[t] h r e e [x] y z}} +#------------------------------------------------------------------------- +# Quick tests of the 'delete-all' command. +# +do_execsql_test 3.1 { + CREATE VIRTUAL TABLE t3 USING fts5(x, content=''); + INSERT INTO t3 VALUES('a b c'); + INSERT INTO t3 VALUES('d e f'); +} + +do_execsql_test 3.2 { + SELECT count(*) FROM t3_docsize; + SELECT count(*) FROM t3_data; +} {2 4} + +do_execsql_test 3.3 { + INSERT INTO t3(t3) VALUES('delete-all'); + SELECT count(*) FROM t3_docsize; + SELECT count(*) FROM t3_data; +} {0 2} + +do_execsql_test 3.4 { + INSERT INTO t3 VALUES('a b c'); + INSERT INTO t3 VALUES('d e f'); + SELECT rowid FROM t3 WHERE t3 MATCH 'e'; +} {2} + +do_execsql_test 3.5 { + SELECT rowid FROM t3 WHERE t3 MATCH 'c'; +} {1} + +do_execsql_test 3.6 { + SELECT count(*) FROM t3_docsize; + SELECT count(*) FROM t3_data; +} {2 4} + +do_execsql_test 3.7 { + CREATE VIRTUAL TABLE t4 USING fts5(x); +} {} +do_catchsql_test 3.8 { + INSERT INTO t4(t4) VALUES('delete-all'); +} {1 {'delete-all' may only be used with a contentless or external content fts5 table}} finish_test diff --git a/ext/fts5/test/fts5rebuild.test b/ext/fts5/test/fts5rebuild.test new file mode 100644 index 0000000000..dfaf28bc6e --- /dev/null +++ b/ext/fts5/test/fts5rebuild.test @@ -0,0 +1,50 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5rebuild + +do_execsql_test 1.1 { + CREATE VIRTUAL TABLE f1 USING fts5(a, b); + INSERT INTO f1(a, b) VALUES('one', 'o n e'); + INSERT INTO f1(a, b) VALUES('two', 't w o'); + INSERT INTO f1(a, b) VALUES('three', 't h r e e'); +} + +do_execsql_test 1.2 { + INSERT INTO f1(f1) VALUES('integrity-check'); +} {} + +do_execsql_test 1.3 { + INSERT INTO f1(f1) VALUES('rebuild'); +} {} + +do_execsql_test 1.4 { + INSERT INTO f1(f1) VALUES('integrity-check'); +} {} + +do_execsql_test 1.5 { + DELETE FROM f1_data; +} {} + +do_catchsql_test 1.6 { + INSERT INTO f1(f1) VALUES('integrity-check'); +} {1 {SQL logic error or missing database}} + +do_execsql_test 1.7 { + INSERT INTO f1(f1) VALUES('rebuild'); + INSERT INTO f1(f1) VALUES('integrity-check'); +} {} + +finish_test + diff --git a/manifest b/manifest index 5c31f642e1..a7855d306f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\siPos\sparameter\sfrom\sthe\stokenizer\scallback.\sFix\sthe\s"tokenchars"\sand\s"separators"\soptions\son\sthe\ssimple\stokenizer. -D 2015-01-06T19:08:26.571 +C Add\sthe\s'rebuild'\sand\s'delete-all'\scommands. +D 2015-01-07T17:11:11.301 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7cd23e4fc91004a6bd081623e1bc6932e44828c0 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -104,21 +104,21 @@ F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl 4199cb887040ee3c3cd59a5171ddb0566904586e F ext/fts5/extract_api_docs.tcl 6320db4a1d0722a4e2069e661381ad75e9889786 -F ext/fts5/fts5.c 9f6f6597410d9fe76db385955ad6be171c454331 -F ext/fts5/fts5.h cfafdf6f43f9402f999334382085e46f89d85ecf -F ext/fts5/fts5Int.h 8b338037a968da542a98bbbcdbb10bcf361ee2fe +F ext/fts5/fts5.c 66ca4324ea89dc727f01ea77eb48e5ba311be032 +F ext/fts5/fts5.h 0f8563e21ffa69cb87be4c2e24652fc41b441850 +F ext/fts5/fts5Int.h 00a8770e34b56f3db7eb29e5b110d2f7623ca959 F ext/fts5/fts5_aux.c 549aef152b0fd46020f5595d861b1fd60b3f9b4f F ext/fts5/fts5_buffer.c 32dd3c950392346ca69a0f1803501766c5c954f9 F ext/fts5/fts5_config.c 33534ca25198cc62c54ff7d285d455c57ad19399 F ext/fts5/fts5_expr.c 0320ae948e82cf7dca800463de7f5b6a808ba7c3 F ext/fts5/fts5_hash.c 63fa8379c5f2ac107d47c2b7d9ac04c95ef8a279 -F ext/fts5/fts5_index.c b58bcfba3fe4e53fbf2dc525ec25aa37b77ac9f0 -F ext/fts5/fts5_storage.c cd72f2839049d5277df0edd0cf5c801f33542b07 +F ext/fts5/fts5_index.c 4e612b2c91a57ec770869b6cc89caeec0f658107 +F ext/fts5/fts5_storage.c 844b9667030370e9bb1daf3f9e862716cddb1a22 F ext/fts5/fts5_tcl.c 1293fac2bb26903fd3d5cdee59c5885ba7e620d5 F ext/fts5/fts5_tokenize.c 4c30cf32c63e59bec5b38533e0a65987df262851 F ext/fts5/fts5_unicode2.c 9c7dd640d1f014bf5c3ee029759adfbb4d7e95a9 F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9 -F ext/fts5/test/fts5_common.tcl 08e939096a07eb77a7a986613e960f31d3cab2cc w test/fts5_common.tcl +F ext/fts5/test/fts5_common.tcl 08e939096a07eb77a7a986613e960f31d3cab2cc F ext/fts5/test/fts5aa.test 3941b54d7585153be0c5cf0026f7dd8cfef13ea9 F ext/fts5/test/fts5ab.test 91a3faac09ad9fab5f71494db6e4071963281536 F ext/fts5/test/fts5ac.test 48181b7c873da0e3b4a3316760fcb90d88e7fbd8 @@ -132,10 +132,11 @@ F ext/fts5/test/fts5aj.test 05b569f5c16ea3098fb1984eec5cf50dbdaae5d8 F ext/fts5/test/fts5ak.test dc7bcd087dea0451ec40bba173962a0ba3a1d8ce F ext/fts5/test/fts5al.test 633fdb3d974629d01ba7734d180dbc2ad8ed772a F ext/fts5/test/fts5auxdata.test c69b86092bf1a157172de5f9169731af3403179b -F ext/fts5/test/fts5content.test ed6a141b1fcaa8fc1cf719492a9e38b29f2a830b +F ext/fts5/test/fts5content.test 4234e0b11e003fe1e80472aa637f70464396fdd0 F ext/fts5/test/fts5ea.test 04695560a444fcc00c3c4f27783bdcfbf71f030c F ext/fts5/test/fts5fault1.test f3f4c6ed15cc7a4dc8d517c0d1969d8e5a35a65c F ext/fts5/test/fts5porter.test 50322599823cb8080a99f0ec0c39f7d0c12bcb5e +F ext/fts5/test/fts5rebuild.test 2a5e98205393487b4a732c8290999af7c0b907b4 F ext/fts5/test/fts5tokenizer.test f951bb9be29232bd057b0ac4d535b879d9cd9a89 F ext/fts5/test/fts5unicode.test 9ae93296e59917c1210336388f6d3b98051b50c9 F ext/fts5/test/fts5unicode2.test 64a5267fd6082fcb46439892ebd0cbaa5c38acee @@ -1271,7 +1272,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ce6a899baff7265a60c880098a9a57ea352b5415 -R ee708c3acc09a58536cb486296b83967 +P 65f0262fb82dbfd9f80233ac7c3108e2f2716c0a +R 3ced0a5a3507e3fd1f8ff1a5ee996e45 U dan -Z 63a8748a3e94829622d4a3b2bf209e0e +Z 06bf9328f4305b86890a4adcb16c3d2f diff --git a/manifest.uuid b/manifest.uuid index ccd2f86400..5026d1fa16 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -65f0262fb82dbfd9f80233ac7c3108e2f2716c0a \ No newline at end of file +0cb2fed525778d96237b5b0943047665e1f636d1 \ No newline at end of file