From: dan Date: Mon, 21 Jul 2014 11:44:47 +0000 (+0000) Subject: Fix the xColumnSize() extension API. X-Git-Tag: version-3.8.11~114^2~160 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=84d6fb394f71d0feea3cd5108daf6bc938c85585;p=thirdparty%2Fsqlite.git Fix the xColumnSize() extension API. FossilOrigin-Name: 19504c4108472d2ad1281221642b8bd06eb69f4e --- diff --git a/ext/fts5/fts5.c b/ext/fts5/fts5.c index eb688a7bc6..745a557128 100644 --- a/ext/fts5/fts5.c +++ b/ext/fts5/fts5.c @@ -619,8 +619,9 @@ static int fts5ApiColumnCount(Fts5Context *pCtx){ } static int fts5ApiColumnAvgSize(Fts5Context *pCtx, int iCol, int *pnToken){ - assert( 0 ); - return 0; + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); + return sqlite3Fts5StorageAvgsize(pTab->pStorage, iCol, pnToken); } static int fts5ApiTokenize( diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index f274baa3f9..fc78d64589 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -267,6 +267,9 @@ int sqlite3Fts5IndexInit(sqlite3*); void sqlite3Fts5IndexPgsz(Fts5Index *p, int pgsz); +int sqlite3Fts5IndexGetAverages(Fts5Index *p, Fts5Buffer *pBuf); +int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int); + /* ** End of interface to code in fts5_index.c. **************************************************************************/ @@ -297,7 +300,7 @@ int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt **); void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*); int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol); -int sqlite3Fts5StorageAvgsize(Fts5Storage *p, int *aCol); +int sqlite3Fts5StorageAvgsize(Fts5Storage *p, int iCol, int *pnAvg); /* diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c index 7345a68f29..deb2c8233e 100644 --- a/ext/fts5/fts5_aux.c +++ b/ext/fts5/fts5_aux.c @@ -43,11 +43,24 @@ static void fts5TestFunction( } memset(&s, 0, sizeof(Fts5Buffer)); + nCol = pApi->xColumnCount(pFts); + + if( zReq==0 ){ + sqlite3Fts5BufferAppendPrintf(&rc, &s, "columnavgsize "); + } + if( 0==zReq || 0==sqlite3_stricmp(zReq, "columnavgsize") ){ + if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "{"); + for(i=0; rc==SQLITE_OK && ixColumnAvgSize(pFts, i, &colsz); + sqlite3Fts5BufferAppendPrintf(&rc, &s, "%s%d", i==0?"":" ", colsz); + } + if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "}"); + } if( zReq==0 ){ sqlite3Fts5BufferAppendPrintf(&rc, &s, "columncount "); } - nCol = pApi->xColumnCount(pFts); if( 0==zReq || 0==sqlite3_stricmp(zReq, "columncount") ){ sqlite3Fts5BufferAppendPrintf(&rc, &s, "%d", nCol); } diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index e6ea440f5f..5050159206 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -734,7 +734,7 @@ static void fts5DataReference(Fts5Data *pData){ /* ** INSERT OR REPLACE a record into the %_data table. */ -static void fts5DataWrite(Fts5Index *p, i64 iRowid, u8 *pData, int nData){ +static void fts5DataWrite(Fts5Index *p, i64 iRowid, const u8 *pData, int nData){ if( p->rc!=SQLITE_OK ) return; if( p->pWriter==0 ){ @@ -2734,6 +2734,7 @@ int sqlite3Fts5IndexOpen( } rc = p->rc; } + sqlite3Fts5IndexSetAverages(p, (const u8*)"", 0); } if( rc ){ @@ -3619,3 +3620,22 @@ void sqlite3Fts5IterClose(Fts5IndexIter *pIter){ } } +/* +** Read the "averages" record into the buffer supplied as the second +** argument. Return SQLITE_OK if successful, or an SQLite error code +** if an error occurs. +*/ +int sqlite3Fts5IndexGetAverages(Fts5Index *p, Fts5Buffer *pBuf){ + fts5DataReadOrBuffer(p, pBuf, FTS5_AVERAGES_ROWID); + return p->rc; +} + +/* +** Replace the current "averages" record with the contents of the buffer +** supplied as the second argument. +*/ +int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8 *pData, int nData){ + fts5DataWrite(p, FTS5_AVERAGES_ROWID, pData, nData); + return p->rc; +} + diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index d3715dac07..fdab68d465 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -17,6 +17,8 @@ struct Fts5Storage { Fts5Config *pConfig; Fts5Index *pIndex; + i64 nTotalRow; /* Total number of rows in FTS table */ + i64 *aTotalSize; /* Total sizes of each column */ sqlite3_stmt *aStmt[9]; }; @@ -168,11 +170,15 @@ int sqlite3Fts5StorageOpen( ){ int rc; Fts5Storage *p; /* New object */ + int nByte; /* Bytes of space to allocate */ - *pp = p = (Fts5Storage*)sqlite3_malloc(sizeof(Fts5Storage)); + nByte = sizeof(Fts5Storage) /* Fts5Storage object */ + + pConfig->nCol * sizeof(i64); /* Fts5Storage.aTotalSize[] */ + *pp = p = (Fts5Storage*)sqlite3_malloc(nByte); if( !p ) return SQLITE_NOMEM; - memset(p, 0, sizeof(Fts5Storage)); + memset(p, 0, nByte); + p->aTotalSize = (i64*)&p[1]; p->pConfig = pConfig; p->pIndex = pIndex; @@ -285,7 +291,9 @@ static int fts5StorageDeleteFromIndex(Fts5Storage *p, i64 iDel){ (void*)&ctx, fts5StorageInsertCallback ); + p->aTotalSize[iCol-1] -= (i64)ctx.szCol; } + p->nTotalRow--; } rc2 = sqlite3_reset(pSeek); if( rc==SQLITE_OK ) rc = rc2; @@ -315,6 +323,62 @@ static int fts5StorageInsertDocsize( return rc; } +/* +** Load the contents of the "averages" record from disk into the +** p->nTotalRow and p->aTotalSize[] variables. +** +** Return SQLITE_OK if successful, or an SQLite error code if an error +** occurs. +*/ +static int fts5StorageLoadTotals(Fts5Storage *p){ + int nCol = p->pConfig->nCol; + Fts5Buffer buf; + int rc; + memset(&buf, 0, sizeof(buf)); + + memset(p->aTotalSize, 0, sizeof(i64) * nCol); + p->nTotalRow = 0; + rc = sqlite3Fts5IndexGetAverages(p->pIndex, &buf); + if( rc==SQLITE_OK && buf.n ){ + int i = 0; + int iCol; + i += getVarint(&buf.p[i], (u64*)&p->nTotalRow); + for(iCol=0; iaTotalSize[iCol]); + } + } + sqlite3_free(buf.p); + + return rc; +} + +/* +** Store the current contents of the p->nTotalRow and p->aTotalSize[] +** variables in the "averages" record on disk. +** +** Return SQLITE_OK if successful, or an SQLite error code if an error +** occurs. +*/ +static int fts5StorageSaveTotals(Fts5Storage *p){ + int nCol = p->pConfig->nCol; + int i; + Fts5Buffer buf; + int rc = SQLITE_OK; + memset(&buf, 0, sizeof(buf)); + + sqlite3Fts5BufferAppendVarint(&rc, &buf, p->nTotalRow); + for(i=0; iaTotalSize[i]); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexSetAverages(p->pIndex, buf.p, buf.n); + } + sqlite3_free(buf.p); + + return rc; +} + + /* ** Insert a new row into the FTS table. */ @@ -333,15 +397,18 @@ int sqlite3Fts5StorageInsert( Fts5Buffer buf; /* Buffer used to build up %_docsize blob */ memset(&buf, 0, sizeof(Fts5Buffer)); + rc = fts5StorageLoadTotals(p); /* Insert the new row into the %_content table. */ - if( eConflict==SQLITE_REPLACE ){ - eStmt = FTS5_STMT_REPLACE_CONTENT; - if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){ - rc = fts5StorageDeleteFromIndex(p, sqlite3_value_int64(apVal[1])); + if( rc==SQLITE_OK ){ + if( eConflict==SQLITE_REPLACE ){ + eStmt = FTS5_STMT_REPLACE_CONTENT; + if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){ + rc = fts5StorageDeleteFromIndex(p, sqlite3_value_int64(apVal[1])); + } + }else{ + eStmt = FTS5_STMT_INSERT_CONTENT; } - }else{ - eStmt = FTS5_STMT_INSERT_CONTENT; } if( rc==SQLITE_OK ){ rc = fts5StorageGetStmt(p, eStmt, &pInsert); @@ -367,7 +434,9 @@ int sqlite3Fts5StorageInsert( fts5StorageInsertCallback ); sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol); + p->aTotalSize[ctx.iCol] += (i64)ctx.szCol; } + p->nTotalRow++; /* Write the %_docsize record */ if( rc==SQLITE_OK ){ @@ -375,6 +444,11 @@ int sqlite3Fts5StorageInsert( } sqlite3_free(buf.p); + /* Write the averages record */ + if( rc==SQLITE_OK ){ + rc = fts5StorageSaveTotals(p); + } + return rc; } @@ -538,7 +612,16 @@ int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol){ return rc; } -int sqlite3Fts5StorageAvgsize(Fts5Storage *p, int *aCol){ - return 0; +int sqlite3Fts5StorageAvgsize(Fts5Storage *p, int iCol, int *pnAvg){ + int rc = fts5StorageLoadTotals(p); + if( rc==SQLITE_OK ){ + int nAvg = 1; + if( p->nTotalRow ){ + nAvg = (int)((p->aTotalSize[iCol] + (p->nTotalRow/2)) / p->nTotalRow); + if( nAvg<1 ) nAvg = 1; + *pnAvg = nAvg; + } + } + return rc; } diff --git a/manifest b/manifest index 6019a0fb5d..d18e5f644b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssimple\stests\sfor\sthe\sxColumnText()\sextension\sapi. -D 2014-07-19T20:27:54.153 +C Fix\sthe\sxColumnSize()\sextension\sAPI. +D 2014-07-21T11:44:47.659 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -103,15 +103,15 @@ F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl dc6f268eb526710e2c6e496c372471d773d0c368 -F ext/fts5/fts5.c fbd94670336eb95a26c705c1e4c188818720c888 +F ext/fts5/fts5.c 5bf93402f9bafa55181dfa70c3a785a41af31025 F ext/fts5/fts5.h 84060c2fe91aa03a783cc993f313d8a5b22cbe11 -F ext/fts5/fts5Int.h 5105506a180942811aa7104867518bc84d36c17a -F ext/fts5/fts5_aux.c 75f9abf7a7ccc7712357f04eeb6297c64095528d +F ext/fts5/fts5Int.h 12d03496152b716e63a5380e396b776fbefa2065 +F ext/fts5/fts5_aux.c 9f0487715cd9933f2268620b41fb47f78a389297 F ext/fts5/fts5_buffer.c 00361d4a70040ebd2c32bc349ab708ff613a1749 F ext/fts5/fts5_config.c 94f1b4cb4de6a7cd5780c14adb0198e289df8cef F ext/fts5/fts5_expr.c 288b3e016253eab69ea8cefbff346a4697b44291 -F ext/fts5/fts5_index.c 9ff3008e903aa9077b0a7a7aa76ab6080eb07a36 -F ext/fts5/fts5_storage.c fcf66173e55927cee0675ecfb1038d0000e4fa10 +F ext/fts5/fts5_index.c 68d2d41b5c6d2f8838c3d6ebdc8b242718b8e997 +F ext/fts5/fts5_storage.c f722b080b9794f9e49cc4d36f0d9fb516cb7f309 F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43 F ext/icu/icu.c d415ccf984defeb9df2c0e1afcfaa2f6dc05eacb @@ -595,11 +595,11 @@ F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7 F test/fts4merge4.test d895b1057a7798b67e03455d0fa50e9ea836c47b F test/fts4noti.test 524807f0c36d49deea7920cdd4cd687408b58849 F test/fts4unicode.test 01ec3fe2a7c3cfff3b4c0581b83caa11b33efa36 -F test/fts5aa.test c8d3b9694f6b2864161c7437408464a535d19343 +F test/fts5aa.test 0f5d29bf0a86b9dff0906c9e166d624c591d3437 F test/fts5ab.test dc04ed48cf93ca957d174406e6c192f2ff4f3397 F test/fts5ac.test 9be418d037763f4cc5d86f4239db41fc86bb4f85 F test/fts5ad.test 2ed38bbc865678cb2905247120d02ebba7f20e07 -F test/fts5ae.test c95b106236ea2f9f151715311ad0a2e45c49ccc1 +F test/fts5ae.test b856782549ae0c56628e0980444fc38cf20b41a0 F test/fts5ea.test ff43b40f8879ba50b82def70f2ab67c195d1a1d4 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d F test/func.test ae97561957aba6ca9e3a7b8a13aac41830d701ef @@ -1195,7 +1195,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 43fcb844726cfeeb1c8a0dbfaa0d2ca22e6ac16c -R fe613ccf4d230813074cd2e3b5b79d5f +P 1e9053abdaf5e128d44504ee00dfd909dc25f378 +R 962590916dedf53a395813418f70a86f U dan -Z 057df6fbedf27e3f40e741066d4f0911 +Z 393e4d18f784466676fdeef71a737c83 diff --git a/manifest.uuid b/manifest.uuid index 7556250b6e..766db6dab7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1e9053abdaf5e128d44504ee00dfd909dc25f378 \ No newline at end of file +19504c4108472d2ad1281221642b8bd06eb69f4e \ No newline at end of file diff --git a/test/fts5aa.test b/test/fts5aa.test index f8b8b54d08..9159cf3bc6 100644 --- a/test/fts5aa.test +++ b/test/fts5aa.test @@ -129,6 +129,7 @@ foreach {i x y} { #------------------------------------------------------------------------- # +breakpoint reset_db do_execsql_test 6.0 { CREATE VIRTUAL TABLE t1 USING fts5(x,y); diff --git a/test/fts5ae.test b/test/fts5ae.test index f5a53625f7..3fa99d3d51 100644 --- a/test/fts5ae.test +++ b/test/fts5ae.test @@ -128,7 +128,7 @@ do_execsql_test 4.1 { } #------------------------------------------------------------------------- -# Test that the xColumnSize() API works. +# Test that the xColumnSize() and xColumnAvgsize() APIs work. # reset_db @@ -156,6 +156,26 @@ do_execsql_test 5.2 { 1 {{a b c d} {e f g h i j}} } +do_execsql_test 5.3 { + SELECT rowid, fts5_test(t5, 'columnavgsize') FROM t5 WHERE t5 MATCH 'a' + ORDER BY rowid DESC; +} { + 3 {2 2} + 2 {2 2} + 1 {2 2} +} + +do_execsql_test 5.4 { + INSERT INTO t5 VALUES('x y z', 'v w x y z'); + SELECT rowid, fts5_test(t5, 'columnavgsize') FROM t5 WHERE t5 MATCH 'a' + ORDER BY rowid DESC; +} { + 3 {2 3} + 2 {2 3} + 1 {2 3} +} + + finish_test