From ef480d37dcc22f4da07d797341c96462f61f9e91 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 21 Mar 2016 09:56:19 +0000 Subject: [PATCH] Change the way fts5 internally allocates segment ids in order to eliminated non-determinism from the module. FossilOrigin-Name: d6e2637df16764aa9723a30ea2eb8a631d28cb2b --- ext/fts5/fts5_index.c | 39 +++++++++++++++++++++++++++--------- ext/fts5/tool/fts5txt2db.tcl | 5 +++-- manifest | 17 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 41 insertions(+), 22 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 323e6cefdc..44fba94c84 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -3453,18 +3453,35 @@ static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){ if( pStruct->nSegment>=FTS5_MAX_SEGMENT ){ p->rc = SQLITE_FULL; }else{ - while( iSegid==0 ){ - int iLvl, iSeg; - sqlite3_randomness(sizeof(u32), (void*)&iSegid); - iSegid = iSegid & ((1 << FTS5_DATA_ID_B)-1); - for(iLvl=0; iLvlnLevel; iLvl++){ - for(iSeg=0; iSegaLevel[iLvl].nSeg; iSeg++){ - if( iSegid==pStruct->aLevel[iLvl].aSeg[iSeg].iSegid ){ - iSegid = 0; - } + /* FTS5_MAX_SEGMENT is currently defined as 2000. So the following + ** array is 63 elements, or 252 bytes, in size. */ + u32 aUsed[(FTS5_MAX_SEGMENT+31) / 32]; + int iLvl, iSeg; + int i; + u32 mask; + memset(aUsed, 0, sizeof(aUsed)); + for(iLvl=0; iLvlnLevel; iLvl++){ + for(iSeg=0; iSegaLevel[iLvl].nSeg; iSeg++){ + int iId = pStruct->aLevel[iLvl].aSeg[iSeg].iSegid; + if( iId<=FTS5_MAX_SEGMENT ){ + aUsed[(iId-1) / 32] |= 1 << ((iId-1) % 32); } } } + + for(i=0; aUsed[i]==0xFFFFFFFF; i++); + mask = aUsed[i]; + for(iSegid=0; mask & (1 << iSegid); iSegid++); + iSegid += 1 + i*32; + +#ifdef SQLITE_DEBUG + for(iLvl=0; iLvlnLevel; iLvl++){ + for(iSeg=0; iSegaLevel[iLvl].nSeg; iSeg++){ + assert( iSegid!=pStruct->aLevel[iLvl].aSeg[iSeg].iSegid ); + } + } + assert( iSegid>0 && iSegid<=FTS5_MAX_SEGMENT ); +#endif } } @@ -3909,7 +3926,9 @@ static void fts5WriteFinish( fts5WriteFlushLeaf(p, pWriter); } *pnLeaf = pLeaf->pgno-1; - fts5WriteFlushBtree(p, pWriter); + if( pLeaf->pgno>1 ){ + fts5WriteFlushBtree(p, pWriter); + } } fts5BufferFree(&pLeaf->term); fts5BufferFree(&pLeaf->buf); diff --git a/ext/fts5/tool/fts5txt2db.tcl b/ext/fts5/tool/fts5txt2db.tcl index d5df971d4d..4766b00b06 100644 --- a/ext/fts5/tool/fts5txt2db.tcl +++ b/ext/fts5/tool/fts5txt2db.tcl @@ -17,6 +17,7 @@ proc process_cmdline {} { {detail "full" "Fts5 detail mode to use"} {repeat 1 "Load each file this many times"} {prefix "" "Fts prefix= option"} + {trans 1 "True to use a transaction"} database file... } { @@ -214,7 +215,7 @@ foreach c [lrange $cols 1 end] { } append sql ")" -db eval BEGIN +if {$A(trans)} { db eval BEGIN } while {$i < $N} { foreach c $cols s $A(colsize) { set R($c) [lrange $tokens $i [expr $i+$s-1]] @@ -222,7 +223,7 @@ db eval BEGIN } db eval $sql } -db eval COMMIT +if {$A(trans)} { db eval COMMIT } diff --git a/manifest b/manifest index 0773bab205..b4be055c28 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sVdbe.aMem\sarray\sso\sthat\sit\sis\szero-based\sinstead\sof\sone-based. -D 2016-03-21T00:38:59.802 +C Change\sthe\sway\sfts5\sinternally\sallocates\ssegment\sids\sin\sorder\sto\seliminated\snon-determinism\sfrom\sthe\smodule. +D 2016-03-21T09:56:19.361 F Makefile.in f53429fb2f313c099283659d0df6f20f932c861f F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc df0bf9ff7f8b3f4dd9fb4cc43f92fe58f6ec5c66 @@ -104,7 +104,7 @@ F ext/fts5/fts5_buffer.c 4c1502d4c956cd092c89ce4480867f9d8bf325cd F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857 F ext/fts5/fts5_expr.c 35e9d92c89e7c7ea0759b73d24da1ecb7630a24b F ext/fts5/fts5_hash.c f3a7217c86eb8f272871be5f6aa1b6798960a337 -F ext/fts5/fts5_index.c d4f0c12e4f04bbc3a06b6da052039f2ce3e45438 +F ext/fts5/fts5_index.c d3759c2f7d878e9e0a392b027a1c6e05c356007d F ext/fts5/fts5_main.c b8501e1a6a11591c53b18ce7aea7e5386cfb0421 F ext/fts5/fts5_storage.c 2a38c6fa5db193a6a00588865134450ef5812daa F ext/fts5/fts5_tcl.c f8731e0508299bd43f1a2eff7dbeaac870768966 @@ -194,7 +194,7 @@ F ext/fts5/test/fts5update.test 57c7012a7919889048947addae10e0613df45529 F ext/fts5/test/fts5version.test 978f59541d8cef7e8591f8be2115ec5ccb863e2e F ext/fts5/test/fts5vocab.test 480d780aa6b699816c5066225fbd86f3a0239477 F ext/fts5/tool/fts5speed.tcl b0056f91a55b2d1a3684ec05729de92b042e2f85 -F ext/fts5/tool/fts5txt2db.tcl 1343745b89ca2a1e975c23f836d0cee410052975 +F ext/fts5/tool/fts5txt2db.tcl 526a9979c963f1c54fd50976a05a502e533a4c59 F ext/fts5/tool/loadfts5.tcl 95b03429ee6b138645703c6ca192c3ac96eaf093 F ext/fts5/tool/mkfts5c.tcl d1c2a9ab8e0ec690a52316f33dd9b1d379942f45 F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c @@ -1457,8 +1457,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d7852c639683a1d305a1e731df3cccafa64b594b c5677ecd5cd2637d92a831ec6bd5b002f8d75626 -R 3918e948fd7d72bc109c3b7a0fa018f3 -T +closed c5677ecd5cd2637d92a831ec6bd5b002f8d75626 -U drh -Z 542f0ebd1ebe0cc42b5001043dab2846 +P c39081e878faccc8552141afa5732a2bf2f77570 +R b7f851b6f7c0c6dc2d06642c2058dff4 +U dan +Z 052aba56922a1e224a7195cfbf595d27 diff --git a/manifest.uuid b/manifest.uuid index d4e8a322fa..775d397701 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c39081e878faccc8552141afa5732a2bf2f77570 \ No newline at end of file +d6e2637df16764aa9723a30ea2eb8a631d28cb2b \ No newline at end of file -- 2.47.2