From: dan Date: Mon, 5 Oct 2015 19:41:16 +0000 (+0000) Subject: Improve performance of prefix queries without a prefix index on fts5 tables. X-Git-Tag: version-3.9.0~37 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=6f832d3dffe83401c6fe359d9bedc967f0f4282a;p=thirdparty%2Fsqlite.git Improve performance of prefix queries without a prefix index on fts5 tables. FossilOrigin-Name: f2f0184e9e1c9f121ee2ac864cd28c8cd8efecb5 --- diff --git a/ext/fts5/fts5_buffer.c b/ext/fts5/fts5_buffer.c index 1a7c0d0f8a..e2dbcbf141 100644 --- a/ext/fts5/fts5_buffer.c +++ b/ext/fts5/fts5_buffer.c @@ -233,13 +233,15 @@ int sqlite3Fts5PoslistWriterAppend( ){ static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32; int rc = SQLITE_OK; - if( (iPos & colmask) != (pWriter->iPrev & colmask) ){ - fts5BufferAppendVarint(&rc, pBuf, 1); - fts5BufferAppendVarint(&rc, pBuf, (iPos >> 32)); - pWriter->iPrev = (iPos & colmask); + if( 0==sqlite3Fts5BufferGrow(&rc, pBuf, 5+5+5) ){ + if( (iPos & colmask) != (pWriter->iPrev & colmask) ){ + pBuf->p[pBuf->n++] = 1; + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32)); + pWriter->iPrev = (iPos & colmask); + } + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-pWriter->iPrev)+2); + pWriter->iPrev = iPos; } - fts5BufferAppendVarint(&rc, pBuf, (iPos - pWriter->iPrev) + 2); - pWriter->iPrev = iPos; return rc; } diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 418127a51c..490dbd7cb7 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -307,9 +307,7 @@ struct Fts5Index { }; struct Fts5DoclistIter { - u8 *a; - int n; - int i; + u8 *aEof; /* Pointer to 1 byte past end of doclist */ /* Output variables. aPoslist==0 at EOF */ i64 iRowid; @@ -3706,10 +3704,15 @@ static int fts5PoslistPrefix(const u8 *aBuf, int nMax){ return ret; } -#define fts5BufferSafeAppendBlob(pBuf, pBlob, nBlob) { \ - assert( pBuf->nSpace>=(pBuf->n+nBlob) ); \ - memcpy(&pBuf->p[pBuf->n], pBlob, nBlob); \ - pBuf->n += nBlob; \ +#define fts5BufferSafeAppendBlob(pBuf, pBlob, nBlob) { \ + assert( (pBuf)->nSpace>=((pBuf)->n+nBlob) ); \ + memcpy(&(pBuf)->p[(pBuf)->n], pBlob, nBlob); \ + (pBuf)->n += nBlob; \ +} + +#define fts5BufferSafeAppendVarint(pBuf, iVal) { \ + (pBuf)->n += sqlite3Fts5PutVarint(&(pBuf)->p[(pBuf)->n], (iVal)); \ + assert( (pBuf)->nSpace>=(pBuf)->n ); \ } /* @@ -3989,22 +3992,28 @@ static void fts5MultiIterPoslist( } static void fts5DoclistIterNext(Fts5DoclistIter *pIter){ - if( pIter->in ){ - int bDummy; - if( pIter->i ){ - i64 iDelta; - pIter->i += fts5GetVarint(&pIter->a[pIter->i], (u64*)&iDelta); - pIter->iRowid += iDelta; + u8 *p = pIter->aPoslist + pIter->nPoslist; + + assert( pIter->aPoslist ); + if( p>=pIter->aEof ){ + pIter->aPoslist = 0; + }else{ + i64 iDelta; + + p += fts5GetVarint(p, (u64*)&iDelta); + pIter->iRowid += iDelta; + + /* Read position list size */ + if( p[0] & 0x80 ){ + int nPos; + p += fts5GetVarint32(p, nPos); + pIter->nPoslist = (nPos>>1); }else{ - pIter->i += fts5GetVarint(&pIter->a[pIter->i], (u64*)&pIter->iRowid); + pIter->nPoslist = ((int)(p[0])) >> 1; + p++; } - pIter->i += fts5GetPoslistSize( - &pIter->a[pIter->i], &pIter->nPoslist, &bDummy - ); - pIter->aPoslist = &pIter->a[pIter->i]; - pIter->i += pIter->nPoslist; - }else{ - pIter->aPoslist = 0; + + pIter->aPoslist = p; } } @@ -4013,27 +4022,34 @@ static void fts5DoclistIterInit( Fts5DoclistIter *pIter ){ memset(pIter, 0, sizeof(*pIter)); - pIter->a = pBuf->p; - pIter->n = pBuf->n; + pIter->aPoslist = pBuf->p; + pIter->aEof = &pBuf->p[pBuf->n]; fts5DoclistIterNext(pIter); } +#if 0 /* ** Append a doclist to buffer pBuf. +** +** This function assumes that space within the buffer has already been +** allocated. */ static void fts5MergeAppendDocid( - int *pRc, /* IN/OUT: Error code */ Fts5Buffer *pBuf, /* Buffer to write to */ i64 *piLastRowid, /* IN/OUT: Previous rowid written (if any) */ i64 iRowid /* Rowid to append */ ){ - if( pBuf->n==0 ){ - fts5BufferAppendVarint(pRc, pBuf, iRowid); - }else{ - fts5BufferAppendVarint(pRc, pBuf, iRowid - *piLastRowid); - } + assert( pBuf->n!=0 || (*piLastRowid)==0 ); + fts5BufferSafeAppendVarint(pBuf, iRowid - *piLastRowid); *piLastRowid = iRowid; } +#endif + +#define fts5MergeAppendDocid(pBuf, iLastRowid, iRowid) { \ + assert( (pBuf)->n!=0 || (iLastRowid)==0 ); \ + fts5BufferSafeAppendVarint((pBuf), (iRowid) - (iLastRowid)); \ + (iLastRowid) = (iRowid); \ +} /* ** Buffers p1 and p2 contain doclists. This function merges the content @@ -4057,53 +4073,60 @@ static void fts5MergePrefixLists( memset(&out, 0, sizeof(out)); memset(&tmp, 0, sizeof(tmp)); + sqlite3Fts5BufferGrow(&p->rc, &out, p1->n + p2->n); fts5DoclistIterInit(p1, &i1); fts5DoclistIterInit(p2, &i2); while( p->rc==SQLITE_OK && (i1.aPoslist!=0 || i2.aPoslist!=0) ){ if( i2.aPoslist==0 || (i1.aPoslist && i1.iRowidrc, &out, &iLastRowid, i1.iRowid); + fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid); /* WRITEPOSLISTSIZE */ - fts5BufferAppendVarint(&p->rc, &out, i1.nPoslist * 2); - fts5BufferAppendBlob(&p->rc, &out, i1.nPoslist, i1.aPoslist); + fts5BufferSafeAppendVarint(&out, i1.nPoslist * 2); + fts5BufferSafeAppendBlob(&out, i1.aPoslist, i1.nPoslist); fts5DoclistIterNext(&i1); } else if( i1.aPoslist==0 || i2.iRowid!=i1.iRowid ){ /* Copy entry from i2 */ - fts5MergeAppendDocid(&p->rc, &out, &iLastRowid, i2.iRowid); + fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid); /* WRITEPOSLISTSIZE */ - fts5BufferAppendVarint(&p->rc, &out, i2.nPoslist * 2); - fts5BufferAppendBlob(&p->rc, &out, i2.nPoslist, i2.aPoslist); + fts5BufferSafeAppendVarint(&out, i2.nPoslist * 2); + fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.nPoslist); fts5DoclistIterNext(&i2); } else{ - Fts5PoslistReader r1; - Fts5PoslistReader r2; - Fts5PoslistWriter writer; + i64 iPos1 = 0; + i64 iPos2 = 0; + int iOff1 = 0; + int iOff2 = 0; + Fts5PoslistWriter writer; memset(&writer, 0, sizeof(writer)); /* Merge the two position lists. */ - fts5MergeAppendDocid(&p->rc, &out, &iLastRowid, i2.iRowid); + fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid); fts5BufferZero(&tmp); - sqlite3Fts5PoslistReaderInit(-1, i1.aPoslist, i1.nPoslist, &r1); - sqlite3Fts5PoslistReaderInit(-1, i2.aPoslist, i2.nPoslist, &r2); - while( p->rc==SQLITE_OK && (r1.bEof==0 || r2.bEof==0) ){ + + sqlite3Fts5PoslistNext64(i1.aPoslist, i1.nPoslist, &iOff1, &iPos1); + sqlite3Fts5PoslistNext64(i2.aPoslist, i2.nPoslist, &iOff2, &iPos2); + + while( p->rc==SQLITE_OK && (iPos1>=0 || iPos2>=0) ){ i64 iNew; - if( r2.bEof || (r1.bEof==0 && r1.iPos=0 && iPos1rc = sqlite3Fts5PoslistWriterAppend(&tmp, &writer, iNew); } /* WRITEPOSLISTSIZE */ - fts5BufferAppendVarint(&p->rc, &out, tmp.n * 2); - fts5BufferAppendBlob(&p->rc, &out, tmp.n, tmp.p); + fts5BufferSafeAppendVarint(&out, tmp.n * 2); + fts5BufferSafeAppendBlob(&out, tmp.p, tmp.n); fts5DoclistIterNext(&i1); fts5DoclistIterNext(&i2); } @@ -4165,14 +4188,19 @@ static void fts5SetupPrefixIter( fts5BufferZero(&aBuf[i]); } } + iLastRowid = 0; } - fts5MergeAppendDocid(&p->rc, &doclist, &iLastRowid, iRowid); - fts5MultiIterPoslist(p, p1, 1, &doclist); + if( 0==sqlite3Fts5BufferGrow(&p->rc, &doclist, 9) ){ + fts5MergeAppendDocid(&doclist, iLastRowid, iRowid); + fts5MultiIterPoslist(p, p1, 1, &doclist); + } } for(i=0; irc==SQLITE_OK ){ + fts5MergePrefixLists(p, &doclist, &aBuf[i]); + } fts5BufferFree(&aBuf[i]); } fts5MultiIterFree(p, p1); diff --git a/manifest b/manifest index a6a5c2b408..5081c72572 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sfts3\sso\sthat\sexpressions\sto\sthe\sleft\sand\sright\sof\sa\sNOT\soperator\sare\sbalanced.\sThis\sprevents\srelatively\ssmall\sexpressions\s(a\sdozen\sterms\sor\sso)\sthat\sare\schildren\sof\sNOT\soperators\sfrom\striggering\sthe\s"expression\stree\sis\stoo\slarge"\serror. -D 2015-10-05T15:39:45.681 +C Improve\sperformance\sof\sprefix\squeries\swithout\sa\sprefix\sindex\son\sfts5\stables. +D 2015-10-05T19:41:16.348 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2143eeef6d0cc26006ae5fc4bb242a4a8b973412 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -108,11 +108,11 @@ F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h 98f802fe41481f9d797fce496f0fefcad72c7782 F ext/fts5/fts5Int.h ff78a77d819a7fc04a7f8b08b0e1ce361a3395e4 F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e -F ext/fts5/fts5_buffer.c 64dcaf36a3ebda9e84b7c3b8788887ec325e12a4 +F ext/fts5/fts5_buffer.c 54b18497395a19dfe1d00f63a3b403e5f93d4fd1 F ext/fts5/fts5_config.c 57ee5fe71578cb494574fc0e6e51acb9a22a8695 F ext/fts5/fts5_expr.c 667faaf14a69a5683ac383acdc8d942cf32c3f93 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 -F ext/fts5/fts5_index.c 00d2593f94ede440ea274f8db21864cf41632aa3 +F ext/fts5/fts5_index.c ca3912a44ef5a173ef098f3454465519bd4b8e88 F ext/fts5/fts5_main.c fe5243d6bbb79217394f0ec7f4f5199ddbc9e7e8 F ext/fts5/fts5_storage.c df061a5caf9e50fbbd43113009b5b248362f4995 F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37 @@ -1391,7 +1391,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 44f1ce30d1e446c9ee5f8bd8a62119e00356aa0e -R 0954493ffc80fb341b0df1cab997ace8 +P d6b66cd7b89fbd964f798d160a34caac0ba7347a +R 0053d2d2b912a0e9b79250c67681b097 U dan -Z 52900cdc8d31460ed8eb8d6276be0e92 +Z c981c12e0ed92ecd6400cdfa9ecf4084 diff --git a/manifest.uuid b/manifest.uuid index a2a1d63f6e..fcb70f5907 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d6b66cd7b89fbd964f798d160a34caac0ba7347a \ No newline at end of file +f2f0184e9e1c9f121ee2ac864cd28c8cd8efecb5 \ No newline at end of file