From: drh Date: Thu, 13 Apr 2017 01:19:30 +0000 (+0000) Subject: Forward port the skip-ahead-distinct branch which was abandoned for some reason X-Git-Tag: version-3.19.0~59^2~5 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=8489bf5aff75791c6426a251c157945f1f858f03;p=thirdparty%2Fsqlite.git Forward port the skip-ahead-distinct branch which was abandoned for some reason that I do not recall. This port should have been achived by a merge of trunk into the previous head of skip-ahead-distinct, but that did not work. So I had to manually "rebase" the changes. FossilOrigin-Name: 132339a1fb0b9664df4d3eefbed6e77ef100ba95a91dcc622da7bd3fcdfcd6af --- diff --git a/manifest b/manifest index 9c3017b58b..9c2c74dfef 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sregression\scaused\sby\sthe\sfix\sfor\sticket\s[6c9b5514077fed34551f98e64c09a1]\s-\ncontrol\scharacters\sallowed\sin\sJSON. -D 2017-04-13T00:12:32.332 +C Forward\sport\sthe\sskip-ahead-distinct\sbranch\swhich\swas\sabandoned\sfor\ssome\sreason\nthat\sI\sdo\snot\srecall.\s\sThis\sport\sshould\shave\sbeen\sachived\sby\sa\smerge\sof\strunk\ninto\sthe\sprevious\shead\sof\sskip-ahead-distinct,\sbut\sthat\sdid\snot\swork.\s\sSo\sI\shad\nto\smanually\s"rebase"\sthe\schanges. +D 2017-04-13T01:19:30.439 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a @@ -401,12 +401,12 @@ F src/printf.c 8757834f1b54dae512fb25eb1acc8e94a0d15dd2290b58f2563f65973265adb2 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 3e518b962d932a997fae373366880fc028c75706 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c e6f9afd8a5ef35bd186e51a6bea6d3d46bc93a530f26a21fe8a0a43dbeca9415 +F src/select.c cc9129bc97d20f3c79b9dfcc555db37eed28a660cc6954bf3b282d8e1e5593a2 F src/shell.c 70f4957b988572315e97c56941fdc81fd35907fee36b7b2e7be5ec4c7e9d065d F src/sqlite.h.in 40233103e3e4e10f8a63523498d0259d232e42aba478e2d3fb914799185aced6 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 58fd0676d3111d02e62e5a35992a7d3da5d3f88753acc174f2d37b774fbbdd28 -F src/sqliteInt.h 9affb53bb405dcea1d86e85198ebaf6232a684cc2b2af6b3c181869f1c8f3e93 +F src/sqliteInt.h f0b6ed4d471bdd555746b58e403bbee5c27a674e0e9a38a2f30dbec9d73d2476 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -482,7 +482,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 40c543f0a2195d1b0dc88ef12142bea690009344 F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71 F src/walker.c b71a992b413b3a022572eccf29ef4b4890223791 -F src/where.c 1d14e18f32231fa7969e718e7b60ef749b0065e2a7e1b6b00883b20732d280f1 +F src/where.c 33fd1fcda81dcbfe5e7622247a2e0196f97f8188cbcd6b9a0e0aa5cdb82d9545 F src/whereInt.h 7a21ef633e26acbf46df04add2eba6e0a2100c78dc5879049e93f981fc3344df F src/wherecode.c 943e32e9dccd0af802e0683ae11071c8bd808364e5908a5fb66758bd404c8681 F src/whereexpr.c e913aaa7b73ffcce66abcea5f197e2c538d48b5df78d0b7bba8ff4d73cc2e745 @@ -658,6 +658,7 @@ F test/descidx2.test 9f1a0c83fd57f8667c82310ca21b30a350888b5d F test/descidx3.test 09ddbe3f5295f482d2f8b687cf6db8bad7acd9a2 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e F test/distinct.test a1783b960ad8c15a77cd9f207be072898db1026c +F test/distinct2.test 2d5245672d259c28a8b706cdc764c4cfa663e2ba F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 F test/e_blobbytes.test 439a945953b35cb6948a552edaec4dc31fd70a05 F test/e_blobclose.test 4b3c8c60c2171164d472059c73e9f3c1844bb66d @@ -1571,7 +1572,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c847543f8bb1376fef52bca72b4191162a32eb7e6c5f0cd1aa0ab116b3183396 -R 0115b666e9627edf0663b399afd00fa5 +P 8e7b611863247a3bf46a96ec4b47d24c0ae0d60c9cee968a1cfd1da157e7c9bb +R de2bc0d85efdd08063075633dcadd275 +T *branch * skip-ahead-distinct +T *sym-skip-ahead-distinct * +T -sym-trunk * U drh -Z d4923dfb1fc052664f2f0021f04650c5 +Z 4afe54ae8fd900ea4ea328fe1c4fe900 diff --git a/manifest.uuid b/manifest.uuid index 90be1e70ef..411dd2c3a5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8e7b611863247a3bf46a96ec4b47d24c0ae0d60c9cee968a1cfd1da157e7c9bb \ No newline at end of file +132339a1fb0b9664df4d3eefbed6e77ef100ba95a91dcc622da7bd3fcdfcd6af \ No newline at end of file diff --git a/src/select.c b/src/select.c index 54265ab2a9..3c35c75041 100644 --- a/src/select.c +++ b/src/select.c @@ -1690,7 +1690,7 @@ int sqlite3ColumnsFromExprList( pColExpr = pColExpr->pRight; assert( pColExpr!=0 ); } - if( pColExpr->op==TK_COLUMN && ALWAYS(pColExpr->pTab!=0) ){ + if( pColExpr->op==TK_COLUMN && pColExpr->pTab!=0 ){ /* For columns use the column name name */ int iCol = pColExpr->iColumn; pTab = pColExpr->pTab; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 40660aed99..eeea78b8d5 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1501,6 +1501,7 @@ struct sqlite3 { #define SQLITE_Transitive 0x0200 /* Transitive constraints */ #define SQLITE_OmitNoopJoin 0x0400 /* Omit unused tables in joins */ #define SQLITE_Stat34 0x0800 /* Use STAT3 or STAT4 data */ +#define SQLITE_SkipAhead 0x1000 /* Skip ahead on DISTINCT */ #define SQLITE_CursorHints 0x2000 /* Add OP_CursorHint opcodes */ #define SQLITE_AllOpts 0xffff /* All optimizations */ diff --git a/src/where.c b/src/where.c index 38183d6849..e79e88c3c5 100644 --- a/src/where.c +++ b/src/where.c @@ -4758,6 +4758,7 @@ WhereInfo *sqlite3WhereBegin( if( (pLoop->wsFlags & WHERE_CONSTRAINT)!=0 && (pLoop->wsFlags & (WHERE_COLUMN_RANGE|WHERE_SKIPSCAN))==0 && (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0 + && pWInfo->eDistinct!=WHERE_DISTINCT_ORDERED ){ sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ); /* Hint to COMDB2 */ } @@ -4848,12 +4849,59 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ pLoop = pLevel->pWLoop; sqlite3VdbeResolveLabel(v, pLevel->addrCont); if( pLevel->op!=OP_Noop ){ - sqlite3VdbeAddOp3(v, pLevel->op, pLevel->p1, pLevel->p2, pLevel->p3); - sqlite3VdbeChangeP5(v, pLevel->p5); - VdbeCoverage(v); - VdbeCoverageIf(v, pLevel->op==OP_Next); - VdbeCoverageIf(v, pLevel->op==OP_Prev); - VdbeCoverageIf(v, pLevel->op==OP_VNext); +#ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT + int n = -1; + int j, k, op; + int r1 = pParse->nMem+1; + if( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED + && (pLoop->wsFlags & WHERE_INDEXED)!=0 + && OptimizationEnabled(db, SQLITE_SkipAhead) + ){ + /* This is the Skip-ahead optimization. When doing a DISTINCT query + ** that has WHERE_DISTINCT_ORDERED, use OP_SkipGT/OP_SkipLT to skip + ** over all duplicate entries, rather than visiting all duplicates + ** using OP_Next/OP_Prev. */ + ExprList *pX = pWInfo->pResultSet; + Index *pIdx = pLoop->u.btree.pIndex; + for(j=0; jnExpr; j++){ + Expr *pE = sqlite3ExprSkipCollate(pX->a[j].pExpr); + if( pE->op==TK_COLUMN ){ + if( pE->iTable!=pLevel->iTabCur ) continue; + k = 1+sqlite3ColumnOfIndex(pIdx, pE->iColumn); + if( k>n ) n = k; + }else if( pIdx->aColExpr ){ + for(k=n+1; knKeyCol; k++){ + Expr *pI = pIdx->aColExpr->a[k].pExpr; + if( pI && sqlite3ExprCompare(pE,pI,0)<2 ){ + n = k+1; + break; + } + } + } + } + } + if( n>0 ){ + for(j=0; jiIdxCur, j, r1+j); + } + pParse->nMem += n; + op = pLevel->op==OP_Prev ? OP_SeekLT : OP_SeekGT; + k = sqlite3VdbeAddOp4Int(v, op, pLevel->iIdxCur, 0, r1, n); + VdbeCoverageIf(v, op==OP_SeekLT); + VdbeCoverageIf(v, op==OP_SeekGT); + sqlite3VdbeAddOp2(v, OP_Goto, 1, pLevel->p2); + sqlite3VdbeJumpHere(v, k); + }else +#endif /* SQLITE_DISABLE_SKIPAHEAD_DISTINCT */ + { + /* The common case: Advance to the next row */ + sqlite3VdbeAddOp3(v, pLevel->op, pLevel->p1, pLevel->p2, pLevel->p3); + sqlite3VdbeChangeP5(v, pLevel->p5); + VdbeCoverage(v); + VdbeCoverageIf(v, pLevel->op==OP_Next); + VdbeCoverageIf(v, pLevel->op==OP_Prev); + VdbeCoverageIf(v, pLevel->op==OP_VNext); + } } if( pLoop->wsFlags & WHERE_IN_ABLE && pLevel->u.in.nIn>0 ){ struct InLoop *pIn; diff --git a/test/distinct2.test b/test/distinct2.test new file mode 100644 index 0000000000..fc42e7556f --- /dev/null +++ b/test/distinct2.test @@ -0,0 +1,88 @@ +# 2016-04-15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this script is DISTINCT queries using the skip-ahead +# optimization. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +set testprefix distinct2 + +do_execsql_test 100 { + CREATE TABLE t1(x INTEGER PRIMARY KEY); + INSERT INTO t1 VALUES(0),(1),(2); + CREATE TABLE t2 AS + SELECT DISTINCT a.x AS aa, b.x AS bb + FROM t1 a, t1 b; + SELECT *, '|' FROM t2 ORDER BY aa, bb; +} {0 0 | 0 1 | 0 2 | 1 0 | 1 1 | 1 2 | 2 0 | 2 1 | 2 2 |} +do_execsql_test 110 { + DROP TABLE t2; + CREATE TABLE t2 AS + SELECT DISTINCT a.x AS aa, b.x AS bb + FROM t1 a, t1 b + WHERE a.x IN t1 AND b.x IN t1; + SELECT *, '|' FROM t2 ORDER BY aa, bb; +} {0 0 | 0 1 | 0 2 | 1 0 | 1 1 | 1 2 | 2 0 | 2 1 | 2 2 |} +do_execsql_test 120 { + CREATE TABLE t102 (i0 TEXT UNIQUE NOT NULL); + INSERT INTO t102 VALUES ('0'),('1'),('2'); + DROP TABLE t2; + CREATE TABLE t2 AS + SELECT DISTINCT * + FROM t102 AS t0 + JOIN t102 AS t4 ON (t2.i0 IN t102) + NATURAL JOIN t102 AS t3 + JOIN t102 AS t1 ON (t0.i0 IN t102) + JOIN t102 AS t2 ON (t2.i0=+t0.i0 OR (t0.i0<>500 AND t2.i0=t1.i0)); + SELECT *, '|' FROM t2 ORDER BY 1, 2, 3, 4, 5; +} {0 0 0 0 | 0 0 1 0 | 0 0 1 1 | 0 0 2 0 | 0 0 2 2 | 0 1 0 0 | 0 1 1 0 | 0 1 1 1 | 0 1 2 0 | 0 1 2 2 | 0 2 0 0 | 0 2 1 0 | 0 2 1 1 | 0 2 2 0 | 0 2 2 2 | 1 0 0 0 | 1 0 0 1 | 1 0 1 1 | 1 0 2 1 | 1 0 2 2 | 1 1 0 0 | 1 1 0 1 | 1 1 1 1 | 1 1 2 1 | 1 1 2 2 | 1 2 0 0 | 1 2 0 1 | 1 2 1 1 | 1 2 2 1 | 1 2 2 2 | 2 0 0 0 | 2 0 0 2 | 2 0 1 1 | 2 0 1 2 | 2 0 2 2 | 2 1 0 0 | 2 1 0 2 | 2 1 1 1 | 2 1 1 2 | 2 1 2 2 | 2 2 0 0 | 2 2 0 2 | 2 2 1 1 | 2 2 1 2 | 2 2 2 2 |} + +do_execsql_test 400 { + CREATE TABLE t4(a,b,c,d,e,f,g,h,i,j); + INSERT INTO t4 VALUES(0,1,2,3,4,5,6,7,8,9); + INSERT INTO t4 SELECT * FROM t4; + INSERT INTO t4 SELECT * FROM t4; + CREATE INDEX t4x ON t4(c,d,e); + SELECT DISTINCT a,b,c FROM t4 WHERE a=0 AND b=1; +} {0 1 2} +do_execsql_test 410 { + SELECT DISTINCT a,b,c,d FROM t4 WHERE a=0 AND b=1; +} {0 1 2 3} +do_execsql_test 411 { + SELECT DISTINCT d,a,b,c FROM t4 WHERE a=0 AND b=1; +} {3 0 1 2} +do_execsql_test 420 { + SELECT DISTINCT a,b,c,d,e FROM t4 WHERE a=0 AND b=1; +} {0 1 2 3 4} +do_execsql_test 430 { + SELECT DISTINCT a,b,c,d,e,f FROM t4 WHERE a=0 AND b=1; +} {0 1 2 3 4 5} + +do_execsql_test 500 { + CREATE TABLE t5(a INT, b INT); + CREATE UNIQUE INDEX t5x ON t5(a+b); + INSERT INTO t5(a,b) VALUES(0,0),(1,0),(1,1),(0,3); + CREATE TEMP TABLE out AS SELECT DISTINCT a+b FROM t5; + SELECT * FROM out ORDER BY 1; +} {0 1 2 3} + +do_execsql_test 600 { + CREATE TABLE t6a(x INTEGER PRIMARY KEY); + INSERT INTO t6a VALUES(1); + CREATE TABLE t6b(y INTEGER PRIMARY KEY); + INSERT INTO t6b VALUES(2),(3); + SELECT DISTINCT x, x FROM t6a, t6b; +} {1 1} + +finish_test