Fix multiple issues with the ORDER BY LIMIT optimization. This is the

proposed resolution to ticket [9936b2fa443fec03ff25].

FossilOrigin-Name: 206720129ed2fa8875a286266d05b99fb2caf8671e4b74b26a6286a2073fcd8b

diff --git a/manifest b/manifest
index 604bfb6dbefe4efb48f65c0b90bd2057ca475b7b..5b87b9772ea72e253aadfc61d08b0b8cdd45d1eb 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sa\smissing\scall\sto\sfree()\sin\sLemon.
-D 2018-09-08T16:55:18.635
+C Fix\smultiple\sissues\swith\sthe\sORDER\sBY\sLIMIT\soptimization.\s\sThis\sis\sthe\nproposed\sresolution\sto\sticket\s[9936b2fa443fec03ff25].
+D 2018-09-08T20:09:46.990
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 6b650013511fd9d8b094203ac268af9220d292cc7d4e1bc9fbca15aacd8c7995
@@ -501,12 +501,12 @@ F src/printf.c 7f6f3cba8e0c49c19e30a1ff4e9aeda6e06814dcbad4b664a69e1b6cb6e7e365
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 352c6af1a99441206ff62a6f7429dbf537827f42c428639695220b9c8639e33b
 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93
-F src/select.c ae7396a314cc1bb1d767947cd57094e3a9ffcbb155ebc1b1c391e028c44a9a04
+F src/select.c ed6192ddd09a97169cb1c6d732b26c0f647b72d5fa8ca401c7ee1180fbbe521a
 F src/shell.c.in 6e0aad854be738a5d0368940459399be211e9ac43aebe92bb9ed46cfe38d0e1f
 F src/sqlite.h.in cdf2a539cd0570322a94bcb97c01c56feb1be0657ec7cfb8273c89d19fff87a9
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 9887b27e69c01e79c2cbe74ef73bf01af5b5703d6a7f0a4371e386d7249cb1c7
-F src/sqliteInt.h c72e50d876b1fd909db70399a228d022a44b6a7326a6031f76594acdfd25b69e
+F src/sqliteInt.h f63f04e3db06b605ead2d9bb8eda8bc3add3baf3da9655954dfb24860e625933
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -586,7 +586,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c df50883d93689d009be5ad9bdc4e53a4ee45fcc291087ec9272569d00b360791
 F src/wal.h 606292549f5a7be50b6227bd685fa76e3a4affad71bb8ac5ce4cb5c79f6a176a
 F src/walker.c ba7225773931760cf60bf22f34d0cce2588df7ce5ce0f215a52eb88234b55ac4
-F src/where.c 155809967fbab889374dedf970ea6561b8fb519fcb165d6ba00776552ecc5cde
+F src/where.c 5192013a21843523f4772087a56e59db73bd4c7401349968120dc0693b7c4eb4
 F src/whereInt.h b90ef9b9707ef750eab2a7a080c48fb4900315033274689def32d0cf5a81ebe4
 F src/wherecode.c 2b6cd1b27736cc803060289e04ecf9849976106f4077aa67d1a2c0e3ec420159
 F src/whereexpr.c d87df2c00ecc0c2ef4409562608d19cec259a6a03ca72b86fc999db9c07ce119
@@ -1052,7 +1052,7 @@ F test/like.test 11cfd7d4ef8625389df9efc46735ff0b0b41d5e62047ef0f3bc24c380d28a7a
 F test/like2.test 3b2ee13149ba4a8a60b59756f4e5d345573852da
 F test/like3.test 3608a2042b6f922f900fbfd5d3ce4e7eca57f7c4
 F test/limit.test 0c99a27a87b14c646a9d583c7c89fd06c352663e
-F test/limit2.test 360982809e03211636d2b18ddbc97d5da06826941370607e4b00e113f827cb5a
+F test/limit2.test 9409b033284642a859fafc95f29a5a6a557bd57c1f0d7c3f554bd64ed69df77e
 F test/loadext.test d077450695ddb5c1ea3ad7d48e5f5850fe732ad9
 F test/loadext2.test 0408380b57adca04004247179837a18e866a74f7
 F test/lock.test be4fe08118fb988fed741f429b7dd5d65e1c90db
@@ -1764,7 +1764,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e812e5d59a699e8b82c51d465d9c0f09df6a1e6996b5499814dca99c5f8020d5
-R 985ed317810844fbd8e283aa0c269697
-U mistachkin
-Z e4dcf479c4692c03c98ec48ee992b041
+P 8b4cf33aafe09d9009119dcbd464b54be9605af5701002ee458819efa6e2e1f9
+R 314bbe63eed86b2216509901a995337d
+U drh
+Z 40fd1d6c79004b4386462b0fab14be9d

diff --git a/manifest.uuid b/manifest.uuid
index ad4a8b272b0a3490bca82a04054c7d7674cc46fb..afcfdbc27d529260761e6d2e4ca5757177f9535e 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-8b4cf33aafe09d9009119dcbd464b54be9605af5701002ee458819efa6e2e1f9
\ No newline at end of file
+206720129ed2fa8875a286266d05b99fb2caf8671e4b74b26a6286a2073fcd8b
\ No newline at end of file

diff --git a/src/select.c b/src/select.c
index f97efa7a728337d1bd122a2f90700ddac9cd20c3..35dc9f1b718f677c20a60f8f36cfa1f5b38fc86f 100644 (file)
--- a/src/select.c
+++ b/src/select.c
@@ -68,8 +68,8 @@ struct SortCtx {
   int labelBkOut;       /* Start label for the block-output subroutine */
   int addrSortIndex;    /* Address of the OP_SorterOpen or OP_OpenEphemeral */
   int labelDone;        /* Jump here when done, ex: LIMIT reached */
+  int labelOBLopt;      /* Jump here when sorter is full */
   u8 sortFlags;         /* Zero or more SORTFLAG_* bits */
-  u8 bOrderedInnerLoop; /* ORDER BY correctly sorts the inner loop */
 #ifdef SQLITE_ENABLE_SORTER_REFERENCES
   u8 nDefer;            /* Number of valid entries in aDefer[] */
   struct DeferredCsr {
@@ -693,10 +693,10 @@ static void pushOntoSorter(
     ** than LIMIT+OFFSET items in the sorter.
     **
     ** If the new record does not need to be inserted into the sorter,
-    ** jump to the next iteration of the loop. Or, if the
-    ** pSort->bOrderedInnerLoop flag is set to indicate that the inner
-    ** loop delivers items in sorted order, jump to the next iteration
-    ** of the outer loop.
+    ** jump to the next iteration of the loop. If the pSort->labelOBLopt
+    ** value is not zero, then it is a label of where to jump.  Otherwise,
+    ** just bypass the row insert logic.  See the header comment on the
+    ** sqlite3WhereOrderByLimitOptLabel() function for additional info.
     */
     int iCsr = pSort->iECursor;
     sqlite3VdbeAddOp2(v, OP_IfNotZero, iLimit, sqlite3VdbeCurrentAddr(v)+4);
@@ -718,9 +718,8 @@ static void pushOntoSorter(
   sqlite3VdbeAddOp4Int(v, op, pSort->iECursor, regRecord,
                        regBase+nOBSat, nBase-nOBSat);
   if( iSkip ){
-    assert( pSort->bOrderedInnerLoop==0 || pSort->bOrderedInnerLoop==1 );
     sqlite3VdbeChangeP2(v, iSkip,
-         sqlite3VdbeCurrentAddr(v) + pSort->bOrderedInnerLoop);
+         pSort->labelOBLopt ? pSort->labelOBLopt : sqlite3VdbeCurrentAddr(v));
   }
 }
 
@@ -6060,7 +6059,7 @@ int sqlite3Select(
     }
     if( sSort.pOrderBy ){
       sSort.nOBSat = sqlite3WhereIsOrdered(pWInfo);
-      sSort.bOrderedInnerLoop = sqlite3WhereOrderedInnerLoop(pWInfo);
+      sSort.labelOBLopt = sqlite3WhereOrderByLimitOptLabel(pWInfo);
       if( sSort.nOBSat==sSort.pOrderBy->nExpr ){
         sSort.pOrderBy = 0;
       }

diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index c33295a4053fd665d25e332f429f1e72da92965f..310460c16a31a7925420d172d28d95d9840c36b3 100644 (file)
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -3923,7 +3923,7 @@ void sqlite3WhereEnd(WhereInfo*);
 LogEst sqlite3WhereOutputRowCount(WhereInfo*);
 int sqlite3WhereIsDistinct(WhereInfo*);
 int sqlite3WhereIsOrdered(WhereInfo*);
-int sqlite3WhereOrderedInnerLoop(WhereInfo*);
+int sqlite3WhereOrderByLimitOptLabel(WhereInfo*);
 int sqlite3WhereIsSorted(WhereInfo*);
 int sqlite3WhereContinueLabel(WhereInfo*);
 int sqlite3WhereBreakLabel(WhereInfo*);

diff --git a/src/where.c b/src/where.c
index 53c362d61b4be1a3795429bedd27fde49e0e2430..edc22b09c3e15eead1482de04b110448324d4e1c 100644 (file)
--- a/src/where.c
+++ b/src/where.c
@@ -67,15 +67,38 @@ int sqlite3WhereIsOrdered(WhereInfo *pWInfo){
 }
 
 /*
-** Return TRUE if the innermost loop of the WHERE clause implementation
-** returns rows in ORDER BY order for complete run of the inner loop.
+** In the ORDER BY LIMIT optimization, if the inner-most loop is known
+** to emit rows in increasing order, and if the last row emitted by the
+** inner-most loop did not fit within the sorter, then we can skip all
+** subsequent rows for the current iteration of the inner loop (because they
+** will not fit in the sorter either) and continue with the second inner
+** loop - the loop immediately outside the inner-most.
 **
-** Across multiple iterations of outer loops, the output rows need not be
-** sorted.  As long as rows are sorted for just the innermost loop, this
-** routine can return TRUE.
+** When a row does not fit in the sorter (because the sorter already
+** holds LIMIT+OFFSET rows that are smaller), then a jump is made to the
+** label returned by this function.
+**
+** If the ORDER BY LIMIT optimization applies, the jump destination should
+** be the continuation for the second-inner-most loop.  If the ORDER BY
+** LIMIT optimization does not apply, then the jump destination should
+** be the continuation for the inner-most loop.
+**
+** It is always safe for this routine to return the continuation of the
+** inner-most loop, in the sense that a correct answer will result.  
+** Returning the continuation the second inner loop is an optimization
+** that might make the code run a little faster, but should not change
+** the final answer.
 */
-int sqlite3WhereOrderedInnerLoop(WhereInfo *pWInfo){
-  return pWInfo->bOrderedInnerLoop;
+int sqlite3WhereOrderByLimitOptLabel(WhereInfo *pWInfo){
+  WhereLevel *pInner;
+  if( !pWInfo->bOrderedInnerLoop ){
+    /* The ORDER BY LIMIT optimization does not apply.  Jump to the 
+    ** continuation of the inner-most loop. */
+    return pWInfo->iContinue;
+  }
+  pInner = &pWInfo->a[pWInfo->nLevel-1];
+  if( pInner->addrNxt ) return pInner->addrNxt;
+  return pInner->addrBrk;
 }
 
 /*
@@ -4248,6 +4271,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
       pWInfo->eDistinct = WHERE_DISTINCT_ORDERED;
     }
   }
+  pWInfo->bOrderedInnerLoop = 0;
   if( pWInfo->pOrderBy ){
     if( pWInfo->wctrlFlags & WHERE_DISTINCTBY ){
       if( pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){

diff --git a/test/limit2.test b/test/limit2.test
index 83c67506f5705a9e727e888b4d357227cefd61a1..c03f39cd9c8720fd4e423437c33a8af432acda62 100644 (file)
--- a/test/limit2.test
+++ b/test/limit2.test
@@ -167,4 +167,51 @@ do_execsql_test 600 {
   SELECT y FROM t1, t2 WHERE a=x AND b<=y ORDER BY b DESC;
 } {3}
 
+# Ticket https://www.sqlite.org/src/info/9936b2fa443fec03 2018-09-08
+# Infinite loop due to the ORDER BY LIMIT optimization.
+#
+do_execsql_test 700 {
+  DROP TABLE IF EXISTS t1;
+  DROP TABLE IF EXISTS t2;
+  CREATE TABLE t1(aa VARCHAR PRIMARY KEY NOT NULL,bb,cc,x VARCHAR(400));
+  INSERT INTO t1(aa,bb,cc) VALUES('maroon','meal','lecture');
+  INSERT INTO t1(aa,bb,cc) VALUES('reality','meal','catsear');
+  CREATE TABLE t2(aa VARCHAR PRIMARY KEY, dd INT DEFAULT 1, ee, x VARCHAR(100));
+  INSERT INTO t2(aa,dd,ee) VALUES('maroon',0,'travel'),('reality',0,'hour');
+  CREATE INDEX t2x1 ON t2(dd,ee);
+  ANALYZE;
+  DROP TABLE IF EXISTS sqlite_stat4;
+  DELETE FROM sqlite_stat1;
+  INSERT INTO sqlite_stat1 VALUES
+    ('t2','t2x1','3 3 3'),
+    ('t2','sqlite_autoindex_t2_1','3 1'),
+    ('t1','sqlite_autoindex_t1_1','2 1');
+  ANALYZE sqlite_master;
+  SELECT *
+    FROM t1 LEFT JOIN t2 ON t1.aa=t2.aa
+   WHERE t1.bb='meal'
+   ORDER BY t2.dd DESC
+   LIMIT 1;
+} {maroon meal lecture {} maroon 0 travel {}}
+do_execsql_test 710 {
+  DROP TABLE t1;
+  DROP TABLE t2;
+  CREATE TABLE t1(aa, bb);
+  INSERT INTO t1 VALUES('maroon','meal');
+  CREATE TABLE t2(cc, dd, ee, x VARCHAR(100));
+  INSERT INTO t2(cc,dd,ee) VALUES('maroon',1,'one');
+  INSERT INTO t2(cc,dd,ee) VALUES('maroon',2,'two');
+  INSERT INTO t2(cc,dd,ee) VALUES('maroon',0,'zero');
+  CREATE INDEX t2ddee ON t2(dd,ee);
+  CREATE INDEX t2cc ON t2(cc);
+   ANALYZE;
+  SELECT t2.cc, t2.dd, t2.ee FROM t1 CROSS JOIN t2 ON t1.aa=t2.cc
+  ORDER BY t2.dd LIMIT 1;
+} {maroon 0 zero}
+do_execsql_test 720 {
+  SELECT t2.cc, t2.dd, t2.ee FROM t1 CROSS JOIN t2 ON t1.aa=t2.cc
+  WHERE t1.bb='meal'
+  ORDER BY t2.dd LIMIT 1;
+} {maroon 0 zero}
+
 finish_test