Refactor the LIKE optimization decision logic so that it uses

sqlite3AtoF() on both boundary keys to determine if the optimization can be
used when the LHS is something that might not have TEXT affinity.
Ticket [ce8717f0885af975].  See also [c94369cae9b561b1],
[b043a54c3de54b28], [fd76310a5e843e07], and [158290c0abafde67].

FossilOrigin-Name: b4a9e09e60213ccff925d09f0b6e549e2a3e3862856c710f108779e2867dec76

diff --git a/manifest b/manifest
index 4de3d8c50ae6ee813a6860b051493b5d3343ab09..89460f0100c6d46e8b554f5614ce5b4b281eb89d 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sa\snew\srequirement\smark\son\sthe\sctime.c\scode.
-D 2019-06-13T16:14:53.717
+C Refactor\sthe\sLIKE\soptimization\sdecision\slogic\sso\sthat\sit\suses\nsqlite3AtoF()\son\sboth\sboundary\skeys\sto\sdetermine\sif\sthe\soptimization\scan\sbe\nused\swhen\sthe\sLHS\sis\ssomething\sthat\smight\snot\shave\sTEXT\saffinity.\nTicket\s[ce8717f0885af975].\s\sSee\salso\s[c94369cae9b561b1],\n[b043a54c3de54b28],\s[fd76310a5e843e07],\sand\s[158290c0abafde67].
+D 2019-06-14T12:28:21.568
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -611,7 +611,7 @@ F src/walker.c 7607f1a68130c028255d8d56094ea602fc402c79e1e35a46e6282849d90d5fe4
 F src/where.c 99c7b718ef846ac952016083aaf4e22ede2290beceaf4730a2df55c023251369
 F src/whereInt.h 1b728f71654ebf8421a1715497a587f02d6f538e819af58dc826908f8577e810
 F src/wherecode.c 37a1004237d630d785c47bba2290eac652a7a8b0047518eba3cb7c808b604c4a
-F src/whereexpr.c d0683adb125754b3edd44d7c73fa2e4ea175eaacd900f1dc6993da2f6f0149cc
+F src/whereexpr.c 5e559bdd24b06e3bc2e68f258bf751302954dc1e432daf71fdd8098a71462326
 F src/window.c 5be2cf7d8763cc97137fc44d015aed8a1a4a56fe9700d7933ed560172617c756
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd
@@ -1095,7 +1095,7 @@ F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200
 F test/lemon-test01.y 58b764610fd934e189ffbb0bbfa33d171b9cb06019b55bdc04d090d6767e11d7
 F test/like.test 5013f18e7242fe118524fcf8e484b8827bcd5906b509d106f3587c7bfcf274ae
 F test/like2.test 3b2ee13149ba4a8a60b59756f4e5d345573852da
-F test/like3.test ac61947ef35bde9d97718bcfa04659a17d9218f1fffc4104b135b3f82ed43836
+F test/like3.test 62bf82ac674b7d4126e73532e1ad96cdcd8e1752a4ed90b28a6f98305f5a66aa
 F test/limit.test 0c99a27a87b14c646a9d583c7c89fd06c352663e
 F test/limit2.test 9409b033284642a859fafc95f29a5a6a557bd57c1f0d7c3f554bd64ed69df77e
 F test/loadext.test faa4f6eed07a5aac35d57fdd7bc07f8fc82464cfd327567c10cf0ba3c86cde04
@@ -1427,7 +1427,7 @@ F test/tkt-5e10420e8d.test 904d1687b3c06d43e5b3555bbcf6802e7c0ffd84
 F test/tkt-5ee23731f.test 9db6e1d7209dc0794948b260d6f82b2b1de83a9f
 F test/tkt-6bfb98dfc0.test 24780633627b5cfc0635a5500c2389ebfb563336
 F test/tkt-752e1646fc.test ea78d88d14fe9866bdd991c634483334639e13bf
-F test/tkt-78e04e52ea.test 1b5be1bac961833a9fd70fe50738cb4064822c61f82c54f7d488435ec806ea62
+F test/tkt-78e04e52ea.test cb44d0f5e7940223be740a39913a1b9b9b30d7e4a17ed3349141f893bae1b8f2
 F test/tkt-7a31705a7e6.test 9e9c057b6a9497c8f7ba7b16871029414ccf6550e7345d9085d6d71c9a56bb6f
 F test/tkt-7bbfb7d442.test 7b2cd79c7a17ae6750e75ec1a7846712a69c9d18
 F test/tkt-80ba201079.test 105a721e6aad0ae3c5946d7615d1e4d03f6145b8
@@ -1606,7 +1606,7 @@ F test/vacuummem.test 7b42abb3208bd82dd23a7536588396f295a314f2
 F test/varint.test bbce22cda8fc4d135bcc2b589574be8410614e62
 F test/veryquick.test 57ab846bacf7b90cf4e9a672721ea5c5b669b661
 F test/view.test 40d54c9ddf5b9fdee692bf936d3352159fe467838f92742aa1d08c7c7d1eac73
-F test/vtab1.test 60b4f70aafa6078d6fdfc11417af3bd216d7ef5eafce16707a6ca3dae5166d20
+F test/vtab1.test 47b935205d7b6290765973abd6fce0e13e94e040017099170f43f81886efa5ba
 F test/vtab2.test 14d4ab26cee13ba6cf5c5601b158e4f57552d3b055cdd9406cf7f711e9c84082
 F test/vtab3.test b45f47d20f225ccc9c28dc915d92740c2dee311e
 F test/vtab4.test 8e73ed268f3d596bc3590f45fc948fb40f28e9c3
@@ -1621,7 +1621,7 @@ F test/vtabC.test 4528f459a13136f982e75614d120aef165f17292
 F test/vtabD.test 05b3f1d77117271671089e48719524b676842e96
 F test/vtabE.test 2a143fe75a11275781d1fd1988d86b66a3f69cb98f4add62e3da8fd0f637b45f
 F test/vtabF.test 1918844c7c902f6a16c8dacf1ec8f84886d6e78b
-F test/vtabH.test 3cf9aa1c1c4381b3b3ac33f933376f06fbb99d2294a83c79b7562d3ed87be450
+F test/vtabH.test e65540eed0f7434cdf0b160374570b51f3e3179548f0fa5e99b1d33f8dcdf9a0
 F test/vtabI.test 751b07636700dbdea328e4265b6077ccd6811a3f
 F test/vtabJ.test d7b73675708cf63cfcb9d443bb451fc01a028347275b7311e51f9fdf3ca6757f
 F test/vtab_alter.test 736e66fb5ec7b4fee58229aa3ada2f27ec58bc58c00edae4836890c3784c6783
@@ -1830,7 +1830,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P f8696b60eec0dcacfe92d9a31cbf1436d674140e5447de0cd1c2f52bff6c2be4
-R d629c2a79ba0d4b863d7e3516a4b3f0e
+P c4b405687b010ee20ec02c42913a0540909d0155c88a4a56194fda99c704279e
+R 169cc5ff08963701bea2f41ccb640ca5
 U drh
-Z 26c2a9c5cf62b22c3542f90299b66f73
+Z 338301df57dfe8eacff0a4b94bfd0060

diff --git a/manifest.uuid b/manifest.uuid
index 251d32c58ea7fc49585cbcf2f336ed30bb82d901..a05adedae2448a4a0c60dde310ecc410f5087fdc 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c4b405687b010ee20ec02c42913a0540909d0155c88a4a56194fda99c704279e
\ No newline at end of file
+b4a9e09e60213ccff925d09f0b6e549e2a3e3862856c710f108779e2867dec76
\ No newline at end of file

diff --git a/src/whereexpr.c b/src/whereexpr.c
index db460652ce1c0b5cd19143f3394bfbd41e591bf7..9ac940c5a06788954912abf67b26163988b88382 100644 (file)
--- a/src/whereexpr.c
+++ b/src/whereexpr.c
@@ -264,28 +264,31 @@ static int isLikeOrGlob(
         zNew[iTo] = 0;
         assert( iTo>0 );
 
-        /* If the RHS begins with a digit, a +/- sign or whitespace, then the
-        ** LHS must be an ordinary column (not a virtual table column) with
-        ** TEXT affinity. Otherwise the LHS might be numeric and "lhs >= rhs"
-        ** would be false even though "lhs LIKE rhs" is true.  But if the RHS
-        ** does not start with a digit or +/-, then "lhs LIKE rhs" will always
-        ** be false if the LHS is numeric and so the optimization still works.
+        /* If the LHS is not an ordinary column with TEXT affinity, then the
+        ** pattern prefix boundaries (both the start and end boundaries) must
+        ** not look like a number.  Otherwise the pattern might be treated as
+        ** a number, which will invalidate the LIKE optimization.
         **
-        ** 2018-09-10 ticket c94369cae9b561b1f996d0054bfab11389f9d033
-        ** The RHS pattern must not be '/%' because the termination condition
-        ** will then become "x<'0'" and if the affinity is numeric, will then
-        ** be converted into "x<0", which is incorrect.
+        ** Getting this right has been a persistent source of bugs in the
+        ** LIKE optimization.  See, for example:
+        **    2018-09-10 https://sqlite.org/src/info/c94369cae9b561b1
+        **    2019-05-02 https://sqlite.org/src/info/b043a54c3de54b28
+        **    2019-06-10 https://sqlite.org/src/info/fd76310a5e843e07
+        **    2019-06-14 https://sqlite.org/src/info/ce8717f0885af975
         */
-        if( sqlite3Isdigit(zNew[0])
-         || sqlite3Isspace(zNew[0])
-         || zNew[0]=='-'
-         || zNew[0]=='+'
-         || zNew[iTo-1]=='0'-1
+        if( pLeft->op!=TK_COLUMN 
+         || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT 
+         || IsVirtual(pLeft->y.pTab)  /* Value might be numeric */
         ){
-          if( pLeft->op!=TK_COLUMN 
-           || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT 
-           || IsVirtual(pLeft->y.pTab)  /* Value might be numeric */
-          ){
+          int isNum;
+          double rDummy;
+          isNum = sqlite3AtoF(zNew, &rDummy, iTo, SQLITE_UTF8);
+          if( isNum<=0 ){
+            zNew[iTo-1]++;
+            isNum = sqlite3AtoF(zNew, &rDummy, iTo, SQLITE_UTF8);
+            zNew[iTo-1]--;
+          }
+          if( isNum>0 ){
             sqlite3ExprDelete(db, pPrefix);
             sqlite3ValueFree(pVal);
             return 0;

diff --git a/test/like3.test b/test/like3.test
index 5e75571b617b51e0136258366271e9f41b791d3f..f64fb637be06a966b670733c93ea6179e46839d7 100644 (file)
--- a/test/like3.test
+++ b/test/like3.test
@@ -198,6 +198,15 @@ do_execsql_test like3-5.400 {
   SELECT * FROM t0 WHERE t0.c0 LIKE './';
 } {./}
 
+# 2019-06-14
+# Ticket https://www.sqlite.org/src/info/ce8717f0885af975
+do_execsql_test like3-5.410 {
+  DROP TABLE IF EXISTS t0;
+  CREATE TABLE t0(c0 INT UNIQUE COLLATE NOCASE);
+  INSERT INTO t0(c0) VALUES ('.1%');
+  SELECT * FROM t0 WHERE t0.c0 LIKE '.1%';
+} {.1%}
+
 
 # 2019-02-27
 # Verify that the LIKE optimization works with an ESCAPE clause when

diff --git a/test/tkt-78e04e52ea.test b/test/tkt-78e04e52ea.test
index 47a1093dd8cac8041406d0a4637339bec24fd62f..a432e6c22ba880c09b4dbd1dc0871b836188e5c1 100644 (file)
--- a/test/tkt-78e04e52ea.test
+++ b/test/tkt-78e04e52ea.test
@@ -41,7 +41,7 @@ do_test tkt-78e04-1.3 {
   }
 } {}
 do_test tkt-78e04-1.4 {
- db eval {EXPLAIN QUERY PLAN SELECT "" FROM "" WHERE "" LIKE '1abc%';}
+ db eval {EXPLAIN QUERY PLAN SELECT "" FROM "" WHERE "" LIKE '1e5%';}
 } {/*SCAN TABLE  USING COVERING INDEX i1*/}
 do_test tkt-78e04-1.5 {
   execsql {

diff --git a/test/vtab1.test b/test/vtab1.test
index 193a53acf7699596e5c6c198f88ac3c89ec94494..8280715cdcbf8424a836d50f4ff2a32aa2ea89e0 100644 (file)
--- a/test/vtab1.test
+++ b/test/vtab1.test
@@ -1311,10 +1311,13 @@ foreach {tn sql res filter} {
     {xFilter {SELECT rowid, a, b FROM 't6' WHERE b >= ? AND b < ?} 8 9}
 
   1.3 "SELECT a FROM e6 WHERE b LIKE '8J%'" {3 4}
-    {xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} 8J%}
+    {xFilter {SELECT rowid, a, b FROM 't6' WHERE b >= ? AND b < ? AND b like ?} 8J 8k 8J%}
 
   1.4 "SELECT a FROM e6 WHERE b LIKE '8j%'" {3 4}
-    {xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} 8j%}
+    {xFilter {SELECT rowid, a, b FROM 't6' WHERE b >= ? AND b < ? AND b like ?} 8J 8k 8j%}
+
+  1.5 "SELECT a FROM e6 WHERE b LIKE '8%'" {3 4}
+    {xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} 8%}
 } {
   set echo_module {}
   do_execsql_test 18.$tn.1 $sql $res
@@ -1323,11 +1326,14 @@ foreach {tn sql res filter} {
 
 do_execsql_test 18.2.0 {  PRAGMA case_sensitive_like = ON }
 foreach {tn sql res filter} {
-  2.1 "SELECT a FROM e6 WHERE b LIKE '8J%'" {3 4}
-    {xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} 8J%}
+  2.1 "SELECT a FROM e6 WHERE b LIKE '8%'" {3 4}
+    {xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} 8%}
 
   2.2 "SELECT a FROM e6 WHERE b LIKE '8j%'" {}
-    {xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} 8j%}
+    {xFilter {SELECT rowid, a, b FROM 't6' WHERE b >= ? AND b < ? AND b like ?} 8j 8k 8j%}
+
+  2.3 "SELECT a FROM e6 WHERE b LIKE '8J%'" {3 4}
+    {xFilter {SELECT rowid, a, b FROM 't6' WHERE b >= ? AND b < ? AND b like ?} 8J 8K 8J%}
 } {
   set echo_module {}
   do_execsql_test 18.$tn.1 $sql $res

diff --git a/test/vtabH.test b/test/vtabH.test
index 56c12544f81abb165217bed785989e6255c9ef36..78b156cb638e643e0292949c086a4955dc899ac9 100644 (file)
--- a/test/vtabH.test
+++ b/test/vtabH.test
@@ -32,13 +32,27 @@ do_execsql_test 1.0 {
 
 foreach {tn sql expect} {
   1 "SELECT * FROM e6 WHERE b LIKE '8abc'" {
-    xBestIndex {SELECT rowid, a, b FROM 't6' WHERE b like ?}
-    xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} 8abc
+    xBestIndex 
+       {SELECT rowid, a, b FROM 't6' WHERE b >= ? AND b < ? AND b like ?}
+    xFilter
+       {SELECT rowid, a, b FROM 't6' WHERE b >= ? AND b < ? AND b like ?}
+       8ABC 8abd 8abc
   }
 
   2 "SELECT * FROM e6 WHERE b GLOB '8abc'" {
+     xBestIndex
+       {SELECT rowid, a, b FROM 't6' WHERE b >= ? AND b < ? AND b glob ?}
+     xFilter
+       {SELECT rowid, a, b FROM 't6' WHERE b >= ? AND b < ? AND b glob ?}
+       8abc 8abd 8abc
+  }
+  3 "SELECT * FROM e6 WHERE b LIKE '8e/'" {
+    xBestIndex {SELECT rowid, a, b FROM 't6' WHERE b like ?}
+    xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} 8e/
+  }
+  4 "SELECT * FROM e6 WHERE b GLOB '8e/'" {
     xBestIndex {SELECT rowid, a, b FROM 't6' WHERE b glob ?}
-    xFilter {SELECT rowid, a, b FROM 't6' WHERE b glob ?} 8abc
+    xFilter {SELECT rowid, a, b FROM 't6' WHERE b glob ?} 8e/
   }
 } {
   do_test 1.$tn {