From: drh Date: Wed, 2 Dec 2020 00:20:00 +0000 (+0000) Subject: Slightly faster tokenization of non-keyword identifiers. X-Git-Tag: version-3.35.0~213 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=c52a1e9f356e0733101e7af3d77959212fbe7611;p=thirdparty%2Fsqlite.git Slightly faster tokenization of non-keyword identifiers. FossilOrigin-Name: 55fa22bd403cc8f0973efea898a7cfa3a32b57c7e2a7a4c30c3f2c72d5396f07 --- c52a1e9f356e0733101e7af3d77959212fbe7611 diff --cc manifest index c47550693d,d3f2807345..c44588e565 --- a/manifest +++ b/manifest @@@ -1,5 -1,5 +1,5 @@@ - C Add\sthe\s--lookaside\sSIZE\sCOUNT\scommand-line\soption\sto\sthe\sdbfuzz2\stesting\stool. - D 2020-12-01T23:18:13.197 -C Improve\sthe\sspeed\sof\sthe\stokenizer\sby\srecognizing\sthat\stokens\sstarting\nwith\sletters\s"_",\s"Y",\sor\s"Z"\scan\snever\sbe\sSQL\skeywords\sand\smust\sbe\sordinary\nidentifiers. -D 2020-11-27T20:56:16.951 ++C Slightly\sfaster\stokenization\sof\snon-keyword\sidentifiers. ++D 2020-12-02T00:20:00.564 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@@ -1390,7 -1390,7 +1390,7 @@@ F test/speed3.test 694affeb910052600743 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 377a0c48e5a92e0b11c1c5ebb1bc9d83a7312c922bc0cb05970ef5d6a96d1f0c --F test/speedtest1.c 849dbcb0bded9f966e43f28e8ce824b2915cb5dd0031a2e85996f7e3de36c2b1 ++F test/speedtest1.c 5e5b805f24cc939656058f6a498f5a2160f9142e4815c54faf758ec798d4cdad F test/spellfix.test 951a6405d49d1a23d6b78027d3877b4a33eeb8221dcab5704b499755bb4f552e F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33 @@@ -1886,7 -1886,10 +1886,8 @@@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a9 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 - P a26b6597e3ae272231b96f9982c3bcc17ddec2f2b6eb4df06a224b91089fed5b - R 2c2b0393b1f5ca930e6c9ce57f506377 -P 8edb983bc87898eff2cd2e7e672a32a47c71b2be9d818513d339e95560d45b2b -R a0457978bc43ec4e3c0a4c4481e2d29f -T *branch * faster-tokenizer -T *sym-faster-tokenizer * -T -sym-trunk * ++P 2466960c0ba02ef9c325e9a5f8603db518e7529547f614c225fef430421e1643 16e281ed6219cc229dec7e3f1b40da2304dc270a74fd6ef78d04a088e30e7026 ++R a8add1c818b1e70c15ae61bdff16cc29 ++T +closed 16e281ed6219cc229dec7e3f1b40da2304dc270a74fd6ef78d04a088e30e7026 U drh - Z 2700eec066243e283a576b1178e7dd55 -Z 7f6094958178f08bfb909f826361ecf4 ++Z 1f1b11cb63e506030abbce26b0c97982 diff --cc manifest.uuid index d124fe677f,c33fe4a84a..290dd4158f --- a/manifest.uuid +++ b/manifest.uuid @@@ -1,1 -1,1 +1,1 @@@ - 2466960c0ba02ef9c325e9a5f8603db518e7529547f614c225fef430421e1643 -16e281ed6219cc229dec7e3f1b40da2304dc270a74fd6ef78d04a088e30e7026 ++55fa22bd403cc8f0973efea898a7cfa3a32b57c7e2a7a4c30c3f2c72d5396f07 diff --cc test/speedtest1.c index 3d7ac4c8e8,3d7ac4c8e8..9e8c396085 --- a/test/speedtest1.c +++ b/test/speedtest1.c @@@ -707,9 -707,9 +707,9 @@@ void testset_main(void) maxb = roundup_allones(sz); speedtest1_begin_test(100, "%d INSERTs into table with no index", n); speedtest1_exec("BEGIN"); -- speedtest1_exec("CREATE%s TABLE t1(a INTEGER %s, b INTEGER %s, c TEXT %s);", ++ speedtest1_exec("CREATE%s TABLE z1(a INTEGER %s, b INTEGER %s, c TEXT %s);", isTemp(9), g.zNN, g.zNN, g.zNN); -- speedtest1_prepare("INSERT INTO t1 VALUES(?1,?2,?3); -- %d times", n); ++ speedtest1_prepare("INSERT INTO z1 VALUES(?1,?2,?3); -- %d times", n); for(i=1; i<=n; i++){ x1 = swizzle(i,maxb); speedtest1_numbername(x1, zNum, sizeof(zNum)); @@@ -726,9 -726,9 +726,9 @@@ speedtest1_begin_test(110, "%d ordered INSERTS with one index/PK", n); speedtest1_exec("BEGIN"); speedtest1_exec( -- "CREATE%s TABLE t2(a INTEGER %s %s, b INTEGER %s, c TEXT %s) %s", ++ "CREATE%s TABLE z2(a INTEGER %s %s, b INTEGER %s, c TEXT %s) %s", isTemp(5), g.zNN, g.zPK, g.zNN, g.zNN, g.zWR); -- speedtest1_prepare("INSERT INTO t2 VALUES(?1,?2,?3); -- %d times", n); ++ speedtest1_prepare("INSERT INTO z2 VALUES(?1,?2,?3); -- %d times", n); for(i=1; i<=n; i++){ x1 = swizzle(i,maxb); speedtest1_numbername(x1, zNum, sizeof(zNum)); @@@ -768,7 -768,7 +768,7 @@@ speedtest1_begin_test(130, "%d SELECTS, numeric BETWEEN, unindexed", n); speedtest1_exec("BEGIN"); speedtest1_prepare( -- "SELECT count(*), avg(b), sum(length(c)), group_concat(c) FROM t1\n" ++ "SELECT count(*), avg(b), sum(length(c)), group_concat(c) FROM z1\n" " WHERE b BETWEEN ?1 AND ?2; -- %d times", n ); for(i=1; i<=n; i++){ @@@ -788,7 -788,7 +788,7 @@@ speedtest1_begin_test(140, "%d SELECTS, LIKE, unindexed", n); speedtest1_exec("BEGIN"); speedtest1_prepare( -- "SELECT count(*), avg(b), sum(length(c)), group_concat(c) FROM t1\n" ++ "SELECT count(*), avg(b), sum(length(c)), group_concat(c) FROM z1\n" " WHERE c LIKE ?1; -- %d times", n ); for(i=1; i<=n; i++){ @@@ -810,7 -810,7 +810,7 @@@ speedtest1_begin_test(142, "%d SELECTS w/ORDER BY, unindexed", n); speedtest1_exec("BEGIN"); speedtest1_prepare( -- "SELECT a, b, c FROM t1 WHERE c LIKE ?1\n" ++ "SELECT a, b, c FROM z1 WHERE c LIKE ?1\n" " ORDER BY a; -- %d times", n ); for(i=1; i<=n; i++){ @@@ -831,7 -831,7 +831,7 @@@ speedtest1_begin_test(145, "%d SELECTS w/ORDER BY and LIMIT, unindexed", n); speedtest1_exec("BEGIN"); speedtest1_prepare( -- "SELECT a, b, c FROM t1 WHERE c LIKE ?1\n" ++ "SELECT a, b, c FROM z1 WHERE c LIKE ?1\n" " ORDER BY a LIMIT 10; -- %d times", n ); for(i=1; i<=n; i++){ @@@ -851,10 -851,10 +851,10 @@@ speedtest1_begin_test(150, "CREATE INDEX five times"); speedtest1_exec("BEGIN;"); -- speedtest1_exec("CREATE UNIQUE INDEX t1b ON t1(b);"); -- speedtest1_exec("CREATE INDEX t1c ON t1(c);"); -- speedtest1_exec("CREATE UNIQUE INDEX t2b ON t2(b);"); -- speedtest1_exec("CREATE INDEX t2c ON t2(c DESC);"); ++ speedtest1_exec("CREATE UNIQUE INDEX t1b ON z1(b);"); ++ speedtest1_exec("CREATE INDEX t1c ON z1(c);"); ++ speedtest1_exec("CREATE UNIQUE INDEX t2b ON z2(b);"); ++ speedtest1_exec("CREATE INDEX t2c ON z2(c DESC);"); speedtest1_exec("CREATE INDEX t3bc ON t3(b,c);"); speedtest1_exec("COMMIT;"); speedtest1_end_test(); @@@ -864,7 -864,7 +864,7 @@@ speedtest1_begin_test(160, "%d SELECTS, numeric BETWEEN, indexed", n); speedtest1_exec("BEGIN"); speedtest1_prepare( -- "SELECT count(*), avg(b), sum(length(c)), group_concat(a) FROM t1\n" ++ "SELECT count(*), avg(b), sum(length(c)), group_concat(a) FROM z1\n" " WHERE b BETWEEN ?1 AND ?2; -- %d times", n ); for(i=1; i<=n; i++){ @@@ -884,7 -884,7 +884,7 @@@ speedtest1_begin_test(161, "%d SELECTS, numeric BETWEEN, PK", n); speedtest1_exec("BEGIN"); speedtest1_prepare( -- "SELECT count(*), avg(b), sum(length(c)), group_concat(a) FROM t2\n" ++ "SELECT count(*), avg(b), sum(length(c)), group_concat(a) FROM z2\n" " WHERE a BETWEEN ?1 AND ?2; -- %d times", n ); for(i=1; i<=n; i++){ @@@ -904,7 -904,7 +904,7 @@@ speedtest1_begin_test(170, "%d SELECTS, text BETWEEN, indexed", n); speedtest1_exec("BEGIN"); speedtest1_prepare( -- "SELECT count(*), avg(b), sum(length(c)), group_concat(a) FROM t1\n" ++ "SELECT count(*), avg(b), sum(length(c)), group_concat(a) FROM z1\n" " WHERE c BETWEEN ?1 AND (?1||'~'); -- %d times", n ); for(i=1; i<=n; i++){ @@@ -930,14 -930,14 +930,14 @@@ isTemp(1), g.zNN, g.zPK, g.zNN, g.zNN, g.zWR); speedtest1_exec("CREATE INDEX t4b ON t4(b)"); speedtest1_exec("CREATE INDEX t4c ON t4(c)"); -- speedtest1_exec("INSERT INTO t4 SELECT * FROM t1"); ++ speedtest1_exec("INSERT INTO t4 SELECT * FROM z1"); speedtest1_exec("COMMIT"); speedtest1_end_test(); n = sz; speedtest1_begin_test(190, "DELETE and REFILL one table", n); -- speedtest1_exec("DELETE FROM t2;"); -- speedtest1_exec("INSERT INTO t2 SELECT * FROM t1;"); ++ speedtest1_exec("DELETE FROM z2;"); ++ speedtest1_exec("INSERT INTO z2 SELECT * FROM z1;"); speedtest1_end_test(); @@@ -947,8 -947,8 +947,8 @@@ speedtest1_begin_test(210, "ALTER TABLE ADD COLUMN, and query"); -- speedtest1_exec("ALTER TABLE t2 ADD COLUMN d DEFAULT 123"); -- speedtest1_exec("SELECT sum(d) FROM t2"); ++ speedtest1_exec("ALTER TABLE z2 ADD COLUMN d DEFAULT 123"); ++ speedtest1_exec("SELECT sum(d) FROM z2"); speedtest1_end_test(); @@@ -956,7 -956,7 +956,7 @@@ speedtest1_begin_test(230, "%d UPDATES, numeric BETWEEN, indexed", n); speedtest1_exec("BEGIN"); speedtest1_prepare( -- "UPDATE t2 SET d=b*2 WHERE b BETWEEN ?1 AND ?2; -- %d times", n ++ "UPDATE z2 SET d=b*2 WHERE b BETWEEN ?1 AND ?2; -- %d times", n ); for(i=1; i<=n; i++){ x1 = speedtest1_random()%maxb; @@@ -973,7 -973,7 +973,7 @@@ speedtest1_begin_test(240, "%d UPDATES of individual rows", n); speedtest1_exec("BEGIN"); speedtest1_prepare( -- "UPDATE t2 SET d=b*3 WHERE a=?1; -- %d times", n ++ "UPDATE z2 SET d=b*3 WHERE a=?1; -- %d times", n ); for(i=1; i<=n; i++){ x1 = speedtest1_random()%sz + 1; @@@ -984,12 -984,12 +984,12 @@@ speedtest1_end_test(); speedtest1_begin_test(250, "One big UPDATE of the whole %d-row table", sz); -- speedtest1_exec("UPDATE t2 SET d=b*4"); ++ speedtest1_exec("UPDATE z2 SET d=b*4"); speedtest1_end_test(); speedtest1_begin_test(260, "Query added column after filling"); -- speedtest1_exec("SELECT sum(d) FROM t2"); ++ speedtest1_exec("SELECT sum(d) FROM z2"); speedtest1_end_test(); @@@ -998,7 -998,7 +998,7 @@@ speedtest1_begin_test(270, "%d DELETEs, numeric BETWEEN, indexed", n); speedtest1_exec("BEGIN"); speedtest1_prepare( -- "DELETE FROM t2 WHERE b BETWEEN ?1 AND ?2; -- %d times", n ++ "DELETE FROM z2 WHERE b BETWEEN ?1 AND ?2; -- %d times", n ); for(i=1; i<=n; i++){ x1 = speedtest1_random()%maxb + 1; @@@ -1027,16 -1027,16 +1027,16 @@@ speedtest1_begin_test(290, "Refill two %d-row tables using REPLACE", sz); -- speedtest1_exec("REPLACE INTO t2(a,b,c) SELECT a,b,c FROM t1"); -- speedtest1_exec("REPLACE INTO t3(a,b,c) SELECT a,b,c FROM t1"); ++ speedtest1_exec("REPLACE INTO z2(a,b,c) SELECT a,b,c FROM z1"); ++ speedtest1_exec("REPLACE INTO t3(a,b,c) SELECT a,b,c FROM z1"); speedtest1_end_test(); speedtest1_begin_test(300, "Refill a %d-row table using (b&1)==(a&1)", sz); -- speedtest1_exec("DELETE FROM t2;"); -- speedtest1_exec("INSERT INTO t2(a,b,c)\n" -- " SELECT a,b,c FROM t1 WHERE (b&1)==(a&1);"); -- speedtest1_exec("INSERT INTO t2(a,b,c)\n" -- " SELECT a,b,c FROM t1 WHERE (b&1)<>(a&1);"); ++ speedtest1_exec("DELETE FROM z2;"); ++ speedtest1_exec("INSERT INTO z2(a,b,c)\n" ++ " SELECT a,b,c FROM z1 WHERE (b&1)==(a&1);"); ++ speedtest1_exec("INSERT INTO z2(a,b,c)\n" ++ " SELECT a,b,c FROM z1 WHERE (b&1)<>(a&1);"); speedtest1_end_test(); @@@ -1044,11 -1044,11 +1044,11 @@@ speedtest1_begin_test(310, "%d four-ways joins", n); speedtest1_exec("BEGIN"); speedtest1_prepare( -- "SELECT t1.c FROM t1, t2, t3, t4\n" ++ "SELECT z1.c FROM z1, z2, t3, t4\n" " WHERE t4.a BETWEEN ?1 AND ?2\n" " AND t3.a=t4.b\n" -- " AND t2.a=t3.b\n" -- " AND t1.c=t2.c" ++ " AND z2.a=t3.b\n" ++ " AND z1.c=z2.c" ); for(i=1; i<=n; i++){ x1 = speedtest1_random()%sz + 1; @@@ -1063,8 -1063,8 +1063,8 @@@ speedtest1_begin_test(320, "subquery in result set", n); speedtest1_prepare( "SELECT sum(a), max(c),\n" -- " avg((SELECT a FROM t2 WHERE 5+t2.b=t1.b) AND rowid=?1 AND x1<=?2"); ++ speedtest1_prepare("SELECT count(*) FROM z1 WHERE x0>=?1 AND x1<=?2"); iStep = mxCoord/n; for(i=0; i=?1 AND y0<=?2"); ++ speedtest1_prepare("SELECT count(*) FROM z1 WHERE y1>=?1 AND y0<=?2"); iStep = mxCoord/n; for(i=0; i