From: dan Date: Fri, 29 Jun 2018 17:44:52 +0000 (+0000) Subject: Instead of using a lemon %fallback directive, have the tokenizer try to figure X-Git-Tag: version-3.25.0~178^2~1^2~4 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=59ff42516cdd4a5c4d5e5d65b45ce7e312bf0472;p=thirdparty%2Fsqlite.git Instead of using a lemon %fallback directive, have the tokenizer try to figure out whether an instance of "WINDOW" should be TK_WINDOW or TK_ID. FossilOrigin-Name: 022079cb0d67be5ac0a50dd9a4d41ee55ce8df681ecd0a544170d75fc8649978 --- diff --git a/manifest b/manifest index 69ff0aa49e..e6513a564a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modifications\sto\sparse.y\sto\sbetter\ssupport\sbackwards\scompatibility\sfor\sthe\n"window"\skeyword. -D 2018-06-28T20:05:04.235 +C Instead\sof\susing\sa\slemon\s%fallback\sdirective,\shave\sthe\stokenizer\stry\sto\sfigure\nout\swhether\san\sinstance\sof\s"WINDOW"\sshould\sbe\sTK_WINDOW\sor\sTK_ID. +D 2018-06-29T17:44:52.692 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 0a3a6c81e6fcb969ff9106e882f0a08547014ba463cb6beca4c4efaecc924ee6 @@ -484,7 +484,7 @@ F src/os_win.c ac29c25cde4cfb4adacc59cdec4aa45698ca0e29164ea127859585ccd9faa354 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 1bb6a57fa0465296a4d6109a1a64610a0e7adde1f3acf3ef539a9d972908ce8f F src/pager.h c571b064df842ec8f2e90855dead9acf4cbe0d1b2c05afe0ef0d0145f7fd0388 -F src/parse.y 9b5eb0e61e615832647a7a77f9e675f1da536a133426327e6812316d83f275e7 +F src/parse.y 216e0e953df076ef89668614e5aab8b489e0c2c24f7ff17cf42e7bea802793a9 F src/pcache.c 135ef0bc6fb2e3b7178d49ab5c9176254c8a691832c1bceb1156b2fbdd0869bd F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880 @@ -500,7 +500,7 @@ F src/shell.c.in 8578421c5fb2a972461b2a996f7173646e55e0dbd2a2eee30c8f5dc7d3dbadf F src/sqlite.h.in 243e0b1d302c237372c06002fe84f731fd22587eccd142d15de834a5296514ff F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9887b27e69c01e79c2cbe74ef73bf01af5b5703d6a7f0a4371e386d7249cb1c7 -F src/sqliteInt.h 74728a0b20e9d2173470186a8ca1e3229c79abb19d1ac4eedc4562e8905d798d +F src/sqliteInt.h c14c850bf0c4dd3860cc5092f140ae9faf93b77737b3febf38454088e1ab3988 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -558,7 +558,7 @@ F src/test_windirent.h 90dfbe95442c9762357fe128dc7ae3dc199d006de93eb33ba3972e0a9 F src/test_window.c add59ee68568868129516999f30a68e8ab2afd276e272aba4f633c9fc52c1bb1 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c -F src/tokenize.c b42aa952b05df665ed10d54f02c467159cd9fcb8cbbe5ff3c9cd74781fa2803c +F src/tokenize.c e25caaeb2ad912a378e528771d843badbaee560f2eb8c25dbface05a32c2e4cf F src/treeview.c 2c5c4bc0a443401db5fd621542150452ddf5055d38edd4eef868bc2b6bfb0260 F src/trigger.c 4ace6d1d5ba9a89822deb287317f33c810440526eafe185c2d8a48c31df1e995 F src/update.c 46dc24c6158446aaab45caee09b6d99327cb479268b83ffeb5b701823da3b67b @@ -1626,6 +1626,7 @@ F test/window3.test 47884c240d0d5234ad9c6da65452cfec1bfa69ec6f9c4158cd9750c3d88d F test/window4.tcl 7cec7e578aa9f78b7265bff8d552cda17a1d8d89f0449d0e74970a527b8846f5 F test/window4.test dcd8767869988e0d23d56bc3f8b46ec116de23127b81b5f66fd48d5529072ed1 F test/window5.test a4835b96d30eb3b81a1dbc683e333e57a614645eb6f2ae476a7ed2addf0b0f1f +F test/window6.test 97bd18d5cccd612b8a93be4017a7a26c2968a1c868a458d0d66d8d8cc62e33f8 F test/windowfault.test 97d5fc404308edb579a5a183e294ed874c844ecf01f0a28ba46df3141ebaee1f F test/with1.test 58475190cd8caaeebea8cfeb2a264ec97a0c492b8ffe9ad20cefbb23df462f96 F test/with2.test e0030e2f0267a910d6c0e4f46f2dfe941c1cc0d4f659ba69b3597728e7e8f1ab @@ -1663,7 +1664,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f F tool/lemon.c 93a321bd82b5c73191e95b2f27203048bf4d054fdab4a257b7a6c9541f31828d -F tool/lempar.c c7371bad027fd516b7eaed36d03353972df35163672637cddd00f472889932f9 +F tool/lempar.c afc03d26f33db130c0ca7d5942000aea7eedd638455845ca76f792818896df19 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca @@ -1743,7 +1744,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c41d7079fb63831bd2f009b64cbf5a423e8119e6b841bfd9bd5a2d85fba3b142 -R 4e74da2117f8f8cceb7bb5b15336dc74 +P 7c4b879bdb10f30260f0fb28fcb559ab0949865d10f4ff2cf13834a326198998 +R 53ad10aee784d2508443a129bf20d430 U dan -Z e15fe89d6d9f69693116ca5d610ea195 +Z a766b00041899c4982e4087a15ea2a82 diff --git a/manifest.uuid b/manifest.uuid index 51740e4ee8..2825dfd995 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7c4b879bdb10f30260f0fb28fcb559ab0949865d10f4ff2cf13834a326198998 \ No newline at end of file +022079cb0d67be5ac0a50dd9a4d41ee55ce8df681ecd0a544170d75fc8649978 \ No newline at end of file diff --git a/src/parse.y b/src/parse.y index 1faa9104a1..0ed0b28bad 100644 --- a/src/parse.y +++ b/src/parse.y @@ -217,8 +217,8 @@ columnname(A) ::= nm(A) typetoken(Y). {sqlite3AddColumn(pParse,&A,&Y);} EXCEPT INTERSECT UNION %endif SQLITE_OMIT_COMPOUND_SELECT %ifndef SQLITE_OMIT_WINDOWFUNC - CURRENT FILTER FOLLOWING OVER PARTITION - PRECEDING RANGE UNBOUNDED + CURRENT FILTER FOLLOWING OVER PARTITION + PRECEDING RANGE UNBOUNDED %endif SQLITE_OMIT_WINDOWFUNC REINDEX RENAME CTIME_KW IF . @@ -257,7 +257,7 @@ columnname(A) ::= nm(A) typetoken(Y). {sqlite3AddColumn(pParse,&A,&Y);} // And "ids" is an identifer-or-string. // -%token_class ids ID|STRING. +%token_class ids ID|STRING. // The name of a column or table can be any of the following: // @@ -265,7 +265,6 @@ columnname(A) ::= nm(A) typetoken(Y). {sqlite3AddColumn(pParse,&A,&Y);} nm(A) ::= id(A). nm(A) ::= STRING(A). nm(A) ::= JOIN_KW(A). -nm(A) ::= WINDOW(A). // A typetoken is really zero or more tokens that form a type name such // as can be found after the column name in a CREATE TABLE statement. @@ -281,8 +280,8 @@ typetoken(A) ::= typename(A) LP signed COMMA signed RP(Y). { A.n = (int)(&Y.z[Y.n] - A.z); } %type typename {Token} -typename(A) ::= nm(A). -typename(A) ::= typename(A) nm(Y). {A.n=Y.n+(int)(Y.z-A.z);} +typename(A) ::= ids(A). +typename(A) ::= typename(A) ids(Y). {A.n=Y.n+(int)(Y.z-A.z);} signed ::= plus_num. signed ::= minus_num. @@ -343,7 +342,7 @@ ccons ::= CHECK LP expr(X) RP. {sqlite3AddCheckConstraint(pParse,X);} ccons ::= REFERENCES nm(T) eidlist_opt(TA) refargs(R). {sqlite3CreateForeignKey(pParse,0,&T,TA,R);} ccons ::= defer_subclause(D). {sqlite3DeferForeignKey(pParse,D);} -ccons ::= COLLATE nm(C). {sqlite3AddCollateType(pParse, &C);} +ccons ::= COLLATE ids(C). {sqlite3AddCollateType(pParse, &C);} // The optional AUTOINCREMENT keyword %type autoinc {int} @@ -989,7 +988,6 @@ expr(A) ::= nm(X) DOT nm(Y) DOT nm(Z). { } term(A) ::= NULL|FLOAT|BLOB(X). {A=tokenExpr(pParse,@X,X); /*A-overwrites-X*/} term(A) ::= STRING(X). {A=tokenExpr(pParse,@X,X); /*A-overwrites-X*/} -term(A) ::= WINDOW(X). {A=tokenExpr(pParse,TK_ID,X);/*A-overwrites-X*/} term(A) ::= INTEGER(X). { A = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &X, 1); } @@ -1013,7 +1011,7 @@ expr(A) ::= VARIABLE(X). { } } } -expr(A) ::= expr(A) COLLATE nm(C). { +expr(A) ::= expr(A) COLLATE ids(C). { A = sqlite3ExprAddCollateToken(pParse, A, &C, 1); } %ifndef SQLITE_OMIT_CAST @@ -1022,7 +1020,7 @@ expr(A) ::= CAST LP expr(E) AS typetoken(T) RP. { sqlite3ExprAttachSubtrees(pParse->db, A, E, 0); } %endif SQLITE_OMIT_CAST -expr(A) ::= nm(X) LP distinct(D) exprlist(Y) RP +expr(A) ::= id(X) LP distinct(D) exprlist(Y) RP %ifndef SQLITE_OMIT_WINDOWFUNC over_opt(Z) %endif @@ -1036,7 +1034,7 @@ expr(A) ::= nm(X) LP distinct(D) exprlist(Y) RP A->flags |= EP_Distinct; } } -expr(A) ::= nm(X) LP STAR RP +expr(A) ::= id(X) LP STAR RP %ifndef SQLITE_OMIT_WINDOWFUNC over_opt(Z) %endif @@ -1445,7 +1443,7 @@ eidlist(A) ::= nm(Y) collate(C) sortorder(Z). { %type collate {int} collate(C) ::= . {C = 0;} -collate(C) ::= COLLATE nm. {C = 1;} +collate(C) ::= COLLATE ids. {C = 1;} ///////////////////////////// The DROP INDEX command ///////////////////////// diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 9fab8f5590..448918a7f6 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4299,6 +4299,7 @@ char sqlite3IndexColumnAffinity(sqlite3*, Index*, int); void sqlite3ParserFree(void*, void(*)(void*)); #endif void sqlite3Parser(void*, int, Token); +int sqlite3ParserFallback(int); #ifdef YYTRACKMAXSTACKDEPTH int sqlite3ParserStackPeak(void*); #endif diff --git a/src/tokenize.c b/src/tokenize.c index 3286f94f54..10ad25e29f 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -188,6 +188,55 @@ const char sqlite3IsEbcdicIdChar[] = { int sqlite3IsIdChar(u8 c){ return IdChar(c); } #endif +/* +** Return the id of the next token in string (*pz). Before returning, set +** (*pz) to point to the byte following the parsed token. +** +** This function assumes that any keywords that start with "w" are +** actually TK_ID. +*/ +static int windowGetToken(const unsigned char **pz){ + int ret; + const unsigned char *z = *pz; + if( z[0]=='w' || z[0]=='W' ){ + do { z++; }while( IdChar(z[0]) ); + ret = TK_ID; + }else{ + z += sqlite3GetToken(z, &ret); + } + *pz = z; + return ret; +} + +/* +** The tokenizer has just parsed the keyword WINDOW. In this case the token +** may really be the keyword (TK_WINDOW), or may be an identifier (TK_ID). +** This function determines which it is by inspecting the next two tokens +** in the input stream. Specifically, the token is TK_WINDOW if the following +** two tokens are: +** +** * TK_ID, or something else that can be used as a window name, and +** * TK_AS. +** +** Instead of using sqlite3GetToken() to parse tokens directly, this function +** uses windowGetToken(). This is to avoid recursion if the input is similar +** to "window window window window". +*/ +static void analyzeWindowKeyword(const unsigned char *z, int *tokenType){ + int t; + assert( *tokenType==TK_WINDOW ); + while( (t = windowGetToken(&z))==TK_SPACE ); + if( t!=TK_ID && t!=TK_STRING + && t!=TK_JOIN_KW && sqlite3ParserFallback(t)!=TK_ID + ){ + *tokenType = TK_ID; + }else{ + while( (t = windowGetToken(&z))==TK_SPACE ); + if( t!=TK_AS ){ + *tokenType = TK_ID; + } + } +} /* ** Return the length (in bytes) of the token that begins at z[0]. @@ -433,7 +482,12 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){ break; } *tokenType = TK_ID; - return keywordCode((char*)z, i, tokenType); + keywordCode((char*)z, i, tokenType); + if( *tokenType==TK_WINDOW ){ + assert( i==6 ); + analyzeWindowKeyword(&z[6], tokenType); + } + return i; } case CC_X: { #ifndef SQLITE_OMIT_BLOB_LITERAL diff --git a/test/window6.test b/test/window6.test new file mode 100644 index 0000000000..a6706fc521 --- /dev/null +++ b/test/window6.test @@ -0,0 +1,117 @@ +# 2018 May 8 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. Specifically, +# it tests the sqlite3_create_window_function() API. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix window6 + +ifcapable !windowfunc { + finish_test + return +} + +set setup { + CREATE TABLE %t1(%a, %b %typename); + INSERT INTO %t1 VALUES(1, 'a'); + INSERT INTO %t1 VALUES(2, 'b'); + INSERT INTO %t1 VALUES(3, 'c'); + INSERT INTO %t1 VALUES(4, 'd'); + INSERT INTO %t1 VALUES(5, 'e'); +} + +foreach {tn vars} { + 1 {} + 2 { set A(%t1) over } + 3 { set A(%a) over } + 4 { + set A(%alias) over + set A(%a) following + set A(%b) over + } + 5 { + set A(%t1) over + set A(%a) following + set A(%b) preceding + set A(%w) current + set A(%alias) filter + set A(%typename) window + } + + 6 { + set A(%a) window + } +} { + set A(%t1) t1 + set A(%a) a + set A(%b) b + set A(%w) w + set A(%alias) alias + set A(%typename) integer + eval $vars + + set MAP [array get A] + set setup_sql [string map $MAP $setup] + reset_db + execsql $setup_sql + + do_execsql_test 1.$tn.1 [string map $MAP { + SELECT group_concat(%a, '.') OVER (ORDER BY %b) FROM %t1 + }] {1 1.2 1.2.3 1.2.3.4 1.2.3.4.5} + + do_execsql_test 1.$tn.2 [string map $MAP { + SELECT sum(%a) OVER %w FROM %t1 WINDOW %w AS (ORDER BY %b) + }] {1 3 6 10 15} + + do_execsql_test 1.$tn.3 [string map $MAP { + SELECT sum(%alias.%a) OVER %w FROM %t1 %alias WINDOW %w AS (ORDER BY %b) + }] {1 3 6 10 15} + + do_execsql_test 1.$tn.4 [string map $MAP { + SELECT sum(%a) %alias FROM %t1 + }] {15} +} + + +proc winproc {args} { return "window: $args" } +db func window winproc +do_execsql_test 2.0 { + SELECT window('hello world'); +} {{window: {hello world}}} + +proc wincmp {a b} { string compare $b $a } +db collate window wincmp +do_execsql_test 3.0 { + CREATE TABLE window(x COLLATE window); + INSERT INTO window VALUES('bob'), ('alice'), ('cate'); + SELECT * FROM window ORDER BY x COLLATE window; +} {cate bob alice} +do_execsql_test 3.1 { + DROP TABLE window; + CREATE TABLE x1(x); + INSERT INTO x1 VALUES('bob'), ('alice'), ('cate'); + CREATE INDEX window ON x1(x COLLATE window); + SELECT * FROM x1 ORDER BY x COLLATE window; +} {cate bob alice} + + +do_execsql_test 4.0 { CREATE TABLE t4(x, y); } + +# do_execsql_test 4.1 { PRAGMA parser_trace = 1 } +do_execsql_test 4.1 { + SELECT * FROM t4 window, t4; +} + + +finish_test + diff --git a/tool/lempar.c b/tool/lempar.c index d625fcb0a4..feb79e3a58 100644 --- a/tool/lempar.c +++ b/tool/lempar.c @@ -1068,3 +1068,17 @@ void Parse( #endif return; } + +/* +** Return the fallback token corresponding to canonical token iToken, or +** 0 if iToken has no fallback. +*/ +int ParseFallback(int iToken){ +#ifdef YYFALLBACK + if( iToken