From: dan Date: Sat, 30 Jun 2018 18:54:56 +0000 (+0000) Subject: Have the tokenizer handle fallback for tokens "OVER" and "FILTER" in the same X-Git-Tag: version-3.25.0~178^2~1^2 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=refs%2Fheads%2Fweak-fallback;p=thirdparty%2Fsqlite.git Have the tokenizer handle fallback for tokens "OVER" and "FILTER" in the same way as it does for "WINDOW". FossilOrigin-Name: 12d819e1c17d8036900352b0989c4bfcbc34193c3735bb9af7ab051f0f129d3d --- diff --git a/manifest b/manifest index a487454ad8..ae9a219479 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\sperformance\srelated\stweaks\sfor\ssqlite3RunParser(). -D 2018-06-29T20:43:33.493 +C Have\sthe\stokenizer\shandle\sfallback\sfor\stokens\s"OVER"\sand\s"FILTER"\sin\sthe\ssame\nway\sas\sit\sdoes\sfor\s"WINDOW". +D 2018-06-30T18:54:56.070 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 0a3a6c81e6fcb969ff9106e882f0a08547014ba463cb6beca4c4efaecc924ee6 @@ -484,7 +484,7 @@ F src/os_win.c ac29c25cde4cfb4adacc59cdec4aa45698ca0e29164ea127859585ccd9faa354 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 1bb6a57fa0465296a4d6109a1a64610a0e7adde1f3acf3ef539a9d972908ce8f F src/pager.h c571b064df842ec8f2e90855dead9acf4cbe0d1b2c05afe0ef0d0145f7fd0388 -F src/parse.y 5a53ee98a8bc76c526ae9b4a05cc4912f6b7b2e2a601a8fccc38b7b3b7830f97 +F src/parse.y 9b57f1d0d3d7578ab2917e07ff5d8def4b0aac571113dd7b7cb8108e7194d025 F src/pcache.c 135ef0bc6fb2e3b7178d49ab5c9176254c8a691832c1bceb1156b2fbdd0869bd F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880 @@ -558,7 +558,7 @@ F src/test_windirent.h 90dfbe95442c9762357fe128dc7ae3dc199d006de93eb33ba3972e0a9 F src/test_window.c add59ee68568868129516999f30a68e8ab2afd276e272aba4f633c9fc52c1bb1 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c -F src/tokenize.c a9b97ce736f958390e51ed9f03a152403a2b77d54e6d5729f1ee1f365878d507 +F src/tokenize.c 0e3e0462f7da08bf95b3da1dca4c01cd2c3ca1d988ed0f9d2f66334a975e4020 F src/treeview.c 2c5c4bc0a443401db5fd621542150452ddf5055d38edd4eef868bc2b6bfb0260 F src/trigger.c 4ace6d1d5ba9a89822deb287317f33c810440526eafe185c2d8a48c31df1e995 F src/update.c 46dc24c6158446aaab45caee09b6d99327cb479268b83ffeb5b701823da3b67b @@ -1626,7 +1626,7 @@ F test/window3.test 47884c240d0d5234ad9c6da65452cfec1bfa69ec6f9c4158cd9750c3d88d F test/window4.tcl 7cec7e578aa9f78b7265bff8d552cda17a1d8d89f0449d0e74970a527b8846f5 F test/window4.test dcd8767869988e0d23d56bc3f8b46ec116de23127b81b5f66fd48d5529072ed1 F test/window5.test a4835b96d30eb3b81a1dbc683e333e57a614645eb6f2ae476a7ed2addf0b0f1f -F test/window6.test 97bd18d5cccd612b8a93be4017a7a26c2968a1c868a458d0d66d8d8cc62e33f8 +F test/window6.test fb4e464993630c3d6023e906aa6705667062600a8341085b7fd8c201f4c2f411 F test/windowfault.test 97d5fc404308edb579a5a183e294ed874c844ecf01f0a28ba46df3141ebaee1f F test/with1.test 58475190cd8caaeebea8cfeb2a264ec97a0c492b8ffe9ad20cefbb23df462f96 F test/with2.test e0030e2f0267a910d6c0e4f46f2dfe941c1cc0d4f659ba69b3597728e7e8f1ab @@ -1744,7 +1744,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P eef61ffab7fa36b126f57bf7028dd35c67ed4617c47145be059f91e58023b0a4 -R 6ded36ebdbd58ad68c3d8e5fd0ba8d3c +P 5eb4776598f5bba7ef21a2c58c03105544da73d642d7ffc146f84eff1993d71e +R 020addcbc17f30026d883f2e135b0a63 +T +closed a3e9e62c6cafac5b5f103edcff77085b463085fca60c515aa16723c3d0006d44 U dan -Z d818b642c548723f779df4a6cae08814 +Z f871d44af5dded308446b721ea515004 diff --git a/manifest.uuid b/manifest.uuid index afa47307eb..01c135be3b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5eb4776598f5bba7ef21a2c58c03105544da73d642d7ffc146f84eff1993d71e \ No newline at end of file +12d819e1c17d8036900352b0989c4bfcbc34193c3735bb9af7ab051f0f129d3d \ No newline at end of file diff --git a/src/parse.y b/src/parse.y index 5097314329..42efe6fb6f 100644 --- a/src/parse.y +++ b/src/parse.y @@ -217,8 +217,7 @@ columnname(A) ::= nm(A) typetoken(Y). {sqlite3AddColumn(pParse,&A,&Y);} EXCEPT INTERSECT UNION %endif SQLITE_OMIT_COMPOUND_SELECT %ifndef SQLITE_OMIT_WINDOWFUNC - CURRENT FILTER FOLLOWING OVER PARTITION - PRECEDING RANGE UNBOUNDED + CURRENT FOLLOWING PARTITION PRECEDING RANGE UNBOUNDED %endif SQLITE_OMIT_WINDOWFUNC REINDEX RENAME CTIME_KW IF . @@ -1599,10 +1598,10 @@ wqlist(A) ::= wqlist(A) COMMA nm(X) eidlist_opt(Y) AS LP select(Z) RP. { %endif SQLITE_OMIT_CTE //////////////////////// WINDOW FUNCTION EXPRESSIONS ///////////////////////// -// These must be at the end of this file. Specifically, the windowdefn_opt -// rule must be the very last in the file. This causes the integer value -// assigned to the TK_WINDOW token to be larger than all other tokens that -// may be output by the tokenizer except TK_SPACE and TK_ILLEGAL. +// These must be at the end of this file. Specifically, the rules that +// introduce tokens WINDOW, OVER and FILTER must appear last. This causes +// the integer values assigned to these tokens to be larger than all other +// tokens that may be output by the tokenizer except TK_SPACE and TK_ILLEGAL. // %ifndef SQLITE_OMIT_WINDOWFUNC %type windowdefn_list {Window*} @@ -1622,9 +1621,6 @@ windowdefn(A) ::= nm(X) AS window(Y). { A = Y; } -%type over_opt {Window*} -%destructor over_opt {sqlite3WindowDelete(pParse->db, $$);} - %type window {Window*} %destructor window {sqlite3WindowDelete(pParse->db, $$);} @@ -1646,12 +1642,6 @@ sqlite3WindowDelete(pParse->db, $$);} %type frame_bound {struct FrameBound} %destructor frame_bound {sqlite3ExprDelete(pParse->db, $$.pExpr);} -over_opt(A) ::= . { A = 0; } -over_opt(A) ::= filter_opt(W) OVER window_or_nm(Z). { - A = Z; - if( A ) A->pFilter = W; -} - window_or_nm(A) ::= window(Z). {A = Z;} window_or_nm(A) ::= nm(Z). { A = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); @@ -1670,8 +1660,6 @@ window(A) ::= LP part_opt(X) orderby_opt(Y) frame_opt(Z) RP. { part_opt(A) ::= PARTITION BY exprlist(X). { A = X; } part_opt(A) ::= . { A = 0; } -filter_opt(A) ::= . { A = 0; } -filter_opt(A) ::= FILTER LP WHERE expr(X) RP. { A = X; } frame_opt(A) ::= . { A = sqlite3WindowAlloc(pParse, TK_RANGE, TK_UNBOUNDED, 0, TK_CURRENT, 0); @@ -1696,5 +1684,16 @@ frame_bound(A) ::= UNBOUNDED FOLLOWING. { A.eType = TK_UNBOUNDED; A.pExpr = 0; } %destructor windowdefn_opt {sqlite3WindowDelete(pParse->db, $$);} windowdefn_opt(A) ::= . { A = 0; } windowdefn_opt(A) ::= WINDOW windowdefn_list(B). { A = B; } + +%type over_opt {Window*} +%destructor over_opt {sqlite3WindowDelete(pParse->db, $$);} +over_opt(A) ::= . { A = 0; } +over_opt(A) ::= filter_opt(W) OVER window_or_nm(Z). { + A = Z; + if( A ) A->pFilter = W; +} + +filter_opt(A) ::= . { A = 0; } +filter_opt(A) ::= FILTER LP WHERE expr(X) RP. { A = X; } %endif // SQLITE_OMIT_WINDOWFUNC diff --git a/src/tokenize.c b/src/tokenize.c index 72f160d59b..15678ed698 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -192,54 +192,76 @@ int sqlite3IsIdChar(u8 c){ return IdChar(c); } /* ** Return the id of the next token in string (*pz). Before returning, set ** (*pz) to point to the byte following the parsed token. -** -** This function assumes that any keywords that start with "w" are -** actually TK_ID. */ -static int windowGetToken(const unsigned char **pz){ - int ret; +static int getToken(const unsigned char **pz){ const unsigned char *z = *pz; - if( z[0]=='w' || z[0]=='W' ){ - do { z++; }while( IdChar(z[0]) ); - ret = TK_ID; - }else{ - z += sqlite3GetToken(z, &ret); + int t; /* Token type to return */ + do { + z += sqlite3GetToken(z, &t); + }while( t==TK_SPACE ); + if( t==TK_ID + || t==TK_STRING + || t==TK_JOIN_KW + || t==TK_WINDOW + || t==TK_OVER + || sqlite3ParserFallback(t)==TK_ID + ){ + t = TK_ID; } *pz = z; - return ret; + return t; } -#endif // SQLITE_OMIT_WINDOWFUNC -#ifndef SQLITE_OMIT_WINDOWFUNC /* -** The tokenizer has just parsed the keyword WINDOW. In this case the token -** may really be the keyword (TK_WINDOW), or may be an identifier (TK_ID). -** This function determines which it is by inspecting the next two tokens -** in the input stream. Specifically, the token is TK_WINDOW if the following -** two tokens are: +** The following three functions are called immediately after the tokenizer +** reads the keywords WINDOW, OVER and FILTER, respectively, to determine +** whether the token should be treated as a keyword or an SQL identifier. +** This cannot be handled by the usual lemon %fallback method, due to +** the ambiguity in some constructions. e.g. +** +** SELECT sum(x) OVER ... +** +** In the above, "OVER" might be a keyword, or it might be an alias for the +** sum(x) expression. If a "%fallback ID OVER" directive were added to +** grammar, then SQLite would always treat "OVER" as an alias, making it +** impossible to call a window-function without a FILTER clause. +** +** WINDOW is treated as a keyword if: +** +** * the following token is an identifier, or a keyword that can fallback +** to being an identifier, and +** * the token after than one is TK_AS. ** -** * TK_ID, or something else that can be used as a window name, and -** * TK_AS. +** OVER is a keyword if: ** -** Instead of using sqlite3GetToken() to parse tokens directly, this function -** uses windowGetToken(). This is to avoid recursion if the input is similar -** to "window window window window". +** * the previous token was TK_RP, and +** * the next token is either TK_LP or an identifier. +** +** FILTER is a keyword if: +** +** * the previous token was TK_RP, and +** * the next token is TK_LP. */ static int analyzeWindowKeyword(const unsigned char *z){ int t; - int ret = TK_WINDOW; - while( (t = windowGetToken(&z))==TK_SPACE ); - if( t!=TK_ID && t!=TK_STRING - && t!=TK_JOIN_KW && sqlite3ParserFallback(t)!=TK_ID - ){ - ret = TK_ID; - }else{ - while( (t = windowGetToken(&z))==TK_SPACE ); - if( t!=TK_AS ){ - ret = TK_ID; - } + t = getToken(&z); + if( t!=TK_ID ) return TK_ID; + t = getToken(&z); + if( t!=TK_AS ) return TK_ID; + return TK_WINDOW; +} +static int analyzeOverKeyword(const unsigned char *z, int lastToken){ + if( lastToken==TK_RP ){ + int t = getToken(&z); + if( t==TK_LP || t==TK_ID ) return TK_OVER; + } + return TK_ID; +} +static int analyzeFilterKeyword(const unsigned char *z, int lastToken){ + if( lastToken==TK_RP && getToken(&z)==TK_LP ){ + return TK_FILTER; } - return ret; + return TK_ID; } #endif // SQLITE_OMIT_WINDOWFUNC @@ -571,7 +593,7 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ } #ifndef SQLITE_OMIT_WINDOWFUNC if( tokenType>=TK_WINDOW ){ - assert( tokenType==TK_SPACE + assert( tokenType==TK_SPACE || tokenType==TK_OVER || tokenType==TK_FILTER || tokenType==TK_ILLEGAL || tokenType==TK_WINDOW ); #else @@ -599,7 +621,14 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ n = 0; #ifndef SQLITE_OMIT_WINDOWFUNC }else if( tokenType==TK_WINDOW ){ + assert( n==6 ); tokenType = analyzeWindowKeyword((const u8*)&zSql[6]); + }else if( tokenType==TK_OVER ){ + assert( n==4 ); + tokenType = analyzeOverKeyword((const u8*)&zSql[4], lastTokenParsed); + }else if( tokenType==TK_FILTER ){ + assert( n==6 ); + tokenType = analyzeFilterKeyword((const u8*)&zSql[6], lastTokenParsed); #endif // SQLITE_OMIT_WINDOWFUNC }else{ sqlite3ErrorMsg(pParse, "unrecognized token: \"%.*s\"", n, zSql); diff --git a/test/window6.test b/test/window6.test index a6706fc521..69b597077e 100644 --- a/test/window6.test +++ b/test/window6.test @@ -22,7 +22,7 @@ ifcapable !windowfunc { } set setup { - CREATE TABLE %t1(%a, %b %typename); + CREATE TABLE %t1(%x, %y %typename); INSERT INTO %t1 VALUES(1, 'a'); INSERT INTO %t1 VALUES(2, 'b'); INSERT INTO %t1 VALUES(3, 'c'); @@ -33,28 +33,28 @@ set setup { foreach {tn vars} { 1 {} 2 { set A(%t1) over } - 3 { set A(%a) over } + 3 { set A(%x) over } 4 { set A(%alias) over - set A(%a) following - set A(%b) over + set A(%x) following + set A(%y) over } 5 { - set A(%t1) over - set A(%a) following - set A(%b) preceding + set A(%t1) over + set A(%x) following + set A(%y) preceding set A(%w) current - set A(%alias) filter - set A(%typename) window + set A(%alias) filter + set A(%typename) window } 6 { - set A(%a) window + set A(%x) window } } { set A(%t1) t1 - set A(%a) a - set A(%b) b + set A(%x) x + set A(%y) y set A(%w) w set A(%alias) alias set A(%typename) integer @@ -66,19 +66,19 @@ foreach {tn vars} { execsql $setup_sql do_execsql_test 1.$tn.1 [string map $MAP { - SELECT group_concat(%a, '.') OVER (ORDER BY %b) FROM %t1 + SELECT group_concat(%x, '.') OVER (ORDER BY %y) FROM %t1 }] {1 1.2 1.2.3 1.2.3.4 1.2.3.4.5} do_execsql_test 1.$tn.2 [string map $MAP { - SELECT sum(%a) OVER %w FROM %t1 WINDOW %w AS (ORDER BY %b) + SELECT sum(%x) OVER %w FROM %t1 WINDOW %w AS (ORDER BY %y) }] {1 3 6 10 15} do_execsql_test 1.$tn.3 [string map $MAP { - SELECT sum(%alias.%a) OVER %w FROM %t1 %alias WINDOW %w AS (ORDER BY %b) + SELECT sum(%alias.%x) OVER %w FROM %t1 %alias WINDOW %w AS (ORDER BY %y) }] {1 3 6 10 15} do_execsql_test 1.$tn.4 [string map $MAP { - SELECT sum(%a) %alias FROM %t1 + SELECT sum(%x) %alias FROM %t1 }] {15} } @@ -112,6 +112,32 @@ do_execsql_test 4.1 { SELECT * FROM t4 window, t4; } +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 5.0 { + CREATE TABLE over(x, over); + CREATE TABLE window(x, window); + INSERT INTO over VALUES(1, 2), (3, 4), (5, 6); + INSERT INTO window VALUES(1, 2), (3, 4), (5, 6); + SELECT sum(x) over FROM over +} {9} + +do_execsql_test 5.1 { + SELECT sum(x) over over FROM over WINDOW over AS () +} {9 9 9} + +do_execsql_test 5.2 { + SELECT sum(over) over over over FROM over over WINDOW over AS (ORDER BY over) +} {2 6 12} + +do_execsql_test 5.3 { + SELECT sum(over) over over over FROM over over WINDOW over AS (ORDER BY over); +} {2 6 12} + +do_execsql_test 5.4 { + SELECT sum(window) OVER window window FROM window window window window AS (ORDER BY window); +} {2 6 12} finish_test