From: dan Date: Sat, 5 Oct 2024 17:37:19 +0000 (+0000) Subject: Experimental change to allow expressions with subtypes to be read from indexes in... X-Git-Tag: version-3.47.0~58^2~5 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=c857b9eb5d9cfab1366fb80e5d1c306e340b6d65;p=thirdparty%2Fsqlite.git Experimental change to allow expressions with subtypes to be read from indexes in situations where they are not used as function parameters. FossilOrigin-Name: ac63f98ad85a4dd1e49cc64b41f0ca0044153972c15d71c669f4bc3ec590e268 --- diff --git a/manifest b/manifest index 3a576a9093..37b5b5e282 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sext/wasm\sto\sthe\stop-level\sclean/distclean\srules\sin\ssuch\sa\sway\sthat\sany\serror\sdue\sto\sa\slack\sof\sgmake\sare\signored. -D 2024-10-05T12:02:17.445 +C Experimental\schange\sto\sallow\sexpressions\swith\ssubtypes\sto\sbe\sread\sfrom\sindexes\sin\ssituations\swhere\sthey\sare\snot\sused\sas\sfunction\sparameters. +D 2024-10-05T17:37:19.432 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -719,7 +719,7 @@ F src/date.c 89ce1ff20512a7fa5070ba6e7dd5c171148ca7d580955795bf97c79c2456144a F src/dbpage.c db1be8adaf1f839ad733c08baeac5c22aa912f7b535865c0c061382602081360 F src/dbstat.c 73362c0df0f40ad5523a6f5501224959d0976757b511299bf892313e79d14f5c F src/delete.c 03a77ba20e54f0f42ebd8eddf15411ed6bdb06a2c472ac4b6b336521bf7cea42 -F src/expr.c 6d5f2c38fe3ec06a7eac599dac822788b36064124e20112a844e9cd5156cb239 +F src/expr.c 9441dc9f9f4dd3a172270d3dbf476d709aa82ca9bab73a5f3bb878654afe4424 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 928ed2517e8732113d2b9821aa37af639688d752f4ea9ac6e0e393d713eeb76f F src/func.c df400a1d3f4625997d4dd8a81951c303e066277c29b861d37e03cd152d7858dd @@ -774,7 +774,7 @@ F src/shell.c.in 981efe98f98a983c1d0193d18528eb2d765207c0c82b67b610be60f17995a43 F src/sqlite.h.in 1def838497ad53c81486649ce79821925d1ac20a9843af317a344d507efe116e F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 -F src/sqliteInt.h 989dca8b25ca11f5c52e5a457cc500042c43b0b3e5fea9a12d9020d0350722cd +F src/sqliteInt.h 40552bd22f4bc3330f2a73633d12f12863c5aa4e409af75f8ebfaeb794b11e4f F src/sqliteLimit.h 6878ab64bdeb8c24a1d762d45635e34b96da21132179023338c93f820eee6728 F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -854,7 +854,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c a0d42bfdef935e1389737152394d08e59e7c48697f40a9fc2e0552cb19dc731f F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c d5006d6b005e4ea7302ad390957a8d41ed83faa177e412f89bc5600a7462a014 -F src/where.c 461d41017d900d4248a268df96d2d30506c4dcc2257f4167c4f46072003ce2cf +F src/where.c 12fe24880901997372b88fd7ca9a21457404ad35201712c02cc57978578abb10 F src/whereInt.h a5d079c346a658b7a6e9e47bb943d021e02fa1e6aed3b964ca112112a4892192 F src/wherecode.c 5172d647798134e7c92536ddffe7e530c393d79b5dedd648b88faf2646c65baf F src/whereexpr.c 44f41ae554c7572e1de1485b3169b233ee04d464b2ee5881687ede3bf07cacfa @@ -1324,6 +1324,7 @@ F test/indexA.test 11d84f6995e6e5b9d8315953fb1b6d29772ee7c7803ee9112715e7e4dd3e4 F test/indexedby.test f21eca4f7a6ffe14c8500a7ad6cd53166666c99e5ccd311842a28bc94a195fe0 F test/indexexpr1.test 24fa85a12da384dd1d56f7b24e593c51a8a54b4c5e2e8bbb9e5fdf1099427faf F test/indexexpr2.test 1c382e81ef996d8ae8b834a74f2a9013dddf59214c32201d7c8a656d739f999a +F test/indexexpr3.test 7689d11a60dca09879f490db5fd26b9171a57e31d97282d9963e944b1536f676 F test/indexfault.test 98d78a8ff1f5335628b62f886a1cb7c7dac1ef6d48fa39c51ec871c87dce9811 F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 F test/insert.test 4e3f0de67aac3c5be1f4aaedbcea11638f1b5cdc9a3115be14d19aa9db7623c6 @@ -2215,8 +2216,11 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6733893f450097e07cbd563d6a46790825fd0689283d60181c09793ce7d5509e -R 749ae8645be170a25b1c641eade23387 -U stephan -Z c3b0bf3081778b6e6721957f472a53c9 +P 2f7eab381e16760952d1c90a9119d2a217933f0136442d8f6eeb6d95e366ca4f +R 05b36287276c4340f7e35170606a310b +T *branch * indexed-subtype-expr +T *sym-indexed-subtype-expr * +T -sym-trunk * +U dan +Z 943b26137ccac8e38cc1774f405e6dc0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 717f299acd..a0670d9728 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2f7eab381e16760952d1c90a9119d2a217933f0136442d8f6eeb6d95e366ca4f +ac63f98ad85a4dd1e49cc64b41f0ca0044153972c15d71c669f4bc3ec590e268 diff --git a/src/expr.c b/src/expr.c index 1b18828dd6..83b1ff56cd 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1164,6 +1164,7 @@ Expr *sqlite3ExprFunction( ){ Expr *pNew; sqlite3 *db = pParse->db; + int ii; assert( pToken ); pNew = sqlite3ExprAlloc(db, TK_FUNCTION, pToken, 1); if( pNew==0 ){ @@ -1178,6 +1179,11 @@ Expr *sqlite3ExprFunction( ){ sqlite3ErrorMsg(pParse, "too many arguments on function %T", pToken); } + if( pList && pParse->nErr==0 ){ + for(ii=0; iinExpr; ii++){ + ExprSetProperty(pList->a[ii].pExpr, EP_FuncArg); + } + } pNew->x.pList = pList; ExprSetProperty(pNew, EP_HasFunc); assert( ExprUseXList(pNew) ); @@ -4554,6 +4560,59 @@ static int exprCodeInlineFunction( return target; } +/* +** Expression Node callback for sqlite3ExprCanReturnSubtype(). +** +** Only a function call is able to return a subtype. So if the node +** is not a function call, return WRC_Prune immediately. +** +** A function call is able to return a subtype if it has the +** SQLITE_RESULT_SUBTYPE property. +** +** Assume that every function is able to pass-through a subtype from +** one of its argument (using sqlite3_result_value()). Most functions +** are not this way, but we don't have a mechanism to distinguish those +** that are from those that are not, so assume they all work this way. +** That means that if one of its arguments is another function and that +** other function is able to return a subtype, then this function is +** able to return a subtype. +*/ +static int exprNodeCanReturnSubtype(Walker *pWalker, Expr *pExpr){ + int n; + FuncDef *pDef; + sqlite3 *db; + if( pExpr->op!=TK_FUNCTION ){ + return WRC_Prune; + } + assert( ExprUseXList(pExpr) ); + db = pWalker->pParse->db; + n = pExpr->x.pList ? pExpr->x.pList->nExpr : 0; + pDef = sqlite3FindFunction(db, pExpr->u.zToken, n, ENC(db), 0); + if( pDef==0 || (pDef->funcFlags & SQLITE_RESULT_SUBTYPE)!=0 ){ + pWalker->eCode = 1; + return WRC_Prune; + } + return WRC_Continue; +} + +/* +** Return TRUE if expression pExpr is able to return a subtype. +** +** A TRUE return does not guarantee that a subtype will be returned. +** It only indicates that a subtype return is possible. False positives +** are acceptable as they only disable an optimization. False negatives, +** on the other hand, can lead to incorrect answers. +*/ +static int sqlite3ExprCanReturnSubtype(Parse *pParse, Expr *pExpr){ + Walker w; + memset(&w, 0, sizeof(w)); + w.pParse = pParse; + w.xExprCallback = exprNodeCanReturnSubtype; + sqlite3WalkExpr(&w, pExpr); + return w.eCode; +} + + /* ** Check to see if pExpr is one of the indexed expressions on pParse->pIdxEpr. ** If it is, then resolve the expression by reading from the index and @@ -4586,6 +4645,17 @@ static SQLITE_NOINLINE int sqlite3IndexedExprLookup( continue; } + + /* Functions that might set a subtype should not be replaced by the + ** value taken from an expression index if they are themselves an + ** argument to another scalar function or aggregate. + ** https://sqlite.org/forum/forumpost/68d284c86b082c3e */ + if( ExprHasProperty(pExpr, EP_FuncArg) + && sqlite3ExprCanReturnSubtype(pParse, pExpr) + ){ + continue; + } + v = pParse->pVdbe; assert( v!=0 ); if( p->bMaybeNullRow ){ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 0e0035ce60..8288786700 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3104,7 +3104,7 @@ struct Expr { #define EP_IsTrue 0x10000000 /* Always has boolean value of TRUE */ #define EP_IsFalse 0x20000000 /* Always has boolean value of FALSE */ #define EP_FromDDL 0x40000000 /* Originates from sqlite_schema */ - /* 0x80000000 // Available */ +#define EP_FuncArg 0x80000000 /* Is a function() argument */ /* The EP_Propagate mask is a set of properties that automatically propagate ** upwards into parent nodes. diff --git a/src/where.c b/src/where.c index 9aaa082cd3..c2fc338247 100644 --- a/src/where.c +++ b/src/where.c @@ -6302,58 +6302,6 @@ static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful( } } -/* -** Expression Node callback for sqlite3ExprCanReturnSubtype(). -** -** Only a function call is able to return a subtype. So if the node -** is not a function call, return WRC_Prune immediately. -** -** A function call is able to return a subtype if it has the -** SQLITE_RESULT_SUBTYPE property. -** -** Assume that every function is able to pass-through a subtype from -** one of its argument (using sqlite3_result_value()). Most functions -** are not this way, but we don't have a mechanism to distinguish those -** that are from those that are not, so assume they all work this way. -** That means that if one of its arguments is another function and that -** other function is able to return a subtype, then this function is -** able to return a subtype. -*/ -static int exprNodeCanReturnSubtype(Walker *pWalker, Expr *pExpr){ - int n; - FuncDef *pDef; - sqlite3 *db; - if( pExpr->op!=TK_FUNCTION ){ - return WRC_Prune; - } - assert( ExprUseXList(pExpr) ); - db = pWalker->pParse->db; - n = pExpr->x.pList ? pExpr->x.pList->nExpr : 0; - pDef = sqlite3FindFunction(db, pExpr->u.zToken, n, ENC(db), 0); - if( pDef==0 || (pDef->funcFlags & SQLITE_RESULT_SUBTYPE)!=0 ){ - pWalker->eCode = 1; - return WRC_Prune; - } - return WRC_Continue; -} - -/* -** Return TRUE if expression pExpr is able to return a subtype. -** -** A TRUE return does not guarantee that a subtype will be returned. -** It only indicates that a subtype return is possible. False positives -** are acceptable as they only disable an optimization. False negatives, -** on the other hand, can lead to incorrect answers. -*/ -static int sqlite3ExprCanReturnSubtype(Parse *pParse, Expr *pExpr){ - Walker w; - memset(&w, 0, sizeof(w)); - w.pParse = pParse; - w.xExprCallback = exprNodeCanReturnSubtype; - sqlite3WalkExpr(&w, pExpr); - return w.eCode; -} - /* ** The index pIdx is used by a query and contains one or more expressions. ** In other words pIdx is an index on an expression. iIdxCur is the cursor @@ -6387,12 +6335,6 @@ static SQLITE_NOINLINE void whereAddIndexedExpr( continue; } if( sqlite3ExprIsConstant(0,pExpr) ) continue; - if( pExpr->op==TK_FUNCTION && sqlite3ExprCanReturnSubtype(pParse,pExpr) ){ - /* Functions that might set a subtype should not be replaced by the - ** value taken from an expression index since the index omits the - ** subtype. https://sqlite.org/forum/forumpost/68d284c86b082c3e */ - continue; - } p = sqlite3DbMallocRaw(pParse->db, sizeof(IndexedExpr)); if( p==0 ) break; p->pIENext = pParse->pIdxEpr; diff --git a/test/indexexpr3.test b/test/indexexpr3.test new file mode 100644 index 0000000000..e3cfcd116a --- /dev/null +++ b/test/indexexpr3.test @@ -0,0 +1,70 @@ +# 2024-10-05 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing indexes on expressions. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix indexexpr3 + + +do_execsql_test 1.0 { + CREATE TABLE t1(a, j); + INSERT INTO t1 VALUES(1, '{x:"one"}'); + INSERT INTO t1 VALUES(2, '{x:"two"}'); + INSERT INTO t1 VALUES(3, '{x:"three"}'); + + CREATE INDEX i1 ON t1( json_extract(j, '$.x') ); + CREATE INDEX i2 ON t1( a, json_extract(j, '$.x') ); +} + +proc do_hasfunction_test {tn sql res} { + set bFunction 0 + db eval "EXPLAIN $sql" x { + if {$x(opcode)=="Function"} { + set bFunction 1 + } + } + + do_execsql_test $tn " + SELECT $bFunction; + $sql + " $res +} + +do_hasfunction_test 1.1 { + SELECT json_extract(j, '$.x') FROM t1 ORDER BY 1; +} { + 0 one three two +} + +do_hasfunction_test 1.2 { + SELECT json_extract(j, '$.x') FROM t1 WHERE a=2 +} { + 0 two +} + +do_hasfunction_test 1.3 { + SELECT coalesce(json_extract(j, '$.x'), 0) FROM t1 WHERE a=2 +} { + 1 two +} + +do_hasfunction_test 1.3 { + SELECT json_extract(j, '$.x') || '.two' FROM t1 WHERE a=2 +} { + 0 two.two +} + + +finish_test +