From: stephan Date: Tue, 24 May 2022 00:22:10 +0000 (+0000) Subject: wasm/JS: added support for scalar UDFs. Fixed a deallocation problem with bind()ed... X-Git-Tag: version-3.39.0~108 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=a240a24ad0b9d0e2023c5e77bcc4fc277db2e5cf;p=thirdparty%2Fsqlite.git wasm/JS: added support for scalar UDFs. Fixed a deallocation problem with bind()ed strings/blobs. FossilOrigin-Name: 325a9ee31ad7abae563c4da5cd8228e151b00aa9afcac7e9bca5efaa9d48e107 --- diff --git a/Makefile.in b/Makefile.in index bd9bc366f1..8a6d04a1de 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1535,7 +1535,7 @@ clean: clean-wasm #emcc_opt = -O2 #emcc_opt = -O3 emcc_opt = -Oz -emcc_flags = $(emcc_opt) -I. $(SHELL_OPT) +emcc_flags = $(emcc_opt) -sALLOW_TABLE_GROWTH -I. $(SHELL_OPT) $(fiddle_module_js): Makefile sqlite3.c shell.c \ $(fiddle_dir)/EXPORTED_RUNTIME_METHODS $(fiddle_dir)/EXPORTED_FUNCTIONS.fiddle emcc -o $@ $(emcc_flags) \ diff --git a/ext/fiddle/EXPORTED_FUNCTIONS.sqlite3 b/ext/fiddle/EXPORTED_FUNCTIONS.sqlite3 index da14513415..45f9266a38 100644 --- a/ext/fiddle/EXPORTED_FUNCTIONS.sqlite3 +++ b/ext/fiddle/EXPORTED_FUNCTIONS.sqlite3 @@ -47,3 +47,4 @@ _sqlite3_value_bytes _sqlite3_value_double _sqlite3_value_text _sqlite3_value_type +_free diff --git a/ext/fiddle/EXPORTED_RUNTIME_METHODS b/ext/fiddle/EXPORTED_RUNTIME_METHODS index a5b33fb786..1bfcc97d40 100644 --- a/ext/fiddle/EXPORTED_RUNTIME_METHODS +++ b/ext/fiddle/EXPORTED_RUNTIME_METHODS @@ -4,3 +4,8 @@ stackAlloc stackSave stackRestore UTF8ToString +removeFunction +addFunction +setValue +getValue +allocate diff --git a/ext/fiddle/sqlite3-api.js b/ext/fiddle/sqlite3-api.js index 25ae514394..22773d8d7d 100644 --- a/ext/fiddle/sqlite3-api.js +++ b/ext/fiddle/sqlite3-api.js @@ -144,6 +144,10 @@ SQLITE_TEXT: 3, SQLITE_BLOB: 4, SQLITE_NULL: 5, + /* create_function() flags */ + SQLITE_DETERMINISTIC: 0x000000800, + SQLITE_DIRECTONLY: 0x000080000, + SQLITE_INNOCUOUS: 0x000200000, /* sqlite encodings, used for creating UDFs, noting that we will only support UTF8. */ SQLITE_UTF8: 1 @@ -257,7 +261,8 @@ this.checkRc(S.sqlite3_open(name, pPtrArg)); this._pDb = getValue(pPtrArg, "i32"); this.filename = name; - this._statements = {/*map of open Stmt _pointers_*/}; + this._statements = {/*map of open Stmt _pointers_ to Stmt*/}; + this._udfs = {/*map of UDF names to wasm function _pointers_*/}; }; /** @@ -297,6 +302,12 @@ return db; }; + /** Returns true if n is a 32-bit (signed) integer, + else false. */ + const isInt32 = function(n){ + return (n===n|0 && n<0xFFFFFFFF) ? true : undefined; + }; + /** Expects to be passed (arguments) from DB.exec() and DB.execMulti(). Does the argument processing/validation, throws @@ -340,6 +351,11 @@ return out; }; + /** If object opts has _its own_ property named p then that + property's value is returned, else dflt is returned. */ + const getOwnOption = (opts, p, dflt)=> + opts.hasOwnProperty(p) ? opts[p] : dflt; + DB.prototype = { /** Expects to be given an sqlite3 API result code. If it is @@ -369,6 +385,9 @@ delete that._statements[k]; if(s && s._pStmt) s.finalize(); }); + Object.values(this._udfs).forEach(Module.removeFunction); + delete this._udfs; + delete this._statements; S.sqlite3_close_v2(this._pDb); delete this._pDb; } @@ -550,7 +569,184 @@ stackRestore(stack); } return this; - }/*execMulti()*/ + }/*execMulti()*/, + /** + Creates a new scalar UDF (User-Defined Function) which is + accessible via SQL code. This function may be called in any + of the following forms: + + - (name, function) + - (name, function, optionsObject) + - (name, optionsObject) + - (optionsObject) + + In the final two cases, the function must be defined as the + 'callback' property of the options object. In the final + case, the function's name must be the 'name' property. + + This can only be used to create scalar functions, not + aggregate or window functions. UDFs cannot be removed from + a DB handle after they're added. + + On success, returns this object. Throws on error. + + When called from SQL, arguments to the UDF, and its result, + will be converted between JS and SQL with as much fidelity + as is feasible, triggering an exception if a type + conversion cannot be determined. Some freedom is afforded + to numeric conversions due to friction between the JS and C + worlds: integers which are larger than 32 bits will be + treated as doubles, as JS does not support 64-bit integers + and it is (as of this writing) illegal to use WASM + functions which take or return 64-bit integers from JS. + + The optional options object may contain flags to modify how + the function is defined: + + - .arity: the number of arguments which SQL calls to this + function expect or require. The default value is the + callback's length property. A value of -1 means that the + function is variadic and may accept any number of + arguments, up to sqlite3's compile-time limits. sqlite3 + will enforce the argument count if is zero or greater. + + The following properties correspond to flags documented at: + + https://sqlite.org/c3ref/create_function.html + + - .deterministic = SQLITE_DETERMINISTIC + - .directOnly = SQLITE_DIRECTONLY + - .innocuous = SQLITE_INNOCUOUS + + + Maintenance reminder: the ability to add new + WASM-accessible functions to the runtime requires that the + WASM build is compiled with emcc's `-sALLOW_TABLE_GROWTH` + flag. + */ + createFunction: function f(name, callback,opt){ + switch(arguments.length){ + case 1: /* (optionsObject) */ + opt = name; + name = opt.name; + callback = opt.callback; + break; + case 2: /* (name, callback|optionsObject) */ + if(!(callback instanceof Function)){ + opt = callback; + callback = opt.callback; + } + break; + default: break; + } + if(!opt) opt = {}; + if(!(callback instanceof Function)){ + toss("Invalid arguments: expecting a callback function."); + }else if('string' !== typeof name){ + toss("Invalid arguments: missing function name."); + } + if(!f._extractArgs){ + /* Static init */ + f._extractArgs = function(argc, pArgv){ + let i, pVal, valType, arg; + const tgt = []; + for(i = 0; i < argc; ++i){ + pVal = getValue(pArgv + (4 * i), "i32"); + valType = S.sqlite3_value_type(pVal); + switch(valType){ + case S.SQLITE_INTEGER: + case S.SQLITE_FLOAT: + arg = S.sqlite3_value_double(pVal); + break; + case SQLITE_TEXT: + arg = S.sqlite3_value_text(pVal); + break; + case SQLITE_BLOB:{ + const n = S.sqlite3_value_bytes(ptr); + const pBlob = S.sqlite3_value_blob(ptr); + arg = new Uint8Array(n); + let i; + for(i = 0; i < n; ++i) arg[i] = HEAP8[pBlob+i]; + break; + } + default: + arg = null; break; + } + tgt.push(arg); + } + return tgt; + }/*_extractArgs()*/; + f._setResult = function(pCx, val){ + switch(typeof val) { + case 'boolean': + S.sqlite3_result_int(pCx, val ? 1 : 0); + break; + case 'number': { + (isInt32(val) + ? S.sqlite3_result_int + : S.sqlite3_result_double)(pCx, val); + break; + } + case 'string': + S.sqlite3_result_text(pCx, val, -1, + -1/*==SQLITE_TRANSIENT*/); + break; + case 'object': + if(null===val) { + S.sqlite3_result_null(pCx); + break; + }else if(undefined!==val.length){ + const pBlob = Module.allocate(val, ALLOC_NORMAL); + S.sqlite3_result_blob(pCx, pBlob, val.length, -1/*==SQLITE_TRANSIENT*/); + Module._free(blobptr); + break; + } + // else fall through + default: + toss("Don't not how to handle this UDF result value:",val); + }; + }/*_setResult()*/; + }/*static init*/ + const wrapper = function(pCx, argc, pArgv){ + try{ + f._setResult(pCx, callback.apply(null, f._extractArgs(argc, pArgv))); + }catch(e){ + S.sqlite3_result_error(pCx, e.message, -1); + } + }; + const pUdf = Module.addFunction(wrapper, "viii"); + let fFlags = 0; + if(getOwnOption(opt, 'deterministic')) fFlags |= S.SQLITE_DETERMINISTIC; + if(getOwnOption(opt, 'directOnly')) fFlags |= S.SQLITE_DIRECTONLY; + if(getOwnOption(opt, 'innocuous')) fFlags |= S.SQLITE_INNOCUOUS; + name = name.toLowerCase(); + try { + this.checkRc(S.sqlite3_create_function_v2( + this._pDb, name, + (opt.hasOwnProperty('arity') ? +opt.arity : callback.length), + S.SQLITE_UTF8 | fFlags, null/*pApp*/, pUdf, + null/*xStep*/, null/*xFinal*/, null/*xDestroy*/)); + }catch(e){ + Module.removeFunction(pUdf); + throw e; + } + if(this._udfs.hasOwnProperty(name)){ + Module.removeFunction(this._udfs[name]); + } + this._udfs[name] = pUdf; + return this; + }/*createFunction()*/, + selectValue: function(sql,bind){ + let stmt, rc; + try { + stmt = this.prepare(sql); + stmt.bind(bind); + if(stmt.step()) rc = stmt.get(0); + }finally{ + if(stmt) stmt.finalize(); + } + return rc; + } }/*DB.prototype*/; @@ -654,7 +850,7 @@ f._ = { string: function(stmt, ndx, val, asBlob){ const bytes = intArrayFromString(val,true); - const pStr = allocate(bytes, ALLOC_NORMAL); + const pStr = Module.allocate(bytes, ALLOC_NORMAL); stmt._allocs.push(pStr); const func = asBlob ? S.sqlite3_bind_blob : S.sqlite3_bind_text; return func(stmt._pStmt, ndx, pStr, bytes.length, 0); @@ -673,12 +869,10 @@ break; } case BindTypes.number: { - const m = ((val === (val|0)) - ? ((val & 0x00000000/*>32 bits*/) - ? S.sqlite3_bind_double - /*It's illegal to bind a 64-bit int - from here*/ - : S.sqlite3_bind_int) + const m = (isInt32(val) + ? S.sqlite3_bind_int + /*It's illegal to bind a 64-bit int + from here*/ : S.sqlite3_bind_double); rc = m(stmt._pStmt, ndx, val); break; @@ -695,7 +889,7 @@ toss("Binding a value as a blob requires", "that it have a length member."); } - const pBlob = allocate(val, ALLOC_NORMAL); + const pBlob = Module.allocate(val, ALLOC_NORMAL); stmt._allocs.push(pBlob); rc = S.sqlite3_bind_blob(stmt._pStmt, ndx, pBlob, len, 0); } @@ -711,7 +905,7 @@ const freeBindMemory = function(stmt){ let m; while(undefined !== (m = stmt._allocs.pop())){ - _free(m); + Module._free(m); } return stmt; }; @@ -775,7 +969,13 @@ Bindable value types: - - null or undefined is bound as NULL. + - null is bound as NULL. + + - undefined as a standalone value is a no-op intended to + simplify certain client-side use cases: passing undefined + as a value to this function will not actually bind + anything. undefined as an array or object property when + binding an array/object is treated as null. - Numbers are bound as either doubles or integers: doubles if they are larger than 32 bits, else double or int32, @@ -818,18 +1018,24 @@ - The statement has been finalized. */ - bind: function(/*[ndx,] value*/){ - if(!affirmStmtOpen(this).parameterCount){ - toss("This statement has no bindable parameters."); - } - this._mayGet = false; + bind: function(/*[ndx,] arg*/){ + affirmStmtOpen(this); let ndx, arg; switch(arguments.length){ case 1: ndx = 1; arg = arguments[0]; break; case 2: ndx = arguments[0]; arg = arguments[1]; break; default: toss("Invalid bind() arguments."); } - if(null===arg || undefined===arg){ + this._mayGet = false; + if(undefined===arg){ + /* It might seem intuitive to bind undefined as NULL + but this approach simplifies certain client-side + uses when passing on arguments between 2+ levels of + functions. */ + return this; + }else if(!this.parameterCount){ + toss("This statement has no bindable parameters."); + }else if(null===arg){ /* bind NULL */ return bindOne(this, ndx, BindTypes.null, arg); } diff --git a/ext/fiddle/testing1.js b/ext/fiddle/testing1.js index a59d2d2cc2..5e8dd662d0 100644 --- a/ext/fiddle/testing1.js +++ b/ext/fiddle/testing1.js @@ -14,14 +14,20 @@ */ const mainTest1 = function(namespace){ + const T = self.SqliteTestUtil; + T.assert(Module._free instanceof Function). + assert(Module.allocate instanceof Function). + assert(Module.addFunction instanceof Function). + assert(Module.removeFunction instanceof Function); + const S = namespace.sqlite3.api; const oo = namespace.sqlite3.SQLite3; - const T = self.SqliteTestUtil; console.log("Loaded module:",S.sqlite3_libversion(), S.sqlite3_sourceid()); const db = new oo.DB(); const log = console.log.bind(console); try { + T.assert(db._pDb); log("DB:",db.filename); log("Build options:",oo.compileOptionUsed()); @@ -89,10 +95,31 @@ INSERT INTO t(a,b) VALUES(1,2),(3,4),(?,?);`, } }); T.assert(6 === counter); - log("Test count:",T.counter); + + log("Testing UDF..."); + db.createFunction("foo",function(a,b){return a+b}); + T.assert(7===db.selectValue("select foo(3,4)")). + assert(5===db.selectValue("select foo(3,?)",2)). + assert(5===db.selectValue("select foo(?,?)",[1,4])). + assert(5===db.selectValue("select foo($a,$b)",{$a:0,$b:5})); + db.createFunction("bar", { + arity: -1, + callback: function(){ + var rc = 0; + for(let i = 0; i < arguments.length; ++i) rc += arguments[i]; + return rc; + } + }); + T.assert(0===db.selectValue("select bar()")). + assert(1===db.selectValue("select bar(1)")). + assert(3===db.selectValue("select bar(1,2)")). + assert(-1===db.selectValue("select bar(1,2,-4)")); + + T.assert('hi' === db.selectValue("select ?",'hi')); }finally{ db.close(); } + log("Total Test count:",T.counter); }; self/*window or worker*/.Module.postRun.push(function(theModule){ diff --git a/manifest b/manifest index f2a75f05f8..98394a4ca7 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C wasm:\sminor\srefactoring\sand\sdoc\supdates. -D 2022-05-23T19:38:57.101 +C wasm/JS:\sadded\ssupport\sfor\sscalar\sUDFs.\sFixed\sa\sdeallocation\sproblem\swith\sbind()ed\sstrings/blobs. +D 2022-05-24T00:22:10.054 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in a192a8de35ba61e6d695a3bd430b021e7cbf7ea473497028540801fe7b659282 +F Makefile.in dd31c34eb86a7869660f9697694d52c8f1c9705fede9717c59559530b6f0cb87 F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241 F Makefile.msc b28a8a7a977e7312f6859f560348e1eb110c21bd6cf9fab0d16537c0a514eef3 F README.md 8b8df9ca852aeac4864eb1e400002633ee6db84065bd01b78c33817f97d31f5e @@ -56,8 +56,8 @@ F ext/expert/sqlite3expert.c 6ca30d73b9ed75bd56d6e0d7f2c962d2affaa72c505458619d0 F ext/expert/sqlite3expert.h ca81efc2679a92373a13a3e76a6138d0310e32be53d6c3bfaedabd158ea8969b F ext/expert/test_expert.c d56c194b769bdc90cf829a14c9ecbc1edca9c850b837a4d0b13be14095c32a72 F ext/fiddle/EXPORTED_FUNCTIONS.fiddle 487fc7c83d45c48326f731c89162ed17ab15767e5efede8999d7d6c6e2d04c0f -F ext/fiddle/EXPORTED_FUNCTIONS.sqlite3 5816adc4d4715b410a9df971c70f55fca610d3a240bd85d2ec34e75483cb54bb -F ext/fiddle/EXPORTED_RUNTIME_METHODS 91d5dcb0168ee056fa1a340cb8ab3c23d922622f8dad39d28919dd8af2b3ade0 +F ext/fiddle/EXPORTED_FUNCTIONS.sqlite3 07b573a1830cb2d38ed347cf2a4139ec3b9c0f69748da6a2d8356b426c807694 +F ext/fiddle/EXPORTED_RUNTIME_METHODS ff64aea52779b0d4a838268275fe02adf6f2fdf4d9ce21c22d104bf3d7597398 F ext/fiddle/Makefile 9277c73e208b9c8093659256c9f07409c877e366480c7c22ec545ee345451d95 F ext/fiddle/SqliteTestUtil.js e3094833660a6ddd40766b802901b5861b37f0b89c6c577ee0ce4c9d36399e61 F ext/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f @@ -65,10 +65,10 @@ F ext/fiddle/fiddle-worker.js e87c17070b979bd057a6849332f2a86660a4255ff7f1b6671e F ext/fiddle/fiddle.html 657c6c3f860c322fba3c69fa4f7a1209e2d2ce44b4bc65a3e154e3a97c047a7c F ext/fiddle/fiddle.js 68f5bb45fc1ae7f8ae3f6b85f465257db514d12bf50ec492259685178c452a88 F ext/fiddle/index.md d9c1c308d8074341bc3b11d1d39073cd77754cb3ca9aeb949f23fdd8323d81cf -F ext/fiddle/sqlite3-api.js c684fc5ce6b6c3e70f33699de2fc4bf9eaf045a217a30125a9da31737a9ca9e7 +F ext/fiddle/sqlite3-api.js 43d750c13ca2426580a57c1f0c8b4e529a1d8af45eda92dcdde6b5d5e4031fcd F ext/fiddle/testing-common.js 723aada13d90a5ee3f0f8f5b5b88e46954becae5d2b04ded811d90106057f4ac F ext/fiddle/testing1.html 026502e5d5e6a250e4101f8e8948708a1295ce831a094d741839ecaf788d8533 -F ext/fiddle/testing1.js c3d529379f901846907b00f62dffe752ff5724fb39791d47b421c4afdab0f58b +F ext/fiddle/testing1.js 7365c6dac4f680f8ebd6ecfcf6475c5c0a0afd61cdaff5b5281e473b79c7424e F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e F ext/fts1/ft_hash.c 3927bd880e65329bdc6f506555b228b28924921b F ext/fts1/ft_hash.h 06df7bba40dadd19597aa400a875dbc2fed705ea @@ -1968,8 +1968,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 107e3497869d757265f2a4235082bf324ba1220075d1096c2a82021a5d348a6c -R e327ce077052e8c8f39099637b67690c +P 6044605b2a712da73600cabb967797a03ed1915dc0ab0b10edbd52525e548196 +R ceb272ebfbe3295f001f5fea9e36326e U stephan -Z 572c126f779e5a94cb3892ddd8508065 +Z 228a27040854e49e332f67b566615c59 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b561a9295a..06b3868814 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6044605b2a712da73600cabb967797a03ed1915dc0ab0b10edbd52525e548196 \ No newline at end of file +325a9ee31ad7abae563c4da5cd8228e151b00aa9afcac7e9bca5efaa9d48e107 \ No newline at end of file