From: stephan Date: Tue, 20 Sep 2022 16:10:39 +0000 (+0000) Subject: Merge kv-vfs-magic-names branch into fiddle-opfs branch and make some kvvfs-relevant... X-Git-Tag: version-3.40.0~169^2~83 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=fa5aac7475494695dbae52e0c60718167597e721;p=thirdparty%2Fsqlite.git Merge kv-vfs-magic-names branch into fiddle-opfs branch and make some kvvfs-relevant tweaks. FossilOrigin-Name: e3d36dcdd37e59f17a07d3611d08744eb86f439fab82a648490dd608bcaa3185 --- fa5aac7475494695dbae52e0c60718167597e721 diff --cc ext/wasm/GNUmakefile index d8cc845230,3133d1230b..e97138c2ca --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@@ -305,103 -262,7 +305,103 @@@ wasm: $(sqlite3.js # End main Emscripten-based module build ######################################################################## -include kvvfs.make +######################################################################## +# batch-runner.js... +dir.sql := sql +speedtest1 := ../../speedtest1 +speedtest1.c := ../../test/speedtest1.c +speedtest1.sql := $(dir.sql)/speedtest1.sql - speedtest1.cliflags := --size 100 --big-transactions ++speedtest1.cliflags := --size 50 --big-transactions +$(speedtest1): + $(MAKE) -C ../.. speedtest1 +$(speedtest1.sql): $(speedtest1) $(MAKEFILE) + $(speedtest1) $(speedtest1.cliflags) --script $@ +batch-runner.list: $(MAKEFILE) $(speedtest1.sql) $(dir.sql)/000-mandelbrot.sql + bash split-speedtest1-script.sh $(dir.sql)/speedtest1.sql + ls -1 $(dir.sql)/*.sql | grep -v speedtest1.sql | sort > $@ +clean-batch: + rm -f batch-runner.list $(dir.sql)/speedtest1*.sql +# ^^^ we don't do this along with 'clean' because we clean/rebuild on +# a regular basis with different -Ox flags and rebuilding the batch +# pieces each time is an unnecessary time sink. +batch: batch-runner.list +all: batch +# end batch-runner.js +######################################################################## +# speedtest1.js... +# speedtest1-common.eflags = emcc flags used by multiple builds of speedtest1 +# speedtest1.eflags = emcc flags used by main build of speedtest1 +speedtest1-common.eflags := -g $(emcc_opt) +speedtest1.eflags := +speedtest1.eflags += -sENVIRONMENT=web +speedtest1-common.eflags += -sINVOKE_RUN=0 +speedtest1-common.eflags += --no-entry +#speedtest1-common.eflags += -flto +speedtest1-common.eflags += -sABORTING_MALLOC +speedtest1-common.eflags += -sINITIAL_MEMORY=128450560 +speedtest1-common.eflags += -sSTRICT_JS +speedtest1-common.eflags += -sMODULARIZE +speedtest1-common.eflags += -Wno-limited-postlink-optimizations +speedtest1-common.eflags += -sEXPORTED_FUNCTIONS=@$(dir.wasm)/EXPORTED_FUNCTIONS.speedtest1 +speedtest1-common.eflags += $(emcc.exportedRuntimeMethods) +speedtest1-common.eflags += -sALLOW_TABLE_GROWTH +speedtest1-common.eflags += -sDYNAMIC_EXECUTION=0 +speedtest1-common.eflags += --minify 0 +speedtest1-common.eflags += -sEXPORT_NAME=sqlite3Speedtest1InitModule +speedtest1-common.eflags += --post-js=$(post-js.js) +ifneq (0,$(enable_bigint)) +speedtest1-common.eflags += -sWASM_BIGINT +endif +speedtest1.exit-runtime0 := -sEXIT_RUNTIME=0 +speedtest1.exit-runtime1 := -sEXIT_RUNTIME=1 +# Re -sEXIT_RUNTIME=1 vs 0: if it's 1 and speedtest1 crashes, we get +# this error from emscripten: +# +# > native function `free` called after runtime exit (use +# NO_EXIT_RUNTIME to keep it alive after main() exits)) +# +# If it's 0 and it crashes, we get: +# +# > stdio streams had content in them that was not flushed. you should +# set EXIT_RUNTIME to 1 (see the FAQ), or make sure to emit a newline +# when you printf etc. +# +# and pending output is not flushed because it didn't end with a +# newline (by design). The lesser of the two evils seems to be +# -sEXIT_RUNTIME=1 but we need EXIT_RUNTIME=0 for the worker-based app +# which runs speedtest1 multiple times. + +EXPORTED_FUNCTIONS.speedtest1: EXPORTED_FUNCTIONS.api + { echo _wasm_main; cat EXPORTED_FUNCTIONS.api; } > $@ +CLEAN_FILES += EXPORTED_FUNCTIONS.speedtest1 +speedtest1.js := speedtest1.js +speedtest1.wasm := $(subst .js,.wasm,$(speedtest1.js)) +speedtest1.cflags := \ + -I. -I.. -I$(dir.top) \ + -DSQLITE_SPEEDTEST1_WASM +speedtest1.cs := $(speedtest1.c) $(sqlite3-wasm.c) $(jaccwabyt_test.c) +$(speedtest1.js): emcc.cflags+= +# speedtest1 notes re. sqlite3-wasm.o vs sqlite3-wasm.c: building against +# the latter (predictably) results in a slightly faster binary, but we're +# close enough to the target speed requirements that the 500ms makes a +# difference. +$(speedtest1.js): $(MAKEFILE) $(speedtest1.cs) $(post-js.js) \ + EXPORTED_FUNCTIONS.speedtest1 + @echo "Building $@ ..." + $(emcc.bin) \ + $(speedtest1.eflags) $(speedtest1-common.eflags) $(speedtest1.cflags) \ + $(SQLITE_OPT) \ + $(speedtest1.exit-runtime0) \ + '-DSQLITE_DEFAULT_UNIX_VFS="unix-none"' \ + -o $@ $(speedtest1.cs) -lm + $(maybe-wasm-strip) $(speedtest1.wasm) + ls -la $@ $(speedtest1.wasm) + +speedtest1: $(speedtest1.js) +all: speedtest1 +CLEAN_FILES += $(speedtest1.js) $(speedtest1.wasm) +# end speedtest1.js +######################################################################## ######################################################################## # fiddle_remote is the remote destination for the fiddle app. It diff --cc ext/wasm/api/sqlite3-api-oo1.js index fb01e98713,9e54733966..25ca34dda9 --- a/ext/wasm/api/sqlite3-api-oo1.js +++ b/ext/wasm/api/sqlite3-api-oo1.js @@@ -67,125 -64,9 +67,137 @@@ self.sqlite3ApiBootstrap.initializers.p this.name = 'SQLite3Error'; } }; - const toss3 = (...args)=>{throw new SQLite3Error(args)}; + const toss3 = (...args)=>{throw new SQLite3Error(...args)}; sqlite3.SQLite3Error = SQLite3Error; + // Documented in DB.checkRc() + const checkSqlite3Rc = function(dbPtr, sqliteResultCode){ + if(sqliteResultCode){ + if(dbPtr instanceof DB) dbPtr = dbPtr.pointer; + throw new SQLite3Error( + "sqlite result code",sqliteResultCode+":", + (dbPtr + ? capi.sqlite3_errmsg(dbPtr) + : capi.sqlite3_errstr(sqliteResultCode)) + ); + } + }; + + /** + A proxy for DB class constructors. It must be called with the + being-construct DB object as its "this". See the DB constructor + for the argument docs. This is split into a separate function + in order to enable simple creation of special-case DB constructors, + e.g. a hypothetical LocalStorageDB or OpfsDB. + + Expects to be passed a configuration object with the following + properties: + + - `.filename`: the db filename. It may be a special name like ":memory:" + or "". + + - `.flags`: as documented in the DB constructor. + + - `.vfs`: as documented in the DB constructor. + + It also accepts those as the first 3 arguments. + */ + const dbCtorHelper = function ctor(...args){ + if(!ctor._name2vfs){ - // Map special filenames which we handle here (instead of in C) - // to some helpful metadata... ++ /** ++ Map special filenames which we handle here (instead of in C) ++ to some helpful metadata... ++ ++ As of 2022-09-20, the C API supports the names :localStorage: ++ and :sessionStorage: for kvvfs. However, C code cannot ++ determine (without embedded JS code, e.g. via Emscripten's ++ EM_JS()) whether the kvvfs is legal in the current browser ++ context (namely the main UI thread). In order to help client ++ code fail early on, instead of it being delayed until they ++ try to read or write a kvvfs-backed db, we'll check for those ++ names here and throw if they're not legal in the current ++ context. ++ */ + ctor._name2vfs = Object.create(null); - const isWorkerThread = (self.window===self /*===running in main window*/) - ? false - : (n)=>toss3("The VFS for",n,"is only available in the main window thread.") ++ const isWorkerThread = ('function'===typeof importScripts/*===running in worker thread*/) ++ ? (n)=>toss3("The VFS for",n,"is only available in the main window thread.") ++ : false; + ctor._name2vfs[':localStorage:'] = { + vfs: 'kvvfs', + filename: isWorkerThread || (()=>'local') + }; + ctor._name2vfs[':sessionStorage:'] = { + vfs: 'kvvfs', + filename: isWorkerThread || (()=>'session') + }; + } + const opt = ctor.normalizeArgs(...args); + let fn = opt.filename, vfsName = opt.vfs, flagsStr = opt.flags; + if(('string'!==typeof fn && 'number'!==typeof fn) + || 'string'!==typeof flagsStr + || (vfsName && ('string'!==typeof vfsName && 'number'!==typeof vfsName))){ + console.error("Invalid DB ctor args",opt,arguments); + toss3("Invalid arguments for DB constructor."); + } + let fnJs = ('number'===typeof fn) ? capi.wasm.cstringToJs(fn) : fn; + const vfsCheck = ctor._name2vfs[fnJs]; + if(vfsCheck){ + vfsName = vfsCheck.vfs; + fn = fnJs = vfsCheck.filename(fnJs); + } + let ptr, oflags = 0; + if( flagsStr.indexOf('c')>=0 ){ + oflags |= capi.SQLITE_OPEN_CREATE | capi.SQLITE_OPEN_READWRITE; + } + if( flagsStr.indexOf('w')>=0 ) oflags |= capi.SQLITE_OPEN_READWRITE; + if( 0===oflags ) oflags |= capi.SQLITE_OPEN_READONLY; + oflags |= capi.SQLITE_OPEN_EXRESCODE; + const stack = capi.wasm.scopedAllocPush(); + try { + const ppDb = capi.wasm.scopedAllocPtr() /* output (sqlite3**) arg */; + const pVfsName = vfsName ? ( + ('number'===typeof vfsName ? vfsName : capi.wasm.scopedAllocCString(vfsName)) + ): 0; + const rc = capi.sqlite3_open_v2(fn, ppDb, oflags, pVfsName); + ptr = capi.wasm.getPtrValue(ppDb); + checkSqlite3Rc(ptr, rc); + }catch( e ){ + if( ptr ) capi.sqlite3_close_v2(ptr); + throw e; + }finally{ + capi.wasm.scopedAllocPop(stack); + } + this.filename = fnJs; + __ptrMap.set(this, ptr); + __stmtMap.set(this, Object.create(null)); + __udfMap.set(this, Object.create(null)); + }; + + /** + A helper for DB constructors. It accepts either a single + config-style object or up to 3 arguments (filename, dbOpenFlags, + dbVfsName). It returns a new object containing: + + { filename: ..., flags: ..., vfs: ... } + + If passed an object, any additional properties it has are copied + as-is into the new object. + */ + dbCtorHelper.normalizeArgs = function(filename,flags = 'c',vfs = null){ + const arg = {}; + if(1===arguments.length && 'object'===typeof arguments[0]){ + const x = arguments[0]; + Object.keys(x).forEach((k)=>arg[k] = x[k]); + if(undefined===arg.flags) arg.flags = 'c'; + if(undefined===arg.vfs) arg.vfs = null; + }else{ + arg.filename = filename; + arg.flags = flags; + arg.vfs = vfs; + } + return arg; + }; + /** The DB class provides a high-level OO wrapper around an sqlite3 db handle. diff --cc ext/wasm/api/sqlite3-api-worker1.js index b41a837e9f,0000000000..d9a943971b mode 100644,000000..100644 --- a/ext/wasm/api/sqlite3-api-worker1.js +++ b/ext/wasm/api/sqlite3-api-worker1.js @@@ -1,624 -1,0 +1,624 @@@ +/* + 2022-07-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file implements the initializer for the sqlite3 "Worker API + #1", a very basic DB access API intended to be scripted from a main + window thread via Worker-style messages. Because of limitations in + that type of communication, this API is minimalistic and only + capable of serving relatively basic DB requests (e.g. it cannot + process nested query loops concurrently). + + This file requires that the core C-style sqlite3 API and OO API #1 + have been loaded. +*/ + +/** + sqlite3.initWorker1API() implements a Worker-based wrapper around + SQLite3 OO API #1, colloquially known as "Worker API #1". + + In order to permit this API to be loaded in worker threads without + automatically registering onmessage handlers, initializing the + worker API requires calling initWorker1API(). If this function is + called from a non-worker thread then it throws an exception. It + must only be called once per Worker. + + When initialized, it installs message listeners to receive Worker + messages and then it posts a message in the form: + + ``` + {type:'sqlite3-api', result:'worker1-ready'} + ``` + + to let the client know that it has been initialized. Clients may + optionally depend on this function not returning until + initialization is complete, as the initialization is synchronous. + In some contexts, however, listening for the above message is + a better fit. + + Note that the worker-based interface can be slightly quirky because + of its async nature. In particular, any number of messages may be posted + to the worker before it starts handling any of them. If, e.g., an + "open" operation fails, any subsequent messages will fail. The + Promise-based wrapper for this API (`sqlite3-worker1-promiser.js`) + is more comfortable to use in that regard. + + The documentation for the input and output worker messages for + this API follows... + + ==================================================================== + Common message format... + + Each message posted to the worker has an operation-independent + envelope and operation-dependent arguments: + + ``` + { + type: string, // one of: 'open', 'close', 'exec', 'config-get' + + messageId: OPTIONAL arbitrary value. The worker will copy it as-is + into response messages to assist in client-side dispatching. + + dbId: a db identifier string (returned by 'open') which tells the + operation which database instance to work on. If not provided, the + first-opened db is used. This is an "opaque" value, with no + inherently useful syntax or information. Its value is subject to + change with any given build of this API and cannot be used as a + basis for anything useful beyond its one intended purpose. + + args: ...operation-dependent arguments... + + // the framework may add other properties for testing or debugging + // purposes. + + } + ``` + + Response messages, posted back to the main thread, look like: + + ``` + { + type: string. Same as above except for error responses, which have the type + 'error', + + messageId: same value, if any, provided by the inbound message + + dbId: the id of the db which was operated on, if any, as returned + by the corresponding 'open' operation. + + result: ...operation-dependent result... + + } + ``` + + ==================================================================== + Error responses + + Errors are reported messages in an operation-independent format: + + ``` + { + type: 'error', + + messageId: ...as above..., + + dbId: ...as above... + + result: { + + operation: type of the triggering operation: 'open', 'close', ... + + message: ...error message text... + + errorClass: string. The ErrorClass.name property from the thrown exception. + + input: the message object which triggered the error. + + stack: _if available_, a stack trace array. + + } + + } + ``` + + + ==================================================================== + "config-get" + + This operation fetches the serializable parts of the sqlite3 API + configuration. + + Message format: + + ``` + { + type: "config-get", + messageId: ...as above..., + args: currently ignored and may be elided. + } + ``` + + Response: + + ``` + { + type: 'config', + messageId: ...as above..., + result: { + + persistentDirName: path prefix, if any, of persistent storage. + An empty string denotes that no persistent storage is available. + + bigIntEnabled: bool. True if BigInt support is enabled. + + persistenceEnabled: true if persistent storage is enabled in the + current environment. Only files stored under persistentDirName + will persist, however. + + } + } + ``` + + + ==================================================================== + "open" a database + + Message format: + + ``` + { + type: "open", + messageId: ...as above..., + args:{ + + filename [=":memory:" or "" (unspecified)]: the db filename. + See the sqlite3.oo1.DB constructor for peculiarities and transformations, + + persistent [=false]: if true and filename is not one of ("", + ":memory:"), prepend sqlite3.capi.sqlite3_web_persistent_dir() + to the given filename so that it is stored in persistent storage + _if_ the environment supports it. If persistent storage is not + supported, the filename is used as-is. + + } + } + ``` + + Response: + + ``` + { + type: 'open', + messageId: ...as above..., + result: { + filename: db filename, possibly differing from the input. + + dbId: an opaque ID value which must be passed in the message + envelope to other calls in this API to tell them which db to + use. If it is not provided to future calls, they will default to + operating on the first-opened db. This property is, for API + consistency's sake, also part of the contaning message envelope. + Only the `open` operation includes it in the `result` property. + + persistent: true if the given filename resides in the + known-persistent storage, else false. This determination is + independent of the `persistent` input argument. + } + } + ``` + + ==================================================================== + "close" a database + + Message format: + + ``` + { + type: "close", + messageId: ...as above... + dbId: ...as above... + args: OPTIONAL: { + + unlink: if truthy, the associated db will be unlinked (removed) + from the virtual filesystems. Failure to unlink is silently + ignored. + + } + } + ``` + + If the dbId does not refer to an opened ID, this is a no-op. The + inability to close a db (because it's not opened) or delete its + file does not trigger an error. + + Response: + + ``` + { + type: 'close', + messageId: ...as above..., + result: { + + filename: filename of closed db, or undefined if no db was closed + + } + } + ``` + + ==================================================================== + "exec" SQL + + All SQL execution is processed through the exec operation. It offers + most of the features of the oo1.DB.exec() method, with a few limitations + imposed by the state having to cross thread boundaries. + + Message format: + + ``` + { + type: "exec", + messageId: ...as above... + dbId: ...as above... + args: string (SQL) or {... see below ...} + } + ``` + + Response: + + ``` + { + type: 'exec', + messageId: ...as above..., + dbId: ...as above... + result: { + input arguments, possibly modified. See below. + } + } + ``` + + The arguments are in the same form accepted by oo1.DB.exec(), with + the exceptions noted below. + + A function-type args.callback property cannot cross + the window/Worker boundary, so is not useful here. If + args.callback is a string then it is assumed to be a + message type key, in which case a callback function will be + applied which posts each row result via: + + postMessage({type: thatKeyType, + rowNumber: 1-based-#, + row: theRow, + columnNames: anArray + }) + + And, at the end of the result set (whether or not any result rows + were produced), it will post an identical message with + (row=undefined, rowNumber=null) to alert the caller than the result + set is completed. Note that a row value of `null` is a legal row + result for certain arg.rowMode values. + + (Design note: we don't use (row=undefined, rowNumber=undefined) to + indicate end-of-results because fetching those would be + indistinguishable from fetching from an empty object unless the + client used hasOwnProperty() (or similar) to distinguish "missing + property" from "property with the undefined value". Similarly, + `null` is a legal value for `row` in some case , whereas the db + layer won't emit a result value of `undefined`.) + + The callback proxy must not recurse into this interface. An exec() + call will type up the Worker thread, causing any recursion attempt + to wait until the first exec() is completed. + + The response is the input options object (or a synthesized one if + passed only a string), noting that options.resultRows and + options.columnNames may be populated by the call to db.exec(). + +*/ +self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ +sqlite3.initWorker1API = function(){ + 'use strict'; + const toss = (...args)=>{throw new Error(args.join(' '))}; - if(self.window === self || 'function' !== typeof importScripts){ ++ if('function' !== typeof importScripts){ + toss("initWorker1API() must be run from a Worker thread."); + } + const self = this.self; + const sqlite3 = this.sqlite3 || toss("Missing this.sqlite3 object."); + const DB = sqlite3.oo1.DB; + + /** + Returns the app-wide unique ID for the given db, creating one if + needed. + */ + const getDbId = function(db){ + let id = wState.idMap.get(db); + if(id) return id; + id = 'db#'+(++wState.idSeq)+'@'+db.pointer; + /** ^^^ can't simply use db.pointer b/c closing/opening may re-use + the same address, which could map pending messages to a wrong + instance. */ + wState.idMap.set(db, id); + return id; + }; + + /** + Internal helper for managing Worker-level state. + */ + const wState = { + /** First-opened db is the default for future operations when no + dbId is provided by the client. */ + defaultDb: undefined, + /** Sequence number of dbId generation. */ + idSeq: 0, + /** Map of DB instances to dbId. */ + idMap: new WeakMap, + /** Temp holder for "transferable" postMessage() state. */ + xfer: [], + open: function(opt){ + const db = new DB(opt.filename); + this.dbs[getDbId(db)] = db; + if(!this.defaultDb) this.defaultDb = db; + return db; + }, + close: function(db,alsoUnlink){ + if(db){ + delete this.dbs[getDbId(db)]; + const filename = db.getFilename(); + db.close(); + if(db===this.defaultDb) this.defaultDb = undefined; + if(alsoUnlink && filename){ + /* This isn't necessarily correct: the db might be using a + VFS other than the default. How do we best resolve this + without having to special-case the kvvfs and opfs + VFSes? */ + sqlite3.capi.wasm.sqlite3_wasm_vfs_unlink(filename); + } + } + }, + /** + Posts the given worker message value. If xferList is provided, + it must be an array, in which case a copy of it passed as + postMessage()'s second argument and xferList.length is set to + 0. + */ + post: function(msg,xferList){ + if(xferList && xferList.length){ + self.postMessage( msg, Array.from(xferList) ); + xferList.length = 0; + }else{ + self.postMessage(msg); + } + }, + /** Map of DB IDs to DBs. */ + dbs: Object.create(null), + /** Fetch the DB for the given id. Throw if require=true and the + id is not valid, else return the db or undefined. */ + getDb: function(id,require=true){ + return this.dbs[id] + || (require ? toss("Unknown (or closed) DB ID:",id) : undefined); + } + }; + + /** Throws if the given db is falsy or not opened. */ + const affirmDbOpen = function(db = wState.defaultDb){ + return (db && db.pointer) ? db : toss("DB is not opened."); + }; + + /** Extract dbId from the given message payload. */ + const getMsgDb = function(msgData,affirmExists=true){ + const db = wState.getDb(msgData.dbId,false) || wState.defaultDb; + return affirmExists ? affirmDbOpen(db) : db; + }; + + const getDefaultDbId = function(){ + return wState.defaultDb && getDbId(wState.defaultDb); + }; + + /** + A level of "organizational abstraction" for the Worker + API. Each method in this object must map directly to a Worker + message type key. The onmessage() dispatcher attempts to + dispatch all inbound messages to a method of this object, + passing it the event.data part of the inbound event object. All + methods must return a plain Object containing any result + state, which the dispatcher may amend. All methods must throw + on error. + */ + const wMsgHandler = { + open: function(ev){ + const oargs = Object.create(null), args = (ev.args || Object.create(null)); + if(args.simulateError){ // undocumented internal testing option + toss("Throwing because of simulateError flag."); + } + const rc = Object.create(null); + const pDir = sqlite3.capi.sqlite3_web_persistent_dir(); + if(!args.filename || ':memory:'===args.filename){ + oargs.filename = args.filename || ''; + }else if(pDir){ + oargs.filename = pDir + ('/'===args.filename[0] ? args.filename : ('/'+args.filename)); + }else{ + oargs.filename = args.filename; + } + const db = wState.open(oargs); + rc.filename = db.filename; + rc.persistent = !!pDir && db.filename.startsWith(pDir); + rc.dbId = getDbId(db); + return rc; + }, + + close: function(ev){ + const db = getMsgDb(ev,false); + const response = { + filename: db && db.filename + }; + if(db){ + wState.close(db, ((ev.args && 'object'===typeof ev.args) + ? !!ev.args.unlink : false)); + } + return response; + }, + + exec: function(ev){ + const rc = ( + 'string'===typeof ev.args + ) ? {sql: ev.args} : (ev.args || Object.create(null)); + if('stmt'===rc.rowMode){ + toss("Invalid rowMode for 'exec': stmt mode", + "does not work in the Worker API."); + }else if(!rc.sql){ + toss("'exec' requires input SQL."); + } + const db = getMsgDb(ev); + if(rc.callback || Array.isArray(rc.resultRows)){ + // Part of a copy-avoidance optimization for blobs + db._blobXfer = wState.xfer; + } + const theCallback = rc.callback; + let rowNumber = 0; + const hadColNames = !!rc.columnNames; + if('string' === typeof theCallback){ + if(!hadColNames) rc.columnNames = []; + /* Treat this as a worker message type and post each + row as a message of that type. */ + rc.callback = function(row,stmt){ + wState.post({ + type: theCallback, + columnNames: rc.columnNames, + rowNumber: ++rowNumber, + row: row + }, wState.xfer); + } + } + try { + db.exec(rc); + if(rc.callback instanceof Function){ + rc.callback = theCallback; + /* Post a sentinel message to tell the client that the end + of the result set has been reached (possibly with zero + rows). */ + wState.post({ + type: theCallback, + columnNames: rc.columnNames, + rowNumber: null /*null to distinguish from "property not set"*/, + row: undefined /*undefined because null is a legal row value + for some rowType values, but undefined is not*/ + }); + } + }finally{ + delete db._blobXfer; + if(rc.callback) rc.callback = theCallback; + } + return rc; + }/*exec()*/, + + 'config-get': function(){ + const rc = Object.create(null), src = sqlite3.config; + [ + 'persistentDirName', 'bigIntEnabled' + ].forEach(function(k){ + if(Object.getOwnPropertyDescriptor(src, k)) rc[k] = src[k]; + }); + rc.persistenceEnabled = !!sqlite3.capi.sqlite3_web_persistent_dir(); + return rc; + }, + + /** + TO(RE)DO, once we can abstract away access to the + JS environment's virtual filesystem. Currently this + always throws. + + Response is (should be) an object: + + { + buffer: Uint8Array (db file contents), + filename: the current db filename, + mimetype: 'application/x-sqlite3' + } + + TODO is to determine how/whether this feature can support + exports of ":memory:" and "" (temp file) DBs. The latter is + ostensibly easy because the file is (potentially) on disk, but + the former does not have a structure which maps directly to a + db file image. We can VACUUM INTO a :memory:/temp db into a + file for that purpose, though. + */ + export: function(ev){ + toss("export() requires reimplementing for portability reasons."); + /** + We need to reimplement this to use the Emscripten FS + interface. That part used to be in the OO#1 API but that + dependency was removed from that level of the API. + */ + /**const db = getMsgDb(ev); + const response = { + buffer: db.exportBinaryImage(), + filename: db.filename, + mimetype: 'application/x-sqlite3' + }; + wState.xfer.push(response.buffer.buffer); + return response;**/ + }/*export()*/, + + toss: function(ev){ + toss("Testing worker exception"); + } + }/*wMsgHandler*/; + + self.onmessage = function(ev){ + ev = ev.data; + let result, dbId = ev.dbId, evType = ev.type; + const arrivalTime = performance.now(); + try { + if(wMsgHandler.hasOwnProperty(evType) && + wMsgHandler[evType] instanceof Function){ + result = wMsgHandler[evType](ev); + }else{ + toss("Unknown db worker message type:",ev.type); + } + }catch(err){ + evType = 'error'; + result = { + operation: ev.type, + message: err.message, + errorClass: err.name, + input: ev + }; + if(err.stack){ + result.stack = ('string'===typeof err.stack) + ? err.stack.split(/\n\s*/) : err.stack; + } + if(0) console.warn("Worker is propagating an exception to main thread.", + "Reporting it _here_ for the stack trace:",err,result); + } + if(!dbId){ + dbId = result.dbId/*from 'open' cmd*/ + || getDefaultDbId(); + } + // Timing info is primarily for use in testing this API. It's not part of + // the public API. arrivalTime = when the worker got the message. + wState.post({ + type: evType, + dbId: dbId, + messageId: ev.messageId, + workerReceivedTime: arrivalTime, + workerRespondTime: performance.now(), + departureTime: ev.departureTime, + // TODO: move the timing bits into... + //timing:{ + // departure: ev.departureTime, + // workerReceived: arrivalTime, + // workerResponse: performance.now(); + //}, + result: result + }, wState.xfer); + }; + self.postMessage({type:'sqlite3-api',result:'worker1-ready'}); +}.bind({self, sqlite3}); +}); diff --cc ext/wasm/speedtest1.html index 275ba7de49,0000000000..08195f2b12 mode 100644,000000..100644 --- a/ext/wasm/speedtest1.html +++ b/ext/wasm/speedtest1.html @@@ -1,168 -1,0 +1,169 @@@ + + + + + + + + + speedtest1.wasm + + +
speedtest1.wasm
+
See also: A Worker-thread variant of this page.
+ +
+
+
Initializing app...
+
+ On a slow internet connection this may take a moment. If this + message displays for "a long time", intialization may have + failed and the JavaScript console may contain clues as to why. +
+
+
Downloading...
+
+ +
+
This page starts running the main exe when it loads, which will + block the UI until it finishes! Adding UI controls to manually configure and start it + are TODO.
+ +
Achtung: running it with the dev tools open may + drastically slow it down. For faster results, keep the dev + tools closed when running it! +
+
Output is delayed/buffered because we cannot update the UI while the + speedtest is running. Output will appear below when ready... +
+ + + + + diff --cc manifest index 8c30d17726,2851e8ad5b..54c3edc79b --- a/manifest +++ b/manifest @@@ -1,5 -1,5 +1,5 @@@ - C Export\sthe\ssqlite3_uri_...()\sfamily\sof\sfunctions\sto\swasm. - D 2022-09-20T14:52:26.772 -C When\scompiled\swith\sSQLITE_OS_KV_OPTIONAL,\sthe\smagic\snames\s":localStorage:"\nand\s":sessionStorage:"\sare\srecognized\sand\sconverted\sto\suse\sthe\skv-vfs. -D 2022-09-20T14:36:53.844 ++C Merge\skv-vfs-magic-names\sbranch\sinto\sfiddle-opfs\sbranch\sand\smake\ssome\skvvfs-relevant\stweaks. ++D 2022-09-20T16:10:39.081 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@@ -474,62 -474,41 +474,62 @@@ F ext/userauth/user-auth.txt e6641021a9 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle 7fb73f7150ab79d83bb45a67d257553c905c78cd3d693101699243f36c5ae6c3 F ext/wasm/EXPORTED_RUNTIME_METHODS.fiddle a004bd5eeeda6d3b28d16779b7f1a80305bfe009dfc7f0721b042967f0d39d02 - F ext/wasm/GNUmakefile e6359a72f044a877ac6ca8f2b04fa643664157f84d81a6d9ad09f384a65160d2 -F ext/wasm/GNUmakefile 12a672ab9125dc860457c2853f7651b98517e424d7a0e9714c89b28c5ff73800 -F ext/wasm/README.md 4b00ae7c7d93c4591251245f0996a319e2651361013c98d2efb0b026771b7331 -F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api c5eaceabb9e759aaae7d3101a4a3e542f96ab2c99d89a80ce20ec18c23115f33 ++F ext/wasm/GNUmakefile 0635cb6e90787b2d06ae51d903444214e8030274554a2406136881fed3c1fad6 +F ext/wasm/README.md e1ee1e7c321c6a250bf78a84ca6f5882890a237a450ba5a0649c7a8399194c52 +F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api 8a724a674bd2089eef9676b434c0ab709da00db33f73a94e4987e90169b1cd14 F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 -F ext/wasm/api/README.md b6d0fb64bfdf7bf9ce6938ea4104228f6f5bbef600f5d910b2f8c8694195988c +F ext/wasm/api/README.md d876597edd2b9542b6ea031adaaff1c042076fde7b670b1dc6d8a87b28a6631b F ext/wasm/api/post-js-footer.js b64319261d920211b8700004d08b956a6c285f3b0bba81456260a713ed04900c F ext/wasm/api/post-js-header.js 0e853b78db83cb1c06b01663549e0e8b4f377f12f5a2d9a4a06cb776c003880b -F ext/wasm/api/sqlite3-api-cleanup.js 149fd63a0400cd1d69548887ffde2ed89c13283384a63c2e9fcfc695e38a9e11 -F ext/wasm/api/sqlite3-api-glue.js 82c09f49c69984009ba5af2b628e67cc26c5dd203d383cd3091d40dab4e6514b -F ext/wasm/api/sqlite3-api-oo1.js e9612cb704c0563c5d71ed2a8dccd95bf6394fa4de3115d1b978dc269c49ab02 -F ext/wasm/api/sqlite3-api-opfs.js c93cdd14f81a26b3a64990515ee05c7e29827fbc8fba4e4c2fef3a37a984db89 -F ext/wasm/api/sqlite3-api-prologue.js 0fb0703d2d8ac89fa2d4dd8f9726b0ea226b8708ac34e5b482df046e147de0eb -F ext/wasm/api/sqlite3-api-worker.js 1124f404ecdf3c14d9f829425cef778cd683911a9883f0809a463c3c7773c9fd +F ext/wasm/api/sqlite3-api-cleanup.js 8564a6077cdcaea9a9f428a019af8a05887f0131e6a2a1e72a7ff1145fadfe77 +F ext/wasm/api/sqlite3-api-glue.js 366d580c8e5bf7fcf4c6dee6f646c31f5549bd417ea03a59a0acca00e8ecce30 - F ext/wasm/api/sqlite3-api-oo1.js 2d13dddf0d2b4168a9249f124134d37924331e5b55e05dba18b6d661fbeefe48 ++F ext/wasm/api/sqlite3-api-oo1.js f974e79d9af8f26bf33928c5730b0988cc706d14f59a5fe36394739b92249841 +F ext/wasm/api/sqlite3-api-opfs.js ce75aba0cbfb600cf839362012d17b7b2984aeac5189586c9a5a8f37a573a929 +F ext/wasm/api/sqlite3-api-prologue.js 6f3a67c4db37e884d33a05e5cf6d9d9bc012226a18c09f33f662fefd99840a63 - F ext/wasm/api/sqlite3-api-worker1.js ee4cf149cbacb63d06b536674f822aa5088b7e022cdffc69f1f36cebe2f9fea0 ++F ext/wasm/api/sqlite3-api-worker1.js 2eeb2a24e1a90322d84a9b88a99919b806623de62792436446099c0988f2030b F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 -F ext/wasm/api/sqlite3-wasm.c 8585793ca8311c7a0618b7e00ed2b3729799c20664a51f196258576e3d475c9e -F ext/wasm/api/sqlite3-worker.js 1325ca8d40129a82531902a3a077b795db2eeaee81746e5a0c811a04b415fa7f -F ext/wasm/common/SqliteTestUtil.js e41a1406f18da9224523fad0c48885caf995b56956a5b9852909c0989e687e90 +F ext/wasm/api/sqlite3-wasm.c 4130e2df9587f4e4c3afc04c3549d682c8a5c0cfe5b22819a0a86edb7f01b9bd +F ext/wasm/batch-runner.html 2857a6db7292ac83d1581af865d643fd34235db2df830d10b43b01388c599e04 +F ext/wasm/batch-runner.js 6f5b86e0b5519a9a941d9f17ee9c5ecdc63f452f157602fe7fdf87f6275a2b49 +F ext/wasm/common/SqliteTestUtil.js 529161a624265ba84271a52db58da022649832fa1c71309fb1e02cc037327a2b F ext/wasm/common/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f -F ext/wasm/common/testing.css 572cf1ffae0b6eb7ca63684d3392bf350217a07b90e7a896e4fa850700c989b0 -F ext/wasm/common/whwasmutil.js 3d9deda1be718e2b10e2b6b474ba6ba857d905be314201ae5b3df5eef79f66aa +F ext/wasm/common/testing.css 3a5143699c2b73a85b962271e1a9b3241b30d90e30d895e4f55665e648572962 +F ext/wasm/common/whwasmutil.js f64f94e2a3e1d8fd2139b4d80f00a3ab504985e38103c45b1ff15ec8b44fcce3 +F ext/wasm/demo-123-worker.html e419b66495d209b5211ec64903b4cfb3ca7df20d652b41fcd28bf018a773234f +F ext/wasm/demo-123.html aa281d33b7eefa755f3122b7b5a18f39a42dc5fb69c8879171bf14b4c37c4ec4 +F ext/wasm/demo-123.js 234655683e35a4543a23de7b10800d76b0369947b33e089e5613171fa7795afb +F ext/wasm/demo-kvvfs1.html 7d4f28873de67f51ac18c584b7d920825139866a96049a49c424d6f5a0ea5e7f +F ext/wasm/demo-kvvfs1.js e884ea35022d772c0d1dd884b40011413696438394f605c6cd4808cfb1642a4a F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f -F ext/wasm/fiddle/fiddle-worker.js 88bc2193a6cb6a3f04d8911bed50a4401fe6f277de7a71ba833865ab64a1b4ae +F ext/wasm/fiddle/fiddle-worker.js bccf46045be8824752876f3eec01c223be0616ccac184bffd0024cfe7a3262b8 F ext/wasm/fiddle/fiddle.html 550c5aafce40bd218de9bf26192749f69f9b10bc379423ecd2e162bcef885c08 -F ext/wasm/fiddle/fiddle.js 812f9954cc7c4b191884ad171f36fcf2d0112d0a7ecfdf6087896833a0c079a8 -F ext/wasm/jaccwabyt/jaccwabyt.js 99b424b4d467d4544e82615b58e2fe07532a898540bf9de2a985f3c21e7082b2 +F ext/wasm/fiddle/fiddle.js 4ffcfc9a235beebaddec689a549e9e0dfad6dca5c1f0b41f03468d7e76480686 +F ext/wasm/index.html 4b74ea334f0bba9923a61246d807a1873784fcef4d3fb342b698613d0a29fd9c +F ext/wasm/jaccwabyt/jaccwabyt.js 0d7f32817456a0f3937fcfd934afeb32154ca33580ab264dab6c285e6dbbd215 F ext/wasm/jaccwabyt/jaccwabyt.md 447cc02b598f7792edaa8ae6853a7847b8178a18ed356afacbdbf312b2588106 F ext/wasm/jaccwabyt/jaccwabyt_test.c 39e4b865a33548f943e2eb9dd0dc8d619a80de05d5300668e9960fff30d0d36f F ext/wasm/jaccwabyt/jaccwabyt_test.exports 5ff001ef975c426ffe88d7d8a6e96ec725e568d2c2307c416902059339c06f19 -F ext/wasm/kvvfs.make 7cc9cf10e744c3ba523c3eaf5c4af47028f3a5bb76db304ea8044a9b2a9d496f -F ext/wasm/kvvfs1.html 2acb241a6110a4ec581adbf07a23d5fc2ef9c7142aa9d60856732a102abc5016 -F ext/wasm/kvvfs1.js 46afaf4faba041bf938355627bc529854295e561f49db3a240c914e75a529338 -F ext/wasm/testing1.html 0bf3ff224628c1f1e3ed22a2dc1837c6c73722ad8c0ad9c8e6fb9e6047667231 -F ext/wasm/testing1.js cba7134901a965743fa9289d82447ab71de4690b1ee5d06f6cb83e8b569d7943 -F ext/wasm/testing2.html 73e5048e666fd6fb28b6e635677a9810e1e139c599ddcf28d687c982134b92b8 -F ext/wasm/testing2.js d37433c601f88ed275712c1cfc92d3fb36c7c22e1ed8c7396fb2359e42238ebc +F ext/wasm/scratchpad-wasmfs-main.html 20cf6f1a8f368e70d01e8c17200e3eaa90f1c8e1029186d836d14b83845fbe06 +F ext/wasm/scratchpad-wasmfs-main.js f0836e3576df7a89390d777bb53e142e559e8a79becfb2a5a976490b05a1c4fa +F ext/wasm/speedtest1-wasmfs.html 9d8cd19eab8854d17f7129aa11607cae6f6d9857c505a4aef13000588583d93e +F ext/wasm/speedtest1-worker.html ede59f2c1884bf72e3d650064604b48703c81848250b19b8063d260aa3a2201d +F ext/wasm/speedtest1-worker.js 11e7f68cedd2a83b0e638f94c1d2f58406ba672a7e88b66bff5d4f4284e8ba16 - F ext/wasm/speedtest1.html 512addeb3c27c94901178b7bcbde83a6f95c093f9ebe16a2959a0aa0d828cf1d ++F ext/wasm/speedtest1.html 8ae6ece128151d01f90579de69cfa06f021acdb760735250ef745eba7ed05d49 +F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x +F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 +F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5 +F ext/wasm/sqlite3-opfs-async-proxy.js 9305d92f32d02983c4528b9c801096cfd8295ca7d24e357d90de9bbcb201d035 +F ext/wasm/sqlite3-worker1-promiser.js 4fd0465688a28a75f1d4ee4406540ba494f49844e3cad0670d0437a001943365 +F ext/wasm/sqlite3-worker1.js 0c1e7626304543969c3846573e080c082bf43bcaa47e87d416458af84f340a9e +F ext/wasm/test-opfs-vfs.html eb69dda21eb414b8f5e3f7c1cc0f774103cc9c0f87b2d28a33419e778abfbab5 +F ext/wasm/test-opfs-vfs.js 753c6b86dd8ce0813121add44726a038ba1b83acebdc8414189cb163faf23e6d +F ext/wasm/testing-worker1-promiser.html 6eaec6e04a56cf24cf4fa8ef49d78ce8905dde1354235c9125dca6885f7ce893 +F ext/wasm/testing-worker1-promiser.js 63448fddfd3b8c89ff667d17c8b31c6c2259dd4647ebbbd28f3a921c48e924da +F ext/wasm/testing1.html 50575755e43232dbe4c2f97c9086b3118eb91ec2ee1fae931e6d7669fb17fcae +F ext/wasm/testing1.js 507001a970fe8a8eb67b6c8d783e1c1daa3db2719f727c4551af29349410e538 +F ext/wasm/testing2.html a66951c38137ff1d687df79466351f3c734fa9c6d9cce71d3cf97c291b2167e3 +F ext/wasm/testing2.js 25584bcc30f19673ce13a6f301f89f8820a59dfe044e0c4f2913941f4097fe3c +F ext/wasm/wasmfs.make 0fbe3b4ef4e5e25ed61d7b581c48e6406dd688443d1b8d4daf94d779a8056c54 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 @@@ -2025,8 -2004,11 +2025,8 @@@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a9 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 - P 25a36920d44544547a84161681cd41e292b4a70df60ac3630791873a79237d98 - R 6ffa39d63e3e386cd82927e67533dcb0 -P 354726aa6c399053785f29104de15091629ce4bc275b9e2205cb3656a9e81cd7 -R 2a2a6e9f98f7c53c15ad912991ea899c -T *branch * kv-vfs-magic-names -T *sym-kv-vfs-magic-names * -T -sym-kv-vfs * -U drh -Z 6a8bdecbcdd0ef58e91af5aa4ad5d7c2 ++P 72bebc848fce5c3b4766017d016ccb2360de2bd0cb3e47e710c80dbcb6b8b707 c5db9262d0388ccb0e84c6a4b4e2e786dd634f13874e4034ba7b175befa4ce90 ++R 1f7897feb6769d430b654c4dfafdd719 +U stephan - Z 98694450e0cd5d4ad65ed4d27ad0649d ++Z 3da2be5bc057dab3ee54a8c0dcea5576 # Remove this line to create a well-formed Fossil manifest. diff --cc manifest.uuid index 9a6bcff385,66706b769e..054082f67b --- a/manifest.uuid +++ b/manifest.uuid @@@ -1,1 -1,1 +1,1 @@@ - 72bebc848fce5c3b4766017d016ccb2360de2bd0cb3e47e710c80dbcb6b8b707 -c5db9262d0388ccb0e84c6a4b4e2e786dd634f13874e4034ba7b175befa4ce90 ++e3d36dcdd37e59f17a07d3611d08744eb86f439fab82a648490dd608bcaa3185