From: stephan Date: Wed, 4 Mar 2026 19:21:09 +0000 (+0000) Subject: Consolidate the last 200 lines of common OPFS VFS code. "opfs" still works, "opfs... X-Git-Tag: major-release~100^2~21 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=b85067c568f40e6020ec7cc17aef5bae501fedcd;p=thirdparty%2Fsqlite.git Consolidate the last 200 lines of common OPFS VFS code. "opfs" still works, "opfs-wl" registers fine but is still otherwise untested. FossilOrigin-Name: 5978ee4902e4223fed6b95bd2d8f489bb300af8b762650e7113d1f3e97519d88 --- diff --git a/ext/wasm/api/opfs-common-shared.c-pp.js b/ext/wasm/api/opfs-common-shared.c-pp.js index feefca5eb4..5150fb8e61 100644 --- a/ext/wasm/api/opfs-common-shared.c-pp.js +++ b/ext/wasm/api/opfs-common-shared.c-pp.js @@ -18,19 +18,18 @@ */ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ 'use strict'; - const toss = sqlite3.util.toss; - const capi = sqlite3.capi; - const util = sqlite3.util; - const wasm = sqlite3.wasm; + const toss = sqlite3.util.toss, + capi = sqlite3.capi, + util = sqlite3.util, + wasm = sqlite3.wasm; /** Generic utilities for working with OPFS. This will get filled out by the Promise setup and, on success, installed as sqlite3.opfs. - This is an internal/private namespace intended for use solely - by the OPFS VFSes and test code for them. The library bootstrapping + This is an internal/private namespace intended for use solely by + the OPFS VFSes and test code for them. The library bootstrapping process removes this object in non-testing contexts. - */ const opfsUtil = sqlite3.opfs = Object.create(null); @@ -421,14 +420,44 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } }; + /** + Must be called by the VFS's main installation routine and passed + the options object that function receives and a reference to that + function itself (which is assumed to have a defaultProxyUri + property set on it. See sqlite3-vfs-opfs{,-wl}.c-pp.js for + examples. + + It throws if OPFS is not available. + + If it returns falsy, it detected that OPFS should be disabled, in + which case the callee should immediately return/resolve to the + sqlite3 object. + + Else it returns a new copy of the options object, fleshed out + with any missing defaults. The caller must: + + - Set up any local state they need. + + - Call opfsUtil.createVfsState(vfsName,opt), where opt is the + object returned by this function. + + - Set up any references they may need to state returned + by the previous step. + + - Call opfvs.doTheThing() + */ opfsUtil.initOptions = function(options, callee){ - options = util.nu(options); const urlParams = new URL(globalThis.location.href).searchParams; if(urlParams.has('opfs-disable')){ //sqlite3.config.warn('Explicitly not installing "opfs" VFS due to opfs-disable flag.'); - options.disableOpfs = true; - return options; + return; + } + try{ + opfsUtil.vfsInstallationFeatureCheck(); + }catch(e){ + return; } + options = util.nu(options); if(undefined===options.verbose){ options.verbose = urlParams.has('opfs-verbose') ? (+urlParams.get('opfs-verbose') || 2) : 1; @@ -449,14 +478,18 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Creates and populates the main state object used by "opfs" and "opfs-wl", and transfered from those to their async counterpart. - Returns an object containing state which we send to the async-api - Worker or share with it. + Returns an object containing state which we send to the async + proxy Worker. - Because the returned object must be serializable to be posted to - the async proxy, after this returns, the caller must: + The returned object's vfs property holds the fully-populated + capi.sqlite3_vfs instance. - - Make a local-scope reference of state.vfs then (delete - state.vfs). That's the capi.sqlite3_vfs instance for the VFS. + After setting up any local state needed, the caller must + call theVfs.doTheThing(X,Y), where X is an object containing + the sqlite3_io_methods to override and Y is a callback which + gets triggered if init succeeds, before the final Promise + decides whether or not to reject. The result of doTheThing() + must be returned from their main installation function. This object must, when it's passed to the async part, contain only cloneable or sharable objects. After the worker's "inited" @@ -1013,6 +1046,228 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ //#include api/opfs-common-inline.c-pp.js //#undef vfs.metrics.enable opfsVfs.initS11n = initS11n; + + /** + To be called by the VFS's main installation routine after it has + wired up enough state to provide its overridden io-method impls + (which must be properties of the ioMethods argument). Returns a + Promise which the installation routine must return. callback must + be a function which performs any post-bootstrap touchups, namely + plugging in a sqlite3.oo1 wrapper. It is passed (sqlite3, opfsVfs), + where opfsVfs is the sqlite3_vfs object which was set up by + opfsUtil.createVfsState(). + */ + opfsVfs.doTheThing = function(ioMethods, callback){ + Object.assign(opfsVfs.ioSyncWrappers, ioMethods); + const thePromise = new Promise(function(promiseResolve_, promiseReject_){ + const loggers = [ + sqlite3.config.error, + sqlite3.config.warn, + sqlite3.config.log + ]; + const logImpl = (level,...args)=>{ + if(options.verbose>level) loggers[level]("OPFS syncer:",...args); + }; + const log = (...args)=>logImpl(2, ...args), + warn = (...args)=>logImpl(1, ...args), + error = (...args)=>logImpl(0, ...args), + capi = sqlite3.capi, + wasm = sqlite3.wasm; + + let promiseWasRejected = undefined; + const promiseReject = (err)=>{ + promiseWasRejected = true; + opfsVfs.dispose(); + return promiseReject_(err); + }; + const promiseResolve = ()=>{ + try{ + callback(sqlite3, opfsVfs); + }catch(e){ + return promiseReject(e); + } + promiseWasRejected = false; + return promiseResolve_(sqlite3); + }; + options.proxyUri += '?vfs='+vfsName; + const W = opfsVfs.worker = +//#if target:es6-bundler-friendly + new Worker(new URL("sqlite3-opfs-async-proxy.js?vfs=opfs", import.meta.url)); +//#elif target:es6-module + new Worker(new URL(options.proxyUri, import.meta.url)); +//#else + new Worker(options.proxyUri); +//#endif + setTimeout(()=>{ + /* At attempt to work around a browser-specific quirk in which + the Worker load is failing in such a way that we neither + resolve nor reject it. This workaround gives that resolve/reject + a time limit and rejects if that timer expires. Discussion: + https://sqlite.org/forum/forumpost/a708c98dcb3ef */ + if(undefined===promiseWasRejected){ + promiseReject( + new Error("Timeout while waiting for OPFS async proxy worker.") + ); + } + }, 4000); + W._originalOnError = W.onerror /* will be restored later */; + W.onerror = function(err){ + // The error object doesn't contain any useful info when the + // failure is, e.g., that the remote script is 404. + error("Error initializing OPFS asyncer:",err); + promiseReject(new Error("Loading OPFS async Worker failed for unknown reasons.")); + }; + + const opRun = opfsVfs.opRun; +//#if nope + /** + Not part of the public API. Only for test/development use. + */ + opfsVfs.debug = { + asyncShutdown: ()=>{ + warn("Shutting down OPFS async listener. The OPFS VFS will no longer work."); + opRun('opfs-async-shutdown'); + }, + asyncRestart: ()=>{ + warn("Attempting to restart OPFS VFS async listener. Might work, might not."); + W.postMessage({type: 'opfs-async-restart'}); + } + }; +//#endif + + const sanityCheck = function(){ + const scope = wasm.scopedAllocPush(); + const sq3File = new capi.sqlite3_file(); + try{ + const fid = sq3File.pointer; + const openFlags = capi.SQLITE_OPEN_CREATE + | capi.SQLITE_OPEN_READWRITE + //| capi.SQLITE_OPEN_DELETEONCLOSE + | capi.SQLITE_OPEN_MAIN_DB; + const pOut = wasm.scopedAlloc(8); + const dbFile = "/sanity/check/file"+randomFilename(8); + const zDbFile = wasm.scopedAllocCString(dbFile); + let rc; + state.s11n.serialize("This is ä string."); + rc = state.s11n.deserialize(); + log("deserialize() says:",rc); + if("This is ä string."!==rc[0]) toss("String d13n error."); + opfsVfs.vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); + rc = wasm.peek(pOut,'i32'); + log("xAccess(",dbFile,") exists ?=",rc); + rc = opfsVfs.vfsSyncWrappers.xOpen(opfsVfs.pointer, zDbFile, + fid, openFlags, pOut); + log("open rc =",rc,"state.sabOPView[xOpen] =", + state.sabOPView[state.opIds.xOpen]); + if(0!==rc){ + error("open failed with code",rc); + return; + } + opfsVfs.vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); + rc = wasm.peek(pOut,'i32'); + if(!rc) toss("xAccess() failed to detect file."); + rc = opfsVfs.ioSyncWrappers.xSync(sq3File.pointer, 0); + if(rc) toss('sync failed w/ rc',rc); + rc = opfsVfs.ioSyncWrappers.xTruncate(sq3File.pointer, 1024); + if(rc) toss('truncate failed w/ rc',rc); + wasm.poke(pOut,0,'i64'); + rc = opfsVfs.ioSyncWrappers.xFileSize(sq3File.pointer, pOut); + if(rc) toss('xFileSize failed w/ rc',rc); + log("xFileSize says:",wasm.peek(pOut, 'i64')); + rc = opfsVfs.ioSyncWrappers.xWrite(sq3File.pointer, zDbFile, 10, 1); + if(rc) toss("xWrite() failed!"); + const readBuf = wasm.scopedAlloc(16); + rc = opfsVfs.ioSyncWrappers.xRead(sq3File.pointer, readBuf, 6, 2); + wasm.poke(readBuf+6,0); + let jRead = wasm.cstrToJs(readBuf); + log("xRead() got:",jRead); + if("sanity"!==jRead) toss("Unexpected xRead() value."); + if(opfsVfs.vfsSyncWrappers.xSleep){ + log("xSleep()ing before close()ing..."); + opfsVfs.vfsSyncWrappers.xSleep(opfsVfs.pointer,2000); + log("waking up from xSleep()"); + } + rc = opfsVfs.ioSyncWrappers.xClose(fid); + log("xClose rc =",rc,"sabOPView =",state.sabOPView); + log("Deleting file:",dbFile); + opfsVfs.vfsSyncWrappers.xDelete(opfsVfs.pointer, zDbFile, 0x1234); + opfsVfs.vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); + rc = wasm.peek(pOut,'i32'); + if(rc) toss("Expecting 0 from xAccess(",dbFile,") after xDelete()."); + warn("End of OPFS sanity checks."); + }finally{ + sq3File.dispose(); + wasm.scopedAllocPop(scope); + } + }/*sanityCheck()*/; + + W.onmessage = function({data}){ + //log("Worker.onmessage:",data); + switch(data.type){ + case 'opfs-unavailable': + /* Async proxy has determined that OPFS is unavailable. There's + nothing more for us to do here. */ + promiseReject(new Error(data.payload.join(' '))); + break; + case 'opfs-async-loaded': + /* Arrives as soon as the asyc proxy finishes loading. + Pass our config and shared state on to the async + worker. */ + delete state.vfs; + W.postMessage({type: 'opfs-async-init', args: util.nu(state)}); + break; + case 'opfs-async-inited': { + /* Indicates that the async partner has received the 'init' + and has finished initializing, so the real work can + begin... */ + if(true===promiseWasRejected){ + break /* promise was already rejected via timer */; + } + try { + sqlite3.vfs.installVfs({ + io: {struct: opfsVfs.ioMethods, methods: opfsVfs.ioSyncWrappers}, + vfs: {struct: opfsVfs, methods: opfsVfs.vfsSyncWrappers} + }); + state.sabOPView = new Int32Array(state.sabOP); + state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize); + state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); + opfsVfs.initS11n(); + delete opfsVfs.initS11n; + if(options.sanityChecks){ + warn("Running sanity checks because of opfs-sanity-check URL arg..."); + sanityCheck(); + } + if(opfsUtil.thisThreadHasOPFS()){ + opfsUtil.getRootDir().then((d)=>{ + W.onerror = W._originalOnError; + delete W._originalOnError; + log("End of OPFS sqlite3_vfs setup.", opfsVfs); + promiseResolve(); + }).catch(promiseReject); + }else{ + promiseResolve(); + } + }catch(e){ + error(e); + promiseReject(e); + } + break; + } + default: { + const errMsg = ( + "Unexpected message from the OPFS async worker: " + + JSON.stringify(data) + ); + error(errMsg); + promiseReject(new Error(errMsg)); + break; + } + }/*switch(data.type)*/ + }/*W.onmessage()*/; + })/*thePromise*/; + return thePromise; + }/*doTheThing()*/; + return state; }/*createVfsState()*/; diff --git a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js index 134b62b107..e62f23ed14 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js @@ -36,9 +36,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const util = sqlite3.util, toss = sqlite3.util.toss; - const opfsUtil = sqlite3.opfs || sqlite3.util.toss("Missing sqlite3.opfs") - /* These get removed from sqlite3 during bootstrap, so we need an - early reference to it. */; + const opfsUtil = sqlite3.opfs || toss("Missing sqlite3.opfs"); /** installOpfsWlVfs() returns a Promise which, on success, installs an @@ -46,7 +44,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ which accept a VFS. It is intended to be called via sqlite3ApiBootstrap.initializers or an equivalent mechanism. - This VFS is essentially a copy of the "opfs" VFS but uses + This VFS is essentially identical to the "opfs" VFS but uses WebLocks for its xLock() and xUnlock() implementations. Quirks specific to this VFS: @@ -54,69 +52,30 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ - Because WebLocks effectively block until they return, they will effectively hang on locks rather than returning SQLITE_BUSY. - - The argument may optionally be a plain object with the following - configuration options: - - - proxyUri: name of the async proxy JS file. - - - verbose (=2): an integer 0-3. 0 disables all logging, 1 enables - logging of errors. 2 enables logging of warnings and errors. 3 - additionally enables debugging info. Logging is performed - via the sqlite3.config.{log|warn|error}() functions. - - - On success, the Promise resolves to the top-most sqlite3 namespace - object. Success does not necessarily mean that it installs the VFS, - as there are legitimate non-error reasons for OPFS not to be - available. + Aside from locking differences in the VFSes, this function + otherwise behaves the same as + sqlite3-vfs-opfs.c-pp.js:installOpfsVfs(). */ -const installOpfsWlVfs = function callee(options){ - try{ - opfsUtil.vfsInstallationFeatureCheck(); - }catch(e){ - return Promise.reject(e); - } +const installOpfsWlVfs = async function callee(options){ options = opfsUtil.initOptions(options, callee); - if( options.disableOpfs ){ - return Promise.resolve(sqlite3); - } - - const thePromise = new Promise(function(promiseResolve_, promiseReject_){ - const loggers = [ - sqlite3.config.error, - sqlite3.config.warn, - sqlite3.config.log - ]; - const logImpl = (level,...args)=>{ - if(options.verbose>level) loggers[level]("OPFS syncer:",...args); - }; - const log = (...args)=>logImpl(2, ...args), - warn = (...args)=>logImpl(1, ...args), - error = (...args)=>logImpl(0, ...args), - capi = sqlite3.capi, - wasm = sqlite3.wasm; - const state = opfsUtil.createVfsState('opfs-wl', options), - opfsVfs = state.vfs, - metrics = opfsVfs.metrics.counters, - mTimeStart = opfsVfs.mTimeStart, - mTimeEnd = opfsVfs.mTimeEnd, - __openFiles = opfsVfs.__openFiles; - delete state.vfs; - - /* At this point, createVfsState() has populated state and - opfsVfs with any code common to both the "opfs" and "opfs-wl" - VFSes. Now comes the VFS-dependent work... */ - - opfsVfs.ioSyncWrappers.xLock = function(pFile, lockType){ + if( !options ) return sqlite3; + const capi = sqlite3.capi, + state = opfsUtil.createVfsState('opfs-wl', options), + opfsVfs = state.vfs, + metrics = opfsVfs.metrics.counters, + mTimeStart = opfsVfs.mTimeStart, + mTimeEnd = opfsVfs.mTimeEnd, + __openFiles = opfsVfs.__openFiles; + /* At this point, createVfsState() has populated state and opfsVfs + with any code common to both the "opfs" and "opfs-wl" VFSes. Now + comes the VFS-dependent work... */ + return opfsVfs.doTheThing(util.nu({ + xLock: function(pFile, lockType){ mTimeStart('xLock'); ++metrics.xLock.count; const f = __openFiles[pFile]; let rc = 0; - /* All OPFS locks are exclusive locks. If xLock() has - previously succeeded, do nothing except record the lock - type. If no lock is active, have the async counterpart - lock the file. */ + /* See notes in sqlite3-vfs-opfs.c-pp.js. */ if( f.lockType ) { f.lockType = lockType; }else{ @@ -141,9 +100,8 @@ const installOpfsWlVfs = function callee(options){ } mTimeEnd(); return rc; - }; - - opfsVfs.ioSyncWrappers.xUnlock =function(pFile,lockType){ + }, + xUnlock: function(pFile,lockType){ mTimeStart('xUnlock'); ++metrics.xUnlock.count; const f = __openFiles[pFile]; @@ -162,64 +120,8 @@ const installOpfsWlVfs = function callee(options){ if( 0===rc ) f.lockType = lockType; mTimeEnd(); return rc; - }; - - - - let promiseWasRejected = undefined; - const promiseReject = (err)=>{ - promiseWasRejected = true; - opfsVfs.dispose(); - return promiseReject_(err); - }; - const promiseResolve = ()=>{ - promiseWasRejected = false; - return promiseResolve_(sqlite3); - }; - options.proxyUri += '?vfs=opfs-wl'; - const W = opfsVfs.worker = -//#if target:es6-bundler-friendly - new Worker(new URL("sqlite3-opfs-async-proxy.js?vfs=opfs-wl", import.meta.url)); -//#elif target:es6-module - new Worker(new URL(options.proxyUri, import.meta.url)); -//#else - new Worker(options.proxyUri); -//#endif - setTimeout(()=>{ - /* At attempt to work around a browser-specific quirk in which - the Worker load is failing in such a way that we neither - resolve nor reject it. This workaround gives that resolve/reject - a time limit and rejects if that timer expires. Discussion: - https://sqlite.org/forum/forumpost/a708c98dcb3ef */ - if(undefined===promiseWasRejected){ - promiseReject( - new Error("Timeout while waiting for OPFS async proxy worker.") - ); - } - }, 4000); - W._originalOnError = W.onerror /* will be restored later */; - W.onerror = function(err){ - // The error object doesn't contain any useful info when the - // failure is, e.g., that the remote script is 404. - error("Error initializing OPFS asyncer:",err); - promiseReject(new Error("Loading OPFS async Worker failed for unknown reasons.")); - }; - - const opRun = opfsVfs.opRun; - /** - Not part of the public API. Only for test/development use. - */ - opfsUtil.debug = { - asyncShutdown: ()=>{ - warn("Shutting down OPFS async listener. The OPFS VFS will no longer work."); - opRun('opfs-async-shutdown'); - }, - asyncRestart: ()=>{ - warn("Attempting to restart OPFS VFS async listener. Might work, might not."); - W.postMessage({type: 'opfs-async-restart'}); - } - }; - + } + }), function(sqlite3, vfs){ if(sqlite3.oo1){ const OpfsWlDb = function(...args){ const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args); @@ -229,156 +131,14 @@ const installOpfsWlVfs = function callee(options){ OpfsWlDb.prototype = Object.create(sqlite3.oo1.DB.prototype); sqlite3.oo1.OpfsWlDb = OpfsWlDb; OpfsWlDb.importDb = opfsUtil.importDb; -//#if nope - sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenCallback( - opfsVfs.pointer, - function(oo1Db, sqlite3){ - /* Set a relatively high default busy-timeout handler to - help OPFS dbs deal with multi-tab/multi-worker - contention. */ - sqlite3.capi.sqlite3_busy_timeout(oo1Db, 10000); - } - ); -//#endif }/*extend sqlite3.oo1*/ - - const sanityCheck = function(){ - const scope = wasm.scopedAllocPush(); - const sq3File = new capi.sqlite3_file(); - try{ - const fid = sq3File.pointer; - const openFlags = capi.SQLITE_OPEN_CREATE - | capi.SQLITE_OPEN_READWRITE - //| capi.SQLITE_OPEN_DELETEONCLOSE - | capi.SQLITE_OPEN_MAIN_DB; - const pOut = wasm.scopedAlloc(8); - const dbFile = "/sanity/check/file"+randomFilename(8); - const zDbFile = wasm.scopedAllocCString(dbFile); - let rc; - state.s11n.serialize("This is ä string."); - rc = state.s11n.deserialize(); - log("deserialize() says:",rc); - if("This is ä string."!==rc[0]) toss("String d13n error."); - opfsVfs.vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); - rc = wasm.peek(pOut,'i32'); - log("xAccess(",dbFile,") exists ?=",rc); - rc = opfsVfs.vfsSyncWrappers.xOpen(opfsVfs.pointer, zDbFile, - fid, openFlags, pOut); - log("open rc =",rc,"state.sabOPView[xOpen] =", - state.sabOPView[state.opIds.xOpen]); - if(0!==rc){ - error("open failed with code",rc); - return; - } - opfsVfs.vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); - rc = wasm.peek(pOut,'i32'); - if(!rc) toss("xAccess() failed to detect file."); - rc = opfsVfs.ioSyncWrappers.xSync(sq3File.pointer, 0); - if(rc) toss('sync failed w/ rc',rc); - rc = opfsVfs.ioSyncWrappers.xTruncate(sq3File.pointer, 1024); - if(rc) toss('truncate failed w/ rc',rc); - wasm.poke(pOut,0,'i64'); - rc = opfsVfs.ioSyncWrappers.xFileSize(sq3File.pointer, pOut); - if(rc) toss('xFileSize failed w/ rc',rc); - log("xFileSize says:",wasm.peek(pOut, 'i64')); - rc = opfsVfs.ioSyncWrappers.xWrite(sq3File.pointer, zDbFile, 10, 1); - if(rc) toss("xWrite() failed!"); - const readBuf = wasm.scopedAlloc(16); - rc = opfsVfs.ioSyncWrappers.xRead(sq3File.pointer, readBuf, 6, 2); - wasm.poke(readBuf+6,0); - let jRead = wasm.cstrToJs(readBuf); - log("xRead() got:",jRead); - if("sanity"!==jRead) toss("Unexpected xRead() value."); - if(opfsVfs.vfsSyncWrappers.xSleep){ - log("xSleep()ing before close()ing..."); - opfsVfs.vfsSyncWrappers.xSleep(opfsVfs.pointer,2000); - log("waking up from xSleep()"); - } - rc = opfsVfs.ioSyncWrappers.xClose(fid); - log("xClose rc =",rc,"sabOPView =",state.sabOPView); - log("Deleting file:",dbFile); - opfsVfs.vfsSyncWrappers.xDelete(opfsVfs.pointer, zDbFile, 0x1234); - opfsVfs.vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); - rc = wasm.peek(pOut,'i32'); - if(rc) toss("Expecting 0 from xAccess(",dbFile,") after xDelete()."); - warn("End of OPFS sanity checks."); - }finally{ - sq3File.dispose(); - wasm.scopedAllocPop(scope); - } - }/*sanityCheck()*/; - - W.onmessage = function({data}){ - //log("Worker.onmessage:",data); - switch(data.type){ - case 'opfs-unavailable': - /* Async proxy has determined that OPFS is unavailable. There's - nothing more for us to do here. */ - promiseReject(new Error(data.payload.join(' '))); - break; - case 'opfs-async-loaded': - /* Arrives as soon as the asyc proxy finishes loading. - Pass our config and shared state on to the async - worker. */ - W.postMessage({type: 'opfs-async-init', args: util.nu(state)}); - break; - case 'opfs-async-inited': { - /* Indicates that the async partner has received the 'init' - and has finished initializing, so the real work can - begin... */ - if(true===promiseWasRejected){ - break /* promise was already rejected via timer */; - } - try { - sqlite3.vfs.installVfs({ - io: {struct: opfsVfs.ioMethods, methods: opfsVfs.ioSyncWrappers}, - vfs: {struct: opfsVfs, methods: opfsVfs.vfsSyncWrappers} - }); - state.sabOPView = new Int32Array(state.sabOP); - state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize); - state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); - opfsVfs.initS11n(); - delete opfsVfs.initS11n; - if(options.sanityChecks){ - warn("Running sanity checks because of opfs-sanity-check URL arg..."); - sanityCheck(); - } - if(opfsUtil.thisThreadHasOPFS()){ - opfsUtil.getRootDir().then((d)=>{ - W.onerror = W._originalOnError; - delete W._originalOnError; - log("End of OPFS-WL sqlite3_vfs setup.", opfsVfs); - promiseResolve(); - }).catch(promiseReject); - }else{ - promiseResolve(); - } - }catch(e){ - error(e); - promiseReject(e); - } - break; - } - default: { - const errMsg = ( - "Unexpected message from the OPFS async worker: " + - JSON.stringify(data) - ); - error(errMsg); - promiseReject(new Error(errMsg)); - break; - } - }/*switch(data.type)*/ - }/*W.onmessage()*/; - })/*thePromise*/; - return thePromise; + })/*doTheThing()*/; }/*installOpfsWlVfs()*/; -installOpfsWlVfs.defaultProxyUri = - "sqlite3-opfs-async-proxy.js"; +installOpfsWlVfs.defaultProxyUri = "sqlite3-opfs-async-proxy.js"; globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ try{ let proxyJs = installOpfsWlVfs.defaultProxyUri; - if( sqlite3?.scriptInfo?.sqlite3Dir ){ + if( sqlite3.scriptInfo?.sqlite3Dir ){ installOpfsWlVfs.defaultProxyUri = sqlite3.scriptInfo.sqlite3Dir + proxyJs; //sqlite3.config.warn("installOpfsWlVfs.defaultProxyUri =",installOpfsWlVfs.defaultProxyUri); @@ -392,6 +152,4 @@ globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ } }); }/*sqlite3ApiBootstrap.initializers.push()*/); -//#else -/* The OPFS VFS parts are elided from builds targeting node.js. */ //#endif target:node diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index 7b72199e65..6fe317ec8d 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -21,10 +21,7 @@ 'use strict'; globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const util = sqlite3.util, - toss = sqlite3.util.toss; - const opfsUtil = sqlite3.opfs || sqlite3.util.toss("Missing sqlite3.opfs") - /* These get removed from sqlite3 during bootstrap, so we need an - early reference to it. */; + opfsUtil = sqlite3.opfs || sqlite3.util.toss("Missing sqlite3.opfs"); /** installOpfsVfs() returns a Promise which, on success, installs an sqlite3_vfs named "opfs", suitable for use with all sqlite3 APIs @@ -75,107 +72,31 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Promise resolves. This is only intended for testing and development of the VFS, not client-side use. + Additionaly, the (officially undocumented) 'opfs-disable' URL + argument will disable OPFS, making this function a no-op. + On success, the Promise resolves to the top-most sqlite3 namespace object. Success does not necessarily mean that it installs the VFS, as there are legitimate non-error reasons for OPFS not to be available. */ -const installOpfsVfs = function callee(options){ - try{ - opfsUtil.vfsInstallationFeatureCheck(); - }catch(e){ - return Promise.reject(e); - } +const installOpfsVfs = async function callee(options){ options = opfsUtil.initOptions(options, callee); - if( options.disableOpfs ){ - return Promise.resolve(sqlite3); - } - - //sqlite3.config.warn("OPFS options =",options,globalThis.location); - const thePromise = new Promise(function(promiseResolve_, promiseReject_){ - const loggers = [ - sqlite3.config.error, - sqlite3.config.warn, - sqlite3.config.log - ]; - const logImpl = (level,...args)=>{ - if(options.verbose>level) loggers[level]("OPFS syncer:",...args); - }; - const log = (...args)=>logImpl(2, ...args), - warn = (...args)=>logImpl(1, ...args), - error = (...args)=>logImpl(0, ...args), - capi = sqlite3.capi, - wasm = sqlite3.wasm; - - const state = opfsUtil.createVfsState('opfs', options), - opfsVfs = state.vfs, - metrics = opfsVfs.metrics.counters, - mTimeStart = opfsVfs.mTimeStart, - mTimeEnd = opfsVfs.mTimeEnd, - __openFiles = opfsVfs.__openFiles; - delete state.vfs; - - /* At this point, createVfsState() has populated state and - opfsVfs with any code common to both the "opfs" and "opfs-wl" - VFSes. Now comes the VFS-dependent work... */ - - let promiseWasRejected = undefined; - const promiseReject = (err)=>{ - promiseWasRejected = true; - opfsVfs.dispose(); - return promiseReject_(err); - }; - const promiseResolve = ()=>{ - promiseWasRejected = false; - return promiseResolve_(sqlite3); - }; - options.proxyUri += '?vfs=opfs'; - const W = opfsVfs.worker = -//#if target:es6-bundler-friendly - new Worker(new URL("sqlite3-opfs-async-proxy.js?vfs=opfs", import.meta.url)); -//#elif target:es6-module - new Worker(new URL(options.proxyUri, import.meta.url)); -//#else - new Worker(options.proxyUri); -//#endif - setTimeout(()=>{ - /* At attempt to work around a browser-specific quirk in which - the Worker load is failing in such a way that we neither - resolve nor reject it. This workaround gives that resolve/reject - a time limit and rejects if that timer expires. Discussion: - https://sqlite.org/forum/forumpost/a708c98dcb3ef */ - if(undefined===promiseWasRejected){ - promiseReject( - new Error("Timeout while waiting for OPFS async proxy worker.") - ); - } - }, 4000); - W._originalOnError = W.onerror /* will be restored later */; - W.onerror = function(err){ - // The error object doesn't contain any useful info when the - // failure is, e.g., that the remote script is 404. - error("Error initializing OPFS asyncer:",err); - promiseReject(new Error("Loading OPFS async Worker failed for unknown reasons.")); - }; - - const opRun = opfsVfs.opRun; -//#if nope - /** - Not part of the public API. Only for test/development use. - */ - opfsUtil.debug = { - asyncShutdown: ()=>{ - warn("Shutting down OPFS async listener. The OPFS VFS will no longer work."); - opRun('opfs-async-shutdown'); - }, - asyncRestart: ()=>{ - warn("Attempting to restart OPFS VFS async listener. Might work, might not."); - W.postMessage({type: 'opfs-async-restart'}); - } - }; -//#endif - - opfsVfs.ioSyncWrappers.xLock = function(pFile,lockType){ + if( !options ) return sqlite3; + const capi = sqlite3.capi, + state = opfsUtil.createVfsState('opfs', options), + opfsVfs = state.vfs, + metrics = opfsVfs.metrics.counters, + mTimeStart = opfsVfs.mTimeStart, + mTimeEnd = opfsVfs.mTimeEnd, + opRun = opfsVfs.opRun, + __openFiles = opfsVfs.__openFiles; + + /* At this point, createVfsState() has populated state and + opfsVfs with any code common to both the "opfs" and "opfs-wl" + VFSes. Now comes the VFS-dependent work... */ + return opfsVfs.doTheThing(util.nu({ + xLock: function(pFile,lockType){ mTimeStart('xLock'); ++metrics.xLock.count; const f = __openFiles[pFile]; @@ -192,9 +113,8 @@ const installOpfsVfs = function callee(options){ } mTimeEnd(); return rc; - }; - - opfsVfs.ioSyncWrappers.xUnlock = function(pFile,lockType){ + }, + xUnlock: function(pFile,lockType){ mTimeStart('xUnlock'); ++metrics.xUnlock.count; const f = __openFiles[pFile]; @@ -206,12 +126,12 @@ const installOpfsVfs = function callee(options){ if( 0===rc ) f.lockType = lockType; mTimeEnd(); return rc; - }; - + } + }), function(sqlite3, vfs){ if(sqlite3.oo1){ const OpfsDb = function(...args){ const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args); - opt.vfs = opfsVfs.$zName; + opt.vfs = vfs.$zName; sqlite3.oo1.DB.dbCtorHelper.call(this, opt); }; OpfsDb.prototype = Object.create(sqlite3.oo1.DB.prototype); @@ -227,144 +147,13 @@ const installOpfsVfs = function callee(options){ } ); }/*extend sqlite3.oo1*/ - - const sanityCheck = function(){ - const scope = wasm.scopedAllocPush(); - const sq3File = new capi.sqlite3_file(); - try{ - const fid = sq3File.pointer; - const openFlags = capi.SQLITE_OPEN_CREATE - | capi.SQLITE_OPEN_READWRITE - //| capi.SQLITE_OPEN_DELETEONCLOSE - | capi.SQLITE_OPEN_MAIN_DB; - const pOut = wasm.scopedAlloc(8); - const dbFile = "/sanity/check/file"+randomFilename(8); - const zDbFile = wasm.scopedAllocCString(dbFile); - let rc; - state.s11n.serialize("This is ä string."); - rc = state.s11n.deserialize(); - log("deserialize() says:",rc); - if("This is ä string."!==rc[0]) toss("String d13n error."); - opfsVfs.vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); - rc = wasm.peek(pOut,'i32'); - log("xAccess(",dbFile,") exists ?=",rc); - rc = opfsVfs.vfsSyncWrappers.xOpen(opfsVfs.pointer, zDbFile, - fid, openFlags, pOut); - log("open rc =",rc,"state.sabOPView[xOpen] =", - state.sabOPView[state.opIds.xOpen]); - if(0!==rc){ - error("open failed with code",rc); - return; - } - opfsVfs.vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); - rc = wasm.peek(pOut,'i32'); - if(!rc) toss("xAccess() failed to detect file."); - rc = opfsVfs.ioSyncWrappers.xSync(sq3File.pointer, 0); - if(rc) toss('sync failed w/ rc',rc); - rc = opfsVfs.ioSyncWrappers.xTruncate(sq3File.pointer, 1024); - if(rc) toss('truncate failed w/ rc',rc); - wasm.poke(pOut,0,'i64'); - rc = opfsVfs.ioSyncWrappers.xFileSize(sq3File.pointer, pOut); - if(rc) toss('xFileSize failed w/ rc',rc); - log("xFileSize says:",wasm.peek(pOut, 'i64')); - rc = opfsVfs.ioSyncWrappers.xWrite(sq3File.pointer, zDbFile, 10, 1); - if(rc) toss("xWrite() failed!"); - const readBuf = wasm.scopedAlloc(16); - rc = opfsVfs.ioSyncWrappers.xRead(sq3File.pointer, readBuf, 6, 2); - wasm.poke(readBuf+6,0); - let jRead = wasm.cstrToJs(readBuf); - log("xRead() got:",jRead); - if("sanity"!==jRead) toss("Unexpected xRead() value."); - if(opfsVfs.vfsSyncWrappers.xSleep){ - log("xSleep()ing before close()ing..."); - opfsVfs.vfsSyncWrappers.xSleep(opfsVfs.pointer,2000); - log("waking up from xSleep()"); - } - rc = opfsVfs.ioSyncWrappers.xClose(fid); - log("xClose rc =",rc,"sabOPView =",state.sabOPView); - log("Deleting file:",dbFile); - opfsVfs.vfsSyncWrappers.xDelete(opfsVfs.pointer, zDbFile, 0x1234); - opfsVfs.vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); - rc = wasm.peek(pOut,'i32'); - if(rc) toss("Expecting 0 from xAccess(",dbFile,") after xDelete()."); - warn("End of OPFS sanity checks."); - }finally{ - sq3File.dispose(); - wasm.scopedAllocPop(scope); - } - }/*sanityCheck()*/; - - W.onmessage = function({data}){ - //log("Worker.onmessage:",data); - switch(data.type){ - case 'opfs-unavailable': - /* Async proxy has determined that OPFS is unavailable. There's - nothing more for us to do here. */ - promiseReject(new Error(data.payload.join(' '))); - break; - case 'opfs-async-loaded': - /* Arrives as soon as the asyc proxy finishes loading. - Pass our config and shared state on to the async - worker. */ - W.postMessage({type: 'opfs-async-init',args: state}); - break; - case 'opfs-async-inited': { - /* Indicates that the async partner has received the 'init' - and has finished initializing, so the real work can - begin... */ - if(true===promiseWasRejected){ - break /* promise was already rejected via timer */; - } - try { - sqlite3.vfs.installVfs({ - io: {struct: opfsVfs.ioMethods, methods: opfsVfs.ioSyncWrappers}, - vfs: {struct: opfsVfs, methods: opfsVfs.vfsSyncWrappers} - }); - state.sabOPView = new Int32Array(state.sabOP); - state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize); - state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); - opfsVfs.initS11n(); - delete opfsVfs.initS11n; - if(options.sanityChecks){ - warn("Running sanity checks because of opfs-sanity-check URL arg..."); - sanityCheck(); - } - if(opfsUtil.thisThreadHasOPFS()){ - opfsUtil.getRootDir().then((d)=>{ - W.onerror = W._originalOnError; - delete W._originalOnError; - log("End of OPFS sqlite3_vfs setup.", opfsVfs); - promiseResolve(); - }).catch(promiseReject); - }else{ - promiseResolve(); - } - }catch(e){ - error(e); - promiseReject(e); - } - break; - } - default: { - const errMsg = ( - "Unexpected message from the OPFS async worker: " + - JSON.stringify(data) - ); - error(errMsg); - promiseReject(new Error(errMsg)); - break; - } - }/*switch(data.type)*/ - }/*W.onmessage()*/; - })/*thePromise*/; - return thePromise; + })/*doTheThing()*/; }/*installOpfsVfs()*/; -installOpfsVfs.defaultProxyUri = - "sqlite3-opfs-async-proxy.js"; +installOpfsVfs.defaultProxyUri = "sqlite3-opfs-async-proxy.js"; globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ try{ let proxyJs = installOpfsVfs.defaultProxyUri; - if( sqlite3?.scriptInfo?.sqlite3Dir ){ + if( sqlite3.scriptInfo?.sqlite3Dir ){ installOpfsVfs.defaultProxyUri = sqlite3.scriptInfo.sqlite3Dir + proxyJs; //sqlite3.config.warn("installOpfsVfs.defaultProxyUri =",installOpfsVfs.defaultProxyUri); @@ -378,6 +167,4 @@ globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ } }); }/*sqlite3ApiBootstrap.initializers.push()*/); -//#else -/* The OPFS VFS parts are elided from builds targeting node.js. */ //#endif target:node diff --git a/manifest b/manifest index 34b1032854..003a670b2e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Factor\sout\sabout\s300\slines\sof\scommon\sOPFS\sVFS\sbootstrapping\scode. -D 2026-03-04T17:54:02.085 +C Consolidate\sthe\slast\s200\slines\sof\scommon\sOPFS\sVFS\scode.\s"opfs"\sstill\sworks,\s"opfs-wl"\sregisters\sfine\sbut\sis\sstill\sotherwise\suntested. +D 2026-03-04T19:21:09.278 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -585,7 +585,7 @@ F ext/wasm/api/README.md a905d5c6bfc3e2df875bd391d6d6b7b48d41b43bdee02ad115b4724 F ext/wasm/api/extern-post-js.c-pp.js d9f42ecbedc784c0d086bc37800e52946a14f7a21600b291daa3f963c314f930 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/opfs-common-inline.c-pp.js 5be8d6d91963849e218221b48206ae55612630bb2cd7f30b1b6fcf7a9e374b76 -F ext/wasm/api/opfs-common-shared.c-pp.js 91b1291447c689a77ffcc3297dc478dd29196311facb063737aeaaf70660a0f0 +F ext/wasm/api/opfs-common-shared.c-pp.js d8ecb1c7f6b29c2eb501ab8da6f9d9867c1ceb8a42c9c883dd53aed8ddfe106a F ext/wasm/api/post-js-footer.js a50c1a2c4d008aede7b2aa1f18891a7ee71437c2f415b8aeb3db237ddce2935b F ext/wasm/api/post-js-header.js f35d2dcf1ab7f22a93d565f8e0b622a2934fc4e743edf3b708e4dd8140eeff55 F ext/wasm/api/pre-js.c-pp.js 9234ea680a2f6a2a177e8dcd934bdc5811a9f8409165433a252b87f4c07bba6f @@ -598,8 +598,8 @@ F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js f0a2aa8712211ff9db2ef548ae8b676b F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 2ccf4322f42063aefc150972943e750c77f7926b866f1639d40eec05df075b6e F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9 -F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js 929bad4b98f176b2d0a8c1509ca833b42a11f5f0871d2b3bb2597b9e29c8ea24 -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 9babe167f28ecd8fe67c97fe0734ec88beecbb61a0580d5218edcb8b3d8670ce +F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js a755ea941f254f89fcd519789097a7401362d9e9dfba19a9bfc972861257c3c5 +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 50a955ef393722d498177ad09c9e2d05bbe8dccae4c40c501482a860ca30017d F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81 F ext/wasm/api/sqlite3-wasm.c 45bb20e19b245136711f9b78584371233975811b6560c29ed9b650e225417e29 F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js aa9715f661fb700459a5a6cb1c32a4d6a770723b47aa9ac0e16c2cf87d622a66 @@ -2191,8 +2191,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P b71c79ef9672c77a72a976ffcd7cbebfaf0ff314dff97b274f7d092de6a7773f -R c0d13d0a3e8b693179cc92fc3fedae39 +P 57adecbab71795b62b1c2e4570ff504f35681e81dd8c94f78ad8e05ef39d36fd +R 18d89b2ee7fb09440d0eac4e9c8dc540 U stephan -Z 3ba0938d21136544be85b7e2c587b666 +Z d66223f806eab9c2d766ad2e9d73c95d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a8ccc41dc5..5c99fbd65b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -57adecbab71795b62b1c2e4570ff504f35681e81dd8c94f78ad8e05ef39d36fd +5978ee4902e4223fed6b95bd2d8f489bb300af8b762650e7113d1f3e97519d88