From: stephan Date: Wed, 4 Mar 2026 11:37:39 +0000 (+0000) Subject: Consolidate much of the OPFS utility code into a new file for use by two of the OPFS... X-Git-Tag: major-release~100^2~25 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=96daad10faab55aca79d53fe78bd5d1bcaaf18df;p=thirdparty%2Fsqlite.git Consolidate much of the OPFS utility code into a new file for use by two of the OPFS VFSes. FossilOrigin-Name: db19a6e9663c3a44996178cb8c35dc4ccd60f48cb4b81b6c214411a56c57def7 --- diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 7e48113ed2..1b7c5269e8 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -911,6 +911,7 @@ ifeq (0,$(wasm-bare-bones)) sqlite3-api.jses += $(dir.api)/sqlite3-vtab-helper.c-pp.js endif sqlite3-api.jses += $(dir.api)/sqlite3-vfs-kvvfs.c-pp.js +sqlite3-api.jses += $(dir.api)/opfs-common-shared.c-pp.js sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs.c-pp.js sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs-sahpool.c-pp.js sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs-wl.c-pp.js diff --git a/ext/wasm/api/opfs-common.c-pp.js b/ext/wasm/api/opfs-common-inline.c-pp.js similarity index 89% rename from ext/wasm/api/opfs-common.c-pp.js rename to ext/wasm/api/opfs-common-inline.c-pp.js index 1a7818927a..dcff8a05bd 100644 --- a/ext/wasm/api/opfs-common.c-pp.js +++ b/ext/wasm/api/opfs-common-inline.c-pp.js @@ -1,26 +1,14 @@ //#if nope /** This file is for preprocessor #include into the "opfs" and - "opfs-wl" impls, as well as their async-proxy part. + "opfs-wl" impls, as well as their async-proxy part. It must be + inlined in those files, as opposed to being a shared copy in the + library, because (A) the async proxy does not load the library and + (B) it references an object which is local to each of those files + but which has a 99% identical structure for each. */ //#endif - -//#if not defined opfs-async-proxy -/** - TODO: move the sqlite3.opfs (private/internal) namespace object - init from sqlite3-vfs-opfs*.js into here. That namespace gets - removed from the sqlite3 namespace in the final stages of library - bootstrapping except in test runs, where it's retained so that - tests can clean up OPFS so their test cases work (the down-side of - them being persistent). -*/ -//#endif not defined opfs-async-proxy - -// This function won't work as-is if we #include it, but the -// missing elements have not yet been identified. const initS11n = function(){ - /* This function is needed by the "opfs" and "opfs-wl" VFSes - and sqlite-opfs-async-proxy.js (used by those of those). */ /** This proxy de/serializes cross-thread function arguments and output-pointer values via the state.sabIO SharedArrayBuffer, diff --git a/ext/wasm/api/opfs-common-shared.c-pp.js b/ext/wasm/api/opfs-common-shared.c-pp.js new file mode 100644 index 0000000000..56af5585bc --- /dev/null +++ b/ext/wasm/api/opfs-common-shared.c-pp.js @@ -0,0 +1,430 @@ +//#if not target:node +/* + 2026-03-04 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file holds code shared by sqlite3-vfs-opfs{,-wl}.c-pp.js. It + creates a private/internal sqlite3.opfs namespace common to the two + and used (only) by them and the test framework. It is not part of + the public API. +*/ +globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ + 'use strict'; + const toss = sqlite3.util.toss; + const toss3 = sqlite3.util.toss3; + const capi = sqlite3.capi; + const util = sqlite3.util; + const wasm = sqlite3.wasm; + + /** + Generic utilities for working with OPFS. This will get filled out + by the Promise setup and, on success, installed as sqlite3.opfs. + + This is an internal/private namespace intended for use solely + by the OPFS VFSes and test code for them. The library bootstrapping + process removes this object in non-testing contexts. + + */ + const opfsUtil = sqlite3.opfs = Object.create(null); + + /** + Returns true if _this_ thread has access to the OPFS APIs. + */ + opfsUtil.thisThreadHasOPFS = ()=>{ + return globalThis.FileSystemHandle && + globalThis.FileSystemDirectoryHandle && + globalThis.FileSystemFileHandle && + globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle && + navigator?.storage?.getDirectory; + }; + + /** + Must be called by the OPFS VFSes immediately after they determine + whether OPFS is available by calling + thisThreadHasOPFS(). Resolves to the OPFS storage root directory + and sets opfsUtil.rootDirectory to that value. + */ + opfsUtil.getRootDir = async function f(){ + return f.promise ??= navigator.storage.getDirectory().then(d=>{ + opfsUtil.rootDirectory = d; + return d; + }).catch(e=>{ + delete f.promise; + throw e; + }); + }; + + /** + Expects an OPFS file path. It gets resolved, such that ".." + components are properly expanded, and returned. If the 2nd arg + is true, the result is returned as an array of path elements, + else an absolute path string is returned. + */ + opfsUtil.getResolvedPath = function(filename,splitIt){ + const p = new URL(filename, "file://irrelevant").pathname; + return splitIt ? p.split('/').filter((v)=>!!v) : p; + }; + + /** + Takes the absolute path to a filesystem element. Returns an + array of [handleOfContainingDir, filename]. If the 2nd argument + is truthy then each directory element leading to the file is + created along the way. Throws if any creation or resolution + fails. + */ + opfsUtil.getDirForFilename = async function f(absFilename, createDirs = false){ + const path = opfsUtil.getResolvedPath(absFilename, true); + const filename = path.pop(); + let dh = await opfsUtil.getRootDir(); + for(const dirName of path){ + if(dirName){ + dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs}); + } + } + return [dh, filename]; + }; + + /** + Creates the given directory name, recursively, in + the OPFS filesystem. Returns true if it succeeds or the + directory already exists, else false. + */ + opfsUtil.mkdir = async function(absDirName){ + try { + await opfsUtil.getDirForFilename(absDirName+"/filepart", true); + return true; + }catch(e){ + //sqlite3.config.warn("mkdir(",absDirName,") failed:",e); + return false; + } + }; + /** + Checks whether the given OPFS filesystem entry exists, + returning true if it does, false if it doesn't or if an + exception is intercepted while trying to make the + determination. + */ + opfsUtil.entryExists = async function(fsEntryName){ + try { + const [dh, fn] = await opfsUtil.getDirForFilename(fsEntryName); + await dh.getFileHandle(fn); + return true; + }catch(e){ + return false; + } + }; + + /** + Generates a random ASCII string len characters long, intended for + use as a temporary file name. + */ + opfsUtil.randomFilename = function f(len=16){ + if(!f._chars){ + f._chars = "abcdefghijklmnopqrstuvwxyz"+ + "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+ + "012346789"; + f._n = f._chars.length; + } + const a = []; + let i = 0; + for( ; i < len; ++i){ + const ndx = Math.random() * (f._n * 64) % f._n | 0; + a[i] = f._chars[ndx]; + } + return a.join(""); + /* + An alternative impl. with an unpredictable length + but much simpler: + + Math.floor(Math.random() * Number.MAX_SAFE_INTEGER).toString(36) + */ + }; + + /** + Returns a promise which resolves to an object which represents + all files and directories in the OPFS tree. The top-most object + has two properties: `dirs` is an array of directory entries + (described below) and `files` is a list of file names for all + files in that directory. + + Traversal starts at sqlite3.opfs.rootDirectory. + + Each `dirs` entry is an object in this form: + + ``` + { name: directoryName, + dirs: [...subdirs], + files: [...file names] + } + ``` + + The `files` and `subdirs` entries are always set but may be + empty arrays. + + The returned object has the same structure but its `name` is + an empty string. All returned objects are created with + Object.create(null), so have no prototype. + + Design note: the entries do not contain more information, + e.g. file sizes, because getting such info is not only + expensive but is subject to locking-related errors. + */ + opfsUtil.treeList = async function(){ + const doDir = async function callee(dirHandle,tgt){ + tgt.name = dirHandle.name; + tgt.dirs = []; + tgt.files = []; + for await (const handle of dirHandle.values()){ + if('directory' === handle.kind){ + const subDir = Object.create(null); + tgt.dirs.push(subDir); + await callee(handle, subDir); + }else{ + tgt.files.push(handle.name); + } + } + }; + const root = Object.create(null); + const dir = await opfsUtil.getRootDir(); + await doDir(dir, root); + return root; + }; + + /** + Irrevocably deletes _all_ files in the current origin's OPFS. + Obviously, this must be used with great caution. It may throw + an exception if removal of anything fails (e.g. a file is + locked), but the precise conditions under which the underlying + APIs will throw are not documented (so we cannot tell you what + they are). + */ + opfsUtil.rmfr = async function(){ + const rd = await opfsUtil.getRootDir(); + const dir = rd, opt = {recurse: true}; + for await (const handle of dir.values()){ + dir.removeEntry(handle.name, opt); + } + }; + + /** + Deletes the given OPFS filesystem entry. As this environment + has no notion of "current directory", the given name must be an + absolute path. If the 2nd argument is truthy, deletion is + recursive (use with caution!). + + The returned Promise resolves to true if the deletion was + successful, else false (but...). The OPFS API reports the + reason for the failure only in human-readable form, not + exceptions which can be type-checked to determine the + failure. Because of that... + + If the final argument is truthy then this function will + propagate any exception on error, rather than returning false. + */ + opfsUtil.unlink = async function(fsEntryName, recursive = false, + throwOnError = false){ + try { + const [hDir, filenamePart] = + await opfsUtil.getDirForFilename(fsEntryName, false); + await hDir.removeEntry(filenamePart, {recursive}); + return true; + }catch(e){ + if(throwOnError){ + throw new Error("unlink(",arguments[0],") failed: "+e.message,{ + cause: e + }); + } + return false; + } + }; + + /** + Traverses the OPFS filesystem, calling a callback for each + entry. The argument may be either a callback function or an + options object with any of the following properties: + + - `callback`: function which gets called for each filesystem + entry. It gets passed 3 arguments: 1) the + FileSystemFileHandle or FileSystemDirectoryHandle of each + entry (noting that both are instanceof FileSystemHandle). 2) + the FileSystemDirectoryHandle of the parent directory. 3) the + current depth level, with 0 being at the top of the tree + relative to the starting directory. If the callback returns a + literal false, as opposed to any other falsy value, traversal + stops without an error. Any exceptions it throws are + propagated. Results are undefined if the callback manipulate + the filesystem (e.g. removing or adding entries) because the + how OPFS iterators behave in the face of such changes is + undocumented. + + - `recursive` [bool=true]: specifies whether to recurse into + subdirectories or not. Whether recursion is depth-first or + breadth-first is unspecified! + + - `directory` [FileSystemDirectoryEntry=sqlite3.opfs.rootDirectory] + specifies the starting directory. + + If this function is passed a function, it is assumed to be the + callback. + + Returns a promise because it has to (by virtue of being async) + but that promise has no specific meaning: the traversal it + performs is synchronous. The promise must be used to catch any + exceptions propagated by the callback, however. + */ + opfsUtil.traverse = async function(opt){ + const defaultOpt = { + recursive: true, + directory: await opfsUtil.getRootDir() + }; + if('function'===typeof opt){ + opt = {callback:opt}; + } + opt = Object.assign(defaultOpt, opt||{}); + const doDir = async function callee(dirHandle, depth){ + for await (const handle of dirHandle.values()){ + if(false === opt.callback(handle, dirHandle, depth)) return false; + else if(opt.recursive && 'directory' === handle.kind){ + if(false === await callee(handle, depth + 1)) break; + } + } + }; + doDir(opt.directory, 0); + }; + + /** + Impl of opfsUtil.importDb() when it's given a function as its + second argument. + */ + const importDbChunked = async function(filename, callback){ + const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true); + const hFile = await hDir.getFileHandle(fnamePart, {create:true}); + let sah = await hFile.createSyncAccessHandle(); + let nWrote = 0, chunk, checkedHeader = false, err = false; + try{ + sah.truncate(0); + while( undefined !== (chunk = await callback()) ){ + if(chunk instanceof ArrayBuffer) chunk = new Uint8Array(chunk); + if( !checkedHeader && 0===nWrote && chunk.byteLength>=15 ){ + util.affirmDbHeader(chunk); + checkedHeader = true; + } + sah.write(chunk, {at: nWrote}); + nWrote += chunk.byteLength; + } + if( nWrote < 512 || 0!==nWrote % 512 ){ + toss("Input size",nWrote,"is not correct for an SQLite database."); + } + if( !checkedHeader ){ + const header = new Uint8Array(20); + sah.read( header, {at: 0} ); + util.affirmDbHeader( header ); + } + sah.write(new Uint8Array([1,1]), {at: 18}/*force db out of WAL mode*/); + return nWrote; + }catch(e){ + await sah.close(); + sah = undefined; + await hDir.removeEntry( fnamePart ).catch(()=>{}); + throw e; + }finally { + if( sah ) await sah.close(); + } + }; + + /** + Asynchronously imports the given bytes (a byte array or + ArrayBuffer) into the given database file. + + Results are undefined if the given db name refers to an opened + db. + + If passed a function for its second argument, its behaviour + changes: imports its data in chunks fed to it by the given + callback function. It calls the callback (which may be async) + repeatedly, expecting either a Uint8Array or ArrayBuffer (to + denote new input) or undefined (to denote EOF). For so long as + the callback continues to return non-undefined, it will append + incoming data to the given VFS-hosted database file. When + called this way, the resolved value of the returned Promise is + the number of bytes written to the target file. + + It very specifically requires the input to be an SQLite3 + database and throws if that's not the case. It does so in + order to prevent this function from taking on a larger scope + than it is specifically intended to. i.e. we do not want it to + become a convenience for importing arbitrary files into OPFS. + + This routine rewrites the database header bytes in the output + file (not the input array) to force disabling of WAL mode. + + On error this throws and the state of the input file is + undefined (it depends on where the exception was triggered). + + On success, resolves to the number of bytes written. + */ + opfsUtil.importDb = async function(filename, bytes){ + if( bytes instanceof Function ){ + return importDbChunked(filename, bytes); + } + if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes); + util.affirmIsDb(bytes); + const n = bytes.byteLength; + const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true); + let sah, err, nWrote = 0; + try { + const hFile = await hDir.getFileHandle(fnamePart, {create:true}); + sah = await hFile.createSyncAccessHandle(); + sah.truncate(0); + nWrote = sah.write(bytes, {at: 0}); + if(nWrote != n){ + toss("Expected to write "+n+" bytes but wrote "+nWrote+"."); + } + sah.write(new Uint8Array([1,1]), {at: 18}) /* force db out of WAL mode */; + return nWrote; + }catch(e){ + if( sah ){ await sah.close(); sah = undefined; } + await hDir.removeEntry( fnamePart ).catch(()=>{}); + throw e; + }finally{ + if( sah ) await sah.close(); + } + }; + + /** + Checks for features required for OPFS VFSes and throws with a + descriptive error message if they're not found. This is intended + to be run as part of async VFS installation steps. + */ + opfsUtil.vfsInstallationFeatureCheck = function(){ + if(!globalThis.SharedArrayBuffer + || !globalThis.Atomics){ + throw new Error("Cannot install OPFS: Missing SharedArrayBuffer and/or Atomics. "+ + "The server must emit the COOP/COEP response headers to enable those. "+ + "See https://sqlite.org/wasm/doc/trunk/persistence.md#coop-coep"); + }else if('undefined'===typeof WorkerGlobalScope){ + throw new Error("The OPFS sqlite3_vfs cannot run in the main thread "+ + "because it requires Atomics.wait()."); + }else if(!globalThis.FileSystemHandle || + !globalThis.FileSystemDirectoryHandle || + !globalThis.FileSystemFileHandle || + !globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle || + !navigator?.storage?.getDirectory){ + throw new newError("Missing required OPFS APIs."); + } + }; + +}/*sqlite3ApiBootstrap.initializers*/); +//#else +/* + The OPFS SAH Pool parts are elided from builds targeting node.js. +*/ +//#endif target:node diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js index 740be8938b..98eeb1639b 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js @@ -46,19 +46,16 @@ theFunc().then(...) is not compatible with the change to synchronous, but we do do not use those APIs that way. i.e. we don't _need_ to change anything for this, but at some point (after Chrome - versions (approximately) 104-107 are extinct) should change our + versions (approximately) 104-107 are extinct) we should change our usage of those methods to remove the "await". */ "use strict"; const wPost = (type,...args)=>postMessage({type, payload:args}); -//#if nope const urlParams = new URL(globalThis.location.href).searchParams; if( !urlParams.has('vfs') ){ throw new Error("Expecting vfs=opfs|opfs-wl URL argument for this worker"); } const isWebLocker = 'opfs-wl'===urlParams.get('vfs'); -const msgKeyPrefix = 'opfs-'; //isWebLocker ? 'opfs-wl-' : 'opfs-'; -//#endif const installAsyncProxy = function(){ const toss = function(...args){throw new Error(args.join(' '))}; if(globalThis.window === globalThis){ @@ -73,6 +70,9 @@ const installAsyncProxy = function(){ this API. */ const state = Object.create(null); +//#define opfs-async-proxy +//#include api/opfs-common-inline.c-pp.js +//#undef opfs-async-proxy /** verbose: @@ -326,7 +326,7 @@ const installAsyncProxy = function(){ } log("Got",opName+"() sync handle for",fh.filenameAbs, 'in',performance.now() - t,'ms'); - if(!fh.xLock && !state.lock/*set by opfs-wl*/){ + if(!isWebLocker && !fh.xLock){ __implicitLocks.add(fh.fid); log("Acquired implicit lock for",opName+"()",fh.fid,fh.filenameAbs); } @@ -611,9 +611,6 @@ const installAsyncProxy = function(){ } }/*vfsAsyncImpls*/; -//#define opfs-async-proxy -//#include api/opfs-common.c-pp.js - /** Starts a new WebLock request. */ diff --git a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js index 6113657021..e2f4ff2f93 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js @@ -20,12 +20,7 @@ This file is intended to be appended to the main sqlite3 JS deliverable somewhere after sqlite3-api-oo1.js. - TODOs (2026-0303): - - - Move pieces of this file which are common to it, - sqlite3-vfs-opfs.c-pp.js, and/or sqlite3-opfs-async-proxy.js into - separate files and #include them in each using the preprocessor. - e.g. the s11n namespace object is duplicated in all three files. + TODOs (2026-03-03): - For purposes of tester1.js we need to figure out which of these VFSes will install the (internal-use-only) sqlite3.opfs utility code @@ -39,8 +34,11 @@ */ 'use strict'; globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ + const opfsUtil = sqlite3.opfs || sqlite3.util.toss("Missing sqlite3.opfs") + /* Gets removed from sqlite3 during bootstrap, so we need an + early reference to it. */; /** - installOpfsVfs() returns a Promise which, on success, installs an + installOpfsWlVfs() returns a Promise which, on success, installs an sqlite3_vfs named "opfs-wl", suitable for use with all sqlite3 APIs which accept a VFS. It is intended to be called via sqlite3ApiBootstrap.initializers or an equivalent mechanism. @@ -67,27 +65,11 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ On success, the Promise resolves to the top-most sqlite3 namespace object. */ -const installOpfsVfs = function callee(options){ - if(!globalThis.SharedArrayBuffer - || !globalThis.Atomics){ - return Promise.reject( - new Error("Cannot install OPFS: Missing SharedArrayBuffer and/or Atomics. "+ - "The server must emit the COOP/COEP response headers to enable those. "+ - "See https://sqlite.org/wasm/doc/trunk/persistence.md#coop-coep") - ); - }else if('undefined'===typeof WorkerGlobalScope){ - return Promise.reject( - new Error("The OPFS sqlite3_vfs cannot run in the main thread "+ - "because it requires Atomics.wait().") - ); - }else if(!globalThis.FileSystemHandle || - !globalThis.FileSystemDirectoryHandle || - !globalThis.FileSystemFileHandle || - !globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle || - !navigator?.storage?.getDirectory){ - return Promise.reject( - new Error("Missing required OPFS APIs.") - ); +const installOpfsWlVfs = function callee(options){ + try{ + opfsUtil.vfsInstallationFeatureCheck(); + }catch(e){ + return Promise.reject(e); } const nu = (...obj)=>Object.assign(Object.create(null),...obj); options = nu(options); @@ -129,31 +111,34 @@ const installOpfsVfs = function callee(options){ const sqlite3_file = capi.sqlite3_file; const sqlite3_io_methods = capi.sqlite3_io_methods; /** - Generic utilities for working with OPFS. This will get filled out - by the Promise setup and, on success, installed as sqlite3.opfs. + State which we send to the async-api Worker or share with it. + This object must initially contain only cloneable or sharable + objects. After the worker's "inited" message arrives, other types + of data may be added to it. - ACHTUNG: do not rely on these APIs in client code. They are - experimental and subject to change or removal as the - OPFS-specific sqlite3_vfs evolves. - */ - const opfsUtil = Object.create(null); + For purposes of Atomics.wait() and Atomics.notify(), we use a + SharedArrayBuffer with one slot reserved for each of the API + proxy's methods. The sync side of the API uses Atomics.wait() + on the corresponding slot and the async side uses + Atomics.notify() on that slot. - /** - Returns true if _this_ thread has access to the OPFS APIs. + The approach of using a single SAB to serialize comms for all + instances might(?) lead to deadlock situations in multi-db + cases. We should probably have one SAB here with a single slot + for locking a per-file initialization step and then allocate a + separate SAB like the above one for each file. That will + require a bit of acrobatics but should be feasible. The most + problematic part is that xOpen() would have to use + postMessage() to communicate its SharedArrayBuffer, and mixing + that approach with Atomics.wait/notify() gets a bit messy. */ - const thisThreadHasOPFS = ()=>{ - return globalThis.FileSystemHandle && - globalThis.FileSystemDirectoryHandle && - globalThis.FileSystemFileHandle && - globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle && - navigator?.storage?.getDirectory; - }; + const state = Object.create(null); + const metrics = Object.create(null); +//#define opfs-has-metrics +//#include api/opfs-common-inline.c-pp.js +//#undef opfs-has-metrics - /** - Not part of the public API. Solely for internal/development - use. - */ - opfsUtil.metrics = { + const vfsMetrics = { dump: function(){ let k, n = 0, t = 0, w = 0; for(k in state.opIds){ @@ -254,29 +239,7 @@ const installOpfsVfs = function callee(options){ environment or the other when sqlite3_os_end() is called (_if_ it gets called at all in a wasm build, which is undefined). */ - /** - State which we send to the async-api Worker or share with it. - This object must initially contain only cloneable or sharable - objects. After the worker's "inited" message arrives, other types - of data may be added to it. - - For purposes of Atomics.wait() and Atomics.notify(), we use a - SharedArrayBuffer with one slot reserved for each of the API - proxy's methods. The sync side of the API uses Atomics.wait() - on the corresponding slot and the async side uses - Atomics.notify() on that slot. - The approach of using a single SAB to serialize comms for all - instances might(?) lead to deadlock situations in multi-db - cases. We should probably have one SAB here with a single slot - for locking a per-file initialization step and then allocate a - separate SAB like the above one for each file. That will - require a bit of acrobatics but should be feasible. The most - problematic part is that xOpen() would have to use - postMessage() to communicate its SharedArrayBuffer, and mixing - that approach with Atomics.wait/notify() gets a bit messy. - */ - const state = Object.create(null); state.verbose = options.verbose; state.littleEndian = (()=>{ const buffer = new ArrayBuffer(2); @@ -331,7 +294,6 @@ const installOpfsVfs = function callee(options){ + state.sabS11nSize/* argument/result serialization block */ ); state.opIds = Object.create(null); - const metrics = Object.create(null); { /* Indexes for use in our SharedArrayBuffer... */ let i = 0; @@ -378,7 +340,7 @@ const installOpfsVfs = function callee(options){ state.sabOP = new SharedArrayBuffer( i * 4/* ==sizeof int32, noting that Atomics.wait() and friends can only function on Int32Array views of an SAB. */); - opfsUtil.metrics.reset(); + vfsMetrics.reset(); } /** SQLITE_xxx constants to export to the async worker @@ -499,6 +461,7 @@ const installOpfsVfs = function callee(options){ return rc; }; +//#if nope /** Not part of the public API. Only for test/development use. */ @@ -512,35 +475,7 @@ const installOpfsVfs = function callee(options){ W.postMessage({type: 'opfs-async-restart'}); } }; - -//#define opfs-has-metrics -//#include api/opfs-common.c-pp.js - - /** - Generates a random ASCII string len characters long, intended for - use as a temporary file name. - */ - const randomFilename = function f(len=16){ - if(!f._chars){ - f._chars = "abcdefghijklmnopqrstuvwxyz"+ - "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+ - "012346789"; - f._n = f._chars.length; - } - const a = []; - let i = 0; - for( ; i < len; ++i){ - const ndx = Math.random() * (f._n * 64) % f._n | 0; - a[i] = f._chars[ndx]; - } - return a.join(""); - /* - An alternative impl. with an unpredictable length - but much simpler: - - Math.floor(Math.random() * Number.MAX_SAFE_INTEGER).toString(36) - */ - }; +//#endif /** Map of sqlite3_file pointers to objects constructed by xOpen(). @@ -770,7 +705,7 @@ const installOpfsVfs = function callee(options){ mTimeStart('xOpen'); let opfsFlags = 0; if(0===zName){ - zName = randomFilename(); + zName = opfsUtil.randomFilename(); }else if(wasm.isPtr(zName)){ if(capi.sqlite3_uri_boolean(zName, "opfs-unlock-asap", 0)){ /* -----------------------^^^^^ MUST pass the untranslated @@ -832,322 +767,6 @@ const installOpfsVfs = function callee(options){ }; } - /** - Expects an OPFS file path. It gets resolved, such that ".." - components are properly expanded, and returned. If the 2nd arg - is true, the result is returned as an array of path elements, - else an absolute path string is returned. - */ - opfsUtil.getResolvedPath = function(filename,splitIt){ - const p = new URL(filename, "file://irrelevant").pathname; - return splitIt ? p.split('/').filter((v)=>!!v) : p; - }; - - /** - Takes the absolute path to a filesystem element. Returns an - array of [handleOfContainingDir, filename]. If the 2nd argument - is truthy then each directory element leading to the file is - created along the way. Throws if any creation or resolution - fails. - */ - opfsUtil.getDirForFilename = async function f(absFilename, createDirs = false){ - const path = opfsUtil.getResolvedPath(absFilename, true); - const filename = path.pop(); - let dh = opfsUtil.rootDirectory; - for(const dirName of path){ - if(dirName){ - dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs}); - } - } - return [dh, filename]; - }; - - /** - Creates the given directory name, recursively, in - the OPFS filesystem. Returns true if it succeeds or the - directory already exists, else false. - */ - opfsUtil.mkdir = async function(absDirName){ - try { - await opfsUtil.getDirForFilename(absDirName+"/filepart", true); - return true; - }catch(e){ - //sqlite3.config.warn("mkdir(",absDirName,") failed:",e); - return false; - } - }; - /** - Checks whether the given OPFS filesystem entry exists, - returning true if it does, false if it doesn't or if an - exception is intercepted while trying to make the - determination. - */ - opfsUtil.entryExists = async function(fsEntryName){ - try { - const [dh, fn] = await opfsUtil.getDirForFilename(fsEntryName); - await dh.getFileHandle(fn); - return true; - }catch(e){ - return false; - } - }; - - /** - Generates a random ASCII string, intended for use as a - temporary file name. Its argument is the length of the string, - defaulting to 16. - */ - opfsUtil.randomFilename = randomFilename; - - /** - Returns a promise which resolves to an object which represents - all files and directories in the OPFS tree. The top-most object - has two properties: `dirs` is an array of directory entries - (described below) and `files` is a list of file names for all - files in that directory. - - Traversal starts at sqlite3.opfs.rootDirectory. - - Each `dirs` entry is an object in this form: - - ``` - { name: directoryName, - dirs: [...subdirs], - files: [...file names] - } - ``` - - The `files` and `subdirs` entries are always set but may be - empty arrays. - - The returned object has the same structure but its `name` is - an empty string. All returned objects are created with - Object.create(null), so have no prototype. - - Design note: the entries do not contain more information, - e.g. file sizes, because getting such info is not only - expensive but is subject to locking-related errors. - */ - opfsUtil.treeList = async function(){ - const doDir = async function callee(dirHandle,tgt){ - tgt.name = dirHandle.name; - tgt.dirs = []; - tgt.files = []; - for await (const handle of dirHandle.values()){ - if('directory' === handle.kind){ - const subDir = Object.create(null); - tgt.dirs.push(subDir); - await callee(handle, subDir); - }else{ - tgt.files.push(handle.name); - } - } - }; - const root = Object.create(null); - await doDir(opfsUtil.rootDirectory, root); - return root; - }; - - /** - Irrevocably deletes _all_ files in the current origin's OPFS. - Obviously, this must be used with great caution. It may throw - an exception if removal of anything fails (e.g. a file is - locked), but the precise conditions under which the underlying - APIs will throw are not documented (so we cannot tell you what - they are). - */ - opfsUtil.rmfr = async function(){ - const dir = opfsUtil.rootDirectory, opt = {recurse: true}; - for await (const handle of dir.values()){ - dir.removeEntry(handle.name, opt); - } - }; - - /** - Deletes the given OPFS filesystem entry. As this environment - has no notion of "current directory", the given name must be an - absolute path. If the 2nd argument is truthy, deletion is - recursive (use with caution!). - - The returned Promise resolves to true if the deletion was - successful, else false (but...). The OPFS API reports the - reason for the failure only in human-readable form, not - exceptions which can be type-checked to determine the - failure. Because of that... - - If the final argument is truthy then this function will - propagate any exception on error, rather than returning false. - */ - opfsUtil.unlink = async function(fsEntryName, recursive = false, - throwOnError = false){ - try { - const [hDir, filenamePart] = - await opfsUtil.getDirForFilename(fsEntryName, false); - await hDir.removeEntry(filenamePart, {recursive}); - return true; - }catch(e){ - if(throwOnError){ - throw new Error("unlink(",arguments[0],") failed: "+e.message,{ - cause: e - }); - } - return false; - } - }; - - /** - Traverses the OPFS filesystem, calling a callback for each - entry. The argument may be either a callback function or an - options object with any of the following properties: - - - `callback`: function which gets called for each filesystem - entry. It gets passed 3 arguments: 1) the - FileSystemFileHandle or FileSystemDirectoryHandle of each - entry (noting that both are instanceof FileSystemHandle). 2) - the FileSystemDirectoryHandle of the parent directory. 3) the - current depth level, with 0 being at the top of the tree - relative to the starting directory. If the callback returns a - literal false, as opposed to any other falsy value, traversal - stops without an error. Any exceptions it throws are - propagated. Results are undefined if the callback manipulate - the filesystem (e.g. removing or adding entries) because the - how OPFS iterators behave in the face of such changes is - undocumented. - - - `recursive` [bool=true]: specifies whether to recurse into - subdirectories or not. Whether recursion is depth-first or - breadth-first is unspecified! - - - `directory` [FileSystemDirectoryEntry=sqlite3.opfs.rootDirectory] - specifies the starting directory. - - If this function is passed a function, it is assumed to be the - callback. - - Returns a promise because it has to (by virtue of being async) - but that promise has no specific meaning: the traversal it - performs is synchronous. The promise must be used to catch any - exceptions propagated by the callback, however. - */ - opfsUtil.traverse = async function(opt){ - const defaultOpt = { - recursive: true, - directory: opfsUtil.rootDirectory - }; - if('function'===typeof opt){ - opt = {callback:opt}; - } - opt = Object.assign(defaultOpt, opt||{}); - const doDir = async function callee(dirHandle, depth){ - for await (const handle of dirHandle.values()){ - if(false === opt.callback(handle, dirHandle, depth)) return false; - else if(opt.recursive && 'directory' === handle.kind){ - if(false === await callee(handle, depth + 1)) break; - } - } - }; - doDir(opt.directory, 0); - }; - - /** - impl of importDb() when it's given a function as its second - argument. - */ - const importDbChunked = async function(filename, callback){ - const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true); - const hFile = await hDir.getFileHandle(fnamePart, {create:true}); - let sah = await hFile.createSyncAccessHandle(); - let nWrote = 0, chunk, checkedHeader = false, err = false; - try{ - sah.truncate(0); - while( undefined !== (chunk = await callback()) ){ - if(chunk instanceof ArrayBuffer) chunk = new Uint8Array(chunk); - if( !checkedHeader && 0===nWrote && chunk.byteLength>=15 ){ - util.affirmDbHeader(chunk); - checkedHeader = true; - } - sah.write(chunk, {at: nWrote}); - nWrote += chunk.byteLength; - } - if( nWrote < 512 || 0!==nWrote % 512 ){ - toss("Input size",nWrote,"is not correct for an SQLite database."); - } - if( !checkedHeader ){ - const header = new Uint8Array(20); - sah.read( header, {at: 0} ); - util.affirmDbHeader( header ); - } - sah.write(new Uint8Array([1,1]), {at: 18}/*force db out of WAL mode*/); - return nWrote; - }catch(e){ - await sah.close(); - sah = undefined; - await hDir.removeEntry( fnamePart ).catch(()=>{}); - throw e; - }finally { - if( sah ) await sah.close(); - } - }; - - /** - Asynchronously imports the given bytes (a byte array or - ArrayBuffer) into the given database file. - - Results are undefined if the given db name refers to an opened - db. - - If passed a function for its second argument, its behaviour - changes: imports its data in chunks fed to it by the given - callback function. It calls the callback (which may be async) - repeatedly, expecting either a Uint8Array or ArrayBuffer (to - denote new input) or undefined (to denote EOF). For so long as - the callback continues to return non-undefined, it will append - incoming data to the given VFS-hosted database file. When - called this way, the resolved value of the returned Promise is - the number of bytes written to the target file. - - It very specifically requires the input to be an SQLite3 - database and throws if that's not the case. It does so in - order to prevent this function from taking on a larger scope - than it is specifically intended to. i.e. we do not want it to - become a convenience for importing arbitrary files into OPFS. - - This routine rewrites the database header bytes in the output - file (not the input array) to force disabling of WAL mode. - - On error this throws and the state of the input file is - undefined (it depends on where the exception was triggered). - - On success, resolves to the number of bytes written. - */ - opfsUtil.importDb = async function(filename, bytes){ - if( bytes instanceof Function ){ - return importDbChunked(filename, bytes); - } - if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes); - util.affirmIsDb(bytes); - const n = bytes.byteLength; - const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true); - let sah, err, nWrote = 0; - try { - const hFile = await hDir.getFileHandle(fnamePart, {create:true}); - sah = await hFile.createSyncAccessHandle(); - sah.truncate(0); - nWrote = sah.write(bytes, {at: 0}); - if(nWrote != n){ - toss("Expected to write "+n+" bytes but wrote "+nWrote+"."); - } - sah.write(new Uint8Array([1,1]), {at: 18}) /* force db out of WAL mode */; - return nWrote; - }catch(e){ - if( sah ){ await sah.close(); sah = undefined; } - await hDir.removeEntry( fnamePart ).catch(()=>{}); - throw e; - }finally{ - if( sah ) await sah.close(); - } - }; - if(sqlite3.oo1){ const OpfsWlDb = function(...args){ const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args); @@ -1270,13 +889,11 @@ const installOpfsVfs = function callee(options){ warn("Running sanity checks because of opfs-sanity-check URL arg..."); sanityCheck(); } - if(thisThreadHasOPFS()){ - navigator.storage.getDirectory().then((d)=>{ + if(opfsUtil.thisThreadHasOPFS()){ + opfsUtil.getRootDir().then((d)=>{ W.onerror = W._originalOnError; delete W._originalOnError; - //sqlite3.opfs = opfsUtil; - opfsUtil.rootDirectory = d; - log("End of OPFS sqlite3_vfs setup.", opfsVfs); + log("End of OPFS-WL sqlite3_vfs setup.", opfsVfs); promiseResolve(); }).catch(promiseReject); }else{ @@ -1301,22 +918,22 @@ const installOpfsVfs = function callee(options){ }/*W.onmessage()*/; })/*thePromise*/; return thePromise; -}/*installOpfsVfs()*/; -installOpfsVfs.defaultProxyUri = +}/*installOpfsWlVfs()*/; +installOpfsWlVfs.defaultProxyUri = "sqlite3-opfs-async-proxy.js"; globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ try{ - let proxyJs = installOpfsVfs.defaultProxyUri; + let proxyJs = installOpfsWlVfs.defaultProxyUri; if( sqlite3?.scriptInfo?.sqlite3Dir ){ - installOpfsVfs.defaultProxyUri = + installOpfsWlVfs.defaultProxyUri = sqlite3.scriptInfo.sqlite3Dir + proxyJs; - //sqlite3.config.warn("installOpfsVfs.defaultProxyUri =",installOpfsVfs.defaultProxyUri); + //sqlite3.config.warn("installOpfsWlVfs.defaultProxyUri =",installOpfsWlVfs.defaultProxyUri); } - return installOpfsVfs().catch((e)=>{ - sqlite3.config.warn("Ignoring inability to install OPFS sqlite3_vfs:",e.message); + return installOpfsWlVfs().catch((e)=>{ + sqlite3.config.warn("Ignoring inability to install OPFS-WL sqlite3_vfs:",e.message); }); }catch(e){ - sqlite3.config.error("installOpfsVfs() exception:",e); + sqlite3.config.error("installOpfsWlVfs() exception:",e); return Promise.reject(e); } }); diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index 216cd83a2a..7ba7427d0a 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -20,7 +20,10 @@ */ 'use strict'; globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ -/** + const opfsUtil = sqlite3.opfs || sqlite3.util.toss("Missing sqlite3.opfs") + /* Gets removed from sqlite3 during bootstrap, so we need an + early reference to it. */; + /** installOpfsVfs() returns a Promise which, on success, installs an sqlite3_vfs named "opfs", suitable for use with all sqlite3 APIs which accept a VFS. It is intended to be called via @@ -74,30 +77,12 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ object. */ const installOpfsVfs = function callee(options){ - if(!globalThis.SharedArrayBuffer - || !globalThis.Atomics){ - return Promise.reject( - new Error("Cannot install OPFS: Missing SharedArrayBuffer and/or Atomics. "+ - "The server must emit the COOP/COEP response headers to enable those. "+ - "See https://sqlite.org/wasm/doc/trunk/persistence.md#coop-coep") - ); - }else if('undefined'===typeof WorkerGlobalScope){ - return Promise.reject( - new Error("The OPFS sqlite3_vfs cannot run in the main thread "+ - "because it requires Atomics.wait().") - ); - }else if(!globalThis.FileSystemHandle || - !globalThis.FileSystemDirectoryHandle || - !globalThis.FileSystemFileHandle || - !globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle || - !navigator?.storage?.getDirectory){ - return Promise.reject( - new Error("Missing required OPFS APIs.") - ); - } - if(!options || 'object'!==typeof options){ - options = Object.create(null); + try{ + opfsUtil.vfsInstallationFeatureCheck(); + }catch(e){ + return Promise.reject(e); } + options = Object.assign(Object.create(null), options); const urlParams = new URL(globalThis.location.href).searchParams; if(urlParams.has('opfs-disable')){ //sqlite3.config.warn('Explicitly not installing "opfs" VFS due to opfs-disable flag.'); @@ -113,12 +98,10 @@ const installOpfsVfs = function callee(options){ if(undefined===options.proxyUri){ options.proxyUri = callee.defaultProxyUri; } - - //sqlite3.config.warn("OPFS options =",options,globalThis.location); - if('function' === typeof options.proxyUri){ options.proxyUri = options.proxyUri(); } + //sqlite3.config.warn("OPFS options =",options,globalThis.location); const thePromise = new Promise(function(promiseResolve_, promiseReject_){ const loggers = [ sqlite3.config.error, @@ -138,32 +121,36 @@ const installOpfsVfs = function callee(options){ const sqlite3_vfs = capi.sqlite3_vfs; const sqlite3_file = capi.sqlite3_file; const sqlite3_io_methods = capi.sqlite3_io_methods; + /** - Generic utilities for working with OPFS. This will get filled out - by the Promise setup and, on success, installed as sqlite3.opfs. + State which we send to the async-api Worker or share with it. + This object must initially contain only cloneable or sharable + objects. After the worker's "inited" message arrives, other types + of data may be added to it. - ACHTUNG: do not rely on these APIs in client code. They are - experimental and subject to change or removal as the - OPFS-specific sqlite3_vfs evolves. - */ - const opfsUtil = Object.create(null); + For purposes of Atomics.wait() and Atomics.notify(), we use a + SharedArrayBuffer with one slot reserved for each of the API + proxy's methods. The sync side of the API uses Atomics.wait() + on the corresponding slot and the async side uses + Atomics.notify() on that slot. - /** - Returns true if _this_ thread has access to the OPFS APIs. + The approach of using a single SAB to serialize comms for all + instances might(?) lead to deadlock situations in multi-db + cases. We should probably have one SAB here with a single slot + for locking a per-file initialization step and then allocate a + separate SAB like the above one for each file. That will + require a bit of acrobatics but should be feasible. The most + problematic part is that xOpen() would have to use + postMessage() to communicate its SharedArrayBuffer, and mixing + that approach with Atomics.wait/notify() gets a bit messy. */ - const thisThreadHasOPFS = ()=>{ - return globalThis.FileSystemHandle && - globalThis.FileSystemDirectoryHandle && - globalThis.FileSystemFileHandle && - globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle && - navigator?.storage?.getDirectory; - }; + const state = Object.create(null); + const metrics = Object.create(null); +//#define opfs-has-metrics +//#include api/opfs-common-inline.c-pp.js +//#undef opfs-has-metrics - /** - Not part of the public API. Solely for internal/development - use. - */ - opfsUtil.metrics = { + const vfsMetrics = { dump: function(){ let k, n = 0, t = 0, w = 0; for(k in state.opIds){ @@ -264,29 +251,6 @@ const installOpfsVfs = function callee(options){ environment or the other when sqlite3_os_end() is called (_if_ it gets called at all in a wasm build, which is undefined). */ - /** - State which we send to the async-api Worker or share with it. - This object must initially contain only cloneable or sharable - objects. After the worker's "inited" message arrives, other types - of data may be added to it. - - For purposes of Atomics.wait() and Atomics.notify(), we use a - SharedArrayBuffer with one slot reserved for each of the API - proxy's methods. The sync side of the API uses Atomics.wait() - on the corresponding slot and the async side uses - Atomics.notify() on that slot. - - The approach of using a single SAB to serialize comms for all - instances might(?) lead to deadlock situations in multi-db - cases. We should probably have one SAB here with a single slot - for locking a per-file initialization step and then allocate a - separate SAB like the above one for each file. That will - require a bit of acrobatics but should be feasible. The most - problematic part is that xOpen() would have to use - postMessage() to communicate its SharedArrayBuffer, and mixing - that approach with Atomics.wait/notify() gets a bit messy. - */ - const state = Object.create(null); state.verbose = options.verbose; state.littleEndian = (()=>{ const buffer = new ArrayBuffer(2); @@ -341,7 +305,6 @@ const installOpfsVfs = function callee(options){ + state.sabS11nSize/* argument/result serialization block */ ); state.opIds = Object.create(null); - const metrics = Object.create(null); { /* Indexes for use in our SharedArrayBuffer... */ let i = 0; @@ -379,7 +342,7 @@ const installOpfsVfs = function callee(options){ state.sabOP = new SharedArrayBuffer( i * 4/* ==sizeof int32, noting that Atomics.wait() and friends can only function on Int32Array views of an SAB. */); - opfsUtil.metrics.reset(); + vfsMetrics.reset(); } /** SQLITE_xxx constants to export to the async worker @@ -495,6 +458,7 @@ const installOpfsVfs = function callee(options){ return rc; }; +//#if nope /** Not part of the public API. Only for test/development use. */ @@ -508,35 +472,7 @@ const installOpfsVfs = function callee(options){ W.postMessage({type: 'opfs-async-restart'}); } }; - -//#define opfs-has-metrics -//#include api/opfs-common.c-pp.js - - /** - Generates a random ASCII string len characters long, intended for - use as a temporary file name. - */ - const randomFilename = function f(len=16){ - if(!f._chars){ - f._chars = "abcdefghijklmnopqrstuvwxyz"+ - "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+ - "012346789"; - f._n = f._chars.length; - } - const a = []; - let i = 0; - for( ; i < len; ++i){ - const ndx = Math.random() * (f._n * 64) % f._n | 0; - a[i] = f._chars[ndx]; - } - return a.join(""); - /* - An alternative impl. with an unpredictable length - but much simpler: - - Math.floor(Math.random() * Number.MAX_SAFE_INTEGER).toString(36) - */ - }; +//#endif /** Map of sqlite3_file pointers to objects constructed by xOpen(). @@ -743,7 +679,7 @@ const installOpfsVfs = function callee(options){ mTimeStart('xOpen'); let opfsFlags = 0; if(0===zName){ - zName = randomFilename(); + zName = opfsUtil.randomFilename(); }else if(wasm.isPtr(zName)){ if(capi.sqlite3_uri_boolean(zName, "opfs-unlock-asap", 0)){ /* -----------------------^^^^^ MUST pass the untranslated @@ -805,322 +741,6 @@ const installOpfsVfs = function callee(options){ }; } - /** - Expects an OPFS file path. It gets resolved, such that ".." - components are properly expanded, and returned. If the 2nd arg - is true, the result is returned as an array of path elements, - else an absolute path string is returned. - */ - opfsUtil.getResolvedPath = function(filename,splitIt){ - const p = new URL(filename, "file://irrelevant").pathname; - return splitIt ? p.split('/').filter((v)=>!!v) : p; - }; - - /** - Takes the absolute path to a filesystem element. Returns an - array of [handleOfContainingDir, filename]. If the 2nd argument - is truthy then each directory element leading to the file is - created along the way. Throws if any creation or resolution - fails. - */ - opfsUtil.getDirForFilename = async function f(absFilename, createDirs = false){ - const path = opfsUtil.getResolvedPath(absFilename, true); - const filename = path.pop(); - let dh = opfsUtil.rootDirectory; - for(const dirName of path){ - if(dirName){ - dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs}); - } - } - return [dh, filename]; - }; - - /** - Creates the given directory name, recursively, in - the OPFS filesystem. Returns true if it succeeds or the - directory already exists, else false. - */ - opfsUtil.mkdir = async function(absDirName){ - try { - await opfsUtil.getDirForFilename(absDirName+"/filepart", true); - return true; - }catch(e){ - //sqlite3.config.warn("mkdir(",absDirName,") failed:",e); - return false; - } - }; - /** - Checks whether the given OPFS filesystem entry exists, - returning true if it does, false if it doesn't or if an - exception is intercepted while trying to make the - determination. - */ - opfsUtil.entryExists = async function(fsEntryName){ - try { - const [dh, fn] = await opfsUtil.getDirForFilename(fsEntryName); - await dh.getFileHandle(fn); - return true; - }catch(e){ - return false; - } - }; - - /** - Generates a random ASCII string, intended for use as a - temporary file name. Its argument is the length of the string, - defaulting to 16. - */ - opfsUtil.randomFilename = randomFilename; - - /** - Returns a promise which resolves to an object which represents - all files and directories in the OPFS tree. The top-most object - has two properties: `dirs` is an array of directory entries - (described below) and `files` is a list of file names for all - files in that directory. - - Traversal starts at sqlite3.opfs.rootDirectory. - - Each `dirs` entry is an object in this form: - - ``` - { name: directoryName, - dirs: [...subdirs], - files: [...file names] - } - ``` - - The `files` and `subdirs` entries are always set but may be - empty arrays. - - The returned object has the same structure but its `name` is - an empty string. All returned objects are created with - Object.create(null), so have no prototype. - - Design note: the entries do not contain more information, - e.g. file sizes, because getting such info is not only - expensive but is subject to locking-related errors. - */ - opfsUtil.treeList = async function(){ - const doDir = async function callee(dirHandle,tgt){ - tgt.name = dirHandle.name; - tgt.dirs = []; - tgt.files = []; - for await (const handle of dirHandle.values()){ - if('directory' === handle.kind){ - const subDir = Object.create(null); - tgt.dirs.push(subDir); - await callee(handle, subDir); - }else{ - tgt.files.push(handle.name); - } - } - }; - const root = Object.create(null); - await doDir(opfsUtil.rootDirectory, root); - return root; - }; - - /** - Irrevocably deletes _all_ files in the current origin's OPFS. - Obviously, this must be used with great caution. It may throw - an exception if removal of anything fails (e.g. a file is - locked), but the precise conditions under which the underlying - APIs will throw are not documented (so we cannot tell you what - they are). - */ - opfsUtil.rmfr = async function(){ - const dir = opfsUtil.rootDirectory, opt = {recurse: true}; - for await (const handle of dir.values()){ - dir.removeEntry(handle.name, opt); - } - }; - - /** - Deletes the given OPFS filesystem entry. As this environment - has no notion of "current directory", the given name must be an - absolute path. If the 2nd argument is truthy, deletion is - recursive (use with caution!). - - The returned Promise resolves to true if the deletion was - successful, else false (but...). The OPFS API reports the - reason for the failure only in human-readable form, not - exceptions which can be type-checked to determine the - failure. Because of that... - - If the final argument is truthy then this function will - propagate any exception on error, rather than returning false. - */ - opfsUtil.unlink = async function(fsEntryName, recursive = false, - throwOnError = false){ - try { - const [hDir, filenamePart] = - await opfsUtil.getDirForFilename(fsEntryName, false); - await hDir.removeEntry(filenamePart, {recursive}); - return true; - }catch(e){ - if(throwOnError){ - throw new Error("unlink(",arguments[0],") failed: "+e.message,{ - cause: e - }); - } - return false; - } - }; - - /** - Traverses the OPFS filesystem, calling a callback for each - entry. The argument may be either a callback function or an - options object with any of the following properties: - - - `callback`: function which gets called for each filesystem - entry. It gets passed 3 arguments: 1) the - FileSystemFileHandle or FileSystemDirectoryHandle of each - entry (noting that both are instanceof FileSystemHandle). 2) - the FileSystemDirectoryHandle of the parent directory. 3) the - current depth level, with 0 being at the top of the tree - relative to the starting directory. If the callback returns a - literal false, as opposed to any other falsy value, traversal - stops without an error. Any exceptions it throws are - propagated. Results are undefined if the callback manipulate - the filesystem (e.g. removing or adding entries) because the - how OPFS iterators behave in the face of such changes is - undocumented. - - - `recursive` [bool=true]: specifies whether to recurse into - subdirectories or not. Whether recursion is depth-first or - breadth-first is unspecified! - - - `directory` [FileSystemDirectoryEntry=sqlite3.opfs.rootDirectory] - specifies the starting directory. - - If this function is passed a function, it is assumed to be the - callback. - - Returns a promise because it has to (by virtue of being async) - but that promise has no specific meaning: the traversal it - performs is synchronous. The promise must be used to catch any - exceptions propagated by the callback, however. - */ - opfsUtil.traverse = async function(opt){ - const defaultOpt = { - recursive: true, - directory: opfsUtil.rootDirectory - }; - if('function'===typeof opt){ - opt = {callback:opt}; - } - opt = Object.assign(defaultOpt, opt||{}); - const doDir = async function callee(dirHandle, depth){ - for await (const handle of dirHandle.values()){ - if(false === opt.callback(handle, dirHandle, depth)) return false; - else if(opt.recursive && 'directory' === handle.kind){ - if(false === await callee(handle, depth + 1)) break; - } - } - }; - doDir(opt.directory, 0); - }; - - /** - impl of importDb() when it's given a function as its second - argument. - */ - const importDbChunked = async function(filename, callback){ - const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true); - const hFile = await hDir.getFileHandle(fnamePart, {create:true}); - let sah = await hFile.createSyncAccessHandle(); - let nWrote = 0, chunk, checkedHeader = false, err = false; - try{ - sah.truncate(0); - while( undefined !== (chunk = await callback()) ){ - if(chunk instanceof ArrayBuffer) chunk = new Uint8Array(chunk); - if( !checkedHeader && 0===nWrote && chunk.byteLength>=15 ){ - util.affirmDbHeader(chunk); - checkedHeader = true; - } - sah.write(chunk, {at: nWrote}); - nWrote += chunk.byteLength; - } - if( nWrote < 512 || 0!==nWrote % 512 ){ - toss("Input size",nWrote,"is not correct for an SQLite database."); - } - if( !checkedHeader ){ - const header = new Uint8Array(20); - sah.read( header, {at: 0} ); - util.affirmDbHeader( header ); - } - sah.write(new Uint8Array([1,1]), {at: 18}/*force db out of WAL mode*/); - return nWrote; - }catch(e){ - await sah.close(); - sah = undefined; - await hDir.removeEntry( fnamePart ).catch(()=>{}); - throw e; - }finally { - if( sah ) await sah.close(); - } - }; - - /** - Asynchronously imports the given bytes (a byte array or - ArrayBuffer) into the given database file. - - Results are undefined if the given db name refers to an opened - db. - - If passed a function for its second argument, its behaviour - changes: imports its data in chunks fed to it by the given - callback function. It calls the callback (which may be async) - repeatedly, expecting either a Uint8Array or ArrayBuffer (to - denote new input) or undefined (to denote EOF). For so long as - the callback continues to return non-undefined, it will append - incoming data to the given VFS-hosted database file. When - called this way, the resolved value of the returned Promise is - the number of bytes written to the target file. - - It very specifically requires the input to be an SQLite3 - database and throws if that's not the case. It does so in - order to prevent this function from taking on a larger scope - than it is specifically intended to. i.e. we do not want it to - become a convenience for importing arbitrary files into OPFS. - - This routine rewrites the database header bytes in the output - file (not the input array) to force disabling of WAL mode. - - On error this throws and the state of the input file is - undefined (it depends on where the exception was triggered). - - On success, resolves to the number of bytes written. - */ - opfsUtil.importDb = async function(filename, bytes){ - if( bytes instanceof Function ){ - return importDbChunked(filename, bytes); - } - if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes); - util.affirmIsDb(bytes); - const n = bytes.byteLength; - const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true); - let sah, err, nWrote = 0; - try { - const hFile = await hDir.getFileHandle(fnamePart, {create:true}); - sah = await hFile.createSyncAccessHandle(); - sah.truncate(0); - nWrote = sah.write(bytes, {at: 0}); - if(nWrote != n){ - toss("Expected to write "+n+" bytes but wrote "+nWrote+"."); - } - sah.write(new Uint8Array([1,1]), {at: 18}) /* force db out of WAL mode */; - return nWrote; - }catch(e){ - if( sah ){ await sah.close(); sah = undefined; } - await hDir.removeEntry( fnamePart ).catch(()=>{}); - throw e; - }finally{ - if( sah ) await sah.close(); - } - }; - if(sqlite3.oo1){ const OpfsDb = function(...args){ const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args); @@ -1241,12 +861,10 @@ const installOpfsVfs = function callee(options){ warn("Running sanity checks because of opfs-sanity-check URL arg..."); sanityCheck(); } - if(thisThreadHasOPFS()){ - navigator.storage.getDirectory().then((d)=>{ + if(opfsUtil.thisThreadHasOPFS()){ + opfsUtil.getRootDir().then((d)=>{ W.onerror = W._originalOnError; delete W._originalOnError; - sqlite3.opfs = opfsUtil; - opfsUtil.rootDirectory = d; log("End of OPFS sqlite3_vfs setup.", opfsVfs); promiseResolve(); }).catch(promiseReject); diff --git a/manifest b/manifest index 037a053c93..f334f15c5a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C A\spotential\sfix\sfor\sthe\sprobable\sbreakage\sof\sbundler-friendly\sbuilds\sin\sthe\sprevious\scheck-in.\sPending\sreview\sfrom\ssomeone\swho\suses\sthose\stools. -D 2026-03-04T00:48:03.758 +C Consolidate\smuch\sof\sthe\sOPFS\sutility\scode\sinto\sa\snew\sfile\sfor\suse\sby\stwo\sof\sthe\sOPFS\sVFSes. +D 2026-03-04T11:37:39.256 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -572,7 +572,7 @@ F ext/session/sessionwor.test 6fd9a2256442cebde5b2284936ae9e0d54bde692d0f5fd009e F ext/session/sqlite3session.c 6ebd02be470f36d41c4bd78927f39d507b62051ba025eacaed9936c769902a07 F ext/session/sqlite3session.h 7404723606074fcb2afdc6b72c206072cdb2b7d8ba097ca1559174a80bc26f7a F ext/session/test_session.c 190110e3bd9463717248dec1272b44fe9943e57b7646d0b4200dcf11e4dccee6 -F ext/wasm/GNUmakefile a0afdb3e6306997de1e123e8b8270affb744519c9e384dbc277d0af6a6ba1c91 +F ext/wasm/GNUmakefile 4496669ca5785c7bfc7af76565f1dd45e8f9ec7529b58c485fd374088162bef1 F ext/wasm/README-dist.txt f01081a850ce38a56706af6b481e3a7878e24e42b314cfcd4b129f0f8427066a F ext/wasm/README.md 2e87804e12c98f1d194b7a06162a88441d33bb443efcfe00dc6565a780d2f259 F ext/wasm/SQLTester/GNUmakefile e0794f676d55819951bbfae45cc5e8d7818dc460492dc317ce7f0d2eca15caff @@ -584,7 +584,8 @@ F ext/wasm/api/EXPORTED_FUNCTIONS.c-pp 7ba933e8f1290cc65459dd371c0c9a031d96bdf14 F ext/wasm/api/README.md a905d5c6bfc3e2df875bd391d6d6b7b48d41b43bdee02ad115b47244781a7e81 F ext/wasm/api/extern-post-js.c-pp.js d9f42ecbedc784c0d086bc37800e52946a14f7a21600b291daa3f963c314f930 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 -F ext/wasm/api/opfs-common.c-pp.js 5540c58aee8fbbf2c67984e0a35e2b680a7603052f2cc442d407da3cc81c4819 +F ext/wasm/api/opfs-common-inline.c-pp.js b9c4e080698792cbc04ce9dd9dda7d8316c6db0262f74820706f98b352b949d5 w ext/wasm/api/opfs-common.c-pp.js +F ext/wasm/api/opfs-common-shared.c-pp.js 3f8f3f2ab4790fdd1e6d1d9224e232cef07f1c8753827c18bbba965dbe98795f F ext/wasm/api/post-js-footer.js a50c1a2c4d008aede7b2aa1f18891a7ee71437c2f415b8aeb3db237ddce2935b F ext/wasm/api/post-js-header.js f35d2dcf1ab7f22a93d565f8e0b622a2934fc4e743edf3b708e4dd8140eeff55 F ext/wasm/api/pre-js.c-pp.js 9234ea680a2f6a2a177e8dcd934bdc5811a9f8409165433a252b87f4c07bba6f @@ -593,12 +594,12 @@ F ext/wasm/api/sqlite3-api-oo1.c-pp.js 45454631265d9ce82685f1a64e1650ee19c8e121c F ext/wasm/api/sqlite3-api-prologue.js ccd8ece4b4580d2a70996218f28e810d70a86f5e2795f4d4a75f0603af24aef6 F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667430f9d45167569633ffc7567938 F ext/wasm/api/sqlite3-license-version-header.js 98d90255a12d02214db634e041c8e7f2f133d9361a8ebf000ba9c9af4c6761cc -F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js 041f24cd61d470072303b0cd1bb73dd2928bf2c96c0fbf3d3f4a39024c4e9ae7 +F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js 25b856508ac94336419133c6ec10594f576b469f85cc69cde4c09cfa06a8e1c7 F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 2ccf4322f42063aefc150972943e750c77f7926b866f1639d40eec05df075b6e F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9 -F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js 8c79bf3ea89102772225e6539c030787ea85203b6cc0354c545df69d81a2d910 -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b2b4c7570ec816c49e7b3293f451992bd7fb7154dcd4cec6fb05d032c6cedf52 +F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js 54f71e563dda30af73ed84ff9de03441537b2e8fb8d2ae2a0b0c8187f51db67a +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 2e2a72a40e2ad6ea92f52eb7adbd925d4acd874ffeecaa00b85234ad49862655 F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81 F ext/wasm/api/sqlite3-wasm.c 45bb20e19b245136711f9b78584371233975811b6560c29ed9b650e225417e29 F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js aa9715f661fb700459a5a6cb1c32a4d6a770723b47aa9ac0e16c2cf87d622a66 @@ -2190,8 +2191,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P 9a471f7491a371052bf1785098ba966dd0d03503e7d8b9fbcd65f07b038e5021 -R 8aa8acc5e3e3610acfaa4dc12ceb0c5d +P 8ea85776116521526d684f221d67e288126e62931d4a0ea7fc7f164cd2d5b2ec +R 82203270087631af34878be94f4d1594 U stephan -Z dc633fdeffefdb99fdb4ea173110ea63 +Z 1548da853f7a79cab3a86616c1bf35cc # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3a2f818401..65aadb1877 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8ea85776116521526d684f221d67e288126e62931d4a0ea7fc7f164cd2d5b2ec +db19a6e9663c3a44996178cb8c35dc4ccd60f48cb4b81b6c214411a56c57def7