From: stephan Date: Sat, 7 Mar 2026 02:19:23 +0000 (+0000) Subject: Cleanups and docs in the new opfs code structure. X-Git-Tag: major-release~100^2~6 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=9be458e61134fc0fb3a442b7c028cc12fc3b96d9;p=thirdparty%2Fsqlite.git Cleanups and docs in the new opfs code structure. FossilOrigin-Name: 3b470c4c7a1fcc710e6b9eae32134c7f6c3f6008b24c7351257f66f5e8f70311 --- diff --git a/ext/wasm/api/opfs-common-shared.c-pp.js b/ext/wasm/api/opfs-common-shared.c-pp.js index 17916426ef..c0b028160e 100644 --- a/ext/wasm/api/opfs-common-shared.c-pp.js +++ b/ext/wasm/api/opfs-common-shared.c-pp.js @@ -14,7 +14,10 @@ This file holds code shared by sqlite3-vfs-opfs{,-wl}.c-pp.js. It creates a private/internal sqlite3.opfs namespace common to the two and used (only) by them and the test framework. It is not part of - the public API. + the public API. The library deletes sqlite3.opfs in its final + bootstrapping steps unless it's specifically told to keep them (for + testing purposes only) using an undocumented and unsupported + mechanism. */ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ 'use strict'; @@ -461,49 +464,36 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ? (+urlParams.get('opfs-verbose') || 2) : 1; options.sanityChecks ??= urlParams.has('opfs-sanity-check'); - if( true ){ - /* Doing this from one scope up does not work */ - opfsUtil.proxyUri = "sqlite3-opfs-async-proxy.js"; - if( sqlite3.scriptInfo?.sqlite3Dir ){ - opfsUtil.proxyUri = ( - sqlite3.scriptInfo.sqlite3Dir + opfsUtil.proxyUri - ); - } - //sqlite3.config.error("proxyUri =",opfsUtil.proxyUri, sqlite3.scriptInfo); + opfsUtil.proxyUri ??= "sqlite3-opfs-async-proxy.js"; + if( sqlite3.scriptInfo?.sqlite3Dir ){ + /* Doing this from one scope up, outside of this function, does + not work. */ + opfsUtil.proxyUri = ( + sqlite3.scriptInfo.sqlite3Dir + opfsUtil.proxyUri + ); } - options.proxyUri ??= opfsUtil.proxyUri; if('function' === typeof options.proxyUri){ options.proxyUri = options.proxyUri(); } - if( false ){ - /* This ends up with the same values for all Worker instances. */ - callee.counter ??= 0; - ++callee.counter; - options.workerId ??= urlParams.get('opfs-async-proxy-id') ?? callee.counter; - }else{ - options.workerId ??= (Math.random() * 10000000) | 0; - } //sqlite3.config.warn("opfsUtil options =",JSON.stringify(options), 'urlParams =', urlParams); return opfsUtil.options = options; }; /** - Creates and populates the main state object used by "opfs" and "opfs-wl", and - transfered from those to their async counterpart. - - Returns an object containing state which we send to the async - proxy Worker. + Creates, populates, and returns the main state object used by the + "opfs" and "opfs-wl" VFSes, and transfered from those to their + async counterparts. The returned object's vfs property holds the fully-populated - capi.sqlite3_vfs instance. + capi.sqlite3_vfs instance, tagged with lots of extra state which + the current VFSes need to have exposed to them. - After setting up any local state needed, the caller must - call theVfs.bindVfs(X,Y), where X is an object containing - the sqlite3_io_methods to override and Y is a callback which - gets triggered if init succeeds, before the final Promise - decides whether or not to reject. The result of bindVfs() - must be returned from their main installation function. + After setting up any local state needed, the caller must call + theVfs.bindVfs(X,Y), where X is an object containing the + sqlite3_io_methods to override and Y is a callback which gets + triggered if init succeeds, before the final Promise decides + whether or not to reject. This object must, when it's passed to the async part, contain only cloneable or sharable objects. After the worker's "inited" @@ -1105,9 +1095,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ let proxyUri = options.proxyUri +( (options.proxyUri.indexOf('?')<0) ? '?' : '&' )+'vfs='+vfsName; - if( options.workerId ){ - proxyUri += '&opfs-async-proxy-id='+encodeURIComponent(options.workerId); - } //sqlite3.config.error("proxyUri",options.proxyUri, (new Error())); const W = opfsVfs.worker = //#if target:es6-bundler-friendly diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js index 6362c6efd4..d09dd2b350 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js @@ -55,8 +55,12 @@ const vfsName = urlParams.get('vfs'); if( !vfsName ){ throw new Error("Expecting vfs=opfs|opfs-wl URL argument for this worker"); } -const workerId = urlParams.get('opfs-async-proxy-id') - ?? 'unnamed'; +/** + We use this to allow us to differentiate debug output from + multiple instances, e.g. multiple Workers to the "opfs" + VFS or both the "opfs" and "opfs-wl" VFSes. +*/ +const workerId = (Math.random() * 10000000) | 0; const isWebLocker = true; //'opfs-wl'===urlParams.get('vfs'); const wPost = (type,...args)=>postMessage({type, payload:args}); const installAsyncProxy = function(){ @@ -287,10 +291,11 @@ const installAsyncProxy = function(){ In order to help alleviate cross-tab contention for a dabase, if an exception is thrown while acquiring the handle, this routine - will wait briefly and try again, up to some fixed number of - times. If acquisition still fails at that point it will give up - and propagate the exception. Client-level code will see that as - an I/O error. + will wait briefly and try again, up to `maxTries` of times. If + acquisition still fails at that point it will give up and + propagate the exception. Client-level code will see that either + as an I/O error or SQLITE_BUSY, depending on the exception and + the context. 2024-06-12: there is a rare race condition here which has been reported a single time: @@ -306,6 +311,12 @@ const installAsyncProxy = function(){ creating a viable test for that condition has proven challenging so far. + Interface quirk: if fh.xLock is falsy and the handle is acquired + then fh.fid is added to __implicitLocks(). If fh.xLock is truthy, + it is not added as an implicit lock. i.e. xLock() impls must set + fh.xLock immediately _before_ calling this and must arrange to + restore it to its previous value if this function throws. + 2026-03-06: - baseWaitTime is the number of milliseconds to wait for the @@ -314,12 +325,11 @@ const installAsyncProxy = function(){ - maxTries is the number of attempt to make, each one spaced out by one additional factor of the baseWaitTime (e.g. 300, then 600, - then 900, the 1200...). This MUST be an integer >0 and defaults - to 6. + then 900, the 1200...). This MUST be an integer >0. Only the Web Locks impl should use the 3rd and 4th parameters. */ - const getSyncHandle = async (fh,opName, baseWaitTime, maxTries)=>{ + const getSyncHandle = async (fh, opName, baseWaitTime, maxTries = 6)=>{ if(!fh.syncHandle){ const t = performance.now(); log("Acquiring sync handle for",fh.filenameAbs); @@ -610,8 +620,15 @@ const installAsyncProxy = function(){ if( isWebLocker ){ /* We require separate xLock() and xUnlock() implementations for the original and Web Lock implementations. The ones in this block - are for the WebLock impl. */ - + are for the WebLock impl. + + The Golden Rule for this impl is: if we have a web lock, we + must also hold the SAH. When "upgrading" an implicit lock to a + requested (explicit) lock, we must remove the SAH from the + __implicitLocks set. When we unlock, we release both the web + lock and the SAH. That invariant must be kept intact or race + conditions on SAHs will ensue. + */ /** Registry of active Web Locks: fid -> { mode, resolveRelease } */ const __activeWebLocks = Object.create(null); @@ -632,8 +649,7 @@ const installAsyncProxy = function(){ storeAndNotify(whichOp, 0); /* Don't do this: existing.mode = requestedMode; - Paraphrased from advice given by a consultanting - developer: + Paraphrased from advice given by a consulting developer: If you hold an exclusive lock and SQLite requests shared, you should keep exiting.mode as exclusive in because the @@ -648,10 +664,11 @@ const installAsyncProxy = function(){ Upgrade path: we must release shared and acquire exclusive. This transition is NOT atomic in Web Locks API. - Except that it _effectively_ is atomic if we don't call - closeSyncHandle(fh), as no other worker can lock that - until we let it go. But we can't do that without leading - to a deadly embrace, so... + It _effectively_ is atomic if we don't call + closeSyncHandle(fh), as no other worker can lock that until + we let it go. But we can't do that without eventually + leading to deadly embrace situations, so we don't do that. + (That's not a hypothetical, it has happened.) */ await closeSyncHandle(fh); existing.resolveRelease(); @@ -664,10 +681,10 @@ const installAsyncProxy = function(){ //error("xLock() initial promise entered..."); navigator.locks.request(lockName, { mode: requestedMode }, async (lock) => { //error("xLock() Web Lock entered.", fh); - fh.xLock = lockType/*must be set before getSyncHandle() is called!*/; __implicitLocks.delete(fid); let rc = 0; try{ + fh.xLock = lockType/*must be set before getSyncHandle() is called!*/; await getSyncHandle(fh, 'xLock', state.asyncIdleWaitTime, 5); }catch(e){ fh.xLock = oldLockType; diff --git a/manifest b/manifest index ecc54ab16c..c12974dd15 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C For\sbackwards\scompatibility,\sensure\sthat\sthe\s"opfs"\sVFS\sdoes\snot\sspecifically\srequire\sAtomics.waitAsync()\s(a\snew\srequirement\sof\s"opfs-wl"),\sbut\smake\suse\sof\sit\sif\savailable.\sOnly\sapply\sjitter\sto\sthe\sconcurrency\stest\sruns\sat\srandom\sintervals. -D 2026-03-07T01:01:13.846 +C Cleanups\sand\sdocs\sin\sthe\snew\sopfs\scode\sstructure. +D 2026-03-07T02:19:23.636 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -585,7 +585,7 @@ F ext/wasm/api/README.md a905d5c6bfc3e2df875bd391d6d6b7b48d41b43bdee02ad115b4724 F ext/wasm/api/extern-post-js.c-pp.js d9f42ecbedc784c0d086bc37800e52946a14f7a21600b291daa3f963c314f930 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/opfs-common-inline.c-pp.js 5be8d6d91963849e218221b48206ae55612630bb2cd7f30b1b6fcf7a9e374b76 -F ext/wasm/api/opfs-common-shared.c-pp.js 612df64a2a159db30786c742c70a09d75b993d034f360367125747d2eca531a5 +F ext/wasm/api/opfs-common-shared.c-pp.js 81a2429027b76df8691e6c134d3fedbf7cb804585ac28165b857ecb2260a99f3 F ext/wasm/api/post-js-footer.js a50c1a2c4d008aede7b2aa1f18891a7ee71437c2f415b8aeb3db237ddce2935b F ext/wasm/api/post-js-header.js f35d2dcf1ab7f22a93d565f8e0b622a2934fc4e743edf3b708e4dd8140eeff55 F ext/wasm/api/pre-js.c-pp.js 9234ea680a2f6a2a177e8dcd934bdc5811a9f8409165433a252b87f4c07bba6f @@ -594,7 +594,7 @@ F ext/wasm/api/sqlite3-api-oo1.c-pp.js 45454631265d9ce82685f1a64e1650ee19c8e121c F ext/wasm/api/sqlite3-api-prologue.js 98fedc159c9239b226d19567d7172300dee5ffce176e5fa2f62dd1f17d088385 F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667430f9d45167569633ffc7567938 F ext/wasm/api/sqlite3-license-version-header.js 98d90255a12d02214db634e041c8e7f2f133d9361a8ebf000ba9c9af4c6761cc -F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js 296b3d3701c3b42918357e4d9e5d82c5321b371115fb51a8a405c2eaeebd9f0f +F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js 5c84524885393b90b57d27446872fe980a48e8b9f5f2bb728e7ba63e905feb3f F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js a61dd2b4d919d2d5d83c5c7e49b89ecbff2525ff81419f6a6dbaecaf3819c490 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9 @@ -2191,8 +2191,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P a9aecc987512d60f2663973f43c769cf086fc14149edfbcb18c0aec9f5aa3dbf -R 825063f48a3ce6fa65e33b9852885bae +P f2175f526c00cfe562e8f332eb197b5ef2c3d6be1fff2aab1566c2c533a293ac +R 226e3a09fddd6452554266d606d70e1e U stephan -Z a8fffcaa59d459a1613276d07e6e1f6d +Z 2eb7dcbdcae5965bf20b0b5469f42aa6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a5dd212704..d013e9cb8f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f2175f526c00cfe562e8f332eb197b5ef2c3d6be1fff2aab1566c2c533a293ac +3b470c4c7a1fcc710e6b9eae32134c7f6c3f6008b24c7351257f66f5e8f70311