From 232446b532f6a88395bf1ca3583942a7a11ac59a Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 4 Mar 2026 20:33:21 +0000 Subject: [PATCH] Get opfs-wl plugged in to the concurrency tester. Somewhat ironically, all competing workers fail with locking errors while worker 1 is busy running off the rails somewhere. Stashing for closer investigation later. FossilOrigin-Name: 3b27310aa29ea84f459974981a600301abac5c705029a289d3872ecacf231da3 --- ext/wasm/api/opfs-common-shared.c-pp.js | 67 +++++++++++++---------- ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js | 3 + ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js | 5 +- ext/wasm/tests/opfs/concurrency/test.js | 12 +++- ext/wasm/tests/opfs/concurrency/worker.js | 16 +++++- manifest | 20 +++---- manifest.uuid | 2 +- 7 files changed, 77 insertions(+), 48 deletions(-) diff --git a/ext/wasm/api/opfs-common-shared.c-pp.js b/ext/wasm/api/opfs-common-shared.c-pp.js index 5150fb8e61..72c366adab 100644 --- a/ext/wasm/api/opfs-common-shared.c-pp.js +++ b/ext/wasm/api/opfs-common-shared.c-pp.js @@ -499,6 +499,20 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const state = util.nu(); state.verbose = options.verbose; + const loggers = [ + sqlite3.config.error, + sqlite3.config.warn, + sqlite3.config.log + ]; + const logImpl = (level,...args)=>{ + if(options.verbose>level) loggers[level]("OPFS syncer:",...args); + }; + const log = (...args)=>logImpl(2, ...args), + warn = (...args)=>logImpl(1, ...args), + error = (...args)=>logImpl(0, ...args), + capi = sqlite3.capi, + wasm = sqlite3.wasm; + const opfsVfs = state.vfs = new capi.sqlite3_vfs(); const opfsIoMethods = opfsVfs.ioMethods = new capi.sqlite3_io_methods(); @@ -571,7 +585,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ of this value is also used for determining how long to wait on lock contention to free up. */ - state.asyncIdleWaitTime = isWebLocker ? 100 : 150; + state.asyncIdleWaitTime = isWebLocker ? 300 : 150; /** Whether the async counterpart should log exceptions to @@ -966,15 +980,17 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ convey error messages from xOpen() because there would be a race condition between sqlite3_open()'s call to xOpen() and this function. */ - warn("OPFS xGetLastError() has nothing sensible to return."); + sqlite3.config.warn("OPFS xGetLastError() has nothing sensible to return."); return 0; }, //xSleep is optionally defined below xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){ mTimeStart('xOpen'); let opfsFlags = 0; - if(0===zName){ - zName = opfsUtil.randomFilename(); + let jzName, zToFree; + if( !zName ){ + jzName = opfsUtil.randomFilename(); + zName = zToFree = wasm.allocCString(jzName); }else if(wasm.isPtr(zName)){ if(capi.sqlite3_uri_boolean(zName, "opfs-unlock-asap", 0)){ /* -----------------------^^^^^ MUST pass the untranslated @@ -984,18 +1000,24 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ if(capi.sqlite3_uri_boolean(zName, "delete-before-open", 0)){ opfsFlags |= state.opfsFlags.OPFS_UNLINK_BEFORE_OPEN; } - zName = wasm.cstrToJs(zName); - //warn("xOpen zName =",zName, "opfsFlags =",opfsFlags); + jzName = wasm.cstrToJs(zName); + //sqlite3.config.warn("xOpen zName =",zName, "opfsFlags =",opfsFlags); + }else{ + sqlite3.config.error("Impossible zName value in xOpen?", zName); + return capi.SQLITE_CANTOPEN; } - const fh = Object.create(null); - fh.fid = pFile; - fh.filename = wasm.cstrToJs(zName); - fh.sab = new SharedArrayBuffer(state.fileBufferSize); - fh.flags = flags; - fh.readOnly = !(capi.SQLITE_OPEN_CREATE & flags) - && !!(flags & capi.SQLITE_OPEN_READONLY); - const rc = opRun('xOpen', pFile, zName, flags, opfsFlags); - if(!rc){ + const fh = util.nu({ + fid: pFile, + filename: jzName, + sab: new SharedArrayBuffer(state.fileBufferSize), + flags: flags, + readOnly: !(capi.SQLITE_OPEN_CREATE & flags) + && !!(flags & capi.SQLITE_OPEN_READONLY) + }); + const rc = opRun('xOpen', pFile, jzName, flags, opfsFlags); + if(rc){ + if( zToFree ) wasm.dealloc(zToFree); + }else{ /* Recall that sqlite3_vfs::xClose() will be called, even on error, unless pFile->pMethods is NULL. */ if(fh.readOnly){ @@ -1004,6 +1026,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ __openFiles[pFile] = fh; fh.sabView = state.sabFileBufView; fh.sq3File = new capi.sqlite3_file(pFile); + if( zToFree ) fh.sq3File.addOnDispose(zToFree); fh.sq3File.$pMethods = opfsIoMethods.pointer; fh.lockType = capi.SQLITE_LOCK_NONE; } @@ -1060,20 +1083,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ opfsVfs.doTheThing = function(ioMethods, callback){ Object.assign(opfsVfs.ioSyncWrappers, ioMethods); const thePromise = new Promise(function(promiseResolve_, promiseReject_){ - const loggers = [ - sqlite3.config.error, - sqlite3.config.warn, - sqlite3.config.log - ]; - const logImpl = (level,...args)=>{ - if(options.verbose>level) loggers[level]("OPFS syncer:",...args); - }; - const log = (...args)=>logImpl(2, ...args), - warn = (...args)=>logImpl(1, ...args), - error = (...args)=>logImpl(0, ...args), - capi = sqlite3.capi, - wasm = sqlite3.wasm; - let promiseWasRejected = undefined; const promiseReject = (err)=>{ promiseWasRejected = true; diff --git a/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js index e3fb722877..0fdd7cb4cc 100644 --- a/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js @@ -1137,9 +1137,12 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }/*.ioJrnl*/ }/*methodOverrides*/; +//#if nope debug("pVfs and friends", pVfs, pIoDb, pIoJrnl, kvvfsMethods, capi.sqlite3_file.structInfo, KVVfsFile.structInfo); +//#endif + try { util.assert( cache.buffer.n>1024*129, "Heap buffer is not large enough" /* Native is SQLITE_KVOS_SZ is 133073 as of this writing */ ); diff --git a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js index e62f23ed14..4b2094ee81 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js @@ -60,6 +60,7 @@ const installOpfsWlVfs = async function callee(options){ options = opfsUtil.initOptions(options, callee); if( !options ) return sqlite3; const capi = sqlite3.capi, + debug = sqlite3.config.debug, state = opfsUtil.createVfsState('opfs-wl', options), opfsVfs = state.vfs, metrics = opfsVfs.metrics.counters, @@ -94,7 +95,7 @@ const installOpfsWlVfs = async function callee(options){ } f.lockType = lockType; }catch(e){ - error("xLock(",arguments,") failed", e, f); + sqlite3.config.error("xLock(",arguments,") failed", e, f); rc = capi.SQLITE_IOERR_LOCK; } } @@ -113,7 +114,7 @@ const installOpfsWlVfs = async function callee(options){ Atomics.notify(view, state.lock.atomicsHandshake); Atomics.wait(view, state.lock.atomicsHandshake, 1); }catch(e){ - error("xUnlock(",pFile,lockType,") failed",e, f); + sqlite3.config.error("xUnlock(",pFile,lockType,") failed",e, f); rc = capi.SQLITE_IOERR_LOCK; } } diff --git a/ext/wasm/tests/opfs/concurrency/test.js b/ext/wasm/tests/opfs/concurrency/test.js index 1848901afe..c2932290b7 100644 --- a/ext/wasm/tests/opfs/concurrency/test.js +++ b/ext/wasm/tests/opfs/concurrency/test.js @@ -55,8 +55,8 @@ const options = Object.create(null); options.sqlite3Dir = urlArgsJs.get('sqlite3.dir'); options.workerCount = ( - urlArgsHtml.has('workers') ? +urlArgsHtml.get('workers') : 3 - ) || 4; + urlArgsHtml.has('workers') ? +urlArgsHtml.get('workers') : 0 + ) || 3; options.opfsVerbose = ( urlArgsHtml.has('verbose') ? +urlArgsHtml.get('verbose') : 1 ) || 1; @@ -69,6 +69,7 @@ options.unlockAsap = ( urlArgsHtml.has('unlock-asap') ? +urlArgsHtml.get('unlock-asap') : 0 ) || 0; + options.vfs = urlArgsHtml.get('vfs') || 'opfs'; options.noUnlink = !!urlArgsHtml.has('no-unlink'); const workers = []; workers.post = (type,...args)=>{ @@ -118,12 +119,16 @@ const eTestLinks = document.querySelector('#testlinks'); const optArgs = function(obj){ const li = []; - for(const k of ['interval','iterations','workers','verbose','unlock-asap']){ + for(const k of [ + 'interval','iterations','unlock-asap', + 'verbose','vfs','workers', + ]){ if( obj.hasOwnProperty(k) ) li.push(k+'='+obj[k]); } return li.join('&'); }; for(const opt of [ + {interval: 500, workers: 3, iterations: 30, vfs: 'opfs-wl'}, {interval: 1000, workers: 5, iterations: 30}, {interval: 500, workers: 5, iterations: 30}, {interval: 250, workers: 3, iterations: 30}, @@ -142,6 +147,7 @@ workers.uri = ( 'worker.js?' + 'sqlite3.dir='+options.sqlite3Dir + + '&vfs='+options.vfs + '&interval='+options.interval + '&iterations='+options.iterations + '&opfs-verbose='+options.opfsVerbose diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index a28d80f202..f68bf3516b 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -6,7 +6,8 @@ globalThis.sqlite3InitModule().then(async function(sqlite3){ const urlArgs = new URL(globalThis.location.href).searchParams; const options = { workerName: urlArgs.get('workerId') || Math.round(Math.random()*10000), - unlockAsap: urlArgs.get('opfs-unlock-asap') || 0 /*EXPERIMENTAL*/ + unlockAsap: urlArgs.get('opfs-unlock-asap') || 0 /*EXPERIMENTAL*/, + vfs: urlArgs.get('vfs') }; const wPost = (type,...payload)=>{ postMessage({type, worker: options.workerName, payload}); @@ -14,7 +15,7 @@ globalThis.sqlite3InitModule().then(async function(sqlite3){ const stdout = (...args)=>wPost('stdout',...args); const stderr = (...args)=>wPost('stderr',...args); if(!sqlite3.opfs){ - stderr("OPFS support not detected. Aborting."); + stderr("This code requires the (private) sqlite3.opfs object. Aborting."); return; } @@ -47,7 +48,16 @@ globalThis.sqlite3InitModule().then(async function(sqlite3){ } }; const run = async function(){ - db = new sqlite3.oo1.OpfsDb({ + const Ctors = Object.assign(Object.create(null),{ + opfs: sqlite3.oo1.OpfsDb, + 'opfs-wl': sqlite3.oo1.OpfsWlDb + }); + const ctor = Ctors[options.vfs]; + if( !ctor ){ + stderr("Invalid VFS name:",vfs); + return; + } + db = new ctor({ filename: 'file:'+dbName+'?opfs-unlock-asap='+options.unlockAsap, flags: 'c' }); diff --git a/manifest b/manifest index 003a670b2e..fa51c7fc6c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Consolidate\sthe\slast\s200\slines\sof\scommon\sOPFS\sVFS\scode.\s"opfs"\sstill\sworks,\s"opfs-wl"\sregisters\sfine\sbut\sis\sstill\sotherwise\suntested. -D 2026-03-04T19:21:09.278 +C Get\sopfs-wl\splugged\sin\sto\sthe\sconcurrency\stester.\sSomewhat\sironically,\sall\scompeting\sworkers\sfail\swith\slocking\serrors\swhile\sworker\s1\sis\sbusy\srunning\soff\sthe\srails\ssomewhere.\sStashing\sfor\scloser\sinvestigation\slater. +D 2026-03-04T20:33:21.840 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -585,7 +585,7 @@ F ext/wasm/api/README.md a905d5c6bfc3e2df875bd391d6d6b7b48d41b43bdee02ad115b4724 F ext/wasm/api/extern-post-js.c-pp.js d9f42ecbedc784c0d086bc37800e52946a14f7a21600b291daa3f963c314f930 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/opfs-common-inline.c-pp.js 5be8d6d91963849e218221b48206ae55612630bb2cd7f30b1b6fcf7a9e374b76 -F ext/wasm/api/opfs-common-shared.c-pp.js d8ecb1c7f6b29c2eb501ab8da6f9d9867c1ceb8a42c9c883dd53aed8ddfe106a +F ext/wasm/api/opfs-common-shared.c-pp.js 1218fb9e72f3a208e62a4caafcf1fb0a41b66ca46df27601e5bfb06220822f24 F ext/wasm/api/post-js-footer.js a50c1a2c4d008aede7b2aa1f18891a7ee71437c2f415b8aeb3db237ddce2935b F ext/wasm/api/post-js-header.js f35d2dcf1ab7f22a93d565f8e0b622a2934fc4e743edf3b708e4dd8140eeff55 F ext/wasm/api/pre-js.c-pp.js 9234ea680a2f6a2a177e8dcd934bdc5811a9f8409165433a252b87f4c07bba6f @@ -596,9 +596,9 @@ F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667 F ext/wasm/api/sqlite3-license-version-header.js 98d90255a12d02214db634e041c8e7f2f133d9361a8ebf000ba9c9af4c6761cc F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js f0a2aa8712211ff9db2ef548ae8b676be3e7c82f61586d03fd8317fbc95bbedd F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d -F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 2ccf4322f42063aefc150972943e750c77f7926b866f1639d40eec05df075b6e +F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js a61dd2b4d919d2d5d83c5c7e49b89ecbff2525ff81419f6a6dbaecaf3819c490 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9 -F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js a755ea941f254f89fcd519789097a7401362d9e9dfba19a9bfc972861257c3c5 +F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js 62d41024ad20c388c022f032d3f25838809cb00915ebeb97b598b9f370fba2c3 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 50a955ef393722d498177ad09c9e2d05bbe8dccae4c40c501482a860ca30017d F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81 F ext/wasm/api/sqlite3-wasm.c 45bb20e19b245136711f9b78584371233975811b6560c29ed9b650e225417e29 @@ -646,8 +646,8 @@ F ext/wasm/tester1-worker.c-pp.html d0032241d0b24d996cf1c4dd0dde364189693af9b5c9 F ext/wasm/tester1.c-pp.html 52d88fe2c6f21a046030a36410b4839b632f4424028197a45a3d5669ea724ddb F ext/wasm/tester1.c-pp.js 6b946cd6d4da130dbae4a401057716d27117ca02cad2ea8c29ae9c46c675d618 F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e -F ext/wasm/tests/opfs/concurrency/test.js d08889a5bb6e61937d0b8cbb78c9efbefbf65ad09f510589c779b7cc6a803a88 -F ext/wasm/tests/opfs/concurrency/worker.js 3e29c07bdd11f4ff9ee50e641e39c254d39243bbb83fb20a255755ee92739d12 +F ext/wasm/tests/opfs/concurrency/test.js 2150850176a1ffbe7f6b19f5cf5db9c77290764d6d54430b2796b6512b676a4a +F ext/wasm/tests/opfs/concurrency/worker.js 537e11f86fdc5d5bc64032aaa48cd027a2f8cf4b020aca6c3c012fec85da2eb1 F ext/wasm/tests/opfs/sahpool/digest-worker.js b0ab6218588f1f0a6d15a363b493ceaf29bfb87804d9e0165915a9996377cf79 F ext/wasm/tests/opfs/sahpool/digest.html 206d08a34dc8bd570b2581d3d9ab3ecad3201b516a598dd096dcf3cf8cd81df8 F ext/wasm/tests/opfs/sahpool/index.html be736567fd92d3ecb9754c145755037cbbd2bca01385e2732294b53f4c842328 @@ -2191,8 +2191,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P 57adecbab71795b62b1c2e4570ff504f35681e81dd8c94f78ad8e05ef39d36fd -R 18d89b2ee7fb09440d0eac4e9c8dc540 +P 5978ee4902e4223fed6b95bd2d8f489bb300af8b762650e7113d1f3e97519d88 +R 0cf8357d9b35fb1b75ea83762f01c38b U stephan -Z d66223f806eab9c2d766ad2e9d73c95d +Z eba11c8254e99ee26ae9feba85ef81c4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5c99fbd65b..15d2b8cd82 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5978ee4902e4223fed6b95bd2d8f489bb300af8b762650e7113d1f3e97519d88 +3b27310aa29ea84f459974981a600301abac5c705029a289d3872ecacf231da3 -- 2.47.3