From: stephan Date: Sat, 7 Mar 2026 03:32:17 +0000 (+0000) Subject: Cleanups and docs in the opfs vfs and its concurrency tester. Experimentally add... X-Git-Tag: major-release~100^2~4 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=7d10030d75122cc26cf5e71b75b258835b3afd72;p=thirdparty%2Fsqlite.git Cleanups and docs in the opfs vfs and its concurrency tester. Experimentally add sqlite3.capi.sqlite3_js_retry_busy(), which runs a callback repeatedly until it _stops_ returning (or throwing) an SQLITE_BUSY error. FossilOrigin-Name: 45c02ed21635f7ef45214ab5ec6230b6b8bd89e35ef5889db318523e1c679fea --- diff --git a/ext/wasm/api/opfs-common-shared.c-pp.js b/ext/wasm/api/opfs-common-shared.c-pp.js index c0b028160e..15eec1ff8d 100644 --- a/ext/wasm/api/opfs-common-shared.c-pp.js +++ b/ext/wasm/api/opfs-common-shared.c-pp.js @@ -107,6 +107,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return false; } }; + /** Checks whether the given OPFS filesystem entry exists, returning true if it does, false if it doesn't or if an @@ -461,7 +462,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ options = util.nu(options); options.vfsName = vfsName; options.verbose ??= urlParams.has('opfs-verbose') - ? (+urlParams.get('opfs-verbose') || 2) : 1; + ? +urlParams.get('opfs-verbose') : 1; options.sanityChecks ??= urlParams.has('opfs-sanity-check'); opfsUtil.proxyUri ??= "sqlite3-opfs-async-proxy.js"; diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 7bb31dc60b..d1055c1cfb 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -1939,6 +1939,62 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( 'sqlite3changeset_old'); }/*changeset/preupdate additions*/ + /** + EXPERIMENTAL. For tentative addition in 3.53.0. + + sqlite3_js_retry_busy(maxTimes,callback[,beforeRetry]) + + Calls the given _synchronous_ callback function. If that function + returns sqlite3.capi.SQLITE_BUSY _or_ throws an SQLite3Error + which a resultCode property of that value then it will suppress + that error and try again, up to the given maximum number of + times. If the callback returns any other value than that, + it is returned. If the maximum number of retries has been + reached, an SQLite3Error with a resultCode value of + sqlite3.capi.SQLITE_BUSY is thrown. If the callback throws any + exception other than the aforementioned BUSY exception, it is + propagated. If it throws a BUSY exception on its final attempt, + that is propagated as well. + + If the beforeRetry argument is given, it must be a _synchronous_ + function. It is called immediately before each retry of the + callback (not for the initial call), passed the attempt number + (so it starts with 2, not 1). If it throws, the exception is + handled as described above. Its result value is ignored. + + To effectively retry "forever", pass a negative maxTimes value, + with the caveat that there is no recovery from that if the code + gets stuck in a deadlock situation. + + TODO: an async variant of this. + */ + capi.sqlite3_js_retry_busy = function(maxTimes, callback, beforeRetry){ + for(let n = 1; n <= maxTimes; ++n){ + try{ + if( beforeRetry && n>1 ) beforeRetry(n); + const rc = callback(); + if( capi.SQLITE_BUSY===rc ){ + if( n===maxTimes ){ + throw new SQLite3Error(rc, [ + "sqlite3_js_retry_busy() max retry attempts (", + maxTimes, + ") reached." + ].join('')); + } + continue; + } + return rc; + }catch(e){ + if( npostMessage({type, payload:args}); const installAsyncProxy = function(){ const toss = function(...args){throw new Error(args.join(' '))}; @@ -101,7 +101,7 @@ const installAsyncProxy = function(){ 2:console.log.bind(console) }; const logImpl = (level,...args)=>{ - if(state.verbose>level) loggers[level]('opfs-async-proxy',workerId+":",...args); + if(state.verbose>level) loggers[level](vfsName+' async-proxy',workerId+":",...args); }; const log = (...args)=>logImpl(2, ...args); const warn = (...args)=>logImpl(1, ...args); @@ -563,7 +563,7 @@ const installAsyncProxy = function(){ rc = state.sq3Codes.SQLITE_IOERR_SHORT_READ; } }catch(e){ - error("xRead() failed",e,fh); + //error("xRead() failed",e,fh); state.s11n.storeException(1,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_READ); } @@ -590,7 +590,7 @@ const installAsyncProxy = function(){ affirmNotRO('xTruncate', fh); await (await getSyncHandle(fh,'xTruncate')).truncate(size); }catch(e){ - error("xTruncate():",e,fh); + //error("xTruncate():",e,fh); state.s11n.storeException(2,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_TRUNCATE); } @@ -608,7 +608,7 @@ const installAsyncProxy = function(){ {at: Number(offset64)}) ) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE; }catch(e){ - error("xWrite():",e,fh); + //error("xWrite():",e,fh); state.s11n.storeException(1,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_WRITE); } @@ -811,6 +811,11 @@ const installAsyncProxy = function(){ const slotWhichOp = opIds.whichOp; const idleWaitTime = state.asyncIdleWaitTime; const hasWaitAsync = !!Atomics.waitAsync; +//#if nope + error("waitLoop init: isWebLocker",isWebLocker, + "idleWaitTime",idleWaitTime, + "hasWaitAsync",hasWaitAsync); +//#endif while(!flagAsyncShutdown){ try { let opId; @@ -834,17 +839,14 @@ const installAsyncProxy = function(){ Safari: 16.4 (2023-03-27) The "opfs" VFS was not born until Chrome was somewhere in - the v104-108 range and did not work with Safari < v17 - (2023-09-18) due to a WebKit bug which restricted OPFS - access from sub-Workers. - - The waitAsync() counterpart of this block has shown to be - slightly more performant for the "opfs" VFS than this - block (whereas "opfs-wl" _requires_ that block), so we - enable it where it's available, regardless of the value - of isWebLocker, with the note that if isWebLocker is true - then the bootstrapping of this script will have failed if - waitAsync() is not available. + the v104-108 range (Summer/Autumn 2022) and did not work + with Safari < v17 (2023-09-18) due to a WebKit bug which + restricted OPFS access from sub-Workers. + + The waitAsync() counterpart of this block can be used by + both "opfs" and "opfs-wl", whereas this block can only be + used by "opfs". Performance comparisons between the two + in high-contention tests have been indecisive. */ if('not-equal'!==Atomics.wait( state.sabOPView, slotWhichOp, 0, state.asyncIdleWaitTime @@ -863,6 +865,9 @@ const installAsyncProxy = function(){ index is a few lines down from here, and that instance is required in order for clear communication between the sync half of this proxy and this half. + + Much later (2026-03-07): that phenomenon is apparently + called a spurious wakeup. */ await releaseImplicitLocks(); continue; @@ -913,6 +918,7 @@ const installAsyncProxy = function(){ } }); initS11n(); + //warn("verbosity =",opt.verbose, state.verbose); log("init state",state); wPost('opfs-async-inited'); waitLoop(); diff --git a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js index c1d34f192d..349819b0e1 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js @@ -59,7 +59,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const installOpfsWlVfs = async function(options){ options = opfsUtil.initOptions('opfs-wl',options); if( !options ) return sqlite3; - options.verbose = 2; const capi = sqlite3.capi, state = opfsUtil.createVfsState(), opfsVfs = state.vfs, diff --git a/ext/wasm/tests/opfs/concurrency/test.js b/ext/wasm/tests/opfs/concurrency/test.js index f8d7f10399..a73bc2e47d 100644 --- a/ext/wasm/tests/opfs/concurrency/test.js +++ b/ext/wasm/tests/opfs/concurrency/test.js @@ -8,6 +8,7 @@ return ''+v; default: break; } + if( v instanceof Date ) return v.toString(); if(null===v) return 'null'; if(v instanceof Error){ v = { @@ -84,11 +85,12 @@ if(workers.counts.passed + workers.counts.failed !== workers.length){ return; } - stdout("Total Worker run time:",calcTime(new Date()),"ms"); if(workers.counts.failed>0){ - logCss('tests-fail',"Finished with",workers.counts.failed,"failure(s)."); + logCss('tests-fail',"Finished with",workers.counts.failed,"failure(s) in", + calcTime(new Date()),"ms"); }else{ - logCss('tests-pass',"All",workers.length,"workers finished."); + logCss('tests-pass',"All",workers.length,"workers finished in", + calcTime(new Date()),"ms"); } }; diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index ccd1df932a..a698fc96f7 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -63,29 +63,29 @@ globalThis.sqlite3InitModule().then(async function(sqlite3){ } while(true){ try{ - if( !db ){ - db = new ctor({ - filename: 'file:'+dbName+'?opfs-unlock-asap='+options.unlockAsap, - flags: 'c' - }); - sqlite3.capi.sqlite3_busy_timeout(db.pointer, 5000); - } - db.transaction((db)=>{ - db.exec([ - "create table if not exists t1(w TEXT UNIQUE ON CONFLICT REPLACE,v);", - "create table if not exists t2(w TEXT UNIQUE ON CONFLICT REPLACE,v);" - ]); + db = new ctor({ + filename: 'file:'+dbName+'?opfs-unlock-asap='+options.unlockAsap, + flags: 'c' }); break; }catch(e){ if(e instanceof sqlite3.SQLite3Error && sqlite3.capi.SQLITE_BUSY===e.resultCode){ - stderr("Retrying",(db ? "db init" : "open()"),"for BUSY:",e.message); + stderr("Retrying open() for BUSY:",e.message); continue; } throw e; } } + sqlite3.capi.sqlite3_busy_timeout(db.pointer, 5000); + sqlite3.capi.sqlite3_js_retry_busy(-1, ()=>{ + db.transaction((db)=>{ + db.exec([ + "create table if not exists t1(w TEXT UNIQUE ON CONFLICT REPLACE,v);", + "create table if not exists t2(w TEXT UNIQUE ON CONFLICT REPLACE,v);" + ]); + }); + }); const maxIterations = urlArgs.has('iterations') ? (+urlArgs.get('iterations') || 10) : 10; @@ -95,24 +95,19 @@ globalThis.sqlite3InitModule().then(async function(sqlite3){ ++interval.count; const prefix = "v(#"+interval.count+")"; stdout("Setting",prefix,"=",tm); - while(true){ - try{ + try{ + sqlite3.capi.sqlite3_js_retry_busy(-1, ()=>{ db.exec({ sql:"INSERT OR REPLACE INTO t1(w,v) VALUES(?,?)", bind: [options.workerName, new Date().getTime()] }); - //stdout("Set",prefix); - break; - }catch(e){ - if(e instanceof sqlite3.SQLite3Error - && sqlite3.capi.SQLITE_BUSY===e.resultCode){ - stderr("Retrying interval #",interval.count,"for BUSY:",e.message); - continue; - } - stderr("Error: ",e.message); - interval.error = e; - throw e; - } + }, (n)=>{ + stderr("Attempt #",n,"for interval #",interval.count,"due to BUSY"); + }); + }catch(e){ + stderr("Error: ",e.message); + interval.error = e; + throw e; } //stdout("doWork()",prefix,"error ",interval.error); }; diff --git a/manifest b/manifest index 640a34ab1e..44ee1e7107 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Elide\smore\sof\sthe\s64-bit\swas\sstuff\sfrom\sthe\sdefault\sbuild\starget. -D 2026-03-07T03:29:26.758 +C Cleanups\sand\sdocs\sin\sthe\sopfs\svfs\sand\sits\sconcurrency\stester.\sExperimentally\sadd\ssqlite3.capi.sqlite3_js_retry_busy(),\swhich\sruns\sa\scallback\srepeatedly\suntil\sit\s_stops_\sreturning\s(or\sthrowing)\san\sSQLITE_BUSY\serror. +D 2026-03-07T03:32:17.361 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -585,20 +585,20 @@ F ext/wasm/api/README.md a905d5c6bfc3e2df875bd391d6d6b7b48d41b43bdee02ad115b4724 F ext/wasm/api/extern-post-js.c-pp.js d9f42ecbedc784c0d086bc37800e52946a14f7a21600b291daa3f963c314f930 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/opfs-common-inline.c-pp.js 5be8d6d91963849e218221b48206ae55612630bb2cd7f30b1b6fcf7a9e374b76 -F ext/wasm/api/opfs-common-shared.c-pp.js 81a2429027b76df8691e6c134d3fedbf7cb804585ac28165b857ecb2260a99f3 +F ext/wasm/api/opfs-common-shared.c-pp.js d58f16b8bf00401d954249d154c91abc4d4f54221f755d774cf5c4e8f83a1a9f F ext/wasm/api/post-js-footer.js a50c1a2c4d008aede7b2aa1f18891a7ee71437c2f415b8aeb3db237ddce2935b F ext/wasm/api/post-js-header.js f35d2dcf1ab7f22a93d565f8e0b622a2934fc4e743edf3b708e4dd8140eeff55 F ext/wasm/api/pre-js.c-pp.js 9234ea680a2f6a2a177e8dcd934bdc5811a9f8409165433a252b87f4c07bba6f F ext/wasm/api/sqlite3-api-glue.c-pp.js 9b33e3ee467791dec4fd1b444b12a8545dfbb6c8b28ac651c7bdc7661a3b5a5c F ext/wasm/api/sqlite3-api-oo1.c-pp.js 45454631265d9ce82685f1a64e1650ee19c8e121c41db98a22b534c15e543cfa -F ext/wasm/api/sqlite3-api-prologue.js 98fedc159c9239b226d19567d7172300dee5ffce176e5fa2f62dd1f17d088385 +F ext/wasm/api/sqlite3-api-prologue.js c616bff3bdff1fcbb6a39bd429a96de9c3e5f1e8c0d287083cea5e98e8b0151c F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667430f9d45167569633ffc7567938 F ext/wasm/api/sqlite3-license-version-header.js 98d90255a12d02214db634e041c8e7f2f133d9361a8ebf000ba9c9af4c6761cc -F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js 5c84524885393b90b57d27446872fe980a48e8b9f5f2bb728e7ba63e905feb3f +F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js c2236c9037ccac368be6eeed3e5f6ed3cefed0d20539c9092139f27a76aaf1ba F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js a61dd2b4d919d2d5d83c5c7e49b89ecbff2525ff81419f6a6dbaecaf3819c490 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9 -F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js f65903e15f49a5475eaf8fd3f35971d4c4c4b43f0a00167f1a2c98389cf84900 +F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js c518aee3c2f537ae73aa3afd67463d1eb63db0559a41d64fe4d1361b9966b52f F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 872c29aa760b05c2703185368a39b4b2e1a8abf15c29ee7e23ee8ca7cdcbed9d F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81 F ext/wasm/api/sqlite3-wasm.c 45bb20e19b245136711f9b78584371233975811b6560c29ed9b650e225417e29 @@ -646,8 +646,8 @@ F ext/wasm/tester1-worker.c-pp.html d0032241d0b24d996cf1c4dd0dde364189693af9b5c9 F ext/wasm/tester1.c-pp.html 52d88fe2c6f21a046030a36410b4839b632f4424028197a45a3d5669ea724ddb F ext/wasm/tester1.c-pp.js a4e79fbf63bb3255d2b8ffc1cd538c115d2f6b599bc324904c80f6644379a284 F ext/wasm/tests/opfs/concurrency/index.html c8ac239f6fb45440adbdddb33a0fde2c61a20799189d60b8926be702a27dd226 -F ext/wasm/tests/opfs/concurrency/test.js 520061e6a624fe0a7c07cf3438cb6b9b7c9dcbd606e644ccb5fb52518c48b342 -F ext/wasm/tests/opfs/concurrency/worker.js 90e51ff3e781a6e08f265cbaaa993bb10eb0b798893b78cf4d8a8a8b3b561392 +F ext/wasm/tests/opfs/concurrency/test.js 46c772bc18abb0fcbb058d57b5aaee9e7938f948ecdd802c6ca0850ad3519f92 +F ext/wasm/tests/opfs/concurrency/worker.js fc985ec86b70b057224e8caaa9c5a11892ea9b980a5c5da21b1efdf5f12bc3f6 F ext/wasm/tests/opfs/sahpool/digest-worker.js b0ab6218588f1f0a6d15a363b493ceaf29bfb87804d9e0165915a9996377cf79 F ext/wasm/tests/opfs/sahpool/digest.html 206d08a34dc8bd570b2581d3d9ab3ecad3201b516a598dd096dcf3cf8cd81df8 F ext/wasm/tests/opfs/sahpool/index.html be736567fd92d3ecb9754c145755037cbbd2bca01385e2732294b53f4c842328 @@ -2191,8 +2191,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P 3b470c4c7a1fcc710e6b9eae32134c7f6c3f6008b24c7351257f66f5e8f70311 -R fd6275af0aa6341e83c30c2e83b929c6 +P 3b74ad9081daee560bf9b400e69a75abcc62dae7c5f55dcf46de84daae50ed9f +R 5251eca545de3a2827b98f8a138c1479 U stephan -Z 401ca0507277c8f3e4ba9832533ffae5 +Z e640850ae89db803157694e7fdf4c165 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cc74046192..d297924b93 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3b74ad9081daee560bf9b400e69a75abcc62dae7c5f55dcf46de84daae50ed9f +45c02ed21635f7ef45214ab5ec6230b6b8bd89e35ef5889db318523e1c679fea