Cleanups and docs in the opfs vfs and its concurrency tester. Experimentally add...

FossilOrigin-Name: 45c02ed21635f7ef45214ab5ec6230b6b8bd89e35ef5889db318523e1c679fea

diff --git a/ext/wasm/api/opfs-common-shared.c-pp.js b/ext/wasm/api/opfs-common-shared.c-pp.js
index c0b028160e46bf6168db7e398c73db78e94eaab4..15eec1ff8d12993b3507976ce7e8fd010f379ce1 100644 (file)
--- a/ext/wasm/api/opfs-common-shared.c-pp.js
+++ b/ext/wasm/api/opfs-common-shared.c-pp.js
@@ -107,6 +107,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
       return false;
     }
   };
+
   /**
      Checks whether the given OPFS filesystem entry exists,
      returning true if it does, false if it doesn't or if an
@@ -461,7 +462,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
     options = util.nu(options);
     options.vfsName = vfsName;
     options.verbose ??= urlParams.has('opfs-verbose')
-      ? (+urlParams.get('opfs-verbose') || 2) : 1;
+      ? +urlParams.get('opfs-verbose') : 1;
     options.sanityChecks ??= urlParams.has('opfs-sanity-check');
 
     opfsUtil.proxyUri ??= "sqlite3-opfs-async-proxy.js";

diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js
index 7bb31dc60b772bef3f71361eb4b675cadeff2c2d..d1055c1cfb1c019d1fbc3bf34044fd77e67ecee0 100644 (file)
--- a/ext/wasm/api/sqlite3-api-prologue.js
+++ b/ext/wasm/api/sqlite3-api-prologue.js
@@ -1939,6 +1939,62 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap(
                                             'sqlite3changeset_old');
   }/*changeset/preupdate additions*/
 
+  /**
+     EXPERIMENTAL. For tentative addition in 3.53.0.
+
+     sqlite3_js_retry_busy(maxTimes,callback[,beforeRetry])
+
+     Calls the given _synchronous_ callback function. If that function
+     returns sqlite3.capi.SQLITE_BUSY _or_ throws an SQLite3Error
+     which a resultCode property of that value then it will suppress
+     that error and try again, up to the given maximum number of
+     times. If the callback returns any other value than that,
+     it is returned. If the maximum number of retries has been
+     reached, an SQLite3Error with a resultCode value of
+     sqlite3.capi.SQLITE_BUSY is thrown. If the callback throws any
+     exception other than the aforementioned BUSY exception, it is
+     propagated. If it throws a BUSY exception on its final attempt,
+     that is propagated as well.
+
+     If the beforeRetry argument is given, it must be a _synchronous_
+     function.  It is called immediately before each retry of the
+     callback (not for the initial call), passed the attempt number
+     (so it starts with 2, not 1). If it throws, the exception is
+     handled as described above. Its result value is ignored.
+
+     To effectively retry "forever", pass a negative maxTimes value,
+     with the caveat that there is no recovery from that if the code
+     gets stuck in a deadlock situation.
+
+     TODO: an async variant of this.
+  */
+  capi.sqlite3_js_retry_busy = function(maxTimes, callback, beforeRetry){
+    for(let n = 1; n <= maxTimes; ++n){
+      try{
+        if( beforeRetry && n>1 ) beforeRetry(n);
+        const rc = callback();
+        if( capi.SQLITE_BUSY===rc ){
+          if( n===maxTimes ){
+            throw new SQLite3Error(rc, [
+              "sqlite3_js_retry_busy() max retry attempts (",
+              maxTimes,
+              ") reached."
+            ].join(''));
+          }
+          continue;
+        }
+        return rc;
+      }catch(e){
+        if( n<maxTimes
+            && (e instanceof SQLite3Error)
+            && e.resultCode===capi.SQLITE_BUSY ){
+          continue;
+        }
+        throw e;
+      }
+    }
+  };
+
   /* The remainder of the API will be set up in later steps. */
   const sqlite3 = {
     WasmAllocError: WasmAllocError,

diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js
index d09dd2b35019dd9f641e451ae832e7596bbece4b..9aed973e267575a483bfdc215f41958537715c65 100644 (file)
--- a/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js
+++ b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js
@@ -61,7 +61,7 @@ if( !vfsName ){
    VFS or both the "opfs" and "opfs-wl" VFSes.
 */
 const workerId = (Math.random() * 10000000) | 0;
-const isWebLocker = true; //'opfs-wl'===urlParams.get('vfs');
+const isWebLocker = 'opfs-wl'===urlParams.get('vfs');
 const wPost = (type,...args)=>postMessage({type, payload:args});
 const installAsyncProxy = function(){
   const toss = function(...args){throw new Error(args.join(' '))};
@@ -101,7 +101,7 @@ const installAsyncProxy = function(){
     2:console.log.bind(console)
   };
   const logImpl = (level,...args)=>{
-    if(state.verbose>level) loggers[level]('opfs-async-proxy',workerId+":",...args);
+    if(state.verbose>level) loggers[level](vfsName+' async-proxy',workerId+":",...args);
   };
   const log =    (...args)=>logImpl(2, ...args);
   const warn =   (...args)=>logImpl(1, ...args);
@@ -563,7 +563,7 @@ const installAsyncProxy = function(){
           rc = state.sq3Codes.SQLITE_IOERR_SHORT_READ;
         }
       }catch(e){
-        error("xRead() failed",e,fh);
+        //error("xRead() failed",e,fh);
         state.s11n.storeException(1,e);
         rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_READ);
       }
@@ -590,7 +590,7 @@ const installAsyncProxy = function(){
         affirmNotRO('xTruncate', fh);
         await (await getSyncHandle(fh,'xTruncate')).truncate(size);
       }catch(e){
-        error("xTruncate():",e,fh);
+        //error("xTruncate():",e,fh);
         state.s11n.storeException(2,e);
         rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_TRUNCATE);
       }
@@ -608,7 +608,7 @@ const installAsyncProxy = function(){
                    {at: Number(offset64)})
         ) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE;
       }catch(e){
-        error("xWrite():",e,fh);
+        //error("xWrite():",e,fh);
         state.s11n.storeException(1,e);
         rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_WRITE);
       }
@@ -811,6 +811,11 @@ const installAsyncProxy = function(){
     const slotWhichOp = opIds.whichOp;
     const idleWaitTime = state.asyncIdleWaitTime;
     const hasWaitAsync = !!Atomics.waitAsync;
+//#if nope
+    error("waitLoop init: isWebLocker",isWebLocker,
+          "idleWaitTime",idleWaitTime,
+          "hasWaitAsync",hasWaitAsync);
+//#endif
     while(!flagAsyncShutdown){
       try {
         let opId;
@@ -834,17 +839,14 @@ const installAsyncProxy = function(){
              Safari: 16.4 (2023-03-27)
 
              The "opfs" VFS was not born until Chrome was somewhere in
-             the v104-108 range and did not work with Safari < v17
-             (2023-09-18) due to a WebKit bug which restricted OPFS
-             access from sub-Workers.
-
-             The waitAsync() counterpart of this block has shown to be
-             slightly more performant for the "opfs" VFS than this
-             block (whereas "opfs-wl" _requires_ that block), so we
-             enable it where it's available, regardless of the value
-             of isWebLocker, with the note that if isWebLocker is true
-             then the bootstrapping of this script will have failed if
-             waitAsync() is not available.
+             the v104-108 range (Summer/Autumn 2022) and did not work
+             with Safari < v17 (2023-09-18) due to a WebKit bug which
+             restricted OPFS access from sub-Workers.
+
+             The waitAsync() counterpart of this block can be used by
+             both "opfs" and "opfs-wl", whereas this block can only be
+             used by "opfs". Performance comparisons between the two
+             in high-contention tests have been indecisive.
           */
           if('not-equal'!==Atomics.wait(
             state.sabOPView, slotWhichOp, 0, state.asyncIdleWaitTime
@@ -863,6 +865,9 @@ const installAsyncProxy = function(){
                index is a few lines down from here, and that instance
                is required in order for clear communication between
                the sync half of this proxy and this half.
+
+               Much later (2026-03-07): that phenomenon is apparently
+               called a spurious wakeup.
             */
             await releaseImplicitLocks();
             continue;
@@ -913,6 +918,7 @@ const installAsyncProxy = function(){
               }
             });
             initS11n();
+            //warn("verbosity =",opt.verbose, state.verbose);
             log("init state",state);
             wPost('opfs-async-inited');
             waitLoop();

diff --git a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js
index c1d34f192d5083cd88ffcad90985eee70427a5c3..349819b0e184f8da95091d68981d1f25fa6ad82a 100644 (file)
--- a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js
+++ b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js
@@ -59,7 +59,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
 const installOpfsWlVfs = async function(options){
   options = opfsUtil.initOptions('opfs-wl',options);
   if( !options ) return sqlite3;
-  options.verbose = 2;
   const capi = sqlite3.capi,
         state = opfsUtil.createVfsState(),
         opfsVfs = state.vfs,

diff --git a/ext/wasm/tests/opfs/concurrency/test.js b/ext/wasm/tests/opfs/concurrency/test.js
index f8d7f103996e76221c772af5124ee25c3a7e9f30..a73bc2e47dedf1f96507d91d1b0cd3e3378991c4 100644 (file)
--- a/ext/wasm/tests/opfs/concurrency/test.js
+++ b/ext/wasm/tests/opfs/concurrency/test.js
@@ -8,6 +8,7 @@
             return ''+v;
           default: break;
       }
+      if( v instanceof Date ) return v.toString();
       if(null===v) return 'null';
       if(v instanceof Error){
         v = {
@@ -84,11 +85,12 @@
     if(workers.counts.passed + workers.counts.failed !== workers.length){
       return;
     }
-    stdout("Total Worker run time:",calcTime(new Date()),"ms");
     if(workers.counts.failed>0){
-      logCss('tests-fail',"Finished with",workers.counts.failed,"failure(s).");
+      logCss('tests-fail',"Finished with",workers.counts.failed,"failure(s) in",
+             calcTime(new Date()),"ms");
     }else{
-      logCss('tests-pass',"All",workers.length,"workers finished.");
+      logCss('tests-pass',"All",workers.length,"workers finished in",
+             calcTime(new Date()),"ms");
     }
   };
 

diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js
index ccd1df932acc10c95f4df6c80cbb5d299f6a9bbb..a698fc96f7d851e9f83c7d9fc41f7f1046e88801 100644 (file)
--- a/ext/wasm/tests/opfs/concurrency/worker.js
+++ b/ext/wasm/tests/opfs/concurrency/worker.js
@@ -63,29 +63,29 @@ globalThis.sqlite3InitModule().then(async function(sqlite3){
     }
     while(true){
       try{
-        if( !db ){
-          db = new ctor({
-            filename: 'file:'+dbName+'?opfs-unlock-asap='+options.unlockAsap,
-              flags: 'c'
-          });
-          sqlite3.capi.sqlite3_busy_timeout(db.pointer, 5000);
-        }
-        db.transaction((db)=>{
-          db.exec([
-            "create table if not exists t1(w TEXT UNIQUE ON CONFLICT REPLACE,v);",
-            "create table if not exists t2(w TEXT UNIQUE ON CONFLICT REPLACE,v);"
-          ]);
+        db = new ctor({
+          filename: 'file:'+dbName+'?opfs-unlock-asap='+options.unlockAsap,
+          flags: 'c'
         });
         break;
       }catch(e){
         if(e instanceof sqlite3.SQLite3Error
            && sqlite3.capi.SQLITE_BUSY===e.resultCode){
-          stderr("Retrying",(db ? "db init" : "open()"),"for BUSY:",e.message);
+          stderr("Retrying open() for BUSY:",e.message);
           continue;
         }
         throw e;
       }
     }
+    sqlite3.capi.sqlite3_busy_timeout(db.pointer, 5000);
+    sqlite3.capi.sqlite3_js_retry_busy(-1, ()=>{
+      db.transaction((db)=>{
+        db.exec([
+          "create table if not exists t1(w TEXT UNIQUE ON CONFLICT REPLACE,v);",
+          "create table if not exists t2(w TEXT UNIQUE ON CONFLICT REPLACE,v);"
+        ]);
+      });
+    });
 
     const maxIterations =
           urlArgs.has('iterations') ? (+urlArgs.get('iterations') || 10) : 10;
@@ -95,24 +95,19 @@ globalThis.sqlite3InitModule().then(async function(sqlite3){
       ++interval.count;
       const prefix = "v(#"+interval.count+")";
       stdout("Setting",prefix,"=",tm);
-      while(true){
-        try{
+      try{
+        sqlite3.capi.sqlite3_js_retry_busy(-1, ()=>{
           db.exec({
             sql:"INSERT OR REPLACE INTO t1(w,v) VALUES(?,?)",
             bind: [options.workerName, new Date().getTime()]
           });
-          //stdout("Set",prefix);
-          break;
-        }catch(e){
-          if(e instanceof sqlite3.SQLite3Error
-             && sqlite3.capi.SQLITE_BUSY===e.resultCode){
-            stderr("Retrying interval #",interval.count,"for BUSY:",e.message);
-            continue;
-          }
-          stderr("Error: ",e.message);
-          interval.error = e;
-          throw e;
-        }
+        }, (n)=>{
+          stderr("Attempt #",n,"for interval #",interval.count,"due to BUSY");
+        });
+      }catch(e){
+        stderr("Error: ",e.message);
+        interval.error = e;
+        throw e;
       }
       //stdout("doWork()",prefix,"error ",interval.error);
     };

diff --git a/manifest b/manifest
index 640a34ab1ef05a8cd86088700e3190f91cfa6c8d..44ee1e71071630316165715496f3f8695074a727 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Elide\smore\sof\sthe\s64-bit\swas\sstuff\sfrom\sthe\sdefault\sbuild\starget.
-D 2026-03-07T03:29:26.758
+C Cleanups\sand\sdocs\sin\sthe\sopfs\svfs\sand\sits\sconcurrency\stester.\sExperimentally\sadd\ssqlite3.capi.sqlite3_js_retry_busy(),\swhich\sruns\sa\scallback\srepeatedly\suntil\sit\s_stops_\sreturning\s(or\sthrowing)\san\sSQLITE_BUSY\serror.
+D 2026-03-07T03:32:17.361
 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
@@ -585,20 +585,20 @@ F ext/wasm/api/README.md a905d5c6bfc3e2df875bd391d6d6b7b48d41b43bdee02ad115b4724
 F ext/wasm/api/extern-post-js.c-pp.js d9f42ecbedc784c0d086bc37800e52946a14f7a21600b291daa3f963c314f930
 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41
 F ext/wasm/api/opfs-common-inline.c-pp.js 5be8d6d91963849e218221b48206ae55612630bb2cd7f30b1b6fcf7a9e374b76
-F ext/wasm/api/opfs-common-shared.c-pp.js 81a2429027b76df8691e6c134d3fedbf7cb804585ac28165b857ecb2260a99f3
+F ext/wasm/api/opfs-common-shared.c-pp.js d58f16b8bf00401d954249d154c91abc4d4f54221f755d774cf5c4e8f83a1a9f
 F ext/wasm/api/post-js-footer.js a50c1a2c4d008aede7b2aa1f18891a7ee71437c2f415b8aeb3db237ddce2935b
 F ext/wasm/api/post-js-header.js f35d2dcf1ab7f22a93d565f8e0b622a2934fc4e743edf3b708e4dd8140eeff55
 F ext/wasm/api/pre-js.c-pp.js 9234ea680a2f6a2a177e8dcd934bdc5811a9f8409165433a252b87f4c07bba6f
 F ext/wasm/api/sqlite3-api-glue.c-pp.js 9b33e3ee467791dec4fd1b444b12a8545dfbb6c8b28ac651c7bdc7661a3b5a5c
 F ext/wasm/api/sqlite3-api-oo1.c-pp.js 45454631265d9ce82685f1a64e1650ee19c8e121c41db98a22b534c15e543cfa
-F ext/wasm/api/sqlite3-api-prologue.js 98fedc159c9239b226d19567d7172300dee5ffce176e5fa2f62dd1f17d088385
+F ext/wasm/api/sqlite3-api-prologue.js c616bff3bdff1fcbb6a39bd429a96de9c3e5f1e8c0d287083cea5e98e8b0151c
 F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667430f9d45167569633ffc7567938
 F ext/wasm/api/sqlite3-license-version-header.js 98d90255a12d02214db634e041c8e7f2f133d9361a8ebf000ba9c9af4c6761cc
-F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js 5c84524885393b90b57d27446872fe980a48e8b9f5f2bb728e7ba63e905feb3f
+F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js c2236c9037ccac368be6eeed3e5f6ed3cefed0d20539c9092139f27a76aaf1ba
 F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d
 F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js a61dd2b4d919d2d5d83c5c7e49b89ecbff2525ff81419f6a6dbaecaf3819c490
 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9
-F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js f65903e15f49a5475eaf8fd3f35971d4c4c4b43f0a00167f1a2c98389cf84900
+F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js c518aee3c2f537ae73aa3afd67463d1eb63db0559a41d64fe4d1361b9966b52f
 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 872c29aa760b05c2703185368a39b4b2e1a8abf15c29ee7e23ee8ca7cdcbed9d
 F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81
 F ext/wasm/api/sqlite3-wasm.c 45bb20e19b245136711f9b78584371233975811b6560c29ed9b650e225417e29
@@ -646,8 +646,8 @@ F ext/wasm/tester1-worker.c-pp.html d0032241d0b24d996cf1c4dd0dde364189693af9b5c9
 F ext/wasm/tester1.c-pp.html 52d88fe2c6f21a046030a36410b4839b632f4424028197a45a3d5669ea724ddb
 F ext/wasm/tester1.c-pp.js a4e79fbf63bb3255d2b8ffc1cd538c115d2f6b599bc324904c80f6644379a284
 F ext/wasm/tests/opfs/concurrency/index.html c8ac239f6fb45440adbdddb33a0fde2c61a20799189d60b8926be702a27dd226
-F ext/wasm/tests/opfs/concurrency/test.js 520061e6a624fe0a7c07cf3438cb6b9b7c9dcbd606e644ccb5fb52518c48b342
-F ext/wasm/tests/opfs/concurrency/worker.js 90e51ff3e781a6e08f265cbaaa993bb10eb0b798893b78cf4d8a8a8b3b561392
+F ext/wasm/tests/opfs/concurrency/test.js 46c772bc18abb0fcbb058d57b5aaee9e7938f948ecdd802c6ca0850ad3519f92
+F ext/wasm/tests/opfs/concurrency/worker.js fc985ec86b70b057224e8caaa9c5a11892ea9b980a5c5da21b1efdf5f12bc3f6
 F ext/wasm/tests/opfs/sahpool/digest-worker.js b0ab6218588f1f0a6d15a363b493ceaf29bfb87804d9e0165915a9996377cf79
 F ext/wasm/tests/opfs/sahpool/digest.html 206d08a34dc8bd570b2581d3d9ab3ecad3201b516a598dd096dcf3cf8cd81df8
 F ext/wasm/tests/opfs/sahpool/index.html be736567fd92d3ecb9754c145755037cbbd2bca01385e2732294b53f4c842328
@@ -2191,8 +2191,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee
 F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
 F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c
-P 3b470c4c7a1fcc710e6b9eae32134c7f6c3f6008b24c7351257f66f5e8f70311
-R fd6275af0aa6341e83c30c2e83b929c6
+P 3b74ad9081daee560bf9b400e69a75abcc62dae7c5f55dcf46de84daae50ed9f
+R 5251eca545de3a2827b98f8a138c1479
 U stephan
-Z 401ca0507277c8f3e4ba9832533ffae5
+Z e640850ae89db803157694e7fdf4c165
 # Remove this line to create a well-formed Fossil manifest.

diff --git a/manifest.uuid b/manifest.uuid
index cc74046192bdfb62db227530b891946594d40da0..d297924b935ffb17cbacf64844a2cb549e9fc24b 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-3b74ad9081daee560bf9b400e69a75abcc62dae7c5f55dcf46de84daae50ed9f
+45c02ed21635f7ef45214ab5ec6230b6b8bd89e35ef5889db318523e1c679fea