OPFS VFS: moved i/o buffer from per-file to the VFS, and related refactoring, in...

FossilOrigin-Name: d4d63e4580ad8d497310608175308c03c517e051d7865cb66aa0b10356612d7d

diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js
index 3fd08d9aeca4f7675eab9acad589d47f47ec8fd6..ffc7669a7f4585bdefeaf4e6b3a10bf2db5e8012 100644 (file)
--- a/ext/wasm/api/sqlite3-api-opfs.js
+++ b/ext/wasm/api/sqlite3-api-opfs.js
@@ -188,17 +188,22 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
     const state = Object.create(null);
     state.verbose = options.verbose;
     state.fileBufferSize =
-      1024 * 64 + 8 /* size of aFileHandle.sab. 64k = max sqlite3 page
-                       size. The additional bytes are space for
-                       holding BigInt results, since we cannot store
-                       those via the Atomics API (which only works on
-                       an Int32Array). */;
+      1024 * 64 /* size of aFileHandle.sab. 64k = max sqlite3 page
+                   size. */;
+    state.sabOffsetS11n = state.fileBufferSize;
+    state.sabIO = new SharedArrayBuffer(
+      state.fileBufferSize
+        + 4096/* arg/result serialization */
+        + 8 /* to be removed - porting crutch */
+    );
     state.fbInt64Offset =
-      state.fileBufferSize - 8 /*spot in fileHandle.sab to store an int64 result */;
+      state.sabIO.byteLength - 8 /*spot in fileHandle.sab to store an int64 result.
+                                  to be removed. Porting crutch. */;
     state.opIds = Object.create(null);
     const metrics = Object.create(null);
     {
       let i = 0;
+      state.opIds.nothing = i++;
       state.opIds.xAccess = i++;
       state.opIds.xClose = i++;
       state.opIds.xDelete = i++;
@@ -211,7 +216,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
       state.opIds.xTruncate = i++;
       state.opIds.xWrite = i++;
       state.opIds.mkdir = i++;
-      state.opSAB = new SharedArrayBuffer(i * 4/*sizeof int32*/);
+      state.sabOP = new SharedArrayBuffer(i * 4/*sizeof int32*/);
       state.opIds.xFileControl = state.opIds.xSync /* special case */;
       opfsUtil.metrics.reset();
     }
@@ -225,7 +230,9 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
       'SQLITE_IOERR_WRITE', 'SQLITE_IOERR_FSYNC',
       'SQLITE_IOERR_TRUNCATE', 'SQLITE_IOERR_DELETE',
       'SQLITE_IOERR_ACCESS', 'SQLITE_IOERR_CLOSE',
-      'SQLITE_IOERR_DELETE'
+      'SQLITE_IOERR_DELETE',
+      'SQLITE_OPEN_CREATE', 'SQLITE_OPEN_DELETEONCLOSE',
+      'SQLITE_OPEN_READONLY'
     ].forEach(function(k){
       state.sq3Codes[k] = capi[k] || toss("Maintenance required: not found:",k);
       state.sq3Codes._reverse[capi[k]] = k;
@@ -236,17 +243,17 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
     /**
        Runs the given operation in the async worker counterpart, waits
        for its response, and returns the result which the async worker
-       writes to the given op's index in state.opSABView. The 2nd argument
+       writes to the given op's index in state.sabOPView. The 2nd argument
        must be a single object or primitive value, depending on the
        given operation's signature in the async API counterpart.
     */
     const opRun = (op,args)=>{
       const t = performance.now();
-      Atomics.store(state.opSABView, state.opIds[op], -1);
+      Atomics.store(state.sabOPView, state.opIds[op], -1);
       wMsg(op, args);
-      Atomics.wait(state.opSABView, state.opIds[op], -1);
+      Atomics.wait(state.sabOPView, state.opIds[op], -1);
       metrics[op].wait += performance.now() - t;
-      return Atomics.load(state.opSABView, state.opIds[op]);
+      return Atomics.load(state.sabOPView, state.opIds[op]);
     };
 
     /**
@@ -574,10 +581,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
         args.fid = pFile;
         args.filename = zName;
         args.sab = new SharedArrayBuffer(state.fileBufferSize);
-        args.fileType = f._.getFileType(args.filename, flags);
-        args.create = !!(flags & capi.SQLITE_OPEN_CREATE);
-        args.deleteOnClose = !!(flags & capi.SQLITE_OPEN_DELETEONCLOSE);
-        args.readOnly = !!(flags & capi.SQLITE_OPEN_READONLY);
+        args.flags = flags;
         const rc = opRun('xOpen', args);
         if(!rc){
           /* Recall that sqlite3_vfs::xClose() will be called, even on
@@ -586,8 +590,8 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
             wasm.setMemValue(pOutFlags, capi.SQLITE_OPEN_READONLY, 'i32');
           }
           __openFiles[pFile] = args;
-          args.sabView = new Uint8Array(args.sab);
-          args.sabViewFileSize = new DataView(args.sab, state.fbInt64Offset, 8);
+          args.sabView = new Uint8Array(state.sabIO, 0, state.fileBufferSize);
+          args.sabViewFileSize = new DataView(state.sabIO, state.fbInt64Offset, 8);
           args.sq3File = new sqlite3_file(pFile);
           args.sq3File.$pMethods = opfsIoMethods.pointer;
           args.ba = new Uint8Array(args.sab);
@@ -615,7 +619,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
          assume it's sane and use it, otherwise install a JS-based
          one. */
       vfsSyncWrappers.xSleep = function(pVfs,ms){
-        Atomics.wait(state.opSABView, state.opIds.xSleep, 0, ms);
+        Atomics.wait(state.sabOPView, state.opIds.xSleep, 0, ms);
         return 0;
       };
     }
@@ -702,8 +706,8 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
         log("xAccess(",dbFile,") exists ?=",rc);
         rc = vfsSyncWrappers.xOpen(opfsVfs.pointer, zDbFile,
                                    fid, openFlags, pOut);
-        log("open rc =",rc,"state.opSABView[xOpen] =",
-            state.opSABView[state.opIds.xOpen]);
+        log("open rc =",rc,"state.sabOPView[xOpen] =",
+            state.sabOPView[state.opIds.xOpen]);
         if(isWorkerErrCode(rc)){
           error("open failed with code",rc);
           return;
@@ -733,7 +737,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
           log("waking up from xSleep()");
         }
         rc = ioSyncWrappers.xClose(fid);
-        log("xClose rc =",rc,"opSABView =",state.opSABView);
+        log("xClose rc =",rc,"sabOPView =",state.sabOPView);
         log("Deleting file:",dbFile);
         vfsSyncWrappers.xDelete(opfsVfs.pointer, zDbFile, 0x1234);
         vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut);
@@ -767,7 +771,8 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
                 toss("BUG: sqlite3_vfs_find() failed for just-installed OPFS VFS");
               }
               capi.sqlite3_vfs_register.addReference(opfsVfs, opfsIoMethods);
-              state.opSABView = new Int32Array(state.opSAB);
+              state.sabOPView = new Int32Array(state.sabOP);
+              state.sabFileBufView = new Uint8Array(state.sabFileBufView, 0, state.fileBufferSize);
               if(options.sanityChecks){
                 warn("Running sanity checks because of opfs-sanity-check URL arg...");
                 sanityCheck();

diff --git a/ext/wasm/sqlite3-opfs-async-proxy.js b/ext/wasm/sqlite3-opfs-async-proxy.js
index fb472488bf71f3a8a7dc9a984a1a4271119edc4a..a1ed4362d3abcdc7b5215387e490767fecfc9ffa 100644 (file)
--- a/ext/wasm/sqlite3-opfs-async-proxy.js
+++ b/ext/wasm/sqlite3-opfs-async-proxy.js
@@ -131,8 +131,8 @@ const getDirForPath = async function f(absFilename, createDirs = false){
 */
 const storeAndNotify = (opName, value)=>{
   log(opName+"() is notify()ing w/ value:",value);
-  Atomics.store(state.opSABView, state.opIds[opName], value);
-  Atomics.notify(state.opSABView, state.opIds[opName]);
+  Atomics.store(state.sabOPView, state.opIds[opName], value);
+  Atomics.notify(state.sabOPView, state.opIds[opName]);
 };
 
 /**
@@ -214,6 +214,12 @@ const vfsAsyncImpls = {
     }
     mTimeEnd();
   },
+  xDelete: async function(...args){
+    mTimeStart('xDelete');
+    const rc = await vfsAsyncImpls.xDeleteNoWait(...args);
+    storeAndNotify('xDelete', rc);
+    mTimeEnd();
+  },
   xDeleteNoWait: async function({filename, syncDir, recursive = false}){
     /* The syncDir flag is, for purposes of the VFS API's semantics,
        ignored here. However, if it has the value 0x1234 then: after
@@ -248,12 +254,6 @@ const vfsAsyncImpls = {
     }
     return rc;
   },
-  xDelete: async function(...args){
-    mTimeStart('xDelete');
-    const rc = await vfsAsyncImpls.xDeleteNoWait(...args);
-    storeAndNotify('xDelete', rc);
-    mTimeEnd();
-  },
   xFileSize: async function(fid){
     mTimeStart('xFileSize');
     log("xFileSize(",arguments,")");
@@ -261,6 +261,9 @@ const vfsAsyncImpls = {
     let sz;
     try{
       sz = await fh.accessHandle.getSize();
+      if(!fh.sabViewFileSize){
+        fh.sabViewFileSize = new DataView(state.sabIO,state.fbInt64Offset,8);
+      }
       fh.sabViewFileSize.setBigInt64(0, BigInt(sz), true);
       sz = 0;
     }catch(e){
@@ -272,16 +275,15 @@ const vfsAsyncImpls = {
   },
   xOpen: async function({
     fid/*sqlite3_file pointer*/,
-    sab/*file-specific SharedArrayBuffer*/,
     filename,
-    fileType = undefined /*mainDb, mainJournal, etc.*/,
-    create = false, readOnly = false, deleteOnClose = false
+    flags
   }){
     const opName = 'xOpen';
     mTimeStart(opName);
+    log(opName+"(",arguments[0],")");
+    const deleteOnClose = (state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags);
+    const create = (state.sq3Codes.SQLITE_OPEN_CREATE & flags);
     try{
-      if(create) readOnly = false;
-      log(opName+"(",arguments[0],")");
       let hDir, filenamePart;
       try {
         [hDir, filenamePart] = await getDirForPath(filename, !!create);
@@ -290,20 +292,8 @@ const vfsAsyncImpls = {
         mTimeEnd();
         return;
       }
-      const hFile = await hDir.getFileHandle(filenamePart, {create: !!create});
-      log(opName,"filenamePart =",filenamePart, 'hDir =',hDir);
-      const fobj = __openFiles[fid] = Object.create(null);
-      fobj.filenameAbs = filename;
-      fobj.filenamePart = filenamePart;
-      fobj.dirHandle = hDir;
-      fobj.fileHandle = hFile;
-      fobj.fileType = fileType;
-      fobj.sab = sab;
-      fobj.sabView = new Uint8Array(sab,0,state.fbInt64Offset);
-      fobj.sabViewFileSize = new DataView(sab,state.fbInt64Offset,8);
-      fobj.create = !!create;
-      fobj.readOnly = !!readOnly;
-      fobj.deleteOnClose = !!deleteOnClose;
+      const hFile = await hDir.getFileHandle(filenamePart, {create});
+      const fobj = Object.create(null);
       /**
          wa-sqlite, at this point, grabs a SyncAccessHandle and
          assigns it to the accessHandle prop of the file state
@@ -311,6 +301,14 @@ const vfsAsyncImpls = {
          places that limitation on it.
       */
       fobj.accessHandle = await hFile.createSyncAccessHandle();
+      __openFiles[fid] = fobj;
+      fobj.filenameAbs = filename;
+      fobj.filenamePart = filenamePart;
+      fobj.dirHandle = hDir;
+      fobj.fileHandle = hFile;
+      fobj.sabView = state.sabFileBufView;
+      fobj.readOnly = create ? false : (state.sq3Codes.SQLITE_OPEN_READONLY & flags);
+      fobj.deleteOnClose = deleteOnClose;
       storeAndNotify(opName, 0);
     }catch(e){
       error(opName,e);
@@ -395,8 +393,10 @@ navigator.storage.getDirectory().then(function(d){
           state.verbose = opt.verbose ?? 2;
           state.fileBufferSize = opt.fileBufferSize;
           state.fbInt64Offset = opt.fbInt64Offset;
-          state.opSAB = opt.opSAB;
-          state.opSABView = new Int32Array(state.opSAB);
+          state.sabOP = opt.sabOP;
+          state.sabOPView = new Int32Array(state.sabOP);
+          state.sabIO = opt.sabIO;
+          state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize);
           state.opIds = opt.opIds;
           state.sq3Codes = opt.sq3Codes;
           Object.keys(vfsAsyncImpls).forEach((k)=>{

diff --git a/manifest b/manifest
index c2229acedec191770b69fc88247b8972e8dade41..00518b64c82905d2714172b62beba17f938b865c 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Further\smetrics\sand\sbuffer-copy\soptimizations\sin\sthe\sOPFS\sproxy,\sbut\swith\slittle\seffect.
-D 2022-09-19T18:22:29.467
+C OPFS\sVFS:\smoved\si/o\sbuffer\sfrom\sper-file\sto\sthe\sVFS,\sand\srelated\srefactoring,\sin\sprep\sfor\sexperimentation\swith\sa\snew\sinter-worker\scomms\smodel.
+D 2022-09-20T01:28:47.162
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -484,7 +484,7 @@ F ext/wasm/api/post-js-header.js 0e853b78db83cb1c06b01663549e0e8b4f377f12f5a2d9a
 F ext/wasm/api/sqlite3-api-cleanup.js 8564a6077cdcaea9a9f428a019af8a05887f0131e6a2a1e72a7ff1145fadfe77
 F ext/wasm/api/sqlite3-api-glue.js 366d580c8e5bf7fcf4c6dee6f646c31f5549bd417ea03a59a0acca00e8ecce30
 F ext/wasm/api/sqlite3-api-oo1.js 2d13dddf0d2b4168a9249f124134d37924331e5b55e05dba18b6d661fbeefe48
-F ext/wasm/api/sqlite3-api-opfs.js df3d085a55be11a0b7bce3d42ea1b721ff8aaba93659d0ec48c3327dde384596
+F ext/wasm/api/sqlite3-api-opfs.js f8027fb4af1c24fcfad31889f8a5ccfa3b96d6e812d3495b13833bef57034046
 F ext/wasm/api/sqlite3-api-prologue.js 0d2639387b94c30f492d4aea6e44fb7b16720808678464559458fd2ae3759655
 F ext/wasm/api/sqlite3-api-worker1.js ee4cf149cbacb63d06b536674f822aa5088b7e022cdffc69f1f36cebe2f9fea0
 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9
@@ -519,7 +519,7 @@ F ext/wasm/speedtest1.html 512addeb3c27c94901178b7bcbde83a6f95c093f9ebe16a2959a0
 F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x
 F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0
 F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5
-F ext/wasm/sqlite3-opfs-async-proxy.js f1a270e7a8adeb80c73e0b345e472e574e694d9eaec6588ce3671438b79eb351
+F ext/wasm/sqlite3-opfs-async-proxy.js 7ebc36915cd61bd4f067a4823307f4d4eb2678a173aaae470c534e8fe9cda650
 F ext/wasm/sqlite3-worker1-promiser.js 4fd0465688a28a75f1d4ee4406540ba494f49844e3cad0670d0437a001943365
 F ext/wasm/sqlite3-worker1.js 0c1e7626304543969c3846573e080c082bf43bcaa47e87d416458af84f340a9e
 F ext/wasm/test-opfs-vfs.html eb69dda21eb414b8f5e3f7c1cc0f774103cc9c0f87b2d28a33419e778abfbab5
@@ -2026,8 +2026,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P fb7f287310d74a3e236125ae9c49b859f9263c29ae85161c1bcf9dd0778d8a51
-R 946ef52a6b1d5b2d6cbf3b063dacd006
+P d1f1fe6f1c60640f7770dfb9245c459a09b8d24ec2ddf664dff77c810bd51f96
+R e5a1ff53ebe902985ab8e87bba2c71fb
 U stephan
-Z f42132e655dd81eba8154f54be0c97de
+Z d05962442428c5dae722ab00dca47e48
 # Remove this line to create a well-formed Fossil manifest.

diff --git a/manifest.uuid b/manifest.uuid
index e76b3ad26deb78a161c3a034dc5fabed448a2717..8d4f713497cedb50ab892059b0c7621b010ebb3a 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d1f1fe6f1c60640f7770dfb9245c459a09b8d24ec2ddf664dff77c810bd51f96
\ No newline at end of file
+d4d63e4580ad8d497310608175308c03c517e051d7865cb66aa0b10356612d7d
\ No newline at end of file