From d0ae50411f36b52e358343e7fd1095ed663513b7 Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 16 Jul 2023 10:02:41 +0000 Subject: [PATCH] Redefine what the opfs-sahpool installation promise resolves to. Fix addCapacity(). Add utility methods to import/export files. FossilOrigin-Name: 809c6f4de3653ad7a7751af45a7a0d6cb20c3ee3be80c69833c729242227d970 --- ext/wasm/api/sqlite3-vfs-opfs-sahpool.js | 135 +++++++++++++++-------- ext/wasm/speedtest1-worker.js | 5 +- manifest | 14 +-- manifest.uuid | 2 +- 4 files changed, 99 insertions(+), 57 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js index 15fc687877..0551523101 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js @@ -56,10 +56,10 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const toss = sqlite3.util.toss; let vfsRegisterResult = undefined; /** - installOpfsSAHPoolVfs() asynchronously initializes the - OPFS SyncAccessHandle Pool VFS. It returns a Promise - which either resolves to the sqlite3 object or rejects - with an Error value. + installOpfsSAHPoolVfs() asynchronously initializes the OPFS + SyncAccessHandle Pool VFS. It returns a Promise which either + resolves to a utility object described below or rejects with an + Error value. Initialization of this VFS is not automatic because its registration requires that it lock all resources it @@ -75,6 +75,12 @@ let vfsRegisterResult = undefined; resolved or rejected Promise. If called while the first call is still pending resolution, a rejected promise with a descriptive error is returned. + + On success, the resulting Promise resolves to a utility object + which can be used to query and manipulate the pool. Its API is... + + TODO + */ sqlite3.installOpfsSAHPoolVfs = async function(){ if(sqlite3===vfsRegisterResult) return Promise.resolve(sqlite3); @@ -112,8 +118,11 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ vfsRegisterResult = err; return Promise.reject(err); }; + /** The PoolUtil object will be the result of the + resolved Promise. */ + const PoolUtil = Object.create(null); const promiseResolve = - ()=>Promise.resolve(vfsRegisterResult = sqlite3); + ()=>Promise.resolve(vfsRegisterResult = PoolUtil); // Config opts for the VFS... const SECTOR_SIZE = 4096; const HEADER_MAX_PATH_SIZE = 512; @@ -184,15 +193,14 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ to the new capacity. */ addCapacity: async function(n){ - const cap = this.getCapacity(); - for(let i = cap; i < cap+n; ++i){ + for(let i = 0; i < n; ++i){ const name = getRandomName(); const h = await this.dirHandle.getFileHandle(name, {create:true}); const ah = await h.createSyncAccessHandle(); this.mapSAHToName.set(ah,name); this.setAssociatedPath(ah, '', 0); } - return i; + return this.getCapacity(); }, /** Removes n entries from the pool's current capacity @@ -399,14 +407,19 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ const rc = this.$error; this.$error = undefined; return rc; + }, + nextAvailableSAH: function(){ + const [rc] = this.availableSAH.keys(); + return rc; } + })/*SAHPool*/; - sqlite3.SAHPool = SAHPool/*only for testing*/; + //sqlite3.SAHPool = SAHPool/*only for testing*/; /** Impls for the sqlite3_io_methods methods. Maintenance reminder: members are in alphabetical order to simplify finding them. */ - const ioSyncWrappers = { + const ioMethods = { xCheckReservedLock: function(pFile,pOut){ log('xCheckReservedLock'); SAHPool.storeErr(); @@ -523,13 +536,13 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ return capi.SQLITE_IOERR; } } - }/*ioSyncWrappers*/; + }/*ioMethods*/; /** Impls for the sqlite3_vfs methods. Maintenance reminder: members are in alphabetical order to simplify finding them. */ - const vfsSyncWrappers = { + const vfsMethods = { xAccess: function(pVfs,zName,flags,pOut){ log(`xAccess ${wasm.cstrToJs(zName)}`); SAHPool.storeErr(); @@ -597,7 +610,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ // File not found so try to create it. if(SAHPool.getFileCount() < SAHPool.getCapacity()) { // Choose an unassociated OPFS file from the pool. - [sah] = SAHPool.availableSAH.keys(); + sah = SAHPool.nextAvailableSAH(); SAHPool.setAssociatedPath(sah, path, flags); }else{ // File pool is full. @@ -621,7 +634,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ return capi.SQLITE_CANTOPEN; } }/*xOpen()*/ - }/*vfsSyncWrappers*/; + }/*vfsMethods*/; if(dVfs){ /* Inherit certain VFS members from the default VFS, @@ -631,7 +644,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ } if(!opfsVfs.$xRandomness){ /* If the default VFS has no xRandomness(), add a basic JS impl... */ - vfsSyncWrappers.xRandomness = function(pVfs, nOut, pOut){ + vfsMethods.xRandomness = function(pVfs, nOut, pOut){ const heap = wasm.heap8u(); let i = 0; for(; i < nOut; ++i) heap[pOut + i] = (Math.random()*255000) & 0xFF; @@ -639,7 +652,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ }; } if(!opfsVfs.$xSleep){ - vfsSyncWrappers.xSleep = (pVfs,ms)=>0; + vfsMethods.xSleep = (pVfs,ms)=>0; } /** @@ -669,6 +682,58 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ if(true!==apiVersionCheck){ return promiseReject(apiVersionCheck); } + + PoolUtil.$SAHPool = SAHPool/* ONLY for testing and debugging */; + PoolUtil.addCapacity = async (n)=>SAHPool.addCapacity(n); + PoolUtil.reduceCapacity = async (n)=>SAHPool.reduceCapacity(n); + PoolUtil.getCapacity = SAHPool.getCapacity.bind(SAHPool); + PoolUtil.getActiveFileCount = SAHPool.getFileCount.bind(SAHPool); + /** + Synchronously reads the contents of the given file into a + Uint8Array and returns it. This will throw if the given name is + not currently in active use or on I/O error. + */ + PoolUtil.exportFile = function(name){ + const sah = SAHPool.mapPathToSAH.get(name) || toss("File not found:",name); + const n = sah.getSize() - HEADER_OFFSET_DATA; + const b = new Uint8Array(n>=0 ? n : 0); + if(n>0) sah.read(b, {at: HEADER_OFFSET_DATA}); + return b; + }; + + /** + The counterpart of exportFile(), this imports the contents of an + SQLite database, provided as a byte array, under the given name, + overwriting any existing content. Throws if the pool has no + available file slots, on I/O error, or if the input does not + appear to be a database. In the latter case, only a cursory + examination is made. + + Note that this routine is _only_ for importing database files, + not arbitrary files, the reason being that this VFS will + automatically clean up any non-database files so importing them + is pointless. + + Returns undefined. + */ + PoolUtil.importDb = function(name, bytes){ + const n = bytes.byteLength; + if(n<512 || n%512!=0){ + toss("Byte array size is invalid for an SQLite db."); + } + const header = "SQLite format 3"; + for(let i = 0; i < header.length; ++i){ + if( header.charCodeAt(i) !== bytes[i] ){ + toss("Input does not contain an SQLite database header."); + } + } + const sah = SAHPool.mapPathToSAH.get(name) + || SAHPool.nextAvailableSAH() + || toss("No available handles to import to."); + sah.write(bytes, {at: HEADER_OFFSET_DATA}); + SAHPool.setAssociatedPath(sah, name, capi.SQLITE_OPEN_MAIN_DB); + }; + return SAHPool.isReady = SAHPool.reset().then(async ()=>{ if(SAHPool.$error){ throw SAHPool.$error; @@ -678,12 +743,11 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ } //log("vfs list:",capi.sqlite3_js_vfs_list()); sqlite3.vfs.installVfs({ - io: {struct: opfsIoMethods, methods: ioSyncWrappers}, - vfs: {struct: opfsVfs, methods: vfsSyncWrappers}, - applyArgcCheck: true + io: {struct: opfsIoMethods, methods: ioMethods}, + vfs: {struct: opfsVfs, methods: vfsMethods} }); - log("opfsVfs",opfsVfs,"opfsIoMethods",opfsIoMethods); - log("vfs list:",capi.sqlite3_js_vfs_list()); + //log("opfsVfs",opfsVfs,"opfsIoMethods",opfsIoMethods); + //log("vfs list:",capi.sqlite3_js_vfs_list()); if(sqlite3.oo1){ const OpfsSAHPoolDb = function(...args){ const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args); @@ -691,39 +755,14 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ sqlite3.oo1.DB.dbCtorHelper.call(this, opt); }; OpfsSAHPoolDb.prototype = Object.create(sqlite3.oo1.DB.prototype); - OpfsSAHPoolDb.addPoolCapacity = async (n)=>SAHPool.addCapacity(n); - OpfsSAHPoolDb.reducePoolCapacity = async (n)=>SAHPool.reduceCapacity(n); - OpfsSAHPoolDb.getPoolCapacity = ()=>SAHPool.getCapacity(); - OpfsSAHPoolDb.getPoolUsage = ()=>SAHPool.getFileCount(); + OpfsSAHPoolDb.PoolUtil; sqlite3.oo1.OpfsSAHPoolDb = OpfsSAHPoolDb; sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenSql( opfsVfs.pointer, function(oo1Db, sqlite3){ sqlite3.capi.sqlite3_exec(oo1Db, [ - /* As of July 2023, the PERSIST journal mode on OPFS is - somewhat slower than DELETE or TRUNCATE (it was faster - before Chrome version 108 or 109). TRUNCATE and DELETE - have very similar performance on OPFS. - - Roy Hashimoto notes that TRUNCATE and PERSIST modes may - decrease OPFS concurrency because multiple connections - can open the journal file in those modes: - - https://github.com/rhashimoto/wa-sqlite/issues/68 - - Given that, and the fact that testing has not revealed - any appreciable difference between performance of - TRUNCATE and DELETE modes on OPFS, we currently (as of - 2023-07-13) default to DELETE mode. - */ + /* See notes in sqlite3-vfs-opfs.js */ "pragma journal_mode=DELETE;", - /* - OPFS benefits hugely from cache on moderate/large - speedtest1 --size 50 and --size 100 workloads. We - currently rely on setting a non-default cache size when - building sqlite3.wasm. If that policy changes, the cache - can be set here. - */ "pragma cache_size=-16384;" ], 0, 0, 0); } diff --git a/ext/wasm/speedtest1-worker.js b/ext/wasm/speedtest1-worker.js index 17169bc553..6cd0c1af00 100644 --- a/ext/wasm/speedtest1-worker.js +++ b/ext/wasm/speedtest1-worker.js @@ -107,7 +107,10 @@ const S = globalThis.S = sqlite3; log("Loaded speedtest1 module. Setting up..."); if(S.installOpfsSAHPoolVfs){ - await S.installOpfsSAHPoolVfs().catch(e=>{ + await S.installOpfsSAHPoolVfs().then(P=>{ + S.SAHPoolUtil = P; + //return P.addCapacity(5).then(log("pool capacity:",P.getCapacity()));; + }).catch(e=>{ logErr("Error setting up opfs-sahpool:",e.message); }); } diff --git a/manifest b/manifest index 524f846225..78b72b420f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\scleanups\sin\sthe\sopfs-sahpool\sVFS. -D 2023-07-15T21:08:48.986 +C Redefine\swhat\sthe\sopfs-sahpool\sinstallation\spromise\sresolves\sto.\sFix\saddCapacity().\sAdd\sutility\smethods\sto\simport/export\sfiles. +D 2023-07-16T10:02:41.870 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -502,7 +502,7 @@ F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b17386 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 961bbc3ccc1fa4e91d6519a96e8811ad7ae60173bd969fee7775dacb6eee1da2 F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js 0032097c168c8fe7e753abc5b35e65323116d04b0dbaaa97176604660b7bb98c +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js 0e982cc4f1b9ed4786086d1115e740b7efd628de5cbaf16caf3a71913f91241b F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js a5c3195203e6085d7aa89fae4b84cf3f3eec4ff4f928c6d0e5d3ef8b14cbc1c0 F ext/wasm/api/sqlite3-wasm.c 12a096d8e58a0af0589142bae5a3c27a0c7e19846755a1a37d2c206352fbedda F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f @@ -540,7 +540,7 @@ F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd84223150 F ext/wasm/speedtest1-wasmfs.html 0e9d335a9b5b5fafe6e1bc8dc0f0ca7e22e6eb916682a2d7c36218bb7d67379d F ext/wasm/speedtest1-wasmfs.mjs ac5cadbf4ffe69e9eaac8b45e8523f030521e02bb67d654c6eb5236d9c456cbe F ext/wasm/speedtest1-worker.html bbcf1e7fd79541040c1a7ca2ebf1cb7793ddaf9900d6bde1784148f11b807c34 -F ext/wasm/speedtest1-worker.js 554b0985f791758e40ff2b1a04b771e315ab84b4e26b4b8a1c7a5ba968086c45 +F ext/wasm/speedtest1-worker.js faa4a06ec21921aaa0e0b672a94b56037da837e16732bdd6545b99f1cadbb32e F ext/wasm/speedtest1.html ff048b4a623aa192e83e143e48f1ce2a899846dd42c023fdedc8772b6e3f07da F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 41bf1fe31f2f3d0daa2bac25dc57262a4b90f22fed6fa97e4e92467c32ae02dc -R a6cacf00d5cb9e3eccf543ab6eefecd8 +P 279e09070918dab7b60c39179ebb7eb931ca6bd4e589b414f436740499a2f910 +R 3700fe28ac7e053b813d4612e5083eb3 U stephan -Z d51af2a044894ea09e404a3df277218e +Z 2285d7e58406d05c66733e55ed60721b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3be8e1f2dd..3cd58eacd0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -279e09070918dab7b60c39179ebb7eb931ca6bd4e589b414f436740499a2f910 \ No newline at end of file +809c6f4de3653ad7a7751af45a7a0d6cb20c3ee3be80c69833c729242227d970 \ No newline at end of file -- 2.47.2