From: stephan Date: Sat, 15 Jul 2023 01:02:38 +0000 (+0000) Subject: More work on porting the sahpool bits. Revert to random backing-store names because... X-Git-Tag: version-3.43.0~122^2~33 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=100bc4429a32381b61e8058c559caba5fd33d0c0;p=thirdparty%2Fsqlite.git More work on porting the sahpool bits. Revert to random backing-store names because it works better with the capacity-reduction algorithm. FossilOrigin-Name: b4e005fd38b06b8d2f2317b955b93807e80a6a18db5f06d7747978d3bfa11411 --- diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 9a05ca5455..2b2aeb5e04 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -46,12 +46,12 @@ # $(eval), or at least centralize the setup of the numerous vars # related to each build variant $(JS_BUILD_MODES). # +#default: all +default: quick SHELL := $(shell which bash 2>/dev/null) MAKEFILE := $(lastword $(MAKEFILE_LIST)) CLEAN_FILES := DISTCLEAN_FILES := ./--dummy-- -default: all -#default: quick release: oz # JS_BUILD_MODES exists solely to reduce repetition in documentation # below. diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js index 410986ff27..6bfa5770cb 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js @@ -63,15 +63,14 @@ 'use strict'; globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const installOpfsVfs = async function(sqlite3){ - const pToss = (...args)=>Promise.reject(new Error(args.join(' '))); if(!globalThis.FileSystemHandle || !globalThis.FileSystemDirectoryHandle || !globalThis.FileSystemFileHandle || !globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle || !navigator?.storage?.getDirectory){ - return pToss("Missing required OPFS APIs."); + return Promise.reject(new Error("Missing required OPFS APIs.")); } - const thePromise = new Promise(function(promiseResolve, promiseReject_){ + return new Promise(async function(promiseResolve, promiseReject_){ const verbosity = 3; const loggers = [ sqlite3.config.error, @@ -130,10 +129,13 @@ const installOpfsVfs = async function(sqlite3){ 'cleanup default VFS wrapper', ()=>(dVfs ? dVfs.dispose() : null) ); - const getFilename = function(ndx){ - return 'sahpool-'+('00'+ndx).substr(-3); - } + const getFilename = false + ? (ndx)=>'sahpool-'+('00'+ndx).substr(-3) + : ()=>Math.random().toString(36).slice(2) + /** + All state for the VFS. + */ const SAHPool = Object.assign(Object.create(null),{ /* OPFS dir in which VFS metadata is stored. */ vfsDir: ".sqlite3-opfs-sahpool", @@ -148,24 +150,33 @@ const installOpfsVfs = async function(sqlite3){ addCapacity: async function(n){ const cap = this.getCapacity(); for(let i = cap; i < cap+n; ++i){ - const name = getFilename(i); + const name = getFilename(i) + /* Reminder: because of how removeCapacity() works, we + really want random names. At this point in the dev + process that fills up the OPFS with randomly-named files + each time the page is reloaded, so delay the return to + random names until we've gotten far enough to eliminate + that problem. */; const h = await this.dirHandle.getFileHandle(name, {create:true}); - let ah = await h.createSyncAccessHandle(); - if(0===i){ - /* Ensure that this client has the "all-synchronous" - OPFS API and fail if they don't. */ - if(undefined !== ah.close()){ - toss("OPFS API is too old for opfs-sahpool:", - "it has an async close() method."); - } - ah = await h.createSyncAccessHandle(); - } + const ah = await h.createSyncAccessHandle(); this.mapAH2Name.set(ah,name); this.setAssociatedPath(ah, '', 0); } }, - setAssociatedPath: function(accessHandle, path, flags){ - // TODO + removeCapacity: async function(n){ + let nRm = 0; + for(const ah of Array.from(this.availableAH)){ + if(nRm === n || this.getFileCount() === this.getCapacity()){ + break; + } + const name = this.mapAH2Name.get(ah); + ah.close(); + await this.dirHandle.removeEntry(name); + this.mapAH2Name.delete(ah); + this.availableAH.delete(ah); + ++nRm; + } + return nRm; }, releaseAccessHandles: function(){ for(const ah of this.mapAH2Name.keys()) ah.close(); @@ -174,7 +185,98 @@ const installOpfsVfs = async function(sqlite3){ this.availableAH.clear(); }, acquireAccessHandles: async function(){ - // TODO + const files = []; + for await (const [name,h] of this.dirHandle){ + if('file'===h.kind){ + files.push([name,h]); + } + } + await Promise.all(files.map(async ([name,h])=>{ + const ah = await h.createSyncAccessHandle() + /*TODO: clean up and fail vfs init on error*/; + this.mapAH2Name.set(ah, name); + const path = this.getAssociatedPath(ah); + if(path){ + this.mapPath2AH.set(path, ah); + }else{ + this.availableAH.add(ah); + } + })); + }, + gapBody: new Uint8Array(HEADER_CORPUS_SIZE), + textDecoder: new TextDecoder(), + getAssociatedPath: function(sah){ + const body = this.gapBody; + sah.read(body, {at: 0}); + // Delete any unexpected files left over by previous + // untimely errors... + const dv = new DataView(body.buffer, body.byteOffset); + const flags = dv.getUint32(HEADER_OFFSET_FLAGS); + if(body[0] && + ((flags & capi.SQLITE_OPEN_DELETEONCLOSE) || + (flags & PERSISTENT_FILE_TYPES)===0)){ + warn(`Removing file with unexpected flags ${flags.toString(16)}`); + this.setAssociatedPath(sah, '', 0); + return ''; + } + + const fileDigest = new Uint32Array(HEADER_DIGEST_SIZE / 4); + sah.read(fileDigest, {at: HEADER_OFFSET_DIGEST}); + const compDigest = this.computeDigest(body); + if(fileDigest.every((v,i) => v===compDigest[i])){ + // Valid digest + const pathBytes = body.findIndex((v)=>0===v); + if(0===pathBytes){ + // This file is unassociated, so ensure that it's empty + // to avoid leaving stale db data laying around. + sah.truncate(HEADER_OFFSET_DATA); + } + return this.textDecoder.decode(body.subarray(0,pathBytes)); + }else{ + // Invalid digest + warn('Disassociating file with bad digest.'); + this.setAssociatedPath(sah, '', 0); + return ''; + } + }, + textEncoder: new TextEncoder(), + setAssociatedPath: function(sah, path, flags){ + const body = this.gapBody; + const enc = this.textEncoder.encodeInto(path, body); + if(HEADER_MAX_PATH_SIZE <= enc.written){ + toss("Path too long:",path); + } + + const dv = new DataView(body.buffer, body.byteOffset); + dv.setUint32(HEADER_OFFSET_FLAGS, flags); + + const digest = this.computeDigest(body); + sah.write(body, {at: 0}); + sah.write(digest, {at: HEADER_OFFSET_DIGEST}); + sah.flush(); + + if(path){ + this.mapPath2AH.set(path, sah); + this.availableAH.delete(sah); + }else{ + // This is not a persistent file, so eliminate the contents. + sah.truncate(HEADER_OFFSET_DATA); + this.mapPath2AH.delete(path); + this.availableAH.add(sah); + } + }, + computeDigest: function(byteArray){ + if(!byteArray[0]){ + // Deleted file + return new Uint32Array([0xfecc5f80, 0xaccec037]); + } + let h1 = 0xdeadbeef; + let h2 = 0x41c6ce57; + for(const v of byteArray){ + h1 = 31 * h1 + (v * 307); + h2 = 31 * h2 + (v * 307); + } + return new Uint32Array([h1>>>0, h2>>>0]); }, reset: async function(){ await this.isReady; @@ -187,8 +289,20 @@ const installOpfsVfs = async function(sqlite3){ this.dirHandle = h; this.releaseAccessHandles(); await this.acquireAccessHandles(); + }, + getPath: function(arg) { + if(wasm.isPtr(arg)) arg = wasm.cstrToJs(arg); + return ((arg instanceof URL) + ? arg + : new URL(arg, 'file://localhost/')).pathname; + }, + deletePath: function(path) { + const sah = this.mapPath2AH.get(path); + if(sah) { + // Un-associate the SQLite path from the OPFS file. + this.setAssociatedPath(sah, '', 0); + } } - // much more TODO })/*SAHPool*/; sqlite3.SAHPool = SAHPool/*only for testing*/; // Much, much more TODO... @@ -201,8 +315,21 @@ const installOpfsVfs = async function(sqlite3){ return 0; }, xClose: function(pFile){ - let rc = 0; - return rc; + const file = SAHPool.mapId2File.get(pFile); + if(file) { + try{ + log(`xClose ${file.path}`); + file.sah.flush(); + SAHPool.mapId2File.delete(pFIle); + if(file.flags & capi.SQLITE_OPEN_DELETEONCLOSE){ + SAHPool.deletePath(file.path); + } + }catch(e){ + error("xClose() failed:",e.message); + return capi.SQLITE_IOERR; + } + } + return 0; }, xDeviceCharacteristics: function(pFile){ return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; @@ -211,32 +338,74 @@ const installOpfsVfs = async function(sqlite3){ return capi.SQLITE_NOTFOUND; }, xFileSize: function(pFile,pSz64){ - let rc = 0; - return rc; + const file = SAHPool.mapId2File(pFile); + const size = file.sah.getSize() - HEADER_OFFSET_DATA; + //log(`xFileSize ${file.path} ${size}`); + wasm.poke64(pSz64, BigInt(size)); + return 0; }, xLock: function(pFile,lockType){ - let rc = capi.SQLITE_IOERR_LOCK; + let rc = capi.SQLITE_IOERR; return rc; }, xRead: function(pFile,pDest,n,offset64){ - let rc = capi.SQLITE_IOERR_READ; - return rc; + const file = SAHPool.mapId2File.get(pFile); + log(`xRead ${file.path} ${n} ${offset64}`); + try { + const nRead = file.sah.read( + pDest, {at: HEADER_OFFSET_DATA + offset64} + ); + if(nRead < n){ + wasm.heap8u().fill(0, pDest + nRead, pDest + n); + return capi.SQLITE_IOERR_SHORT_READ; + } + return 0; + }catch(e){ + error("xRead() failed:",e.message); + return capi.SQLITE_IOERR; + } + }, + xSectorSize: function(pFile){ + return SECTOR_SIZE; }, xSync: function(pFile,flags){ - let rc = capi.SQLITE_IOERR_FSYNC; - return rc; + const file = SAHPool.mapId2File.get(pFile); + //log(`xSync ${file.path} ${flags}`); + try{ + file.sah.flush(); + return 0; + }catch(e){ + error("xSync() failed:",e.message); + return capi.SQLITE_IOERR; + } }, xTruncate: function(pFile,sz64){ - let rc = capi.SQLITE_IOERR_TRUNCATE; - return rc; - }, - xUnlock: function(pFile,lockType){ - let rc = capi.SQLITE_IOERR_UNLOCK; - return rc; + const file = SAHPool.mapId2File.get(pFile); + //log(`xTruncate ${file.path} ${iSize}`); + try{ + file.sah.truncate(HEADER_OFFSET_DATA + Number(sz64)); + return 0; + }catch(e){ + error("xTruncate() failed:",e.message); + return capi.SQLITE_IOERR; + } }, + /**xUnlock: function(pFile,lockType){ + return capi.SQLITE_IOERR; + },*/ xWrite: function(pFile,pSrc,n,offset64){ - let rc = capi.SQLITE_IOERR_WRITE; - return rc; + const file = SAHPool.mapId2File(pFile); + //log(`xWrite ${file.path} ${n} ${offset64}`); + try{ + const nBytes = file.sah.write( + pSrc, { at: HEADER_OFFSET_DATA + Number(offset64) } + ); + return nBytes === n ? 0 : capi.SQLITE_IOERR; + return 0; + }catch(e){ + error("xWrite() failed:",e.message); + return capi.SQLITE_IOERR; + } } }/*ioSyncWrappers*/; @@ -246,8 +415,9 @@ const installOpfsVfs = async function(sqlite3){ */ const vfsSyncWrappers = { xAccess: function(pVfs,zName,flags,pOut){ - const rc = capi.SQLITE_ERROR; - return rc; + const name = this.getPath(zName); + wasm.poke32(pOut, SAHPool.mapPath2AH.has(name) ? 1 : 0); + return 0; }, xCurrentTime: function(pVfs,pOut){ wasm.poke(pOut, 2440587.5 + (new Date().getTime()/86400000), @@ -260,8 +430,13 @@ const installOpfsVfs = async function(sqlite3){ return 0; }, xDelete: function(pVfs, zName, doSyncDir){ - const rc = capi.SQLITE_ERROR; - return rc; + try{ + SAHPool.deletePath(SAHPool.getPath(zName)); + return 0; + }catch(e){ + error("Error xDelete()ing file:",e.message); + return capi.SQLITE_IOERR_DELETE; + } }, xFullPathname: function(pVfs,zName,nOut,pOut){ const i = wasm.cstrncpy(pOut, zName, nOut); @@ -301,6 +476,31 @@ const installOpfsVfs = async function(sqlite3){ }; } + /** + Ensure that the client has a "fully-sync" SAH impl, + else reject the promise. Returns true on success, + else false. + */ + const affirmHasSyncAPI = async function(){ + try { + const dh = await navigator.storage.getDirectory(); + const fn = '.opfs-sahpool-sync-check-'+Math.random().toString(36).slice(2); + const fh = await dh.getFileHandle(fn, { create: true }); + const ah = await fh.createSyncAccessHandle(); + const close = ah.close(); + await close; + await dh.removeEntry(fn); + if(close?.then){ + toss("The local OPFS API is too old for opfs-sahpool:", + "it has an async FileSystemSyncAccessHandle.close() method."); + } + return true; + }catch(e){ + promiseReject(e); + return false; + } + }; + if(!(await affirmHasSyncAPI())) return; SAHPool.isReady = SAHPool.reset().then(async ()=>{ if(0===SAHPool.getCapacity()){ await SAHPool.addCapacity(DEFAULT_CAPACITY); @@ -314,8 +514,7 @@ const installOpfsVfs = async function(sqlite3){ log("opfs-sahpool VFS initialized."); promiseResolve(sqlite3); }).catch(promiseReject); - })/*thePromise*/; - return thePromise; + })/*return Promise*/; }/*installOpfsVfs()*/; globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 9af2856022..8b71198669 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -1288,7 +1288,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; if(1){ const vfsList = capi.sqlite3_js_vfs_list(); T.assert(vfsList.length>1); - log("vfsList =",vfsList); + //log("vfsList =",vfsList); wasm.scopedAllocCall(()=>{ const vfsArg = (v)=>wasm.xWrap.testConvertArg('sqlite3_vfs*',v); for(const v of vfsList){ diff --git a/manifest b/manifest index 0e36e7381d..d8a87ebc0b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Switch\sopfs-sahpool\sto\suse\sdeterministic\sbacking-store\sfile\snames.\sDelay\sVFS\sregistration\suntil\safter\sthe\spool's\sfiles\sare\sall\sopened.\sFail\svfs\sinit\sif\sthe\sclient's\sOPFS\sAPI\shas\san\sasync\sFileSystemSyncAccessHandle.close()\smethod\s(indicating\sthat\sit's\soutdated). -D 2023-07-14T23:02:58.852 +C More\swork\son\sporting\sthe\ssahpool\sbits.\sRevert\sto\srandom\sbacking-store\snames\sbecause\sit\sworks\sbetter\swith\sthe\scapacity-reduction\salgorithm. +D 2023-07-15T01:02:38.317 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -482,7 +482,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile b3d23a5f85dd9baff57c525a8fbd8f8ba2cdb41f2988fc93fb5e7746a276d8ab +F ext/wasm/GNUmakefile 1712a1fbf615dea49dfdc9cf9057c268df3a6887930aec8d9249f02a2059f2b2 F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api d6a5078f48a5301ed17b9a30331075d9b2506e1360c1f0dee0c7816c10acd9ab @@ -502,7 +502,7 @@ F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b17386 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 961bbc3ccc1fa4e91d6519a96e8811ad7ae60173bd969fee7775dacb6eee1da2 F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js a54fee5e4112dce167bbcddd496f7817846cc269f859237460a835fe693b9c39 +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js 54beb56c4cc403c4bc9cf38704186e330b38fe6f24a86516a21d9d6239769b05 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 891f3a18d9ac9b0422b32fd975319dfcd0af5a8ca392f0cce850524e51b49c87 F ext/wasm/api/sqlite3-wasm.c 12a096d8e58a0af0589142bae5a3c27a0c7e19846755a1a37d2c206352fbedda F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f @@ -549,7 +549,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js f09266873e1a34d9bdb6d3981ec8c9e382f31f215c9fd2f9016d2394b8ae9b7b F ext/wasm/tester1-worker.html ebc4b820a128963afce328ecf63ab200bd923309eb939f4110510ab449e9814c F ext/wasm/tester1.c-pp.html 1c1bc78b858af2019e663b1a31e76657b73dc24bede28ca92fbe917c3a972af2 -F ext/wasm/tester1.c-pp.js c021334b946f657251b2cc2cde11b2ff6c4f059fd5a0f4f07ea2d7564409c2cd +F ext/wasm/tester1.c-pp.js 4420eb97b6b4fc79e4e156b4b8010dd9f373365f4230dd76d823fb04ce28ffde F ext/wasm/tests/opfs/concurrency/index.html 0802373d57034d51835ff6041cda438c7a982deea6079efd98098d3e42fbcbc1 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ef96e6b586825a2b3ed011174309cba8ce1031876c86dc59ed87ab9bbc64f57f -R 1c797ce49382346884871d0f3bb17cf9 +P 7c6697ededee9a64962ac6fd78934c6d6e39258b9558a03c1a6c02bf3be1759e +R ce448f7649c77f7ac682168b6f7b78d5 U stephan -Z de96eab25d9db47795591a43ac723ba9 +Z 9bbeb2300d5ae5fa66a5962c456126f8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 34a7aa0e8c..04ee2beedf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7c6697ededee9a64962ac6fd78934c6d6e39258b9558a03c1a6c02bf3be1759e \ No newline at end of file +b4e005fd38b06b8d2f2317b955b93807e80a6a18db5f06d7747978d3bfa11411 \ No newline at end of file