From 68230ef670ea6f6019240b8e246b7ba82f6d1d77 Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 25 Nov 2025 05:44:03 +0000 Subject: [PATCH] kvvfs internal cleanups. Experimentally add async event listeners to kvvfs to explore backing up a kvvfs a page at a time. FossilOrigin-Name: f355fd484947a645206c9b9c2fd6fe691455dece7fb1aa5b72cb51a86b39474f --- ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js | 215 ++++++++++++++++++++----- ext/wasm/tester1.c-pp.js | 62 +++++++ manifest | 14 +- manifest.uuid | 2 +- 4 files changed, 245 insertions(+), 48 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js index 003844aeed..f409abaa61 100644 --- a/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js @@ -226,9 +226,14 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ KVVfsFile instances currently using this storage. Managed by xOpen() and xClose(). */ - files: [] + files: [], + listeners: [] }); + const installStorageAndJournal = (store)=> + cache.storagePool[store.jzClass] = + cache.storagePool[store.jzClass+'-journal'] = store; + /** Map of JS-stringified KVVfsFile::zClass names to reference-counted Storage objects. These objects are created in @@ -260,10 +265,10 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ cache.storagePool[k+'-journal'] = orig; } - cache.setError = (e=undefined)=>{ + cache.setError = (e=undefined, dfltErrCode=capi.SQLITE_ERROR)=>{ if( e ){ cache.lastError = e; - return (e.resultCode | 0) || capi.SQLITE_ERROR; + return (e.resultCode | 0) || dfltErrCode; } delete cache.lastError; return 0; @@ -275,6 +280,34 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return e; }; + const noop = ()=>{}; + + /** + Listener events and their argument(s): + + 'open': number of opened handles on this storage. + + 'close': number of opened handles on this storage. + + 'write': key, value + + 'delete': key + */ + const notifyListners = async function(eventName,store,...args){ + store.listeners.forEach((v)=>{ + const f = v?.[eventName]; + if( !f ) return; + const ev = Object.create(null); + ev.storageName = store.jxClass; + ev.type = eventName; + ev.data = ((args.length===1) ? args[0] : args); + try{f(ev)?.catch?.(noop)} + catch(e){ + warn("notifyListener",store.jzClass,eventName,e); + } + }); + }; + /** Returns the storage object mapped to the given string zClass (C-string pointer or JS string). @@ -424,7 +457,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } if( !store ) return -1; const zXKey = zKeyForStorage(store, zClass, zKey); - if(!zXKey) return -3/*OOM*/; + //if(!zXKey) return -3/*OOM*/; const jXKey = wasm.cstrToJs(zXKey); //debug("xRcrdRead zXKey", jzClass, wasm.cstrToJs(zXKey), store ); const jV = store.storage.getItem(jXKey); @@ -471,6 +504,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ heap[wasm.ptr.add(zBuf, nV)] = 0; return nBuf; }catch(e){ + error("kvrecordRead()",e); cache.setError(e); return -2; } @@ -481,15 +515,15 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const jzClass = wasm.cstrToJs(zClass); const store = storageForZClass(jzClass); const zXKey = zKeyForStorage(store, zClass, zKey); - if(!zXKey) return SQLITE_NOMEM; - store.storage.setItem( - wasm.cstrToJs(zXKey), - wasm.cstrToJs(zData) - ); + //if(!zXKey) return SQLITE_NOMEM; + const jxKey = wasm.cstrToJs(zXKey); + const jData = wasm.cstrToJs(zData); + store.storage.setItem(jxKey, jData); + notifyListners('write', store, jxKey, jData); return 0; }catch(e){ error("kvrecordWrite()",e); - return capi.SQLITE_IOERR; + return cache.setError(e, capi.SQLITE_IOERR); } }, @@ -497,12 +531,14 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ try { const store = storageForZClass(zClass); const zXKey = zKeyForStorage(store, zClass, zKey); - if(!zXKey) return capi.SQLITE_NOMEM; - store.storage.removeItem(wasm.cstrToJs(zXKey)); + //if(!zXKey) return capi.SQLITE_NOMEM; + const jxKey = wasm.cstrToJs(zXKey); + store.storage.removeItem(jxKey); + notifyListners('delete', store, jxKey); return 0; }catch(e){ error("kvrecordDelete()",e); - return capi.SQLITE_IOERR; + return cache.setError(e, capi.SQLITE_IOERR); } } }/*recordHandler*/, @@ -575,18 +611,15 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ wasm.poke32(pOutFlags, flags | sqlite3.SQLITE_OPEN_CREATE); util.assert( !f.$isJournal, "Opening a journal before its db? "+jzClass ); /* Map both zName and zName-journal to the same storage. */ - const other = f.$isJournal - ? jzClass.replace(cache.rxJournalSuffix,'') - : jzClass + '-journal'; - s = cache.storagePool[jzClass] - = cache.storagePool[other] - = newStorageObj(jzClass); + const nm = jzClass.replace(cache.rxJournalSuffix,''); + s = newStorageObj(nm); + installStorageAndJournal(s); s.files.push(f); s.deleteAtRefc0 = deleteAt0; - debug("xOpen installed storage handles [", - jzClass, other,"]", s); + debug("xOpen installed storage handle [",nm, nm+"-journal","]", s); } pFileHandles.set(pProtoFile, {storage: s, file: f, jzClass}); + notifyListners('open', s, s.files.length); return 0; }catch(e){ warn("xOpen:",e); @@ -733,6 +766,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } originalIoMethods(h.file).xClose(pFile); h.file.dispose(); + notifyListners('close', s, s.files.length); }else{ /* Can happen if xOpen fails */ } @@ -941,6 +975,20 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return sz * 2 /* because JS uses 2-byte char encoding */; }; + /** + Throws if storage name n is not valid for use as a storage name. + This is intended for the high-level APIs, not the low-level ones. + */ + const validateStorageName = function(n){ + if( cache.rxJournalSuffix.test(n) ){ + toss3(capi.SQLITE_MISUSE, "Storage names may not have a '-journal' suffix."); + } + if( n.length>23 ){ + toss3(capi.SQLITE_RANGE, "Storage name is too long."); + } + // TODO: check all of kvvfs's name constraints + }; + /** Copies the entire contents of the given transient storage object into a JSON-friendly form. The returned object is structured as @@ -1044,11 +1092,12 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ - Malformed input object. - If it throws after starting the import then it clears the - storage before returning, to avoid leaving the db in an - undefined state. It may throw for any of the above-listed - conditions before reaching that step, in which case the db is - not modified. + If it throws after starting the import then it clears the storage + before returning, to avoid leaving the db in an undefined + state. It may throw for any of the above-listed conditions before + reaching that step, in which case the db is not modified. If + exp.name refers to a new storage name then if it throws, the name + does not get installed. */ capi.sqlite3_js_kvvfs_import = function(exp, overwrite=false){ if( !exp?.timestamp @@ -1059,7 +1108,9 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ toss3(capi.SQLITE_MISUSE, "Malformed export object."); } //warn("importFromObject() is incomplete"); + validateStorageName(exp.name); let store = storageForZClass(exp.name); + const isNew = !store; if( store ){ if( !overwrite ){ //warn("Storage exists:",arguments,store); @@ -1075,19 +1126,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } capi.sqlite3_js_kvvfs_clear(exp.name); }else{ - if( cache.rxJournalSuffix.test(exp.name) ){ - /* kvvfs's xOpen() specifically prohibits that db files have a - suffix of "-journal" because it has a very specific meaning - in kvvfs. We report it here, rather than waiting on a - pending xOpen() to catch it, because xOpen() has no way of - reporting an error message. */ - toss3(capi.SQLITE_MISUSE, - "Cowardly refusing to create storage with a", - "'-journal' suffix."); - } store = newStorageObj(exp.name); - cache.storagePool[exp.name] = - cache.storagePool[exp.name+'-journal'] = store; //warn("Installing new storage:",store); } //debug("Importing store",store.poolEntry.files.length, store); @@ -1100,14 +1139,110 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ s.setItem(keyPrefix+'sz', exp.size); if( exp.journal ) s.setItem(keyPrefix+'jrnl', exp.journal); exp.pages.forEach((v,ndx)=>s.setItem(keyPrefix+(ndx+1), v)); - //s.getItem("")/*kludge: for KVVfsStorage to reset its keys*/; + if( isNew ) installStorageAndJournal(store); }catch(e){ - capi.sqlite3_js_kvvfs_clear(exp.name); + if( !isNew ){ + try{capi.sqlite3_js_kvvfs_clear(exp.name);} + catch(ee){/*ignored*/} + } throw e; } return this; }; + /** + Adds an event listener to a kvvfs storage object. The idea is + that this can be used to asynchronously back up one kvvfs storage + object to another or another channel entirely. (The caveat in the + latter case is that kvvfs's format is not readily consumable by + downstream code.) + + Its argument must be an object with the following properties: + + - storage: the name of the kvvfs storage object. + + - reserve [=false]: if true, sqlite3_js_kvvfs_reserve() is used + to ensure that the storage exists. + + - events: an object which may have any of the following + callback function properties: open, close, write, delete. + + Each one of the events callbacks will be called asynchronously + when the given storage performs those operations. They may be + asynchronous. All exceptions, including those via Promises, are + ignored but may trigger warning output on the console. + + Each callback gets passed a single object with the following + properties: + + .type = the same as the name of the callback + + .storageName = the name of the storage object + + .data = callback-dependent: + + - 'open' and 'close' get an integer, the number of + currently-opened handles on the storage. + + - 'write' gets a length-two array holding the key and value which + were written (both strings). + + - 'delete' gets the string-type key of the deleted record. + */ + capi.sqlite3_js_kvvfs_listen = function(opt){ + if( !opt || 'object'!==typeof opt ){ + toss3(capi.SQLITE_MISUSE, "Expecting a listener object."); + } + let store = storageForZClass(opt.storage); + if( !store ){ + if( opt.storage && opt.reserve ){ + capi.sqlite3_js_kvvfs_reserve(opt.storage); + store = storageForZClass(opt.storage); + util.assert(store, + "Unexpectedly cannot fetch reserved storage " + +opt.storage); + }else{ + toss3(capi.SQLITE_NOTFOUND,"No such storage:",opt.storage); + } + } + if( opt.events ){ + store.listeners.push(opt.events); + } + }; + + /** + Removes all kvvfs event listeners for the given options + object. It must be passed the same object instance which was + passed to sqlite3_js_kvvfs_listen(). + + This has no side effects if opt is invalid or is not a match for + any listeners. + */ + capi.sqlite3_js_kvvfs_unlisten = function(opt){ + const store = storageForZClass(opt?.storage); + if( store && opt.events ){ + store.listeners = store.listeners.filter((v)=>v!==opt.events); + } + }; + + /** + If no kvvfs storage exists with the given name, one is + installed. If one exists, its reference count is increased so + that it won't be freed by the closing of a database or journal + file. + + Throws if the name is not valid for a new storage object. + */ + capi.sqlite3_js_kvvfs_reserve = function(name){ + let store = storageForZClass(name); + if( store ){ + ++store.refc; + return; + } + validateStorageName(name); + installStorageAndJournal(newStorageObj(name)); + }; + if(sqlite3?.oo1?.DB){ /** Functionally equivalent to DB(storageName,'c','kvvfs') except diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 11eb440e66..f08f487a55 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -3059,6 +3059,68 @@ globalThis.sqlite3InitModule = sqlite3InitModule; } } }/*concurrent transient kvvfs*/) + .t({ + name: 'kvvfs listeners (experiment)', + test: function(sqlite3){ + let db; + try { + const filename = 'listen'; + const DB = sqlite3.oo1.DB; + const sqlSetup = [ + 'create table kvvfs(a);', + 'insert into kvvfs(a) values(1),(2),(3)' + ]; + const counts = Object.assign(Object.create(null),{ + open: 0, close: 0, delete: 0, write: 0 + }); + const listener = { + storage: filename, + reserve: true, + events: { + 'open': (ev)=>{ + ++counts[ev.type]; + T.assert('number'===typeof ev.data); + }, + 'close': (ev)=>{ + ++counts[ev.type]; + T.assert('number'===typeof ev.data); + }, + 'delete': (ev)=>{ + ++counts[ev.type]; + T.assert('string'===typeof ev.data); + }, + 'write': (ev)=>{ + ++counts[ev.type]; + T.assert(Array.isArray(ev.data)) + .assert('string'===typeof ev.data[0]) + .assert('string'===typeof ev.data[1]); + } + } + }; + capi.sqlite3_js_kvvfs_listen(listener); + const dbFileRaw = 'file:'+filename+'?vfs=kvvfs&delete-on-close=1'; + db = new DB(dbFileRaw); + db.exec(sqlSetup); + db.close(); + console.debug("kvvfs listener counts:",counts); + T.assert( counts.open ) + .assert( counts.close ) + .assert( counts.delete ) + .assert( counts.write ) + .assert( counts.open===counts.close ); + const before = JSON.stringify(counts); + capi.sqlite3_js_kvvfs_unlisten(listener); + db = new DB(dbFileRaw); + db.exec("delete from kvvfs"); + db.close(); + const after = JSON.stringify(counts); + T.assert( before===after, "Expecting no events after unlistening." ); + }finally{ + db?.close?.(); + } + + } + })/*kvvfs listeners //#if enable-see .t({ name: 'kvvfs SEE encryption in sessionStorage', diff --git a/manifest b/manifest index c56d923a27..7e2ad3f333 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Factor\sout\ssome\snow-superfluous\sJS-side\skvvfs\scode.\sFactor\sout\sa\ssuperfluous\sallocation.\sShorten\sthe\spublic\sAPI\snames\sof\sthe\snew\smethods. -D 2025-11-25T04:07:50.280 +C kvvfs\sinternal\scleanups.\sExperimentally\sadd\sasync\sevent\slisteners\sto\skvvfs\sto\sexplore\sbacking\sup\sa\skvvfs\sa\spage\sat\sa\stime. +D 2025-11-25T05:44:03.837 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -600,7 +600,7 @@ F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 9654b565b346dc609b75d15337f20acfa7af7d9d558da1afeb9b6d8eaa404966 F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d -F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js d6336194ad6b0516fc452f1dee4b3fdb59df09b2c7f5de5d00f3ce1f27c080de +F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 4ca1a1b99f0e6e8fcf37eed52487a1f850d557123607030b9f9b4f5e573b8589 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js a2eea6442556867b589e04107796c6e1d04a472219529eeb45b7cd221d7d048b F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 88ce2078267a2d1af57525a32d896295f4a8db7664de0e17e82dc9ff006ed8d3 F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 9097074724172e31e56ce20ccd7482259cf72a76124213cbc9469d757676da86 @@ -647,7 +647,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 1618670e466f424aa289859fe0ec8ded223e42e9e69b5c851f809baaaca1a00c F ext/wasm/tester1-worker.c-pp.html 0e432ec2c0d99cd470484337066e8d27e7aee4641d97115338f7d962bf7b081a F ext/wasm/tester1.c-pp.html 52d88fe2c6f21a046030a36410b4839b632f4424028197a45a3d5669ea724ddb -F ext/wasm/tester1.c-pp.js 433d6c9417f920ae5fd2fb09ff665f01893ef6b70d96e622a3e8efe097c87de1 +F ext/wasm/tester1.c-pp.js 157df4565b400704ce1b605e41d706ebc97555e719131f714e39e2498eeb1ba6 F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e F ext/wasm/tests/opfs/concurrency/test.js d08889a5bb6e61937d0b8cbb78c9efbefbf65ad09f510589c779b7cc6a803a88 F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2178,8 +2178,8 @@ F tool/version-info.c 33d0390ef484b3b1cb685d59362be891ea162123cea181cb8e6d2cf6dd F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 357cc42633efb85c3ca9bc3d6d46430e1ecaf2825e6bdd7d7b4e0f6865d0b599 -R 44c9ff67e944e4eb1b2a540fd288593c +P be435b668f1aee56fc4965592c207de25283de238fe89002f1a68ba0567aca65 +R a40d02997945aa1084905b681811107d U stephan -Z f1beb3859dfbeb8ae21ff433c8ab6e9d +Z 28f21285f9b3750dac51a6c728b3cdad # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2518f8dff1..79e5fb3f7b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -be435b668f1aee56fc4965592c207de25283de238fe89002f1a68ba0567aca65 +f355fd484947a645206c9b9c2fd6fe691455dece7fb1aa5b72cb51a86b39474f -- 2.47.3