From: stephan Date: Sun, 30 Nov 2025 07:08:45 +0000 (+0000) Subject: Add sync events to kvvfs.listen() so that streaming can maybe get a better sense... X-Git-Tag: artiphishell~117^2~15 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=9dd6979f3eba1155a5d8e9cf43f29aa00f864335;p=thirdparty%2Fsqlite.git Add sync events to kvvfs.listen() so that streaming can maybe get a better sense of when it's okay to process the stream. FossilOrigin-Name: a4b6dadffa1b43aadebfa06a1de14d763efd18b16c4a3e87f1bd039fbff2524d --- diff --git a/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js index 626aabd9bf..6b50be885b 100644 --- a/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js @@ -404,7 +404,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const kvvfsEncode = wasm.exports.sqlite3__wasm_kvvfs_encode; /** - Listener events and their argument(s): + Listener events and their argument(s) (via the callback(ev) + ev.data member): 'open': number of opened handles on this storage. @@ -413,11 +414,14 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ 'write': key, value 'delete': key + + 'sync': true if it's from xSync(), false if it's from + xFileControl(). */ const notifyListeners = async function(eventName,store,...args){ if( store.listeners ){ //cache.rxPageNoSuffix ??= /(\d+)$/; - if( store.keyPrefix ){ + if( store.keyPrefix && args[0] ){ args[0] = args[0].replace(store.keyPrefix,''); } let u8enc, z0, z1, wcache; @@ -451,9 +455,13 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ wcache[args[0]] = ev.data[1] = heap.slice(z1, wasm.ptr.add(z1,rc)); + }else{ + continue; } }else{ - ev.data = ((args.length===1) ? args[0] : args); + ev.data = args.length + ? ((args.length===1) ? args[0] : args) + : undefined; } try{f(ev)?.catch?.(catchForNotify)} catch(e){ @@ -721,10 +729,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ //no if( true===deleteAt0 ) s.deleteAtRefc0 = true; s.files.push(f); }else{ - /* TODO: a url flag which tells it to keep the storage - around forever so that future xOpen()s get the same - Storage-ish objects. We can accomplish that by - simply increasing the refcount once more. */ wasm.poke32(pOutFlags, flags | sqlite3.SQLITE_OPEN_CREATE); util.assert( !f.$isJournal, "Opening a journal before its db? "+jzClass ); /* Map both zName and zName-journal to the same storage. */ @@ -735,7 +739,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ s.deleteAtRefc0 = deleteAt0 || !!(capi.SQLITE_OPEN_DELETEONCLOSE & flags); //debug("xOpen installed storage handle [",nm, nm+"-journal","]", s); } - pFileHandles.set(pProtoFile, {storage: s, file: f, jzClass}); + pFileHandles.set(pProtoFile, {store: s, file: f, jzClass}); notifyListeners('open', s, s.files.length); return 0; }catch(e){ @@ -748,10 +752,12 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ cache.popError(); try{ const jzName = wasm.cstrToJs(zName); - //debug("xDelete", jzName, storageForZClass(jzName)); if( cache.rxJournalSuffix.test(jzName) ){ recordHandler.xRcrdDelete(zName, cache.zKeyJrnl); - } + }/* + else: historically not done, but maybe otherwise delete + all db pages from storageForZClass(zName)? + */ return 0; }catch(e){ warn("xDelete",e); @@ -829,12 +835,13 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ioDb:{ /* sqlite3_kvvfs_methods::pIoDb's methods */ xClose: function(pFile){ + cache.popError(); try{ const h = pFileHandles.get(pFile); //debug("xClose", pFile, h); if( h ){ pFileHandles.delete(pFile); - const s = storageForZClass(h.jzClass); + const s = h.store;//storageForZClass(h.jzClass); s.files = s.files.filter((v)=>v!==h.file); if( --s.refc<=0 && s.deleteAtRefc0 ){ deleteStorage(s); @@ -853,9 +860,11 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }, xFileControl: function(pFile, opId, pArg){ + cache.popError(); try{ const h = pFileHandles.get(pFile); util.assert(h, "Missing KVVfsFile handle"); + //debug("xFileControl",h,opId); if( opId===capi.SQLITE_FCNTL_PRAGMA ){ /* pArg== length-3 (char**) */ const zName = wasm.peekPtr(wasm.ptr.add(pArg, wasm.ptr.size)); @@ -881,18 +890,36 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } } } - return originalIoMethods(h.file).xFileControl(pFile, opId, pArg); + const rc = originalIoMethods(h.file).xFileControl(pFile, opId, pArg); + if( 0==rc && capi.SQLITE_FCNTL_SYNC===opId ){ + notifyListeners('sync', h.store, false); + } + return rc; }catch(e){ error("xFileControl",e); return cache.setError(e); } }, + xSync: function(pFile,flags){ + cache.popError(); + try{ + const h = pFileHandles.get(pFile); + //debug("xSync",h); + util.assert(h, "Missing KVVfsFile handle"); + const rc = originalIoMethods(h.file).xSync(pFile, flags); + if( 0==rc ) notifyListeners('sync', h.store, true); + return rc; + }catch(e){ + error("xSync",e); + return cache.setError(e); + } + } + //#if nope xRead: function(pFile,pTgt,n,iOff64){}, xWrite: function(pFile,pSrc,n,iOff64){}, xTruncate: function(pFile,i64){}, - xSync: function(pFile,flags){}, xFileSize: function(pFile,pi64Out){}, xLock: function(pFile,iLock){}, xUnlock: function(pFile,iLock){}, @@ -1439,6 +1466,12 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ - 'delete' gets the string-type key of the deleted record. + - 'sync' receives + one argument: true if it was triggered by db file's + xSync(), false if it was triggered by xFileControl() (which + triggers before the xSync() and also triggers is the DB has + PRAGMA SYNCHRONOUS=OFF. + The arguments to 'write', and keys to 'delete', are in one of the following forms: diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 3c51730020..f42dcb2bee 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -3260,9 +3260,8 @@ globalThis.sqlite3InitModule = sqlite3InitModule; ]; const sqlCount = "select count(*) from kvvfs"; const sqlSelectSchema = "select * from sqlite_schema"; - const counts = Object.assign(Object.create(null),{ - open: 0, close: 0, delete: 0, write: 0 - }); + const counts = Object.create(null); + const incr = (key)=>counts[key] = 1 + (counts[key] ?? 0); const pglog = Object.create(null); pglog.pages = []; pglog.jrnl = undefined; @@ -3277,6 +3276,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; throw e; } }; + const listener = { storage: filename, reserve: true, @@ -3285,7 +3285,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; events: { 'open': (ev)=>{ //console.warn('open',ev); - ++counts[ev.type]; + incr(ev.type); T.assert(filename===ev.storageName) .assert('number'===typeof ev.data); }, @@ -3293,13 +3293,13 @@ globalThis.sqlite3InitModule = sqlite3InitModule; //^^^ if this is async, we can't time the test for // pglog.exception without far more hoop-jumping. //console.warn('close',ev); - ++counts[ev.type]; + incr(ev.type); T.assert('number'===typeof ev.data); toss(); }, 'delete': (ev)=>{ //console.warn('delete',ev); - ++counts[ev.type]; + incr(ev.type); T.assert('string'===typeof ev.data); switch(ev.data){ case 'jrnl': @@ -3317,9 +3317,12 @@ globalThis.sqlite3InitModule = sqlite3InitModule; } } }, + 'sync': (ev)=>{ + incr(ev.data ? 'xSync' : 'xFileControlSync'); + }, 'write': (ev)=>{ //console.warn('write',ev); - ++counts[ev.type]; + incr(ev.type); const key = ev.data[0], val = ev.data[1]; T.assert(Array.isArray(ev.data)) .assert('string'===typeof key); @@ -3350,6 +3353,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; } } }; + kvvfs.listen(listener); const dbFileRaw = 'file:'+filename+'?vfs=kvvfs&delete-on-close=1'; const expOpt = { @@ -3362,12 +3366,14 @@ globalThis.sqlite3InitModule = sqlite3InitModule; "Unexpected empty schema"); db.close(); debug("kvvfs listener counts:",counts); - T.assert( counts.open ) - .assert( counts.close ) - .assert( listener.elideJournal ? !counts.delete : counts.delete ) - .assert( counts.write ) - .assert( counts.open===counts.close ) - .assert( pglog.elideJournal + T.assert( counts.open ); + T.assert( counts.close ); + T.assert( listener.elideJournal ? !counts.delete : counts.delete ); + T.assert( counts.write ); + T.assert( counts.xSync ); + T.assert( counts.xFileControlSync>=counts.xSync ); + T.assert( counts.open===counts.close ); + T.assert( pglog.elideJournal ? (undefined===pglog.jrnl) : (null===pglog.jrnl), "Unexpected pglog.jrnl value: "+pglog.jrnl ); @@ -3377,8 +3383,8 @@ globalThis.sqlite3InitModule = sqlite3InitModule; //debug("kvvfs listener pageLog", pglog); } const before = JSON.stringify(counts); - T.assert( kvvfs.unlisten(listener) ) - .assert( !kvvfs.unlisten(listener) ); + T.assert( kvvfs.unlisten(listener) ); + T.assert( !kvvfs.unlisten(listener) ); db = new DB(dbFileRaw); T.assert( db.selectObjects(sqlSelectSchema)?.length>0 ); const exp = kvvfs.export(expOpt); diff --git a/manifest b/manifest index 2d269d57af..319bc9f3de 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C More\swork\son\sthe\skvvfs\sevents.\sDemonstrate\sre-importing\sa\sstreamed-out\sset\sof\sdb\spages. -D 2025-11-30T06:20:28.183 +C Add\ssync\sevents\sto\skvvfs.listen()\sso\sthat\sstreaming\scan\smaybe\sget\sa\sbetter\ssense\sof\swhen\sit's\sokay\sto\sprocess\sthe\sstream. +D 2025-11-30T07:08:45.236 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -600,7 +600,7 @@ F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 9654b565b346dc609b75d15337f20acfa7af7d9d558da1afeb9b6d8eaa404966 F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d -F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 4785535033eff734b4a9115edcf4d09b58b51a5830cc3a54f2ded9fff51a8c75 +F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 92e838b2b1b3039d14ab43f23a99c553df5bd967525457335e2d498f4b72f632 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js a2eea6442556867b589e04107796c6e1d04a472219529eeb45b7cd221d7d048b F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 88ce2078267a2d1af57525a32d896295f4a8db7664de0e17e82dc9ff006ed8d3 F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81 @@ -647,7 +647,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 1618670e466f424aa289859fe0ec8ded223e42e9e69b5c851f809baaaca1a00c F ext/wasm/tester1-worker.c-pp.html 0e432ec2c0d99cd470484337066e8d27e7aee4641d97115338f7d962bf7b081a F ext/wasm/tester1.c-pp.html 52d88fe2c6f21a046030a36410b4839b632f4424028197a45a3d5669ea724ddb -F ext/wasm/tester1.c-pp.js dfffa59662eb6ae04be40b65c34becb864edb74d1439f3f8a850c0102546ebe6 +F ext/wasm/tester1.c-pp.js cb0b1a07bd19af014e62343b973c61be312ef91f612b83db074b1542969d64e3 F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e F ext/wasm/tests/opfs/concurrency/test.js d08889a5bb6e61937d0b8cbb78c9efbefbf65ad09f510589c779b7cc6a803a88 F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2180,8 +2180,8 @@ F tool/version-info.c 33d0390ef484b3b1cb685d59362be891ea162123cea181cb8e6d2cf6dd F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P bbd08b67ceeaae2a3a82655da5a3983f4e9057424de4223f1505ba74c36238af -R 67172ddc08db9584ebb1618c06504a62 +P b8fee839b0199d1d2714c99612ea4b4d943d8b0033d742eaa66e1575ce46f0d2 +R fb9f8ede3e2ffe33cbb590ee3d7d0cdc U stephan -Z f1288e031bc5ab35fe9740e0ceeda37e +Z a9c442087c9256ab328a4323b6416791 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 033877a2d0..611d42e9a1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b8fee839b0199d1d2714c99612ea4b4d943d8b0033d742eaa66e1575ce46f0d2 +a4b6dadffa1b43aadebfa06a1de14d763efd18b16c4a3e87f1bd039fbff2524d