From: stephan Date: Sun, 30 Nov 2025 05:20:17 +0000 (+0000) Subject: Extend the kvvfs.listen() config to enable posting of raw binary db pages instead... X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=9f83f4c5634ca14d184dd6f9f518a546be679635;p=thirdparty%2Fsqlite.git Extend the kvvfs.listen() config to enable posting of raw binary db pages instead of the kvvfs-encoding. This is much more expensive but was added to... Demonstrate basic async streaming of kvvfs db page-level changes via logging of kvvfs write/delete ops. FossilOrigin-Name: 0f2bad285577c26f1185dcafd3b8ca2f16e74aa9dc40e6e23867150bccee4602 --- diff --git a/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js index 64d96bb5e6..4f0af5d186 100644 --- a/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js @@ -396,7 +396,12 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return e; }; - const noop = ()=>{}; + const catchForNotify = (e)=>{ + warn("kvvfs.listener handler threw:",e); + }; + + const kvvfsDecode = wasm.exports.sqlite3__wasm_kvvfs_decode; + const kvvfsEncode = wasm.exports.sqlite3__wasm_kvvfs_encode; /** Listener events and their argument(s): @@ -411,16 +416,48 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ */ const notifyListeners = async function(eventName,store,...args){ if( store.listeners ){ - const ev = Object.create(null); - ev.storageName = store.jzClass; - ev.type = eventName; + //cache.rxPageNoSuffix ??= /(\d+)$/; + if( store.keyPrefix ){ + args[0] = args[0].replace(store.keyPrefix,''); + } + let u8enc, z0, z1, wcache; for(const ear of store.listeners){ - const f = ear?.[eventName]; + const ev = Object.create(null); + ev.storageName = store.jzClass; + ev.type = eventName; + const decodePages = ear.decodePages; + const f = ear.events[eventName]; if( f ){ - ev.data = ((args.length===1) ? args[0] : args); - try{f(ev)?.catch?.(noop)} + if( ear.elideJournal && args[0]==='jrnl' ){ + continue; + } + if( 'write'===eventName && ear.decodePages && +args[0]>0 ){ + /* Decode pages to Uint8Array. */ + ev.data = [args[0]]; + if( wcache?.[args[0]] ){ + ev.data[1] = wcache[args[0]]; + continue; + } + u8enc ??= new TextEncoder('utf-8'); + z0 ??= cache.memBuffer(10); + z1 ??= cache.memBuffer(11); + const u = u8enc.encode(args[1]); + const heap = wasm.heap8u(); + heap.set(u, z0); + heap[wasm.ptr.add(z0, u.length)] = 0; + const rc = kvvfsDecode(z0, z1, cache.buffer.n); + if( rc>0 ){ + wcache ??= Object.create(null); + wcache[args[0]] + = ev.data[1] + = heap.slice(z1, wasm.ptr.add(z1,rc)); + } + }else{ + ev.data = ((args.length===1) ? args[0] : args); + } + try{f(ev)?.catch?.(catchForNotify)} catch(e){ - warn("notifyListeners",store.jzClass,eventName,e); + warn("notifyListeners [",store.jzClass,"]",eventName,e); } } } @@ -483,12 +520,12 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const kvvfsMakeKey = wasm.exports.sqlite3__wasm_kvvfsMakeKey; /** - Returns a C string from sqlite3__wasm_kvvfsMakeKey() OR returns - zKey. In the former case the memory is static, so must be copied - before a second call. zKey MUST be a pointer passed to a - VFS/file method, to allow us to avoid an alloc and/or an - snprintf(). It requires C-string arguments for zClass and - zKey. zClass may be NULL but zKey may not. + Returns a C string from kvvfsMakeKey() OR returns zKey. In the + former case the memory is static, so must be copied before a + second call. zKey MUST be a pointer passed to a VFS/file method, + to allow us to avoid an alloc and/or an snprintf(). It requires + C-string arguments for zClass and zKey. zClass may be NULL but + zKey may not. */ const zKeyForStorage = (store, zClass, zKey)=>{ //debug("zKeyForStorage(",store, wasm.cstrToJs(zClass), wasm.cstrToJs(zKey)); @@ -615,8 +652,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ xRcrdWrite: (zClass, zKey, zData)=>{ try { - const jzClass = wasm.cstrToJs(zClass); - const store = storageForZClass(jzClass); + const store = storageForZClass(zClass); const jxKey = jsKeyForStorage(store, zClass, zKey); const jData = wasm.cstrToJs(zData); store.storage.setItem(jxKey, jData); @@ -1056,7 +1092,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ (JSON-friendly). - "includeJournal" (bool=false). If true and the db has a current - journal, it is exported as well. + journal, it is exported as well. (Kvvfs journals are stored as a + single record within the db's storage object.) The returned object is structured as follows... @@ -1071,7 +1108,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ - "size": the unencoded db size. - - "journal": if includeJournal is true and this db has a + - "journal": if options.includeJournal is true and this db has a journal, it is stored as a string here, otherwise this property is not set. @@ -1094,16 +1131,16 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ let opt; if( 1===args.length && 'object'===typeof args[0] ){ opt = args[0]; - }else{ - opt = { + }else if(args.length){ + opt = Object.assign(Object.create(null),{ name: args[0], //expandPages: true - }; + }); } - const store = storageForZClass(opt.name); + const store = opt ? storageForZClass(opt.name) : null; if( !store ){ toss3(capi.SQLITE_NOTFOUND, - "There is no kvvfs storage named",opt.name); + "There is no kvvfs storage named",opt?.name); } //debug("store to export=",store); const s = store.storage; @@ -1147,7 +1184,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } heap[wasm.ptr.add(z, i)] = 0; //debug("Decoding",i,"page bytes"); - const nDec = wasm.exports.sqlite3__wasm_kvvfs_decode( + const nDec = kvvfsDecode( z, zDec, cache.buffer.n ); if( cache.fixedPageSize !== nDec ){ @@ -1202,10 +1239,15 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ if( !exp?.timestamp || !exp.name || undefined===exp.size - || exp.size<0 || exp.size>=0x7fffffff || !Array.isArray(exp.pages) ){ toss3(capi.SQLITE_MISUSE, "Malformed export object."); + }else if( !exp.size + || (exp.size !== (exp.size | 0)) + || (exp.size % cache.fixedPageSize) + || exp.size>=0x7fffffff ){ + toss3(capi.SQLITE_RANGE, "Invalid db size: "+exp.size); } + validateStorageName(exp.name); let store = storageForZClass(exp.name); const isNew = !store; @@ -1245,16 +1287,15 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ if( cache.fixedPageSize !== n ){ util.toss3(capi.SQLITE_RANGE,"Unexpected page size:", n); } - zEnc = cache.memBuffer(1); + zEnc ??= cache.memBuffer(1); const zBin = cache.memBuffer(0), - heap = wasm.heap8u(); - /* Copy u to the heap and encode the heaped copy. This is - _presumably_ faster than porting the encoding algo to - JS, which would involve many, many more function calls. */ - let i; - for(i=0; iv!==opt.events); + store.listeners = store.listeners.filter((v)=>v!==opt); if( !store.listeners.length ){ store.listeners = undefined; } diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 7b395098dd..72d207e3b2 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -2912,7 +2912,11 @@ globalThis.sqlite3InitModule = sqlite3InitModule; if( 0 ){ const scope = wasm.scopedAllocPush(); try{ - const pg = "53514C69746520666F726D61742033b20b0101b402020d02d02l01d04l01jb02b2E91E00Dd011FD3b1FD3dxl2B010617171701377461626C656B767666736B7676667302435245415445205441424C45206B76766673286129"; + const pg = [ + "53514C69746520666F726D61742033b20b0101b402020d02d02l01d0", + "4l01jb02b2E91E00Dd011FD3b1FD3dxl2B010617171701377461626C", + "656B767666736B7676667302435245415445205441424C45206B76766673286129" + ].join(''); const n = pg.length; const pI = wasm.scopedAlloc( n+1 ); const nO = 8192 * 2; @@ -3240,6 +3244,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; } } }/*concurrent transient kvvfs*/) + .t({ name: 'kvvfs listeners (experiment)', test: function(sqlite3){ @@ -3254,9 +3259,25 @@ globalThis.sqlite3InitModule = sqlite3InitModule; const counts = Object.assign(Object.create(null),{ open: 0, close: 0, delete: 0, write: 0 }); + const pglog = Object.create(null); + pglog.pages = []; + pglog.jrnl = undefined; + pglog.size = undefined; + pglog.elideJournal = true; + //pglog.decodePages = true; + pglog.exception = new Error("Testing that exceptions from listeners do not interfere"); + const toss = ()=>{ + if( pglog.exception ){ + const e = pglog.exception; + delete pglog.exception; + throw e; + } + }; const listener = { storage: filename, reserve: true, + elideJournal: pglog.elideJournal, + decodePages: pglog.decodePages, events: { 'open': (ev)=>{ //console.warn('open',ev); @@ -3265,21 +3286,58 @@ globalThis.sqlite3InitModule = sqlite3InitModule; .assert('number'===typeof ev.data); }, 'close': (ev)=>{ + //^^^ if this is async, we can't time the test for + // pglog.exception without far more hoop-jumping. //console.warn('close',ev); ++counts[ev.type]; T.assert('number'===typeof ev.data); + toss(); }, 'delete': (ev)=>{ //console.warn('delete',ev); ++counts[ev.type]; T.assert('string'===typeof ev.data); + switch(ev.data){ + case 'jrnl': + T.assert(!pglog.elideJournal); + pglog.jrnl = null; + break; + default: + T.assert( +ev.data>0, "Expecting positive db page number" ); + pglog[+ev.data] = undefined; + break; + } }, 'write': (ev)=>{ //console.warn('write',ev); ++counts[ev.type]; + const key = ev.data[0], val = ev.data[1]; T.assert(Array.isArray(ev.data)) - .assert('string'===typeof ev.data[0]) - .assert('string'===typeof ev.data[1]); + .assert('string'===typeof key); + switch( key ){ + case 'jrnl': + T.assert(!pglog.elideJournal); + pglog.jrnl = val; + break; + case 'sz':{ + const sz = +val; + T.assert( sz>0, "Expecting a db page number" ); + if( sz < pglog.sz ){ + pglog.pages.length = sz / pglog.pages.length; + } + pglog.size = sz; + break; + } + default: + T.assert( +key>0, "Expecting a positive db page number" ); + pglog.pages[+key] = val; + if( pglog.decodePages ){ + T.assert( val instanceof Uint8Array ); + }else{ + T.assert( 'string'===typeof val ); + } + break; + } } } }; @@ -3291,9 +3349,18 @@ globalThis.sqlite3InitModule = sqlite3InitModule; debug("kvvfs listener counts:",counts); T.assert( counts.open ) .assert( counts.close ) - .assert( counts.delete ) + .assert( listener.elideJournal ? !counts.delete : counts.delete ) .assert( counts.write ) - .assert( counts.open===counts.close ); + .assert( counts.open===counts.close ) + .assert( pglog.elideJournal + ? (undefined===pglog.jrnl) + : (null===pglog.jrnl), + "Unexpected pglog.jrnl value: "+pglog.jrnl ); + if( 1 ){ + T.assert(undefined===pglog.pages[0], "Expecting empty slot 0"); + pglog.pages.shift(); + debug("kvvfs listener pageLog", pglog); + } const before = JSON.stringify(counts); sqlite3.kvvfs.unlisten(listener); db = new DB(dbFileRaw); diff --git a/manifest b/manifest index 2a8e11a0c6..fad204cc8a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Extend\skvvfs\sexport\sto\soptionally\sexport\sthe\sraw\sbinary\sdb\spages\sas\sa\slist\sof\sUint8Array\sinstead\sof\skvvfs-encoded\sstrings.\sThis\sis\stypically\smuch\slarger\sbut\sthe\spages\scan\sthen\sbe\sused\sas-is. -D 2025-11-30T03:02:06.188 +C Extend\sthe\skvvfs.listen()\sconfig\sto\senable\sposting\sof\sraw\sbinary\sdb\spages\sinstead\sof\sthe\skvvfs-encoding.\sThis\sis\smuch\smore\sexpensive\sbut\swas\sadded\sto...\sDemonstrate\sbasic\sasync\sstreaming\sof\skvvfs\sdb\spage-level\schanges\svia\slogging\sof\skvvfs\swrite/delete\sops. +D 2025-11-30T05:20:17.833 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -600,7 +600,7 @@ F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 9654b565b346dc609b75d15337f20acfa7af7d9d558da1afeb9b6d8eaa404966 F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d -F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js e01f80208b7f00a47045bc92e5c0b40c6e2232fc058f2dd81ed41d46b1cfd323 +F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 3d07cc5dd8b20fa81401c9bea93ab794fad4412dd4982280e695f7fb2f0bffbf F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js a2eea6442556867b589e04107796c6e1d04a472219529eeb45b7cd221d7d048b F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 88ce2078267a2d1af57525a32d896295f4a8db7664de0e17e82dc9ff006ed8d3 F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81 @@ -647,7 +647,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 1618670e466f424aa289859fe0ec8ded223e42e9e69b5c851f809baaaca1a00c F ext/wasm/tester1-worker.c-pp.html 0e432ec2c0d99cd470484337066e8d27e7aee4641d97115338f7d962bf7b081a F ext/wasm/tester1.c-pp.html 52d88fe2c6f21a046030a36410b4839b632f4424028197a45a3d5669ea724ddb -F ext/wasm/tester1.c-pp.js d5bf9dd8be21377981e20974fa798ae9839ee7a279189a773299532a844a8138 +F ext/wasm/tester1.c-pp.js 322e86c7b1627b61f9decdc741052aa52d62733af004055c6b9850365aa7f5bb F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e F ext/wasm/tests/opfs/concurrency/test.js d08889a5bb6e61937d0b8cbb78c9efbefbf65ad09f510589c779b7cc6a803a88 F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2180,8 +2180,8 @@ F tool/version-info.c 33d0390ef484b3b1cb685d59362be891ea162123cea181cb8e6d2cf6dd F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ed9ab366d1c1880d3c06edce6c0c33ad30c7ae59725c1ec1fe3f620be1835630 -R 8530361058afe18d87abefbdbe6a3870 +P a4f59496a53a079f8f73e4cde68f47dbd13d2d74de2ad11bc716e7e5c00f1ec0 +R 657f4aadff300838a0780e1d6467cc6f U stephan -Z 197caaef7aceb5323c153f1905283e67 +Z 248974dfe9e189a43e97fc46a03783b5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ac3cbc9ad1..ffbfc63fea 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a4f59496a53a079f8f73e4cde68f47dbd13d2d74de2ad11bc716e7e5c00f1ec0 +0f2bad285577c26f1185dcafd3b8ca2f16e74aa9dc40e6e23867150bccee4602