From 3dbe1f2f9637732bbe8c699a0d09b2f0083c1027 Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 30 Nov 2025 06:20:28 +0000 Subject: [PATCH] More work on the kvvfs events. Demonstrate re-importing a streamed-out set of db pages. FossilOrigin-Name: b8fee839b0199d1d2714c99612ea4b4d943d8b0033d742eaa66e1575ce46f0d2 --- ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js | 22 +++++--- ext/wasm/tester1.c-pp.js | 73 ++++++++++++++++++++------ manifest | 14 ++--- manifest.uuid | 2 +- 4 files changed, 79 insertions(+), 32 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js index 91cc8111a0..626aabd9bf 100644 --- a/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js @@ -1086,7 +1086,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ - "name" (string) required. The storage to export. - - "expandPages" (bool=false). If true, the .pages result property + - "decodePages" (bool=false). If true, the .pages result property holdes Uint8Array objects holding the raw binary-format db pages. The default is to use kvvfs-encoded string pages (JSON-friendly). @@ -1134,7 +1134,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }else if(args.length){ opt = Object.assign(Object.create(null),{ name: args[0], - //expandPages: true + //decodePages: true }); } const store = opt ? storageForZClass(opt.name) : null; @@ -1172,7 +1172,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ if( !util.isInt32(kk) || kk<=0 ){ toss3(capi.SQLITE_RANGE, "Malformed kvvfs key: "+k); } - if( opt.expandPages ){ + if( opt.decodePages ){ const spg = s.getItem(k), n = spg.length, z = cache.memBuffer(0), @@ -1199,7 +1199,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } } } - if( opt.expandPages ) cache.memBufferFree(1); + if( opt.decodePages ) cache.memBufferFree(1); /* Now sort the page numbers and move them into an array. In JS property keys are always strings, so we have to coerce them to numbers so we can get them sorted properly for the array. */ @@ -1439,14 +1439,20 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ - 'delete' gets the string-type key of the deleted record. - The arguments to 'write' and 'delete' are in one of the + The arguments to 'write', and keys to 'delete', are in one of the following forms: - - 'sz' = the unencoded db size as a string + - 'sz' = the unencoded db size as a string. This specific key is + key is never deleted, so is only ever passed to 'write' events. - - 'jrnl' = the current db journal as a string + - 'jrnl' = the current db journal as a kvvfs-encoded string. This + journal format is not useful anywhere except in the kvvfs + internals. These events are not fired if opt.elideJournal is + true. - - '[1-9][0-9]*' (a db page number) = an encoded db page + - '[1-9][0-9]*' (a db page number) = a db page. Its type depends + on opt.decodePages. These may be written and deleted in arbitrary + order. If a page is deleted, the db is shrinking. For 'local' and 'session' storage, all of those keys have a prefix of 'kvvfs-local-' resp. 'kvvfs-session-'. This is required diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index e1ede17db0..3c51730020 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -3098,7 +3098,11 @@ globalThis.sqlite3InitModule = sqlite3InitModule; duo = new JDb(filename); duo.exec('insert into kvvfs(a) values(4),(5),(6)'); T.assert(6 === db.selectValue(sqlCount)); - let exp = exportDb(filename); + const expOpt = { + name: filename, + decodePages: true + }; + let exp = exportDb(expOpt); let expectRows = 6; debug("exported db",exp); db.close(); @@ -3171,7 +3175,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; if( 1 ){ const pageSize = 1024 * 16; if( 0 ){ - debug("Export before vacuum", exportDb(filename)); + debug("Export before vacuum", exportDb(expOpt)); debug("page size before vacuum", db.selectArray( "select page_size from pragma_page_size()" @@ -3205,10 +3209,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; // It fails at this point for non-8k sizes T.assert(expectRows === duo.selectValue(sqlCount), "Unexpected record count."); - exp = exportDb({ - name: filename, - expandPages: true - }); + exp = exportDb(expOpt); debug("Exported page-expanded db",exp); if( 0 ){ debug("vacuumed export",exp); @@ -3248,14 +3249,17 @@ globalThis.sqlite3InitModule = sqlite3InitModule; .t({ name: 'kvvfs listeners (experiment)', test: function(sqlite3){ + const kvvfs = sqlite3.kvvfs; + const filename = 'listen'; let db; try { - const filename = 'listen'; const DB = sqlite3.oo1.DB; const sqlSetup = [ 'create table kvvfs(a);', 'insert into kvvfs(a) values(1),(2),(3)' ]; + const sqlCount = "select count(*) from kvvfs"; + const sqlSelectSchema = "select * from sqlite_schema"; const counts = Object.assign(Object.create(null),{ open: 0, close: 0, delete: 0, write: 0 }); @@ -3264,7 +3268,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; pglog.jrnl = undefined; pglog.size = undefined; pglog.elideJournal = true; - //pglog.decodePages = true; + pglog.decodePages = true; pglog.exception = new Error("Testing that exceptions from listeners do not interfere"); const toss = ()=>{ if( pglog.exception ){ @@ -3302,10 +3306,15 @@ globalThis.sqlite3InitModule = sqlite3InitModule; T.assert(!pglog.elideJournal); pglog.jrnl = null; break; - default: - T.assert( +ev.data>0, "Expecting positive db page number" ); - pglog[+ev.data] = undefined; + default:{ + const n = +ev.data>0; + T.assert( n, "Expecting positive db page number" ); + if( n < pglog.pages.length ){ + pglog.size = undefined; + } + pglog.pages[n] = undefined; break; + } } }, 'write': (ev)=>{ @@ -3341,10 +3350,16 @@ globalThis.sqlite3InitModule = sqlite3InitModule; } } }; - sqlite3.kvvfs.listen(listener); + kvvfs.listen(listener); const dbFileRaw = 'file:'+filename+'?vfs=kvvfs&delete-on-close=1'; + const expOpt = { + name: filename, + //decodePages: true + }; db = new DB(dbFileRaw); db.exec(sqlSetup); + T.assert(db.selectObjects(sqlSelectSchema)?.length>0, + "Unexpected empty schema"); db.close(); debug("kvvfs listener counts:",counts); T.assert( counts.open ) @@ -3359,19 +3374,45 @@ globalThis.sqlite3InitModule = sqlite3InitModule; if( 1 ){ T.assert(undefined===pglog.pages[0], "Expecting empty slot 0"); pglog.pages.shift(); - debug("kvvfs listener pageLog", pglog); + //debug("kvvfs listener pageLog", pglog); } const before = JSON.stringify(counts); - T.assert( sqlite3.kvvfs.unlisten(listener) ) - .assert( !sqlite3.kvvfs.unlisten(listener) ); + T.assert( kvvfs.unlisten(listener) ) + .assert( !kvvfs.unlisten(listener) ); db = new DB(dbFileRaw); + T.assert( db.selectObjects(sqlSelectSchema)?.length>0 ); + const exp = kvvfs.export(expOpt); + const expectRows = db.selectValue(sqlCount); db.exec("delete from kvvfs"); db.close(); const after = JSON.stringify(counts); T.assert( before===after, "Expecting no events after unlistening." ); - sqlite3.kvvfs.unlink(filename); + if( 0 ){ + exp = kvvfs.export(expOpt); + debug("Post-delete export:",exp); + } + if( 1 ){ + // Replace the storage with the pglog state... + const bogoExp = { + name: filename, + size: pglog.size, + timestamp: Date.now(), + pages: pglog.pages + }; + //debug("exp",exp); + //debug("bogoExp",bogoExp); + kvvfs.import(bogoExp, true); + //debug("Re-exported", kvvfs.export(expOpt)); + db = new DB(dbFileRaw); + // Failing on the next line despite exports looking good + T.assert(db.selectObjects(sqlSelectSchema)?.length>0, + "Empty schema on imported db"); + T.assert(expectRows===db.selectValue(sqlCount)); + db.close(); + } }finally{ db?.close?.(); + kvvfs.unlink(filename); } } })/*kvvfs listeners */ diff --git a/manifest b/manifest index f53d98af9b..2d269d57af 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Give\skvvfs.unlisten()\sa\suseful\sresult\svalue. -D 2025-11-30T05:25:44.700 +C More\swork\son\sthe\skvvfs\sevents.\sDemonstrate\sre-importing\sa\sstreamed-out\sset\sof\sdb\spages. +D 2025-11-30T06:20:28.183 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -600,7 +600,7 @@ F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 9654b565b346dc609b75d15337f20acfa7af7d9d558da1afeb9b6d8eaa404966 F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d -F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 962b0f903da386f278669be52bf233e12f1d2bc1bbeb9b45753f2423e4b61d15 +F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 4785535033eff734b4a9115edcf4d09b58b51a5830cc3a54f2ded9fff51a8c75 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js a2eea6442556867b589e04107796c6e1d04a472219529eeb45b7cd221d7d048b F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 88ce2078267a2d1af57525a32d896295f4a8db7664de0e17e82dc9ff006ed8d3 F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81 @@ -647,7 +647,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 1618670e466f424aa289859fe0ec8ded223e42e9e69b5c851f809baaaca1a00c F ext/wasm/tester1-worker.c-pp.html 0e432ec2c0d99cd470484337066e8d27e7aee4641d97115338f7d962bf7b081a F ext/wasm/tester1.c-pp.html 52d88fe2c6f21a046030a36410b4839b632f4424028197a45a3d5669ea724ddb -F ext/wasm/tester1.c-pp.js 0f351a6f21dcf9fc40e90b4778ec1b56db7c24763ef00bab29cecb0382cf6b2d +F ext/wasm/tester1.c-pp.js dfffa59662eb6ae04be40b65c34becb864edb74d1439f3f8a850c0102546ebe6 F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e F ext/wasm/tests/opfs/concurrency/test.js d08889a5bb6e61937d0b8cbb78c9efbefbf65ad09f510589c779b7cc6a803a88 F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2180,8 +2180,8 @@ F tool/version-info.c 33d0390ef484b3b1cb685d59362be891ea162123cea181cb8e6d2cf6dd F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 0f2bad285577c26f1185dcafd3b8ca2f16e74aa9dc40e6e23867150bccee4602 -R a91c43886f7953b96b0dc2b39d55f65b +P bbd08b67ceeaae2a3a82655da5a3983f4e9057424de4223f1505ba74c36238af +R 67172ddc08db9584ebb1618c06504a62 U stephan -Z 624b197b64a025faddc0668a7a1d59b0 +Z f1288e031bc5ab35fe9740e0ceeda37e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b5e09ce20a..033877a2d0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bbd08b67ceeaae2a3a82655da5a3983f4e9057424de4223f1505ba74c36238af +b8fee839b0199d1d2714c99612ea4b4d943d8b0033d742eaa66e1575ce46f0d2 -- 2.47.3