From: stephan Date: Sun, 30 Nov 2025 03:02:06 +0000 (+0000) Subject: Extend kvvfs export to optionally export the raw binary db pages as a list of Uint8Ar... X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=768c364aaa6548be7e65bfccc40bd52d7334b299;p=thirdparty%2Fsqlite.git Extend kvvfs export to optionally export the raw binary db pages as a list of Uint8Array instead of kvvfs-encoded strings. This is typically much larger but the pages can then be used as-is. FossilOrigin-Name: a4f59496a53a079f8f73e4cde68f47dbd13d2d74de2ad11bc716e7e5c00f1ec0 --- diff --git a/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js index fd4703a2cb..64d96bb5e6 100644 --- a/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js @@ -116,12 +116,38 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ util.assert( 32<=kvvfsMethods.$nKeySize, "unexpected kvvfsMethods.$nKeySize: "+kvvfsMethods.$nKeySize); const cache = Object.assign(Object.create(null),{ + fixedPageSize: 8192/*used in some validation*/, rxJournalSuffix: /-journal$/, zKeyJrnl: wasm.allocCString("jrnl"), zKeySz: wasm.allocCString("sz"), - keySize: kvvfsMethods.$nKeySize + keySize: kvvfsMethods.$nKeySize, + buffer: Object.assign(Object.create(null),{ + n: kvvfsMethods.$nBufferSize, + pool: Object.create(null) + }) }); + /** + A wasm.alloc()'d buffer large enough for all kvvfs + encoding/decoding needs (cache.buffer.n). + + We leak this one-time alloc because we've no better option. + sqlite3_vfs does not have a finalizer, so we've no place to hook + in the cleanup. We "could" extend sqlite3_shutdown() to have a + cleanup list for stuff like this but that function is never + used in JS, so it's hardly worth it. + */ + cache.memBuffer = (id=0)=>cache.buffer.pool[id] ??= wasm.alloc(cache.buffer.n); + + /** Freeds the buffer with the given id. */ + cache.memBufferFree = (id)=>{ + const b = cache.buffer.pool[id]; + if( b ){ + wasm.dealloc(b); + delete cache.buffer.pool[id]; + } + }; + const debug = sqlite3.__isUnderTest ? function(){sqlite3.config.debug("kvvfs:", ...arguments)} : function(){}; @@ -297,7 +323,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const other = cache.rxJournalSuffix.test(store.jzClass) ? store.jzClass.replace(cache.rxJournalSuffix,'') : store.jzClass+'-journal'; - debug("cleaning up storage handles [", store.jzClass, other,"]",store); + //debug("cleaning up storage handles [", store.jzClass, other,"]",store); delete cache.storagePool[store.jzClass]; delete cache.storagePool[other]; if( !sqlite3.__isUnderTest ){ @@ -568,16 +594,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ debug("xRcrdRead", nBuf, zClass, wasm.cstrToJs(zClass), wasm.cstrToJs(zKey), nV, jV, store); } - const zV = cache.keybuf.mem ??= wasm.alloc.impl(cache.keybuf.n) - /* We leak this one-time alloc because we've no better - option. sqlite3_vfs does not have a finalizer, so - we've no place to hook in the cleanup. We "could" - extend sqlite3_shutdown() to have a cleanup list for - stuff like this but (A) that function is never used in - JS and (B) its cleanup would leave cache.keybuf - pointing to stale memory, so if the library were used - after sqlite3_shutdown() then we'd corrupt memory. */; - if( !zV ) return -3 /*OOM*/; + const zV = cache.memBuffer(0); + //if( !zV ) return -3 /*OOM*/; const heap = wasm.heap8(); let i; for(i = 0; i < nV; ++i){ @@ -634,7 +652,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ xOpen: function(pProtoVfs,zName,pProtoFile,flags,pOutFlags){ cache.popError(); try{ - //cache.zReadBuf ??= wasm.malloc(kvvfsMethods.$nBufferSize); if( !zName ){ zName = (cache.zEmpty ??= wasm.allocCString("")); } @@ -680,7 +697,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ installStorageAndJournal(s); s.files.push(f); s.deleteAtRefc0 = deleteAt0 || !!(capi.SQLITE_OPEN_DELETEONCLOSE & flags); - debug("xOpen installed storage handle [",nm, nm+"-journal","]", s); + //debug("xOpen installed storage handle [",nm, nm+"-journal","]", s); } pFileHandles.set(pProtoFile, {storage: s, file: f, jzClass}); notifyListeners('open', s, s.files.length); @@ -874,9 +891,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ kvvfsMethods, capi.sqlite3_file.structInfo, KVVfsFile.structInfo); try { - cache.keybuf = Object.create(null); - cache.keybuf.n = kvvfsMethods.$nBufferSize; - util.assert( cache.keybuf.n>1024*129, "Key buffer is not large enough" + util.assert( cache.buffer.n>1024*129, "Heap buffer is not large enough" /* Native is SQLITE_KVOS_SZ is 133073 as of this writing */ ); for(const e of Object.entries(methodOverrides.recordHandler)){ // Overwrite kvvfsMethods's callbacks @@ -1022,9 +1037,28 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }; /** - Copies the entire contents of the given transient storage object - into a JSON-friendly form. The returned object is structured as - follows... + Exports a kvvfs storage object to an object, optionally + JSON-friendly. + + Usages: + + thisfunc(storageName); + thisfunc(options); + + In the latter case, the options object must be an object with + the following properties: + + - "name" (string) required. The storage to export. + + - "expandPages" (bool=false). If true, the .pages result property + holdes Uint8Array objects holding the raw binary-format db + pages. The default is to use kvvfs-encoded string pages + (JSON-friendly). + + - "includeJournal" (bool=false). If true and the db has a current + journal, it is exported as well. + + The returned object is structured as follows... - "name": the name of the storage. This is 'local' or 'session' for localStorage resp. sessionStorage, and an arbitrary name for @@ -1056,11 +1090,20 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Added in version @kvvfs-v2-added-in@. */ - const sqlite3_js_kvvfs_export = function(storageName,includeJournal=true){ - const store = storageForZClass(storageName); + const sqlite3_js_kvvfs_export = function callee(...args){ + let opt; + if( 1===args.length && 'object'===typeof args[0] ){ + opt = args[0]; + }else{ + opt = { + name: args[0], + //expandPages: true + }; + } + const store = storageForZClass(opt.name); if( !store ){ toss3(capi.SQLITE_NOTFOUND, - "There is no kvvfs storage named",storageName); + "There is no kvvfs storage named",opt.name); } //debug("store to export=",store); const s = store.storage; @@ -1070,6 +1113,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ pages: [] }); const pages = Object.create(null); + let xpages; const keyPrefix = kvvfsKeyPrefix(rc.name); const rxTail = keyPrefix ? /^kvvfs-[^-]+-(\w+)/ /* X... part of kvvfs-NAME-X... */ @@ -1081,7 +1125,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ let kk = (keyPrefix ? rxTail.exec(k) : undefined)?.[1] ?? k; switch( kk ){ case 'jrnl': - if( includeJournal ) rc.journal = s.getItem(k); + if( opt.includeJournal ) rc.journal = s.getItem(k); break; case 'sz': rc.size = +s.getItem(k); @@ -1091,11 +1135,34 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ if( !util.isInt32(kk) || kk<=0 ){ toss3(capi.SQLITE_RANGE, "Malformed kvvfs key: "+k); } - pages[kk] = s.getItem(k); + if( opt.expandPages ){ + const spg = s.getItem(k), + n = spg.length, + z = cache.memBuffer(0), + zDec = cache.memBuffer(1), + heap = wasm.heap8u()/* MUST be inited last*/; + let i = 0; + for( ; i < n; ++i ){ + heap[wasm.ptr.add(z, i)] = spg.codePointAt(i) & 0xff; + } + heap[wasm.ptr.add(z, i)] = 0; + //debug("Decoding",i,"page bytes"); + const nDec = wasm.exports.sqlite3__wasm_kvvfs_decode( + z, zDec, cache.buffer.n + ); + if( cache.fixedPageSize !== nDec ){ + util.toss3(capi.SQLITE_ERROR,"Unexpected decoded page size:",nDec); + } + //debug("Decoded",nDec,"page bytes"); + pages[kk] = heap.slice(zDec, wasm.ptr.add(zDec, nDec)); + }else{ + pages[kk] = s.getItem(k); + } break; } } } + if( opt.expandPages ) cache.memBufferFree(1); /* Now sort the page numbers and move them into an array. In JS property keys are always strings, so we have to coerce them to numbers so we can get them sorted properly for the array. */ @@ -1139,7 +1206,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ || !Array.isArray(exp.pages) ){ toss3(capi.SQLITE_MISUSE, "Malformed export object."); } - //warn("importFromObject() is incomplete"); validateStorageName(exp.name); let store = storageForZClass(exp.name); const isNew = !store; @@ -1156,7 +1222,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ toss3(capi.SQLITE_IOERR_ACCESS, "Cannot import db storage while it is in use."); } - sqlite3_js_kvvfs_clear(exp.name, true); + sqlite3_js_kvvfs_clear(exp.name); }else{ store = newStorageObj(exp.name); //warn("Installing new storage:",store); @@ -1164,13 +1230,43 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ //debug("Importing store",store.poolEntry.files.length, store); //debug("object to import:",exp); const keyPrefix = kvvfsKeyPrefix(exp.name); + let zEnc; try{ /* Force the native KVVfsFile instances to re-read the db and page size. */; const s = store.storage; s.setItem(keyPrefix+'sz', exp.size); if( exp.journal ) s.setItem(keyPrefix+'jrnl', exp.journal); - exp.pages.forEach((v,ndx)=>s.setItem(keyPrefix+(ndx+1), v)); + if( exp.pages[0] instanceof Uint8Array ){ + /* raw binary pages */ + //debug("pages",exp.pages); + exp.pages.forEach((u,ndx)=>{ + const n = u.length; + if( cache.fixedPageSize !== n ){ + util.toss3(capi.SQLITE_RANGE,"Unexpected page size:", n); + } + zEnc = cache.memBuffer(1); + const zBin = cache.memBuffer(0), + heap = wasm.heap8u(); + /* Copy u to the heap and encode the heaped copy. This is + _presumably_ faster than porting the encoding algo to + JS, which would involve many, many more function calls. */ + let i; + for(i=0; is.setItem(keyPrefix+(ndx+1), v)); + } if( isNew ) installStorageAndJournal(store); }catch(e){ if( !isNew ){ @@ -1178,6 +1274,10 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ catch(ee){/*ignored*/} } throw e; + }finally{ + if( zEnc ){ + cache.memBufferFree(1); + } } return this; }; @@ -1468,10 +1568,10 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } }); Object.assign(Object.create(ProtoCursor),{ - rowid: 0, - names: Object.keys(cache.storagePool) - .filter(v=>!cache.rxJournalSuffix.test(v)) - }) + rowid: 0, + names: Object.keys(cache.storagePool) + .filter(v=>!cache.rxJournalSuffix.test(v)) + }); const cursorState = function(cursor, reset){ const o = (cursor instanceof capi.sqlite3_vtab_cursor) ? cursor diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index 56661f3b79..a0d12fc6ea 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -1794,6 +1794,10 @@ SQLITE_WASM_EXPORT int sqlite3__wasm_kvvfs_decode(const char *a, char *aOut, int nOut){ return kvvfsDecode(a, aOut, nOut); } +SQLITE_WASM_EXPORT +int sqlite3__wasm_kvvfs_encode(const char *a, int nA, char *aOut){ + return kvvfsEncode(a, nA, aOut); +} #if defined(__EMSCRIPTEN__) && defined(SQLITE_ENABLE_WASMFS) diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 4ebc4168c8..7b395098dd 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -2908,6 +2908,28 @@ globalThis.sqlite3InitModule = sqlite3InitModule; 'size', 'clear'] ){ T.assert( k[n] instanceof Function ); } + + if( 0 ){ + const scope = wasm.scopedAllocPush(); + try{ + const pg = "53514C69746520666F726D61742033b20b0101b402020d02d02l01d04l01jb02b2E91E00Dd011FD3b1FD3dxl2B010617171701377461626C656B767666736B7676667302435245415445205441424C45206B76766673286129"; + const n = pg.length; + const pI = wasm.scopedAlloc( n+1 ); + const nO = 8192 * 2; + const pO = wasm.scopedAlloc( nO ); + const heap = wasm.heap8u(); + let i; + for( i=0; i 0, "Missing db size" ) .assert( exp?.pages?.length > 0, "Missing db pages" ); @@ -3179,7 +3201,11 @@ globalThis.sqlite3InitModule = sqlite3InitModule; // It fails at this point for non-8k sizes T.assert(expectRows === duo.selectValue(sqlCount), "Unexpected record count."); - exp = exportDb(filename); + exp = exportDb({ + name: filename, + expandPages: true + }); + debug("Exported page-expanded db",exp); if( 0 ){ debug("vacuumed export",exp); } @@ -3281,6 +3307,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; } } })/*kvvfs listeners */ + .t({ name: 'kvvfs vtab', predicate: (sqlite3)=>!!sqlite3.kvvfs.create_module, @@ -3308,6 +3335,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; } } })/* kvvfs vtab */ + //#if enable-see .t({ name: 'kvvfs SEE encryption in sessionStorage', diff --git a/manifest b/manifest index 09955709e9..2a8e11a0c6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\scomment\sexplaining\sthe\snew\skvvfsDecode()\sreturn\s-1\sfrom\s[cd81cb70525e]. -D 2025-11-29T23:41:34.784 +C Extend\skvvfs\sexport\sto\soptionally\sexport\sthe\sraw\sbinary\sdb\spages\sas\sa\slist\sof\sUint8Array\sinstead\sof\skvvfs-encoded\sstrings.\sThis\sis\stypically\smuch\slarger\sbut\sthe\spages\scan\sthen\sbe\sused\sas-is. +D 2025-11-30T03:02:06.188 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -600,11 +600,11 @@ F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 9654b565b346dc609b75d15337f20acfa7af7d9d558da1afeb9b6d8eaa404966 F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d -F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 52c79574b05ee39ec01de1ba63a1ed36bece0858b817d2c405e36e19864610ad +F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js e01f80208b7f00a47045bc92e5c0b40c6e2232fc058f2dd81ed41d46b1cfd323 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js a2eea6442556867b589e04107796c6e1d04a472219529eeb45b7cd221d7d048b F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 88ce2078267a2d1af57525a32d896295f4a8db7664de0e17e82dc9ff006ed8d3 F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81 -F ext/wasm/api/sqlite3-wasm.c d74ea827a644598090cf450a0bb38564332e658829d3e924f471558de669a336 +F ext/wasm/api/sqlite3-wasm.c 3fd63b99dec39665ba89fcca6d001bb404ac9ff0be8c52e4e07f52749a8ba57a F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bda1c75bd674a92a0e27cc2f3d46dbbf21e422413f8046814515a0bd7409328a F ext/wasm/api/sqlite3-worker1.c-pp.js 802d69ead8c38dc1be52c83afbfc77e757da8a91a2e159e7ed3ecda8b8dba2e7 F ext/wasm/c-pp-lite.c f38254fba42561728c2e4764a7ba8d68700091e7c2f4418112868c0daba16783 @@ -647,7 +647,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 1618670e466f424aa289859fe0ec8ded223e42e9e69b5c851f809baaaca1a00c F ext/wasm/tester1-worker.c-pp.html 0e432ec2c0d99cd470484337066e8d27e7aee4641d97115338f7d962bf7b081a F ext/wasm/tester1.c-pp.html 52d88fe2c6f21a046030a36410b4839b632f4424028197a45a3d5669ea724ddb -F ext/wasm/tester1.c-pp.js 3398107cad5c548443d1978adf8d8b3ca48dd6baa0410ae26b24b016d39cf157 +F ext/wasm/tester1.c-pp.js d5bf9dd8be21377981e20974fa798ae9839ee7a279189a773299532a844a8138 F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e F ext/wasm/tests/opfs/concurrency/test.js d08889a5bb6e61937d0b8cbb78c9efbefbf65ad09f510589c779b7cc6a803a88 F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -718,7 +718,7 @@ F src/notify.c 57c2d1a2805d6dee32acd5d250d928ab94e02d76369ae057dee7d445fd64e878 F src/os.c 509452169d5ea739723e213b8e2481cf0e587f0e88579a912d200db5269f5f6d F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63 F src/os_common.h 6c0eb8dd40ef3e12fe585a13e709710267a258e2c8dd1c40b1948a1d14582e06 -F src/os_kv.c 110ef88f5ee976124a08a03c51ac47f46cc9904739929e77016d7ab0ca9004b0 +F src/os_kv.c e7d96727db5b67e39d590a68cc61c86daf4c093c36c011a09ebfb521182ec28d F src/os_setup.h 8efc64eda6a6c2f221387eefc2e7e45fd5a3d5c8337a7a83519ba4fbd2957ae2 F src/os_unix.c 7945ede1e85b2d1b910e1b4af9ba342e964b1e30e79f4176480a60736445cb36 F src/os_win.c a89b501fc195085c7d6c9eec7f5bd782625e94bb2a96b000f4d009703df1083f @@ -2180,8 +2180,8 @@ F tool/version-info.c 33d0390ef484b3b1cb685d59362be891ea162123cea181cb8e6d2cf6dd F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 989d097b1324cf712107bb81697fa8e9044aea0e7feacf0e6b6561e216f07989 -R 465351f3f0514d80bd2330c01d6416cc +P ed9ab366d1c1880d3c06edce6c0c33ad30c7ae59725c1ec1fe3f620be1835630 +R 8530361058afe18d87abefbdbe6a3870 U stephan -Z ab4d411f734d9d3878ae160021ac295e +Z 197caaef7aceb5323c153f1905283e67 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index eb886b1eee..ac3cbc9ad1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ed9ab366d1c1880d3c06edce6c0c33ad30c7ae59725c1ec1fe3f620be1835630 +a4f59496a53a079f8f73e4cde68f47dbd13d2d74de2ad11bc716e7e5c00f1ec0 diff --git a/src/os_kv.c b/src/os_kv.c index 2fd9e183d7..1fd1c8e8ce 100644 --- a/src/os_kv.c +++ b/src/os_kv.c @@ -383,7 +383,10 @@ sqlite3_kvvfs_methods sqlite3KvvfsMethods = { ** of hexadecimal and base-26 numbers, it is always clear where ** one stops and the next begins. */ -static int kvvfsEncode(const char *aData, int nData, char *aOut){ +#ifndef SQLITE_WASM +static +#endif +int kvvfsEncode(const char *aData, int nData, char *aOut){ int i, j; const unsigned char *a = (const unsigned char*)aData; for(i=j=0; i