util.assert( 32<=kvvfsMethods.$nKeySize, "unexpected kvvfsMethods.$nKeySize: "+kvvfsMethods.$nKeySize);
const cache = Object.assign(Object.create(null),{
+ fixedPageSize: 8192/*used in some validation*/,
rxJournalSuffix: /-journal$/,
zKeyJrnl: wasm.allocCString("jrnl"),
zKeySz: wasm.allocCString("sz"),
- keySize: kvvfsMethods.$nKeySize
+ keySize: kvvfsMethods.$nKeySize,
+ buffer: Object.assign(Object.create(null),{
+ n: kvvfsMethods.$nBufferSize,
+ pool: Object.create(null)
+ })
});
+ /**
+ A wasm.alloc()'d buffer large enough for all kvvfs
+ encoding/decoding needs (cache.buffer.n).
+
+ We leak this one-time alloc because we've no better option.
+ sqlite3_vfs does not have a finalizer, so we've no place to hook
+ in the cleanup. We "could" extend sqlite3_shutdown() to have a
+ cleanup list for stuff like this but that function is never
+ used in JS, so it's hardly worth it.
+ */
+ cache.memBuffer = (id=0)=>cache.buffer.pool[id] ??= wasm.alloc(cache.buffer.n);
+
+ /** Freeds the buffer with the given id. */
+ cache.memBufferFree = (id)=>{
+ const b = cache.buffer.pool[id];
+ if( b ){
+ wasm.dealloc(b);
+ delete cache.buffer.pool[id];
+ }
+ };
+
const debug = sqlite3.__isUnderTest
? function(){sqlite3.config.debug("kvvfs:", ...arguments)}
: function(){};
const other = cache.rxJournalSuffix.test(store.jzClass)
? store.jzClass.replace(cache.rxJournalSuffix,'')
: store.jzClass+'-journal';
- debug("cleaning up storage handles [", store.jzClass, other,"]",store);
+ //debug("cleaning up storage handles [", store.jzClass, other,"]",store);
delete cache.storagePool[store.jzClass];
delete cache.storagePool[other];
if( !sqlite3.__isUnderTest ){
debug("xRcrdRead", nBuf, zClass, wasm.cstrToJs(zClass),
wasm.cstrToJs(zKey), nV, jV, store);
}
- const zV = cache.keybuf.mem ??= wasm.alloc.impl(cache.keybuf.n)
- /* We leak this one-time alloc because we've no better
- option. sqlite3_vfs does not have a finalizer, so
- we've no place to hook in the cleanup. We "could"
- extend sqlite3_shutdown() to have a cleanup list for
- stuff like this but (A) that function is never used in
- JS and (B) its cleanup would leave cache.keybuf
- pointing to stale memory, so if the library were used
- after sqlite3_shutdown() then we'd corrupt memory. */;
- if( !zV ) return -3 /*OOM*/;
+ const zV = cache.memBuffer(0);
+ //if( !zV ) return -3 /*OOM*/;
const heap = wasm.heap8();
let i;
for(i = 0; i < nV; ++i){
xOpen: function(pProtoVfs,zName,pProtoFile,flags,pOutFlags){
cache.popError();
try{
- //cache.zReadBuf ??= wasm.malloc(kvvfsMethods.$nBufferSize);
if( !zName ){
zName = (cache.zEmpty ??= wasm.allocCString(""));
}
installStorageAndJournal(s);
s.files.push(f);
s.deleteAtRefc0 = deleteAt0 || !!(capi.SQLITE_OPEN_DELETEONCLOSE & flags);
- debug("xOpen installed storage handle [",nm, nm+"-journal","]", s);
+ //debug("xOpen installed storage handle [",nm, nm+"-journal","]", s);
}
pFileHandles.set(pProtoFile, {storage: s, file: f, jzClass});
notifyListeners('open', s, s.files.length);
kvvfsMethods, capi.sqlite3_file.structInfo,
KVVfsFile.structInfo);
try {
- cache.keybuf = Object.create(null);
- cache.keybuf.n = kvvfsMethods.$nBufferSize;
- util.assert( cache.keybuf.n>1024*129, "Key buffer is not large enough"
+ util.assert( cache.buffer.n>1024*129, "Heap buffer is not large enough"
/* Native is SQLITE_KVOS_SZ is 133073 as of this writing */ );
for(const e of Object.entries(methodOverrides.recordHandler)){
// Overwrite kvvfsMethods's callbacks
};
/**
- Copies the entire contents of the given transient storage object
- into a JSON-friendly form. The returned object is structured as
- follows...
+ Exports a kvvfs storage object to an object, optionally
+ JSON-friendly.
+
+ Usages:
+
+ thisfunc(storageName);
+ thisfunc(options);
+
+ In the latter case, the options object must be an object with
+ the following properties:
+
+ - "name" (string) required. The storage to export.
+
+ - "expandPages" (bool=false). If true, the .pages result property
+ holdes Uint8Array objects holding the raw binary-format db
+ pages. The default is to use kvvfs-encoded string pages
+ (JSON-friendly).
+
+ - "includeJournal" (bool=false). If true and the db has a current
+ journal, it is exported as well.
+
+ The returned object is structured as follows...
- "name": the name of the storage. This is 'local' or 'session'
for localStorage resp. sessionStorage, and an arbitrary name for
Added in version @kvvfs-v2-added-in@.
*/
- const sqlite3_js_kvvfs_export = function(storageName,includeJournal=true){
- const store = storageForZClass(storageName);
+ const sqlite3_js_kvvfs_export = function callee(...args){
+ let opt;
+ if( 1===args.length && 'object'===typeof args[0] ){
+ opt = args[0];
+ }else{
+ opt = {
+ name: args[0],
+ //expandPages: true
+ };
+ }
+ const store = storageForZClass(opt.name);
if( !store ){
toss3(capi.SQLITE_NOTFOUND,
- "There is no kvvfs storage named",storageName);
+ "There is no kvvfs storage named",opt.name);
}
//debug("store to export=",store);
const s = store.storage;
pages: []
});
const pages = Object.create(null);
+ let xpages;
const keyPrefix = kvvfsKeyPrefix(rc.name);
const rxTail = keyPrefix
? /^kvvfs-[^-]+-(\w+)/ /* X... part of kvvfs-NAME-X... */
let kk = (keyPrefix ? rxTail.exec(k) : undefined)?.[1] ?? k;
switch( kk ){
case 'jrnl':
- if( includeJournal ) rc.journal = s.getItem(k);
+ if( opt.includeJournal ) rc.journal = s.getItem(k);
break;
case 'sz':
rc.size = +s.getItem(k);
if( !util.isInt32(kk) || kk<=0 ){
toss3(capi.SQLITE_RANGE, "Malformed kvvfs key: "+k);
}
- pages[kk] = s.getItem(k);
+ if( opt.expandPages ){
+ const spg = s.getItem(k),
+ n = spg.length,
+ z = cache.memBuffer(0),
+ zDec = cache.memBuffer(1),
+ heap = wasm.heap8u()/* MUST be inited last*/;
+ let i = 0;
+ for( ; i < n; ++i ){
+ heap[wasm.ptr.add(z, i)] = spg.codePointAt(i) & 0xff;
+ }
+ heap[wasm.ptr.add(z, i)] = 0;
+ //debug("Decoding",i,"page bytes");
+ const nDec = wasm.exports.sqlite3__wasm_kvvfs_decode(
+ z, zDec, cache.buffer.n
+ );
+ if( cache.fixedPageSize !== nDec ){
+ util.toss3(capi.SQLITE_ERROR,"Unexpected decoded page size:",nDec);
+ }
+ //debug("Decoded",nDec,"page bytes");
+ pages[kk] = heap.slice(zDec, wasm.ptr.add(zDec, nDec));
+ }else{
+ pages[kk] = s.getItem(k);
+ }
break;
}
}
}
+ if( opt.expandPages ) cache.memBufferFree(1);
/* Now sort the page numbers and move them into an array. In JS
property keys are always strings, so we have to coerce them to
numbers so we can get them sorted properly for the array. */
|| !Array.isArray(exp.pages) ){
toss3(capi.SQLITE_MISUSE, "Malformed export object.");
}
- //warn("importFromObject() is incomplete");
validateStorageName(exp.name);
let store = storageForZClass(exp.name);
const isNew = !store;
toss3(capi.SQLITE_IOERR_ACCESS,
"Cannot import db storage while it is in use.");
}
- sqlite3_js_kvvfs_clear(exp.name, true);
+ sqlite3_js_kvvfs_clear(exp.name);
}else{
store = newStorageObj(exp.name);
//warn("Installing new storage:",store);
//debug("Importing store",store.poolEntry.files.length, store);
//debug("object to import:",exp);
const keyPrefix = kvvfsKeyPrefix(exp.name);
+ let zEnc;
try{
/* Force the native KVVfsFile instances to re-read the db
and page size. */;
const s = store.storage;
s.setItem(keyPrefix+'sz', exp.size);
if( exp.journal ) s.setItem(keyPrefix+'jrnl', exp.journal);
- exp.pages.forEach((v,ndx)=>s.setItem(keyPrefix+(ndx+1), v));
+ if( exp.pages[0] instanceof Uint8Array ){
+ /* raw binary pages */
+ //debug("pages",exp.pages);
+ exp.pages.forEach((u,ndx)=>{
+ const n = u.length;
+ if( cache.fixedPageSize !== n ){
+ util.toss3(capi.SQLITE_RANGE,"Unexpected page size:", n);
+ }
+ zEnc = cache.memBuffer(1);
+ const zBin = cache.memBuffer(0),
+ heap = wasm.heap8u();
+ /* Copy u to the heap and encode the heaped copy. This is
+ _presumably_ faster than porting the encoding algo to
+ JS, which would involve many, many more function calls. */
+ let i;
+ for(i=0; i<n; ++i ) heap[wasm.ptr.add(zBin,i)] = u[i];
+ heap[wasm.ptr.add(zBin,i)] = 0;
+ const rc = wasm.exports.sqlite3__wasm_kvvfs_encode(zBin, i, zEnc);
+ util.assert( rc < cache.buffer.n,
+ "Impossibly long output - possibly smashed the heap" );
+ util.assert( 0===wasm.peek8(wasm.ptr.add(zEnc,rc)),
+ "Expecting NUL-terminated encoded output" );
+ const jenc = wasm.cstrToJs(zEnc);
+ //debug("(un)encoded page:",u,jenc);
+ s.setItem(keyPrefix+(ndx+1), jenc);
+ });
+ }else if( exp.pages[0] ){
+ /* kvvfs-encoded pages */
+ exp.pages.forEach((v,ndx)=>s.setItem(keyPrefix+(ndx+1), v));
+ }
if( isNew ) installStorageAndJournal(store);
}catch(e){
if( !isNew ){
catch(ee){/*ignored*/}
}
throw e;
+ }finally{
+ if( zEnc ){
+ cache.memBufferFree(1);
+ }
}
return this;
};
}
});
Object.assign(Object.create(ProtoCursor),{
- rowid: 0,
- names: Object.keys(cache.storagePool)
- .filter(v=>!cache.rxJournalSuffix.test(v))
- })
+ rowid: 0,
+ names: Object.keys(cache.storagePool)
+ .filter(v=>!cache.rxJournalSuffix.test(v))
+ });
const cursorState = function(cursor, reset){
const o = (cursor instanceof capi.sqlite3_vtab_cursor)
? cursor
-C Add\sa\scomment\sexplaining\sthe\snew\skvvfsDecode()\sreturn\s-1\sfrom\s[cd81cb70525e].
-D 2025-11-29T23:41:34.784
+C Extend\skvvfs\sexport\sto\soptionally\sexport\sthe\sraw\sbinary\sdb\spages\sas\sa\slist\sof\sUint8Array\sinstead\sof\skvvfs-encoded\sstrings.\sThis\sis\stypically\smuch\slarger\sbut\sthe\spages\scan\sthen\sbe\sused\sas-is.
+D 2025-11-30T03:02:06.188
F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89
F ext/wasm/api/sqlite3-opfs-async-proxy.js 9654b565b346dc609b75d15337f20acfa7af7d9d558da1afeb9b6d8eaa404966
F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d
-F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 52c79574b05ee39ec01de1ba63a1ed36bece0858b817d2c405e36e19864610ad
+F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js e01f80208b7f00a47045bc92e5c0b40c6e2232fc058f2dd81ed41d46b1cfd323
F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js a2eea6442556867b589e04107796c6e1d04a472219529eeb45b7cd221d7d048b
F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 88ce2078267a2d1af57525a32d896295f4a8db7664de0e17e82dc9ff006ed8d3
F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81
-F ext/wasm/api/sqlite3-wasm.c d74ea827a644598090cf450a0bb38564332e658829d3e924f471558de669a336
+F ext/wasm/api/sqlite3-wasm.c 3fd63b99dec39665ba89fcca6d001bb404ac9ff0be8c52e4e07f52749a8ba57a
F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bda1c75bd674a92a0e27cc2f3d46dbbf21e422413f8046814515a0bd7409328a
F ext/wasm/api/sqlite3-worker1.c-pp.js 802d69ead8c38dc1be52c83afbfc77e757da8a91a2e159e7ed3ecda8b8dba2e7
F ext/wasm/c-pp-lite.c f38254fba42561728c2e4764a7ba8d68700091e7c2f4418112868c0daba16783
F ext/wasm/test-opfs-vfs.js 1618670e466f424aa289859fe0ec8ded223e42e9e69b5c851f809baaaca1a00c
F ext/wasm/tester1-worker.c-pp.html 0e432ec2c0d99cd470484337066e8d27e7aee4641d97115338f7d962bf7b081a
F ext/wasm/tester1.c-pp.html 52d88fe2c6f21a046030a36410b4839b632f4424028197a45a3d5669ea724ddb
-F ext/wasm/tester1.c-pp.js 3398107cad5c548443d1978adf8d8b3ca48dd6baa0410ae26b24b016d39cf157
+F ext/wasm/tester1.c-pp.js d5bf9dd8be21377981e20974fa798ae9839ee7a279189a773299532a844a8138
F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e
F ext/wasm/tests/opfs/concurrency/test.js d08889a5bb6e61937d0b8cbb78c9efbefbf65ad09f510589c779b7cc6a803a88
F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2
F src/os.c 509452169d5ea739723e213b8e2481cf0e587f0e88579a912d200db5269f5f6d
F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63
F src/os_common.h 6c0eb8dd40ef3e12fe585a13e709710267a258e2c8dd1c40b1948a1d14582e06
-F src/os_kv.c 110ef88f5ee976124a08a03c51ac47f46cc9904739929e77016d7ab0ca9004b0
+F src/os_kv.c e7d96727db5b67e39d590a68cc61c86daf4c093c36c011a09ebfb521182ec28d
F src/os_setup.h 8efc64eda6a6c2f221387eefc2e7e45fd5a3d5c8337a7a83519ba4fbd2957ae2
F src/os_unix.c 7945ede1e85b2d1b910e1b4af9ba342e964b1e30e79f4176480a60736445cb36
F src/os_win.c a89b501fc195085c7d6c9eec7f5bd782625e94bb2a96b000f4d009703df1083f
F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7
F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 989d097b1324cf712107bb81697fa8e9044aea0e7feacf0e6b6561e216f07989
-R 465351f3f0514d80bd2330c01d6416cc
+P ed9ab366d1c1880d3c06edce6c0c33ad30c7ae59725c1ec1fe3f620be1835630
+R 8530361058afe18d87abefbdbe6a3870
U stephan
-Z ab4d411f734d9d3878ae160021ac295e
+Z 197caaef7aceb5323c153f1905283e67
# Remove this line to create a well-formed Fossil manifest.
projects / thirdparty / sqlite.git / commitdiff
