return e;
};
- const noop = ()=>{};
+ const catchForNotify = (e)=>{
+ warn("kvvfs.listener handler threw:",e);
+ };
+
+ const kvvfsDecode = wasm.exports.sqlite3__wasm_kvvfs_decode;
+ const kvvfsEncode = wasm.exports.sqlite3__wasm_kvvfs_encode;
/**
Listener events and their argument(s):
*/
const notifyListeners = async function(eventName,store,...args){
if( store.listeners ){
- const ev = Object.create(null);
- ev.storageName = store.jzClass;
- ev.type = eventName;
+ //cache.rxPageNoSuffix ??= /(\d+)$/;
+ if( store.keyPrefix ){
+ args[0] = args[0].replace(store.keyPrefix,'');
+ }
+ let u8enc, z0, z1, wcache;
for(const ear of store.listeners){
- const f = ear?.[eventName];
+ const ev = Object.create(null);
+ ev.storageName = store.jzClass;
+ ev.type = eventName;
+ const decodePages = ear.decodePages;
+ const f = ear.events[eventName];
if( f ){
- ev.data = ((args.length===1) ? args[0] : args);
- try{f(ev)?.catch?.(noop)}
+ if( ear.elideJournal && args[0]==='jrnl' ){
+ continue;
+ }
+ if( 'write'===eventName && ear.decodePages && +args[0]>0 ){
+ /* Decode pages to Uint8Array. */
+ ev.data = [args[0]];
+ if( wcache?.[args[0]] ){
+ ev.data[1] = wcache[args[0]];
+ continue;
+ }
+ u8enc ??= new TextEncoder('utf-8');
+ z0 ??= cache.memBuffer(10);
+ z1 ??= cache.memBuffer(11);
+ const u = u8enc.encode(args[1]);
+ const heap = wasm.heap8u();
+ heap.set(u, z0);
+ heap[wasm.ptr.add(z0, u.length)] = 0;
+ const rc = kvvfsDecode(z0, z1, cache.buffer.n);
+ if( rc>0 ){
+ wcache ??= Object.create(null);
+ wcache[args[0]]
+ = ev.data[1]
+ = heap.slice(z1, wasm.ptr.add(z1,rc));
+ }
+ }else{
+ ev.data = ((args.length===1) ? args[0] : args);
+ }
+ try{f(ev)?.catch?.(catchForNotify)}
catch(e){
- warn("notifyListeners",store.jzClass,eventName,e);
+ warn("notifyListeners [",store.jzClass,"]",eventName,e);
}
}
}
const kvvfsMakeKey = wasm.exports.sqlite3__wasm_kvvfsMakeKey;
/**
- Returns a C string from sqlite3__wasm_kvvfsMakeKey() OR returns
- zKey. In the former case the memory is static, so must be copied
- before a second call. zKey MUST be a pointer passed to a
- VFS/file method, to allow us to avoid an alloc and/or an
- snprintf(). It requires C-string arguments for zClass and
- zKey. zClass may be NULL but zKey may not.
+ Returns a C string from kvvfsMakeKey() OR returns zKey. In the
+ former case the memory is static, so must be copied before a
+ second call. zKey MUST be a pointer passed to a VFS/file method,
+ to allow us to avoid an alloc and/or an snprintf(). It requires
+ C-string arguments for zClass and zKey. zClass may be NULL but
+ zKey may not.
*/
const zKeyForStorage = (store, zClass, zKey)=>{
//debug("zKeyForStorage(",store, wasm.cstrToJs(zClass), wasm.cstrToJs(zKey));
xRcrdWrite: (zClass, zKey, zData)=>{
try {
- const jzClass = wasm.cstrToJs(zClass);
- const store = storageForZClass(jzClass);
+ const store = storageForZClass(zClass);
const jxKey = jsKeyForStorage(store, zClass, zKey);
const jData = wasm.cstrToJs(zData);
store.storage.setItem(jxKey, jData);
(JSON-friendly).
- "includeJournal" (bool=false). If true and the db has a current
- journal, it is exported as well.
+ journal, it is exported as well. (Kvvfs journals are stored as a
+ single record within the db's storage object.)
The returned object is structured as follows...
- "size": the unencoded db size.
- - "journal": if includeJournal is true and this db has a
+ - "journal": if options.includeJournal is true and this db has a
journal, it is stored as a string here, otherwise this property
is not set.
let opt;
if( 1===args.length && 'object'===typeof args[0] ){
opt = args[0];
- }else{
- opt = {
+ }else if(args.length){
+ opt = Object.assign(Object.create(null),{
name: args[0],
//expandPages: true
- };
+ });
}
- const store = storageForZClass(opt.name);
+ const store = opt ? storageForZClass(opt.name) : null;
if( !store ){
toss3(capi.SQLITE_NOTFOUND,
- "There is no kvvfs storage named",opt.name);
+ "There is no kvvfs storage named",opt?.name);
}
//debug("store to export=",store);
const s = store.storage;
}
heap[wasm.ptr.add(z, i)] = 0;
//debug("Decoding",i,"page bytes");
- const nDec = wasm.exports.sqlite3__wasm_kvvfs_decode(
+ const nDec = kvvfsDecode(
z, zDec, cache.buffer.n
);
if( cache.fixedPageSize !== nDec ){
if( !exp?.timestamp
|| !exp.name
|| undefined===exp.size
- || exp.size<0 || exp.size>=0x7fffffff
|| !Array.isArray(exp.pages) ){
toss3(capi.SQLITE_MISUSE, "Malformed export object.");
+ }else if( !exp.size
+ || (exp.size !== (exp.size | 0))
+ || (exp.size % cache.fixedPageSize)
+ || exp.size>=0x7fffffff ){
+ toss3(capi.SQLITE_RANGE, "Invalid db size: "+exp.size);
}
+
validateStorageName(exp.name);
let store = storageForZClass(exp.name);
const isNew = !store;
if( cache.fixedPageSize !== n ){
util.toss3(capi.SQLITE_RANGE,"Unexpected page size:", n);
}
- zEnc = cache.memBuffer(1);
+ zEnc ??= cache.memBuffer(1);
const zBin = cache.memBuffer(0),
- heap = wasm.heap8u();
- /* Copy u to the heap and encode the heaped copy. This is
- _presumably_ faster than porting the encoding algo to
- JS, which would involve many, many more function calls. */
- let i;
- for(i=0; i<n; ++i ) heap[wasm.ptr.add(zBin,i)] = u[i];
- heap[wasm.ptr.add(zBin,i)] = 0;
- const rc = wasm.exports.sqlite3__wasm_kvvfs_encode(zBin, i, zEnc);
+ heap = wasm.heap8u()/*MUST be inited last*/;
+ /* Copy u to the heap and encode the heap copy via C. This
+ is _presumably_ faster than porting the encoding algo to
+ JS. */
+ heap.set(u, zBin);
+ heap[wasm.ptr.add(zBin,n)] = 0;
+ const rc = kvvfsEncode(zBin, n, zEnc);
util.assert( rc < cache.buffer.n,
"Impossibly long output - possibly smashed the heap" );
util.assert( 0===wasm.peek8(wasm.ptr.add(zEnc,rc)),
- events: an object which may have any of the following
callback function properties: open, close, write, delete.
+ - decodePages [=false]: if true, write events will receive each
+ db page write in the form of a Uint8Array holding the raw binary
+ db page. The default is to emit the kvvfs-format page because it
+ requires no extra work, we already have it in hand, and it's
+ often smaller. It's not great for interchange, though.
+
+ - elideJournal [=false]: if true, writes and deletes of
+ "jrnl" records are elided (no event is sent).
+
+ Passing the same object ot sqlite3_js_kvvfs_unlisten() will
+ remove the listener.
+
Each one of the events callbacks will be called asynchronously
when the given storage performs those operations. They may be
- asynchronous. All exceptions, including those via Promises, are
- ignored but may trigger warning output on the console.
+ asynchronous functions but are not required to be (the events are
+ fired async either way, but making the event callbacks async may
+ be advantageous when multiple listeners are involved). All
+ exceptions, including those via Promises, are ignored but may (or
+ may not) trigger warning output on the console.
Each callback gets passed a single object with the following
properties:
currently-opened handles on the storage.
- 'write' gets a length-two array holding the key and value which
- were written (both strings).
+ were written. The key is always a string, even if it's a db page
+ number. For db-page records, the value's type depends on
+ opt.decodePages. All others, including the journal, are strings
+ (the latter, being in a kvvfs-specific format, is delivered in
+ its kvvfs-native format). More details below.
- 'delete' gets the string-type key of the deleted record.
- Passing the same object ot sqlite3_js_kvvfs_unlisten() will
- remove the listener.
-
- The arguments to 'write' and 'delete' are in one of the following
- forms:
+ The arguments to 'write' and 'delete' are in one of the
+ following forms:
- 'sz' = the unencoded db size as a string
}
}
if( opt.events ){
- (store.listeners ??= []).push(opt.events);
+ (store.listeners ??= []).push(opt);
}
};
const sqlite3_js_kvvfs_unlisten = function(opt){
const store = storageForZClass(opt?.storage);
if( store?.listeners && opt.events ){
- store.listeners = store.listeners.filter((v)=>v!==opt.events);
+ store.listeners = store.listeners.filter((v)=>v!==opt);
if( !store.listeners.length ){
store.listeners = undefined;
}
if( 0 ){
const scope = wasm.scopedAllocPush();
try{
- const pg = "53514C69746520666F726D61742033b20b0101b402020d02d02l01d04l01jb02b2E91E00Dd011FD3b1FD3dxl2B010617171701377461626C656B767666736B7676667302435245415445205441424C45206B76766673286129";
+ const pg = [
+ "53514C69746520666F726D61742033b20b0101b402020d02d02l01d0",
+ "4l01jb02b2E91E00Dd011FD3b1FD3dxl2B010617171701377461626C",
+ "656B767666736B7676667302435245415445205441424C45206B76766673286129"
+ ].join('');
const n = pg.length;
const pI = wasm.scopedAlloc( n+1 );
const nO = 8192 * 2;
}
}
}/*concurrent transient kvvfs*/)
+
.t({
name: 'kvvfs listeners (experiment)',
test: function(sqlite3){
const counts = Object.assign(Object.create(null),{
open: 0, close: 0, delete: 0, write: 0
});
+ const pglog = Object.create(null);
+ pglog.pages = [];
+ pglog.jrnl = undefined;
+ pglog.size = undefined;
+ pglog.elideJournal = true;
+ //pglog.decodePages = true;
+ pglog.exception = new Error("Testing that exceptions from listeners do not interfere");
+ const toss = ()=>{
+ if( pglog.exception ){
+ const e = pglog.exception;
+ delete pglog.exception;
+ throw e;
+ }
+ };
const listener = {
storage: filename,
reserve: true,
+ elideJournal: pglog.elideJournal,
+ decodePages: pglog.decodePages,
events: {
'open': (ev)=>{
//console.warn('open',ev);
.assert('number'===typeof ev.data);
},
'close': (ev)=>{
+ //^^^ if this is async, we can't time the test for
+ // pglog.exception without far more hoop-jumping.
//console.warn('close',ev);
++counts[ev.type];
T.assert('number'===typeof ev.data);
+ toss();
},
'delete': (ev)=>{
//console.warn('delete',ev);
++counts[ev.type];
T.assert('string'===typeof ev.data);
+ switch(ev.data){
+ case 'jrnl':
+ T.assert(!pglog.elideJournal);
+ pglog.jrnl = null;
+ break;
+ default:
+ T.assert( +ev.data>0, "Expecting positive db page number" );
+ pglog[+ev.data] = undefined;
+ break;
+ }
},
'write': (ev)=>{
//console.warn('write',ev);
++counts[ev.type];
+ const key = ev.data[0], val = ev.data[1];
T.assert(Array.isArray(ev.data))
- .assert('string'===typeof ev.data[0])
- .assert('string'===typeof ev.data[1]);
+ .assert('string'===typeof key);
+ switch( key ){
+ case 'jrnl':
+ T.assert(!pglog.elideJournal);
+ pglog.jrnl = val;
+ break;
+ case 'sz':{
+ const sz = +val;
+ T.assert( sz>0, "Expecting a db page number" );
+ if( sz < pglog.sz ){
+ pglog.pages.length = sz / pglog.pages.length;
+ }
+ pglog.size = sz;
+ break;
+ }
+ default:
+ T.assert( +key>0, "Expecting a positive db page number" );
+ pglog.pages[+key] = val;
+ if( pglog.decodePages ){
+ T.assert( val instanceof Uint8Array );
+ }else{
+ T.assert( 'string'===typeof val );
+ }
+ break;
+ }
}
}
};
debug("kvvfs listener counts:",counts);
T.assert( counts.open )
.assert( counts.close )
- .assert( counts.delete )
+ .assert( listener.elideJournal ? !counts.delete : counts.delete )
.assert( counts.write )
- .assert( counts.open===counts.close );
+ .assert( counts.open===counts.close )
+ .assert( pglog.elideJournal
+ ? (undefined===pglog.jrnl)
+ : (null===pglog.jrnl),
+ "Unexpected pglog.jrnl value: "+pglog.jrnl );
+ if( 1 ){
+ T.assert(undefined===pglog.pages[0], "Expecting empty slot 0");
+ pglog.pages.shift();
+ debug("kvvfs listener pageLog", pglog);
+ }
const before = JSON.stringify(counts);
sqlite3.kvvfs.unlisten(listener);
db = new DB(dbFileRaw);
-C Extend\skvvfs\sexport\sto\soptionally\sexport\sthe\sraw\sbinary\sdb\spages\sas\sa\slist\sof\sUint8Array\sinstead\sof\skvvfs-encoded\sstrings.\sThis\sis\stypically\smuch\slarger\sbut\sthe\spages\scan\sthen\sbe\sused\sas-is.
-D 2025-11-30T03:02:06.188
+C Extend\sthe\skvvfs.listen()\sconfig\sto\senable\sposting\sof\sraw\sbinary\sdb\spages\sinstead\sof\sthe\skvvfs-encoding.\sThis\sis\smuch\smore\sexpensive\sbut\swas\sadded\sto...\sDemonstrate\sbasic\sasync\sstreaming\sof\skvvfs\sdb\spage-level\schanges\svia\slogging\sof\skvvfs\swrite/delete\sops.
+D 2025-11-30T05:20:17.833
F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89
F ext/wasm/api/sqlite3-opfs-async-proxy.js 9654b565b346dc609b75d15337f20acfa7af7d9d558da1afeb9b6d8eaa404966
F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d
-F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js e01f80208b7f00a47045bc92e5c0b40c6e2232fc058f2dd81ed41d46b1cfd323
+F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 3d07cc5dd8b20fa81401c9bea93ab794fad4412dd4982280e695f7fb2f0bffbf
F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js a2eea6442556867b589e04107796c6e1d04a472219529eeb45b7cd221d7d048b
F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 88ce2078267a2d1af57525a32d896295f4a8db7664de0e17e82dc9ff006ed8d3
F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81
F ext/wasm/test-opfs-vfs.js 1618670e466f424aa289859fe0ec8ded223e42e9e69b5c851f809baaaca1a00c
F ext/wasm/tester1-worker.c-pp.html 0e432ec2c0d99cd470484337066e8d27e7aee4641d97115338f7d962bf7b081a
F ext/wasm/tester1.c-pp.html 52d88fe2c6f21a046030a36410b4839b632f4424028197a45a3d5669ea724ddb
-F ext/wasm/tester1.c-pp.js d5bf9dd8be21377981e20974fa798ae9839ee7a279189a773299532a844a8138
+F ext/wasm/tester1.c-pp.js 322e86c7b1627b61f9decdc741052aa52d62733af004055c6b9850365aa7f5bb
F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e
F ext/wasm/tests/opfs/concurrency/test.js d08889a5bb6e61937d0b8cbb78c9efbefbf65ad09f510589c779b7cc6a803a88
F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2
F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7
F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P ed9ab366d1c1880d3c06edce6c0c33ad30c7ae59725c1ec1fe3f620be1835630
-R 8530361058afe18d87abefbdbe6a3870
+P a4f59496a53a079f8f73e4cde68f47dbd13d2d74de2ad11bc716e7e5c00f1ec0
+R 657f4aadff300838a0780e1d6467cc6f
U stephan
-Z 197caaef7aceb5323c153f1905283e67
+Z 248974dfe9e189a43e97fc46a03783b5
# Remove this line to create a well-formed Fossil manifest.
projects / thirdparty / sqlite.git / commitdiff
