From: stephan Date: Sun, 30 Nov 2025 18:43:59 +0000 (+0000) Subject: kvvfs docs and 64-bit fixes. X-Git-Tag: artiphishell~117^2~12 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=8ff58b4f41ed23f0b63cd8949208de4ba644a77c;p=thirdparty%2Fsqlite.git kvvfs docs and 64-bit fixes. FossilOrigin-Name: cf58e17fa2dc2e183a6ea1d41795c701efb303c9b378aa9b90953c9b568c621a --- diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index aba900318e..0c7d499da1 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -1267,6 +1267,8 @@ speedtest1: b-speedtest1 # # Generate 64-bit variants of speedtest1*.{js,html} # +# TODO: preprocess these like we do the rest. +# define gen-st64 $(2): $(1) @$$(call b.echo,speedtest164,$$(emo.disk)$(emo.lock) Creating from $$<) diff --git a/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js index dc58a6565b..a578a4c4e1 100644 --- a/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js @@ -115,21 +115,66 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ); util.assert( 32<=kvvfsMethods.$nKeySize, "unexpected kvvfsMethods.$nKeySize: "+kvvfsMethods.$nKeySize); + /** + Most of the VFS-internal state. + */ const cache = Object.assign(Object.create(null),{ - fixedPageSize: 8192/*used in some validation*/, + /** + Bug: kvvfs is currently fixed at a page size of 8kb because + changing it is leaving corrupted dbs for reasons as yet + unknown. This value is used in certain validation to ensure + that if that limitation is lifted, it will break potentially + affected code. + */ + fixedPageSize: 8192, + /** Regex matching journal file names. */ rxJournalSuffix: /-journal$/, + /** Frequently-used C-string. */ zKeyJrnl: wasm.allocCString("jrnl"), + /** Frequently-used C-string. */ zKeySz: wasm.allocCString("sz"), + /** + The maximum size of a kvvfs record key. It is historically only + 32, a limitation currently retained only because it's convenient to + do so (the underlying code has outgrown the need for the artifically + low limit). + + We cache this value here because the end of this init code will + dispose of kvvfsMethods, invalidating it. + */ keySize: kvvfsMethods.$nKeySize, + /** + WASM heap memory buffers to optimize out some frequent + allocations. + */ buffer: Object.assign(Object.create(null),{ + /** + The size of each buffer in this.pool. + + kvvfsMethods.$nBufferSize is slightly larger than the output + space needed for a kvvfs-encoded 64kb db page in a worse-cast + encoding (128kb). It is not suitable for arbitrary buffer + use, only page de/encoding. As the VFS system has no hook + into library finalization, these buffers are effectively + leaked except in the few places which use memBufferFree(). + */ n: kvvfsMethods.$nBufferSize, + /** + Map of buffer ids to wasm.alloc()'d pointers of size + this.n. (Re)used by various internals. + + Buffer ids 0 and 1 are used in the API internals. Other + names are used in higher-level APIs. + + See memBuffer() and memBufferFree(). + */ pool: Object.create(null) }) }); /** - A wasm.alloc()'d buffer large enough for all kvvfs - encoding/decoding needs (cache.buffer.n). + Returns a (cached) wasm.alloc()'d buffer of cache.buffer.n size, + throwing on OOM. We leak this one-time alloc because we've no better option. sqlite3_vfs does not have a finalizer, so we've no place to hook @@ -139,7 +184,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ */ cache.memBuffer = (id=0)=>cache.buffer.pool[id] ??= wasm.alloc(cache.buffer.n); - /** Freeds the buffer with the given id. */ + /** Frees the buffer with the given id. */ cache.memBufferFree = (id)=>{ const b = cache.buffer.pool[id]; if( b ){ @@ -148,19 +193,21 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } }; + const noop = ()=>{}; const debug = sqlite3.__isUnderTest - ? function(){sqlite3.config.debug("kvvfs:", ...arguments)} - : function(){}; - const warn = function(){sqlite3.config.warn("kvvfs:", ...arguments)}; - const error = function(){sqlite3.config.error("kvvfs:", ...arguments)}; + ? (...args)=>sqlite3.config.debug("kvvfs:", ...args) + : noop; + const warn = (...args)=>sqlite3.config.warn("kvvfs:", ...args); + const error = (...args)=>sqlite3.config.error("kvvfs:", ...args); /** Implementation of JS's Storage interface for use as backing store of the kvvfs. Storage is a native class and its constructor cannot be legally called from JS, making it impossible to directly subclass Storage. This class implements (only) the - Storage interface, however, to make it a drop-in replacement for - localStorage/sessionStorage. + Storage interface, to make it a drop-in replacement for + localStorage/sessionStorage. (Any behavioral discrepancies are to + be considered bugs.) This impl simply proxies a plain, prototype-less Object, suitable for JSON-ing. @@ -266,26 +313,31 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /** Create a new instance of the objects which go into - cache.storagePool. + cache.storagePool, with a refcount of 1. If passed a Storage-like + object as its second argument, it is used for the storage, + otherwise it creates a new KVVfsStorage object. */ - const newStorageObj = (name,storage)=>Object.assign(Object.create(null),{ + const newStorageObj = (name,storage=undefined)=>Object.assign(Object.create(null),{ /** JS string value of this KVVfsFile::$zClass. i.e. the storage's name. */ jzClass: name, /** - Refcount to keep dbs and journals pointing to the same storage - for the life of both. Managed by xOpen() and xClose(). + Refcount. This keeps dbs and journals pointing to the same + storage for the life of both and enables kvvfs to behave more + like a conventional filesystem (a stepping stone towards + downstream API goals). Managed by xOpen() and xClose(). */ refc: 1, /** - deleteAtRefc0 objects will be removed by xClose() when refc - reaches 0. The others will persist, to give the illusion of - real back-end storage. Managed by xOpen(). By default this is - false but the delete-on-close=1 flag can be used to set this to - true. - */ + If true, this storage will be removed by xClose() or + sqlite3_js_kvvfs_unlink() when refc reaches 0. The others will + persist when refc==0, to give the illusion of real back-end + storage. Managed by xOpen() and sqlite3_js_kvvfs_reserve(). By + default this is false but the delete-on-close=1 flag can be + used to set this to true. + */ deleteAtRefc0: false, /** The backing store. Must implement the Storage interface. @@ -294,10 +346,10 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /** The storage prefix used for kvvfs keys. It is "kvvfs-STORAGENAME-" for local/session storage and an empty - string for transient storage. local/session storage must use - the long form (A) for backwards compatibility and (B) so that - kvvfs can coexist with non-db client data in those backends. - Neither (A) nor (B) are concerns for KVVfsStorage objects. + string for other storage. local/session storage must use the + long form (A) for backwards compatibility and (B) so that kvvfs + can coexist with non-db client data in those backends. Neither + (A) nor (B) are concerns for KVVfsStorage objects. This prefix mirrors the one generated by os_kv.c's kvrecordMakeKey() and must stay in sync with that one. @@ -310,14 +362,17 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ files: [], /** If set, it's an array of objects with various event - callbacks. See sqlite3_js_kvvfs_listen(). + callbacks. See sqlite3_js_kvvfs_listen(). When there are no + listeners, this member is set to undefined (instead of an empty + array) to allow us to more easily optimize out calls to + notifyListeners() for the common case of no listeners. */ listeners: undefined }); /** - Deletes the cache.storagePool entries for store and its - db/journal counterpart. + Deletes the cache.storagePool entries for store (a + cache.storagePool entry) and its db/journal counterpart. */ const deleteStorage = function(store){ const other = cache.rxJournalSuffix.test(store.jzClass) @@ -402,6 +457,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return e; }; + /** Exception handler for notifyListeners(). */ const catchForNotify = (e)=>{ warn("kvvfs.listener handler threw:",e); }; @@ -425,58 +481,63 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ xFileControl(). */ const notifyListeners = async function(eventName,store,...args){ - if( store.listeners ){ - //cache.rxPageNoSuffix ??= /(\d+)$/; - if( store.keyPrefix && args[0] ){ - args[0] = args[0].replace(store.keyPrefix,''); - } - let u8enc, z0, z1, wcache; - for(const ear of store.listeners){ - const ev = Object.create(null); - ev.storageName = store.jzClass; - ev.type = eventName; - const decodePages = ear.decodePages; - const f = ear.events[eventName]; - if( f ){ - if( !ear.includeJournal && args[0]==='jrnl' ){ - continue; - } - if( 'write'===eventName && ear.decodePages && +args[0]>0 ){ - /* Decode pages to Uint8Array. */ - ev.data = [args[0]]; - if( wcache?.[args[0]] ){ - ev.data[1] = wcache[args[0]]; + try{ + if( store.listeners ){ + //cache.rxPageNoSuffix ??= /(\d+)$/; + if( store.keyPrefix && args[0] ){ + args[0] = args[0].replace(store.keyPrefix,''); + } + let u8enc, z0, z1, wcache; + for(const ear of store.listeners){ + const ev = Object.create(null); + ev.storageName = store.jzClass; + ev.type = eventName; + const decodePages = ear.decodePages; + const f = ear.events[eventName]; + if( f ){ + if( !ear.includeJournal && args[0]==='jrnl' ){ continue; } - u8enc ??= new TextEncoder('utf-8'); - z0 ??= cache.memBuffer(10); - z1 ??= cache.memBuffer(11); - const u = u8enc.encode(args[1]); - const heap = wasm.heap8u(); - heap.set(u, z0); - heap[wasm.ptr.add(z0, u.length)] = 0; - const rc = kvvfsDecode(z0, z1, cache.buffer.n); - if( rc>0 ){ - wcache ??= Object.create(null); - wcache[args[0]] - = ev.data[1] - = heap.slice(z1, wasm.ptr.add(z1,rc)); + if( 'write'===eventName && ear.decodePages && +args[0]>0 ){ + /* Decode pages to Uint8Array, caching the result in + wcache in case we have more listeners. */ + ev.data = [args[0]]; + if( wcache?.[args[0]] ){ + ev.data[1] = wcache[args[0]]; + continue; + } + u8enc ??= new TextEncoder('utf-8'); + z0 ??= cache.memBuffer(10); + z1 ??= cache.memBuffer(11); + const u = u8enc.encode(args[1]); + const heap = wasm.heap8u(); + heap.set(u, Number(z0)); + heap[wasm.ptr.addn(z0, u.length)] = 0; + const rc = kvvfsDecode(z0, z1, cache.buffer.n); + if( rc>0 ){ + wcache ??= Object.create(null); + wcache[args[0]] + = ev.data[1] + = heap.slice(Number(z1), wasm.ptr.addn(z1,rc)); + }else{ + continue; + } }else{ - continue; + ev.data = args.length + ? ((args.length===1) ? args[0] : args) + : undefined; + } + try{f(ev)?.catch?.(catchForNotify)} + catch(e){ + warn("notifyListeners [",store.jzClass,"]",eventName,e); } - }else{ - ev.data = args.length - ? ((args.length===1) ? args[0] : args) - : undefined; - } - try{f(ev)?.catch?.(catchForNotify)} - catch(e){ - warn("notifyListeners [",store.jzClass,"]",eventName,e); } } } + }catch(e){ + catchForNotify(e); } - }; + }/*notifyListeners()*/; /** Returns the storage object mapped to the given string zClass @@ -1223,7 +1284,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ util.toss3(capi.SQLITE_ERROR,"Unexpected decoded page size:",nDec); } //debug("Decoded",nDec,"page bytes"); - pages[kk] = heap.slice(zDec, wasm.ptr.add(zDec, nDec)); + pages[kk] = heap.slice(Number(zDec), wasm.ptr.addn(zDec, nDec)); }else{ pages[kk] = s.getItem(k); } @@ -1325,8 +1386,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /* Copy u to the heap and encode the heap copy via C. This is _presumably_ faster than porting the encoding algo to JS. */ - heap.set(u, zBin); - heap[wasm.ptr.add(zBin,n)] = 0; + heap.set(u, Number(zBin)); + heap[wasm.ptr.addn(zBin,n)] = 0; const rc = kvvfsEncode(zBin, n, zEnc); util.assert( rc < cache.buffer.n, "Impossibly long output - possibly smashed the heap" ); diff --git a/ext/wasm/index.html b/ext/wasm/index.html index 55e4cdb75c..fd65dbc339 100644 --- a/ext/wasm/index.html +++ b/ext/wasm/index.html @@ -105,6 +105,11 @@ (32-bit, 64-bit): an interactive Worker-thread variant of speedtest1. +
  • speedtest1-worker?vfs=kvvfs + (32-bit, + 64-bit): + speedtest1-worker with the + kvvfs VFS preselected and configured for a moderate workload.
  • speedtest1-worker?vfs=opfs (32-bit, 64-bit): diff --git a/manifest b/manifest index 90e76f0bf3..71c54c614c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Optimize\sout\sa\skvvfs\sevent\snotification\scall\sin\sthe\scommon\scase\swhere\sthere\sare\sno\slisteners. -D 2025-11-30T16:37:47.709 +C kvvfs\sdocs\sand\s64-bit\sfixes. +D 2025-11-30T18:43:59.989 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -578,7 +578,7 @@ F ext/session/sessionwor.test 6fd9a2256442cebde5b2284936ae9e0d54bde692d0f5fd009e F ext/session/sqlite3session.c b3de195ce668cace9b324599bf6255a70290cbfb5451e826e946f3aee6e64c54 F ext/session/sqlite3session.h 7404723606074fcb2afdc6b72c206072cdb2b7d8ba097ca1559174a80bc26f7a F ext/session/test_session.c 8766b5973a6323934cb51248f621c3dc87ad2a98f023c3cc280d79e7d78d36fb -F ext/wasm/GNUmakefile 2fe52720144ab481755c6df45f46dcf6c540f8622a5387ed280f1d0b26a6d28f +F ext/wasm/GNUmakefile 0b40ca7fc6310a1375b813dde2b26b549e6a650c10045418fe8440619a3e826c F ext/wasm/README-dist.txt f01081a850ce38a56706af6b481e3a7878e24e42b314cfcd4b129f0f8427066a F ext/wasm/README.md 2e87804e12c98f1d194b7a06162a88441d33bb443efcfe00dc6565a780d2f259 F ext/wasm/SQLTester/GNUmakefile e0794f676d55819951bbfae45cc5e8d7818dc460492dc317ce7f0d2eca15caff @@ -600,7 +600,7 @@ F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 9654b565b346dc609b75d15337f20acfa7af7d9d558da1afeb9b6d8eaa404966 F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d -F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 278c7882968b01e9fc8269ae94f725e85acce1dc0a38e91b71397a17d8c1876e +F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js bc83c21b2211f1bcaa2b200a08411d86962426284987a39477543bc45bd64260 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js a2eea6442556867b589e04107796c6e1d04a472219529eeb45b7cd221d7d048b F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 88ce2078267a2d1af57525a32d896295f4a8db7664de0e17e82dc9ff006ed8d3 F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81 @@ -627,7 +627,7 @@ F ext/wasm/fiddle/fiddle-worker.js 7798af02e672e088ff192716f80626c8895e19301a65b F ext/wasm/fiddle/fiddle.js 84fd75967e0af8b69d3dd849818342227d0f81d13db92e0dcbc63649b31a4893 F ext/wasm/fiddle/index.c-pp.html 72c7e5517217960b3809648429ea396a7cbad0ffb2c92f6a2f5703abecb27317 F ext/wasm/index-dist.html db23748044e286773f2768eec287669501703b5d5f72755e8db73607dc54d290 -F ext/wasm/index.html 54e27db740695ab2cb296e02d42c4c66b3f11b65797340d19fa6590f5b287da1 +F ext/wasm/index.html 475bc283338749db4e3fbf24cf3f5aa020cc85a1fffb780d400a915fcb5f1756 F ext/wasm/jaccwabyt/jaccwabyt.js 4e2b797dc170851c9c530c3567679f4aa509eec0fab73b466d945b00b356574b F ext/wasm/jaccwabyt/jaccwabyt.md 6aa90fa1a973d0ad10d077088bea163b241d8470c75eafdef87620a1de1dea41 F ext/wasm/mkdist.sh 64d53f469c823ed311f6696f69cec9093f745e467334b34f5ceabdf9de3c5b28 x @@ -2180,8 +2180,8 @@ F tool/version-info.c 33d0390ef484b3b1cb685d59362be891ea162123cea181cb8e6d2cf6dd F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7f0eca9d0aeb49c86a785ae930d235902bbd0f15877cc8f6083daac8efb2d1c1 -R bd382764c774d080ec17e2301611eab2 +P 8405c19d32f1e8b7273953a038f8bdfd4ea1a6548300bac5421cdf6fc6840285 +R 2f3b7cfa7869c544f8a6af729f308175 U stephan -Z 249c02695d60433ade4ac626997fb17e +Z d109a9becc744b2aad4397bfe3d32fc6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4b986bda31..8a7bb22797 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8405c19d32f1e8b7273953a038f8bdfd4ea1a6548300bac5421cdf6fc6840285 +cf58e17fa2dc2e183a6ea1d41795c701efb303c9b378aa9b90953c9b568c621a