From: stephan Date: Sun, 23 Nov 2025 20:05:21 +0000 (+0000) Subject: Docs, cleanups, and dead code removal. X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=606b1bd1539e9bf8cd2fd42e710ebe844c443d2e;p=thirdparty%2Fsqlite.git Docs, cleanups, and dead code removal. FossilOrigin-Name: 49db59aa9c74e49d878adc8671b0d32db8f1f898bde29d046ce0e368d8987868 --- diff --git a/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js index 08b67116cb..bf6d6b4e65 100644 --- a/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js @@ -19,6 +19,64 @@ Documentation home page: https://sqlite.org/wasm */ + +/** + kvvfs - the Key/Value VFS - is an SQLite3 VFS which delegates + storage of its pages and metadata to a key-value store. + + It was conceived in order to support JS's localStorage and + sessionStorage objects. Its native implementation uses files as + key/value storage (one file per record) but the JS implementation + replaces a few methods so that it can use the aforementioned + objects as storage. + + It uses a bespoke ASCII encoding to store each db page as a + separate record and stores some metadata, like the db's encoded + size and its journal, as individual records. + + kvvfs is significantly less efficient than a plain in-memory db but + it also, as a side effect of its design, offers a JSON-friendly + interchange format for exporting and importing databases. + + kvvfs is _not_ designed for heavy db loads. It is relatively + malloc()-heavy, having to de/allocate frequently, and it + spends much of its time converting the raw db pages into and out of + an ASCII encoding. + + But it _does_ work and is "performant enough" for db work of the + scale of a db which will fit within sessionStorage or localStorage + (just 2-3mb). + + "Version 2" extends it to support using Storage-like objects as + backing storage, Storage being the JS class which localStorage and + sessionStorage both derive from. This essentially moves the backing + store from whatever localStorage and sessionStorage use to an + in-memory object. + + This effort is primarily a stepping stone towards eliminating, if + it proves possible, the POSIX I/O API dependencies in SQLite's WASM + builds. That is: if this VFS works properly, it can be set as the + default VFS and we can eliminate the "unix" VFS from the JS/WASM + builds (as opposed to server-wise/WASI builds). That still, as of + 2025-11-23, a ways away, but it's the main driver for version 2 of + kvvfs. + + Version 2 remains compatible with version 1 databases and always + writes localStorage/sessionStorage metadata in the v1 format, so + such dbs can be manipulated freely by either version. For transient + storage objects (new in version 2), the format of its record keys + is simpified, requiring less space than v1 keys by eliding + redundant (in this context) info from the keys. + + Another benefit of v2 is its ability to export dbs into a + JSON-friendly (but not human-friendly) format. + + A potential, as-yet-unproven, benefit, would be the ability to plug + arbitrary Storage-compatible objects in so that clients could, + e.g. asynchronously post updates to db pages to some back-end for + backups. +*/ + globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ 'use strict'; /* We unregister the kvvfs VFS from Worker threads later on. */ @@ -57,12 +115,14 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Implementation of JS's Storage interface for use as backing store of the kvvfs. Storage is a native class and its constructor cannot be legally called from JS, making it impossible to - directly subclass Storage. + directly subclass Storage. This class implements the Storage + interface, however, to make it a drop-in replacement for + localStorage/sessionStorage. This impl simply proxies a plain, prototype-less Object, suitable for JSON-ing. */ - class TransientStorage { + class KVVfsStorage { #map; #keys; #getKeys(){return this.#keys ??= Object.keys(this.#map);} @@ -101,7 +161,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ get length() { return this.#getKeys().length; } - }/*TransientStorage*/; + }/*KVVfsStorage*/; /** Map of JS-stringified KVVfsFile::zClass names to @@ -115,7 +175,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /* Start off with mappings for well-known names. */ localThread: { refc: 3/*never reaches 0*/, - s: new TransientStorage, + s: new KVVfsStorage, files: [/*KVVfsFile instances currently using this storage*/] } }); @@ -207,6 +267,30 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return rc; }; +//#if nope + // fileForDb() works but we don't have a current need for it. + /** + Expects an (sqlite3*). Uses sqlite3_file_control() to extract its + (sqlite3_file*). On success it returns a new KVVfsFile instance + wrapping that pointer, which the caller must eventual call + dispose() on (which won't free the underlying pointer, just the + wrapper). + */ + const fileForDb = function(pDb){ + const stack = pstack.pointer; + try{ + const pOut = pstack.allocPtr(); + return wasm.exports.sqlite3_file_control( + pDb, wasm.ptr.null, capi.SQLITE_FCNTL_FILE_POINTER, pOut + ) + ? null + : new KVVfsFile(wasm.peekPtr(pOut)); + }finally{ + pstack.restore(stack); + } + }; +//#endif nope + /** Clears all storage used by the kvvfs DB backend, deleting any DB(s) stored there. @@ -492,25 +576,18 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ if( rc ) return rc; const f = new KVVfsFile(pProtoFile); util.assert(f.$zClass, "Missing f.$zClass"); - const jzClass = wasm.cstrToJs(zName);//f.$zClass); + const jzClass = wasm.cstrToJs(zName); let s = cache.jzClassToStorage[jzClass]; //debug("xOpen", jzClass, s); if( s ){ ++s.refc; s.files.push(f); - if( false && !s.keyPrefix ){ - /* this messes up the recordHandler methods. They have only - the key, not the sqlite3_file object, so cannot map - a prefixless key to a storage object. */ - f.$zClass = wasm.ptr.null; - } }else{ /* TODO: a url flag which tells it to keep the storage around forever so that future xOpen()s get the same Storage-ish objects. We can accomplish that by simply increasing the refcount once more. */ util.assert( !f.$isJournal, "Opening a journal before its db? "+jzClass ); - //breaks stuff f.$zClass = wasm.ptr.null /* causes the "kvvfs-" prefix to be elided from keys */; const other = f.$isJournal ? jzClass.replace(cache.rxJournalSuffix,'') : jzClass + '-journal'; @@ -522,7 +599,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ will follow soon enough and bump the refcount. If we start at 2 here, that pending open will increment it again. */, - s: new TransientStorage, + s: new KVVfsStorage, keyPrefix: '', files: [f] }); @@ -619,12 +696,12 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ // these impls work but there's currently no pressing need _not_ use // the native impls. xCurrentTime: function(pVfs,pOut){ - wasm.poke64f(pOut, 2440587.5 + (new Date().getTime()/86400000)); + wasm.poke64f(pOut, 2440587.5 + (Date.now()/86400000)); return 0; }, xCurrentTimeInt64: function(pVfs,pOut){ - wasm.poke64(pOut, (2440587.5 * 86400000) + new Date().getTime()); + wasm.poke64(pOut, (2440587.5 * 86400000) + Date.now()); return 0; } //#endif @@ -857,7 +934,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const s = store.s; const rc = Object.assign(Object.create(null),{ name: this.filename, - timestamp: (new Date()).valueOf(), + timestamp: Date.now(), pages: [] }); const pages = Object.create(null); @@ -904,16 +981,27 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ invoking this while the db is in active use invokes undefined behavior. - Throws on error. Returns this object on success. + Returns this object on success. Throws on error. Error + conditions include: + + - This db is closed. + + - Other handles to the same storage object are opened. + Performing this page-by-page import would invoke undefined + behavior on them. - FIXMEs: + - A transaction is active. - - We need the page size in the export so that we can reset it, - if needed, on the import. + Those are the error case it can easily cover. The room for + undefined behavior in wiping a db's storage out from under it + is a whole other potential minefield. - - We need to ensure that the native-size KVVfsFile::szDb and - KVVfsFile::szPage get set to -1 for all open instances so that - they re-read the db size. + If it throws after starting the input then it clears the + storage before returning, to avoid leaving the db in an + undefined state. It has no inherent error conditions during the + input phase beyond out-of-memory but it may throw for any of + the above-listed conditions before reaching that step, in which + case the db is not modified. */ jdb.prototype.importFromObject = function(exp){ this.affirmOpen(); @@ -923,12 +1011,23 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ || !Array.isArray(exp.pages) ){ util.toss3(capi.SQLITE_MISUSE, "Malformed export object."); } + if( s.files.length>1 ){ + util.toss3(capi.SQLITE_IOERR_ACCESS, + "Cannot import a db when multiple handles to it", + "are opened."); + }else if( capi.sqlite3_txn_state(this.pointer, null)>0 ){ + util.toss3(capi.SQLITE_MISUSE, + "Cannot import the db while a transaction is active."); + } warn("importFromObject() is incomplete"); - this.clearStorage(); const store = kvvfsWhich(this.filename); - util.assert(store?.s, "Somehow missing a storage object for",this.filename); + util.assert(store?.s, "Somehow missing a storage object for", this.filename); const keyPrefix = kvvfsKeyPrefix(this.filename); + this.clearStorage(); try{ + s.files.forEach((f)=>f.$szPage = $f.$szDb = -1) + /* Force the native KVVfsFile instances to re-read the db + and page size. */; s.setItem(keyPrefix+'sz', exp.size); if( exp.journal ) s.setItem(keyPrefix+'jrnl', exp.journal); exp.pages.forEach((v,ndx)=>s.setItem(keyPrefix+(ndx+1))); diff --git a/manifest b/manifest index 73065f8e2c..62673c4340 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Get\sthe\skvvfs\sv2\sworking\sin\s64-bit\swasm\sbuilds\s(a\s0\svs\snull\svs\sBigInt\sissue). -D 2025-11-23T16:54:16.202 +C Docs,\scleanups,\sand\sdead\scode\sremoval. +D 2025-11-23T20:05:21.758 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -600,7 +600,7 @@ F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 9654b565b346dc609b75d15337f20acfa7af7d9d558da1afeb9b6d8eaa404966 F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d -F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 6886aa3a355ea7332190877fac6e5d569ee0c158cf467b45b0fc876e5d305d33 +F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 1aaea0e80bbc992a365088b2f0c7ddb8fe9ec61d539fd75bc457db879e7a7eaa F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 26cb41d5a62f46a106b6371eb00fef02de3cdbfaa51338ba087a45f53028e0d0 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 88ce2078267a2d1af57525a32d896295f4a8db7664de0e17e82dc9ff006ed8d3 F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 9097074724172e31e56ce20ccd7482259cf72a76124213cbc9469d757676da86 @@ -2178,8 +2178,8 @@ F tool/version-info.c 33d0390ef484b3b1cb685d59362be891ea162123cea181cb8e6d2cf6dd F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f63014f70febf82976a99f9f1ce6d793e2ca8d1dd2f72622152b64b2d65f8adc -R 9d104150c6613ed7753bd1fb4ac1b002 +P 4811742688bd8ae847c0a13ca9b395b2d2edd24ce4e7760a5d1b0adf1c7f332d +R e3580b09ff0203465b33f3cc14e105b3 U stephan -Z cb3a60effd1de39bc1ff05e3b6dc575c +Z b07b7b7559951d4d97e0ab23e9e648e7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fc86bee876..335d8ffd1f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4811742688bd8ae847c0a13ca9b395b2d2edd24ce4e7760a5d1b0adf1c7f332d +49db59aa9c74e49d878adc8671b0d32db8f1f898bde29d046ce0e368d8987868