From: stephan Date: Fri, 21 Nov 2025 18:55:33 +0000 (+0000) Subject: More work towards using JS generic Storage objects as db page storage via kvvfs. X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=7c07260353b7630a10463e7c8ef37e5d591b37b0;p=thirdparty%2Fsqlite.git More work towards using JS generic Storage objects as db page storage via kvvfs. FossilOrigin-Name: 90a33941c69b3581feaed271542f0238ca81ee34fe5b353ca7da48b81ac73a5f --- diff --git a/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js index 3cb98d299a..03d170e6ab 100644 --- a/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js @@ -20,33 +20,112 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ 'use strict'; /* We unregister the kvvfs VFS from Worker threads later on. */ - const util = sqlite3.util, - capi = sqlite3.capi, - wasm = sqlite3.wasm, + const capi = sqlite3.capi, sqlite3_kvvfs_methods = capi.sqlite3_kvvfs_methods, - pKvvfs = sqlite3.capi.sqlite3_vfs_find("kvvfs"); - delete capi.sqlite3_kvvfs_methods /* this is JS plumbing, not part - of the public API */; - if( !pKvvfs ) return /* built without kvvfs */; + KVVfsFile = capi.KVVfsFile, + pKvvfs = sqlite3.capi.sqlite3_vfs_find("kvvfs") + + /* These are JS plumbing, not part of the public API */ + delete capi.sqlite3_kvvfs_methods; + delete capi.KVVfsFile; + + if( !pKvvfs ) return /* nothing to do */; + + const util = sqlite3.util, + wasm = sqlite3.wasm; if( !util.isUIThread() ){ - /* One test currently relies on this VFS not being visible - in Workers. If we add generic object storage, we can - retain this VFS in Workers. */ + /* One test currently relies on this VFS not being visible in + Workers. Once we add generic object storage, we can retain this + VFS in Workers, we just can't provide local/sessionStorage + access there. */ capi.sqlite3_vfs_unregister(pKvvfs); return; } - /** - Internal helper for sqlite3_js_kvvfs_clear() and friends. - Its argument should be one of ('local','session',""). - */ + const __hop = (o,k)=>Object.prototype.hasOwnProperty.call(o,k); + /** + Implementation of JS's Storage interface for use + as backing store of the kvvfs. + */ + class ObjectStorage extends Storage { + #map; + #keys; + #getKeys(){return this.#keys ??= Object.keys(this.#map);} + + constructor(){ + super(); + this.clear(); + } + + key(n){ + const k = this.#getKeys(); + return n((115/*=='s'*/===wasm.peek(zClass)) - ? sessionStorage : localStorage); + const kvvfsStorage = function(zClass){ + const s = wasm.cstrToJs(zClass); + if( !this.rxSession ){ + this.rxSession = /^session(-journal)?$/; + this.rxLocal = /^local(-journal)?$/; + } + if( this.rxSession.test(s) ) return sessionStorage; + if( this.rxLocal.test(s) ) return localStorage; + return kvvfsStorage.jClassToStorage.get(s); + }.bind(Object.create(null)); { /* Override native sqlite3_kvvfs_methods */ const kvvfsMethods = new sqlite3_kvvfs_methods( @@ -130,14 +217,14 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ backing store. The interface docs for these methods are in - src/os_kv.c:kvstorageRead(), kvstorageWrite(), and - kvstorageDelete(). + src/os_kv.c:kvrecordRead(), kvrecordWrite(), and + kvrecordDelete(). */ for(const e of Object.entries({ - xStorageRead: (zClass, zKey, zBuf, nBuf)=>{ + xRcrdRead: (zClass, zKey, zBuf, nBuf)=>{ const stack = pstack.pointer, astack = wasm.scopedAllocPush(); - try { + try{ const zXKey = kvvfsMakeKey(zClass,zKey); if(!zXKey) return -3/*OOM*/; const jKey = wasm.cstrToJs(zXKey); @@ -159,37 +246,37 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ wasm.poke(wasm.ptr.add(zBuf, nBuf, -1), 0); return nBuf - 1; }catch(e){ - sqlite3.config.error("kvstorageRead()",e); + sqlite3.config.error("kvrecordRead()",e); return -2; }finally{ pstack.restore(stack); wasm.scopedAllocPop(astack); } }, - xStorageWrite: (zClass, zKey, zData)=>{ + xRcrdWrite: (zClass, zKey, zData)=>{ const stack = pstack.pointer; try { const zXKey = kvvfsMakeKey(zClass,zKey); - if(!zXKey) return 1/*OOM*/; + if(!zXKey) return SQLITE_NOMEM; const jKey = wasm.cstrToJs(zXKey); kvvfsStorage(zClass).setItem(jKey, wasm.cstrToJs(zData)); return 0; }catch(e){ - sqlite3.config.error("kvstorageWrite()",e); + sqlite3.config.error("kvrecordWrite()",e); return capi.SQLITE_IOERR; }finally{ pstack.restore(stack); } }, - xStorageDelete: (zClass, zKey)=>{ + xRcrdDelete: (zClass, zKey)=>{ const stack = pstack.pointer; try { const zXKey = kvvfsMakeKey(zClass,zKey); - if(!zXKey) return 1/*OOM*/; + if(!zXKey) return capi.SQLITE_NOMEM; kvvfsStorage(zClass).removeItem(wasm.cstrToJs(zXKey)); return 0; }catch(e){ - sqlite3.config.error("kvstorageDelete()",e); + sqlite3.config.error("kvrecordDelete()",e); return capi.SQLITE_IOERR; }finally{ pstack.restore(stack); @@ -219,11 +306,13 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ in Worker threads and non-browser builds. They could optionally be exported to/from JSON. */ - const eventualTodo = 1 || { + const eventualTodo = { vfsMethods:{ +//#if nope /** */ - xOpen: function(pProtoVfs,zName,pProtoFile,flags,pOutFlags){}, + xOpen: function(pProtoVfs,zName,pProtoFile,flags,pOutFlags){ + }, xDelete: function(pVfs, zName, iSyncFlag){}, xAccess:function(pProtoVfs, zPath, flags, pResOut){}, xFullPathname: function(pVfs, zPath, nOut, zOut){}, @@ -238,8 +327,10 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ for(; i < nOut; ++i) heap[npOut + i] = (Math.random()*255000) & 0xFF; return i; } +//#endif }, +//#if nope /** kvvfs has separate impls for some of the I/O methods, depending on whether it's a db or journal file. @@ -276,6 +367,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ xDeviceCharacteristics: todoIOMethodsDb.xDeviceCharacteristics } } +//#endif }/*eventualTodo*/; if(sqlite3?.oo1?.DB){ diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index 98a9c2bff4..a83bd52f7b 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -1088,14 +1088,29 @@ const char * sqlite3__wasm_enum_json(void){ #undef CurrentStruct #define CurrentStruct sqlite3_kvvfs_methods + /* From os_kv.c */ StructBinder { - M(xRcrdRead, "i(sspi)"); - M(xRcrdWrite, "i(sss)"); - M(xRcrdDelete, "i(s)"); - M(nKeySize, "i"); - M(pVfs, "p"); - M(pIoDb, "p"); - M(pIoJrnl, "p"); + M(xRcrdRead, "i(sspi)"); + M(xRcrdWrite, "i(sss)"); + M(xRcrdDelete, "i(ss)"); + M(nKeySize, "i"); + M(pVfs, "p"); + M(pIoDb, "p"); + M(pIoJrnl, "p"); + } _StructBinder; +#undef CurrentStruct + +#define CurrentStruct KVVfsFile + /* From os_kv.c */ + StructBinder { + M(base, "p")/*sqlite3_file base*/; + M(zClass, "s"); + M(isJournal, "i"); + M(nJrnl, "i")/*actually unsigned!*/; + M(aJrnl, "p"); + M(szPage, "i"); + M(szDb, "j"); + M(aData, "p"); } _StructBinder; #undef CurrentStruct @@ -1152,7 +1167,13 @@ const char * sqlite3__wasm_enum_json(void){ ** sqlite3_index_info, we have to uplift those into constructs we ** can access by type name. These structs _must_ match their ** in-sqlite3_index_info counterparts byte for byte. - */ + ** + ** 2025-11-21: this uplifing is no longer necessary, as Jaccwabyt + ** can now handle nested structs, but "it ain't broke" so there's + ** no pressing need to rewire this. Also, it's conceivable that + ** rewiring it might break downstream vtab impls, so it shouldn't + ** be rewired. + */ typedef struct { int iColumn; unsigned char op; @@ -1566,7 +1587,7 @@ int sqlite3__wasm_posix_create_file( const char *zFilename, ** ** Allocates sqlite3KvvfsMethods.nKeySize bytes from ** sqlite3__wasm_pstack_alloc() and returns 0 if that allocation fails, -** else it passes that string to kvstorageMakeKey() and returns a +** else it passes that string to kvrecordMakeKey() and returns a ** NUL-terminated pointer to that string. It is up to the caller to ** use sqlite3__wasm_pstack_restore() to free the returned pointer. */ @@ -1577,7 +1598,7 @@ char * sqlite3__wasm_kvvfsMakeKeyOnPstack(const char *zClass, char *zKeyOut = (char *)sqlite3__wasm_pstack_alloc(sqlite3KvvfsMethods.nKeySize); if(zKeyOut){ - kvstorageMakeKey(zClass, zKeyIn, zKeyOut); + kvrecordMakeKey(zClass, zKeyIn, zKeyOut); } return zKeyOut; } diff --git a/manifest b/manifest index da6435427b..9e8a6bcc6a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\skvvfs-specific\sfilename\svalidation\sfrom\sthe\soo1.DB\sctor. -D 2025-11-21T18:49:58.635 +C More\swork\stowards\susing\sJS\sgeneric\sStorage\sobjects\sas\sdb\spage\sstorage\svia\skvvfs. +D 2025-11-21T18:55:33.689 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -600,11 +600,11 @@ F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 9654b565b346dc609b75d15337f20acfa7af7d9d558da1afeb9b6d8eaa404966 F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d -F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js c4e6009351a2841ade6ecab247c9883974eccd8d5e2d8c00d4bcc9999023b0ad +F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js e84a6ec93acb2f6f2a19378ec7d3317de65d37dc75b13b70ad2b51b512268468 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 26cb41d5a62f46a106b6371eb00fef02de3cdbfaa51338ba087a45f53028e0d0 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js aa330fa0e8ef35cbd92eb0d52e05fbaa07e61540c5cb164e693c82428ce1d763 F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 9097074724172e31e56ce20ccd7482259cf72a76124213cbc9469d757676da86 -F ext/wasm/api/sqlite3-wasm.c 2179f102b6c851070f4e780d71f8294e2aa53ba010f1e7cfd8fc101b81df8b0e +F ext/wasm/api/sqlite3-wasm.c dea6f331a03a49b8613d1fd5afcb606e1d06e8ab1df8dd65ee967fbfdd43ca1a F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bda1c75bd674a92a0e27cc2f3d46dbbf21e422413f8046814515a0bd7409328a F ext/wasm/api/sqlite3-worker1.c-pp.js 802d69ead8c38dc1be52c83afbfc77e757da8a91a2e159e7ed3ecda8b8dba2e7 F ext/wasm/c-pp-lite.c 943be1a36774d58385dca32de36fc18d4f432fe79f7aa35e6c85dd6a6b825bd0 @@ -717,7 +717,7 @@ F src/notify.c 57c2d1a2805d6dee32acd5d250d928ab94e02d76369ae057dee7d445fd64e878 F src/os.c 509452169d5ea739723e213b8e2481cf0e587f0e88579a912d200db5269f5f6d F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63 F src/os_common.h 6c0eb8dd40ef3e12fe585a13e709710267a258e2c8dd1c40b1948a1d14582e06 -F src/os_kv.c 2963455c752a20eb60d6d497a9bace5a6cc83af4ecd40ed0e33fdfb370864fce +F src/os_kv.c 26191ac1cb171ec3fceade6018ef66309cefba65e41f5ffe4c5385f2cb682ba7 F src/os_setup.h 8efc64eda6a6c2f221387eefc2e7e45fd5a3d5c8337a7a83519ba4fbd2957ae2 F src/os_unix.c 7945ede1e85b2d1b910e1b4af9ba342e964b1e30e79f4176480a60736445cb36 F src/os_win.c a89b501fc195085c7d6c9eec7f5bd782625e94bb2a96b000f4d009703df1083f @@ -2178,8 +2178,8 @@ F tool/version-info.c 33d0390ef484b3b1cb685d59362be891ea162123cea181cb8e6d2cf6dd F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 190ef4a94005b0feebe865a960d846a1b60ec1b267b15d5a24749f174a6169bc -R 27101f2463500f36d7a13f96ac32ad77 +P 206275292217be4ff317d4c9186ecaf863ca69295e2f995ed175aa65d9ad11dc +R 113fe1254ac4b59119cf9c28e45f222e U stephan -Z ec5d23038f38ba629b0ed0cf97f470b5 +Z bcdce04b0f6728fd8539031f7a059a7e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d092c0c636..585f31466a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -206275292217be4ff317d4c9186ecaf863ca69295e2f995ed175aa65d9ad11dc +90a33941c69b3581feaed271542f0238ca81ee34fe5b353ca7da48b81ac73a5f diff --git a/src/os_kv.c b/src/os_kv.c index 4f282d6a98..f4c1fc84b6 100644 --- a/src/os_kv.c +++ b/src/os_kv.c @@ -42,6 +42,11 @@ typedef struct KVVfsFile KVVfsFile; /* A single open file. There are only two files represented by this ** VFS - the database and the rollback journal. +** +** Maintenance reminder: if this struct changes in any way, the JSON +** rendering of its structure must be updated in +** sqlite3-wasm.c:sqlite3__wasm_enum_json(). There are no binary +** compatibility concerns, so it does not need an iVersion member. */ struct KVVfsFile { sqlite3_file base; /* IO methods */ @@ -90,11 +95,7 @@ static int kvvfsSleep(sqlite3_vfs*, int microseconds); static int kvvfsCurrentTime(sqlite3_vfs*, double*); static int kvvfsCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); -static -#ifndef SQLITE_WASM -const -#endif -sqlite3_vfs sqlite3OsKvvfsObject = { +static sqlite3_vfs sqlite3OsKvvfsObject = { 1, /* iVersion */ sizeof(KVVfsFile), /* szOsFile */ 1024, /* mxPathname */ @@ -118,11 +119,7 @@ sqlite3_vfs sqlite3OsKvvfsObject = { /* Methods for sqlite3_file objects referencing a database file */ -static -#ifndef SQLITE_WASM -const -#endif -sqlite3_io_methods kvvfs_db_io_methods = { +static sqlite3_io_methods kvvfs_db_io_methods = { 1, /* iVersion */ kvvfsClose, /* xClose */ kvvfsReadDb, /* xRead */ @@ -146,11 +143,7 @@ sqlite3_io_methods kvvfs_db_io_methods = { /* Methods for sqlite3_file objects referencing a rollback journal */ -static -#ifndef SQLITE_WASM -const -#endif -sqlite3_io_methods kvvfs_jrnl_io_methods = { +static sqlite3_io_methods kvvfs_jrnl_io_methods = { 1, /* iVersion */ kvvfsClose, /* xClose */ kvvfsReadJrnl, /* xRead */ @@ -193,7 +186,8 @@ static void kvrecordMakeKey( const char *zKeyIn, char *zKeyOut ){ - sqlite3_snprintf(KVRECORD_KEY_SZ, zKeyOut, "kvvfs-%s-%s", zClass, zKeyIn); + sqlite3_snprintf(KVRECORD_KEY_SZ, zKeyOut, "kvvfs-%s-%s", + zClass, zKeyIn ? zKeyIn : ""); } /* Write content into a key. zClass is the particular namespace of the @@ -299,14 +293,13 @@ static int kvrecordRead( ** Maintenance reminder: if this struct changes in any way, the JSON ** rendering of its structure must be updated in ** sqlite3-wasm.c:sqlite3__wasm_enum_json(). There are no binary -** compatibility concerns, so it does not need an iVersion -** member. +** compatibility concerns, so it does not need an iVersion member. */ typedef struct sqlite3_kvvfs_methods sqlite3_kvvfs_methods; struct sqlite3_kvvfs_methods { + int (*xRcrdRead)(const char*, const char *zKey, char *zBuf, int nBuf); int (*xRcrdWrite)(const char*, const char *zKey, const char *zData); int (*xRcrdDelete)(const char*, const char *zKey); - int (*xRcrdRead)(const char*, const char *zKey, char *zBuf, int nBuf); const int nKeySize; #ifndef SQLITE_WASM # define MAYBE_CONST const @@ -334,9 +327,9 @@ struct sqlite3_kvvfs_methods { const #endif sqlite3_kvvfs_methods sqlite3KvvfsMethods = { - .xRcrdRead = kvrecordRead, - .xRcrdWrite = kvrecordWrite, - .xRcrdDelete = kvrecordDelete, + .xRcrdRead = kvrecordRead, + .xRcrdWrite = kvrecordWrite, + .xRcrdDelete = kvrecordDelete, .nKeySize = KVRECORD_KEY_SZ, .pVfs = &sqlite3OsKvvfsObject, .pIoDb = &kvvfs_db_io_methods, @@ -499,7 +492,7 @@ static sqlite3_int64 kvvfsReadFileSize(KVVfsFile *pFile){ char zData[50]; zData[0] = 0; sqlite3KvvfsMethods.xRcrdRead(pFile->zClass, "sz", zData, - sizeof(zData)-1); + sizeof(zData)-1); return strtoll(zData, 0, 0); } static int kvvfsWriteFileSize(KVVfsFile *pFile, sqlite3_int64 sz){ @@ -586,7 +579,7 @@ static int kvvfsReadDb( } sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); got = sqlite3KvvfsMethods.xRcrdRead(pFile->zClass, zKey, - aData, SQLITE_KVOS_SZ-1); + aData, SQLITE_KVOS_SZ-1); if( got<0 ){ n = 0; }else{ @@ -844,33 +837,39 @@ static int kvvfsOpen( KVVfsFile *pFile = (KVVfsFile*)pProtoFile; if( zName==0 ) zName = ""; SQLITE_KV_LOG(("xOpen(\"%s\")\n", zName)); - if( strcmp(zName, "local")==0 - || strcmp(zName, "session")==0 - ){ - pFile->isJournal = 0; - pFile->base.pMethods = &kvvfs_db_io_methods; - }else - if( strcmp(zName, "local-journal")==0 - || strcmp(zName, "session-journal")==0 - ){ + assert(!pFile->zClass); + assert(!pFile->aData); + pFile->aData = 0; + pFile->aJrnl = 0; + pFile->nJrnl = 0; + pFile->szPage = -1; + pFile->szDb = -1; + if( 0==sqlite3_strglob("*-journal", zName) ){ pFile->isJournal = 1; pFile->base.pMethods = &kvvfs_jrnl_io_methods; + if( 0==strcmp("session-journal",zName) ){ + pFile->zClass = "session"; + }else if( 0==strcmp("local-journal",zName) ){ + pFile->zClass = "local"; + } }else{ - return SQLITE_CANTOPEN; + pFile->isJournal = 0; + pFile->base.pMethods = &kvvfs_db_io_methods; + if( 0==strcmp("session",zName) || 0==strcmp("local",zName) ){ + pFile->zClass = zName; + } } - if( zName[0]=='s' ){ - pFile->zClass = "session"; - }else{ - pFile->zClass = "local"; + if( !pFile->zClass ){ +#ifndef SQLITE_WASM + return SQLITE_CANTOPEN; +#else + pFile->zClass = zName; +#endif } pFile->aData = sqlite3_malloc64(SQLITE_KVOS_SZ); if( pFile->aData==0 ){ return SQLITE_NOMEM; } - pFile->aJrnl = 0; - pFile->nJrnl = 0; - pFile->szPage = -1; - pFile->szDb = -1; return SQLITE_OK; } @@ -880,13 +879,17 @@ static int kvvfsOpen( ** returning. */ static int kvvfsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + int rc /* The JS impl can fail with OOM in argument conversion */; if( strcmp(zPath, "local-journal")==0 ){ - sqlite3KvvfsMethods.xRcrdDelete("local", "jrnl"); + rc = sqlite3KvvfsMethods.xRcrdDelete("local", "jrnl"); }else if( strcmp(zPath, "session-journal")==0 ){ - sqlite3KvvfsMethods.xRcrdDelete("session", "jrnl"); + rc = sqlite3KvvfsMethods.xRcrdDelete("session", "jrnl"); } - return SQLITE_OK; + else{ + rc = 0; + } + return rc; } /*