From: stephan Date: Tue, 2 Dec 2025 15:47:32 +0000 (+0000) Subject: Add more logging to kvvfs to try to trace down why it cannot recover from a page... X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=22b1bc9997e358009c689e5d69cb69f89499ba51;p=thirdparty%2Fsqlite.git Add more logging to kvvfs to try to trace down why it cannot recover from a page size change. FossilOrigin-Name: 0bf0b8a98a2cc5128aa0e510ef2fe411a6859ce807d6159175f5eaf3bc14183b --- diff --git a/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js index bfefd3228c..499c1448d6 100644 --- a/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js @@ -642,6 +642,34 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const originalIoMethods = (kvvfsFile)=> originalMethods[kvvfsFile.$isJournal ? 'ioJrnl' : 'ioDb']; + /** + Public interface for kvvfs v2. The capi.sqlite3_js_kvvfs_...() + routines remain in place for v1. Some members of this class proxy + to those functions but use different default argument values in + some cases. + */ + const kvvfs = sqlite3.kvvfs = Object.create(null); + if( sqlite3.__isUnderTest ){ + /* For inspection via the dev tools console. */ + sqlite3.kvvfs.test = Object.assign(Object.create(null),{ + pFileHandles, + cache, + storageForZClass, + KVVfsStorage + }); + sqlite3.kvvfs.log = Object.assign(Object.create(null),{ + xOpen: false, + xClose: false, + xWrite: false, + xRead: false, + xSync: false, + xFileControl: false, + xRcrdRead: false, + xRcrdWrite: false, + xRcrdDelete: false, + }); + } + const pVfs = new capi.sqlite3_vfs(kvvfsMethods.$pVfs); const pIoDb = new capi.sqlite3_io_methods(kvvfsMethods.$pIoDb); const pIoJrnl = new capi.sqlite3_io_methods(kvvfsMethods.$pIoJrnl); @@ -680,13 +708,9 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ try{ const jzClass = wasm.cstrToJs(zClass); const store = storageForZClass(jzClass); - if( 0 ){ - debug("xRcrdRead", jzClass, wasm.cstrToJs(zKey), - zBuf, nBuf, store ); - } if( !store ) return -1; const jXKey = jsKeyForStorage(store, zClass, zKey); - //debug("xRcrdRead zXKey", jzClass, wasm.cstrToJs(zXKey), store ); + kvvfs?.log?.xRcrdRead && warn("xRcrdRead", jzClass, jXKey, nBuf, store ); const jV = store.storage.getItem(jXKey); if(null===jV) return -1; const nV = jV.length /* We are relying 100% on v being @@ -734,6 +758,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const store = storageForZClass(zClass); const jxKey = jsKeyForStorage(store, zClass, zKey); const jData = wasm.cstrToJs(zData); + kvvfs?.log?.xRcrdWrite && warn("xRcrdWrite",jxKey, store); store.storage.setItem(jxKey, jData); store.listeners && notifyListeners('write', store, jxKey, jData); return 0; @@ -747,6 +772,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ try { const store = storageForZClass(zClass); const jxKey = jsKeyForStorage(store, zClass, zKey); + kvvfs?.log?.xRcrdDelete && warn("xRcrdDelete",jxKey, store); store.storage.removeItem(jxKey); store.listeners && notifyListeners('delete', store, jxKey); return 0; @@ -774,7 +800,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ zName = zToFree; } const jzClass = wasm.cstrToJs(zName); - //debug("xOpen",jzClass); + kvvfs?.log?.xOpen && debug("xOpen",jzClass,"flags =",flags); validateStorageName(jzClass, true); if( (flags & (capi.SQLITE_OPEN_MAIN_DB | capi.SQLITE_OPEN_TEMP_DB @@ -805,6 +831,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ++s.refc; //no if( true===deleteAt0 ) s.deleteAtRefc0 = true; s.files.push(f); + wasm.poke32(pOutFlags, flags); }else{ wasm.poke32(pOutFlags, flags | capi.SQLITE_OPEN_CREATE); util.assert( !f.$isJournal, "Opening a journal before its db? "+jzClass ); @@ -917,7 +944,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ cache.popError(); try{ const h = pFileHandles.get(pFile); - //debug("xClose", pFile, h); + kvvfs?.log?.xClose && debug("xClose", pFile, h); if( h ){ pFileHandles.delete(pFile); const s = h.store;//storageForZClass(h.jzClass); @@ -943,8 +970,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ try{ const h = pFileHandles.get(pFile); util.assert(h, "Missing KVVfsFile handle"); - //debug("xFileControl",h,opId); - if( opId===capi.SQLITE_FCNTL_PRAGMA ){ + kvvfs?.log?.xFileControl && debug("xFileControl",h,opId); + if( 0 && opId===capi.SQLITE_FCNTL_PRAGMA ){ /* pArg== length-3 (char**) */ const zName = wasm.peekPtr(wasm.ptr.add(pArg, wasm.ptr.size)); //const argv = wasm.cArgvToJs(3, pArg); @@ -956,13 +983,15 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ vacuum breaks the db. With this, it continues working but does not actually change the page size. */ - //debug("xFileControl pragma", h, - // "NOT setting page size to", wasm.cstrToJs(zVal)); + warn("xFileControl pragma", h, + "NOT setting page size to", wasm.cstrToJs(zVal)); h.file.$szPage = -1; - return 0;//corrupts capi.SQLITE_NOTFOUND; + if( h.file.$aJournal ){ + warn("This file has a journal of", h.file.$nJrnl, "bytes"); + } + return 0/*corrupts, but not until much later? capi.SQLITE_NOTFOUND*/; }else if( 0 && h.file.$szPage>0 ){ - // This works but is superfluous - //debug("xFileControl", h, "getting page size",h.file.$szPage); + warn("xFileControl", h, "getting page size",h.file.$szPage); wasm.pokePtr(pArg, wasm.allocCString(""+h.file.$szPage)); // memory now owned by sqlite. return 0;//capi.SQLITE_NOTFOUND; @@ -984,20 +1013,45 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ cache.popError(); try{ const h = pFileHandles.get(pFile); - //debug("xSync",h); + kvvfs?.log?.xSync && debug("xSync", h); util.assert(h, "Missing KVVfsFile handle"); - const rc = originalIoMethods(h.file).xSync(pFile, flags); + const rc = originalMethods.ioDb.xSync(pFile, flags); if( 0==rc && h.store.listeners ) notifyListeners('sync', h.store, true); return rc; }catch(e){ error("xSync",e); return cache.setError(e); } - } + }, + +//#if not nope + xRead: function(pFile,pTgt,n,iOff64){ + cache.popError(); + try{ + const h = pFileHandles.get(pFile); + util.assert(h, "Missing KVVfsFile handle"); + kvvfs?.log?.xRead && debug("xRead", n, iOff64, h); + return originalIoMethods(h.file).xRead(pFile, pTgt, n, iOff64); + }catch(e){ + error("xRead",e); + return cache.setError(e); + } + }, + xWrite: function(pFile,pSrc,n,iOff64){ + cache.popError(); + try{ + const h = pFileHandles.get(pFile); + util.assert(h, "Missing KVVfsFile handle"); + kvvfs?.log?.xWrite && debug("xWrite", n, iOff64, h); + return originalIoMethods(h.file).xWrite(pFile, pSrc, n, iOff64); + }catch(e){ + error("xRead",e); + return cache.setError(e); + } + }, +//#endif nope //#if nope - xRead: function(pFile,pTgt,n,iOff64){}, - xWrite: function(pFile,pSrc,n,iOff64){}, xTruncate: function(pFile,i64){}, xFileSize: function(pFile,pi64Out){}, xLock: function(pFile,iLock){}, @@ -1635,33 +1689,16 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return false; }; - /** - Public interface for kvvfs v2. The capi.sqlite3_js_kvvfs_...() - routines remain in place for v1. Some members of this class proxy - to those functions but use different default argument values in - some cases. - */ - sqlite3.kvvfs = Object.assign(Object.create(null),{ - reserve: sqlite3_js_kvvfs_reserve, - import: sqlite3_js_kvvfs_import, - export: sqlite3_js_kvvfs_export, - unlink: sqlite3_js_kvvfs_unlink, - listen: sqlite3_js_kvvfs_listen, - unlisten: sqlite3_js_kvvfs_unlisten, - exists: (name)=>!!storageForZClass(name), - estimateSize: sqlite3_js_kvvfs_size, - clear: sqlite3_js_kvvfs_clear - }); + sqlite3.kvvfs.reserve = sqlite3_js_kvvfs_reserve; + sqlite3.kvvfs.import = sqlite3_js_kvvfs_import; + sqlite3.kvvfs.export = sqlite3_js_kvvfs_export; + sqlite3.kvvfs.unlink = sqlite3_js_kvvfs_unlink; + sqlite3.kvvfs.listen = sqlite3_js_kvvfs_listen; + sqlite3.kvvfs.unlisten = sqlite3_js_kvvfs_unlisten; + sqlite3.kvvfs.exists = (name)=>!!storageForZClass(name); + sqlite3.kvvfs.estimateSize = sqlite3_js_kvvfs_size; + sqlite3.kvvfs.clear = sqlite3_js_kvvfs_clear; - if( sqlite3.__isUnderTest ){ - /* For inspection via the dev tools console. */ - sqlite3.kvvfs.test = { - pFileHandles, - cache, - storageForZClass, - KVVfsStorage - }; - } if( globalThis.Storage ){ /** diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 3c2086bf41..028088518e 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -3080,7 +3080,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; name: 'concurrent transient kvvfs', //predicate: ()=>false, test: function(sqlite3){ - const filename = '💾👷'; + const filename = 'myStorage'; const kvvfs = sqlite3.kvvfs; const DB = sqlite3.oo1.DB; const JDb = sqlite3.oo1.JsStorageDb; @@ -3148,8 +3148,10 @@ globalThis.sqlite3InitModule = sqlite3InitModule; importDb(exp, true); db = new JDb({ - filename - //flags: 't' + filename, + flags: 'c' + /* BUG: without the 'c' flag, the db works until we try to + vacuum, at which point it fails with "read only db". */ }); duo = new JDb(filename); T.assert(expectRows === duo.selectValue(sqlCount)); @@ -3179,7 +3181,8 @@ globalThis.sqlite3InitModule = sqlite3InitModule; q2.finalize(); if( 1 ){ - const pageSize = 1024 * 16; + error("Begin vacuum/page size test..."); + const pageSize = 1024 * 8; if( 0 ){ debug("Export before vacuum", exportDb(expOpt)); debug("page size before vacuum", @@ -3188,11 +3191,15 @@ globalThis.sqlite3InitModule = sqlite3InitModule; )); } db.exec([ - "delete from kvvfs where a=1;", - "pragma page_size="+pageSize+";", - "vacuum;" + "BEGIN;", + "insert into kvvfs(a) values(randomblob(16000/*>pg size*/));", + "COMMIT;", + "delete from kvvfs where octet_length(a)>100;", + //"pragma page_size="+pageSize+";", + "vacuum;", + "select 1;" ]); - --expectRows; + expectRows; if( 0 ){ debug("page size after", db.selectArray( @@ -3220,6 +3227,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; if( 0 ){ debug("vacuumed export",exp); } + error("End vacuum/page size test."); }else{ expectRows = 6; } diff --git a/manifest b/manifest index a0a3c3db04..af64acc0c0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Begrudingly\sallow\ssqlite3_js_kvvfs_clear()\sto\swork\sfor\sopened\sstorage\sonly\sfor\sthe\slocal/session\sstorage\scases\sand\sonly\s(one\sof\sthe\sdemo\sapps\sreminds\sme)\sfor\sbackwards\scompatiblity.\sLikewise,\sthe\sJsStorageDb.clearStorage()\sinherited\smethod\sis\snow\sdeprecated\sbecause\sit\srequires\san\sopened\sdb\s(but\scontinues\sto\swork\sfor\sthose\stwo\sstores),\swhereas\sits\s'static'\smethod\sof\sthe\ssame\sname\sis\sunaffected.\sRename\skvvfs.size()\sto\skvvfs.estimateSize(). -D 2025-12-01T22:20:54.328 +C Add\smore\slogging\sto\skvvfs\sto\stry\sto\strace\sdown\swhy\sit\scannot\srecover\sfrom\sa\spage\ssize\schange. +D 2025-12-02T15:47:32.634 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -600,7 +600,7 @@ F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 9654b565b346dc609b75d15337f20acfa7af7d9d558da1afeb9b6d8eaa404966 F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d -F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 00168b874ee56360e7f6584fd860328c105dd868c2a165252d7e09141d3389dc +F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 3146bcb9830ed41027909305d54f41505724606409dc443edabb2f6f5ce65103 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js a2eea6442556867b589e04107796c6e1d04a472219529eeb45b7cd221d7d048b F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 88ce2078267a2d1af57525a32d896295f4a8db7664de0e17e82dc9ff006ed8d3 F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81 @@ -647,7 +647,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 1618670e466f424aa289859fe0ec8ded223e42e9e69b5c851f809baaaca1a00c F ext/wasm/tester1-worker.c-pp.html 0e432ec2c0d99cd470484337066e8d27e7aee4641d97115338f7d962bf7b081a F ext/wasm/tester1.c-pp.html 52d88fe2c6f21a046030a36410b4839b632f4424028197a45a3d5669ea724ddb -F ext/wasm/tester1.c-pp.js 94519a9d3dc190277b0404f20492d7a6622787eea817f0e556cbd1487c9b3c7d +F ext/wasm/tester1.c-pp.js 57d465d0313178524e1749fbd7f651383b210a245d3815eb9cd4614daa58095e F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e F ext/wasm/tests/opfs/concurrency/test.js d08889a5bb6e61937d0b8cbb78c9efbefbf65ad09f510589c779b7cc6a803a88 F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2180,8 +2180,8 @@ F tool/version-info.c 33d0390ef484b3b1cb685d59362be891ea162123cea181cb8e6d2cf6dd F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9cbd5be803aba28916440cfa70c54588e7891d2073caebfa2b4dfcdd434b5c49 -R 9056d5b5f899babad2cf312ebbd008e1 +P 09e9255828ed6a7ccbe18f701f4a88b859cfdbd1ddca7ac5dac09744542c25fa +R 26de08b1230e759818946db5817cbd59 U stephan -Z 8a4b0d044f27cf08a66abc91dd8344f9 +Z a2609685c3f2eaa0fbe4894a2be65512 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 496915546b..1521b54239 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -09e9255828ed6a7ccbe18f701f4a88b859cfdbd1ddca7ac5dac09744542c25fa +0bf0b8a98a2cc5128aa0e510ef2fe411a6859ce807d6159175f5eaf3bc14183b