From: stephan Date: Wed, 26 Nov 2025 13:55:20 +0000 (+0000) Subject: Internal kvvfs docs and cleanups. Wasm speedtest1: remove the hard-coded --size=2... X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=a781caa9ad2da0452249f70b0a123e4622262cf1;p=thirdparty%2Fsqlite.git Internal kvvfs docs and cleanups. Wasm speedtest1: remove the hard-coded --size=2 limit for kvvfs and use a transient storage object instead of session storage (which is limited to --size=2). For small job sizes kvvfs is roughly 3x the speed of memdb and that gap grows as the --size increases, with kvvfs _really_ bogging down on the --size=50 jsonb tests. FossilOrigin-Name: 43173267849f7f4b77a9ee15ab586e9deb0e290071149857fe18421fb948a715 --- diff --git a/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js index bb4abc2790..46baef17a0 100644 --- a/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js @@ -194,26 +194,35 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /** Throws if storage name n is not valid for use as a storage name. - This is intended for the high-level APIs, not the low-level ones. + Much of this goes back to kvvfs having a fixed buffer size for + its keys, and the storage name needing to be encoded in the keys + for local/session storage. We disallow non-ASCII to avoid + problems with truncated multibyte characters at the end of the + key buffer. + + The second argument must only be true when called from xOpen() - + it makes names with a "-journal" suffix legal. */ const validateStorageName = function(n,mayBeJournal=false){ + if( kvvfsIsPersistentName(n) ) return; + const len = n.length; + if( !len ) toss3(capi.SQLITE_RANGE, "Empty name is not permitted."); let maxLen = cache.keySize - 1; if( cache.rxJournalSuffix.test(n) ){ if( !mayBeJournal ){ - toss3(capi.SQLITE_MISUSE, "Storage names may not have a '-journal' suffix."); + toss3(capi.SQLITE_MISUSE, + "Storage names may not have a '-journal' suffix."); } }else{ maxLen -= 8 /* "-journal" */; } - const len = n.length; - if( !len ) toss3(capi.SQLITE_RANGE,"Empty name is not permitted."); if( len > maxLen ){ toss3(capi.SQLITE_RANGE, "Storage name is too long. Limit =", maxLen); } let i; for( i = 0; i < len; ++i ){ const ch = n.codePointAt(i); - if( ch<45 || (ch & 0x80) ){ + if( ch<45 || ch >126 ){ toss3(capi.SQLITE_RANGE, "Illegal character ("+ch+"d) in storage name:",n); } @@ -323,6 +332,10 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } } + cache.builtinStorageNames = Object.keys(cache.storagePool); + + const isBuiltinName = (n)=>cache.builtinStorageNames.indexOf(n)>-1; + /* Add "-journal" twins for each cache.storagePool entry... */ for(const k of Object.keys(cache.storagePool)){ /* Journals in kvvfs are are stored as individual records within @@ -854,9 +867,15 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /* Native is SQLITE_KVOS_SZ is 133073 as of this writing */ ); for(const e of Object.entries(methodOverrides.recordHandler)){ // Overwrite kvvfsMethods's callbacks - recordHandler[e[0]] = e[1]; - kvvfsMethods[kvvfsMethods.memberKey(e[0])] = - wasm.installFunction(kvvfsMethods.memberSignature(e[0]), e[1]); + const k = e[0], f = e[1]; + recordHandler[k] = f; + if( 0 ){ + // bug: this should work + kvvfsMethods.installMethod(k, f); + }else{ + kvvfsMethods[kvvfsMethods.memberKey(k)] = + wasm.installFunction(kvvfsMethods.memberSignature(k), f); + } } for(const e of Object.entries(methodOverrides.vfs)){ // Overwrite some pVfs entries and stash the original impls @@ -1179,7 +1198,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ reference count by 1. This is a no-op if name ends in "-journal" or refers to a - built-in storage object ('local', 'session', or 'localThread'). + built-in storage object. It will not lower the refcount below the number of currently-opened db/journal files for the storage (so that it @@ -1196,7 +1215,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const store = storageForZClass(name); if( !store || kvvfsIsPersistentName(store.jzClass) - || 'localThread'===store.jzClass + || isBuiltinName(store.jzClass) || cache.rxJournalSuffix.test(name) ) return false; if( store.refc > store.files.length || 0===store.files.length ){ if( --store.refc<=0 ){ diff --git a/ext/wasm/speedtest1.html b/ext/wasm/speedtest1.html index cce6171850..73ea724e11 100644 --- a/ext/wasm/speedtest1.html +++ b/ext/wasm/speedtest1.html @@ -66,80 +66,83 @@ }; wasmfsDir._ = undefined; - const eOut = document.querySelector('#test-output'); - const log2 = function(cssClass,...args){ + const eOut = document.querySelector('#test-output'); + const log2 = function(cssClass,...args){ const ln = document.createElement('div'); if(cssClass) ln.classList.add(cssClass); ln.append(document.createTextNode(args.join(' '))); eOut.append(ln); //this.e.output.lastElementChild.scrollIntoViewIfNeeded(); - }; - const logList = []; - const dumpLogList = function(){ + }; + const logList = []; + const dumpLogList = function(){ logList.forEach((v)=>log2('',v)); logList.length = 0; - }; - /* we cannot update DOM while speedtest is running unless we run - speedtest in a worker thread. */; - const log = (...args)=>{ + }; + /* we cannot update DOM while speedtest is running unless we run + speedtest in a worker thread. */; + const log = (...args)=>{ console.log(...args); logList.push(args.join(' ')); - }; - const logErr = function(...args){ + }; + const logErr = function(...args){ console.error(...args); logList.push('ERROR: '+args.join(' ')); - }; + }; - const runTests = function(sqlite3){ + const runTests = function(sqlite3){ const capi = sqlite3.capi, wasm = sqlite3.wasm; //console.debug('sqlite3 =',sqlite3); const pDir = wasmfsDir(wasm); if(pDir){ - console.warn("Persistent storage:",pDir); + console.warn("Persistent storage:",pDir); } const scope = wasm.scopedAllocPush(); let dbFile = pDir+"/speedtest1.db"; const urlParams = new URL(self.location.href).searchParams; const argv = ["speedtest1"]; if(urlParams.has('flags')){ - argv.push(...(urlParams.get('flags').split(','))); + argv.push(...(urlParams.get('flags').split(','))); } let forceSize = 0; let vfs, pVfs = 0; if(urlParams.has('vfs')){ - vfs = urlParams.get('vfs'); - pVfs = capi.sqlite3_vfs_find(vfs); - if(!pVfs){ - log2('error',"Unknown VFS:",vfs); - return; - } - argv.push("--vfs", vfs); - log2('',"Using VFS:",vfs); - if('kvvfs' === vfs){ - forceSize = 2 /* >2 is too big as of mid-2025 */; - dbFile = 'session'; - log2('warning',"kvvfs VFS: forcing --size",forceSize, - "and filename '"+dbFile+"'."); - capi.sqlite3_js_kvvfs_clear(dbFile); - } + vfs = urlParams.get('vfs'); + pVfs = capi.sqlite3_vfs_find(vfs); + if(!pVfs){ + log2('error',"Unknown VFS:",vfs); + return; + } + argv.push("--vfs", vfs); + log2('',"Using VFS:",vfs); + if('kvvfs' === vfs){ + //forceSize = 10 + /* --size > 2 is too big for local/session storage + as of mid-2025, but kvvfs v2 (late 2025) + lets us use transient storage. */; + dbFile = 'transient'; + log2('warning',"kvvfs VFS: forcing --size",forceSize, + "and filename '"+dbFile+"'."); + capi.sqlite3_js_kvvfs_clear(dbFile); + } } if(forceSize){ - argv.push('--size',forceSize); + argv.push('--size',forceSize); }else{ - [ - 'size' - ].forEach(function(k){ - const v = urlParams.get(k); - if(v) argv.push('--'+k, urlParams[k]); - }); + [ + 'size' + ].forEach(function(k){ + const v = urlParams.get(k); + if(v) argv.push('--'+k, urlParams[k]); + }); } argv.push( - "--singlethread", - //"--nomutex", - //"--nosync", - //"--memdb", // note that memdb trumps the filename arg - "--nomemstat" + "--singlethread", + //"--nomutex", + //"--nosync", + //"--memdb", // note that memdb trumps the filename arg + "--nomemstat" ); argv.push("--big-transactions"/*important for tests 410 and 510!*/, dbFile); @@ -154,19 +157,19 @@ "Give it time..."); logList.length = 0; setTimeout(function(){ - wasm.xCall('wasm_main', argv.length, - wasm.scopedAllocMainArgv(argv)); - wasm.scopedAllocPop(scope); - if('kvvfs'===vfs){ - logList.unshift("KVVFS "+dbFile+" size = "+ - capi.sqlite3_js_kvvfs_size(dbFile)); - } - if(pDir) wasm.sqlite3_wasm_vfs_unlink(pVfs,dbFile); - logList.unshift("Done running native main(). Output:"); - dumpLogList(); - log2('',"WASM heap size:",sqlite3.wasm.heap8().byteLength,"bytes"); + wasm.xCall('wasm_main', argv.length, + wasm.scopedAllocMainArgv(argv)); + wasm.scopedAllocPop(scope); + if('kvvfs'===vfs){ + logList.unshift("KVVFS "+dbFile+" size = "+ + capi.sqlite3_js_kvvfs_size(dbFile)); + } + if(pDir) wasm.sqlite3_wasm_vfs_unlink(pVfs,dbFile); + logList.unshift("Done running native main(). Output:"); + dumpLogList(); + log2('',"WASM heap size:",sqlite3.wasm.heap8().byteLength,"bytes"); }, 50); - }/*runTests()*/; + }/*runTests()*/; self.sqlite3TestModule.print = log; self.sqlite3TestModule.printErr = logErr; diff --git a/manifest b/manifest index 697dd8965e..382973b7e7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\ssome\sdebug\soutput. -D 2025-11-25T19:06:05.531 +C Internal\skvvfs\sdocs\sand\scleanups.\sWasm\sspeedtest1:\sremove\sthe\shard-coded\s--size=2\slimit\sfor\skvvfs\sand\suse\sa\stransient\sstorage\sobject\sinstead\sof\ssession\sstorage\s(which\sis\slimited\sto\s--size=2).\sFor\ssmall\sjob\ssizes\skvvfs\sis\sroughly\s3x\sthe\sspeed\sof\smemdb\sand\sthat\sgap\sgrows\sas\sthe\s--size\sincreases,\swith\skvvfs\s_really_\sbogging\sdown\son\sthe\s--size=50\sjsonb\stests. +D 2025-11-26T13:55:20.070 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -600,7 +600,7 @@ F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 9654b565b346dc609b75d15337f20acfa7af7d9d558da1afeb9b6d8eaa404966 F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d -F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js c12563c1c5ce2ff89bf93b23561266115f9ebad4f85206249079fe448eedbc84 +F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js cfa7feabd5db5fd62faef81a2f5e80064e4ee768b33453d9d9e2b38fbf26d447 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js a2eea6442556867b589e04107796c6e1d04a472219529eeb45b7cd221d7d048b F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 88ce2078267a2d1af57525a32d896295f4a8db7664de0e17e82dc9ff006ed8d3 F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 9097074724172e31e56ce20ccd7482259cf72a76124213cbc9469d757676da86 @@ -639,7 +639,7 @@ F ext/wasm/speedtest1-wasmfs.html 0e9d335a9b5b5fafe6e1bc8dc0f0ca7e22e6eb916682a2 F ext/wasm/speedtest1-wasmfs.mjs 60dd5842f6d2a70a6d0bef12633a11491bde6984aff75a37c2040980d8cbf36a F ext/wasm/speedtest1-worker.html 068d4190f304fa1c34e6501a1b3a4c32fe8d8dac93c2d0f53d667a1cb386eedc F ext/wasm/speedtest1-worker.js 958a2d3c710bf8e82567277f656193a0248216db99a3c2c86966124b84309efb -F ext/wasm/speedtest1.html c90d63dfa795f0cb1ad188de587be9024b1ff73b4adc5fdf7efc0d781be94d03 +F ext/wasm/speedtest1.html 5a913c7355dbaa72178484258d0b62b66d0d8134bcb496da90423c2402f462d4 F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5 @@ -2178,8 +2178,8 @@ F tool/version-info.c 33d0390ef484b3b1cb685d59362be891ea162123cea181cb8e6d2cf6dd F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a5c61fa56770a6fffcb4c76db2256c71b2d5f0beee6fc2c4800c40bf7ef995c8 -R f60751816e53c5cb5fe13336e4dd5985 +P 9fc6b811d6b0e268be18b768b6a69381606557a1702064b29f0ff4ca7c505cac +R 7bb0e65fe8224ec5fb003bd562b40cb9 U stephan -Z 12c53cfb6a000b9c5315603ad9a9ed06 +Z c4c5f600a67a1abe0d9c03c1534e75d2 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1cf88ca693..77565dd33e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9fc6b811d6b0e268be18b768b6a69381606557a1702064b29f0ff4ca7c505cac +43173267849f7f4b77a9ee15ab586e9deb0e290071149857fe18421fb948a715