From: stephan Date: Sat, 24 Sep 2022 10:12:19 +0000 (+0000) Subject: Reworked out the OPFS async proxy metrics are fetched so that they play more nicely... X-Git-Tag: version-3.40.0~169^2~72 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=56fae744d402211a773ef892fbed351c98da7be4;p=thirdparty%2Fsqlite.git Reworked out the OPFS async proxy metrics are fetched so that they play more nicely with the tight event-polling loop. FossilOrigin-Name: ef503ced5c2ca842be9aea9ef13719a378ed3020e884032db09afee1b8eba0a1 --- diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index abdf90d49b..f69e162c7e 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -142,7 +142,8 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) "metrics for",self.location.href,":",metrics, "\nTotal of",n,"op(s) for",t, "ms (incl. "+w+" ms of waiting on the async side)"); - console.log("Serialization metrics:",JSON.stringify(metrics.s11n,0,2)); + console.log("Serialization metrics:",metrics.s11n); + opRun('async-metrics'); }, reset: function(){ let k; @@ -160,7 +161,17 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) //].forEach((k)=>r(metrics[k] = Object.create(null))); } }/*metrics*/; - + const promiseReject = function(err){ + opfsVfs.dispose(); + return promiseReject_(err); + }; + const W = new Worker(options.proxyUri); + W._originalOnError = W.onerror /* will be restored later */; + W.onerror = function(err){ + // The error object doesn't contain any useful info when the + // failure is, e.g., that the remote script is 404. + promiseReject(new Error("Loading OPFS async Worker failed for unknown reasons.")); + }; const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/; const dVfs = pDVfs ? new sqlite3_vfs(pDVfs) @@ -193,18 +204,6 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) gets called at all in a wasm build, which is undefined). */ - const promiseReject = function(err){ - opfsVfs.dispose(); - return promiseReject_(err); - }; - const W = new Worker(options.proxyUri); - W._originalOnError = W.onerror /* will be restored later */; - W.onerror = function(err){ - // The error object doesn't contain any useful info when the - // failure is, e.g., that the remote script is 404. - promiseReject(new Error("Loading OPFS async Worker failed for unknown reasons.")); - }; - /** State which we send to the async-api Worker or share with it. This object must initially contain only cloneable or sharable @@ -227,11 +226,19 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) const state = Object.create(null); state.verbose = options.verbose; state.littleEndian = true; - /** If true, the async counterpart should log exceptions to + /** Whether the async counterpart should log exceptions to the serialization channel. That produces a great deal of noise for seemingly innocuous things like xAccess() checks - for missing files. */ - state.asyncS11nExceptions = false; + for missing files, so this option may have one of 3 values: + + 0 = no exception logging + + 1 = only log exceptions for "significant" ops like xOpen(), + xRead(), and xWrite(). + + 2 = log all exceptions. + */ + state.asyncS11nExceptions = 1; /* Size of file I/O buffer block. 64k = max sqlite3 page size. */ state.fileBufferSize = 1024 * 64; @@ -270,6 +277,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) state.opIds.xClose = i++; state.opIds.xDelete = i++; state.opIds.xDeleteNoWait = i++; + state.opIds.xFileControl = i++; state.opIds.xFileSize = i++; state.opIds.xOpen = i++; state.opIds.xRead = i++; @@ -278,7 +286,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) state.opIds.xTruncate = i++; state.opIds.xWrite = i++; state.opIds.mkdir = i++; - state.opIds.xFileControl = i++; + state.opIds['async-metrics'] = i++; state.sabOP = new SharedArrayBuffer(i * 4/*sizeof int32*/); opfsUtil.metrics.reset(); } @@ -857,6 +865,27 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) defaulting to 16. */ opfsUtil.randomFilename = randomFilename; + + /** + Re-registers the OPFS VFS. This is intended only for odd use + cases which have to call sqlite3_shutdown() as part of their + initialization process, which will unregister the VFS + registered by installOpfsVfs(). If passed a truthy value, the + OPFS VFS is registered as the default VFS, else it is not made + the default. Returns the result of the the + sqlite3_vfs_register() call. + + Design note: the problem of having to re-register things after + a shutdown/initialize pair is more general. How to best plug + that in to the library is unclear. In particular, we cannot + hook in to any C-side calls to sqlite3_initialize(), so we + cannot add an after-initialize callback mechanism. + */ + opfsUtil.reregisterVfs = (asDefault=false)=>{ + return capi.wasm.exports.sqlite3_vfs_register( + opfsVfs.pointer, asDefault ? 1 : 0 + ); + }; if(sqlite3.oo1){ opfsUtil.OpfsDb = function(...args){ @@ -940,7 +969,6 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) wasm.scopedAllocPop(scope); } }/*sanityCheck()*/; - W.onmessage = function({data}){ //log("Worker.onmessage:",data); diff --git a/ext/wasm/sqlite3-opfs-async-proxy.js b/ext/wasm/sqlite3-opfs-async-proxy.js index 6c0765c63d..fa5cd2d289 100644 --- a/ext/wasm/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/sqlite3-opfs-async-proxy.js @@ -84,11 +84,10 @@ metrics.dump = ()=>{ } console.log(self.location.href, "metrics for",self.location.href,":\n", - JSON.stringify(metrics,0,2) - /*dev console can't expand this object!*/, + metrics, "\nTotal of",n,"op(s) for",t,"ms", "approx",w,"ms spent waiting on OPFS APIs."); - console.log("Serialization metrics:",JSON.stringify(metrics.s11n,0,2)); + console.log("Serialization metrics:",metrics.s11n); }; warn("This file is very much experimental and under construction.", @@ -122,7 +121,7 @@ const getResolvedPath = function(filename,splitIt){ truthy then each directory element leading to the file is created along the way. Throws if any creation or resolution fails. */ -const getDirForPath = async function f(absFilename, createDirs = false){ +const getDirForFilename = async function f(absFilename, createDirs = false){ const path = getResolvedPath(absFilename, true); const filename = path.pop(); let dh = state.rootDir; @@ -183,14 +182,20 @@ const wTimeEnd = ()=>( to simplify finding them. */ const vfsAsyncImpls = { - mkdir: async function(dirname){ + 'async-metrics': async ()=>{ + mTimeStart('async-metrics'); + metrics.dump(); + storeAndNotify('async-metrics', 0); + mTimeEnd(); + }, + mkdir: async (dirname)=>{ mTimeStart('mkdir'); let rc = 0; wTimeStart('mkdir'); try { - await getDirForPath(dirname+"/filepart", true); + await getDirForFilename(dirname+"/filepart", true); }catch(e){ - state.s11n.storeException(e); + state.s11n.storeException(2,e); rc = state.sq3Codes.SQLITE_IOERR; }finally{ wTimeEnd(); @@ -198,7 +203,7 @@ const vfsAsyncImpls = { storeAndNotify('mkdir', rc); mTimeEnd(); }, - xAccess: async function(filename){ + xAccess: async (filename)=>{ mTimeStart('xAccess'); /* OPFS cannot support the full range of xAccess() queries sqlite3 calls for. We can essentially just tell if the file is @@ -214,10 +219,10 @@ const vfsAsyncImpls = { let rc = 0; wTimeStart('xAccess'); try{ - const [dh, fn] = await getDirForPath(filename); + const [dh, fn] = await getDirForFilename(filename); await dh.getFileHandle(fn); }catch(e){ - state.s11n.storeException(e); + state.s11n.storeException(2,e); rc = state.sq3Codes.SQLITE_IOERR; }finally{ wTimeEnd(); @@ -269,7 +274,7 @@ const vfsAsyncImpls = { wTimeStart('xDelete'); try { while(filename){ - const [hDir, filenamePart] = await getDirForPath(filename, false); + const [hDir, filenamePart] = await getDirForFilename(filename, false); if(!filenamePart) break; await hDir.removeEntry(filenamePart, {recursive}); if(0x1234 !== syncDir) break; @@ -278,7 +283,7 @@ const vfsAsyncImpls = { filename = filename.join('/'); } }catch(e){ - state.s11n.storeException(e); + state.s11n.storeException(2,e); rc = state.sq3Codes.SQLITE_IOERR_DELETE; } wTimeEnd(); @@ -294,7 +299,7 @@ const vfsAsyncImpls = { state.s11n.serialize(Number(sz)); sz = 0; }catch(e){ - state.s11n.storeException(e); + state.s11n.storeException(2,e); sz = state.sq3Codes.SQLITE_IOERR; } wTimeEnd(); @@ -310,7 +315,7 @@ const vfsAsyncImpls = { try{ let hDir, filenamePart; try { - [hDir, filenamePart] = await getDirForPath(filename, !!create); + [hDir, filenamePart] = await getDirForFilename(filename, !!create); }catch(e){ storeAndNotify(opName, state.sql3Codes.SQLITE_NOTFOUND); mTimeEnd(); @@ -339,7 +344,7 @@ const vfsAsyncImpls = { }catch(e){ wTimeEnd(); error(opName,e); - state.s11n.storeException(e); + state.s11n.storeException(1,e); storeAndNotify(opName, state.sq3Codes.SQLITE_IOERR); } mTimeEnd(); @@ -361,7 +366,7 @@ const vfsAsyncImpls = { } }catch(e){ error("xRead() failed",e,fh); - state.s11n.storeException(e); + state.s11n.storeException(1,e); rc = state.sq3Codes.SQLITE_IOERR_READ; } storeAndNotify('xRead',rc); @@ -376,7 +381,7 @@ const vfsAsyncImpls = { wTimeStart('xSync'); await fh.accessHandle.flush(); }catch(e){ - state.s11n.storeException(e); + state.s11n.storeException(2,e); }finally{ wTimeEnd(); } @@ -394,7 +399,7 @@ const vfsAsyncImpls = { await fh.accessHandle.truncate(size); }catch(e){ error("xTruncate():",e,fh); - state.s11n.storeException(e); + state.s11n.storeException(2,e); rc = state.sq3Codes.SQLITE_IOERR_TRUNCATE; } wTimeEnd(); @@ -414,7 +419,7 @@ const vfsAsyncImpls = { ) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE; }catch(e){ error("xWrite():",e,fh); - state.s11n.storeException(e); + state.s11n.storeException(1,e); rc = state.sq3Codes.SQLITE_IOERR_WRITE; }finally{ wTimeEnd(); @@ -519,7 +524,11 @@ const initS11n = ()=>{ }; state.s11n.storeException = state.asyncS11nExceptions - ? ((e)=>state.s11n.serialize(e.message)) + ? ((priority,e)=>{ + if(priority<=state.asyncS11nExceptions){ + state.s11n.serialize(e.message); + } + }) : ()=>{}; return state.s11n; @@ -533,10 +542,8 @@ const waitLoop = async function f(){ const o = Object.create(null); opHandlers[state.opIds[k]] = o; o.key = k; - o.f = vi;// || toss("No vfsAsyncImpls[",k,"]"); + o.f = vi || toss("No vfsAsyncImpls[",k,"]"); } - let metricsTimer = self.location.port>=1024 ? performance.now() : 0; - // ^^^ in dev environment, dump out these metrics one time after a delay. while(true){ try { if('timed-out'===Atomics.wait(state.sabOPView, state.opIds.whichOp, 0, 500)){ @@ -545,7 +552,7 @@ const waitLoop = async function f(){ const opId = Atomics.load(state.sabOPView, state.opIds.whichOp); Atomics.store(state.sabOPView, state.opIds.whichOp, 0); const hnd = opHandlers[opId] ?? toss("No waitLoop handler for whichOp #",opId); - const args = state.s11n.deserialize(); + const args = state.s11n.deserialize() || []; state.s11n.serialize()/* clear s11n to keep the caller from confusing this with an exception string written by the upcoming operation */; @@ -554,14 +561,6 @@ const waitLoop = async function f(){ else error("Missing callback for opId",opId); }catch(e){ error('in waitLoop():',e.message); - }finally{ - // We can't call metrics.dump() from the dev console because this - // thread is continually tied up in Atomics.wait(), so let's - // do, for dev purposes only, a dump one time after 60 seconds. - if(metricsTimer && (performance.now() > metricsTimer + 60000)){ - metrics.dump(); - metricsTimer = 0; - } } }; }; diff --git a/manifest b/manifest index a911a21d4f..5dc3d9e030 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Refactoring\stowards\sgetting\sfiddle\sto\ssupport\sOPFS\sas\sa\sfirst-class\scitizen.\sCertain\soperations,\se.g.\simport,\sexport,\sand\sunlink,\sare\snot\sOPFS-aware. -D 2022-09-24T07:36:45.332 +C Reworked\sout\sthe\sOPFS\sasync\sproxy\smetrics\sare\sfetched\sso\sthat\sthey\splay\smore\snicely\swith\sthe\stight\sevent-polling\sloop. +D 2022-09-24T10:12:19.409 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -484,7 +484,7 @@ F ext/wasm/api/post-js-header.js 2e5c886398013ba2af88028ecbced1e4b22dc96a86467f1 F ext/wasm/api/sqlite3-api-cleanup.js 8564a6077cdcaea9a9f428a019af8a05887f0131e6a2a1e72a7ff1145fadfe77 F ext/wasm/api/sqlite3-api-glue.js cfff894bdf98a6c579975d09dd45471b0e3399f08a6f9e44a22646e8403196ed F ext/wasm/api/sqlite3-api-oo1.js f974e79d9af8f26bf33928c5730b0988cc706d14f59a5fe36394739b92249841 -F ext/wasm/api/sqlite3-api-opfs.js d623ea3519cd81fe18e243adfdd07cd1fa4b07ff3b0fd0d2b269beb0e127acb3 +F ext/wasm/api/sqlite3-api-opfs.js 5585dc80aea9df54c3d5d3a6c62771bf741f21b23706330ba62571c57ec07abf F ext/wasm/api/sqlite3-api-prologue.js a50ba8618e81a10a4fecd70f8723a7295cfcc0babd6df1dd018e7c5db2904aac F ext/wasm/api/sqlite3-api-worker1.js 2eeb2a24e1a90322d84a9b88a99919b806623de62792436446099c0988f2030b F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 @@ -519,7 +519,7 @@ F ext/wasm/speedtest1.html 8ae6ece128151d01f90579de69cfa06f021acdb760735250ef745 F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5 -F ext/wasm/sqlite3-opfs-async-proxy.js 483b6326e268589ed3749a4d1a60b4ec2afa8ff7c218a09d39430e3bde89862e +F ext/wasm/sqlite3-opfs-async-proxy.js fe4b8268eea9acaec633ebd1dd3f85dae7c461c5c68985ab1075d9560b1db8e8 F ext/wasm/sqlite3-worker1-promiser.js 4fd0465688a28a75f1d4ee4406540ba494f49844e3cad0670d0437a001943365 F ext/wasm/sqlite3-worker1.js 0c1e7626304543969c3846573e080c082bf43bcaa47e87d416458af84f340a9e F ext/wasm/test-opfs-vfs.html eb69dda21eb414b8f5e3f7c1cc0f774103cc9c0f87b2d28a33419e778abfbab5 @@ -2026,8 +2026,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e1249369d5ec1c582c280b1f578b35d53637fdf1cbd97c16d5ed95b136b83e56 -R bfb613620806c41d29eb36b1d39427ef +P 1b923ed6438d7fef4508936e0c4bc026a368721698b1539961e3fb3140a185cb +R 2784cada1344182947069d02e01bbe66 U stephan -Z 135731a0096fbab4e7ac0c6ece9bf8ed +Z 8970d31318ed191d3ed86820eec133d3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ea285c7d79..7609818f5d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1b923ed6438d7fef4508936e0c4bc026a368721698b1539961e3fb3140a185cb \ No newline at end of file +ef503ced5c2ca842be9aea9ef13719a378ed3020e884032db09afee1b8eba0a1 \ No newline at end of file