From: stephan Date: Mon, 21 Nov 2022 05:18:24 +0000 (+0000) Subject: More tweaking of OPFS concurrency measures and the related test app. X-Git-Tag: version-3.41.0~396 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=b38ac0986e86d56115396c36355d4e751cc4f7f5;p=thirdparty%2Fsqlite.git More tweaking of OPFS concurrency measures and the related test app. FossilOrigin-Name: a8d4da1501d411085ec2fd48c4a056c8b1d97ef3c3203c5b403a854ac2864870 --- diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index 58cf8ca3c7..c208932e17 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -220,22 +220,19 @@ class GetSyncHandleError extends Error { } }; GetSyncHandleError.convertRc = (e,rc)=>{ - if(1){ - /* This approach returns SQLITE_LOCKED to the C API - when getSyncHandle() fails but makes the very - wild assumption that such a failure _is_ a locking - error. In practice that appears to be the most - common error, by far, but we cannot unambiguously + if(0){ + /* This approach makes the very wild assumption that such a + failure _is_ a locking error. In practice that appears to be + the most common error, by far, but we cannot unambiguously distinguish that from other errors. - This approach demonstrably reduces concurrency-related - errors but is highly questionable. + This approach is highly questionable. */ return (e instanceof GetSyncHandleError) - ? state.sq3Codes.SQLITE_LOCKED + ? state.sq3Codes.SQLITE_IOERR_LOCK : rc; }else{ - return ec; + return rc; } } /** @@ -253,7 +250,7 @@ const getSyncHandle = async (fh)=>{ if(!fh.syncHandle){ const t = performance.now(); log("Acquiring sync handle for",fh.filenameAbs); - const maxTries = 4, msBase = 300; + const maxTries = 6, msBase = 300; let i = 1, ms = msBase; for(; true; ms = msBase * ++i){ try { @@ -271,7 +268,7 @@ const getSyncHandle = async (fh)=>{ } warn("Error getting sync handle. Waiting",ms, "ms and trying again.",fh.filenameAbs,e); - //await closeAutoLocks(); + await closeAutoLocks(); Atomics.wait(state.sabOPView, state.opIds.retry, 0, ms); } } diff --git a/ext/wasm/tests/opfs/concurrency/index.html b/ext/wasm/tests/opfs/concurrency/index.html index 79a46692cc..a082dfe997 100644 --- a/ext/wasm/tests/opfs/concurrency/index.html +++ b/ext/wasm/tests/opfs/concurrency/index.html @@ -18,7 +18,21 @@

OPFS concurrency tester using multiple independent Workers. - This app is incomplete. + Disclaimer: concurrency in OPFS is currently a pain point + and timing/concurrency mitigation in this environment is + highly unpredictable! +

+

+ URL flags: pass a number of workers using + the workers=N URL flag and the worker work interval + as interval=N (milliseconds). Enable OPFS VFS + verbosity with verbose=1-3 (output goes to the + dev console). +

+

Achtung: if it does not start to do anything within a couple of + seconds, check the dev console: Chrome often fails with "cannot allocate + WasmMemory" at startup. Closing and re-opening the tab usually resolves + it.

diff --git a/ext/wasm/tests/opfs/concurrency/test.js b/ext/wasm/tests/opfs/concurrency/test.js index b80dad24c4..8b75ea4c78 100644 --- a/ext/wasm/tests/opfs/concurrency/test.js +++ b/ext/wasm/tests/opfs/concurrency/test.js @@ -57,6 +57,12 @@ options.workerCount = ( urlArgsHtml.has('workers') ? +urlArgsHtml.get('workers') : 3 ) || 3; + options.opfsVerbose = ( + urlArgsHtml.has('verbose') ? +urlArgsHtml.get('verbose') : 1 + ) || 1; + options.interval = ( + urlArgsHtml.has('interval') ? +urlArgsHtml.get('interval') : 750 + ) || 750; const workers = []; workers.post = (type,...args)=>{ for(const w of workers) w.postMessage({type, payload:args}); @@ -91,7 +97,8 @@ workers.uri = ( 'worker.js?' + 'sqlite3.dir='+options.sqlite3Dir - + '&opfs-verbose=2' + + '&interval='+options.interval + + '&opfs-verbose='+options.opfsVerbose ); for(let i = 0; i < options.workerCount; ++i){ stdout("Launching worker..."); diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index 9aaa2f4c78..85a5cf19b7 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -8,7 +8,6 @@ self.sqlite3InitModule().then(async function(sqlite3){ }; const stdout = (...args)=>wPost('stdout',...args); const stderr = (...args)=>wPost('stderr',...args); - const postErr = (...args)=>wPost('error',...args); if(!sqlite3.opfs){ stderr("OPFS support not detected. Aborting."); return; @@ -19,15 +18,33 @@ self.sqlite3InitModule().then(async function(sqlite3){ }; const dbName = 'concurrency-tester.db'; - if((new URL(self.location.href).searchParams).has('unlink-db')){ + const urlArgs = new URL(self.location.href).searchParams; + if(urlArgs.has('unlink-db')){ await sqlite3.opfs.unlink(dbName); stdout("Unlinked",dbName); } wPost('loaded'); - + let db; + const interval = Object.assign(Object.create(null),{ + delay: urlArgs.has('interval') ? (+urlArgs.get('interval') || 750) : 750, + handle: undefined, + count: 0 + }); + const finish = ()=>{ + if(db){ + if(!db.pointer) return; + db.close(); + } + if(interval.error){ + wPost('failed',"Ending work after interval #"+interval.count, + "due to error:",interval.error); + }else{ + wPost('finished',"Ending work after",interval.count,"intervals."); + } + }; const run = async function(){ - const db = new sqlite3.opfs.OpfsDb(dbName,'c'); - //sqlite3.capi.sqlite3_busy_timeout(db.pointer, 2000); + db = new sqlite3.opfs.OpfsDb(dbName,'c'); + sqlite3.capi.sqlite3_busy_timeout(db.pointer, 5000); db.transaction((db)=>{ db.exec([ "create table if not exists t1(w TEXT UNIQUE ON CONFLICT REPLACE,v);", @@ -36,11 +53,6 @@ self.sqlite3InitModule().then(async function(sqlite3){ }); const maxIterations = 10; - const interval = Object.assign(Object.create(null),{ - delay: 500, - handle: undefined, - count: 0 - }); stdout("Starting interval-based db updates with delay of",interval.delay,"ms."); const doWork = async ()=>{ const tm = new Date().getTime(); @@ -57,15 +69,6 @@ self.sqlite3InitModule().then(async function(sqlite3){ interval.error = e; } }; - const finish = ()=>{ - db.close(); - if(interval.error){ - wPost('failed',"Ending work after interval #"+interval.count, - "due to error:",interval.error); - }else{ - wPost('finished',"Ending work after",interval.count,"intervals."); - } - }; if(1){/*use setInterval()*/ interval.handle = setInterval(async ()=>{ await doWork(); @@ -89,7 +92,10 @@ self.sqlite3InitModule().then(async function(sqlite3){ self.onmessage = function({data}){ switch(data.type){ - case 'run': run().catch((e)=>postErr(e.message)); + case 'run': run().catch((e)=>{ + if(!interval.error) interval.error = e; + finish(); + }); break; default: stderr("Unhandled message type '"+data.type+"'."); diff --git a/manifest b/manifest index 75a05f7f6c..0d16ed6e6d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Resolve\smissing\sSQLITE_LOCKED\sresult\scode\swhich\striggered\sa\snew\s(since\slast\scheckin)\sexception\sin\sthe\sOPFS\sVFS.\sImprove\soutput\sof\sthe\sOPFS\scontention\stester\sapp. -D 2022-11-21T04:12:38.735 +C More\stweaking\sof\sOPFS\sconcurrency\smeasures\sand\sthe\srelated\stest\sapp. +D 2022-11-21T05:18:24.071 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -506,7 +506,7 @@ F ext/wasm/api/sqlite3-api-opfs.js 38d368e33f470f9ba196f1a2b0c9ce076c930c70df233 F ext/wasm/api/sqlite3-api-prologue.js 08e96d26d329e8c1e08813fe0b84ee93e0e78b087efdd6eb2809ae2672902437 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 -F ext/wasm/api/sqlite3-opfs-async-proxy.js 021af8b3d1754e308c09eebee5f8d235fb245bea1f9b1c1414141cc2ebd5649c +F ext/wasm/api/sqlite3-opfs-async-proxy.js 1ec10873f1d59d305f6f3b435c50a1b75d693d5fb739b226f3da46fcbb11261a F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8fc8f47680df0e9a6c0f2f03cb004148645ecc983aa216daba09cb21f7e092a2 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -552,9 +552,9 @@ F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d96 F ext/wasm/tester1-worker.html 5ef353348c37cf2e4fd0b23da562d3275523e036260b510734e9a3239ba8c987 F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 F ext/wasm/tester1.c-pp.js 0c129495d057c77788b59715152d51f9bf9002ebbcce759ef8b028272ce3519d -F ext/wasm/tests/opfs/concurrency/index.html c7cf329e5b206dd8226d94ab9fec02f5f350d8ed69a57c96d84e876afd3d3d1b -F ext/wasm/tests/opfs/concurrency/test.js 6f7d49d97c27906f8f5a39d6da176122350f375cd811e51182128bc2db74e464 -F ext/wasm/tests/opfs/concurrency/worker.js 571bcc525520fedfc71e25486265247633a4bf8aee554923aa7219399ed749fd +F ext/wasm/tests/opfs/concurrency/index.html bb9b0f6da86df34c67fa506db9c45b7c4cf0045a211611cc6b8d2b53fa983481 +F ext/wasm/tests/opfs/concurrency/test.js cea04456ffae7bbfd726ef0e859d2afc2a51ead66940fc4a2499884ac44d16dd +F ext/wasm/tests/opfs/concurrency/worker.js a92fa4da1431d3c6db3e1dd0b98794e09d1021e23a8c490335410cce283b11c1 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 F ext/wasm/wasmfs.make 8fea9b4f3cde06141de1fc4c586ab405bd32c3f401554f4ebb18c797401a678d F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 96f76e7616f8157a342b9e1c42f7b1feab200d182268871a2b25f67d4ee2564c -R 37b708b698da9468667dd15ac1e5f125 +P 2debbbca33bd4170a1dc4dbb5eb3e68523e51d289b06c551e5560ac4e32e433b +R cdfdba12cf28ae07442f6b59e52570f8 U stephan -Z 415bce67c5fc2cb5709db14ccdfe0252 +Z bce379ce157581c096a0eabd8577e81e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7de2c9eef7..8ac00ec32b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2debbbca33bd4170a1dc4dbb5eb3e68523e51d289b06c551e5560ac4e32e433b \ No newline at end of file +a8d4da1501d411085ec2fd48c4a056c8b1d97ef3c3203c5b403a854ac2864870 \ No newline at end of file