From: stephan Date: Tue, 4 Oct 2022 17:06:51 +0000 (+0000) Subject: Replace time-based auto-unlock of opfs sync handles with lock acquisition/release... X-Git-Tag: version-3.40.0~169^2~11 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=9a55773b2f2b5e94afa59ced504e2078af2c565b;p=thirdparty%2Fsqlite.git Replace time-based auto-unlock of opfs sync handles with lock acquisition/release via sqlite3_io_methods::xLock/xUnlock(). FossilOrigin-Name: 2625b7cfe1640c1d7e779ec1f37db970541598c0dc3e22e5eecf3c772d95ad40 --- diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index 1a329c8e51..40e615eb86 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -169,6 +169,7 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){ W.onerror = function(err){ // The error object doesn't contain any useful info when the // failure is, e.g., that the remote script is 404. + error("Error initializing OPFS asyncer:",err); promiseReject(new Error("Loading OPFS async Worker failed for unknown reasons.")); }; const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/; @@ -202,7 +203,6 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){ environment or the other when sqlite3_os_end() is called (_if_ it gets called at all in a wasm build, which is undefined). */ - /** State which we send to the async-api Worker or share with it. This object must initially contain only cloneable or sharable @@ -224,7 +224,12 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){ */ const state = Object.create(null); state.verbose = options.verbose; - state.littleEndian = true; + state.littleEndian = (()=>{ + const buffer = new ArrayBuffer(2); + new DataView(buffer).setInt16(0, 256, true /* littleEndian */); + // Int16Array uses the platform's endianness. + return new Int16Array(buffer)[0] === 256; + })(); /** Whether the async counterpart should log exceptions to the serialization channel. That produces a great deal of noise for seemingly innocuous things like xAccess() checks @@ -265,7 +270,7 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){ between both workers. This worker writes to it and the other listens for changes. */ state.opIds.whichOp = i++; - /* Slot for storing return values. This work listens to that + /* Slot for storing return values. This worker listens to that slot and the other worker writes to it. */ state.opIds.rc = i++; /* Each function gets an ID which this worker writes to @@ -278,11 +283,13 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){ state.opIds.xDeleteNoWait = i++; state.opIds.xFileControl = i++; state.opIds.xFileSize = i++; + state.opIds.xLock = i++; state.opIds.xOpen = i++; state.opIds.xRead = i++; state.opIds.xSleep = i++; state.opIds.xSync = i++; state.opIds.xTruncate = i++; + state.opIds.xUnlock = i++; state.opIds.xWrite = i++; state.opIds.mkdir = i++; state.opIds['opfs-async-metrics'] = i++; @@ -290,7 +297,6 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){ state.sabOP = new SharedArrayBuffer(i * 4/*sizeof int32*/); opfsUtil.metrics.reset(); } - /** SQLITE_xxx constants to export to the async worker counterpart... @@ -304,10 +310,17 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){ 'SQLITE_IOERR_TRUNCATE', 'SQLITE_IOERR_DELETE', 'SQLITE_IOERR_ACCESS', 'SQLITE_IOERR_CLOSE', 'SQLITE_IOERR_DELETE', + 'SQLITE_LOCK_NONE', + 'SQLITE_LOCK_SHARED', + 'SQLITE_LOCK_RESERVED', + 'SQLITE_LOCK_PENDING', + 'SQLITE_LOCK_EXCLUSIVE', 'SQLITE_OPEN_CREATE', 'SQLITE_OPEN_DELETEONCLOSE', 'SQLITE_OPEN_READONLY' - ].forEach(function(k){ - state.sq3Codes[k] = capi[k] || toss("Maintenance required: not found:",k); + ].forEach((k)=>{ + if(undefined === (state.sq3Codes[k] = capi[k])){ + toss("Maintenance required: not found:",k); + } }); /** @@ -605,7 +618,8 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){ xCheckReservedLock: function(pFile,pOut){ // Exclusive lock is automatically acquired when opened //warn("xCheckReservedLock(",arguments,") is a no-op"); - wasm.setMemValue(pOut,1,'i32'); + const f = __openFiles[pFile]; + wasm.setMemValue(pOut, f.lockMode ? 1 : 0, 'i32'); return 0; }, xClose: function(pFile){ @@ -643,9 +657,17 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){ return rc; }, xLock: function(pFile,lockType){ - //2022-09: OPFS handles lock when opened - //warn("xLock(",arguments,") is a no-op"); - return 0; + mTimeStart('xLock'); + const f = __openFiles[pFile]; + let rc = 0; + if( capi.SQLITE_LOCK_NONE === f.lockType ) { + rc = opRun('xLock', pFile, lockType); + if( 0===rc ) f.lockType = lockType; + }else{ + f.lockType = lockType; + } + mTimeEnd(); + return rc; }, xRead: function(pFile,pDest,n,offset64){ /* int (*xRead)(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst) */ @@ -676,9 +698,16 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){ return rc; }, xUnlock: function(pFile,lockType){ - //2022-09: OPFS handles lock when opened - //warn("xUnlock(",arguments,") is a no-op"); - return 0; + mTimeStart('xUnlock'); + const f = __openFiles[pFile]; + let rc = 0; + if( capi.SQLITE_LOCK_NONE === lockType + && f.lockType ){ + rc = opRun('xUnlock', pFile, lockType); + } + if( 0===rc ) f.lockType = lockType; + mTimeEnd(); + return rc; }, xWrite: function(pFile,pSrc,n,offset64){ /* int (*xWrite)(sqlite3_file*, const void*, int iAmt, sqlite3_int64 iOfst) */ @@ -696,7 +725,7 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){ return rc; } }/*ioSyncWrappers*/; - + /** Impls for the sqlite3_vfs methods. Maintenance reminder: members are in alphabetical order to simplify finding them. @@ -790,6 +819,7 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){ fh.sabView = state.sabFileBufView; fh.sq3File = new sqlite3_file(pFile); fh.sq3File.$pMethods = opfsIoMethods.pointer; + fh.lockType = capi.SQLITE_LOCK_NONE; } mTimeEnd(); return rc; @@ -1061,5 +1091,12 @@ installOpfsVfs.defaultProxyUri = //self.location.pathname.replace(/[^/]*$/, "sqlite3-opfs-async-proxy.js"); "sqlite3-opfs-async-proxy.js"; //console.warn("sqlite3.installOpfsVfs.defaultProxyUri =",sqlite3.installOpfsVfs.defaultProxyUri); -self.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>installOpfsVfs()); +self.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ + try{ + return installOpfsVfs(); + }catch(e){ + console.error("installOpfsVfs() exception:",e); + throw e; + } +}); }/*sqlite3ApiBootstrap.initializers.push()*/); diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 516f53f52e..5b80c439bb 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -1312,14 +1312,25 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( // Is it okay to resolve these in parallel or do we need them // to resolve in order? We currently only have 1, so it // makes no difference. - lip = lip.map((f)=>f(sqlite3).catch(()=>{})); + lip = lip.map((f)=>f(sqlite3).catch((e)=>{ + console.error("An async sqlite3 initializer failed:",e); + })); //let p = lip.shift(); //while(lip.length) p = p.then(lip.shift()); //return p.then(()=>sqlite3); return Promise.all(lip).then(()=>sqlite3); } }; - sqlite3ApiBootstrap.initializers.forEach((f)=>f(sqlite3)); + try{ + sqlite3ApiBootstrap.initializers.forEach((f)=>{ + f(sqlite3); + }); + }catch(e){ + /* If we don't report this here, it can get completely swallowed + up and disappear into the abyss of Promises and Workers. */ + console.error("sqlite3 bootstrap initializer threw:",e); + throw e; + } delete sqlite3ApiBootstrap.initializers; sqlite3ApiBootstrap.sqlite3 = sqlite3; return sqlite3; diff --git a/ext/wasm/sqlite3-opfs-async-proxy.js b/ext/wasm/sqlite3-opfs-async-proxy.js index d9310d025f..fd4c7f2b80 100644 --- a/ext/wasm/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/sqlite3-opfs-async-proxy.js @@ -41,6 +41,9 @@ if(self.window === self){ }else if(!navigator.storage.getDirectory){ toss("This API requires navigator.storage.getDirectory."); } + +//warn("This file is very much experimental and under construction.",self.location.pathname); + /** Will hold state copied to this object from the syncronous side of this API. @@ -97,8 +100,6 @@ metrics.dump = ()=>{ console.log("Serialization metrics:",metrics.s11n); }; -//warn("This file is very much experimental and under construction.",self.location.pathname); - /** Map of sqlite3_file pointers (integers) to metadata related to a given OPFS file handles. The pointers are, in this side of the @@ -142,8 +143,7 @@ const getDirForFilename = async function f(absFilename, createDirs = false){ /** Returns the sync access handle associated with the given file handle object (which must be a valid handle object), lazily opening - it if needed. Timestamps the handle for use in relinquishing it - during idle time. + it if needed. In order to help alleviate cross-tab contention for a dabase, if an exception is thrown while acquiring the handle, this routine @@ -177,13 +177,12 @@ const getSyncHandle = async (fh)=>{ } log("Got sync handle for",fh.filenameAbs,'in',performance.now() - t,'ms'); } - fh.syncHandleTime = performance.now(); return fh.syncHandle; }; const closeSyncHandle = async (fh)=>{ if(fh.syncHandle){ - //warn("Closing sync handle for",fh.filenameAbs); + log("Closing sync handle for",fh.filenameAbs); const h = fh.syncHandle; delete fh.syncHandle; return h.close(); @@ -239,6 +238,7 @@ const wTimeEnd = ()=>( */ let flagAsyncShutdown = false; + /** Asynchronous wrappers for sqlite3_vfs and sqlite3_io_methods methods. Maintenance reminder: members are in alphabetical order @@ -373,6 +373,20 @@ const vfsAsyncImpls = { storeAndNotify('xFileSize', sz); mTimeEnd(); }, + xLock: async function(fid,lockType){ + mTimeStart('xLock'); + const fh = __openFiles[fid]; + let rc = 0; + if( !fh.syncHandle ){ + try { await getSyncHandle(fh) } + catch(e){ + state.s11n.storeException(1,e); + rc = state.sq3Codes.SQLITE_IOERR; + } + } + storeAndNotify('xLock',rc); + mTimeEnd(); + }, xOpen: async function(fid/*sqlite3_file pointer*/, filename, flags){ const opName = 'xOpen'; mTimeStart(opName); @@ -473,6 +487,23 @@ const vfsAsyncImpls = { storeAndNotify('xTruncate',rc); mTimeEnd(); }, + xUnlock: async function(fid,lockType){ + mTimeStart('xUnlock'); + let rc = 0; + const fh = __openFiles[fid]; + if( state.sq3Codes.SQLITE_LOCK_NONE===lockType + && fh.syncHandle ){ + try { await closeSyncHandle(fh) } + catch(e){ + state.s11n.storeException(1,e); + rc = state.sq3Codes.SQLITE_IOERR; + /* Maybe we want to not report this? "Destructors do not + throw." */ + } + } + storeAndNotify('xUnlock',rc); + mTimeEnd(); + }, xWrite: async function(fid,n,offset){ mTimeStart('xWrite'); let rc; @@ -495,7 +526,7 @@ const vfsAsyncImpls = { storeAndNotify('xWrite',rc); mTimeEnd(); } -}; +}/*vfsAsyncImpls*/; const initS11n = ()=>{ /** @@ -617,21 +648,7 @@ const waitLoop = async function f(){ We need to wake up periodically to give the thread a chance to do other things. */ - const waitTime = 500; - /** - relinquishTime defines the_approximate_ number of ms after which - a db sync access handle will be relinquished so that we do not - hold a persistent lock on it. When the following loop times out - while waiting, every (approximate) increment of this value it - will relinquish any db handles which have been idle for at least - this much time. - - Reaquisition of a sync handle seems to take an average of - 0.6-0.9ms on this dev machine but takes anywhere from 1-3ms every - once in a while (maybe 1 time in 5 or 10). Outliers as long as - 7ms have been witnessed, but they're rare. - */ - const relinquishTime = 500; + const waitTime = 1000; let lastOpTime = performance.now(); let now; while(!flagAsyncShutdown){ @@ -639,21 +656,6 @@ const waitLoop = async function f(){ if('timed-out'===Atomics.wait( state.sabOPView, state.opIds.whichOp, 0, waitTime )){ - if(relinquishTime && - (lastOpTime + relinquishTime <= (now = performance.now()))){ - for(const fh of Object.values(__openFiles)){ - if(fh.syncHandle && ( - now - relinquishTime >= fh.syncHandleTime - )){ - log("Relinquishing for timeout:",fh.filenameAbs); - await closeSyncHandle(fh) - /* Testing shows that we have to wait on this async - op to finish, else we might try to re-open it - before the close has run. The FS layer does not - retain the order those operations, apparently. */; - } - } - } continue; } lastOpTime = performance.now(); @@ -719,4 +721,4 @@ navigator.storage.getDirectory().then(function(d){ } }; wMsg('opfs-async-loaded'); -}).catch((e)=>error(e)); +}).catch((e)=>error("error initializing OPFS asyncer:",e)); diff --git a/ext/wasm/test-opfs-vfs.js b/ext/wasm/test-opfs-vfs.js index 2b22a7e96c..583871b7d1 100644 --- a/ext/wasm/test-opfs-vfs.js +++ b/ext/wasm/test-opfs-vfs.js @@ -34,10 +34,6 @@ const tryOpfsVfs = async function(sqlite3){ const wait = async (ms)=>{ return new Promise((resolve)=>setTimeout(resolve, ms)); }; - const waitForRelinquish = async ()=>{ - log("Waiting briefly to test sync handle relinquishing..."); - return wait(1500); - }; const urlArgs = new URL(self.location.href).searchParams; const dbFile = "my-persistent.db"; @@ -45,13 +41,11 @@ const tryOpfsVfs = async function(sqlite3){ const db = new opfs.OpfsDb(dbFile,'ct'); log("db file:",db.filename); - await waitForRelinquish(); try{ if(opfs.entryExists(dbFile)){ let n = db.selectValue("select count(*) from sqlite_schema"); log("Persistent data found. sqlite_schema entry count =",n); } - await waitForRelinquish(); db.transaction((db)=>{ db.exec({ sql:[ @@ -63,7 +57,6 @@ const tryOpfsVfs = async function(sqlite3){ (performance.now() |0) / 4] }); }); - await waitForRelinquish(); log("count(*) from t =",db.selectValue("select count(*) from t")); // Some sanity checks of the opfs utility functions... diff --git a/manifest b/manifest index 629bbcb37d..2bda31e933 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Tweaks\sto\sthe\sopfs\sasync\swait/relinquish\stimes. -D 2022-10-04T11:14:23.298 +C Replace\stime-based\sauto-unlock\sof\sopfs\ssync\shandles\swith\slock\sacquisition/release\svia\ssqlite3_io_methods::xLock/xUnlock(). +D 2022-10-04T17:06:51.812 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -487,8 +487,8 @@ F ext/wasm/api/pre-js.js 2db711eb637991b383fc6b5c0f3df65ec48a7201e5730e304beba8d F ext/wasm/api/sqlite3-api-cleanup.js 5d22d1d3818ecacb23bfa223d5970cd0617d8cdbb48c8bc4bbd463f05b021a99 F ext/wasm/api/sqlite3-api-glue.js a5a1bd620e2e2c26b6e843cf439548b35c5bb5ed21b24b89a412e0b0a8592c42 F ext/wasm/api/sqlite3-api-oo1.js ac1e08d36bdfb5aa0a2d75b7d4c892fd51819d34c932370c3282810672bcc086 -F ext/wasm/api/sqlite3-api-opfs.js 0db417af18fdcd39171366d9ac9e7bf57c5236aea4d2de26c759e65a22b7c446 -F ext/wasm/api/sqlite3-api-prologue.js baf88cf253a35153674b068b5125238993ea2a7804152d238b5e40140b506ef5 +F ext/wasm/api/sqlite3-api-opfs.js 78fd272d6ec4fe6da93910b978f693fff63beed8e0e5e7dab42a25dbdfefca4b +F ext/wasm/api/sqlite3-api-prologue.js 587e6aa373bd85e83bcc3fb82b3636b01aaf2caefa39679e02574e766560d711 F ext/wasm/api/sqlite3-api-worker1.js 7f4f46cb6b512a48572d7567233896e6a9c46570c44bdc3d13419730c7c221c8 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 0c8cb242b80b8de27b74f05ed781f427958e04c9339c4720f30c3a7e78ee6242 @@ -522,11 +522,11 @@ F ext/wasm/speedtest1.html e4cb5d722b494104fc1249e7c008ca018f820a784833c51004c95 F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5 -F ext/wasm/sqlite3-opfs-async-proxy.js 0861d0eff1ffb441b4ad98781a37825252ae2f08482cc5b29c27f34bcf952049 +F ext/wasm/sqlite3-opfs-async-proxy.js 40fc1f779e18e08b31404ef832381ffd36b9d11447007aaf96223198324e4467 F ext/wasm/sqlite3-worker1-promiser.js 307d7837420ca6a9d3780dfc81194f1c0715637e6d9540e935514086b96913d8 F ext/wasm/sqlite3-worker1.js 466e9bd39409ab03f3e00999887aaffc11e95b416e2689596e3d7f1516673fdf F ext/wasm/test-opfs-vfs.html eb69dda21eb414b8f5e3f7c1cc0f774103cc9c0f87b2d28a33419e778abfbab5 -F ext/wasm/test-opfs-vfs.js 0115d56f3f2a6475040dffe75a6851a353b1690708795944cf0fe1890bc7ff54 +F ext/wasm/test-opfs-vfs.js 56c3d725044c668fa7910451e96c1195d25ad95825f9ac79f747a7759d1973d0 F ext/wasm/testing-worker1-promiser.html 6eaec6e04a56cf24cf4fa8ef49d78ce8905dde1354235c9125dca6885f7ce893 F ext/wasm/testing-worker1-promiser.js bd788e33c1807e0a6dda9c9a9d784bd3350ca49c9dd8ae2cc8719b506b6e013e F ext/wasm/testing1.html 50575755e43232dbe4c2f97c9086b3118eb91ec2ee1fae931e6d7669fb17fcae @@ -2029,8 +2029,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9d488081fc82a9abe3e81c7897fe6b29b6b337f62d0c62cb9cec7517bd54b53f -R 616ce8132e7f98fd04bd3d25f88d4168 +P 35f33c23e5849de1c43c4499ee0a7fa11d26ae34949c1e820c3fa8e8873f9c2b +R 447ab40d8f9d9bacdb3bbf1fc5e8416f U stephan -Z da44b1ca0ef51fa028d53641db451066 +Z 1f6ebc0e7b8d0de3ff6621f210427c47 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index db383fdaae..b47bd0f8b4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -35f33c23e5849de1c43c4499ee0a7fa11d26ae34949c1e820c3fa8e8873f9c2b \ No newline at end of file +2625b7cfe1640c1d7e779ec1f37db970541598c0dc3e22e5eecf3c772d95ad40 \ No newline at end of file