From: stephan Date: Tue, 20 Sep 2022 01:28:47 +0000 (+0000) Subject: OPFS VFS: moved i/o buffer from per-file to the VFS, and related refactoring, in... X-Git-Tag: version-3.40.0~169^2~87 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=c4b87be3e8eea84384d9dd9d2dce2405bcbeae2c;p=thirdparty%2Fsqlite.git OPFS VFS: moved i/o buffer from per-file to the VFS, and related refactoring, in prep for experimentation with a new inter-worker comms model. FossilOrigin-Name: d4d63e4580ad8d497310608175308c03c517e051d7865cb66aa0b10356612d7d --- diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index 3fd08d9aec..ffc7669a7f 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -188,17 +188,22 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) const state = Object.create(null); state.verbose = options.verbose; state.fileBufferSize = - 1024 * 64 + 8 /* size of aFileHandle.sab. 64k = max sqlite3 page - size. The additional bytes are space for - holding BigInt results, since we cannot store - those via the Atomics API (which only works on - an Int32Array). */; + 1024 * 64 /* size of aFileHandle.sab. 64k = max sqlite3 page + size. */; + state.sabOffsetS11n = state.fileBufferSize; + state.sabIO = new SharedArrayBuffer( + state.fileBufferSize + + 4096/* arg/result serialization */ + + 8 /* to be removed - porting crutch */ + ); state.fbInt64Offset = - state.fileBufferSize - 8 /*spot in fileHandle.sab to store an int64 result */; + state.sabIO.byteLength - 8 /*spot in fileHandle.sab to store an int64 result. + to be removed. Porting crutch. */; state.opIds = Object.create(null); const metrics = Object.create(null); { let i = 0; + state.opIds.nothing = i++; state.opIds.xAccess = i++; state.opIds.xClose = i++; state.opIds.xDelete = i++; @@ -211,7 +216,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) state.opIds.xTruncate = i++; state.opIds.xWrite = i++; state.opIds.mkdir = i++; - state.opSAB = new SharedArrayBuffer(i * 4/*sizeof int32*/); + state.sabOP = new SharedArrayBuffer(i * 4/*sizeof int32*/); state.opIds.xFileControl = state.opIds.xSync /* special case */; opfsUtil.metrics.reset(); } @@ -225,7 +230,9 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) 'SQLITE_IOERR_WRITE', 'SQLITE_IOERR_FSYNC', 'SQLITE_IOERR_TRUNCATE', 'SQLITE_IOERR_DELETE', 'SQLITE_IOERR_ACCESS', 'SQLITE_IOERR_CLOSE', - 'SQLITE_IOERR_DELETE' + 'SQLITE_IOERR_DELETE', + 'SQLITE_OPEN_CREATE', 'SQLITE_OPEN_DELETEONCLOSE', + 'SQLITE_OPEN_READONLY' ].forEach(function(k){ state.sq3Codes[k] = capi[k] || toss("Maintenance required: not found:",k); state.sq3Codes._reverse[capi[k]] = k; @@ -236,17 +243,17 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) /** Runs the given operation in the async worker counterpart, waits for its response, and returns the result which the async worker - writes to the given op's index in state.opSABView. The 2nd argument + writes to the given op's index in state.sabOPView. The 2nd argument must be a single object or primitive value, depending on the given operation's signature in the async API counterpart. */ const opRun = (op,args)=>{ const t = performance.now(); - Atomics.store(state.opSABView, state.opIds[op], -1); + Atomics.store(state.sabOPView, state.opIds[op], -1); wMsg(op, args); - Atomics.wait(state.opSABView, state.opIds[op], -1); + Atomics.wait(state.sabOPView, state.opIds[op], -1); metrics[op].wait += performance.now() - t; - return Atomics.load(state.opSABView, state.opIds[op]); + return Atomics.load(state.sabOPView, state.opIds[op]); }; /** @@ -574,10 +581,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) args.fid = pFile; args.filename = zName; args.sab = new SharedArrayBuffer(state.fileBufferSize); - args.fileType = f._.getFileType(args.filename, flags); - args.create = !!(flags & capi.SQLITE_OPEN_CREATE); - args.deleteOnClose = !!(flags & capi.SQLITE_OPEN_DELETEONCLOSE); - args.readOnly = !!(flags & capi.SQLITE_OPEN_READONLY); + args.flags = flags; const rc = opRun('xOpen', args); if(!rc){ /* Recall that sqlite3_vfs::xClose() will be called, even on @@ -586,8 +590,8 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) wasm.setMemValue(pOutFlags, capi.SQLITE_OPEN_READONLY, 'i32'); } __openFiles[pFile] = args; - args.sabView = new Uint8Array(args.sab); - args.sabViewFileSize = new DataView(args.sab, state.fbInt64Offset, 8); + args.sabView = new Uint8Array(state.sabIO, 0, state.fileBufferSize); + args.sabViewFileSize = new DataView(state.sabIO, state.fbInt64Offset, 8); args.sq3File = new sqlite3_file(pFile); args.sq3File.$pMethods = opfsIoMethods.pointer; args.ba = new Uint8Array(args.sab); @@ -615,7 +619,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) assume it's sane and use it, otherwise install a JS-based one. */ vfsSyncWrappers.xSleep = function(pVfs,ms){ - Atomics.wait(state.opSABView, state.opIds.xSleep, 0, ms); + Atomics.wait(state.sabOPView, state.opIds.xSleep, 0, ms); return 0; }; } @@ -702,8 +706,8 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) log("xAccess(",dbFile,") exists ?=",rc); rc = vfsSyncWrappers.xOpen(opfsVfs.pointer, zDbFile, fid, openFlags, pOut); - log("open rc =",rc,"state.opSABView[xOpen] =", - state.opSABView[state.opIds.xOpen]); + log("open rc =",rc,"state.sabOPView[xOpen] =", + state.sabOPView[state.opIds.xOpen]); if(isWorkerErrCode(rc)){ error("open failed with code",rc); return; @@ -733,7 +737,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) log("waking up from xSleep()"); } rc = ioSyncWrappers.xClose(fid); - log("xClose rc =",rc,"opSABView =",state.opSABView); + log("xClose rc =",rc,"sabOPView =",state.sabOPView); log("Deleting file:",dbFile); vfsSyncWrappers.xDelete(opfsVfs.pointer, zDbFile, 0x1234); vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); @@ -767,7 +771,8 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri) toss("BUG: sqlite3_vfs_find() failed for just-installed OPFS VFS"); } capi.sqlite3_vfs_register.addReference(opfsVfs, opfsIoMethods); - state.opSABView = new Int32Array(state.opSAB); + state.sabOPView = new Int32Array(state.sabOP); + state.sabFileBufView = new Uint8Array(state.sabFileBufView, 0, state.fileBufferSize); if(options.sanityChecks){ warn("Running sanity checks because of opfs-sanity-check URL arg..."); sanityCheck(); diff --git a/ext/wasm/sqlite3-opfs-async-proxy.js b/ext/wasm/sqlite3-opfs-async-proxy.js index fb472488bf..a1ed4362d3 100644 --- a/ext/wasm/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/sqlite3-opfs-async-proxy.js @@ -131,8 +131,8 @@ const getDirForPath = async function f(absFilename, createDirs = false){ */ const storeAndNotify = (opName, value)=>{ log(opName+"() is notify()ing w/ value:",value); - Atomics.store(state.opSABView, state.opIds[opName], value); - Atomics.notify(state.opSABView, state.opIds[opName]); + Atomics.store(state.sabOPView, state.opIds[opName], value); + Atomics.notify(state.sabOPView, state.opIds[opName]); }; /** @@ -214,6 +214,12 @@ const vfsAsyncImpls = { } mTimeEnd(); }, + xDelete: async function(...args){ + mTimeStart('xDelete'); + const rc = await vfsAsyncImpls.xDeleteNoWait(...args); + storeAndNotify('xDelete', rc); + mTimeEnd(); + }, xDeleteNoWait: async function({filename, syncDir, recursive = false}){ /* The syncDir flag is, for purposes of the VFS API's semantics, ignored here. However, if it has the value 0x1234 then: after @@ -248,12 +254,6 @@ const vfsAsyncImpls = { } return rc; }, - xDelete: async function(...args){ - mTimeStart('xDelete'); - const rc = await vfsAsyncImpls.xDeleteNoWait(...args); - storeAndNotify('xDelete', rc); - mTimeEnd(); - }, xFileSize: async function(fid){ mTimeStart('xFileSize'); log("xFileSize(",arguments,")"); @@ -261,6 +261,9 @@ const vfsAsyncImpls = { let sz; try{ sz = await fh.accessHandle.getSize(); + if(!fh.sabViewFileSize){ + fh.sabViewFileSize = new DataView(state.sabIO,state.fbInt64Offset,8); + } fh.sabViewFileSize.setBigInt64(0, BigInt(sz), true); sz = 0; }catch(e){ @@ -272,16 +275,15 @@ const vfsAsyncImpls = { }, xOpen: async function({ fid/*sqlite3_file pointer*/, - sab/*file-specific SharedArrayBuffer*/, filename, - fileType = undefined /*mainDb, mainJournal, etc.*/, - create = false, readOnly = false, deleteOnClose = false + flags }){ const opName = 'xOpen'; mTimeStart(opName); + log(opName+"(",arguments[0],")"); + const deleteOnClose = (state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags); + const create = (state.sq3Codes.SQLITE_OPEN_CREATE & flags); try{ - if(create) readOnly = false; - log(opName+"(",arguments[0],")"); let hDir, filenamePart; try { [hDir, filenamePart] = await getDirForPath(filename, !!create); @@ -290,20 +292,8 @@ const vfsAsyncImpls = { mTimeEnd(); return; } - const hFile = await hDir.getFileHandle(filenamePart, {create: !!create}); - log(opName,"filenamePart =",filenamePart, 'hDir =',hDir); - const fobj = __openFiles[fid] = Object.create(null); - fobj.filenameAbs = filename; - fobj.filenamePart = filenamePart; - fobj.dirHandle = hDir; - fobj.fileHandle = hFile; - fobj.fileType = fileType; - fobj.sab = sab; - fobj.sabView = new Uint8Array(sab,0,state.fbInt64Offset); - fobj.sabViewFileSize = new DataView(sab,state.fbInt64Offset,8); - fobj.create = !!create; - fobj.readOnly = !!readOnly; - fobj.deleteOnClose = !!deleteOnClose; + const hFile = await hDir.getFileHandle(filenamePart, {create}); + const fobj = Object.create(null); /** wa-sqlite, at this point, grabs a SyncAccessHandle and assigns it to the accessHandle prop of the file state @@ -311,6 +301,14 @@ const vfsAsyncImpls = { places that limitation on it. */ fobj.accessHandle = await hFile.createSyncAccessHandle(); + __openFiles[fid] = fobj; + fobj.filenameAbs = filename; + fobj.filenamePart = filenamePart; + fobj.dirHandle = hDir; + fobj.fileHandle = hFile; + fobj.sabView = state.sabFileBufView; + fobj.readOnly = create ? false : (state.sq3Codes.SQLITE_OPEN_READONLY & flags); + fobj.deleteOnClose = deleteOnClose; storeAndNotify(opName, 0); }catch(e){ error(opName,e); @@ -395,8 +393,10 @@ navigator.storage.getDirectory().then(function(d){ state.verbose = opt.verbose ?? 2; state.fileBufferSize = opt.fileBufferSize; state.fbInt64Offset = opt.fbInt64Offset; - state.opSAB = opt.opSAB; - state.opSABView = new Int32Array(state.opSAB); + state.sabOP = opt.sabOP; + state.sabOPView = new Int32Array(state.sabOP); + state.sabIO = opt.sabIO; + state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize); state.opIds = opt.opIds; state.sq3Codes = opt.sq3Codes; Object.keys(vfsAsyncImpls).forEach((k)=>{ diff --git a/manifest b/manifest index c2229acede..00518b64c8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\smetrics\sand\sbuffer-copy\soptimizations\sin\sthe\sOPFS\sproxy,\sbut\swith\slittle\seffect. -D 2022-09-19T18:22:29.467 +C OPFS\sVFS:\smoved\si/o\sbuffer\sfrom\sper-file\sto\sthe\sVFS,\sand\srelated\srefactoring,\sin\sprep\sfor\sexperimentation\swith\sa\snew\sinter-worker\scomms\smodel. +D 2022-09-20T01:28:47.162 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -484,7 +484,7 @@ F ext/wasm/api/post-js-header.js 0e853b78db83cb1c06b01663549e0e8b4f377f12f5a2d9a F ext/wasm/api/sqlite3-api-cleanup.js 8564a6077cdcaea9a9f428a019af8a05887f0131e6a2a1e72a7ff1145fadfe77 F ext/wasm/api/sqlite3-api-glue.js 366d580c8e5bf7fcf4c6dee6f646c31f5549bd417ea03a59a0acca00e8ecce30 F ext/wasm/api/sqlite3-api-oo1.js 2d13dddf0d2b4168a9249f124134d37924331e5b55e05dba18b6d661fbeefe48 -F ext/wasm/api/sqlite3-api-opfs.js df3d085a55be11a0b7bce3d42ea1b721ff8aaba93659d0ec48c3327dde384596 +F ext/wasm/api/sqlite3-api-opfs.js f8027fb4af1c24fcfad31889f8a5ccfa3b96d6e812d3495b13833bef57034046 F ext/wasm/api/sqlite3-api-prologue.js 0d2639387b94c30f492d4aea6e44fb7b16720808678464559458fd2ae3759655 F ext/wasm/api/sqlite3-api-worker1.js ee4cf149cbacb63d06b536674f822aa5088b7e022cdffc69f1f36cebe2f9fea0 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 @@ -519,7 +519,7 @@ F ext/wasm/speedtest1.html 512addeb3c27c94901178b7bcbde83a6f95c093f9ebe16a2959a0 F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5 -F ext/wasm/sqlite3-opfs-async-proxy.js f1a270e7a8adeb80c73e0b345e472e574e694d9eaec6588ce3671438b79eb351 +F ext/wasm/sqlite3-opfs-async-proxy.js 7ebc36915cd61bd4f067a4823307f4d4eb2678a173aaae470c534e8fe9cda650 F ext/wasm/sqlite3-worker1-promiser.js 4fd0465688a28a75f1d4ee4406540ba494f49844e3cad0670d0437a001943365 F ext/wasm/sqlite3-worker1.js 0c1e7626304543969c3846573e080c082bf43bcaa47e87d416458af84f340a9e F ext/wasm/test-opfs-vfs.html eb69dda21eb414b8f5e3f7c1cc0f774103cc9c0f87b2d28a33419e778abfbab5 @@ -2026,8 +2026,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fb7f287310d74a3e236125ae9c49b859f9263c29ae85161c1bcf9dd0778d8a51 -R 946ef52a6b1d5b2d6cbf3b063dacd006 +P d1f1fe6f1c60640f7770dfb9245c459a09b8d24ec2ddf664dff77c810bd51f96 +R e5a1ff53ebe902985ab8e87bba2c71fb U stephan -Z f42132e655dd81eba8154f54be0c97de +Z d05962442428c5dae722ab00dca47e48 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e76b3ad26d..8d4f713497 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d1f1fe6f1c60640f7770dfb9245c459a09b8d24ec2ddf664dff77c810bd51f96 \ No newline at end of file +d4d63e4580ad8d497310608175308c03c517e051d7865cb66aa0b10356612d7d \ No newline at end of file