From: stephan Date: Thu, 16 Jun 2022 23:55:35 +0000 (+0000) Subject: fiddle: initial proof of concept of using Emscripten's IndexedDB virtual filesystem... X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=97c37b7b0e4124396344252f4f3ed849eeb769d1;p=thirdparty%2Fsqlite.git fiddle: initial proof of concept of using Emscripten's IndexedDB virtual filesystem for a persistent db. There's still much work to do to integrate this into something useful for clients but the concept is now proven. FossilOrigin-Name: 65c152d3354f1623bf1a6b05ea584b0e4b670eea92052a2df3e1ce57bf3f8b67 --- diff --git a/Makefile.in b/Makefile.in index da75b76a59..3ebf8bc3d1 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1540,7 +1540,7 @@ $(fiddle_module_js): Makefile sqlite3.c shell.c \ emcc -o $@ $(emcc_flags) \ -sEXPORT_NAME=initFiddleModule \ -sEXPORTED_FUNCTIONS=@$(fiddle_dir_abs)/EXPORTED_FUNCTIONS.fiddle \ - sqlite3.c shell.c + sqlite3.c shell.c -lidbfs.js gzip < $@ > $@.gz gzip < $(fiddle_dir)/fiddle-module.wasm > $(fiddle_dir)/fiddle-module.wasm.gz $(sqlite3_wasm_js): Makefile sqlite3.c \ diff --git a/ext/fiddle/fiddle-worker.js b/ext/fiddle/fiddle-worker.js index ca562323ce..5f3a1c0d26 100644 --- a/ext/fiddle/fiddle-worker.js +++ b/ext/fiddle/fiddle-worker.js @@ -75,6 +75,10 @@ {type:'shellExec', data: 'select * from sqlite_master'} + If the data property is an array, it gets glued together with a + newline separating each entry and is provided to the shell as a + single input string. + - More TBD as the higher-level db layer develops. */ @@ -103,7 +107,10 @@ }; const stdout = function(){wMsg('stdout', Array.prototype.slice.call(arguments));}; - const stderr = function(){wMsg('stderr', Array.prototype.slice.call(arguments));}; + const stderr = function f(){ + if(0===f._disabled) wMsg('stderr', Array.prototype.slice.call(arguments)); + }; + stderr._disabled = 0; self.onerror = function(/*message, source, lineno, colno, error*/) { const err = arguments[4]; @@ -120,6 +127,8 @@ }; const Sqlite3Shell = { + module: undefined/*gets set to the wasm module object*/, + idbDir: undefined/*mount point for IDBFS, if available. Includes trailing slash. */, /** Returns the name of the currently-opened db. */ dbFilename: function f(){ if(!f._) f._ = fiddleModule.cwrap('fiddle_db_filename', "string", ['string']); @@ -129,6 +138,17 @@ Runs the given text through the shell as if it had been typed in by a user. Fires a working/start event before it starts and working/end event when it finishes. + + sql may===null, in which case the db connection will be set + up if it hasn't already been, but that case is otherwise a + no-op. + + Client code should call this once with a null SQL value to + trigger the initial setup. + + If the argument is an array, it is pasted together with + `\n` as a separator and submitted as a single input to the + shell. */ exec: function f(sql){ if(!f._) f._ = fiddleModule.cwrap('fiddle_exec', null, ['string']); @@ -136,6 +156,7 @@ stderr("shell module has exit()ed. Cannot run SQL."); return; } + if(Array.isArray(sql)) sql = sql.join('\n'); wMsg('working','start'); try { if(f._running){ @@ -146,6 +167,9 @@ } } finally { delete f._running; + if(this.module.FS.filesystems.IDBFS){ + this.module.FS.syncfs(false,function(){}); + } wMsg('working','end'); } }, @@ -270,12 +294,12 @@ When work is finished, a message with a text value of null is submitted. - After a message with text==null is posted, the module may later - post messages about fatal problems, e.g. an exit() being - triggered, so it is recommended that UI elements for posting - status messages not be outright removed from the DOM when - text==null, and that they instead be hidden until/unless - text!=null. + After a message with text===null is posted, the module may + later post messages about fatal problems, e.g. an exit() + being triggered, so it is recommended that UI elements for + posting status messages not be outright removed from the + DOM when text===null, and that they instead be hidden + until/unless text!==null. */ setStatus: function f(text){ if(!f.last) f.last = { step: 0, text: '' }; @@ -296,6 +320,61 @@ emcc ... -sMODULARIZE=1 -sEXPORT_NAME=initFiddleModule */ initFiddleModule(fiddleModule).then(function(thisModule){ - wMsg('fiddle-ready'); + Sqlite3Shell.module = thisModule; + const FS = thisModule.FS; + if(FS.filesystems.IDBFS){ + /* Set up IndexedDB backing store. See: + + https://emscripten.org/docs/api_reference/Filesystem-API.html#FS.syncfs + */ + Sqlite3Shell.idbDir = '/idbfs/'; + FS.mkdir(Sqlite3Shell.idbDir); + FS.mount(FS.filesystems.IDBFS, {}, Sqlite3Shell.idbDir); + const dbFile = Sqlite3Shell.idbDir+"fiddle.sqlite3"; + /* + When FS.syncfs() is called, we often (but not always) + see this warning on stderr during app startup: + + warning: 2 FS.syncfs operations in flight at once, + probably just doing extra work + + Squelching that requires replacing FS.syncfs() with a + proxy which disables fiddleModule.printErr for the + duration of the call. _Sigh_. To that end... + */ + const oldsyncfs = FS.syncfs; + FS.syncfs = function f(){ + const args = Array.prototype.slice.call(arguments,0); + let cbNdx = -1, i = 0; + for(; i < args.length; ++i){ + if(args[i] instanceof Function){ cbNdx = i; break; } + } + if(cbNdx>=0){ + const cb = args[cbNdx]; + args[cbNdx] = function(){ + --stderr._disabled; + cb.apply(this,Array.prototype.slice.call(arguments,0)); + } + }else{ + args.push(function(){--stderr._disabled;}); + } + ++stderr._disabled; + return oldsyncfs.apply(this,args); + }; + /* Tell the shell to use a persistent db instead of the + default transient one... */ + FS.syncfs(true,function(err){ + if(!err){ + Sqlite3Shell.exec([ + ".print 'Opening IndexedDB-backed db:' "+dbFile, + ".open '"+dbFile+"'", + ".databases" + ]); + } + wMsg('fiddle-ready'); + }); + }else{ + wMsg('fiddle-ready'); + } }); })(); diff --git a/ext/fiddle/fiddle.js b/ext/fiddle/fiddle.js index 619ce4eca8..30dedb5dc9 100644 --- a/ext/fiddle/fiddle.js +++ b/ext/fiddle/fiddle.js @@ -240,11 +240,6 @@ } if(arguments.length > 1) text = Array.prototype.slice.call(arguments).join(' '); else if(1===arguments.length && Array.isArray(text)) text = text.join(' '); - // These replacements are necessary if you render to raw HTML - //text = text.replace(/&/g, "&"); - //text = text.replace(//g, ">"); - //text = text.replace('\n', '
', 'g'); if(null===text){/*special case: clear output*/ f._.value = ''; return; @@ -300,7 +295,7 @@ */ resetDb: function(){ if(window.confirm("Really destroy all content and tables " - +"in the (transient) db?")){ + +"in the db?")){ this.wMsg('db-reset'); } return this; @@ -775,7 +770,6 @@ SELECT group_concat(rtrim(t),x'0a') as Mandelbrot FROM a;`} }); })()/* example queries */; - SF.echo(null/*clear any output generated by the init process*/); if(window.jQuery && window.jQuery.terminal){ /* Set up the terminal-style view... */ const eTerm = window.jQuery('#view-terminal').empty(); @@ -796,14 +790,12 @@ SELECT group_concat(rtrim(t),x'0a') as Mandelbrot FROM a;`} }, false); btnToggleView.click()/*default to terminal view*/; } - SF.dbExec(null/*init the db and output the header*/); - SF.echo('This experimental app is provided in the hope that it', + SF.dbExec(null/*init the db and output the header (if not done already)*/); + SF.echo('\nThis experimental app is provided in the hope that it', 'may prove interesting or useful but is not an officially', 'supported deliverable of the sqlite project. It is subject to', 'any number of changes or outright removal at any time.\n'); delete ForceResizeKludge.$disabled; ForceResizeKludge(); - - btnShellExec.click(); }/*onSFLoaded()*/; })(); diff --git a/manifest b/manifest index c2eeb441fd..913fc5d508 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Minor\sfix\sto\sthe\squery\sinvariant\stesting\slogic\sof\sfuzzcheck. -D 2022-06-16T20:29:36.245 +C fiddle:\sinitial\sproof\sof\sconcept\sof\susing\sEmscripten's\sIndexedDB\svirtual\sfilesystem\sfor\sa\spersistent\sdb.\sThere's\sstill\smuch\swork\sto\sdo\sto\sintegrate\sthis\sinto\ssomething\suseful\sfor\sclients\sbut\sthe\sconcept\sis\snow\sproven. +D 2022-06-16T23:55:35.640 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in bccb0ed3f05fc41aee15da77c844c48b5da419cbb9af35b8a147536c9ad1c822 +F Makefile.in fdde76a9109885ed1df783fd3b69ecf4342a9389119de8989d39f34af7a9d09e F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241 F Makefile.msc de7cb3e095ce2fdc33513ccd76ebdaeda1483d0ddab0410fe65cbdeadd4c0ee1 F README.md 8b8df9ca852aeac4864eb1e400002633ee6db84065bd01b78c33817f97d31f5e @@ -61,9 +61,9 @@ F ext/fiddle/EXPORTED_RUNTIME_METHODS b831017ba67ba993b34a27400cef2f6095bd6789c0 F ext/fiddle/Makefile e25d34a0e1324f771d64c09c592601b97219282011587e6ce410fa8acdedb913 F ext/fiddle/SqliteTestUtil.js 559731c3e8e0de330ec7d292e6c1846566408caee6637acc8a119ac338a8781c F ext/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f -F ext/fiddle/fiddle-worker.js 88bc2193a6cb6a3f04d8911bed50a4401fe6f277de7a71ba833865ab64a1b4ae +F ext/fiddle/fiddle-worker.js fa60a65f30247a14c671132f311d04b0bd51e7dbcaa5fa89c242f15f36af28ca F ext/fiddle/fiddle.html 550c5aafce40bd218de9bf26192749f69f9b10bc379423ecd2e162bcef885c08 -F ext/fiddle/fiddle.js 812f9954cc7c4b191884ad171f36fcf2d0112d0a7ecfdf6087896833a0c079a8 +F ext/fiddle/fiddle.js 3e3192f23dd4d29dc36743c0280a729b496851230807c06e9fdc8656c5f7d96f F ext/fiddle/index.md d9c1c308d8074341bc3b11d1d39073cd77754cb3ca9aeb949f23fdd8323d81cf F ext/fiddle/sqlite3-api.js ccf4bd0c1c5bbb3be3469573423d6c53991941bec497eac63e9f17ea13bf8952 F ext/fiddle/sqlite3-worker.js a9c2b614beca187dbdd8c053ec2770cc61ec1ac9c0ec6398ceb49a79f705a421 @@ -1977,8 +1977,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3a461f61b47e6ba6d5dcc2b7470ebde512b57bc68086f65050e07b06f42b7351 -R d0f84b140700e490433349a70f8e673c -U drh -Z 3d24c9b621cf5763642f52e3d8949e70 +P 447e62a0946f5d77b7358adcabaeb23a7012cdfbfa1ef6082734cd9b45b2699d +R 389664f8ef5d055d1478dcc341b64542 +T *branch * fiddle-indexeddb +T *sym-fiddle-indexeddb * +T -sym-trunk * Cancelled\sby\sbranch. +U stephan +Z c98265dfc1cf4e337cd12b40eb80c496 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5a0c5b8a70..3a96fea92a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -447e62a0946f5d77b7358adcabaeb23a7012cdfbfa1ef6082734cd9b45b2699d \ No newline at end of file +65c152d3354f1623bf1a6b05ea584b0e4b670eea92052a2df3e1ce57bf3f8b67 \ No newline at end of file