From: stephan Date: Mon, 22 Apr 2024 16:46:37 +0000 (+0000) Subject: Extend the JS/WASM SEE build support by (A) filtering SEE-related bits out of the... X-Git-Tag: version-3.46.0~58 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=0a42e9913b233d433bd526c4c76825d0e1785bb6;p=thirdparty%2Fsqlite.git Extend the JS/WASM SEE build support by (A) filtering SEE-related bits out of the JS when not building with SEE and (B) accepting an optional key/textkey/hexkey option to the sqlite3.oo1.DB and subclass constructors to create/open SEE-encrypted databases with. Demonstrate SEE in the test app using the kvvfs. This obviates the changes made in [5c505ee8a7]. FossilOrigin-Name: 8fbda563d2f56f8dd3f695a5711e4356de79035f332270db45d4b33ed52fdfd2 --- diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 6e7b49875f..1a119f9505 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -335,19 +335,26 @@ DISTCLEAN_FILES += $(bin.stripccomments) # # Note that the SQLITE_... build flags used here have NO EFFECT on the # JS/WASM build. They are solely for use with $(bin.c-pp) itself. +# +# -D... flags which should be included in all invocations should be +# appended to $(C-PP.FILTER.global). bin.c-pp := ./c-pp $(bin.c-pp): c-pp.c $(sqlite3.c) $(MAKEFILE) $(CC) -O0 -o $@ c-pp.c $(sqlite3.c) '-DCMPP_DEFAULT_DELIM="//#"' -I$(dir.top) \ -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_UTF16 \ -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_OMIT_WAL -DSQLITE_THREADSAFE=0 \ -DSQLITE_TEMP_STORE=3 +C-PP.FILTER.global ?= +ifeq (1,$(SQLITE_C_IS_SEE)) + C-PP.FILTER.global += -Denable-see +endif define C-PP.FILTER # Create $2 from $1 using $(bin.c-pp) # $1 = Input file: c-pp -f $(1).js # $2 = Output file: c-pp -o $(2).js # $3 = optional c-pp -D... flags $(2): $(1) $$(MAKEFILE) $$(bin.c-pp) - $$(bin.c-pp) -f $(1) -o $$@ $(3) + $$(bin.c-pp) -f $(1) -o $$@ $(3) $(C-PP.FILTER.global) CLEAN_FILES += $(2) endef # /end C-PP.FILTER diff --git a/ext/wasm/api/sqlite3-api-glue.js b/ext/wasm/api/sqlite3-api-glue.js index b05d7a765a..cec0a8c0af 100644 --- a/ext/wasm/api/sqlite3-api-glue.js +++ b/ext/wasm/api/sqlite3-api-glue.js @@ -329,7 +329,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ wasm.bindingSignatures.push(["sqlite3_normalized_sql", "string", "sqlite3_stmt*"]); } - if(wasm.exports.sqlite3_activate_see instanceof Function){ +//#if enable-see + if(wasm.exports.sqlite3_key_v2 instanceof Function){ /** This code is capable of using an SEE build but note that an SEE WASM build is generally incompatible with SEE's license @@ -346,6 +347,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ["sqlite3_activate_see", undefined, "string"] ); } +//#endif enable-see + /** Functions which require BigInt (int64) support are separated from the others because we need to conditionally bind them or apply @@ -627,7 +630,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ["sqlite3__wasm_vfs_create_file", "int", "sqlite3_vfs*","string","*", "int"], ["sqlite3__wasm_posix_create_file", "int", "string","*", "int"], - ["sqlite3__wasm_vfs_unlink", "int", "sqlite3_vfs*","string"] + ["sqlite3__wasm_vfs_unlink", "int", "sqlite3_vfs*","string"], + ["sqlite3__wasm_qfmt_token","string:dealloc", "string","int"] ]; /** diff --git a/ext/wasm/api/sqlite3-api-oo1.js b/ext/wasm/api/sqlite3-api-oo1.js index 06d1df43f9..a5dfcec958 100644 --- a/ext/wasm/api/sqlite3-api-oo1.js +++ b/ext/wasm/api/sqlite3-api-oo1.js @@ -87,6 +87,94 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ */ const __vfsPostOpenSql = Object.create(null); + /** + Converts ArrayBuffer or Uint8Array ba into a string of hex + digits. + */ + const byteArrayToHex = function(ba){ + if( ba instanceof ArrayBuffer ){ + ba = new Uint8Array(ba); + } + const li = []; + const digits = "0123456789abcdef"; + for( const d of ba ){ + li.push( digits[(d & 0xf0) >> 4], digits[d & 0x0f] ); + } + return li.join(''); + }; + +//#if enable-see + /** + Internal helper to apply an SEE key to a just-opened + database. Requires that db be-a DB object which has just been + opened, opt be the options object processed by its ctor, and opt + must have either the key, hexkey, or textkey properties, either + as a string, an ArrayBuffer, or a Uint8Array. + + This is a no-op in non-SEE builds. It throws on error and returns + without side effects if its key/textkey options are not of valid + types. + + Returns true if it applies the key, else a falsy value. + */ + const dbCtorApplySEEKey = function(db,opt){ + if( !capi.sqlite3_key_v2 ) return; + let keytype; + let key; + const check = (opt.key ? 1 : 0) + (opt.hexkey ? 1 : 0) + (opt.textkey ? 1 : 0); + if( !check ) return; + else if( check>1 ) toss3("Only ONE of (key, hexkey, textkey) may be provided."); + if( opt.key ){ + /* It is not legal to bind an argument to PRAGMA key=?, so we + convert it to a hexkey... */ + keytype = 'key'; + key = opt.key; + if('string'===typeof key){ + key = new TextEncoder('utf-8').encode(key); + } + if((key instanceof ArrayBuffer) || (key instanceof Uint8Array)){ + key = byteArrayToHex(key); + keytype = 'hexkey'; + }else{ + toss3("Invalid value for the 'key' option. Expecting a string, ArrayBuffer, or Uint8Array."); + return; + } + }else if( opt.textkey ){ + /* For textkey we need it to be in string form, so convert it to + a string if it's a byte array... */ + keytype = 'textkey'; + key = opt.textkey; + if(key instanceof ArrayBuffer){ + key = new Uint8Array(key); + } + if(key instanceof Uint8Array){ + key = new TextDecoder('utf-8').decode(key); + }else if('string'!==typeof key){ + toss3("Invalid value for the 'textkey' option. Expecting a string, ArrayBuffer, or Uint8Array."); + } + }else if( opt.hexkey ){ + keytype = 'hexkey'; + key = opt.hexkey; + if((key instanceof ArrayBuffer) || (key instanceof Uint8Array)){ + key = byteArrayToHex(key); + }else if('string'!==typeof key){ + toss3("Invalid value for the 'hexkey' option. Expecting a string, ArrayBuffer, or Uint8Array."); + } + /* else assume it's valid hex codes */; + }else{ + return; + } + let stmt; + try{ + stmt = db.prepare("PRAGMA "+keytype+"="+util.sqlite3__wasm_qfmt_token(key, 1)); + stmt.step(); + }finally{ + if(stmt) stmt.finalize(); + } + return true; + }; +//#endif enable-see + /** A proxy for DB class constructors. It must be called with the being-construct DB object as its "this". See the DB constructor @@ -175,28 +263,22 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ __ptrMap.set(this, pDb); __stmtMap.set(this, Object.create(null)); try{ +//#if enable-see + dbCtorApplySEEKey(this,opt); +//#endif // Check for per-VFS post-open SQL/callback... - const pVfs = capi.sqlite3_js_db_vfs(pDb); - if(!pVfs) toss3("Internal error: cannot get VFS for new db handle."); + const pVfs = capi.sqlite3_js_db_vfs(pDb) + || toss3("Internal error: cannot get VFS for new db handle."); const postInitSql = __vfsPostOpenSql[pVfs]; if(postInitSql){ - if(capi.sqlite3_activate_see){ - /** - In SEE-capable builds we have to avoid running any db - code before the client has an opportunity to apply their - decryption key. If we first run any db code, e.g. pragma - journal_mode=..., then it will fail with SQLITE_NOTADB - and the db handle will be left in an unusuable - state. Note that at this point we do not actually know - whether the db is encrypted, but if a client has gone out - of their way to create an SEE build, it seems safe to - assume that they are using the encryption. - */ - sqlite3.config.warn( - "Disabling execution of on-open() db code "+ - "because this is an SEE build. DB: "+fnJs - ); - }else if(postInitSql instanceof Function){ + /** + Reminder: if this db is encrypted and the client did _not_ pass + in the key, any init code will fail, causing the ctor to throw. + We don't actually know whether the db is encrypted, so we cannot + sensibly apply any heuristics which skip the init code only for + encrypted databases for which no key has yet been supplied. + */ + if(postInitSql instanceof Function){ postInitSql(this, sqlite3); }else{ checkSqlite3Rc( @@ -298,6 +380,20 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ - `flags`: open-mode flags - `vfs`: the VFS fname +//#if enable-see + And, for SEE-capable builds, optionally ONE of the following: + + - `key`, `hexkey`, or `textkey`: encryption key as a string, + ArrayBuffer, or Uint8Array. These flags function as documented + for the SEE pragmas of the same names. + + In non-SEE builds, these options are ignored. In SEE builds, + `PRAGMA key/textkey/hexkey=X` is executed immediately after + opening the db. If more than one of the options is provided, + or any option has an invalid argument type, an exception is + thrown. +//#endif enable-see + The `filename` and `vfs` arguments may be either JS strings or C-strings allocated via WASM. `flags` is required to be a JS string (because it's specific to this API, which is specific @@ -1562,7 +1658,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ they are larger than 32 bits, else double or int32, depending on whether they have a fractional part. Booleans are bound as integer 0 or 1. It is not expected the distinction of binding - doubles which have no fractional parts is integers is + doubles which have no fractional parts and integers is significant for the majority of clients due to sqlite3's data typing model. If [BigInt] support is enabled then this routine will bind BigInt values as 64-bit integers if they'll @@ -1946,16 +2042,26 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Functionally equivalent to DB(storageName,'c','kvvfs') except that it throws if the given storage name is not one of 'local' or 'session'. + + As of version 3.46, the argument may optionally be an options + object in the form: + + { + filename: 'session'|'local', + ... etc. (all options supported by the DB ctor) + } + + noting that the 'vfs' option supported by main DB + constructor is ignored here: the vfs is always 'kvvfs'. */ sqlite3.oo1.JsStorageDb = function(storageName='session'){ + const opt = dbCtorHelper.normalizeArgs(...arguments); + storageName = opt.filename; if('session'!==storageName && 'local'!==storageName){ toss3("JsStorageDb db name must be one of 'session' or 'local'."); } - dbCtorHelper.call(this, { - filename: storageName, - flags: 'c', - vfs: "kvvfs" - }); + opt.vfs = 'kvvfs'; + dbCtorHelper.call(this, opt); }; const jdb = sqlite3.oo1.JsStorageDb; jdb.prototype = Object.create(DB.prototype); diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index cc1db6723a..d315b43d6b 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -1678,6 +1678,25 @@ int sqlite3__wasm_config_j(int op, sqlite3_int64 arg){ return sqlite3_config(op, arg); } +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings. +** +** If z is not NULL, returns the result of passing z to +** sqlite3_mprintf()'s %Q modifier (if addQuotes is true) or %q (if +** addQuotes is 0). Returns NULL if z is NULL or on OOM. +*/ +SQLITE_WASM_EXPORT +char * sqlite3__wasm_qfmt_token(char *z, int addQuotes){ + char * rc = 0; + if( z ){ + rc = addQuotes + ? sqlite3_mprintf("%Q", z) + : sqlite3_mprintf("%q", z); + } + return rc; +} + #if 0 // Pending removal after verification of a workaround discussed in the // forum post linked to below. diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index fdde986355..e78897bedf 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -1482,7 +1482,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; /*step() skipped intentionally*/.reset(true); } finally { T.assert(0===st.finalize()) - .assert(undefined===st.finalize()); + .assert(undefined===st.finalize()); } try { @@ -2587,7 +2587,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; const pVfs = capi.sqlite3_vfs_find('kvvfs'); T.assert(pVfs); const JDb = this.JDb = sqlite3.oo1.JsStorageDb; - const unlink = this.kvvfsUnlink = ()=>{JDb.clearStorage(filename)}; + const unlink = this.kvvfsUnlink = ()=>JDb.clearStorage(this.kvvfsDbFile); unlink(); let db = new JDb(filename); try { @@ -2605,6 +2605,54 @@ globalThis.sqlite3InitModule = sqlite3InitModule; } } }/*kvvfs sanity checks*/) +//#if enable-see + .t({ + name: 'kvvfs with SEE encryption', + predicate: ()=>(isUIThread() + || "Only available in main thread."), + test: function(sqlite3){ + this.kvvfsUnlink(); + let db; + try{ + db = new this.JDb({ + filename: this.kvvfsDbFile, + key: 'foo' + }); + db.exec([ + "create table t(a,b);", + "insert into t(a,b) values(1,2),(3,4)" + ]); + db.close(); + let err; + try{ + db = new this.JDb({ + filename: this.kvvfsDbFile, + flags: 'ct' + }); + T.assert(db) /* opening is fine, but... */; + db.exec("select 1 from sqlite_schema"); + console.warn("sessionStorage =",sessionStorage); + }catch(e){ + err = e; + }finally{ + db.close(); + } + T.assert(err,"Expecting an exception") + .assert(sqlite3.capi.SQLITE_NOTADB==err.resultCode, + "Expecting NOTADB"); + db = new sqlite3.oo1.DB({ + filename: this.kvvfsDbFile, + vfs: 'kvvfs', + hexkey: new Uint8Array([0x66,0x6f,0x6f]) // equivalent: '666f6f' + }); + T.assert( 4===db.selectValue('select sum(a) from t') ); + }finally{ + if( db ) db.close(); + this.kvvfsUnlink(); + } + } + })/*kvvfs with SEE*/ +//#endif enable-see ;/* end kvvfs tests */ //////////////////////////////////////////////////////////////////////// diff --git a/manifest b/manifest index c203544e0b..a0cb0217b3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Extra\srobustness\sin\sthe\scode\sthat\scauses\scursors\sto\sreturn\sNULL\swhen\sthey\nare\sparticipating\sin\san\sOUTER\sJOIN. -D 2024-04-22T13:31:24.188 +C Extend\sthe\sJS/WASM\sSEE\sbuild\ssupport\sby\s(A)\sfiltering\sSEE-related\sbits\sout\sof\sthe\sJS\swhen\snot\sbuilding\swith\sSEE\sand\s(B)\saccepting\san\soptional\skey/textkey/hexkey\soption\sto\sthe\ssqlite3.oo1.DB\sand\ssubclass\sconstructors\sto\screate/open\sSEE-encrypted\sdatabases\swith.\sDemonstrate\sSEE\sin\sthe\stest\sapp\susing\sthe\skvvfs.\sThis\sobviates\sthe\schanges\smade\sin\s[5c505ee8a7]. +D 2024-04-22T16:46:37.381 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -587,7 +587,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt ca7e9ee82ca4e1c1744295f8184dd70edfae1992865d26c64303f539eb6c084c F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 4bb4cf70a8153dd5b5fee17d724075c54174da630b424bbcf48c744633396f62 +F ext/wasm/GNUmakefile 0d5ccc8a4814716c1c789bb3069dd4d71d5ef0a97bbea074ac182fbfb85a3ca8 F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576 F ext/wasm/README.md a8a2962c3aebdf8d2104a9102e336c5554e78fc6072746e5daf9c61514e7d193 F ext/wasm/SQLTester/GNUmakefile e0794f676d55819951bbfae45cc5e8d7818dc460492dc317ce7f0d2eca15caff @@ -605,8 +605,8 @@ F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08 F ext/wasm/api/post-js-header.js 04dc12c3edd666b64a1b4ef3b6690c88dcc653f26451fd4734472d8e29c1c122 F ext/wasm/api/pre-js.c-pp.js ad906703f7429590f2fbf5e6498513bf727a1a4f0ebfa057afb08161d7511219 F ext/wasm/api/sqlite3-api-cleanup.js d235ad237df6954145404305040991c72ef8b1881715d2a650dda7b3c2576d0e -F ext/wasm/api/sqlite3-api-glue.js 2d35660c52dcb4bb16d00c56553d34e7caa6ad30083938b515e6f9aa0b312fbb -F ext/wasm/api/sqlite3-api-oo1.js 5b61a9ea9465d75a6086f89273778cad0c3c1794a59c23cce3363e06a1f78bfb +F ext/wasm/api/sqlite3-api-glue.js c744f4b919e1254c898b467573858671a1c8797c2490d0eca2fdbadf2d0ac74b +F ext/wasm/api/sqlite3-api-oo1.js 708934dd9919863bb67e2a54ba6604b05835ba3779d4dc4486218c8512eb0771 F ext/wasm/api/sqlite3-api-prologue.js 93a72b07b2a5d964d2edc76a90b439ece49298bd7ba60a1c6ae5d4878213701e F ext/wasm/api/sqlite3-api-worker1.js 8d9c0562831f62218170a3373468d8a0b7a6503b5985e309b69bf71187b525cf F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 @@ -615,7 +615,7 @@ F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 8433ee332d5f5e39fb19427fccb7bad7f44aa99b5504daad3343fc128c311e78 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 3c72f1a0e6a7343c8c882d29d01bb440f10be12c844651605b486e76f3d6cc8c F ext/wasm/api/sqlite3-vtab-helper.c-pp.js a2fcbc3fecdd0eea229283584ebc122f29d98194083675dbe5cb2cf3a17fe309 -F ext/wasm/api/sqlite3-wasm.c afba6827a49151b564af5cf588a6bbd0401b16ef5cbe3269c66f676fee9ca92c +F ext/wasm/api/sqlite3-wasm.c 3f744dc45ac4be8125d58364448bdc9c082f332a88cec211bfd0143ad0acb373 F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bd89edfe42a4d7122a6d6d405c5423cf00aabba1f76f6ea8e2dba9c628ddd91a F ext/wasm/api/sqlite3-worker1.c-pp.js 5e8706c2c4af2a57fbcdc02f4e7ef79869971bc21bb8ede777687786ce1c92d5 F ext/wasm/batch-runner-sahpool.html e9a38fdeb36a13eac7b50241dfe7ae066fe3f51f5c0b0151e7baee5fce0d07a7 @@ -662,7 +662,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 1618670e466f424aa289859fe0ec8ded223e42e9e69b5c851f809baaaca1a00c F ext/wasm/tester1-worker.html ebc4b820a128963afce328ecf63ab200bd923309eb939f4110510ab449e9814c F ext/wasm/tester1.c-pp.html 1c1bc78b858af2019e663b1a31e76657b73dc24bede28ca92fbe917c3a972af2 -F ext/wasm/tester1.c-pp.js 18331ec28d7e63c8e262a9872a8da3964d37b7ac22eabe0016af93f3c6f74cc4 +F ext/wasm/tester1.c-pp.js 9f8ae7c716ad66523cd6238fe947826c82b6a3b5c1e9d528f9f39ad9c9280ac7 F ext/wasm/tests/opfs/concurrency/index.html 0802373d57034d51835ff6041cda438c7a982deea6079efd98098d3e42fbcbc1 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2184,8 +2184,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5c505ee8a73f4b4a7053d98a12024d98340676f6ae9982311f9f88a9b46c8ae2 -R cf33b03e4c9b76c0999c9981aab311dd -U drh -Z 8e2859a20a59f6d407b535859ed9b4c0 +P 672c2869ef48e08447d37b0d76a1850cdafbe30ca1906ec98c55e3ab496fd9a6 +R 69f9c602594d1ff3e9a25e02a2d41151 +U stephan +Z 6ab53eec83d52fe93f0c77e2c64383e8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 41b06bb769..61afb9adfe 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -672c2869ef48e08447d37b0d76a1850cdafbe30ca1906ec98c55e3ab496fd9a6 \ No newline at end of file +8fbda563d2f56f8dd3f695a5711e4356de79035f332270db45d4b33ed52fdfd2 \ No newline at end of file