From: stephan Date: Tue, 21 Jul 2026 12:57:48 +0000 (+0000) Subject: An initial attempt at embedding sqlite3.wasm into sqlite3.js using base64. This produ... X-Git-Url: http://git.ipfire.org/gitweb/?a=commitdiff_plain;h=379e111bff35f8c6d742cb7ae11bb6c4944ff47c;p=thirdparty%2Fsqlite.git An initial attempt at embedding sqlite3.wasm into sqlite3.js using base64. This produces a chicken-egg situation in the build because the input file where the blob belongs is injested by emscripten (which generates the wasm file from which that blob is to be generated), so it does not work as-is but is being set aside for later. We'll need to add a second preprocessing step which uses a different #directive delimiter to keep it hidden inside the first pass, then squeeze the wasm file in through there. FossilOrigin-Name: 538199cf62f07a2fe7ee5775c9f01aff1644d638bc0916cf631d61376462b5cb --- diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index f47383f4ce..0bc910462d 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -217,6 +217,9 @@ b.cp = $(call b.mkdir@); \ # error before we get to handle that filename juggling. Most JS files # defensively set a file-local @policy. D.common = +ifeq (1,$(base64.wasm)) +D.common += -Dbase64.wasm +endif # # $(call b.c-pp.shcmd,LOGTAG,src,dest,-Dx=y...) diff --git a/ext/wasm/api/extern-post-js.c-pp.js b/ext/wasm/api/extern-post-js.c-pp.js index b2e760d6a0..a3a02c8019 100644 --- a/ext/wasm/api/extern-post-js.c-pp.js +++ b/ext/wasm/api/extern-post-js.c-pp.js @@ -56,7 +56,9 @@ const toExportForESM = Module.instantiateWasm() is found in pre-js.c-pp.js. */ - wasmFilename: '@sqlite3.wasm@' /* replaced by the build process */ +//#@ push delimiter "<<<" ">>>" + wasmFilename: '<<>>' /* replaced by the build process */ +//#@ pop delimiter }); sIMS.debugModule = sIMS.urlParams.has('sqlite3.debugModule') diff --git a/ext/wasm/api/pre-js.c-pp.js b/ext/wasm/api/pre-js.c-pp.js index 98ed6fa72d..5c2e7a921e 100644 --- a/ext/wasm/api/pre-js.c-pp.js +++ b/ext/wasm/api/pre-js.c-pp.js @@ -124,6 +124,28 @@ return this.emscriptenInstantiateWasm(imports, onSuccess); } const sims = this; +//#if defined sqlite3.wasm.base64 + let b64Wasm = ` +//#base64 +//#include -raw [arg sqlite3.wasm.base64] +//#/base64 +`; + const bytes = Uint8Array.fromBase64(b64Wasm); + b64Wasm = null; + const finalThen = (arg) => { + arg.imports = imports; + sims.instantiateWasm = arg; /* used by sqlite3-api-prologue.c-pp.js */ + onSuccess(arg.instance, arg.module); + }; + + WebAssembly.instantiate(bytes.buffer, imports) + .then(finalThen) + .catch((err) => { + console.error("Failed to instantiate embedded sqlite3.wasm", err); + }); + + return {}/* Emscripten interprets this to wait on async instantiation */; +//#else const uri = Module.locateFile( sims.wasmFilename, ( ('undefined'===typeof scriptDirectory/*var defined by Emscripten glue*/) @@ -147,6 +169,7 @@ .then(bytes => WebAssembly.instantiate(bytes, imports)) .then(finalThen) return loadWasm(); +//#/if sqlite3.wasm.base64 }.bind(sIMS); //#/if Module.instantiateWasm and not wasmfs })(Module); diff --git a/ext/wasm/libcmpp.c b/ext/wasm/libcmpp.c index 2450b7e397..f41554a28d 100644 --- a/ext/wasm/libcmpp.c +++ b/ext/wasm/libcmpp.c @@ -21,7 +21,7 @@ #define CMPP_LIB_VERSION "2.0.x" #define CMPP_LIB_VERSION_HASH "a53a63923506aba565e91571c0f6bdb678e7dcc05dc9f7e281a17e68bf95aa46" #define CMPP_LIB_VERSION_TIMESTAMP "2026-07-21 10:38:56.344 UTC" -#define CMPP_LIB_CONFIG_TIMESTAMP "2026-07-21 10:39 GMT" +#define CMPP_LIB_CONFIG_TIMESTAMP "2026-07-21 11:23 GMT" #define CMPP_VERSION CMPP_LIB_VERSION " " CMPP_LIB_VERSION_HASH " @ " CMPP_LIB_VERSION_TIMESTAMP #define CMPP_PLATFORM_EXT_DLL ".so" #define CMPP_MODULE_PATH ".:/usr/local/lib/cmpp" @@ -10207,6 +10207,195 @@ const cmpp_outputer cmpp_outputer_obuf = { .cleanup = cmpp_outputer_cleanup_f_obuf }; #endif + +/* base64 pieces copied from sqlite3.org/src/file/ext/misc/base64.c on + 2026-07-19. */ +#define PC 0x80 /* pad character */ +#define WS 0x81 /* whitespace */ +#define ND 0x82 /* Not above or digit-value */ +#define PAD_CHAR '=' + +#ifndef U8_TYPEDEF +typedef unsigned char u8; +#define U8_TYPEDEF +#endif + +static const char b64Numerals[64+1] += "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +#define BX_NUMERAL(dv) (b64Numerals[(u8)(dv)]) +/* Width of base64 lines. Should be an integer multiple of 4. */ +#define B64_DARK_MAX 72 + +/* Encode a byte buffer into base64 text with linefeeds appended to limit +** encoded group lengths to B64_DARK_MAX or to terminate the last group. +*/ +static char* toBase64( u8 const *pIn, cmpp_size_t nbIn, char *pOut ){ + int nCol = 0; + while( nbIn >= 3 ){ + /* Do the bit-shuffle, exploiting unsigned input to avoid masking. */ + pOut[0] = BX_NUMERAL(pIn[0]>>2); + pOut[1] = BX_NUMERAL(((pIn[0]<<4)|(pIn[1]>>4))&0x3f); + pOut[2] = BX_NUMERAL(((pIn[1]&0xf)<<2)|(pIn[2]>>6)); + pOut[3] = BX_NUMERAL(pIn[2]&0x3f); + pOut += 4; + nbIn -= 3; + pIn += 3; + if( (nCol += 4)>=B64_DARK_MAX || nbIn<=0 ){ + *pOut++ = '\n'; + nCol = 0; + } + } + if( nbIn > 0 ){ + signed char nco = nbIn+1; + cmpp_ssize_t nbe; + unsigned long qv = *pIn++; + for( nbe=1; nbe<3; ++nbe ){ + qv <<= 8; + if( (cmpp_size_t)nbe=0; --nbe ){ + char ce = (nbe>= 6; + pOut[nbe] = ce; + } + pOut += 4; + *pOut++ = '\n'; + } + *pOut = 0; + return pOut; +} + +#if 0 /* from-base-64. We don't yet have a #base64 -d[ecode] */ +/* Decoding table, ASCII (7-bit) value to base 64 digit value or other */ +static const u8 b64DigitValues[128] = { + /* HT LF VT FF CR */ + ND,ND,ND,ND, ND,ND,ND,ND, ND,WS,WS,WS, WS,WS,ND,ND, + /* US */ + ND,ND,ND,ND, ND,ND,ND,ND, ND,ND,ND,ND, ND,ND,ND,ND, + /*sp + / */ + WS,ND,ND,ND, ND,ND,ND,ND, ND,ND,ND,62, ND,ND,ND,63, + /* 0 1 5 9 = */ + 52,53,54,55, 56,57,58,59, 60,61,ND,ND, ND,PC,ND,ND, + /* A O */ + ND, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10, 11,12,13,14, + /* P Z */ + 15,16,17,18, 19,20,21,22, 23,24,25,ND, ND,ND,ND,ND, + /* a o */ + ND,26,27,28, 29,30,31,32, 33,34,35,36, 37,38,39,40, + /* p z */ + 41,42,43,44, 45,46,47,48, 49,50,51,ND, ND,ND,ND,ND +}; +#define BX_DV_PROTO(c) \ + ((((u8)(c))<0x80)? (u8)(b64DigitValues[(u8)(c)]) : 0x80) +#define IS_BX_DIGIT(bdp) (((u8)(bdp))<0x80) +#define IS_BX_WS(bdp) ((bdp)==WS) +#define IS_BX_PAD(bdp) ((bdp)==PC) +/* Skip over text which is not base64 numeral(s). */ +static char * skipNonB64( char *s, int nc ){ + char c; + while( nc-- > 0 && (c = *s) && !IS_BX_DIGIT(BX_DV_PROTO(c)) ) ++s; + return s; +} + +/* Decode base64 text into a byte buffer. */ +static u8* fromBase64( char *pIn, int ncIn, u8 *pOut ){ + if( ncIn>0 && pIn[ncIn-1]=='\n' ) --ncIn; + while( ncIn>0 && *pIn!=PAD_CHAR ){ + static signed char nboi[] = { 0, 0, 1, 2, 3 }; + char *pUse = skipNonB64(pIn, ncIn); + unsigned long qv = 0L; + int nti, nbo, nac; + ncIn -= (pUse - pIn); + pIn = pUse; + nti = (ncIn>4)? 4 : ncIn; + ncIn -= nti; + nbo = nboi[nti]; + if( nbo==0 ) break; + for( nac=0; nac<4; ++nac ){ + char c = (nac>8) & 0xff; + deliberate_fall_through; /* FALLTHRU */ + case 1: + pOut[0] = (qv>>16) & 0xff; + break; + } + pOut += nbo; + } + return pOut; +} +#endif /* base64 decode */ + +/** + Base64-encodes the contents of bIn and replaces bIn's contents + (if any) with these. + + Potential TODO is have it append to bOut, but it's simpler to + replace it and that's all our use cases currently need. +*/ +int cmpp__b_base64_encode(cmpp *pp, cmpp_b const * bIn, cmpp_b * bOut){ + cmpp_size_t nc; + cmpp_size_t const nv = bIn->n; + char *cBuf; + unsigned char const *bBuf; + cmpp_size_t const nvMax = 1024 * 1024 * 1024; + if( ppCode ) return ppCode; + nc = 4*((nv+2)/3); /* quads needed */ + nc += (nc+(B64_DARK_MAX-1))/B64_DARK_MAX + 1; /* LFs and a 0-terminator */ + if( nvMax < nc ){ + return cmpp_err_set(pp, CMPP_RC_RANGE, + "Blob expanded to base64 too big."); + } + bBuf = bIn->z; + cmpp_b_reuse(bOut); + if( !bBuf || !bIn->z ){ + /* Is encoding 0 bytes misuse? */ + return 0; + } + if( 0==cmpp_b_reserve3(pp, bOut, nc) ){ + cBuf = (char*)bOut->z; + bOut->n = (cmpp_size_t)(toBase64(bIn->z, bIn->n, cBuf) - cBuf); + assert( bOut->n < bOut->nAlloc ); + cBuf[bOut->n] = 0; + } + return ppCode; +} + +#undef PC +#undef WS +#undef ND +#undef PAD_CHAR +#undef UB_TYPEDEF +#undef BX_DV_PROTO +#undef IS_BX_DIGIT +#undef IS_BX_WS +#undef IS_BX_PAD +#undef BX_NUMERAL +#undef B64_DARK_MAX /* ** 2025-11-07: ** @@ -13208,6 +13397,78 @@ end: cmpp_b_return(dx->pp, bR); } +/** Impl for #base64. */ +static void cmpp_dx_f_base64(cmpp_dx *dx){ + cmpp_api_init(dx->pp); + cmpp_b ob = cmpp_b_empty; + cmpp_flag32_t consumeFlags = cmpp_dx_consume_F_PROCESS_OTHER_D; + bool comma = false; + bool quote = false; + cmpp_arg const *arg = 0; + for( arg = dx->args.arg0; arg; arg = arg->next ){ + +#define M(STR) cmpp_arg_equals(arg, STR) + if( M("-comma") ) comma = true; + else if( M("-quote") ) quote = true; + else{ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "Unhandled argument: %s", arg->z); + return; + } +#undef M + } + if( cmpp_dx_consume_b(dx, &ob, &dx->d->closer, 1, consumeFlags) ){ + goto end; + }else{ + cmpp_b * const b64 = cmpp_b_borrow(dx->pp); + if( !b64 ) goto end; + int cmpp__b_base64_encode(cmpp *, cmpp_b const *, cmpp_b *)/* in b.c */; + cmpp__b_base64_encode(dx->pp, &ob, b64); + if( cmpp_dx_err_check(dx) ){ + cmpp_b_return(dx->pp, b64); + goto end; + } + cmpp_b_swap(&ob, b64); + cmpp_b_return(dx->pp, b64); + } + unsigned char const * zEnd = ob.z + ob.n; +#define out(P,N) if( cmpp_dx_out_raw(dx, P, N) ) goto end + if( !quote ){ + cmpp_dx_out_raw(dx, ob.z, ob.n); + zEnd = ob.z /* skip the next loop */; + } + for( unsigned char const * z = ob.z; z < zEnd; ++z ){ + if( quote ){ + out("\"", 1); + } + bool closed = false; + for( ;zzDotWasm - ? pB->zDotWasm - : pB->zBaseName; + char const * const zDotWasm = pB + ? (pB->zDotWasm ? pB->zDotWasm : zBaseName) + : 0; /* ** See BuildDef::zDotWasm for _why_ we do this. _What_ we're doing ** is generate $(pre-js.BUILDNAME.js) as above, but: @@ -637,20 +637,22 @@ static void mk_pre_post(char const *zBuildName, BuildDef const * pB){ */ pf("$(pre-js.%s.js): $(pre-js.in.js) $(bin.c-pp) $(MAKEFILE_LIST)", zBuildName); - if( pB->zDotWasm ){ + if( zDotWasm ){ pf(" $(dir.dout)/%s.wasm" /* This .wasm is from some other build, so this may trigger a full build of the reference copy. */, - pB->zDotWasm); + zDotWasm); } ps(""); pf("\t@$(call b.mkdir@); $(call b.c-pp.shcmd," "%s," "$(pre-js.in.js)," "$(pre-js.%s.js)," - "$(c-pp.D.%s)" C_PP_D_CUSTOM_INSTANTIATE + "$(c-pp.D.%s) -Dsqlite3.wasm.base64=$(dir.dout)/%s.wasm" + C_PP_D_CUSTOM_INSTANTIATE ")\n", - zBuildName, zBuildName, zBuildName); + zBuildName, zBuildName, zBuildName, + zDotWasm/*null is harmless here*/); } ps("\n# --post-js=..."); @@ -684,7 +686,8 @@ static void mk_pre_post(char const *zBuildName, BuildDef const * pB){ "$(c-pp.D.%s) --@policy=error -Dsqlite3.wasm=%s.wasm" "))", zBuildName, zBuildName, zBuildName, - zBaseName); + (pB->zDotWasm ? pB->zDotWasm : zBaseName) + ); }else{ pf("$(eval $(call b.c-pp.target," "%s," diff --git a/manifest b/manifest index 178c4de68e..267f72f9fa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Latest\supstream\sext/wasm/libcmpp.c,\sprimarily\sto\smake\sit\sfilcc-legal\s(only\sthe\sfinal\schunk\sin\sthe\sdiff\s-\sthe\srest\sis\sminor\schanges\saccumulated\ssince\sthe\slast\supdate). -D 2026-07-21T10:44:28.794 +C An\sinitial\sattempt\sat\sembedding\ssqlite3.wasm\sinto\ssqlite3.js\susing\sbase64.\sThis\sproduces\sa\schicken-egg\ssituation\sin\sthe\sbuild\sbecause\sthe\sinput\sfile\swhere\sthe\sblob\sbelongs\sis\sinjested\sby\semscripten\s(which\sgenerates\sthe\swasm\sfile\sfrom\swhich\sthat\sblob\sis\sto\sbe\sgenerated),\sso\sit\sdoes\snot\swork\sas-is\sbut\sis\sbeing\sset\saside\sfor\slater.\sWe'll\sneed\sto\sadd\sa\ssecond\spreprocessing\sstep\swhich\suses\sa\sdifferent\s#directive\sdelimiter\sto\skeep\sit\shidden\sinside\sthe\sfirst\spass,\sthen\ssqueeze\sthe\swasm\sfile\sin\sthrough\sthere. +D 2026-07-21T12:57:48.902 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -582,7 +582,7 @@ F ext/session/sessionwor.test 6fd9a2256442cebde5b2284936ae9e0d54bde692d0f5fd009e F ext/session/sqlite3session.c 673a2c71a3e1fc909301ed15daa30485d774419d015eb7688c79396e6cba6378 F ext/session/sqlite3session.h ca7c4422c1514a95056cc8d333217df6b1829d39058126b1de85d10cd62d7a9c F ext/session/test_session.c f465f13c661919f8ba4485a9d0690c4df5bff1cc79e962cfdafa9ff376d5119f -F ext/wasm/GNUmakefile 92936a75dc15057dae3a87d96fe8282b22e663edb139852551da9179b437f1b3 +F ext/wasm/GNUmakefile 6c8c97611d44410eefd8a2c7e578c9ac13ad2b230e88ba3039fe9cdd464bc411 F ext/wasm/README-dist.txt f01081a850ce38a56706af6b481e3a7878e24e42b314cfcd4b129f0f8427066a F ext/wasm/README.md 2e87804e12c98f1d194b7a06162a88441d33bb443efcfe00dc6565a780d2f259 F ext/wasm/SQLTester/GNUmakefile e0794f676d55819951bbfae45cc5e8d7818dc460492dc317ce7f0d2eca15caff @@ -592,13 +592,13 @@ F ext/wasm/SQLTester/index.html 64f3435084c7d6139b08d1f2a713828a73f68de2ae6a3112 F ext/wasm/SQLTester/touint8array.c 2d5ece04ec1393a6a60c4bf96385bda5e1a10ad49f3038b96460fc5e5aa7e536 F ext/wasm/api/EXPORTED_FUNCTIONS.c-pp 189935d0106bca86661efc991ab13e1d5d6e1f55ec34dc7b5055a99321397edd F ext/wasm/api/README.md a905d5c6bfc3e2df875bd391d6d6b7b48d41b43bdee02ad115b47244781a7e81 -F ext/wasm/api/extern-post-js.c-pp.js 80accc53cc6ea1e61c721595f42ba95baa7c7ea636807d9507e69403301f8c54 +F ext/wasm/api/extern-post-js.c-pp.js cba0f9735f741ce266aa0c52e6e1224f9fb027f20253292d3e71480787675a05 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/opfs-common-inline.c-pp.js 496ca858af09b7fef2efaece467960611d35f57254078424bcdeac42ded9e85d F ext/wasm/api/opfs-common-shared.c-pp.js 9a9019bcc070d7c90829af5f5eb81d25135bf9543944a691631c0184f0c62416 F ext/wasm/api/post-js-footer.js a50c1a2c4d008aede7b2aa1f18891a7ee71437c2f415b8aeb3db237ddce2935b F ext/wasm/api/post-js-header.js f35d2dcf1ab7f22a93d565f8e0b622a2934fc4e743edf3b708e4dd8140eeff55 -F ext/wasm/api/pre-js.c-pp.js 949c1a9bdee2457d2a8d9ba9b362634b19d3aa71241fad348aab01ba2cde0c1a +F ext/wasm/api/pre-js.c-pp.js c50ae08280f3ba27f4a466e27b612f09adf86b46ae66040aebf49d1842d572b9 F ext/wasm/api/sqlite3-api-glue.c-pp.js 31a721ada7225838a61310a9f3f797fa5275353f8e9b0ae769d85b437be061f5 F ext/wasm/api/sqlite3-api-oo1.c-pp.js 35e4727010f15fd72ead0dd1eb4e3c2c9bb1cc60e51544cbdff1f7c14f209de2 F ext/wasm/api/sqlite3-api-prologue.js 0084e15d66fbcd75cacbaa58e3b473d5e57082c30f3122be7fdadff5589cf6b6 @@ -636,9 +636,9 @@ F ext/wasm/index-dist.html db23748044e286773f2768eec287669501703b5d5f72755e8db73 F ext/wasm/index.html 5bf6cf1b0a3c8b9f5f54d77f2219d7ae87a15162055ce308109c49b1dcab4239 F ext/wasm/jaccwabyt/jaccwabyt.js 4e2b797dc170851c9c530c3567679f4aa509eec0fab73b466d945b00b356574b F ext/wasm/jaccwabyt/jaccwabyt.md 6aa90fa1a973d0ad10d077088bea163b241d8470c75eafdef87620a1de1dea41 -F ext/wasm/libcmpp.c e8ad0f9213af8edb19859e2b584ab6d5801a0e5e19fd715c9add269dfa720ee3 +F ext/wasm/libcmpp.c 5296a720aa95795617335fa0b3b4da5ea48150132251b85ca131bc6f2a21b066 F ext/wasm/mkdist.sh f8883b077a2ca47cf92e6f0ce305fbf72ca648c3501810125056c4b09c2d5554 x -F ext/wasm/mkwasmbuilds.c 60706fb66db7f1e24d876e3cb2493484cfb27e7b103c7741c384505dd678769c +F ext/wasm/mkwasmbuilds.c 3402400b432f7bff5eb9f4c65e99636b9c928a92cec37d8a9b6790ed578e57bd F ext/wasm/module-symbols.html e54f42112e0aac2a31f850ab33e7f2630a2ea4f63496f484a12469a2501e07e2 F ext/wasm/scratchpad-wasmfs.html a3d7388f3c4b263676b58b526846e9d02dfcb4014ff29d3a5040935286af5b96 F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd842231505895eff00dbd57c63 @@ -2217,8 +2217,11 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh a554d13f6e5cf3760f041b87939e3d616ec6961859c3245e8ef701d1eafc2ca2 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P 099af17c6f82d0a1357f4d3ec5967fb5c664ec532ddfa674017cb4d8041d45c3 -R 1b770928cd957473d7d5119d2d70cf68 +P 06f3bbc5bddc2e763042f8b487a3deedf47c54b7fa785ba6fd6abd524b4cbe3a +R 17e18b90e1608e97dcf702df348ee49d +T *branch * wasm-base64 +T *sym-wasm-base64 * +T -sym-trunk * Cancelled\sby\sbranch. U stephan -Z 98cce33760d403d951c7ab42f002d518 +Z 745a489ee2ca72e98a604082d182941a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.tags b/manifest.tags index bec971799f..391cf1d356 100644 --- a/manifest.tags +++ b/manifest.tags @@ -1,2 +1,2 @@ -branch trunk -tag trunk +branch wasm-base64 +tag wasm-base64 diff --git a/manifest.uuid b/manifest.uuid index 55033a6931..db956d6be1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -06f3bbc5bddc2e763042f8b487a3deedf47c54b7fa785ba6fd6abd524b4cbe3a +538199cf62f07a2fe7ee5775c9f01aff1644d638bc0916cf631d61376462b5cb