From: stephan <stephan@noemail.net>
Date: Wed, 16 Jul 2025 17:00:38 +0000 (+0000)
Subject: Build .wasm/.js files into a build-mode-specific subdir, the goal being to be able... 
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=ea17024eedbb10e3d25761dff132ed02c7e9281a;p=thirdparty%2Fsqlite.git

Build .wasm/.js files into a build-mode-specific subdir, the goal being to be able to build them in parallel. This works, but (A) adds a bit of build ugliness and (B) is not actually parallelizing, so needs a closer look after a break.

FossilOrigin-Name: 5e11d054122a246b14bb399acb08bff80a16060d648fb6337689d623afc00474
---

diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile
index 87c2ca2fcb..3da2fb67bb 100644
--- a/ext/wasm/GNUmakefile
+++ b/ext/wasm/GNUmakefile
@@ -47,6 +47,7 @@ MAKING_CLEAN = $(if $(filter %clean,$(MAKECMDGOALS)),1,0)
 .PHONY: clean distclean
 clean:
 	-rm -f $(CLEAN_FILES)
+	-rm -fr $(CLEAN_DIRS)
 distclean: clean
 	-rm -f $(DISTCLEAN_FILES)
 
@@ -165,12 +166,14 @@ dir.dout = $(dir.wasm)/jswasm
 dir.tmp = $(dir.wasm)/bld
 dir.wasmfs = $(dir.dout)
 
+MKDIR.dout = $(dir.dout)
 MKDIR.bld = $(dir.tmp)
-$(MKDIR.bld):
-	@mkdir -p $@ $(dir.dout)
+$(MKDIR.bld) $(MKDIR.dout):
+	@mkdir -p $@
 
 CLEAN_FILES += *~ $(dir.jacc)/*~ $(dir.api)/*~ $(dir.common)/*~ $(dir.fiddle)/*~ \
-  $(dir.fiddle-debug)/* $(dir.dout)/* $(dir.tmp)/*
+  $(dir.fiddle-debug)/*
+CLEAN_DIRS = $(dir.tmp) $(dir.dout)
 
 ########################################################################
 # Set up sqlite3.c and sqlite3.h...
@@ -552,7 +555,7 @@ SOAP.js.bld = $(dir.dout)/$(notdir $(SOAP.js))
 #   not part of the amalgamation.
 #
 sqlite3-api.ext.jses = $(SOAP.js.bld)
-$(SOAP.js.bld): $(SOAP.js)
+$(SOAP.js.bld): $(SOAP.js) $(MKDIR.dout)
 	cp $< $@
 
 ########################################################################
@@ -887,10 +890,10 @@ EXPORTED_FUNCTIONS.fiddle = $(dir.tmp)/EXPORTED_FUNCTIONS.fiddle
 # together. i.e.  we're building $(sqlite3.wasm) multiple times, but
 # that's unavoidable (and harmless, but is a significant waste of
 # build time).
-$(sqlite3.wasm): $(sqlite3.js)
-$(sqlite3.mjs): $(sqlite3.js)
-$(dir.dout)/sqlite3-bundler-friendly.mjs: $(sqlite3.mjs)
-$(dir.dout)/sqlite3-node.mjs: $(sqlite3.mjs)
+#$(sqlite3.wasm): $(sqlite3.js)
+#$(sqlite3.mjs): $(sqlite3.js)
+#$(dir.dout)/sqlite3-bundler-friendly.mjs: $(sqlite3.mjs)
+#$(dir.dout)/sqlite3-node.mjs: $(sqlite3.mjs)
 #CLEAN_FILES += $(sqlite3.wasm)
 
 ifneq (1,$(MAKING_CLEAN))
diff --git a/ext/wasm/mkwasmbuilds.c b/ext/wasm/mkwasmbuilds.c
index d70de042e6..511c05d34d 100644
--- a/ext/wasm/mkwasmbuilds.c
+++ b/ext/wasm/mkwasmbuilds.c
@@ -59,19 +59,26 @@ static const char * zBanner =
 ** to breakage in some of the flag checks.
 */
 enum LibModeFlags {
-  /* Sentinel value */
-  LIBMODE_PLAIN = 0,
+  /* The canonical build */
+  LIBMODE_MAIN = 0x01,
   /* Indicates an ESM module build. */
-  LIBMODE_ESM = 0x01,
+  LIBMODE_ESM = 0x02,
   /* Indicates a "bundler-friendly" build mode. */
-  LIBMODE_BUNDLER_FRIENDLY = 0x02,
-  /* Indicates to _not_ add this build to the 'all' target. */
-  LIBMODE_DONT_ADD_TO_ALL = 0x04,
+  LIBMODE_BUNDLER_FRIENDLY = 0x04,
+  /* Indicates that this is an unsupported build mode. These builds
+     are not added to the 'all' target. */
+  LIBMODE_UNSUPPORTED = 0x08,
   /* Indicates a node.js-for-node.js build (untested and
   ** unsupported). */
-  LIBMODE_NODEJS = 0x08,
+  LIBMODE_NODEJS = 0x10,
   /* Indicates a wasmfs build (untested and unsupported). */
-  LIBMODE_WASMFS = 0x10
+  LIBMODE_WASMFS = 0x20,
+  /* Indicates that this build creates a file named "sqlite3.wasm" as
+     a side-effect of creating the .js/.mjs file. */
+  LIBMODE_SQLITE3_WASM = 0x40,
+  /* Indicates that the generated .js/.mjs file needs to be patched to
+     rename its reference to "sqlite3-XYZ.wasm" to "sqlite3.wasm". */
+  LIBMODE_PATCH_WASM_NAME = 0x80
 };
 
 /*
@@ -83,7 +90,7 @@ struct BuildDef {
   const char *zName;      /* Name from JS_BUILD_NAMES */
   const char *zMode;      /* Name from JS_BUILD_MODES */
   int flags;              /* Flags from LibModeFlags */
-  const char *zJsOut;     /* Name of generated sqlite3.js/.mjs */
+  const char *zJsOut;     /* Base name of generated sqlite3.js/.mjs */
   /* TODO: dynamically determine zJsOut based on zName, zMode, and
      flags. */
   const char *zCmppD;     /* Extra -D... flags for c-pp */
@@ -101,30 +108,31 @@ typedef struct BuildDef BuildDef;
 */
 const BuildDef aBuildDefs[] = {
   {/* Core build */
-    "sqlite3", "vanilla", LIBMODE_PLAIN, "$(sqlite3.js)", 0, 0},
+    "sqlite3", "vanilla", LIBMODE_MAIN | LIBMODE_SQLITE3_WASM,
+    "sqlite3.js", 0, 0},
 
   {/* Core ESM */
-   "sqlite3", "esm", LIBMODE_ESM, "$(sqlite3.mjs)",
-   "-Dtarget=es6-module", 0},
+   "sqlite3", "esm", LIBMODE_ESM | LIBMODE_SQLITE3_WASM,
+   "sqlite3.mjs",  "-Dtarget=es6-module", 0},
 
   {/* Core bundler-friend. Untested and "not really" supported, but
    ** required by the downstream npm subproject. */
     "sqlite3", "bundler-friendly",
-    LIBMODE_BUNDLER_FRIENDLY | LIBMODE_ESM,
-    "$(dir.dout)/sqlite3-bundler-friendly.mjs",
+    LIBMODE_BUNDLER_FRIENDLY | LIBMODE_PATCH_WASM_NAME | LIBMODE_ESM,
+    "sqlite3-bundler-friendly.mjs",
     "$(c-pp.D.sqlite3-esm) -Dtarget=es6-bundler-friendly", 0},
 
   {/* node.js mode. Untested and unsupported. */
-    "sqlite3", "node", LIBMODE_NODEJS | LIBMODE_DONT_ADD_TO_ALL,
-    "$(dir.dout)/sqlite3-node.mjs",
+    "sqlite3", "node", LIBMODE_NODEJS | LIBMODE_UNSUPPORTED,
+    "sqlite3-node.mjs",
     "$(c-pp.D.sqlite3-bundler-friendly) -Dtarget=node", 0},
 
   {/* The wasmfs build is optional, untested, unsupported, and
    ** needs to be invoked conditionally using info we don't have
    ** here. */
     "sqlite3-wasmfs", "esm" ,
-    LIBMODE_WASMFS | LIBMODE_ESM | LIBMODE_DONT_ADD_TO_ALL,
-    "$(dir.wasmfs)/sqlite3-wasmfs.mjs",
+    LIBMODE_WASMFS | LIBMODE_ESM | LIBMODE_UNSUPPORTED,
+    "sqlite3-wasmfs.mjs",
     "$(c-pp.D.sqlite3-bundler-friendly) -Dwasmfs",
     "-sEXPORT_ES6 -sUSE_ES6_IMPORT_META"},
 
@@ -137,7 +145,9 @@ const BuildDef aBuildDefs[] = {
 */
 static void mk_prologue(void){
   /* A 0-terminated list of makefile vars which we expect to have been
-  ** set up by this point in the build process. */
+  ** set up by this point in the build process. There are some which
+  ** we can't check here because they are not defined until after
+  ** these rules are included, but this set covers the basics. */
   char const * aRequiredVars[] = {
     "dir.top",
     "dir.api", "dir.dout", "dir.tmp",
@@ -367,6 +377,7 @@ static void mk_fiddle(void){
 ** from JS_BUILD_NAMES resp. JS_BUILD_MODES.
 */
 static void mk_lib_mode(const BuildDef * pB){
+  char aOutBuf[1024] = {0};
   const char * zWasmOut = "$(basename $@).wasm"
     /* The various targets named X.js or X.mjs (zJsOut) also generate
     ** X.wasm, and we need that part of the name to perform some
@@ -382,13 +393,31 @@ static void mk_lib_mode(const BuildDef * pB){
      pB->zCmppD ? pB->zCmppD : "<none>");
   pf("$(info Setting up build [%s-%s]: %s)\n", zNM, pB->zJsOut);
   mk_pre_post(zNM, pB->zCmppD);
+
+  snprintf(aOutBuf, sizeof(aOutBuf), "$(dir.tmp)/%s-%s", zNM);
+
+  /* Set up build-specific output dir. Several of the builds generate
+     a file named sqlite3.wasm, which precludes us building them in
+     parallel. We use a build-specific dir to enable parallization of
+     those builds. All such builds will generate a byte-for-byte
+     identical sqlite3.wasm so long as all Emscripten-related flags
+     which influence that file are the same (i.e. where pB->zEmcc is
+     the same), which is the case for most of the affected builds. */
+  pf("dir.tmp.%s-%s = %s\n", zNM, aOutBuf);
+  pf("$(dir.tmp.%s-%s)/.mkdir:\n\tmkdir -p $(dir.tmp.%s-%s)\n\ttouch $@\n",
+     zNM, zNM);
+  pf("clean-%s-%s:\n\trm -fr $(dir.tmp.%s-%s)\n"
+     "clean: clean-%s-%s\n", zNM, zNM, zNM);
+
   pf("\nemcc.flags.%s.%s ?=\n", zNM);
   if( pB->zEmcc && pB->zEmcc[0] ){
     pf("emcc.flags.%s.%s += %s\n", zNM, pB->zEmcc);
   }
 
   /* target pB->zJsOut */
-  pf("%s: $(MAKEFILE_LIST) $(sqlite3-wasm.cfiles) $(EXPORTED_FUNCTIONS.api) "
+  pf("$(dir.tmp.%s-%s)/%s: "
+     "$(dir.tmp.%s-%s)/.mkdir "
+     "$(MAKEFILE_LIST) $(sqlite3-wasm.cfiles) $(EXPORTED_FUNCTIONS.api) "
      "$(pre-post-%s-%s.deps) "
      "$(sqlite3-api.ext.jses)"
      /* ^^^ maintenance reminder: we set these as deps so that they
@@ -397,8 +426,8 @@ static void mk_lib_mode(const BuildDef * pB){
         are still compiling, which is especially helpful when running
         builds with long build times (like -Oz). */
      "\n",
-     pB->zJsOut, zNM);
-  pf("\t@echo \"Building $@ ...\"\n");
+     zNM, pB->zJsOut, zNM, zNM);
+  pf("\t@echo \"Building [%s-%s] %s ...\"\n", zNM, pB->zJsOut);
   pf("\t$(bin.emcc) -o $@ $(emcc_opt_full) $(emcc.flags) \\\n");
   pf("\t\t$(emcc.jsflags) -sENVIRONMENT=$(emcc.environment.%s) \\\n",
      pB->zMode);
@@ -417,26 +446,29 @@ static void mk_lib_mode(const BuildDef * pB){
   }
   pf("\t@chmod -x %s; \\\n"
      "\t\t$(maybe-wasm-strip) %s;\n",
-     zWasmOut, zWasmOut);
+     zWasmOut, zWasmOut)
+    /* For whatever reasons, .wasm files get built with their +x bit
+       set. That upsets althttpd, which uses +x as an indication that
+       the file is a CGI script. There's no apparent useful reason to
+       have +x set on these, so we -x them. */;
   pf("\t@$(call SQLITE.CALL.WASM-OPT,%s)\n", zWasmOut);
-  pf("\t@sed -i -e '/^.*= *_sqlite.*= *createExportWrapper/d' %s || exit; \\\n"
-     /*  ^^^^^^ reminder: Mac/BSD sed has no -i flag */
-     "\t\techo 'Stripped out createExportWrapper() parts.'\n",
-     pB->zJsOut) /* Our JS code installs bindings of each WASM export. The
-                generated Emscripten JS file does the same using its
-                own framework, but we don't use those results and can
-                speed up lib init, and reduce memory cost
-                considerably, by stripping them out. */;
+  pf(/* Our JS code installs bindings of each WASM export. The
+        generated Emscripten JS file does the same using its own
+        framework, but we don't use those results and can speed up lib
+        init, and reduce memory cost a bit, by stripping them out. Of
+        of this writing, this strips approximately 25kb of JS code. */
+    "\t@sed -i -e '/^.*= *_sqlite.*= *createExportWrapper/d' $@ || exit; \\\n"
+    /*  ^^^^^^ reminder: Mac/BSD sed has no -i flag */
+    "\t\techo 'Stripped out createExportWrapper() parts.'\n");
   /*
-  ** The above $(bin.emcc) call will write zJsOut and will create a
-  ** like-named .wasm file (zWasmOut). That .wasm file name gets
+  ** The above $(bin.emcc) call will write pB->zJsOut and will create
+  ** a like-named .wasm file (zWasmOut). That .wasm file name gets
   ** hard-coded into zJsOut so we need to, for some cases, patch
   ** zJsOut to use the name sqlite3.wasm instead. Note that the
-  ** resulting .wasm file is identical for all builds for which zEmcc
-  ** is empty.
+  ** resulting .wasm file is identical for all builds for which
+  ** pB->zEmcc is empty.
   */
-  if( (LIBMODE_BUNDLER_FRIENDLY & pB->flags)
-      || (LIBMODE_NODEJS & pB->flags) ){
+  if( (LIBMODE_PATCH_WASM_NAME & pB->flags) ){
     pf("\t@echo 'Patching $@ for %s.wasm...'; \\\n", pB->zName);
     pf("\t\trm -f %s; \\\n", zWasmOut);
     pf("\t\tsed -i -e 's/%s-%s.wasm/%s.wasm/g' $@ || exit;\n",
@@ -457,9 +489,15 @@ static void mk_lib_mode(const BuildDef * pB){
   }else{
     pf("\t@ls -la %s $@\n", zWasmOut);
   }
-  if( 0==(LIBMODE_DONT_ADD_TO_ALL & pB->flags) ){
-    pf("all: %s\n", pB->zJsOut);
+  pf("\n$(dir.dout)/%s: $(dir.tmp.%s-%s)/%s\n", pB->zJsOut, zNM, pB->zJsOut);
+  pf("\t@cp -p $(dir.tmp.%s-%s)/%s $(dir.dout)/.\n", zNM, pB->zJsOut);
+  if( (LIBMODE_SQLITE3_WASM & pB->flags) ){
+    pf("\t@cp -p $(dir.tmp.%s-%s)/*.wasm $(dir.dout)/.\n", zNM);
+  }
+  if( 0==(LIBMODE_UNSUPPORTED & pB->flags) ){
+    pf("all: $(dir.dout)/%s\n", pB->zJsOut);
   }
+  pf("%s-%s: $(dir.tmp.%s-%s)/%s\n", zNM, zNM, pB->zJsOut);
   pf("# End build [%s-%s]%s", zNM, zBanner);
 #undef zNM
 }
diff --git a/manifest b/manifest
index 5485ecf3fa..fd2c8cc345 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\ssome\sstray\smakefile\sdebug\soutput.
-D 2025-07-16T16:56:33.482
+C Build\s.wasm/.js\sfiles\sinto\sa\sbuild-mode-specific\ssubdir,\sthe\sgoal\sbeing\sto\sbe\sable\sto\sbuild\sthem\sin\sparallel.\sThis\sworks,\sbut\s(A)\sadds\sa\sbit\sof\sbuild\sugliness\sand\s(B)\sis\snot\sactually\sparallelizing,\sso\sneeds\sa\scloser\slook\safter\sa\sbreak.
+D 2025-07-16T17:00:38.236
 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
@@ -622,7 +622,7 @@ F ext/session/sqlite3session.c 19e14bcca2fbc63a8022ffd708ea6e6986c4003a1e9bbca9b
 F ext/session/sqlite3session.h b81e8536ce4b83babafd700f4ff67017804b6c1d71df963b30d3972958e7f4a7
 F ext/session/test_session.c 8766b5973a6323934cb51248f621c3dc87ad2a98f023c3cc280d79e7d78d36fb
 F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c
-F ext/wasm/GNUmakefile b9217715b615bfe51a8c13f6fa9889fb7b734e0f26bffc656ff52652e9906cc1
+F ext/wasm/GNUmakefile 55f580cc174cec7980a6ec6ab337fb0c9f199d6dba44b41eddf88268143d7522
 F ext/wasm/README-dist.txt f01081a850ce38a56706af6b481e3a7878e24e42b314cfcd4b129f0f8427066a
 F ext/wasm/README.md b89605f65661cf35bf034ff6d43e448cc169b8017fc105d498e33b81218b482c
 F ext/wasm/SQLTester/GNUmakefile e0794f676d55819951bbfae45cc5e8d7818dc460492dc317ce7f0d2eca15caff
@@ -683,7 +683,7 @@ F ext/wasm/index-dist.html 56132399702b15d70c474c3f1952541e25cb0922942868f70daf1
 F ext/wasm/index.html bcaa00eca521b372a6a62c7e7b17a870b0fcdf3e418a5921df1fd61e5344080d
 F ext/wasm/jaccwabyt/jaccwabyt.js 6e4f26d0edb5c2e7d381b7eff1924832a040a12274afab2d1e1789027e9f6c5c
 F ext/wasm/jaccwabyt/jaccwabyt.md 1128e3563e7eff90b5a373395251fc76cb32386fad1fea6075b0f34a8f1b9bdf
-F ext/wasm/mkwasmbuilds.c f5c143c10aeb7a519f7c08f98f90927f43c1991af3373bc6f80d8631a58acd42
+F ext/wasm/mkwasmbuilds.c 809878ea14f739f82102ff71671bad0c52e241fb8c1f0b9fd52c5d9daf9acec8
 F ext/wasm/module-symbols.html dc476b403369b26a1a23773e13b80f41b9a49f0825e81435fe3600a7cfbbe337
 F ext/wasm/scratchpad-wasmfs.html a3d7388f3c4b263676b58b526846e9d02dfcb4014ff29d3a5040935286af5b96
 F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd842231505895eff00dbd57c63
@@ -2213,8 +2213,11 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350
 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7
 F tool/warnings.sh 1ad0169b022b280bcaaf94a7fa231591be96b514230ab5c98fbf15cd7df842dd
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P d4203311a2f39189ed8f30d519468aed8983af7772a5b247e7557d3e1936064e
-R ba4aacc796c389ef7dfc5c4dcd68fec3
+P 7ef22c3d11088210d2267375ec188bd352b067614200394b9877f2e40dc12bb2
+R 7d952852b2799a65acea3c84253af2d1
+T *branch * wasm-build-parallel
+T *sym-wasm-build-parallel *
+T -sym-trunk * Cancelled\sby\sbranch.
 U stephan
-Z 4b13663095b96ba0896611543740e3b6
+Z 6d80506eb04a928ca3c1919e5188abeb
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 3d8ab5ceef..93e5fd5b36 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7ef22c3d11088210d2267375ec188bd352b067614200394b9877f2e40dc12bb2
+5e11d054122a246b14bb399acb08bff80a16060d648fb6337689d623afc00474