Introduce --run-build-scripts (-R) to always run build scripts

Currently we run "mkosi -t none" in systemd and mkosi-kernel to run
the build script(s) without rebuilding the entire image. Using the
"none" output format for this purpose is a hack. It also breaks when
using History=yes as running mkosi -t none will change the output
format in the history after which running mkosi qemu will fail saying
it can't boot the "none" output format.

Let's introduce a --run-build-scripts argument which will make us always
run the build scripts regardless of whether the image has already been
built or not. At the same time, remove various behaviors from the "none"
format that were solely added to enable the hack of using it to run build
scripts without rebuilding the image:

- Clean up outputs from the previous build when the "none" output format
  is used

diff --git a/mkosi/__init__.py b/mkosi/__init__.py
index db5713771109d0e22cffc0b868acc71c7055068e..55f0e3ff8ea12f8ec8bea051968c5179dfdc26b6 100644 (file)
--- a/mkosi/__init__.py
+++ b/mkosi/__init__.py
@@ -1342,9 +1342,9 @@ def finalize_default_initrd(
 
     initrd = dataclasses.replace(initrd, image="default-initrd")
 
-    if initrd.is_incremental() and initrd.expand_key_specifiers(initrd.cache_key) == config.expand_key_specifiers(
-        config.cache_key
-    ):
+    if initrd.is_incremental() and initrd.expand_key_specifiers(
+        initrd.cache_key
+    ) == config.expand_key_specifiers(config.cache_key):
         die(
             f"Image '{config.image}' and its default initrd image have the same cache key '{config.expand_key_specifiers(config.cache_key)}'",  # noqa: E501
             hint="Add the &I specifier to the cache key to avoid this issue",
@@ -3905,7 +3905,10 @@ def build_image(context: Context) -> None:
         check_root_populated(context)
         run_build_scripts(context)
 
-        if context.config.output_format == OutputFormat.none:
+        if context.config.output_format == OutputFormat.none or (
+            context.args.run_build_scripts
+            and (context.config.output_dir_or_cwd() / context.config.output).exists()
+        ):
             return
 
         if wantrepo:
@@ -4676,7 +4679,7 @@ def run_clean(args: Args, config: Config) -> None:
     sandbox = functools.partial(config.sandbox, tools=False)
 
     if args.verb == Verb.clean:
-        remove_outputs = config.output_format != OutputFormat.none
+        remove_outputs = True
         remove_build_cache = args.force > 0 or args.wipe_build_dir
         remove_image_cache = args.force > 0
         remove_package_cache = args.force > 1
@@ -5020,7 +5023,7 @@ def run_verb(args: Args, images: Sequence[Config], *, resources: Path) -> None:
             not (tools.output_dir_or_cwd() / tools.output).exists()
             or (tools.is_incremental() and not have_cache(tools))
         )
-        and (args.verb != Verb.build or last.output_format == OutputFormat.none)
+        and args.verb != Verb.build
         and not args.force
     ):
         die(
@@ -5088,6 +5091,7 @@ def run_verb(args: Args, images: Sequence[Config], *, resources: Path) -> None:
         and output.exists()
         and not output.is_symlink()
         and last.output_format != OutputFormat.none
+        and not args.run_build_scripts
     ):
         logging.info(f"Output path {output} exists already. (Use --force to rebuild.)")
         return
@@ -5153,7 +5157,11 @@ def run_verb(args: Args, images: Sequence[Config], *, resources: Path) -> None:
     if not have_history(args):
         images = resolve_deps(images[:-1], last.dependencies) + [last]
 
-    if not (last.output_dir_or_cwd() / last.output).exists() or last.output_format == OutputFormat.none:
+    if (
+        args.run_build_scripts
+        or last.output_format == OutputFormat.none
+        or not (last.output_dir_or_cwd() / last.output).exists()
+    ):
         for config in images:
             if any(
                 source.type != KeySourceType.file
@@ -5166,9 +5174,15 @@ def run_verb(args: Args, images: Sequence[Config], *, resources: Path) -> None:
                 join_new_session_keyring()
                 break
 
-        with complete_step("Validating certificates and keys"):
-            for config in images:
-                validate_certificates_and_keys(config)
+        validate = [
+            c
+            for c in images
+            if c.output_format != OutputFormat.none and not (c.output_dir_or_cwd() / c.output).exists()
+        ]
+        if validate:
+            with complete_step("Validating certificates and keys"):
+                for config in validate:
+                    validate_certificates_and_keys(config)
 
         ensure_directories_exist(last)
 
@@ -5186,27 +5200,34 @@ def run_verb(args: Args, images: Sequence[Config], *, resources: Path) -> None:
                 prefix="mkosi-packages-",
             ) as package_dir,
         ):
-            sync_repository_metadata(
-                args,
-                images,
-                resources=resources,
-                keyring_dir=Path(keyring_dir),
-                metadata_dir=Path(metadata_dir),
-            )
-
             for config in images:
                 run_sync_scripts(config)
 
+            synced = False
+
             for config in images:
-                # If the output format is "none" and there are no build scripts, there's nothing to
-                # do so exit early.
-                if config.output_format == OutputFormat.none and not config.build_scripts:
+                # If the output format is "none" or we're rebuilding and there are no build scripts, there's
+                # nothing to do so exit early.
+                if (
+                    config.output_format == OutputFormat.none
+                    or (args.run_build_scripts and (config.output_dir_or_cwd() / config.output).exists())
+                ) and not config.build_scripts:
                     continue
 
                 check_tools(config, Verb.build)
-
                 check_inputs(config)
                 ensure_directories_exist(config)
+
+                if not synced:
+                    sync_repository_metadata(
+                        args,
+                        images,
+                        resources=resources,
+                        keyring_dir=Path(keyring_dir),
+                        metadata_dir=Path(metadata_dir),
+                    )
+                    synced = True
+
                 fork_and_wait(
                     run_build,
                     args,
@@ -5217,6 +5238,9 @@ def run_verb(args: Args, images: Sequence[Config], *, resources: Path) -> None:
                     package_dir=Path(package_dir),
                 )
 
+            if not synced:
+                logging.info("All images have already been built and do not have any build scripts")
+
         if args.auto_bump:
             bump_image_version()
 

diff --git a/mkosi/config.py b/mkosi/config.py
index 82029006e8b903319a8992b9b2fadb0dde7ac423..fd04104dd43ec8428b4e9e9004cfc4e159779022 100644 (file)
--- a/mkosi/config.py
+++ b/mkosi/config.py
@@ -1676,6 +1676,7 @@ class Args:
     doc_format: DocFormat
     json: bool
     wipe_build_dir: bool
+    run_build_scripts: bool
 
     @classmethod
     def default(cls) -> "Args":
@@ -4172,6 +4173,13 @@ def create_argument_parser(chdir: bool = True) -> argparse.ArgumentParser:
         action="store_true",
         default=False,
     )
+    parser.add_argument(
+        "-R",
+        "--run-build-scripts",
+        help="Run build scripts even if the image is not rebuilt",
+        action="store_true",
+        default=False,
+    )
     # These can be removed once mkosi v15 is available in LTS distros and compatibility with <= v14
     # is no longer needed in build infrastructure (e.g.: OBS).
     parser.add_argument(

diff --git a/tests/test_json.py b/tests/test_json.py
index 7bf5ecfd1239da4244f817cfaebdeb3a43277e75..b5ff8e81f0521d821dfc9f87368c08ec403e9603 100644 (file)
--- a/tests/test_json.py
+++ b/tests/test_json.py
@@ -66,6 +66,7 @@ def test_args(path: Optional[Path]) -> None:
             "GenkeyValidDays": "100",
             "Json": false,
             "Pager": true,
+            "RunBuildScripts": true,
             "Verb": "build",
             "WipeBuildDir": true
         }}
@@ -86,6 +87,7 @@ def test_args(path: Optional[Path]) -> None:
         genkey_valid_days="100",
         json=False,
         pager=True,
+        run_build_scripts=True,
         verb=Verb.build,
         wipe_build_dir=True,
     )