From 04fa0b962a2af5410fe454ef3ac4b4cadf54991c Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Sat, 21 Sep 2024 18:07:52 +0200 Subject: [PATCH] Add Incremental=strict This ensures cached images already exist and fails the build if they don't exist yet. --- mkosi/__init__.py | 23 +++++++++++++++++++++++ mkosi/config.py | 19 +++++++++++++++---- mkosi/resources/man/mkosi.md | 24 ++++++++++++++---------- tests/test_json.py | 5 +++-- 4 files changed, 55 insertions(+), 16 deletions(-) diff --git a/mkosi/__init__.py b/mkosi/__init__.py index 4739ce649..cf9886e86 100644 --- a/mkosi/__init__.py +++ b/mkosi/__init__.py @@ -56,6 +56,7 @@ from mkosi.config import ( Config, ConfigFeature, DocFormat, + Incremental, JsonEncoder, KeySourceType, ManifestFormat, @@ -4243,6 +4244,22 @@ def run_verb(args: Args, images: Sequence[Config], *, resources: Path) -> None: check_workspace_directory(last) + if last.incremental == Incremental.strict: + if args.force > 1: + die( + "Cannot remove incremental caches when building with Incremental=strict", + hint="Build once with -i yes to update the image cache", + ) + + for config in images: + if have_cache(config): + continue + + die( + f"Strict incremental mode is enabled but the cache for image {config.name()} is out-of-date", + hint="Build once with -i yes to update the image cache", + ) + # If we're doing an incremental build and the cache is not out of date, don't clean up the # tools tree so that we can reuse the previous one. if tools and ( @@ -4250,6 +4267,12 @@ def run_verb(args: Args, images: Sequence[Config], *, resources: Path) -> None: or ((args.verb == Verb.build or args.force > 0) and not have_cache(tools)) or needs_build(args, tools, force=2) ): + if tools.incremental == Incremental.strict: + die( + "Tools tree does not exist or is out-of-date but the strict incremental mode is enabled", + hint="Build once with -i yes to update the tools tree", + ) + run_clean(args, tools, resources=resources) # First, process all directory removals because otherwise if different images share directories diff --git a/mkosi/config.py b/mkosi/config.py index 18c9e7696..277aae5f6 100644 --- a/mkosi/config.py +++ b/mkosi/config.py @@ -306,6 +306,15 @@ class Vmm(StrEnum): vmspawn = enum.auto() +class Incremental(StrEnum): + yes = enum.auto() + no = enum.auto() + strict = enum.auto() + + def __bool__(self) -> bool: + return self != Incremental.no + + class Architecture(StrEnum): alpha = enum.auto() arc = enum.auto() @@ -1599,7 +1608,7 @@ class Config: tools_tree_sandbox_trees: list[ConfigTree] tools_tree_packages: list[str] tools_tree_certificates: bool - incremental: bool + incremental: Incremental cacheonly: Cacheonly sandbox_trees: list[ConfigTree] workspace_dir: Optional[Path] @@ -2882,12 +2891,13 @@ SETTINGS = ( ConfigSetting( dest="incremental", short="-i", - metavar="BOOL", nargs="?", section="Build", - parse=config_parse_boolean, + parse=config_make_enum_parser_with_boolean(Incremental, yes=Incremental.yes, no=Incremental.no), + default=Incremental.no, help="Make use of and generate intermediary cache images", scope=SettingScope.universal, + choices=Incremental.values(), ), ConfigSetting( dest="cacheonly", @@ -4478,7 +4488,7 @@ def summary(config: Config) -> str: Tools Tree Packages: {line_join_list(config.tools_tree_packages)} Tools Tree Certificates: {yes_no(config.tools_tree_certificates)} - Incremental: {yes_no(config.incremental)} + Incremental: {config.incremental} Use Only Package Cache: {config.cacheonly} Sandbox Trees: {line_join_list(config.sandbox_trees)} Workspace Directory: {config.workspace_dir_or_default()} @@ -4659,6 +4669,7 @@ def json_type_transformer(refcls: Union[type[Args], type[Config]]) -> Callable[[ OutputFormat: enum_transformer, QemuFirmware: enum_transformer, SecureBootSignTool: enum_transformer, + Incremental: enum_transformer, Optional[Distribution]: optional_enum_transformer, list[ManifestFormat]: enum_list_transformer, Verb: enum_transformer, diff --git a/mkosi/resources/man/mkosi.md b/mkosi/resources/man/mkosi.md index 14b9e945f..d164e3e82 100644 --- a/mkosi/resources/man/mkosi.md +++ b/mkosi/resources/man/mkosi.md @@ -1309,16 +1309,20 @@ boolean argument: either `1`, `yes`, or `true` to enable, or `0`, `no`, Otherwise, these directories are picked up from the host. `Incremental=`, `--incremental=`, `-i` -: Enable incremental build mode. In this mode, a copy of the OS image is - created immediately after all OS packages are installed and the - prepare scripts have executed but before the `mkosi.build` scripts are - invoked (or anything that happens after it). On subsequent invocations - of `mkosi` with the `-i` switch this cached image may be used to skip - the OS package installation, thus drastically speeding up repetitive - build times. Note that while there is some rudimentary cache - invalidation, it is definitely not perfect. In order to force - rebuilding of the cached image, combine `-i` with `-ff` to ensure the - cached image is first removed and then re-created. +: Takes either `strict` or a boolean value as its argument. Enables + incremental build mode. In this mode, a copy of the OS image is created + immediately after all OS packages are installed and the prepare scripts + have executed but before the `mkosi.build` scripts are invoked (or + anything that happens after it). On subsequent invocations of `mkosi` + with the `-i` switch this cached image may be used to skip the OS package + installation, thus drastically speeding up repetitive build times. Note + that while there is some rudimentary cache invalidation, it is definitely + not perfect. In order to force a rebuild of the cached image, combine + `-i` with `-ff` to ensure the cached image is first removed and then + re-created. + + If set to `strict`, the build fails if previously built cached image does + not exist. `CacheOnly=`, `--cache-only=` : Takes one of `auto`, `metadata`, `always` or `never`. Defaults to diff --git a/tests/test_json.py b/tests/test_json.py index 4dad9a9f2..8b56e7334 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -19,6 +19,7 @@ from mkosi.config import ( ConfigFeature, ConfigTree, DocFormat, + Incremental, KeySource, KeySourceType, ManifestFormat, @@ -147,7 +148,7 @@ def test_config() -> None: "Image": "default", "ImageId": "myimage", "ImageVersion": "5", - "Incremental": false, + "Incremental": "no", "InitrdPackages": [ "clevis" ], @@ -404,7 +405,7 @@ def test_config() -> None: image="default", image_id="myimage", image_version="5", - incremental=False, + incremental=Incremental.no, initrd_packages=["clevis"], initrd_volatile_packages=["abc"], initrds=[Path("/efi/initrd1"), Path("/efi/initrd2")], -- 2.47.2