From 2694f533346a5fef91048b24a969d5b97820d717 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Wed, 31 Jan 2024 14:13:37 +0100 Subject: [PATCH] Implement CacheOnly=metadata We make CacheOnly= take an enum but keep backwards compat with the boolean argument as well. CacheOnly=metadata means we'll download packages but we won't sync repository metadata. We also enable this in the kernel-install so that our built initrds use exactly the same package versions as the host system. While we're at it we rename the internal variable to cacheonly instead of cache_only (to match dnf's --cacheonly option). We keep the user facing stuff the same to not break backwards compat. We also make all of our enum functions take StrEnum as argument instead of the generic enum.Enum. --- NEWS.md | 2 ++ kernel-install/50-mkosi.install | 1 + mkosi/__init__.py | 7 ++--- mkosi/config.py | 45 +++++++++++++++++++++++++-------- mkosi/installer/dnf.py | 4 +-- mkosi/resources/mkosi.md | 11 +++++--- tests/test_json.py | 5 ++-- 7 files changed, 53 insertions(+), 22 deletions(-) diff --git a/NEWS.md b/NEWS.md index 8357ff6c9..24a11fc17 100644 --- a/NEWS.md +++ b/NEWS.md @@ -30,6 +30,8 @@ to make sure the repository metadata in `/mkosi` is not cleaned up, otherwise any extension images using this image as their base tree will not be able to install additional packages. +- Implemented `CacheOnly=metadata`. Note that in the JSON output, the + value of `CacheOnly=` will now be a string instead of a boolean. ## v20.2 diff --git a/kernel-install/50-mkosi.install b/kernel-install/50-mkosi.install index 290c9dd56..33d032892 100644 --- a/kernel-install/50-mkosi.install +++ b/kernel-install/50-mkosi.install @@ -134,6 +134,7 @@ def main() -> None: "--output", output, "--workspace-dir=/var/tmp", "--package-cache-dir=/var", + "--cache-only=metadata", "--output-dir", context.staging_area, "--extra-tree", f"/usr/lib/modules/{context.kernel_version}:/usr/lib/modules/{context.kernel_version}", "--extra-tree=/usr/lib/firmware:/usr/lib/firmware", diff --git a/mkosi/__init__.py b/mkosi/__init__.py index 582e782e3..f58fedeb8 100644 --- a/mkosi/__init__.py +++ b/mkosi/__init__.py @@ -28,6 +28,7 @@ from mkosi.config import ( Args, BiosBootloader, Bootloader, + Cacheonly, Compression, Config, ConfigFeature, @@ -1497,7 +1498,7 @@ def build_default_initrd(context: Context) -> Path: *(["--compress-output", str(context.config.compress_output)] if context.config.compress_output else []), "--compress-level", str(context.config.compress_level), "--with-network", str(context.config.with_network), - "--cache-only", str(context.config.cache_only), + "--cache-only", str(context.config.cacheonly), "--output-dir", str(context.workspace / "initrd"), *(["--workspace-dir", str(context.config.workspace_dir)] if context.config.workspace_dir else []), *(["--cache-dir", str(context.config.cache_dir)] if context.config.cache_dir else []), @@ -3479,7 +3480,7 @@ def finalize_default_tools(args: Args, config: Config, *, resources: Path) -> Co *(["--release", config.tools_tree_release] if config.tools_tree_release else []), *(["--mirror", config.tools_tree_mirror] if config.tools_tree_mirror else []), "--repository-key-check", str(config.repository_key_check), - "--cache-only", str(config.cache_only), + "--cache-only", str(config.cacheonly), *(["--output-dir", str(config.output_dir)] if config.output_dir else []), *(["--workspace-dir", str(config.workspace_dir)] if config.workspace_dir else []), *(["--cache-dir", str(config.cache_dir)] if config.cache_dir else []), @@ -3594,7 +3595,7 @@ def rchown_package_manager_dirs(config: Config) -> Iterator[None]: def sync_repository_metadata(args: Args, config: Config, *, resources: Path) -> None: - if have_cache(config) or config.cache_only or config.base_trees: + if have_cache(config) or config.cacheonly != Cacheonly.none or config.base_trees: return with ( diff --git a/mkosi/config.py b/mkosi/config.py index f8277066a..1cfbbdd45 100644 --- a/mkosi/config.py +++ b/mkosi/config.py @@ -227,6 +227,12 @@ class ShimBootloader(StrEnum): unsigned = enum.auto() +class Cacheonly(StrEnum): + always = enum.auto() + none = enum.auto() + metadata = enum.auto() + + class QemuFirmware(StrEnum): auto = enum.auto() linux = enum.auto() @@ -620,8 +626,8 @@ def config_default_kernel_command_line(namespace: argparse.Namespace) -> list[st return [f"console={namespace.architecture.default_serial_tty()}"] -def make_enum_parser(type: type[enum.Enum]) -> Callable[[str], enum.Enum]: - def parse_enum(value: str) -> enum.Enum: +def make_enum_parser(type: type[StrEnum]) -> Callable[[str], StrEnum]: + def parse_enum(value: str) -> StrEnum: try: return type(value) except ValueError: @@ -630,15 +636,28 @@ def make_enum_parser(type: type[enum.Enum]) -> Callable[[str], enum.Enum]: return parse_enum -def config_make_enum_parser(type: type[enum.Enum]) -> ConfigParseCallback: - def config_parse_enum(value: Optional[str], old: Optional[enum.Enum]) -> Optional[enum.Enum]: +def config_make_enum_parser(type: type[StrEnum]) -> ConfigParseCallback: + def config_parse_enum(value: Optional[str], old: Optional[StrEnum]) -> Optional[StrEnum]: return make_enum_parser(type)(value) if value else None return config_parse_enum -def config_make_enum_matcher(type: type[enum.Enum]) -> ConfigMatchCallback: - def config_match_enum(match: str, value: enum.Enum) -> bool: +def config_make_enum_parser_with_boolean(type: type[StrEnum], *, yes: StrEnum, no: StrEnum) -> ConfigParseCallback: + def config_parse_enum(value: Optional[str], old: Optional[StrEnum]) -> Optional[StrEnum]: + if not value: + return None + + if value in type.values(): + return type(value) + + return yes if parse_boolean(value) else no + + return config_parse_enum + + +def config_make_enum_matcher(type: type[StrEnum]) -> ConfigMatchCallback: + def config_match_enum(match: str, value: StrEnum) -> bool: return make_enum_parser(type)(match) == value return config_match_enum @@ -1160,7 +1179,7 @@ class Config: local_mirror: Optional[str] repository_key_check: bool repositories: list[str] - cache_only: bool + cacheonly: Cacheonly package_manager_trees: list[ConfigTree] output_format: OutputFormat @@ -1638,10 +1657,13 @@ SETTINGS = ( help="Repositories to use", ), ConfigSetting( - dest="cache_only", - metavar="BOOL", + dest="cacheonly", + long="--cache-only", + name="CacheOnly", + metavar="CACHEONLY", section="Distribution", - parse=config_parse_boolean, + parse=config_make_enum_parser_with_boolean(Cacheonly, yes=Cacheonly.always, no=Cacheonly.none), + default=Cacheonly.none, help="Only use the package cache when installing packages", ), ConfigSetting( @@ -3423,7 +3445,7 @@ def summary(config: Config) -> str: Local Mirror (build): {none_to_none(config.local_mirror)} Repo Signature/Key check: {yes_no(config.repository_key_check)} Repositories: {line_join_list(config.repositories)} - Use Only Package Cache: {yes_no(config.cache_only)} + Use Only Package Cache: {config.cacheonly} Package Manager Trees: {line_join_tree_list(config.package_manager_trees)} {bold("OUTPUT")}: @@ -3676,6 +3698,7 @@ def json_type_transformer(refcls: Union[type[Args], type[Config]]) -> Callable[[ DocFormat: enum_transformer, list[QemuDrive]: config_drive_transformer, GenericVersion: generic_version_transformer, + Cacheonly: enum_transformer, } def json_transformer(key: str, val: Any) -> Any: diff --git a/mkosi/installer/dnf.py b/mkosi/installer/dnf.py index 9cf9ad735..6d4c6f2d8 100644 --- a/mkosi/installer/dnf.py +++ b/mkosi/installer/dnf.py @@ -3,7 +3,7 @@ import textwrap from collections.abc import Iterable, Sequence from pathlib import Path -from mkosi.config import Config +from mkosi.config import Cacheonly, Config from mkosi.context import Context from mkosi.installer import PackageManager from mkosi.installer.rpm import RpmRepository, fixup_rpmdb_location, rpm_cmd, setup_rpm @@ -122,7 +122,7 @@ class Dnf(PackageManager): opt = "--enable-repo" if dnf.endswith("dnf5") else "--enablerepo" cmdline += [f"{opt}={repo}" for repo in context.config.repositories] - if context.config.cache_only: + if context.config.cacheonly == Cacheonly.always: cmdline += ["--cacheonly"] else: cmdline += ["--setopt=metadata_expire=never"] diff --git a/mkosi/resources/mkosi.md b/mkosi/resources/mkosi.md index 5476228b3..87405097e 100644 --- a/mkosi/resources/mkosi.md +++ b/mkosi/resources/mkosi.md @@ -589,10 +589,13 @@ boolean argument: either `1`, `yes`, or `true` to enable, or `0`, `no`, `CacheOnly=`, `--cache-only=` -: If specified, the package manager is instructed not to contact the - network for updating package data. This provides a minimal level of - reproducibility, as long as the package cache is already fully - populated. +: Takes one of `none`, `metadata` or `always`. If `always`, the package + manager is instructed not to contact the network. This provides a + minimal level of reproducibility, as long as the package cache is + already fully populated. If set to `metadata`, the package manager can + still download packages, but we won't sync the repository metadata. If + set to `none`, the repository metadata is synced and packages can be + downloaded during the build. `PackageManagerTrees=`, `--package-manager-tree=` diff --git a/tests/test_json.py b/tests/test_json.py index aede03b29..d6da93f61 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -13,6 +13,7 @@ from mkosi.config import ( Args, BiosBootloader, Bootloader, + Cacheonly, Compression, Config, ConfigFeature, @@ -105,7 +106,7 @@ def test_config() -> None: ], "BuildSourcesEphemeral": true, "CacheDirectory": "/is/this/the/cachedir", - "CacheOnly": true, + "CacheOnly": "always", "Checksum": false, "CleanPackageMetadata": "auto", "CompressLevel": 3, @@ -309,7 +310,7 @@ def test_config() -> None: build_sources = [ConfigTree(Path("/qux"), Path("/frob"))], build_sources_ephemeral = True, cache_dir = Path("/is/this/the/cachedir"), - cache_only = True, + cacheonly = Cacheonly.always, checksum = False, clean_package_metadata = ConfigFeature.auto, compress_level = 3, -- 2.47.2