From: Daan De Meyer Date: Thu, 13 Feb 2025 23:34:36 +0000 (+0100) Subject: Add BuildKey= and CacheKey= settings X-Git-Tag: v26~384^2~1 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=9b31dde1cafe5e7ce1b007cf93754355c32a8598;p=thirdparty%2Fmkosi.git Add BuildKey= and CacheKey= settings Let's give users more control over how many different cache and build subdirectories we maintain by introducing CacheKey= and BuildKey= with support for delayed specifiers. --- diff --git a/mkosi/__init__.py b/mkosi/__init__.py index 901105c38..e1f19bee2 100644 --- a/mkosi/__init__.py +++ b/mkosi/__init__.py @@ -69,6 +69,7 @@ from mkosi.config import ( Verity, Vmm, cat_config, + expand_delayed_specifiers, format_bytes, have_history, parse_boolean, @@ -839,7 +840,7 @@ def run_build_scripts(context: Context) -> None: "--bind", context.artifacts, "/work/artifacts", "--bind", context.package_dir, "/work/packages", *( - ["--bind", os.fspath(context.config.build_dir), "/work/build"] + ["--bind", os.fspath(context.config.build_subdir), "/work/build"] if context.config.build_dir else [] ), @@ -909,7 +910,7 @@ def run_postinst_scripts(context: Context) -> None: "--bind", context.artifacts, "/work/artifacts", "--bind", context.package_dir, "/work/packages", *( - ["--ro-bind", os.fspath(context.config.build_dir), "/work/build"] + ["--ro-bind", os.fspath(context.config.build_subdir), "/work/build"] if context.config.build_dir else [] ), @@ -978,7 +979,7 @@ def run_finalize_scripts(context: Context) -> None: "--bind", context.artifacts, "/work/artifacts", "--bind", context.package_dir, "/work/packages", *( - ["--ro-bind", os.fspath(context.config.build_dir), "/work/build"] + ["--ro-bind", os.fspath(context.config.build_subdir), "/work/build"] if context.config.build_dir else [] ), @@ -1298,6 +1299,7 @@ def finalize_default_initrd( *(["--output-directory", os.fspath(output_dir)] if output_dir else []), *(["--workspace-directory", os.fspath(config.workspace_dir)] if config.workspace_dir else []), *(["--cache-directory", os.fspath(config.cache_dir)] if config.cache_dir else []), + "--cache-key", config.cache_key or '-', *(["--package-cache-directory", os.fspath(config.package_cache_dir)] if config.package_cache_dir else []), # noqa: E501 *(["--local-mirror", str(config.local_mirror)] if config.local_mirror else []), "--incremental", str(config.incremental), @@ -1332,11 +1334,21 @@ def finalize_default_initrd( "--include=mkosi-initrd", ] # fmt: skip - _, [config] = parse_config(cmdline + ["build"], resources=resources) + _, [initrd] = parse_config(cmdline + ["build"], resources=resources) - run_configure_scripts(config) + run_configure_scripts(initrd) - return dataclasses.replace(config, image="default-initrd") + initrd = dataclasses.replace(initrd, image="default-initrd") + + if initrd.incremental and initrd.expand_key_specifiers(initrd.cache_key) == config.expand_key_specifiers( + config.cache_key + ): + die( + f"Image '{config.image}' and its default initrd image have the same cache key '{config.expand_key_specifiers(config.cache_key)}'", # noqa: E501 + hint="Add the &I specifier to the cache key to avoid this issue", + ) + + return initrd def build_default_initrd(context: Context) -> Path: @@ -2026,15 +2038,7 @@ def expand_kernel_specifiers(text: str, kver: str, token: str, roothash: str, bo "c": boot_count, } - def replacer(match: re.Match[str]) -> str: - m = match.group("specifier") - if specifier := specifiers.get(m): - return specifier - - logging.warning(f"Unknown specifier '&{m}' found in {text}, ignoring") - return "" - - return re.sub(r"&(?P[&a-zA-Z])", replacer, text) + return expand_delayed_specifiers(specifiers, text) def finalize_bootloader_entry_format( @@ -2563,38 +2567,22 @@ def print_output_size(path: Path) -> None: def cache_tree_paths(config: Config) -> tuple[Path, Path, Path]: - if config.image == "tools": - key = "tools" - else: - fragments = [config.distribution, config.release, config.architecture, config.image] - key = "~".join(str(s) for s in fragments) - assert config.cache_dir return ( - config.cache_dir / f"{key}.cache", - config.cache_dir / f"{key}.build.cache", - config.cache_dir / f"{key}.manifest", + config.cache_dir / f"{config.expand_key_specifiers(config.cache_key)}.cache", + config.cache_dir / f"{config.expand_key_specifiers(config.cache_key)}.build.cache", + config.cache_dir / f"{config.expand_key_specifiers(config.cache_key)}.manifest", ) def keyring_cache(config: Config) -> Path: - if config.image == "tools": - key = "tools" - else: - key = f"{'~'.join(str(s) for s in (config.distribution, config.release, config.architecture))}" - assert config.cache_dir - return config.cache_dir / f"{key}.keyring.cache" + return config.cache_dir / f"{config.expand_key_specifiers(config.cache_key)}.keyring.cache" def metadata_cache(config: Config) -> Path: - if config.image == "tools": - key = "tools" - else: - key = f"{'~'.join(str(s) for s in (config.distribution, config.release, config.architecture))}" - assert config.cache_dir - return config.cache_dir / f"{key}.metadata.cache" + return config.cache_dir / f"{config.expand_key_specifiers(config.cache_key)}.metadata.cache" def check_inputs(config: Config) -> None: @@ -4219,8 +4207,8 @@ def run_shell(args: Args, config: Config) -> None: cmdline += ["--bind", f"{src}:{dst}:norbind,{uidmap}"] if config.build_dir: - uidmap = "rootidmap" if config.build_dir.stat().st_uid != 0 else "noidmap" - cmdline += ["--bind", f"{config.build_dir}:/work/build:norbind,{uidmap}"] + uidmap = "rootidmap" if config.build_subdir.stat().st_uid != 0 else "noidmap" + cmdline += ["--bind", f"{config.build_subdir}:/work/build:norbind,{uidmap}"] for tree in config.runtime_trees: target = Path("/root/src") / (tree.target or "") @@ -4501,6 +4489,7 @@ def finalize_default_tools(config: Config, *, resources: Path) -> Config: *(["--output-directory", os.fspath(config.output_dir)] if config.output_dir else []), *(["--workspace-directory", os.fspath(config.workspace_dir)] if config.workspace_dir else []), *(["--cache-directory", os.fspath(config.cache_dir)] if config.cache_dir else []), + "--cache-key=tools", *(["--package-cache-directory", os.fspath(config.package_cache_dir)] if config.package_cache_dir else []), # noqa: E501 "--incremental", str(config.incremental), *([f"--package={package}" for package in config.tools_tree_packages]), @@ -4716,11 +4705,11 @@ def run_clean(args: Args, config: Config) -> None: if ( remove_build_cache and config.build_dir - and config.build_dir.exists() - and any(config.build_dir.iterdir()) + and config.build_subdir.exists() + and any(config.build_subdir.iterdir()) ): with complete_step(f"Clearing out build directory of {config.image} image…"): - rmtree(*config.build_dir.iterdir(), sandbox=sandbox) + rmtree(*config.build_subdir.iterdir(), sandbox=sandbox) if remove_image_cache and config.cache_dir: if config.image in ("main", "tools"): @@ -4757,11 +4746,13 @@ def ensure_directories_exist(config: Config) -> None: p.mkdir(parents=True, exist_ok=True) if config.build_dir: - st = config.build_dir.stat() + config.build_subdir.mkdir(exist_ok=True) + + st = config.build_subdir.stat() # Discard setuid/setgid bits if set as these are inherited and can leak into the image. if stat.S_IMODE(st.st_mode) & (stat.S_ISGID | stat.S_ISUID): - config.build_dir.chmod(stat.S_IMODE(st.st_mode) & ~(stat.S_ISGID | stat.S_ISUID)) + config.build_subdir.chmod(stat.S_IMODE(st.st_mode) & ~(stat.S_ISGID | stat.S_ISUID)) def sync_repository_metadata( @@ -5103,6 +5094,14 @@ def run_verb(args: Args, images: Sequence[Config], *, resources: Path) -> None: check_workspace_directory(last) + if last.incremental: + for a, b in itertools.combinations(images, 2): + if a.expand_key_specifiers(a.cache_key) == b.expand_key_specifiers(b.cache_key): + die( + f"Image {a.image} and {b.image} have the same cache key '{a.expand_key_specifiers(a.cache_key)}'", # noqa: E501 + hint="Add the &I specifier to the cache key to avoid this issue", + ) + if last.incremental == Incremental.strict: if args.force > 1: die( diff --git a/mkosi/config.py b/mkosi/config.py index 3844eca17..21675e643 100644 --- a/mkosi/config.py +++ b/mkosi/config.py @@ -592,6 +592,18 @@ class ArtifactOutput(StrEnum): ] +def expand_delayed_specifiers(specifiers: dict[str, str], text: str) -> str: + def replacer(match: re.Match[str]) -> str: + m = match.group("specifier") + if (specifier := specifiers.get(m)) is not None: + return specifier + + logging.warning(f"Unknown specifier '&{m}' found in {text}, ignoring") + return "" + + return re.sub(r"&(?P[&a-zA-Z])", replacer, text) + + def try_parse_boolean(s: str) -> Optional[bool]: "Parse 1/true/yes/y/t/on as true and 0/false/no/n/f/off/None as false" @@ -1925,8 +1937,10 @@ class Config: sandbox_trees: list[ConfigTree] workspace_dir: Optional[Path] cache_dir: Optional[Path] + cache_key: str package_cache_dir: Optional[Path] build_dir: Optional[Path] + build_key: str use_subvolumes: ConfigFeature repart_offline: bool history: bool @@ -2150,6 +2164,16 @@ class Config: self.output_tar, ] + @property + def build_subdir(self) -> Path: + assert self.build_dir + subdir = self.expand_key_specifiers(self.build_key) + + if subdir == "-": + return self.build_dir + + return self.build_dir / subdir + def cache_manifest(self) -> dict[str, Any]: return { "distribution": self.distribution, @@ -2171,8 +2195,26 @@ class Config: ), } + def expand_key_specifiers(self, key: str) -> str: + specifiers = { + "&": "&", + "d": str(self.distribution), + "r": self.release, + "a": str(self.architecture), + "i": self.image_id or "", + "v": self.image_version or "", + "I": self.image, + } + + return expand_delayed_specifiers(specifiers, key) + def to_dict(self) -> dict[str, Any]: - return dataclasses.asdict(self, dict_factory=dict_with_capitalised_keys_factory) + d = dataclasses.asdict(self, dict_factory=dict_with_capitalised_keys_factory) + + if self.build_dir: + d["BuildSubdirectory"] = self.build_subdir + + return d def to_json(self, *, indent: Optional[int] = 4, sort_keys: bool = True) -> str: """Dump Config as JSON string.""" @@ -2197,6 +2239,8 @@ class Config: return s.dest return "_".join(part.lower() for part in FALLBACK_NAME_TO_DEST_SPLITTER.split(k)) + j.pop("BuildSubdirectory", None) + for k, v in j.items(): k = key_transformer(k) @@ -3425,6 +3469,15 @@ SETTINGS: list[ConfigSetting[Any]] = [ help="Incremental cache directory", scope=SettingScope.universal, ), + ConfigSetting( + dest="cache_key", + metavar="KEY", + section="Build", + parse=config_parse_string, + help="Cache key to use within cache directory", + default="&d~&r~&a~&I", + scope=SettingScope.inherit, + ), ConfigSetting( dest="package_cache_dir", long="--package-cache-directory", @@ -3449,6 +3502,15 @@ SETTINGS: list[ConfigSetting[Any]] = [ help="Path to use as persistent build directory", scope=SettingScope.universal, ), + ConfigSetting( + dest="build_key", + metavar="KEY", + section="Build", + parse=config_parse_string, + help="Build key to use within build directory", + default="&d~&r~&a", + scope=SettingScope.inherit, + ), ConfigSetting( dest="use_subvolumes", metavar="FEATURE", @@ -4647,7 +4709,7 @@ def parse_config( setattr(config, s.dest, context.finalize_value(s)) if prev: - return args, (load_config(config),) + return args, (Config.from_namespace(config),) images = [] @@ -4729,9 +4791,9 @@ def parse_config( if dependencies is not None: setattr(config, "dependencies", dependencies) - main = load_config(config) + main = Config.from_namespace(config) - subimages = [load_config(ns) for ns in images] + subimages = [Config.from_namespace(ns) for ns in images] subimages = resolve_deps(subimages, main.dependencies) return args, tuple(subimages + [main]) @@ -4745,19 +4807,6 @@ def finalize_term() -> str: return term if sys.stderr.isatty() else "dumb" -def load_config(config: argparse.Namespace) -> Config: - # Make sure we don't modify the input namespace. - config = copy.deepcopy(config) - - if ( - config.build_dir - and config.build_dir.name != f"{config.distribution}~{config.release}~{config.architecture}" - ): - config.build_dir /= f"{config.distribution}~{config.release}~{config.architecture}" - - return Config.from_namespace(config) - - def yes_no(b: bool) -> str: return "yes" if b else "no" @@ -4990,8 +5039,10 @@ def summary(config: Config) -> str: Sandbox Trees: {line_join_list(config.sandbox_trees)} Workspace Directory: {config.workspace_dir_or_default()} Cache Directory: {none_to_none(config.cache_dir)} + Cache Key: {config.cache_key} Package Cache Directory: {none_to_default(config.package_cache_dir)} Build Directory: {none_to_none(config.build_dir)} + Build Key: {config.build_key} Use Subvolumes: {config.use_subvolumes} Repart Offline: {yes_no(config.repart_offline)} Save History: {yes_no(config.history)} diff --git a/mkosi/mounts.py b/mkosi/mounts.py index 5416d57a0..4bc571343 100644 --- a/mkosi/mounts.py +++ b/mkosi/mounts.py @@ -81,7 +81,7 @@ def finalize_source_mounts( hint="Configure BuildDirectory= or create mkosi.builddir.", ) - upperdir = config.build_dir / f"mkosi.buildovl.{src.name}" + upperdir = config.build_subdir / f"mkosi.buildovl.{src.name}" upperdir.mkdir(mode=src.stat().st_mode, exist_ok=True) else: upperdir = Path( diff --git a/mkosi/qemu.py b/mkosi/qemu.py index b69e5a408..0fedb9a98 100644 --- a/mkosi/qemu.py +++ b/mkosi/qemu.py @@ -1401,7 +1401,7 @@ def run_qemu(args: Args, config: Config) -> None: add_virtiofs_mount(sock, dst, cmdline, credentials, tag=src.name) if config.build_dir: - sock = stack.enter_context(start_virtiofsd(config, config.build_dir)) + sock = stack.enter_context(start_virtiofsd(config, config.build_subdir)) add_virtiofs_mount(sock, "/work/build", cmdline, credentials, tag="build") for tree in config.runtime_trees: diff --git a/mkosi/resources/man/mkosi.1.md b/mkosi/resources/man/mkosi.1.md index 01fd694cf..d1363bf5e 100644 --- a/mkosi/resources/man/mkosi.1.md +++ b/mkosi/resources/man/mkosi.1.md @@ -1440,6 +1440,28 @@ boolean argument: either `1`, `yes`, or `true` to enable, or `0`, `no`, found in the local directory it is automatically used for this purpose. +`CacheKey=`, `--cache-key=` +: Specifies the subdirectory within the cache directory where to store + the cached image. This may include both the regular specifiers (see + **Specifiers**) and special delayed specifiers, that are expanded + after config parsing has finished, instead of during config parsing, + which are described below. The default format for this parameter is + `&d~&r~&a~&I`. + + The following specifiers may be used: + + | Specifier | Value | + |-----------|----------------------------------------------------| + | `&&` | `&` character | + | `&d` | `Distribution=` | + | `&r` | `Release=` | + | `&a` | `Architecture=` | + | `&i` | `ImageId=` | + | `&v` | `ImageVersion=` | + | `&I` | Subimage name within mkosi.images/ or `main` | + + Note that all images within a build must have a unique cache key. + `PackageCacheDirectory=`, `--package-cache-dir=` : Takes a path to a directory to use as the package cache directory for the distribution package manager used. If unset, but a `mkosi.pkgcache/` directory is found in the local directory it is automatically @@ -1458,6 +1480,18 @@ boolean argument: either `1`, `yes`, or `true` to enable, or `0`, `no`, in the local directory it is automatically used for this purpose (also see the **Files** section below). +`BuildKey=`, `--build-key=` +: Specifies the subdirectory within the build directory where to store + incremental build artifacts. This may include both the regular + specifiers (see **Specifiers**) and special delayed specifiers, that + are expanded after config parsing has finished, instead of during + config parsing, which are the same delayed specifiers that are + supported by `CacheKey=`. The default format for this parameter is + `&d~&r~&a`. + + To disable usage of a build subdirectory completely, assign a + literal `-` to this setting. + `UseSubvolumes=`, `--use-subvolumes=` : Takes a boolean or `auto`. Enables or disables use of btrfs subvolumes for directory tree outputs. If enabled, **mkosi** will create the root directory as @@ -2786,6 +2820,8 @@ down to subimages but can be overridden: - `ImageId=` - `ImageVersion=` - `SectorSize=` +- `CacheKey=` +- `BuildKey=` Images can refer to outputs of images they depend on. Specifically, for the following options, **mkosi** will only check whether the inputs diff --git a/mkosi/vmspawn.py b/mkosi/vmspawn.py index e7e996f35..a96ff2b03 100644 --- a/mkosi/vmspawn.py +++ b/mkosi/vmspawn.py @@ -80,7 +80,7 @@ def run_vmspawn(args: Args, config: Config) -> None: cmdline += ["--bind", f"{src}:{dst}"] if config.build_dir: - cmdline += ["--bind", f"{config.build_dir}:/work/build"] + cmdline += ["--bind", f"{config.build_subdir}:/work/build"] for tree in config.runtime_trees: target = Path("/root/src") / (tree.target or "") diff --git a/tests/test_json.py b/tests/test_json.py index 0c56a4a07..a9749b914 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -105,7 +105,8 @@ def test_config() -> None: "BiosBootloader": "none", "Bootable": "disabled", "Bootloader": "grub", - "BuildDirectory": null, + "BuildDirectory": "abc", + "BuildKey": "abc", "BuildPackages": [ "pkg1", "pkg2" @@ -120,9 +121,11 @@ def test_config() -> None: } ], "BuildSourcesEphemeral": "yes", + "BuildSubdirectory": "abc/abc", "CDROM": false, "CPUs": 2, "CacheDirectory": "/is/this/the/cachedir", + "CacheKey": "qed", "CacheOnly": "always", "Checksum": false, "CleanPackageMetadata": "auto", @@ -435,12 +438,14 @@ def test_config() -> None: bios_bootloader=BiosBootloader.none, bootable=ConfigFeature.disabled, bootloader=Bootloader.grub, - build_dir=None, + build_dir=Path("abc"), + build_key="abc", build_packages=["pkg1", "pkg2"], build_scripts=[Path("/path/to/buildscript")], build_sources_ephemeral=BuildSourcesEphemeral.yes, build_sources=[ConfigTree(Path("/qux"), Path("/frob"))], cache_dir=Path("/is/this/the/cachedir"), + cache_key="qed", cacheonly=Cacheonly.always, cdrom=False, checksum=False,