Add BuildKey= and CacheKey= settings

Let's give users more control over how many different cache and
build subdirectories we maintain by introducing CacheKey= and
BuildKey= with support for delayed specifiers.

diff --git a/mkosi/__init__.py b/mkosi/__init__.py
index 901105c3857c86f901aa4c1d1eb6b7adfc208554..e1f19bee279168131a0c374536dac936651f25c7 100644 (file)
--- a/mkosi/__init__.py
+++ b/mkosi/__init__.py
@@ -69,6 +69,7 @@ from mkosi.config import (
     Verity,
     Vmm,
     cat_config,
+    expand_delayed_specifiers,
     format_bytes,
     have_history,
     parse_boolean,
@@ -839,7 +840,7 @@ def run_build_scripts(context: Context) -> None:
                     "--bind", context.artifacts, "/work/artifacts",
                     "--bind", context.package_dir, "/work/packages",
                     *(
-                        ["--bind", os.fspath(context.config.build_dir), "/work/build"]
+                        ["--bind", os.fspath(context.config.build_subdir), "/work/build"]
                         if context.config.build_dir
                         else []
                     ),
@@ -909,7 +910,7 @@ def run_postinst_scripts(context: Context) -> None:
                     "--bind", context.artifacts, "/work/artifacts",
                     "--bind", context.package_dir, "/work/packages",
                     *(
-                        ["--ro-bind", os.fspath(context.config.build_dir), "/work/build"]
+                        ["--ro-bind", os.fspath(context.config.build_subdir), "/work/build"]
                         if context.config.build_dir
                         else []
                     ),
@@ -978,7 +979,7 @@ def run_finalize_scripts(context: Context) -> None:
                     "--bind", context.artifacts, "/work/artifacts",
                     "--bind", context.package_dir, "/work/packages",
                     *(
-                        ["--ro-bind", os.fspath(context.config.build_dir), "/work/build"]
+                        ["--ro-bind", os.fspath(context.config.build_subdir), "/work/build"]
                         if context.config.build_dir
                         else []
                     ),
@@ -1298,6 +1299,7 @@ def finalize_default_initrd(
         *(["--output-directory", os.fspath(output_dir)] if output_dir else []),
         *(["--workspace-directory", os.fspath(config.workspace_dir)] if config.workspace_dir else []),
         *(["--cache-directory", os.fspath(config.cache_dir)] if config.cache_dir else []),
+        "--cache-key", config.cache_key or '-',
         *(["--package-cache-directory", os.fspath(config.package_cache_dir)] if config.package_cache_dir else []),  # noqa: E501
         *(["--local-mirror", str(config.local_mirror)] if config.local_mirror else []),
         "--incremental", str(config.incremental),
@@ -1332,11 +1334,21 @@ def finalize_default_initrd(
         "--include=mkosi-initrd",
     ]  # fmt: skip
 
-    _, [config] = parse_config(cmdline + ["build"], resources=resources)
+    _, [initrd] = parse_config(cmdline + ["build"], resources=resources)
 
-    run_configure_scripts(config)
+    run_configure_scripts(initrd)
 
-    return dataclasses.replace(config, image="default-initrd")
+    initrd = dataclasses.replace(initrd, image="default-initrd")
+
+    if initrd.incremental and initrd.expand_key_specifiers(initrd.cache_key) == config.expand_key_specifiers(
+        config.cache_key
+    ):
+        die(
+            f"Image '{config.image}' and its default initrd image have the same cache key '{config.expand_key_specifiers(config.cache_key)}'",  # noqa: E501
+            hint="Add the &I specifier to the cache key to avoid this issue",
+        )
+
+    return initrd
 
 
 def build_default_initrd(context: Context) -> Path:
@@ -2026,15 +2038,7 @@ def expand_kernel_specifiers(text: str, kver: str, token: str, roothash: str, bo
         "c": boot_count,
     }
 
-    def replacer(match: re.Match[str]) -> str:
-        m = match.group("specifier")
-        if specifier := specifiers.get(m):
-            return specifier
-
-        logging.warning(f"Unknown specifier '&{m}' found in {text}, ignoring")
-        return ""
-
-    return re.sub(r"&(?P<specifier>[&a-zA-Z])", replacer, text)
+    return expand_delayed_specifiers(specifiers, text)
 
 
 def finalize_bootloader_entry_format(
@@ -2563,38 +2567,22 @@ def print_output_size(path: Path) -> None:
 
 
 def cache_tree_paths(config: Config) -> tuple[Path, Path, Path]:
-    if config.image == "tools":
-        key = "tools"
-    else:
-        fragments = [config.distribution, config.release, config.architecture, config.image]
-        key = "~".join(str(s) for s in fragments)
-
     assert config.cache_dir
     return (
-        config.cache_dir / f"{key}.cache",
-        config.cache_dir / f"{key}.build.cache",
-        config.cache_dir / f"{key}.manifest",
+        config.cache_dir / f"{config.expand_key_specifiers(config.cache_key)}.cache",
+        config.cache_dir / f"{config.expand_key_specifiers(config.cache_key)}.build.cache",
+        config.cache_dir / f"{config.expand_key_specifiers(config.cache_key)}.manifest",
     )
 
 
 def keyring_cache(config: Config) -> Path:
-    if config.image == "tools":
-        key = "tools"
-    else:
-        key = f"{'~'.join(str(s) for s in (config.distribution, config.release, config.architecture))}"
-
     assert config.cache_dir
-    return config.cache_dir / f"{key}.keyring.cache"
+    return config.cache_dir / f"{config.expand_key_specifiers(config.cache_key)}.keyring.cache"
 
 
 def metadata_cache(config: Config) -> Path:
-    if config.image == "tools":
-        key = "tools"
-    else:
-        key = f"{'~'.join(str(s) for s in (config.distribution, config.release, config.architecture))}"
-
     assert config.cache_dir
-    return config.cache_dir / f"{key}.metadata.cache"
+    return config.cache_dir / f"{config.expand_key_specifiers(config.cache_key)}.metadata.cache"
 
 
 def check_inputs(config: Config) -> None:
@@ -4219,8 +4207,8 @@ def run_shell(args: Args, config: Config) -> None:
                 cmdline += ["--bind", f"{src}:{dst}:norbind,{uidmap}"]
 
             if config.build_dir:
-                uidmap = "rootidmap" if config.build_dir.stat().st_uid != 0 else "noidmap"
-                cmdline += ["--bind", f"{config.build_dir}:/work/build:norbind,{uidmap}"]
+                uidmap = "rootidmap" if config.build_subdir.stat().st_uid != 0 else "noidmap"
+                cmdline += ["--bind", f"{config.build_subdir}:/work/build:norbind,{uidmap}"]
 
         for tree in config.runtime_trees:
             target = Path("/root/src") / (tree.target or "")
@@ -4501,6 +4489,7 @@ def finalize_default_tools(config: Config, *, resources: Path) -> Config:
         *(["--output-directory", os.fspath(config.output_dir)] if config.output_dir else []),
         *(["--workspace-directory", os.fspath(config.workspace_dir)] if config.workspace_dir else []),
         *(["--cache-directory", os.fspath(config.cache_dir)] if config.cache_dir else []),
+        "--cache-key=tools",
         *(["--package-cache-directory", os.fspath(config.package_cache_dir)] if config.package_cache_dir else []),  # noqa: E501
         "--incremental", str(config.incremental),
         *([f"--package={package}" for package in config.tools_tree_packages]),
@@ -4716,11 +4705,11 @@ def run_clean(args: Args, config: Config) -> None:
     if (
         remove_build_cache
         and config.build_dir
-        and config.build_dir.exists()
-        and any(config.build_dir.iterdir())
+        and config.build_subdir.exists()
+        and any(config.build_subdir.iterdir())
     ):
         with complete_step(f"Clearing out build directory of {config.image} image…"):
-            rmtree(*config.build_dir.iterdir(), sandbox=sandbox)
+            rmtree(*config.build_subdir.iterdir(), sandbox=sandbox)
 
     if remove_image_cache and config.cache_dir:
         if config.image in ("main", "tools"):
@@ -4757,11 +4746,13 @@ def ensure_directories_exist(config: Config) -> None:
         p.mkdir(parents=True, exist_ok=True)
 
     if config.build_dir:
-        st = config.build_dir.stat()
+        config.build_subdir.mkdir(exist_ok=True)
+
+        st = config.build_subdir.stat()
 
         # Discard setuid/setgid bits if set as these are inherited and can leak into the image.
         if stat.S_IMODE(st.st_mode) & (stat.S_ISGID | stat.S_ISUID):
-            config.build_dir.chmod(stat.S_IMODE(st.st_mode) & ~(stat.S_ISGID | stat.S_ISUID))
+            config.build_subdir.chmod(stat.S_IMODE(st.st_mode) & ~(stat.S_ISGID | stat.S_ISUID))
 
 
 def sync_repository_metadata(
@@ -5103,6 +5094,14 @@ def run_verb(args: Args, images: Sequence[Config], *, resources: Path) -> None:
 
         check_workspace_directory(last)
 
+        if last.incremental:
+            for a, b in itertools.combinations(images, 2):
+                if a.expand_key_specifiers(a.cache_key) == b.expand_key_specifiers(b.cache_key):
+                    die(
+                        f"Image {a.image} and {b.image} have the same cache key '{a.expand_key_specifiers(a.cache_key)}'",  # noqa: E501
+                        hint="Add the &I specifier to the cache key to avoid this issue",
+                    )
+
         if last.incremental == Incremental.strict:
             if args.force > 1:
                 die(

diff --git a/mkosi/config.py b/mkosi/config.py
index 3844eca1726089ba79e2abb082bfb4dd83bf07be..21675e6431ca1571a4b4a20b7911d352edd51505 100644 (file)
--- a/mkosi/config.py
+++ b/mkosi/config.py
@@ -592,6 +592,18 @@ class ArtifactOutput(StrEnum):
         ]
 
 
+def expand_delayed_specifiers(specifiers: dict[str, str], text: str) -> str:
+    def replacer(match: re.Match[str]) -> str:
+        m = match.group("specifier")
+        if (specifier := specifiers.get(m)) is not None:
+            return specifier
+
+        logging.warning(f"Unknown specifier '&{m}' found in {text}, ignoring")
+        return ""
+
+    return re.sub(r"&(?P<specifier>[&a-zA-Z])", replacer, text)
+
+
 def try_parse_boolean(s: str) -> Optional[bool]:
     "Parse 1/true/yes/y/t/on as true and 0/false/no/n/f/off/None as false"
 
@@ -1925,8 +1937,10 @@ class Config:
     sandbox_trees: list[ConfigTree]
     workspace_dir: Optional[Path]
     cache_dir: Optional[Path]
+    cache_key: str
     package_cache_dir: Optional[Path]
     build_dir: Optional[Path]
+    build_key: str
     use_subvolumes: ConfigFeature
     repart_offline: bool
     history: bool
@@ -2150,6 +2164,16 @@ class Config:
             self.output_tar,
         ]
 
+    @property
+    def build_subdir(self) -> Path:
+        assert self.build_dir
+        subdir = self.expand_key_specifiers(self.build_key)
+
+        if subdir == "-":
+            return self.build_dir
+
+        return self.build_dir / subdir
+
     def cache_manifest(self) -> dict[str, Any]:
         return {
             "distribution": self.distribution,
@@ -2171,8 +2195,26 @@ class Config:
             ),
         }
 
+    def expand_key_specifiers(self, key: str) -> str:
+        specifiers = {
+            "&": "&",
+            "d": str(self.distribution),
+            "r": self.release,
+            "a": str(self.architecture),
+            "i": self.image_id or "",
+            "v": self.image_version or "",
+            "I": self.image,
+        }
+
+        return expand_delayed_specifiers(specifiers, key)
+
     def to_dict(self) -> dict[str, Any]:
-        return dataclasses.asdict(self, dict_factory=dict_with_capitalised_keys_factory)
+        d = dataclasses.asdict(self, dict_factory=dict_with_capitalised_keys_factory)
+
+        if self.build_dir:
+            d["BuildSubdirectory"] = self.build_subdir
+
+        return d
 
     def to_json(self, *, indent: Optional[int] = 4, sort_keys: bool = True) -> str:
         """Dump Config as JSON string."""
@@ -2197,6 +2239,8 @@ class Config:
                 return s.dest
             return "_".join(part.lower() for part in FALLBACK_NAME_TO_DEST_SPLITTER.split(k))
 
+        j.pop("BuildSubdirectory", None)
+
         for k, v in j.items():
             k = key_transformer(k)
 
@@ -3425,6 +3469,15 @@ SETTINGS: list[ConfigSetting[Any]] = [
         help="Incremental cache directory",
         scope=SettingScope.universal,
     ),
+    ConfigSetting(
+        dest="cache_key",
+        metavar="KEY",
+        section="Build",
+        parse=config_parse_string,
+        help="Cache key to use within cache directory",
+        default="&d~&r~&a~&I",
+        scope=SettingScope.inherit,
+    ),
     ConfigSetting(
         dest="package_cache_dir",
         long="--package-cache-directory",
@@ -3449,6 +3502,15 @@ SETTINGS: list[ConfigSetting[Any]] = [
         help="Path to use as persistent build directory",
         scope=SettingScope.universal,
     ),
+    ConfigSetting(
+        dest="build_key",
+        metavar="KEY",
+        section="Build",
+        parse=config_parse_string,
+        help="Build key to use within build directory",
+        default="&d~&r~&a",
+        scope=SettingScope.inherit,
+    ),
     ConfigSetting(
         dest="use_subvolumes",
         metavar="FEATURE",
@@ -4647,7 +4709,7 @@ def parse_config(
         setattr(config, s.dest, context.finalize_value(s))
 
     if prev:
-        return args, (load_config(config),)
+        return args, (Config.from_namespace(config),)
 
     images = []
 
@@ -4729,9 +4791,9 @@ def parse_config(
     if dependencies is not None:
         setattr(config, "dependencies", dependencies)
 
-    main = load_config(config)
+    main = Config.from_namespace(config)
 
-    subimages = [load_config(ns) for ns in images]
+    subimages = [Config.from_namespace(ns) for ns in images]
     subimages = resolve_deps(subimages, main.dependencies)
 
     return args, tuple(subimages + [main])
@@ -4745,19 +4807,6 @@ def finalize_term() -> str:
     return term if sys.stderr.isatty() else "dumb"
 
 
-def load_config(config: argparse.Namespace) -> Config:
-    # Make sure we don't modify the input namespace.
-    config = copy.deepcopy(config)
-
-    if (
-        config.build_dir
-        and config.build_dir.name != f"{config.distribution}~{config.release}~{config.architecture}"
-    ):
-        config.build_dir /= f"{config.distribution}~{config.release}~{config.architecture}"
-
-    return Config.from_namespace(config)
-
-
 def yes_no(b: bool) -> str:
     return "yes" if b else "no"
 
@@ -4990,8 +5039,10 @@ def summary(config: Config) -> str:
                       Sandbox Trees: {line_join_list(config.sandbox_trees)}
                 Workspace Directory: {config.workspace_dir_or_default()}
                     Cache Directory: {none_to_none(config.cache_dir)}
+                          Cache Key: {config.cache_key}
             Package Cache Directory: {none_to_default(config.package_cache_dir)}
                     Build Directory: {none_to_none(config.build_dir)}
+                          Build Key: {config.build_key}
                      Use Subvolumes: {config.use_subvolumes}
                      Repart Offline: {yes_no(config.repart_offline)}
                        Save History: {yes_no(config.history)}

diff --git a/mkosi/mounts.py b/mkosi/mounts.py
index 5416d57a00c67ed6f07657ea9d81af4fb0d474dc..4bc571343999336ab6a7e738d6568605f4e8a512 100644 (file)
--- a/mkosi/mounts.py
+++ b/mkosi/mounts.py
@@ -81,7 +81,7 @@ def finalize_source_mounts(
                             hint="Configure BuildDirectory= or create mkosi.builddir.",
                         )
 
-                    upperdir = config.build_dir / f"mkosi.buildovl.{src.name}"
+                    upperdir = config.build_subdir / f"mkosi.buildovl.{src.name}"
                     upperdir.mkdir(mode=src.stat().st_mode, exist_ok=True)
                 else:
                     upperdir = Path(

diff --git a/mkosi/qemu.py b/mkosi/qemu.py
index b69e5a408e77520e3b4a80094038a0766b6b0903..0fedb9a9898d2d2fd7568c5cd2ad01f01e07493b 100644 (file)
--- a/mkosi/qemu.py
+++ b/mkosi/qemu.py
@@ -1401,7 +1401,7 @@ def run_qemu(args: Args, config: Config) -> None:
                 add_virtiofs_mount(sock, dst, cmdline, credentials, tag=src.name)
 
             if config.build_dir:
-                sock = stack.enter_context(start_virtiofsd(config, config.build_dir))
+                sock = stack.enter_context(start_virtiofsd(config, config.build_subdir))
                 add_virtiofs_mount(sock, "/work/build", cmdline, credentials, tag="build")
 
         for tree in config.runtime_trees:

diff --git a/mkosi/resources/man/mkosi.1.md b/mkosi/resources/man/mkosi.1.md
index 01fd694cfefe484df709f49d90b9b13d8299b326..d1363bf5e86f55edc53c1fe7c1c44c6a502fe4da 100644 (file)
--- a/mkosi/resources/man/mkosi.1.md
+++ b/mkosi/resources/man/mkosi.1.md
@@ -1440,6 +1440,28 @@ boolean argument: either `1`, `yes`, or `true` to enable, or `0`, `no`,
     found in the local directory it is automatically used for this
     purpose.
 
+`CacheKey=`, `--cache-key=`
+:   Specifies the subdirectory within the cache directory where to store
+    the cached image. This may include both the regular specifiers (see
+    **Specifiers**) and special delayed specifiers, that are expanded
+    after config parsing has finished, instead of during config parsing,
+    which are described below. The default format for this parameter is
+    `&d~&r~&a~&I`.
+
+    The following specifiers may be used:
+
+    | Specifier | Value                                              |
+    |-----------|----------------------------------------------------|
+    | `&&`      | `&` character                                      |
+    | `&d`      | `Distribution=`                                    |
+    | `&r`      | `Release=`                                         |
+    | `&a`      | `Architecture=`                                    |
+    | `&i`      | `ImageId=`                                         |
+    | `&v`      | `ImageVersion=`                                    |
+    | `&I`      | Subimage name within mkosi.images/ or `main`       |
+
+    Note that all images within a build must have a unique cache key.
+
 `PackageCacheDirectory=`, `--package-cache-dir=`
 :   Takes a path to a directory to use as the package cache directory for the distribution package manager
     used. If unset, but a `mkosi.pkgcache/` directory is found in the local directory it is automatically
@@ -1458,6 +1480,18 @@ boolean argument: either `1`, `yes`, or `true` to enable, or `0`, `no`,
     in the local directory it is automatically used for this purpose (also
     see the **Files** section below).
 
+`BuildKey=`, `--build-key=`
+:   Specifies the subdirectory within the build directory where to store
+    incremental build artifacts. This may include both the regular
+    specifiers (see **Specifiers**) and special delayed specifiers, that
+    are expanded after config parsing has finished, instead of during
+    config parsing, which are the same delayed specifiers that are
+    supported by `CacheKey=`. The default format for this parameter is
+    `&d~&r~&a`.
+
+    To disable usage of a build subdirectory completely, assign a
+    literal `-` to this setting.
+
 `UseSubvolumes=`, `--use-subvolumes=`
 :   Takes a boolean or `auto`. Enables or disables use of btrfs subvolumes for
     directory tree outputs. If enabled, **mkosi** will create the root directory as
@@ -2786,6 +2820,8 @@ down to subimages but can be overridden:
 - `ImageId=`
 - `ImageVersion=`
 - `SectorSize=`
+- `CacheKey=`
+- `BuildKey=`
 
 Images can refer to outputs of images they depend on. Specifically,
 for the following options, **mkosi** will only check whether the inputs

diff --git a/mkosi/vmspawn.py b/mkosi/vmspawn.py
index e7e996f353a87653000d32727450ac97fedc0c3e..a96ff2b0347adca0d2763abb964e4a638823e9c2 100644 (file)
--- a/mkosi/vmspawn.py
+++ b/mkosi/vmspawn.py
@@ -80,7 +80,7 @@ def run_vmspawn(args: Args, config: Config) -> None:
                 cmdline += ["--bind", f"{src}:{dst}"]
 
             if config.build_dir:
-                cmdline += ["--bind", f"{config.build_dir}:/work/build"]
+                cmdline += ["--bind", f"{config.build_subdir}:/work/build"]
 
         for tree in config.runtime_trees:
             target = Path("/root/src") / (tree.target or "")

diff --git a/tests/test_json.py b/tests/test_json.py
index 0c56a4a07c65a16b5523aa23c972770a077a25b7..a9749b914b40f44bbd2a72662b882bfd9cbc4feb 100644 (file)
--- a/tests/test_json.py
+++ b/tests/test_json.py
@@ -105,7 +105,8 @@ def test_config() -> None:
             "BiosBootloader": "none",
             "Bootable": "disabled",
             "Bootloader": "grub",
-            "BuildDirectory": null,
+            "BuildDirectory": "abc",
+            "BuildKey": "abc",
             "BuildPackages": [
                 "pkg1",
                 "pkg2"
@@ -120,9 +121,11 @@ def test_config() -> None:
                 }
             ],
             "BuildSourcesEphemeral": "yes",
+            "BuildSubdirectory": "abc/abc",
             "CDROM": false,
             "CPUs": 2,
             "CacheDirectory": "/is/this/the/cachedir",
+            "CacheKey": "qed",
             "CacheOnly": "always",
             "Checksum": false,
             "CleanPackageMetadata": "auto",
@@ -435,12 +438,14 @@ def test_config() -> None:
         bios_bootloader=BiosBootloader.none,
         bootable=ConfigFeature.disabled,
         bootloader=Bootloader.grub,
-        build_dir=None,
+        build_dir=Path("abc"),
+        build_key="abc",
         build_packages=["pkg1", "pkg2"],
         build_scripts=[Path("/path/to/buildscript")],
         build_sources_ephemeral=BuildSourcesEphemeral.yes,
         build_sources=[ConfigTree(Path("/qux"), Path("/frob"))],
         cache_dir=Path("/is/this/the/cachedir"),
+        cache_key="qed",
         cacheonly=Cacheonly.always,
         cdrom=False,
         checksum=False,