From: Daan De Meyer Date: Fri, 26 Jan 2024 22:05:06 +0000 (+0100) Subject: Use shared package cache directory by default X-Git-Tag: v21~79^2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=refs%2Fpull%2F2333%2Fhead;p=thirdparty%2Fmkosi.git Use shared package cache directory by default Instead of having a separate package cache for each mkosi project, let's default to having a shared package cache directory that can also be configured separately from the incremental cache directory. --- diff --git a/mkosi/__init__.py b/mkosi/__init__.py index 794d6011d..79dadbd1b 100644 --- a/mkosi/__init__.py +++ b/mkosi/__init__.py @@ -52,6 +52,7 @@ from mkosi.installer import ( clean_package_manager_metadata, finalize_package_manager_mounts, package_manager_scripts, + rchown_package_manager_cache_dirs, ) from mkosi.kmod import gen_required_kernel_modules, process_kernel_modules from mkosi.log import ARG_DEBUG, complete_step, die, log_notice, log_step @@ -1486,7 +1487,8 @@ def build_initrd(context: Context) -> Path: "--cache-only", str(context.config.cache_only), "--output-dir", str(context.workspace / "initrd"), *(["--workspace-dir", str(context.config.workspace_dir)] if context.config.workspace_dir else []), - "--cache-dir", str(context.cache_dir), + *(["--cache-dir", str(context.config.cache_dir)] if context.config.cache_dir else []), + *(["--package-cache-dir", str(context.config.package_cache_dir)] if context.config.package_cache_dir else []), *(["--local-mirror", str(context.config.local_mirror)] if context.config.local_mirror else []), "--incremental", str(context.config.incremental), "--acl", str(context.config.acl), @@ -3415,6 +3417,7 @@ def finalize_default_tools(args: Args, config: Config, *, resources: Path) -> Co *(["--output-dir", str(config.output_dir)] if config.output_dir else []), *(["--workspace-dir", str(config.workspace_dir)] if config.workspace_dir else []), *(["--cache-dir", str(config.cache_dir)] if config.cache_dir else []), + *(["--package-cache-dir", str(config.package_cache_dir)] if config.package_cache_dir else []), "--incremental", str(config.incremental), "--acl", str(config.acl), *([f"--package={package}" for package in config.tools_tree_packages]), @@ -3496,11 +3499,16 @@ def run_clean(args: Args, config: Config) -> None: with complete_step(f"Clearing out build directory of {config.name()} image…"): rmtree(*config.build_dir.iterdir()) - if remove_package_cache and config.cache_dir and config.cache_dir.exists() and any(config.cache_dir.iterdir()): + if ( + remove_package_cache and + config.package_cache_dir and + config.package_cache_dir.exists() and + any(config.package_cache_dir.iterdir()) + ): with complete_step(f"Clearing out package cache of {config.name()} image…"): rmtree( *( - config.cache_dir / p / d + config.package_cache_dir / p / d for p in ("cache", "lib") for d in ("apt", "dnf", "libdnf5", "pacman", "zypp") ), @@ -3521,25 +3529,26 @@ def run_build(args: Args, config: Config, *, resources: Path) -> None: if Path(d).exists(): run(["mount", "--rbind", d, d, "--options", "ro"]) + # Create these as the invoking user to make sure they're owned by the user running mkosi. + for p in ( + config.output_dir, + config.cache_dir, + config.package_cache_dir_or_default(), + config.package_state_dir_or_default(), + config.build_dir, + config.workspace_dir, + ): + if p: + INVOKING_USER.mkdir(p) + with ( complete_step(f"Building {config.name()} image"), prepend_to_environ_path(config), + acl_toggle_build(config, INVOKING_USER.uid), + rchown_package_manager_cache_dirs(config), ): - # After tools have been mounted, check if we have what we need check_tools(config, Verb.build) - - # Create these as the invoking user to make sure they're owned by the user running mkosi. - for p in ( - config.output_dir, - config.cache_dir, - config.build_dir, - config.workspace_dir, - ): - if p: - run(["mkdir", "--parents", p], user=INVOKING_USER.uid, group=INVOKING_USER.gid) - - with acl_toggle_build(config, INVOKING_USER.uid): - build_image(args, config, resources=resources) + build_image(args, config, resources=resources) def run_verb(args: Args, images: Sequence[Config], *, resources: Path) -> None: diff --git a/mkosi/config.py b/mkosi/config.py index c7db14264..6ed481d0b 100644 --- a/mkosi/config.py +++ b/mkosi/config.py @@ -1171,6 +1171,7 @@ class Config: output_dir: Optional[Path] workspace_dir: Optional[Path] cache_dir: Optional[Path] + package_cache_dir: Optional[Path] build_dir: Optional[Path] image_id: Optional[str] image_version: Optional[str] @@ -1296,24 +1297,17 @@ class Config: if self.workspace_dir: return self.workspace_dir - if (cache := os.getenv("XDG_CACHE_HOME")) and Path(cache).exists(): - return Path(cache) - - # If we're running from /home and there's a cache or output directory in /home, we want to use a workspace - # directory in /home as well as /home might be on a separate partition or subvolume which means that to take - # advantage of reflinks and such, the workspace directory has to be on the same partition/subvolume. - if ( - Path.cwd().is_relative_to(INVOKING_USER.home()) and - (INVOKING_USER.home() / ".cache").exists() and - ( - self.cache_dir and self.cache_dir.is_relative_to(INVOKING_USER.home()) or - self.output_dir and self.output_dir.is_relative_to(INVOKING_USER.home()) - ) - ): - return INVOKING_USER.home() / ".cache" + if (cache := INVOKING_USER.cache_dir()) and cache != Path("/var/cache"): + return cache return Path("/var/tmp") + def package_cache_dir_or_default(self) -> Path: + return self.package_cache_dir or INVOKING_USER.cache_dir() + + def package_state_dir_or_default(self) -> Path: + return self.package_cache_dir or INVOKING_USER.state_dir() + def tools(self) -> Path: return self.tools_tree or Path("/") @@ -1737,7 +1731,15 @@ SETTINGS = ( section="Output", parse=config_make_path_parser(required=False), paths=("mkosi.cache",), - help="Package cache path", + help="Incremental cache directory", + ), + ConfigSetting( + dest="package_cache_dir", + metavar="PATH", + name="PackageCacheDirectory", + section="Output", + parse=config_make_path_parser(required=False), + help="Package cache directory", ), ConfigSetting( dest="build_dir", @@ -3433,6 +3435,7 @@ def summary(config: Config) -> str: Output Directory: {config.output_dir_or_cwd()} Workspace Directory: {config.workspace_dir_or_default()} Cache Directory: {none_to_none(config.cache_dir)} + Package Cache Directory: {none_to_default(config.package_cache_dir)} Build Directory: {none_to_none(config.build_dir)} Image ID: {config.image_id} Image Version: {config.image_version} diff --git a/mkosi/context.py b/mkosi/context.py index 4214c421d..1f81b1383 100644 --- a/mkosi/context.py +++ b/mkosi/context.py @@ -37,7 +37,6 @@ class Context: self.pkgmngr.mkdir() self.packages.mkdir() self.install_dir.mkdir(exist_ok=True) - self.cache_dir.mkdir(parents=True, exist_ok=True) @property def root(self) -> Path: @@ -55,10 +54,6 @@ class Context: def packages(self) -> Path: return self.workspace / "packages" - @property - def cache_dir(self) -> Path: - return self.config.cache_dir or (self.workspace / "cache") - @property def install_dir(self) -> Path: return self.workspace / "dest" diff --git a/mkosi/distributions/debian.py b/mkosi/distributions/debian.py index dfdc3b9a3..2207179cf 100644 --- a/mkosi/distributions/debian.py +++ b/mkosi/distributions/debian.py @@ -167,7 +167,7 @@ class Installer(DistributionInstaller): with ( # The deb paths will be in the form of "/var/cache/apt/" so we transform them to the corresponding # path in mkosi's package cache directory. - open(context.cache_dir / Path(deb).relative_to("/var"), "rb") as i, + open(context.config.package_cache_dir_or_default() / Path(deb).relative_to("/var/cache"), "rb") as i, tempfile.NamedTemporaryFile() as o ): run(["dpkg-deb", "--fsys-tarfile", "/dev/stdin"], stdin=i, stdout=o, sandbox=context.sandbox()) diff --git a/mkosi/installer/__init__.py b/mkosi/installer/__init__.py index 3b52a02f6..4256d42b2 100644 --- a/mkosi/installer/__init__.py +++ b/mkosi/installer/__init__.py @@ -1,13 +1,17 @@ # SPDX-License-Identifier: LGPL-2.1+ +import contextlib import os +from collections.abc import Iterator from pathlib import Path -from mkosi.config import ConfigFeature +from mkosi.config import Config, ConfigFeature from mkosi.context import Context +from mkosi.log import complete_step from mkosi.sandbox import apivfs_cmd, finalize_crypto_mounts from mkosi.tree import rmtree from mkosi.types import PathString +from mkosi.user import INVOKING_USER from mkosi.util import flatten @@ -74,16 +78,27 @@ def finalize_package_manager_mounts(context: Context) -> list[PathString]: ] mounts += flatten( - ["--bind", context.cache_dir / d, Path("/var") / d] - for d in ( - "lib/apt", - "cache/apt", - f"cache/{dnf_subdir(context)}", - f"lib/{dnf_subdir(context)}", - "cache/pacman/pkg", - "cache/zypp", - ) - if (context.cache_dir / d).exists() + ["--bind", context.config.package_cache_dir_or_default() / d, Path("/var/cache") / d] + for d in ("apt", dnf_subdir(context), "pacman/pkg", "zypp") + if (context.config.package_cache_dir_or_default() / d).exists() + ) + + mounts += flatten( + ["--bind", context.config.package_state_dir_or_default() / d, Path("/var/lib") / d] + for d in ("apt", dnf_subdir(context)) + if (context.config.package_state_dir_or_default() / d).exists() ) return mounts + + +@contextlib.contextmanager +def rchown_package_manager_cache_dirs(config: Config) -> Iterator[None]: + try: + yield + finally: + if INVOKING_USER.is_regular_user(): + with complete_step("Fixing ownership of package manager cache directories"): + for p in ("apt", "dnf", "libdnf5", "pacman", "zypp"): + for d in (config.package_cache_dir_or_default(), config.package_state_dir_or_default()): + INVOKING_USER.rchown(d / p) diff --git a/mkosi/installer/apt.py b/mkosi/installer/apt.py index 81d38961d..465bd5617 100644 --- a/mkosi/installer/apt.py +++ b/mkosi/installer/apt.py @@ -9,6 +9,7 @@ from mkosi.mounts import finalize_ephemeral_source_mounts from mkosi.run import find_binary, run from mkosi.sandbox import apivfs_cmd from mkosi.types import PathString +from mkosi.user import INVOKING_USER from mkosi.util import sort_packages, umask @@ -43,8 +44,8 @@ def setup_apt(context: Context, repos: Iterable[AptRepository]) -> None: (context.root / "var/lib/dpkg").mkdir(parents=True, exist_ok=True) (context.root / "var/lib/dpkg/status").touch() - (context.cache_dir / "lib/apt").mkdir(exist_ok=True, parents=True) - (context.cache_dir / "cache/apt").mkdir(exist_ok=True, parents=True) + INVOKING_USER.mkdir(context.config.package_cache_dir_or_default() / "apt") + INVOKING_USER.mkdir(context.config.package_state_dir_or_default() / "apt") # We have a special apt.conf outside of pkgmngr dir that only configures "Dir::Etc" that we pass to APT_CONFIG to # tell apt it should read config files from /etc/apt in case this is overridden by distributions. This is required diff --git a/mkosi/installer/dnf.py b/mkosi/installer/dnf.py index d195470d0..abc6a395b 100644 --- a/mkosi/installer/dnf.py +++ b/mkosi/installer/dnf.py @@ -10,6 +10,7 @@ from mkosi.mounts import finalize_ephemeral_source_mounts from mkosi.run import find_binary, run from mkosi.sandbox import apivfs_cmd from mkosi.types import PathString +from mkosi.user import INVOKING_USER from mkosi.util import sort_packages @@ -30,8 +31,8 @@ def setup_dnf(context: Context, repositories: Iterable[RpmRepository], filelists (context.pkgmngr / "etc/dnf/vars").mkdir(exist_ok=True, parents=True) (context.pkgmngr / "etc/yum.repos.d").mkdir(exist_ok=True, parents=True) - (context.cache_dir / "cache" / dnf_subdir(context)).mkdir(exist_ok=True, parents=True) - (context.cache_dir / "lib" / dnf_subdir(context)).mkdir(exist_ok=True, parents=True) + INVOKING_USER.mkdir(context.config.package_cache_dir_or_default() / dnf_subdir(context)) + INVOKING_USER.mkdir(context.config.package_state_dir_or_default() / dnf_subdir(context)) config = context.pkgmngr / "etc/dnf/dnf.conf" diff --git a/mkosi/installer/pacman.py b/mkosi/installer/pacman.py index 941fb8a00..8ea5524fb 100644 --- a/mkosi/installer/pacman.py +++ b/mkosi/installer/pacman.py @@ -10,6 +10,7 @@ from mkosi.mounts import finalize_ephemeral_source_mounts from mkosi.run import run from mkosi.sandbox import apivfs_cmd from mkosi.types import PathString +from mkosi.user import INVOKING_USER from mkosi.util import sort_packages, umask from mkosi.versioncomp import GenericVersion @@ -31,7 +32,7 @@ def setup_pacman(context: Context, repositories: Iterable[PacmanRepository]) -> with umask(~0o755): (context.root / "var/lib/pacman").mkdir(exist_ok=True, parents=True) - (context.cache_dir / "cache/pacman/pkg").mkdir(parents=True, exist_ok=True) + INVOKING_USER.mkdir(context.config.package_cache_dir_or_default() / "pacman/pkg") config = context.pkgmngr / "etc/pacman.conf" if config.exists(): diff --git a/mkosi/installer/zypper.py b/mkosi/installer/zypper.py index 2996d046c..c3237aea7 100644 --- a/mkosi/installer/zypper.py +++ b/mkosi/installer/zypper.py @@ -10,6 +10,7 @@ from mkosi.mounts import finalize_ephemeral_source_mounts from mkosi.run import run from mkosi.sandbox import apivfs_cmd from mkosi.types import PathString +from mkosi.user import INVOKING_USER from mkosi.util import sort_packages @@ -17,7 +18,7 @@ def setup_zypper(context: Context, repos: Iterable[RpmRepository]) -> None: config = context.pkgmngr / "etc/zypp/zypp.conf" config.parent.mkdir(exist_ok=True, parents=True) - (context.cache_dir / "cache/zypp").mkdir(exist_ok=True, parents=True) + INVOKING_USER.mkdir(context.config.package_cache_dir_or_default() / "zypp") # rpm.install.excludedocs can only be configured in zypp.conf so we append # to any user provided config file. Let's also bump the refresh delay to diff --git a/mkosi/resources/mkosi.md b/mkosi/resources/mkosi.md index 1ca1e63ca..ac3d44c4c 100644 --- a/mkosi/resources/mkosi.md +++ b/mkosi/resources/mkosi.md @@ -695,10 +695,17 @@ boolean argument: either `1`, `yes`, or `true` to enable, or `0`, `no`, `CacheDirectory=`, `--cache-dir=` -: Takes a path to a directory to use as package cache for the - distribution package manager used. If this option is not used, but a - `mkosi.cache/` directory is found in the local directory it is - automatically used for this purpose. +: Takes a path to a directory to use as the incremental cache directory + for the incremental images produced when the `Incremental=` option is + enabled. If this option is not used, but a `mkosi.cache/` directory is + found in the local directory it is automatically used for this + purpose. + +`PackageCacheDirectory=`, `--package-cache-dir` + +: Takes a path to a directory to use as the package cache directory for + the distribution package manager used. If unset, a suitable directory + in the user's home directory or system is used. `BuildDirectory=`, `--build-dir=` diff --git a/mkosi/user.py b/mkosi/user.py index c468b43b3..27371d9b9 100644 --- a/mkosi/user.py +++ b/mkosi/user.py @@ -9,7 +9,7 @@ import pwd from pathlib import Path from mkosi.log import die -from mkosi.run import spawn +from mkosi.run import run, spawn from mkosi.util import flock SUBRANGE = 65536 @@ -39,6 +39,41 @@ class INVOKING_USER: def home(cls) -> Path: return Path(f"~{cls.name()}").expanduser() + @classmethod + def is_regular_user(cls) -> bool: + return cls.uid >= 1000 + + @classmethod + def cache_dir(cls) -> Path: + if (cache := os.getenv("XDG_CACHE_HOME") or (cache := os.getenv("CACHE_DIRECTORY"))): + return Path(cache) + + if (cls.is_regular_user() and Path.cwd().is_relative_to(INVOKING_USER.home())): + return INVOKING_USER.home() / ".cache" + + return Path("/var/cache") + + @classmethod + def state_dir(cls) -> Path: + if (state := os.getenv("XDG_STATE_HOME") or (state := os.getenv("STATE_DIRECTORY"))): + return Path(state) + + if (cls.is_regular_user() and Path.cwd().is_relative_to(INVOKING_USER.home())): + return INVOKING_USER.home() / ".local/state" + + return Path("/var/lib") + + @classmethod + def mkdir(cls, path: Path) -> None: + user = cls.uid if cls.is_regular_user() and path.is_relative_to(cls.home()) else os.getuid() + group = cls.gid if cls.is_regular_user() and path.is_relative_to(cls.home()) else os.getgid() + run(["mkdir", "--parents", path], user=user, group=group) + + @classmethod + def rchown(cls, path: Path) -> None: + if cls.is_regular_user() and path.is_relative_to(INVOKING_USER.home()) and path.exists(): + run(["chown", "--recursive", f"{INVOKING_USER.uid}:{INVOKING_USER.gid}", path]) + def read_subrange(path: Path) -> int: uid = str(os.getuid()) diff --git a/tests/test_json.py b/tests/test_json.py index b6dc04a28..aede03b29 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -182,6 +182,7 @@ def test_config() -> None: "Output": "outfile", "OutputDirectory": "/your/output/here", "Overlay": true, + "PackageCacheDirectory": "/a/b/c", "PackageDirectories": [], "PackageManagerTrees": [ { @@ -356,6 +357,7 @@ def test_config() -> None: output_dir = Path("/your/output/here"), output_format = OutputFormat.uki, overlay = True, + package_cache_dir = Path("/a/b/c"), package_directories = [], package_manager_trees = [ConfigTree(Path("/foo/bar"), None)], packages = [],