From: Daan De Meyer Date: Thu, 1 Feb 2024 10:48:28 +0000 (+0100) Subject: Store repository metadata snapshot in /var X-Git-Tag: v21~75^2~1 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=18ffab63208c44602993e644b37c2e71e5554ea9;p=thirdparty%2Fmkosi.git Store repository metadata snapshot in /var This makes us more compatible with third-party base trees as the repository metadata is stored in the canonical locations by using /var. This also means that the repository metadata becomes subject to the regular CleanPackageMetadata= cleanup rules. --- diff --git a/NEWS.md b/NEWS.md index 24a11fc17..4b656020a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -24,12 +24,13 @@ `CacheDirectory=` is only used for incremental cached images now. - Repository metadata is now synced once at the start of each image build and never during an image build. Each image includes a snapshot - of the repository metadata in `/mkosi` so that incremental images and - extension images can reuse the same snapshot. When building an image - intended to be used with `BaseTrees=`, disable `CleanPackageMetadata=` - to make sure the repository metadata in `/mkosi` is not cleaned up, - otherwise any extension images using this image as their base tree - will not be able to install additional packages. + of the repository metadata in the canonical locations in `/var` so + that incremental images and extension images can reuse the same + snapshot. When building an image intended to be used with + `BaseTrees=`, disable `CleanPackageMetadata=` to make sure the + repository metadata in `/var` is not cleaned up, otherwise any + extension images using this image as their base tree will not be able + to install additional packages. - Implemented `CacheOnly=metadata`. Note that in the JSON output, the value of `CacheOnly=` will now be a string instead of a boolean. diff --git a/mkosi/__init__.py b/mkosi/__init__.py index 39090194a..2b9253621 100644 --- a/mkosi/__init__.py +++ b/mkosi/__init__.py @@ -2949,11 +2949,15 @@ def copy_repository_metadata(context: Context) -> None: if have_cache(context.config) or context.config.base_trees: return - if context.package_cache_dir.exists() and any(context.package_cache_dir.iterdir()): - return - subdir = context.config.distribution.package_manager(context.config).subdir(context.config) + # Don't copy anything if the repository metadata directories are already populated. + if ( + any((context.package_cache_dir / "cache" / subdir).glob("*")) or + any((context.package_cache_dir / "lib" / subdir).glob("*")) + ): + return + for d in ("cache", "lib"): src = context.config.package_cache_dir_or_default() / d / subdir if not src.exists(): diff --git a/mkosi/context.py b/mkosi/context.py index 2220ed10d..799e00b8e 100644 --- a/mkosi/context.py +++ b/mkosi/context.py @@ -27,7 +27,7 @@ class Context: self.config = config self.workspace = workspace self.resources = resources - self.package_cache_dir = package_cache_dir or (self.root / "mkosi") + self.package_cache_dir = package_cache_dir or (self.root / "var") with umask(~0o755): # Using a btrfs subvolume as the upperdir in an overlayfs results in EXDEV so make sure we create diff --git a/mkosi/installer/__init__.py b/mkosi/installer/__init__.py index 95674fab5..d79049745 100644 --- a/mkosi/installer/__init__.py +++ b/mkosi/installer/__init__.py @@ -7,12 +7,16 @@ from mkosi.config import Config, ConfigFeature, OutputFormat from mkosi.context import Context from mkosi.run import find_binary from mkosi.sandbox import finalize_crypto_mounts -from mkosi.tree import move_tree, rmtree +from mkosi.tree import copy_tree, rmtree from mkosi.types import PathString from mkosi.util import flatten class PackageManager: + @classmethod + def executable(cls, config: Config) -> str: + return "custom" + @classmethod def subdir(cls, config: Config) -> Path: return Path("custom") @@ -73,26 +77,31 @@ def clean_package_manager_metadata(context: Context) -> None: ): # Instead of removing the package cache directory from the image, we move it to the workspace so it stays # available for later steps and is automatically removed along with the workspace when the build finishes. - context.package_cache_dir = move_tree( - context.package_cache_dir, context.workspace / "package-cache-dir", - tools=context.config.tools(), - sandbox=context.sandbox( - options=[ - "--bind", context.package_cache_dir.parent, context.package_cache_dir.parent, - "--bind", context.workspace, context.workspace, - ], - ), - ) + subdir = context.config.distribution.package_manager(context.config).subdir(context.config) + + for d in ("cache", "lib"): + src = context.package_cache_dir / d / subdir + if not src.exists(): + continue + + dst = context.workspace / "package-cache-dir" / d / subdir + dst.mkdir(parents=True, exist_ok=True) + + copy_tree(src, dst, sandbox=context.sandbox(options=["--ro-bind", src, src, "--bind", dst, dst])) + + context.package_cache_dir = context.workspace / "package-cache-dir" if context.config.clean_package_metadata == ConfigFeature.disabled: return always = context.config.clean_package_metadata == ConfigFeature.enabled + executable = context.config.distribution.package_manager(context.config).executable(context.config) + subdir = context.config.distribution.package_manager(context.config).subdir(context.config) - for tool, paths in (("rpm", ["var/lib/rpm", "usr/lib/sysimage/rpm"]), - ("dnf5", ["usr/lib/sysimage/libdnf5"]), - ("dpkg", ["var/lib/dpkg"]), - ("pacman", ["var/lib/pacman"])): + for tool, paths in (("rpm", ["var/lib/rpm", "usr/lib/sysimage/rpm"]), + ("dnf5", ["usr/lib/sysimage/libdnf5"]), + ("dpkg", ["var/lib/dpkg"]), + (executable, [f"var/lib/{subdir}", f"var/cache/{subdir}"])): if always or not find_binary(tool, root=context.root): rmtree(*(context.root / p for p in paths), sandbox=context.sandbox(options=["--bind", context.root, context.root])) diff --git a/mkosi/installer/apt.py b/mkosi/installer/apt.py index a498aaad0..2a532d14f 100644 --- a/mkosi/installer/apt.py +++ b/mkosi/installer/apt.py @@ -34,6 +34,10 @@ class Apt(PackageManager): """ ) + @classmethod + def executable(cls, config: Config) -> str: + return "apt" + @classmethod def subdir(cls, config: Config) -> Path: return Path("apt") @@ -70,8 +74,6 @@ class Apt(PackageManager): (context.root / "var/lib/dpkg").mkdir(parents=True, exist_ok=True) (context.root / "var/lib/dpkg/status").touch() - (context.package_cache_dir / "lib/apt/lists/partial").mkdir(parents=True, exist_ok=True) - # We have a special apt.conf outside of pkgmngr dir that only configures "Dir::Etc" that we pass to APT_CONFIG # to tell apt it should read config files from /etc/apt in case this is overridden by distributions. This is # required because apt parses CLI configuration options after parsing its configuration files and as such we diff --git a/mkosi/installer/pacman.py b/mkosi/installer/pacman.py index 2dc4dd369..c1906dce5 100644 --- a/mkosi/installer/pacman.py +++ b/mkosi/installer/pacman.py @@ -21,6 +21,10 @@ class Pacman(PackageManager): id: str url: str + @classmethod + def executable(cls, config: Config) -> str: + return "pacman" + @classmethod def subdir(cls, config: Config) -> Path: return Path("pacman") diff --git a/mkosi/installer/zypper.py b/mkosi/installer/zypper.py index fd34efe7b..d419147e4 100644 --- a/mkosi/installer/zypper.py +++ b/mkosi/installer/zypper.py @@ -16,6 +16,10 @@ from mkosi.util import sort_packages class Zypper(PackageManager): + @classmethod + def executable(cls, config: Config) -> str: + return "zypper" + @classmethod def subdir(cls, config: Config) -> Path: return Path("zypp") diff --git a/mkosi/resources/mkosi.md b/mkosi/resources/mkosi.md index 839d8a8a0..37f5fa44f 100644 --- a/mkosi/resources/mkosi.md +++ b/mkosi/resources/mkosi.md @@ -986,18 +986,11 @@ boolean argument: either `1`, `yes`, or `true` to enable, or `0`, `no`, `CleanPackageMetadata=`, `--clean-package-metadata=` : Enable/disable removal of package manager databases and repository - metadata in `/mkosi` at the end of installation. Can be specified as - `true`, `false`, or `auto` (the default). With `auto`, package manager - databases will be removed if the respective package manager executable - is *not* present at the end of the installation. - -: Note that when not building a tar or directory image, the repository - metadata in `/mkosi` is always removed, regardless of this setting as - it is only useful for building extensions using `BaseTrees=`. - -: Note that when set to `auto`, repository metadata in `/mkosi` is - removed regardless of whether the respective package manager - executable is present or not. + metadata at the end of installation. Can be specified as `true`, + `false`, or `auto` (the default). With `auto`, package manager + databases and repository metadata will be removed if the respective + package manager executable is *not* present at the end of the + installation. `PrepareScripts=`, `--prepare-script=`