From: Daan De Meyer Date: Tue, 9 Jul 2024 12:24:10 +0000 (+0200) Subject: Introduce VolatilePackageDirectories= X-Git-Tag: v24~46^2~1 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=2d8ea594b1b44777f90b03e82f370ac604ff1482;p=thirdparty%2Fmkosi.git Introduce VolatilePackageDirectories= For packages that should not be available when building the cached image but should be available when installing volatile packages. Additionally, we have build scripts write their packages to a separate directory instead of the repository directory and later on pass these packages as a separate volatile package directory to the default initrd build. This will allow us to introduce proper caching for package directories in the next commit. --- diff --git a/mkosi/__init__.py b/mkosi/__init__.py index 0bbb42c15..2ba97a922 100644 --- a/mkosi/__init__.py +++ b/mkosi/__init__.py @@ -591,7 +591,7 @@ def run_prepare_scripts(context: Context, build: bool) -> None: Mount(json, "/work/config.json", ro=True), Mount(context.root, "/buildroot"), Mount(context.artifacts, "/work/artifacts"), - Mount(context.repository, "/work/packages"), + Mount(context.packages, "/work/packages"), *context.config.distribution.package_manager(context.config).mounts(context), ], options=["--dir", "/work/src", "--chdir", "/work/src"], @@ -674,7 +674,7 @@ def run_build_scripts(context: Context) -> None: Mount(context.install_dir, "/work/dest"), Mount(context.staging, "/work/out"), Mount(context.artifacts, "/work/artifacts"), - Mount(context.repository, "/work/packages"), + Mount(context.packages, "/work/packages"), *( [Mount(context.config.build_dir, "/work/build")] if context.config.build_dir @@ -688,10 +688,6 @@ def run_build_scripts(context: Context) -> None: ), ) - if context.want_local_repo() and context.config.output_format != OutputFormat.none: - with complete_step("Rebuilding local package repository"): - context.config.distribution.createrepo(context) - def run_postinst_scripts(context: Context) -> None: if not context.config.postinst_scripts: @@ -752,7 +748,7 @@ def run_postinst_scripts(context: Context) -> None: Mount(context.root, "/buildroot"), Mount(context.staging, "/work/out"), Mount(context.artifacts, "/work/artifacts"), - Mount(context.repository, "/work/packages"), + Mount(context.packages, "/work/packages"), *context.config.distribution.package_manager(context.config).mounts(context), ], options=["--dir", "/work/src", "--chdir", "/work/src"], @@ -819,7 +815,7 @@ def run_finalize_scripts(context: Context) -> None: Mount(context.root, "/buildroot"), Mount(context.staging, "/work/out"), Mount(context.artifacts, "/work/artifacts"), - Mount(context.repository, "/work/packages"), + Mount(context.packages, "/work/packages"), *context.config.distribution.package_manager(context.config).mounts(context), ], options=["--dir", "/work/src", "--chdir", "/work/src"], @@ -1652,19 +1648,17 @@ def install_package_manager_trees(context: Context) -> None: install_tree(context.config, tree.source, context.pkgmngr, target=tree.target, preserve=False) -def install_package_directories(context: Context) -> None: - if not context.config.package_directories: +def install_package_directories(context: Context, directories: Sequence[Path]) -> None: + directories = [d for d in directories if any(d.iterdir())] + + if not directories: return with complete_step("Copying in extra packages…"): - for d in context.config.package_directories: + for d in directories: for p in itertools.chain(*(d.glob(glob) for glob in PACKAGE_GLOBS)): shutil.copy(p, context.repository, follow_symlinks=True) - if context.want_local_repo(): - with complete_step("Building local package repository"): - context.config.distribution.createrepo(context) - def install_extra_trees(context: Context) -> None: if not context.config.extra_trees: @@ -1786,7 +1780,9 @@ def finalize_default_initrd( "--acl", str(config.acl), *(f"--package={package}" for package in config.initrd_packages), *(f"--volatile-package={package}" for package in config.initrd_volatile_packages), - *(["--package-directory", str(package_dir)] if package_dir else []), + *(f"--package-directory={d}" for d in config.package_directories), + *(f"--volatile-package-directory={d}" for d in config.volatile_package_directories), + *([f"--volatile-package-directory={package_dir}"] if package_dir else []), "--output", "initrd", *(["--image-id", config.image_id] if config.image_id else []), *(["--image-version", config.image_version] if config.image_version else []), @@ -1834,7 +1830,7 @@ def build_default_initrd(context: Context) -> Path: context.config, resources=context.resources, output_dir=context.workspace, - package_dir=context.repository, + package_dir=context.packages, ) assert config.output_dir @@ -3818,6 +3814,16 @@ def copy_repository_metadata(context: Context) -> None: sandbox=sandbox, ) +@contextlib.contextmanager +def createrepo(context: Context) -> Iterator[None]: + st = context.repository.stat() + try: + yield + finally: + if context.repository.stat().st_mtime_ns != st.st_mtime_ns: + with complete_step("Rebuilding local package repository"): + context.config.distribution.createrepo(context) + def build_image(context: Context) -> None: manifest = Manifest(context) if context.config.manifest_format else None @@ -3831,7 +3837,8 @@ def build_image(context: Context) -> None: copy_repository_metadata(context) context.config.distribution.setup(context) - install_package_directories(context) + with createrepo(context): + install_package_directories(context, context.config.package_directories) if not cached: install_skeleton_trees(context) @@ -3853,6 +3860,10 @@ def build_image(context: Context) -> None: rmtree(context.root) return + with createrepo(context): + install_package_directories(context, context.config.volatile_package_directories) + install_package_directories(context, [context.packages]) + install_volatile_packages(context) install_build_dest(context) install_extra_trees(context) diff --git a/mkosi/config.py b/mkosi/config.py index acb0f1f5a..b8bcdd4be 100644 --- a/mkosi/config.py +++ b/mkosi/config.py @@ -1415,6 +1415,7 @@ class Config: build_packages: list[str] volatile_packages: list[str] package_directories: list[Path] + volatile_package_directories: list[Path] with_recommends: bool with_docs: bool @@ -2190,6 +2191,15 @@ SETTINGS = ( help="Specify a directory containing extra packages", universal=True, ), + ConfigSetting( + dest="volatile_package_directories", + long="--volatile-package-directory", + metavar="PATH", + section="Content", + parse=config_make_list_parser(delimiter=",", parse=make_path_parser()), + help="Specify a directory containing extra volatile packages", + universal=True, + ), ConfigSetting( dest="with_recommends", metavar="BOOL", @@ -4048,6 +4058,8 @@ def summary(config: Config) -> str: Packages: {line_join_list(config.packages)} Build Packages: {line_join_list(config.build_packages)} Volatile Packages: {line_join_list(config.volatile_packages)} + Package Directories: {line_join_list(config.package_directories)} + Volatile Package Directories: {line_join_list(config.volatile_package_directories)} With Documentation: {yes_no(config.with_docs)} Base Trees: {line_join_list(config.base_trees)} diff --git a/mkosi/context.py b/mkosi/context.py index 2bbec1258..761358130 100644 --- a/mkosi/context.py +++ b/mkosi/context.py @@ -45,6 +45,7 @@ class Context: self.staging.mkdir() self.pkgmngr.mkdir() self.repository.mkdir() + self.packages.mkdir() self.artifacts.mkdir() self.install_dir.mkdir() @@ -64,6 +65,10 @@ class Context: def repository(self) -> Path: return self.workspace / "repository" + @property + def packages(self) -> Path: + return self.workspace / "packages" + @property def artifacts(self) -> Path: return self.workspace / "artifacts" @@ -115,6 +120,3 @@ class Context: ], extra=extra, ) - - def want_local_repo(self) -> bool: - return any(self.repository.iterdir()) diff --git a/mkosi/resources/mkosi.md b/mkosi/resources/mkosi.md index 3ce7af2ea..ef3480783 100644 --- a/mkosi/resources/mkosi.md +++ b/mkosi/resources/mkosi.md @@ -772,9 +772,15 @@ boolean argument: either `1`, `yes`, or `true` to enable, or `0`, `no`, running scripts. If the `mkosi.packages/` directory is found in the local directory it is also used for this purpose. - Note that this local repository is also made available when running - scripts. Build scripts can add more packages to the local repository - by placing the built packages in `$PACKAGEDIR`. + Build scripts can add more packages to the local repository by + placing the built packages in `$PACKAGEDIR`. + +`VolatilePackageDirectories=`, `--volatile-package-directory=` + +: Like `PackageDirectories=`, but the packages in these directories + are only made available in the local repository just before volatile + packages are installed. Specifically, if `Incremental=` is enabled, + the packages from these directories will not be cached. `WithRecommends=`, `--with-recommends=` : Configures whether to install recommended or weak dependencies, @@ -2470,6 +2476,7 @@ overridden): - `RepartOffline=` - `UseSubvolumes=` - `PackageDirectories=` +- `VolatilePackageDirectories=` - `SourceDateEpoch=` - `VerityKey=` - `VerityKeySource=` diff --git a/tests/test_json.py b/tests/test_json.py index 60b3a2a9a..45145d9fa 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -342,6 +342,9 @@ def test_config() -> None: "Type": "file" }, "VirtualMachineMonitor": "qemu", + "VolatilePackageDirectories": [ + "def" + ], "VolatilePackages": [ "abc" ], @@ -500,6 +503,7 @@ def test_config() -> None: verity_certificate=Path("/path/to/cert"), verity_key=None, verity_key_source=KeySource(type=KeySource.Type.file), + volatile_package_directories=[Path("def")], volatile_packages=["abc"], with_docs=True, with_network=False,