From 6ab5173069dbbb29c9b152c56654000e96dc3522 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Mon, 15 Jan 2024 22:24:08 +0100 Subject: [PATCH] Add PackageDirectories= Let's make it possible to serve local packages as a local repository so that users don't have to put local paths in their Packages= setting. We'll also allow adding more packages to this local repository in the build script so that these can be installed in the initrd when we build it or in a postinst or finalize script. --- mkosi/__init__.py | 29 +++++++++++++++++-- mkosi/config.py | 9 ++++++ mkosi/context.py | 5 ++++ mkosi/distributions/__init__.py | 7 +++++ mkosi/distributions/arch.py | 11 ++++++- mkosi/distributions/centos.py | 6 +++- mkosi/distributions/debian.py | 6 +++- mkosi/distributions/fedora.py | 6 +++- mkosi/distributions/mageia.py | 6 +++- mkosi/distributions/openmandriva.py | 6 +++- mkosi/distributions/opensuse.py | 11 +++++-- mkosi/installer/__init__.py | 1 + mkosi/installer/apt.py | 18 ++++++++++++ mkosi/installer/dnf.py | 20 +++++++++++++ mkosi/installer/pacman.py | 14 +++++++++ mkosi/installer/zypper.py | 20 +++++++++++++ .../mkosi-tools/mkosi.conf.d/10-arch.conf | 1 + .../mkosi.conf.d/10-centos-fedora/mkosi.conf | 2 ++ .../mkosi.conf.d/10-debian-ubuntu.conf | 2 ++ .../mkosi-tools/mkosi.conf.d/10-opensuse.conf | 1 + mkosi/resources/mkosi.md | 15 ++++++++++ tests/test_json.py | 2 ++ 22 files changed, 187 insertions(+), 11 deletions(-) diff --git a/mkosi/__init__.py b/mkosi/__init__.py index 41569f0ea..63038d555 100644 --- a/mkosi/__init__.py +++ b/mkosi/__init__.py @@ -411,12 +411,13 @@ def run_prepare_scripts(context: Context, build: bool) -> None: env = dict( ARCHITECTURE=str(context.config.architecture), BUILDROOT=str(context.root), - CHROOT_SCRIPT="/work/prepare", + SRCDIR="/work/src", CHROOT_SRCDIR="/work/src", + PACKAGEDIR="/work/packages", + SCRIPT="/work/prepare", + CHROOT_SCRIPT="/work/prepare", MKOSI_UID=str(INVOKING_USER.uid), MKOSI_GID=str(INVOKING_USER.gid), - SCRIPT="/work/prepare", - SRCDIR="/work/src", WITH_DOCS=one_zero(context.config.with_docs), WITH_NETWORK=one_zero(context.config.with_network), WITH_TESTS=one_zero(context.config.with_tests), @@ -485,6 +486,7 @@ def run_build_scripts(context: Context) -> None: CHROOT_OUTPUTDIR="/work/out", SRCDIR="/work/src", CHROOT_SRCDIR="/work/src", + PACKAGEDIR="/work/packages", SCRIPT="/work/build-script", CHROOT_SCRIPT="/work/build-script", MKOSI_UID=str(INVOKING_USER.uid), @@ -552,6 +554,10 @@ def run_build_scripts(context: Context) -> None: ) + (chroot if script.suffix == ".chroot" else []), ) + if any(context.packages.iterdir()): + with complete_step("Rebuilding local package repository"): + context.config.distribution.createrepo(context) + def run_postinst_scripts(context: Context) -> None: if not context.config.postinst_scripts: @@ -566,6 +572,7 @@ def run_postinst_scripts(context: Context) -> None: CHROOT_SCRIPT="/work/postinst", SRCDIR="/work/src", CHROOT_SRCDIR="/work/src", + PACKAGEDIR="/work/packages", MKOSI_UID=str(INVOKING_USER.uid), MKOSI_GID=str(INVOKING_USER.gid), ) @@ -624,6 +631,7 @@ def run_finalize_scripts(context: Context) -> None: CHROOT_OUTPUTDIR="/work/out", SRCDIR="/work/src", CHROOT_SRCDIR="/work/src", + PACKAGEDIR="/work/packages", SCRIPT="/work/finalize", CHROOT_SCRIPT="/work/finalize", MKOSI_UID=str(INVOKING_USER.uid), @@ -1385,6 +1393,19 @@ def install_package_manager_trees(context: Context) -> None: install_tree(context, tree.source, context.pkgmngr, target=tree.target, preserve=False) +def install_package_directories(context: Context) -> None: + if not context.config.package_directories: + return + + with complete_step("Copying in extra packages…"): + for d in context.config.package_directories: + install_tree(context, d, context.packages) + + if any(context.packages.iterdir()): + with complete_step("Initializing local package repository…"): + context.config.distribution.createrepo(context) + + def install_extra_trees(context: Context) -> None: if not context.config.extra_trees: return @@ -1459,6 +1480,7 @@ def build_initrd(context: Context) -> Path: "--incremental", str(context.config.incremental), "--acl", str(context.config.acl), *(f"--package={package}" for package in context.config.initrd_packages), + "--package-directory", str(context.packages), "--output", f"{context.config.output}-initrd", *(["--image-id", context.config.image_id] if context.config.image_id else []), *(["--image-version", context.config.image_version] if context.config.image_version else []), @@ -2799,6 +2821,7 @@ def build_image(args: Args, config: Config) -> None: with setup_workspace(args, config) as workspace: context = Context(args, config, workspace) install_package_manager_trees(context) + install_package_directories(context) with mount_base_trees(context): install_base_trees(context) diff --git a/mkosi/config.py b/mkosi/config.py index 451eefb8b..245d95e1a 100644 --- a/mkosi/config.py +++ b/mkosi/config.py @@ -1108,6 +1108,7 @@ class Config: packages: list[str] build_packages: list[str] + package_directories: list[Path] with_recommends: bool with_docs: bool @@ -1746,6 +1747,14 @@ SETTINGS = ( parse=config_make_list_parser(delimiter=","), help="Additional packages needed for build scripts", ), + ConfigSetting( + dest="package_directories", + long="--package-directory", + metavar="PATH", + section="Content", + parse=config_make_list_parser(delimiter=",", parse=make_path_parser()), + help="Specify a directory containing extra packages", + ), ConfigSetting( dest="with_recommends", metavar="BOOL", diff --git a/mkosi/context.py b/mkosi/context.py index ba4eb8ce7..959e8880d 100644 --- a/mkosi/context.py +++ b/mkosi/context.py @@ -34,6 +34,7 @@ class Context: self.staging.mkdir() self.pkgmngr.mkdir() + self.packages.mkdir() self.install_dir.mkdir(exist_ok=True) self.cache_dir.mkdir(parents=True, exist_ok=True) @@ -49,6 +50,10 @@ class Context: def pkgmngr(self) -> Path: return self.workspace / "pkgmngr" + @property + def packages(self) -> Path: + return self.workspace / "packages" + @property def cache_dir(self) -> Path: return self.config.cache_dir or (self.workspace / "cache") diff --git a/mkosi/distributions/__init__.py b/mkosi/distributions/__init__.py index 2c4374ae6..3dd15e81c 100644 --- a/mkosi/distributions/__init__.py +++ b/mkosi/distributions/__init__.py @@ -67,6 +67,10 @@ class DistributionInstaller: def grub_prefix(cls) -> str: return "grub" + @classmethod + def createrepo(cls, context: "Context") -> None: + raise NotImplementedError + class Distribution(StrEnum): # Please consult docs/distribution-policy.md and contact one @@ -143,6 +147,9 @@ class Distribution(StrEnum): def grub_prefix(self) -> str: return self.installer().grub_prefix() + def createrepo(self, context: "Context") -> None: + return self.installer().createrepo(context) + def installer(self) -> type[DistributionInstaller]: modname = str(self).replace('-', '_') mod = importlib.import_module(f"mkosi.distributions.{modname}") diff --git a/mkosi/distributions/arch.py b/mkosi/distributions/arch.py index 697a34b52..2d4a4b388 100644 --- a/mkosi/distributions/arch.py +++ b/mkosi/distributions/arch.py @@ -5,7 +5,12 @@ from collections.abc import Sequence from mkosi.config import Architecture from mkosi.context import Context from mkosi.distributions import Distribution, DistributionInstaller, PackageType -from mkosi.installer.pacman import PacmanRepository, invoke_pacman, setup_pacman +from mkosi.installer.pacman import ( + PacmanRepository, + createrepo_pacman, + invoke_pacman, + setup_pacman, +) from mkosi.log import die @@ -30,6 +35,10 @@ class Installer(DistributionInstaller): def default_tools_tree_distribution(cls) -> Distribution: return Distribution.arch + @classmethod + def createrepo(cls, context: "Context") -> None: + return createrepo_pacman(context) + @classmethod def setup(cls, context: Context) -> None: if context.config.local_mirror: diff --git a/mkosi/distributions/centos.py b/mkosi/distributions/centos.py index 0e389f100..0dc252329 100644 --- a/mkosi/distributions/centos.py +++ b/mkosi/distributions/centos.py @@ -12,7 +12,7 @@ from mkosi.distributions import ( PackageType, join_mirror, ) -from mkosi.installer.dnf import invoke_dnf, setup_dnf +from mkosi.installer.dnf import createrepo_dnf, invoke_dnf, setup_dnf from mkosi.installer.rpm import RpmRepository, find_rpm_gpgkey from mkosi.log import complete_step, die from mkosi.tree import rmtree @@ -58,6 +58,10 @@ class Installer(DistributionInstaller): def grub_prefix(cls) -> str: return "grub2" + @classmethod + def createrepo(cls, context: Context) -> None: + return createrepo_dnf(context) + @classmethod def setup(cls, context: Context) -> None: if GenericVersion(context.config.release) <= 7: diff --git a/mkosi/distributions/debian.py b/mkosi/distributions/debian.py index 8c56722f9..43314825f 100644 --- a/mkosi/distributions/debian.py +++ b/mkosi/distributions/debian.py @@ -9,7 +9,7 @@ from mkosi.archive import extract_tar from mkosi.config import Architecture from mkosi.context import Context from mkosi.distributions import Distribution, DistributionInstaller, PackageType -from mkosi.installer.apt import invoke_apt, setup_apt +from mkosi.installer.apt import createrepo_apt, invoke_apt, setup_apt from mkosi.log import die from mkosi.run import run from mkosi.util import umask @@ -77,6 +77,10 @@ class Installer(DistributionInstaller): def setup(cls, context: Context) -> None: setup_apt(context, cls.repositories(context)) + @classmethod + def createrepo(cls, context: "Context") -> None: + return createrepo_apt(context) + @classmethod def install(cls, context: Context) -> None: # Instead of using debootstrap, we replicate its core functionality here. Because dpkg does not have diff --git a/mkosi/distributions/fedora.py b/mkosi/distributions/fedora.py index dc865819d..16ccb49e7 100644 --- a/mkosi/distributions/fedora.py +++ b/mkosi/distributions/fedora.py @@ -10,7 +10,7 @@ from mkosi.distributions import ( PackageType, join_mirror, ) -from mkosi.installer.dnf import invoke_dnf, setup_dnf +from mkosi.installer.dnf import createrepo_dnf, invoke_dnf, setup_dnf from mkosi.installer.rpm import RpmRepository, find_rpm_gpgkey from mkosi.log import die @@ -40,6 +40,10 @@ class Installer(DistributionInstaller): def grub_prefix(cls) -> str: return "grub2" + @classmethod + def createrepo(cls, context: Context) -> None: + return createrepo_dnf(context) + @classmethod def setup(cls, context: Context) -> None: gpgurls = ( diff --git a/mkosi/distributions/mageia.py b/mkosi/distributions/mageia.py index 56324e063..a79bf5b2a 100644 --- a/mkosi/distributions/mageia.py +++ b/mkosi/distributions/mageia.py @@ -11,7 +11,7 @@ from mkosi.distributions import ( PackageType, join_mirror, ) -from mkosi.installer.dnf import invoke_dnf, setup_dnf +from mkosi.installer.dnf import createrepo_dnf, invoke_dnf, setup_dnf from mkosi.installer.rpm import RpmRepository, find_rpm_gpgkey from mkosi.log import die @@ -37,6 +37,10 @@ class Installer(DistributionInstaller): def default_tools_tree_distribution(cls) -> Distribution: return Distribution.mageia + @classmethod + def createrepo(cls, context: "Context") -> None: + return createrepo_dnf(context) + @classmethod def setup(cls, context: Context) -> None: gpgurls = ( diff --git a/mkosi/distributions/openmandriva.py b/mkosi/distributions/openmandriva.py index 935cf7958..df177e73a 100644 --- a/mkosi/distributions/openmandriva.py +++ b/mkosi/distributions/openmandriva.py @@ -11,7 +11,7 @@ from mkosi.distributions import ( PackageType, join_mirror, ) -from mkosi.installer.dnf import invoke_dnf, setup_dnf +from mkosi.installer.dnf import createrepo_dnf, invoke_dnf, setup_dnf from mkosi.installer.rpm import RpmRepository, find_rpm_gpgkey from mkosi.log import die @@ -37,6 +37,10 @@ class Installer(DistributionInstaller): def default_tools_tree_distribution(cls) -> Distribution: return Distribution.openmandriva + @classmethod + def createrepo(cls, context: "Context") -> None: + return createrepo_dnf(context) + @classmethod def setup(cls, context: Context) -> None: mirror = context.config.mirror or "http://mirror.openmandriva.org" diff --git a/mkosi/distributions/opensuse.py b/mkosi/distributions/opensuse.py index 8b5fbfae1..e893fc719 100644 --- a/mkosi/distributions/opensuse.py +++ b/mkosi/distributions/opensuse.py @@ -8,9 +8,9 @@ from pathlib import Path from mkosi.config import Architecture from mkosi.context import Context from mkosi.distributions import Distribution, DistributionInstaller, PackageType -from mkosi.installer.dnf import invoke_dnf, setup_dnf +from mkosi.installer.dnf import createrepo_dnf, invoke_dnf, setup_dnf from mkosi.installer.rpm import RpmRepository -from mkosi.installer.zypper import invoke_zypper, setup_zypper +from mkosi.installer.zypper import createrepo_zypper, invoke_zypper, setup_zypper from mkosi.log import die from mkosi.run import find_binary, run from mkosi.sandbox import finalize_crypto_mounts @@ -41,6 +41,13 @@ class Installer(DistributionInstaller): def grub_prefix(cls) -> str: return "grub2" + @classmethod + def createrepo(cls, context: "Context") -> None: + if find_binary("zypper", root=context.config.tools()): + createrepo_zypper(context) + else: + createrepo_dnf(context) + @classmethod def setup(cls, context: Context) -> None: release = context.config.release diff --git a/mkosi/installer/__init__.py b/mkosi/installer/__init__.py index 52dcc150c..3b52a02f6 100644 --- a/mkosi/installer/__init__.py +++ b/mkosi/installer/__init__.py @@ -70,6 +70,7 @@ def finalize_package_manager_mounts(context: Context) -> list[PathString]: *(["--ro-bind", m, m] if (m := context.config.local_mirror) else []), *(["--ro-bind", os.fspath(p), os.fspath(p)] if (p := context.workspace / "apt.conf").exists() else []), *finalize_crypto_mounts(tools=context.config.tools()), + "--bind", context.packages, "/work/packages", ] mounts += flatten( diff --git a/mkosi/installer/apt.py b/mkosi/installer/apt.py index bc8298049..60d54ee9a 100644 --- a/mkosi/installer/apt.py +++ b/mkosi/installer/apt.py @@ -119,3 +119,21 @@ def invoke_apt( ), env=context.config.environment, ) + + +def createrepo_apt(context: Context) -> None: + with (context.packages / "Packages").open("w") as f: + run(["dpkg-scanpackages", context.packages], + stdout=f, sandbox=context.sandbox(options=["--ro-bind", context.packages, context.packages])) + + (context.pkgmngr / "etc/apt/sources.list.d").mkdir(parents=True, exist_ok=True) + (context.pkgmngr / "etc/apt/sources.list.d/mkosi-packages.sources").write_text( + f"""\ + Enabled: yes + Types: deb + URIs: file:///work/packages + Suites: {context.config.release} + Components: main + Trusted: yes + """ + ) diff --git a/mkosi/installer/dnf.py b/mkosi/installer/dnf.py index 86c15c47e..a5adf2da7 100644 --- a/mkosi/installer/dnf.py +++ b/mkosi/installer/dnf.py @@ -150,3 +150,23 @@ def invoke_dnf(context: Context, command: str, packages: Iterable[str], apivfs: for p in (context.root / "var/log").iterdir(): if any(p.name.startswith(prefix) for prefix in ("dnf", "hawkey", "yum")): p.unlink() + + +def createrepo_dnf(context: Context) -> None: + run(["createrepo_c", context.packages], + sandbox=context.sandbox(options=["--bind", context.packages, context.packages])) + + (context.pkgmngr / "etc/yum.repos.d").mkdir(parents=True, exist_ok=True) + (context.pkgmngr / "etc/yum.repos.d/mkosi-packages.repo").write_text( + textwrap.dedent( + """\ + [mkosi-packages] + name=mkosi-packages + gpgcheck=0 + enabled=1 + baseurl=file:///work/packages + metadata_expire=0 + priority=50 + """ + ) + ) diff --git a/mkosi/installer/pacman.py b/mkosi/installer/pacman.py index 9028359ed..1669d6613 100644 --- a/mkosi/installer/pacman.py +++ b/mkosi/installer/pacman.py @@ -107,3 +107,17 @@ def invoke_pacman( ), env=context.config.environment, ) + + +def createrepo_pacman(context: Context) -> None: + run(["repo-add", context.packages / "mkosi-packages.db.tar", *context.packages.glob("*.pkg.tar*")]) + + with (context.pkgmngr / "etc/pacman.conf").open("a") as f: + f.write( + textwrap.dedent( + """\ + [mkosi-packages] + Server = file:///work/packages + """ + ) + ) diff --git a/mkosi/installer/zypper.py b/mkosi/installer/zypper.py index 708a71cd0..339815032 100644 --- a/mkosi/installer/zypper.py +++ b/mkosi/installer/zypper.py @@ -97,3 +97,23 @@ def invoke_zypper( ) fixup_rpmdb_location(context) + + +def createrepo_zypper(context: Context) -> None: + run(["createrepo_c", context.packages], + sandbox=context.sandbox(options=["--bind", context.packages, context.packages])) + + (context.pkgmngr / "etc/zypp/repos.d").mkdir(parents=True, exist_ok=True) + (context.pkgmngr / "etc/zypp/repos.d/mkosi-packages.repo").write_text( + textwrap.dedent( + """\ + [mkosi-packages] + name=mkosi-packages + gpgcheck=0 + enabled=1 + baseurl=file:///work/packages + autorefresh=0 + priority=50 + """ + ) + ) diff --git a/mkosi/resources/mkosi-tools/mkosi.conf.d/10-arch.conf b/mkosi/resources/mkosi-tools/mkosi.conf.d/10-arch.conf index 6ebc654d4..6edab862e 100644 --- a/mkosi/resources/mkosi-tools/mkosi.conf.d/10-arch.conf +++ b/mkosi/resources/mkosi-tools/mkosi.conf.d/10-arch.conf @@ -11,6 +11,7 @@ Packages= btrfs-progs curl debian-archive-keyring + dpkg edk2-ovmf erofs-utils grub diff --git a/mkosi/resources/mkosi-tools/mkosi.conf.d/10-centos-fedora/mkosi.conf b/mkosi/resources/mkosi-tools/mkosi.conf.d/10-centos-fedora/mkosi.conf index c1d042270..dd78d62c2 100644 --- a/mkosi/resources/mkosi-tools/mkosi.conf.d/10-centos-fedora/mkosi.conf +++ b/mkosi/resources/mkosi-tools/mkosi.conf.d/10-centos-fedora/mkosi.conf @@ -10,10 +10,12 @@ Distribution=|fedora [Content] Packages= apt + createrepo_c curl-minimal debian-keyring distribution-gpg-keys dnf-plugins-core + dpkg-dev grub2-tools openssh-clients policycoreutils diff --git a/mkosi/resources/mkosi-tools/mkosi.conf.d/10-debian-ubuntu.conf b/mkosi/resources/mkosi-tools/mkosi.conf.d/10-debian-ubuntu.conf index a506728fc..980db4041 100644 --- a/mkosi/resources/mkosi-tools/mkosi.conf.d/10-debian-ubuntu.conf +++ b/mkosi/resources/mkosi-tools/mkosi.conf.d/10-debian-ubuntu.conf @@ -10,8 +10,10 @@ Packages= apt archlinux-keyring btrfs-progs + createrepo-c curl debian-archive-keyring + dpkg-dev erofs-utils grub2 libtss2-dev diff --git a/mkosi/resources/mkosi-tools/mkosi.conf.d/10-opensuse.conf b/mkosi/resources/mkosi-tools/mkosi.conf.d/10-opensuse.conf index ce4040b99..25f874b6d 100644 --- a/mkosi/resources/mkosi-tools/mkosi.conf.d/10-opensuse.conf +++ b/mkosi/resources/mkosi-tools/mkosi.conf.d/10-opensuse.conf @@ -7,6 +7,7 @@ Distribution=opensuse Packages= btrfs-progs ca-certificates-mozilla + createrepo_c curl distribution-gpg-keys dnf-plugins-core diff --git a/mkosi/resources/mkosi.md b/mkosi/resources/mkosi.md index f8ce46917..7be757f08 100644 --- a/mkosi/resources/mkosi.md +++ b/mkosi/resources/mkosi.md @@ -865,6 +865,17 @@ boolean argument: either `1`, `yes`, or `true` to enable, or `0`, `no`, `mkosi.build` scripts require to operate. Note that packages listed here will be absent in the final image. +`PackageDirectories=`, `--package-directory=` + +: Specify directories containing extra packages to be made available during + the build. `mkosi` will create a local repository containing all + packages in these directories and make it available when installing packages or + running scripts. + +: Note that this local repository is also made available when running + scripts. Build scripts can add more packages to the local repository + by placing the built packages in `$PACKAGEDIR`. + `WithRecommends=`, `--with-recommends=` : Configures whether to install recommended or weak dependencies, @@ -1821,6 +1832,10 @@ Scripts executed by mkosi receive the following environment variables: artifacts generated during the build. `$CHROOT_OUTPUTDIR` contains the value that `$OUTPUTDIR` will have after invoking `mkosi-chroot`. +* `$PACKAGEDIR` points to the directory containing the local package + repository. Build scripts can add more packages to the local + repository by writing the packages to `$PACKAGEDIR`. + * `$BUILDROOT` is the root directory of the image being built, optionally with the build overlay mounted on top depending on the script that's being executed. diff --git a/tests/test_json.py b/tests/test_json.py index 232f64de3..061709b57 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -180,6 +180,7 @@ def test_config() -> None: "Output": "outfile", "OutputDirectory": "/your/output/here", "Overlay": true, + "PackageDirectories": [], "PackageManagerTrees": [ { "source": "/foo/bar", @@ -351,6 +352,7 @@ def test_config() -> None: output_dir = Path("/your/output/here"), output_format = OutputFormat.uki, overlay = True, + package_directories = [], package_manager_trees = [ConfigTree(Path("/foo/bar"), None)], packages = [], passphrase = None, -- 2.47.2