]> git.ipfire.org Git - thirdparty/mkosi.git/commitdiff
Add PackageDirectories= 2287/head
authorDaan De Meyer <daan.j.demeyer@gmail.com>
Mon, 15 Jan 2024 21:24:08 +0000 (22:24 +0100)
committerDaan De Meyer <daan.j.demeyer@gmail.com>
Tue, 16 Jan 2024 11:19:03 +0000 (12:19 +0100)
Let's make it possible to serve local packages as a local repository
so that users don't have to put local paths in their Packages= setting.
We'll also allow adding more packages to this local repository in the
build script so that these can be installed in the initrd when we build
it or in a postinst or finalize script.

22 files changed:
mkosi/__init__.py
mkosi/config.py
mkosi/context.py
mkosi/distributions/__init__.py
mkosi/distributions/arch.py
mkosi/distributions/centos.py
mkosi/distributions/debian.py
mkosi/distributions/fedora.py
mkosi/distributions/mageia.py
mkosi/distributions/openmandriva.py
mkosi/distributions/opensuse.py
mkosi/installer/__init__.py
mkosi/installer/apt.py
mkosi/installer/dnf.py
mkosi/installer/pacman.py
mkosi/installer/zypper.py
mkosi/resources/mkosi-tools/mkosi.conf.d/10-arch.conf
mkosi/resources/mkosi-tools/mkosi.conf.d/10-centos-fedora/mkosi.conf
mkosi/resources/mkosi-tools/mkosi.conf.d/10-debian-ubuntu.conf
mkosi/resources/mkosi-tools/mkosi.conf.d/10-opensuse.conf
mkosi/resources/mkosi.md
tests/test_json.py

index 41569f0eabb6d4d0065304eb2f9ae1f31e5d98c9..63038d5559085ad96547c2a81c97bc2b8f462129 100644 (file)
@@ -411,12 +411,13 @@ def run_prepare_scripts(context: Context, build: bool) -> None:
     env = dict(
         ARCHITECTURE=str(context.config.architecture),
         BUILDROOT=str(context.root),
-        CHROOT_SCRIPT="/work/prepare",
+        SRCDIR="/work/src",
         CHROOT_SRCDIR="/work/src",
+        PACKAGEDIR="/work/packages",
+        SCRIPT="/work/prepare",
+        CHROOT_SCRIPT="/work/prepare",
         MKOSI_UID=str(INVOKING_USER.uid),
         MKOSI_GID=str(INVOKING_USER.gid),
-        SCRIPT="/work/prepare",
-        SRCDIR="/work/src",
         WITH_DOCS=one_zero(context.config.with_docs),
         WITH_NETWORK=one_zero(context.config.with_network),
         WITH_TESTS=one_zero(context.config.with_tests),
@@ -485,6 +486,7 @@ def run_build_scripts(context: Context) -> None:
         CHROOT_OUTPUTDIR="/work/out",
         SRCDIR="/work/src",
         CHROOT_SRCDIR="/work/src",
+        PACKAGEDIR="/work/packages",
         SCRIPT="/work/build-script",
         CHROOT_SCRIPT="/work/build-script",
         MKOSI_UID=str(INVOKING_USER.uid),
@@ -552,6 +554,10 @@ def run_build_scripts(context: Context) -> None:
                     ) + (chroot if script.suffix == ".chroot" else []),
                 )
 
+    if any(context.packages.iterdir()):
+        with complete_step("Rebuilding local package repository"):
+            context.config.distribution.createrepo(context)
+
 
 def run_postinst_scripts(context: Context) -> None:
     if not context.config.postinst_scripts:
@@ -566,6 +572,7 @@ def run_postinst_scripts(context: Context) -> None:
         CHROOT_SCRIPT="/work/postinst",
         SRCDIR="/work/src",
         CHROOT_SRCDIR="/work/src",
+        PACKAGEDIR="/work/packages",
         MKOSI_UID=str(INVOKING_USER.uid),
         MKOSI_GID=str(INVOKING_USER.gid),
     )
@@ -624,6 +631,7 @@ def run_finalize_scripts(context: Context) -> None:
         CHROOT_OUTPUTDIR="/work/out",
         SRCDIR="/work/src",
         CHROOT_SRCDIR="/work/src",
+        PACKAGEDIR="/work/packages",
         SCRIPT="/work/finalize",
         CHROOT_SCRIPT="/work/finalize",
         MKOSI_UID=str(INVOKING_USER.uid),
@@ -1385,6 +1393,19 @@ def install_package_manager_trees(context: Context) -> None:
             install_tree(context, tree.source, context.pkgmngr, target=tree.target, preserve=False)
 
 
+def install_package_directories(context: Context) -> None:
+    if not context.config.package_directories:
+        return
+
+    with complete_step("Copying in extra packages…"):
+        for d in context.config.package_directories:
+            install_tree(context, d, context.packages)
+
+    if any(context.packages.iterdir()):
+        with complete_step("Initializing local package repository…"):
+            context.config.distribution.createrepo(context)
+
+
 def install_extra_trees(context: Context) -> None:
     if not context.config.extra_trees:
         return
@@ -1459,6 +1480,7 @@ def build_initrd(context: Context) -> Path:
         "--incremental", str(context.config.incremental),
         "--acl", str(context.config.acl),
         *(f"--package={package}" for package in context.config.initrd_packages),
+        "--package-directory", str(context.packages),
         "--output", f"{context.config.output}-initrd",
         *(["--image-id", context.config.image_id] if context.config.image_id else []),
         *(["--image-version", context.config.image_version] if context.config.image_version else []),
@@ -2799,6 +2821,7 @@ def build_image(args: Args, config: Config) -> None:
     with setup_workspace(args, config) as workspace:
         context = Context(args, config, workspace)
         install_package_manager_trees(context)
+        install_package_directories(context)
 
         with mount_base_trees(context):
             install_base_trees(context)
index 451eefb8b0b81b80bc0426140368846d17e0d350..245d95e1a9c0630a4388d7558e431d63f77684ab 100644 (file)
@@ -1108,6 +1108,7 @@ class Config:
 
     packages: list[str]
     build_packages: list[str]
+    package_directories: list[Path]
     with_recommends: bool
     with_docs: bool
 
@@ -1746,6 +1747,14 @@ SETTINGS = (
         parse=config_make_list_parser(delimiter=","),
         help="Additional packages needed for build scripts",
     ),
+    ConfigSetting(
+        dest="package_directories",
+        long="--package-directory",
+        metavar="PATH",
+        section="Content",
+        parse=config_make_list_parser(delimiter=",", parse=make_path_parser()),
+        help="Specify a directory containing extra packages",
+    ),
     ConfigSetting(
         dest="with_recommends",
         metavar="BOOL",
index ba4eb8ce70fa5f0f0eb460e2690b895748925ee9..959e8880d84a48e3244a0ae0f333fdaa85608b57 100644 (file)
@@ -34,6 +34,7 @@ class Context:
 
         self.staging.mkdir()
         self.pkgmngr.mkdir()
+        self.packages.mkdir()
         self.install_dir.mkdir(exist_ok=True)
         self.cache_dir.mkdir(parents=True, exist_ok=True)
 
@@ -49,6 +50,10 @@ class Context:
     def pkgmngr(self) -> Path:
         return self.workspace / "pkgmngr"
 
+    @property
+    def packages(self) -> Path:
+        return self.workspace / "packages"
+
     @property
     def cache_dir(self) -> Path:
         return self.config.cache_dir or (self.workspace / "cache")
index 2c4374ae62143752394ba27236cb48a153d1628f..3dd15e81c3d4af538f28cb62ed19c0651d945515 100644 (file)
@@ -67,6 +67,10 @@ class DistributionInstaller:
     def grub_prefix(cls) -> str:
         return "grub"
 
+    @classmethod
+    def createrepo(cls, context: "Context") -> None:
+        raise NotImplementedError
+
 
 class Distribution(StrEnum):
     # Please consult docs/distribution-policy.md and contact one
@@ -143,6 +147,9 @@ class Distribution(StrEnum):
     def grub_prefix(self) -> str:
         return self.installer().grub_prefix()
 
+    def createrepo(self, context: "Context") -> None:
+        return self.installer().createrepo(context)
+
     def installer(self) -> type[DistributionInstaller]:
         modname = str(self).replace('-', '_')
         mod = importlib.import_module(f"mkosi.distributions.{modname}")
index 697a34b526b2d2fc2ab2a836f6ba962b3d9c2c0c..2d4a4b388882e2a0edf724e5005739d555b2cb6d 100644 (file)
@@ -5,7 +5,12 @@ from collections.abc import Sequence
 from mkosi.config import Architecture
 from mkosi.context import Context
 from mkosi.distributions import Distribution, DistributionInstaller, PackageType
-from mkosi.installer.pacman import PacmanRepository, invoke_pacman, setup_pacman
+from mkosi.installer.pacman import (
+    PacmanRepository,
+    createrepo_pacman,
+    invoke_pacman,
+    setup_pacman,
+)
 from mkosi.log import die
 
 
@@ -30,6 +35,10 @@ class Installer(DistributionInstaller):
     def default_tools_tree_distribution(cls) -> Distribution:
         return Distribution.arch
 
+    @classmethod
+    def createrepo(cls, context: "Context") -> None:
+        return createrepo_pacman(context)
+
     @classmethod
     def setup(cls, context: Context) -> None:
         if context.config.local_mirror:
index 0e389f1002085ce5f14d99533dc9eff5efa3a2c8..0dc252329f940aaac1485507950bac8cf2824ad7 100644 (file)
@@ -12,7 +12,7 @@ from mkosi.distributions import (
     PackageType,
     join_mirror,
 )
-from mkosi.installer.dnf import invoke_dnf, setup_dnf
+from mkosi.installer.dnf import createrepo_dnf, invoke_dnf, setup_dnf
 from mkosi.installer.rpm import RpmRepository, find_rpm_gpgkey
 from mkosi.log import complete_step, die
 from mkosi.tree import rmtree
@@ -58,6 +58,10 @@ class Installer(DistributionInstaller):
     def grub_prefix(cls) -> str:
         return "grub2"
 
+    @classmethod
+    def createrepo(cls, context: Context) -> None:
+        return createrepo_dnf(context)
+
     @classmethod
     def setup(cls, context: Context) -> None:
         if GenericVersion(context.config.release) <= 7:
index 8c56722f9cd1226ebd536b3805546b9704351291..43314825f54624fe2694124c4e9146cb54a182d1 100644 (file)
@@ -9,7 +9,7 @@ from mkosi.archive import extract_tar
 from mkosi.config import Architecture
 from mkosi.context import Context
 from mkosi.distributions import Distribution, DistributionInstaller, PackageType
-from mkosi.installer.apt import invoke_apt, setup_apt
+from mkosi.installer.apt import createrepo_apt, invoke_apt, setup_apt
 from mkosi.log import die
 from mkosi.run import run
 from mkosi.util import umask
@@ -77,6 +77,10 @@ class Installer(DistributionInstaller):
     def setup(cls, context: Context) -> None:
         setup_apt(context, cls.repositories(context))
 
+    @classmethod
+    def createrepo(cls, context: "Context") -> None:
+        return createrepo_apt(context)
+
     @classmethod
     def install(cls, context: Context) -> None:
         # Instead of using debootstrap, we replicate its core functionality here. Because dpkg does not have
index dc865819d5daf6244e25e4b1ae61ad775dbb51ee..16ccb49e7af20650a71320823546a84e7f263c93 100644 (file)
@@ -10,7 +10,7 @@ from mkosi.distributions import (
     PackageType,
     join_mirror,
 )
-from mkosi.installer.dnf import invoke_dnf, setup_dnf
+from mkosi.installer.dnf import createrepo_dnf, invoke_dnf, setup_dnf
 from mkosi.installer.rpm import RpmRepository, find_rpm_gpgkey
 from mkosi.log import die
 
@@ -40,6 +40,10 @@ class Installer(DistributionInstaller):
     def grub_prefix(cls) -> str:
         return "grub2"
 
+    @classmethod
+    def createrepo(cls, context: Context) -> None:
+        return createrepo_dnf(context)
+
     @classmethod
     def setup(cls, context: Context) -> None:
         gpgurls = (
index 56324e063f4692b1e6efb31e552fb38e40d2e163..a79bf5b2ac369800d510e7ec624831a4ac17714c 100644 (file)
@@ -11,7 +11,7 @@ from mkosi.distributions import (
     PackageType,
     join_mirror,
 )
-from mkosi.installer.dnf import invoke_dnf, setup_dnf
+from mkosi.installer.dnf import createrepo_dnf, invoke_dnf, setup_dnf
 from mkosi.installer.rpm import RpmRepository, find_rpm_gpgkey
 from mkosi.log import die
 
@@ -37,6 +37,10 @@ class Installer(DistributionInstaller):
     def default_tools_tree_distribution(cls) -> Distribution:
         return Distribution.mageia
 
+    @classmethod
+    def createrepo(cls, context: "Context") -> None:
+        return createrepo_dnf(context)
+
     @classmethod
     def setup(cls, context: Context) -> None:
         gpgurls = (
index 935cf79586499262f67186dbb8cb73041e952334..df177e73ad34a2857170d576310a685ec138975e 100644 (file)
@@ -11,7 +11,7 @@ from mkosi.distributions import (
     PackageType,
     join_mirror,
 )
-from mkosi.installer.dnf import invoke_dnf, setup_dnf
+from mkosi.installer.dnf import createrepo_dnf, invoke_dnf, setup_dnf
 from mkosi.installer.rpm import RpmRepository, find_rpm_gpgkey
 from mkosi.log import die
 
@@ -37,6 +37,10 @@ class Installer(DistributionInstaller):
     def default_tools_tree_distribution(cls) -> Distribution:
         return Distribution.openmandriva
 
+    @classmethod
+    def createrepo(cls, context: "Context") -> None:
+        return createrepo_dnf(context)
+
     @classmethod
     def setup(cls, context: Context) -> None:
         mirror = context.config.mirror or "http://mirror.openmandriva.org"
index 8b5fbfae1667d267e6408176c8bbe8f8a1a480ea..e893fc7195aaee29b0b4e47d9f5810627d44db78 100644 (file)
@@ -8,9 +8,9 @@ from pathlib import Path
 from mkosi.config import Architecture
 from mkosi.context import Context
 from mkosi.distributions import Distribution, DistributionInstaller, PackageType
-from mkosi.installer.dnf import invoke_dnf, setup_dnf
+from mkosi.installer.dnf import createrepo_dnf, invoke_dnf, setup_dnf
 from mkosi.installer.rpm import RpmRepository
-from mkosi.installer.zypper import invoke_zypper, setup_zypper
+from mkosi.installer.zypper import createrepo_zypper, invoke_zypper, setup_zypper
 from mkosi.log import die
 from mkosi.run import find_binary, run
 from mkosi.sandbox import finalize_crypto_mounts
@@ -41,6 +41,13 @@ class Installer(DistributionInstaller):
     def grub_prefix(cls) -> str:
         return "grub2"
 
+    @classmethod
+    def createrepo(cls, context: "Context") -> None:
+        if find_binary("zypper", root=context.config.tools()):
+            createrepo_zypper(context)
+        else:
+            createrepo_dnf(context)
+
     @classmethod
     def setup(cls, context: Context) -> None:
         release = context.config.release
index 52dcc150c556f96bfead6f0d4af35142c4647f35..3b52a02f6c903bd46d4628fceb0bdd482c2c2bdf 100644 (file)
@@ -70,6 +70,7 @@ def finalize_package_manager_mounts(context: Context) -> list[PathString]:
         *(["--ro-bind", m, m] if (m := context.config.local_mirror) else []),
         *(["--ro-bind", os.fspath(p), os.fspath(p)] if (p := context.workspace / "apt.conf").exists() else []),
         *finalize_crypto_mounts(tools=context.config.tools()),
+        "--bind", context.packages, "/work/packages",
     ]
 
     mounts += flatten(
index bc8298049d0de8e8726b27675abc20a13ae84ce6..60d54ee9a5dbd5d4c30463fb6534a4ad10faf022 100644 (file)
@@ -119,3 +119,21 @@ def invoke_apt(
             ),
             env=context.config.environment,
         )
+
+
+def createrepo_apt(context: Context) -> None:
+    with (context.packages / "Packages").open("w") as f:
+        run(["dpkg-scanpackages", context.packages],
+            stdout=f, sandbox=context.sandbox(options=["--ro-bind", context.packages, context.packages]))
+
+    (context.pkgmngr / "etc/apt/sources.list.d").mkdir(parents=True, exist_ok=True)
+    (context.pkgmngr / "etc/apt/sources.list.d/mkosi-packages.sources").write_text(
+        f"""\
+        Enabled: yes
+        Types: deb
+        URIs: file:///work/packages
+        Suites: {context.config.release}
+        Components: main
+        Trusted: yes
+        """
+    )
index 86c15c47eafcece323461f02fd7cf571593fa7d4..a5adf2da7b541a80923345c14abf82779b5d928a 100644 (file)
@@ -150,3 +150,23 @@ def invoke_dnf(context: Context, command: str, packages: Iterable[str], apivfs:
     for p in (context.root / "var/log").iterdir():
         if any(p.name.startswith(prefix) for prefix in ("dnf", "hawkey", "yum")):
             p.unlink()
+
+
+def createrepo_dnf(context: Context) -> None:
+    run(["createrepo_c", context.packages],
+        sandbox=context.sandbox(options=["--bind", context.packages, context.packages]))
+
+    (context.pkgmngr / "etc/yum.repos.d").mkdir(parents=True, exist_ok=True)
+    (context.pkgmngr / "etc/yum.repos.d/mkosi-packages.repo").write_text(
+        textwrap.dedent(
+            """\
+            [mkosi-packages]
+            name=mkosi-packages
+            gpgcheck=0
+            enabled=1
+            baseurl=file:///work/packages
+            metadata_expire=0
+            priority=50
+            """
+        )
+    )
index 9028359ed00b3e1852b7c14b1662891c526af352..1669d6613b416cfd980fb227e238b5684bcd227b 100644 (file)
@@ -107,3 +107,17 @@ def invoke_pacman(
             ),
             env=context.config.environment,
         )
+
+
+def createrepo_pacman(context: Context) -> None:
+    run(["repo-add", context.packages / "mkosi-packages.db.tar", *context.packages.glob("*.pkg.tar*")])
+
+    with (context.pkgmngr / "etc/pacman.conf").open("a") as f:
+        f.write(
+            textwrap.dedent(
+                """\
+                [mkosi-packages]
+                Server = file:///work/packages
+                """
+            )
+        )
index 708a71cd072375321b34b05fe4b4b89e62ed0d53..339815032798b95ea652e6ac461bd8dbb8a7d629 100644 (file)
@@ -97,3 +97,23 @@ def invoke_zypper(
         )
 
     fixup_rpmdb_location(context)
+
+
+def createrepo_zypper(context: Context) -> None:
+    run(["createrepo_c", context.packages],
+        sandbox=context.sandbox(options=["--bind", context.packages, context.packages]))
+
+    (context.pkgmngr / "etc/zypp/repos.d").mkdir(parents=True, exist_ok=True)
+    (context.pkgmngr / "etc/zypp/repos.d/mkosi-packages.repo").write_text(
+        textwrap.dedent(
+            """\
+            [mkosi-packages]
+            name=mkosi-packages
+            gpgcheck=0
+            enabled=1
+            baseurl=file:///work/packages
+            autorefresh=0
+            priority=50
+            """
+        )
+    )
index 6ebc654d45b796f94e7aad875d6c98b22c3f3d51..6edab862ed95ab0d3cb6f3cc46725a34e1a2b5ab 100644 (file)
@@ -11,6 +11,7 @@ Packages=
         btrfs-progs
         curl
         debian-archive-keyring
+        dpkg
         edk2-ovmf
         erofs-utils
         grub
index c1d0422704ed8b931a643372204a524ec3800669..dd78d62c2dac0c3e0615a795fc7814d44b4fa2f4 100644 (file)
@@ -10,10 +10,12 @@ Distribution=|fedora
 [Content]
 Packages=
         apt
+        createrepo_c
         curl-minimal
         debian-keyring
         distribution-gpg-keys
         dnf-plugins-core
+        dpkg-dev
         grub2-tools
         openssh-clients
         policycoreutils
index a506728fc38b9a859414671ec3e62abec12cc83e..980db404100fc36701270c2b309a328fa7fb50b7 100644 (file)
@@ -10,8 +10,10 @@ Packages=
         apt
         archlinux-keyring
         btrfs-progs
+        createrepo-c
         curl
         debian-archive-keyring
+        dpkg-dev
         erofs-utils
         grub2
         libtss2-dev
index ce4040b9905473d1e62cba7f3512a06648aee657..25f874b6d6c3149bf8b235f5d389c4a71152afb0 100644 (file)
@@ -7,6 +7,7 @@ Distribution=opensuse
 Packages=
         btrfs-progs
         ca-certificates-mozilla
+        createrepo_c
         curl
         distribution-gpg-keys
         dnf-plugins-core
index f8ce46917ca43acd6d160ef2cd9f2c102ee5198b..7be757f0879dd717e34d4be8bb6a25636cfc39f6 100644 (file)
@@ -865,6 +865,17 @@ boolean argument: either `1`, `yes`, or `true` to enable, or `0`, `no`,
   `mkosi.build` scripts require to operate. Note that packages listed
   here will be absent in the final image.
 
+`PackageDirectories=`, `--package-directory=`
+
+: Specify directories containing extra packages to be made available during
+  the build. `mkosi` will create a local repository containing all
+  packages in these directories and make it available when installing packages or
+  running scripts.
+
+: Note that this local repository is also made available when running
+  scripts. Build scripts can add more packages to the local repository
+  by placing the built packages in `$PACKAGEDIR`.
+
 `WithRecommends=`, `--with-recommends=`
 
 : Configures whether to install recommended or weak dependencies,
@@ -1821,6 +1832,10 @@ Scripts executed by mkosi receive the following environment variables:
   artifacts generated during the build. `$CHROOT_OUTPUTDIR` contains the
   value that `$OUTPUTDIR` will have after invoking `mkosi-chroot`.
 
+* `$PACKAGEDIR` points to the directory containing the local package
+  repository. Build scripts can add more packages to the local
+  repository by writing the packages to `$PACKAGEDIR`.
+
 * `$BUILDROOT` is the root directory of the image being built,
   optionally with the build overlay mounted on top depending on the
   script that's being executed.
index 232f64de3880946d53b0a208167cd027f5451b23..061709b57a42ab842be6ae1caa47e73c5e67dce0 100644 (file)
@@ -180,6 +180,7 @@ def test_config() -> None:
             "Output": "outfile",
             "OutputDirectory": "/your/output/here",
             "Overlay": true,
+            "PackageDirectories": [],
             "PackageManagerTrees": [
                 {
                     "source": "/foo/bar",
@@ -351,6 +352,7 @@ def test_config() -> None:
         output_dir = Path("/your/output/here"),
         output_format = OutputFormat.uki,
         overlay = True,
+        package_directories = [],
         package_manager_trees = [ConfigTree(Path("/foo/bar"), None)],
         packages = [],
         passphrase = None,