Introduce VolatilePackageDirectories=

For packages that should not be available when building the cached
image but should be available when installing volatile packages.

Additionally, we have build scripts write their packages to a separate
directory instead of the repository directory and later on pass these
packages as a separate volatile package directory to the default initrd
build.

This will allow us to introduce proper caching for package directories
in the next commit.

diff --git a/mkosi/__init__.py b/mkosi/__init__.py
index 0bbb42c1503a153682fd02290a1fb8f9594b209c..2ba97a922e7274f9b1236e7d4980520cf923e619 100644 (file)
--- a/mkosi/__init__.py
+++ b/mkosi/__init__.py
@@ -591,7 +591,7 @@ def run_prepare_scripts(context: Context, build: bool) -> None:
                             Mount(json, "/work/config.json", ro=True),
                             Mount(context.root, "/buildroot"),
                             Mount(context.artifacts, "/work/artifacts"),
-                            Mount(context.repository, "/work/packages"),
+                            Mount(context.packages, "/work/packages"),
                             *context.config.distribution.package_manager(context.config).mounts(context),
                         ],
                         options=["--dir", "/work/src", "--chdir", "/work/src"],
@@ -674,7 +674,7 @@ def run_build_scripts(context: Context) -> None:
                             Mount(context.install_dir, "/work/dest"),
                             Mount(context.staging, "/work/out"),
                             Mount(context.artifacts, "/work/artifacts"),
-                            Mount(context.repository, "/work/packages"),
+                            Mount(context.packages, "/work/packages"),
                             *(
                                 [Mount(context.config.build_dir, "/work/build")]
                                 if context.config.build_dir
@@ -688,10 +688,6 @@ def run_build_scripts(context: Context) -> None:
                     ),
                 )
 
-    if context.want_local_repo() and context.config.output_format != OutputFormat.none:
-        with complete_step("Rebuilding local package repository"):
-            context.config.distribution.createrepo(context)
-
 
 def run_postinst_scripts(context: Context) -> None:
     if not context.config.postinst_scripts:
@@ -752,7 +748,7 @@ def run_postinst_scripts(context: Context) -> None:
                             Mount(context.root, "/buildroot"),
                             Mount(context.staging, "/work/out"),
                             Mount(context.artifacts, "/work/artifacts"),
-                            Mount(context.repository, "/work/packages"),
+                            Mount(context.packages, "/work/packages"),
                             *context.config.distribution.package_manager(context.config).mounts(context),
                         ],
                         options=["--dir", "/work/src", "--chdir", "/work/src"],
@@ -819,7 +815,7 @@ def run_finalize_scripts(context: Context) -> None:
                             Mount(context.root, "/buildroot"),
                             Mount(context.staging, "/work/out"),
                             Mount(context.artifacts, "/work/artifacts"),
-                            Mount(context.repository, "/work/packages"),
+                            Mount(context.packages, "/work/packages"),
                             *context.config.distribution.package_manager(context.config).mounts(context),
                         ],
                         options=["--dir", "/work/src", "--chdir", "/work/src"],
@@ -1652,19 +1648,17 @@ def install_package_manager_trees(context: Context) -> None:
             install_tree(context.config, tree.source, context.pkgmngr, target=tree.target, preserve=False)
 
 
-def install_package_directories(context: Context) -> None:
-    if not context.config.package_directories:
+def install_package_directories(context: Context, directories: Sequence[Path]) -> None:
+    directories = [d for d in directories if any(d.iterdir())]
+
+    if not directories:
         return
 
     with complete_step("Copying in extra packages…"):
-        for d in context.config.package_directories:
+        for d in directories:
             for p in itertools.chain(*(d.glob(glob) for glob in PACKAGE_GLOBS)):
                 shutil.copy(p, context.repository, follow_symlinks=True)
 
-    if context.want_local_repo():
-        with complete_step("Building local package repository"):
-            context.config.distribution.createrepo(context)
-
 
 def install_extra_trees(context: Context) -> None:
     if not context.config.extra_trees:
@@ -1786,7 +1780,9 @@ def finalize_default_initrd(
         "--acl", str(config.acl),
         *(f"--package={package}" for package in config.initrd_packages),
         *(f"--volatile-package={package}" for package in config.initrd_volatile_packages),
-        *(["--package-directory", str(package_dir)] if package_dir else []),
+        *(f"--package-directory={d}" for d in config.package_directories),
+        *(f"--volatile-package-directory={d}" for d in config.volatile_package_directories),
+        *([f"--volatile-package-directory={package_dir}"] if package_dir else []),
         "--output", "initrd",
         *(["--image-id", config.image_id] if config.image_id else []),
         *(["--image-version", config.image_version] if config.image_version else []),
@@ -1834,7 +1830,7 @@ def build_default_initrd(context: Context) -> Path:
         context.config,
         resources=context.resources,
         output_dir=context.workspace,
-        package_dir=context.repository,
+        package_dir=context.packages,
     )
 
     assert config.output_dir
@@ -3818,6 +3814,16 @@ def copy_repository_metadata(context: Context) -> None:
                     sandbox=sandbox,
                 )
 
+@contextlib.contextmanager
+def createrepo(context: Context) -> Iterator[None]:
+    st = context.repository.stat()
+    try:
+        yield
+    finally:
+        if context.repository.stat().st_mtime_ns != st.st_mtime_ns:
+            with complete_step("Rebuilding local package repository"):
+                context.config.distribution.createrepo(context)
+
 
 def build_image(context: Context) -> None:
     manifest = Manifest(context) if context.config.manifest_format else None
@@ -3831,7 +3837,8 @@ def build_image(context: Context) -> None:
         copy_repository_metadata(context)
 
         context.config.distribution.setup(context)
-        install_package_directories(context)
+        with createrepo(context):
+            install_package_directories(context, context.config.package_directories)
 
         if not cached:
             install_skeleton_trees(context)
@@ -3853,6 +3860,10 @@ def build_image(context: Context) -> None:
             rmtree(context.root)
             return
 
+        with createrepo(context):
+            install_package_directories(context, context.config.volatile_package_directories)
+            install_package_directories(context, [context.packages])
+
         install_volatile_packages(context)
         install_build_dest(context)
         install_extra_trees(context)

diff --git a/mkosi/config.py b/mkosi/config.py
index acb0f1f5acd483316f81f19d168f03e95d990986..b8bcdd4be76a7ba752564367cc4af94b0e2ebd42 100644 (file)
--- a/mkosi/config.py
+++ b/mkosi/config.py
@@ -1415,6 +1415,7 @@ class Config:
     build_packages: list[str]
     volatile_packages: list[str]
     package_directories: list[Path]
+    volatile_package_directories: list[Path]
     with_recommends: bool
     with_docs: bool
 
@@ -2190,6 +2191,15 @@ SETTINGS = (
         help="Specify a directory containing extra packages",
         universal=True,
     ),
+    ConfigSetting(
+        dest="volatile_package_directories",
+        long="--volatile-package-directory",
+        metavar="PATH",
+        section="Content",
+        parse=config_make_list_parser(delimiter=",", parse=make_path_parser()),
+        help="Specify a directory containing extra volatile packages",
+        universal=True,
+    ),
     ConfigSetting(
         dest="with_recommends",
         metavar="BOOL",
@@ -4048,6 +4058,8 @@ def summary(config: Config) -> str:
                            Packages: {line_join_list(config.packages)}
                      Build Packages: {line_join_list(config.build_packages)}
                   Volatile Packages: {line_join_list(config.volatile_packages)}
+                Package Directories: {line_join_list(config.package_directories)}
+       Volatile Package Directories: {line_join_list(config.volatile_package_directories)}
                  With Documentation: {yes_no(config.with_docs)}
 
                          Base Trees: {line_join_list(config.base_trees)}

diff --git a/mkosi/context.py b/mkosi/context.py
index 2bbec125858836784399de5fe6624dff570e41f1..761358130918b77f11976cafeba414c21b3c2c4e 100644 (file)
--- a/mkosi/context.py
+++ b/mkosi/context.py
@@ -45,6 +45,7 @@ class Context:
         self.staging.mkdir()
         self.pkgmngr.mkdir()
         self.repository.mkdir()
+        self.packages.mkdir()
         self.artifacts.mkdir()
         self.install_dir.mkdir()
 
@@ -64,6 +65,10 @@ class Context:
     def repository(self) -> Path:
         return self.workspace / "repository"
 
+    @property
+    def packages(self) -> Path:
+        return self.workspace / "packages"
+
     @property
     def artifacts(self) -> Path:
         return self.workspace / "artifacts"
@@ -115,6 +120,3 @@ class Context:
             ],
             extra=extra,
         )
-
-    def want_local_repo(self) -> bool:
-        return any(self.repository.iterdir())

diff --git a/mkosi/resources/mkosi.md b/mkosi/resources/mkosi.md
index 3ce7af2ead355692e95eba433d43a2f748f92470..ef348078323febe7eaa9bc17bcc605ff226bbec4 100644 (file)
--- a/mkosi/resources/mkosi.md
+++ b/mkosi/resources/mkosi.md
@@ -772,9 +772,15 @@ boolean argument: either `1`, `yes`, or `true` to enable, or `0`, `no`,
     running scripts. If the `mkosi.packages/` directory is found in the local
     directory it is also used for this purpose.
 
-    Note that this local repository is also made available when running
-    scripts. Build scripts can add more packages to the local repository
-    by placing the built packages in `$PACKAGEDIR`.
+    Build scripts can add more packages to the local repository by
+    placing the built packages in `$PACKAGEDIR`.
+
+`VolatilePackageDirectories=`, `--volatile-package-directory=`
+
+:   Like `PackageDirectories=`, but the packages in these directories
+    are only made available in the local repository just before volatile
+    packages are installed. Specifically, if `Incremental=` is enabled,
+    the packages from these directories will not be cached.
 
 `WithRecommends=`, `--with-recommends=`
 :   Configures whether to install recommended or weak dependencies,
@@ -2470,6 +2476,7 @@ overridden):
 - `RepartOffline=`
 - `UseSubvolumes=`
 - `PackageDirectories=`
+- `VolatilePackageDirectories=`
 - `SourceDateEpoch=`
 - `VerityKey=`
 - `VerityKeySource=`

diff --git a/tests/test_json.py b/tests/test_json.py
index 60b3a2a9a4368d6ac35ae67c93f8b3a62ddec407..45145d9fa4039ecf0bde48bc8d85019e7d390aa2 100644 (file)
--- a/tests/test_json.py
+++ b/tests/test_json.py
@@ -342,6 +342,9 @@ def test_config() -> None:
                 "Type": "file"
             },
             "VirtualMachineMonitor": "qemu",
+            "VolatilePackageDirectories": [
+                "def"
+            ],
             "VolatilePackages": [
                 "abc"
             ],
@@ -500,6 +503,7 @@ def test_config() -> None:
         verity_certificate=Path("/path/to/cert"),
         verity_key=None,
         verity_key_source=KeySource(type=KeySource.Type.file),
+        volatile_package_directories=[Path("def")],
         volatile_packages=["abc"],
         with_docs=True,
         with_network=False,