Rename use_mirror_verbatim to local_mirror, support Debian/Arch/SUSE

With some distros the mirror has to be configured inside the image
(apt, zypper), so having a boolean flag is not enough.
We need to use the local mirror only for the local image build, but
configure the actual remote repositories in the image configuration,
so that it can be used once installed.

diff --git a/mkosi.md b/mkosi.md
index 66942022f39304a1bf49ff4a74a70ecff4e084cd..bdfcb55f60e8b32580f4ecde2b3413717ee5c0b6 100644 (file)
--- a/mkosi.md
+++ b/mkosi.md
@@ -315,12 +315,15 @@ a boolean argument: either "1", "yes", or "true" to enable, or "0",
 : The mirror to use for downloading the distribution packages. Expects
   a mirror URL as argument.
 
-`UseMirrorVerbatim=`, `--use-mirror-verbatim=`
+`LocalMirror=`, `--local-mirror=`
 
-: When `--mirror` is used it will be used as a plain and direct mirror instead
+: The mirror will be used as a local, plain and direct mirror instead
   of using it as a prefix for the full set of repositories normally supported
-  by distributions like Fedora and CentOS. Useful for fully offline builds with
-  a single repository. Only supported for RPM-based distributions for now.
+  by distributions. Useful for fully offline builds with a single repository.
+  Supported on deb/rpm/arch based distributions. Overrides `--mirror=` but only
+  for the local mkosi build, it will not be configured inside the final image,
+  `--mirror=` (or the default repository) will be configured inside the final
+  image instead.
 
 `Repositories=`, `--repositories=`
 

diff --git a/mkosi/__init__.py b/mkosi/__init__.py
index 5cd0e631c81059c46677d3c91a07076f6764b3fe..871bc7ec893ade1298c078924580059ef77a0a1f 100644 (file)
--- a/mkosi/__init__.py
+++ b/mkosi/__init__.py
@@ -1969,7 +1969,8 @@ def invoke_dnf(
     if args.repositories:
         cmdline += ["--disablerepo=*"] + [f"--enablerepo={repo}" for repo in args.repositories]
 
-    if args.with_network == "never":
+    # TODO: this breaks with a local, offline repository created with 'createrepo'
+    if args.with_network == "never" and not args.local_mirror:
         cmdline += ["-C"]
 
     if not args.architecture_is_native():
@@ -2095,8 +2096,8 @@ def parse_fedora_release(release: str) -> Tuple[str, str]:
 def install_fedora(args: MkosiArgs, root: Path, do_run_build_script: bool) -> None:
     release, releasever = parse_fedora_release(args.release)
 
-    if args.use_mirror_verbatim and args.mirror:
-        release_url = f"baseurl={args.mirror}"
+    if args.local_mirror:
+        release_url = f"baseurl={args.local_mirror}"
         updates_url = None
     elif args.mirror:
         baseurl = urllib.parse.urljoin(args.mirror, f"releases/{release}/Everything/$basearch/os/")
@@ -2159,8 +2160,8 @@ def install_fedora(args: MkosiArgs, root: Path, do_run_build_script: bool) -> No
 
 @complete_step("Installing Mageia…")
 def install_mageia(args: MkosiArgs, root: Path, do_run_build_script: bool) -> None:
-    if args.use_mirror_verbatim and args.mirror:
-        release_url = f"baseurl={args.mirror}"
+    if args.local_mirror:
+        release_url = f"baseurl={args.local_mirror}"
         updates_url = None
     elif args.mirror:
         baseurl = f"{args.mirror}/distrib/{args.release}/x86_64/media/core/"
@@ -2208,8 +2209,8 @@ def install_openmandriva(args: MkosiArgs, root: Path, do_run_build_script: bool)
     else:
         release_model = release
 
-    if args.use_mirror_verbatim and args.mirror:
-        release_url = f"baseurl={args.mirror}"
+    if args.local_mirror:
+        release_url = f"baseurl={args.local_mirror}"
         updates_url = None
     elif args.mirror:
         baseurl = f"{args.mirror}/{release_model}/repository/{args.architecture}/main"
@@ -2305,8 +2306,8 @@ def install_centos_variant_repos(args: MkosiArgs, root: Path, epel_release: int)
     gpgpath, gpgurl = centos_variant_gpg_locations(args.distribution, epel_release)
     epel_gpgpath, epel_gpgurl = epel_gpg_locations(epel_release)
 
-    if args.use_mirror_verbatim and args.mirror:
-        appstream_url = f"baseurl={args.mirror}"
+    if args.local_mirror:
+        appstream_url = f"baseurl={args.local_mirror}"
         baseos_url = extras_url = powertools_url = epel_url = None
     elif args.mirror:
         appstream_url = f"baseurl={args.mirror}/{directory}/{args.release}/AppStream/$basearch/os"
@@ -2342,8 +2343,8 @@ def install_centos_stream_repos(args: MkosiArgs, root: Path, epel_release: int)
 
     release = f"{epel_release}-stream"
 
-    if args.use_mirror_verbatim and args.mirror:
-        appstream_url = f"baseurl={args.mirror}"
+    if args.local_mirror:
+        appstream_url = f"baseurl={args.local_mirror}"
         baseos_url = crb_url = epel_url = None
     elif args.mirror:
         appstream_url = f"baseurl={args.mirror}/centos-stream/{release}/AppStream/$basearch/os"
@@ -2425,6 +2426,22 @@ def debootstrap_knows_arg(arg: str) -> bool:
     return bytes("invalid option", "UTF-8") not in run(["debootstrap", arg], stdout=PIPE, check=False).stdout
 
 
+@contextlib.contextmanager
+def mount_apt_local_mirror(args: MkosiArgs, root: Path) -> Iterator[None]:
+    # Ensure apt inside the image can see the local mirror outside of it
+    mirror = args.local_mirror or args.mirror
+    if not mirror or not mirror.startswith("file:"):
+        yield
+        return
+
+    # Strip leading '/' as Path() does not behave well when concatenating
+    mirror_dir = mirror[5:].lstrip("/")
+
+    with complete_step("Mounting apt local mirror…", "Unmounting apt local mirror…"):
+        with mount_bind(Path("/") / mirror_dir, root / mirror_dir):
+            yield
+
+
 def invoke_apt(
     args: MkosiArgs,
     root: Path,
@@ -2449,10 +2466,30 @@ def invoke_apt(
 
     # Overmount /etc/apt on the host with an empty directory, so that apt doesn't parse any configuration
     # from it.
-    with mount_bind(workspace(root) / "apt", Path("/") / "etc/apt"), mount_api_vfs(args, root):
+    with mount_bind(workspace(root) / "apt", Path("/") / "etc/apt"), mount_apt_local_mirror(args, root), mount_api_vfs(args, root):
         return run(cmdline, env=env, text=True, **kwargs)
 
 
+def add_apt_auxiliary_repos(args: MkosiArgs, root: Path, repos: Set[str]) -> None:
+    if args.release in ("unstable", "sid"):
+        return
+
+    if args.distribution == Distribution.ubuntu:
+        updates = f"deb http://archive.ubuntu.com/ubuntu {args.release}-updates {' '.join(repos)}"
+    else:
+        updates = f"deb http://deb.debian.org/debian {args.release}-updates {' '.join(repos)}"
+
+    root.joinpath(f"etc/apt/sources.list.d/{args.release}-updates.list").write_text(f"{updates}\n")
+
+    if args.distribution == Distribution.ubuntu:
+        security = f"deb http://archive.ubuntu.com/ubuntu {args.release}-security {' '.join(repos)}"
+    elif args.release in ("stretch", "buster"):
+        security = f"deb http://security.debian.org/debian-security/ {args.release}/updates main"
+    else:
+        security = f"deb https://security.debian.org/debian-security {args.release}-security main"
+
+    root.joinpath(f"etc/apt/sources.list.d/{args.release}-security.list").write_text(f"{security}\n")
+
 def install_debian_or_ubuntu(args: MkosiArgs, root: Path, *, do_run_build_script: bool) -> None:
     # Either the image builds or it fails and we restart, we don't need safety fsyncs when bootstrapping
     # Add it before debootstrap, as the second stage already uses dpkg from the chroot
@@ -2482,8 +2519,9 @@ def install_debian_or_ubuntu(args: MkosiArgs, root: Path, *, do_run_build_script
         if debootstrap_knows_arg("--no-check-valid-until"):
             cmdline += ["--no-check-valid-until"]
 
-        assert args.mirror is not None
-        cmdline += [args.release, root, args.mirror]
+        mirror = args.local_mirror or args.mirror
+        assert mirror is not None
+        cmdline += [args.release, root, mirror]
         run(cmdline)
 
     # Install extra packages via the secondary APT run, because it is smarter and can deal better with any
@@ -2551,22 +2589,11 @@ def install_debian_or_ubuntu(args: MkosiArgs, root: Path, *, do_run_build_script
             if "VERSION_ID" not in os_release and "BUILD_ID" not in os_release:
                 f.write(f"BUILD_ID=mkosi-{args.release}\n")
 
-    if args.release not in ("unstable", "sid"):
-        if args.distribution == Distribution.ubuntu:
-            updates = f"deb http://archive.ubuntu.com/ubuntu {args.release}-updates {' '.join(repos)}"
-        else:
-            updates = f"deb http://deb.debian.org/debian {args.release}-updates {' '.join(repos)}"
-
-        root.joinpath(f"etc/apt/sources.list.d/{args.release}-updates.list").write_text(f"{updates}\n")
-
-        if args.distribution == Distribution.ubuntu:
-            security = f"deb http://archive.ubuntu.com/ubuntu {args.release}-security {' '.join(repos)}"
-        elif args.release in ("stretch", "buster"):
-            security = f"deb http://security.debian.org/debian-security/ {args.release}/updates main"
-        else:
-            security = f"deb https://security.debian.org/debian-security {args.release}-security main"
-
-        root.joinpath(f"etc/apt/sources.list.d/{args.release}-security.list").write_text(f"{security}\n")
+    if not args.local_mirror:
+        add_apt_auxiliary_repos(args, root, repos)
+    else:
+        # Add a single local offline repository, and then remove it after apt has ran
+        root.joinpath("etc/apt/sources.list.d/mirror.list").write_text(f"deb [trusted=yes] {args.local_mirror} {args.release} main\n")
 
     install_skeleton_trees(args, root, False, late=True)
 
@@ -2578,6 +2605,16 @@ def install_debian_or_ubuntu(args: MkosiArgs, root: Path, *, do_run_build_script
 
     invoke_apt(args, root, "get", "install", ["--assume-yes", "--no-install-recommends", *extra_packages])
 
+    # Now clean up and add the real repositories, so that the image is ready
+    if args.local_mirror:
+        if args.distribution == Distribution.ubuntu:
+            main_repo = f"deb http://archive.ubuntu.com/ubuntu {args.release} {' '.join(repos)}\n"
+        else:
+            main_repo = f"deb http://deb.debian.org/debian {args.release} {' '.join(repos)}\n"
+        root.joinpath("etc/apt/sources.list").write_text(main_repo)
+        root.joinpath("etc/apt/sources.list.d/mirror.list").unlink()
+        add_apt_auxiliary_repos(args, root, repos)
+
     policyrcd.unlink()
     dpkg_io_conf.unlink()
     if not args.with_docs and args.base_image is not None:
@@ -2626,7 +2663,9 @@ def install_arch(args: MkosiArgs, root: Path, do_run_build_script: bool) -> None
     if args.release is not None:
         MkosiPrinter.info("Distribution release specification is not supported for Arch Linux, ignoring.")
 
-    if args.mirror:
+    if args.local_mirror:
+        server = f"Server = {args.local_mirror}"
+    elif args.mirror:
         if platform.machine() == "aarch64":
             server = f"Server = {args.mirror}/$arch/$repo"
         else:
@@ -2679,12 +2718,27 @@ def install_arch(args: MkosiArgs, root: Path, do_run_build_script: bool) -> None
 
                 [core]
                 {server}
+                """
+            )
+        )
 
-                [extra]
-                {server}
+        if not args.local_mirror:
+            f.write(
+                dedent(
+                    f"""\
 
-                [community]
-                {server}
+                    [extra]
+                    {server}
+
+                    [community]
+                    {server}
+                    """
+                )
+            )
+
+        f.write(
+            dedent(
+                f"""\
 
                 {f"Include = {args.repos_dir}/*" if args.repos_dir else ""}
                 """
@@ -2759,10 +2813,7 @@ def install_opensuse(args: MkosiArgs, root: Path, do_run_build_script: bool) ->
 
     # If the release looks like a timestamp, it's Tumbleweed. 13.x is legacy (14.x won't ever appear). For
     # anything else, let's default to Leap.
-    if args.use_mirror_verbatim and args.mirror:
-        release_url = args.mirror
-        updates_url = None
-    elif release.isdigit() or release == "tumbleweed":
+    if release.isdigit() or release == "tumbleweed":
         release_url = f"{args.mirror}/tumbleweed/repo/oss/"
         updates_url = f"{args.mirror}/update/tumbleweed/"
     elif release == "leap":
@@ -2781,8 +2832,12 @@ def install_opensuse(args: MkosiArgs, root: Path, do_run_build_script: bool) ->
     # Configure the repositories: we need to enable packages caching here to make sure that the package cache
     # stays populated after "zypper install".
     run(["zypper", "--root", root, "addrepo", "-ck", release_url, "repo-oss"])
-    if updates_url is not None:
-        run(["zypper", "--root", root, "addrepo", "-ck", updates_url, "repo-update"])
+    run(["zypper", "--root", root, "addrepo", "-ck", updates_url, "repo-update"])
+
+    # If we need to use a local mirror, create a temporary repository definition
+    # that doesn't get in the image, as it is valid only at image build time.
+    if args.local_mirror:
+        run(["zypper", "--reposd-dir", workspace(root) / "zypper-repos.d", "--root", root, "addrepo", "-ck", args.local_mirror, "local-mirror"])
 
     if not args.with_docs:
         root.joinpath("etc/zypp/zypp.conf").write_text("rpm.install.excludedocs = yes\n")
@@ -2813,8 +2868,11 @@ def install_opensuse(args: MkosiArgs, root: Path, do_run_build_script: bool) ->
     if not do_run_build_script and args.ssh:
         add_packages(args, packages, "openssh-server")
 
-    cmdline: List[PathString] = [
-        "zypper",
+    cmdline: List[PathString] = ["zypper"]
+    # --reposd-dir needs to be before the verb
+    if args.local_mirror:
+        cmdline += ["--reposd-dir", workspace(root) / "zypper-repos.d"]
+    cmdline += [
         "--root",
         root,
         "--gpg-auto-import-keys",
@@ -4972,12 +5030,7 @@ def create_parser() -> ArgumentParserMkosi:
     group.add_argument("-r", "--release", help="Distribution release to install")
     group.add_argument("--architecture", help="Override the architecture of installation", default=platform.machine())
     group.add_argument("-m", "--mirror", help="Distribution mirror to use")
-    group.add_argument(
-        "--use-mirror-verbatim",
-        metavar="BOOL",
-        action=BooleanAction,
-        help="Use a single flat and plain mirror for RPM based distributions",
-        default=False,
+    group.add_argument("--local-mirror", help="Use a single local, flat and plain mirror to build the image",
     )
 
     group.add_argument(
@@ -6709,7 +6762,9 @@ def print_summary(args: MkosiArgs) -> None:
     MkosiPrinter.info("                   Release: " + none_to_na(args.release))
     MkosiPrinter.info("              Architecture: " + args.architecture)
     if args.mirror is not None:
-        MkosiPrinter.info(f"         {'(verbatim)' if args.use_mirror_verbatim else '          '} Mirror: " + args.mirror)
+        MkosiPrinter.info("                    Mirror: " + args.mirror)
+    if args.local_mirror is not None:
+        MkosiPrinter.info("      Local Mirror (build): " + args.local_mirror)
     if args.repositories is not None and len(args.repositories) > 0:
         MkosiPrinter.info("              Repositories: " + ",".join(args.repositories))
     MkosiPrinter.info("     Use Host Repositories: " + yes_no(args.use_host_repositories))

diff --git a/mkosi/backend.py b/mkosi/backend.py
index 2ae1358b448390df7ccb50c3ba4991b07f3cb63f..da5ceb8899827d51df93f1e5b313d899d0ebaad1 100644 (file)
--- a/mkosi/backend.py
+++ b/mkosi/backend.py
@@ -460,7 +460,7 @@ class MkosiArgs:
     distribution: Distribution
     release: str
     mirror: Optional[str]
-    use_mirror_verbatim: bool
+    local_mirror: Optional[str]
     repositories: List[str]
     use_host_repositories: bool
     repos_dir: Optional[str]

diff --git a/tests/test_config_parser.py b/tests/test_config_parser.py
index d7ffeecc44bf347087683b9bb6f620d68a800151..49968050a946c20d26c69f5c11718740ddcc4ab6 100644 (file)
--- a/tests/test_config_parser.py
+++ b/tests/test_config_parser.py
@@ -81,9 +81,9 @@ class MkosiConfig:
             "install_dir": None,
             "kernel_command_line": ["rhgb", "selinux=0", "audit=0"],
             "key": None,
+            "local_mirror": None,
             "manifest_format": None,
             "mirror": None,
-            "use_mirror_verbatim": False,
             "mksquashfs_tool": [],
             "no_chown": False,
             "nspawn_settings": None,