]> git.ipfire.org Git - thirdparty/mkosi.git/commitdiff
Add new Snapshot= setting 3802/head
authorDaanDeMeyer <daan.j.demeyer@gmail.com>
Thu, 10 Jul 2025 10:21:11 +0000 (12:21 +0200)
committerDaan De Meyer <daan.j.demeyer@gmail.com>
Thu, 11 Sep 2025 09:38:36 +0000 (11:38 +0200)
In systemd CI, we often run into issues caused by updates to third-party
components like the kernel package in rolling release distributions
like Arch Linux or Fedora Rawhide. When these happen, the corresponding
CI job starts failing on every PR and bisecting the distribution to figure
out when the breakage was introduced is rather tedious.

To mitigate this problem, we need to be able to pin the rolling release
distributions to a specific snapshot which we control. This allows us to
update the pinned snapshot in a PR created by a bot, so that any failures
introduced by moving to a newer snapshot will be limited to the PR that bumps
the snapshot. Any regressions can then be debugged and fixed before merging
the PR that switches us to the new snapshot.

To make this possible, let's introduce a new Snapshot= setting and implement
it for every distribution that has a snapshot concept or something that maps
to it. Per distribution:

- Debian => snapshot.debian.org (unlimited)
- Ubuntu => snapshot.ubuntu.com (unlimited)
- Arch => archive.archlinux.org (unlimited)
- OpenSUSE => download.opensuse.org/history (limited to a month of snapshots)
- CentOS => composes.stream.centos.org (limited to 3 weeks of snapshots)
- Fedora => https://kojipkgs.fedoraproject.org (limited to 2 weeks of snapshots)

Additionally, for CentOS, we also support using composes from mirror.facebook.net
which keeps them around forever so we get unlimited snapshots there as well for
CentOS Stream.

We also add a latest-snapshot verb to be able to easily figure out the latest
snapshot so it can be bumped regularly via a CI workflow. Because we do not track
sufficient information from config files to be able to insert the updated snapshot
into the right config file ourselves, we output it on stdout instead and leave it to
users to insert it into the right config file.

21 files changed:
mkosi/__init__.py
mkosi/config.py
mkosi/curl.py
mkosi/distributions/__init__.py
mkosi/distributions/alma.py
mkosi/distributions/arch.py
mkosi/distributions/azure.py
mkosi/distributions/centos.py
mkosi/distributions/debian.py
mkosi/distributions/fedora.py
mkosi/distributions/kali.py
mkosi/distributions/mageia.py
mkosi/distributions/openmandriva.py
mkosi/distributions/opensuse.py
mkosi/distributions/rhel.py
mkosi/distributions/rhel_ubi.py
mkosi/distributions/rocky.py
mkosi/distributions/ubuntu.py
mkosi/installer/apt.py
mkosi/resources/man/mkosi.1.md
tests/test_json.py

index f2805e9ef8bb641e676a7a598c18f5e954124569..df81cd4e6b8535a43bb2c52eac776cc73a9dfd8f 100644 (file)
@@ -1344,6 +1344,7 @@ def finalize_default_initrd(
         f"--release={config.release}",
         f"--architecture={config.architecture}",
         *([f"--mirror={config.mirror}"] if config.mirror else []),
+        *([f"--snapshot={config.snapshot}"] if config.snapshot else []),
         f"--repository-key-check={config.repository_key_check}",
         f"--repository-key-fetch={config.repository_key_fetch}",
         *([f"--repositories={repository}" for repository in config.repositories]),
@@ -4243,6 +4244,10 @@ def run_box(args: Args, config: Config) -> None:
         )
 
 
+def run_latest_snapshot(args: Args, config: Config) -> None:
+    print(config.distribution.latest_snapshot(config))
+
+
 def run_shell(args: Args, config: Config) -> None:
     opname = "acquire shell in" if args.verb == Verb.shell else "boot"
     if config.output_format not in (OutputFormat.directory, OutputFormat.disk):
@@ -5103,6 +5108,10 @@ def run_verb(args: Args, tools: Optional[Config], images: Sequence[Config], *, r
         finalize_image_version(args, last)
         return
 
+    if args.verb == Verb.latest_snapshot:
+        run_latest_snapshot(args, last)
+        return
+
     if args.verb == Verb.clean:
         if tools and args.force > 0:
             run_clean(args, tools)
index 3c09ffb038c959931ac33c2914a1a6d032f9a555..c006fd81c10f23951c53d71dfb82efcb3df00619 100644 (file)
@@ -90,6 +90,7 @@ class Verb(StrEnum):
     box = enum.auto()
     sandbox = enum.auto()
     init = enum.auto()
+    latest_snapshot = enum.auto()
 
     def supports_cmdline(self) -> bool:
         return self in (
@@ -111,7 +112,14 @@ class Verb(StrEnum):
         )
 
     def needs_tools(self) -> bool:
-        return self in (Verb.box, Verb.sandbox, Verb.journalctl, Verb.coredumpctl, Verb.ssh)
+        return self in (
+            Verb.box,
+            Verb.sandbox,
+            Verb.journalctl,
+            Verb.coredumpctl,
+            Verb.ssh,
+            Verb.latest_snapshot,
+        )
 
     def needs_build(self) -> bool:
         return self in (
@@ -1954,6 +1962,7 @@ class Config:
     release: str
     architecture: Architecture
     mirror: Optional[str]
+    snapshot: Optional[str]
     local_mirror: Optional[str]
     repository_key_check: bool
     repository_key_fetch: bool
@@ -2335,6 +2344,7 @@ class Config:
             "distribution": self.distribution,
             "release": self.release,
             "mirror": self.mirror,
+            "snapshot": self.snapshot,
             "architecture": self.architecture,
             # Caching the package manager used does not matter for the default tools tree because we don't
             # cache the package manager metadata for the tools tree either. In fact, it can cause issues as
@@ -2673,6 +2683,15 @@ SETTINGS: list[ConfigSetting[Any]] = [
         help="Distribution mirror to use",
         scope=SettingScope.universal,
     ),
+    ConfigSetting(
+        dest="snapshot",
+        section="Distribution",
+        help="Distribution snapshot to use",
+        path_suffixes=("snapshot",),
+        path_read_text=True,
+        scope=SettingScope.universal,
+        tools=True,
+    ),
     ConfigSetting(
         dest="local_mirror",
         section="Distribution",
@@ -4992,7 +5011,7 @@ def have_history(args: Args) -> bool:
     if args.directory is None:
         return False
 
-    if args.verb in (Verb.clean, Verb.sandbox):
+    if args.verb in (Verb.clean, Verb.sandbox, Verb.latest_snapshot):
         return False
 
     if args.verb == Verb.summary and args.force > 0:
@@ -5413,6 +5432,7 @@ def summary(config: Config) -> str:
                             Release: {bold(none_to_na(config.release))}
                        Architecture: {config.architecture}
                              Mirror: {none_to_default(config.mirror)}
+                           Snapshot: {none_to_none(config.snapshot)}
                Local Mirror (build): {none_to_none(config.local_mirror)}
            Repo Signature/Key check: {yes_no(config.repository_key_check)}
               Fetch Repository Keys: {yes_no(config.repository_key_fetch)}
index 6560bf987605ee10949e0db286757fd2bc14880b..0db38229af278aeff134cfd7d4a97b97f723fc81 100644 (file)
@@ -1,19 +1,42 @@
 # SPDX-License-Identifier: LGPL-2.1-or-later
 
+import os
+import subprocess
 from pathlib import Path
+from typing import Optional, overload
 
 from mkosi.config import Config
 from mkosi.mounts import finalize_certificate_mounts
 from mkosi.run import run, workdir
 
 
-def curl(config: Config, url: str, output_dir: Path, log: bool = True) -> None:
-    run(
+@overload
+def curl(
+    config: Config,
+    url: str,
+    *,
+    output_dir: Optional[Path],
+    log: bool = True,
+) -> None: ...
+
+
+@overload
+def curl(
+    config: Config,
+    url: str,
+    *,
+    output_dir: None = None,
+    log: bool = True,
+) -> str: ...
+
+
+def curl(config: Config, url: str, *, output_dir: Optional[Path] = None, log: bool = True) -> Optional[str]:
+    result = run(
         [
             "curl",
             "--location",
-            "--output-dir", workdir(output_dir),
-            "--remote-name",
+            *(["--output-dir", workdir(output_dir)] if output_dir else []),
+            *(["--remote-name"] if output_dir else []),
             "--no-progress-meter",
             "--fail",
             *(["--silent"] if not log else []),
@@ -24,9 +47,15 @@ def curl(config: Config, url: str, output_dir: Path, log: bool = True) -> None:
             *(["--proxy-key", "/proxy.clientkey"] if config.proxy_client_key else []),
             url,
         ],
+        stdout=None if output_dir else subprocess.PIPE,
         sandbox=config.sandbox(
             network=True,
-            options=["--bind", output_dir, workdir(output_dir), *finalize_certificate_mounts(config)],
+            options=[
+                *(["--bind", os.fspath(output_dir), workdir(output_dir)] if output_dir else []),
+                *finalize_certificate_mounts(config)
+            ],
         ),
         log=log,
     )  # fmt: skip
+
+    return None if output_dir else result.stdout
index 381becc2937c47b98d5465c013951246f6dfaa6f..1023229a8a4305c5d880870505e572793d2c2f98 100644 (file)
@@ -6,6 +6,7 @@ import urllib.parse
 from pathlib import Path
 from typing import TYPE_CHECKING, Optional, cast
 
+from mkosi.log import die
 from mkosi.util import StrEnum, read_env_file
 
 if TYPE_CHECKING:
@@ -66,6 +67,10 @@ class DistributionInstaller:
     def grub_prefix(cls) -> str:
         return "grub"
 
+    @classmethod
+    def latest_snapshot(cls, config: "Config") -> str:
+        die(f"{cls.pretty_name()} does not support snapshots")
+
 
 class Distribution(StrEnum):
     # Please consult docs/distribution-policy.md and contact one
@@ -148,6 +153,9 @@ class Distribution(StrEnum):
     def createrepo(self, context: "Context") -> None:
         return self.installer().package_manager(context.config).createrepo(context)
 
+    def latest_snapshot(self, config: "Config") -> str:
+        return self.installer().latest_snapshot(config)
+
     def installer(self) -> type[DistributionInstaller]:
         modname = str(self).replace("-", "_")
         mod = importlib.import_module(f"mkosi.distributions.{modname}")
index 4e668f29170e89a0ebe42deba5ab8c689a7675e7..3920be7760a4141ee5c4f228c52629d3ae2785bb 100644 (file)
@@ -3,6 +3,7 @@
 from mkosi.context import Context
 from mkosi.distributions import centos, join_mirror
 from mkosi.installer.rpm import RpmRepository, find_rpm_gpgkey
+from mkosi.log import die
 
 
 class Installer(centos.Installer):
@@ -28,6 +29,9 @@ class Installer(centos.Installer):
         gpgurls: tuple[str, ...],
         repo: str,
     ) -> list[RpmRepository]:
+        if context.config.snapshot:
+            die(f"Snapshot= is not supported for {cls.pretty_name()}")
+
         if context.config.mirror:
             url = f"baseurl={join_mirror(context.config.mirror, f'$releasever/{repo}/$basearch/os')}"
         else:
index 8373306a72afad0bac9377fc07bd1c8deb770051..6ce0ccf40ea501e8dea02de2b2e3a9e3ec2497b9 100644 (file)
@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: LGPL-2.1-or-later
 
+import datetime
 import tempfile
 from collections.abc import Iterable
 from pathlib import Path
@@ -8,7 +9,7 @@ from mkosi.archive import extract_tar
 from mkosi.config import Architecture, Config
 from mkosi.context import Context
 from mkosi.curl import curl
-from mkosi.distributions import DistributionInstaller, PackageType
+from mkosi.distributions import DistributionInstaller, PackageType, join_mirror
 from mkosi.installer.pacman import Pacman, PacmanRepository
 from mkosi.log import complete_step, die
 
@@ -44,7 +45,7 @@ class Installer(DistributionInstaller):
                 curl(
                     context.config,
                     "https://archlinux.org/packages/core/any/archlinux-keyring/download",
-                    Path(d),
+                    output_dir=Path(d),
                 )
                 extract_tar(
                     next(Path(d).iterdir()),
@@ -69,9 +70,24 @@ class Installer(DistributionInstaller):
             yield PacmanRepository("core", context.config.local_mirror)
         else:
             if context.config.architecture.is_arm_variant():
-                url = f"{context.config.mirror or 'http://mirror.archlinuxarm.org'}/$arch/$repo"
+                if context.config.snapshot and not context.config.mirror:
+                    die("There is no known public mirror for snapshots of Arch Linux ARM")
+
+                mirror = context.config.mirror or "http://mirror.archlinuxarm.org"
+            else:
+                if context.config.mirror:
+                    mirror = context.config.mirror
+                elif context.config.snapshot:
+                    mirror = "https://archive.archlinux.org"
+                else:
+                    mirror = "https://geo.mirror.pkgbuild.com"
+
+            if context.config.snapshot:
+                url = join_mirror(mirror, f"repos/{context.config.snapshot}/$repo/os/$arch")
+            elif context.config.architecture.is_arm_variant():
+                url = join_mirror(mirror, "$arch/$repo")
             else:
-                url = f"{context.config.mirror or 'https://geo.mirror.pkgbuild.com'}/$repo/os/$arch"
+                url = join_mirror(mirror, "$repo/os/$arch")
 
             # Testing repositories have to go before regular ones to to take precedence.
             repos = [
@@ -107,3 +123,10 @@ class Installer(DistributionInstaller):
             die(f"Architecture {a} is not supported by Arch Linux")
 
         return a
+
+    @classmethod
+    def latest_snapshot(cls, config: Config) -> str:
+        url = join_mirror(config.mirror or "https://archive.archlinux.org", "repos/last/lastsync")
+        return datetime.datetime.fromtimestamp(int(curl(config, url)), datetime.timezone.utc).strftime(
+            "%Y/%m/%d"
+        )
index f66d8a450809eee1408c9e0ceb0672fb5e758903..4049b85ec508fcc081222a2d9d032ff87f2a09b2 100644 (file)
@@ -37,6 +37,9 @@ class Installer(fedora.Installer):
 
     @classmethod
     def repositories(cls, context: Context) -> Iterable[RpmRepository]:
+        if context.config.snapshot:
+            die(f"Snapshot= is not supported for {cls.pretty_name()}")
+
         gpgurls = (
             find_rpm_gpgkey(
                 context,
index 7c770c8f5193ab81ae26cfa26e393aa1fe00d102..3c6d1623becc6aa12aaf6287e16d67e2748f2e4c 100644 (file)
@@ -4,6 +4,7 @@ from collections.abc import Iterable
 
 from mkosi.config import Architecture, Config
 from mkosi.context import Context
+from mkosi.curl import curl
 from mkosi.distributions import (
     Distribution,
     DistributionInstaller,
@@ -13,6 +14,7 @@ from mkosi.distributions import (
 from mkosi.installer.dnf import Dnf
 from mkosi.installer.rpm import RpmRepository, find_rpm_gpgkey, setup_rpm
 from mkosi.log import die
+from mkosi.util import startswith
 from mkosi.versioncomp import GenericVersion
 
 CENTOS_SIG_REPO_PRIORITY = 50
@@ -132,38 +134,67 @@ class Installer(DistributionInstaller):
         gpgurls: tuple[str, ...],
         repo: str,
     ) -> Iterable[RpmRepository]:
+        mirror = context.config.mirror
+        if not mirror and context.config.snapshot:
+            mirror = "https://composes.stream.centos.org"
+
+        if context.config.snapshot and mirror not in (
+            "https://composes.stream.centos.org",
+            "https://mirror.facebook.net/centos-composes",
+        ):
+            die(
+                f"Snapshot= is only supported for {cls.pretty_name()} if Mirror=https://composes.stream.centos.org"
+            )
+
+        if (
+            mirror in ("https://composes.stream.centos.org", "https://mirror.facebook.net/centos-composes")
+            and not context.config.snapshot
+        ):
+            die(f"Snapshot= must be used on {cls.pretty_name()} if Mirror={mirror}")
+
         if context.config.local_mirror:
             yield RpmRepository(repo, f"baseurl={context.config.local_mirror}", gpgurls)
 
-        elif mirror := context.config.mirror:
+        elif mirror:
+            if mirror == "https://composes.stream.centos.org":
+                subdir = f"stream-{context.config.release}/production"
+            elif mirror == "https://mirror.facebook.net/centos-composes":
+                subdir = context.config.release
+            elif repo == "extras":
+                subdir = "SIGs/$stream"
+            else:
+                subdir = "$stream"
+
+            if context.config.snapshot:
+                subdir += f"/CentOS-Stream-{context.config.release}-{context.config.snapshot}/compose"
+
             if repo == "extras":
                 yield RpmRepository(
                     repo.lower(),
-                    f"baseurl={join_mirror(mirror, f'SIGs/$stream/{repo}/$basearch/extras-common')}",
+                    f"baseurl={join_mirror(mirror, f'{subdir}/{repo}/$basearch/extras-common')}",
                     gpgurls,
                 )
                 yield RpmRepository(
                     f"{repo.lower()}-source",
-                    f"baseurl={join_mirror(mirror, f'SIGs/$stream/{repo}/source/extras-common')}",
+                    f"baseurl={join_mirror(mirror, f'{subdir}/{repo}/source/extras-common')}",
                     gpgurls,
                     enabled=False,
                 )
-
             else:
                 yield RpmRepository(
                     repo.lower(),
-                    f"baseurl={join_mirror(mirror, f'$stream/{repo}/$basearch/os')}",
+                    f"baseurl={join_mirror(mirror, f'{subdir}/{repo}/$basearch/os')}",
                     gpgurls,
                 )
                 yield RpmRepository(
                     f"{repo.lower()}-debuginfo",
-                    f"baseurl={join_mirror(mirror, f'$stream/{repo}/$basearch/debug/tree')}",
+                    f"baseurl={join_mirror(mirror, f'{subdir}/{repo}/$basearch/debug/tree')}",
                     gpgurls,
                     enabled=False,
                 )
                 yield RpmRepository(
                     f"{repo.lower()}-source",
-                    f"baseurl={join_mirror(mirror, f'$stream/{repo}/source/tree')}",
+                    f"baseurl={join_mirror(mirror, f'{subdir}/{repo}/source/tree')}",
                     gpgurls,
                     enabled=False,
                 )
@@ -211,8 +242,9 @@ class Installer(DistributionInstaller):
 
         yield from cls.repository_variants(context, gpgurls, "BaseOS")
         yield from cls.repository_variants(context, gpgurls, "AppStream")
-        yield from cls.repository_variants(context, gpgurls, "extras")
         yield from cls.repository_variants(context, gpgurls, "CRB")
+        if not context.config.snapshot:
+            yield from cls.repository_variants(context, gpgurls, "extras")
 
         yield from cls.epel_repositories(context)
         yield from cls.sig_repositories(context)
@@ -422,3 +454,20 @@ class Installer(DistributionInstaller):
                         enabled=False,
                         priority=CENTOS_SIG_REPO_PRIORITY,
                     )
+
+    @classmethod
+    def latest_snapshot(cls, config: Config) -> str:
+        mirror = config.mirror or "https://composes.stream.centos.org"
+
+        if mirror == "https://mirror.facebook.net/centos-composes":
+            subdir = config.release
+        else:
+            subdir = f"stream-{config.release}/production"
+
+        url = join_mirror(mirror, f"{subdir}/latest-CentOS-Stream/compose/.composeinfo")
+
+        for line in curl(config, url).splitlines():
+            if snapshot := startswith(line, f"id = CentOS-Stream-{config.release}-"):
+                return snapshot
+
+        die("composeinfo is missing compose ID field")
index 4ceec90da8802827620b73637489e5cdc0edafc6..5873c54930afffda301d139639a72815c19ac8ac 100644 (file)
@@ -1,14 +1,17 @@
 # SPDX-License-Identifier: LGPL-2.1-or-later
 
 import itertools
+import json
 import tempfile
 from collections.abc import Iterable
 from pathlib import Path
+from typing import cast
 
 from mkosi.archive import extract_tar
 from mkosi.config import Architecture, Config
 from mkosi.context import Context
-from mkosi.distributions import DistributionInstaller, PackageType
+from mkosi.curl import curl
+from mkosi.distributions import DistributionInstaller, PackageType, join_mirror
 from mkosi.installer.apt import Apt, AptRepository
 from mkosi.log import die
 from mkosi.run import run, workdir
@@ -51,19 +54,33 @@ class Installer(DistributionInstaller):
             )
             return
 
-        mirror = context.config.mirror or "http://deb.debian.org/debian"
+        if context.config.mirror:
+            mirror = context.config.mirror
+        elif context.config.snapshot:
+            mirror = "https://snapshot.debian.org"
+        else:
+            mirror = "http://deb.debian.org"
+
+        if context.config.snapshot:
+            url = join_mirror(mirror, f"archive/debian/{context.config.snapshot}")
+        else:
+            url = join_mirror(mirror, "debian")
+
         signedby = Path("/usr/share/keyrings/debian-archive-keyring.gpg")
 
         yield AptRepository(
             types=types,
-            url=mirror,
+            url=url,
             suite=context.config.release,
             components=components,
             signedby=signedby,
         )
 
         # Debug repos are typically not mirrored.
-        url = "http://deb.debian.org/debian-debug"
+        if context.config.snapshot:
+            url = join_mirror(mirror, f"archive/debian-debug/{context.config.snapshot}")
+        else:
+            url = join_mirror(mirror, "debian-debug")
 
         yield AptRepository(
             types=types,
@@ -76,18 +93,24 @@ class Installer(DistributionInstaller):
         if context.config.release in ("unstable", "sid"):
             return
 
-        yield AptRepository(
-            types=types,
-            url=mirror,
-            suite=f"{context.config.release}-updates",
-            components=components,
-            signedby=signedby,
-        )
+        if not context.config.snapshot:
+            yield AptRepository(
+                types=types,
+                url=join_mirror(mirror, "debian"),
+                suite=f"{context.config.release}-updates",
+                components=components,
+                signedby=signedby,
+            )
+
+        # Security updates repos are never mirrored.
+        if context.config.snapshot:
+            url = join_mirror(mirror, f"archive/debian-security/{context.config.snapshot}")
+        else:
+            url = join_mirror(mirror, "debian-security")
 
         yield AptRepository(
             types=types,
-            # Security updates repos are never mirrored.
-            url="http://security.debian.org/debian-security",
+            url=url,
             suite=f"{context.config.release}-security",
             components=components,
             signedby=signedby,
@@ -220,6 +243,11 @@ class Installer(DistributionInstaller):
 
         return a
 
+    @classmethod
+    def latest_snapshot(cls, config: Config) -> str:
+        url = join_mirror(config.mirror or "https://snapshot.debian.org", "mr/timestamp")
+        return cast(str, json.loads(curl(config, url))["result"]["debian"][-1])
+
 
 def install_apt_sources(context: Context, repos: Iterable[AptRepository]) -> None:
     sources = context.root / f"etc/apt/sources.list.d/{context.config.release}.sources"
index 2da823b28064fee54ad6834cf4635e55f57ecb78..598b06e9dd37b8d91ec798918b24e890b946533d 100644 (file)
@@ -29,7 +29,11 @@ def read_remote_rawhide_key_symlink(context: Context) -> str:
     # let's fetch it from distribution-gpg-keys on github if necessary, which is generally up-to-date.
     with tempfile.TemporaryDirectory() as d:
         # The rawhide key is a symlink and github doesn't redirect those to the actual file for some reason
-        curl(context.config, f"{DISTRIBUTION_GPG_KEYS_UPSTREAM}/RPM-GPG-KEY-fedora-rawhide-primary", Path(d))
+        curl(
+            context.config,
+            f"{DISTRIBUTION_GPG_KEYS_UPSTREAM}/RPM-GPG-KEY-fedora-rawhide-primary",
+            output_dir=Path(d),
+        )
         return (Path(d) / "RPM-GPG-KEY-fedora-rawhide-primary").read_text()
 
 
@@ -59,7 +63,7 @@ def find_fedora_rpm_gpgkeys(context: Context) -> Iterable[str]:
                 curl(
                     context.config,
                     f"{DISTRIBUTION_GPG_KEYS_UPSTREAM}/RPM-GPG-KEY-fedora-{version + 1}-primary",
-                    Path(d),
+                    output_dir=Path(d),
                     log=False,
                 )
 
@@ -138,6 +142,23 @@ class Installer(DistributionInstaller):
     def repositories(cls, context: Context) -> Iterable[RpmRepository]:
         gpgurls = find_fedora_rpm_gpgkeys(context)
 
+        if context.config.snapshot and context.config.release != "rawhide":
+            die(f"Snapshot= is only supported for rawhide on {cls.pretty_name()}")
+
+        mirror = context.config.mirror
+        if not mirror and context.config.snapshot:
+            mirror = "https://kojipkgs.fedoraproject.org"
+
+        if context.config.snapshot and mirror != "https://kojipkgs.fedoraproject.org":
+            die(
+                f"Snapshot= is only supported for {cls.pretty_name()} if Mirror=https://kojipkgs.fedoraproject.org"
+            )
+
+        if mirror == "https://kojipkgs.fedoraproject.org" and not context.config.snapshot:
+            die(
+                f"Snapshot= must be used on {cls.pretty_name()} if Mirror=https://kojipkgs.fedoraproject.org"
+            )
+
         if context.config.local_mirror:
             yield RpmRepository("fedora", f"baseurl={context.config.local_mirror}", gpgurls)
             return
@@ -152,20 +173,29 @@ class Installer(DistributionInstaller):
                     f"{repo.lower()}-debuginfo", f"{url}/$basearch/debug/tree", gpgurls, enabled=False
                 )
                 yield RpmRepository(f"{repo.lower()}-source", f"{url}/source/tree", gpgurls, enabled=False)
-        elif m := context.config.mirror:
-            directory = "development" if context.config.release == "rawhide" else "releases"
-            url = f"baseurl={join_mirror(m, f'linux/{directory}/$releasever/Everything')}"
+        elif mirror:
+            if mirror == "https://kojipkgs.fedoraproject.org":
+                subdir = f"compose/{context.config.release}"
+            else:
+                subdir = "linux/"
+                subdir += "development" if context.config.release == "rawhide" else "releases"
+                subdir += "/$releasever"
+
+            if context.config.snapshot:
+                subdir += f"/Fedora-{context.config.release.capitalize()}-{context.config.snapshot}/compose"
+
+            url = f"baseurl={join_mirror(mirror, f'{subdir}/Everything')}"
             yield RpmRepository("fedora", f"{url}/$basearch/os", gpgurls)
             yield RpmRepository("fedora-debuginfo", f"{url}/$basearch/debug/tree", gpgurls, enabled=False)
             yield RpmRepository("fedora-source", f"{url}/source/tree", gpgurls, enabled=False)
 
             if context.config.release != "rawhide":
-                url = f"baseurl={join_mirror(m, 'linux/updates/$releasever/Everything')}"
+                url = f"baseurl={join_mirror(mirror, 'linux/updates/$releasever/Everything')}"
                 yield RpmRepository("updates", f"{url}/$basearch", gpgurls)
                 yield RpmRepository("updates-debuginfo", f"{url}/$basearch/debug", gpgurls, enabled=False)
                 yield RpmRepository("updates-source", f"{url}/source/tree", gpgurls, enabled=False)
 
-                url = f"baseurl={join_mirror(m, 'linux/updates/testing/$releasever/Everything')}"
+                url = f"baseurl={join_mirror(mirror, 'linux/updates/testing/$releasever/Everything')}"
                 yield RpmRepository("updates-testing", f"{url}/$basearch", gpgurls, enabled=False)
                 yield RpmRepository(
                     "updates-testing-debuginfo", f"{url}/$basearch/debug", gpgurls, enabled=False
@@ -227,3 +257,13 @@ class Installer(DistributionInstaller):
             die(f"Architecture {a} is not supported by Fedora")
 
         return a
+
+    @classmethod
+    def latest_snapshot(cls, config: Config) -> str:
+        mirror = config.mirror or "https://kojipkgs.fedoraproject.org"
+
+        url = join_mirror(
+            mirror, f"compose/{config.release}/latest-Fedora-{config.release.capitalize()}/COMPOSE_ID"
+        )
+
+        return curl(config, url).removeprefix(f"Fedora-{config.release.capitalize()}-").strip()
index 73370509efab954e4362b0c104175f421d2f1d34..0f64cddbe06a6e2e3d1310f59c0c4ce8ed84ff9b 100644 (file)
@@ -25,6 +25,9 @@ class Installer(debian.Installer):
 
     @classmethod
     def repositories(cls, context: Context, local: bool = True) -> Iterable[AptRepository]:
+        if context.config.snapshot:
+            die(f"Snapshot= is not supported for {cls.pretty_name()}")
+
         if context.config.local_mirror and local:
             yield AptRepository(
                 types=("deb",),
index 65224f69e963c5cbad60176b4dcae544b81fa7fd..b95cacb5b7ec7e301cd4ce629724abb6a294ef40 100644 (file)
@@ -29,6 +29,9 @@ class Installer(fedora.Installer):
 
     @classmethod
     def repositories(cls, context: Context) -> Iterable[RpmRepository]:
+        if context.config.snapshot:
+            die(f"Snapshot= is not supported for {cls.pretty_name()}")
+
         gpgurls = (
             find_rpm_gpgkey(
                 context,
index 2bfc2a225de7e259a07d57dd882af04ed48cba81..1ea47a05ea32149fc139fedf176f80c1978582d4 100644 (file)
@@ -29,6 +29,9 @@ class Installer(fedora.Installer):
 
     @classmethod
     def repositories(cls, context: Context) -> Iterable[RpmRepository]:
+        if context.config.snapshot:
+            die(f"Snapshot= is not supported for {cls.pretty_name()}")
+
         mirror = context.config.mirror or "http://mirror.openmandriva.org"
 
         gpgurls = (
index 943dbd44b45c9b1bd31ede030b92e62a9a1746ab..5327c9e305473b259e73734440e7e52943b245d8 100644 (file)
@@ -68,7 +68,7 @@ class Installer(DistributionInstaller):
         zypper = cls.package_manager(context.config) is Zypper
         mirror = context.config.mirror or "https://download.opensuse.org"
 
-        if context.config.release == "tumbleweed" or context.config.release.isdigit():
+        if context.config.release == "tumbleweed":
             gpgkeys = tuple(
                 p
                 for key in ("RPM-GPG-KEY-openSUSE-Tumbleweed", "RPM-GPG-KEY-openSUSE")
@@ -97,7 +97,12 @@ class Installer(DistributionInstaller):
                     ),
                 )  # fmt: skip
 
-            if context.config.release == "tumbleweed":
+            if context.config.snapshot:
+                if context.config.architecture != Architecture.x86_64:
+                    die(f"Snapshot= is only supported for x86-64 on {cls.pretty_name()}")
+
+                subdir = f"history/{context.config.snapshot}"
+            else:
                 if context.config.architecture == Architecture.x86_64:
                     subdir = ""
                 elif context.config.architecture == Architecture.arm64:
@@ -116,11 +121,6 @@ class Installer(DistributionInstaller):
                     subdir = "ports/riscv"
                 else:
                     die(f"{context.config.architecture} not supported by openSUSE Tumbleweed")
-            else:
-                if context.config.architecture != Architecture.x86_64:
-                    die(f"Old snapshots are only supported for x86-64 on {cls.pretty_name()}")
-
-                subdir = f"history/{context.config.release}"
 
             for repo in ("oss", "non-oss"):
                 url = join_mirror(mirror, f"{subdir}/tumbleweed/repo/{repo}")
@@ -131,7 +131,7 @@ class Installer(DistributionInstaller):
                     enabled=repo == "oss",
                 )
 
-                if context.config.release == "tumbleweed":
+                if not context.config.snapshot:
                     for d in ("debug", "source"):
                         url = join_mirror(mirror, f"{subdir}/{d}/tumbleweed/repo/{repo}")
                         yield RpmRepository(
@@ -141,7 +141,7 @@ class Installer(DistributionInstaller):
                             enabled=False,
                         )
 
-            if context.config.release == "tumbleweed":
+            if not context.config.snapshot:
                 url = join_mirror(mirror, f"{subdir}/update/tumbleweed")
                 yield RpmRepository(
                     id="oss-update",
@@ -157,6 +157,9 @@ class Installer(DistributionInstaller):
                     enabled=False,
                 )
         else:
+            if context.config.snapshot:
+                die(f"Snapshot= is only supported for Tumbleweed on {cls.pretty_name()}")
+
             if (
                 context.config.release in ("current", "stable", "leap")
                 and context.config.architecture != Architecture.x86_64
@@ -248,12 +251,17 @@ class Installer(DistributionInstaller):
 
         return a
 
+    @classmethod
+    def latest_snapshot(cls, config: Config) -> str:
+        url = join_mirror(config.mirror or "https://download.opensuse.org", "history/latest")
+        return curl(config, url).strip()
+
 
 def fetch_gpgurls(context: Context, repourl: str) -> tuple[str, ...]:
     gpgurls = [f"{repourl}/repodata/repomd.xml.key"]
 
     with tempfile.TemporaryDirectory() as d:
-        curl(context.config, f"{repourl}/repodata/repomd.xml", Path(d))
+        curl(context.config, f"{repourl}/repodata/repomd.xml", output_dir=Path(d))
         xml = (Path(d) / "repomd.xml").read_text()
 
     root = ElementTree.fromstring(xml)
index 5a1bb0ba2963443ddf28d90ebe854862a3f949b2..04c2860d6705dc83284b3d24f499ec37767fd839 100644 (file)
@@ -112,6 +112,9 @@ class Installer(centos.Installer):
 
     @classmethod
     def repositories(cls, context: Context) -> Iterable[RpmRepository]:
+        if context.config.snapshot:
+            die(f"Snapshot= is not supported for {cls.pretty_name()}")
+
         gpgurls = cls.gpgurls(context)
         yield from cls.repository_variants(context, gpgurls, "baseos")
         yield from cls.repository_variants(context, gpgurls, "appstream")
index 66eacd21f314100cf36850e03ed317c3cbae109c..6e4e8cd9907b4d9e7a8a0c3924fd16089f5c7163 100644 (file)
@@ -5,6 +5,7 @@ from collections.abc import Iterable
 from mkosi.context import Context
 from mkosi.distributions import centos, join_mirror
 from mkosi.installer.rpm import RpmRepository, find_rpm_gpgkey
+from mkosi.log import die
 
 
 class Installer(centos.Installer):
@@ -55,6 +56,9 @@ class Installer(centos.Installer):
 
     @classmethod
     def repositories(cls, context: Context) -> Iterable[RpmRepository]:
+        if context.config.snapshot:
+            die(f"Snapshot= is not supported for {cls.pretty_name()}")
+
         gpgurls = cls.gpgurls(context)
         yield from cls.repository_variants(context, gpgurls, "baseos")
         yield from cls.repository_variants(context, gpgurls, "appstream")
index bebefae69b9dfda261972582459798e3b649bd86..5829121bf29468fce8ada7c2373092543817dbb0 100644 (file)
@@ -3,6 +3,7 @@
 from mkosi.context import Context
 from mkosi.distributions import centos, join_mirror
 from mkosi.installer.rpm import RpmRepository, find_rpm_gpgkey
+from mkosi.log import die
 
 
 class Installer(centos.Installer):
@@ -28,6 +29,9 @@ class Installer(centos.Installer):
         gpgurls: tuple[str, ...],
         repo: str,
     ) -> list[RpmRepository]:
+        if context.config.snapshot:
+            die(f"Snapshot= is not supported for {cls.pretty_name()}")
+
         if context.config.mirror:
             url = f"baseurl={join_mirror(context.config.mirror, f'$releasever/{repo}/$basearch/os')}"
         else:
index 745b34c6d10121368b26362a445ec6e6ce4caf65..889705614e95cf23b64120b3d625fed7d221ab36 100644 (file)
@@ -1,11 +1,17 @@
 # SPDX-License-Identifier: LGPL-2.1-or-later
 
+import datetime
+import locale
 from collections.abc import Iterable
 from pathlib import Path
 
+from mkosi.config import Config
 from mkosi.context import Context
-from mkosi.distributions import Distribution, debian
+from mkosi.curl import curl
+from mkosi.distributions import Distribution, debian, join_mirror
 from mkosi.installer.apt import AptRepository
+from mkosi.log import die
+from mkosi.util import startswith
 
 
 class Installer(debian.Installer):
@@ -53,6 +59,7 @@ class Installer(debian.Installer):
             suite=context.config.release,
             components=components,
             signedby=signedby,
+            snapshot=context.config.snapshot,
         )
 
         yield AptRepository(
@@ -61,6 +68,7 @@ class Installer(debian.Installer):
             suite=f"{context.config.release}-updates",
             components=components,
             signedby=signedby,
+            snapshot=context.config.snapshot,
         )
 
         # Security updates repos are never mirrored. But !x86 are on the ports server.
@@ -75,4 +83,26 @@ class Installer(debian.Installer):
             suite=f"{context.config.release}-security",
             components=components,
             signedby=signedby,
+            snapshot=context.config.snapshot,
         )
+
+    @classmethod
+    def latest_snapshot(cls, config: Config) -> str:
+        mirror = config.mirror or "http://snapshot.ubuntu.com"
+        release = curl(config, join_mirror(mirror, f"ubuntu/dists/{config.release}-updates/Release"))
+
+        for line in release.splitlines():
+            if date := startswith(line, "Date: "):
+                # %a and %b parse the abbreviated day of the week and the abbreviated month which are both
+                # locale-specific so set the locale to C explicitly to make sure we try to parse the english
+                # abbreviations used in the Release file.
+                lc = locale.setlocale(locale.LC_TIME)
+                try:
+                    locale.setlocale(locale.LC_TIME, "C")
+                    return datetime.datetime.strptime(date, "%a, %d %b %Y %H:%M:%S %Z").strftime(
+                        "%Y%m%dT%H%M%SZ"
+                    )
+                finally:
+                    locale.setlocale(locale.LC_TIME, lc)
+
+        die("Release file is missing Date field")
index 76f747b905a2941b0a84b3f3d19c0497f7e0f0e7..cff959010dc2af43e3e9cffd5f40b5e96ea7b683 100644 (file)
@@ -22,6 +22,7 @@ class AptRepository:
     suite: str
     components: tuple[str, ...]
     signedby: Optional[Path]
+    snapshot: Optional[str] = None
 
     def __str__(self) -> str:
         return textwrap.dedent(
@@ -31,6 +32,7 @@ class AptRepository:
             Suites: {self.suite}
             Components: {" ".join(self.components)}
             {"Signed-By" if self.signedby else "Trusted"}: {self.signedby or "yes"}
+            {f"Snapshot: {self.snapshot}" if self.snapshot else ""}
 
             """
         )
index a10d1976bf388445b95cd87a4de1095dc6664309..e41663852f875c60aa4bde48a6fcd05424556467 100644 (file)
@@ -48,6 +48,8 @@ mkosi — Build Bespoke OS Images
 
 `mkosi [options…] completion [shell]`
 
+`mkosi [options…] latest-snapshot`
+
 `mkosi [options…] help`
 
 # DESCRIPTION
@@ -224,6 +226,14 @@ The following command line verbs are known:
     See the documentation for `ToolsTreeProfiles=` for a list of
     available profiles.
 
+`latest-snapshot`
+:   Output the latest available snapshot in the configured mirror.
+
+    This verb is useful to automatically bump snapshots every so often.
+    Note that this verb only outputs the latest snapshot. It's up to the
+    caller to ensure that the snapshot is written to the intended configuration
+    file.
+
 `help`
 :   This verb is equivalent to the `--help` switch documented below: it
     shows a brief usage explanation.
@@ -502,7 +512,7 @@ boolean argument: either `1`, `yes`, or `true` to enable, or `0`, `no`,
 
     |                | x86-64                            | aarch64                        |
     |----------------|-----------------------------------|--------------------------------|
-    | `debian`       | http://deb.debian.org/debian      |                                |
+    | `debian`       | http://deb.debian.org             |                                |
     | `arch`         | https://geo.mirror.pkgbuild.com   | http://mirror.archlinuxarm.org |
     | `opensuse`     | http://download.opensuse.org      |                                |
     | `kali`         | http://http.kali.org/kali         |                                |
@@ -516,6 +526,26 @@ boolean argument: either `1`, `yes`, or `true` to enable, or `0`, `no`,
     | `openmandriva` | http://mirrors.openmandriva.org   |                                |
     | `azure`        | https://packages.microsoft.com/   |                                |
 
+`Snapshot=`
+:   Download packages from the given snapshot instead of downloading the latest
+    distribution packages from the given mirror. Takes a snapshot ID (the format
+    of the snapshot ID differs per distribution), use the `latest-snapshot` verb
+    to figure out the latest available snapshot.
+
+    If this setting is configured and `Mirror=` is not explicitly configured, different
+    default mirrors are used:
+
+    |                | x86-64                             | aarch64                        |
+    |----------------|------------------------------------|--------------------------------|
+    | `debian`       | https://snapshot.debian.org        |                                |
+    | `arch`         | https://archive.archlinux.org      | http://mirror.archlinuxarm.org |
+    | `opensuse`     | http://download.opensuse.org       |                                |
+    | `ubuntu`       | http://archive.ubuntu.com          | http://ports.ubuntu.com        |
+    | `centos`       | https://composes.stream.centos.org |                                |
+    | `fedora`       | https://kojipkgs.fedoraproject.org |                                |
+
+    For any distribution not listed above, snapshots are not supported.
+
 `LocalMirror=`, `--local-mirror=`
 :   The mirror will be used as a local, plain and direct mirror instead
     of using it as a prefix for the full set of repositories normally supported
index 6fd59fe493112cc9990b994a535813a0d3cf169f..929ec2aeb7f20b685c8b094886edc1dde02bf466 100644 (file)
@@ -370,6 +370,7 @@ def test_config() -> None:
                     "Target": "/qux"
                 }
             ],
+            "Snapshot": "snapshot",
             "SourceDateEpoch": 12345,
             "Splash": "/splash",
             "SplitArtifacts": [
@@ -577,6 +578,7 @@ def test_config() -> None:
         sign_expected_pcr=ConfigFeature.disabled,
         sign=False,
         skeleton_trees=[ConfigTree(Path("/foo/bar"), Path("/")), ConfigTree(Path("/bar/baz"), Path("/qux"))],
+        snapshot="snapshot",
         source_date_epoch=12345,
         splash=Path("/splash"),
         split_artifacts=[ArtifactOutput.uki, ArtifactOutput.kernel],