From c5324fcdd26863c5349785f765fa34af0a45e394 Mon Sep 17 00:00:00 2001 From: DaanDeMeyer Date: Thu, 10 Jul 2025 12:21:11 +0200 Subject: [PATCH] Add new Snapshot= setting In systemd CI, we often run into issues caused by updates to third-party components like the kernel package in rolling release distributions like Arch Linux or Fedora Rawhide. When these happen, the corresponding CI job starts failing on every PR and bisecting the distribution to figure out when the breakage was introduced is rather tedious. To mitigate this problem, we need to be able to pin the rolling release distributions to a specific snapshot which we control. This allows us to update the pinned snapshot in a PR created by a bot, so that any failures introduced by moving to a newer snapshot will be limited to the PR that bumps the snapshot. Any regressions can then be debugged and fixed before merging the PR that switches us to the new snapshot. To make this possible, let's introduce a new Snapshot= setting and implement it for every distribution that has a snapshot concept or something that maps to it. Per distribution: - Debian => snapshot.debian.org (unlimited) - Ubuntu => snapshot.ubuntu.com (unlimited) - Arch => archive.archlinux.org (unlimited) - OpenSUSE => download.opensuse.org/history (limited to a month of snapshots) - CentOS => composes.stream.centos.org (limited to 3 weeks of snapshots) - Fedora => https://kojipkgs.fedoraproject.org (limited to 2 weeks of snapshots) Additionally, for CentOS, we also support using composes from mirror.facebook.net which keeps them around forever so we get unlimited snapshots there as well for CentOS Stream. We also add a latest-snapshot verb to be able to easily figure out the latest snapshot so it can be bumped regularly via a CI workflow. Because we do not track sufficient information from config files to be able to insert the updated snapshot into the right config file ourselves, we output it on stdout instead and leave it to users to insert it into the right config file. --- mkosi/__init__.py | 9 ++++ mkosi/config.py | 24 ++++++++++- mkosi/curl.py | 39 ++++++++++++++--- mkosi/distributions/__init__.py | 8 ++++ mkosi/distributions/alma.py | 4 ++ mkosi/distributions/arch.py | 31 ++++++++++++-- mkosi/distributions/azure.py | 3 ++ mkosi/distributions/centos.py | 65 +++++++++++++++++++++++++---- mkosi/distributions/debian.py | 54 ++++++++++++++++++------ mkosi/distributions/fedora.py | 54 ++++++++++++++++++++---- mkosi/distributions/kali.py | 3 ++ mkosi/distributions/mageia.py | 3 ++ mkosi/distributions/openmandriva.py | 3 ++ mkosi/distributions/opensuse.py | 28 ++++++++----- mkosi/distributions/rhel.py | 3 ++ mkosi/distributions/rhel_ubi.py | 4 ++ mkosi/distributions/rocky.py | 4 ++ mkosi/distributions/ubuntu.py | 32 +++++++++++++- mkosi/installer/apt.py | 2 + mkosi/resources/man/mkosi.1.md | 32 +++++++++++++- tests/test_json.py | 2 + 21 files changed, 356 insertions(+), 51 deletions(-) diff --git a/mkosi/__init__.py b/mkosi/__init__.py index f2805e9ef..df81cd4e6 100644 --- a/mkosi/__init__.py +++ b/mkosi/__init__.py @@ -1344,6 +1344,7 @@ def finalize_default_initrd( f"--release={config.release}", f"--architecture={config.architecture}", *([f"--mirror={config.mirror}"] if config.mirror else []), + *([f"--snapshot={config.snapshot}"] if config.snapshot else []), f"--repository-key-check={config.repository_key_check}", f"--repository-key-fetch={config.repository_key_fetch}", *([f"--repositories={repository}" for repository in config.repositories]), @@ -4243,6 +4244,10 @@ def run_box(args: Args, config: Config) -> None: ) +def run_latest_snapshot(args: Args, config: Config) -> None: + print(config.distribution.latest_snapshot(config)) + + def run_shell(args: Args, config: Config) -> None: opname = "acquire shell in" if args.verb == Verb.shell else "boot" if config.output_format not in (OutputFormat.directory, OutputFormat.disk): @@ -5103,6 +5108,10 @@ def run_verb(args: Args, tools: Optional[Config], images: Sequence[Config], *, r finalize_image_version(args, last) return + if args.verb == Verb.latest_snapshot: + run_latest_snapshot(args, last) + return + if args.verb == Verb.clean: if tools and args.force > 0: run_clean(args, tools) diff --git a/mkosi/config.py b/mkosi/config.py index 3c09ffb03..c006fd81c 100644 --- a/mkosi/config.py +++ b/mkosi/config.py @@ -90,6 +90,7 @@ class Verb(StrEnum): box = enum.auto() sandbox = enum.auto() init = enum.auto() + latest_snapshot = enum.auto() def supports_cmdline(self) -> bool: return self in ( @@ -111,7 +112,14 @@ class Verb(StrEnum): ) def needs_tools(self) -> bool: - return self in (Verb.box, Verb.sandbox, Verb.journalctl, Verb.coredumpctl, Verb.ssh) + return self in ( + Verb.box, + Verb.sandbox, + Verb.journalctl, + Verb.coredumpctl, + Verb.ssh, + Verb.latest_snapshot, + ) def needs_build(self) -> bool: return self in ( @@ -1954,6 +1962,7 @@ class Config: release: str architecture: Architecture mirror: Optional[str] + snapshot: Optional[str] local_mirror: Optional[str] repository_key_check: bool repository_key_fetch: bool @@ -2335,6 +2344,7 @@ class Config: "distribution": self.distribution, "release": self.release, "mirror": self.mirror, + "snapshot": self.snapshot, "architecture": self.architecture, # Caching the package manager used does not matter for the default tools tree because we don't # cache the package manager metadata for the tools tree either. In fact, it can cause issues as @@ -2673,6 +2683,15 @@ SETTINGS: list[ConfigSetting[Any]] = [ help="Distribution mirror to use", scope=SettingScope.universal, ), + ConfigSetting( + dest="snapshot", + section="Distribution", + help="Distribution snapshot to use", + path_suffixes=("snapshot",), + path_read_text=True, + scope=SettingScope.universal, + tools=True, + ), ConfigSetting( dest="local_mirror", section="Distribution", @@ -4992,7 +5011,7 @@ def have_history(args: Args) -> bool: if args.directory is None: return False - if args.verb in (Verb.clean, Verb.sandbox): + if args.verb in (Verb.clean, Verb.sandbox, Verb.latest_snapshot): return False if args.verb == Verb.summary and args.force > 0: @@ -5413,6 +5432,7 @@ def summary(config: Config) -> str: Release: {bold(none_to_na(config.release))} Architecture: {config.architecture} Mirror: {none_to_default(config.mirror)} + Snapshot: {none_to_none(config.snapshot)} Local Mirror (build): {none_to_none(config.local_mirror)} Repo Signature/Key check: {yes_no(config.repository_key_check)} Fetch Repository Keys: {yes_no(config.repository_key_fetch)} diff --git a/mkosi/curl.py b/mkosi/curl.py index 6560bf987..0db38229a 100644 --- a/mkosi/curl.py +++ b/mkosi/curl.py @@ -1,19 +1,42 @@ # SPDX-License-Identifier: LGPL-2.1-or-later +import os +import subprocess from pathlib import Path +from typing import Optional, overload from mkosi.config import Config from mkosi.mounts import finalize_certificate_mounts from mkosi.run import run, workdir -def curl(config: Config, url: str, output_dir: Path, log: bool = True) -> None: - run( +@overload +def curl( + config: Config, + url: str, + *, + output_dir: Optional[Path], + log: bool = True, +) -> None: ... + + +@overload +def curl( + config: Config, + url: str, + *, + output_dir: None = None, + log: bool = True, +) -> str: ... + + +def curl(config: Config, url: str, *, output_dir: Optional[Path] = None, log: bool = True) -> Optional[str]: + result = run( [ "curl", "--location", - "--output-dir", workdir(output_dir), - "--remote-name", + *(["--output-dir", workdir(output_dir)] if output_dir else []), + *(["--remote-name"] if output_dir else []), "--no-progress-meter", "--fail", *(["--silent"] if not log else []), @@ -24,9 +47,15 @@ def curl(config: Config, url: str, output_dir: Path, log: bool = True) -> None: *(["--proxy-key", "/proxy.clientkey"] if config.proxy_client_key else []), url, ], + stdout=None if output_dir else subprocess.PIPE, sandbox=config.sandbox( network=True, - options=["--bind", output_dir, workdir(output_dir), *finalize_certificate_mounts(config)], + options=[ + *(["--bind", os.fspath(output_dir), workdir(output_dir)] if output_dir else []), + *finalize_certificate_mounts(config) + ], ), log=log, ) # fmt: skip + + return None if output_dir else result.stdout diff --git a/mkosi/distributions/__init__.py b/mkosi/distributions/__init__.py index 381becc29..1023229a8 100644 --- a/mkosi/distributions/__init__.py +++ b/mkosi/distributions/__init__.py @@ -6,6 +6,7 @@ import urllib.parse from pathlib import Path from typing import TYPE_CHECKING, Optional, cast +from mkosi.log import die from mkosi.util import StrEnum, read_env_file if TYPE_CHECKING: @@ -66,6 +67,10 @@ class DistributionInstaller: def grub_prefix(cls) -> str: return "grub" + @classmethod + def latest_snapshot(cls, config: "Config") -> str: + die(f"{cls.pretty_name()} does not support snapshots") + class Distribution(StrEnum): # Please consult docs/distribution-policy.md and contact one @@ -148,6 +153,9 @@ class Distribution(StrEnum): def createrepo(self, context: "Context") -> None: return self.installer().package_manager(context.config).createrepo(context) + def latest_snapshot(self, config: "Config") -> str: + return self.installer().latest_snapshot(config) + def installer(self) -> type[DistributionInstaller]: modname = str(self).replace("-", "_") mod = importlib.import_module(f"mkosi.distributions.{modname}") diff --git a/mkosi/distributions/alma.py b/mkosi/distributions/alma.py index 4e668f291..3920be776 100644 --- a/mkosi/distributions/alma.py +++ b/mkosi/distributions/alma.py @@ -3,6 +3,7 @@ from mkosi.context import Context from mkosi.distributions import centos, join_mirror from mkosi.installer.rpm import RpmRepository, find_rpm_gpgkey +from mkosi.log import die class Installer(centos.Installer): @@ -28,6 +29,9 @@ class Installer(centos.Installer): gpgurls: tuple[str, ...], repo: str, ) -> list[RpmRepository]: + if context.config.snapshot: + die(f"Snapshot= is not supported for {cls.pretty_name()}") + if context.config.mirror: url = f"baseurl={join_mirror(context.config.mirror, f'$releasever/{repo}/$basearch/os')}" else: diff --git a/mkosi/distributions/arch.py b/mkosi/distributions/arch.py index 8373306a7..6ce0ccf40 100644 --- a/mkosi/distributions/arch.py +++ b/mkosi/distributions/arch.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: LGPL-2.1-or-later +import datetime import tempfile from collections.abc import Iterable from pathlib import Path @@ -8,7 +9,7 @@ from mkosi.archive import extract_tar from mkosi.config import Architecture, Config from mkosi.context import Context from mkosi.curl import curl -from mkosi.distributions import DistributionInstaller, PackageType +from mkosi.distributions import DistributionInstaller, PackageType, join_mirror from mkosi.installer.pacman import Pacman, PacmanRepository from mkosi.log import complete_step, die @@ -44,7 +45,7 @@ class Installer(DistributionInstaller): curl( context.config, "https://archlinux.org/packages/core/any/archlinux-keyring/download", - Path(d), + output_dir=Path(d), ) extract_tar( next(Path(d).iterdir()), @@ -69,9 +70,24 @@ class Installer(DistributionInstaller): yield PacmanRepository("core", context.config.local_mirror) else: if context.config.architecture.is_arm_variant(): - url = f"{context.config.mirror or 'http://mirror.archlinuxarm.org'}/$arch/$repo" + if context.config.snapshot and not context.config.mirror: + die("There is no known public mirror for snapshots of Arch Linux ARM") + + mirror = context.config.mirror or "http://mirror.archlinuxarm.org" + else: + if context.config.mirror: + mirror = context.config.mirror + elif context.config.snapshot: + mirror = "https://archive.archlinux.org" + else: + mirror = "https://geo.mirror.pkgbuild.com" + + if context.config.snapshot: + url = join_mirror(mirror, f"repos/{context.config.snapshot}/$repo/os/$arch") + elif context.config.architecture.is_arm_variant(): + url = join_mirror(mirror, "$arch/$repo") else: - url = f"{context.config.mirror or 'https://geo.mirror.pkgbuild.com'}/$repo/os/$arch" + url = join_mirror(mirror, "$repo/os/$arch") # Testing repositories have to go before regular ones to to take precedence. repos = [ @@ -107,3 +123,10 @@ class Installer(DistributionInstaller): die(f"Architecture {a} is not supported by Arch Linux") return a + + @classmethod + def latest_snapshot(cls, config: Config) -> str: + url = join_mirror(config.mirror or "https://archive.archlinux.org", "repos/last/lastsync") + return datetime.datetime.fromtimestamp(int(curl(config, url)), datetime.timezone.utc).strftime( + "%Y/%m/%d" + ) diff --git a/mkosi/distributions/azure.py b/mkosi/distributions/azure.py index f66d8a450..4049b85ec 100644 --- a/mkosi/distributions/azure.py +++ b/mkosi/distributions/azure.py @@ -37,6 +37,9 @@ class Installer(fedora.Installer): @classmethod def repositories(cls, context: Context) -> Iterable[RpmRepository]: + if context.config.snapshot: + die(f"Snapshot= is not supported for {cls.pretty_name()}") + gpgurls = ( find_rpm_gpgkey( context, diff --git a/mkosi/distributions/centos.py b/mkosi/distributions/centos.py index 7c770c8f5..3c6d1623b 100644 --- a/mkosi/distributions/centos.py +++ b/mkosi/distributions/centos.py @@ -4,6 +4,7 @@ from collections.abc import Iterable from mkosi.config import Architecture, Config from mkosi.context import Context +from mkosi.curl import curl from mkosi.distributions import ( Distribution, DistributionInstaller, @@ -13,6 +14,7 @@ from mkosi.distributions import ( from mkosi.installer.dnf import Dnf from mkosi.installer.rpm import RpmRepository, find_rpm_gpgkey, setup_rpm from mkosi.log import die +from mkosi.util import startswith from mkosi.versioncomp import GenericVersion CENTOS_SIG_REPO_PRIORITY = 50 @@ -132,38 +134,67 @@ class Installer(DistributionInstaller): gpgurls: tuple[str, ...], repo: str, ) -> Iterable[RpmRepository]: + mirror = context.config.mirror + if not mirror and context.config.snapshot: + mirror = "https://composes.stream.centos.org" + + if context.config.snapshot and mirror not in ( + "https://composes.stream.centos.org", + "https://mirror.facebook.net/centos-composes", + ): + die( + f"Snapshot= is only supported for {cls.pretty_name()} if Mirror=https://composes.stream.centos.org" + ) + + if ( + mirror in ("https://composes.stream.centos.org", "https://mirror.facebook.net/centos-composes") + and not context.config.snapshot + ): + die(f"Snapshot= must be used on {cls.pretty_name()} if Mirror={mirror}") + if context.config.local_mirror: yield RpmRepository(repo, f"baseurl={context.config.local_mirror}", gpgurls) - elif mirror := context.config.mirror: + elif mirror: + if mirror == "https://composes.stream.centos.org": + subdir = f"stream-{context.config.release}/production" + elif mirror == "https://mirror.facebook.net/centos-composes": + subdir = context.config.release + elif repo == "extras": + subdir = "SIGs/$stream" + else: + subdir = "$stream" + + if context.config.snapshot: + subdir += f"/CentOS-Stream-{context.config.release}-{context.config.snapshot}/compose" + if repo == "extras": yield RpmRepository( repo.lower(), - f"baseurl={join_mirror(mirror, f'SIGs/$stream/{repo}/$basearch/extras-common')}", + f"baseurl={join_mirror(mirror, f'{subdir}/{repo}/$basearch/extras-common')}", gpgurls, ) yield RpmRepository( f"{repo.lower()}-source", - f"baseurl={join_mirror(mirror, f'SIGs/$stream/{repo}/source/extras-common')}", + f"baseurl={join_mirror(mirror, f'{subdir}/{repo}/source/extras-common')}", gpgurls, enabled=False, ) - else: yield RpmRepository( repo.lower(), - f"baseurl={join_mirror(mirror, f'$stream/{repo}/$basearch/os')}", + f"baseurl={join_mirror(mirror, f'{subdir}/{repo}/$basearch/os')}", gpgurls, ) yield RpmRepository( f"{repo.lower()}-debuginfo", - f"baseurl={join_mirror(mirror, f'$stream/{repo}/$basearch/debug/tree')}", + f"baseurl={join_mirror(mirror, f'{subdir}/{repo}/$basearch/debug/tree')}", gpgurls, enabled=False, ) yield RpmRepository( f"{repo.lower()}-source", - f"baseurl={join_mirror(mirror, f'$stream/{repo}/source/tree')}", + f"baseurl={join_mirror(mirror, f'{subdir}/{repo}/source/tree')}", gpgurls, enabled=False, ) @@ -211,8 +242,9 @@ class Installer(DistributionInstaller): yield from cls.repository_variants(context, gpgurls, "BaseOS") yield from cls.repository_variants(context, gpgurls, "AppStream") - yield from cls.repository_variants(context, gpgurls, "extras") yield from cls.repository_variants(context, gpgurls, "CRB") + if not context.config.snapshot: + yield from cls.repository_variants(context, gpgurls, "extras") yield from cls.epel_repositories(context) yield from cls.sig_repositories(context) @@ -422,3 +454,20 @@ class Installer(DistributionInstaller): enabled=False, priority=CENTOS_SIG_REPO_PRIORITY, ) + + @classmethod + def latest_snapshot(cls, config: Config) -> str: + mirror = config.mirror or "https://composes.stream.centos.org" + + if mirror == "https://mirror.facebook.net/centos-composes": + subdir = config.release + else: + subdir = f"stream-{config.release}/production" + + url = join_mirror(mirror, f"{subdir}/latest-CentOS-Stream/compose/.composeinfo") + + for line in curl(config, url).splitlines(): + if snapshot := startswith(line, f"id = CentOS-Stream-{config.release}-"): + return snapshot + + die("composeinfo is missing compose ID field") diff --git a/mkosi/distributions/debian.py b/mkosi/distributions/debian.py index 4ceec90da..5873c5493 100644 --- a/mkosi/distributions/debian.py +++ b/mkosi/distributions/debian.py @@ -1,14 +1,17 @@ # SPDX-License-Identifier: LGPL-2.1-or-later import itertools +import json import tempfile from collections.abc import Iterable from pathlib import Path +from typing import cast from mkosi.archive import extract_tar from mkosi.config import Architecture, Config from mkosi.context import Context -from mkosi.distributions import DistributionInstaller, PackageType +from mkosi.curl import curl +from mkosi.distributions import DistributionInstaller, PackageType, join_mirror from mkosi.installer.apt import Apt, AptRepository from mkosi.log import die from mkosi.run import run, workdir @@ -51,19 +54,33 @@ class Installer(DistributionInstaller): ) return - mirror = context.config.mirror or "http://deb.debian.org/debian" + if context.config.mirror: + mirror = context.config.mirror + elif context.config.snapshot: + mirror = "https://snapshot.debian.org" + else: + mirror = "http://deb.debian.org" + + if context.config.snapshot: + url = join_mirror(mirror, f"archive/debian/{context.config.snapshot}") + else: + url = join_mirror(mirror, "debian") + signedby = Path("/usr/share/keyrings/debian-archive-keyring.gpg") yield AptRepository( types=types, - url=mirror, + url=url, suite=context.config.release, components=components, signedby=signedby, ) # Debug repos are typically not mirrored. - url = "http://deb.debian.org/debian-debug" + if context.config.snapshot: + url = join_mirror(mirror, f"archive/debian-debug/{context.config.snapshot}") + else: + url = join_mirror(mirror, "debian-debug") yield AptRepository( types=types, @@ -76,18 +93,24 @@ class Installer(DistributionInstaller): if context.config.release in ("unstable", "sid"): return - yield AptRepository( - types=types, - url=mirror, - suite=f"{context.config.release}-updates", - components=components, - signedby=signedby, - ) + if not context.config.snapshot: + yield AptRepository( + types=types, + url=join_mirror(mirror, "debian"), + suite=f"{context.config.release}-updates", + components=components, + signedby=signedby, + ) + + # Security updates repos are never mirrored. + if context.config.snapshot: + url = join_mirror(mirror, f"archive/debian-security/{context.config.snapshot}") + else: + url = join_mirror(mirror, "debian-security") yield AptRepository( types=types, - # Security updates repos are never mirrored. - url="http://security.debian.org/debian-security", + url=url, suite=f"{context.config.release}-security", components=components, signedby=signedby, @@ -220,6 +243,11 @@ class Installer(DistributionInstaller): return a + @classmethod + def latest_snapshot(cls, config: Config) -> str: + url = join_mirror(config.mirror or "https://snapshot.debian.org", "mr/timestamp") + return cast(str, json.loads(curl(config, url))["result"]["debian"][-1]) + def install_apt_sources(context: Context, repos: Iterable[AptRepository]) -> None: sources = context.root / f"etc/apt/sources.list.d/{context.config.release}.sources" diff --git a/mkosi/distributions/fedora.py b/mkosi/distributions/fedora.py index 2da823b28..598b06e9d 100644 --- a/mkosi/distributions/fedora.py +++ b/mkosi/distributions/fedora.py @@ -29,7 +29,11 @@ def read_remote_rawhide_key_symlink(context: Context) -> str: # let's fetch it from distribution-gpg-keys on github if necessary, which is generally up-to-date. with tempfile.TemporaryDirectory() as d: # The rawhide key is a symlink and github doesn't redirect those to the actual file for some reason - curl(context.config, f"{DISTRIBUTION_GPG_KEYS_UPSTREAM}/RPM-GPG-KEY-fedora-rawhide-primary", Path(d)) + curl( + context.config, + f"{DISTRIBUTION_GPG_KEYS_UPSTREAM}/RPM-GPG-KEY-fedora-rawhide-primary", + output_dir=Path(d), + ) return (Path(d) / "RPM-GPG-KEY-fedora-rawhide-primary").read_text() @@ -59,7 +63,7 @@ def find_fedora_rpm_gpgkeys(context: Context) -> Iterable[str]: curl( context.config, f"{DISTRIBUTION_GPG_KEYS_UPSTREAM}/RPM-GPG-KEY-fedora-{version + 1}-primary", - Path(d), + output_dir=Path(d), log=False, ) @@ -138,6 +142,23 @@ class Installer(DistributionInstaller): def repositories(cls, context: Context) -> Iterable[RpmRepository]: gpgurls = find_fedora_rpm_gpgkeys(context) + if context.config.snapshot and context.config.release != "rawhide": + die(f"Snapshot= is only supported for rawhide on {cls.pretty_name()}") + + mirror = context.config.mirror + if not mirror and context.config.snapshot: + mirror = "https://kojipkgs.fedoraproject.org" + + if context.config.snapshot and mirror != "https://kojipkgs.fedoraproject.org": + die( + f"Snapshot= is only supported for {cls.pretty_name()} if Mirror=https://kojipkgs.fedoraproject.org" + ) + + if mirror == "https://kojipkgs.fedoraproject.org" and not context.config.snapshot: + die( + f"Snapshot= must be used on {cls.pretty_name()} if Mirror=https://kojipkgs.fedoraproject.org" + ) + if context.config.local_mirror: yield RpmRepository("fedora", f"baseurl={context.config.local_mirror}", gpgurls) return @@ -152,20 +173,29 @@ class Installer(DistributionInstaller): f"{repo.lower()}-debuginfo", f"{url}/$basearch/debug/tree", gpgurls, enabled=False ) yield RpmRepository(f"{repo.lower()}-source", f"{url}/source/tree", gpgurls, enabled=False) - elif m := context.config.mirror: - directory = "development" if context.config.release == "rawhide" else "releases" - url = f"baseurl={join_mirror(m, f'linux/{directory}/$releasever/Everything')}" + elif mirror: + if mirror == "https://kojipkgs.fedoraproject.org": + subdir = f"compose/{context.config.release}" + else: + subdir = "linux/" + subdir += "development" if context.config.release == "rawhide" else "releases" + subdir += "/$releasever" + + if context.config.snapshot: + subdir += f"/Fedora-{context.config.release.capitalize()}-{context.config.snapshot}/compose" + + url = f"baseurl={join_mirror(mirror, f'{subdir}/Everything')}" yield RpmRepository("fedora", f"{url}/$basearch/os", gpgurls) yield RpmRepository("fedora-debuginfo", f"{url}/$basearch/debug/tree", gpgurls, enabled=False) yield RpmRepository("fedora-source", f"{url}/source/tree", gpgurls, enabled=False) if context.config.release != "rawhide": - url = f"baseurl={join_mirror(m, 'linux/updates/$releasever/Everything')}" + url = f"baseurl={join_mirror(mirror, 'linux/updates/$releasever/Everything')}" yield RpmRepository("updates", f"{url}/$basearch", gpgurls) yield RpmRepository("updates-debuginfo", f"{url}/$basearch/debug", gpgurls, enabled=False) yield RpmRepository("updates-source", f"{url}/source/tree", gpgurls, enabled=False) - url = f"baseurl={join_mirror(m, 'linux/updates/testing/$releasever/Everything')}" + url = f"baseurl={join_mirror(mirror, 'linux/updates/testing/$releasever/Everything')}" yield RpmRepository("updates-testing", f"{url}/$basearch", gpgurls, enabled=False) yield RpmRepository( "updates-testing-debuginfo", f"{url}/$basearch/debug", gpgurls, enabled=False @@ -227,3 +257,13 @@ class Installer(DistributionInstaller): die(f"Architecture {a} is not supported by Fedora") return a + + @classmethod + def latest_snapshot(cls, config: Config) -> str: + mirror = config.mirror or "https://kojipkgs.fedoraproject.org" + + url = join_mirror( + mirror, f"compose/{config.release}/latest-Fedora-{config.release.capitalize()}/COMPOSE_ID" + ) + + return curl(config, url).removeprefix(f"Fedora-{config.release.capitalize()}-").strip() diff --git a/mkosi/distributions/kali.py b/mkosi/distributions/kali.py index 73370509e..0f64cddbe 100644 --- a/mkosi/distributions/kali.py +++ b/mkosi/distributions/kali.py @@ -25,6 +25,9 @@ class Installer(debian.Installer): @classmethod def repositories(cls, context: Context, local: bool = True) -> Iterable[AptRepository]: + if context.config.snapshot: + die(f"Snapshot= is not supported for {cls.pretty_name()}") + if context.config.local_mirror and local: yield AptRepository( types=("deb",), diff --git a/mkosi/distributions/mageia.py b/mkosi/distributions/mageia.py index 65224f69e..b95cacb5b 100644 --- a/mkosi/distributions/mageia.py +++ b/mkosi/distributions/mageia.py @@ -29,6 +29,9 @@ class Installer(fedora.Installer): @classmethod def repositories(cls, context: Context) -> Iterable[RpmRepository]: + if context.config.snapshot: + die(f"Snapshot= is not supported for {cls.pretty_name()}") + gpgurls = ( find_rpm_gpgkey( context, diff --git a/mkosi/distributions/openmandriva.py b/mkosi/distributions/openmandriva.py index 2bfc2a225..1ea47a05e 100644 --- a/mkosi/distributions/openmandriva.py +++ b/mkosi/distributions/openmandriva.py @@ -29,6 +29,9 @@ class Installer(fedora.Installer): @classmethod def repositories(cls, context: Context) -> Iterable[RpmRepository]: + if context.config.snapshot: + die(f"Snapshot= is not supported for {cls.pretty_name()}") + mirror = context.config.mirror or "http://mirror.openmandriva.org" gpgurls = ( diff --git a/mkosi/distributions/opensuse.py b/mkosi/distributions/opensuse.py index 943dbd44b..5327c9e30 100644 --- a/mkosi/distributions/opensuse.py +++ b/mkosi/distributions/opensuse.py @@ -68,7 +68,7 @@ class Installer(DistributionInstaller): zypper = cls.package_manager(context.config) is Zypper mirror = context.config.mirror or "https://download.opensuse.org" - if context.config.release == "tumbleweed" or context.config.release.isdigit(): + if context.config.release == "tumbleweed": gpgkeys = tuple( p for key in ("RPM-GPG-KEY-openSUSE-Tumbleweed", "RPM-GPG-KEY-openSUSE") @@ -97,7 +97,12 @@ class Installer(DistributionInstaller): ), ) # fmt: skip - if context.config.release == "tumbleweed": + if context.config.snapshot: + if context.config.architecture != Architecture.x86_64: + die(f"Snapshot= is only supported for x86-64 on {cls.pretty_name()}") + + subdir = f"history/{context.config.snapshot}" + else: if context.config.architecture == Architecture.x86_64: subdir = "" elif context.config.architecture == Architecture.arm64: @@ -116,11 +121,6 @@ class Installer(DistributionInstaller): subdir = "ports/riscv" else: die(f"{context.config.architecture} not supported by openSUSE Tumbleweed") - else: - if context.config.architecture != Architecture.x86_64: - die(f"Old snapshots are only supported for x86-64 on {cls.pretty_name()}") - - subdir = f"history/{context.config.release}" for repo in ("oss", "non-oss"): url = join_mirror(mirror, f"{subdir}/tumbleweed/repo/{repo}") @@ -131,7 +131,7 @@ class Installer(DistributionInstaller): enabled=repo == "oss", ) - if context.config.release == "tumbleweed": + if not context.config.snapshot: for d in ("debug", "source"): url = join_mirror(mirror, f"{subdir}/{d}/tumbleweed/repo/{repo}") yield RpmRepository( @@ -141,7 +141,7 @@ class Installer(DistributionInstaller): enabled=False, ) - if context.config.release == "tumbleweed": + if not context.config.snapshot: url = join_mirror(mirror, f"{subdir}/update/tumbleweed") yield RpmRepository( id="oss-update", @@ -157,6 +157,9 @@ class Installer(DistributionInstaller): enabled=False, ) else: + if context.config.snapshot: + die(f"Snapshot= is only supported for Tumbleweed on {cls.pretty_name()}") + if ( context.config.release in ("current", "stable", "leap") and context.config.architecture != Architecture.x86_64 @@ -248,12 +251,17 @@ class Installer(DistributionInstaller): return a + @classmethod + def latest_snapshot(cls, config: Config) -> str: + url = join_mirror(config.mirror or "https://download.opensuse.org", "history/latest") + return curl(config, url).strip() + def fetch_gpgurls(context: Context, repourl: str) -> tuple[str, ...]: gpgurls = [f"{repourl}/repodata/repomd.xml.key"] with tempfile.TemporaryDirectory() as d: - curl(context.config, f"{repourl}/repodata/repomd.xml", Path(d)) + curl(context.config, f"{repourl}/repodata/repomd.xml", output_dir=Path(d)) xml = (Path(d) / "repomd.xml").read_text() root = ElementTree.fromstring(xml) diff --git a/mkosi/distributions/rhel.py b/mkosi/distributions/rhel.py index 5a1bb0ba2..04c2860d6 100644 --- a/mkosi/distributions/rhel.py +++ b/mkosi/distributions/rhel.py @@ -112,6 +112,9 @@ class Installer(centos.Installer): @classmethod def repositories(cls, context: Context) -> Iterable[RpmRepository]: + if context.config.snapshot: + die(f"Snapshot= is not supported for {cls.pretty_name()}") + gpgurls = cls.gpgurls(context) yield from cls.repository_variants(context, gpgurls, "baseos") yield from cls.repository_variants(context, gpgurls, "appstream") diff --git a/mkosi/distributions/rhel_ubi.py b/mkosi/distributions/rhel_ubi.py index 66eacd21f..6e4e8cd99 100644 --- a/mkosi/distributions/rhel_ubi.py +++ b/mkosi/distributions/rhel_ubi.py @@ -5,6 +5,7 @@ from collections.abc import Iterable from mkosi.context import Context from mkosi.distributions import centos, join_mirror from mkosi.installer.rpm import RpmRepository, find_rpm_gpgkey +from mkosi.log import die class Installer(centos.Installer): @@ -55,6 +56,9 @@ class Installer(centos.Installer): @classmethod def repositories(cls, context: Context) -> Iterable[RpmRepository]: + if context.config.snapshot: + die(f"Snapshot= is not supported for {cls.pretty_name()}") + gpgurls = cls.gpgurls(context) yield from cls.repository_variants(context, gpgurls, "baseos") yield from cls.repository_variants(context, gpgurls, "appstream") diff --git a/mkosi/distributions/rocky.py b/mkosi/distributions/rocky.py index bebefae69..5829121bf 100644 --- a/mkosi/distributions/rocky.py +++ b/mkosi/distributions/rocky.py @@ -3,6 +3,7 @@ from mkosi.context import Context from mkosi.distributions import centos, join_mirror from mkosi.installer.rpm import RpmRepository, find_rpm_gpgkey +from mkosi.log import die class Installer(centos.Installer): @@ -28,6 +29,9 @@ class Installer(centos.Installer): gpgurls: tuple[str, ...], repo: str, ) -> list[RpmRepository]: + if context.config.snapshot: + die(f"Snapshot= is not supported for {cls.pretty_name()}") + if context.config.mirror: url = f"baseurl={join_mirror(context.config.mirror, f'$releasever/{repo}/$basearch/os')}" else: diff --git a/mkosi/distributions/ubuntu.py b/mkosi/distributions/ubuntu.py index 745b34c6d..889705614 100644 --- a/mkosi/distributions/ubuntu.py +++ b/mkosi/distributions/ubuntu.py @@ -1,11 +1,17 @@ # SPDX-License-Identifier: LGPL-2.1-or-later +import datetime +import locale from collections.abc import Iterable from pathlib import Path +from mkosi.config import Config from mkosi.context import Context -from mkosi.distributions import Distribution, debian +from mkosi.curl import curl +from mkosi.distributions import Distribution, debian, join_mirror from mkosi.installer.apt import AptRepository +from mkosi.log import die +from mkosi.util import startswith class Installer(debian.Installer): @@ -53,6 +59,7 @@ class Installer(debian.Installer): suite=context.config.release, components=components, signedby=signedby, + snapshot=context.config.snapshot, ) yield AptRepository( @@ -61,6 +68,7 @@ class Installer(debian.Installer): suite=f"{context.config.release}-updates", components=components, signedby=signedby, + snapshot=context.config.snapshot, ) # Security updates repos are never mirrored. But !x86 are on the ports server. @@ -75,4 +83,26 @@ class Installer(debian.Installer): suite=f"{context.config.release}-security", components=components, signedby=signedby, + snapshot=context.config.snapshot, ) + + @classmethod + def latest_snapshot(cls, config: Config) -> str: + mirror = config.mirror or "http://snapshot.ubuntu.com" + release = curl(config, join_mirror(mirror, f"ubuntu/dists/{config.release}-updates/Release")) + + for line in release.splitlines(): + if date := startswith(line, "Date: "): + # %a and %b parse the abbreviated day of the week and the abbreviated month which are both + # locale-specific so set the locale to C explicitly to make sure we try to parse the english + # abbreviations used in the Release file. + lc = locale.setlocale(locale.LC_TIME) + try: + locale.setlocale(locale.LC_TIME, "C") + return datetime.datetime.strptime(date, "%a, %d %b %Y %H:%M:%S %Z").strftime( + "%Y%m%dT%H%M%SZ" + ) + finally: + locale.setlocale(locale.LC_TIME, lc) + + die("Release file is missing Date field") diff --git a/mkosi/installer/apt.py b/mkosi/installer/apt.py index 76f747b90..cff959010 100644 --- a/mkosi/installer/apt.py +++ b/mkosi/installer/apt.py @@ -22,6 +22,7 @@ class AptRepository: suite: str components: tuple[str, ...] signedby: Optional[Path] + snapshot: Optional[str] = None def __str__(self) -> str: return textwrap.dedent( @@ -31,6 +32,7 @@ class AptRepository: Suites: {self.suite} Components: {" ".join(self.components)} {"Signed-By" if self.signedby else "Trusted"}: {self.signedby or "yes"} + {f"Snapshot: {self.snapshot}" if self.snapshot else ""} """ ) diff --git a/mkosi/resources/man/mkosi.1.md b/mkosi/resources/man/mkosi.1.md index a10d1976b..e41663852 100644 --- a/mkosi/resources/man/mkosi.1.md +++ b/mkosi/resources/man/mkosi.1.md @@ -48,6 +48,8 @@ mkosi — Build Bespoke OS Images `mkosi [options…] completion [shell]` +`mkosi [options…] latest-snapshot` + `mkosi [options…] help` # DESCRIPTION @@ -224,6 +226,14 @@ The following command line verbs are known: See the documentation for `ToolsTreeProfiles=` for a list of available profiles. +`latest-snapshot` +: Output the latest available snapshot in the configured mirror. + + This verb is useful to automatically bump snapshots every so often. + Note that this verb only outputs the latest snapshot. It's up to the + caller to ensure that the snapshot is written to the intended configuration + file. + `help` : This verb is equivalent to the `--help` switch documented below: it shows a brief usage explanation. @@ -502,7 +512,7 @@ boolean argument: either `1`, `yes`, or `true` to enable, or `0`, `no`, | | x86-64 | aarch64 | |----------------|-----------------------------------|--------------------------------| - | `debian` | http://deb.debian.org/debian | | + | `debian` | http://deb.debian.org | | | `arch` | https://geo.mirror.pkgbuild.com | http://mirror.archlinuxarm.org | | `opensuse` | http://download.opensuse.org | | | `kali` | http://http.kali.org/kali | | @@ -516,6 +526,26 @@ boolean argument: either `1`, `yes`, or `true` to enable, or `0`, `no`, | `openmandriva` | http://mirrors.openmandriva.org | | | `azure` | https://packages.microsoft.com/ | | +`Snapshot=` +: Download packages from the given snapshot instead of downloading the latest + distribution packages from the given mirror. Takes a snapshot ID (the format + of the snapshot ID differs per distribution), use the `latest-snapshot` verb + to figure out the latest available snapshot. + + If this setting is configured and `Mirror=` is not explicitly configured, different + default mirrors are used: + + | | x86-64 | aarch64 | + |----------------|------------------------------------|--------------------------------| + | `debian` | https://snapshot.debian.org | | + | `arch` | https://archive.archlinux.org | http://mirror.archlinuxarm.org | + | `opensuse` | http://download.opensuse.org | | + | `ubuntu` | http://archive.ubuntu.com | http://ports.ubuntu.com | + | `centos` | https://composes.stream.centos.org | | + | `fedora` | https://kojipkgs.fedoraproject.org | | + + For any distribution not listed above, snapshots are not supported. + `LocalMirror=`, `--local-mirror=` : The mirror will be used as a local, plain and direct mirror instead of using it as a prefix for the full set of repositories normally supported diff --git a/tests/test_json.py b/tests/test_json.py index 6fd59fe49..929ec2aeb 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -370,6 +370,7 @@ def test_config() -> None: "Target": "/qux" } ], + "Snapshot": "snapshot", "SourceDateEpoch": 12345, "Splash": "/splash", "SplitArtifacts": [ @@ -577,6 +578,7 @@ def test_config() -> None: sign_expected_pcr=ConfigFeature.disabled, sign=False, skeleton_trees=[ConfigTree(Path("/foo/bar"), Path("/")), ConfigTree(Path("/bar/baz"), Path("/qux"))], + snapshot="snapshot", source_date_epoch=12345, splash=Path("/splash"), split_artifacts=[ArtifactOutput.uki, ArtifactOutput.kernel], -- 2.47.3