From: Arran Cudbard-Bell Date: Sat, 25 Jul 2026 22:27:59 +0000 (-0600) Subject: scripts/docker: snapshot dnf mirrorlists at image build time X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=7ca0f783d0545d4523385dd89f3ca810feb151fd;p=thirdparty%2Ffreeradius-server.git scripts/docker: snapshot dnf mirrorlists at image build time The mirrors.rockylinux.org / mirrors.fedoraproject.org redirectors occasionally answer with an empty mirror set, which dnf reports as "No URLs in mirrorlist" and treats as fatal rather than retryable, so each mirror list is now fetched once at image build time into /etc/dnf/mirrorlists/ and the enabled repos point at the snapshot with mirrorlist=file:// (explicitly supported by librepo, mirrors tried in file order). The origin CDN leads each snapshot so the fetched mirrors are failover only, minor-versioned paths are rewritten to the major-version symlink form so a snapshot survives point releases, and the unused metalink-only epel-cisco-openh264 repo is disabled. --- diff --git a/scripts/docker/bin/dnf-mirrorlist-snapshot b/scripts/docker/bin/dnf-mirrorlist-snapshot new file mode 100755 index 00000000000..6cd07129bd7 --- /dev/null +++ b/scripts/docker/bin/dnf-mirrorlist-snapshot @@ -0,0 +1,99 @@ +#!/bin/sh +# +# Snapshot dnf mirror lists at image build time. +# +# The mirrors.rockylinux.org / mirrors.fedoraproject.org redirectors +# occasionally answer with an empty mirror set, which dnf reports as +# "No URLs in mirrorlist" and treats as fatal rather than retryable. +# Fetching the mirror list once here, and pointing each enabled repo +# at the local snapshot with mirrorlist=file://, takes the redirectors +# out of the loop for every later dnf invocation. +# +# Each snapshot leads with the origin CDN, so the fetched mirrors only +# see traffic when the origin fails. Minor-versioned path segments in +# fetched entries (.../10.2/...) are rewritten to the major-version +# symlink form so a snapshot survives point releases, which delete old +# minor trees from the mirrors. If the redirector fetch fails or +# returns nothing, the snapshot degrades to the origin entry alone, +# equivalent to pinning the origin as a single baseurl. +# +# librepo handles file:// mirrorlists explicitly (lr_is_local_path, +# lr_prepend_url_protocol) and tries the mirrors in file order. +# +# Usage: dnf-mirrorlist-snapshot rocky|epel +# +# rocky - snapshot the repos the builds enable (baseos, appstream, +# crb, extras); disabled repos keep their remote mirrorlist +# lines and are never fetched. +# epel - snapshot epel and epel-testing; run after epel-release is +# installed. Also disables the EL9-only epel-cisco-openh264 +# repo, which is metalink-only and unused by our builds. + +set -e + +arch="$(uname -m)" +relver="$(. /etc/os-release; echo "${VERSION_ID%%.*}")" + +list_dir=/etc/dnf/mirrorlists +repo_dir=/etc/yum.repos.d + +# +# snapshot
... +# +# Writes / (origin first, fetched mirrors after, +# origin host de-duplicated), then rewrites the = line of +# the [
] block in the given repo files to point at the +# snapshot. +# +snapshot() { + name="$1"; sect="$2"; key="$3"; mlurl="$4"; origin="$5"; shift 5 + + list="${list_dir}/${name}" + origin_host="$(printf '%s' "${origin}" | cut -d/ -f3)" + + { + echo "${origin}" + curl -sf --retry 3 --retry-delay 5 "${mlurl}" \ + | grep '^http' \ + | sed -e "s|/${relver}\.[0-9]*/|/${relver}/|" \ + -e "\\|//${origin_host}/|d" \ + || true + } > "${list}" + + sed -i "/^\[${sect}\]/,/^\[/ s|^${key}=.*|mirrorlist=file://${list}|" "$@" +} + +mkdir -p "${list_dir}" + +case "$1" in +rocky) + for pair in baseos:BaseOS appstream:AppStream crb:CRB extras:extras; do + sect="${pair%%:*}"; repo="${pair##*:}" + snapshot "rocky-${sect}" "${sect}" mirrorlist \ + "https://mirrors.rockylinux.org/mirrorlist?arch=${arch}&repo=${repo}-${relver}" \ + "http://dl.rockylinux.org/pub/rocky/${relver}/${repo}/${arch}/os/" \ + "${repo_dir}"/rocky*.repo + done + ;; + +epel) + snapshot epel epel metalink \ + "https://mirrors.fedoraproject.org/mirrorlist?repo=epel-${relver}&arch=${arch}" \ + "https://dl.fedoraproject.org/pub/epel/${relver}/Everything/${arch}/" \ + "${repo_dir}/epel.repo" + + snapshot epel-testing epel-testing metalink \ + "https://mirrors.fedoraproject.org/mirrorlist?repo=epel-testing-${relver}&arch=${arch}" \ + "https://dl.fedoraproject.org/pub/epel/testing/${relver}/Everything/${arch}/" \ + "${repo_dir}/epel-testing.repo" + + if [ -e "${repo_dir}/epel-cisco-openh264.repo" ]; then + sed -i 's|^enabled=1|enabled=0|' "${repo_dir}/epel-cisco-openh264.repo" + fi + ;; + +*) + echo "Usage: $0 rocky|epel" >&2 + exit 1 + ;; +esac diff --git a/scripts/docker/build/rocky10/Dockerfile.ci b/scripts/docker/build/rocky10/Dockerfile.ci index 9227e8c7682..c74b59d2b65 100644 --- a/scripts/docker/build/rocky10/Dockerfile.ci +++ b/scripts/docker/build/rocky10/Dockerfile.ci @@ -13,6 +13,14 @@ FROM ${from} # (minrate) stays on so a stalled in-flight download still gets killed. # RUN printf 'retries=3\ntimeout=15\n' >> /etc/dnf/dnf.conf + +# +# Snapshot the Rocky mirror lists at build time so no later dnf +# invocation depends on the mirrors.rockylinux.org redirector (see +# the script header for the full story). +# +COPY scripts/docker/bin/dnf-mirrorlist-snapshot /usr/local/bin/dnf-mirrorlist-snapshot +RUN dnf-mirrorlist-snapshot rocky # # Refresh dnf metadata and install the build toolchain shared by the # production, crossbuild, and CI base templates. CRB is enabled because @@ -34,6 +42,11 @@ RUN dnf install -y \ RUN dnf config-manager --set-enabled crb +# +# Snapshot the EPEL mirror lists, which only exist once epel-release +# is installed (the script itself arrives via common.dnf.retries.m4). +# +RUN dnf-mirrorlist-snapshot epel # # Build libkqueue from source as proper RPMs (libkqueue, libkqueue-devel, # libkqueue-debuginfo) so downstream dnf-builddep / Requires steps find diff --git a/scripts/docker/build/rocky10/Dockerfile.crossbuild b/scripts/docker/build/rocky10/Dockerfile.crossbuild index 76b8ba03a9c..3f79aed23a3 100644 --- a/scripts/docker/build/rocky10/Dockerfile.crossbuild +++ b/scripts/docker/build/rocky10/Dockerfile.crossbuild @@ -13,6 +13,14 @@ FROM ${from} AS build # (minrate) stays on so a stalled in-flight download still gets killed. # RUN printf 'retries=3\ntimeout=15\n' >> /etc/dnf/dnf.conf + +# +# Snapshot the Rocky mirror lists at build time so no later dnf +# invocation depends on the mirrors.rockylinux.org redirector (see +# the script header for the full story). +# +COPY scripts/docker/bin/dnf-mirrorlist-snapshot /usr/local/bin/dnf-mirrorlist-snapshot +RUN dnf-mirrorlist-snapshot rocky # # Refresh dnf metadata and install the build toolchain shared by the # production, crossbuild, and CI base templates. CRB is enabled because @@ -34,6 +42,11 @@ RUN dnf install -y \ RUN dnf config-manager --set-enabled crb +# +# Snapshot the EPEL mirror lists, which only exist once epel-release +# is installed (the script itself arrives via common.dnf.retries.m4). +# +RUN dnf-mirrorlist-snapshot epel # # Build libkqueue from source as proper RPMs (libkqueue, libkqueue-devel, # libkqueue-debuginfo) so downstream dnf-builddep / Requires steps find diff --git a/scripts/docker/build/rocky10/Dockerfile.service b/scripts/docker/build/rocky10/Dockerfile.service index c0e166a2808..6f7bd65f0d0 100644 --- a/scripts/docker/build/rocky10/Dockerfile.service +++ b/scripts/docker/build/rocky10/Dockerfile.service @@ -13,6 +13,14 @@ FROM ${from} AS build # (minrate) stays on so a stalled in-flight download still gets killed. # RUN printf 'retries=3\ntimeout=15\n' >> /etc/dnf/dnf.conf + +# +# Snapshot the Rocky mirror lists at build time so no later dnf +# invocation depends on the mirrors.rockylinux.org redirector (see +# the script header for the full story). +# +COPY scripts/docker/bin/dnf-mirrorlist-snapshot /usr/local/bin/dnf-mirrorlist-snapshot +RUN dnf-mirrorlist-snapshot rocky # # Refresh dnf metadata and install the build toolchain shared by the # production, crossbuild, and CI base templates. CRB is enabled because @@ -34,6 +42,11 @@ RUN dnf install -y \ RUN dnf config-manager --set-enabled crb +# +# Snapshot the EPEL mirror lists, which only exist once epel-release +# is installed (the script itself arrives via common.dnf.retries.m4). +# +RUN dnf-mirrorlist-snapshot epel # # Build libkqueue from source as proper RPMs (libkqueue, libkqueue-devel, # libkqueue-debuginfo) so downstream dnf-builddep / Requires steps find @@ -148,6 +161,14 @@ FROM ${from} # RUN printf 'retries=3\ntimeout=15\n' >> /etc/dnf/dnf.conf +# +# Snapshot the Rocky mirror lists at build time so no later dnf +# invocation depends on the mirrors.rockylinux.org redirector (see +# the script header for the full story). +# +COPY scripts/docker/bin/dnf-mirrorlist-snapshot /usr/local/bin/dnf-mirrorlist-snapshot +RUN dnf-mirrorlist-snapshot rocky + COPY --from=build /root/rpms /tmp/ # @@ -188,6 +209,12 @@ RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-10. && dnf install -y dnf-utils \ && dnf config-manager --enable epel-testing +# +# Snapshot the EPEL mirror lists, which only exist once epel-release +# is installed (the script itself arrives via common.dnf.retries.m4). +# +RUN dnf-mirrorlist-snapshot epel + ARG radiusd_uid=95 ARG radiusd_gid=95 diff --git a/scripts/docker/build/rocky9/Dockerfile.ci b/scripts/docker/build/rocky9/Dockerfile.ci index 45dda5647ff..749a337325d 100644 --- a/scripts/docker/build/rocky9/Dockerfile.ci +++ b/scripts/docker/build/rocky9/Dockerfile.ci @@ -13,6 +13,14 @@ FROM ${from} # (minrate) stays on so a stalled in-flight download still gets killed. # RUN printf 'retries=3\ntimeout=15\n' >> /etc/dnf/dnf.conf + +# +# Snapshot the Rocky mirror lists at build time so no later dnf +# invocation depends on the mirrors.rockylinux.org redirector (see +# the script header for the full story). +# +COPY scripts/docker/bin/dnf-mirrorlist-snapshot /usr/local/bin/dnf-mirrorlist-snapshot +RUN dnf-mirrorlist-snapshot rocky # # Refresh dnf metadata and install the build toolchain shared by the # production, crossbuild, and CI base templates. CRB is enabled because @@ -34,6 +42,11 @@ RUN dnf install -y \ RUN dnf config-manager --set-enabled crb +# +# Snapshot the EPEL mirror lists, which only exist once epel-release +# is installed (the script itself arrives via common.dnf.retries.m4). +# +RUN dnf-mirrorlist-snapshot epel # # Build libkqueue from source as proper RPMs (libkqueue, libkqueue-devel, # libkqueue-debuginfo) so downstream dnf-builddep / Requires steps find diff --git a/scripts/docker/build/rocky9/Dockerfile.crossbuild b/scripts/docker/build/rocky9/Dockerfile.crossbuild index b2b6a85f6da..8c804083aaf 100644 --- a/scripts/docker/build/rocky9/Dockerfile.crossbuild +++ b/scripts/docker/build/rocky9/Dockerfile.crossbuild @@ -13,6 +13,14 @@ FROM ${from} AS build # (minrate) stays on so a stalled in-flight download still gets killed. # RUN printf 'retries=3\ntimeout=15\n' >> /etc/dnf/dnf.conf + +# +# Snapshot the Rocky mirror lists at build time so no later dnf +# invocation depends on the mirrors.rockylinux.org redirector (see +# the script header for the full story). +# +COPY scripts/docker/bin/dnf-mirrorlist-snapshot /usr/local/bin/dnf-mirrorlist-snapshot +RUN dnf-mirrorlist-snapshot rocky # # Refresh dnf metadata and install the build toolchain shared by the # production, crossbuild, and CI base templates. CRB is enabled because @@ -34,6 +42,11 @@ RUN dnf install -y \ RUN dnf config-manager --set-enabled crb +# +# Snapshot the EPEL mirror lists, which only exist once epel-release +# is installed (the script itself arrives via common.dnf.retries.m4). +# +RUN dnf-mirrorlist-snapshot epel # # Build libkqueue from source as proper RPMs (libkqueue, libkqueue-devel, # libkqueue-debuginfo) so downstream dnf-builddep / Requires steps find diff --git a/scripts/docker/build/rocky9/Dockerfile.service b/scripts/docker/build/rocky9/Dockerfile.service index 1b4117031dd..6843939c98c 100644 --- a/scripts/docker/build/rocky9/Dockerfile.service +++ b/scripts/docker/build/rocky9/Dockerfile.service @@ -13,6 +13,14 @@ FROM ${from} AS build # (minrate) stays on so a stalled in-flight download still gets killed. # RUN printf 'retries=3\ntimeout=15\n' >> /etc/dnf/dnf.conf + +# +# Snapshot the Rocky mirror lists at build time so no later dnf +# invocation depends on the mirrors.rockylinux.org redirector (see +# the script header for the full story). +# +COPY scripts/docker/bin/dnf-mirrorlist-snapshot /usr/local/bin/dnf-mirrorlist-snapshot +RUN dnf-mirrorlist-snapshot rocky # # Refresh dnf metadata and install the build toolchain shared by the # production, crossbuild, and CI base templates. CRB is enabled because @@ -34,6 +42,11 @@ RUN dnf install -y \ RUN dnf config-manager --set-enabled crb +# +# Snapshot the EPEL mirror lists, which only exist once epel-release +# is installed (the script itself arrives via common.dnf.retries.m4). +# +RUN dnf-mirrorlist-snapshot epel # # Build libkqueue from source as proper RPMs (libkqueue, libkqueue-devel, # libkqueue-debuginfo) so downstream dnf-builddep / Requires steps find @@ -148,6 +161,14 @@ FROM ${from} # RUN printf 'retries=3\ntimeout=15\n' >> /etc/dnf/dnf.conf +# +# Snapshot the Rocky mirror lists at build time so no later dnf +# invocation depends on the mirrors.rockylinux.org redirector (see +# the script header for the full story). +# +COPY scripts/docker/bin/dnf-mirrorlist-snapshot /usr/local/bin/dnf-mirrorlist-snapshot +RUN dnf-mirrorlist-snapshot rocky + COPY --from=build /root/rpms /tmp/ # @@ -188,6 +209,12 @@ RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.n && dnf install -y dnf-utils \ && dnf config-manager --enable epel-testing +# +# Snapshot the EPEL mirror lists, which only exist once epel-release +# is installed (the script itself arrives via common.dnf.retries.m4). +# +RUN dnf-mirrorlist-snapshot epel + ARG radiusd_uid=95 ARG radiusd_gid=95 diff --git a/scripts/docker/m4/common.dnf.mirrorlist.epel.m4 b/scripts/docker/m4/common.dnf.mirrorlist.epel.m4 new file mode 100644 index 00000000000..9664931987c --- /dev/null +++ b/scripts/docker/m4/common.dnf.mirrorlist.epel.m4 @@ -0,0 +1,5 @@ +# +# Snapshot the EPEL mirror lists, which only exist once epel-release +# is installed (the script itself arrives via common.dnf.retries.m4). +# +RUN dnf-mirrorlist-snapshot epel diff --git a/scripts/docker/m4/common.dnf.retries.m4 b/scripts/docker/m4/common.dnf.retries.m4 index 2933abe1745..531936defca 100644 --- a/scripts/docker/m4/common.dnf.retries.m4 +++ b/scripts/docker/m4/common.dnf.retries.m4 @@ -5,3 +5,11 @@ # (minrate) stays on so a stalled in-flight download still gets killed. # RUN printf 'retries=3\ntimeout=15\n' >> /etc/dnf/dnf.conf + +# +# Snapshot the Rocky mirror lists at build time so no later dnf +# invocation depends on the mirrors.rockylinux.org redirector (see +# the script header for the full story). +# +COPY scripts/docker/bin/dnf-mirrorlist-snapshot /usr/local/bin/dnf-mirrorlist-snapshot +RUN dnf-mirrorlist-snapshot rocky diff --git a/scripts/docker/m4/common.rpm.toolchain.m4 b/scripts/docker/m4/common.rpm.toolchain.m4 index 9aad0539455..8fb2bee8589 100644 --- a/scripts/docker/m4/common.rpm.toolchain.m4 +++ b/scripts/docker/m4/common.rpm.toolchain.m4 @@ -19,4 +19,7 @@ RUN dnf install -y \ RUN dnf config-manager --set-enabled crb +dnl EPEL mirror lists can only be snapshotted after the epel-release +dnl install above puts the repo files in place. +include(`common.dnf.mirrorlist.epel.m4')dnl include(`common.rpm.libkqueue.m4')dnl diff --git a/scripts/docker/m4/service.rpm.m4 b/scripts/docker/m4/service.rpm.m4 index bd4f862b37b..b47b1b0b428 100644 --- a/scripts/docker/m4/service.rpm.m4 +++ b/scripts/docker/m4/service.rpm.m4 @@ -87,6 +87,10 @@ RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-OS_ && dnf install -y dnf-utils \ && dnf config-manager --enable epel-testing +dnl EPEL mirror lists can only be snapshotted after the epel-release +dnl install above puts the repo files in place. +include(`common.dnf.mirrorlist.epel.m4')dnl + ARG radiusd_uid=95 ARG radiusd_gid=95