scripts/docker: snapshot dnf mirrorlists at image build time

The mirrors.rockylinux.org / mirrors.fedoraproject.org redirectors
occasionally answer with an empty mirror set, which dnf reports as "No
URLs in mirrorlist" and treats as fatal rather than retryable, so each
mirror list is now fetched once at image build time into
/etc/dnf/mirrorlists/<repo> and the enabled repos point at the
snapshot with mirrorlist=file:// (explicitly supported by librepo,
mirrors tried in file order). The origin CDN leads each snapshot so
the fetched mirrors are failover only, minor-versioned paths are
rewritten to the major-version symlink form so a snapshot survives
point releases, and the unused metalink-only epel-cisco-openh264 repo
is disabled.

diff --git a/scripts/docker/bin/dnf-mirrorlist-snapshot b/scripts/docker/bin/dnf-mirrorlist-snapshot
new file mode 100755 (executable)
index 0000000..6cd0712
--- /dev/null
+++ b/scripts/docker/bin/dnf-mirrorlist-snapshot
@@ -0,0 +1,99 @@
+#!/bin/sh
+#
+#  Snapshot dnf mirror lists at image build time.
+#
+#  The mirrors.rockylinux.org / mirrors.fedoraproject.org redirectors
+#  occasionally answer with an empty mirror set, which dnf reports as
+#  "No URLs in mirrorlist" and treats as fatal rather than retryable.
+#  Fetching the mirror list once here, and pointing each enabled repo
+#  at the local snapshot with mirrorlist=file://, takes the redirectors
+#  out of the loop for every later dnf invocation.
+#
+#  Each snapshot leads with the origin CDN, so the fetched mirrors only
+#  see traffic when the origin fails. Minor-versioned path segments in
+#  fetched entries (.../10.2/...) are rewritten to the major-version
+#  symlink form so a snapshot survives point releases, which delete old
+#  minor trees from the mirrors. If the redirector fetch fails or
+#  returns nothing, the snapshot degrades to the origin entry alone,
+#  equivalent to pinning the origin as a single baseurl.
+#
+#  librepo handles file:// mirrorlists explicitly (lr_is_local_path,
+#  lr_prepend_url_protocol) and tries the mirrors in file order.
+#
+#  Usage: dnf-mirrorlist-snapshot rocky|epel
+#
+#    rocky - snapshot the repos the builds enable (baseos, appstream,
+#            crb, extras); disabled repos keep their remote mirrorlist
+#            lines and are never fetched.
+#    epel  - snapshot epel and epel-testing; run after epel-release is
+#            installed. Also disables the EL9-only epel-cisco-openh264
+#            repo, which is metalink-only and unused by our builds.
+
+set -e
+
+arch="$(uname -m)"
+relver="$(. /etc/os-release; echo "${VERSION_ID%%.*}")"
+
+list_dir=/etc/dnf/mirrorlists
+repo_dir=/etc/yum.repos.d
+
+#
+#  snapshot <list-name> <section> <line-key> <mirrorlist-url> <origin-url> <repo-file>...
+#
+#  Writes <list-dir>/<list-name> (origin first, fetched mirrors after,
+#  origin host de-duplicated), then rewrites the <line-key>= line of
+#  the [<section>] block in the given repo files to point at the
+#  snapshot.
+#
+snapshot() {
+       name="$1"; sect="$2"; key="$3"; mlurl="$4"; origin="$5"; shift 5
+
+       list="${list_dir}/${name}"
+       origin_host="$(printf '%s' "${origin}" | cut -d/ -f3)"
+
+       {
+               echo "${origin}"
+               curl -sf --retry 3 --retry-delay 5 "${mlurl}" \
+                   | grep '^http' \
+                   | sed -e "s|/${relver}\.[0-9]*/|/${relver}/|" \
+                         -e "\\|//${origin_host}/|d" \
+                   || true
+       } > "${list}"
+
+       sed -i "/^\[${sect}\]/,/^\[/ s|^${key}=.*|mirrorlist=file://${list}|" "$@"
+}
+
+mkdir -p "${list_dir}"
+
+case "$1" in
+rocky)
+       for pair in baseos:BaseOS appstream:AppStream crb:CRB extras:extras; do
+               sect="${pair%%:*}"; repo="${pair##*:}"
+               snapshot "rocky-${sect}" "${sect}" mirrorlist \
+                   "https://mirrors.rockylinux.org/mirrorlist?arch=${arch}&repo=${repo}-${relver}" \
+                   "http://dl.rockylinux.org/pub/rocky/${relver}/${repo}/${arch}/os/" \
+                   "${repo_dir}"/rocky*.repo
+       done
+       ;;
+
+epel)
+       snapshot epel epel metalink \
+           "https://mirrors.fedoraproject.org/mirrorlist?repo=epel-${relver}&arch=${arch}" \
+           "https://dl.fedoraproject.org/pub/epel/${relver}/Everything/${arch}/" \
+           "${repo_dir}/epel.repo"
+
+       snapshot epel-testing epel-testing metalink \
+           "https://mirrors.fedoraproject.org/mirrorlist?repo=epel-testing-${relver}&arch=${arch}" \
+           "https://dl.fedoraproject.org/pub/epel/testing/${relver}/Everything/${arch}/" \
+           "${repo_dir}/epel-testing.repo"
+
+       if [ -e "${repo_dir}/epel-cisco-openh264.repo" ]; then
+               sed -i 's|^enabled=1|enabled=0|' "${repo_dir}/epel-cisco-openh264.repo"
+       fi
+       ;;
+
+*)
+       echo "Usage: $0 rocky|epel" >&2
+       exit 1
+       ;;
+esac

diff --git a/scripts/docker/build/rocky10/Dockerfile.ci b/scripts/docker/build/rocky10/Dockerfile.ci
index 9227e8c7682736fa550b327095b40b0f1de35345..c74b59d2b65a387a1d0f81da019d6a5b870afabf 100644 (file)
--- a/scripts/docker/build/rocky10/Dockerfile.ci
+++ b/scripts/docker/build/rocky10/Dockerfile.ci
@@ -13,6 +13,14 @@ FROM ${from}
 #  (minrate) stays on so a stalled in-flight download still gets killed.
 #
 RUN printf 'retries=3\ntimeout=15\n' >> /etc/dnf/dnf.conf
+
+#
+#  Snapshot the Rocky mirror lists at build time so no later dnf
+#  invocation depends on the mirrors.rockylinux.org redirector (see
+#  the script header for the full story).
+#
+COPY scripts/docker/bin/dnf-mirrorlist-snapshot /usr/local/bin/dnf-mirrorlist-snapshot
+RUN dnf-mirrorlist-snapshot rocky
 #
 #  Refresh dnf metadata and install the build toolchain shared by the
 #  production, crossbuild, and CI base templates. CRB is enabled because
@@ -34,6 +42,11 @@ RUN dnf install -y \
 
 RUN dnf config-manager --set-enabled crb
 
+#
+#  Snapshot the EPEL mirror lists, which only exist once epel-release
+#  is installed (the script itself arrives via common.dnf.retries.m4).
+#
+RUN dnf-mirrorlist-snapshot epel
 #
 #  Build libkqueue from source as proper RPMs (libkqueue, libkqueue-devel,
 #  libkqueue-debuginfo) so downstream dnf-builddep / Requires steps find

diff --git a/scripts/docker/build/rocky10/Dockerfile.crossbuild b/scripts/docker/build/rocky10/Dockerfile.crossbuild
index 76b8ba03a9c0734aff46b51008478a2246ef22a8..3f79aed23a36cb0b2b8ea766d3e5bd4279d7be61 100644 (file)
--- a/scripts/docker/build/rocky10/Dockerfile.crossbuild
+++ b/scripts/docker/build/rocky10/Dockerfile.crossbuild
@@ -13,6 +13,14 @@ FROM ${from} AS build
 #  (minrate) stays on so a stalled in-flight download still gets killed.
 #
 RUN printf 'retries=3\ntimeout=15\n' >> /etc/dnf/dnf.conf
+
+#
+#  Snapshot the Rocky mirror lists at build time so no later dnf
+#  invocation depends on the mirrors.rockylinux.org redirector (see
+#  the script header for the full story).
+#
+COPY scripts/docker/bin/dnf-mirrorlist-snapshot /usr/local/bin/dnf-mirrorlist-snapshot
+RUN dnf-mirrorlist-snapshot rocky
 #
 #  Refresh dnf metadata and install the build toolchain shared by the
 #  production, crossbuild, and CI base templates. CRB is enabled because
@@ -34,6 +42,11 @@ RUN dnf install -y \
 
 RUN dnf config-manager --set-enabled crb
 
+#
+#  Snapshot the EPEL mirror lists, which only exist once epel-release
+#  is installed (the script itself arrives via common.dnf.retries.m4).
+#
+RUN dnf-mirrorlist-snapshot epel
 #
 #  Build libkqueue from source as proper RPMs (libkqueue, libkqueue-devel,
 #  libkqueue-debuginfo) so downstream dnf-builddep / Requires steps find

diff --git a/scripts/docker/build/rocky10/Dockerfile.service b/scripts/docker/build/rocky10/Dockerfile.service
index c0e166a2808b6eca85dcc3c90af5d474e999ec44..6f7bd65f0d0919a0a95a7ba8d63e3dfd680d475e 100644 (file)
--- a/scripts/docker/build/rocky10/Dockerfile.service
+++ b/scripts/docker/build/rocky10/Dockerfile.service
@@ -13,6 +13,14 @@ FROM ${from} AS build
 #  (minrate) stays on so a stalled in-flight download still gets killed.
 #
 RUN printf 'retries=3\ntimeout=15\n' >> /etc/dnf/dnf.conf
+
+#
+#  Snapshot the Rocky mirror lists at build time so no later dnf
+#  invocation depends on the mirrors.rockylinux.org redirector (see
+#  the script header for the full story).
+#
+COPY scripts/docker/bin/dnf-mirrorlist-snapshot /usr/local/bin/dnf-mirrorlist-snapshot
+RUN dnf-mirrorlist-snapshot rocky
 #
 #  Refresh dnf metadata and install the build toolchain shared by the
 #  production, crossbuild, and CI base templates. CRB is enabled because
@@ -34,6 +42,11 @@ RUN dnf install -y \
 
 RUN dnf config-manager --set-enabled crb
 
+#
+#  Snapshot the EPEL mirror lists, which only exist once epel-release
+#  is installed (the script itself arrives via common.dnf.retries.m4).
+#
+RUN dnf-mirrorlist-snapshot epel
 #
 #  Build libkqueue from source as proper RPMs (libkqueue, libkqueue-devel,
 #  libkqueue-debuginfo) so downstream dnf-builddep / Requires steps find
@@ -148,6 +161,14 @@ FROM ${from}
 #
 RUN printf 'retries=3\ntimeout=15\n' >> /etc/dnf/dnf.conf
 
+#
+#  Snapshot the Rocky mirror lists at build time so no later dnf
+#  invocation depends on the mirrors.rockylinux.org redirector (see
+#  the script header for the full story).
+#
+COPY scripts/docker/bin/dnf-mirrorlist-snapshot /usr/local/bin/dnf-mirrorlist-snapshot
+RUN dnf-mirrorlist-snapshot rocky
+
 COPY --from=build /root/rpms /tmp/
 
 #
@@ -188,6 +209,12 @@ RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-10.
     && dnf install -y dnf-utils \
     && dnf config-manager --enable epel-testing
 
+#
+#  Snapshot the EPEL mirror lists, which only exist once epel-release
+#  is installed (the script itself arrives via common.dnf.retries.m4).
+#
+RUN dnf-mirrorlist-snapshot epel
+
 ARG radiusd_uid=95
 ARG radiusd_gid=95
 

diff --git a/scripts/docker/build/rocky9/Dockerfile.ci b/scripts/docker/build/rocky9/Dockerfile.ci
index 45dda5647ff0ec5a1384510c4cde92065b819fef..749a337325d044280b396576dd25266a9ed432e6 100644 (file)
--- a/scripts/docker/build/rocky9/Dockerfile.ci
+++ b/scripts/docker/build/rocky9/Dockerfile.ci
@@ -13,6 +13,14 @@ FROM ${from}
 #  (minrate) stays on so a stalled in-flight download still gets killed.
 #
 RUN printf 'retries=3\ntimeout=15\n' >> /etc/dnf/dnf.conf
+
+#
+#  Snapshot the Rocky mirror lists at build time so no later dnf
+#  invocation depends on the mirrors.rockylinux.org redirector (see
+#  the script header for the full story).
+#
+COPY scripts/docker/bin/dnf-mirrorlist-snapshot /usr/local/bin/dnf-mirrorlist-snapshot
+RUN dnf-mirrorlist-snapshot rocky
 #
 #  Refresh dnf metadata and install the build toolchain shared by the
 #  production, crossbuild, and CI base templates. CRB is enabled because
@@ -34,6 +42,11 @@ RUN dnf install -y \
 
 RUN dnf config-manager --set-enabled crb
 
+#
+#  Snapshot the EPEL mirror lists, which only exist once epel-release
+#  is installed (the script itself arrives via common.dnf.retries.m4).
+#
+RUN dnf-mirrorlist-snapshot epel
 #
 #  Build libkqueue from source as proper RPMs (libkqueue, libkqueue-devel,
 #  libkqueue-debuginfo) so downstream dnf-builddep / Requires steps find

diff --git a/scripts/docker/build/rocky9/Dockerfile.crossbuild b/scripts/docker/build/rocky9/Dockerfile.crossbuild
index b2b6a85f6dadbc66d0100e2d5b59cb707a066f3d..8c804083aaf252a7cde9bcc6be14bb31b7f89fca 100644 (file)
--- a/scripts/docker/build/rocky9/Dockerfile.crossbuild
+++ b/scripts/docker/build/rocky9/Dockerfile.crossbuild
@@ -13,6 +13,14 @@ FROM ${from} AS build
 #  (minrate) stays on so a stalled in-flight download still gets killed.
 #
 RUN printf 'retries=3\ntimeout=15\n' >> /etc/dnf/dnf.conf
+
+#
+#  Snapshot the Rocky mirror lists at build time so no later dnf
+#  invocation depends on the mirrors.rockylinux.org redirector (see
+#  the script header for the full story).
+#
+COPY scripts/docker/bin/dnf-mirrorlist-snapshot /usr/local/bin/dnf-mirrorlist-snapshot
+RUN dnf-mirrorlist-snapshot rocky
 #
 #  Refresh dnf metadata and install the build toolchain shared by the
 #  production, crossbuild, and CI base templates. CRB is enabled because
@@ -34,6 +42,11 @@ RUN dnf install -y \
 
 RUN dnf config-manager --set-enabled crb
 
+#
+#  Snapshot the EPEL mirror lists, which only exist once epel-release
+#  is installed (the script itself arrives via common.dnf.retries.m4).
+#
+RUN dnf-mirrorlist-snapshot epel
 #
 #  Build libkqueue from source as proper RPMs (libkqueue, libkqueue-devel,
 #  libkqueue-debuginfo) so downstream dnf-builddep / Requires steps find

diff --git a/scripts/docker/build/rocky9/Dockerfile.service b/scripts/docker/build/rocky9/Dockerfile.service
index 1b4117031ddfa508aa5e0f774b3277493dca6006..6843939c98c674d265824668f7f006bb0f86571e 100644 (file)
--- a/scripts/docker/build/rocky9/Dockerfile.service
+++ b/scripts/docker/build/rocky9/Dockerfile.service
@@ -13,6 +13,14 @@ FROM ${from} AS build
 #  (minrate) stays on so a stalled in-flight download still gets killed.
 #
 RUN printf 'retries=3\ntimeout=15\n' >> /etc/dnf/dnf.conf
+
+#
+#  Snapshot the Rocky mirror lists at build time so no later dnf
+#  invocation depends on the mirrors.rockylinux.org redirector (see
+#  the script header for the full story).
+#
+COPY scripts/docker/bin/dnf-mirrorlist-snapshot /usr/local/bin/dnf-mirrorlist-snapshot
+RUN dnf-mirrorlist-snapshot rocky
 #
 #  Refresh dnf metadata and install the build toolchain shared by the
 #  production, crossbuild, and CI base templates. CRB is enabled because
@@ -34,6 +42,11 @@ RUN dnf install -y \
 
 RUN dnf config-manager --set-enabled crb
 
+#
+#  Snapshot the EPEL mirror lists, which only exist once epel-release
+#  is installed (the script itself arrives via common.dnf.retries.m4).
+#
+RUN dnf-mirrorlist-snapshot epel
 #
 #  Build libkqueue from source as proper RPMs (libkqueue, libkqueue-devel,
 #  libkqueue-debuginfo) so downstream dnf-builddep / Requires steps find
@@ -148,6 +161,14 @@ FROM ${from}
 #
 RUN printf 'retries=3\ntimeout=15\n' >> /etc/dnf/dnf.conf
 
+#
+#  Snapshot the Rocky mirror lists at build time so no later dnf
+#  invocation depends on the mirrors.rockylinux.org redirector (see
+#  the script header for the full story).
+#
+COPY scripts/docker/bin/dnf-mirrorlist-snapshot /usr/local/bin/dnf-mirrorlist-snapshot
+RUN dnf-mirrorlist-snapshot rocky
+
 COPY --from=build /root/rpms /tmp/
 
 #
@@ -188,6 +209,12 @@ RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.n
     && dnf install -y dnf-utils \
     && dnf config-manager --enable epel-testing
 
+#
+#  Snapshot the EPEL mirror lists, which only exist once epel-release
+#  is installed (the script itself arrives via common.dnf.retries.m4).
+#
+RUN dnf-mirrorlist-snapshot epel
+
 ARG radiusd_uid=95
 ARG radiusd_gid=95
 

diff --git a/scripts/docker/m4/common.dnf.mirrorlist.epel.m4 b/scripts/docker/m4/common.dnf.mirrorlist.epel.m4
new file mode 100644 (file)
index 0000000..9664931
--- /dev/null
+++ b/scripts/docker/m4/common.dnf.mirrorlist.epel.m4
@@ -0,0 +1,5 @@
+#
+#  Snapshot the EPEL mirror lists, which only exist once epel-release
+#  is installed (the script itself arrives via common.dnf.retries.m4).
+#
+RUN dnf-mirrorlist-snapshot epel

diff --git a/scripts/docker/m4/common.dnf.retries.m4 b/scripts/docker/m4/common.dnf.retries.m4
index 2933abe1745eeed0ba128b077daa580dc633d4d0..531936defca5ca7acfc54fa2662fae5033f83ecb 100644 (file)
--- a/scripts/docker/m4/common.dnf.retries.m4
+++ b/scripts/docker/m4/common.dnf.retries.m4
@@ -5,3 +5,11 @@
 #  (minrate) stays on so a stalled in-flight download still gets killed.
 #
 RUN printf 'retries=3\ntimeout=15\n' >> /etc/dnf/dnf.conf
+
+#
+#  Snapshot the Rocky mirror lists at build time so no later dnf
+#  invocation depends on the mirrors.rockylinux.org redirector (see
+#  the script header for the full story).
+#
+COPY scripts/docker/bin/dnf-mirrorlist-snapshot /usr/local/bin/dnf-mirrorlist-snapshot
+RUN dnf-mirrorlist-snapshot rocky

diff --git a/scripts/docker/m4/common.rpm.toolchain.m4 b/scripts/docker/m4/common.rpm.toolchain.m4
index 9aad05394550dfbbc443aff3d51040f8c67ca1b0..8fb2bee858905314621fc91a10af70d72c185ee8 100644 (file)
--- a/scripts/docker/m4/common.rpm.toolchain.m4
+++ b/scripts/docker/m4/common.rpm.toolchain.m4
@@ -19,4 +19,7 @@ RUN dnf install -y \
 
 RUN dnf config-manager --set-enabled crb
 
+dnl  EPEL mirror lists can only be snapshotted after the epel-release
+dnl  install above puts the repo files in place.
+include(`common.dnf.mirrorlist.epel.m4')dnl
 include(`common.rpm.libkqueue.m4')dnl

diff --git a/scripts/docker/m4/service.rpm.m4 b/scripts/docker/m4/service.rpm.m4
index bd4f862b37bdfca2a68df6f42b8090a7821831f8..b47b1b0b428194f78c4c80f173c0abaea0819631 100644 (file)
--- a/scripts/docker/m4/service.rpm.m4
+++ b/scripts/docker/m4/service.rpm.m4
@@ -87,6 +87,10 @@ RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-OS_
     && dnf install -y dnf-utils \
     && dnf config-manager --enable epel-testing
 
+dnl  EPEL mirror lists can only be snapshotted after the epel-release
+dnl  install above puts the repo files in place.
+include(`common.dnf.mirrorlist.epel.m4')dnl
+
 ARG radiusd_uid=95
 ARG radiusd_gid=95