From: Daan De Meyer <daan.j.demeyer@gmail.com>
Date: Thu, 19 Oct 2023 18:06:06 +0000 (+0200)
Subject: Make join_mirror() more robust and use it everywhere
X-Git-Tag: v19~60^2
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=refs%2Fpull%2F1998%2Fhead;p=thirdparty%2Fmkosi.git

Make join_mirror() more robust and use it everywhere
---

diff --git a/mkosi/distributions/__init__.py b/mkosi/distributions/__init__.py
index 858e093b7..3141ed445 100644
--- a/mkosi/distributions/__init__.py
+++ b/mkosi/distributions/__init__.py
@@ -3,6 +3,7 @@
 import enum
 import importlib
 import re
+import urllib.parse
 from collections.abc import Sequence
 from typing import TYPE_CHECKING, Optional, cast
 
@@ -177,3 +178,12 @@ def detect_distribution() -> tuple[Optional[Distribution], Optional[str]]:
         version_id = version_codename or extracted_codename
 
     return d, version_id
+
+
+def join_mirror(mirror: str, link: str) -> str:
+    # urljoin() behaves weirdly if the base does not end with a / or the path starts with a / so fix them up as needed.
+    if not mirror.endswith("/"):
+        mirror = f"{mirror}/"
+    link = link.removeprefix("/")
+
+    return urllib.parse.urljoin(mirror, link)
diff --git a/mkosi/distributions/alma.py b/mkosi/distributions/alma.py
index ff90c23bf..8d67da6f1 100644
--- a/mkosi/distributions/alma.py
+++ b/mkosi/distributions/alma.py
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: LGPL-2.1+
 
-from mkosi.distributions import centos
+from mkosi.distributions import centos, join_mirror
 from mkosi.installer.dnf import Repo, find_rpm_gpgkey
 from mkosi.state import MkosiState
 
@@ -23,7 +23,7 @@ class Installer(centos.Installer):
     @classmethod
     def repository_variants(cls, state: MkosiState, repo: str) -> list[Repo]:
         if state.config.mirror:
-            url = f"baseurl={state.config.mirror}/almalinux/$releasever/{repo}/$basearch/os"
+            url = f"baseurl={join_mirror(state.config.mirror, f'almalinux/$releasever/{repo}/$basearch/os')}"
         else:
             url = f"mirrorlist=https://mirrors.almalinux.org/mirrorlist/$releasever/{repo.lower()}"
 
diff --git a/mkosi/distributions/centos.py b/mkosi/distributions/centos.py
index 320d4c047..a04ebcdc9 100644
--- a/mkosi/distributions/centos.py
+++ b/mkosi/distributions/centos.py
@@ -2,12 +2,16 @@
 
 import os
 import shutil
-import urllib.parse
 from collections.abc import Iterable, Sequence
 from pathlib import Path
 
 from mkosi.architecture import Architecture
-from mkosi.distributions import Distribution, DistributionInstaller, PackageType
+from mkosi.distributions import (
+    Distribution,
+    DistributionInstaller,
+    PackageType,
+    join_mirror,
+)
 from mkosi.installer.dnf import Repo, find_rpm_gpgkey, invoke_dnf, setup_dnf
 from mkosi.log import complete_step, die
 from mkosi.state import MkosiState
@@ -28,9 +32,6 @@ def move_rpm_db(root: Path) -> None:
             newdb.symlink_to(os.path.relpath(olddb, start=newdb.parent))
 
 
-def join_mirror(mirror: str, link: str) -> str:
-    return urllib.parse.urljoin(mirror, link)
-
 
 class Installer(DistributionInstaller):
     @classmethod
diff --git a/mkosi/distributions/fedora.py b/mkosi/distributions/fedora.py
index 04198e48b..3837c6e8f 100644
--- a/mkosi/distributions/fedora.py
+++ b/mkosi/distributions/fedora.py
@@ -1,10 +1,14 @@
 # SPDX-License-Identifier: LGPL-2.1+
 
-import urllib.parse
 from collections.abc import Sequence
 
 from mkosi.architecture import Architecture
-from mkosi.distributions import Distribution, DistributionInstaller, PackageType
+from mkosi.distributions import (
+    Distribution,
+    DistributionInstaller,
+    PackageType,
+    join_mirror,
+)
 from mkosi.installer.dnf import Repo, find_rpm_gpgkey, invoke_dnf, setup_dnf
 from mkosi.log import die
 from mkosi.state import MkosiState
@@ -93,7 +97,7 @@ class Installer(DistributionInstaller):
         elif state.config.release == "eln":
             mirror = state.config.mirror or "https://odcs.fedoraproject.org/composes/production/latest-Fedora-ELN/compose"
             for repo in ("Appstream", "BaseOS", "Extras", "CRB"):
-                url = f"baseurl={urllib.parse.urljoin(mirror, repo)}"
+                url = f"baseurl={join_mirror(mirror, repo)}"
                 repos += [
                     Repo(repo.lower(), f"{url}/$basearch/os", gpgurls),
                     Repo(repo.lower(), f"{url}/$basearch/debug/tree", gpgurls, enabled=False),
@@ -101,7 +105,7 @@ class Installer(DistributionInstaller):
                 ]
         elif state.config.mirror:
             directory = "development" if state.config.release == "rawhide" else "releases"
-            url = f"baseurl={urllib.parse.urljoin(state.config.mirror, f'{directory}/$releasever/Everything')}"
+            url = f"baseurl={join_mirror(state.config.mirror, f'{directory}/$releasever/Everything')}"
             repos += [
                 Repo("fedora", f"{url}/$basearch/os", gpgurls),
                 Repo("fedora-debuginfo", f"{url}/$basearch/debug/tree", gpgurls, enabled=False),
@@ -109,14 +113,14 @@ class Installer(DistributionInstaller):
             ]
 
             if state.config.release != "rawhide":
-                url = f"baseurl={urllib.parse.urljoin(state.config.mirror, 'updates/$releasever/Everything')}"
+                url = f"baseurl={join_mirror(state.config.mirror, 'updates/$releasever/Everything')}"
                 repos += [
                     Repo("updates", f"{url}/$basearch", gpgurls),
                     Repo("updates-debuginfo", f"{url}/$basearch/debug", gpgurls, enabled=False),
                     Repo("updates-source", f"{url}/SRPMS", gpgurls, enabled=False),
                 ]
 
-                url = f"baseurl={urllib.parse.urljoin(state.config.mirror, 'updates/testing/$releasever/Everything')}"
+                url = f"baseurl={join_mirror(state.config.mirror, 'updates/testing/$releasever/Everything')}"
                 repos += [
                     Repo("updates-testing", f"{url}/$basearch", gpgurls, enabled=False),
                     Repo("updates-testing-debuginfo", f"{url}/$basearch/debug", gpgurls, enabled=False),
diff --git a/mkosi/distributions/gentoo.py b/mkosi/distributions/gentoo.py
index 78b0ca4f2..5ed0fe1de 100644
--- a/mkosi/distributions/gentoo.py
+++ b/mkosi/distributions/gentoo.py
@@ -9,7 +9,12 @@ from pathlib import Path
 
 from mkosi.architecture import Architecture
 from mkosi.archive import extract_tar
-from mkosi.distributions import Distribution, DistributionInstaller, PackageType
+from mkosi.distributions import (
+    Distribution,
+    DistributionInstaller,
+    PackageType,
+    join_mirror,
+)
 from mkosi.log import ARG_DEBUG, complete_step, die
 from mkosi.run import apivfs_cmd, bwrap, chroot_cmd, run
 from mkosi.state import MkosiState
@@ -88,7 +93,7 @@ class Installer(DistributionInstaller):
 
         mirror = state.config.mirror or "https://distfiles.gentoo.org"
         # http://distfiles.gentoo.org/releases/amd64/autobuilds/latest-stage3.txt
-        stage3tsf_path_url = urllib.parse.urljoin(
+        stage3tsf_path_url = join_mirror(
             mirror.partition(" ")[0],
             f"releases/{arch}/autobuilds/latest-stage3.txt",
         )
@@ -104,7 +109,7 @@ class Installer(DistributionInstaller):
             else:
                 die("profile names changed upstream?")
 
-        stage3_url = urllib.parse.urljoin(mirror, f"releases/{arch}/autobuilds/{stage3_latest}")
+        stage3_url = join_mirror(mirror, f"releases/{arch}/autobuilds/{stage3_latest}")
         stage3_tar = state.cache_dir / "stage3.tar"
         stage3 = state.cache_dir / "stage3"
 
diff --git a/mkosi/distributions/mageia.py b/mkosi/distributions/mageia.py
index 4f4efdcd7..3bcfa24c6 100644
--- a/mkosi/distributions/mageia.py
+++ b/mkosi/distributions/mageia.py
@@ -1,11 +1,15 @@
 # SPDX-License-Identifier: LGPL-2.1+
 
 import shutil
-import urllib.parse
 from collections.abc import Sequence
 
 from mkosi.architecture import Architecture
-from mkosi.distributions import Distribution, DistributionInstaller, PackageType
+from mkosi.distributions import (
+    Distribution,
+    DistributionInstaller,
+    PackageType,
+    join_mirror,
+)
 from mkosi.installer.dnf import Repo, find_rpm_gpgkey, invoke_dnf, setup_dnf
 from mkosi.log import die
 from mkosi.state import MkosiState
@@ -47,7 +51,7 @@ class Installer(DistributionInstaller):
         if state.config.local_mirror:
             repos += [Repo("core-release", f"baseurl={state.config.local_mirror}", gpgurls)]
         elif state.config.mirror:
-            url = f"baseurl={urllib.parse.urljoin(state.config.mirror, 'distrib/$releasever/$basearch/media/core/')}"
+            url = f"baseurl={join_mirror(state.config.mirror, 'distrib/$releasever/$basearch/media/core/')}"
             repos += [
                 Repo("core-release", f"{url}/release", gpgurls),
                 Repo("core-updates", f"{url}/updates/", gpgurls)
diff --git a/mkosi/distributions/openmandriva.py b/mkosi/distributions/openmandriva.py
index 88b243ff3..6c6fee120 100644
--- a/mkosi/distributions/openmandriva.py
+++ b/mkosi/distributions/openmandriva.py
@@ -1,11 +1,15 @@
 # SPDX-License-Identifier: LGPL-2.1+
 
 import shutil
-import urllib.parse
 from collections.abc import Sequence
 
 from mkosi.architecture import Architecture
-from mkosi.distributions import Distribution, DistributionInstaller, PackageType
+from mkosi.distributions import (
+    Distribution,
+    DistributionInstaller,
+    PackageType,
+    join_mirror,
+)
 from mkosi.installer.dnf import Repo, find_rpm_gpgkey, invoke_dnf, setup_dnf
 from mkosi.log import die
 from mkosi.state import MkosiState
@@ -49,7 +53,7 @@ class Installer(DistributionInstaller):
         if state.config.local_mirror:
             repos += [Repo("main-release", f"baseurl={state.config.local_mirror}", gpgurls)]
         else:
-            url = f"baseurl={urllib.parse.urljoin(mirror, '$releasever/repository/$basearch/main')}"
+            url = f"baseurl={join_mirror(mirror, '$releasever/repository/$basearch/main')}"
             repos += [
                 Repo("main-release", f"{url}/release", gpgurls),
                 Repo("main-updates", f"{url}/updates", gpgurls),
diff --git a/mkosi/distributions/rhel.py b/mkosi/distributions/rhel.py
index 36665048f..5557c08d7 100644
--- a/mkosi/distributions/rhel.py
+++ b/mkosi/distributions/rhel.py
@@ -4,7 +4,7 @@ from collections.abc import Iterable
 from pathlib import Path
 from typing import Any, Optional
 
-from mkosi.distributions import centos
+from mkosi.distributions import centos, join_mirror
 from mkosi.installer.dnf import Repo, find_rpm_gpgkey
 from mkosi.log import die
 from mkosi.state import MkosiState
@@ -90,19 +90,19 @@ class Installer(centos.Installer):
             major = int(float(v))
             yield Repo(
                 f"rhel-{v}-{repo}-rpms",
-                f"baseurl={centos.join_mirror(mirror, f'rhel{major}/{v}/$basearch/{repo}/os')}",
+                f"baseurl={join_mirror(mirror, f'rhel{major}/{v}/$basearch/{repo}/os')}",
                 enabled=True,
                 **common,
             )
             yield Repo(
                 f"rhel-{v}-{repo}-debug-rpms",
-                f"baseurl={centos.join_mirror(mirror, f'rhel{major}/{v}/$basearch/{repo}/debug')}",
+                f"baseurl={join_mirror(mirror, f'rhel{major}/{v}/$basearch/{repo}/debug')}",
                 enabled=False,
                 **common,
             )
             yield Repo(
                 f"rhel-{v}-{repo}-source",
-                f"baseurl={centos.join_mirror(mirror, f'rhel{major}/{v}/$basearch/{repo}/source')}",
+                f"baseurl={join_mirror(mirror, f'rhel{major}/{v}/$basearch/{repo}/source')}",
                 enabled=False,
                 **common,
             )
diff --git a/mkosi/distributions/rhel_ubi.py b/mkosi/distributions/rhel_ubi.py
index bd2218ff9..d24776e8c 100644
--- a/mkosi/distributions/rhel_ubi.py
+++ b/mkosi/distributions/rhel_ubi.py
@@ -2,7 +2,7 @@
 
 from collections.abc import Iterable
 
-from mkosi.distributions import centos
+from mkosi.distributions import centos, join_mirror
 from mkosi.installer.dnf import Repo, find_rpm_gpgkey
 from mkosi.state import MkosiState
 
@@ -34,18 +34,18 @@ class Installer(centos.Installer):
             v = state.config.release
             yield Repo(
                 f"ubi-{v}-{repo}-rpms",
-                f"baseurl={centos.join_mirror(mirror, f'ubi{v}/{v}/$basearch/{repo}/os')}",
+                f"baseurl={join_mirror(mirror, f'ubi{v}/{v}/$basearch/{repo}/os')}",
                 cls.gpgurls(state),
             )
             yield Repo(
                 f"ubi-{v}-{repo}-debug-rpms",
-                f"baseurl={centos.join_mirror(mirror, f'ubi{v}/{v}/$basearch/{repo}/debug')}",
+                f"baseurl={join_mirror(mirror, f'ubi{v}/{v}/$basearch/{repo}/debug')}",
                 cls.gpgurls(state),
                 enabled=False,
             )
             yield Repo(
                 f"ubi-{v}-{repo}-source",
-                f"baseurl={centos.join_mirror(mirror, f'ubi{v}/{v}/$basearch/{repo}/source')}",
+                f"baseurl={join_mirror(mirror, f'ubi{v}/{v}/$basearch/{repo}/source')}",
                 cls.gpgurls(state),
                 enabled=False,
             )
diff --git a/mkosi/distributions/rocky.py b/mkosi/distributions/rocky.py
index e09bf03df..98c7c7b7c 100644
--- a/mkosi/distributions/rocky.py
+++ b/mkosi/distributions/rocky.py
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: LGPL-2.1+
 
-from mkosi.distributions import centos
+from mkosi.distributions import centos, join_mirror
 from mkosi.installer.dnf import Repo, find_rpm_gpgkey
 from mkosi.state import MkosiState
 
@@ -23,7 +23,7 @@ class Installer(centos.Installer):
     @classmethod
     def repository_variants(cls, state: MkosiState, repo: str) -> list[Repo]:
         if state.config.mirror:
-            url = f"baseurl={state.config.mirror}/rocky/$releasever/{repo}/$basearch/os"
+            url = f"baseurl={join_mirror(state.config.mirror, f'rocky/$releasever/{repo}/$basearch/os')}"
         else:
             url = f"mirrorlist=https://mirrors.rockylinux.org/mirrorlist?arch=$basearch&repo={repo}-$releasever"