From: Remi Gacogne <remi.gacogne@powerdns.com>
Date: Fri, 22 Mar 2024 14:38:08 +0000 (+0100)
Subject: Move the version (and hashes) of external dependencies to JSON files
X-Git-Tag: rec-5.1.0-alpha1~86^2~1
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=4ccb0f786bab6e752ce8f6bc8223d428754c0081;p=thirdparty%2Fpdns.git

Move the version (and hashes) of external dependencies to JSON files
---

diff --git a/.github/workflows/build-and-test-all.yml b/.github/workflows/build-and-test-all.yml
index ed0ee3ad02..ba0669bfb2 100644
--- a/.github/workflows/build-and-test-all.yml
+++ b/.github/workflows/build-and-test-all.yml
@@ -250,7 +250,7 @@ jobs:
           restore-keys: dnsdist-${{ matrix.features }}-${{ matrix.sanitizers }}-ccache-
       - run: inv ci-install-rust ${{ env.REPO_HOME }}
         working-directory: ./pdns/dnsdistdist/
-      - run: inv ci-build-and-install-quiche
+      - run: inv ci-build-and-install-quiche ${{ env.REPO_HOME }}
         working-directory: ./pdns/dnsdistdist/
       - run: inv ci-autoconf
         working-directory: ./pdns/dnsdistdist/
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
index 22f11414fc..17391461f7 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -156,7 +156,7 @@ jobs:
     - run: inv ci-install-rust ${{ env.REPO_HOME }}
       if: matrix.product == 'dnsdist'
       working-directory: ./pdns/dnsdistdist/
-    - run: inv ci-build-and-install-quiche
+    - run: inv ci-build-and-install-quiche ${{ env.REPO_HOME }}
       if: matrix.product == 'dnsdist'
       working-directory: ./pdns/dnsdistdist/
     - name: Configure dnsdist
diff --git a/.github/workflows/misc-dailies.yml b/.github/workflows/misc-dailies.yml
index 00c4b09754..b452528fd7 100644
--- a/.github/workflows/misc-dailies.yml
+++ b/.github/workflows/misc-dailies.yml
@@ -73,6 +73,7 @@ jobs:
       COVERITY_TOKEN: ${{ secrets.coverity_dnsdist_token }}
       SANITIZERS:
       UNIT_TESTS: no
+      REPO_HOME: ${{ github.workspace }}
     steps:
       - uses: PowerDNS/pdns/set-ubuntu-mirror@meta
       - uses: actions/checkout@v4
@@ -86,7 +87,7 @@ jobs:
       - run: inv coverity-clang-configure
       - run: inv ci-autoconf
         working-directory: ./pdns/dnsdistdist/
-      - run: inv ci-build-and-install-quiche
+      - run: inv ci-build-and-install-quiche ${{ env.REPO_HOME }}
         working-directory: ./pdns/dnsdistdist/
       - run: inv ci-dnsdist-configure full
         working-directory: ./pdns/dnsdistdist/
diff --git a/Dockerfile-dnsdist b/Dockerfile-dnsdist
index 2b96ab3ef9..a62c7a720e 100644
--- a/Dockerfile-dnsdist
+++ b/Dockerfile-dnsdist
@@ -42,16 +42,16 @@ RUN if [ "${DOCKER_FAKE_RELEASE}" = "YES" ]; then \
 
 
 RUN mkdir /libh2o && cd /libh2o && \
-      apt-get update && apt-get install -y cmake curl libssl-dev zlib1g-dev && \
-      curl -f -L https://github.com/PowerDNS/h2o/archive/refs/tags/v2.2.6+pdns2.tar.gz | tar xz && \
-      CFLAGS='-fPIC' cmake -DWITH_PICOTLS=off -DWITH_BUNDLED_SSL=off -DWITH_MRUBY=off -DCMAKE_INSTALL_PREFIX=/opt ./h2o-2.2.6-pdns2 && \
-      make install
+      apt-get update && apt-get install -y cmake curl jq libssl-dev zlib1g-dev && \
+      cd /source/builder-support/helpers/ && \
+      ./install_h2o.sh
 
 RUN mkdir /quiche && cd /quiche && \
     apt-get install -y libclang-dev && \
     apt-get clean && \
-    /source/builder-support/helpers/install_rust.sh && \
-    /source/builder-support/helpers/install_quiche.sh
+    cd /source/builder-support/helpers/ && \
+    ./install_rust.sh && \
+    ./install_quiche.sh
 
 RUN mkdir /build && \
     LUAVER=$([ -z "${NO_LUA_JIT##*$(dpkg --print-architecture)*}" ] && echo 'lua5.3' || echo 'luajit') && \
diff --git a/Dockerfile-recursor b/Dockerfile-recursor
index 9b118cd866..cf94ac420d 100644
--- a/Dockerfile-recursor
+++ b/Dockerfile-recursor
@@ -13,7 +13,7 @@ ENV NO_LUA_JIT="s390x arm64"
 RUN apt-get update && apt-get -y dist-upgrade && apt-get clean
 
 # devscripts gives us mk-build-deps (and a lot of other stuff)
-RUN apt-get install -y --no-install-recommends devscripts equivs git curl && apt-get clean
+RUN apt-get install -y --no-install-recommends devscripts equivs git curl jq && apt-get clean
 
 COPY builder-support /source/builder-support
 
@@ -29,7 +29,8 @@ COPY .git /source/.git
 COPY builder/helpers/set-configure-ac-version.sh /usr/local/bin
 
 COPY builder-support/helpers/install_rust.sh /source/install_rust.sh
-RUN /source/install_rust.sh
+COPY builder-support/helpers/rust.json /source/rust.json
+RUN cd /source/ && ./install_rust.sh
 
 # build and install (TODO: before we hit this line, rearrange /source structure if we are coming from a tarball)
 WORKDIR /source/pdns/recursordist
diff --git a/builder-support/dockerfiles/Dockerfile.debbuild b/builder-support/dockerfiles/Dockerfile.debbuild
index 71d30f6ee8..bbf65dc45a 100644
--- a/builder-support/dockerfiles/Dockerfile.debbuild
+++ b/builder-support/dockerfiles/Dockerfile.debbuild
@@ -18,9 +18,8 @@ RUN mv pdns-recursor*.deb /dist; mv pdns-recursor*.ddeb /dist || true
 @IF [ -n "$M_dnsdist$M_all" ]
 RUN mkdir /libh2o && cd /libh2o && \
       apt-get update && apt-get install -y cmake curl libssl-dev zlib1g-dev && \
-      curl -f -L https://github.com/PowerDNS/h2o/archive/refs/tags/v2.2.6+pdns2.tar.gz | tar xz && \
-      CFLAGS='-fPIC' cmake -DWITH_PICOTLS=off -DWITH_BUNDLED_SSL=off -DWITH_MRUBY=off -DCMAKE_INSTALL_PREFIX=/opt ./h2o-2.2.6-pdns2 && \
-      make install
+      cd /pdns/builder-support/helpers/ && \
+      ./install_h2o.sh
 
 RUN builder/helpers/build-debs.sh dnsdist-${BUILDER_VERSION}
 
diff --git a/builder-support/dockerfiles/Dockerfile.debbuild-prepare b/builder-support/dockerfiles/Dockerfile.debbuild-prepare
index 64a5d401c5..caf1847a4a 100644
--- a/builder-support/dockerfiles/Dockerfile.debbuild-prepare
+++ b/builder-support/dockerfiles/Dockerfile.debbuild-prepare
@@ -1,6 +1,6 @@
 FROM dist-base as package-builder
 ARG APT_URL
-RUN DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends devscripts dpkg-dev build-essential python3-venv equivs curl
+RUN DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends devscripts dpkg-dev build-essential python3-venv equivs curl jq
 
 RUN mkdir /dist /pdns
 WORKDIR /pdns
@@ -9,13 +9,13 @@ ADD builder/helpers/ /pdns/builder/helpers/
 ADD builder-support/helpers/ /pdns/builder-support/helpers/
 
 @IF [ -n "$M_recursor$M_all" ]
-RUN /pdns/builder-support/helpers/install_rust.sh
+RUN cd /pdns/builder-support/helpers/ && ./install_rust.sh
 @ENDIF
 
 @IF [ -n "$M_dnsdist$M_all" ]
-RUN /pdns/builder-support/helpers/install_rust.sh
+RUN cd /pdns/builder-support/helpers/ && ./install_rust.sh
 RUN DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends git cmake clang
-RUN /pdns/builder-support/helpers/install_quiche.sh
+RUN cd /pdns/builder-support/helpers/ && ./install_quiche.sh
 @ENDIF
 
 # Used for -p option to only build specific packages
diff --git a/builder-support/dockerfiles/Dockerfile.recursor b/builder-support/dockerfiles/Dockerfile.recursor
index 088df9710c..214ef9e622 100644
--- a/builder-support/dockerfiles/Dockerfile.recursor
+++ b/builder-support/dockerfiles/Dockerfile.recursor
@@ -13,7 +13,8 @@ ADD builder-support/gen-version /pdns-recursor/pdns/recursordist/builder-support
 WORKDIR /pdns-recursor/pdns/recursordist
 
 ADD builder-support/helpers/ /pdns/builder-support/helpers/
-RUN /pdns/builder-support/helpers/install_rust.sh
+RUN cd /pdns/builder-support/helpers/ && \
+    ./install_rust.sh
 
 RUN mkdir /sdist
 
diff --git a/builder-support/dockerfiles/Dockerfile.rpmbuild b/builder-support/dockerfiles/Dockerfile.rpmbuild
index 048e6ad2ea..dee001a198 100644
--- a/builder-support/dockerfiles/Dockerfile.rpmbuild
+++ b/builder-support/dockerfiles/Dockerfile.rpmbuild
@@ -1,10 +1,10 @@
 FROM dist-base as package-builder
 RUN touch /var/lib/rpm/* && if $(grep -q 'release 7' /etc/redhat-release); then \
       yum upgrade -y && \
-      yum install -y rpm-build rpmdevtools python2 python3 curl "@Development Tools"; \
+      yum install -y rpm-build rpmdevtools python2 python3 curl jq "@Development Tools"; \
     else \
       yum upgrade -y && \
-      yum install --allowerasing -y rpm-build rpmdevtools python3 curl "@Development Tools"; \
+      yum install --allowerasing -y rpm-build rpmdevtools python3 curl jq "@Development Tools"; \
     fi
 
 RUN mkdir /dist /pdns
@@ -16,16 +16,17 @@ ADD builder/helpers/ /pdns/builder/helpers/
 ADD builder-support/helpers/ /pdns/builder-support/helpers/
 
 @IF [ -n "$M_recursor$M_all" ]
-RUN /pdns/builder-support/helpers/install_rust.sh
+RUN cd pdns/builder-support/helpers/ && ./install_rust.sh
 @ENDIF
 
 @IF [ -n "$M_dnsdist$M_all" ]
-RUN /pdns/builder-support/helpers/install_rust.sh
 # We do not build Quiche (DNS over QUIC support) on el-7 because the clang
 # version is too old to build the 'boring-sys' crate needed by Quiche
 RUN if ! $(grep -q 'release 7' /etc/redhat-release); then \
+      cd /pdns/builder-support/helpers/ && ./install_rust.sh; \
       yum install -y git cmake clang; \
-      /pdns/builder-support/helpers/install_quiche.sh; \
+      cd /pdns/builder-support/helpers/ && \
+      ./install_quiche.sh; \
     fi
 @ENDIF
 
@@ -62,9 +63,8 @@ RUN touch /var/lib/rpm/* &&  if $(grep -q 'release 7' /etc/redhat-release); then
 # this is fine because --allowerasing is only there to deal with libcurl conflicting with libcurl-minimal on some el9 images
 RUN touch /var/lib/rpm/* && mkdir /libh2o && cd /libh2o && \
       yum install -y --allowerasing curl libcurl openssl-devel cmake || yum install -y curl libcurl openssl-devel cmake && \
-      curl -f -L https://github.com/PowerDNS/h2o/archive/refs/tags/v2.2.6+pdns2.tar.gz | tar xz && \
-      CFLAGS='-fPIC' cmake -DWITH_PICOTLS=off -DWITH_BUNDLED_SSL=off -DWITH_MRUBY=off -DCMAKE_INSTALL_PREFIX=/opt ./h2o-2.2.6-pdns2 && \
-      make install
+      cd /pdns/builder-support/helpers/ && \
+      ./install_h2o.sh
 
 RUN touch /var/lib/rpm/* && if $(grep -q 'release 7' /etc/redhat-release); then \
       scl enable devtoolset-11 -- builder/helpers/build-specs.sh builder-support/specs/dnsdist.spec; \
diff --git a/builder-support/helpers/h2o.json b/builder-support/helpers/h2o.json
new file mode 100644
index 0000000000..bff2e6de21
--- /dev/null
+++ b/builder-support/helpers/h2o.json
@@ -0,0 +1,4 @@
+{
+  "version": "2.2.6-pdns2",
+  "SHA256SUM": "e25959c3f9a102e7a332ca0bb8b3f533eb14919d5a60ca999730c2ebee4b548f"
+}
diff --git a/builder-support/helpers/install_h2o.sh b/builder-support/helpers/install_h2o.sh
new file mode 100755
index 0000000000..7c704606cd
--- /dev/null
+++ b/builder-support/helpers/install_h2o.sh
@@ -0,0 +1,20 @@
+#!/bin/sh
+set -v
+set -e
+
+readonly H2O_VERSION=$(jq -r .version < h2o.json)
+readonly H2O_TARBALL="v${H2O_VERSION}.tar.gz"
+readonly H2O_TARBALL_URL="https://github.com/PowerDNS/h2o/archive/refs/tags/${H2O_TARBALL}"
+readonly H2O_TARBALL_HASH=$(jq -r .SHA256SUM < h2o.json)
+
+cd /tmp
+echo $0: Downloading $H2O_TARBALL
+curl -f -L -o "${H2O_TARBALL}" "${H2O_TARBALL_URL}"
+
+# Line below should echo two spaces between digest and name
+echo "${H2O_TARBALL_HASH}"  "${H2O_TARBALL}" | sha256sum -c -
+tar xf "${H2O_TARBALL}"
+CFLAGS='-fPIC' cmake -DWITH_PICOTLS=off -DWITH_BUNDLED_SSL=off -DWITH_MRUBY=off -DCMAKE_INSTALL_PREFIX=/opt ./h2o-${H2O_VERSION}
+make -j $(nproc)
+make install
+rm -rf "${H2O_TARBALL}" "h2o-${H2O_VERSION}"
diff --git a/builder-support/helpers/install_quiche.sh b/builder-support/helpers/install_quiche.sh
index 4bdb547e8a..b09793fcd0 100755
--- a/builder-support/helpers/install_quiche.sh
+++ b/builder-support/helpers/install_quiche.sh
@@ -2,10 +2,10 @@
 set -v
 set -e
 
-readonly QUICHE_VERSION='0.20.1'
+readonly QUICHE_VERSION=$(jq -r .version < quiche.json)
 readonly QUICHE_TARBALL="${QUICHE_VERSION}.tar.gz"
 readonly QUICHE_TARBALL_URL="https://github.com/cloudflare/quiche/archive/${QUICHE_TARBALL}"
-readonly QUICHE_TARBALL_HASH='9c460d8ecf6c80c06bf9b42f91201ef33f912e2615a871ff2d0e50197b901c71'
+readonly QUICHE_TARBALL_HASH=$(jq -r .SHA256SUM < quiche.json)
 
 INSTALL_PREFIX=/usr
 SOEXT=so
@@ -19,8 +19,9 @@ if [ $(uname) = Darwin ]; then
 fi
 
 cd /tmp
-echo $0: Downloading $QUICHE_TARBALL
+echo $0: Downloading ${QUICHE_TARBALL}
 curl -L -o "${QUICHE_TARBALL}" "${QUICHE_TARBALL_URL}"
+echo $0: Checking that the hash of ${QUICHE_TARBALL} is ${QUICHE_TARBALL_HASH}
 # Line below should echo two spaces between digest and name
 echo "${QUICHE_TARBALL_HASH}"  "${QUICHE_TARBALL}" | sha256sum -c -
 tar xf "${QUICHE_TARBALL}"
diff --git a/builder-support/helpers/install_rust.sh b/builder-support/helpers/install_rust.sh
index 542b09c67e..bd1b4ade19 100755
--- a/builder-support/helpers/install_rust.sh
+++ b/builder-support/helpers/install_rust.sh
@@ -5,7 +5,8 @@ set -e
 ARCH=$(arch)
 
 # Default version
-RUST_VERSION=rust-1.75.0-$ARCH-unknown-linux-gnu
+RUST_VERSION_NUMBER=$(jq -r .version < rust.json)
+RUST_VERSION=rust-$RUST_VERSION_NUMBER-$ARCH-unknown-linux-gnu
 
 if [ $# -ge 1 ]; then
     RUST_VERSION=$1
@@ -15,8 +16,8 @@ fi
 SITE=https://downloads.powerdns.com/rust
 RUST_TARBALL=$RUST_VERSION.tar.gz
 
-SHA256SUM_x86_64=473978b6f8ff216389f9e89315211c6b683cf95a966196e7914b46e8cf0d74f6
-SHA256SUM_aarch64=30828cd904fcfb47f1ac43627c7033c903889ea4aca538f53dcafbb3744a9a73
+SHA256SUM_x86_64=$(jq -r .SHA256SUM_x86_64 < rust.json)
+SHA256SUM_aarch64=$(jq -r .SHA256SUM_aarch64 < rust.json)
 
 NAME=SHA256SUM_$ARCH
 eval VALUE=\$$NAME
@@ -35,6 +36,7 @@ fi
 #
 cd /tmp
 echo $0: Downloading $RUST_TARBALL
+echo $0: Expecting hash $VALUE
 
 curl -f -o $RUST_TARBALL $SITE/$RUST_TARBALL
 # Line below should echo two spaces between digest and name
diff --git a/builder-support/helpers/quiche.json b/builder-support/helpers/quiche.json
new file mode 100644
index 0000000000..e3025b32b3
--- /dev/null
+++ b/builder-support/helpers/quiche.json
@@ -0,0 +1,4 @@
+{
+  "version": "0.20.1",
+  "SHA256SUM": "9c460d8ecf6c80c06bf9b42f91201ef33f912e2615a871ff2d0e50197b901c71"
+}
diff --git a/builder-support/helpers/rust.json b/builder-support/helpers/rust.json
new file mode 100644
index 0000000000..ac9e47ffe4
--- /dev/null
+++ b/builder-support/helpers/rust.json
@@ -0,0 +1,5 @@
+{
+  "version": "1.75.0",
+  "SHA256SUM_x86_64": "473978b6f8ff216389f9e89315211c6b683cf95a966196e7914b46e8cf0d74f6",
+  "SHA256SUM_aarch64": "30828cd904fcfb47f1ac43627c7033c903889ea4aca538f53dcafbb3744a9a73"
+}
diff --git a/builder-support/specs/pdns-recursor.spec b/builder-support/specs/pdns-recursor.spec
index 8408500e98..df4b6b9682 100644
--- a/builder-support/specs/pdns-recursor.spec
+++ b/builder-support/specs/pdns-recursor.spec
@@ -4,6 +4,7 @@ Release: %{getenv:BUILDER_RPM_RELEASE}%{?dist}
 Summary: Modern, advanced and high performance recursing/non authoritative name server
 Group: System Environment/Daemons
 License: GPLv2
+Vendor: PowerDNS.COM BV
 URL: https://powerdns.com
 Source0: %{name}-%{getenv:BUILDER_VERSION}.tar.bz2
 
diff --git a/builder-support/specs/pdns.spec b/builder-support/specs/pdns.spec
index 9f265eda1d..ac02b68d52 100644
--- a/builder-support/specs/pdns.spec
+++ b/builder-support/specs/pdns.spec
@@ -7,6 +7,7 @@ Release: %{getenv:BUILDER_RPM_RELEASE}%{dist}
 Summary: A modern, advanced and high performance authoritative-only nameserver
 Group: System Environment/Daemons
 License: GPLv2
+Vendor: PowerDNS.COM BV
 URL: https://powerdns.com
 Source0: %{name}-%{getenv:BUILDER_VERSION}.tar.bz2
 
diff --git a/tasks.py b/tasks.py
index ae920dc55e..f3e72b3d4f 100644
--- a/tasks.py
+++ b/tasks.py
@@ -1,6 +1,7 @@
 from invoke import task
 from invoke.exceptions import Failure, UnexpectedExit
 
+import json
 import os
 import sys
 import time
@@ -8,8 +9,6 @@ import time
 auth_backend_ip_addr = os.getenv('AUTH_BACKEND_IP_ADDR', '127.0.0.1')
 
 clang_version = os.getenv('CLANG_VERSION', '13')
-quiche_version = '0.20.1'
-quiche_hash = '9c460d8ecf6c80c06bf9b42f91201ef33f912e2615a871ff2d0e50197b901c71'
 
 all_build_deps = [
     'ccache',
@@ -174,7 +173,8 @@ def install_clang_runtime(c):
 
 @task
 def ci_install_rust(c, repo):
-    c.sudo(f'{repo}/builder-support/helpers/install_rust.sh')
+    with c.cd(f'{repo}/builder-support/helpers/'):
+        c.run('sudo sh install_rust.sh')
 
 def install_libdecaf(c, product):
     c.run('git clone https://git.code.sf.net/p/ed448goldilocks/code /tmp/libdecaf')
@@ -943,7 +943,12 @@ def coverity_upload(c, email, project, tarball):
             https://scan.coverity.com/builds?project={project}', hide=True)
 
 @task
-def ci_build_and_install_quiche(c):
+def ci_build_and_install_quiche(c, repo):
+    with open(f'{repo}/builder-support/helpers/quiche.json') as quiche_json:
+        quiche_data = json.load(quiche_json)
+        quiche_version = quiche_data['version']
+        quiche_hash = quiche_data['SHA256SUM']
+
     # we have to pass -L because GitHub will do a redirect, sadly
     c.run(f'curl -L -o quiche-{quiche_version}.tar.gz https://github.com/cloudflare/quiche/archive/{quiche_version}.tar.gz')
     # Line below should echo two spaces between digest and name