dnsdist-1.8.x: Backport the workflow to build packages from tags

diff --git a/.github/workflows/build-packages.yml b/.github/workflows/build-packages.yml
new file mode 100644 (file)
index 0000000..12bedec
--- /dev/null
+++ b/.github/workflows/build-packages.yml
@@ -0,0 +1,212 @@
+---
+name: Build packages
+
+on:
+  workflow_call:
+    inputs:
+      product:
+        required: true
+        description: Product to build
+        type: string
+      os:
+        required: false
+        description: OSes to build for, space separated
+        type: string
+        # please remember to update the pkghashes below when you
+        # update this list, as well as the one in builder-dispatch.yml
+        default: >
+          el-7
+          el-8
+          el-9
+          debian-buster
+          debian-bullseye
+          debian-bookworm
+          ubuntu-focal
+          ubuntu-jammy
+      ref:
+        description: git ref to checkout
+        type: string
+        default: master
+        required: false
+      is_release:
+        description: is this a release build?
+        type: string
+        required: false
+        default: 'NO'
+    secrets:
+      DOWNLOADS_AUTOBUILT_SECRET:
+        required: true
+      DOWNLOADS_AUTOBUILT_RSYNCTARGET:
+        required: true
+      DOWNLOADS_AUTOBUILT_HOSTKEY:
+        required: true
+
+permissions: # least privileges, see https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions
+  contents: read
+
+jobs:
+  prepare:
+    name: generate OS list
+    runs-on: ubuntu-20.04
+    outputs:
+      oslist: ${{ steps.get-oslist.outputs.oslist }}
+    steps:
+      # instead of jo, we could use jq here, which avoids running apt, and thus would be faster.
+      # but, as this whole workflow needs at least 30 minutes to run, I prefer spending a few seconds here
+      # so that the command remains readable, because jo is simpler to use.
+      - run: sudo apt-get update && sudo apt-get -y install jo
+      - id: get-oslist
+        run: echo "oslist=$(jo -a ${{ inputs.os }})" >> "$GITHUB_OUTPUT"
+  build:
+    needs: prepare
+    name: build ${{ inputs.product }} (${{ inputs.ref }}) for ${{ matrix.os }}
+    # on a ubuntu-20.04 VM
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        os: ${{fromJson(needs.prepare.outputs.oslist)}}
+      fail-fast: false
+    outputs:
+      version: ${{ steps.getversion.outputs.version }}
+      pkghashes-el-7: ${{ steps.pkghashes.outputs.pkghashes-el-7 }}
+      pkghashes-el-8: ${{ steps.pkghashes.outputs.pkghashes-el-8 }}
+      pkghashes-el-9: ${{ steps.pkghashes.outputs.pkghashes-el-9 }}
+      pkghashes-debian-buster: ${{ steps.pkghashes.outputs.pkghashes-debian-buster }}
+      pkghashes-debian-bullseye: ${{ steps.pkghashes.outputs.pkghashes-debian-bullseye }}
+      pkghashes-debian-bookworm: ${{ steps.pkghashes.outputs.pkghashes-debian-bookworm }}
+      pkghashes-ubuntu-focal: ${{ steps.pkghashes.outputs.pkghashes-ubuntu-focal }}
+      pkghashes-ubuntu-jammy: ${{ steps.pkghashes.outputs.pkghashes-ubuntu-jammy }}
+      srchashes: ${{ steps.srchashes.outputs.srchashes }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # for correct version numbers
+          submodules: recursive
+          ref: ${{ inputs.ref }}
+      # this builds packages and runs our unit tests (make check)
+      - run: IS_RELEASE=${{ inputs.is_release}} builder/build.sh -v -m ${{ inputs.product }} ${{ matrix.os }}
+      - name: Get version number
+        run: |
+          echo "version=$(readlink builder/tmp/latest)" >> $GITHUB_OUTPUT
+        id: getversion
+      - name: Upload packages as GH artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ inputs.product }}-${{ matrix.os }}-${{ steps.getversion.outputs.version }}
+          path: built_pkgs/
+          retention-days: 7
+      - name: Normalize package name
+        id: normalize-name
+        run: |
+          if [ "x${{ inputs.product }}" = "xauthoritative" ]; then
+            echo "normalized-package-name=pdns" >> $GITHUB_OUTPUT
+          elif [ "x${{ inputs.product }}" = "xrecursor" ]; then
+            echo "normalized-package-name=pdns-recursor" >> $GITHUB_OUTPUT
+          else
+            echo "normalized-package-name=${{ inputs.product }}" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Extract packages from the tarball
+        # so we get provenance for individual packages (and the JSON package manifests from the builder)
+        id: extract
+        run: |
+          mkdir -m 700 -p ./packages/
+          tar xvf ./built_pkgs/*/*/${{ steps.normalize-name.outputs.normalized-package-name }}-${{ steps.getversion.outputs.version }}-${{ matrix.os }}.tar.bz2 -C ./packages/ --transform='s/.*\///'
+      - name: Generate package hashes for provenance
+        shell: bash
+        id: pkghashes
+        run: |
+          echo "pkghashes-${{ matrix.os }}=$(sha256sum ./packages/*.rpm ./packages/*.deb ./packages/*.json | base64 -w0)" >> $GITHUB_OUTPUT
+      - name: Generate source hash for provenance
+        shell: bash
+        id: srchashes
+        run: |
+          echo "srchashes=$(sha256sum ./built_pkgs/*/*/${{ steps.normalize-name.outputs.normalized-package-name }}-${{ steps.getversion.outputs.version }}.tar.bz2 ./packages/*.json | base64 -w0)" >> $GITHUB_OUTPUT
+      - name: Upload packages to downloads.powerdns.com
+        env:
+          SSHKEY: ${{ secrets.DOWNLOADS_AUTOBUILT_SECRET }}
+          RSYNCTARGET: ${{ secrets.DOWNLOADS_AUTOBUILT_RSYNCTARGET }}
+          HOSTKEY: ${{ secrets.DOWNLOADS_AUTOBUILT_HOSTKEY }}
+        if:
+          "${{ env.SSHKEY != '' }}"
+        run: |
+          mkdir -m 700 -p ~/.ssh
+          echo "$SSHKEY" > ~/.ssh/id_ed25519
+          chmod 600 ~/.ssh/id_ed25519
+          echo "$HOSTKEY" > ~/.ssh/known_hosts
+          rsync -4rlptD built_pkgs/* "$RSYNCTARGET"
+
+  check-hashes:
+    needs: build
+    name: Check if hashes were created for all requested targets
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Get list of outputs from build jobs
+        run: echo '${{ toJSON(needs.build.outputs) }}' | jq 'keys[]' | grep -v version | tee /tmp/build-outputs.txt
+      - name: Get list of OS inputs
+        run: for i in ${{ inputs.os }}; do echo "\"pkghashes-$i\""; done | sort | tee /tmp/os-inputs.txt; echo "\"srchashes\"" | tee -a /tmp/os-inputs.txt
+      - name: Fail if there is a hash missing
+        run: if ! diff -q /tmp/build-outputs.txt /tmp/os-inputs.txt; then exit 1; fi
+
+  provenance-pkgs:
+    needs: [prepare, build]
+    name: Generate provenance for ${{ inputs.product }} (${{ inputs.ref }}) for ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: ${{fromJson(needs.prepare.outputs.oslist)}}
+    permissions:
+      actions: read   # To read the workflow path.
+      id-token: write # To sign the provenance.
+      contents: write # To be able to upload assets as release artifacts
+    uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v1.4.0
+    with:
+      base64-subjects: "${{ needs.build.outputs[format('pkghashes-{0}', matrix.os)] }}"
+      upload-assets: false
+      provenance-name: "${{ inputs.product }}-${{ needs.build.outputs.version }}-${{ matrix.os}}.intoto.jsonl"
+
+  provenance-src:
+    needs: build
+    name: Generate provenance for ${{ inputs.product }} (${{ inputs.ref }}) source tarball
+    permissions:
+      actions: read   # To read the workflow path.
+      id-token: write # To sign the provenance.
+      contents: write # To be able to upload assets as release artifacts
+    uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v1.4.0
+    with:
+      base64-subjects: "${{ needs.build.outputs.srchashes }}"
+      upload-assets: false
+      provenance-name: "${{ inputs.product }}-${{ needs.build.outputs.version }}-src.intoto.jsonl"
+
+  upload-provenance:
+    needs: [prepare, build, provenance-src, provenance-pkgs]
+    name: Upload the provenance artifacts to downloads.powerdns.com
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        os: ${{fromJson(needs.prepare.outputs.oslist)}}
+    steps:
+      - name: Download source tarball provenance for ${{ inputs.product }} (${{ inputs.ref }})
+        id: download-src-provenance
+        uses: actions/download-artifact@v3
+        with:
+          name: "${{ inputs.product }}-${{ needs.build.outputs.version }}-src.intoto.jsonl"
+      - name: Download provenance for ${{ inputs.product }} (${{ inputs.ref }}) for ${{ matrix.os }}
+        id: download-provenance
+        uses: actions/download-artifact@v3
+        with:
+          name: "${{ inputs.product }}-${{ needs.build.outputs.version }}-${{ matrix.os}}.intoto.jsonl"
+      - name: Upload provenance artifacts to downloads.powerdns.com
+        id: upload-provenance
+        env:
+          SSHKEY: ${{ secrets.DOWNLOADS_AUTOBUILT_SECRET }}
+          RSYNCTARGET: ${{ secrets.DOWNLOADS_AUTOBUILT_RSYNCTARGET }}
+          HOSTKEY: ${{ secrets.DOWNLOADS_AUTOBUILT_HOSTKEY }}
+        if:
+          "${{ env.SSHKEY != '' }}"
+        shell: bash
+        run: |
+          mkdir -m 700 -p ~/.ssh
+          echo "$SSHKEY" > ~/.ssh/id_ed25519
+          chmod 600 ~/.ssh/id_ed25519
+          echo "$HOSTKEY" > ~/.ssh/known_hosts
+          rsync -4rlptD ${{steps.download-src-provenance.outputs.download-path}}/*.jsonl ${{steps.download-provenance.outputs.download-path}}/*.jsonl "${RSYNCTARGET}/${{ inputs.product }}/${{ needs.build.outputs.version }}/"

diff --git a/.github/workflows/build-tags.yml b/.github/workflows/build-tags.yml
new file mode 100644 (file)
index 0000000..6431ec9
--- /dev/null
+++ b/.github/workflows/build-tags.yml
@@ -0,0 +1,46 @@
+---
+name: Build packages for tags
+
+on:
+  push:
+    tags:
+    - 'auth-*'
+    - 'dnsdist-*'
+    - 'rec-*'
+
+jobs:
+  call-build-packages-auth:
+    uses: PowerDNS/pdns/.github/workflows/build-packages.yml@master
+    if: startsWith(github.ref_name, 'auth')
+    with:
+      is_release: 'YES'
+      product: 'authoritative'
+      ref: ${{ github.ref_name }}
+    secrets:
+      DOWNLOADS_AUTOBUILT_SECRET: ${{ secrets.DOWNLOADS_AUTOBUILT_SECRET }}
+      DOWNLOADS_AUTOBUILT_RSYNCTARGET: ${{ secrets.DOWNLOADS_AUTOBUILT_RSYNCTARGET }}
+      DOWNLOADS_AUTOBUILT_HOSTKEY: ${{ secrets.DOWNLOADS_AUTOBUILT_HOSTKEY }}
+
+  call-build-packages-dnsdist:
+    uses: PowerDNS/pdns/.github/workflows/build-packages.yml@master
+    if: startsWith(github.ref_name, 'dnsdist')
+    with:
+      is_release: 'YES'
+      product: 'dnsdist'
+      ref: ${{ github.ref_name }}
+    secrets:
+      DOWNLOADS_AUTOBUILT_SECRET: ${{ secrets.DOWNLOADS_AUTOBUILT_SECRET }}
+      DOWNLOADS_AUTOBUILT_RSYNCTARGET: ${{ secrets.DOWNLOADS_AUTOBUILT_RSYNCTARGET }}
+      DOWNLOADS_AUTOBUILT_HOSTKEY: ${{ secrets.DOWNLOADS_AUTOBUILT_HOSTKEY }}
+
+  call-build-packages-rec:
+    uses: PowerDNS/pdns/.github/workflows/build-packages.yml@master
+    if: startsWith(github.ref_name, 'rec')
+    with:
+      is_release: 'YES'
+      product: 'recursor'
+      ref: ${{ github.ref_name }}
+    secrets:
+      DOWNLOADS_AUTOBUILT_SECRET: ${{ secrets.DOWNLOADS_AUTOBUILT_SECRET }}
+      DOWNLOADS_AUTOBUILT_RSYNCTARGET: ${{ secrets.DOWNLOADS_AUTOBUILT_RSYNCTARGET }}
+      DOWNLOADS_AUTOBUILT_HOSTKEY: ${{ secrets.DOWNLOADS_AUTOBUILT_HOSTKEY }}

diff --git a/.github/workflows/builder.yml b/.github/workflows/builder.yml
index 3b9484da1eb3f8af0cccdc85e824e3a418662d65..2666adde81626eeea09541584f4a49f806af3a17 100644 (file)
--- a/.github/workflows/builder.yml
+++ b/.github/workflows/builder.yml
@@ -36,7 +36,8 @@ jobs:
       # this builds packages and runs our unit test (make check)
       - run: builder/build.sh -v -m ${{ matrix.product }} ${{ matrix.os }}
       - name: Get version number
-        run: 'echo ::set-output name=version::$(readlink builder/tmp/latest)'
+        run: |
+          echo "version=$(readlink builder/tmp/latest)" >> $GITHUB_OUTPUT
         id: getversion
       - name: Upload packages
         uses: actions/upload-artifact@v3

diff --git a/builder-support/dockerfiles/Dockerfile.debbuild b/builder-support/dockerfiles/Dockerfile.debbuild
index 5b350d666ae2e0437146c933abc4b23e69f63220..46b315d74b0ac2e3cce51cc64759139ca819b9e4 100644 (file)
--- a/builder-support/dockerfiles/Dockerfile.debbuild
+++ b/builder-support/dockerfiles/Dockerfile.debbuild
@@ -20,3 +20,7 @@ RUN builder/helpers/build-debs.sh dnsdist-${BUILDER_VERSION}
 
 RUN mv dnsdist*.deb /dist; mv dnsdist*.ddeb /dist || true
 @ENDIF
+
+# Generate provenance
+RUN apt-get install -y python-apt || apt-get install -y python3-apt
+@EVAL RUN python2 builder/helpers/generate-deb-provenance.py /dist/packages-${BUILDER_TARGET}.json || python3 builder/helpers/generate-deb-provenance.py /dist/packages-${BUILDER_TARGET}.json

diff --git a/builder-support/dockerfiles/Dockerfile.rpmbuild b/builder-support/dockerfiles/Dockerfile.rpmbuild
index b21923a435881e9c07b08ec8cdf9222772b4acee..c18859c2609c8767162f002e44558b3b5c57c051 100644 (file)
--- a/builder-support/dockerfiles/Dockerfile.rpmbuild
+++ b/builder-support/dockerfiles/Dockerfile.rpmbuild
@@ -1,7 +1,11 @@
 FROM dist-base as package-builder
-RUN touch /var/lib/rpm/* && \
-    yum upgrade -y && \
-    yum install -y rpm-build rpmdevtools python3 "@Development Tools"
+RUN touch /var/lib/rpm/* && if $(grep -q 'release 7' /etc/redhat-release); then \
+      yum upgrade -y && \
+      yum install -y rpm-build rpmdevtools python2 python3 "@Development Tools"; \
+    else \
+      yum upgrade -y && \
+      yum install -y rpm-build rpmdevtools python3 "@Development Tools"; \
+    fi
 
 RUN mkdir /dist /pdns
 WORKDIR /pdns
@@ -54,6 +58,14 @@ RUN touch /var/lib/rpm/* && if $(grep -q 'release 7' /etc/redhat-release); then
     fi
 @ENDIF
 
+# Generate provenance
+@IF [ ${BUILDER_TARGET} = el-7 || ${BUILDER_TARGET} = centos-7 ]
+@EVAL RUN python builder/helpers/generate-yum-provenance.py /dist/packages-${BUILDER_TARGET}.json || python3 builder/helpers/generate-yum-provenance.py /dist/packages-${BUILDER_TARGET}.json
+@ENDIF
+@IF [ ${BUILDER_TARGET} != el-7 && ${BUILDET_TARGET} != centos-7 ]
+@EVAL RUN python builder/helpers/generate-dnf-provenance.py /dist/packages-${BUILDER_TARGET}.json || python3 builder/helpers/generate-dnf-provenance.py /dist/packages-${BUILDER_TARGET}.json
+@ENDIF
+
 # mv across layers with overlay2 is buggy in some kernel versions (results in empty dirs)
 # See: https://github.com/moby/moby/issues/33733
 #RUN mv /root/rpmbuild/RPMS/* /dist/