gh actions - misc-dailies.yml: validate only pinned pip packages are installed

diff --git a/.github/workflows/misc-dailies.yml b/.github/workflows/misc-dailies.yml
index c11ebc27f70efcb2215be95251cb54e823cf0d93..4f5d6cf5bf88fc7f73e8d3e15eb777275eafa97f 100644 (file)
--- a/.github/workflows/misc-dailies.yml
+++ b/.github/workflows/misc-dailies.yml
@@ -68,3 +68,67 @@ jobs:
     secrets:
       COVERITY_TOKEN: ${{ secrets.coverity_rec_token }}
       COVERITY_EMAIL: ${{ secrets.coverity_email }}
+
+  # The jobs below check that only the pinned version of Python3 packages are installed with pip. Running in a container, a
+  # pip proxy registers all requests for installing packages with pip.  Then, the downloaded packages and their versions are compared
+  # with the list used for the installation (i.e. docs/requirements.txt, pdns/recursordist/docs/requirements.txt, etc). If a package
+  # is missing or a version does not match the one expected, this job fails, which makes the workflow fail.
+  #
+  # The pinned version plus hashes are generated using pip-compile using an input file that includes the original list of packages
+  # (pip-compile --generate-hashes -U requirements.in). "pip-compile" can be installed via pip-tools with Python 3.11, which is the version
+  # used in the CI. Any other Python version would end up with different versions for packages and could result in workflow failures.
+  #
+  # One recurring error thrown by this validation is when a new version of a pinned package is released for a "setup-requires" dependency
+  # of one of the packages in the list (see https://github.com/PowerDNS/pdns/pull/14596). The package version in “requirements.in” should
+  # be modified to solve this issue. In some cases, it is enough to generate again the list of packages, making sure to add the -U flag
+  # to force the upgrade: "pip-compile --generate-hashes -U requirements.in" (this could include upgrading other packages).
+
+  list-pip-requirement-files:
+    if: ${{ vars.SCHEDULED_MISC_DAILIES }}
+    runs-on: ubuntu-22.04
+    outputs:
+      req-files: ${{ steps.get-list-requirements.outputs.files }}
+    steps:
+    - uses: actions/checkout@v4
+    - name: Get all requirements.txt files and export them as outputs
+      id: get-list-requirements
+      run: |
+        echo "files=$(find . -name 'requirements.txt' | jq -R -s -c 'split("\n")[:-1]')" >> "$GITHUB_OUTPUT"
+
+  validate-pip-hashes:
+    if: ${{ vars.SCHEDULED_MISC_DAILIES }}
+    name: ${{ matrix.requirements-file }} - Validate list of packages and hashes
+    runs-on: ubuntu-22.04
+    needs: list-pip-requirement-files
+    env:
+      SERVICE_IP_ADDR: 127.0.0.1
+    services:
+      database:
+        image: epicwink/proxpi
+        ports:
+          - 5000:5000
+        options: >-
+          --restart always
+    strategy:
+      fail-fast: false
+      matrix:
+        requirements-file: ${{ fromJson(needs.list-pip-requirement-files.outputs.req-files) }}
+    steps:
+      - run: echo "${{ matrix.requirements-file }}"
+      - uses: PowerDNS/pdns/set-ubuntu-mirror@meta
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      # Configure pip index-url set to proxpi
+      - run: pip config set global.index-url http://${{ env.SERVICE_IP_ADDR }}:5000/index/
+      - run: pip config set global.trusted-host ${{ env.SERVICE_IP_ADDR }}
+      - id: proxpi-docker
+        run: echo "id=$(docker ps | grep "epicwink/proxpi" | awk '{print $1}')" >> "$GITHUB_OUTPUT"
+      - run: pip install -r ${{ matrix.requirements-file }}
+      - name: Get the list of packages requested to the pip proxy
+        run: |
+          docker logs ${{ steps.proxpi-docker.outputs.id }} 2>&1 | grep whl | awk '{print $8}' | cut -d "/" -f 4 | awk -F'-' '{print $1"=="$2}' | sort -u --ignore-case | sed 's/_/-/' | egrep -v "pip==|setuptools==|wheel==|setuptools-git==" > /tmp/proxpi.log
+          cat /tmp/proxpi.log
+      - name: check only listed packages were installed
+        run: for i in `cat /tmp/proxpi.log`; do grep -qq -i $i ${{ matrix.requirements-file }} || ( echo "$i not found" && exit 1 ); done