From: Remi Gacogne <remi.gacogne@powerdns.com>
Date: Mon, 25 Mar 2024 13:51:05 +0000 (+0100)
Subject: Attempt to generate SBOMs after building packages
X-Git-Tag: rec-5.1.0-alpha1~76^2~5
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=44503ded5e8e6f088d009177f5f951d408467b58;p=thirdparty%2Fpdns.git

Attempt to generate SBOMs after building packages
---

diff --git a/builder-support/dockerfiles/Dockerfile.rpmsbom b/builder-support/dockerfiles/Dockerfile.rpmsbom
new file mode 100644
index 0000000000..00e6ea51c5
--- /dev/null
+++ b/builder-support/dockerfiles/Dockerfile.rpmsbom
@@ -0,0 +1,23 @@
+RUN touch /var/lib/rpm/* && \
+    yum upgrade -y && yum install -y python3
+ADD builder-support/helpers/ /pdns/builder-support/helpers/
+
+@IF [ -n "$M_recursor$M_all" ]
+RUN cd /pdns/builder-support/helpers/ && python3 generate-sbom-dnf.py /dist/pdns-recursor-${BUILDER_VERSION}-${BUILDER_TARGET}.cyclonedx.json pdns-recursor rust.json
+@ENDIF
+
+@IF [ -n "$M_dnsdist$M_all" ]
+RUN cd /pdns/builder-support/helpers/; \
+    if ! $(grep -q 'release 7' /etc/redhat-release); then \
+       python3 generate-sbom-dnf.py /dist/dnsdist-${BUILDER_VERSION}-${BUILDER_TARGET}.cyclonedx.json dnsdist rust.json quiche.json h2o.json ; \
+    else \
+       python3 generate-sbom-dnf.py /dist/dnsdist-${BUILDER_VERSION}-${BUILDER_TARGET}.cyclonedx.json dnsdist h2o.json ; \
+    fi
+@ENDIF
+
+@IF [ -n "$M_authoritative$M_all" ]
+RUN cd /pdns/builder-support/helpers/ && \
+    for pkg in pdns pdns-backend-geoip pdns-backend-ldap pdns-backend-lmdb pdns-backend-lua2 pdns-backend-mysql pdns-backend-odbc pdns-backend-postgresql pdns-backend-remote pdns-backend-sqlite pdns-backend-tinydns pdns-backend-ixfrdist pdns-tools; do \
+      python3 generate-sbom-dnf.py /dist/${pkg}-${BUILDER_VERSION}-${BUILDER_TARGET}.cyclonedx.json ${pkg}; \
+    done
+@ENDIF
diff --git a/builder-support/dockerfiles/Dockerfile.rpmtest b/builder-support/dockerfiles/Dockerfile.rpmtest
new file mode 100644
index 0000000000..87587499ae
--- /dev/null
+++ b/builder-support/dockerfiles/Dockerfile.rpmtest
@@ -0,0 +1,14 @@
+# Install the built rpms and test them
+FROM dist-base as dist
+
+# If you want to install extra packages or do generic configuration,
+# do it before the COPY. Either here, or in the dist-base layer.
+
+COPY --from=package-builder /dist /dist
+
+# Install built packages with dependencies
+RUN yum localinstall -y /dist/*/*.rpm
+
+# Generate SBOM
+# We should probably guard this behind "${IS_RELEASE}" = "YES"
+@EXEC [ "$skipsbom" = "" ] && include Dockerfile.rpmsbom
diff --git a/builder-support/dockerfiles/Dockerfile.target.amazon-2 b/builder-support/dockerfiles/Dockerfile.target.amazon-2
index cac43d12fe..01933467b2 100644
--- a/builder-support/dockerfiles/Dockerfile.target.amazon-2
+++ b/builder-support/dockerfiles/Dockerfile.target.amazon-2
@@ -11,5 +11,5 @@ RUN touch /var/lib/rpm/* && amazon-linux-extras install epel -y
 @INCLUDE Dockerfile.rpmbuild
 
 # Do a test install and verify
-# Can be skipped with skiptests=1 in the environment
-# @EXEC [ "$skiptests" = "" ] && include Dockerfile.rpmtest
+# Can be skipped with skippackagetest=1 in the environment
+@EXEC [ "$skippackagetest" = "" ] && include Dockerfile.rpmtest
diff --git a/builder-support/dockerfiles/Dockerfile.target.amazon-2023 b/builder-support/dockerfiles/Dockerfile.target.amazon-2023
index f47d6d712c..0dfee6b86c 100644
--- a/builder-support/dockerfiles/Dockerfile.target.amazon-2023
+++ b/builder-support/dockerfiles/Dockerfile.target.amazon-2023
@@ -10,5 +10,5 @@ ARG BUILDER_CACHE_BUSTER=
 @INCLUDE Dockerfile.rpmbuild
 
 # Do a test install and verify
-# Can be skipped with skiptests=1 in the environment
-# @EXEC [ "$skiptests" = "" ] && include Dockerfile.rpmtest
+# Can be skipped with skippackagetest=1 in the environment
+@EXEC [ "$skippackagetest" = "" ] && include Dockerfile.rpmtest
diff --git a/builder-support/dockerfiles/Dockerfile.target.centos-7 b/builder-support/dockerfiles/Dockerfile.target.centos-7
index 9bd7c2a124..24a5dff806 100644
--- a/builder-support/dockerfiles/Dockerfile.target.centos-7
+++ b/builder-support/dockerfiles/Dockerfile.target.centos-7
@@ -18,5 +18,5 @@ RUN touch /var/lib/rpm/* && yum install -y --nogpgcheck devtoolset-11-gcc-c++
 @INCLUDE Dockerfile.rpmbuild
 
 # Do a test install and verify
-# Can be skipped with skiptests=1 in the environment
-# @EXEC [ "$skiptests" = "" ] && include Dockerfile.rpmtest
+# Can be skipped with skippackagetest=1 in the environment
+@EXEC [ "$skippackagetest" = "" ] && include Dockerfile.rpmtest
diff --git a/builder-support/dockerfiles/Dockerfile.target.centos-8 b/builder-support/dockerfiles/Dockerfile.target.centos-8
index ae0e4c9ab9..237ff5ad09 100644
--- a/builder-support/dockerfiles/Dockerfile.target.centos-8
+++ b/builder-support/dockerfiles/Dockerfile.target.centos-8
@@ -30,5 +30,5 @@ RUN touch /var/lib/rpm/* && dnf install -y epel-release && \
 @INCLUDE Dockerfile.rpmbuild
 
 # Do a test install and verify
-# Can be skipped with skiptests=1 in the environment
-# @EXEC [ "$skiptests" = "" ] && include Dockerfile.rpmtest
+# Can be skipped with skippackagetest=1 in the environment
+@EXEC [ "$skippackagetest" = "" ] && include Dockerfile.rpmtest
diff --git a/builder-support/dockerfiles/Dockerfile.target.centos-9-stream b/builder-support/dockerfiles/Dockerfile.target.centos-9-stream
index 873f63fde1..6b9cda656b 100644
--- a/builder-support/dockerfiles/Dockerfile.target.centos-9-stream
+++ b/builder-support/dockerfiles/Dockerfile.target.centos-9-stream
@@ -15,5 +15,5 @@ RUN touch /var/lib/rpm/* && dnf install -y epel-release && \
 @INCLUDE Dockerfile.rpmbuild
 
 # Do a test install and verify
-# Can be skipped with skiptests=1 in the environment
-# @EXEC [ "$skiptests" = "" ] && include Dockerfile.rpmtest
+# Can be skipped with skippackagetest=1 in the environment
+@EXEC [ "$skippackagetest" = "" ] && include Dockerfile.rpmtest
diff --git a/builder-support/dockerfiles/Dockerfile.target.el-9 b/builder-support/dockerfiles/Dockerfile.target.el-9
index c5766a84e4..5e35f9193c 100644
--- a/builder-support/dockerfiles/Dockerfile.target.el-9
+++ b/builder-support/dockerfiles/Dockerfile.target.el-9
@@ -22,5 +22,5 @@ RUN touch /var/lib/rpm/* && dnf install -y https://dl.fedoraproject.org/pub/epel
 @INCLUDE Dockerfile.rpmbuild
 
 # Do a test install and verify
-# Can be skipped with skiptests=1 in the environment
-# @EXEC [ "$skiptests" = "" ] && include Dockerfile.rpmtest
+# Can be skipped with skippackagetest=1 in the environment
+@EXEC [ "$skippackagetest" = "" ] && include Dockerfile.rpmtest
diff --git a/builder-support/dockerfiles/Dockerfile.target.oraclelinux-8 b/builder-support/dockerfiles/Dockerfile.target.oraclelinux-8
index 63dac11e52..923759aa8a 100644
--- a/builder-support/dockerfiles/Dockerfile.target.oraclelinux-8
+++ b/builder-support/dockerfiles/Dockerfile.target.oraclelinux-8
@@ -22,5 +22,5 @@ RUN touch /var/lib/rpm/* && dnf install -y https://dl.fedoraproject.org/pub/epel
 @INCLUDE Dockerfile.rpmbuild
 
 # Do a test install and verify
-# Can be skipped with skiptests=1 in the environment
-# @EXEC [ "$skiptests" = "" ] && include Dockerfile.rpmtest
+# Can be skipped with skippackagetest=1 in the environment
+@EXEC [ "$skippackagetest" = "" ] && include Dockerfile.rpmtest
diff --git a/builder-support/helpers/generate-sbom-dnf.py b/builder-support/helpers/generate-sbom-dnf.py
new file mode 100755
index 0000000000..6ba91e5cc4
--- /dev/null
+++ b/builder-support/helpers/generate-sbom-dnf.py
@@ -0,0 +1,159 @@
+#!/usr/bin/env python3
+"""
+This script uses dnf to generate a Software Bill of Materials
+(SBOM) in CycloneDX Protocol Buffer format.
+"""
+import datetime
+import dnf
+import json
+import os
+import sys
+import uuid
+
+def licenseToSPDXIdentifier(licenseName):
+    licenseMap = {
+        'BSD': 'BSD-3-Clause',
+        'GPLv2': 'GPL-2.0-only',
+        'GPLv2+': 'GPL-2.0-or-later',
+        'LGPLv2+': 'LGPL-2.0-or-later',
+        'MIT': 'MIT',
+        'OpenLDAP': 'OLDAP-2.8',
+        }
+    if licenseName in licenseMap:
+        return licenseMap[licenseName]
+    return None
+
+def getPackageDatabase():
+    with dnf.Base() as db:
+        conf = db.conf
+        conf.installroot = '/'
+        conf.substitutions.update_from_etc('/')
+        db.read_all_repos()
+
+        db.fill_sack(load_system_repo='auto', load_available_repos=True)
+        query = db.sack.query()
+        return query.installed()
+
+def getPackageInformations(pkgDB, packageName):
+    matches = pkgDB.filter(name=packageName).run()
+    if len(matches) == 0:
+        print(f'-> Package {packageName} not found')
+        return None
+    return matches[0]
+
+def addDependencyToSBOM(sbom, appInfos, pkg):
+    bomRef = 'lib:' + pkg.name
+    component = { 'name': pkg.name, 'bom-ref': bomRef, 'type': 'library'}
+    if pkg.release:
+        component['version'] = (pkg.version if pkg.epoch == 0 else str(pkg.epoch) + ':' + pkg.version) + '-' + pkg.release
+    else:
+        component['version'] = (pkg.version if pkg.epoch == 0 else str(pkg.epoch) + ':' + pkg.version)
+    if hasattr(pkg, 'vendor') and pkg.vendor is not None:
+        component['supplier'] = {'name': pkg.vendor}
+    if hasattr(pkg, 'publisher') and pkg.publisher is not None:
+        component['publisher'] = pkg.publisher
+    spdxLicense = licenseToSPDXIdentifier(pkg.license)
+    if spdxLicense is None:
+        component['licenses'] = [{'license': {'name': pkg.license}}]
+    else:
+        component['licenses'] = [{'license': {'id': spdxLicense}}]
+    if hasattr(pkg, 'sha256') and pkg.sha256 is not None:
+        component['hashes'] = [{'alg': 'SHA-256', 'content': pkg.sha256}]
+
+    sbom['components'].append(component)
+
+def processDependencies(pkgDB, sbom, appInfos, depRelations):
+    seenDeps = {}
+    for require in appInfos.requires:
+        depName = require.name.split('(')[0]
+        if depName in ['/bin/sh', 'config', 'ld-linux-x86-64.so.2', 'rpmlib', 'rtld']:
+            continue
+        if depName in seenDeps:
+            continue
+        seenDeps[depName] = True
+
+        matches = pkgDB.filter(name=depName).run()
+        if len(matches) == 0:
+            flags = []
+            matches = pkgDB.filter(*flags, provides__glob=[require.name]).run()
+            if len(matches) == 0:
+                print(f'Unable to find a match for {depName}')
+                continue
+        if len(matches) > 1:
+            print(f'Got {len(matches)} matches for {depName}')
+
+        dep = matches[0]
+        depRef = 'lib:' + dep.name
+        if depRef in seenDeps:
+            continue
+        seenDeps[depRef] = True
+
+        addDependencyToSBOM(sbom, appInfos, dep)
+        depRelations['pkg:' + appInfos.name].append(depRef)
+
+class StaticLibDep(object):
+    pass
+
+def processAdditionalDependencies(sbom, appInfos, additionalDeps, depRelations):
+    for additionalDepFile in additionalDeps:
+        with open(additionalDepFile) as depDataFile:
+            depData = json.load(depDataFile)
+            pkg = StaticLibDep()
+            pkg.name = os.path.splitext(os.path.basename(additionalDepFile))[0]
+            pkg.version = depData['version']
+            pkg.epoch = 0
+            pkg.release = None
+            pkg.supplier = 'PowerDNS.COM BV'
+            if 'license' in depData:
+                pkg.license = depData['license']
+            if 'publisher' in depData:
+                pkg.publisher = depData['publisher']
+            if 'SHA256SUM' in depData:
+                pkg.sha256 = depData['SHA256SUM']
+            elif 'SHA256SUM_x86_64' in depData:
+                pkg.sha256 = depData['SHA256SUM_x86_64']
+
+            depRef = 'lib:' + pkg.name
+            addDependencyToSBOM(sbom, appInfos, pkg)
+            depRelations['pkg:' + appInfos.name].append(depRef)
+
+def generateSBOM(packageName, additionalDeps):
+    sbom = { 'bomFormat': 'CycloneDX', 'specVersion': '1.5', 'version': 1 }
+    sbom['serialNumber'] = 'urn:uuid:' + str(uuid.uuid4())
+    depRelations = {}
+
+    pkgDB = getPackageDatabase()
+    appName = packageName
+    appInfos = getPackageInformations(pkgDB, packageName)
+    component = { 'name': appName, 'bom-ref': 'pkg:' + appName, 'type': 'application'}
+    component['version'] = appInfos.version
+    component['supplier'] = {'name': appInfos.vendor if appInfos.vendor != '<NULL>' else 'PowerDNS.COM BV', 'url': ['https://www.powerdns.com']}
+    component['licenses'] = [{'license': {'id': licenseToSPDXIdentifier(appInfos.license)}}]
+    depRelations['pkg:' + appName] = []
+
+    sbom['metadata'] = { 'timestamp': datetime.datetime.now(tz=datetime.timezone.utc).isoformat(),
+                         'authors': [{'name': 'PowerDNS.COM BV'}],
+                         'component': component }
+    sbom['components'] = []
+    sbom['dependencies'] = []
+
+    processDependencies(pkgDB, sbom, appInfos, depRelations)
+    processAdditionalDependencies(sbom, appInfos, additionalDeps, depRelations)
+
+    for pkg, deps in depRelations.items():
+        sbom['dependencies'].append({'ref': pkg, 'dependsOn': deps})
+
+    return sbom
+
+if __name__ == "__main__":
+    if len(sys.argv) < 3:
+        sys.exit('Usage: %s <output file> <package> [static dependencies ...]' % (sys.argv[0]))
+
+    staticDeps = []
+    if len(sys.argv) > 3:
+        staticDeps = sys.argv[3:]
+
+    sbom = generateSBOM(sys.argv[2], staticDeps)
+
+    with open(sys.argv[1], "w") as f:
+        f.write(json.dumps(sbom))