]> git.ipfire.org Git - thirdparty/Python/cpython.git/commitdiff
gh-114250: Fetch metadata for pip and its vendored dependencies from PyPI (#114450)
authorSeth Michael Larson <seth@python.org>
Fri, 26 Jan 2024 09:48:13 +0000 (03:48 -0600)
committerGitHub <noreply@github.com>
Fri, 26 Jan 2024 09:48:13 +0000 (09:48 +0000)
Misc/sbom.spdx.json
Tools/build/generate_sbom.py

index 5b3cd04ffa7f74bae0d0f850453c6977c40e4cf6..94566772338b1082b74f8efd2e5ddba77dc3ae32 100644 (file)
       "primaryPackagePurpose": "SOURCE",
       "versionInfo": "2.5.1"
     },
+    {
+      "SPDXID": "SPDXRef-PACKAGE-cachecontrol",
+      "checksums": [
+        {
+          "algorithm": "SHA256",
+          "checksumValue": "95dedbec849f46dda3137866dc28b9d133fc9af55f5b805ab1291833e4457aa4"
+        }
+      ],
+      "downloadLocation": "https://files.pythonhosted.org/packages/1d/e3/a22348e6226dcd585d5a4b5f0175b3a16dabfd3912cbeb02f321d00e56c7/cachecontrol-0.13.1-py3-none-any.whl",
+      "externalRefs": [
+        {
+          "referenceCategory": "PACKAGE_MANAGER",
+          "referenceLocator": "pkg:pypi/cachecontrol@0.13.1",
+          "referenceType": "purl"
+        }
+      ],
+      "licenseConcluded": "MIT",
+      "name": "cachecontrol",
+      "primaryPackagePurpose": "SOURCE",
+      "versionInfo": "0.13.1"
+    },
+    {
+      "SPDXID": "SPDXRef-PACKAGE-colorama",
+      "checksums": [
+        {
+          "algorithm": "SHA256",
+          "checksumValue": "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"
+        }
+      ],
+      "downloadLocation": "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl",
+      "externalRefs": [
+        {
+          "referenceCategory": "PACKAGE_MANAGER",
+          "referenceLocator": "pkg:pypi/colorama@0.4.6",
+          "referenceType": "purl"
+        }
+      ],
+      "licenseConcluded": "MIT",
+      "name": "colorama",
+      "primaryPackagePurpose": "SOURCE",
+      "versionInfo": "0.4.6"
+    },
+    {
+      "SPDXID": "SPDXRef-PACKAGE-distlib",
+      "checksums": [
+        {
+          "algorithm": "SHA256",
+          "checksumValue": "f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e"
+        }
+      ],
+      "downloadLocation": "https://files.pythonhosted.org/packages/76/cb/6bbd2b10170ed991cf64e8c8b85e01f2fb38f95d1bc77617569e0b0b26ac/distlib-0.3.6-py2.py3-none-any.whl",
+      "externalRefs": [
+        {
+          "referenceCategory": "PACKAGE_MANAGER",
+          "referenceLocator": "pkg:pypi/distlib@0.3.6",
+          "referenceType": "purl"
+        }
+      ],
+      "licenseConcluded": "MIT",
+      "name": "distlib",
+      "primaryPackagePurpose": "SOURCE",
+      "versionInfo": "0.3.6"
+    },
+    {
+      "SPDXID": "SPDXRef-PACKAGE-distro",
+      "checksums": [
+        {
+          "algorithm": "SHA256",
+          "checksumValue": "99522ca3e365cac527b44bde033f64c6945d90eb9f769703caaec52b09bbd3ff"
+        }
+      ],
+      "downloadLocation": "https://files.pythonhosted.org/packages/f4/2c/c90a3adaf0ddb70afe193f5ebfb539612af57cffe677c3126be533df3098/distro-1.8.0-py3-none-any.whl",
+      "externalRefs": [
+        {
+          "referenceCategory": "PACKAGE_MANAGER",
+          "referenceLocator": "pkg:pypi/distro@1.8.0",
+          "referenceType": "purl"
+        }
+      ],
+      "licenseConcluded": "MIT",
+      "name": "distro",
+      "primaryPackagePurpose": "SOURCE",
+      "versionInfo": "1.8.0"
+    },
+    {
+      "SPDXID": "SPDXRef-PACKAGE-msgpack",
+      "checksums": [
+        {
+          "algorithm": "SHA256",
+          "checksumValue": "525228efd79bb831cf6830a732e2e80bc1b05436b086d4264814b4b2955b2fa9"
+        }
+      ],
+      "downloadLocation": "https://files.pythonhosted.org/packages/9f/4a/36d936e54cf71e23ad276564465f6a54fb129e3d61520b76e13e0bb29167/msgpack-1.0.5-cp310-cp310-macosx_10_9_universal2.whl",
+      "externalRefs": [
+        {
+          "referenceCategory": "PACKAGE_MANAGER",
+          "referenceLocator": "pkg:pypi/msgpack@1.0.5",
+          "referenceType": "purl"
+        }
+      ],
+      "licenseConcluded": "MIT",
+      "name": "msgpack",
+      "primaryPackagePurpose": "SOURCE",
+      "versionInfo": "1.0.5"
+    },
+    {
+      "SPDXID": "SPDXRef-PACKAGE-packaging",
+      "checksums": [
+        {
+          "algorithm": "SHA256",
+          "checksumValue": "ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"
+        }
+      ],
+      "downloadLocation": "https://files.pythonhosted.org/packages/05/8e/8de486cbd03baba4deef4142bd643a3e7bbe954a784dc1bb17142572d127/packaging-21.3-py3-none-any.whl",
+      "externalRefs": [
+        {
+          "referenceCategory": "PACKAGE_MANAGER",
+          "referenceLocator": "pkg:pypi/packaging@21.3",
+          "referenceType": "purl"
+        }
+      ],
+      "licenseConcluded": "MIT",
+      "name": "packaging",
+      "primaryPackagePurpose": "SOURCE",
+      "versionInfo": "21.3"
+    },
+    {
+      "SPDXID": "SPDXRef-PACKAGE-platformdirs",
+      "checksums": [
+        {
+          "algorithm": "SHA256",
+          "checksumValue": "cec7b889196b9144d088e4c57d9ceef7374f6c39694ad1577a0aab50d27ea28c"
+        }
+      ],
+      "downloadLocation": "https://files.pythonhosted.org/packages/9e/d8/563a9fc17153c588c8c2042d2f0f84a89057cdb1c30270f589c88b42d62c/platformdirs-3.8.1-py3-none-any.whl",
+      "externalRefs": [
+        {
+          "referenceCategory": "PACKAGE_MANAGER",
+          "referenceLocator": "pkg:pypi/platformdirs@3.8.1",
+          "referenceType": "purl"
+        }
+      ],
+      "licenseConcluded": "MIT",
+      "name": "platformdirs",
+      "primaryPackagePurpose": "SOURCE",
+      "versionInfo": "3.8.1"
+    },
+    {
+      "SPDXID": "SPDXRef-PACKAGE-pyparsing",
+      "checksums": [
+        {
+          "algorithm": "SHA256",
+          "checksumValue": "d554a96d1a7d3ddaf7183104485bc19fd80543ad6ac5bdb6426719d766fb06c1"
+        }
+      ],
+      "downloadLocation": "https://files.pythonhosted.org/packages/a4/24/6ae4c9c45cf99d96b06b5d99e25526c060303171fb0aea9da2bfd7dbde93/pyparsing-3.1.0-py3-none-any.whl",
+      "externalRefs": [
+        {
+          "referenceCategory": "PACKAGE_MANAGER",
+          "referenceLocator": "pkg:pypi/pyparsing@3.1.0",
+          "referenceType": "purl"
+        }
+      ],
+      "licenseConcluded": "MIT",
+      "name": "pyparsing",
+      "primaryPackagePurpose": "SOURCE",
+      "versionInfo": "3.1.0"
+    },
+    {
+      "SPDXID": "SPDXRef-PACKAGE-pyproject-hooks",
+      "checksums": [
+        {
+          "algorithm": "SHA256",
+          "checksumValue": "283c11acd6b928d2f6a7c73fa0d01cb2bdc5f07c57a2eeb6e83d5e56b97976f8"
+        }
+      ],
+      "downloadLocation": "https://files.pythonhosted.org/packages/d5/ea/9ae603de7fbb3df820b23a70f6aff92bf8c7770043254ad8d2dc9d6bcba4/pyproject_hooks-1.0.0-py3-none-any.whl",
+      "externalRefs": [
+        {
+          "referenceCategory": "PACKAGE_MANAGER",
+          "referenceLocator": "pkg:pypi/pyproject-hooks@1.0.0",
+          "referenceType": "purl"
+        }
+      ],
+      "licenseConcluded": "MIT",
+      "name": "pyproject-hooks",
+      "primaryPackagePurpose": "SOURCE",
+      "versionInfo": "1.0.0"
+    },
+    {
+      "SPDXID": "SPDXRef-PACKAGE-requests",
+      "checksums": [
+        {
+          "algorithm": "SHA256",
+          "checksumValue": "58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"
+        }
+      ],
+      "downloadLocation": "https://files.pythonhosted.org/packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl",
+      "externalRefs": [
+        {
+          "referenceCategory": "PACKAGE_MANAGER",
+          "referenceLocator": "pkg:pypi/requests@2.31.0",
+          "referenceType": "purl"
+        }
+      ],
+      "licenseConcluded": "MIT",
+      "name": "requests",
+      "primaryPackagePurpose": "SOURCE",
+      "versionInfo": "2.31.0"
+    },
+    {
+      "SPDXID": "SPDXRef-PACKAGE-certifi",
+      "checksums": [
+        {
+          "algorithm": "SHA256",
+          "checksumValue": "92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"
+        }
+      ],
+      "downloadLocation": "https://files.pythonhosted.org/packages/4c/dd/2234eab22353ffc7d94e8d13177aaa050113286e93e7b40eae01fbf7c3d9/certifi-2023.7.22-py3-none-any.whl",
+      "externalRefs": [
+        {
+          "referenceCategory": "PACKAGE_MANAGER",
+          "referenceLocator": "pkg:pypi/certifi@2023.7.22",
+          "referenceType": "purl"
+        }
+      ],
+      "licenseConcluded": "MIT",
+      "name": "certifi",
+      "primaryPackagePurpose": "SOURCE",
+      "versionInfo": "2023.7.22"
+    },
+    {
+      "SPDXID": "SPDXRef-PACKAGE-chardet",
+      "checksums": [
+        {
+          "algorithm": "SHA256",
+          "checksumValue": "362777fb014af596ad31334fde1e8c327dfdb076e1960d1694662d46a6917ab9"
+        }
+      ],
+      "downloadLocation": "https://files.pythonhosted.org/packages/74/8f/8fc49109009e8d2169d94d72e6b1f4cd45c13d147ba7d6170fb41f22b08f/chardet-5.1.0-py3-none-any.whl",
+      "externalRefs": [
+        {
+          "referenceCategory": "PACKAGE_MANAGER",
+          "referenceLocator": "pkg:pypi/chardet@5.1.0",
+          "referenceType": "purl"
+        }
+      ],
+      "licenseConcluded": "MIT",
+      "name": "chardet",
+      "primaryPackagePurpose": "SOURCE",
+      "versionInfo": "5.1.0"
+    },
+    {
+      "SPDXID": "SPDXRef-PACKAGE-idna",
+      "checksums": [
+        {
+          "algorithm": "SHA256",
+          "checksumValue": "90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"
+        }
+      ],
+      "downloadLocation": "https://files.pythonhosted.org/packages/fc/34/3030de6f1370931b9dbb4dad48f6ab1015ab1d32447850b9fc94e60097be/idna-3.4-py3-none-any.whl",
+      "externalRefs": [
+        {
+          "referenceCategory": "PACKAGE_MANAGER",
+          "referenceLocator": "pkg:pypi/idna@3.4",
+          "referenceType": "purl"
+        }
+      ],
+      "licenseConcluded": "MIT",
+      "name": "idna",
+      "primaryPackagePurpose": "SOURCE",
+      "versionInfo": "3.4"
+    },
+    {
+      "SPDXID": "SPDXRef-PACKAGE-rich",
+      "checksums": [
+        {
+          "algorithm": "SHA256",
+          "checksumValue": "8f87bc7ee54675732fa66a05ebfe489e27264caeeff3728c945d25971b6485ec"
+        }
+      ],
+      "downloadLocation": "https://files.pythonhosted.org/packages/fc/1e/482e5eec0b89b593e81d78f819a9412849814e22225842b598908e7ac560/rich-13.4.2-py3-none-any.whl",
+      "externalRefs": [
+        {
+          "referenceCategory": "PACKAGE_MANAGER",
+          "referenceLocator": "pkg:pypi/rich@13.4.2",
+          "referenceType": "purl"
+        }
+      ],
+      "licenseConcluded": "MIT",
+      "name": "rich",
+      "primaryPackagePurpose": "SOURCE",
+      "versionInfo": "13.4.2"
+    },
+    {
+      "SPDXID": "SPDXRef-PACKAGE-pygments",
+      "checksums": [
+        {
+          "algorithm": "SHA256",
+          "checksumValue": "db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1"
+        }
+      ],
+      "downloadLocation": "https://files.pythonhosted.org/packages/34/a7/37c8d68532ba71549db4212cb036dbd6161b40e463aba336770e80c72f84/Pygments-2.15.1-py3-none-any.whl",
+      "externalRefs": [
+        {
+          "referenceCategory": "PACKAGE_MANAGER",
+          "referenceLocator": "pkg:pypi/pygments@2.15.1",
+          "referenceType": "purl"
+        }
+      ],
+      "licenseConcluded": "MIT",
+      "name": "pygments",
+      "primaryPackagePurpose": "SOURCE",
+      "versionInfo": "2.15.1"
+    },
+    {
+      "SPDXID": "SPDXRef-PACKAGE-typing-extensions",
+      "checksums": [
+        {
+          "algorithm": "SHA256",
+          "checksumValue": "440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"
+        }
+      ],
+      "downloadLocation": "https://files.pythonhosted.org/packages/ec/6b/63cc3df74987c36fe26157ee12e09e8f9db4de771e0f3404263117e75b95/typing_extensions-4.7.1-py3-none-any.whl",
+      "externalRefs": [
+        {
+          "referenceCategory": "PACKAGE_MANAGER",
+          "referenceLocator": "pkg:pypi/typing_extensions@4.7.1",
+          "referenceType": "purl"
+        }
+      ],
+      "licenseConcluded": "MIT",
+      "name": "typing_extensions",
+      "primaryPackagePurpose": "SOURCE",
+      "versionInfo": "4.7.1"
+    },
+    {
+      "SPDXID": "SPDXRef-PACKAGE-resolvelib",
+      "checksums": [
+        {
+          "algorithm": "SHA256",
+          "checksumValue": "d2da45d1a8dfee81bdd591647783e340ef3bcb104b54c383f70d422ef5cc7dbf"
+        }
+      ],
+      "downloadLocation": "https://files.pythonhosted.org/packages/d2/fc/e9ccf0521607bcd244aa0b3fbd574f71b65e9ce6a112c83af988bbbe2e23/resolvelib-1.0.1-py2.py3-none-any.whl",
+      "externalRefs": [
+        {
+          "referenceCategory": "PACKAGE_MANAGER",
+          "referenceLocator": "pkg:pypi/resolvelib@1.0.1",
+          "referenceType": "purl"
+        }
+      ],
+      "licenseConcluded": "MIT",
+      "name": "resolvelib",
+      "primaryPackagePurpose": "SOURCE",
+      "versionInfo": "1.0.1"
+    },
+    {
+      "SPDXID": "SPDXRef-PACKAGE-setuptools",
+      "checksums": [
+        {
+          "algorithm": "SHA256",
+          "checksumValue": "11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f"
+        }
+      ],
+      "downloadLocation": "https://files.pythonhosted.org/packages/c7/42/be1c7bbdd83e1bfb160c94b9cafd8e25efc7400346cf7ccdbdb452c467fa/setuptools-68.0.0-py3-none-any.whl",
+      "externalRefs": [
+        {
+          "referenceCategory": "PACKAGE_MANAGER",
+          "referenceLocator": "pkg:pypi/setuptools@68.0.0",
+          "referenceType": "purl"
+        }
+      ],
+      "licenseConcluded": "MIT",
+      "name": "setuptools",
+      "primaryPackagePurpose": "SOURCE",
+      "versionInfo": "68.0.0"
+    },
+    {
+      "SPDXID": "SPDXRef-PACKAGE-six",
+      "checksums": [
+        {
+          "algorithm": "SHA256",
+          "checksumValue": "8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
+        }
+      ],
+      "downloadLocation": "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl",
+      "externalRefs": [
+        {
+          "referenceCategory": "PACKAGE_MANAGER",
+          "referenceLocator": "pkg:pypi/six@1.16.0",
+          "referenceType": "purl"
+        }
+      ],
+      "licenseConcluded": "MIT",
+      "name": "six",
+      "primaryPackagePurpose": "SOURCE",
+      "versionInfo": "1.16.0"
+    },
+    {
+      "SPDXID": "SPDXRef-PACKAGE-tenacity",
+      "checksums": [
+        {
+          "algorithm": "SHA256",
+          "checksumValue": "2f277afb21b851637e8f52e6a613ff08734c347dc19ade928e519d7d2d8569b0"
+        }
+      ],
+      "downloadLocation": "https://files.pythonhosted.org/packages/e7/b0/c23bd61e1b32c9b96fbca996c87784e196a812da8d621d8d04851f6c8181/tenacity-8.2.2-py3-none-any.whl",
+      "externalRefs": [
+        {
+          "referenceCategory": "PACKAGE_MANAGER",
+          "referenceLocator": "pkg:pypi/tenacity@8.2.2",
+          "referenceType": "purl"
+        }
+      ],
+      "licenseConcluded": "MIT",
+      "name": "tenacity",
+      "primaryPackagePurpose": "SOURCE",
+      "versionInfo": "8.2.2"
+    },
+    {
+      "SPDXID": "SPDXRef-PACKAGE-tomli",
+      "checksums": [
+        {
+          "algorithm": "SHA256",
+          "checksumValue": "939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"
+        }
+      ],
+      "downloadLocation": "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl",
+      "externalRefs": [
+        {
+          "referenceCategory": "PACKAGE_MANAGER",
+          "referenceLocator": "pkg:pypi/tomli@2.0.1",
+          "referenceType": "purl"
+        }
+      ],
+      "licenseConcluded": "MIT",
+      "name": "tomli",
+      "primaryPackagePurpose": "SOURCE",
+      "versionInfo": "2.0.1"
+    },
+    {
+      "SPDXID": "SPDXRef-PACKAGE-truststore",
+      "checksums": [
+        {
+          "algorithm": "SHA256",
+          "checksumValue": "e37a5642ae9fc48caa8f120b6283d77225d600d224965a672c9e8ef49ce4bb4c"
+        }
+      ],
+      "downloadLocation": "https://files.pythonhosted.org/packages/20/56/7811d5439b6a56374f274a8672d8f18b4deadadeb3a9f0c86424b98b6f96/truststore-0.8.0-py3-none-any.whl",
+      "externalRefs": [
+        {
+          "referenceCategory": "PACKAGE_MANAGER",
+          "referenceLocator": "pkg:pypi/truststore@0.8.0",
+          "referenceType": "purl"
+        }
+      ],
+      "licenseConcluded": "MIT",
+      "name": "truststore",
+      "primaryPackagePurpose": "SOURCE",
+      "versionInfo": "0.8.0"
+    },
+    {
+      "SPDXID": "SPDXRef-PACKAGE-webencodings",
+      "checksums": [
+        {
+          "algorithm": "SHA256",
+          "checksumValue": "a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78"
+        }
+      ],
+      "downloadLocation": "https://files.pythonhosted.org/packages/f4/24/2a3e3df732393fed8b3ebf2ec078f05546de641fe1b667ee316ec1dcf3b7/webencodings-0.5.1-py2.py3-none-any.whl",
+      "externalRefs": [
+        {
+          "referenceCategory": "PACKAGE_MANAGER",
+          "referenceLocator": "pkg:pypi/webencodings@0.5.1",
+          "referenceType": "purl"
+        }
+      ],
+      "licenseConcluded": "MIT",
+      "name": "webencodings",
+      "primaryPackagePurpose": "SOURCE",
+      "versionInfo": "0.5.1"
+    },
+    {
+      "SPDXID": "SPDXRef-PACKAGE-urllib3",
+      "checksums": [
+        {
+          "algorithm": "SHA256",
+          "checksumValue": "94a757d178c9be92ef5539b8840d48dc9cf1b2709c9d6b588232a055c524458b"
+        }
+      ],
+      "downloadLocation": "https://files.pythonhosted.org/packages/48/fe/a5c6cc46e9fe9171d7ecf0f33ee7aae14642f8d74baa7af4d7840f9358be/urllib3-1.26.17-py2.py3-none-any.whl",
+      "externalRefs": [
+        {
+          "referenceCategory": "PACKAGE_MANAGER",
+          "referenceLocator": "pkg:pypi/urllib3@1.26.17",
+          "referenceType": "purl"
+        }
+      ],
+      "licenseConcluded": "MIT",
+      "name": "urllib3",
+      "primaryPackagePurpose": "SOURCE",
+      "versionInfo": "1.26.17"
+    },
     {
       "SPDXID": "SPDXRef-PACKAGE-pip",
       "checksums": [
     }
   ],
   "relationships": [
+    {
+      "relatedSpdxElement": "SPDXRef-PACKAGE-cachecontrol",
+      "relationshipType": "DEPENDS_ON",
+      "spdxElementId": "SPDXRef-PACKAGE-pip"
+    },
+    {
+      "relatedSpdxElement": "SPDXRef-PACKAGE-certifi",
+      "relationshipType": "DEPENDS_ON",
+      "spdxElementId": "SPDXRef-PACKAGE-pip"
+    },
+    {
+      "relatedSpdxElement": "SPDXRef-PACKAGE-chardet",
+      "relationshipType": "DEPENDS_ON",
+      "spdxElementId": "SPDXRef-PACKAGE-pip"
+    },
+    {
+      "relatedSpdxElement": "SPDXRef-PACKAGE-colorama",
+      "relationshipType": "DEPENDS_ON",
+      "spdxElementId": "SPDXRef-PACKAGE-pip"
+    },
+    {
+      "relatedSpdxElement": "SPDXRef-PACKAGE-distlib",
+      "relationshipType": "DEPENDS_ON",
+      "spdxElementId": "SPDXRef-PACKAGE-pip"
+    },
+    {
+      "relatedSpdxElement": "SPDXRef-PACKAGE-distro",
+      "relationshipType": "DEPENDS_ON",
+      "spdxElementId": "SPDXRef-PACKAGE-pip"
+    },
+    {
+      "relatedSpdxElement": "SPDXRef-PACKAGE-idna",
+      "relationshipType": "DEPENDS_ON",
+      "spdxElementId": "SPDXRef-PACKAGE-pip"
+    },
+    {
+      "relatedSpdxElement": "SPDXRef-PACKAGE-msgpack",
+      "relationshipType": "DEPENDS_ON",
+      "spdxElementId": "SPDXRef-PACKAGE-pip"
+    },
+    {
+      "relatedSpdxElement": "SPDXRef-PACKAGE-packaging",
+      "relationshipType": "DEPENDS_ON",
+      "spdxElementId": "SPDXRef-PACKAGE-pip"
+    },
+    {
+      "relatedSpdxElement": "SPDXRef-PACKAGE-platformdirs",
+      "relationshipType": "DEPENDS_ON",
+      "spdxElementId": "SPDXRef-PACKAGE-pip"
+    },
+    {
+      "relatedSpdxElement": "SPDXRef-PACKAGE-pygments",
+      "relationshipType": "DEPENDS_ON",
+      "spdxElementId": "SPDXRef-PACKAGE-pip"
+    },
+    {
+      "relatedSpdxElement": "SPDXRef-PACKAGE-pyparsing",
+      "relationshipType": "DEPENDS_ON",
+      "spdxElementId": "SPDXRef-PACKAGE-pip"
+    },
+    {
+      "relatedSpdxElement": "SPDXRef-PACKAGE-pyproject-hooks",
+      "relationshipType": "DEPENDS_ON",
+      "spdxElementId": "SPDXRef-PACKAGE-pip"
+    },
+    {
+      "relatedSpdxElement": "SPDXRef-PACKAGE-requests",
+      "relationshipType": "DEPENDS_ON",
+      "spdxElementId": "SPDXRef-PACKAGE-pip"
+    },
+    {
+      "relatedSpdxElement": "SPDXRef-PACKAGE-resolvelib",
+      "relationshipType": "DEPENDS_ON",
+      "spdxElementId": "SPDXRef-PACKAGE-pip"
+    },
+    {
+      "relatedSpdxElement": "SPDXRef-PACKAGE-rich",
+      "relationshipType": "DEPENDS_ON",
+      "spdxElementId": "SPDXRef-PACKAGE-pip"
+    },
+    {
+      "relatedSpdxElement": "SPDXRef-PACKAGE-setuptools",
+      "relationshipType": "DEPENDS_ON",
+      "spdxElementId": "SPDXRef-PACKAGE-pip"
+    },
+    {
+      "relatedSpdxElement": "SPDXRef-PACKAGE-six",
+      "relationshipType": "DEPENDS_ON",
+      "spdxElementId": "SPDXRef-PACKAGE-pip"
+    },
+    {
+      "relatedSpdxElement": "SPDXRef-PACKAGE-tenacity",
+      "relationshipType": "DEPENDS_ON",
+      "spdxElementId": "SPDXRef-PACKAGE-pip"
+    },
+    {
+      "relatedSpdxElement": "SPDXRef-PACKAGE-tomli",
+      "relationshipType": "DEPENDS_ON",
+      "spdxElementId": "SPDXRef-PACKAGE-pip"
+    },
+    {
+      "relatedSpdxElement": "SPDXRef-PACKAGE-truststore",
+      "relationshipType": "DEPENDS_ON",
+      "spdxElementId": "SPDXRef-PACKAGE-pip"
+    },
+    {
+      "relatedSpdxElement": "SPDXRef-PACKAGE-typing-extensions",
+      "relationshipType": "DEPENDS_ON",
+      "spdxElementId": "SPDXRef-PACKAGE-pip"
+    },
+    {
+      "relatedSpdxElement": "SPDXRef-PACKAGE-urllib3",
+      "relationshipType": "DEPENDS_ON",
+      "spdxElementId": "SPDXRef-PACKAGE-pip"
+    },
+    {
+      "relatedSpdxElement": "SPDXRef-PACKAGE-webencodings",
+      "relationshipType": "DEPENDS_ON",
+      "spdxElementId": "SPDXRef-PACKAGE-pip"
+    },
     {
       "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-COPYING",
       "relationshipType": "CONTAINS",
index 317d48fee3a9d4052f44f1f994c0bf59151a4a3d..aceb13f141cba407e2a9adf520fa7e015a915d3e 100644 (file)
@@ -8,6 +8,7 @@ import pathlib
 import subprocess
 import sys
 import typing
+import zipfile
 from urllib.request import urlopen
 
 CPYTHON_ROOT_DIR = pathlib.Path(__file__).parent.parent.parent
@@ -16,10 +17,16 @@ CPYTHON_ROOT_DIR = pathlib.Path(__file__).parent.parent.parent
 # the license expression is a valid SPDX license expression:
 # See: https://spdx.org/licenses
 ALLOWED_LICENSE_EXPRESSIONS = {
-    "MIT",
-    "CC0-1.0",
     "Apache-2.0",
+    "Apache-2.0 OR BSD-2-Clause",
     "BSD-2-Clause",
+    "BSD-3-Clause",
+    "CC0-1.0",
+    "ISC",
+    "LGPL-2.1-only",
+    "MIT",
+    "MPL-2.0",
+    "Python-2.0.1",
 }
 
 # Properties which are required for our purposes.
@@ -31,14 +38,13 @@ REQUIRED_PROPERTIES_PACKAGE = frozenset([
     "checksums",
     "licenseConcluded",
     "externalRefs",
-    "originator",
     "primaryPackagePurpose",
 ])
 
 
 class PackageFiles(typing.NamedTuple):
     """Structure for describing the files of a package"""
-    include: list[str]
+    include: list[str] | None
     exclude: list[str] | None = None
 
 
@@ -118,62 +124,209 @@ def filter_gitignored_paths(paths: list[str]) -> list[str]:
     return sorted([line.split()[-1] for line in git_check_ignore_lines if line.startswith("::")])
 
 
+def fetch_package_metadata_from_pypi(project: str, version: str, filename: str | None = None) -> tuple[str, str] | None:
+    """
+    Fetches the SHA256 checksum and download location from PyPI.
+    If we're given a filename then we match with that, otherwise we use wheels.
+    """
+    # Get pip's download location from PyPI. Check that the checksum is correct too.
+    try:
+        raw_text = urlopen(f"https://pypi.org/pypi/{project}/{version}/json").read()
+        release_metadata = json.loads(raw_text)
+        url: dict[str, typing.Any]
+
+        # Look for a matching artifact filename and then check
+        # its remote checksum to the local one.
+        for url in release_metadata["urls"]:
+            # pip can only use Python-only dependencies, so there's
+            # no risk of picking the 'incorrect' wheel here.
+            if (
+                (filename is None and url["packagetype"] == "bdist_wheel")
+                or (filename is not None and url["filename"] == filename)
+            ):
+                break
+        else:
+            raise ValueError(f"No matching filename on PyPI for '{filename}'")
+
+        # Successfully found the download URL for the matching artifact.
+        download_url = url["url"]
+        checksum_sha256 = url["digests"]["sha256"]
+        return download_url, checksum_sha256
+
+    except (OSError, ValueError) as e:
+        # Fail if we're running in CI where we should have an internet connection.
+        error_if(
+            "CI" in os.environ,
+            f"Couldn't fetch metadata for project '{project}' from PyPI: {e}"
+        )
+        return None
+
+
+def find_ensurepip_pip_wheel() -> pathlib.Path | None:
+    """Try to find the pip wheel bundled in ensurepip. If missing return None"""
+
+    ensurepip_bundled_dir = CPYTHON_ROOT_DIR / "Lib/ensurepip/_bundled"
+
+    pip_wheels = []
+    try:
+        for wheel_filename in os.listdir(ensurepip_bundled_dir):
+            if wheel_filename.startswith("pip-"):
+                pip_wheels.append(wheel_filename)
+            else:
+                print(f"Unexpected wheel in ensurepip: '{wheel_filename}'")
+                sys.exit(1)
+
+    # Ignore this error, likely caused by downstream distributors
+    # deleting the 'ensurepip/_bundled' directory.
+    except FileNotFoundError:
+        pass
+
+    if len(pip_wheels) == 0:
+        return None
+    elif len(pip_wheels) > 1:
+        print("Multiple pip wheels detected in 'Lib/ensurepip/_bundled'")
+        sys.exit(1)
+    # Otherwise return the one pip wheel.
+    return ensurepip_bundled_dir / pip_wheels[0]
+
+
+def maybe_remove_pip_and_deps_from_sbom(sbom_data: dict[str, typing.Any]) -> None:
+    """
+    Removes pip and its dependencies from the SBOM data
+    if the pip wheel is removed from ensurepip. This is done
+    by redistributors of Python and pip.
+    """
+
+    # If there's a wheel we don't remove anything.
+    if find_ensurepip_pip_wheel() is not None:
+        return
+
+    # Otherwise we traverse the relationships
+    # to find dependent packages to remove.
+    sbom_pip_spdx_id = spdx_id("SPDXRef-PACKAGE-pip")
+    sbom_spdx_ids_to_remove = {sbom_pip_spdx_id}
+
+    # Find all package SPDXIDs that pip depends on.
+    for sbom_relationship in sbom_data["relationships"]:
+        if (
+            sbom_relationship["relationshipType"] == "DEPENDS_ON"
+            and sbom_relationship["spdxElementId"] == sbom_pip_spdx_id
+        ):
+            sbom_spdx_ids_to_remove.add(sbom_relationship["relatedSpdxElement"])
+
+    # Remove all the packages and relationships.
+    sbom_data["packages"] = [
+        sbom_package for sbom_package in sbom_data["packages"]
+        if sbom_package["SPDXID"] not in sbom_spdx_ids_to_remove
+    ]
+    sbom_data["relationships"] = [
+        sbom_relationship for sbom_relationship in sbom_data["relationships"]
+        if sbom_relationship["relatedSpdxElement"] not in sbom_spdx_ids_to_remove
+    ]
+
+
 def discover_pip_sbom_package(sbom_data: dict[str, typing.Any]) -> None:
     """pip is a part of a packaging ecosystem (Python, surprise!) so it's actually
     automatable to discover the metadata we need like the version and checksums
-    so let's do that on behalf of our friends at the PyPA.
+    so let's do that on behalf of our friends at the PyPA. This function also
+    discovers vendored packages within pip and fetches their metadata.
     """
     global PACKAGE_TO_FILES
 
-    ensurepip_bundled_dir = CPYTHON_ROOT_DIR / "Lib/ensurepip/_bundled"
-    pip_wheels = []
-
-    # Find the hopefully one pip wheel in the bundled directory.
-    for wheel_filename in os.listdir(ensurepip_bundled_dir):
-        if wheel_filename.startswith("pip-"):
-            pip_wheels.append(wheel_filename)
-    if len(pip_wheels) != 1:
-        print("Zero or multiple pip wheels detected in 'Lib/ensurepip/_bundled'")
-        sys.exit(1)
-    pip_wheel_filename = pip_wheels[0]
+    pip_wheel_filepath = find_ensurepip_pip_wheel()
+    if pip_wheel_filepath is None:
+        return  # There's no pip wheel, nothing to discover.
 
     # Add the wheel filename to the list of files so the SBOM file
     # and relationship generator can work its magic on the wheel too.
     PACKAGE_TO_FILES["pip"] = PackageFiles(
-        include=[f"Lib/ensurepip/_bundled/{pip_wheel_filename}"]
+        include=[str(pip_wheel_filepath.relative_to(CPYTHON_ROOT_DIR))]
     )
 
     # Wheel filename format puts the version right after the project name.
-    pip_version = pip_wheel_filename.split("-")[1]
+    pip_version = pip_wheel_filepath.name.split("-")[1]
     pip_checksum_sha256 = hashlib.sha256(
-        (ensurepip_bundled_dir / pip_wheel_filename).read_bytes()
+        pip_wheel_filepath.read_bytes()
     ).hexdigest()
 
-    # Get pip's download location from PyPI. Check that the checksum is correct too.
-    try:
-        raw_text = urlopen(f"https://pypi.org/pypi/pip/{pip_version}/json").read()
-        pip_release_metadata = json.loads(raw_text)
-        url: dict[str, typing.Any]
+    pip_metadata = fetch_package_metadata_from_pypi(
+        project="pip",
+        version=pip_version,
+        filename=pip_wheel_filepath.name,
+    )
+    # We couldn't fetch any metadata from PyPI,
+    # so we give up on verifying if we're not in CI.
+    if pip_metadata is None:
+        return
+
+    pip_download_url, pip_actual_sha256 = pip_metadata
+    if pip_actual_sha256 != pip_checksum_sha256:
+        raise ValueError("Unexpected")
+
+    # Parse 'pip/_vendor/vendor.txt' from the wheel for sub-dependencies.
+    with zipfile.ZipFile(pip_wheel_filepath) as whl:
+        vendor_txt_data = whl.read("pip/_vendor/vendor.txt").decode()
+
+        # With this version regex we're assuming that pip isn't using pre-releases.
+        # If any version doesn't match we get a failure below, so we're safe doing this.
+        version_pin_re = re.compile(r"^([a-zA-Z0-9_.-]+)==([0-9.]*[0-9])$")
+        sbom_pip_dependency_spdx_ids = set()
+        for line in vendor_txt_data.splitlines():
+            line = line.partition("#")[0].strip()  # Strip comments and whitespace.
+            if not line:  # Skip empty lines.
+                continue
+
+            # Non-empty lines we must be able to match.
+            match = version_pin_re.match(line)
+            error_if(match is None, f"Couldn't parse line from pip vendor.txt: '{line}'")
+            assert match is not None  # Make mypy happy.
+
+            # Parse out and normalize the project name.
+            project_name, project_version = match.groups()
+            project_name = project_name.lower()
+
+            # At this point if pip's metadata fetch succeeded we should
+            # expect this request to also succeed.
+            project_metadata = (
+                fetch_package_metadata_from_pypi(project_name, project_version)
+            )
+            assert project_metadata is not None
+            project_download_url, project_checksum_sha256 = project_metadata
+
+            # Update our SBOM data with what we received from PyPI.
+            # Don't overwrite any existing values.
+            sbom_project_spdx_id = spdx_id(f"SPDXRef-PACKAGE-{project_name}")
+            sbom_pip_dependency_spdx_ids.add(sbom_project_spdx_id)
+            for package in sbom_data["packages"]:
+                if package["SPDXID"] != sbom_project_spdx_id:
+                    continue
 
-        # Look for a matching artifact filename and then check
-        # its remote checksum to the local one.
-        for url in pip_release_metadata["urls"]:
-            if url["filename"] == pip_wheel_filename:
+                # Only thing missing from this blob is the `licenseConcluded`,
+                # that needs to be triaged by human maintainers if the list changes.
+                package.update({
+                    "SPDXID": sbom_project_spdx_id,
+                    "name": project_name,
+                    "versionInfo": project_version,
+                    "downloadLocation": project_download_url,
+                    "checksums": [
+                        {"algorithm": "SHA256", "checksumValue": project_checksum_sha256}
+                    ],
+                    "externalRefs": [
+                        {
+                            "referenceCategory": "PACKAGE_MANAGER",
+                            "referenceLocator": f"pkg:pypi/{project_name}@{project_version}",
+                            "referenceType": "purl",
+                        },
+                    ],
+                    "primaryPackagePurpose": "SOURCE"
+                })
                 break
-        else:
-            raise ValueError(f"No matching filename on PyPI for '{pip_wheel_filename}'")
-        if url["digests"]["sha256"] != pip_checksum_sha256:
-            raise ValueError(f"Local pip checksum doesn't match artifact on PyPI")
-
-        # Successfully found the download URL for the matching artifact.
-        pip_download_url = url["url"]
 
-    except (OSError, ValueError) as e:
-        print(f"Couldn't fetch pip's metadata from PyPI: {e}")
-        sys.exit(1)
+            PACKAGE_TO_FILES[project_name] = PackageFiles(include=None)
 
     # Remove pip from the existing SBOM packages if it's there
     # and then overwrite its entry with our own generated one.
+    sbom_pip_spdx_id = spdx_id("SPDXRef-PACKAGE-pip")
     sbom_data["packages"] = [
         sbom_package
         for sbom_package in sbom_data["packages"]
@@ -181,7 +334,7 @@ def discover_pip_sbom_package(sbom_data: dict[str, typing.Any]) -> None:
     ]
     sbom_data["packages"].append(
         {
-            "SPDXID": spdx_id("SPDXRef-PACKAGE-pip"),
+            "SPDXID": sbom_pip_spdx_id,
             "name": "pip",
             "versionInfo": pip_version,
             "originator": "Organization: Python Packaging Authority",
@@ -205,12 +358,27 @@ def discover_pip_sbom_package(sbom_data: dict[str, typing.Any]) -> None:
             "primaryPackagePurpose": "SOURCE",
         }
     )
+    for sbom_dep_spdx_id in sorted(sbom_pip_dependency_spdx_ids):
+        sbom_data["relationships"].append({
+            "spdxElementId": sbom_pip_spdx_id,
+            "relatedSpdxElement": sbom_dep_spdx_id,
+            "relationshipType": "DEPENDS_ON"
+        })
 
 
 def main() -> None:
     sbom_path = CPYTHON_ROOT_DIR / "Misc/sbom.spdx.json"
     sbom_data = json.loads(sbom_path.read_bytes())
 
+    # Check if pip should be removed if the wheel is missing.
+    # We can't reset the SBOM relationship data until checking this.
+    maybe_remove_pip_and_deps_from_sbom(sbom_data)
+
+    # We regenerate all of this information. Package information
+    # should be preserved though since that is edited by humans.
+    sbom_data["files"] = []
+    sbom_data["relationships"] = []
+
     # Insert pip's SBOM metadata from the wheel.
     discover_pip_sbom_package(sbom_data)
 
@@ -227,9 +395,10 @@ def main() -> None:
             "name" not in package,
             "Package is missing the 'name' field"
         )
+        missing_required_keys = REQUIRED_PROPERTIES_PACKAGE - set(package.keys())
         error_if(
-            set(package.keys()) != REQUIRED_PROPERTIES_PACKAGE,
-            f"Package '{package['name']}' is missing required fields",
+            bool(missing_required_keys),
+            f"Package '{package['name']}' is missing required fields: {missing_required_keys}",
         )
         error_if(
             package["SPDXID"] != spdx_id(f"SPDXRef-PACKAGE-{package['name']}"),
@@ -257,15 +426,11 @@ def main() -> None:
             f"License identifier '{license_concluded}' not in SBOM tool allowlist"
         )
 
-    # Regenerate file information from current data.
-    sbom_files = []
-    sbom_relationships = []
-
     # We call 'sorted()' here a lot to avoid filesystem scan order issues.
     for name, files in sorted(PACKAGE_TO_FILES.items()):
         package_spdx_id = spdx_id(f"SPDXRef-PACKAGE-{name}")
         exclude = files.exclude or ()
-        for include in sorted(files.include):
+        for include in sorted(files.include or ()):
             # Find all the paths and then filter them through .gitignore.
             paths = glob.glob(include, root_dir=CPYTHON_ROOT_DIR, recursive=True)
             paths = filter_gitignored_paths(paths)
@@ -285,7 +450,7 @@ def main() -> None:
                 checksum_sha256 = hashlib.sha256(data).hexdigest()
 
                 file_spdx_id = spdx_id(f"SPDXRef-FILE-{path}")
-                sbom_files.append({
+                sbom_data["files"].append({
                     "SPDXID": file_spdx_id,
                     "fileName": path,
                     "checksums": [
@@ -295,15 +460,13 @@ def main() -> None:
                 })
 
                 # Tie each file back to its respective package.
-                sbom_relationships.append({
+                sbom_data["relationships"].append({
                     "spdxElementId": package_spdx_id,
                     "relatedSpdxElement": file_spdx_id,
                     "relationshipType": "CONTAINS",
                 })
 
     # Update the SBOM on disk
-    sbom_data["files"] = sbom_files
-    sbom_data["relationships"] = sbom_relationships
     sbom_path.write_text(json.dumps(sbom_data, indent=2, sort_keys=True))