From a1466d15420d9e5354d16a4c1d0c6b204a008569 Mon Sep 17 00:00:00 2001 From: Ramesh Padmanabhaiah <22363102+codeforester@users.noreply.github.com> Date: Wed, 28 Oct 2026 00:20:13 +0530 Subject: [PATCH 1/3] security: verify published PyPI artifact bytes --- .github/workflows/package.yml | 27 ++++++++++++- scripts/verify_pypi_metadata.py | 62 ++++++++++++++++++++++++++++++ tests/test_package_workflow.py | 2 + tests/test_verify_pypi_metadata.py | 44 ++++++++++++++++++++- 4 files changed, 132 insertions(+), 3 deletions(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index d47ddb7..7d2571b 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -400,8 +400,31 @@ jobs: with: python-version: "3.13" - - name: Verify PyPI description and version - run: python scripts/verify_pypi_metadata.py --version "${{ needs.build.outputs.version }}" + - name: Download reviewed distributions + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: base-cli-dist-${{ github.run_id }} + path: dist + + - name: Download release metadata + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: base-cli-release-metadata-${{ github.run_id }} + path: dist + + - name: Verify PyPI metadata and published bytes + run: | + python scripts/verify_pypi_metadata.py \ + --version "${{ needs.build.outputs.version }}" \ + --artifacts-dir dist \ + --evidence-output dist/PYPI-BYTE-VERIFICATION.json + + - name: Upload PyPI byte verification evidence + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: base-cli-pypi-byte-evidence-${{ github.run_id }} + path: dist/PYPI-BYTE-VERIFICATION.json + if-no-files-found: error release: name: Create GitHub Release diff --git a/scripts/verify_pypi_metadata.py b/scripts/verify_pypi_metadata.py index f3005db..81ea7ba 100644 --- a/scripts/verify_pypi_metadata.py +++ b/scripts/verify_pypi_metadata.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +import hashlib import json import time import urllib.error @@ -40,6 +41,59 @@ def validate_metadata(metadata: dict[str, Any], version: str, readme: str) -> li return errors +def validate_artifact_hashes(metadata: dict[str, Any], artifacts_dir: Path) -> tuple[list[str], dict[str, Any]]: + """Compare PyPI file hashes with the reviewed distribution and SHA256SUMS.""" + + errors: list[str] = [] + artifacts = sorted((*artifacts_dir.glob("*.whl"), *artifacts_dir.glob("*.tar.gz"))) + observed = {path.name: hashlib.sha256(path.read_bytes()).hexdigest() for path in artifacts} + urls = metadata.get("urls") + if not isinstance(urls, list): + return ["PyPI response has no file URL list"], {} + published: dict[str, str] = {} + for entry in urls: + if not isinstance(entry, dict): + errors.append("PyPI file URL entry is not an object") + continue + filename = entry.get("filename") + digest = (entry.get("digests") or {}).get("sha256") if isinstance(entry.get("digests"), dict) else None + if not isinstance(filename, str) or not isinstance(digest, str): + errors.append("PyPI file URL entry has no filename and SHA-256 digest") + continue + published[filename] = digest + if set(published) != set(observed): + errors.append( + f"PyPI files differ from reviewed artifacts: expected {sorted(observed)!r}, observed {sorted(published)!r}" + ) + for filename, digest in observed.items(): + if published.get(filename) != digest: + errors.append(f"PyPI SHA-256 for {filename} does not match the reviewed artifact") + + checksums_path = artifacts_dir / "SHA256SUMS" + checksums: dict[str, str] = {} + if not checksums_path.is_file(): + errors.append("reviewed SHA256SUMS is missing") + else: + for line in checksums_path.read_text(encoding="utf-8").splitlines(): + fields = line.split(maxsplit=1) + if len(fields) == 2: + checksums[fields[1]] = fields[0] + if checksums != observed: + errors.append("reviewed SHA256SUMS does not match the reviewed artifacts") + + evidence = { + "package": "base-cli", + "version": metadata.get("info", {}).get("version") if isinstance(metadata.get("info"), dict) else None, + "files": [ + {"filename": filename, "sha256": observed[filename], "pypi_sha256": published.get(filename)} + for filename in sorted(observed) + ], + "sha256sums_verified": checksums == observed, + "result": not errors, + } + return errors, evidence + + def _fetch(url: str, attempts: int, delay: float) -> dict[str, Any]: last_error: Exception | None = None for attempt in range(1, attempts + 1): @@ -62,9 +116,17 @@ def main() -> None: parser.add_argument("--readme", type=Path, default=Path("README.md")) parser.add_argument("--attempts", type=int, default=12) parser.add_argument("--delay", type=float, default=5.0) + parser.add_argument("--artifacts-dir", type=Path) + parser.add_argument("--evidence-output", type=Path) args = parser.parse_args() metadata = _fetch(f"https://pypi.org/pypi/base-cli/{args.version}/json", args.attempts, args.delay) errors = validate_metadata(metadata, args.version, args.readme.read_text(encoding="utf-8")) + evidence: dict[str, Any] | None = None + if args.artifacts_dir is not None: + artifact_errors, evidence = validate_artifact_hashes(metadata, args.artifacts_dir) + errors.extend(artifact_errors) + if args.evidence_output is not None and evidence is not None: + args.evidence_output.write_text(json.dumps(evidence, indent=2, sort_keys=True) + "\n", encoding="utf-8") if errors: for error in errors: print(f"PyPI metadata validation failed: {error}") diff --git a/tests/test_package_workflow.py b/tests/test_package_workflow.py index 38dd089..7426579 100644 --- a/tests/test_package_workflow.py +++ b/tests/test_package_workflow.py @@ -63,6 +63,8 @@ def test_package_workflow_gates_tag_publication_on_release_documentation() -> No assert "python scripts/validate_release_docs.py" in workflow assert "scripts/verify_pypi_metadata.py" in workflow assert "name: Verify published PyPI metadata" in workflow + assert "--artifacts-dir dist" in workflow + assert "PYPI-BYTE-VERIFICATION.json" in workflow assert "needs: [build, smoke, provenance, publish, attest, verify_pypi]" in workflow diff --git a/tests/test_verify_pypi_metadata.py b/tests/test_verify_pypi_metadata.py index cd4b515..fc20658 100644 --- a/tests/test_verify_pypi_metadata.py +++ b/tests/test_verify_pypi_metadata.py @@ -1,11 +1,13 @@ from __future__ import annotations +import hashlib import sys import unittest from pathlib import Path +from tempfile import TemporaryDirectory sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from scripts.verify_pypi_metadata import validate_metadata +from scripts.verify_pypi_metadata import validate_artifact_hashes, validate_metadata README = """\ # base-cli @@ -29,6 +31,46 @@ def test_rejects_prepublication_description(self) -> None: errors = validate_metadata(metadata, "1.2.3", README) self.assertTrue(any("pre-publication wording" in error for error in errors)) + def test_verifies_pypi_file_hashes_against_reviewed_artifacts_and_checksums(self) -> None: + with TemporaryDirectory() as directory: + root = Path(directory) + wheel = root / "base_cli-1.2.3-py3-none-any.whl" + sdist = root / "base_cli-1.2.3.tar.gz" + wheel.write_bytes(b"wheel") + sdist.write_bytes(b"sdist") + checksums = "\n".join( + f"{hashlib.sha256(path.read_bytes()).hexdigest()} {path.name}" for path in (wheel, sdist) + ) + (root / "SHA256SUMS").write_text(checksums + "\n", encoding="utf-8") + metadata = { + "info": {"version": "1.2.3"}, + "urls": [ + {"filename": path.name, "digests": {"sha256": hashlib.sha256(path.read_bytes()).hexdigest()}} + for path in (wheel, sdist) + ], + } + errors, evidence = validate_artifact_hashes(metadata, root) + + self.assertEqual(errors, []) + self.assertTrue(evidence["result"]) + + def test_rejects_pypi_hash_mismatch(self) -> None: + with TemporaryDirectory() as directory: + root = Path(directory) + wheel = root / "base_cli-1.2.3-py3-none-any.whl" + wheel.write_bytes(b"wheel") + digest = hashlib.sha256(wheel.read_bytes()).hexdigest() + (root / "SHA256SUMS").write_text(f"{digest} {wheel.name}\n", encoding="utf-8") + metadata = { + "info": {"version": "1.2.3"}, + "urls": [{"filename": wheel.name, "digests": {"sha256": "0" * 64}}], + } + + errors, evidence = validate_artifact_hashes(metadata, root) + + self.assertTrue(any("does not match" in error for error in errors)) + self.assertFalse(evidence["result"]) + if __name__ == "__main__": unittest.main() From 66ede322c7ac0f1f320a06de3ec493246a53c0cc Mon Sep 17 00:00:00 2001 From: Ramesh Padmanabhaiah <22363102+codeforester@users.noreply.github.com> Date: Wed, 28 Oct 2026 07:39:56 +0530 Subject: [PATCH 2/3] ci: retain PyPI verification evidence on failure --- .github/workflows/package.yml | 1 + tests/test_package_workflow.py | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 7d2571b..336de9c 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -420,6 +420,7 @@ jobs: --evidence-output dist/PYPI-BYTE-VERIFICATION.json - name: Upload PyPI byte verification evidence + if: ${{ always() }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: base-cli-pypi-byte-evidence-${{ github.run_id }} diff --git a/tests/test_package_workflow.py b/tests/test_package_workflow.py index 7426579..ce6aedb 100644 --- a/tests/test_package_workflow.py +++ b/tests/test_package_workflow.py @@ -65,6 +65,7 @@ def test_package_workflow_gates_tag_publication_on_release_documentation() -> No assert "name: Verify published PyPI metadata" in workflow assert "--artifacts-dir dist" in workflow assert "PYPI-BYTE-VERIFICATION.json" in workflow + assert "if: ${{ always() }}" in workflow assert "needs: [build, smoke, provenance, publish, attest, verify_pypi]" in workflow From 0d10b6cce06c888077fea589bdcc77a08d5fe730 Mon Sep 17 00:00:00 2001 From: Ramesh Padmanabhaiah <22363102+codeforester@users.noreply.github.com> Date: Wed, 28 Oct 2026 08:26:15 +0530 Subject: [PATCH 3/3] security: preserve PyPI verification evidence --- .github/workflows/package.yml | 2 +- scripts/verify_pypi_metadata.py | 48 +++++++++++++++++------------- tests/test_verify_pypi_metadata.py | 15 ++++++++++ 3 files changed, 43 insertions(+), 22 deletions(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 336de9c..2cf4aba 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -425,7 +425,7 @@ jobs: with: name: base-cli-pypi-byte-evidence-${{ github.run_id }} path: dist/PYPI-BYTE-VERIFICATION.json - if-no-files-found: error + if-no-files-found: ignore release: name: Create GitHub Release diff --git a/scripts/verify_pypi_metadata.py b/scripts/verify_pypi_metadata.py index 81ea7ba..e9c06eb 100644 --- a/scripts/verify_pypi_metadata.py +++ b/scripts/verify_pypi_metadata.py @@ -4,7 +4,6 @@ from __future__ import annotations import argparse -import hashlib import json import time import urllib.error @@ -12,6 +11,11 @@ from pathlib import Path from typing import Any +try: + from .release_metadata_helpers import sha256_file +except ImportError: # pragma: no cover - direct script execution + from release_metadata_helpers import sha256_file + def _normalized(value: str) -> str: return value.replace("\r\n", "\n").strip() @@ -46,26 +50,27 @@ def validate_artifact_hashes(metadata: dict[str, Any], artifacts_dir: Path) -> t errors: list[str] = [] artifacts = sorted((*artifacts_dir.glob("*.whl"), *artifacts_dir.glob("*.tar.gz"))) - observed = {path.name: hashlib.sha256(path.read_bytes()).hexdigest() for path in artifacts} + reviewed = {path.name: sha256_file(path) for path in artifacts} urls = metadata.get("urls") - if not isinstance(urls, list): - return ["PyPI response has no file URL list"], {} published: dict[str, str] = {} - for entry in urls: - if not isinstance(entry, dict): - errors.append("PyPI file URL entry is not an object") - continue - filename = entry.get("filename") - digest = (entry.get("digests") or {}).get("sha256") if isinstance(entry.get("digests"), dict) else None - if not isinstance(filename, str) or not isinstance(digest, str): - errors.append("PyPI file URL entry has no filename and SHA-256 digest") - continue - published[filename] = digest - if set(published) != set(observed): + if not isinstance(urls, list): + errors.append("PyPI response has no file URL list") + else: + for entry in urls: + if not isinstance(entry, dict): + errors.append("PyPI file URL entry is not an object") + continue + filename = entry.get("filename") + digest = (entry.get("digests") or {}).get("sha256") if isinstance(entry.get("digests"), dict) else None + if not isinstance(filename, str) or not isinstance(digest, str): + errors.append("PyPI file URL entry has no filename and SHA-256 digest") + continue + published[filename] = digest + if set(published) != set(reviewed): errors.append( - f"PyPI files differ from reviewed artifacts: expected {sorted(observed)!r}, observed {sorted(published)!r}" + f"PyPI files differ from reviewed artifacts: expected {sorted(reviewed)!r}, observed {sorted(published)!r}" ) - for filename, digest in observed.items(): + for filename, digest in reviewed.items(): if published.get(filename) != digest: errors.append(f"PyPI SHA-256 for {filename} does not match the reviewed artifact") @@ -78,18 +83,19 @@ def validate_artifact_hashes(metadata: dict[str, Any], artifacts_dir: Path) -> t fields = line.split(maxsplit=1) if len(fields) == 2: checksums[fields[1]] = fields[0] - if checksums != observed: + if checksums != reviewed: errors.append("reviewed SHA256SUMS does not match the reviewed artifacts") evidence = { "package": "base-cli", "version": metadata.get("info", {}).get("version") if isinstance(metadata.get("info"), dict) else None, "files": [ - {"filename": filename, "sha256": observed[filename], "pypi_sha256": published.get(filename)} - for filename in sorted(observed) + {"filename": filename, "sha256": reviewed[filename], "pypi_sha256": published.get(filename)} + for filename in sorted(reviewed) ], - "sha256sums_verified": checksums == observed, + "sha256sums_verified": checksums == reviewed, "result": not errors, + "errors": errors, } return errors, evidence diff --git a/tests/test_verify_pypi_metadata.py b/tests/test_verify_pypi_metadata.py index fc20658..043c9a5 100644 --- a/tests/test_verify_pypi_metadata.py +++ b/tests/test_verify_pypi_metadata.py @@ -71,6 +71,21 @@ def test_rejects_pypi_hash_mismatch(self) -> None: self.assertTrue(any("does not match" in error for error in errors)) self.assertFalse(evidence["result"]) + def test_preserves_reviewed_hashes_when_pypi_urls_are_malformed(self) -> None: + with TemporaryDirectory() as directory: + root = Path(directory) + wheel = root / "base_cli-1.2.3-py3-none-any.whl" + wheel.write_bytes(b"wheel") + digest = hashlib.sha256(wheel.read_bytes()).hexdigest() + (root / "SHA256SUMS").write_text(f"{digest} {wheel.name}\n", encoding="utf-8") + + errors, evidence = validate_artifact_hashes({"info": {"version": "1.2.3"}, "urls": {}}, root) + + self.assertTrue(any("no file URL list" in error for error in errors)) + self.assertEqual(evidence["files"], [{"filename": wheel.name, "pypi_sha256": None, "sha256": digest}]) + self.assertEqual(evidence["errors"], errors) + self.assertFalse(evidence["result"]) + if __name__ == "__main__": unittest.main()