diff --git a/.github/workflows/doxygen.yml b/.github/workflows/doxygen.yml index 4d5cd64..1f8c7cc 100644 --- a/.github/workflows/doxygen.yml +++ b/.github/workflows/doxygen.yml @@ -63,7 +63,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 45 permissions: - contents: read + contents: write pages: write id-token: write environment: @@ -88,6 +88,19 @@ jobs: RELEASE_TAG: ${{ needs.check.outputs.tag }} RELEASE_ID: ${{ needs.check.outputs.release_id }} run: bash scripts/build-docs.sh + - name: Package this publication for durable downloads + env: + GENERATION: ${{ github.run_id }}-${{ github.run_attempt }} + run: python3 scripts/package_docs.py site bundle --generation "$GENERATION" + - name: Publish versioned documentation download + env: + GH_TOKEN: ${{ github.token }} + GENERATION: ${{ github.run_id }}-${{ github.run_attempt }} + run: | + gh release create "docs-$GENERATION" bundle/docs.tar.gz bundle/manifest.json \ + --repo "$GITHUB_REPOSITORY" --target "$GITHUB_SHA" \ + --prerelease --latest=false --title "Documentation publication $GENERATION" \ + --notes "Documentation bundle for this publication. Discover the active publication through https://fastled.github.io/dox/publication.json." - name: Configure Pages uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5 - name: Upload Pages artifact diff --git a/.gitignore b/.gitignore index 9462490..35cc8c6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ .build/ fastled/ site/ +bundle/ +__pycache__/ diff --git a/README.md b/README.md index 7798357..3550326 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,31 @@ artifact with one-day retention; it is never committed to this repository. The deployed `release.json` records the release ID, tag, source commit, and Doxygen version. No personal token or cross-repository write access is needed. +## Durable documentation downloads + +The active site's [publication.json](https://fastled.github.io/dox/publication.json) +identifies a public, versioned download of that exact publication. Mirrors should +discover downloads through this manifest rather than GitHub's latest-release API. +Its `schema_version` is `1`; `generation` is the GitHub run ID and attempt joined +with a hyphen. The `release` object contains `id`, `tag_name`, `source_sha`, +`doxygen_version`, and the same `generation`. The `archive` object contains the +public `url`, hexadecimal `sha256`, and byte `size` of `docs.tar.gz`. + +Each build creates a `docs-` documentation prerelease containing +`docs.tar.gz` and `manifest.json`. These assets are uploaded before Pages deploys; +the public site selects the generation only after deployment succeeds. Failed +deployments therefore leave the previous publication selected. Consumers should +verify the archive's size and SHA256 before extracting its files and directories. +The archive includes `release.json` with the matching generation and excludes +`publication.json` to avoid a self-referential checksum. The standalone +`manifest.json` and deployed `publication.json` are identical. + +Publication assets remain available beyond the Actions artifact retention period. +Generated HTML and archives are never committed to Git. A forced rebuild of the +same FastLED tag creates a new generation; an unchanged daily check still skips +building and publishing. The publish job uses repository write permission to +create its documentation release; the release check retains read permission. + For a local build on Linux x86-64, install Git, curl, jq, tar, Python 3, and Graphviz, then: ```bash @@ -73,8 +98,8 @@ The successful daily check records a small status snapshot on `automation/release-checks` at most once every 30 days. This provides repository activity before GitHub's 60-day inactivity limit. The audit history contains only JSON and never triggers a documentation build or deployment. The audit job -alone has repository write permission; release checking and building keep their -existing permissions. +has repository write permission, as does the separate publish job for its public +download release. Release checking retains read permission. If GitHub disables the schedule after a prolonged outage, open the workflow in Actions and choose **Enable workflow**, then run it manually. See diff --git a/scripts/package_docs.py b/scripts/package_docs.py new file mode 100644 index 0000000..ad49cfe --- /dev/null +++ b/scripts/package_docs.py @@ -0,0 +1,76 @@ +"""Package a release site for durable downloads and independent mirrors.""" + +import argparse +import hashlib +import json +import re +import tarfile +from pathlib import Path + + +def package_docs(site, output_dir, generation): + if not re.fullmatch(r"[1-9][0-9]*-[1-9][0-9]*", generation): + raise ValueError( + "generation must be a GitHub run ID and attempt: -" + ) + site = Path(site) + output_dir = Path(output_dir) + if site.is_symlink(): + raise ValueError("site must not be a symlink") + site = site.resolve(strict=True) + output_dir = output_dir.resolve() + if output_dir == site or site in output_dir.parents: + raise ValueError("output directory must be outside the site") + paths = sorted(site.rglob("*")) + for path in paths: + if path.is_symlink(): + raise ValueError(f"site contains a symlink: {path.relative_to(site)}") + if not (path.is_file() or path.is_dir()): + raise ValueError( + f"site contains a non-file entry: {path.relative_to(site)}" + ) + + release = json.loads((site / "release.json").read_text()) + for key in ("id", "tag_name", "source_sha", "doxygen_version"): + if key not in release: + raise ValueError(f"release.json is missing {key}") + release["generation"] = generation + (site / "release.json").write_text(json.dumps(release, indent=2) + "\n") + output_dir.mkdir(parents=True, exist_ok=True) + archive_path = output_dir / "docs.tar.gz" + # Symlinks were rejected above; dereference hard links into regular files. + with tarfile.open(archive_path, "w:gz", dereference=True) as archive: + for path in paths: + if path != site / "publication.json": + archive.add(path, arcname=path.relative_to(site), recursive=False) + digest = hashlib.sha256() + with archive_path.open("rb") as archive: + for chunk in iter(lambda: archive.read(1024 * 1024), b""): + digest.update(chunk) + manifest = { + "schema_version": 1, + "generation": generation, + "release": release, + "archive": { + "url": f"https://github.com/FastLED/dox/releases/download/docs-{generation}/docs.tar.gz", + "sha256": digest.hexdigest(), + "size": archive_path.stat().st_size, + }, + } + content = json.dumps(manifest, indent=2) + "\n" + (output_dir / "manifest.json").write_text(content) + (site / "publication.json").write_text(content) + return manifest + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("site", type=Path) + parser.add_argument("output_dir", type=Path) + parser.add_argument("--generation", required=True) + args = parser.parse_args() + package_docs(args.site, args.output_dir, args.generation) + + +if __name__ == "__main__": + main() diff --git a/scripts/test_package_docs.py b/scripts/test_package_docs.py new file mode 100644 index 0000000..30c1afa --- /dev/null +++ b/scripts/test_package_docs.py @@ -0,0 +1,104 @@ +"""Validate the public publication bundle contract using a small real site.""" + +import hashlib +import json +import os +import tarfile +import tempfile +import unittest +from pathlib import Path + +from package_docs import package_docs + + +class PackageTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.site = Path(self.temp.name) / "site" + self.output = Path(self.temp.name) / "bundle" + self.site.mkdir() + self.release = { + "id": 416153555, + "tag_name": "3.10.8", + "source_sha": "f" * 40, + "doxygen_version": "1.13.2", + } + (self.site / "release.json").write_text(json.dumps(self.release)) + (self.site / "index.html").write_text("

FastLED

") + (self.site / ".nojekyll").touch() + (self.site / "search").mkdir() + (self.site / "search" / "search.js").write_text("// search") + (self.site / "publication.json").write_text('{"old": true}') + + def test_archive_and_manifest_describe_identical_release(self): + manifest = package_docs(self.site, self.output, "12345-2") + self.assertEqual(manifest["schema_version"], 1) + self.assertEqual(manifest["generation"], "12345-2") + self.assertEqual(manifest["release"], dict(self.release, generation="12345-2")) + archive = (self.output / "docs.tar.gz").read_bytes() + self.assertEqual( + manifest["archive"], + { + "url": "https://github.com/FastLED/dox/releases/download/docs-12345-2/docs.tar.gz", + "sha256": hashlib.sha256(archive).hexdigest(), + "size": len(archive), + }, + ) + self.assertEqual( + (self.site / "publication.json").read_bytes(), + (self.output / "manifest.json").read_bytes(), + ) + self.assertEqual( + json.loads((self.output / "manifest.json").read_text()), manifest + ) + with tarfile.open(self.output / "docs.tar.gz") as bundle: + self.assertEqual( + set(bundle.getnames()), + { + ".nojekyll", + "index.html", + "release.json", + "search", + "search/search.js", + }, + ) + self.assertTrue(all(member.isfile() or member.isdir() for member in bundle)) + self.assertEqual( + json.load(bundle.extractfile("release.json")), manifest["release"] + ) + + def test_rejects_unsafe_generation_before_modifying_release(self): + original = (self.site / "release.json").read_bytes() + for generation in ("../123", "1/2", "1-2\n", "1-2;echo", "", "1", "a-2", "0-1"): + with self.subTest(generation=generation), self.assertRaises(ValueError): + package_docs(self.site, self.output, generation) + self.assertEqual((self.site / "release.json").read_bytes(), original) + + def test_rejects_symlinks_including_excluded_manifest(self): + for name in ("linked.html", "publication.json"): + with self.subTest(name=name): + target = self.site / name + if target.exists(): + target.rename(target.with_suffix(".old")) + target.symlink_to(self.site / "index.html") + with self.assertRaisesRegex(ValueError, "symlink"): + package_docs(self.site, self.output, "12345-1") + target.rename(target.with_suffix(".rejected")) + # Move rejected symlinks outside the site before the next case. + target.with_suffix(".rejected").rename(Path(self.temp.name) / name) + + def test_rejects_output_inside_site(self): + with self.assertRaises(ValueError): + package_docs(self.site, self.site / "bundle", "12345-1") + + def test_hardlinks_are_archived_as_regular_files(self): + os.link(self.site / "index.html", self.site / "copy.html") + package_docs(self.site, self.output, "12345-1") + with tarfile.open(self.output / "docs.tar.gz") as bundle: + self.assertTrue(bundle.getmember("index.html").isfile()) + self.assertTrue(bundle.getmember("copy.html").isfile()) + + +if __name__ == "__main__": + unittest.main()