fix: bound tar.gz metadata decompression

docs/reference/catalogs.md

@@ -109,6 +109,20 @@ configuration does not mean its sources or components were approved. Run the
 relevant `catalog list` command and inspect every install-allowed source before
 installing or running components from an unfamiliar project.
 
+Shared archive validation limits the decompressed tar.gz stream to 54 MiB,
+including tar headers, padding, and GNU/PAX metadata as well as file contents.
+Each GNU/PAX metadata payload is also capped at 64 KiB before it is read or
+parsed, with a cumulative budget of 1 MiB for all metadata payloads per archive.
+Global PAX payloads have a tighter cumulative budget of 8 KiB per
+archive because their fields are copied into retained members. Metadata chains
+are limited to 32 headers, and negative metadata sizes are rejected.
+These limits apply during format detection and extraction; ordinary member-size
+and total-file-size limits still apply. Packages that exceed these limits must
+be reduced before installation.
+Old GNU sparse archives and GNU sparse 1.0 archives are rejected before their
+separately stored maps are parsed; sparse maps contained in bounded PAX 0.0/0.1
+metadata remain supported.
+
 ## Host a private catalog
 
 1. Review and approve the entries and release artifacts before registering an

src/specify_cli/_download_security.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import gzip
 import io
 import re
 import socket
@@ -32,6 +33,14 @@
 # ``ZipFile`` reads this whole structure into memory. Four MiB leaves roughly
 # 8 KiB of filename/extra/comment metadata for each of the 512 allowed entries.
 MAX_ZIP_CENTRAL_DIRECTORY_BYTES = 4 * 1024 * 1024
+# Include tar headers, padding, and GNU/PAX metadata, not just file payloads.
+MAX_TAR_STREAM_BYTES = MAX_ZIP_TOTAL_BYTES + 4 * 1024 * 1024
+MAX_TAR_METADATA_BYTES = 64 * 1024
+# Per-member metadata dictionaries are retained during archive validation.
+MAX_TAR_TOTAL_METADATA_BYTES = 1 * 1024 * 1024
+# Global fields are copied into every retained member.
+MAX_TAR_GLOBAL_PAX_BYTES = 8 * 1024
+MAX_TAR_METADATA_DEPTH = 32
 READ_CHUNK_SIZE = 64 * 1024
 
 # Tighter ceilings for responses that are read fully into memory and parsed as
@@ -92,6 +101,114 @@
 }
 
 
+class _TarLimitError(tarfile.ReadError):
+    pass
+
+
+def is_archive_safety_error(exc: Exception) -> bool:
+    """Identify a tar safety signal retained by shared error translation."""
+    return isinstance(exc, _TarLimitError) or isinstance(exc.__cause__, _TarLimitError)
+
+
+class _BoundedTarInfo(tarfile.TarInfo):
+    """Check every metadata header, including headers parsed recursively."""
+
+    def _proc_member(self, archive: tarfile.TarFile) -> tarfile.TarInfo:
+        if self.type == tarfile.GNUTYPE_SPARSE:
+            raise _TarLimitError("Unsupported GNU sparse metadata in tar.gz archive")
+        if self.type in (
+            tarfile.GNUTYPE_LONGNAME,
+            tarfile.GNUTYPE_LONGLINK,
+            tarfile.XHDTYPE,
+            tarfile.XGLTYPE,
+            tarfile.SOLARIS_XHDTYPE,
+        ):
+            if self.size < 0:
+                raise _TarLimitError("tar.gz metadata payload has negative size")
+            if self.size > MAX_TAR_METADATA_BYTES:
+                raise _TarLimitError(
+                    "tar.gz metadata payload exceeds maximum size "
+                    f"of {MAX_TAR_METADATA_BYTES} bytes"
+                )
+            if not isinstance(archive, _BoundedTarFile):
+                raise TypeError("Metadata validation requires a bounded tar archive")
+            with archive._metadata_header(self):
+                return super()._proc_member(archive)
+        return super()._proc_member(archive)
+
+    def _proc_gnusparse_10(
+        self,
+        next_member: tarfile.TarInfo,
+        pax_headers: dict[str, str],
+        archive: tarfile.TarFile,
+    ) -> NoReturn:
+        # This map lives in file data, outside the bounded PAX header payload.
+        raise _TarLimitError("Unsupported GNU sparse metadata in tar.gz archive")
+
+
+class _BoundedTarFile(tarfile.TarFile):
+    tarinfo = _BoundedTarInfo
+    _metadata_bytes: int = 0
+    _global_pax_bytes: int = 0
+    _metadata_depth: int = 0
+
+    @contextmanager
+    def _metadata_header(self, member: tarfile.TarInfo) -> Iterator[None]:
+        if self._metadata_depth >= MAX_TAR_METADATA_DEPTH:
+            raise _TarLimitError(
+                "tar.gz metadata chain exceeds maximum depth "
+                f"of {MAX_TAR_METADATA_DEPTH}"
+            )
+        if self._metadata_bytes + member.size > MAX_TAR_TOTAL_METADATA_BYTES:
+            raise _TarLimitError(
+                "tar.gz archive metadata exceeds maximum cumulative size "
+                f"of {MAX_TAR_TOTAL_METADATA_BYTES} bytes"
+            )
+        if member.type == tarfile.XGLTYPE:
+            if self._global_pax_bytes + member.size > MAX_TAR_GLOBAL_PAX_BYTES:
+                raise _TarLimitError(
+                    "tar.gz global PAX metadata exceeds maximum cumulative size "
+                    f"of {MAX_TAR_GLOBAL_PAX_BYTES} bytes"
+                )
+            self._global_pax_bytes += member.size
+        self._metadata_bytes += member.size
+        self._metadata_depth += 1
+        try:
+            yield
+        finally:
+            self._metadata_depth -= 1
+
+
+class _BoundedTarGzipFile(gzip.GzipFile):
+    """Bound decompressed offsets before tarfile reads metadata or skips data."""
+
+    def _check_position(self, position: int) -> None:
+        if position > MAX_TAR_STREAM_BYTES:
+            raise _TarLimitError(
+                "tar.gz archive exceeds maximum decompressed stream size "
+                f"of {MAX_TAR_STREAM_BYTES} bytes"
+            )
+
+    def read(self, size: int = -1) -> bytes:
+        position = super().seek(0, io.SEEK_CUR)
+        if size >= 0:
+            self._check_position(position + size)
+        else:
+            size = MAX_TAR_STREAM_BYTES - position + 1
+        data = super().read(size)
+        self._check_position(position + len(data))
+        return data
+
+    def seek(self, offset: int, whence: int = io.SEEK_SET) -> int:
+        if whence == io.SEEK_CUR:
+            offset += super().seek(0, io.SEEK_CUR)
+        elif whence != io.SEEK_SET:
+            # SEEK_END would decompress the whole stream before checking it.
+            raise io.UnsupportedOperation("Unbounded tar.gz seek")
+        self._check_position(offset)
+        return super().seek(offset)
+
+
 def archive_format_from_name(name: str) -> ArchiveFormat | None:
     """Return the supported archive format declared by a path or URL."""
     try:
@@ -178,15 +295,22 @@ def detect_archive_format(
             is_tar_gz = False
             if is_gzip:
                 try:
-                    with tarfile.open(fileobj=archive_file, mode="r:gz"):
+                    with (
+                        _BoundedTarGzipFile(fileobj=archive_file, mode="rb") as source,
+                        _BoundedTarFile.open(fileobj=source, mode="r:"),
+                    ):
                         is_tar_gz = True
-                except _TAR_DECOMPRESSION_ERRORS:
+                except _TarLimitError:
+                    raise
+                except (*_TAR_DECOMPRESSION_ERRORS, gzip.BadGzipFile):
                     # A truncated gzip stream raises a bare EOFError here rather
                     # than a TarError, so catching only TarError let it escape
                     # this probe as a raw exception instead of leaving
                     # ``is_tar_gz`` False and reporting the format mismatch.
                     pass
             archive_file.seek(0)
+        except _TarLimitError as exc:
+            _raise_from(error_type, str(exc), exc)
         except OSError as exc:
             _raise_from(error_type, f"Invalid archive: {archive_path}", exc)
 
@@ -1087,18 +1211,23 @@ def safe_extract_tar(
     except OSError as exc:
         _raise_from(error_type, f"Invalid tar extraction target: {target_dir}", exc)
 
-    try:
-        if archive_file is not None:
-            archive_file.seek(0)
-        archive = tarfile.open(
-            archive_path if archive_file is None else None,
-            mode="r:gz",
-            fileobj=archive_file,
-        )
-    except (*_TAR_DECOMPRESSION_ERRORS, OSError) as exc:
-        _raise_from(error_type, f"Invalid tar.gz archive: {archive_path}", exc)
+    with ExitStack() as stack:
+        try:
+            if archive_file is not None:
+                archive_file.seek(0)
+            source = stack.enter_context(
+                _BoundedTarGzipFile(
+                    archive_path if archive_file is None else None,
+                    mode="rb",
+                    fileobj=archive_file,
+                )
+            )
+            archive = stack.enter_context(
+                _BoundedTarFile.open(fileobj=source, mode="r:")
+            )
+        except (*_TAR_DECOMPRESSION_ERRORS, OSError) as exc:
+            _raise_from(error_type, f"Invalid tar.gz archive: {archive_path}: {exc}", exc)
 
-    with archive:
         validated: list[tuple[tarfile.TarInfo, str, bool]] = []
         validated_paths: dict[tuple[str, ...], tuple[str, bool]] = {}
         total_size = 0
@@ -1170,7 +1299,7 @@ def safe_extract_tar(
         except (*_TAR_DECOMPRESSION_ERRORS, OSError) as exc:
             _raise_from(
                 error_type,
-                f"Invalid tar.gz archive: {archive_path}",
+                f"Invalid tar.gz archive: {archive_path}: {exc}",
                 exc,
             )
 

src/specify_cli/extensions/_commands.py

@@ -24,6 +24,7 @@
 from .._download_security import (
     archive_format_from_name,
     detect_archive_format,
+    is_archive_safety_error,
     is_https_or_localhost_http,
     read_response_limited,
 )
@@ -254,6 +255,8 @@ def install_extension_from_url(
                 error_type=ExtensionError,
             )
         except ExtensionError as exc:
+            if is_archive_safety_error(exc):
+                raise
             raise ExtensionError(
                 f"{url} did not return a ZIP archive or tar.gz/tgz archive "
                 f"(got {len(archive_data)} bytes). This usually means the request "

src/specify_cli/workflows/_commands.py

@@ -23,6 +23,7 @@
     archive_format_from_name,
     archive_suffix as archive_suffix,
     detect_archive_format,
+    is_archive_safety_error,
     is_https_or_localhost_http,
     is_safe_download_redirect,
     read_response_limited,
@@ -296,7 +297,9 @@ def _sniff_workflow_archive_format(data: bytes):
             Path("workflow-download"),
             archive_file=BytesIO(data),
         )
-    except ValueError:
+    except ValueError as exc:
+        if is_archive_safety_error(exc):
+            raise
         return None
 
 

tests/archive_helpers.py

@@ -0,0 +1,104 @@
+"""Small compressed archives exercising shared tar safety rejections."""
+
+import gzip
+import io
+import tarfile
+
+
+def tar_metadata_record(metadata_type: bytes, size: int, key: str = "comment") -> bytes:
+    info = tarfile.TarInfo("metadata")
+    info.type = metadata_type
+    info.size = size
+    if metadata_type in (tarfile.GNUTYPE_LONGNAME, tarfile.GNUTYPE_LONGLINK):
+        payload = b"file.txt\0"[:size].ljust(size, b"\0")
+    elif size:
+        prefix = f"{size} {key}=".encode()
+        payload = prefix + b"x" * (size - len(prefix) - 1) + b"\n"
+    else:
+        payload = b""
+    return info.tobuf() + payload + b"\0" * (-size % tarfile.BLOCKSIZE)
+
+
+def tar_global_pax_archive(sizes: list[int], *, interleaved: bool) -> bytes:
+    records = []
+    for index, size in enumerate(sizes):
+        records.append(tar_metadata_record(tarfile.XGLTYPE, size, f"field{index}"))
+        if interleaved:
+            records.append(tarfile.TarInfo(f"file{index}.txt").tobuf())
+    if not interleaved:
+        records.append(tarfile.TarInfo("file.txt").tobuf())
+    return gzip.compress(b"".join(records) + b"\0" * tarfile.BLOCKSIZE)
+
+
+def tar_metadata_chain_archive(
+    count: int, metadata_type: bytes = tarfile.XHDTYPE, *, size: int = 32
+) -> bytes:
+    plain = tar_metadata_record(metadata_type, size) * count
+    return gzip.compress(
+        plain + tarfile.TarInfo("file.txt").tobuf() + b"\0" * tarfile.BLOCKSIZE
+    )
+
+
+def tar_negative_metadata_archive(metadata_type: bytes) -> bytes:
+    info = tarfile.TarInfo("metadata")
+    info.type = metadata_type
+    info.size = -512
+    return gzip.compress(
+        info.tobuf(format=tarfile.GNU_FORMAT) + b"x" * 65537
+    )
+
+
+def tar_safety_archive(kind: str) -> bytes:
+    if kind == "total-metadata":
+        return tar_metadata_chain_archive(17, size=65536)
+    if kind == "global-pax":
+        return tar_global_pax_archive([1024] * 9, interleaved=False)
+    if kind == "metadata-chain":
+        return tar_metadata_chain_archive(400)
+    if kind == "negative-metadata":
+        return tar_negative_metadata_archive(tarfile.XHDTYPE)
+    if kind in ("stream", "metadata"):
+        size = 4096 if kind == "stream" else 65537
+        prefix = f"{size} comment=".encode()
+        payload = prefix + b"x" * (size - len(prefix) - 1) + b"\n"
+        info = tarfile.TarInfo("metadata")
+        info.type = tarfile.XHDTYPE
+        info.size = size
+        plain = info.tobuf() + payload + b"\0" * (-size % tarfile.BLOCKSIZE)
+        plain += tarfile.TarInfo("file.txt").tobuf()
+    elif kind == "gnu-sparse":
+        info = tarfile.TarInfo("file.txt")
+        info.type = tarfile.GNUTYPE_SPARSE
+        header = bytearray(info.tobuf(format=tarfile.GNU_FORMAT))
+        header[482] = 1  # The old GNU format stores map extensions after this header.
+        header[483:495] = tarfile.itn(2, 12)
+        header[148:156] = b" " * 8
+        header[148:156] = f"{sum(header):06o}\0 ".encode()
+        blocks = []
+        for index in range(160):
+            block = bytearray(tarfile.BLOCKSIZE)
+            for entry in range(21):
+                offset = entry * 24
+                block[offset:offset + 24] = tarfile.itn(1, 12) * 2
+            block[504] = int(index < 159)
+            blocks.append(bytes(block))
+        plain = bytes(header) + b"".join(blocks)
+    elif kind == "pax-sparse-1.0":
+        sparse_map = (b"20000\n" + b"1\n1\n" * 20000)
+        payload = sparse_map.ljust(
+            (len(sparse_map) + 511) // 512 * 512, b"\0"
+        )
+        info = tarfile.TarInfo("file.txt")
+        info.size = len(payload)
+        info.pax_headers = {
+            "GNU.sparse.major": "1",
+            "GNU.sparse.minor": "0",
+            "GNU.sparse.realsize": "2",
+        }
+        buffer = io.BytesIO()
+        with tarfile.open(fileobj=buffer, mode="w", format=tarfile.PAX_FORMAT) as archive:
+            archive.addfile(info, io.BytesIO(payload))
+        plain = buffer.getvalue()
+    else:
+        raise ValueError(f"Unknown test archive kind: {kind}")
+    return gzip.compress(plain + b"\0" * tarfile.BLOCKSIZE)