Add Handbook.md benchmark tasks

2026-06-24 12:44:34 -07:00
commit 72f9a65917
1800 changed files with 323589 additions and 0 deletions
@@ -0,0 +1,20 @@
+[build-system]
+build-backend = "hatchling.build"
+requires = [ "hatchling" ]
+
+[project]
+name = "read-file-safe"
+version = "0.0.1"
+description = """\
+  Safe file reading: binary detection, incremental UTF-8 decode with lossy fallback, byte-offset pagination — never \
+  raises\
+  """
+requires-python = ">=3.13"
+classifiers = [
+  "Programming Language :: Python :: 3 :: Only",
+  "Programming Language :: Python :: 3.13",
+  "Programming Language :: Python :: 3.14",
+]
+dependencies = [
+  "pydantic>=2,<3",
+]
@@ -0,0 +1,17 @@
+from .core import (
+    DEFAULT_READ_LIMIT_BYTES,
+    HEADER_SNIFF_BYTES,
+    ReadFileSafeResult,
+    assemble_read_result,
+    error_result,
+    read_file_safe,
+)
+
+__all__ = [
+    "DEFAULT_READ_LIMIT_BYTES",
+    "HEADER_SNIFF_BYTES",
+    "ReadFileSafeResult",
+    "assemble_read_result",
+    "error_result",
+    "read_file_safe",
+]
@@ -0,0 +1,158 @@
+"""Safe file reading: binary format detection, incremental UTF-8 decode with lossy fallback,
+and byte-offset pagination. Never raises — all errors are returned as structured dicts."""
+
+from __future__ import annotations
+
+import codecs
+import os
+from pathlib import Path
+from typing import TypedDict
+
+
+class ReadFileSafeResult(TypedDict):
+    content: str
+    returncode: int
+    file_path: str
+    offset: int
+    next_offset: int
+    total_bytes: int
+    truncated: bool
+    warning: str
+    stderr: str
+
+
+DEFAULT_READ_LIMIT_BYTES = 200_000
+HEADER_SNIFF_BYTES = 8_192
+
+_LOSSY_DECODE_WARNING = (
+    "Some bytes could not be decoded as UTF-8 and were replaced with '\\N{REPLACEMENT CHARACTER}' (U+FFFD) "
+    "characters. The file may be binary or use a non-UTF-8 encoding — check the "
+    "content, and consider using readPDF or bash with `file <path>` / `iconv` "
+    "if it looks garbled."
+)
+
+
+def error_result(display_path: str, offset: int, message: str) -> ReadFileSafeResult:
+    return ReadFileSafeResult(
+        content="",
+        returncode=1,
+        file_path=display_path,
+        offset=offset,
+        next_offset=offset,
+        total_bytes=0,
+        truncated=False,
+        warning="",
+        stderr=message,
+    )
+
+
+def _binary_error_from_header(header: bytes) -> str | None:
+    """Return a user-facing error if the header indicates a non-UTF-8 binary format."""
+    if not header:
+        return None
+    if header[:4] in (b"\xff\xfe\x00\x00", b"\x00\x00\xfe\xff"):
+        return (
+            "File is UTF-32 encoded; readFile only supports UTF-8. "
+            "Convert first via bash, e.g.: "
+            "`iconv -f UTF-32 -t UTF-8 <path> > <path>.utf8`"
+        )
+    if header[:2] in (b"\xff\xfe", b"\xfe\xff"):
+        return (
+            "File is UTF-16 encoded; readFile only supports UTF-8. "
+            "Convert first via bash, e.g.: "
+            "`iconv -f UTF-16 -t UTF-8 <path> > <path>.utf8`"
+        )
+    if b"\x00" in header:
+        return (
+            "File appears to be binary (null bytes in header). readFile is text-only. "
+            "Use readPDF for PDFs, or bash with e.g. `xxd` / `file <path>` "
+            "if you need raw bytes or to identify the format."
+        )
+    return None
+
+
+def _decode_utf8_slice(raw: bytes, is_final: bool) -> tuple[str, int, bool]:
+    """Decode one UTF-8 page. Returns (content, bytes_consumed, lossy).
+
+    Two-stage strategy:
+      1. Strict incremental decode — rewinds cleanly at mid-codepoint boundaries.
+      2. Lossy fallback — kicks in when strict fails; invalid bytes become U+FFFD.
+    """
+    if not raw:
+        return "", 0, False
+
+    decoder = codecs.getincrementaldecoder("utf-8")(errors="strict")
+    try:
+        content = decoder.decode(raw, final=is_final)
+        pending, _ = decoder.getstate()
+        consumed = len(raw) - len(pending)
+        if consumed > 0:
+            return content, consumed, False
+    except UnicodeDecodeError:
+        pass
+
+    return raw.decode("utf-8", errors="replace"), len(raw), True
+
+
+def assemble_read_result(
+    display_path: str,
+    total_bytes: int,
+    header: bytes,
+    raw: bytes,
+    start: int,
+    limit: int | None,
+) -> ReadFileSafeResult:
+    """Build a ``ReadFileSafeResult`` from already-read bytes.
+
+    Pure (no I/O): the caller supplies ``total_bytes``, the ``header`` (first
+    ``HEADER_SNIFF_BYTES`` for binary detection) and ``raw`` (the slice at
+    ``start`` of length ``limit``). Split out from :func:`read_file_safe` so a
+    caller that must perform the actual read under a different privilege (e.g.
+    syntara reading as the unprivileged sandbox user) can reuse the decoding /
+    binary-sniff / pagination logic without this module doing the ``open()``.
+    """
+    binary_err = _binary_error_from_header(header)
+    if binary_err:
+        return error_result(display_path, start, binary_err)
+
+    is_final = limit is None or len(raw) < limit
+    content, consumed, lossy = _decode_utf8_slice(raw, is_final)
+    next_offset = start + consumed
+
+    return ReadFileSafeResult(
+        content=content,
+        returncode=0,
+        file_path=display_path,
+        offset=start,
+        next_offset=next_offset,
+        total_bytes=total_bytes,
+        truncated=next_offset < total_bytes,
+        warning=_LOSSY_DECODE_WARNING if lossy else "",
+        stderr="",
+    )
+
+
+def read_file_safe(
+    display_path: str,
+    resolved_path: Path | str,
+    offset: int = 0,
+    limit: int | None = DEFAULT_READ_LIMIT_BYTES,
+) -> ReadFileSafeResult:
+    """Read a file at an already-resolved absolute path.
+
+    `display_path` is used only for reporting (returned as `file_path` in the result).
+    The caller is responsible for path resolution and security checks.
+    """
+    resolved_path = Path(resolved_path)
+    try:
+        total_bytes = os.path.getsize(resolved_path)
+        start = min(offset, total_bytes)
+
+        with open(resolved_path, "rb") as f:
+            header = f.read(min(HEADER_SNIFF_BYTES, total_bytes))
+            f.seek(start)
+            raw = f.read() if limit is None else f.read(limit)
+
+        return assemble_read_result(display_path, total_bytes, header, raw, start, limit)
+    except Exception as e:
+        return error_result(display_path, offset, str(e))
@@ -0,0 +1,193 @@
+"""Tests for the read_file_safe package core functions."""
+
+import os
+import shutil
+import stat
+import tempfile
+import unittest
+from pathlib import Path
+
+from read_file_safe import DEFAULT_READ_LIMIT_BYTES, read_file_safe
+from read_file_safe.core import _binary_error_from_header, _decode_utf8_slice
+
+
+class BinaryHeaderDetectionTests(unittest.TestCase):
+    def test_empty_header_is_not_binary(self):
+        assert _binary_error_from_header(b"") is None
+
+    def test_utf32_le_bom_detected(self):
+        err = _binary_error_from_header(b"\xff\xfe\x00\x00rest")
+        assert err is not None
+        assert "UTF-32" in err
+
+    def test_utf32_be_bom_detected(self):
+        err = _binary_error_from_header(b"\x00\x00\xfe\xffrest")
+        assert err is not None
+        assert "UTF-32" in err
+
+    def test_utf16_le_bom_detected(self):
+        err = _binary_error_from_header(b"\xff\xferest")
+        assert err is not None
+        assert "UTF-16" in err
+
+    def test_utf16_be_bom_detected(self):
+        err = _binary_error_from_header(b"\xfe\xffrest")
+        assert err is not None
+        assert "UTF-16" in err
+
+    def test_null_byte_detected_as_binary(self):
+        err = _binary_error_from_header(b"hello\x00world")
+        assert err is not None
+        assert "binary" in err.lower()
+
+    def test_clean_utf8_passes(self):
+        assert _binary_error_from_header(b"hello world\n") is None
+
+    def test_utf32_takes_priority_over_utf16(self):
+        # UTF-32 LE BOM starts with UTF-16 LE BOM bytes — UTF-32 must win
+        err = _binary_error_from_header(b"\xff\xfe\x00\x00extra")
+        assert err is not None
+        assert "UTF-32" in err
+
+
+class DecodeUtf8SliceTests(unittest.TestCase):
+    def test_empty_bytes_returns_empty(self):
+        content, consumed, lossy = _decode_utf8_slice(b"", is_final=True)
+        assert content == ""
+        assert consumed == 0
+        assert lossy is False
+
+    def test_clean_ascii(self):
+        content, consumed, lossy = _decode_utf8_slice(b"hello", is_final=True)
+        assert content == "hello"
+        assert consumed == 5
+        assert lossy is False
+
+    def test_clean_utf8_multibyte(self):
+        raw = "héllo".encode()
+        content, consumed, lossy = _decode_utf8_slice(raw, is_final=True)
+        assert content == "héllo"
+        assert consumed == len(raw)
+        assert lossy is False
+
+    def test_mid_codepoint_split_rewinds(self):
+        # "é" is 2 bytes (0xc3 0xa9); slice after first byte only
+        raw = b"\xc3"  # incomplete é
+        content, consumed, _lossy = _decode_utf8_slice(raw, is_final=False)
+        # Strict decoder buffers it as pending — consumed == 0, falls through to lossy
+        # OR strict decoder returns 0 consumed and we fall back to lossy
+        # Either way we get output without crashing
+        assert isinstance(content, str)
+        assert isinstance(consumed, int)
+
+    def test_invalid_utf8_falls_back_lossy(self):
+        raw = b"\xff\xfe"  # invalid UTF-8 (not a BOM context here, just bad bytes)
+        content, consumed, lossy = _decode_utf8_slice(raw, is_final=True)
+        assert lossy is True
+        assert consumed == len(raw)
+        assert "�" in content
+
+
+class ReadFileSafeTests(unittest.TestCase):
+    def setUp(self):
+        self.tmpdir = tempfile.mkdtemp(prefix="read-file-safe-test-")
+
+    def tearDown(self):
+        shutil.rmtree(self.tmpdir, ignore_errors=True)
+
+    def _write(self, name: str, content: bytes) -> Path:
+        p = Path(self.tmpdir) / name
+        p.write_bytes(content)
+        return p
+
+    def test_happy_path_small_file(self):
+        p = self._write("hello.txt", b"hello world")
+        result = read_file_safe("hello.txt", p)
+        assert result["returncode"] == 0
+        assert result["content"] == "hello world"
+        assert result["total_bytes"] == 11
+        assert result["truncated"] is False
+        assert result["warning"] == ""
+        assert result["stderr"] == ""
+
+    def test_empty_file(self):
+        p = self._write("empty.txt", b"")
+        result = read_file_safe("empty.txt", p)
+        assert result["returncode"] == 0
+        assert result["content"] == ""
+        assert result["total_bytes"] == 0
+        assert result["truncated"] is False
+
+    def test_nonexistent_file_returns_error(self):
+        result = read_file_safe("missing.txt", Path(self.tmpdir) / "missing.txt")
+        assert result["returncode"] == 1
+        assert result["stderr"] != ""
+
+    def test_directory_as_path_returns_error(self):
+        result = read_file_safe("dir", Path(self.tmpdir))
+        assert result["returncode"] == 1
+
+    def test_binary_file_returns_error(self):
+        p = self._write("bin.dat", b"\x00\x01\x02binary content")
+        result = read_file_safe("bin.dat", p)
+        assert result["returncode"] == 1
+        assert "binary" in result["stderr"].lower()
+
+    def test_utf16_file_returns_error(self):
+        p = self._write("utf16.txt", "hello".encode("utf-16"))
+        result = read_file_safe("utf16.txt", p)
+        assert result["returncode"] == 1
+        assert "UTF-16" in result["stderr"]
+
+    def test_offset_and_limit(self):
+        p = self._write("abc.txt", b"abcdefghij")
+        result = read_file_safe("abc.txt", p, offset=3, limit=4)
+        assert result["content"] == "defg"
+        assert result["offset"] == 3
+        assert result["next_offset"] == 7
+        assert result["truncated"] is True
+
+    def test_null_limit_reads_whole_file(self):
+        p = self._write("full.txt", b"full content")
+        result = read_file_safe("full.txt", p, limit=None)
+        assert result["content"] == "full content"
+        assert result["truncated"] is False
+
+    def test_offset_past_eof_returns_empty(self):
+        p = self._write("short.txt", b"hi")
+        result = read_file_safe("short.txt", p, offset=100)
+        assert result["returncode"] == 0
+        assert result["content"] == ""
+        assert result["truncated"] is False
+
+    def test_lossy_decode_sets_warning(self):
+        # Latin-1 bytes that aren't valid UTF-8
+        p = self._write("latin.txt", bytes(range(0x80, 0x90)))
+        result = read_file_safe("latin.txt", p)
+        assert result["returncode"] == 0
+        assert result["warning"] != ""
+        assert "�" in result["content"]
+
+    def test_display_path_used_in_result(self):
+        p = self._write("real.txt", b"content")
+        result = read_file_safe("/shown/path.txt", p)
+        assert result["file_path"] == "/shown/path.txt"
+
+    def test_default_limit_constant_is_sane(self):
+        assert DEFAULT_READ_LIMIT_BYTES > 0
+        assert DEFAULT_READ_LIMIT_BYTES <= 1_000_000
+
+    @unittest.skipIf(os.getuid() == 0, "root bypasses permission checks")
+    def test_permission_denied_returns_error(self):
+        p = self._write("secret.txt", b"secret")
+        p.chmod(0o000)
+        try:
+            result = read_file_safe("secret.txt", p)
+            assert result["returncode"] == 1
+            assert result["stderr"] != ""
+        finally:
+            p.chmod(stat.S_IRUSR | stat.S_IWUSR)
+
+
+if __name__ == "__main__":
+    unittest.main()