fix: bound generic text reads

graphify/detect.py

@@ -309,7 +309,8 @@ def _looks_like_paper(path: Path) -> bool:
     """Heuristic: does this text file read like an academic paper?"""
     try:
         # Only scan first 3000 chars for speed
-        text = path.read_text(encoding="utf-8", errors="ignore")[:3000]
+        with open(_os_path(path), encoding="utf-8", errors="ignore") as f:
+            text = f.read(3000)
         hits = sum(1 for pattern in _PAPER_SIGNALS if pattern.search(text))
         return hits >= _PAPER_SIGNAL_THRESHOLD
     except Exception:
@@ -802,7 +803,16 @@ def count_words(path: Path) -> int:
         if not stat.S_ISREG(os.stat(_os_path(path)).st_mode):
             return 0
         with open(_os_path(path), encoding="utf-8", errors="ignore") as f:
-            return len(f.read().split())
+            words = 0
+            in_word = False
+            while chunk := f.read(64 * 1024):
+                for char in chunk:
+                    if char.isspace():
+                        in_word = False
+                    elif not in_word:
+                        words += 1
+                        in_word = True
+            return words
     except Exception:
         return 0
 

graphify/llm.py

@@ -533,8 +533,9 @@ def _file_to_text(path: Path) -> str:
     """
     if path.suffix.lower() == ".pdf":
         from graphify.detect import extract_pdf_text
-        return extract_pdf_text(path)
-    return path.read_text(encoding="utf-8", errors="replace")
+        return extract_pdf_text(path)[:_FILE_CHAR_CAP]
+    with path.open(encoding="utf-8", errors="replace") as f:
+        return f.read(_FILE_CHAR_CAP)
 
 
 def _resolve_under_root(path: Path, root: Path) -> Path | None:
@@ -2095,7 +2096,7 @@ def _estimate_file_tokens(unit: "Path | FileSlice") -> int:
         return chars // _CHARS_PER_TOKEN
     else:
         try:
-            content = path.read_text(encoding="utf-8", errors="replace")[:_FILE_CHAR_CAP]
+            content = _file_to_text(path)
         except OSError:
             return 0
 

tests/test_detect.py

@@ -1,6 +1,8 @@
 import os
 import subprocess
 import unicodedata
+from io import StringIO
+
 import pytest
 from pathlib import Path
 from graphify.detect import classify_file, count_words, detect, detect_incremental, save_manifest, FileType, _looks_like_paper, _is_ignored, _load_graphifyignore, _is_sensitive
@@ -73,6 +75,41 @@ def test_count_words_sample_md():
     words = count_words(FIXTURES / "sample.md")
     assert words > 5
 
+
+def test_count_words_streams_with_split_semantics(tmp_path, monkeypatch):
+    path = tmp_path / "words.txt"
+    path.touch()
+    text = "\talpha  beta\n\u2003gamma\r\ndelta "
+    read_sizes = []
+
+    class GuardedReader(StringIO):
+        def read(self, size=-1):
+            read_sizes.append(size)
+            assert size > 0
+            return super().read(size)
+
+    monkeypatch.setattr(detect_mod, "open", lambda *args, **kwargs: GuardedReader(text), raising=False)
+
+    assert count_words(path) == len(text.split())
+    assert read_sizes and all(size > 0 for size in read_sizes)
+
+
+def test_looks_like_paper_reads_only_its_prefix(tmp_path, monkeypatch):
+    path = tmp_path / "paper.md"
+    text = "Abstract. We propose a method for arXiv. " + ("tail " * 1000)
+    read_sizes = []
+
+    class GuardedReader(StringIO):
+        def read(self, size=-1):
+            read_sizes.append(size)
+            assert size == 3000
+            return super().read(size)
+
+    monkeypatch.setattr(detect_mod, "open", lambda *args, **kwargs: GuardedReader(text), raising=False)
+
+    assert _looks_like_paper(path)
+    assert read_sizes == [3000]
+
 def test_detect_finds_fixtures():
     result = detect(FIXTURES)
     assert result["total_files"] >= 2

tests/test_file_slice.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from io import StringIO
 from pathlib import Path
 
 import pytest
@@ -128,6 +129,32 @@ def test_estimate_tokens_for_slice_scales_with_range(tmp_path):
     assert llm._estimate_file_tokens(small) < llm._estimate_file_tokens(big)
 
 
+def test_whole_file_text_paths_read_at_the_char_cap(tmp_path, monkeypatch):
+    path = tmp_path / "large.py"
+    text = "x" * (llm._FILE_CHAR_CAP + 1)
+    read_sizes = []
+
+    class GuardedReader(StringIO):
+        def read(self, size=-1):
+            read_sizes.append(size)
+            assert size == llm._FILE_CHAR_CAP
+            return super().read(size)
+
+    class Tokenizer:
+        def encode(self, text, *, disallowed_special):
+            return [0] * len(text)
+
+    monkeypatch.setattr(Path, "open", lambda self, *args, **kwargs: GuardedReader(text))
+    monkeypatch.setattr(llm, "_TOKENIZER", Tokenizer())
+
+    assert llm._file_to_text(path) == text[:llm._FILE_CHAR_CAP]
+    assert llm._dispatched_source_text([path], tmp_path) == {path.resolve(): text[:llm._FILE_CHAR_CAP]}
+    assert llm._estimate_file_tokens(path) == llm._FILE_CHAR_CAP + (
+        llm._PER_FILE_OVERHEAD_CHARS // llm._CHARS_PER_TOKEN
+    )
+    assert read_sizes == [llm._FILE_CHAR_CAP] * 3
+
+
 def test_partition_keeps_slices_as_text(tmp_path):
     f = tmp_path / "a.md"
     fs = FileSlice(f, 0, 5, 0, 1)