diff --git a/graphify/detect.py b/graphify/detect.py index 1adad00bb..43a0e26d0 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -309,7 +309,8 @@ def _looks_like_paper(path: Path) -> bool: """Heuristic: does this text file read like an academic paper?""" try: # Only scan first 3000 chars for speed - text = path.read_text(encoding="utf-8", errors="ignore")[:3000] + with open(_os_path(path), encoding="utf-8", errors="ignore") as f: + text = f.read(3000) hits = sum(1 for pattern in _PAPER_SIGNALS if pattern.search(text)) return hits >= _PAPER_SIGNAL_THRESHOLD except Exception: @@ -802,7 +803,16 @@ def count_words(path: Path) -> int: if not stat.S_ISREG(os.stat(_os_path(path)).st_mode): return 0 with open(_os_path(path), encoding="utf-8", errors="ignore") as f: - return len(f.read().split()) + words = 0 + in_word = False + while chunk := f.read(64 * 1024): + for char in chunk: + if char.isspace(): + in_word = False + elif not in_word: + words += 1 + in_word = True + return words except Exception: return 0 diff --git a/graphify/llm.py b/graphify/llm.py index 75f637881..294d99acd 100644 --- a/graphify/llm.py +++ b/graphify/llm.py @@ -533,8 +533,9 @@ def _file_to_text(path: Path) -> str: """ if path.suffix.lower() == ".pdf": from graphify.detect import extract_pdf_text - return extract_pdf_text(path) - return path.read_text(encoding="utf-8", errors="replace") + return extract_pdf_text(path)[:_FILE_CHAR_CAP] + with path.open(encoding="utf-8", errors="replace") as f: + return f.read(_FILE_CHAR_CAP) def _resolve_under_root(path: Path, root: Path) -> Path | None: @@ -2095,7 +2096,7 @@ def _estimate_file_tokens(unit: "Path | FileSlice") -> int: return chars // _CHARS_PER_TOKEN else: try: - content = path.read_text(encoding="utf-8", errors="replace")[:_FILE_CHAR_CAP] + content = _file_to_text(path) except OSError: return 0 diff --git a/tests/test_detect.py b/tests/test_detect.py index 1bf6b056b..3e66f9806 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -1,6 +1,8 @@ import os import subprocess import unicodedata +from io import StringIO + import pytest from pathlib import Path from graphify.detect import classify_file, count_words, detect, detect_incremental, save_manifest, FileType, _looks_like_paper, _is_ignored, _load_graphifyignore, _is_sensitive @@ -73,6 +75,41 @@ def test_count_words_sample_md(): words = count_words(FIXTURES / "sample.md") assert words > 5 + +def test_count_words_streams_with_split_semantics(tmp_path, monkeypatch): + path = tmp_path / "words.txt" + path.touch() + text = "\talpha beta\n\u2003gamma\r\ndelta " + read_sizes = [] + + class GuardedReader(StringIO): + def read(self, size=-1): + read_sizes.append(size) + assert size > 0 + return super().read(size) + + monkeypatch.setattr(detect_mod, "open", lambda *args, **kwargs: GuardedReader(text), raising=False) + + assert count_words(path) == len(text.split()) + assert read_sizes and all(size > 0 for size in read_sizes) + + +def test_looks_like_paper_reads_only_its_prefix(tmp_path, monkeypatch): + path = tmp_path / "paper.md" + text = "Abstract. We propose a method for arXiv. " + ("tail " * 1000) + read_sizes = [] + + class GuardedReader(StringIO): + def read(self, size=-1): + read_sizes.append(size) + assert size == 3000 + return super().read(size) + + monkeypatch.setattr(detect_mod, "open", lambda *args, **kwargs: GuardedReader(text), raising=False) + + assert _looks_like_paper(path) + assert read_sizes == [3000] + def test_detect_finds_fixtures(): result = detect(FIXTURES) assert result["total_files"] >= 2 diff --git a/tests/test_file_slice.py b/tests/test_file_slice.py index 619ffc1f4..b4af4d600 100644 --- a/tests/test_file_slice.py +++ b/tests/test_file_slice.py @@ -1,5 +1,6 @@ from __future__ import annotations +from io import StringIO from pathlib import Path import pytest @@ -128,6 +129,32 @@ def test_estimate_tokens_for_slice_scales_with_range(tmp_path): assert llm._estimate_file_tokens(small) < llm._estimate_file_tokens(big) +def test_whole_file_text_paths_read_at_the_char_cap(tmp_path, monkeypatch): + path = tmp_path / "large.py" + text = "x" * (llm._FILE_CHAR_CAP + 1) + read_sizes = [] + + class GuardedReader(StringIO): + def read(self, size=-1): + read_sizes.append(size) + assert size == llm._FILE_CHAR_CAP + return super().read(size) + + class Tokenizer: + def encode(self, text, *, disallowed_special): + return [0] * len(text) + + monkeypatch.setattr(Path, "open", lambda self, *args, **kwargs: GuardedReader(text)) + monkeypatch.setattr(llm, "_TOKENIZER", Tokenizer()) + + assert llm._file_to_text(path) == text[:llm._FILE_CHAR_CAP] + assert llm._dispatched_source_text([path], tmp_path) == {path.resolve(): text[:llm._FILE_CHAR_CAP]} + assert llm._estimate_file_tokens(path) == llm._FILE_CHAR_CAP + ( + llm._PER_FILE_OVERHEAD_CHARS // llm._CHARS_PER_TOKEN + ) + assert read_sizes == [llm._FILE_CHAR_CAP] * 3 + + def test_partition_keeps_slices_as_text(tmp_path): f = tmp_path / "a.md" fs = FileSlice(f, 0, 5, 0, 1)