Modularize GraphRAG smoke runner

scripts/graphrag_smoke/__init__.py

@@ -0,0 +1 @@
+"""GraphRAG Docker smoke modules."""

scripts/graphrag_smoke/benchmark.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+import json
+import subprocess
+from pathlib import Path
+from typing import Any
+
+from .common import rel
+from .context import REPORT_JSON, REPORT_MD, ROOT_DIR
+from .models import StatusState
+
+
+
+def run_scored_report(fixture_path: Path, manifest_path: Path, status: StatusState) -> dict[str, Any]:
+    """Score the generated smoke fixture through the real-world job runner."""
+
+    run_cmd = [
+        "cargo",
+        "run",
+        "-p",
+        "elf-eval",
+        "--bin",
+        "real_world_job_benchmark",
+        "--",
+        "run",
+        "--fixtures",
+        str(fixture_path),
+        "--out",
+        str(REPORT_JSON),
+        "--run-id",
+        "real-world-memory-live-graphrag",
+        "--adapter-id",
+        "graphrag_docker_smoke",
+        "--adapter-name",
+        "GraphRAG Docker smoke adapter",
+        "--adapter-behavior",
+        "docker_python_cli_api_smoke",
+        "--adapter-storage-status",
+        status.setup,
+        "--adapter-runtime-status",
+        status.overall,
+        "--adapter-notes",
+        "Generated by the cost-bounded GraphRAG Docker smoke; pass or wrong_result requires GraphRAG output tables mapped to generated evidence ids, while provider/setup limits remain typed.",
+        "--external-adapter-manifest",
+        str(manifest_path),
+    ]
+    publish_cmd = [
+        "cargo",
+        "run",
+        "-p",
+        "elf-eval",
+        "--bin",
+        "real_world_job_benchmark",
+        "--",
+        "publish",
+        "--report",
+        str(REPORT_JSON),
+        "--out",
+        str(REPORT_MD),
+    ]
+
+    subprocess.run(run_cmd, cwd=ROOT_DIR, check=True)
+    subprocess.run(publish_cmd, cwd=ROOT_DIR, check=True)
+
+    report = json.loads(REPORT_JSON.read_text(encoding="utf-8"))
+
+    return {
+        "json": rel(REPORT_JSON),
+        "markdown": rel(REPORT_MD),
+        "summary": report.get("summary", {}),
+        "suites": report.get("suites", []),
+    }
+
+
+def scored_benchmark(report: dict[str, Any] | None) -> dict[str, Any]:
+    """Extract the post-score benchmark status from a real_world_job report."""
+
+    if report is None:
+        return {
+            "schema": "elf.scored_benchmark_status/v1",
+            "source": "real_world_job_benchmark",
+            "status": "pending",
+            "reason": "The smoke materialization was written before benchmark scoring completed.",
+        }
+
+    summary = report.get("summary", {})
+    counts = {
+        status: int(summary.get(status, 0) or 0)
+        for status in (
+            "pass",
+            "wrong_result",
+            "lifecycle_fail",
+            "incomplete",
+            "blocked",
+            "not_encoded",
+        )
+    }
+    status = next((name for name, count in counts.items() if name != "pass" and count > 0), "pass")
+
+    return {
+        "schema": "elf.scored_benchmark_status/v1",
+        "source": "real_world_job_benchmark",
+        "status": status,
+        "counts": counts,
+        "job_count": int(summary.get("job_count", 0) or 0),
+        "mean_score": summary.get("mean_score"),
+        "evidence_coverage": summary.get("evidence_coverage"),
+    }

scripts/graphrag_smoke/common.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+import json
+import shutil
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from .context import FIXTURE_DIR, LOG_DIR, MANIFEST_OUT, OUT, OUTPUT_CAPTURE_DIR, REPORT_DIR, REPORT_JSON, REPORT_MD, ROOT_DIR, SUMMARY_OUT, WORK_DIR
+
+
+
+def utc_now() -> str:
+    """Return an RFC3339 UTC timestamp."""
+
+    return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
+
+
+def rel(path: Path) -> str:
+    """Return a repository-relative path when possible."""
+
+    try:
+        return str(path.resolve().relative_to(ROOT_DIR))
+    except ValueError:
+        return str(path)
+
+
+def mkdirs() -> None:
+    """Create output directories."""
+
+    for path in (REPORT_DIR, WORK_DIR, FIXTURE_DIR, OUTPUT_CAPTURE_DIR, LOG_DIR):
+        path.mkdir(parents=True, exist_ok=True)
+
+
+def write_json(path: Path, payload: Any) -> None:
+    """Write stable, pretty JSON."""
+
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+
+
+def dir_size(path: Path) -> int:
+    """Return total file size for a directory or file."""
+
+    if not path.exists():
+        return 0
+    if path.is_file():
+        return path.stat().st_size
+
+    return sum(item.stat().st_size for item in path.rglob("*") if item.is_file())
+
+
+def file_count(path: Path) -> int:
+    """Return file count for a directory."""
+
+    if not path.exists():
+        return 0
+
+    return sum(1 for item in path.rglob("*") if item.is_file())
+
+
+def command_available(command: str) -> bool:
+    """Return whether a command is on PATH."""
+
+    return shutil.which(command) is not None
+
+
+def slug(value: str) -> str:
+    """Return a small ASCII slug."""
+
+    out: list[str] = []
+    last_dash = False
+
+    for char in value.lower():
+        if char.isascii() and char.isalnum():
+            out.append(char)
+            last_dash = False
+        elif not last_dash and out:
+            out.append("-")
+            last_dash = True
+
+    while out and out[-1] == "-":
+        out.pop()
+
+    return "".join(out) or "item"

scripts/graphrag_smoke/context.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+
+SCRIPT_DIR = Path(__file__).resolve().parent.parent
+ROOT_DIR = SCRIPT_DIR.parent
+REPORT_DIR = Path(
+    os.environ.get(
+        "ELF_GRAPHRAG_SMOKE_REPORT_DIR",
+        ROOT_DIR / "tmp" / "real-world-memory" / "graphrag-smoke",
+    )
+)
+WORK_DIR = Path(os.environ.get("ELF_GRAPHRAG_SMOKE_WORK_DIR", REPORT_DIR / "work"))
+OUT = Path(os.environ.get("ELF_GRAPHRAG_SMOKE_OUT", REPORT_DIR / "graphrag-smoke.json"))
+MANIFEST_OUT = Path(
+    os.environ.get(
+        "ELF_GRAPHRAG_SMOKE_MANIFEST_OUT",
+        REPORT_DIR / "memory_projects_manifest.graphrag-smoke.json",
+    )
+)
+SUMMARY_OUT = Path(os.environ.get("ELF_GRAPHRAG_SMOKE_SUMMARY_OUT", REPORT_DIR / "summary.json"))
+REPORT_JSON = Path(os.environ.get("ELF_GRAPHRAG_SMOKE_REPORT_JSON", REPORT_DIR / "graphrag-report.json"))
+REPORT_MD = Path(os.environ.get("ELF_GRAPHRAG_SMOKE_REPORT_MD", REPORT_DIR / "graphrag-report.md"))
+FIXTURE_DIR = REPORT_DIR / "graphrag-fixtures"
+OUTPUT_CAPTURE_DIR = REPORT_DIR / "graphrag-output"
+LOG_DIR = REPORT_DIR / "logs"
+
+RUN_ID = os.environ.get(
+    "ELF_GRAPHRAG_SMOKE_RUN_ID",
+    f"graphrag-docker-smoke-{datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S')}",
+)
+RUN_LIVE = os.environ.get("ELF_GRAPHRAG_SMOKE_RUN", "0") == "1"
+ALLOW_HOST = os.environ.get("ELF_GRAPHRAG_SMOKE_ALLOW_HOST", "0") == "1"
+INSTALL_GRAPHRAG = os.environ.get("ELF_GRAPHRAG_SMOKE_INSTALL", "1") == "1"
+GRAPH_RAG_VERSION = os.environ.get("ELF_GRAPHRAG_VERSION", "3.1.0")
+GRAPH_RAG_PACKAGE = os.environ.get("ELF_GRAPHRAG_PACKAGE", f"graphrag=={GRAPH_RAG_VERSION}")
+GRAPH_RAG_REF = os.environ.get("ELF_GRAPHRAG_REF", f"pypi:{GRAPH_RAG_PACKAGE}")
+CHAT_MODEL = os.environ.get("ELF_GRAPHRAG_CHAT_MODEL", "gpt-4o-mini")
+EMBEDDING_MODEL = os.environ.get("ELF_GRAPHRAG_EMBEDDING_MODEL", "text-embedding-3-small")
+API_BASE = os.environ.get("ELF_GRAPHRAG_API_BASE", "")
+API_KEY = os.environ.get("ELF_GRAPHRAG_API_KEY", os.environ.get("GRAPHRAG_API_KEY", ""))
+INDEX_METHOD = os.environ.get("ELF_GRAPHRAG_INDEX_METHOD", "fast")
+QUERY_METHOD = os.environ.get("ELF_GRAPHRAG_QUERY_METHOD", "local")
+TIMEOUT_SECONDS = int(os.environ.get("ELF_GRAPHRAG_TIMEOUT_SECONDS", "900"))
+MAX_DOCS = max(1, min(int(os.environ.get("ELF_GRAPHRAG_MAX_DOCS", "3")), 3))
+MAX_INPUT_CHARS = max(400, min(int(os.environ.get("ELF_GRAPHRAG_MAX_INPUT_CHARS", "2400")), 6000))
+
+TABLES = (
+    "documents",
+    "text_units",
+    "communities",
+    "community_reports",
+    "entities",
+    "relationships",
+)

scripts/graphrag_smoke/corpus.py

@@ -0,0 +1,83 @@
+from __future__ import annotations
+
+import csv
+from pathlib import Path
+
+from .common import slug
+from .context import MAX_DOCS, MAX_INPUT_CHARS, REPORT_DIR
+
+
+
+def generated_corpus() -> list[dict[str, str]]:
+    """Return the bounded generated-public corpus."""
+
+    docs = [
+        {
+            "evidence_id": "graphrag-smoke-nova-observatory",
+            "title": "Nova Observatory memo",
+            "text": (
+                "Evidence ID graphrag-smoke-nova-observatory. Nova Observatory "
+                "operates the public Aurora Index review. The Aurora Index links "
+                "skyglow measurements to open weather station readings for civic "
+                "science audits. The GraphRAG smoke must map this source document "
+                "and its text unit back to the Nova Observatory evidence id."
+            ),
+        },
+        {
+            "evidence_id": "graphrag-smoke-aurora-index",
+            "title": "Aurora Index field note",
+            "text": (
+                "Evidence ID graphrag-smoke-aurora-index. The Aurora Index uses "
+                "Nova Observatory calibration notes when explaining why a public "
+                "skyglow reading changed. The GraphRAG smoke must keep the Aurora "
+                "Index source document and text unit evidence id recoverable."
+            ),
+        },
+        {
+            "evidence_id": "graphrag-smoke-stale-trap",
+            "title": "Retired skyglow note",
+            "text": (
+                "Evidence ID graphrag-smoke-stale-trap. Retired note: Nova "
+                "Observatory previously used the obsolete Zenith Ledger. This note "
+                "is a distractor and must not be used as the primary answer."
+            ),
+        },
+    ]
+    trimmed: list[dict[str, str]] = []
+    used_chars = 0
+
+    for doc in docs[:MAX_DOCS]:
+        remaining = MAX_INPUT_CHARS - used_chars
+
+        if remaining <= 0:
+            break
+
+        text = doc["text"][:remaining].strip()
+        used_chars += len(text)
+        trimmed.append({**doc, "text": text})
+
+    return trimmed
+
+
+def write_corpus(project_dir: Path, corpus: list[dict[str, str]]) -> Path:
+    """Write GraphRAG plain text input plus a CSV mapping copy."""
+
+    input_dir = project_dir / "input"
+    input_dir.mkdir(parents=True, exist_ok=True)
+    csv_path = REPORT_DIR / "generated-corpus.csv"
+
+    with csv_path.open("w", newline="", encoding="utf-8") as handle:
+        writer = csv.DictWriter(handle, fieldnames=("evidence_id", "title", "text"))
+        writer.writeheader()
+
+        for item in corpus:
+            writer.writerow(item)
+
+    for item in corpus:
+        file_name = f"{slug(item['evidence_id'])}.txt"
+        (input_dir / file_name).write_text(
+            f"Title: {item['title']}\nEvidence ID: {item['evidence_id']}\n\n{item['text']}\n",
+            encoding="utf-8",
+        )
+
+    return csv_path