diff --git a/scripts/persistent_eval/README.md b/scripts/persistent_eval/README.md new file mode 100644 index 000000000..5f321c3b3 --- /dev/null +++ b/scripts/persistent_eval/README.md @@ -0,0 +1,109 @@ +# Persistent evaluator experiment + +KernelBot's warm Modal function still starts fresh Python processes for submission +import and evaluation. This experiment measures how much wall time can be saved +by retaining one Python interpreter and CUDA context across trusted submissions. +It does not modify or enable the production runner. + +## Run + +From the repository root, with Modal authentication configured: + +```sh +uv run --no-project --with modal==1.5.5 --with pyyaml python \ + scripts/persistent_eval/launch.py --workload inline \ + --output-dir /tmp/kernelbot-inline-results +``` + +Use `--workload triton` for the other existing vector-add example. The output +directory must not exist. The default is four submissions per batch, two rounds +in fresh/persistent then persistent/fresh order, on one ephemeral T4 with eight +CPU cores and 32 GiB RAM. `--count 1 --rounds 1` makes a smaller smoke test. +Every run also checks correct → incorrect → correct code in one persistent +worker. For CUDA this keeps the same `load_inline(name="add_cuda")` name and +changes addition to subtraction; the middle request must fail correctness. + +The launcher saves complete results, phase output, source hashes, compiled-library +names/hashes, machine metadata, and logs before stopping the sandbox. Result +export is compressed, chunked, and checksummed to avoid provider log-line limits. +Each completed batch is checkpointed separately so its full details can be +recovered from retained provider logs if the local connection drops. +GPU cost estimates exclude CPU/RAM and are approximate, using $0.000164/T4-second. + +## What changes + +Both modes call the repository's `run_config`. The experiment coordinator caches +`SystemInfo` once for both modes. Each submission has a separate working directory +and cold Torch extension, Triton, and CUDA disk caches. For inline CUDA, the normal +preliminary import compiles the extension, and the benchmark request reloads it. +The compiler is limited to two jobs per submission. + +```mermaid +flowchart LR + C[run_config] --> I[Submission import] + I --> B[Benchmark request] + B --> F[Fresh: eval.py plus spawned GPU worker] + B --> P[Persistent: socket to the same Python and CUDA process] + F --> E[Existing correctness and CUDA-event timing bodies] + P --> E +``` + +Fresh mode uses `run_program` unchanged. Persistent mode replaces only that +transport with a Unix socket request, reloads task/reference/submission modules, +and calls the existing `run_benchmarking` body with an in-process `Pool.apply` +adapter. This removes both interpreter startup and the evaluator's spawned GPU +worker. The worker synchronizes CUDA and clears unused allocator cache after +successful requests. GPU measurements are sequential in both modes; submitting +an entire batch at once is not required. The persistent worker owns the sandbox's +assigned GPU; this experiment does not need a GPU admission lock. + +Batch time includes worker startup/teardown, compilation, validation, and all +submission requests, but excludes Modal image/container startup and the once-per- +experiment machine probe. Each request records compile/import and benchmark wall +time separately. Kernel times remain a distinct metric from evaluation latency. + +## Scope and review points + +This is a benchmark prototype for the bundled trusted examples, not isolation +for arbitrary uploaded code. It covers single-GPU benchmark mode only. The +persistent transport does not implement production per-request timeout handling, +CUDA-fault recovery, process-state isolation, or unloading native extensions. +Modules and allocator cleanup do not reset all process state. Python output is +captured per request, while compiler subprocess output remains in the sandbox +log; the existing stdout-based `CompileResult` classification can therefore +differ between modes. Phase timings and generated `.so` files are recorded +separately to verify that compilation happened. A production design +needs bounded worker recycling and scoring consistency checks. The sandbox has +an overall 30-minute timeout and is terminated by the launcher on exit. + +The minimal pinned CUDA 12.8.1 / Torch 2.7.1 image is deliberately consistent with +the earlier experiment; it is not KernelBot's production image. Results on this +small vector-add workload should not be extrapolated to compilation-heavy or +long-running competition tasks without measuring them. + +[Inline CUDA results](RESULTS.md): **19–28% less evaluation wall time**, with +compilation still dominating. Reported kernel medians shifted 12–15% in the +complete repeat, so score equivalence remains unresolved. Earlier separate Triton measurements were 51.3 seconds fresh versus +12.5 seconds persistent per four-submission batch (two rounds). Those results +motivated this experiment; they are not an inline CUDA claim. That experiment +also saw an approximately 8.5% shift in the smallest shape's reported kernel +time, which needs investigation before using persistence for ranked scores. + +## Local checks + +```sh +uv run --no-project --with pytest --with modal==1.5.5 --with pyyaml \ + pytest scripts/persistent_eval/test_experiment.py -q +uv run --no-project --with ruff ruff check . --exclude examples/ --line-length 120 +``` + +To audit a completed run and emit compact shareable JSON: + +```sh +python scripts/persistent_eval/summarize.py /tmp/kernelbot-inline-results/results.json \ + > /tmp/kernelbot-inline-summary.json +``` + +The GPU run supplies the actual correctness/reload and performance evidence. +The CPU tests cover source/config preservation and evidence transport, including +large outputs and checksum rejection. diff --git a/scripts/persistent_eval/RESULTS.md b/scripts/persistent_eval/RESULTS.md new file mode 100644 index 000000000..b68836cbc --- /dev/null +++ b/scripts/persistent_eval/RESULTS.md @@ -0,0 +1,96 @@ +# Inline CUDA: persistence saves 19–28% of evaluation wall time + +Measured on 2026-09-04 using KernelBot's bundled inline-CUDA FP16 vector-add +example, three benchmark shapes (1024², 2048², 4096²), and cold private build +caches for every submission. GPU measurements remain sequential. + +## Turnaround + +Each batch contains four submissions and includes worker startup/teardown. +Compilation and correctness checks are included; Modal image/container startup +and the once-per-experiment machine probe are excluded. + +| Experiment | Fresh processes | Persistent worker | Less wall time | +|---|---:|---:|---:| +| Initial allocation, median of two counterbalanced rounds | 173.8 s | 125.4 s | 27.8% | +| Complete repeat on another allocation, one fresh/persistent round | 151.1 s | 122.2 s | 19.1% | + +That is approximately **29–48 seconds saved per four-submission batch** on this +workload. Each comparison uses the same assigned GPU and CPU resources for both +modes. Absolute times vary across the two allocations; do not pool those times +into a single baseline. This small experiment is not a production workload +estimate or a confidence interval. + +## What still takes time + +Per-request phase medians from the complete repeat: + +| Phase | Fresh processes | Persistent worker | +|---|---:|---:| +| Compile and import submission | 31.74 s | 29.85 s | +| Benchmark request, including validation and setup | 5.89 s | 0.41 s | + +Keeping Python/CUDA alive removes most benchmark-request startup. Rebuilding +C++/CUDA dominates the remaining time. Every inline request produced its own +native library; persistent requests used `add_cuda.so`, then `add_cuda_v1.so`, +`add_cuda_v2.so`, and `add_cuda_v3.so` in separate cold build directories. +The phase medians are individual-request statistics, not additive batch totals. + +## Kernel scores are not yet equivalent + +The same correctness and CUDA-event timing function bodies produced different +reported kernel times. Below are medians of each submission's reported mean in +the complete repeat. These are GPU microseconds, distinct from evaluation wall +seconds above. + +| Shape | Fresh, µs | Persistent, µs | Persistent change | +|---|---:|---:|---:| +| 1024² | 36.02 | 40.28 | +11.8% | +| 2048² | 108.37 | 123.65 | +14.1% | +| 4096² | 407.96 | 468.47 | +14.8% | + +These 12–15% shifts require investigation before using the prototype for ranked +scores. This one ordered repeat does not separate process-state effects from +clock/thermal drift or measurement variability; GPU clocks were not pinned. +There is no claim that the kernel itself became faster or that the two execution +modes are score-equivalent. + +## Correctness and code replacement + +All 24 timed submissions passed across the two experiments. The complete repeat +also ran **correct addition → incorrect subtraction → correct addition** in one +persistent worker, retaining the requested extension name `add_cuda`. Observed +results were **pass → fail → pass**. The incorrect variant compiled successfully +and failed with numerical mismatches on all three shapes; this was not a compiler +or import failure. Restored code produced correct results again. + +The worker still does not isolate arbitrary user code, unload native modules, +implement production request timeouts, or recover from a poisoned CUDA context. +This PR remains an experiment with no production runner changes. + +## Evidence and reproduction + +- [Initial timing observations](results/inline-initial-timings.json): two complete + rounds, recovered from provider logs after a local DNS/connection failure. The + final reload check was interrupted. Full phase/kernel records were not exported, + so this file explicitly marks its limited detail. +- [Complete repeat](results/inline-repeat.json): all per-submission/phase timings, + correctness output, kernel statistics, native-library names/hashes, and a hash + of the full local result. Compiler logs are omitted from this compact artifact. +- [Repeat manifest](results/inline-repeat-manifest.json): runtime/source hashes, + pinned image, resources, and stopped sandbox metadata. The uploaded source + hashes match the PR runtime files at commit `4de31428`. +- [Run instructions](README.md): reproducible fresh/persistent comparison and + correct/incorrect/correct check. The default runs two counterbalanced rounds; + this complete repeat used `--rounds 1` after the original two-round matrix. + +The complete repeat used one Tesla T4, eight reserved CPU cores, 32 GiB RAM, +CUDA 12.8.1 compiler image, Torch 2.7.1, and `MAX_JOBS=2`. It took 406 seconds +including local launch/admission overhead, with an estimated $0.067 GPU-only +cost at $0.000164/T4-second; CPU/RAM are excluded. Both experiment sandboxes are +stopped. No production deployment, submissions, or database writes were made. + +Validation: repository-wide Ruff and three CPU regression tests pass. The +summarizer additionally audits the actual GPU result, requiring valid timing +statistics, successful compilation, native-library artifacts, and numerical +mismatch evidence for the negative reload test. diff --git a/scripts/persistent_eval/experiment.py b/scripts/persistent_eval/experiment.py new file mode 100644 index 000000000..21c1585b1 --- /dev/null +++ b/scripts/persistent_eval/experiment.py @@ -0,0 +1,169 @@ +"""Compare fresh and persistent processes with cold per-submission caches.""" + +import argparse +import contextlib +import hashlib +import json +import os +import statistics +import subprocess +import sys +import time +from pathlib import Path + +from export_result import emit_result +from persistent import running_server + + +def run_job(root, index, workload, mode): + """Use a new coordinator and private build directory for each submission.""" + directory = root / f"job-{index}-{workload}" + directory.mkdir(parents=True) + env = dict( + os.environ, + MAX_JOBS="2", + OMP_NUM_THREADS="1", + MKL_NUM_THREADS="1", + OPENBLAS_NUM_THREADS="1", + TORCH_CUDA_ARCH_LIST="7.5", + TORCH_EXTENSIONS_DIR=str(directory / "torch-cache"), + TRITON_CACHE_DIR=str(directory / "triton-cache"), + CUDA_CACHE_PATH=str(directory / "cuda-cache"), + ) + started = time.monotonic() + child = subprocess.run( + [sys.executable, "/experiment/worker.py", f"/experiment/{workload}.json", mode], + cwd=directory, + env=env, + capture_output=True, + text=True, + timeout=420, + ) + wall_s = time.monotonic() - started + result_file = directory / "result.json" + row = json.loads(result_file.read_text()) if result_file.exists() else {"passed": False} + artifacts = [ + { + "name": str(p.relative_to(directory)), + "bytes": p.stat().st_size, + "sha256": hashlib.sha256(p.read_bytes()).hexdigest(), + } + for p in sorted(directory.rglob("*.so")) + ] + row.update( + index=index, + workload=workload, + mode=mode, + wall_s=wall_s, + returncode=child.returncode, + stdout=child.stdout, + stderr=child.stderr, + compiled_extensions=artifacts, + ) + print("JOB", json.dumps({k: row[k] for k in ("index", "workload", "mode", "wall_s", "passed")}), flush=True) + return row + + +def run_batch(root, index, mode, workload, count): + """Include worker startup and teardown; GPU measurements stay sequential.""" + started = time.monotonic() + directory = root / f"batch-{index}-{mode}" + server = running_server() if mode == "persistent" else contextlib.nullcontext() + with server: + jobs = [run_job(directory, i, workload, mode) for i in range(count)] + result = { + "index": index, + "mode": mode, + "wall_s": time.monotonic() - started, + "passed": all(j["passed"] for j in jobs), + "jobs": jobs, + } + print("BATCH", json.dumps({k: v for k, v in result.items() if k != "jobs"}), flush=True) + return result + + +def check_reload(root, workload): + """Require correct/wrong/correct with the same inline extension name.""" + wrong = json.loads(Path(f"/experiment/{workload}.json").read_text()) + before, after = ( + ("C[idx] = A[idx] + B[idx]", "C[idx] = A[idx] - B[idx]") if workload == "inline" else ("C = A + B", "C = A - B") + ) + assert wrong["sources"]["submission.py"].count(before) == 1 + wrong["sources"]["submission.py"] = wrong["sources"]["submission.py"].replace(before, after) + Path("/experiment/incorrect.json").write_text(json.dumps(wrong)) + with running_server(): + jobs = [ + run_job(root / "reload", i, name, "persistent") for i, name in enumerate([workload, "incorrect", workload]) + ] + observed = [j["passed"] for j in jobs] + # A compiler/import failure is not evidence of detecting incorrect math. + wrong_runs = jobs[1].get("result", {}).get("runs", {}) + wrong_results = [r.get("run", {}) for r in wrong_runs.values()] + correctness_rejected = any(r and r.get("result", {}).get("check") == "fail" for r in wrong_results) + result = { + "expected": [True, False, True], + "observed": observed, + "jobs": jobs, + "correctness_rejected": correctness_rejected, + "passed": observed == [True, False, True] and correctness_rejected, + } + print("RELOAD", json.dumps({k: v for k, v in result.items() if k != "jobs"}), flush=True) + return result + + +def main(): + """Run counterbalanced rounds and export checksummed complete evidence.""" + parser = argparse.ArgumentParser() + parser.add_argument("--workload", choices=["inline", "triton"], default="inline") + parser.add_argument("--count", type=int, default=4) + parser.add_argument("--rounds", type=int, default=2) + args = parser.parse_args() + assert args.count > 0 and args.rounds > 0 + root = Path("/experiment/output") + root.mkdir() + info = subprocess.check_output( + [ + sys.executable, + "-c", + "import json,dataclasses; from libkernelbot.run_eval import make_system_info; " + "print(json.dumps(dataclasses.asdict(make_system_info())))", + ], + text=True, + ) + Path("/experiment/system.json").write_text(info) + assert json.loads(info)["device_count"] == 1 + hardware = subprocess.check_output( + ["nvidia-smi", "--query-gpu=name,uuid,driver_version,memory.total", "--format=csv"], text=True + ) + print("HARDWARE", hardware.strip(), flush=True) + batches = [] + for round_index in range(args.rounds): + order = ["fresh", "persistent"] if round_index % 2 == 0 else ["persistent", "fresh"] + for mode in order: + batches.append(run_batch(root, len(batches), mode, args.workload, args.count)) + emit_result({"checkpoint": "batch", "batch": batches[-1]}) + if not batches[-1]["passed"]: + break + if not batches[-1]["passed"]: + break + reload_check = check_reload(root, args.workload) if all(b["passed"] for b in batches) else None + summary = { + mode: statistics.median(b["wall_s"] for b in batches if b["mode"] == mode) + for mode in {b["mode"] for b in batches} + } + result = { + "system": json.loads(info), + "hardware": hardware, + "arguments": vars(args), + "batches": batches, + "summary": summary, + "reload": reload_check, + "passed": all(b["passed"] for b in batches) and bool(reload_check and reload_check["passed"]), + } + print("SUMMARY", json.dumps(summary), flush=True) + emit_result(result) + return 0 if result["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/persistent_eval/export_result.py b/scripts/persistent_eval/export_result.py new file mode 100644 index 000000000..2825dc30c --- /dev/null +++ b/scripts/persistent_eval/export_result.py @@ -0,0 +1,15 @@ +"""Export complete evidence without exceeding the provider's log line limit.""" + +import base64 +import gzip +import hashlib +import json + + +def emit_result(result): + """Emit compressed chunks and a checksum for reliable local reconstruction.""" + raw = json.dumps(result).encode() + encoded = base64.b64encode(gzip.compress(raw)).decode() + for offset in range(0, len(encoded), 24000): + print("EXPERIMENT_CHUNK=" + encoded[offset : offset + 24000], flush=True) + print("EXPERIMENT_END=" + hashlib.sha256(raw).hexdigest(), flush=True) diff --git a/scripts/persistent_eval/launch.py b/scripts/persistent_eval/launch.py new file mode 100644 index 000000000..9e3161f34 --- /dev/null +++ b/scripts/persistent_eval/launch.py @@ -0,0 +1,190 @@ +"""Build a minimal image and run the trusted-example experiment on one Modal T4.""" + +import argparse +import base64 +import gzip +import hashlib +import json +import shutil +import subprocess +import tempfile +import threading +import time +from pathlib import Path + +import modal +import yaml + + +def prepare_payload(payload, repo): + """Copy the current evaluator and examples without importing GPU dependencies.""" + scripts = Path(__file__).resolve().parent + for name in ("worker.py", "persistent.py", "experiment.py", "export_result.py"): + shutil.copyfile(scripts / name, payload / name) + library = payload / "source/libkernelbot" + library.mkdir(parents=True) + (library / "__init__.py").touch() + for name in ("run_eval.py", "consts.py"): + shutil.copyfile(repo / "src/libkernelbot" / name, library / name) + example = repo / "examples/vectoradd_py" + task = yaml.safe_load((example / "task.yml").read_text()) + for workload, submission in [("inline", "submission_cuda_inline.py"), ("triton", "submission_triton.py")]: + sources = { + entry["name"]: ( + example / (submission if entry["source"] == "@SUBMISSION@" else entry["source"]) + ).read_text() + for entry in task["files"] + } + config = { + "main": task["config"]["main"], + "sources": sources, + "lang": "py", + "arch": 75, + "benchmarks": task["benchmarks"][:3], + "tests": task["tests"], + "mode": "benchmark", + "test_timeout": 180, + "benchmark_timeout": 180, + "ranked_timeout": 180, + "ranking_by": "mean", + "seed": None, + "multi_gpu": False, + } + (payload / f"{workload}.json").write_text(json.dumps(config)) + return { + str(p.relative_to(payload)): hashlib.sha256(p.read_bytes()).hexdigest() + for p in sorted(payload.rglob("*")) + if p.is_file() + } + + +def decode_result(chunks, expected_sha256): + """Reject incomplete or corrupted exported evidence.""" + raw = gzip.decompress(base64.b64decode("".join(chunks))) + if hashlib.sha256(raw).hexdigest() != expected_sha256: + raise ValueError("Result checksum mismatch") + return json.loads(raw) + + +def save_decoded(decoded, output): + """Persist each completed batch independently of the final report.""" + checkpoint = decoded.get("checkpoint") == "batch" + destination = output / f"batch-{decoded['batch']['index']}.json" if checkpoint else output / "results.json" + destination.write_text(json.dumps(decoded, indent=2)) + print("SAVED", str(destination), flush=True) + return checkpoint + + +def collect(sandbox, output): + """Save logs and reconstruct a complete result despite Modal's line limit.""" + errors = [] + + def stderr_reader(): + try: + with (output / "stderr.log").open("w") as log: + for line in sandbox.stderr: + log.write(line) + log.flush() + print(line, end="", flush=True) + except Exception as error: + errors.append(error) + + reader = threading.Thread(target=stderr_reader, daemon=True) + reader.start() + chunks = [] + result = None + with (output / "stdout.log").open("w") as log: + for line in sandbox.stdout: + log.write(line) + log.flush() + if line.startswith("EXPERIMENT_CHUNK="): + chunks.append(line.split("=", 1)[1].strip()) + elif line.startswith("EXPERIMENT_END="): + decoded = decode_result(chunks, line.split("=", 1)[1].strip()) + chunks.clear() + if not save_decoded(decoded, output): + result = decoded + else: + print(line, end="", flush=True) + finish_stream(sandbox, reader, errors) + if result is None: + raise RuntimeError("Sandbox exited without complete results; see saved logs") + return result + + +def finish_stream(sandbox, reader, errors): + """Verify both output streams completed before accepting the result.""" + sandbox.wait() + reader.join(timeout=10) + if errors: + raise RuntimeError("Failed to preserve stderr") from errors[0] + if reader.is_alive(): + raise RuntimeError("stderr stream did not finish") + + +def main(): + """Run an ephemeral sandbox and retain local evidence before cleanup.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--workload", choices=["inline", "triton"], default="inline") + parser.add_argument("--count", type=int, default=4) + parser.add_argument("--rounds", type=int, default=2) + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args() + if args.count < 1 or args.rounds < 1: + parser.error("count and rounds must be positive") + output = args.output_dir.resolve() + output.mkdir(parents=True, exist_ok=False) + repo = Path(__file__).resolve().parents[2] + with tempfile.TemporaryDirectory(prefix="kernelbot-persistent-") as temporary: + payload = Path(temporary) + hashes = prepare_payload(payload, repo) + manifest = { + "kernelbot_head": subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip(), + "payload_sha256": hashes, + "arguments": {**vars(args), "output_dir": str(output)}, + "image": "nvidia/cuda:12.8.1-devel-ubuntu22.04", + "python": "3.11", + "packages": ["torch==2.7.1", "numpy==2.2.6", "ninja==1.11.1.4"], + "gpu": "T4", + "cpu": 8, + "memory_mib": 32768, + } + (output / "manifest.json").write_text(json.dumps(manifest, indent=2)) + image = ( + modal.Image.from_registry(manifest["image"], add_python="3.11") + .apt_install("g++") + .pip_install(*manifest["packages"]) + .add_local_dir(payload, "/experiment", copy=True) + .env({"PYTHONPATH": "/experiment:/experiment/source", "PYTHONUNBUFFERED": "1"}) + ) + command = [ + "python", + "/experiment/experiment.py", + "--workload", + args.workload, + "--count", + str(args.count), + "--rounds", + str(args.rounds), + ] + app = modal.App("kernelbot-persistent-evaluator-experiment") + with modal.enable_output(), app.run(): + started = time.monotonic() + sandbox = modal.Sandbox.create(*command, image=image, app=app, gpu="T4", cpu=8, memory=32768, timeout=1800) + print("SANDBOX", sandbox.object_id, flush=True) + metadata = {"id": sandbox.object_id, "command": command} + (output / "sandbox.json").write_text(json.dumps(metadata, indent=2)) + try: + result = collect(sandbox, output) + return 0 if sandbox.returncode == 0 and result["passed"] else 1 + finally: + elapsed = time.monotonic() - started + sandbox.terminate() + sandbox.detach() + metadata.update(elapsed_s=elapsed, stopped=True, gpu_cost_estimate_usd=elapsed * 0.000164) + (output / "sandbox.json").write_text(json.dumps(metadata, indent=2)) + print("ELAPSED", elapsed, "GPU_COST_ESTIMATE_USD", elapsed * 0.000164, flush=True) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/persistent_eval/persistent.py b/scripts/persistent_eval/persistent.py new file mode 100644 index 000000000..96ec92b1a --- /dev/null +++ b/scripts/persistent_eval/persistent.py @@ -0,0 +1,169 @@ +"""Trusted-example-only prototype of a persistent KernelBot GPU evaluator.""" + +import contextlib +import gc +import importlib.util +import io +import json +import os +import runpy +import socket +import subprocess +import sys +import time +import traceback +from pathlib import Path + +SOCKET = "/tmp/kernelbot-persistent-evaluator.sock" + + +@contextlib.contextmanager +def running_server(): + """Own the experiment worker, including cleanup when a request fails.""" + Path(SOCKET).unlink(missing_ok=True) + server = subprocess.Popen([sys.executable, str(Path(__file__).resolve())]) + try: + deadline = time.monotonic() + 10 + while not Path(SOCKET).exists(): + if server.poll() is not None or time.monotonic() > deadline: + raise RuntimeError("Persistent evaluator failed to start") + time.sleep(0.02) + yield server + finally: + if server.poll() is None: + try: + request({"stop": True}) + server.wait(timeout=10) + except (OSError, ValueError, subprocess.TimeoutExpired): + server.kill() + server.wait() + Path(SOCKET).unlink(missing_ok=True) + + +class DirectPool: + """Run existing single-GPU evaluation functions in the persistent process.""" + + def apply(self, function, args): + """Use the unchanged evaluator body without starting another process.""" + return function(*args) + + +class ResultLogger: + """Collect the same key/value results normally written through POPCORN_FD.""" + + def __init__(self): + """Initialize one request's output dictionary.""" + self.values = {} + + def log(self, key, value): + """Preserve KernelBot's string result representation.""" + self.values[str(key)] = str(value) + + +def request(payload): + """Send a local request to the evaluator and decode its full response.""" + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as connection: + connection.settimeout(240) + connection.connect(SOCKET) + connection.sendall(json.dumps(payload).encode() + b"\n") + with connection.makefile("rb") as handle: + return json.loads(handle.readline()) + + +def run_program(args, seed, timeout, multi_gpu=False, extra_env=None): + """Replace only the execution transport for this diagnostic experiment.""" + from libkernelbot.run_eval import RunResult + + if multi_gpu: + raise ValueError("Persistent prototype covers single-GPU trusted examples only") + environment = { + key: value + for key, value in os.environ.items() + if key + in { + "TORCH_EXTENSIONS_DIR", + "TRITON_CACHE_DIR", + "CUDA_CACHE_PATH", + "TORCH_CUDA_ARCH_LIST", + "OMP_NUM_THREADS", + "MKL_NUM_THREADS", + "OPENBLAS_NUM_THREADS", + "MAX_JOBS", + } + } + response = request({"args": args, "seed": seed, "cwd": os.getcwd(), "env": environment}) + return RunResult(**response) + + +def evaluate(payload): + """Load a trusted submission and use the original correctness/timing bodies.""" + args = payload["args"] + os.chdir(payload["cwd"]) + os.environ.update(payload["env"]) + sys.path.insert(0, os.getcwd()) + for name in ("task", "utils", "reference", "submission"): + sys.modules.pop(name, None) + stdout, stderr = io.StringIO(), io.StringIO() + logger = ResultLogger() + started = time.monotonic() + code = 0 + try: + with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + if args[1] == "submission.py": + runpy.run_path("submission.py", run_name="__main__") + else: + if args[2] != "benchmark": + raise ValueError("This prototype exercises benchmark mode only") + spec = importlib.util.spec_from_file_location("kb_persistent_eval", Path("eval.py")) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + module.set_seed(payload["seed"] or 42) + tests = module.get_test_cases(args[3], payload["seed"]) + code = module.run_benchmarking(logger, DirectPool(), tests) + del module + sys.modules.pop(spec.name, None) + import torch + + if torch.cuda.is_initialized(): + torch.cuda.synchronize() + gc.collect() + torch.cuda.empty_cache() + except Exception: + code = 1 + stderr.write(traceback.format_exc()) + finally: + sys.path.pop(0) + return { + "success": code in (0, 112), + "passed": logger.values.get("check") == "pass", + "command": " ".join(args), + "stdout": stdout.getvalue(), + "stderr": stderr.getvalue(), + "exit_code": code, + "duration": time.monotonic() - started, + "result": logger.values, + } + + +def serve(): + """Serve sequential requests on one GPU, keeping Torch and CUDA initialized.""" + Path(SOCKET).unlink(missing_ok=True) + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as server: + server.bind(SOCKET) + server.listen() + while True: + connection, _ = server.accept() + with connection: + with connection.makefile("rb") as handle: + payload = json.loads(handle.readline()) + if payload.get("stop"): + connection.sendall(b"{}\n") + break + result = evaluate(payload) + connection.sendall(json.dumps(result).encode() + b"\n") + Path(SOCKET).unlink(missing_ok=True) + + +if __name__ == "__main__": + serve() diff --git a/scripts/persistent_eval/results/inline-initial-timings.json b/scripts/persistent_eval/results/inline-initial-timings.json new file mode 100644 index 000000000..daecf9a2d --- /dev/null +++ b/scripts/persistent_eval/results/inline-initial-timings.json @@ -0,0 +1,196 @@ +{ + "detail_available": false, + "timing_matrix_completed": true, + "reload_check_completed": false, + "note": "Local Modal connection failed. Recovered provider logs retain all batch/job totals, but full phase/kernel records were not exported. A separate complete repeat supplies detailed validation.", + "summary": { + "fresh": 173.755196909, + "persistent": 125.39272825049997 + }, + "wall_time_reduction_pct": 27.833681822955015, + "batches": [ + { + "index": 0, + "mode": "fresh", + "wall_s": 174.68469984499998, + "passed": true, + "jobs": [ + { + "index": 0, + "workload": "inline", + "mode": "fresh", + "wall_s": 45.389421504000005, + "passed": true + }, + { + "index": 1, + "workload": "inline", + "mode": "fresh", + "wall_s": 42.62965532900001, + "passed": true + }, + { + "index": 2, + "workload": "inline", + "mode": "fresh", + "wall_s": 43.23664459599999, + "passed": true + }, + { + "index": 3, + "workload": "inline", + "mode": "fresh", + "wall_s": 43.40593621899998, + "passed": true + } + ] + }, + { + "index": 1, + "mode": "persistent", + "wall_s": 126.86751236599997, + "passed": true, + "jobs": [ + { + "index": 0, + "workload": "inline", + "mode": "persistent", + "wall_s": 36.18477429000001, + "passed": true + }, + { + "index": 1, + "workload": "inline", + "mode": "persistent", + "wall_s": 30.022353893000002, + "passed": true + }, + { + "index": 2, + "workload": "inline", + "mode": "persistent", + "wall_s": 29.765507914999972, + "passed": true + }, + { + "index": 3, + "workload": "inline", + "mode": "persistent", + "wall_s": 29.783142566000038, + "passed": true + } + ] + }, + { + "index": 2, + "mode": "persistent", + "wall_s": 123.91794413499997, + "passed": true, + "jobs": [ + { + "index": 0, + "workload": "inline", + "mode": "persistent", + "wall_s": 35.96695079, + "passed": true + }, + { + "index": 1, + "workload": "inline", + "mode": "persistent", + "wall_s": 29.173396095999976, + "passed": true + }, + { + "index": 2, + "workload": "inline", + "mode": "persistent", + "wall_s": 28.79137326900002, + "passed": true + }, + { + "index": 3, + "workload": "inline", + "mode": "persistent", + "wall_s": 29.17625037900001, + "passed": true + } + ] + }, + { + "index": 3, + "mode": "fresh", + "wall_s": 172.825693973, + "passed": true, + "jobs": [ + { + "index": 0, + "workload": "inline", + "mode": "fresh", + "wall_s": 43.52770579700001, + "passed": true + }, + { + "index": 1, + "workload": "inline", + "mode": "fresh", + "wall_s": 43.25928978500002, + "passed": true + }, + { + "index": 2, + "workload": "inline", + "mode": "fresh", + "wall_s": 42.60960218399998, + "passed": true + }, + { + "index": 3, + "workload": "inline", + "mode": "fresh", + "wall_s": 43.40977090000001, + "passed": true + } + ] + } + ], + "incomplete_reload_jobs": [ + { + "index": 0, + "workload": "inline", + "mode": "persistent", + "wall_s": 34.296041403000004, + "passed": true + } + ], + "manifest": { + "kernelbot_head": "727212cdcf1b9b4d587c12f6d1484b3fd54549d0", + "payload_sha256": { + "experiment.py": "5ec5ce2f1de758316aee28db182707843d24ba0a8f143becc3d578925a32f7e5", + "export_result.py": "0832549b69ca38641e020c2b6d7949cf6281e48d4df126c27b0ad0930da13ba3", + "inline.json": "06cfdd41217715960265e525afa4322b4ef21f7112e51374ab0e3ebd3c5c6eb6", + "persistent.py": "658713543c8c96dac3e7baf64fb767ce0f39ad811bc86ae4db41d0bbfc0e1e02", + "source/libkernelbot/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "source/libkernelbot/consts.py": "3226ad414d94862a536336c867690015a93776d068e2b33ec8d3deb613f3a0b2", + "source/libkernelbot/run_eval.py": "873a5f1e0f1365b0e87189c2ce0836a229ac6391b8c6a75ff021c67f2ee431a9", + "triton.json": "4054d6994f19dd92166b1c52b5f53c58392364dbdc14c62e274798efa14db008", + "worker.py": "28d044c03dce42bc4d07c23f563a373754c8eb696c944403dcfb34575e57771a" + }, + "arguments": { + "workload": "inline", + "count": 4, + "rounds": 2 + }, + "image": "nvidia/cuda:12.8.1-devel-ubuntu22.04", + "python": "3.11", + "packages": [ + "torch==2.7.1", + "numpy==2.2.6", + "ninja==1.11.1.4" + ], + "gpu": "T4", + "cpu": 8, + "memory_mib": 32768 + }, + "recovered_log_sha256": "68cfec430511a78858f385f62ba81f28783d9be1b87a36cb19a5f3d932e29a88" +} diff --git a/scripts/persistent_eval/results/inline-repeat-manifest.json b/scripts/persistent_eval/results/inline-repeat-manifest.json new file mode 100644 index 000000000..4b7e32575 --- /dev/null +++ b/scripts/persistent_eval/results/inline-repeat-manifest.json @@ -0,0 +1,47 @@ +{ + "kernelbot_head": "adfe557c0ca1e38b8a391c66de733391ef2aaeed", + "payload_sha256": { + "experiment.py": "ca5c047b9f3499b5b036c1a92c0ad6f07909efe6ad5aafbc17a79fd84f573f3b", + "export_result.py": "0832549b69ca38641e020c2b6d7949cf6281e48d4df126c27b0ad0930da13ba3", + "inline.json": "06cfdd41217715960265e525afa4322b4ef21f7112e51374ab0e3ebd3c5c6eb6", + "persistent.py": "658713543c8c96dac3e7baf64fb767ce0f39ad811bc86ae4db41d0bbfc0e1e02", + "source/libkernelbot/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "source/libkernelbot/consts.py": "3226ad414d94862a536336c867690015a93776d068e2b33ec8d3deb613f3a0b2", + "source/libkernelbot/run_eval.py": "873a5f1e0f1365b0e87189c2ce0836a229ac6391b8c6a75ff021c67f2ee431a9", + "triton.json": "4054d6994f19dd92166b1c52b5f53c58392364dbdc14c62e274798efa14db008", + "worker.py": "28d044c03dce42bc4d07c23f563a373754c8eb696c944403dcfb34575e57771a" + }, + "arguments": { + "workload": "inline", + "count": 4, + "rounds": 1 + }, + "image": "nvidia/cuda:12.8.1-devel-ubuntu22.04", + "python": "3.11", + "packages": [ + "torch==2.7.1", + "numpy==2.2.6", + "ninja==1.11.1.4" + ], + "gpu": "T4", + "cpu": 8, + "memory_mib": 32768, + "payload_matches_commit": "4de31428", + "note": "The launcher started before the payload commit was made; payload hashes were checked against the committed files after completion.", + "sandbox": { + "id": "sb-xkqyVw79gSxClLEL8rCfVJ", + "command": [ + "python", + "/experiment/experiment.py", + "--workload", + "inline", + "--count", + "4", + "--rounds", + "1" + ], + "elapsed_s": 406.38916991600126, + "stopped": true, + "gpu_cost_estimate_usd": 0.06664782386622421 + } +} diff --git a/scripts/persistent_eval/results/inline-repeat.json b/scripts/persistent_eval/results/inline-repeat.json new file mode 100644 index 000000000..6b2bd72bf --- /dev/null +++ b/scripts/persistent_eval/results/inline-repeat.json @@ -0,0 +1,622 @@ +{ + "passed": true, + "system": { + "gpu": "Tesla T4", + "device_count": 1, + "cpu": "GenuineIntel", + "runtime": "CUDA", + "platform": "Linux-4.19.0-gvisor-x86_64-with-glibc2.35", + "torch": "2.7.1+cu126" + }, + "arguments": { + "workload": "inline", + "count": 4, + "rounds": 1 + }, + "medians": { + "fresh": { + "batch_wall_s": 151.086733423, + "job_wall_s": 37.8477659165, + "phase_wall_s": { + "compile-import": 31.737814234999995, + "benchmark": 5.890799048500007 + }, + "kernel_mean_us": { + "size: 1024; seed: 54352": 36.022720308974385, + "size: 2048; seed: 93246": 108.3745938339816, + "size: 4096; seed: 6256": 407.9626699288686 + } + }, + "persistent": { + "batch_wall_s": 122.18059408700003, + "job_wall_s": 30.315491877999975, + "phase_wall_s": { + "compile-import": 29.849093452499986, + "benchmark": 0.4067544014999953 + }, + "kernel_mean_us": { + "size: 1024; seed: 54352": 40.280799847096205, + "size: 2048; seed: 93246": 123.65123234233077, + "size: 4096; seed: 6256": 468.4746749699116 + } + } + }, + "wall_time_reduction_pct": 19.132149250371956, + "batches": [ + { + "index": 0, + "mode": "fresh", + "wall_s": 151.086733423, + "passed": true, + "jobs": [ + { + "index": 0, + "workload": "inline", + "mode": "fresh", + "wall_s": 37.717555915000005, + "passed": true, + "compiled_extensions": [ + { + "name": "torch-cache/add_cuda/add_cuda.so", + "bytes": 1371104, + "sha256": "3d00123e9ac3434ae37ebc665fd7010431cd6e01467d19eb35289b6bd57a4520" + } + ], + "phases": [ + { + "phase": "compile-import", + "wall_s": 31.784614422999997, + "exit_code": 0 + }, + { + "phase": "benchmark", + "wall_s": 5.846300309, + "exit_code": 0 + } + ], + "benchmark": { + "benchmark-count": "3", + "benchmark.0.spec": "size: 1024; seed: 54352", + "benchmark.0.runs": "100", + "benchmark.0.mean": "36704.32057231665", + "benchmark.0.std": "1092.9279454774917", + "benchmark.0.err": "109.29279454774917", + "benchmark.0.best": "35135.99932193756", + "benchmark.0.worst": "42784.00167822838", + "benchmark.1.spec": "size: 2048; seed: 93246", + "benchmark.1.runs": "3", + "benchmark.1.mean": "126965.33401807149", + "benchmark.1.std": "18.471047698708272", + "benchmark.1.err": "10.664264361063639", + "benchmark.1.best": "126944.00548934937", + "benchmark.1.worst": "126975.99828243256", + "benchmark.2.spec": "size: 4096; seed: 6256", + "benchmark.2.runs": "3", + "benchmark.2.mean": "486666.66944821674", + "benchmark.2.std": "774.1920274015364", + "benchmark.2.err": "446.97997542473917", + "benchmark.2.best": "485792.01102256775", + "benchmark.2.worst": "487264.0073299408", + "check": "pass" + } + }, + { + "index": 1, + "workload": "inline", + "mode": "fresh", + "wall_s": 36.851178094, + "passed": true, + "compiled_extensions": [ + { + "name": "torch-cache/add_cuda/add_cuda.so", + "bytes": 1371104, + "sha256": "8bdb931cca19eb984771a700b03a77e724ed6ac59fab0e0c5415bc4a9a12e165" + } + ], + "phases": [ + { + "phase": "compile-import", + "wall_s": 30.934483453, + "exit_code": 0 + }, + { + "phase": "benchmark", + "wall_s": 5.830919796999993, + "exit_code": 0 + } + ], + "benchmark": { + "benchmark-count": "3", + "benchmark.0.spec": "size: 1024; seed: 54352", + "benchmark.0.runs": "100", + "benchmark.0.mean": "35341.120045632124", + "benchmark.0.std": "3527.4678772128505", + "benchmark.0.err": "352.74678772128505", + "benchmark.0.best": "30719.999223947525", + "benchmark.0.worst": "38911.99827194214", + "benchmark.1.spec": "size: 2048; seed: 93246", + "benchmark.1.runs": "27", + "benchmark.1.mean": "108018.96232145804", + "benchmark.1.std": "552.1387463821106", + "benchmark.1.err": "106.25915128457802", + "benchmark.1.best": "106976.00245475769", + "benchmark.1.worst": "108543.9994931221", + "benchmark.2.spec": "size: 4096; seed: 6256", + "benchmark.2.runs": "3", + "benchmark.2.mean": "407594.67085202533", + "benchmark.2.std": "102.87334472452157", + "benchmark.2.err": "59.3939532691397", + "benchmark.2.best": "407519.99616622925", + "benchmark.2.worst": "407712.01252937317", + "check": "pass" + } + }, + { + "index": 2, + "workload": "inline", + "mode": "fresh", + "wall_s": 37.977975918, + "passed": true, + "compiled_extensions": [ + { + "name": "torch-cache/add_cuda/add_cuda.so", + "bytes": 1371104, + "sha256": "7622c6fb31a80edff4fd1756177a8ac0d464ce022036fba9a6e31b3e12e5bb48" + } + ], + "phases": [ + { + "phase": "compile-import", + "wall_s": 31.691014046999996, + "exit_code": 0 + }, + { + "phase": "benchmark", + "wall_s": 6.196681572000003, + "exit_code": 0 + } + ], + "benchmark": { + "benchmark-count": "3", + "benchmark.0.spec": "size: 1024; seed: 54352", + "benchmark.0.runs": "100", + "benchmark.0.mean": "33994.23988536", + "benchmark.0.std": "3488.7067277221295", + "benchmark.0.err": "348.87067277221297", + "benchmark.0.best": "30719.999223947525", + "benchmark.0.worst": "39935.99861860275", + "benchmark.1.spec": "size: 2048; seed: 93246", + "benchmark.1.runs": "24", + "benchmark.1.mean": "108711.99922015269", + "benchmark.1.std": "511.5864351976797", + "benchmark.1.err": "104.42714379697729", + "benchmark.1.best": "108159.99656915665", + "benchmark.1.worst": "110207.99726247787", + "benchmark.2.spec": "size: 4096; seed: 6256", + "benchmark.2.runs": "3", + "benchmark.2.mean": "408330.66900571185", + "benchmark.2.std": "563.7189195717535", + "benchmark.2.err": "325.4632699620369", + "benchmark.2.best": "407680.0048351288", + "benchmark.2.worst": "408672.0049381256", + "check": "pass" + } + }, + { + "index": 3, + "workload": "inline", + "mode": "fresh", + "wall_s": 38.51741364899999, + "passed": true, + "compiled_extensions": [ + { + "name": "torch-cache/add_cuda/add_cuda.so", + "bytes": 1371104, + "sha256": "d1bfce49d9fc1d7f4f5b076ced7917c76b29d0b1808a328aba0dfd1ece00fcd5" + } + ], + "phases": [ + { + "phase": "compile-import", + "wall_s": 32.48583579000001, + "exit_code": 0 + }, + { + "phase": "benchmark", + "wall_s": 5.935297788000014, + "exit_code": 0 + } + ], + "benchmark": { + "benchmark-count": "3", + "benchmark.0.spec": "size: 1024; seed: 54352", + "benchmark.0.runs": "100", + "benchmark.0.mean": "40641.59948378801", + "benchmark.0.std": "1395.5323521220882", + "benchmark.0.err": "139.55323521220882", + "benchmark.0.best": "39391.99820160866", + "benchmark.0.worst": "53631.99859857559", + "benchmark.1.spec": "size: 2048; seed: 93246", + "benchmark.1.runs": "37", + "benchmark.1.mean": "108037.1884478105", + "benchmark.1.std": "648.2360525955978", + "benchmark.1.err": "106.56935058153861", + "benchmark.1.best": "106880.00172376633", + "benchmark.1.worst": "110367.99848079681", + "benchmark.2.spec": "size: 4096; seed: 6256", + "benchmark.2.runs": "3", + "benchmark.2.mean": "407477.32917467755", + "benchmark.2.std": "102.87334472452157", + "benchmark.2.err": "59.3939532691397", + "benchmark.2.best": "407359.9874973297", + "benchmark.2.worst": "407552.00386047363", + "check": "pass" + } + } + ] + }, + { + "index": 1, + "mode": "persistent", + "wall_s": 122.18059408700003, + "passed": true, + "jobs": [ + { + "index": 0, + "workload": "inline", + "mode": "persistent", + "wall_s": 31.75274639, + "passed": true, + "compiled_extensions": [ + { + "name": "torch-cache/add_cuda/add_cuda.so", + "bytes": 1371104, + "sha256": "990d1e099828fa7dd18a7ef42d839c59e638a312f4ec210ff9586fb5038d10a8" + } + ], + "phases": [ + { + "phase": "compile-import", + "wall_s": 30.905686301999992, + "exit_code": 0 + }, + { + "phase": "benchmark", + "wall_s": 0.7587213730000144, + "exit_code": 0 + } + ], + "benchmark": { + "benchmark-count": "3", + "benchmark.0.spec": "size: 1024; seed: 54352", + "benchmark.0.runs": "4", + "benchmark.0.mean": "38879.9998909235", + "benchmark.0.std": "69.12670269459139", + "benchmark.0.err": "34.56335134729569", + "benchmark.0.best": "38784.001022577286", + "benchmark.0.worst": "38943.99851560593", + "benchmark.1.spec": "size: 2048; seed: 93246", + "benchmark.1.runs": "20", + "benchmark.1.mean": "136451.19965076447", + "benchmark.1.std": "589.3107837663108", + "benchmark.1.err": "131.77389723751503", + "benchmark.1.best": "135391.9953107834", + "benchmark.1.worst": "137375.99551677704", + "benchmark.2.spec": "size: 4096; seed: 6256", + "benchmark.2.runs": "3", + "benchmark.2.mean": "523861.3486289978", + "benchmark.2.std": "826.0229714363858", + "benchmark.2.err": "476.9045849156119", + "benchmark.2.best": "522912.02545166016", + "benchmark.2.worst": "524416.0294532776", + "check": "pass" + } + }, + { + "index": 1, + "workload": "inline", + "mode": "persistent", + "wall_s": 29.038929069999995, + "passed": true, + "compiled_extensions": [ + { + "name": "torch-cache/add_cuda/add_cuda_v1.so", + "bytes": 1371112, + "sha256": "70688b07348f9bdff40fa7fdb9a4646378e579d4de73cc306800a83e347a1cc2" + } + ], + "phases": [ + { + "phase": "compile-import", + "wall_s": 28.522874734, + "exit_code": 0 + }, + { + "phase": "benchmark", + "wall_s": 0.425054369999998, + "exit_code": 0 + } + ], + "benchmark": { + "benchmark-count": "3", + "benchmark.0.spec": "size: 1024; seed: 54352", + "benchmark.0.runs": "100", + "benchmark.0.mean": "36071.35981321335", + "benchmark.0.std": "4543.822084910424", + "benchmark.0.err": "454.3822084910424", + "benchmark.0.best": "31520.001590251923", + "benchmark.0.worst": "43007.999658584595", + "benchmark.1.spec": "size: 2048; seed: 93246", + "benchmark.1.runs": "49", + "benchmark.1.mean": "110851.26503389709", + "benchmark.1.std": "763.9683026068492", + "benchmark.1.err": "109.1383289438356", + "benchmark.1.best": "109375.99837779999", + "benchmark.1.worst": "112640.00087976456", + "benchmark.2.spec": "size: 4096; seed: 6256", + "benchmark.2.runs": "4", + "benchmark.2.mean": "413088.00131082535", + "benchmark.2.std": "792.9197520986974", + "benchmark.2.err": "396.4598760493487", + "benchmark.2.best": "411936.0148906708", + "benchmark.2.worst": "413695.9910392761", + "check": "pass" + } + }, + { + "index": 2, + "workload": "inline", + "mode": "persistent", + "wall_s": 30.949702426999977, + "passed": true, + "compiled_extensions": [ + { + "name": "torch-cache/add_cuda/add_cuda_v2.so", + "bytes": 1371112, + "sha256": "8869b7f80b1a1ae78d36a4b279a1af9882d6ec45e01188e0de805da246e34154" + } + ], + "phases": [ + { + "phase": "compile-import", + "wall_s": 30.496586076, + "exit_code": 0 + }, + { + "phase": "benchmark", + "wall_s": 0.3597627510000052, + "exit_code": 0 + } + ], + "benchmark": { + "benchmark-count": "3", + "benchmark.0.spec": "size: 1024; seed: 54352", + "benchmark.0.runs": "100", + "benchmark.0.mean": "42088.95988762379", + "benchmark.0.std": "1158.9204020498435", + "benchmark.0.err": "115.89204020498434", + "benchmark.0.best": "32191.99925661087", + "benchmark.0.worst": "43007.999658584595", + "benchmark.1.spec": "size: 2048; seed: 93246", + "benchmark.1.runs": "3", + "benchmark.1.mean": "108650.66697200139", + "benchmark.1.std": "147.8027943473722", + "benchmark.1.err": "85.33398310343425", + "benchmark.1.best": "108479.99900579453", + "benchmark.1.worst": "108736.00095510483", + "benchmark.2.spec": "size: 4096; seed: 6256", + "benchmark.2.runs": "3", + "benchmark.2.mean": "407925.3375530243", + "benchmark.2.std": "18.47965088813478", + "benchmark.2.err": "10.669231414794924", + "benchmark.2.best": "407903.9990901947", + "benchmark.2.worst": "407936.0067844391", + "check": "pass" + } + }, + { + "index": 3, + "workload": "inline", + "mode": "persistent", + "wall_s": 29.681281328999972, + "passed": true, + "compiled_extensions": [ + { + "name": "torch-cache/add_cuda/add_cuda_v3.so", + "bytes": 1371112, + "sha256": "ec1c9109ebf531ad21d9410ded81c7269df7b322d3aa4ae94c3035f0c6e9248b" + } + ], + "phases": [ + { + "phase": "compile-import", + "wall_s": 29.201600828999972, + "exit_code": 0 + }, + { + "phase": "benchmark", + "wall_s": 0.3884544329999926, + "exit_code": 0 + } + ], + "benchmark": { + "benchmark-count": "3", + "benchmark.0.spec": "size: 1024; seed: 54352", + "benchmark.0.runs": "100", + "benchmark.0.mean": "41681.59980326891", + "benchmark.0.std": "552.8994604089672", + "benchmark.0.err": "55.28994604089672", + "benchmark.0.best": "40927.99872159958", + "benchmark.0.worst": "43007.999658584595", + "benchmark.1.spec": "size: 2048; seed: 93246", + "benchmark.1.runs": "4", + "benchmark.1.mean": "147528.0001759529", + "benchmark.1.std": "229.82920232469908", + "benchmark.1.err": "114.91460116234954", + "benchmark.1.best": "147392.00472831726", + "benchmark.1.worst": "147872.00093269348", + "benchmark.2.spec": "size: 4096; seed: 6256", + "benchmark.2.runs": "4", + "benchmark.2.mean": "570695.9962844849", + "benchmark.2.std": "917.231329145202", + "benchmark.2.err": "458.615664572601", + "benchmark.2.best": "569343.9841270447", + "benchmark.2.worst": "571359.9920272827", + "check": "pass" + } + } + ] + } + ], + "reload": { + "observed": [ + true, + false, + true + ], + "jobs": [ + { + "index": 0, + "workload": "inline", + "mode": "persistent", + "wall_s": 33.52144904900001, + "passed": true, + "compiled_extensions": [ + { + "name": "torch-cache/add_cuda/add_cuda.so", + "bytes": 1371104, + "sha256": "7da8350dcf94fc6e7963e982984c6111a49978c884d8053ae85fee41964e2242" + } + ], + "phases": [ + { + "phase": "compile-import", + "wall_s": 32.60032794399996, + "exit_code": 0 + }, + { + "phase": "benchmark", + "wall_s": 0.825036434000026, + "exit_code": 0 + } + ], + "benchmark": { + "benchmark-count": "3", + "benchmark.0.spec": "size: 1024; seed: 54352", + "benchmark.0.runs": "14", + "benchmark.0.mean": "38852.57111064025", + "benchmark.0.std": "137.41183467847603", + "benchmark.0.err": "36.72485758963425", + "benchmark.0.best": "38623.99980425835", + "benchmark.0.worst": "39103.999733924866", + "benchmark.1.spec": "size: 2048; seed: 93246", + "benchmark.1.runs": "34", + "benchmark.1.mean": "136321.8821146909", + "benchmark.1.std": "777.4748487729736", + "benchmark.1.err": "133.335836548451", + "benchmark.1.best": "135135.99336147308", + "benchmark.1.worst": "138751.99854373932", + "benchmark.2.spec": "size: 4096; seed: 6256", + "benchmark.2.runs": "3", + "benchmark.2.mean": "524224.0031560262", + "benchmark.2.std": "84.65060660382474", + "benchmark.2.err": "48.87305050978333", + "benchmark.2.best": "524128.0198097229", + "benchmark.2.worst": "524287.99867630005", + "check": "pass" + } + }, + { + "index": 1, + "workload": "incorrect", + "mode": "persistent", + "wall_s": 29.96964817600002, + "passed": false, + "compiled_extensions": [ + { + "name": "torch-cache/add_cuda/add_cuda_v1.so", + "bytes": 1371112, + "sha256": "1893cb07903c015a4d5662610d619689299dcd60258693884500df927d304eec" + } + ], + "phases": [ + { + "phase": "compile-import", + "wall_s": 29.727761948000023, + "exit_code": 0 + }, + { + "phase": "benchmark", + "wall_s": 0.13815245300003198, + "exit_code": 112 + } + ], + "benchmark": { + "benchmark-count": "3", + "benchmark.0.spec": "size: 1024; seed: 54352", + "benchmark.0.status": "fail", + "benchmark.0.error": "mismatch found! custom implementation doesn't match reference: Number of mismatched elements: 1048369", + "benchmark.1.spec": "size: 2048; seed: 93246", + "benchmark.1.status": "fail", + "benchmark.1.error": "mismatch found! custom implementation doesn't match reference: Number of mismatched elements: 4193350", + "benchmark.2.spec": "size: 4096; seed: 6256", + "benchmark.2.status": "fail", + "benchmark.2.error": "mismatch found! custom implementation doesn't match reference: Number of mismatched elements: 16773409", + "check": "fail" + } + }, + { + "index": 2, + "workload": "inline", + "mode": "persistent", + "wall_s": 29.147953131999998, + "passed": true, + "compiled_extensions": [ + { + "name": "torch-cache/add_cuda/add_cuda_v2.so", + "bytes": 1371112, + "sha256": "b590b53a9eedc64579365cc97dbc904d3ef6ab1455a26821d24a84f097251761" + } + ], + "phases": [ + { + "phase": "compile-import", + "wall_s": 28.662332048000053, + "exit_code": 0 + }, + { + "phase": "benchmark", + "wall_s": 0.3931625609999969, + "exit_code": 0 + } + ], + "benchmark": { + "benchmark-count": "3", + "benchmark.0.spec": "size: 1024; seed: 54352", + "benchmark.0.runs": "100", + "benchmark.0.mean": "35824.320428073406", + "benchmark.0.std": "1848.9114611217553", + "benchmark.0.err": "184.89114611217553", + "benchmark.0.best": "34784.00036692619", + "benchmark.0.worst": "42656.00070357323", + "benchmark.1.spec": "size: 2048; seed: 93246", + "benchmark.1.runs": "30", + "benchmark.1.mean": "121565.86547692616", + "benchmark.1.std": "649.4707776925375", + "benchmark.1.err": "118.57659846087527", + "benchmark.1.best": "120768.0031657219", + "benchmark.1.worst": "122815.99640846252", + "benchmark.2.spec": "size: 4096; seed: 6256", + "benchmark.2.runs": "3", + "benchmark.2.mean": "465418.65666707355", + "benchmark.2.std": "510.32935870079126", + "benchmark.2.err": "294.6387926212709", + "benchmark.2.best": "464991.9867515564", + "benchmark.2.worst": "465983.9868545532", + "check": "pass" + } + } + ] + }, + "raw_results_sha256": "54e44c186b65359163278bf769109753597ff6eb73630b1f1ec23f97965c4069" +} diff --git a/scripts/persistent_eval/summarize.py b/scripts/persistent_eval/summarize.py new file mode 100644 index 000000000..a689d1e2a --- /dev/null +++ b/scripts/persistent_eval/summarize.py @@ -0,0 +1,95 @@ +"""Validate saved measurements and produce a compact, shareable evidence file.""" + +import argparse +import hashlib +import json +import math +import statistics +from pathlib import Path + + +def compact_job(job, workload): + """Retain timings, correctness, and extension provenance without compiler logs.""" + result = job["result"]["runs"]["benchmark"]["run"]["result"] + assert int(result["benchmark-count"]) == 3 + if job["passed"]: + assert job["returncode"] == 0 and result["check"] == "pass" + # Successful shapes log statistics; only failing shapes have a status key. + assert all(f"benchmark.{i}.status" not in result for i in range(3)) + assert all( + math.isfinite(float(result[f"benchmark.{i}.mean"])) and float(result[f"benchmark.{i}.mean"]) > 0 + for i in range(3) + ) + assert len(job["phases"]) == 2 + assert all(phase["exit_code"] == 0 for phase in job["phases"]) + if workload == "inline": + assert job["compiled_extensions"], "Cold inline job must produce a native library" + return { + **{key: job[key] for key in ("index", "workload", "mode", "wall_s", "passed", "compiled_extensions")}, + "phases": [{k: p[k] for k in ("phase", "wall_s", "exit_code")} for p in job["phases"]], + "benchmark": result, + } + + +def summarize(raw): + """Audit correctness and keep every observation behind the aggregate medians.""" + assert raw["passed"] and raw["reload"]["passed"] + assert raw["reload"]["observed"] == [True, False, True] + assert len(raw["batches"]) == raw["arguments"]["rounds"] * 2 + workload = raw["arguments"]["workload"] + batches = [] + for batch in raw["batches"]: + assert batch["passed"] and len(batch["jobs"]) == raw["arguments"]["count"] + batches.append( + { + **{k: batch[k] for k in ("index", "mode", "wall_s", "passed")}, + "jobs": [compact_job(j, workload) for j in batch["jobs"]], + } + ) + medians = {} + for mode in ("fresh", "persistent"): + selected = [b for b in batches if b["mode"] == mode] + jobs = [j for b in selected for j in b["jobs"]] + medians[mode] = { + "batch_wall_s": statistics.median(b["wall_s"] for b in selected), + "job_wall_s": statistics.median(j["wall_s"] for j in jobs), + "phase_wall_s": { + phase: statistics.median(p["wall_s"] for j in jobs for p in j["phases"] if p["phase"] == phase) + for phase in ("compile-import", "benchmark") + }, + "kernel_mean_us": { + jobs[0]["benchmark"][f"benchmark.{i}.spec"]: statistics.median( + float(j["benchmark"][f"benchmark.{i}.mean"]) / 1000 for j in jobs + ) + for i in range(3) + }, + } + reload_jobs = [compact_job(j, workload) for j in raw["reload"]["jobs"]] + wrong = reload_jobs[1]["benchmark"] + assert all( + wrong[f"benchmark.{i}.status"] == "fail" and "mismatch" in wrong[f"benchmark.{i}.error"] for i in range(3) + ) + return { + "passed": True, + "system": {k: v for k, v in raw["system"].items() if k != "hostname"}, + "arguments": raw["arguments"], + "medians": medians, + "wall_time_reduction_pct": 100 * (1 - medians["persistent"]["batch_wall_s"] / medians["fresh"]["batch_wall_s"]), + "batches": batches, + "reload": {"observed": raw["reload"]["observed"], "jobs": reload_jobs}, + } + + +def main(): + """Read the complete local result and write audited JSON to stdout.""" + parser = argparse.ArgumentParser() + parser.add_argument("results", type=Path) + args = parser.parse_args() + data = args.results.read_bytes() + summary = summarize(json.loads(data)) + summary["raw_results_sha256"] = hashlib.sha256(data).hexdigest() + print(json.dumps(summary, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/persistent_eval/test_experiment.py b/scripts/persistent_eval/test_experiment.py new file mode 100644 index 000000000..dfe495222 --- /dev/null +++ b/scripts/persistent_eval/test_experiment.py @@ -0,0 +1,54 @@ +"""CPU regression checks for reproducibility and lossless result export.""" + +import hashlib +import json +import random +from pathlib import Path +from types import SimpleNamespace + +import pytest +from export_result import emit_result +from launch import collect, decode_result, prepare_payload + + +def test_large_evidence_roundtrip_and_checksum(capsys): + """Outputs larger than a provider log line must survive multiple chunks.""" + result = {"log": random.Random(42).randbytes(150000).hex(), "passed": False} + emit_result(result) + lines = capsys.readouterr().out.splitlines() + chunks = [line.split("=", 1)[1] for line in lines if line.startswith("EXPERIMENT_CHUNK=")] + checksum = lines[-1].split("=", 1)[1] + assert len(chunks) > 1 + assert max(map(len, lines)) < 25000 + assert decode_result(chunks, checksum) == result + with pytest.raises(ValueError, match="checksum"): + decode_result(chunks, "0" * 64) + + +def test_payload_preserves_evaluator_and_submission(tmp_path): + """Build from current repository sources and keep the stated benchmark shapes.""" + repo = Path(__file__).resolve().parents[2] + hashes = prepare_payload(tmp_path, repo) + for name in ("run_eval.py", "consts.py"): + source = (repo / "src/libkernelbot" / name).read_bytes() + assert (tmp_path / "source/libkernelbot" / name).read_bytes() == source + assert hashes[f"source/libkernelbot/{name}"] == hashlib.sha256(source).hexdigest() + for workload, filename in [("inline", "submission_cuda_inline.py"), ("triton", "submission_triton.py")]: + config = json.loads((tmp_path / f"{workload}.json").read_text()) + assert config["sources"]["submission.py"] == (repo / "examples/vectoradd_py" / filename).read_text() + assert config["sources"]["eval.py"] == (repo / "examples/eval.py").read_text() + assert [case["size"] for case in config["benchmarks"]] == [1024, 2048, 4096] + assert config["mode"] == "benchmark" and not config["multi_gpu"] + + +def test_checkpoint_stream_then_final_report(tmp_path, capsys): + """Multiple framed exports must not concatenate into a corrupt final result.""" + for index in range(2): + emit_result({"checkpoint": "batch", "batch": {"index": index, "passed": True}}) + final = {"passed": True, "summary": {"fresh": 20, "persistent": 10}} + emit_result(final) + sandbox = SimpleNamespace(stdout=capsys.readouterr().out.splitlines(True), stderr=[], wait=lambda: None) + assert collect(sandbox, tmp_path) == final + assert json.loads((tmp_path / "batch-0.json").read_text())["batch"]["index"] == 0 + assert json.loads((tmp_path / "batch-1.json").read_text())["batch"]["index"] == 1 + assert json.loads((tmp_path / "results.json").read_text()) == final diff --git a/scripts/persistent_eval/worker.py b/scripts/persistent_eval/worker.py new file mode 100644 index 000000000..a3710c6c9 --- /dev/null +++ b/scripts/persistent_eval/worker.py @@ -0,0 +1,56 @@ +"""Instrument the real run_config boundary for trusted example submissions.""" + +import dataclasses +import json +import sys +import time +from pathlib import Path + +from libkernelbot import run_eval + + +def main(): + """Execute one submission with fresh processes or a persistent RPC worker.""" + config = json.loads(Path(sys.argv[1]).read_text()) + mode = sys.argv[2] + system = run_eval.SystemInfo(**json.loads(Path("/experiment/system.json").read_text())) + run_eval.make_system_info = lambda: system + original = run_eval.run_program + if mode == "persistent": + from persistent import run_program + + original = run_program + phases = [] + + def measured_program(args, seed, timeout, multi_gpu=False, extra_env=None): + started = time.monotonic() + result = original(args, seed, timeout, multi_gpu, extra_env) + phases.append( + { + "phase": args[2] if len(args) > 2 else "compile-import", + "wall_s": time.monotonic() - started, + "exit_code": int(result.exit_code), + "stdout": result.stdout, + "stderr": result.stderr, + } + ) + return result + + run_eval.run_program = measured_program + try: + result = run_eval.run_config(config) + row = { + "passed": result.success and all(r.run and r.run.passed for r in result.runs.values()), + "result": dataclasses.asdict(result), + } + except Exception: + import traceback + + row = {"passed": False, "error": traceback.format_exc()} + row["phases"] = phases + Path("result.json").write_text(json.dumps(row, default=str)) + return 0 if row["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main())