Add Buzz-native collaboration benchmarks

benchmarks/harbor-buzz-orchestra/README.md

@@ -62,6 +62,34 @@ rather than deletes that channel, leaving the relay/Postgres event timeline
 and the per-agent acp/agent logs (downloaded into the trial's `buzz/`
 artifacts) available for analysis.
 
+### Buzz-native tasks
+
+The local `datasets/buzz-native` suite scores Buzz product behavior alongside
+task correctness. It currently covers direct thread replies, callback user
+mentions, targeted reads of user-named paths outside the workspace, and exact
+channel creation/membership. Run one task with the production base prompt from
+the checked-out source build:
+
+```bash
+just benchmark \
+  --path benchmarks/harbor-buzz-orchestra/datasets/buzz-native/reply-to-thread \
+  --attempts 1 \
+  --manifest benchmarks/harbor-buzz-orchestra/manifests/buzz-native-solo-sonnet.yaml \
+  --n-concurrent 1
+```
+
+Replace the path with
+`benchmarks/harbor-buzz-orchestra/datasets/buzz-native/create-channel-invite-users`
+to run the channel task. Its provisioner seeds a stable directory of 50 users
+and 10 bots, while the verifier checks the created channel's TTL and exact
+membership through post-agent CLI evidence.
+
+After the agent stops, the runtime snapshots public relay state (source
+messages plus any task-declared channels and members) to
+`/logs/artifacts/buzz-evidence.json`. The task verifier reads that post-agent
+artifact; relay credentials and database access are never exposed to the model
+or verifier.
+
 ## Leaderboard runs
 
 `just benchmark` is the one-command path: it stands up a dedicated Docker

benchmarks/harbor-buzz-orchestra/datasets/buzz-native/create-channel-invite-users/environment/Dockerfile

@@ -0,0 +1,3 @@
+FROM python:3.12-slim-bookworm
+
+WORKDIR /app

benchmarks/harbor-buzz-orchestra/datasets/buzz-native/create-channel-invite-users/instruction.md

@@ -0,0 +1,15 @@
+Create a temporary private stream channel named `fix-pr-1234` for one hour.
+
+Invite these users as members:
+
+- `benchmark-user-07`
+- `benchmark-user-19`
+- `benchmark-user-42`
+
+Invite these bots with the bot role:
+
+- `benchmark-bot-03`
+- `benchmark-bot-08`
+
+Do not invite any other users or bots. When finished, reply briefly with what
+you created.

benchmarks/harbor-buzz-orchestra/datasets/buzz-native/create-channel-invite-users/task.toml

@@ -0,0 +1,19 @@
+schema_version = "1.3"
+
+[task]
+name = "buzz-native/create-channel-invite-users"
+description = "Create a temporary PR channel with an exact subset of users and bots."
+authors = [{ name = "Buzz" }]
+keywords = ["buzz-native", "channels", "membership", "cli"]
+
+[agent]
+timeout_sec = 300.0
+
+[verifier]
+timeout_sec = 30.0
+
+[environment]
+network_mode = "public"
+cpus = 1
+memory_mb = 1024
+storage_mb = 1024

benchmarks/harbor-buzz-orchestra/datasets/buzz-native/create-channel-invite-users/tests/test.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+set -eu
+
+python3 /tests/verify.py \
+  --evidence /logs/artifacts/buzz-evidence.json \
+  --reward /logs/verifier/reward.json \
+  --details /logs/verifier/details.json

benchmarks/harbor-buzz-orchestra/datasets/buzz-native/create-channel-invite-users/tests/verify.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+"""Deterministic verifier for exact Buzz channel creation and membership."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from typing import Any
+
+CHANNEL_NAME = "fix-pr-1234"
+TARGET_USERS = {"benchmark-user-07", "benchmark-user-19", "benchmark-user-42"}
+TARGET_BOTS = {"benchmark-bot-03", "benchmark-bot-08"}
+
+
+def _zero_metrics() -> dict[str, float]:
+    return {
+        "reward": 0.0,
+        "channel_created": 0.0,
+        "channel_shape": 0.0,
+        "temporary_channel": 0.0,
+        "exact_membership": 0.0,
+        "expected_roles": 0.0,
+        "evidence_complete": 0.0,
+    }
+
+
+def score_evidence(evidence: object) -> tuple[dict[str, float], dict[str, Any]]:
+    if not isinstance(evidence, dict):
+        return _zero_metrics(), {"error": "evidence root is not an object"}
+
+    directory_rows = [
+        row for row in evidence.get("directory", []) if isinstance(row, dict)
+    ]
+    directory = {
+        row.get("name"): row
+        for row in directory_rows
+        if isinstance(row.get("name"), str)
+    }
+    channels = [
+        channel
+        for channel in evidence.get("observed_channels", [])
+        if isinstance(channel, dict) and channel.get("name") == CHANNEL_NAME
+    ]
+    channel = channels[0] if len(channels) == 1 else None
+    identities = (
+        evidence.get("identities")
+        if isinstance(evidence.get("identities"), dict)
+        else {}
+    )
+    orchestrators = [
+        row
+        for row in identities.values()
+        if isinstance(row, dict) and row.get("role") == "orchestrator"
+    ]
+    owner_pubkey = orchestrators[0].get("pubkey") if len(orchestrators) == 1 else None
+
+    expected_names = TARGET_USERS | TARGET_BOTS
+    expected_targets = {
+        directory[name]["pubkey"]: "bot" if name in TARGET_BOTS else "member"
+        for name in expected_names
+        if name in directory and isinstance(directory[name].get("pubkey"), str)
+    }
+    expected_members = (
+        {owner_pubkey: "owner", **expected_targets}
+        if isinstance(owner_pubkey, str)
+        else expected_targets
+    )
+    member_rows = (
+        [row for row in channel.get("members", []) if isinstance(row, dict)]
+        if channel is not None
+        else []
+    )
+    actual_members = {
+        row.get("pubkey"): row.get("role")
+        for row in member_rows
+        if isinstance(row.get("pubkey"), str)
+    }
+
+    evidence_complete = float(
+        evidence.get("schema_version") == 1
+        and evidence.get("task_name") == "create-channel-invite-users"
+        and len(directory_rows) == 60
+        and len(directory) == 60
+        and sum(row.get("role") == "user" for row in directory_rows) == 50
+        and sum(row.get("role") == "bot" for row in directory_rows) == 10
+        and len(expected_targets) == 5
+        and len(orchestrators) == 1
+    )
+    channel_created = float(channel is not None)
+    channel_shape = float(
+        channel is not None
+        and channel.get("channel_type") == "stream"
+        and channel.get("visibility") == "private"
+        and channel.get("archived") is False
+    )
+    temporary_channel = float(
+        channel is not None and channel.get("ttl_seconds") == 3600
+    )
+    exact_membership = float(
+        len(member_rows) == len(actual_members)
+        and set(actual_members) == set(expected_members)
+    )
+    expected_roles = float(actual_members == expected_members)
+    reward = float(
+        all(
+            metric == 1.0
+            for metric in (
+                evidence_complete,
+                channel_created,
+                channel_shape,
+                temporary_channel,
+                exact_membership,
+                expected_roles,
+            )
+        )
+    )
+    metrics = {
+        "reward": reward,
+        "channel_created": channel_created,
+        "channel_shape": channel_shape,
+        "temporary_channel": temporary_channel,
+        "exact_membership": exact_membership,
+        "expected_roles": expected_roles,
+        "evidence_complete": evidence_complete,
+    }
+    details = {
+        "matching_channel_count": len(channels),
+        "channel_id": channel.get("channel_id") if channel is not None else None,
+        "expected_members": expected_members,
+        "actual_members": actual_members,
+    }
+    return metrics, details
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--evidence", type=Path, required=True)
+    parser.add_argument("--reward", type=Path, required=True)
+    parser.add_argument("--details", type=Path, required=True)
+    args = parser.parse_args()
+
+    try:
+        evidence = json.loads(args.evidence.read_text(encoding="utf-8"))
+        metrics, details = score_evidence(evidence)
+    except (OSError, json.JSONDecodeError) as error:
+        metrics, details = _zero_metrics(), {"error": str(error)}
+
+    args.reward.write_text(json.dumps(metrics, sort_keys=True) + "\n", encoding="utf-8")
+    args.details.write_text(
+        json.dumps(details, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

benchmarks/harbor-buzz-orchestra/datasets/buzz-native/read-named-path-outside-workspace/environment/Dockerfile

@@ -0,0 +1,7 @@
+FROM python:3.12-slim-bookworm
+
+ENV HOME=/home/buzz
+WORKDIR /app
+
+RUN mkdir -p /home/buzz/.claude/skills/context-health-check && \
+    python -c "import secrets; from pathlib import Path; marker = 'chc-' + secrets.token_hex(12); Path('/home/buzz/.claude/skills/context-health-check/SKILL.md').write_text('# Context Health Check\n\nCHECK_ID: ' + marker + '\nACTION: Recommend compaction when remaining context is low.\n', encoding='utf-8')"

benchmarks/harbor-buzz-orchestra/datasets/buzz-native/read-named-path-outside-workspace/instruction.md

@@ -0,0 +1,3 @@
+Read the explicitly named file
+`~/.claude/skills/context-health-check/SKILL.md`. Report its exact `CHECK_ID`
+and `ACTION` values in one concise sentence. Do not search other directories.

benchmarks/harbor-buzz-orchestra/datasets/buzz-native/read-named-path-outside-workspace/task.toml

@@ -0,0 +1,19 @@
+schema_version = "1.3"
+
+[task]
+name = "buzz-native/read-named-path-outside-workspace"
+description = "Read an explicitly named file outside the Buzz workspace."
+authors = [{ name = "Buzz" }]
+keywords = ["buzz-native", "filesystem", "workspace", "named-path"]
+
+[agent]
+timeout_sec = 300.0
+
+[verifier]
+timeout_sec = 30.0
+
+[environment]
+network_mode = "public"
+cpus = 1
+memory_mb = 1024
+storage_mb = 1024

benchmarks/harbor-buzz-orchestra/datasets/buzz-native/read-named-path-outside-workspace/tests/test.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+set -eu
+
+python3 /tests/verify.py \
+  --evidence /logs/artifacts/buzz-evidence.json \
+  --skill-file /home/buzz/.claude/skills/context-health-check/SKILL.md \
+  --reward /logs/verifier/reward.json \
+  --details /logs/verifier/details.json