From 70633697224ac8fd501675f3537d556d27518cd4 Mon Sep 17 00:00:00 2001 From: ajosh0504 Date: Tue, 14 Jul 2026 16:16:20 -0700 Subject: [PATCH 1/4] Add Remote MCP performance-triage demo An end-to-end demo of an AI agent triaging a payments performance incident via the MongoDB Remote MCP plugin: a slow checkout status-poll query (missing index) is diagnosed with explain() and the Atlas Performance Advisor, fixed with a compound index, and verified. - seed_payments.py: seeds a realistic ~300k payments collection (no index on session_id; sized so a COLLSCAN is ~4s warm / ~30s cold, under the MCP 60s cap) - generate_load.py: trickle load to keep the Performance Advisor recommendation fresh and the cache warm - post_alert.py: posts an Atlas-styled Query Targeting alert to Slack - README.md: story, setup, run, and reset/reseed instructions --- apps/remote-mcp-perf-triage/README.md | 168 +++++++++++++++++ apps/remote-mcp-perf-triage/generate_load.py | 125 ++++++++++++ apps/remote-mcp-perf-triage/post_alert.py | 150 +++++++++++++++ apps/remote-mcp-perf-triage/requirements.txt | 1 + apps/remote-mcp-perf-triage/seed_payments.py | 189 +++++++++++++++++++ 5 files changed, 633 insertions(+) create mode 100644 apps/remote-mcp-perf-triage/README.md create mode 100644 apps/remote-mcp-perf-triage/generate_load.py create mode 100644 apps/remote-mcp-perf-triage/post_alert.py create mode 100644 apps/remote-mcp-perf-triage/requirements.txt create mode 100644 apps/remote-mcp-perf-triage/seed_payments.py diff --git a/apps/remote-mcp-perf-triage/README.md b/apps/remote-mcp-perf-triage/README.md new file mode 100644 index 00000000..96b143c5 --- /dev/null +++ b/apps/remote-mcp-perf-triage/README.md @@ -0,0 +1,168 @@ +# Remote MCP — Performance Triage Demo + +A live demo of an AI agent triaging a production performance incident end-to-end, +entirely through the **MongoDB Remote MCP** plugin. A vague, cluster-level alert +lands in Slack; the agent connects to Atlas, pinpoints the offending collection and +query, consults the Atlas **Performance Advisor**, creates the missing index, and +verifies the fix — without the developer ever leaving the chat or writing a query +by hand. + +## The story + +**Setup (world before the incident):** An e-commerce app uses MongoDB Atlas as the +system of record for payments. After a user clicks *Pay*, the checkout page polls +MongoDB until the payment status flips to `completed`: + +```js +db.payments.findOne({ session_id: "sess_...", status: "completed" }) +``` + +The `payments` collection has grown to millions of documents, and there is **no +index on `session_id`**. Every poll is a full collection scan. + +**Incident:** Checkout hangs on "Processing your payment…" and times out. A +**Query Targeting** alert (scanned objects / returned too high) fires from Atlas +into a Slack channel. The alert is cluster-level and vague — it does *not* name the +collection, query, or index. Pinpointing that is the agent's job. + +**Triage (live, via MCP):** +1. Agent inspects the slow query and runs `explain()` → `COLLSCAN`, every document examined (~3.5–4 s). +2. Agent consults the **Performance Advisor** → confirms a missing index on `session_id`. +3. Agent creates the index `{ session_id: 1, status: 1 }`. +4. Agent re-runs `explain()` → `IXSCAN`, ~1 doc examined, sub-millisecond. Checkout recovers. + +**Takeaway:** a conversational agent traversed from a *business symptom* (failed +checkouts in Slack) to a *database root cause* and fix — cross-layer triage that +would normally require a developer who knows exactly where to look. + +## What's real vs. staged + +- **Real:** the seeded collection, the slow query, the `explain()` output, the + Performance Advisor recommendation, and the index creation. The agent does + genuine work against a live M10 cluster over MCP. +- **Staged:** only the *delivery* of the Slack alert (`post_alert.py`). We mimic the + appearance of an Atlas alert rather than configuring Atlas alerting, so the alert + fires on cue with no evaluation lag. The demo begins from "the alert has arrived." + +## Files + +| File | Purpose | +|------|---------| +| `seed_payments.py` | Seeds a large, realistic `payments` collection (no index on `session_id`). | +| `generate_load.py` | Runs the checkout status-poll query repeatedly to feed Performance Advisor. | +| `post_alert.py` | Posts an Atlas-styled "Query Targeting" alert to Slack via an incoming webhook. | +| `requirements.txt` | `pymongo` (for the seed and load scripts). | + +## Prerequisites + +- An Atlas **M10** cluster (dedicated tier — required for the full Performance Advisor). +- A **read-write database user** + connection string for the seed/load scripts. + This is a direct MongoDB connection, separate from the Remote MCP OAuth grant. +- The **Remote MCP** plugin connected to the same Atlas org/project, with MCP access + enabled for the org (Org Admin → App Authorizations). +- A **Slack incoming webhook** URL for the alert (a personal workspace works fine). + +## Setup + +```bash +pip install -r requirements.txt +export MONGODB_URI="mongodb+srv://:@cluster0.<...>.mongodb-dev.net/" +``` + +### 1. Seed the data + +```bash +python seed_payments.py # ~300,000 docs, ~1.6 KB gateway payload each +``` + +**Sizing (measured on M10):** 300k docs of ~2 KB is the sweet spot. A COLLSCAN of +the poll query runs **~30 s cold** and settles to **~3.5–4 s warm** — clearly slow +and dramatic in `explain()`, yet safely under the MongoDB MCP server's **60 s +`maxTimeMS` cap**. Do **not** seed millions: a scan that large can exceed the 60 s +cap, which makes the agent's `explain()`/`find()` **error out** during the demo +instead of returning stats. Bigger is worse, not better. + +Document size lives in a realistic `gateway_response` field (an opaque base64 +payload — screenshot-safe, and high-entropy so WiredTiger's compression can't shrink +the collection and let it be served entirely from cache). + +### 2. Keep the slow-query pattern fresh (trickle load) + +Performance Advisor builds its recommendation from slow queries in a rolling ~24h +window. The recommendation only persists while slow queries **keep recurring** AND +the index is **still absent** — so keep a gentle trickle running until showtime, and +**do not create the index before the demo**. + +```bash +python generate_load.py # default trickle: 3 queries every 5 min, until Ctrl+C +``` + +Recency matters more than volume, and the M10 has a burstable CPU, so a light trickle +is enough (and kinder to the cluster) — no need to hammer it. To run unattended: + +```bash +nohup python generate_load.py > load.log 2>&1 & # background; tail -f load.log to watch +``` + +Notes: +- Give Performance Advisor ~15–30 min of traffic to first surface the recommendation. +- Warm the cache before demoing (let the trickle run a bit) so scans are ~4 s, not ~30 s. +- Confirm readiness with `atlas-get-performance-advisor` (expect a suggested index on + `{ session_id: 1, status: 1 }` for `ecommerce.payments`). + +## Running the demo + +1. Post the alert to Slack: + ```bash + export SLACK_WEBHOOK_URL="https://hooks.slack.com/services/XXX/YYY/ZZZ" + python post_alert.py + ``` +2. In the AI client (Claude Code / Codex) connected via Remote MCP, hand the agent + the incident and let it triage: inspect the query, `explain()`, read the + Performance Advisor, create the index, and re-verify with `explain()`. +3. (Optional closing beat) Post a resolved/green alert to mirror Atlas detecting recovery. + +## The fix + +```js +db.payments.createIndex({ session_id: 1, status: 1 }) +``` + +`session_id` is the selective field; the compound index also covers the `status` +predicate. After creation the poll query uses `IXSCAN`, examines ~1 document, and +completes sub-millisecond — well under any checkout timeout. + +## Reset (to re-run the demo) + +### Light reset (usual case) + +If the collection is intact and you just created the index during the demo, drop the +index so the slow condition returns: + +```js +db.payments.dropIndex("session_id_1_status_1") +``` + +Then make sure the trickle is running again ahead of the next run: + +```bash +nohup python generate_load.py > load.log 2>&1 & # background; tail -f load.log to watch +``` + +### Full reseed ritual (fresh data) + +If you need to rebuild the collection from scratch: + +```bash +python seed_payments.py --drop # drop, then seed a fresh 300k +nohup python generate_load.py > load.log 2>&1 & # warm cache + rebuild Advisor recommendation +``` + +**Always use `--drop` when reseeding.** Running `seed_payments.py` without it *appends* +another 300k (→ 600k total), which roughly doubles scan time and pushes the cold case +into the MCP server's 60 s `maxTimeMS` cap — the failure mode 300k is sized to avoid. + +After any reseed, remember: +- The cache is cold again (first scans ~30 s, settling to ~4 s) — let the trickle warm it. +- The Performance Advisor recommendation resets with the collection — give the trickle + ~15–30 min to rebuild it, then confirm with `atlas-get-performance-advisor` before going live. diff --git a/apps/remote-mcp-perf-triage/generate_load.py b/apps/remote-mcp-perf-triage/generate_load.py new file mode 100644 index 00000000..9aad9c41 --- /dev/null +++ b/apps/remote-mcp-perf-triage/generate_load.py @@ -0,0 +1,125 @@ +""" +Generate checkout status-poll traffic against the unindexed `payments` collection. + +This stands in for real production traffic. It runs the checkout page's status-poll +query on an interval so that (a) Atlas Performance Advisor observes the slow query +pattern and surfaces an index recommendation, and (b) the pattern stays fresh in the +Advisor's rolling ~24h window right up to demo time. + +The query mirrors what a checkout page runs while waiting for a payment to confirm: + db.payments.findOne({ session_id: , status: "completed" }) + +It queries session_ids with no completed record yet (still "pending") — realistic, +since the page polls before the processor webhook flips the status. With no index on +session_id, every poll is a full COLLSCAN. + +No maxTimeMS here: we WANT each query to complete slowly so it lands in the slow-query +log (>100 ms) that Performance Advisor analyzes. The "checkout timing out" symptom is +a narrative prop delivered via the staged Slack alert (post_alert.py). + +TRICKLE MODE (default): recency matters more than volume for Performance Advisor, and +the M10 has a burstable CPU, so by default this runs a small BURST of queries every +INTERVAL seconds rather than hammering continuously. That keeps the recommendation +alive and the cache warm while being gentle on the cluster. + +Usage: + export MONGODB_URI="mongodb+srv://:@host/" + + # Default trickle: 3 queries every 5 minutes, forever (Ctrl+C to stop). + python generate_load.py + + # Custom trickle cadence. + python generate_load.py --burst 5 --interval 900 + + # Continuous (hammer) mode: back-to-back queries, no sleep between bursts. + python generate_load.py --interval 0 + + # Run unattended in the background, logging to a file: + nohup python generate_load.py > load.log 2>&1 & + # ...check on it later: tail -f load.log ; stop it: kill %1 (or the PID) + + # Or via cron — a burst every 15 minutes (no long-running process): + # */15 * * * * cd /path/to/apps/remote-mcp-perf-triage && \ + # MONGODB_URI="mongodb+srv://..." python generate_load.py --burst 3 --interval 0 --duration 30 +""" + +import argparse +import os +import secrets +import sys +import time +from datetime import datetime + +try: + from pymongo import MongoClient +except ImportError: + sys.exit("pymongo not installed. Run: pip install -r requirements.txt") + +DB_NAME = "ecommerce" +COLLECTION_NAME = "payments" + +POLL_FILTER_STATUS = "completed" + + +def run_query(coll): + """Run one checkout status-poll query; return its latency in ms. + + A random session_id that has no 'completed' record forces a full COLLSCAN. + """ + session_id = f"sess_{secrets.token_hex(12)}" + t0 = time.perf_counter() + coll.find_one({"session_id": session_id, "status": POLL_FILTER_STATUS}) + return (time.perf_counter() - t0) * 1000 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--burst", type=int, default=3, + help="number of queries to run per cycle") + parser.add_argument("--interval", type=float, default=300, + help="seconds between the start of each burst " + "(0 = continuous / no sleep between bursts)") + parser.add_argument("--duration", type=float, default=0, + help="total seconds to run (0 = run until Ctrl+C)") + args = parser.parse_args() + + uri = os.environ.get("MONGODB_URI") + if not uri: + sys.exit('ERROR: set MONGODB_URI, e.g.\n export MONGODB_URI="mongodb+srv://user:pass@host/"') + + client = MongoClient(uri, appname="perf-triage-demo-load") + coll = client[DB_NAME][COLLECTION_NAME] + client.admin.command("ping") + + mode = "continuous" if args.interval == 0 else f"trickle ({args.burst} queries / {args.interval:.0f}s)" + limit = "until Ctrl+C" if args.duration == 0 else f"{args.duration:.0f}s" + print(f"Polling {DB_NAME}.{COLLECTION_NAME} — {mode}, {limit}. " + "Each query is a full COLLSCAN (no index on session_id).") + + deadline = time.time() + args.duration if args.duration > 0 else None + total = 0 + try: + while deadline is None or time.time() < deadline: + cycle_start = time.time() + latencies = [run_query(coll) for _ in range(args.burst)] + total += len(latencies) + + avg = sum(latencies) / len(latencies) + slow = 100 * sum(1 for x in latencies if x > 100) / len(latencies) + ts = datetime.now().strftime("%H:%M:%S") + print(f" {ts} burst of {len(latencies)}: avg {avg:7.1f} ms | " + f"{slow:3.0f}% over 100 ms | {total:,} total", flush=True) + + if args.interval > 0: + sleep_for = args.interval - (time.time() - cycle_start) + if sleep_for > 0 and (deadline is None or time.time() + sleep_for < deadline): + time.sleep(sleep_for) + elif deadline is not None: + break # not enough time left for another cycle + except KeyboardInterrupt: + pass + print(f"\nStopped after {total:,} queries.") + + +if __name__ == "__main__": + main() diff --git a/apps/remote-mcp-perf-triage/post_alert.py b/apps/remote-mcp-perf-triage/post_alert.py new file mode 100644 index 00000000..3ad6780b --- /dev/null +++ b/apps/remote-mcp-perf-triage/post_alert.py @@ -0,0 +1,150 @@ +""" +Post a MongoDB Atlas-styled alert to a Slack channel via an incoming webhook. + +This mimics the appearance of a real Atlas "Query Targeting: Scanned Objects / +Returned" alert notification without configuring or triggering Atlas alerting. +It is a demo prop: the substance of the demo (the agent's live triage against a +real slow query + Performance Advisor) is genuine; only the alert delivery is staged. + +Usage: + export SLACK_WEBHOOK_URL="https://hooks.slack.com/services/XXX/YYY/ZZZ" + python post_alert.py + + # Optional overrides: + python post_alert.py --project "Apoorva Test" --cluster "payments-prod" \ + --ratio 2847 --atlas-url "https://cloud-dev.mongodb.com/v2/#/alerts" +""" + +import argparse +import json +import os +import sys +import urllib.request +from datetime import datetime, timezone + +# MongoDB leaf logo (used as the Slack sender icon so the message reads as "from Atlas") +MONGODB_ICON_URL = "https://www.mongodb.com/assets/images/global/favicon.ico" + +# Atlas alert accent color for a triggered alert (Atlas uses a red/amber bar) +ATLAS_ALERT_COLOR = "#B71C1C" + + +def build_payload(project, cluster, ratio, threshold, atlas_url): + """Construct a Slack message that mirrors an Atlas Query Targeting alert.""" + fired_at = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") + + # Block Kit body rendered inside a colored attachment so we get the accent bar. + blocks = [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "🔴 Alert: Query Targeting", + "emoji": True, + }, + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": ( + "*Query Targeting: Scanned Objects / Returned has gone above " + f"{threshold}*\n" + "Queries are scanning far more documents than they return, " + "indicating a missing or ineffective index." + ), + }, + }, + { + "type": "section", + "fields": [ + {"type": "mrkdwn", "text": f"*Project*\n{project}"}, + {"type": "mrkdwn", "text": f"*Cluster*\n{cluster}"}, + {"type": "mrkdwn", "text": f"*Metric*\nScanned / Returned"}, + {"type": "mrkdwn", "text": f"*Current Value*\n{ratio:,} : 1"}, + {"type": "mrkdwn", "text": f"*Condition*\n> {threshold} : 1"}, + {"type": "mrkdwn", "text": f"*Fired At*\n{fired_at}"}, + ], + }, + { + "type": "actions", + "elements": [ + { + "type": "button", + "text": {"type": "plain_text", "text": "View Alert in Atlas"}, + "url": atlas_url, + "style": "danger", + } + ], + }, + ] + + return { + "username": "MongoDB Atlas", + "icon_url": MONGODB_ICON_URL, + "attachments": [ + { + "color": ATLAS_ALERT_COLOR, + "blocks": blocks, + "fallback": ( + f"[{project}/{cluster}] Query Targeting: Scanned Objects / " + f"Returned has gone above {threshold} (current {ratio}:1)" + ), + } + ], + } + + +def post(webhook_url, payload): + data = json.dumps(payload).encode("utf-8") + req = urllib.request.Request( + webhook_url, data=data, headers={"Content-Type": "application/json"} + ) + with urllib.request.urlopen(req) as resp: + return resp.status, resp.read().decode("utf-8") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--project", default="Apoorva Test", help="Atlas project name") + parser.add_argument( + "--cluster", default="payments-prod", help="Atlas cluster name shown in alert" + ) + parser.add_argument( + "--ratio", + type=int, + default=2847, + help="Current scanned:returned ratio to display", + ) + parser.add_argument( + "--threshold", + type=int, + default=1000, + help="Alert threshold (scanned:returned) to display", + ) + parser.add_argument( + "--atlas-url", + default="https://cloud-dev.mongodb.com/v2#/alerts", + help="URL the 'View Alert in Atlas' button links to", + ) + args = parser.parse_args() + + webhook_url = os.environ.get("SLACK_WEBHOOK_URL") + if not webhook_url: + sys.exit( + "ERROR: set SLACK_WEBHOOK_URL, e.g.\n" + ' export SLACK_WEBHOOK_URL="https://hooks.slack.com/services/XXX/YYY/ZZZ"' + ) + + payload = build_payload( + args.project, args.cluster, args.ratio, args.threshold, args.atlas_url + ) + status, body = post(webhook_url, payload) + if status == 200: + print("Alert posted to Slack.") + else: + sys.exit(f"Slack returned {status}: {body}") + + +if __name__ == "__main__": + main() diff --git a/apps/remote-mcp-perf-triage/requirements.txt b/apps/remote-mcp-perf-triage/requirements.txt new file mode 100644 index 00000000..569647ea --- /dev/null +++ b/apps/remote-mcp-perf-triage/requirements.txt @@ -0,0 +1 @@ +pymongo>=4.6 diff --git a/apps/remote-mcp-perf-triage/seed_payments.py b/apps/remote-mcp-perf-triage/seed_payments.py new file mode 100644 index 00000000..3e3b696a --- /dev/null +++ b/apps/remote-mcp-perf-triage/seed_payments.py @@ -0,0 +1,189 @@ +""" +Seed a large, realistic `payments` collection on an Atlas M10 for the +performance-triage demo. + +Goal: make a full collection scan genuinely slow so that (a) the checkout +status-poll query trips the slow-query log that feeds Performance Advisor, and +(b) a live explain() shows a real COLLSCAN examining every document. + +Two design constraints, satisfied together: + +1. The scan must be reliably slow. Empirically on an M10 (measured), 300,000 + documents of ~2 KB is the sweet spot: a COLLSCAN runs ~30 s cold and settles + to ~3.5-4 s once the cache is warm. That is clearly slow and dramatic in + explain(), yet stays safely under the MongoDB MCP server's 60 s maxTimeMS cap. + NOTE: do NOT seed millions here — a scan of that size can exceed the 60 s cap, + making the agent's explain()/find() ERROR out during the demo instead of + returning stats. Bigger is worse, not better. + + The document bytes live in a realistic `gateway_response` field (an opaque + base64 payload — exactly what payment processors return and apps store for + reconciliation). It is high-entropy so WiredTiger's snappy compression can't + shrink the collection, AND screenshot-safe: if a document appears on screen + during the demo, it reads as a real payment record, not obvious junk padding. + +2. There is deliberately NO index on `session_id` (the field the checkout poll + filters on). Only the default _id index exists. + +Tip: after seeding, warm the cache (run generate_load.py or a few scans) so the +demo sees the ~4 s warm time rather than the ~30 s cold time. + +Usage: + export MONGODB_URI="mongodb+srv://:@cluster0.z7basj.mongodb-dev.net/" + python seed_payments.py # defaults: 300,000 docs, ~1.6 KB gateway blob + python seed_payments.py --docs 300000 --blob-bytes 1600 + python seed_payments.py --drop # drop the collection first (fresh reseed) +""" + +import argparse +import base64 +import os +import random +import secrets +import sys +import time +from datetime import datetime, timedelta, timezone + +try: + from pymongo import MongoClient, InsertOne +except ImportError: + sys.exit("pymongo not installed. Run: pip install -r requirements.txt") + +DB_NAME = "ecommerce" +COLLECTION_NAME = "payments" + +# Realistic reference data. These enum-ish fields are a small fraction of each +# document, so their compressibility doesn't matter — the bulk is the random blob. +STATUS_WEIGHTS = [("completed", 0.92), ("pending", 0.05), ("failed", 0.03)] +PAYMENT_METHODS = ["card", "paypal", "apple_pay", "google_pay", "bank_transfer"] +CURRENCIES = ["USD", "EUR", "GBP", "CAD"] +CARD_BRANDS = ["visa", "mastercard", "amex", "discover"] +GATEWAYS = ["stripe", "adyen", "braintree"] +CITIES = [ + ("Austin", "TX", "US"), ("Seattle", "WA", "US"), ("Denver", "CO", "US"), + ("London", "", "GB"), ("Toronto", "ON", "CA"), ("Berlin", "", "DE"), + ("Dublin", "", "IE"), ("New York", "NY", "US"), +] +PRODUCTS = [ + ("SKU-1001", "Wireless Headphones"), ("SKU-1002", "USB-C Cable"), + ("SKU-1003", "Laptop Stand"), ("SKU-1004", "Mechanical Keyboard"), + ("SKU-1005", "4K Monitor"), ("SKU-1006", "Webcam"), + ("SKU-1007", "Desk Lamp"), ("SKU-1008", "Notebook"), +] + + +def weighted_status(): + r = random.random() + cumulative = 0.0 + for status, weight in STATUS_WEIGHTS: + cumulative += weight + if r <= cumulative: + return status + return "completed" + + +def gateway_blob(blob_bytes): + """An opaque, high-entropy base64 payload — realistic AND incompressible. + + base64 of random bytes has no repeated structure, so WiredTiger's snappy + compression can't shrink it: the collection stays large on disk. + """ + raw = os.urandom(max(1, blob_bytes * 3 // 4)) # base64 expands ~4/3 + return base64.b64encode(raw).decode("ascii") + + +def make_doc(blob_bytes, base_time): + """Build one realistic payment document.""" + city, state, country = random.choice(CITIES) + n_items = random.randint(1, 3) + line_items = [] + for _ in range(n_items): + sku, name = random.choice(PRODUCTS) + line_items.append({ + "sku": sku, + "name": name, + "qty": random.randint(1, 4), + "unit_price": random.randint(500, 20_000), + }) + amount = sum(i["qty"] * i["unit_price"] for i in line_items) + created = base_time - timedelta(seconds=random.randint(0, 90 * 24 * 3600)) + return { + # High-cardinality session id — the checkout poll filters on this (unindexed). + "session_id": f"sess_{secrets.token_hex(12)}", + "order_id": f"ord_{random.randint(100000, 999999)}", + "user_id": f"usr_{random.randint(1, 500_000)}", + "amount": amount, + "currency": random.choice(CURRENCIES), + "status": weighted_status(), + "payment_method": random.choice(PAYMENT_METHODS), + "card": { + "brand": random.choice(CARD_BRANDS), + "last4": f"{random.randint(0, 9999):04d}", + "exp_month": random.randint(1, 12), + "exp_year": random.randint(2026, 2031), + "fingerprint": secrets.token_hex(8), + }, + "billing_address": { + "city": city, "state": state, "country": country, + "postal_code": f"{random.randint(10000, 99999)}", + }, + "line_items": line_items, + "gateway": random.choice(GATEWAYS), + # The bulk of the document size: an opaque processor response payload. + "gateway_response": gateway_blob(blob_bytes), + "risk_score": random.randint(0, 99), + "created_at": created, + "updated_at": created + timedelta(seconds=random.randint(1, 30)), + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--docs", type=int, default=300_000, help="number of documents to insert") + parser.add_argument("--blob-bytes", type=int, default=1600, + help="approx size of the gateway_response payload per doc") + parser.add_argument("--batch", type=int, default=5000, help="insert batch size") + parser.add_argument("--drop", action="store_true", help="drop the collection before seeding") + args = parser.parse_args() + + uri = os.environ.get("MONGODB_URI") + if not uri: + sys.exit('ERROR: set MONGODB_URI, e.g.\n export MONGODB_URI="mongodb+srv://user:pass@host/"') + + client = MongoClient(uri, appname="perf-triage-demo-seed") + coll = client[DB_NAME][COLLECTION_NAME] + + # Fail fast on bad credentials / network before the long insert loop. + client.admin.command("ping") + + if args.drop: + print(f"Dropping {DB_NAME}.{COLLECTION_NAME} ...") + coll.drop() + + print(f"Seeding {args.docs:,} docs (~{args.blob_bytes}B gateway payload each) into " + f"{DB_NAME}.{COLLECTION_NAME} — NO index on session_id.") + base_time = datetime.now(timezone.utc) + start = time.time() + inserted = 0 + + while inserted < args.docs: + n = min(args.batch, args.docs - inserted) + ops = [InsertOne(make_doc(args.blob_bytes, base_time)) for _ in range(n)] + coll.bulk_write(ops, ordered=False) + inserted += n + if inserted % 100_000 == 0 or inserted == args.docs: + elapsed = time.time() - start + rate = inserted / elapsed if elapsed else 0 + print(f" {inserted:,}/{args.docs:,} ({rate:,.0f} docs/s, {elapsed:,.0f}s elapsed)") + + stats = client[DB_NAME].command("collstats", COLLECTION_NAME) + size_gb = stats.get("size", 0) / 1e9 + storage_gb = stats.get("storageSize", 0) / 1e9 + print(f"\nDone. Logical size ~{size_gb:.2f} GB, on-disk storage ~{storage_gb:.2f} GB.") + print(f"Indexes present: {list(coll.index_information().keys())} (expect only _id_)") + print("Next: run generate_load.py to warm the cache and feed Performance Advisor, " + "then confirm scan time with an explain() (expect ~30 s cold, ~4 s warm on M10).") + + +if __name__ == "__main__": + main() From aedf330376d27cd84f64fa528b787f75e889d2f3 Mon Sep 17 00:00:00 2001 From: ajosh0504 Date: Wed, 29 Jul 2026 14:56:23 -0700 Subject: [PATCH 2/4] Performance traige demo --- apps/README.md | 1 + apps/remote-mcp-perf-triage/.gitignore | 6 + apps/remote-mcp-perf-triage/README.md | 163 ++++-- apps/remote-mcp-perf-triage/checkout_app.py | 472 ++++++++++++++++++ apps/remote-mcp-perf-triage/generate_load.py | 58 ++- apps/remote-mcp-perf-triage/requirements.txt | 3 + apps/remote-mcp-perf-triage/seed_payments.py | 44 +- .../remote-mcp-perf-triage/trigger_chatgpt.py | 329 ++++++++++++ 8 files changed, 1027 insertions(+), 49 deletions(-) create mode 100644 apps/remote-mcp-perf-triage/.gitignore create mode 100644 apps/remote-mcp-perf-triage/checkout_app.py create mode 100644 apps/remote-mcp-perf-triage/trigger_chatgpt.py diff --git a/apps/README.md b/apps/README.md index 728ea30d..d49c5828 100644 --- a/apps/README.md +++ b/apps/README.md @@ -12,3 +12,4 @@ Javascript and Python apps and demos showcasing how to use MongoDB in GenAI appl | Voice OpenAI Rentals Agent | MongoDB Atlas, Python, Flask, OpenAI | Voice interaction agent for real estate listings using vector search and real-time voice processing | [![View Code](https://img.shields.io/badge/view-code-blue?logo=github)](voice-openai-mongo-rentals-agent) | | Video Intelligence App | MongoDB Atlas, FastAPI, React, OpenAI, VoyageAI | AI-powered video search system allowing users to upload videos, extract insights, and search content using natural language queries | [![View Code](https://img.shields.io/badge/view-code-blue?logo=github)](video-intelligence) | | Voice AI Agents with Memory | MongoDB Atlas, VoyageAI, Gemini Live | Voice AI Agents powerd with MongoDB Hybrid searchable memory| [![View Code](https://img.shields.io/badge/view-code-blue?logo=github)](voice-memory-demo) | +| Remote MCP Performance Triage | MongoDB Atlas, Python, MongoDB Remote MCP, ChatGPT Workspace Agents | A ChatGPT Workspace Agent triages a simulated PagerDuty incident end-to-end over Remote MCP — finding the slow query, consulting Performance Advisor, and proposing the missing index for human approval | [![View Code](https://img.shields.io/badge/view-code-blue?logo=github)](remote-mcp-perf-triage) | diff --git a/apps/remote-mcp-perf-triage/.gitignore b/apps/remote-mcp-perf-triage/.gitignore new file mode 100644 index 00000000..ce38d252 --- /dev/null +++ b/apps/remote-mcp-perf-triage/.gitignore @@ -0,0 +1,6 @@ +__pycache__/ +*.py[cod] +.env +.venv/ +load.log +checkout.log diff --git a/apps/remote-mcp-perf-triage/README.md b/apps/remote-mcp-perf-triage/README.md index 96b143c5..8f2f114b 100644 --- a/apps/remote-mcp-perf-triage/README.md +++ b/apps/remote-mcp-perf-triage/README.md @@ -1,11 +1,10 @@ # Remote MCP — Performance Triage Demo -A live demo of an AI agent triaging a production performance incident end-to-end, -entirely through the **MongoDB Remote MCP** plugin. A vague, cluster-level alert -lands in Slack; the agent connects to Atlas, pinpoints the offending collection and -query, consults the Atlas **Performance Advisor**, creates the missing index, and -verifies the fix — without the developer ever leaving the chat or writing a query -by hand. +A live demo of a ChatGPT Workspace Agent triaging a production performance incident +end-to-end through the **MongoDB Remote MCP** plugin. A simulated PagerDuty incident +triggers the agent through the Workspace Agents API. The agent connects to Atlas, +pinpoints the offending collection and query, consults the Atlas **Performance +Advisor**, proposes the missing index, and verifies the approved fix. ## The story @@ -17,40 +16,48 @@ MongoDB until the payment status flips to `completed`: db.payments.findOne({ session_id: "sess_...", status: "completed" }) ``` -The `payments` collection has grown to millions of documents, and there is **no +The `payments` collection has grown to hundreds of thousands of documents, and there is **no index on `session_id`**. Every poll is a full collection scan. -**Incident:** Checkout hangs on "Processing your payment…" and times out. A -**Query Targeting** alert (scanned objects / returned too high) fires from Atlas -into a Slack channel. The alert is cluster-level and vague — it does *not* name the -collection, query, or index. Pinpointing that is the agent's job. +**Incident:** Checkout hangs on "Processing your payment…" and times out. A Datadog +monitor observes an elevated payment-confirmation timeout rate, causing +PagerDuty to open a SEV-2 incident. The incident contains application symptoms but +does *not* identify MongoDB, a collection, a query, or an index. Pinpointing the +database cause is the agent's job. **Triage (live, via MCP):** -1. Agent inspects the slow query and runs `explain()` → `COLLSCAN`, every document examined (~3.5–4 s). +1. Agent inspects the slow query and runs `explain()` → `COLLSCAN`, every document examined (~5 s). 2. Agent consults the **Performance Advisor** → confirms a missing index on `session_id`. -3. Agent creates the index `{ session_id: 1, status: 1 }`. -4. Agent re-runs `explain()` → `IXSCAN`, ~1 doc examined, sub-millisecond. Checkout recovers. +3. Agent proposes the index `{ session_id: 1, status: 1 }` and waits for approval. +4. After approval, the agent creates the index and re-runs `explain()` → `IXSCAN`, + ~1 doc examined, sub-millisecond. Checkout recovers. **Takeaway:** a conversational agent traversed from a *business symptom* (failed -checkouts in Slack) to a *database root cause* and fix — cross-layer triage that +checkouts in PagerDuty) to a *database root cause* and fix — cross-layer triage that would normally require a developer who knows exactly where to look. ## What's real vs. staged - **Real:** the seeded collection, the slow query, the `explain()` output, the Performance Advisor recommendation, and the index creation. The agent does - genuine work against a live M10 cluster over MCP. -- **Staged:** only the *delivery* of the Slack alert (`post_alert.py`). We mimic the - appearance of an Atlas alert rather than configuring Atlas alerting, so the alert - fires on cue with no evaluation lag. The demo begins from "the alert has arrived." + genuine work against a live M10 cluster over MCP. The checkout page's hang is also + real — it runs the same unindexed poll query and genuinely exceeds its timeout. +- **Staged:** PagerDuty itself. `trigger_chatgpt.py` acts as the adapter that sends a + PagerDuty incident resource to the Workspace Agents API. The incident contains + application symptoms, but no database namespace, query shape, root cause, or index + recommendation. Also staged: the payment processor's confirmation webhook, which + `checkout_app.py` simulates with a ~2 s delay, and PagerDuty's on-call resolution — + the assignee is a fixed name, not resolved from a real schedule. ## Files | File | Purpose | |------|---------| -| `seed_payments.py` | Seeds a large, realistic `payments` collection (no index on `session_id`). | +| `checkout_app.py` | The Contoso checkout page. Really hangs on the slow poll, then fires the incident to ChatGPT. | +| `seed_payments.py` | Seeds a large, realistic `payments` collection (no index on `session_id`); `--drop-index` resets the demo. | | `generate_load.py` | Runs the checkout status-poll query repeatedly to feed Performance Advisor. | | `post_alert.py` | Posts an Atlas-styled "Query Targeting" alert to Slack via an incoming webhook. | +| `trigger_chatgpt.py` | Sends a realistic PagerDuty-style incident to a published ChatGPT Workspace Agent and prints the conversation URL. | | `requirements.txt` | `pymongo` (for the seed and load scripts). | ## Prerequisites @@ -60,28 +67,46 @@ would normally require a developer who knows exactly where to look. This is a direct MongoDB connection, separate from the Remote MCP OAuth grant. - The **Remote MCP** plugin connected to the same Atlas org/project, with MCP access enabled for the org (Org Admin → App Authorizations). -- A **Slack incoming webhook** URL for the alert (a personal workspace works fine). +- A published ChatGPT Workspace Agent with an API channel and the MongoDB MCP plugin. +- A Workspace Agent access token. An admin must enable Workspace Agents and + **Allow users to create personal access tokens** under Admin > Permissions & roles. +- Optional: a **Slack incoming webhook** URL if you also want the alert shown in Slack. ## Setup ```bash pip install -r requirements.txt -export MONGODB_URI="mongodb+srv://:@cluster0.<...>.mongodb-dev.net/" ``` +Add the direct database connection used by the seed and load scripts to `.env`: + +```dotenv +MONGODB_URI="mongodb+srv://:@cluster0.<...>.mongodb.net/" +``` + +This URI must point to the same Atlas cluster exposed to the Workspace Agent through +MongoDB MCP. + ### 1. Seed the data ```bash python seed_payments.py # ~300,000 docs, ~1.6 KB gateway payload each ``` -**Sizing (measured on M10):** 300k docs of ~2 KB is the sweet spot. A COLLSCAN of -the poll query runs **~30 s cold** and settles to **~3.5–4 s warm** — clearly slow -and dramatic in `explain()`, yet safely under the MongoDB MCP server's **60 s +**Sizing (measured on M10, 2 GB RAM):** 300k docs of ~2 KB is the sweet spot. A +COLLSCAN of the poll query runs **~9 s cold** and settles to **~5 s warm** — clearly +slow and dramatic in `explain()`, yet safely under the MongoDB MCP server's **60 s `maxTimeMS` cap**. Do **not** seed millions: a scan that large can exceed the 60 s cap, which makes the agent's `explain()`/`find()` **error out** during the demo instead of returning stats. Bigger is worse, not better. +**The scan is only slow if the collection outgrows the WiredTiger cache.** That cache +is ~50% of host RAM, so a 2 GB M10 gives ~537 MB against this collection's ~0.63 GB — +scans hit disk and take seconds. On a larger tier the whole collection fits in cache +and the same query returns in **~200 ms**, which quietly kills the demo's drama (it +was measured at 222 ms on a 4 GB host). Check `hostInfo.memSizeMB` if the scan comes +back suspiciously fast; the fix is a smaller tier, not more documents. + Document size lives in a realistic `gateway_response` field (an opaque base64 payload — screenshot-safe, and high-entropy so WiredTiger's compression can't shrink the collection and let it be served entirely from cache). @@ -106,21 +131,84 @@ nohup python generate_load.py > load.log 2>&1 & # background; tail -f load.log Notes: - Give Performance Advisor ~15–30 min of traffic to first surface the recommendation. -- Warm the cache before demoing (let the trickle run a bit) so scans are ~4 s, not ~30 s. +- Warm the cache before demoing (let the trickle run a bit) so scans are ~5 s, not ~9 s. - Confirm readiness with `atlas-get-performance-advisor` (expect a suggested index on `{ session_id: 1, status: 1 }` for `ecommerce.payments`). ## Running the demo -1. Post the alert to Slack: +### The checkout page (recommended opening) + +`checkout_app.py` gives the demo a visual first act with no terminal on screen. It +serves a Contoso checkout that runs the **real** status-poll query against the **real** +unindexed collection — so the hang isn't staged, it's the bug: + +```bash +python checkout_app.py # http://127.0.0.1:8000 +python checkout_app.py --no-incident # rehearse without paging the agent +``` + +Click **Submit payment**. A real `pending` payment is inserted, a simulated processor +confirms it ~2 s later, and the page polls for the confirmation. Pre-index each poll is +a ~6 s COLLSCAN, so the page blows its 10 s budget and fails — then fires the PagerDuty +incident to the Workspace Agent automatically. The status panel shows each poll's +latency, so the audience sees *why* it hung. + +After the agent's index is approved, click **Submit payment** again: polls drop to +~20 ms and checkout confirms in ~2.5 s. That's the demo's closing beat. + +The incident fires **once**, so a rehearsal or a double-click doesn't spend the demo or +litter your workspace with conversations. The **Re-arm** control in the status panel +resets it. The ChatGPT token stays server-side; the browser never sees it. + +Each poll passes its *remaining* budget as `maxTimeMS`, so a slow scan can't outlive the +checkout timeout. Without that a 6 s poll starting at t=9.9 s would finish at t=15.9 s — +after the gateway confirmed the payment — and a checkout that should fail would succeed. + +Because the page is a server, it survives laptop sleep: start it before you leave, wake +the machine on stage with the tab already open, and click. Nothing to type. + +### Triggering from the CLI instead + +1. In ChatGPT, publish the incident agent to an API channel and copy its stable + `agtch_...` trigger identifier. Connect the MongoDB MCP plugin to the agent. Keep + its instructions generic: investigate incoming MongoDB incidents, gather evidence, + and require human approval before changes. Do not give it this demo's root cause. +2. Create a ChatGPT access token under Admin > Access tokens with the **Workspace + Agents** scope, then add both values to the git-ignored `.env` file: + ```dotenv + AGENT_ACCESS_TOKEN="..." + WORKSPACE_AGENT_TRIGGER_ID="agtch_..." + ``` +3. Trigger a new incident conversation: ```bash - export SLACK_WEBHOOK_URL="https://hooks.slack.com/services/XXX/YYY/ZZZ" - python post_alert.py + python trigger_chatgpt.py --wait ``` -2. In the AI client (Claude Code / Codex) connected via Remote MCP, hand the agent - the incident and let it triage: inspect the query, `explain()`, read the - Performance Advisor, create the index, and re-verify with `explain()`. -3. (Optional closing beat) Post a resolved/green alert to mirror Atlas detecting recovery. + Open the printed `https://chatgpt.com/c/...` URL. The agent receives only the + simulated alert fields and must discover the database issue through MongoDB MCP. +4. Approve the proposed database action in that conversation, then let the agent + re-run `explain()` and summarize the result. + +Preview the complete PagerDuty incident resource and Workspace Agents request without +credentials or a network call: + +```bash +python trigger_chatgpt.py --dry-run +``` + +`post_alert.py` remains optional if you also want a separate Atlas-style alert in Slack. + +The Workspace Agents API accepts a caller-defined `conversation_key`. The simulator +uses the PagerDuty incident ID for that key, so future webhook events for the same +incident can continue in one conversation. The separate event ID becomes the +`Idempotency-Key`, preventing a retried webhook from creating a duplicate task: + +```bash +python trigger_chatgpt.py \ + --incident-id QDEMO123456789 \ + --event-id 01DEMO123456789 \ + --wait +``` ## The fix @@ -139,10 +227,13 @@ completes sub-millisecond — well under any checkout timeout. If the collection is intact and you just created the index during the demo, drop the index so the slow condition returns: -```js -db.payments.dropIndex("session_id_1_status_1") +```bash +python seed_payments.py --drop-index # drops the index, keeps the 300k docs ``` +This exits before the insert loop, so your data is untouched. (Equivalent by hand: +`db.payments.dropIndex("session_id_1_status_1")`.) + Then make sure the trickle is running again ahead of the next run: ```bash @@ -163,6 +254,6 @@ another 300k (→ 600k total), which roughly doubles scan time and pushes the co into the MCP server's 60 s `maxTimeMS` cap — the failure mode 300k is sized to avoid. After any reseed, remember: -- The cache is cold again (first scans ~30 s, settling to ~4 s) — let the trickle warm it. +- The cache is cold again (first scans ~9 s, settling to ~5 s) — let the trickle warm it. - The Performance Advisor recommendation resets with the collection — give the trickle ~15–30 min to rebuild it, then confirm with `atlas-get-performance-advisor` before going live. diff --git a/apps/remote-mcp-perf-triage/checkout_app.py b/apps/remote-mcp-perf-triage/checkout_app.py new file mode 100644 index 00000000..8a82e71b --- /dev/null +++ b/apps/remote-mcp-perf-triage/checkout_app.py @@ -0,0 +1,472 @@ +"""Contoso checkout page — the demo's opening act, and a real reproduction of the bug. + +Nothing here is simulated except the payment processor. The page runs the SAME +status-poll query the demo is about, against the SAME unindexed collection: + + db.payments.findOne({ session_id: ..., status: "completed" }) + +Pre-index that poll is a ~6 s COLLSCAN, so the page genuinely blows its client-side +budget and genuinely fails. Post-index it returns in ~1 ms and the page succeeds. +The hang is not staged; it is the bug. + +Flow when you click "Submit payment": + 1. POST /api/pay inserts a real payment doc with status="pending", and + schedules a background task to flip it to "completed" + after GATEWAY_DELAY_S (stands in for the processor webhook). + 2. GET /api/status runs the real poll query. The browser calls this in a loop + and shows each attempt's latency in the status panel. + 3. On timeout, the browser calls POST /api/incident, which sends the PagerDuty + incident to the ChatGPT Workspace Agent — ONCE. A "Re-arm" + control resets it so a rehearsal doesn't spend the demo. + +The ChatGPT access token stays server-side; the browser never sees it. + +Usage: + python checkout_app.py # http://127.0.0.1:8000 + python checkout_app.py --port 9000 + python checkout_app.py --no-incident # rehearse without paging anyone +""" + +import argparse +import asyncio +import os +import secrets +import sys +import time +from datetime import datetime, timedelta, timezone + +try: + from fastapi import FastAPI + from fastapi.responses import HTMLResponse, JSONResponse + from pymongo import MongoClient + from pymongo.errors import ExecutionTimeout + from dotenv import load_dotenv + import uvicorn +except ImportError: + sys.exit("Missing deps. Run: pip install -r requirements.txt") + +# Reuse the incident builder and API client rather than duplicating the payload. +import trigger_chatgpt + +load_dotenv() + +DB_NAME = "ecommerce" +COLLECTION_NAME = "payments" + +# How long the fake processor takes to confirm. Realistic (real gateways take +# 1-3 s) and well under CLIENT_TIMEOUT_S, so post-index the page succeeds fast. +GATEWAY_DELAY_S = 3.0 + +# The user-facing budget: how long the shopper watches the spinner before the page +# gives up. Real checkouts often allow 30 s, but that is a long silence to narrate +# on stage — 12 s reads as clearly broken while staying brisk. Correctness does not +# depend on this value (see POLL_DEADLINE_MS), so it is safe to tune for pacing. +# Must stay comfortably above GATEWAY_DELAY_S or the post-index SUCCESS case breaks. +CLIENT_TIMEOUT_S = 12.0 + +# Gap between polls. Real checkout pages poll every 2-3 s rather than hammering. +# With a 12 s budget this yields 3 visible poll lines, ~5 s apart. +POLL_INTERVAL_MS = 2_500 + +# Per-request deadline for ONE poll, applied as maxTimeMS. Real services set a +# per-call deadline (API gateway, service SLO) far below the overall user budget, +# so having one is normal — but this VALUE is calibrated deliberately: +# +# Measured COLLSCANs on this cluster range 4.0-9.0 s. 2.5 s sits below the +# fastest of them, so EVERY pre-index poll is killed server-side before it can +# complete. No poll ever observes the gateway's confirmation, so checkout +# ALWAYS fails — regardless of cluster load on the day. +# +# Without this, the outcome depends on scan time vs. the overall budget: a poll +# that completes after the gateway confirms would find the record and checkout +# would SUCCEED, silently killing the demo. A 4.0 s burst was measured, so that +# edge is real, not theoretical. Post-index polls take ~20 ms and are unaffected. +POLL_DEADLINE_MS = 2_500 + +app = FastAPI(title="Contoso Checkout") + +# Module state. Single-process, single-presenter demo; no locking needed. +state = { + "incident_armed": True, + "incident_enabled": True, + "last_incident": None, +} + +client: MongoClient | None = None + + +def coll(): + return client[DB_NAME][COLLECTION_NAME] + + +ORDER = [ + {"sku": "SKU-1001", "name": "Wireless Headphones", "qty": 1, "unit_price": 14900}, + {"sku": "SKU-1002", "name": "USB-C Cable", "qty": 1, "unit_price": 1200}, +] +ORDER_TOTAL = sum(i["qty"] * i["unit_price"] for i in ORDER) + + +async def confirm_payment_later(session_id: str): + """Stand in for the payment processor's confirmation webhook.""" + await asyncio.sleep(GATEWAY_DELAY_S) + await asyncio.to_thread( + coll().update_one, + {"session_id": session_id}, + {"$set": {"status": "completed", "updated_at": datetime.now(timezone.utc)}}, + ) + + +@app.post("/api/pay") +async def pay(): + """Create a real pending payment, then let the 'processor' confirm it.""" + session_id = f"sess_{secrets.token_hex(12)}" + now = datetime.now(timezone.utc) + doc = { + "session_id": session_id, + "order_id": f"ord_{secrets.randbelow(900000) + 100000}", + "user_id": "usr_408122", + "amount": ORDER_TOTAL, + "currency": "USD", + "status": "pending", + "payment_method": "card", + "card": {"brand": "visa", "last4": "4242", "exp_month": 11, + "exp_year": 2029, "fingerprint": secrets.token_hex(8)}, + "billing_address": {"city": "Seattle", "state": "WA", + "country": "US", "postal_code": "98104"}, + "line_items": ORDER, + "gateway": "stripe", + "risk_score": 4, + "created_at": now, + "updated_at": now, + } + await asyncio.to_thread(coll().insert_one, doc) + asyncio.create_task(confirm_payment_later(session_id)) + return {"session_id": session_id, "amount": ORDER_TOTAL} + + +@app.get("/api/status") +async def status(session_id: str, budget_ms: int = POLL_DEADLINE_MS): + """The checkout poll. THIS is the slow query the whole demo is about. + + The query is bounded by whichever is SMALLER: this poll's own deadline + (POLL_DEADLINE_MS) or the checkout's remaining budget passed in as budget_ms. + Two different limits, both real: + + * the per-poll deadline is the service's own request SLO, and it's what + makes the pre-index failure deterministic (see POLL_DEADLINE_MS); + * the remaining-budget cap stops the last poll of a run from overrunning + the user-facing timeout, which would let a scan finish AFTER the gateway + confirms and turn a should-fail checkout into a success. + """ + budget_ms = max(1, min(budget_ms, POLL_DEADLINE_MS, 60_000)) + t0 = time.perf_counter() + try: + found = await asyncio.to_thread( + coll().find_one, + {"session_id": session_id, "status": "completed"}, + max_time_ms=budget_ms, + ) + killed = False + except ExecutionTimeout: + found, killed = None, True + elapsed_ms = (time.perf_counter() - t0) * 1000 + return { + "confirmed": found is not None, + "timed_out": killed, + "elapsed_ms": round(elapsed_ms, 1), + "order_id": found.get("order_id") if found else None, + } + + +@app.post("/api/incident") +async def incident(): + """Page the on-call via the ChatGPT Workspace Agent — once per arming.""" + if not state["incident_enabled"]: + return JSONResponse({"fired": False, "reason": "disabled"}, status_code=200) + if not state["incident_armed"]: + return JSONResponse({"fired": False, "reason": "already_fired"}, status_code=200) + + token = os.environ.get("AGENT_ACCESS_TOKEN") + trigger_id = os.environ.get("WORKSPACE_AGENT_TRIGGER_ID") + if not token or not trigger_id: + return JSONResponse( + {"fired": False, "reason": "missing_credentials"}, status_code=200 + ) + + # Build the same PagerDuty resource the CLI sends, with the same defaults. + args = trigger_chatgpt.main_defaults() + incident_id = trigger_chatgpt.pagerduty_id() + event_id = secrets.token_hex(16) + payload = trigger_chatgpt.build_pagerduty_incident(args, incident_id) + + try: + response = await asyncio.to_thread( + trigger_chatgpt.trigger_agent, + trigger_id, token, payload, f"pagerduty-{incident_id}", event_id, + ) + except (RuntimeError, KeyError) as exc: + return JSONResponse({"fired": False, "reason": str(exc)}, status_code=200) + + state["incident_armed"] = False + state["last_incident"] = { + "incident_id": incident_id, + "conversation_url": response.get("conversation_url"), + } + return { + "fired": True, + "incident_id": incident_id, + "conversation_url": response.get("conversation_url"), + } + + +@app.post("/api/rearm") +async def rearm(): + state["incident_armed"] = True + return {"armed": True} + + +@app.get("/api/config") +async def config(): + return { + "client_timeout_s": CLIENT_TIMEOUT_S, + "poll_interval_ms": POLL_INTERVAL_MS, + "incident_enabled": state["incident_enabled"], + "incident_armed": state["incident_armed"], + } + + +@app.get("/", response_class=HTMLResponse) +async def index(): + # Loading the checkout page re-arms the incident, so a browser refresh is the + # reset — no demo control on the shopper's screen. Deliberately scoped to this + # HTML route: /api/* calls never re-arm, so the polling loop can't rearm + # mid-run and fire a second incident. + state["incident_armed"] = True + return PAGE + + +PAGE = """ + + + +Contoso — Checkout + + + +
contoso.
+
+
+

Order summary

+
Wireless Headphones$149.00
+
USB-C Cable$12.00
+
Total$161.00
+
+ +
•••• •••• •••• 4242
+
+ +
+
Processing your payment…
+
Please don't close this window.
+
+
+ +
+

Checkout status poll

+
idle — awaiting payment
+
+
+ + + +""" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=8000) + parser.add_argument("--no-incident", action="store_true", + help="rehearse the checkout without paging the agent") + args = parser.parse_args() + + uri = os.environ.get("MONGODB_URI") + if not uri: + sys.exit('ERROR: set MONGODB_URI in .env') + + global client + client = MongoClient(uri, appname="perf-triage-demo-checkout") + client.admin.command("ping") # fail fast, before the stage + + state["incident_enabled"] = not args.no_incident + if args.no_incident: + print("Incident dispatch DISABLED (--no-incident): checkout will fail quietly.") + elif not os.environ.get("AGENT_ACCESS_TOKEN"): + print("WARNING: AGENT_ACCESS_TOKEN unset — checkout will fail but page no one.") + + print(f"Checkout page: http://{args.host}:{args.port}") + uvicorn.run(app, host=args.host, port=args.port, log_level="warning") + + +if __name__ == "__main__": + main() diff --git a/apps/remote-mcp-perf-triage/generate_load.py b/apps/remote-mcp-perf-triage/generate_load.py index 9aad9c41..5f9f354a 100644 --- a/apps/remote-mcp-perf-triage/generate_load.py +++ b/apps/remote-mcp-perf-triage/generate_load.py @@ -52,9 +52,13 @@ try: from pymongo import MongoClient + from pymongo.errors import PyMongoError + from dotenv import load_dotenv except ImportError: sys.exit("pymongo not installed. Run: pip install -r requirements.txt") +load_dotenv() + DB_NAME = "ecommerce" COLLECTION_NAME = "payments" @@ -65,6 +69,8 @@ def run_query(coll): """Run one checkout status-poll query; return its latency in ms. A random session_id that has no 'completed' record forces a full COLLSCAN. + Raises PyMongoError on connection trouble; the caller decides whether to + keep going. """ session_id = f"sess_{secrets.token_hex(12)}" t0 = time.perf_counter() @@ -89,7 +95,12 @@ def main(): client = MongoClient(uri, appname="perf-triage-demo-load") coll = client[DB_NAME][COLLECTION_NAME] - client.admin.command("ping") + # Fail loudly at launch on a bad URI or firewall — that's a config error to fix, + # not something to retry. Mid-run failures are handled in the loop below. + try: + client.admin.command("ping") + except PyMongoError as exc: + sys.exit(f"ERROR: cannot reach MongoDB — {type(exc).__name__}: {exc}") mode = "continuous" if args.interval == 0 else f"trickle ({args.burst} queries / {args.interval:.0f}s)" limit = "until Ctrl+C" if args.duration == 0 else f"{args.duration:.0f}s" @@ -98,17 +109,41 @@ def main(): deadline = time.time() + args.duration if args.duration > 0 else None total = 0 + errors = 0 try: while deadline is None or time.time() < deadline: cycle_start = time.time() - latencies = [run_query(coll) for _ in range(args.burst)] - total += len(latencies) - - avg = sum(latencies) / len(latencies) - slow = 100 * sum(1 for x in latencies if x > 100) / len(latencies) ts = datetime.now().strftime("%H:%M:%S") - print(f" {ts} burst of {len(latencies)}: avg {avg:7.1f} ms | " - f"{slow:3.0f}% over 100 ms | {total:,} total", flush=True) + + # Survive transient trouble instead of dying. This process is meant to + # run unattended for hours before a demo, across laptop sleep, wifi + # handoffs and Atlas blips — any of which raises PyMongoError. Losing + # the trickle means Performance Advisor's recommendation goes stale, + # so a failed burst is logged and skipped, never fatal. pymongo + # reconnects on its own; we just have to keep asking. + latencies = [] + failures = [] + for _ in range(args.burst): + try: + latencies.append(run_query(coll)) + except PyMongoError as exc: + failures.append(type(exc).__name__) + + total += len(latencies) + errors += len(failures) + + if latencies: + avg = sum(latencies) / len(latencies) + slow = 100 * sum(1 for x in latencies if x > 100) / len(latencies) + suffix = f" | {len(failures)} failed" if failures else "" + print(f" {ts} burst of {len(latencies)}: avg {avg:7.1f} ms | " + f"{slow:3.0f}% over 100 ms | {total:,} total{suffix}", + flush=True) + else: + # Whole burst failed — almost always a dropped connection. + print(f" {ts} burst FAILED ({', '.join(sorted(set(failures)))}) — " + f"will retry next cycle | {total:,} total, {errors} errors", + flush=True) if args.interval > 0: sleep_for = args.interval - (time.time() - cycle_start) @@ -116,9 +151,14 @@ def main(): time.sleep(sleep_for) elif deadline is not None: break # not enough time left for another cycle + elif not latencies: + # Continuous mode with a dead connection would spin as fast as the + # driver can fail. Back off so the log stays readable. + time.sleep(5) except KeyboardInterrupt: pass - print(f"\nStopped after {total:,} queries.") + tail = f", {errors} errors" if errors else "" + print(f"\nStopped after {total:,} queries{tail}.") if __name__ == "__main__": diff --git a/apps/remote-mcp-perf-triage/requirements.txt b/apps/remote-mcp-perf-triage/requirements.txt index 569647ea..62831438 100644 --- a/apps/remote-mcp-perf-triage/requirements.txt +++ b/apps/remote-mcp-perf-triage/requirements.txt @@ -1 +1,4 @@ pymongo>=4.6 +python-dotenv>=1.0 +fastapi>=0.110 +uvicorn>=0.27 diff --git a/apps/remote-mcp-perf-triage/seed_payments.py b/apps/remote-mcp-perf-triage/seed_payments.py index 3e3b696a..ab051643 100644 --- a/apps/remote-mcp-perf-triage/seed_payments.py +++ b/apps/remote-mcp-perf-triage/seed_payments.py @@ -8,14 +8,21 @@ Two design constraints, satisfied together: -1. The scan must be reliably slow. Empirically on an M10 (measured), 300,000 - documents of ~2 KB is the sweet spot: a COLLSCAN runs ~30 s cold and settles - to ~3.5-4 s once the cache is warm. That is clearly slow and dramatic in +1. The scan must be reliably slow. Empirically on an M10 with 2 GB RAM (measured), + 300,000 documents of ~2 KB is the sweet spot: a COLLSCAN runs ~9 s cold and + settles to ~5 s once the cache is warm. That is clearly slow and dramatic in explain(), yet stays safely under the MongoDB MCP server's 60 s maxTimeMS cap. NOTE: do NOT seed millions here — a scan of that size can exceed the 60 s cap, making the agent's explain()/find() ERROR out during the demo instead of returning stats. Bigger is worse, not better. + What makes the scan slow is that the collection (~0.63 GB) OUTGROWS the + WiredTiger cache (~50% of host RAM, so ~537 MB on a 2 GB M10), forcing reads + from disk. On a bigger tier the collection fits entirely in cache and the same + query returns in ~200 ms — measured at 222 ms on a 4 GB host — which kills the + demo. If scans come back suspiciously fast, check hostInfo.memSizeMB: the fix + is a smaller tier, NOT more documents. + The document bytes live in a realistic `gateway_response` field (an opaque base64 payload — exactly what payment processors return and apps store for reconciliation). It is high-entropy so WiredTiger's snappy compression can't @@ -33,6 +40,7 @@ python seed_payments.py # defaults: 300,000 docs, ~1.6 KB gateway blob python seed_payments.py --docs 300000 --blob-bytes 1600 python seed_payments.py --drop # drop the collection first (fresh reseed) + python seed_payments.py --drop-index # light reset: drop the demo index, keep the data """ import argparse @@ -46,12 +54,19 @@ try: from pymongo import MongoClient, InsertOne + from dotenv import load_dotenv except ImportError: sys.exit("pymongo not installed. Run: pip install -r requirements.txt") +load_dotenv() + DB_NAME = "ecommerce" COLLECTION_NAME = "payments" +# The index the agent creates during the demo, and that --drop-index removes to +# restore the slow condition. Name is MongoDB's default for { session_id: 1, status: 1 }. +DEMO_INDEX_NAME = "session_id_1_status_1" + # Realistic reference data. These enum-ish fields are a small fraction of each # document, so their compressibility doesn't matter — the bulk is the random blob. STATUS_WEIGHTS = [("completed", 0.92), ("pending", 0.05), ("failed", 0.03)] @@ -144,6 +159,9 @@ def main(): help="approx size of the gateway_response payload per doc") parser.add_argument("--batch", type=int, default=5000, help="insert batch size") parser.add_argument("--drop", action="store_true", help="drop the collection before seeding") + parser.add_argument("--drop-index", action="store_true", + help="drop the demo index and exit WITHOUT reseeding " + "(the light reset after a demo has created it)") args = parser.parse_args() uri = os.environ.get("MONGODB_URI") @@ -156,6 +174,24 @@ def main(): # Fail fast on bad credentials / network before the long insert loop. client.admin.command("ping") + if args.drop_index: + # Light reset: restore the slow condition on an intact collection. Exits + # before the insert loop, so the existing 300k documents are untouched. + if args.drop: + sys.exit("ERROR: --drop-index and --drop are mutually exclusive. " + "--drop-index keeps the data; --drop destroys it.") + existing = list(coll.index_information().keys()) + if DEMO_INDEX_NAME in existing: + print(f"Dropping index {DEMO_INDEX_NAME} from {DB_NAME}.{COLLECTION_NAME} ...") + coll.drop_index(DEMO_INDEX_NAME) + else: + print(f"Index {DEMO_INDEX_NAME} not present — nothing to drop.") + print(f"Indexes now: {list(coll.index_information().keys())} (expect only _id_)") + print(f"Documents kept: {coll.estimated_document_count():,}") + print("Next: make sure generate_load.py is running so Performance Advisor " + "rebuilds its recommendation before the next demo.") + return + if args.drop: print(f"Dropping {DB_NAME}.{COLLECTION_NAME} ...") coll.drop() @@ -182,7 +218,7 @@ def main(): print(f"\nDone. Logical size ~{size_gb:.2f} GB, on-disk storage ~{storage_gb:.2f} GB.") print(f"Indexes present: {list(coll.index_information().keys())} (expect only _id_)") print("Next: run generate_load.py to warm the cache and feed Performance Advisor, " - "then confirm scan time with an explain() (expect ~30 s cold, ~4 s warm on M10).") + "then confirm scan time with an explain() (expect ~9 s cold, ~5 s warm on a 2 GB M10).") if __name__ == "__main__": diff --git a/apps/remote-mcp-perf-triage/trigger_chatgpt.py b/apps/remote-mcp-perf-triage/trigger_chatgpt.py new file mode 100644 index 00000000..56ab6a8a --- /dev/null +++ b/apps/remote-mcp-perf-triage/trigger_chatgpt.py @@ -0,0 +1,329 @@ +"""Send a simulated PagerDuty incident to a ChatGPT Workspace Agent. + +The input mirrors a PagerDuty incident resource created from a Datadog monitor. It +includes only application symptoms, never the database namespace, query shape, +root cause, or remediation. + +Usage: + export AGENT_ACCESS_TOKEN="..." + export WORKSPACE_AGENT_TRIGGER_ID="agtch_..." + python trigger_chatgpt.py +""" + +import argparse +import hashlib +import json +import os +import secrets +import string +import sys +import time +import urllib.error +import urllib.request +from datetime import datetime, timezone + +try: + from dotenv import load_dotenv +except ImportError: + sys.exit("python-dotenv not installed. Run: pip install -r requirements.txt") + +load_dotenv() + +API_ROOT = "https://api.chatgpt.com/v1/workspace_agents" +BETA_HEADER = "workspace_agent_runs=v1" +WAIT_STOP_STATUSES = {"completed", "failed", "suspended"} + +# The trigger POST has been measured at ~5.5 s, but tail latency is much worse: a +# live run once died on a 30 s read timeout, then succeeded three times in a row. +# Generous timeout + retries, because failing this call on stage is the one error +# the audience actually sees. +REQUEST_TIMEOUT_S = 90 +MAX_ATTEMPTS = 3 +RETRY_BACKOFF_S = 2.0 + +# Retrying a POST is only safe because we always send Idempotency-Key: the server +# dedupes, so a retry after a timeout resumes the original request rather than +# creating a second conversation. Never retry a mutating call without that key. +RETRY_STATUS = {408, 429, 500, 502, 503, 504} + + +def request_json(url, token, method="GET", payload=None, idempotency_key=None): + data = json.dumps(payload).encode("utf-8") if payload is not None else None + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + "OpenAI-Beta": BETA_HEADER, + } + if idempotency_key: + headers["Idempotency-Key"] = idempotency_key + + request = urllib.request.Request( + url, + data=data, + headers=headers, + method=method, + ) + + last_error = None + for attempt in range(1, MAX_ATTEMPTS + 1): + try: + with urllib.request.urlopen(request, timeout=REQUEST_TIMEOUT_S) as response: + body = response.read().decode("utf-8") + return response.status, json.loads(body) + except urllib.error.HTTPError as exc: + body = exc.read().decode("utf-8", errors="replace") + # 4xx other than 408/429 is our bug (bad token, bad trigger id) — + # retrying just delays the error message. + if exc.code not in RETRY_STATUS: + raise RuntimeError( + f"ChatGPT API returned HTTP {exc.code}: {body}" + ) from exc + last_error = RuntimeError(f"ChatGPT API returned HTTP {exc.code}: {body}") + except (urllib.error.URLError, TimeoutError, OSError) as exc: + reason = getattr(exc, "reason", exc) + last_error = RuntimeError(f"Could not reach the ChatGPT API: {reason}") + + if attempt < MAX_ATTEMPTS: + delay = RETRY_BACKOFF_S * attempt + print( + f" request failed ({last_error}); retrying in {delay:.0f}s " + f"[attempt {attempt + 1}/{MAX_ATTEMPTS}]", + file=sys.stderr, + flush=True, + ) + time.sleep(delay) + + raise last_error + + +def pagerduty_id(prefix="P", length=7): + alphabet = string.ascii_uppercase + string.digits + return prefix + "".join(secrets.choice(alphabet) for _ in range(length - 1)) + + +def build_pagerduty_incident(args, incident_id): + created_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + account_url = f"https://{args.account_subdomain}.pagerduty.com" + + return { + "id": incident_id, + "type": "incident", + "self": f"https://api.pagerduty.com/incidents/{incident_id}", + "html_url": f"{account_url}/incidents/{incident_id}", + "number": args.incident_number, + "status": "triggered", + "incident_key": hashlib.sha256(incident_id.encode("utf-8")).hexdigest()[:32], + "created_at": created_at, + "reopened_at": None, + "title": ( + f"[Datadog] {args.service} payment confirmation timeout rate is " + f"{args.timeout_rate:g}% in {args.environment}/{args.region}" + ), + "incident_type": {"name": "major"}, + "service": { + "html_url": f"{account_url}/services/{args.service_id}", + "id": args.service_id, + "self": f"https://api.pagerduty.com/services/{args.service_id}", + "summary": args.service, + "type": "service_reference", + }, + "assignees": [ + { + "html_url": f"{account_url}/users/{args.assignee_id}", + "id": args.assignee_id, + "self": f"https://api.pagerduty.com/users/{args.assignee_id}", + "summary": args.assignee, + "type": "user_reference", + } + ], + "escalation_policy": { + "html_url": f"{account_url}/escalation_policies/{args.policy_id}", + "id": args.policy_id, + "self": f"https://api.pagerduty.com/escalation_policies/{args.policy_id}", + "summary": args.escalation_policy, + "type": "escalation_policy_reference", + }, + "teams": [ + { + "html_url": f"{account_url}/teams/{args.team_id}", + "id": args.team_id, + "self": f"https://api.pagerduty.com/teams/{args.team_id}", + "summary": args.team, + "type": "team_reference", + } + ], + "priority": { + "html_url": f"{account_url}/account/incident_priorities", + "id": args.priority_id, + "self": f"https://api.pagerduty.com/priorities/{args.priority_id}", + "summary": "P2", + "type": "priority_reference", + }, + "urgency": "high", + "conference_bridge": None, + "resolve_reason": None, + } + + +def build_trigger_payload(incident, conversation_key): + return { + "conversation_key": conversation_key, + "input": "PagerDuty incident:\n```json\n" + + json.dumps(incident, indent=2) + + "\n```", + } + + +def trigger_agent(trigger_id, token, incident, conversation_key, event_id): + payload = build_trigger_payload(incident, conversation_key) + url = f"{API_ROOT}/{trigger_id}/trigger" + status, response = request_json( + url, + token, + method="POST", + payload=payload, + idempotency_key=event_id, + ) + if status != 202: + raise RuntimeError(f"Expected HTTP 202, received {status}: {response}") + return response + + +def wait_for_run(trigger_id, run_id, token, poll_interval, timeout): + url = f"{API_ROOT}/{trigger_id}/runs/{run_id}" + deadline = time.monotonic() + timeout + last_status = None + + while time.monotonic() < deadline: + _, run = request_json(url, token) + status = run.get("status", "unknown") + if status != last_status: + print(f"Agent run: {status}") + last_status = status + if status in WAIT_STOP_STATUSES: + return run + time.sleep(poll_interval) + + raise RuntimeError(f"Agent run did not settle within {timeout:.0f} seconds") + + +def build_parser(): + parser = argparse.ArgumentParser(description=__doc__) + # Contoso is Microsoft's official placeholder company, used here so a public + # repo never puts a real organization's name on a fabricated incident. + parser.add_argument("--account-subdomain", default="contoso") + parser.add_argument("--service", default="Checkout API") + parser.add_argument("--service-id", default="PF9KMXH") + parser.add_argument("--environment", default="production") + parser.add_argument("--region", default="us-west-1") + parser.add_argument("--timeout-rate", type=float, default=18.6) + parser.add_argument("--assignee", default="Dana Whitfield") + parser.add_argument("--team", default="Payments Platform") + parser.add_argument( + "--escalation-policy", + default="Checkout — Production", + help="PagerDuty escalation policy name shown on the incident", + ) + parser.add_argument("--assignee-id", default="PTUXL6G") + parser.add_argument("--policy-id", default="PUS0KTE") + parser.add_argument("--team-id", default="PFCVPS0") + parser.add_argument("--priority-id", default="PSO75BM") + parser.add_argument("--incident-number", type=int, default=4821) + parser.add_argument( + "--incident-id", + help="PagerDuty-style P... ID; shared by all events for one incident", + ) + parser.add_argument( + "--event-id", + help="unique webhook event ID; reuse only when retrying the same event", + ) + parser.add_argument( + "--wait", + action="store_true", + help="poll until the run completes, fails, or is suspended for human action", + ) + parser.add_argument("--poll-interval", type=float, default=2.0) + parser.add_argument("--timeout", type=float, default=180.0) + parser.add_argument( + "--dry-run", + action="store_true", + help="print the Workspace Agents API request without sending it", + ) + return parser + + +def main_defaults(): + """The incident defaults as a namespace, with no CLI parsing. + + Lets checkout_app.py build the exact same PagerDuty resource this CLI sends, + so the two entry points can never drift apart. + """ + return build_parser().parse_args([]) + + +def main(): + parser = build_parser() + args = parser.parse_args() + + incident_id = args.incident_id or pagerduty_id() + event_id = args.event_id or secrets.token_hex(16) + conversation_key = f"pagerduty-{incident_id}" + incident = build_pagerduty_incident(args, incident_id) + + if args.dry_run: + preview = { + "method": "POST", + "url": f"{API_ROOT}/agtch_.../trigger", + "headers": { + "Authorization": "Bearer ", + "Content-Type": "application/json", + "OpenAI-Beta": BETA_HEADER, + "Idempotency-Key": event_id, + }, + "body": build_trigger_payload(incident, conversation_key), + } + print(json.dumps(preview, indent=2)) + return 0 + + token = os.environ.get("AGENT_ACCESS_TOKEN") + trigger_id = os.environ.get("WORKSPACE_AGENT_TRIGGER_ID") + if not token: + sys.exit("ERROR: set AGENT_ACCESS_TOKEN") + if not trigger_id: + sys.exit("ERROR: set WORKSPACE_AGENT_TRIGGER_ID to the agtch_... API channel ID") + if not trigger_id.startswith("agtch_"): + sys.exit("ERROR: WORKSPACE_AGENT_TRIGGER_ID must begin with agtch_") + + try: + response = trigger_agent( + trigger_id, + token, + incident, + conversation_key, + event_id, + ) + print(f"PagerDuty incident: {incident_id}") + print(f"Webhook event: {event_id}") + print(f"Conversation: {response['conversation_url']}") + + run_id = response.get("agent_trigger_run_id") + if run_id: + print(f"Run ID: {run_id}") + if args.wait and run_id: + run = wait_for_run( + trigger_id, + run_id, + token, + args.poll_interval, + args.timeout, + ) + if run.get("error"): + print(f"Run error: {json.dumps(run['error'])}") + return 1 if run.get("status") == "failed" else 0 + return 0 + except (RuntimeError, KeyError) as exc: + sys.exit(f"ERROR: {exc}") + + +if __name__ == "__main__": + raise SystemExit(main()) From 795a6f0c0858d3dfc7ec70d57cf76afd22c5bdb7 Mon Sep 17 00:00:00 2001 From: ajosh0504 Date: Wed, 29 Jul 2026 15:43:46 -0700 Subject: [PATCH 3/4] Removing stale file --- apps/remote-mcp-perf-triage/README.md | 33 ++- apps/remote-mcp-perf-triage/checkout_app.py | 58 ++++- apps/remote-mcp-perf-triage/generate_load.py | 3 +- apps/remote-mcp-perf-triage/post_alert.py | 150 ------------ .../static/mongodb-logo.png | Bin 0 -> 20490 bytes .../remote-mcp-perf-triage/trigger_chatgpt.py | 7 +- apps/remote-mcp-perf-triage/trigger_slack.py | 215 ++++++++++++++++++ 7 files changed, 296 insertions(+), 170 deletions(-) delete mode 100644 apps/remote-mcp-perf-triage/post_alert.py create mode 100644 apps/remote-mcp-perf-triage/static/mongodb-logo.png create mode 100644 apps/remote-mcp-perf-triage/trigger_slack.py diff --git a/apps/remote-mcp-perf-triage/README.md b/apps/remote-mcp-perf-triage/README.md index 8f2f114b..fdba1c07 100644 --- a/apps/remote-mcp-perf-triage/README.md +++ b/apps/remote-mcp-perf-triage/README.md @@ -53,12 +53,12 @@ would normally require a developer who knows exactly where to look. | File | Purpose | |------|---------| -| `checkout_app.py` | The Contoso checkout page. Really hangs on the slow poll, then fires the incident to ChatGPT. | +| `checkout_app.py` | The Leafy Electronics checkout page. Really hangs on the slow poll, then fires the incident to ChatGPT. | | `seed_payments.py` | Seeds a large, realistic `payments` collection (no index on `session_id`); `--drop-index` resets the demo. | | `generate_load.py` | Runs the checkout status-poll query repeatedly to feed Performance Advisor. | -| `post_alert.py` | Posts an Atlas-styled "Query Targeting" alert to Slack via an incoming webhook. | | `trigger_chatgpt.py` | Sends a realistic PagerDuty-style incident to a published ChatGPT Workspace Agent and prints the conversation URL. | -| `requirements.txt` | `pymongo` (for the seed and load scripts). | +| `trigger_slack.py` | Posts that *same* incident to a Slack channel, on demand — shows the fan-out to a second surface. | +| `requirements.txt` | `pymongo`, `python-dotenv`, and FastAPI/uvicorn for the checkout page. | ## Prerequisites @@ -140,7 +140,7 @@ Notes: ### The checkout page (recommended opening) `checkout_app.py` gives the demo a visual first act with no terminal on screen. It -serves a Contoso checkout that runs the **real** status-poll query against the **real** +serves a Leafy Electronics checkout that runs the **real** status-poll query against the **real** unindexed collection — so the hang isn't staged, it's the bug: ```bash @@ -196,7 +196,30 @@ credentials or a network call: python trigger_chatgpt.py --dry-run ``` -`post_alert.py` remains optional if you also want a separate Atlas-style alert in Slack. +### Showing the Slack fan-out (optional) + +The same incident can land in Slack as well as ChatGPT — one webhook event, two +surfaces. This never happens automatically; run it when you want the beat: + +```bash +export SLACK_WEBHOOK_URL="https://hooks.slack.com/services/XXX/YYY/ZZZ" +python trigger_slack.py --dry-run # preview, sends nothing +python trigger_slack.py # post it +``` + +To make it obviously *one* incident on screen, pass the same ID to both and link the +Slack message back to the agent's conversation: + +```bash +python trigger_chatgpt.py --incident-id PY1Z69L +python trigger_slack.py --incident-id PY1Z69L \ + --conversation-url "https://chatgpt.com/c/..." # adds an "Open agent triage" button +``` + +The incident comes from `trigger_chatgpt.build_pagerduty_incident()`, so the two +channels cannot drift apart — only the rendering differs. Note that nothing reads the +Slack message back: it shows the incident *reaching* Slack, not a conversation with +the agent there. A real deployment would need a Slack app wired to the agent for that. The Workspace Agents API accepts a caller-defined `conversation_key`. The simulator uses the PagerDuty incident ID for that key, so future webhook events for the same diff --git a/apps/remote-mcp-perf-triage/checkout_app.py b/apps/remote-mcp-perf-triage/checkout_app.py index 8a82e71b..f42fa855 100644 --- a/apps/remote-mcp-perf-triage/checkout_app.py +++ b/apps/remote-mcp-perf-triage/checkout_app.py @@ -1,4 +1,5 @@ -"""Contoso checkout page — the demo's opening act, and a real reproduction of the bug. +"""Leafy Electronics checkout page — the demo's opening act, and a real reproduction +of the bug. Nothing here is simulated except the payment processor. The page runs the SAME status-poll query the demo is about, against the SAME unindexed collection: @@ -34,10 +35,12 @@ import sys import time from datetime import datetime, timedelta, timezone +from pathlib import Path try: from fastapi import FastAPI from fastapi.responses import HTMLResponse, JSONResponse + from fastapi.staticfiles import StaticFiles from pymongo import MongoClient from pymongo.errors import ExecutionTimeout from dotenv import load_dotenv @@ -83,7 +86,13 @@ # edge is real, not theoretical. Post-index polls take ~20 ms and are unaffected. POLL_DEADLINE_MS = 2_500 -app = FastAPI(title="Contoso Checkout") +app = FastAPI(title="Leafy Electronics Checkout") + +# The MongoDB leaf, copied from the Leafy Roasters inventory demo so both apps use +# the identical asset. Resolved relative to this file, not the working directory, +# so the app can be started from anywhere. +STATIC_DIR = Path(__file__).parent / "static" +app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static") # Module state. Single-process, single-presenter demo; no locking needed. state = { @@ -249,18 +258,37 @@ async def index(): -Contoso — Checkout +Leafy Electronics — Checkout -
contoso.
+
+
+ +
+ Leafy Electronics + Checkout +
+
+

Order summary

diff --git a/apps/remote-mcp-perf-triage/generate_load.py b/apps/remote-mcp-perf-triage/generate_load.py index 5f9f354a..0830c76c 100644 --- a/apps/remote-mcp-perf-triage/generate_load.py +++ b/apps/remote-mcp-perf-triage/generate_load.py @@ -15,7 +15,8 @@ No maxTimeMS here: we WANT each query to complete slowly so it lands in the slow-query log (>100 ms) that Performance Advisor analyzes. The "checkout timing out" symptom is -a narrative prop delivered via the staged Slack alert (post_alert.py). +demonstrated for real by checkout_app.py, which runs this same query and genuinely +exceeds its timeout. TRICKLE MODE (default): recency matters more than volume for Performance Advisor, and the M10 has a burstable CPU, so by default this runs a small BURST of queries every diff --git a/apps/remote-mcp-perf-triage/post_alert.py b/apps/remote-mcp-perf-triage/post_alert.py deleted file mode 100644 index 3ad6780b..00000000 --- a/apps/remote-mcp-perf-triage/post_alert.py +++ /dev/null @@ -1,150 +0,0 @@ -""" -Post a MongoDB Atlas-styled alert to a Slack channel via an incoming webhook. - -This mimics the appearance of a real Atlas "Query Targeting: Scanned Objects / -Returned" alert notification without configuring or triggering Atlas alerting. -It is a demo prop: the substance of the demo (the agent's live triage against a -real slow query + Performance Advisor) is genuine; only the alert delivery is staged. - -Usage: - export SLACK_WEBHOOK_URL="https://hooks.slack.com/services/XXX/YYY/ZZZ" - python post_alert.py - - # Optional overrides: - python post_alert.py --project "Apoorva Test" --cluster "payments-prod" \ - --ratio 2847 --atlas-url "https://cloud-dev.mongodb.com/v2/#/alerts" -""" - -import argparse -import json -import os -import sys -import urllib.request -from datetime import datetime, timezone - -# MongoDB leaf logo (used as the Slack sender icon so the message reads as "from Atlas") -MONGODB_ICON_URL = "https://www.mongodb.com/assets/images/global/favicon.ico" - -# Atlas alert accent color for a triggered alert (Atlas uses a red/amber bar) -ATLAS_ALERT_COLOR = "#B71C1C" - - -def build_payload(project, cluster, ratio, threshold, atlas_url): - """Construct a Slack message that mirrors an Atlas Query Targeting alert.""" - fired_at = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") - - # Block Kit body rendered inside a colored attachment so we get the accent bar. - blocks = [ - { - "type": "header", - "text": { - "type": "plain_text", - "text": "🔴 Alert: Query Targeting", - "emoji": True, - }, - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": ( - "*Query Targeting: Scanned Objects / Returned has gone above " - f"{threshold}*\n" - "Queries are scanning far more documents than they return, " - "indicating a missing or ineffective index." - ), - }, - }, - { - "type": "section", - "fields": [ - {"type": "mrkdwn", "text": f"*Project*\n{project}"}, - {"type": "mrkdwn", "text": f"*Cluster*\n{cluster}"}, - {"type": "mrkdwn", "text": f"*Metric*\nScanned / Returned"}, - {"type": "mrkdwn", "text": f"*Current Value*\n{ratio:,} : 1"}, - {"type": "mrkdwn", "text": f"*Condition*\n> {threshold} : 1"}, - {"type": "mrkdwn", "text": f"*Fired At*\n{fired_at}"}, - ], - }, - { - "type": "actions", - "elements": [ - { - "type": "button", - "text": {"type": "plain_text", "text": "View Alert in Atlas"}, - "url": atlas_url, - "style": "danger", - } - ], - }, - ] - - return { - "username": "MongoDB Atlas", - "icon_url": MONGODB_ICON_URL, - "attachments": [ - { - "color": ATLAS_ALERT_COLOR, - "blocks": blocks, - "fallback": ( - f"[{project}/{cluster}] Query Targeting: Scanned Objects / " - f"Returned has gone above {threshold} (current {ratio}:1)" - ), - } - ], - } - - -def post(webhook_url, payload): - data = json.dumps(payload).encode("utf-8") - req = urllib.request.Request( - webhook_url, data=data, headers={"Content-Type": "application/json"} - ) - with urllib.request.urlopen(req) as resp: - return resp.status, resp.read().decode("utf-8") - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--project", default="Apoorva Test", help="Atlas project name") - parser.add_argument( - "--cluster", default="payments-prod", help="Atlas cluster name shown in alert" - ) - parser.add_argument( - "--ratio", - type=int, - default=2847, - help="Current scanned:returned ratio to display", - ) - parser.add_argument( - "--threshold", - type=int, - default=1000, - help="Alert threshold (scanned:returned) to display", - ) - parser.add_argument( - "--atlas-url", - default="https://cloud-dev.mongodb.com/v2#/alerts", - help="URL the 'View Alert in Atlas' button links to", - ) - args = parser.parse_args() - - webhook_url = os.environ.get("SLACK_WEBHOOK_URL") - if not webhook_url: - sys.exit( - "ERROR: set SLACK_WEBHOOK_URL, e.g.\n" - ' export SLACK_WEBHOOK_URL="https://hooks.slack.com/services/XXX/YYY/ZZZ"' - ) - - payload = build_payload( - args.project, args.cluster, args.ratio, args.threshold, args.atlas_url - ) - status, body = post(webhook_url, payload) - if status == 200: - print("Alert posted to Slack.") - else: - sys.exit(f"Slack returned {status}: {body}") - - -if __name__ == "__main__": - main() diff --git a/apps/remote-mcp-perf-triage/static/mongodb-logo.png b/apps/remote-mcp-perf-triage/static/mongodb-logo.png new file mode 100644 index 0000000000000000000000000000000000000000..c87c2a79929ca00ca1fa3f3c40248bfa01b69267 GIT binary patch literal 20490 zcmeIa`9IX%`#64P?6MUhvQ?xAWh?7YRQ3o{iD|b*WC>v^kwj%lkxZ#{Bc?)9m=+|_ zAfjy9vPB}Yf3Gv$yyWXoV49;jB;yN4nTwbn31IsLXR#l znLEINkahabZAQCK5r*4byqxTd4d;&8FYiy9nIAY76Y=Gc>4{g|aq4H;R=Z|&%Z_lb zrb-jOstk&Ml_uO=+Nn%@M zpB6^GP3Lk0~~a-a{Cn;F*<6&`m#$aie{kcMe@)T3EH350Tvi@-nLlXdO( zr*O6+8Wj=R>D#a90q)ZQZf=CVwfGz_$)av*wc%h$|gJI6dX>BcDm&E*bg6s=*yHM5X!!;KeIVy?r`*Ch1(^C(Tb4FheSRc7J)8~GjzYVmVnr#_JsB<~S#;|y>WM-txQTNfV8E{s)hw##ubA$7= zRyx=8#&izHetj<AisJ**kNx!I?oTp9n6l zL@4QrF*o{nJfKL%^0egqJ;5dD!7X4W%{jTm@gKo^-=DTbpPux8FAzXPG+Q8n<_ydY z`{mpwTge8XNHw5oq_n;g$1RhG)FuuP(69P49Oz@_Vu_=69k6M46}u?1Nj@;)+_p^y zev=0*Y?4neOv?u6dELR_t|0iQ^2n%tm@20=9wH_MOG+KREWItEEXKDVi1@Zc<-~pRL7uy z(GjV^=-l5F@&~wL3IJEMGnmbK-=3ZNewN<;TMy=7U3ho0C1m;RuC&uMKvMCl0FbWy z)n4Y-m_Uv)*p}kTstp11X@KQhuirY^v6#Q@>>G~jB zHfE2Ih%p#W;(_*%<FfNrEm!0%^2+zDGIyDJ3^1rhbkW4vfX%yzD; zkE`w+>6r^_o&A}LkhnNttfmLzJ=U>*&V4PJWu({+BP;OA%An|uC62GMmk)09vV{hYQT)903S#g`AaxDe^X4frxM`Z!;|RN!;Ia5pw1zOx{K{<71PN$uGxPIEW< zqA5bCUmtK}dXKkfubef~?h7wWZaaw3{5=q5f1a33sn|aoymKoUn{TNs%jSKsr z`5$@p0l^1Wy0|P@z)%t%y47ITY}7v8X--)sTmhjkYk}fI?||QKJ74|Na`#T;~+G3k_47#=v5oksg$)vc$$dr$y*#sHg| z<664YT*DTuMjLJJx?Ktj)eg7}hD~|RTJ37M!)F7q+3v1(;ZPoMm1zcaCz4W+Jzj^e;Qb7_KSA1nHEA+0x{Kg zY`}xlmI1+Su+|DJfz4E z_r5D<{k260Y|P}EBd6}=r*XvAJDRl-sP>q3jT+xvl#U-EUeX!&<&_c$phFm-M=B0u(SI!z4r_xA-uF>AWm)7AEi>qGDXi@fO)ENyg z{PG6Gu(zgP4^Ju_D!H@#zc$2>!V1!M^m5>x>1o}s%fFj7h{wwO39+rlT%zd2)46fS zS4y7)$5P%MbJ4vago?206t%rLq5Ex$SbU}XOac#q`W(wPwMII0B}75yy0yYY^t1sS zx0ClDyQNzCY?4DK*Z_+kI%@?UOIazfqg;8AxO49tqr!Jp5r{E_rI_d2CA+WpurccN z2aW3tv|4#r@TLVfn!{2k>Uv`oT#m<%;<4jZ+4C_!2ER~C@!~=-o)LX+A{Fq#)b6f{ zKF9ZJ?(i#@j@Y(ti2qr4!QQhbFI%5?oalgltk=8EL5>B*>yL-8RAb*VnN-Y1Dgj$0 z#bWC(6% zM#@2V%6r`i-!N6mTVK5keU=biY?gv%*u=1$jZeI!{4{uk;AMz32K=`<7yiq6kJMeo z|9Tpc8qmg%q*Eq00#sbSKrEu20`e5wUQC&tGI^xp{G~=RoeSA$fO6YBW<~Kl;=4?U zEWF^nskRWm(?Z3@HI*K`3<}n651Z6O)Go3dJL?Ni-h;QBOPj!19sw&NMIVhy2c<9& z%bT8yOxlAKaR4<_A1!h3@*%|n5n*~0GhEbHV6W8p{*tpw7#l?e_-9RUUgt%~CKNB? z8Mmi+opAs*!)#dw13y73MLme8-F`k?l9DHoQhL_oW<^l&0~i&F11DeF4_>b)8MtW; za@eY+JGc83N8qzlmo>b@BoKoFSqk-b@YvUot0{SG6ukFRcg`DB!hY%MI< zy->Fkp>q@vapb|tQ{S?tc*$QBL;Y89v!Qdcm;msSPNU27d%H^fTI9J9AJ)dm&hm_N zF_I(a&}a40_*quQT9OGmHZ$^g%Dq&_b%s~x#D5)3$q`t729l_~r#RADqJ2(FJ;CY{ zj5RP6D4$sMB3OtNr9$lL!|7H329Pb&F**C(u?{Z!iod~WA`T$WxK5s0o04AkM2B0_ zUn1j4;6a>RQ=K1yJ1&{P3QqUYy}H(g3o&(p1O2I>A$OuH37*bz^S{TKvw|oX$L5JD z2t?CQ@8?mntqy4fX$PcR0bCAYGGeZ+dDdvuT$=Xwm;(=iwjGp|DfVQ$j7x+g7s1md zj%%#@>oEeFI17yIKgVL^L4#?+#P+|u+{Yre^D(F^M~S+bOZhM(L>LufAO1y4_hSf0 ziT_x;XZsvr{SYS>U;GMmeutv@qe()j@ExdzXG7zlEHe{Na=@PYe5O!806 zxRE4QC6$_mA$?eAb2f-~3zl)xpD0{Av3&Pe4bx+zZ$`#ogY!@wrlw6tH5xsr1e}3% z(ssa^Bz81vw(iYS!beE+! zf5!)u2%;x48pJKzW7{&p5j$XC+EjN_$%|{`vGDZ!^>Qc&uaO-**5A46g$_6Mc~A3Y zR`$o*0n#u2hePy;2Jy5K;pdf*jVYIdp5*k%09r@BZwq^kmtjDc^3da=d7zMN@>d&| zU|v=i>-h^&Ofn|yS-TJNbugrB8$%`@H5yU$#nR}fF)r@jH zG=kQ$%%Ug(b}{7LCl}nf^QeVA>gPugT^87++=ZRM^O|$W)tqHqfQ2$nC6v1=w|5Mc z62S%h$3P_^VWmEQFJQRbnIHxG4s7Q|Nw9EHjlIw86oGl~Z@9_D_TzQZzEyzv(C2p! zfw~U@Y^TD-FJ~-w^$Vb<*k(RK9*ULS&xIg)mnnt3u>Z>ab@}pnM@Hm2KQ5{ymXo1v ztAJ;MaUePs3xIyFF8|KZ!XH231&_dj$L(OjJnTADXUFQMiu0(_HqfKG5qP$!!Scux z5qLh{x?i_z6_}{|8&hb$5C$k#KDx~933HG`K}+_}K5Gfp;NA3m(NwQC0WHVew>&L{ zLX%9>E3tOc4uV%2ob0MQ(-lDLP%+3zsp>}5i=dydPb~lbt%H|NC4l|4=spTLErD$0 zK*4tp4UJ>;{ z1!Wy3E}_o67nsV3_GY78`0Fi5CZO{+=?bCC(boSUwF)EklZ6zeS;gS>AEbay56tUj z*hIb_vLmC9+o&8QeTbMTr9|FG!ZoSHAe43cjTBYq{l9jlSmpZDik0w+)M8{@Yc1EK z-l}5)_YjR27^Ij!phP?T*PU|!Z%ES?E(Cc=-J3+}PfV5PpKT#h+*3PRS)y9TgA@~i z_3xV2%&U>6>-)zM^(@1t#q!7JA)edh-WB6~@UC*BZZk}TSWew^ zqm(%KM7XBM2a(Ps7O~mfYUz$iElMvFrVy6v{Flo@z-5ts8JQ({RI?keh_oFT+v?$r z5~%wurx5HVE!(59{ZfrVuV~6O)0wZ7Xu-+p=aG#7Oik?eof!nP>seokS zcssrB(~vh2?QH9WvkYZs!c!E8P-bn1Z=YnR?;K_sfa z_r4-ZO&*4K^DlK4MbI=ICeiN8tzA1wpix`gMcg>oggClB#iL8ZaWiZh*4wx4F9@sR0WQLb=(sbYh0qKJe zp@v}!gEFdgS=Ak;I-dWJIwVo4O*f_3(FCR~%)Q|WMHJS>!rkIEx-}yB-Aiq5R^~Rt zp$jLyh$#%_HM1J`#qJshRkXf~s0PuBt>fktN7?PjGYN40euIUpctqN%vD^OMs1S%{O!o9d8!&7xNu;+R zPfK3Z9NDrxr0!;@4C4EnK8oDVb+Z@qWb+d2*)JzlxnC)|$C;MVYCtL`8wQiSc z!D?XpLFM0chJ(?URyKC)2QTG{iZsbWnhSYi<*YG35Gq6|N_@sf_W|s*JOYZmL5j?P zuF>%izNmR6#F^AaMZ_uo;?d5%NXtB07p1Dtx~ir?{!0Do@o&ws8DZd7Cj*3_Ae7hv z`+FgnD0w(kh_+HJeE?^hRDK}}fxZIL7DaC^#dlU33FrRyWq8i;(lpjBs3Qs+I0>eH z?|0N|+b6)_#M&8B^Q^mxV0Q1l)So4IW(lO*fG`tRR@!Xa1#EPVj37TNP>ZLDPuUW! zZ|EW^yDkAzAhbSi#VsKJf3(0;G4IVI+Ol`8gnq0)UN3&O;)fx*b8f zq~jRMQuz^-Zw;*y(*#I%p>9Pp0Zn%8n4S(pHkp0HvWRvBSob;k)xbIQ3l!o{Q!`|O z1hX1}lTot-v;jlStw;!@4#p;U5YR*1E(9#Rwv4iLh9-{}8HX6~%LpATHEp&KAfmmn zp`qz#{QiKb&w5f2Da00$B#+8`H#s>OiWr8mwE?W!HpLQSREaTSRI_4-D8vreO2x%3 zP_3;hAW$8kP3|}ys-_z%L^XUDGB)*k9#J*Hmy=zv)J~-B3R6WCOTb|-#e|)jT4ajo z2EgE<8#Rtwe>x+^uYjDTM7j#@a?{(x)hW=#O&})#_x!(eq1M5!11u^iCa{9TF#u9N z{HCSH2r({#Ddg17$f6jDOt5ALC`eMIfFUt&gE}PSP&V*^E-`ctDIZ%~+hGGjH^UPu zU_!OTkxv510a!q>hY8hh#~645RwF%_kW&FIC6Lyi);rw7-!1}S_T<&<6kIpDb_h`U zgdgzk@wo+L0*SzGEUARs8a&>W8^4X26_u# zu6cjg%KU;yrqCmp+r!tHDnRU7fj< z_+veuUawF5E`IqwA;-06FFQjCxH#w4z9rt0@^bK|9{2aRf)fW{7q=R|YZLN_exBw2UN_{ygt^NBj^n&z3xwv-46Y?qC7@cxBFB~Sxn6@hnO62p2JMF0 z2Wi_Os1{s}*h??;EB({Oc)^%AB$AVff&FjZYVzWemY6SDF2eubX-jGn9yvz>uta6* z=Nf@2)rG+dRLty2Cz3mE!pxI8i5EN#i=x-(1&2PB#q;J>#01R+*)?gzl)4va%u3QP=cKq!vQgJVYfzqXR?AYM;nQNDssf#6 zA7p)1my`zDqM0H!!o@6UJ46u8w}ieV+~;oLVl->9@1?1tFV};3K+(+X#BG0Vt9|!$epVX)l7{XnK6!qn2L6ij|e*n(*^N~;?EmK$|78WVEp})-8XsEu1;9%B(_D)tf zBO8!8>3D5w38Kn*{Dk<#YR8WVx$ZB1_&|S;YD80S38JmXl3SCh{nAtuY;dpE%;kUk z4s(n^WsC>Df|(Es@wC~2*c_dZj!j$s-8X+TJIATw>esfHOd`E6&_S(oUO$@;9#hG z@w|lbiN0{f|IzE-&wJ*(_!2rzU0^NY!FhlvY9-4~)4Lx-SPuo#2j_*kZ|j1g^@aZi zL+igZaFDtequ=ryeWOc~IQlFL@%E8|)nD`rgAFs2rX z>Y+zh6w>M0%VnbjKnGZoj!)Z{#HE-9BmCBV=U}GYyif3S^d<9<24MMeJFg}Ck`GFd zqUy`IkO>fws1m!dfhwSB$U)z~Wt8*ZzEs9puX_y~VGXOwxXAX9FtGqkPZFDRTe;~p zI~_VQ1N&5L8Kx|pZ%+>HzjlJR&>Q$9jquh_ktoH}Fc?iMJ zV{Bl3b|iBHKz<1b-Aj@By-Khoo`cLRfSj5@x{6s8d`Oj=%X5$BSgZenpa|PB8o8K6 z3#qd!asZw>mq1YLx*MX@pHi5Mw-O~YQMzMwZUgQ}mRy~Ok~!Tshz63`DY{(MqY_LJNHooF8i2AOqf!5i7IMwwP6pX%Q$Y{g_M!{n6X)E0L@PODuwclW9f23DP2MX1+ZFoX&^AcLrW3)s^d!~#)@&OR= zGXbYdv~3wZ1L}S%mWSM|Hn1ZyHsT#KhP32(0pbn;P~xyWbkzfXiFvAd>%*Z-I|mJ@JVJMZUY2Er(T+WF@9BCYW`hg|}fh)RQsj z<_CIsX>g+Ok@g7_X>JY`Ov)krYieSI3qsXcAOjjym7vQ;;X&ak`15ux7=1J?6&IjZ`8w!5+>P#YxK_ zZFRZf%dUt+)V8XIWBm_XS)2pL!Vl&g>1vLUai(tyC)3)BhvUyc%gJ7;8FxMCOY?_L z+EW(x>b;;SJFy}h={*{Y%ztyVxm(Dwv5>wa(tv{9!M9uSL=((d!fkrPzX>rE0F`$= zeG)nawIYqG?deSU%m|{$PN4Hn!Aua<>brDEINYIu4n$`$H2B>`RhBl(OF+^#LB7Sw zr)hvCZyMB#L|b#An{` zOlM6_tQI@fZ~ShDJGjkdpd`o|z0Cc)UVs=$&7U)5OVUk>I^eRmgxhjmoZ{;??` z?d=8;`dZ<55Ey7pVixud>xAQ((ZdneR9hg>?8qxNo%;T~v+eJs+Yz>58)ac>W~^xY zzGqmC@a7}&9k8PSjE&_+A6Y=;ZI#^`ek4JJGB(HAV!3G|o@Zx2b!s`wQ<_wlzGt)H z;ZjD4aB^bvxTjh#KQ}2(N~x4t+;f|swk$X3PsGTsd*YuFUw{G4{!alihp9c(tmc9|i>L@*ee7sE^R^W_U#9vy1q*_+T61m}`4 zd7%C+M5qjfMG=y2%8xLj;!T+B=M_&--0w@2W5vaLPaFu}t2T({tsN?1N=L0iiLigL zDQ+#=8R=YqhU1}|UJl_;OznQ8`iq%TVWu)sw@sFh6&A-Za`B?x@h8>!X;RNW;IS+a z&nw3zy}(~8CN3yZCBR4q)%a=ruPFVJYlVor;h@3h+{#5s#*A-_Qbc#12K86(umqP4 z0Z`@#j|zHlUWmpC>M5ZZ6B*E><(PCch4bOt`zr|Uu;F>|^3<@OLVp_UBtEM_SYYKH z%K;UX=sI;HN&_ti?fOW8^SD>S?&L3rH^ny1!Er~^wFpO)gg>o{JzW)L@Qm-;nS!3V zt;mK0;BN5kHaEzr`6x1BwrM|~4cr7HX|7Zk@bCzCHS#-xc4B7mc!3F-vVoXXfu6R{qWBuy~#f}*{lzH9{x^XmsYYav;*PMNbq*;T9m zrk0zeXCDiaR#fb_Fd;G>0iUcn9e#@D7R!v|I;ScAAc91~gA0I@LUN{N&_VwB+iXi5 z5-{NqfIcb=@SSPYn8~84`YyHn|8?vXbJ%N)b%SXG}E^#i}mSmFIn!g1oJSEN}}Cc1`xm%pBOnPucRf=jy6J6lwN% zhHZ~Pb?WtNXTRHZ3fRCI7U`kLv)7aR&sQkpEaIrT?_Yc=_9a1u5%IKOd7Zdel}!B`bxK|j9E zib!<_C**!DL0Xx-V^ED@_G%+}TT?iP4TzW2n34cR%NrwS&-aY&L^e2+s;Th4#eU-H z;Wp7iuerrEcGL*Y^`j@$#{6a(0h$nn19yDT)#DMN60q8-Y>8hh!2r7Ot46R@pe(8mwQUMv~?T9IT&EN)C+ChcKT$xJjku}?M2-`Tk3L4Vyp5u zH?;3cDG$QQ!{S}8C54AK@plFvYIup9@V46oHo_(G_h<6Z@3k<(af0%)Mj<$hvc!t` zXQv*xWkHwfaq0OIIzG)imF8uNoFS@r24=tsD3hxjxXSPb#>|amdA!LIncFpGXWJh5 zH#kqaABv~?58gLBaF0^Ee}oI&gqRBO%q*)hjI)gKwbR()|@sy*~CFBGdJ>yqy6Sp-n zD_MINJ@=%5(@9hWDIPVo_jfufPL?EyY7D^5p+EJgiiQ(N>#&{y4=RFejGEd8mt3c(xB~ilX#9V^|iu%M#guzhqgMZQ2-TVqHqOmiK`XS9cQ^| z?tUAHC$ym}P*Qu`Ynk@gZui!CkB$M4-Ab0>x?`fq3(T$4Yh3e{$jnaU`UMzB1i>ns z3W^qNlHzE4WbkiQ@TgzFN>bEM4chu9E&>^=U%Au6rMK8oe)k$gF#>HCJr;Xq@Gk6a z=rCs)UZhq+&TlPG^@C_AIbH(^P<)!W|M}pC_HX8aOT(~4FM0J{g>~L44EL@m>Rz%8 zI|b)j*AB%BQ44S@=uhiPiUx@6xGv0sq>kFF8QUm0c%@!hOTJu##)g$PIteSbS<@^4KzcV`DXGY+yAo5y3nfoxXgIPjV2^-My!S<~U$ z{Ohb(<#yJGro^13wz;jIYK#;MHnYYe2Sk?!K@6=(gZmRjEpDwL0V@%m6L+=f8v`4pV zlyj5JVQ&7;8@e1j8&H#)jsSAS_BnZ}_7*$6c#TGeh*(~4NO-|s0g?wEdCa-*R@WLD zpA|<$BRJB0NZ?7&YtuHH$l10VL#gd(;sK3-4Ty%1jz0U__l+p5bf|+$TcnpkboL7= z>NW6+{jbCbTR%mp@i^4oE#pA>kgvfV#AkMA6OTS}+qY+5gjP-Umvkwvf4`_ywKnrm!FZFs)FjG&bYld>Lu6Q;lsuCj)Z@N)YF~6$+ z2s!2IhXAC7t@#wt$0`X(5}!nz>@z7Ff8~NAM?huIfe1aDo+SxK%387&wYUyH zSxJqBSV-S9b=k_|z{?HsKQ5J;S0)P=AUd{ctq&ri^O+1OuY)tZ)D7UD=|SV`l$G{Q z?0L73j}C(u-i>wK&qK{vk@v+fiZX^E4RUrakpSiqa`R6 zM?~F#%iq2>iV;E+pB8l$H~umyGHG(*W=w))o8jE}1iut{qR!+&$Y_fXVqE*Tcfuhf ziq{!Vwy7H+#?bdW+L4*+`);0nSEE65gdBq&3J4c^7@SXf9UXx};cS;GjWG`Gt~y4T zzZ1o)0w?A)8O(tWNQUli=IM`lQNH{Y7&KGNQcG?LNfm2W)JdiFXg>b_#v3cS&;(S) zs9|vR=OAxP``aLieF-_L^@70O=?LXN9~aclbBIBab*A&<#Bvl09H$w=(doptnTo${ z7mEXRT(D~^>=P3vdv*ZR#i`C%FIDf;AGD1;LNOc6D}jmq5Ju4aA>NpW>KrEJKoRS5 z^$eVIe<}R6?!oa}XLq#WUTdIprbCl@aV(63YII`k2OO4NQUfG4+FSpLnL<%rr-D!^meHnGzc*D3d zc;kGJzf~pvpbRKR9-2B8X6MZt^AmC=lc%^_4R@D|h_;Q--0@szg}XFxSFby7bq{K* z3MI?!?y`!i?>PrI@`2xKdT@E|*|7H+Ti8qc$xAH;C~_h`d;ovI{V@-*`0BVup%v9+ zyXhg}9K?rX&4J=@Ki`&aqKU*3dOvkQ2}R;m;6&QcD}aCL#{9Pv!sn|DRujbW zvE^%Rom6X4ZYkI8;rG*D+~4DJ@tY4z{TX;hX*T0!1mS!YQMoA@j=PWu_U$X$Q>LEZ zbz=lan2M~Cf(v3a5g`9v-Qn{iIqQBYG6oyUmazFrWIU?cjV?kRnYI#)d{y~Q5>LFF zS`+h)80efLcJ`0&Mz2$LNdBoo`B6Wyin$1fcKCh4J6}y7%JE1zGmoO$A5K4m67fxj zAk$;k#w)aK?)j~px&=O31X5U15H{!}9`i`uSyud_Lo$&b)zP%US7@JaJ zI-PxVS+!T2+>>@>Zb3X7xQ)Qy2k#ZPd1`0aEI_53{p|I7iOLq6kOlWl9A z)mimZQcK0R^z*WCZ8i4OhM1iLlI7o7TVPU)Nb!@Ex0k+7s7Pl=BKTB&8hReIW-fo_ z{7}8#@!}T(-uzK@6#b65f~Ah!DUzZFgKj=JXB;RBLZ>K-JY}<43PSj0(_c%oxKGeF ztHiS~Z-U=1X*;%+|L%X3?LD~=3C}WKSy)^1{%aO*|`K|!ujw6eK+otqa z%XXiL^$B+tBl6}Z-G|2{PZ0GThW5c;jmDG`zg74903w|Sz437{X2~y2~7Y{xKonFTD*h4M%uGgqo{058s=>`^6M{^dfQod8>J0~BIekzuT zw>ZdOGCe$COAD&26y{F5_|{1$F62WT8|#3H<^@*(>v{xR+~w;c+OO{oD&uQ<+T=$q z-CUr@T`ji-|3bIh$8`qK^*vqkYNe3o1kYj0fWrp(AX+h~M}FxFm& zlIW{mttynWdB{>VqandzVQEWEgS2koceS~Py_NWQ6_J%Ht2 z7>l1rMgQ9U=k>>D-_h#It6f2Adh2TLdl zn=k@Py2n3E956ZpZ{Nn*q|m>epyo4l&NZ2C3?18>{Ut=|2-qD>$Y_B+tM0DMZU})3#GuloPko`}111kY`8ISTk4^kD=A+AzC)SQ) ztF!8E%^*yPpy*S=0GFb0WJC1!whi%jcDj9y`t2o>ryn zYo}W8Tz{O4`Vhk|toan*H?-4p`#Uyndi<5+O7$A`&2T%N_2Md2&Bgj1`0F9K7(r$_?4VLUx?`~L5( zd*OQ{PUc)q6^a1Z8H}COq3GX1pWDi~7vwgZZorLbfmngIkN>(iEg}3K5`4emRWFK_ zkB1-p%#VU$7lJ}HT@JmQ_9pmQyXmsdWYCP~=>_sXA(f8}I|lb(D_r~qPw@UcA(+gw z4U{u)tjKU<#@;Nm`7D6uAupwtrsip;Rn8i1zxl&4Z$b}C2~pcIAB5oH({#B@nnm;Z zRz6wAvd0DRly1KoA7aD4==Q0h{^$fA4^2@@!H~PABOt&B_j6F>;r7Oa_{RfUmKNIf znbA|{9pQ~1=(Tvt#1I$Q2s2*JLVJGIVUOz57svj*tAGTOdizGY#dU>I-5c~(pj z{_n@f5DC&OxSu}b&Qq~odHP_zz@OKRTeV$cKztOA;5WR1!ONYu_z0m$h@VIvORqXk z=3P?6adS!eugm|uoV>TAEKF$KEv{<1gF}{rL3GGAaAyT+Z1X17}tks{`U*(R6dl)#1S%ImEmog4{R?i?zLNV|X_o0Omn{5$w$h#}2Recv>e(Xh<~H`Jb2OjwRmXje8Icvq-T`?+p;!DZ!TF4m*b58FoO{ zM|_iBlSS=W55+1)o;qc&)5lQ_{`M zjpA(UZ;rmRKlKY-b|W}hQY=gfdLc7z*D8L&r_E3a5ZjH<)(V2*_3~zG!L1ieI(p}{ zF}}E4Gc}3~%qd8*xW7@n?BfcJ`xBP%ycE1z7G_=mWMwW9P(PX1QtiyM3YH$Qc1}gG zk3ko*g_Q2R+L>dPnZo&X)eDLFfdj(^W9mc%R585aiIwl3#Qg~`Ej9TEKECn4JH7M9 z=#o4{4F_{XWvOCEK8YTzQ&0Zd__{h_6MRoHwKK0LX4jmxgT+Eu1QRFyz`-<^IsW2ckgb^BA;eH`0ihRK!Kl~U7fhhp4mu@f=k-AaZmMRbPKI3_x zH|7f|eRJchh0GU^!WR#jw}LJ-zHvDlcZo}gf{~xZ$Op6;&RAUdwfHnpd#A0#9u)bZ z#$vna_A71u&GMbgoQz)cz7gU(#;L{gv7K_)CYm;ejXI!J_N8tbKH} zw_SnnNZ{%3PM569VlJ7riUAqNg@KIkW>Sv^xpsw5D?H9N?D%rJ$@9MB+Qp#l7zZoB zq0svxx#0A&6^%tghWksz_DH!*AC$VNVyaaT_AJPM8S3QY27$C}4i71alTL1Ok-ypW zPqOxH#nb`ySfAC8Zzv4bc9g)$_^~%j15u~v{E1)2?}z+83^*ND-&c93H*MumTxELb z%I>0zp7&2_T7D#;n{tFDXmd!a*suE;Jv^g2HAj;VO*ejeWG-m%T>QxUz*LV*Lqn(h zl@O}E4(c%+dEwV=T%PlVjcpO1U!U5WTRHpG5uSgt5rKByRZVVtx*Ky&i&TG!g3i^r zb6;ih2b2A$p9b-(r9QbtUe|5E;-HpGW%fBagaUQ|&Az9846HYs-u!q=%msPSpSgd) z$Pb@b80oE964+v)O0ZHTdYM<8vhNCe-#%{xta^-JJ@x$TPLdpQc@(;|#q-cI58gxF zn@a|h*E%=O{8CULqBZyp(A(Y;UG1iD=gXd231r81XL5*Wp2O93&X&R_x|X9WSKu^G zHd7|o`9jd?gpu>F_OxE;P1D}zuU9&gEs*;0F4WyH zI=TL!rJYmvOvkse6Z1hT54#)Z5!zvfX{nZYbTB|^;aB6(L!J9Ej?NP7=(8-?!?cu( zf!MFKkX=LNZ|^30XXs6@xt1LjJ{@>DuhpqC+vBR~7WkhzU7F?s zZk^J?IB&ym7rh(I&y9yW#q>Tb6Yz#V6$6?;+80*5$#P@crQHjZ<)bY#-F_Dqj1O-t z(fB=lZ#$wM#X~}DOKpCey?Gb-eCUAQhELt|U$PN$y#%t%U-!*tDr(oz+KuV_8;|mO z+BwCTKY9@=4u79!88*2i`XdJCb1s~`diKzFc<{^)TinnUm!BWA8A|q9@Vb*MNI(m> zKvwx9ExUdWjd}iz9{8q~8!in*(AUDN2{gO<+x{H&zIJGbv)YG>$2CgW$z5R+t}~^_ zM#B_A+_U&e4Lv>hd5q>mzS6*N4hRinoosdyG$;{25^%E2k@q1V5zT(!l|%Hh+K%~v zC${A7q2}!loCK;BTpmX=KRO3{6@T^3WS@K_Sn+#w{F4{~O|t$hgy1*tyr#0^FX=n) zm8*{IhgaM}Kw8)(ACcLE&w_!}G-!h}0z~#N#a|M|52}g)`2LH)A*S@--zGBRwho`O zc^6w zIAS1q=g2)M){%~3v8u<&r8s;EH(820H9^tQaUXc*-5jX?XJ~D;86(2m__eHWQkD=XHMU{=)zkUw?tK0+6pfHK2|9H0#qaDa zX!`_z8L6ELF#P&`vXvqnWC&ChY@XrQvSF#?e!qYWe%z5Mak}oqSP-JwVL1QPJM-<; z_k3J(1gM`w;()1#CH18zNoJ^m*YI7Blf*%kVSDF!J@p z@5mS68Ba0K4*6ej_!w!Rgc$o2fJ0Iqu32vQFNgFeV+LmDPdx5O057Q{4C@xVAh z&KM&Xhgno94Nv`C8<(Q~6uhX%vy@jh3!8UK?(t_w&C4*UPRC^_TeJkxge9iHL1Ocj zQrBJwkZJ&4yrbf6xNK;U8fbDQCdpfVp7-`IBSZ_qEPocSC!|U5f*V0caPz5-z|}nq zJR9LnF2;zJp4tKg9V2~b5#B00`i!}>*E{|J3q}~&o9>+KV7Q!*jA85ra&COC zP%Zo=1gwvz?h9PCl(GbVM*x;ToL>El35F)AS$f%%g$Y)E&%~7D#A?8VfeB!ku2_}i zna+hMfj~NSXW_YlX@yr14ozXv>)=if+u}uEux1_(8Oy`CNaBeV-A+m&7*A(Rfs`_oV`&9ZpdMT*42HTFu} o`|JRk3Ne6&gkAy15w>ZPmSPRu+>GDR#}e%nXO z*`8p)*fn^pfxYi?@Qa^&lj0vxJ$-cgjQfzyV(STal6%XqF54! z{TU0s?JZ5=20I(tftj>f{^g9bRD#7>0_wpe+_Vjlxhxt3ntW;}{`^nH)FY2Q4l4*K z9y?#F_5PoP$#5o#oaON7$K7Jcd?j3n%EU;o-2!JgSFkHOQGZ(NjY?zjtfUh{Rrp)x z@C*loJx>;*{c1cje@F}s#d)6~pfzitJ!dL)&~Z=zu1!$yVm@~?f0gKX1yO{Njhl{1 zQn_Oryk-7E%FF8BH5>_LXle(>W3D1sfsPI8w?5`T_Fz%!&Ic^HV5k1SKmW^t|0@R= aUh^_`>-RD=oxd;O!8=W?w&faA&i;R^16gta literal 0 HcmV?d00001 diff --git a/apps/remote-mcp-perf-triage/trigger_chatgpt.py b/apps/remote-mcp-perf-triage/trigger_chatgpt.py index 56ab6a8a..38009362 100644 --- a/apps/remote-mcp-perf-triage/trigger_chatgpt.py +++ b/apps/remote-mcp-perf-triage/trigger_chatgpt.py @@ -209,9 +209,10 @@ def wait_for_run(trigger_id, run_id, token, poll_interval, timeout): def build_parser(): parser = argparse.ArgumentParser(description=__doc__) - # Contoso is Microsoft's official placeholder company, used here so a public - # repo never puts a real organization's name on a fabricated incident. - parser.add_argument("--account-subdomain", default="contoso") + # Leafy Electronics is an invented company (MongoDB's leaf, and a sibling to the + # Leafy Roasters demo), so a public repo never puts a real organization's name on + # a fabricated incident. + parser.add_argument("--account-subdomain", default="leafyelectronics") parser.add_argument("--service", default="Checkout API") parser.add_argument("--service-id", default="PF9KMXH") parser.add_argument("--environment", default="production") diff --git a/apps/remote-mcp-perf-triage/trigger_slack.py b/apps/remote-mcp-perf-triage/trigger_slack.py new file mode 100644 index 00000000..e984515f --- /dev/null +++ b/apps/remote-mcp-perf-triage/trigger_slack.py @@ -0,0 +1,215 @@ +"""Post the SAME PagerDuty incident that goes to ChatGPT into a Slack channel. + +The point is the fan-out: one incident, two surfaces. It shows the agent isn't +bolted to one chat client — the same webhook event that opens a ChatGPT +conversation can just as easily land in Slack, where the on-call engineer already +lives. Pair it with trigger_chatgpt.py in the demo: + + python trigger_chatgpt.py --incident-id PY1Z69L # opens the agent conversation + python trigger_slack.py --incident-id PY1Z69L # same incident, in Slack + +Passing the same --incident-id to both makes the two views obviously the same +incident on screen. Omit it and each call invents its own. + +The incident payload comes from trigger_chatgpt.build_pagerduty_incident(), so +the two channels can never drift apart: same title, service, assignee, team, +escalation policy, priority. Only the rendering differs. + +This is the staged part of the demo. A real deployment would have PagerDuty POST +its webhook to an adapter that fans out to both the Workspace Agents API and +Slack; this script stands in for that adapter's Slack leg. Nothing here reads the +Slack message back — it's a delivery surface, not a second trigger path. + +Like the ChatGPT payload, this carries application symptoms only — no database +namespace, query shape, root cause, or index recommendation. The agent still has to +discover those itself. + +Usage: + # Add to the git-ignored .env (or export it): + # SLACK_WEBHOOK_URL="https://hooks.slack.com/services/XXX/YYY/ZZZ" + + python trigger_slack.py # post the incident + python trigger_slack.py --dry-run # print the payload, send nothing + python trigger_slack.py --incident-id PY1Z69L # match a ChatGPT trigger + python trigger_slack.py --conversation-url https://chatgpt.com/c/... +""" + +import argparse +import json +import os +import sys +import urllib.error +import urllib.request +from datetime import datetime, timezone + +try: + from dotenv import load_dotenv +except ImportError: + sys.exit("python-dotenv not installed. Run: pip install -r requirements.txt") + +# Reuse the incident builder so Slack and ChatGPT get identical incidents. +import trigger_chatgpt + +load_dotenv() + +# PagerDuty's accent red for a triggered, high-urgency incident. +PAGERDUTY_ALERT_COLOR = "#CD3B48" + +# Rendered as the Slack sender avatar so the message reads as "from PagerDuty". +PAGERDUTY_ICON_URL = "https://avatars.slack-edge.com/2019-11-19/822144368room_72.png" + + +def build_slack_payload(incident, conversation_url=None): + """Render a PagerDuty incident resource as a Slack message. + + Mirrors the shape of PagerDuty's own Slack integration: title as the header, + a field grid of the routing metadata, and a button through to the incident. + """ + # created_at is ISO-8601 with microseconds; trim to whole seconds for display. + created = ( + datetime.fromisoformat(incident["created_at"].replace("Z", "+00:00")) + .strftime("%Y-%m-%d %H:%M:%S UTC") + ) + service = incident["service"]["summary"] + assignee = incident["assignees"][0]["summary"] + team = incident["teams"][0]["summary"] + policy = incident["escalation_policy"]["summary"] + priority = incident["priority"]["summary"] + + blocks = [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": f"🚨 Triggered: {priority} incident", + "emoji": True, + }, + }, + { + "type": "section", + "text": {"type": "mrkdwn", "text": f"*{incident['title']}*"}, + }, + { + "type": "section", + "fields": [ + {"type": "mrkdwn", "text": f"*Incident*\n#{incident['number']} · {incident['id']}"}, + {"type": "mrkdwn", "text": f"*Service*\n{service}"}, + {"type": "mrkdwn", "text": f"*Assigned to*\n{assignee}"}, + {"type": "mrkdwn", "text": f"*Team*\n{team}"}, + {"type": "mrkdwn", "text": f"*Escalation policy*\n{policy}"}, + {"type": "mrkdwn", "text": f"*Urgency*\n{incident['urgency'].title()}"}, + ], + }, + { + "type": "context", + "elements": [ + {"type": "mrkdwn", "text": f"Triggered {created} · status *{incident['status']}*"} + ], + }, + ] + + elements = [ + { + "type": "button", + "text": {"type": "plain_text", "text": "View in PagerDuty"}, + "url": incident["html_url"], + "style": "danger", + } + ] + # When the ChatGPT trigger has already run, link the two surfaces together so + # the audience can see it is one incident rather than two. + if conversation_url: + elements.append( + { + "type": "button", + "text": {"type": "plain_text", "text": "Open agent triage"}, + "url": conversation_url, + } + ) + blocks.append({"type": "actions", "elements": elements}) + + return { + "username": "PagerDuty", + "icon_url": PAGERDUTY_ICON_URL, + "attachments": [ + { + "color": PAGERDUTY_ALERT_COLOR, + "blocks": blocks, + "fallback": f"[{priority}] {incident['title']} ({incident['id']})", + } + ], + } + + +def post(webhook_url, payload): + data = json.dumps(payload).encode("utf-8") + req = urllib.request.Request( + webhook_url, data=data, headers={"Content-Type": "application/json"} + ) + try: + with urllib.request.urlopen(req, timeout=30) as resp: + return resp.status, resp.read().decode("utf-8") + except urllib.error.HTTPError as exc: + body = exc.read().decode("utf-8", errors="replace") + raise RuntimeError(f"Slack returned HTTP {exc.code}: {body}") from exc + except urllib.error.URLError as exc: + raise RuntimeError(f"Could not reach Slack: {exc.reason}") from exc + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--incident-id", + help="PagerDuty-style incident ID. Pass the same value you gave " + "trigger_chatgpt.py so both surfaces show one incident.", + ) + parser.add_argument( + "--conversation-url", + help="ChatGPT conversation URL from trigger_chatgpt.py; adds an " + "'Open agent triage' button linking the two surfaces.", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="print the Slack payload without sending it", + ) + args, _ = parser.parse_known_args() + + # Same defaults as the ChatGPT trigger: Leafy Electronics, Checkout API, + # Dana Whitfield. + incident_args = trigger_chatgpt.main_defaults() + incident_id = args.incident_id or trigger_chatgpt.pagerduty_id() + incident = trigger_chatgpt.build_pagerduty_incident(incident_args, incident_id) + payload = build_slack_payload(incident, args.conversation_url) + + if args.dry_run: + print(json.dumps(payload, indent=2)) + return 0 + + webhook_url = os.environ.get("SLACK_WEBHOOK_URL") + if not webhook_url: + sys.exit( + "ERROR: SLACK_WEBHOOK_URL is not set. Add it to .env (git-ignored):\n" + ' SLACK_WEBHOOK_URL="https://hooks.slack.com/services/XXX/YYY/ZZZ"\n' + "Create one at https://api.slack.com/apps → your app → Incoming Webhooks." + ) + if not webhook_url.startswith("https://hooks.slack.com/services/"): + sys.exit( + f"ERROR: SLACK_WEBHOOK_URL does not look like an incoming webhook:\n" + f" {webhook_url[:40]}...\n" + "Expected https://hooks.slack.com/services/T.../B.../..." + ) + + try: + status, body = post(webhook_url, payload) + except RuntimeError as exc: + sys.exit(f"ERROR: {exc}") + + if status == 200: + print(f"Incident {incident_id} posted to Slack.") + return 0 + sys.exit(f"Slack returned {status}: {body}") + + +if __name__ == "__main__": + raise SystemExit(main()) From 71321e94872c7e636aa980df1f7eabd37283f956 Mon Sep 17 00:00:00 2001 From: ajosh0504 Date: Wed, 29 Jul 2026 15:49:21 -0700 Subject: [PATCH 4/4] Precommit checks pass --- apps/remote-mcp-perf-triage/checkout_app.py | 53 ++++++-- apps/remote-mcp-perf-triage/generate_load.py | 61 ++++++--- apps/remote-mcp-perf-triage/seed_payments.py | 116 ++++++++++++------ .../remote-mcp-perf-triage/trigger_chatgpt.py | 4 +- apps/remote-mcp-perf-triage/trigger_slack.py | 23 ++-- 5 files changed, 179 insertions(+), 78 deletions(-) diff --git a/apps/remote-mcp-perf-triage/checkout_app.py b/apps/remote-mcp-perf-triage/checkout_app.py index f42fa855..76d3b59b 100644 --- a/apps/remote-mcp-perf-triage/checkout_app.py +++ b/apps/remote-mcp-perf-triage/checkout_app.py @@ -34,17 +34,17 @@ import secrets import sys import time -from datetime import datetime, timedelta, timezone +from datetime import datetime, timezone from pathlib import Path try: + import uvicorn + from dotenv import load_dotenv from fastapi import FastAPI from fastapi.responses import HTMLResponse, JSONResponse from fastapi.staticfiles import StaticFiles from pymongo import MongoClient from pymongo.errors import ExecutionTimeout - from dotenv import load_dotenv - import uvicorn except ImportError: sys.exit("Missing deps. Run: pip install -r requirements.txt") @@ -103,6 +103,10 @@ client: MongoClient | None = None +# Strong references to in-flight "processor confirms the payment" tasks. Without +# this asyncio can garbage-collect them mid-sleep (see RUF006). +_gateway_tasks: set[asyncio.Task] = set() + def coll(): return client[DB_NAME][COLLECTION_NAME] @@ -138,10 +142,19 @@ async def pay(): "currency": "USD", "status": "pending", "payment_method": "card", - "card": {"brand": "visa", "last4": "4242", "exp_month": 11, - "exp_year": 2029, "fingerprint": secrets.token_hex(8)}, - "billing_address": {"city": "Seattle", "state": "WA", - "country": "US", "postal_code": "98104"}, + "card": { + "brand": "visa", + "last4": "4242", + "exp_month": 11, + "exp_year": 2029, + "fingerprint": secrets.token_hex(8), + }, + "billing_address": { + "city": "Seattle", + "state": "WA", + "country": "US", + "postal_code": "98104", + }, "line_items": ORDER, "gateway": "stripe", "risk_score": 4, @@ -149,7 +162,12 @@ async def pay(): "updated_at": now, } await asyncio.to_thread(coll().insert_one, doc) - asyncio.create_task(confirm_payment_later(session_id)) + # Keep a strong reference: asyncio only holds a weak one, so an unreferenced + # task can be garbage-collected before it runs — which here would mean the + # payment never gets confirmed and even the post-index checkout fails. + task = asyncio.create_task(confirm_payment_later(session_id)) + _gateway_tasks.add(task) + task.add_done_callback(_gateway_tasks.discard) return {"session_id": session_id, "amount": ORDER_TOTAL} @@ -193,7 +211,9 @@ async def incident(): if not state["incident_enabled"]: return JSONResponse({"fired": False, "reason": "disabled"}, status_code=200) if not state["incident_armed"]: - return JSONResponse({"fired": False, "reason": "already_fired"}, status_code=200) + return JSONResponse( + {"fired": False, "reason": "already_fired"}, status_code=200 + ) token = os.environ.get("AGENT_ACCESS_TOKEN") trigger_id = os.environ.get("WORKSPACE_AGENT_TRIGGER_ID") @@ -211,7 +231,11 @@ async def incident(): try: response = await asyncio.to_thread( trigger_chatgpt.trigger_agent, - trigger_id, token, payload, f"pagerduty-{incident_id}", event_id, + trigger_id, + token, + payload, + f"pagerduty-{incident_id}", + event_id, ) except (RuntimeError, KeyError) as exc: return JSONResponse({"fired": False, "reason": str(exc)}, status_code=200) @@ -482,13 +506,16 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--host", default="127.0.0.1") parser.add_argument("--port", type=int, default=8000) - parser.add_argument("--no-incident", action="store_true", - help="rehearse the checkout without paging the agent") + parser.add_argument( + "--no-incident", + action="store_true", + help="rehearse the checkout without paging the agent", + ) args = parser.parse_args() uri = os.environ.get("MONGODB_URI") if not uri: - sys.exit('ERROR: set MONGODB_URI in .env') + sys.exit("ERROR: set MONGODB_URI in .env") global client client = MongoClient(uri, appname="perf-triage-demo-checkout") diff --git a/apps/remote-mcp-perf-triage/generate_load.py b/apps/remote-mcp-perf-triage/generate_load.py index 0830c76c..898342cd 100644 --- a/apps/remote-mcp-perf-triage/generate_load.py +++ b/apps/remote-mcp-perf-triage/generate_load.py @@ -52,9 +52,9 @@ from datetime import datetime try: + from dotenv import load_dotenv from pymongo import MongoClient from pymongo.errors import PyMongoError - from dotenv import load_dotenv except ImportError: sys.exit("pymongo not installed. Run: pip install -r requirements.txt") @@ -81,18 +81,29 @@ def run_query(coll): def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--burst", type=int, default=3, - help="number of queries to run per cycle") - parser.add_argument("--interval", type=float, default=300, - help="seconds between the start of each burst " - "(0 = continuous / no sleep between bursts)") - parser.add_argument("--duration", type=float, default=0, - help="total seconds to run (0 = run until Ctrl+C)") + parser.add_argument( + "--burst", type=int, default=3, help="number of queries to run per cycle" + ) + parser.add_argument( + "--interval", + type=float, + default=300, + help="seconds between the start of each burst " + "(0 = continuous / no sleep between bursts)", + ) + parser.add_argument( + "--duration", + type=float, + default=0, + help="total seconds to run (0 = run until Ctrl+C)", + ) args = parser.parse_args() uri = os.environ.get("MONGODB_URI") if not uri: - sys.exit('ERROR: set MONGODB_URI, e.g.\n export MONGODB_URI="mongodb+srv://user:pass@host/"') + sys.exit( + 'ERROR: set MONGODB_URI, e.g.\n export MONGODB_URI="mongodb+srv://user:pass@host/"' + ) client = MongoClient(uri, appname="perf-triage-demo-load") coll = client[DB_NAME][COLLECTION_NAME] @@ -103,10 +114,16 @@ def main(): except PyMongoError as exc: sys.exit(f"ERROR: cannot reach MongoDB — {type(exc).__name__}: {exc}") - mode = "continuous" if args.interval == 0 else f"trickle ({args.burst} queries / {args.interval:.0f}s)" + mode = ( + "continuous" + if args.interval == 0 + else f"trickle ({args.burst} queries / {args.interval:.0f}s)" + ) limit = "until Ctrl+C" if args.duration == 0 else f"{args.duration:.0f}s" - print(f"Polling {DB_NAME}.{COLLECTION_NAME} — {mode}, {limit}. " - "Each query is a full COLLSCAN (no index on session_id).") + print( + f"Polling {DB_NAME}.{COLLECTION_NAME} — {mode}, {limit}. " + "Each query is a full COLLSCAN (no index on session_id)." + ) deadline = time.time() + args.duration if args.duration > 0 else None total = 0 @@ -137,18 +154,24 @@ def main(): avg = sum(latencies) / len(latencies) slow = 100 * sum(1 for x in latencies if x > 100) / len(latencies) suffix = f" | {len(failures)} failed" if failures else "" - print(f" {ts} burst of {len(latencies)}: avg {avg:7.1f} ms | " - f"{slow:3.0f}% over 100 ms | {total:,} total{suffix}", - flush=True) + print( + f" {ts} burst of {len(latencies)}: avg {avg:7.1f} ms | " + f"{slow:3.0f}% over 100 ms | {total:,} total{suffix}", + flush=True, + ) else: # Whole burst failed — almost always a dropped connection. - print(f" {ts} burst FAILED ({', '.join(sorted(set(failures)))}) — " - f"will retry next cycle | {total:,} total, {errors} errors", - flush=True) + print( + f" {ts} burst FAILED ({', '.join(sorted(set(failures)))}) — " + f"will retry next cycle | {total:,} total, {errors} errors", + flush=True, + ) if args.interval > 0: sleep_for = args.interval - (time.time() - cycle_start) - if sleep_for > 0 and (deadline is None or time.time() + sleep_for < deadline): + if sleep_for > 0 and ( + deadline is None or time.time() + sleep_for < deadline + ): time.sleep(sleep_for) elif deadline is not None: break # not enough time left for another cycle diff --git a/apps/remote-mcp-perf-triage/seed_payments.py b/apps/remote-mcp-perf-triage/seed_payments.py index ab051643..85b583c5 100644 --- a/apps/remote-mcp-perf-triage/seed_payments.py +++ b/apps/remote-mcp-perf-triage/seed_payments.py @@ -53,8 +53,8 @@ from datetime import datetime, timedelta, timezone try: - from pymongo import MongoClient, InsertOne from dotenv import load_dotenv + from pymongo import InsertOne, MongoClient except ImportError: sys.exit("pymongo not installed. Run: pip install -r requirements.txt") @@ -75,15 +75,24 @@ CARD_BRANDS = ["visa", "mastercard", "amex", "discover"] GATEWAYS = ["stripe", "adyen", "braintree"] CITIES = [ - ("Austin", "TX", "US"), ("Seattle", "WA", "US"), ("Denver", "CO", "US"), - ("London", "", "GB"), ("Toronto", "ON", "CA"), ("Berlin", "", "DE"), - ("Dublin", "", "IE"), ("New York", "NY", "US"), + ("Austin", "TX", "US"), + ("Seattle", "WA", "US"), + ("Denver", "CO", "US"), + ("London", "", "GB"), + ("Toronto", "ON", "CA"), + ("Berlin", "", "DE"), + ("Dublin", "", "IE"), + ("New York", "NY", "US"), ] PRODUCTS = [ - ("SKU-1001", "Wireless Headphones"), ("SKU-1002", "USB-C Cable"), - ("SKU-1003", "Laptop Stand"), ("SKU-1004", "Mechanical Keyboard"), - ("SKU-1005", "4K Monitor"), ("SKU-1006", "Webcam"), - ("SKU-1007", "Desk Lamp"), ("SKU-1008", "Notebook"), + ("SKU-1001", "Wireless Headphones"), + ("SKU-1002", "USB-C Cable"), + ("SKU-1003", "Laptop Stand"), + ("SKU-1004", "Mechanical Keyboard"), + ("SKU-1005", "4K Monitor"), + ("SKU-1006", "Webcam"), + ("SKU-1007", "Desk Lamp"), + ("SKU-1008", "Notebook"), ] @@ -114,12 +123,14 @@ def make_doc(blob_bytes, base_time): line_items = [] for _ in range(n_items): sku, name = random.choice(PRODUCTS) - line_items.append({ - "sku": sku, - "name": name, - "qty": random.randint(1, 4), - "unit_price": random.randint(500, 20_000), - }) + line_items.append( + { + "sku": sku, + "name": name, + "qty": random.randint(1, 4), + "unit_price": random.randint(500, 20_000), + } + ) amount = sum(i["qty"] * i["unit_price"] for i in line_items) created = base_time - timedelta(seconds=random.randint(0, 90 * 24 * 3600)) return { @@ -139,7 +150,9 @@ def make_doc(blob_bytes, base_time): "fingerprint": secrets.token_hex(8), }, "billing_address": { - "city": city, "state": state, "country": country, + "city": city, + "state": state, + "country": country, "postal_code": f"{random.randint(10000, 99999)}", }, "line_items": line_items, @@ -154,19 +167,32 @@ def make_doc(blob_bytes, base_time): def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--docs", type=int, default=300_000, help="number of documents to insert") - parser.add_argument("--blob-bytes", type=int, default=1600, - help="approx size of the gateway_response payload per doc") + parser.add_argument( + "--docs", type=int, default=300_000, help="number of documents to insert" + ) + parser.add_argument( + "--blob-bytes", + type=int, + default=1600, + help="approx size of the gateway_response payload per doc", + ) parser.add_argument("--batch", type=int, default=5000, help="insert batch size") - parser.add_argument("--drop", action="store_true", help="drop the collection before seeding") - parser.add_argument("--drop-index", action="store_true", - help="drop the demo index and exit WITHOUT reseeding " - "(the light reset after a demo has created it)") + parser.add_argument( + "--drop", action="store_true", help="drop the collection before seeding" + ) + parser.add_argument( + "--drop-index", + action="store_true", + help="drop the demo index and exit WITHOUT reseeding " + "(the light reset after a demo has created it)", + ) args = parser.parse_args() uri = os.environ.get("MONGODB_URI") if not uri: - sys.exit('ERROR: set MONGODB_URI, e.g.\n export MONGODB_URI="mongodb+srv://user:pass@host/"') + sys.exit( + 'ERROR: set MONGODB_URI, e.g.\n export MONGODB_URI="mongodb+srv://user:pass@host/"' + ) client = MongoClient(uri, appname="perf-triage-demo-seed") coll = client[DB_NAME][COLLECTION_NAME] @@ -178,26 +204,36 @@ def main(): # Light reset: restore the slow condition on an intact collection. Exits # before the insert loop, so the existing 300k documents are untouched. if args.drop: - sys.exit("ERROR: --drop-index and --drop are mutually exclusive. " - "--drop-index keeps the data; --drop destroys it.") + sys.exit( + "ERROR: --drop-index and --drop are mutually exclusive. " + "--drop-index keeps the data; --drop destroys it." + ) existing = list(coll.index_information().keys()) if DEMO_INDEX_NAME in existing: - print(f"Dropping index {DEMO_INDEX_NAME} from {DB_NAME}.{COLLECTION_NAME} ...") + print( + f"Dropping index {DEMO_INDEX_NAME} from {DB_NAME}.{COLLECTION_NAME} ..." + ) coll.drop_index(DEMO_INDEX_NAME) else: print(f"Index {DEMO_INDEX_NAME} not present — nothing to drop.") - print(f"Indexes now: {list(coll.index_information().keys())} (expect only _id_)") + print( + f"Indexes now: {list(coll.index_information().keys())} (expect only _id_)" + ) print(f"Documents kept: {coll.estimated_document_count():,}") - print("Next: make sure generate_load.py is running so Performance Advisor " - "rebuilds its recommendation before the next demo.") + print( + "Next: make sure generate_load.py is running so Performance Advisor " + "rebuilds its recommendation before the next demo." + ) return if args.drop: print(f"Dropping {DB_NAME}.{COLLECTION_NAME} ...") coll.drop() - print(f"Seeding {args.docs:,} docs (~{args.blob_bytes}B gateway payload each) into " - f"{DB_NAME}.{COLLECTION_NAME} — NO index on session_id.") + print( + f"Seeding {args.docs:,} docs (~{args.blob_bytes}B gateway payload each) into " + f"{DB_NAME}.{COLLECTION_NAME} — NO index on session_id." + ) base_time = datetime.now(timezone.utc) start = time.time() inserted = 0 @@ -210,15 +246,23 @@ def main(): if inserted % 100_000 == 0 or inserted == args.docs: elapsed = time.time() - start rate = inserted / elapsed if elapsed else 0 - print(f" {inserted:,}/{args.docs:,} ({rate:,.0f} docs/s, {elapsed:,.0f}s elapsed)") + print( + f" {inserted:,}/{args.docs:,} ({rate:,.0f} docs/s, {elapsed:,.0f}s elapsed)" + ) stats = client[DB_NAME].command("collstats", COLLECTION_NAME) size_gb = stats.get("size", 0) / 1e9 storage_gb = stats.get("storageSize", 0) / 1e9 - print(f"\nDone. Logical size ~{size_gb:.2f} GB, on-disk storage ~{storage_gb:.2f} GB.") - print(f"Indexes present: {list(coll.index_information().keys())} (expect only _id_)") - print("Next: run generate_load.py to warm the cache and feed Performance Advisor, " - "then confirm scan time with an explain() (expect ~9 s cold, ~5 s warm on a 2 GB M10).") + print( + f"\nDone. Logical size ~{size_gb:.2f} GB, on-disk storage ~{storage_gb:.2f} GB." + ) + print( + f"Indexes present: {list(coll.index_information().keys())} (expect only _id_)" + ) + print( + "Next: run generate_load.py to warm the cache and feed Performance Advisor, " + "then confirm scan time with an explain() (expect ~9 s cold, ~5 s warm on a 2 GB M10)." + ) if __name__ == "__main__": diff --git a/apps/remote-mcp-perf-triage/trigger_chatgpt.py b/apps/remote-mcp-perf-triage/trigger_chatgpt.py index 38009362..d864188f 100644 --- a/apps/remote-mcp-perf-triage/trigger_chatgpt.py +++ b/apps/remote-mcp-perf-triage/trigger_chatgpt.py @@ -291,7 +291,9 @@ def main(): if not token: sys.exit("ERROR: set AGENT_ACCESS_TOKEN") if not trigger_id: - sys.exit("ERROR: set WORKSPACE_AGENT_TRIGGER_ID to the agtch_... API channel ID") + sys.exit( + "ERROR: set WORKSPACE_AGENT_TRIGGER_ID to the agtch_... API channel ID" + ) if not trigger_id.startswith("agtch_"): sys.exit("ERROR: WORKSPACE_AGENT_TRIGGER_ID must begin with agtch_") diff --git a/apps/remote-mcp-perf-triage/trigger_slack.py b/apps/remote-mcp-perf-triage/trigger_slack.py index e984515f..2d571459 100644 --- a/apps/remote-mcp-perf-triage/trigger_slack.py +++ b/apps/remote-mcp-perf-triage/trigger_slack.py @@ -40,7 +40,7 @@ import sys import urllib.error import urllib.request -from datetime import datetime, timezone +from datetime import datetime try: from dotenv import load_dotenv @@ -66,10 +66,9 @@ def build_slack_payload(incident, conversation_url=None): a field grid of the routing metadata, and a button through to the incident. """ # created_at is ISO-8601 with microseconds; trim to whole seconds for display. - created = ( - datetime.fromisoformat(incident["created_at"].replace("Z", "+00:00")) - .strftime("%Y-%m-%d %H:%M:%S UTC") - ) + created = datetime.fromisoformat( + incident["created_at"].replace("Z", "+00:00") + ).strftime("%Y-%m-%d %H:%M:%S UTC") service = incident["service"]["summary"] assignee = incident["assignees"][0]["summary"] team = incident["teams"][0]["summary"] @@ -92,7 +91,10 @@ def build_slack_payload(incident, conversation_url=None): { "type": "section", "fields": [ - {"type": "mrkdwn", "text": f"*Incident*\n#{incident['number']} · {incident['id']}"}, + { + "type": "mrkdwn", + "text": f"*Incident*\n#{incident['number']} · {incident['id']}", + }, {"type": "mrkdwn", "text": f"*Service*\n{service}"}, {"type": "mrkdwn", "text": f"*Assigned to*\n{assignee}"}, {"type": "mrkdwn", "text": f"*Team*\n{team}"}, @@ -103,7 +105,10 @@ def build_slack_payload(incident, conversation_url=None): { "type": "context", "elements": [ - {"type": "mrkdwn", "text": f"Triggered {created} · status *{incident['status']}*"} + { + "type": "mrkdwn", + "text": f"Triggered {created} · status *{incident['status']}*", + } ], }, ] @@ -161,12 +166,12 @@ def main(): parser.add_argument( "--incident-id", help="PagerDuty-style incident ID. Pass the same value you gave " - "trigger_chatgpt.py so both surfaces show one incident.", + "trigger_chatgpt.py so both surfaces show one incident.", ) parser.add_argument( "--conversation-url", help="ChatGPT conversation URL from trigger_chatgpt.py; adds an " - "'Open agent triage' button linking the two surfaces.", + "'Open agent triage' button linking the two surfaces.", ) parser.add_argument( "--dry-run",