From 916b5e4dd0d2ce9cceed40db84ba41b2eb15a649 Mon Sep 17 00:00:00 2001 From: Daniel Ehrhardt Date: Tue, 6 Oct 2026 08:41:27 +0200 Subject: [PATCH] Heartbeats and a run watchdog Agents can switch on a heartbeat: they wake on their own rhythm (15 min to daily, optional working hours and weekdays), wake themselves on tickets that stand still, failed or never started (in the ticket's own chat, with what changed since), run a standing checklist in a Heartbeat chat, and leave a trail of beats. Quiet beats start no run. The watchdog stops runs that show no sign of life or repeat the same step with the same result, with a report of where they stood. Board tickets try again once; other runs are escalated with a notification. --- README.md | 6 + .../components/agents/detail/chats-tab.tsx | 3 +- .../agents/detail/heartbeat-tab.tsx | 745 ++++++++++++++++++ .../components/chat/conversation-header.tsx | 7 +- .../src/components/layout/recent-chats.tsx | 1 + .../src/components/runs/run-status.tsx | 3 +- .../src/components/settings/ai-section.tsx | 33 +- apps/desktop/src/index.css | 6 + apps/desktop/src/lib/api.ts | 6 + apps/desktop/src/lib/queryKeys.ts | 3 + apps/desktop/src/lib/realtime.ts | 4 +- .../src/pages/activity/activity-page.tsx | 1 + .../desktop/src/pages/agents/agent-detail.tsx | 5 +- docs/screenshots/heartbeat-beats.png | Bin 0 -> 452899 bytes docs/screenshots/heartbeat.png | Bin 0 -> 377061 bytes packages/core/src/agents/service.ts | 17 + packages/core/src/backup/backup.ts | 1 + packages/core/src/cloud/scope.ts | 2 + packages/core/src/db/migrations.ts | 37 + packages/core/src/index.ts | 6 + packages/core/src/mcp/tools.ts | 24 +- packages/core/src/memory/dreaming.ts | 1 + packages/core/src/remote/mirror.ts | 4 +- packages/core/src/runner/prompt.ts | 4 +- packages/core/src/runner/runner.ts | 61 +- packages/core/src/runner/stream.ts | 5 + packages/core/src/runner/watchdog.ts | 228 ++++++ packages/core/src/server/routes/agents.ts | 19 + packages/core/src/services/conversations.ts | 1 + packages/core/src/services/heartbeats.ts | 485 ++++++++++++ packages/core/src/services/settings.ts | 5 +- packages/core/src/tasks/service.ts | 40 +- packages/core/test/composio.test.ts | 1 + packages/core/test/credentials.test.ts | 1 + packages/core/test/heartbeats.test.ts | 299 +++++++ packages/core/test/mcp-servers.test.ts | 1 + packages/core/test/totp.test.ts | 1 + packages/shared/src/api.ts | 3 + packages/shared/src/budget.ts | 1 + packages/shared/src/events.ts | 2 + packages/shared/src/heartbeat.ts | 176 +++++ packages/shared/src/index.ts | 1 + packages/shared/src/models.ts | 13 +- packages/shared/src/retry.ts | 4 +- packages/shared/src/tasks.ts | 1 + 45 files changed, 2242 insertions(+), 25 deletions(-) create mode 100644 apps/desktop/src/components/agents/detail/heartbeat-tab.tsx create mode 100644 docs/screenshots/heartbeat-beats.png create mode 100644 docs/screenshots/heartbeat.png create mode 100644 packages/core/src/runner/watchdog.ts create mode 100644 packages/core/src/services/heartbeats.ts create mode 100644 packages/core/test/heartbeats.test.ts create mode 100644 packages/shared/src/heartbeat.ts diff --git a/README.md b/README.md index 4d6c92e4..3964a83b 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ needs to be useful: | 📋 **Task board** | A Trello-style Kanban board per workspace: create tickets, assign them to agents, drag them across *Backlog → Todo → In progress → In review → Done*. A ticket that lands in *Todo* starts its agent. Every ticket works in its own git worktree of the workspace's repository (a cloned URL or a local folder), on its own branch, so agents running side by side never get in each other's way. When the agent on a **Coding** ticket is done, Godmode pushes its branch and opens a pull request — merged pull requests close their ticket. Reply to a ticket to ask for changes; the pull request updates. Archive finished tickets (one by one, or a whole *Done* column) to keep the board focused — they keep everything and come back in one click. Tickets carry a **priority**, a **due date** and **labels** (filter by them; urgent ones start first and the agent is told the deadline). Every ticket keeps a **timeline** — each delivery with its full result, your feedback, the agent's notes, blocks and pull requests — and shows what it **cost** and how long the agent worked. Review in one click (**Approve** / **Request changes** / **Reopen**); A run that fails on a hiccup (an API error, a crash, a restart) is **tried again on its own** — twice at most — so work gets done without you; *Blocked* is left for what needs you, and says why — the agent needs something, the run kept failing — and offers the matching way on. A ticket whose agent set itself a follow-up shows *Waiting — continues tomorrow at 10:00* instead of landing in review. A ticket can **wait for** others (*Reconcile October* waits for *#3 Get the bank statements*): it stays in Todo, says so, and starts by itself once they are delivered — with their results in its brief. Big tickets split into **parts**: a lead hands them to its team (or you add them), its ticket waits until they're delivered and then continues with their results — approve the whole and its parts are done too. | | ⚡ **Automations** | Work starts when it happens: on a schedule (“weekdays at 08:00”), when something happens in a connected app (a new email, a Slack message, a calendar event, a Notion update), when a condition you describe comes true (“competitor pricing changes”), or when a webhook is called. Describe it in one sentence and Godmode sets it up. | | ⏰ **Follow-ups** | When a task needs waiting — a reply to an email, a delivery, a build, office hours — the agent sets itself a time and picks the chat up again on its own, with all the context, like a coworker who says “I'll check back tomorrow at 10”. You see when it comes back and can continue now, move it or cancel it. | +| **Heartbeats & watchdog** | Agents with real work keep moving between your prompts. Switch on an agent's **heartbeat** (every 15 minutes to once a day, optionally only in working hours on weekdays) and it wakes on its own rhythm: it picks up its tickets that stand still, tries a failed one again, starts one that never did — each in the ticket's own chat, told exactly what changed since it last worked on it — and runs its standing **checklist**. Every beat ends with a trail on the agent's *Heartbeat* tab; a beat with nothing to do is quiet and costs nothing. The **watchdog** looks at every run while it works: a run that shows no sign of life, or calls the same tool with the same result over and over, is stopped with a precise report of where it stood. Board tickets try again once on their own; anything else is escalated to you. | | 🤝 **Delegation** | Agents hand tasks to peer agents or spawn short-lived subagents. The Godmode agent can list, check, create and configure all agents. Follow a handoff both ways: open the teammate's chat from the handoff card, and jump back to the chat that asked from there. | | 🧰 **Claude Code & MCP** | Build your team from the terminal: connect Claude Code — or Cursor, Codex, Claude Desktop, any app that speaks MCP — and say *“Create a Godmode agent that…”*. One click in **Settings → Claude Code & MCP** adds Godmode to Claude Code for every project. The app gets Godmode's own management tools (agents, automations, tasks, runs, VMs) through a key of its own, with full or look-only access; scripts reach the same tools with `godmode call`. Passwords, 2FA codes and settings stay out of its reach. | | 🏢 **Team & org chart** | Give every agent a role (*Bookkeeper*, *Research analyst*) and a lead; agents are told who does what and who they report to, so work goes to the right one. The **org chart** shows the whole team with what each is doing right now. Status tells the truth — *Working in 2 chats*, *Queued*, *Needs your answer*, *Last run failed* (one click opens the run) — and each agent's page shows **what's on its plate**: live work, waiting chats, board tickets, follow-ups and its next automations. Duplicate an agent in one click; messages from automations, the board or other agents are labelled as such, never shown as yours. Or start a **whole team** in one click — *Back office*, *Marketing*, *Sales*, *Product & QA*: a lead and its reports, wired up, with their schedules. | @@ -161,6 +162,10 @@ needs to be useful: An agent waiting for a signed contract, set to continue tomorrow at 09:00
Follow-ups — the agent sets itself a time to continue; move it, cancel it or continue now The agent picked the chat up again at the time it set and finished the task
Back on it — at that time the agent picks the chat up again, with all the context + + An agent's Heartbeat tab: on, beating every hour in working hours, with its board
Heartbeat — the agent wakes on its own rhythm to move its tickets forward and run its checklist + Recent beats and the watchdog's interventions with their reports
Beats & watchdog — every beat leaves a trail; stalled or looping runs are stopped with a precise report + A paused chat with a Continue button above the message box
Pause & continue — the agent finishes its step, stands still and picks the work up where it stopped A chat waiting for Claude's session limit to reset, set to continue by itself
Usage limit — the work waits for the reset and continues by itself, with what you wrote meanwhile @@ -507,6 +512,7 @@ See [CONTRIBUTING.md](CONTRIBUTING.md). - [x] Working folders, Claude Code slash commands, live browser preview in chat, desktop auto-update - [x] Computer use: share a window (background control via Cua Driver), a display, every monitor or a browser tab - [x] Automations: schedules, app events (Composio triggers), plain-language conditions and webhooks +- [x] Heartbeats: agents wake on their own rhythm to move their tickets forward and run a checklist; a watchdog stops stalled and looping runs - [x] Agents working in a dedicated macOS VM (Tart / Virtualization.framework): shell, files and screen, assigned per agent, chat or workspace - [x] API tools: any API with a key (Nano Banana, OpenAI, ElevenLabs…) for agents, global / workspace / agent - [x] SSH servers: agents run commands, edit files and copy files on remote machines — password or key, sudo, pinned host keys diff --git a/apps/desktop/src/components/agents/detail/chats-tab.tsx b/apps/desktop/src/components/agents/detail/chats-tab.tsx index d8a59f80..21e17906 100644 --- a/apps/desktop/src/components/agents/detail/chats-tab.tsx +++ b/apps/desktop/src/components/agents/detail/chats-tab.tsx @@ -2,7 +2,7 @@ import { useEffect, useState, type ComponentType } from "react"; import { Link } from "react-router"; import { motion } from "motion/react"; import { formatDistanceToNowStrict } from "date-fns"; -import { Archive, ArchiveRestore, MessageSquare, MessagesSquare, Moon, Pin, Plug, Plus, Search, Share2, SquareKanban, Trash2, Workflow } from "lucide-react"; +import { Archive, ArchiveRestore, HeartPulse, MessageSquare, MessagesSquare, Moon, Pin, Plug, Plus, Search, Share2, SquareKanban, Trash2, Workflow } from "lucide-react"; import type { Agent, Conversation, ConversationOrigin } from "@godmode/shared"; import { errorMessage } from "@/lib/api"; import { useArchivedConversations, useConversations } from "@/lib/hooks"; @@ -31,6 +31,7 @@ const ORIGIN: Record(value: T, ms = 250): T { diff --git a/apps/desktop/src/components/agents/detail/heartbeat-tab.tsx b/apps/desktop/src/components/agents/detail/heartbeat-tab.tsx new file mode 100644 index 00000000..018ada70 --- /dev/null +++ b/apps/desktop/src/components/agents/detail/heartbeat-tab.tsx @@ -0,0 +1,745 @@ +import { useEffect, useRef, useState, type ReactNode } from "react"; +import { Link, useNavigate } from "react-router"; +import { motion } from "motion/react"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { toast } from "sonner"; +import { format, isThisYear, isToday, isYesterday } from "date-fns"; +import { BellRing, ChevronDown, HeartPulse, Hourglass, ListChecks, Repeat2, RotateCcw, ShieldCheck, Timer, Zap, type LucideIcon } from "lucide-react"; +import { + HEARTBEAT_INTERVALS, + MAX_HEARTBEAT_CHECKLIST_LENGTH, + formatUsd, + heartbeatIntervalLabel, + heartbeatIntervalText, + normalizeHeartbeat, + type Agent, + type AgentHeartbeat, + type AgentHeartbeatInput, + type AgentHeartbeatState, + type HeartbeatBeat, + type HeartbeatOutcome, + type HeartbeatWakeReason, + type WatchdogEvent, + type WatchdogKind, +} from "@godmode/shared"; +import { api, errorMessage, isLicenseRequired } from "@/lib/api"; +import { qk } from "@/lib/queryKeys"; +import { useSettings } from "@/lib/hooks"; +import { cn } from "@/lib/utils"; +import { Markdown } from "@/components/chat/markdown"; +import { Button } from "@/components/ui/button"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { Skeleton } from "@/components/ui/skeleton"; +import { Spinner } from "@/components/ui/spinner"; +import { Switch } from "@/components/ui/switch"; +import { Textarea } from "@/components/ui/textarea"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; +import { useNow } from "@/components/runs/run-row"; +import { Segmented, SettingRow, SettingsGroup } from "@/components/settings/settings-kit"; +import { plural, upcomingWhen } from "./dream-parts"; + +const BEATS_PREVIEW = 10; +const WORKING_HOURS = { from: 8, to: 18 }; + +/** Realtime ("heartbeats" entity) keeps it fresh; polling covers the board, and a timer refetches right after the next beat. */ +function useHeartbeatState(agentId: string) { + const qc = useQueryClient(); + const q = useQuery({ + queryKey: qk.agentHeartbeat(agentId), + queryFn: () => api.agents.heartbeat(agentId), + refetchInterval: 60_000, + }); + const next = q.data?.nextAt ?? null; + useEffect(() => { + if (!next) return; + const wait = Date.parse(next) - Date.now() + 5_000; + if (wait <= 0 || wait > 2_147_483_647) return; + const timer = setTimeout(() => qc.invalidateQueries({ queryKey: qk.agentHeartbeat(agentId) }), wait); + return () => clearTimeout(timer); + }, [next, agentId, qc]); + return q; +} + +/** Optimistic: the agent's heartbeat changes right away and rolls back when the core refuses. */ +function useSaveHeartbeat(agent: Agent) { + const qc = useQueryClient(); + return useMutation({ + mutationFn: (heartbeat: AgentHeartbeatInput) => api.agents.update(agent.id, { heartbeat }), + onMutate: async (patch) => { + await qc.cancelQueries({ queryKey: qk.agent(agent.id) }); + const prev = qc.getQueryData(qk.agent(agent.id)); + if (prev) qc.setQueryData(qk.agent(agent.id), { ...prev, heartbeat: { ...normalizeHeartbeat(prev.heartbeat), ...patch } }); + return { prev }; + }, + onError: (err, _patch, ctx) => { + if (ctx?.prev) qc.setQueryData(qk.agent(agent.id), ctx.prev); + toast.error("Couldn't save the heartbeat", { description: errorMessage(err) }); + }, + onSuccess: (updated) => { + qc.setQueryData(qk.agent(agent.id), updated); + qc.invalidateQueries({ queryKey: qk.agents }); + qc.invalidateQueries({ queryKey: qk.agentHeartbeat(agent.id) }); + }, + }); +} + +type SaveHeartbeat = ReturnType; + +export function HeartbeatTab({ agent }: { agent: Agent }) { + const q = useHeartbeatState(agent.id); + const save = useSaveHeartbeat(agent); + const hb = normalizeHeartbeat(agent.heartbeat); + + return ( +
+ q.refetch()} save={save} /> + +
+ + +
+
+ ); +} + +/* ------------------------------------------------------------------ */ +/* Hero: on/off, where it stands, "Wake now" and the board */ +/* ------------------------------------------------------------------ */ + +function Hero({ + agent, + hb, + state, + error, + onRetry, + save, +}: { + agent: Agent; + hb: AgentHeartbeat; + state: AgentHeartbeatState | undefined; + error: unknown; + onRetry: () => void; + save: SaveHeartbeat; +}) { + const qc = useQueryClient(); + const navigate = useNavigate(); + const now = useNow(hb.enabled, 30_000); + const live = hb.enabled && agent.enabled; + + const beat = useMutation({ + mutationFn: () => api.agents.beat(agent.id), + onSuccess: (b) => { + qc.setQueryData(qk.agentHeartbeat(agent.id), (old) => (old ? { ...old, beats: [b, ...old.beats.filter((x) => x.id !== b.id)] } : old)); + qc.invalidateQueries({ queryKey: qk.agentHeartbeat(agent.id) }); + qc.invalidateQueries({ queryKey: qk.runs }); + if (b.outcome === "failed") toast.error("The beat couldn't start", { description: b.summary }); + else if (b.outcome === "woke") + toast.success(`${agent.name} is on it`, { + description: b.summary, + action: b.conversationId ? { label: "Watch", onClick: () => navigate(`/chat/${b.conversationId}`) } : undefined, + }); + else toast(b.outcome === "quiet" ? "All quiet" : "Beat skipped", { description: b.summary }); + }, + onError: (err) => !isLicenseRequired(err) && toast.error(`Couldn't wake ${agent.name}`, { description: errorMessage(err) }), + }); + + const titleId = `heartbeat-${agent.id}`; + const wake = ( + + ); + + return ( +
+
+
+ +
+
+

+ Heartbeat +

+

+ Wakes {agent.name} on its own rhythm to move its tickets forward and run its checklist — and leaves a trail. +

+ {error ? ( +

+ Couldn't load the heartbeat: {errorMessage(error)} + +

+ ) : ( +

+ + {statusLine(agent, hb, state, now)} +

+ )} +
+
+ + {agent.enabled ? ( + wake + ) : ( + + + + {wake} + + + Switch {agent.name} on first + + )} + +
+
+ +
+ ); +} + +/** "Beats every hour · 8:00 AM–6:00 PM · weekdays · next at 10:30 AM" or "Off". */ +function statusLine(agent: Agent, hb: AgentHeartbeat, state: AgentHeartbeatState | undefined, now: number) { + if (!hb.enabled) return "Off"; + const parts = [`Beats ${heartbeatIntervalText(hb.intervalMinutes)}`]; + if (hb.hours) parts.push(`${hourLabel(hb.hours.from)}–${hourLabel(hb.hours.to)}`); + if (hb.weekdays) parts.push("weekdays"); + if (!agent.enabled) parts.push(`waits while ${agent.name} is switched off`); + else if (state?.nextAt) parts.push(nextLabel(state.nextAt, now)); + else if (state) parts.push("no hour to wake in"); + return parts.join(" · "); +} + +function nextLabel(iso: string, now: number) { + const d = new Date(iso); + if (d.getTime() <= now) return "due now"; + return isToday(d) ? `next at ${format(d, "p")}` : `next ${upcomingWhen(iso, now)}`; +} + +function hourLabel(hour: number) { + return format(new Date(2000, 0, 1, hour % 24), "p"); +} + +/** A heart trace: a bright pulse runs along it while the heartbeat is on, a flat line while it is off. */ +const EKG_PATH = "M0 16H40l3-3 3 3h5l3 6 5-19 4 17 3-4h4l3-2 3 2H120"; + +function Ekg({ live, className }: { live: boolean; className?: string }) { + return ( + + {live ? ( + <> + + + + ) : ( + + )} + + ); +} + +const BOARD: { key: keyof AgentHeartbeatState["board"]; label: string; tone?: string }[] = [ + { key: "working", label: "Working" }, + { key: "waiting", label: "Waiting" }, + { key: "waitingOnYou", label: "Waiting on you", tone: "text-warning" }, + { key: "blocked", label: "Blocked", tone: "text-destructive" }, +]; + +function BoardStrip({ agentId, board }: { agentId: string; board: AgentHeartbeatState["board"] | undefined }) { + return ( +
+ {BOARD.map((b, i) => { + const n = board?.[b.key]; + return ( + = 2 && "border-t @xl:border-t-0", + i === 2 && "@xl:border-l", + )} + > +
{b.label}
+ {n == null ? ( + + ) : ( +
0 ? b.tone : "text-muted-foreground/70")}>{n}
+ )} + + ); + })} +
+ ); +} + +/* ------------------------------------------------------------------ */ +/* Rhythm: interval, hours, weekdays and the standing checklist */ +/* ------------------------------------------------------------------ */ + +function Rhythm({ hb, save }: { hb: AgentHeartbeat; save: SaveHeartbeat }) { + const hours = hb.hours; + return ( + } description="How often it beats, and when."> + + save.mutate({ intervalMinutes: Number(v) })} + options={HEARTBEAT_INTERVALS.map((m) => ({ value: String(m), label: {heartbeatIntervalLabel(m)} }))} + /> + + +
+ {hours && ( + + h !== hours.to % 24)} + onChange={(from) => save.mutate({ hours: { ...hours, from } })} + /> + to + h % 24 !== hours.from)} + onChange={(to) => save.mutate({ hours: { ...hours, to } })} + /> + + )} + save.mutate({ hours: v === "hours" ? WORKING_HOURS : null })} + options={[ + { value: "any", label: Any time }, + { value: "hours", label: Working hours }, + ]} + /> +
+ + + save.mutate({ weekdays })} /> + + + + + + ); +} + +function range(from: number, to: number) { + return Array.from({ length: to - from + 1 }, (_, i) => from + i); +} + +function HourSelect({ value, hours, onChange, "aria-label": ariaLabel }: { value: number; hours: number[]; onChange: (hour: number) => void; "aria-label": string }) { + return ( + + ); +} + +/** Saves when it loses focus; "Saved" shows for a moment after. */ +function ChecklistField({ value, save }: { value: string; save: SaveHeartbeat }) { + const [draft, setDraft] = useState(value); + const [saved, setSaved] = useState(false); + const focused = useRef(false); + + useEffect(() => { + if (!focused.current) setDraft(value); + }, [value]); + + useEffect(() => { + if (!saved) return; + const timer = setTimeout(() => setSaved(false), 2000); + return () => clearTimeout(timer); + }, [saved]); + + const commit = () => { + focused.current = false; + const next = draft.trim(); + if (next === value.trim()) return; + setDraft(next); + setSaved(false); + save.mutate({ checklist: next }, { onSuccess: () => setSaved(true) }); + }; + + const pending = save.isPending && save.variables?.checklist !== undefined; + const nearLimit = draft.length > MAX_HEARTBEAT_CHECKLIST_LENGTH * 0.9; + + return ( +
+