From f71113635b25aa9a208aa30788d237c3faa6827a Mon Sep 17 00:00:00 2001 From: r3dbars Date: Sun, 31 May 2026 03:55:20 -0500 Subject: [PATCH 1/2] Count dictation completions in PostHog value probe --- Tests/RepoCommandContractTests.swift | 21 +++++++++++++++++++++ docs/ops-credentials.md | 2 +- scripts/ops/health-probe.sh | 2 +- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/Tests/RepoCommandContractTests.swift b/Tests/RepoCommandContractTests.swift index 41797315..52a15043 100644 --- a/Tests/RepoCommandContractTests.swift +++ b/Tests/RepoCommandContractTests.swift @@ -99,6 +99,27 @@ func testRepoCommandContract() { ) } + runSuite("Repo command contract - PostHog health probe counts emitted first-value events") { + let probe = readRepoTextFile("scripts/ops/health-probe.sh") + let docs = readRepoTextFile("docs/ops-credentials.md") + + for event in [ + "dictation_completed", + "onboarding_first_dictation_saved", + "meeting_transcript_saved", + "onboarding_agent_cta_clicked" + ] { + assertTrue( + probe.contains(event) && docs.contains(event), + "PostHog first-value probe and docs should include \(event)" + ) + } + assertTrue( + probe.contains("first_value_events=") && probe.contains("first_value_events_7d"), + "PostHog probe should keep the aggregate first-value event count" + ) + } + runSuite("Repo command contract - build bundles only the runtime Parakeet model") { let contents = readRepoTextFile("scripts/entrypoints/build.sh") assertTrue( diff --git a/docs/ops-credentials.md b/docs/ops-credentials.md index 08a762a7..8a17b1e5 100644 --- a/docs/ops-credentials.md +++ b/docs/ops-credentials.md @@ -37,7 +37,7 @@ Only report aggregate counts, status codes, deployment IDs, and issue titles. Fo ### PostHog Probe Shape -The PostHog probe reports aggregate 7-day counts for active devices, workflow events, onboarding events, and first-value events. It also prints a 7-day daily active-device trend so operators can see whether DAU is rising, flat, or missing without inspecting user-level data. First-value events are limited to `onboarding_first_dictation_saved`, `meeting_transcript_saved`, and `onboarding_agent_cta_clicked`, so the health lane can see whether users reached a saved Markdown artifact or agent payoff without exposing transcript text, file paths, titles, or user identifiers. +The PostHog probe reports aggregate 7-day counts for active devices, workflow events, onboarding events, and first-value events. It also prints a 7-day daily active-device trend so operators can see whether DAU is rising, flat, or missing without inspecting user-level data. First-value events are limited to `dictation_completed`, `onboarding_first_dictation_saved`, `meeting_transcript_saved`, and `onboarding_agent_cta_clicked`, so the health lane can see whether users reached successful dictation, a saved Markdown meeting artifact, or agent payoff without exposing transcript text, file paths, titles, or user identifiers. If `POSTHOG_HOST` points at the app ingest host, such as `https://us.i.posthog.com`, the probe normalizes it to the matching PostHog API host before running HogQL. The probe only sends `POSTHOG_PERSONAL_API_KEY` to HTTPS PostHog API hosts by default. Set `POSTHOG_ALLOW_UNTRUSTED_HOST=1` only when using a trusted self-hosted PostHog endpoint. diff --git a/scripts/ops/health-probe.sh b/scripts/ops/health-probe.sh index 839012f1..9b774b2b 100755 --- a/scripts/ops/health-probe.sh +++ b/scripts/ops/health-probe.sh @@ -127,7 +127,7 @@ probe_posthog() { local workflow_events onboarding_events first_value_events query daily_query payload daily_payload workflow_events="'app_launched','app_unclean_shutdown_detected','app_session_stall_detected','onboarding_completed','dictation_started','dictation_start_failed','dictation_completed','dictation_cancelled','dictation_no_speech','dictation_audio_route_recovery_timeout','meeting_recording_started','meeting_recording_start_failed','meeting_recording_stopped','meeting_recording_cancelled','meeting_transcript_saved','meeting_transcript_failed','meeting_transcript_skipped'" onboarding_events="'onboarding_shown','onboarding_step_viewed','onboarding_permission_cta_clicked','onboarding_permission_status_changed','onboarding_model_state_changed','onboarding_primary_cta_clicked','onboarding_first_dictation_started','onboarding_first_dictation_saved','onboarding_first_dictation_stop_clicked','onboarding_first_dictation_empty','onboarding_meeting_dry_run_clicked','onboarding_agent_cta_clicked','onboarding_reporting_toggle_changed','onboarding_completed','onboarding_dismissed'" - first_value_events="'onboarding_first_dictation_saved','meeting_transcript_saved','onboarding_agent_cta_clicked'" + first_value_events="'dictation_completed','onboarding_first_dictation_saved','meeting_transcript_saved','onboarding_agent_cta_clicked'" query="select uniq(distinct_id) as devices_7d, sum(case when event in ($workflow_events) then 1 else 0 end) as workflow_events_7d, sum(case when event in ($onboarding_events) then 1 else 0 end) as onboarding_events_7d, sum(case when event in ($first_value_events) then 1 else 0 end) as first_value_events_7d from events where timestamp >= now() - interval 7 day" daily_query="select toDate(timestamp) as day, uniq(distinct_id) as active_devices from events where timestamp >= now() - interval 7 day and event in ($workflow_events) group by day order by day asc" payload=$(jq -cn --arg query "$query" '{query: {kind: "HogQLQuery", query: $query}, refresh: "blocking"}') From ad9254077d3adac87d966ee8c6a6b03f37628357 Mon Sep 17 00:00:00 2001 From: r3dbars Date: Sun, 31 May 2026 05:19:51 -0500 Subject: [PATCH 2/2] Tighten PostHog first-value probe contract --- Tests/RepoCommandContractTests.swift | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Tests/RepoCommandContractTests.swift b/Tests/RepoCommandContractTests.swift index 52a15043..86e7b405 100644 --- a/Tests/RepoCommandContractTests.swift +++ b/Tests/RepoCommandContractTests.swift @@ -102,6 +102,11 @@ func testRepoCommandContract() { runSuite("Repo command contract - PostHog health probe counts emitted first-value events") { let probe = readRepoTextFile("scripts/ops/health-probe.sh") let docs = readRepoTextFile("docs/ops-credentials.md") + let firstValueEvents = sourceSlice( + probe, + from: "first_value_events=", + to: " query=" + ) for event in [ "dictation_completed", @@ -110,7 +115,7 @@ func testRepoCommandContract() { "onboarding_agent_cta_clicked" ] { assertTrue( - probe.contains(event) && docs.contains(event), + firstValueEvents.contains(event) && docs.contains(event), "PostHog first-value probe and docs should include \(event)" ) }