feat(evi): capture cross-channel feedback in a Postgres store

apps/evi/agent/channels/linear.ts

@@ -1,6 +1,29 @@
 import { connectLinearCredentials } from '@vercel/connect/eve'
 import { linearChannel } from 'eve/channels/linear'
+import { captureReaction } from '../lib/feedback'
 
 export default linearChannel({
   credentials: connectLinearCredentials('linear/evi'),
+  // A thumbs up/down reaction on a comment becomes a verdict on that message.
+  // Only fires when the Linear webhook subscription for this app includes the
+  // CommentReaction type; when it does not, no reaction webhooks arrive and
+  // this handler is a silent no-op, so the channel keeps working either way.
+  onDataWebhook: async (event) => {
+    if (event.type !== 'CommentReaction' || event.action !== 'create') return
+    const data = (event.raw?.data ?? event.raw) as Record<string, unknown> | undefined
+    const emoji = typeof data?.reactionEmoji === 'string'
+      ? data.reactionEmoji
+      : typeof data?.emoji === 'string'
+        ? data.emoji
+        : ''
+    const commentId = typeof data?.commentId === 'string' ? data.commentId : undefined
+    const userId = typeof data?.userId === 'string' ? data.userId : undefined
+    await captureReaction({
+      channel: 'linear',
+      emoji,
+      author: userId ?? 'linear:unknown',
+      added: true,
+      messageRef: commentId,
+    })
+  },
 })

apps/evi/agent/channels/photon.ts

@@ -1,7 +1,22 @@
 import { connectPhotonCredentials } from '@vercel/connect/eve'
 import { photonIMessageChannel } from 'eve/channels/photon'
+import { captureReaction } from '../lib/feedback'
 import { MAINTAINER_PHONE } from '../lib/trust'
 
+/**
+ * The inbound reaction (tapback) the repo's eve patch bridges out of the
+ * Chat SDK. Kept local so the patch does not have to widen the package's
+ * public index.
+ */
+interface PhotonReactionEvent {
+  emoji: string
+  rawEmoji?: string
+  added: boolean
+  messageId: string
+  threadId: string
+  userName: string
+}
+
 const MAX_VALUE_LENGTH = 160
 
 function formatValue(value: unknown) {
@@ -25,6 +40,17 @@ export default photonIMessageChannel({
       },
     }
   },
+  // A thumbs up/down tapback on an iMessage becomes a verdict on that message.
+  onReaction: async (reaction: PhotonReactionEvent) => {
+    await captureReaction({
+      channel: 'imessage',
+      emoji: reaction.emoji,
+      author: reaction.userName || 'imessage:unknown',
+      added: reaction.added,
+      messageRef: reaction.messageId,
+      threadRef: reaction.threadId,
+    })
+  },
   events: {
     async 'input.requested'(event, channel) {
       if (!channel.thread || event.requests.length === 0) return

apps/evi/agent/lib/feedback.test.ts

@@ -0,0 +1,83 @@
+import { afterEach, describe, expect, it } from 'vitest'
+import { captureReaction, feedbackStats, NO_DB_ERROR, saveFeedback, verdictFromReactionEmoji } from './feedback'
+
+const NAMES = ['DATABASE_URL', 'POSTGRES_URL', 'POSTGRESQL_URL'] as const
+
+describe('verdictFromReactionEmoji', () => {
+  it('maps positive reactions and their aliases', () => {
+    for (const emoji of ['👍', '+1', 'thumbs_up', 'thumbsup', 'THUMBS-UP']) {
+      expect(verdictFromReactionEmoji(emoji)).toBe('positive')
+    }
+  })
+
+  it('maps negative reactions and their aliases', () => {
+    for (const emoji of ['👎', '-1', 'thumbs_down', 'thumbsdown']) {
+      expect(verdictFromReactionEmoji(emoji)).toBe('negative')
+    }
+  })
+
+  it('ignores reactions that are not a thumb', () => {
+    for (const emoji of ['❤️', '👀', '🚀', '', 'rocket', 'eyes']) {
+      expect(verdictFromReactionEmoji(emoji)).toBeNull()
+    }
+  })
+})
+
+describe('captureReaction', () => {
+  afterEach(() => {
+    for (const name of NAMES) delete process.env[name]
+  })
+
+  it('returns null for a reaction that is not a thumb', async () => {
+    for (const name of NAMES) delete process.env[name]
+    expect(await captureReaction({ channel: 'imessage', emoji: '🚀', author: 'imessage:x', added: true })).toBeNull()
+  })
+
+  it('returns null for a removal so a capture source no-ops', async () => {
+    for (const name of NAMES) delete process.env[name]
+    expect(await captureReaction({ channel: 'imessage', emoji: '👍', author: 'imessage:x', added: false })).toBeNull()
+  })
+
+  it('records a verdict through saveFeedback when the emoji is a thumb', async () => {
+    for (const name of NAMES) delete process.env[name]
+    const result = await captureReaction({ channel: 'linear', emoji: 'thumbs_down', author: 'linear:user', added: true })
+    expect(result).toEqual({ success: false, error: NO_DB_ERROR })
+  })
+})
+
+describe('saveFeedback', () => {
+  afterEach(() => {
+    for (const name of NAMES) delete process.env[name]
+  })
+
+  it('degrades with an unavailable error when no database is configured', async () => {
+    for (const name of NAMES) delete process.env[name]
+    const result = await saveFeedback({
+      channel: 'linear',
+      verdict: 'negative',
+      author: 'linear:user',
+      source: 'reaction',
+    })
+    expect(result).toEqual({ success: false, error: NO_DB_ERROR })
+  })
+})
+
+describe('feedbackStats', () => {
+  afterEach(() => {
+    for (const name of NAMES) delete process.env[name]
+  })
+
+  it('reports an empty store when no database is configured', async () => {
+    for (const name of NAMES) delete process.env[name]
+    const stats = await feedbackStats(new Date())
+    expect(stats).toEqual({
+      total: 0,
+      positive: 0,
+      negative: 0,
+      reactions: 0,
+      written: 0,
+      byChannel: {},
+      recentNegativeWritten: [],
+    })
+  })
+})

apps/evi/agent/lib/feedback.ts

@@ -0,0 +1,157 @@
+import { desc, gt } from 'drizzle-orm'
+import { feedback } from '../../db/schema'
+import { getDb, isDbConfigured } from './db'
+
+export type FeedbackVerdict = 'positive' | 'negative'
+export type FeedbackSource = 'reaction' | 'written'
+
+export interface SaveFeedbackInput {
+  channel: string
+  verdict: FeedbackVerdict
+  author: string
+  source: FeedbackSource
+  text?: string
+  messageRef?: string
+  threadRef?: string
+  sessionRef?: string
+}
+
+export type SaveFeedbackResult =
+  | { success: true; id: string }
+  | { success: false; error: string }
+
+/** Shown to a caller when the store is absent; local runs without the DB keep working. */
+export const NO_DB_ERROR = 'The feedback store is not configured in this environment.'
+
+const POSITIVE_REACTIONS = new Set(['👍', '+1', 'thumbs_up', 'thumbsup', 'thumbs-up'])
+const NEGATIVE_REACTIONS = new Set(['👎', '-1', 'thumbs_down', 'thumbsdown', 'thumbs-down'])
+
+/**
+ * Maps a reaction emoji (or its common shortcode aliases) to a binary verdict.
+ * Returns null for anything that is neither a thumb up nor a thumb down, so
+ * unrelated reactions (hearts, eyes, ...) are ignored by the capture.
+ */
+export function verdictFromReactionEmoji(emoji: string): FeedbackVerdict | null {
+  const normalized = emoji.trim().toLowerCase()
+  if (POSITIVE_REACTIONS.has(normalized)) return 'positive'
+  if (NEGATIVE_REACTIONS.has(normalized)) return 'negative'
+  return null
+}
+
+/**
+ * Shared entry point for every reaction path (iMessage tapback, Linear webhook,
+ * future GitHub). Maps the emoji to a verdict and records it. Returns null when
+ * the emoji is not a thumb or when the event is a removal, so a capture source
+ * treats "nothing to store" as a no-op rather than an error.
+ */
+export async function captureReaction(
+  input: Omit<SaveFeedbackInput, 'verdict' | 'source'> & { emoji: string; added?: boolean },
+): Promise<SaveFeedbackResult | null> {
+  if (input.added === false) return null
+  const verdict = verdictFromReactionEmoji(input.emoji)
+  if (!verdict) return null
+  return await saveFeedback({ ...input, verdict, source: 'reaction' })
+}
+
+/**
+ * Persists one feedback row. Returns an unavailable error when no database is
+ * configured rather than throwing, mirroring the `isDbConfigured()` guard the
+ * rest of the agent uses so a feature degrades instead of crashing.
+ */
+export async function saveFeedback(input: SaveFeedbackInput): Promise<SaveFeedbackResult> {
+  if (!isDbConfigured()) return { success: false, error: NO_DB_ERROR }
+  const db = getDb()
+  if (!db) return { success: false, error: NO_DB_ERROR }
+  try {
+    const rows = await db
+      .insert(feedback)
+      .values({
+        channel: input.channel,
+        verdict: input.verdict,
+        author: input.author,
+        source: input.source,
+        text: input.text ?? null,
+        messageRef: input.messageRef ?? null,
+        threadRef: input.threadRef ?? null,
+        sessionRef: input.sessionRef ?? null,
+      })
+      .returning({ id: feedback.id })
+    return { success: true, id: rows[0]?.id ?? '' }
+  } catch (error) {
+    console.error('[evi:feedback] failed to save feedback', error)
+    return { success: false, error: 'Feedback could not be stored.' }
+  }
+}
+
+/** A stored feedback row as read back from the store. */
+export interface FeedbackRow {
+  id: string
+  channel: string
+  source: string
+  verdict: string
+  author: string
+  text: string | null
+  messageRef: string | null
+  threadRef: string | null
+  sessionRef: string | null
+  createdAt: Date
+}
+
+/**
+ * Rows created since `since`, newest first. Degrades to an empty list when the
+ * store is not configured, so callers (stats, the weekly review) treat a
+ * missing DB the same as "no feedback yet".
+ */
+export async function listFeedbackSince(since: Date): Promise<FeedbackRow[]> {
+  if (!isDbConfigured()) return []
+  const db = getDb()
+  if (!db) return []
+  return await db
+    .select()
+    .from(feedback)
+    .where(gt(feedback.createdAt, since))
+    .orderBy(desc(feedback.createdAt))
+}
+
+/** Aggregates over the rows captured since `since`, for the admin stats tool. */
+export interface FeedbackStats {
+  total: number
+  positive: number
+  negative: number
+  reactions: number
+  written: number
+  byChannel: Record<string, number>
+  /** The most recent written negative feedback with reasons, for the weekly review. */
+  recentNegativeWritten: {
+    id: string
+    channel: string
+    text: string
+    createdAt: Date
+    messageRef: string | null
+  }[]
+}
+
+const RECENT_NEGATIVE_LIMIT = 10
+
+export async function feedbackStats(since: Date): Promise<FeedbackStats> {
+  const rows = await listFeedbackSince(since)
+  const byChannel: Record<string, number> = {}
+  let positive = 0
+  let negative = 0
+  let reactions = 0
+  let written = 0
+  const recentNegativeWritten: FeedbackStats['recentNegativeWritten'] = []
+
+  for (const row of rows) {
+    byChannel[row.channel] = (byChannel[row.channel] ?? 0) + 1
+    if (row.verdict === 'positive') positive += 1
+    else negative += 1
+    if (row.source === 'reaction') reactions += 1
+    else written += 1
+    if (row.source === 'written' && row.verdict === 'negative' && row.text && recentNegativeWritten.length < RECENT_NEGATIVE_LIMIT) {
+      recentNegativeWritten.push({ id: row.id, channel: row.channel, text: row.text, createdAt: row.createdAt, messageRef: row.messageRef })
+    }
+  }
+
+  return { total: rows.length, positive, negative, reactions, written, byChannel, recentNegativeWritten }
+}

apps/evi/agent/tools/feedback.ts

@@ -0,0 +1,69 @@
+import { defineDynamic, defineTool } from 'eve/tools'
+import { z } from 'zod'
+import { channelName } from '../lib/channel'
+import { feedbackStats, NO_DB_ERROR, saveFeedback } from '../lib/feedback'
+import { canAccessAdminTools, isAutonomous } from '../lib/trust'
+
+// Feedback capture is a small, user-requested write: it stays off the approval
+// cards. Autonomous first-responder turns never see it (they process untrusted
+// text and have no one to confirm with). The stats tool is admin-only. Keep
+// executes inline in the resolver (docs/notes.md).
+export default defineDynamic({
+  events: {
+    'turn.started': (_event, ctx) => {
+      if (isAutonomous(ctx.session.auth.current)) return null
+      const channel = channelName(ctx.channel.kind)
+      const sessionRef = ctx.session.id
+      const admin = canAccessAdminTools(ctx.session.auth.current)
+      return {
+        feedback__record: defineTool({
+          description:
+            'Record written feedback about an Evi answer on behalf of the person in this conversation. Call it only when that person explicitly gives feedback (for example "this answer was wrong because X" or "that was really helpful"), never to record your own judgement. The entry is stored in the feedback store and feeds the weekly self-review and evals. Capture their reason as text when they give one.',
+          inputSchema: z.object({
+            verdict: z.enum(['positive', 'negative']).describe('Whether the feedback is positive or negative.'),
+            text: z.string().min(1).describe('What the person said about the answer, in their words, including the reason.'),
+            messageRef: z.string().optional().describe('Optional platform identifier of the specific message or comment being rated.'),
+          }),
+          async execute(input, toolCtx) {
+            if (isAutonomous(toolCtx.session.auth.current)) {
+              return { success: false as const, error: 'Feedback capture is not available in this session.' }
+            }
+            const author = toolCtx.session.auth.current?.principalId ?? 'unknown'
+            const result = await saveFeedback({
+              channel,
+              verdict: input.verdict,
+              author,
+              text: input.text,
+              messageRef: input.messageRef,
+              sessionRef,
+              source: 'written',
+            })
+            if (!result.success && result.error === NO_DB_ERROR) {
+              return { success: false as const, error: NO_DB_ERROR }
+            }
+            return result
+          },
+        }),
+        ...(admin
+          ? {
+            feedback__stats: defineTool({
+              description:
+                  'Read feedback statistics from the store since a number of days ago: totals, positive/negative split, reaction vs written counts, per-channel counts, and the most recent written negative feedback with its reasons. Use it to review how Evi is doing, or as input to the weekly self-review or evals.',
+              inputSchema: z.object({
+                sinceDays: z.number().int().min(1).max(365).default(7).describe('How far back to aggregate, in days. Defaults to 7.'),
+              }),
+              async execute(input, toolCtx) {
+                if (!canAccessAdminTools(toolCtx.session.auth.current)) {
+                  return { success: false as const, error: 'Feedback statistics are only available to admin sessions.' }
+                }
+                const since = new Date(Date.now() - input.sinceDays * 24 * 60 * 60 * 1000)
+                const stats = await feedbackStats(since)
+                return { success: true as const, sinceDays: input.sinceDays, ...stats }
+              },
+            }),
+          }
+          : {}),
+      }
+    },
+  },
+})

apps/evi/db/migrations/0000_silly_pixie.sql

@@ -0,0 +1,14 @@
+CREATE TABLE "feedback" (
+	"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
+	"channel" text NOT NULL,
+	"source" text NOT NULL,
+	"verdict" text NOT NULL,
+	"author" text NOT NULL,
+	"text" text,
+	"message_ref" text,
+	"thread_ref" text,
+	"session_ref" text,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE INDEX "feedback_created_at_idx" ON "feedback" USING btree ("created_at");