feat(alerts): render Discord alerts as rich embeds, not a JSON dump

src/services/alerts/discord-embed.ts

@@ -0,0 +1,149 @@
+import { ErrorLevel } from "@domain/shared"
+import { recordExceptionInCurrentSpan } from "@services/tracing"
+import axios, { isAxiosError } from "axios"
+
+/**
+ * Shared Discord rich-embed contract for the alert senders (discord.ts and
+ * ops-events.ts). One place for Discord's documented limits, the field builder,
+ * the aggregate clamp, and the rate-limit-aware POST — so a future Discord
+ * change (a new limit, different retry logic) is made once, not twice.
+ *
+ * Discord embed limits: 25 fields, each value <= 1024 chars, each name <= 256,
+ * title <= 256, description <= 4096, and — the one the per-field caps do NOT
+ * bound — an aggregate <= 6000 chars summed across title + description + every
+ * field name + every field value (+ author/footer text). clampEmbedToBudget
+ * enforces both the aggregate AND the 25-field cap; postEmbed applies it before
+ * every POST.
+ */
+
+export const MAX_FIELDS = 25
+export const MAX_FIELD_VALUE = 1024
+export const MAX_TITLE = 256
+export const MAX_DESCRIPTION = 4096
+export const MAX_EMBED_TOTAL = 6000
+
+// A little under Discord's hard 6000 aggregate, leaving headroom for embed
+// scaffolding that also counts toward the limit (author line, timestamp label).
+const AGGREGATE_BUDGET = 5900
+
+const POST_TIMEOUT_MS = 5_000
+
+export type DiscordEmbedField = { name: string; value: string; inline: boolean }
+
+export type DiscordEmbed = {
+  author?: { name: string }
+  title: string
+  description?: string
+  color: number
+  fields: DiscordEmbedField[]
+  timestamp: string
+}
+
+// Trim a string to at most `max` characters, marking the cut with an ellipsis
+// so a reader can tell it was truncated. The ellipsis counts toward the cap:
+// the result length is never greater than `max`.
+export const truncate = (value: string, max: number): string =>
+  value.length > max ? value.slice(0, max - 1) + "…" : value
+
+// A closure that appends non-empty, name-capped, value-capped fields to the
+// given array. Shared so both alert senders build fields identically.
+export const makeFieldBuilder =
+  (fields: DiscordEmbedField[]) =>
+  (name: string, value: string | undefined | null, inline = true): void => {
+    if (value === undefined || value === null || value.length === 0) return
+    fields.push({
+      name: truncate(name, MAX_TITLE),
+      value: truncate(value, MAX_FIELD_VALUE),
+      inline,
+    })
+  }
+
+const fieldCost = (field: DiscordEmbedField): number =>
+  field.name.length + field.value.length
+
+// Keep an embed under Discord's aggregate character limit AND its 25-field cap.
+// The per-field caps bound each part but not their sum, so a caller that stuffs
+// a large value into one field can still build a >6000-char embed; likewise a
+// caller that emits one field per item can exceed 25 fields. Either overflow
+// makes Discord 400 the whole embed (which postEmbed then swallows as a Warn,
+// silently dropping the alert). We trim the description if it alone blows the
+// budget, then drop trailing fields once the running total OR the field count
+// would exceed the limit. Centralizing the count cap here protects every caller
+// (both discord.ts and ops-events.ts) without each having to slice itself.
+export const clampEmbedToBudget = (embed: DiscordEmbed): DiscordEmbed => {
+  const scaffold = (embed.author?.name.length ?? 0) + embed.title.length
+  let remaining = AGGREGATE_BUDGET - scaffold
+
+  let description = embed.description
+  if (description !== undefined) {
+    if (remaining <= 1) {
+      description = undefined
+    } else if (description.length > remaining) {
+      description = truncate(description, remaining)
+    }
+    remaining -= description?.length ?? 0
+  }
+
+  const fields: DiscordEmbedField[] = []
+  for (const field of embed.fields) {
+    if (fields.length >= MAX_FIELDS) break
+    const cost = fieldCost(field)
+    if (cost > remaining) break
+    remaining -= cost
+    fields.push(field)
+  }
+
+  if (fields.length === embed.fields.length && description === embed.description) {
+    return embed
+  }
+  return { ...embed, description, fields }
+}
+
+const sleep = (ms: number): Promise<void> =>
+  new Promise((resolve) => setTimeout(resolve, ms))
+
+// Discord rate-limit responses carry retry_after in seconds (possibly float),
+// in the body and/or the Retry-After header. Returns the wait in ms for a 429,
+// or undefined for any other (terminal) error.
+const retryAfterMs = (error: unknown): number | undefined => {
+  if (!isAxiosError(error) || error.response?.status !== 429) return undefined
+  const body = error.response.data as { retry_after?: number } | undefined
+  const retryAfter = body?.retry_after ?? Number(error.response.headers?.["retry-after"])
+  if (typeof retryAfter !== "number" || !Number.isFinite(retryAfter)) return 1_000
+  return Math.ceil(retryAfter * 1000)
+}
+
+/**
+ * POST a single rich embed to a Discord incoming webhook. Best-effort: never
+ * throws. Clamps the embed to Discord's aggregate limit first; on HTTP 429 it
+ * honors Discord's retry_after once; any other failure (or a failed retry) is
+ * recorded as a Warn and swallowed.
+ */
+export const postEmbed = async (
+  url: string,
+  embed: DiscordEmbed,
+  timeoutMs: number = POST_TIMEOUT_MS,
+): Promise<void> => {
+  const body = { embeds: [clampEmbedToBudget(embed)] }
+  const post = () =>
+    axios.post(url, body, {
+      timeout: timeoutMs,
+      headers: { "Content-Type": "application/json" },
+    })
+
+  try {
+    await post()
+  } catch (error) {
+    const waitMs = retryAfterMs(error)
+    if (waitMs === undefined) {
+      recordExceptionInCurrentSpan({ error, level: ErrorLevel.Warn })
+      return
+    }
+    await sleep(waitMs)
+    try {
+      await post()
+    } catch (retryError) {
+      recordExceptionInCurrentSpan({ error: retryError, level: ErrorLevel.Warn })
+    }
+  }
+}

src/services/alerts/discord.ts

@@ -1,34 +1,75 @@
 import { ALERT_DISCORD_WEBHOOK_URL } from "@config"
-import { ErrorLevel } from "@domain/shared"
-import { recordExceptionInCurrentSpan } from "@services/tracing"
-import axios from "axios"
 
 import { BridgeAlert } from "./index.types"
+import {
+  DiscordEmbed,
+  DiscordEmbedField,
+  MAX_DESCRIPTION,
+  MAX_TITLE,
+  makeFieldBuilder,
+  postEmbed,
+  truncate,
+} from "./discord-embed"
 
-// Discord caps message content at 2000 chars; leave headroom.
-const DISCORD_CONTENT_MAX = 1900
+// Warm red for a page, amber for a warning — the embed's left border colour.
+const COLOR_CRITICAL = 0xe01e5a
+const COLOR_WARNING = 0xf2c744
 
-// Discord incoming webhook ({ content }).
+// Friendly label for the embed author line, so a Fygaro alert doesn't read as a
+// "Bridge alert". Falls back to the raw source for anything unmapped.
+const SOURCE_LABEL: Record<string, string> = {
+  "fygaro-webhook": "Fygaro",
+  "bridge-webhook": "Bridge",
+  "bridge-api": "Bridge API",
+  "ibex": "IBEX",
+  "erpnext-audit": "ERPNext",
+}
+
+// Turn a context key (snake_case / camelCase) into a Title Case field name:
+// "transaction_id" -> "Transaction Id", "balanceUsd" -> "Balance Usd".
+const prettyKey = (key: string): string =>
+  key
+    .replace(/([a-z])([A-Z])/g, "$1 $2")
+    .replace(/[_-]+/g, " ")
+    .replace(/\b\w/g, (c) => c.toUpperCase())
+
+const contextFields = (
+  context: Record<string, unknown> | undefined,
+): DiscordEmbedField[] => {
+  const fields: DiscordEmbedField[] = []
+  if (!context) return fields
+  const field = makeFieldBuilder(fields)
+  for (const [key, raw] of Object.entries(context)) {
+    if (raw === undefined || raw === null) continue
+    field(prettyKey(key), String(raw))
+  }
+  return fields
+}
+
+// Discord incoming webhook — a single rich embed (colour by severity, title,
+// detail as the description, and each context entry as its own field) instead
+// of a raw JSON dump. Delivery is best-effort via postEmbed: the embed is
+// clamped to Discord's aggregate limit, a 429 is retried once, and any other
+// failure is swallowed as a Warn.
 export const sendDiscord = async (alert: BridgeAlert): Promise<void> => {
   if (!ALERT_DISCORD_WEBHOOK_URL) return
 
-  const label = alert.severity === "critical" ? "[CRITICAL]" : "[WARNING]"
-  let content = `${label} **Bridge alert** - ${alert.title}\nsource: \`${alert.source}\` | severity: \`${alert.severity}\``
-  if (alert.detail) content += `\n${alert.detail}`
-  if (alert.context) {
-    content += "\n```json\n" + JSON.stringify(alert.context, null, 2) + "\n```"
-  }
-  if (content.length > DISCORD_CONTENT_MAX) {
-    content = content.slice(0, DISCORD_CONTENT_MAX) + "..."
-  }
+  const fields: DiscordEmbedField[] = []
+  const field = makeFieldBuilder(fields)
+  field("Source", alert.source)
+  field("Severity", alert.severity)
+  fields.push(...contextFields(alert.context))
 
-  try {
-    await axios.post(
-      ALERT_DISCORD_WEBHOOK_URL,
-      { content },
-      { timeout: 5000, headers: { "Content-Type": "application/json" } },
-    )
-  } catch (error) {
-    recordExceptionInCurrentSpan({ error, level: ErrorLevel.Warn })
+  const embed: DiscordEmbed = {
+    author: { name: SOURCE_LABEL[alert.source] ?? alert.source },
+    title: truncate(alert.title, MAX_TITLE),
+    description: alert.detail ? truncate(alert.detail, MAX_DESCRIPTION) : undefined,
+    color: alert.severity === "critical" ? COLOR_CRITICAL : COLOR_WARNING,
+    // Field count is capped centrally by clampEmbedToBudget (applied in
+    // postEmbed), so no per-caller slice is needed here.
+    fields,
+    timestamp: new Date().toISOString(),
   }
+
+  await postEmbed(ALERT_DISCORD_WEBHOOK_URL, embed)
 }

src/services/alerts/ops-events.ts

@@ -1,7 +1,13 @@
 import { NETWORK, OPS_DISCORD_WEBHOOK_URL } from "@config"
 import { ErrorLevel, JMDAmount, USDAmount, USDTAmount } from "@domain/shared"
 import { recordExceptionInCurrentSpan } from "@services/tracing"
-import axios, { isAxiosError } from "axios"
+
+import {
+  DiscordEmbed,
+  DiscordEmbedField,
+  makeFieldBuilder,
+  postEmbed,
+} from "./discord-embed"
 
 /**
  * Fire-and-forget ops event feed: posts color-coded Discord embeds to
@@ -95,20 +101,10 @@ const titleCase = (phrase: string): string =>
     .map((word) => (word === "otp" ? "OTP" : word))
     .join(" ")
 
-type DiscordEmbedField = { name: string; value: string; inline: boolean }
-type DiscordEmbed = {
-  title: string
-  color: number
-  fields: DiscordEmbedField[]
-  timestamp: string
-}
-
 export const buildEmbed = (event: OpsEvent): DiscordEmbed => {
   const flowTitle = event.flow[0].toUpperCase() + event.flow.slice(1)
   const fields: DiscordEmbedField[] = []
-  const field = (name: string, value: string | undefined, inline = true) => {
-    if (value) fields.push({ name, value, inline })
-  }
+  const field = makeFieldBuilder(fields)
 
   field("account", event.accountId && truncateId(event.accountId))
   field("user", event.userId && truncateId(event.userId))
@@ -146,53 +142,22 @@ const queue: OpsEvent[] = []
 let droppedCount = 0
 let draining: Promise<void> | undefined
 
-const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))
-
-// Discord rate limit responses carry retry_after in seconds (possibly float),
-// in the body and/or the Retry-After header.
-const retryAfterMs = (error: unknown): number | undefined => {
-  if (!isAxiosError(error) || error.response?.status !== 429) return undefined
-  const body = error.response.data as { retry_after?: number } | undefined
-  const retryAfter = body?.retry_after ?? Number(error.response.headers?.["retry-after"])
-  if (typeof retryAfter !== "number" || !Number.isFinite(retryAfter)) return 1_000
-  return Math.ceil(retryAfter * 1000)
-}
-
-const postEmbed = async (embed: DiscordEmbed): Promise<void> => {
-  const post = () =>
-    axios.post(
-      OPS_DISCORD_WEBHOOK_URL as string,
-      { embeds: [embed] },
-      { timeout: FETCH_TIMEOUT_MS, headers: { "Content-Type": "application/json" } },
-    )
-
-  try {
-    await post()
-  } catch (error) {
-    const waitMs = retryAfterMs(error)
-    if (waitMs === undefined) {
-      recordExceptionInCurrentSpan({ error, level: ErrorLevel.Warn })
-      return
-    }
-    await sleep(waitMs)
-    try {
-      await post()
-    } catch (retryError) {
-      recordExceptionInCurrentSpan({ error: retryError, level: ErrorLevel.Warn })
-    }
-  }
-}
+// Deliver via the shared, rate-limit-aware sender. It clamps to Discord's
+// aggregate limit and honors a 429's retry_after once; FETCH_TIMEOUT_MS keeps
+// the ops feed's tighter 3s timeout.
+const send = (embed: DiscordEmbed): Promise<void> =>
+  postEmbed(OPS_DISCORD_WEBHOOK_URL as string, embed, FETCH_TIMEOUT_MS)
 
 const drain = async (): Promise<void> => {
   try {
     while (queue.length > 0) {
       const event = queue.shift()
-      if (event) await postEmbed(buildEmbed(event))
+      if (event) await send(buildEmbed(event))
 
       if (queue.length === 0 && droppedCount > 0) {
         const dropped = droppedCount
         droppedCount = 0
-        await postEmbed(droppedSummaryEmbed(dropped))
+        await send(droppedSummaryEmbed(dropped))
       }
     }
   } catch (error) {