feat(fygaro): signature-failure alerts + bankowner float monitoring

dev/config/base-config.yaml

@@ -27,6 +27,11 @@ fygaro:
     # Phase gate: with credit OFF the webhook only records the payment and
     # notifies ops; the treasury -> user transfer stays manual.
     enabled: false
+  float:
+    # Alert (cron) when the bankowner treasury USDT balance — the auto-credit
+    # funding source — drops below this floor, in USD. Default ~4x the $500
+    # auto-credit limit so ops has runway to top up before the well runs dry.
+    floorUsd: 2000
 
 bridge:
   enabled: false # flags-off baseline; enable per environment via config overrides

src/config/schema.ts

@@ -802,6 +802,17 @@ export const configSchema = {
           // the treasury -> user transfer stays manual.
           default: { enabled: false },
         },
+        float: {
+          type: "object",
+          properties: {
+            // Alert when the bankowner treasury USDT balance (the auto-credit
+            // funding source) drops below this floor, in USD. Default ~4x the
+            // $500 auto-credit limit so ops has runway to top up.
+            floorUsd: { type: "number", default: 2000 },
+          },
+          additionalProperties: false,
+          default: { floorUsd: 2000 },
+        },
       },
       additionalProperties: false,
       // Default OFF baseline (mirrors topup); enable per environment via overrides.

src/config/schema.types.d.ts

@@ -80,6 +80,9 @@ type FygaroConfig = {
   credit: {
     enabled: boolean
   }
+  float: {
+    floorUsd: number
+  }
 }
 
 type CashoutEmail = {

src/servers/cron.ts

@@ -24,6 +24,7 @@ import {
   reconcileBridgeAndIbexDeposits,
   reconcileBridgeAndIbexWithdrawals,
 } from "@services/bridge/reconciliation"
+import { checkFygaroTreasuryFloat } from "@services/fygaro/float-monitor"
 
 import { elapsedSinceTimestamp, sleep } from "@utils"
 import { rebalancingInternalChannels } from "@services/lnd/rebalancing"
@@ -88,6 +89,13 @@ const reconcileBridgeWithdrawalsJob = async () => {
   if (result instanceof Error) throw result
 }
 
+// Bankowner treasury float check for Fygaro auto-credit. Runs every cron
+// invocation (~15 min via the k8s CronJob schedule, matching the reconcile
+// cadence above); self-guards on FygaroConfig.enabled and never throws.
+const checkFygaroFloatJob = async () => {
+  await checkFygaroTreasuryFloat()
+}
+
 const main = async () => {
   console.log("cronjob started")
   const start = new Date()
@@ -125,6 +133,7 @@ const main = async () => {
     ...(cronConfig.swapEnabled ? [swapOutJob] : []),
     reconcileBridgeDepositsJob,
     reconcileBridgeWithdrawalsJob,
+    checkFygaroFloatJob,
     deleteExpiredPaymentFlows,
     deleteExpiredInvoices,
     ...(cronConfig.lndTasksEnabled

src/services/alerts/dedup-key.ts

@@ -37,6 +37,16 @@ export const generateDedupKey = {
   // non-USD, non-positive net). Distinct from fygaroCreditFailed so a "skipped"
   // warning never suppresses a later "credit attempt failed" critical.
   fygaroNotCredited: (transactionId: string) => `fygaro:not-credited:${transactionId}`,
+  // Static keys (no per-request suffix) so the built-in TTL dedup rate-limits
+  // these to one alert per window rather than one per failing request/poll.
+  fygaroSignatureFailure: () => "fygaro:signature-failure",
+  // Distinct from fygaroSignatureFailure so a stuck server clock (every real
+  // webhook 401ing on skew) surfaces as its own warning instead of being
+  // collapsed into — or masked by — the "wrong secret" alert. Still static so
+  // replayed old webhooks collapse to one warning per window.
+  fygaroClockSkew: () => "fygaro:clock-skew",
+  fygaroFloatLow: () => "fygaro:float-low",
+  fygaroFloatExhausted: () => "fygaro:float-exhausted",
 }
 
 export const normalizeDedupKey = (key: string): string =>

src/services/fygaro/float-monitor.ts

@@ -0,0 +1,146 @@
+import { FygaroConfig } from "@config"
+import { USDAmount, USDTAmount } from "@domain/shared"
+import { getBalanceForWallet } from "@app/wallets/get-balance-for-wallet"
+import { alertBridge, generateDedupKey } from "@services/alerts"
+import { baseLogger } from "@services/logger"
+import { redis } from "@services/redis"
+
+import {
+  FygaroCreditError,
+  resolveFygaroTreasuryFundingWallet,
+} from "./webhook-server/credit-topup"
+
+/**
+ * Proactive bankowner treasury float check (the main value of the float-
+ * monitoring work). Auto-credit sends card top-ups from the bankowner treasury;
+ * if that float quietly drains, every subsequent credit fails one-at-a-time. A
+ * periodic check that pages BEFORE the well runs dry turns that into a single
+ * "top up soon" heads-up instead of a stream of exhausted-credit incidents.
+ *
+ * Contract: this is registered as a cron task and MUST NOT throw — a thrown
+ * task fails the whole cron run (exit 99 -> CrashLoopBackOff). Every failure
+ * path here logs and returns. Only runs when the fygaro feature is enabled AND
+ * auto-credit is on: during the record-only phase (fygaro.enabled=true,
+ * credit.enabled=false) nothing ever spends from the treasury, so paging "top
+ * up bankowner" would be premature noise that contradicts its own instruction.
+ */
+const DEFAULT_FLOOR_USD = 2000
+
+// Cross-run rate limit for the float-low warning. alertBridge's own TTL dedup
+// is a process-local Map, but this monitor runs as a one-shot cron Job (a fresh
+// process every ~15-min run), so that dedup resets each run and cannot suppress
+// anything across runs. Without this a treasury sitting below the floor would
+// page every run (~4/hr) instead of the ~1/hr the alert layer implies. A Redis
+// NX marker with a 1h TTL gives real cross-run suppression that survives the
+// process restarts. A Redis error falls through to alerting: over-notifying is
+// the safe failure mode for a draining float, never staying silent.
+const FLOAT_LOW_ALERT_WINDOW_SECONDS = 3600
+const FLOAT_LOW_ALERT_MARKER = "fygaro:float-low:alerted"
+
+const claimFloatLowAlertSlot = async (): Promise<boolean> => {
+  try {
+    const set = await redis.set(
+      FLOAT_LOW_ALERT_MARKER,
+      "1",
+      "EX",
+      FLOAT_LOW_ALERT_WINDOW_SECONDS,
+      "NX",
+    )
+    return set === "OK"
+  } catch (err) {
+    baseLogger.warn(
+      { err },
+      "Fygaro float check: dedup marker unavailable, alerting anyway",
+    )
+    return true
+  }
+}
+
+export const checkFygaroTreasuryFloat = async (): Promise<void> => {
+  // Gate on both the feature flag and the auto-credit master gate. With
+  // credit.enabled=false the webhook only records payments and the treasury ->
+  // user transfer stays manual, so there is nothing for this monitor to fund
+  // yet — running it would page every window over a float no credit touches.
+  if (!FygaroConfig.enabled || !FygaroConfig.credit?.enabled) return
+
+  const floorUsd = FygaroConfig.float?.floorUsd ?? DEFAULT_FLOOR_USD
+
+  try {
+    // Read the balance of the SAME wallet auto-credit actually spends from —
+    // resolved through the shared resolver credit-topup uses — so the monitored
+    // account is provably the funding source. In flash's IBEX-custodial model
+    // each walletId is its own IBEX account, so reading the bankowner account's
+    // default wallet would read a DIFFERENT account's balance (typically the USD
+    // wallet) and misread it as the USDT float: a drained USDT float would
+    // hide behind a funded USD wallet (no page ever fires) and a low USD wallet
+    // would false-alarm as "USDT float low".
+    const funding = await resolveFygaroTreasuryFundingWallet()
+    if (funding instanceof FygaroCreditError) {
+      // Could not resolve the treasury or its funding wallet. Log and bail; the
+      // next run re-reads. Never alert here (a distinct alert would fight the
+      // float-low dedup) and never crash the cron.
+      baseLogger.error(
+        { step: funding.step, detail: funding.message },
+        "Fygaro float check: could not resolve the bankowner treasury funding wallet",
+      )
+      return
+    }
+
+    const fundingWallet = funding.fundingWallet
+    // Read through the SAME helper the rest of the app uses so this monitor
+    // inherits IBEX's documented drain semantics instead of re-deriving them.
+    // Crucially, IBEX returns HTTP 404 for a drained / never-funded account and
+    // getBalanceForWallet maps that 404 -> ZERO (get-balance-for-wallet.ts). The
+    // old direct Ibex.getAccountDetails read surfaced that 404 as an IbexError,
+    // hit the `instanceof Error` bail, and returned WITHOUT alerting — silently
+    // missing the very empty-float condition this monitor exists to catch. Via
+    // the helper both drain signals (404 and an absent `balance` field) collapse
+    // to ZERO, so a dry treasury correctly trips the floor.
+    const balance = await getBalanceForWallet({
+      walletId: fundingWallet.id,
+      currency: fundingWallet.currency,
+    })
+
+    if (balance instanceof Error) {
+      // A genuine read blip (a non-404 IBEX error, or an unexpected throw the
+      // helper wraps) must never crash the cron and must never be mistaken for a
+      // low balance. Log and bail; the next run re-reads. (A distinct alert here
+      // would fight the float-low dedup.)
+      baseLogger.error(
+        { err: balance },
+        "Fygaro float check: could not read bankowner treasury balance",
+      )
+      return
+    }
+
+    // Score in the funding wallet's own currency so the USD fallback is not
+    // read as an empty USDT float (or vice versa). A ZERO from either drain
+    // signal above lands here as 0 and correctly trips the floor.
+    const balanceUsd =
+      balance instanceof USDTAmount
+        ? Number(balance.asNumber())
+        : balance instanceof USDAmount
+          ? Number(balance.asDollars())
+          : 0
+
+    if (balanceUsd < floorUsd) {
+      baseLogger.warn({ balanceUsd, floorUsd }, "Fygaro treasury float below floor")
+      // Rate-limit across cron runs via the Redis marker (see above); without
+      // it a persistent low float would page every ~15-min run.
+      if (await claimFloatLowAlertSlot()) {
+        alertBridge({
+          dedupKey: generateDedupKey.fygaroFloatLow(),
+          source: "fygaro-webhook",
+          severity: "warning",
+          title: "Fygaro treasury float low — top up bankowner",
+          detail: `balance=$${balanceUsd.toFixed(2)} floor=$${floorUsd.toFixed(2)}`,
+          context: { balance_usd: balanceUsd, floor_usd: floorUsd },
+        })
+      }
+    }
+  } catch (err) {
+    // Belt-and-suspenders: a resolver throw or any unexpected error is
+    // swallowed so the cron run still succeeds.
+    baseLogger.error({ err }, "Fygaro float check errored")
+  }
+}

src/services/fygaro/webhook-server/credit-topup.ts

@@ -1,7 +1,21 @@
 import { PaymentSendStatus } from "@domain/bitcoin/lightning"
 import { WalletCurrency } from "@domain/shared"
+import { InsufficientBalanceError } from "@domain/errors"
 import { AccountsRepository, WalletsRepository } from "@services/mongoose"
 import { baseLogger } from "@services/logger"
+import { InsufficientIbexBalance } from "@services/ibex/errors"
+
+// The treasury can't cover the send. On the flash IBEX-custodial path an
+// under-funded bankowner surfaces as InsufficientIbexBalance (ibex/errors.ts
+// maps the IBEX "insufficient balance" ApiError); InsufficientBalanceError is
+// the domain-level equivalent kept for defence in depth. Matched by class, not
+// by message, so a generic send error with a coincidental wording is NOT
+// misread as float exhaustion.
+const isInsufficientTreasuryBalance = (err: Error): boolean =>
+  err instanceof InsufficientIbexBalance || err instanceof InsufficientBalanceError
+
+/** Distinct credit-failure step signalling the treasury float is exhausted. */
+export const INSUFFICIENT_TREASURY_FLOAT_STEP = "insufficient-treasury-float"
 
 /**
  * Credits a verified Fygaro card payment to the payer's Flash account:
@@ -32,6 +46,56 @@ const walletsFor = async (accountId: AccountId): Promise<Wallet[]> => {
   return wallets instanceof Error ? [] : wallets
 }
 
+export type FygaroTreasuryFunding = {
+  account: Account
+  fundingWallet: Wallet
+}
+
+/**
+ * Resolves the bankowner treasury account and the exact wallet auto-credit
+ * spends from: the USDT cash wallet on every account (see
+ * accounts/create-account.ts), falling back to the legacy USD wallet.
+ *
+ * This is the SINGLE source of truth for treasury funding-wallet selection.
+ * Both the credit path (creditFygaroTopup, below) and the proactive float
+ * monitor (float-monitor.ts) resolve the funding wallet through here, so the
+ * monitor can never silently drift onto a different wallet than the one credits
+ * actually drain — the round-1 "monitor the exact wallet auto-credit spends
+ * from" invariant is now enforced by shared code, not by two copies staying
+ * byte-identical.
+ */
+export const resolveFygaroTreasuryFundingWallet = async (): Promise<
+  FygaroTreasuryFunding | FygaroCreditError
+> => {
+  const account = await AccountsRepository().findByRole(TREASURY_ROLE)
+  if (account instanceof Error) {
+    return new FygaroCreditError(
+      "resolve-treasury",
+      `no account holds the '${TREASURY_ROLE}' role`,
+    )
+  }
+
+  const wallets = await WalletsRepository().listByAccountId(account.id)
+  if (wallets instanceof Error) {
+    return new FygaroCreditError(
+      "list-treasury-wallets",
+      "could not list treasury wallets",
+    )
+  }
+
+  const fundingWallet =
+    wallets.find((w) => w.currency === WalletCurrency.Usdt) ??
+    wallets.find((w) => w.currency === WalletCurrency.Usd)
+  if (!fundingWallet) {
+    return new FygaroCreditError(
+      "resolve-treasury-wallet",
+      "treasury account has no USDT or USD wallet",
+    )
+  }
+
+  return { account, fundingWallet }
+}
+
 export const creditFygaroTopup = async ({
   recipientAccountId,
   amountCents,
@@ -47,26 +111,13 @@ export const creditFygaroTopup = async ({
     return new FygaroCreditError("validate-amount", `invalid amount: ${amountCents}`)
   }
 
-  const treasuryAccount = await AccountsRepository().findByRole(TREASURY_ROLE)
-  if (treasuryAccount instanceof Error) {
-    return new FygaroCreditError(
-      "resolve-treasury",
-      `no account holds the '${TREASURY_ROLE}' role`,
-    )
-  }
-
-  // Prefer the USDT wallet (the active cash wallet on every account — see
-  // accounts/create-account.ts), falling back to the legacy USD wallet.
-  const treasuryWallets = await walletsFor(treasuryAccount.id)
-  const fundingWallet =
-    treasuryWallets.find((w) => w.currency === WalletCurrency.Usdt) ??
-    treasuryWallets.find((w) => w.currency === WalletCurrency.Usd)
-  if (!fundingWallet) {
-    return new FygaroCreditError(
-      "resolve-treasury-wallet",
-      "treasury account has no USDT or USD wallet",
-    )
+  // Resolve the treasury funding wallet through the shared resolver so the
+  // wallet credits drain from is provably the same one float-monitor watches.
+  const funding = await resolveFygaroTreasuryFundingWallet()
+  if (funding instanceof FygaroCreditError) {
+    return funding
   }
+  const { fundingWallet } = funding
 
   // Recipients must hold a wallet in the funding wallet's currency —
   // send-intraledger rejects cross-currency sends.
@@ -98,6 +149,13 @@ export const creditFygaroTopup = async ({
       { err: result, transactionId, recipientAccountId },
       "fygaro credit: intraledger send returned an error",
     )
+    // Distinguish "the treasury is empty" (an ops top-up problem) from every
+    // other send failure (a bug to debug) so payment.ts can raise the right
+    // alert. The payment is still recorded and the row stays Fiat Received
+    // either way — no double-spend risk.
+    if (isInsufficientTreasuryBalance(result)) {
+      return new FygaroCreditError(INSUFFICIENT_TREASURY_FLOAT_STEP, result.message)
+    }
     return new FygaroCreditError("intraledger-send", result.message)
   }
   if (result === PaymentSendStatus.Success) {