fix(webapp): strip null bytes from idempotency and debounce keys at trigger (#4527)

## What

A trigger request carrying a Unicode NUL (`U+0000`) in the **idempotency
key** or **debounce key** reached `prisma.taskRun.create()` and failed
the insert, so the caller got an opaque 500 and the run was never
created.

These two keys are stored in `jsonb` columns (`idempotencyKeyOptions`,
`debounce`), and Postgres rejects a NUL inside a `jsonb` value with
`SQLSTATE 22P05` ("unsupported Unicode escape sequence ... cannot be
converted to text"). This fix strips the NUL from both keys at the
single trigger-input chokepoint (`#buildEngineTriggerInput`), which
every trigger path flows through (single, batch item, mollified, and
drainer replay).

Stripping matches the existing precedent for run errors and task events.
It does not change dedup behaviour: the idempotency **dedup identity**
is the hashed key (a clean 64-char digest), computed independently of
the raw key we clean, so dedup keeps working exactly as before. For
debounce the key is used directly, so the cleaned key also becomes the
grouping key, an acceptable change for input that is already malformed.

## Why not payload / metadata / tags

Those are `text` columns fed by `JSON.stringify`, which escapes a NUL to
a safe escape sequence, so they do not hit this failure on the normal
JSON path. (A raw NUL in a `text` column throws a different code,
`22021`, and is not what triggers this issue.) The observed failures are
the `jsonb` `22P05` variant, which is only reachable via the two key
fields.

## Evidence

Red then green (containerTest, real Postgres): with the fix reverted,
triggering through the real service with a NUL in
`idempotencyKeyOptions.key` / `debounce.key` fails with the exact
`22P05` signature; with the fix, the run is created and the stored key
has the NUL removed.

Full-stack e2e (isolated stack, real HTTP): `POST
/api/v1/tasks/:taskId/trigger` with a NUL inside
`idempotencyKeyOptions.key` (`"acme<NUL>inc"`) and, separately,
`debounce.key` (`"grp<NUL>1"`):

- both returned `HTTP 200` with a created run (previously `500`)
- stored `idempotencyKeyOptions` = `{ "key": "acmeinc", "scope": "run"
}` (7 chars, NUL removed)
- stored `debounce.key` = `"grp1"` (4 chars, NUL removed)
- both runs render in the dashboard

Unit tests cover the helper (strip, no-op fast path, object-reference
reuse, null/undefined pass-through).

## Rollout / rollback

Server-only webapp change, no flag. Zero behaviour change for clean
input; only affects inputs that previously 500'd. Rollback is a straight
revert, no data migration.

## Known limitation

A raw NUL in a plain-string idempotency key (not created via
`idempotencyKeys.create()`) lands in a `text` column and throws `22021`
instead. That variant is not addressed here because stripping it would
change the dedup identity, so it warrants a separate decision. Not
observed in practice.

refs TRI-13030

Author: ericallam

.server-changes/strip-null-bytes-trigger-keys.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: fix
+---
+
+Fixed a rare error where triggering a task could fail if the idempotency key or debounce key contained an invalid null character. The character is now removed automatically and the run is created as normal.

apps/webapp/app/runEngine/services/triggerTask.server.nullBytes.test.ts

@@ -0,0 +1,110 @@
+import { describe, expect, vi } from "vitest";
+
+vi.mock("~/db.server", () => ({
+  prisma: {},
+  $replica: {},
+  runOpsNewPrisma: {},
+  runOpsLegacyPrisma: {},
+  runOpsNewReplica: {},
+  runOpsLegacyReplica: {},
+}));
+vi.mock("~/v3/runOpsMigration/splitMode.server", () => ({ isSplitEnabled: async () => false }));
+vi.mock("~/services/platform.v3.server", async (importOriginal) => {
+  const actual = (await importOriginal()) as Record<string, unknown>;
+  return {
+    ...actual,
+    getEntitlement: vi.fn(),
+  };
+});
+
+import { setupAuthenticatedEnvironment } from "@internal/run-engine/tests";
+import { assertNonNullable, containerTest } from "@internal/testcontainers";
+import { trace } from "@opentelemetry/api";
+import { IdempotencyKeyConcern } from "~/runEngine/concerns/idempotencyKeys.server";
+import { DefaultQueueManager } from "~/runEngine/concerns/queues.server";
+import { RunEngineTriggerTaskService } from "./triggerTask.server";
+import {
+  buildEngine,
+  CapturingParentRunValidator,
+  MockPayloadProcessor,
+  MockTraceEventConcern,
+} from "./triggerTask.server.test.helpers";
+
+vi.setConfig({ testTimeout: 60_000 });
+
+const NUL = String.fromCharCode(0);
+
+function buildService(engine: any, prisma: any) {
+  return new RunEngineTriggerTaskService({
+    engine,
+    prisma,
+    payloadProcessor: new MockPayloadProcessor(),
+    queueConcern: new DefaultQueueManager(prisma, engine),
+    idempotencyKeyConcern: new IdempotencyKeyConcern(prisma, engine, new MockTraceEventConcern()),
+    validator: new CapturingParentRunValidator(),
+    traceEventConcern: new MockTraceEventConcern(),
+    tracer: trace.getTracer("test", "0.0.0"),
+    metadataMaximumSize: 1024 * 1024 * 1,
+  });
+}
+
+describe("RunEngineTriggerTaskService null-byte sanitization", () => {
+  containerTest(
+    "strips a NUL from idempotencyKeyOptions.key so the jsonb insert does not 22P05",
+    async ({ prisma, redisOptions }) => {
+      const engine = buildEngine(prisma, redisOptions);
+
+      try {
+        const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
+        const service = buildService(engine, prisma);
+
+        const result = await service.call({
+          taskId: "nul-idem-task",
+          environment,
+          body: {
+            payload: { kind: "idem" },
+            options: {
+              idempotencyKey: "a".repeat(64),
+              idempotencyKeyOptions: { key: `acme${NUL}inc`, scope: "run" },
+            },
+          },
+        });
+        assertNonNullable(result);
+
+        const row = await prisma.taskRun.findUniqueOrThrow({ where: { id: result.run.id } });
+        expect(row.idempotencyKeyOptions).toEqual({ key: "acmeinc", scope: "run" });
+      } finally {
+        await engine.quit();
+      }
+    }
+  );
+
+  containerTest(
+    "strips a NUL from debounce.key so the jsonb insert does not 22P05",
+    async ({ prisma, redisOptions }) => {
+      const engine = buildEngine(prisma, redisOptions);
+
+      try {
+        const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
+        const service = buildService(engine, prisma);
+
+        const result = await service.call({
+          taskId: "nul-debounce-task",
+          environment,
+          body: {
+            payload: { kind: "debounce" },
+            options: {
+              debounce: { key: `grp${NUL}1`, delay: "1s" },
+            },
+          },
+        });
+        assertNonNullable(result);
+
+        const row = await prisma.taskRun.findUniqueOrThrow({ where: { id: result.run.id } });
+        expect((row.debounce as { key: string }).key).toBe("grp1");
+      } finally {
+        await engine.quit();
+      }
+    }
+  );
+});

apps/webapp/app/runEngine/services/triggerTask.server.ts

@@ -25,6 +25,7 @@ import type { PrismaClientOrTransaction } from "@trigger.dev/database";
 import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
 import { parseDelay } from "~/utils/delays";
+import { removeNullBytesFromKey } from "~/utils/nullBytes";
 import { handleMetadataPacket } from "~/utils/packets";
 import { startSpan } from "~/v3/tracing.server";
 import { resolveRunIdMintKind } from "~/v3/engineVersion.server";
@@ -906,7 +907,7 @@ export class RunEngineTriggerTaskService {
       environment: args.environment,
       idempotencyKey: args.idempotencyKey,
       idempotencyKeyExpiresAt: args.idempotencyKey ? args.idempotencyKeyExpiresAt : undefined,
-      idempotencyKeyOptions: args.body.options?.idempotencyKeyOptions,
+      idempotencyKeyOptions: removeNullBytesFromKey(args.body.options?.idempotencyKeyOptions),
       taskIdentifier: args.taskId,
       payload: args.payloadPacket.data ?? "",
       payloadType: args.payloadPacket.dataType,
@@ -971,7 +972,7 @@ export class RunEngineTriggerTaskService {
       planType: args.planType,
       realtimeStreamsVersion: args.options.realtimeStreamsVersion,
       streamBasinName: args.environment.organization.streamBasinName,
-      debounce: args.body.options?.debounce,
+      debounce: removeNullBytesFromKey(args.body.options?.debounce),
       annotations: args.annotations,
     };
   }

apps/webapp/app/utils/nullBytes.test.ts

@@ -0,0 +1,36 @@
+import { describe, expect, it } from "vitest";
+import { removeNullBytes, removeNullBytesFromKey } from "./nullBytes";
+
+describe("removeNullBytes", () => {
+  it("strips every NUL from a string", () => {
+    expect(removeNullBytes(`a\u0000b\u0000c`)).toBe("abc");
+  });
+
+  it("returns the same reference when there is no NUL", () => {
+    const clean = "acme-inc";
+    expect(removeNullBytes(clean)).toBe(clean);
+  });
+
+  it("passes through undefined and null", () => {
+    expect(removeNullBytes(undefined)).toBeUndefined();
+    expect(removeNullBytes(null)).toBeNull();
+  });
+});
+
+describe("removeNullBytesFromKey", () => {
+  it("strips a NUL from the key while preserving other fields", () => {
+    expect(removeNullBytesFromKey({ key: `k\u00001`, scope: "run" })).toEqual({
+      key: "k1",
+      scope: "run",
+    });
+  });
+
+  it("returns the same object reference when the key is clean", () => {
+    const opts = { key: "clean", scope: "run" };
+    expect(removeNullBytesFromKey(opts)).toBe(opts);
+  });
+
+  it("passes through undefined", () => {
+    expect(removeNullBytesFromKey(undefined)).toBeUndefined();
+  });
+});

apps/webapp/app/utils/nullBytes.ts

@@ -0,0 +1,26 @@
+/**
+ * Removes Unicode NUL (U+0000) from a string. Postgres cannot store a NUL in a
+ * `text` column (SQLSTATE 22021) and rejects a `\u0000` escape when a JSON value
+ * is stored as `jsonb` (SQLSTATE 22P05), so a caller-supplied NUL reaching
+ * `taskRun.create()` fails the insert. The `indexOf` guard keeps the common
+ * (NUL-free) case allocation-free on the trigger hot path.
+ */
+export function removeNullBytes<T extends string | undefined | null>(value: T): T {
+  if (typeof value !== "string" || value.indexOf("\u0000") === -1) {
+    return value;
+  }
+  return value.replace(/\u0000/g, "") as T;
+}
+
+/**
+ * Returns `value` with a NUL-stripped `key`, reusing the original object when no
+ * NUL is present. Used for the user-supplied idempotency-key and debounce
+ * options, whose `key` lands in a `jsonb` column on the TaskRun row.
+ */
+export function removeNullBytesFromKey<T extends { key: string } | undefined>(value: T): T {
+  if (!value) {
+    return value;
+  }
+  const cleaned = removeNullBytes(value.key);
+  return cleaned === value.key ? value : { ...value, key: cleaned };
+}