fix(run-store): fall back to legacy on an idempotency-key reset when the new DB matches nothing

Routing a predicate reset to the new database ONLY broke resets during the rollout window: a key
minted before an org flipped still lives on a legacy-resident run (idempotency TTL up to 30 days),
so a new-only reset matched zero rows, returned 404, and left the stale key deduping child triggers.
The reset now checks the new database first and falls back to legacy when nothing matched, so it
clears the key wherever the run lives; a fully-drained env still never touches legacy.

Author: d-cs

internal-packages/run-store/src/runOpsStore.envScopedResidency.test.ts

@@ -9,7 +9,9 @@ import type { RunStore } from "./types.js";
 type Call = { method: string; args: unknown[] };
 type FakeStore = RunStore & { slot: "new" | "legacy"; calls: Call[] };
 
-function fakeStore(slot: "new" | "legacy"): FakeStore {
+// `clearCount` lets a test say "this store matched N rows for the reset", so the NEW-first-then-fallback
+// path can be exercised (NEW matches 0 → fall back to LEGACY).
+function fakeStore(slot: "new" | "legacy", clearCount = slot === "new" ? 1 : 0): FakeStore {
   const calls: Call[] = [];
   const rec =
     (method: string, result: unknown) =>
@@ -21,13 +23,13 @@ function fakeStore(slot: "new" | "legacy"): FakeStore {
     slot,
     calls,
     upsertWaitpointTag: rec("upsertWaitpointTag", { id: slot, slot }),
-    clearIdempotencyKey: rec("clearIdempotencyKey", { count: slot === "new" ? 1 : 0 }),
+    clearIdempotencyKey: rec("clearIdempotencyKey", { count: clearCount }),
   } as unknown as FakeStore;
 }
 
-function buildRouter() {
-  const newStore = fakeStore("new");
-  const legacyStore = fakeStore("legacy");
+function buildRouter(newClearCount?: number, legacyClearCount?: number) {
+  const newStore = fakeStore("new", newClearCount);
+  const legacyStore = fakeStore("legacy", legacyClearCount);
   const router = new RoutingRunStore({
     new: newStore,
     legacy: legacyStore,
@@ -52,9 +54,9 @@ describe("RoutingRunStore.upsertWaitpointTag — residency hint for a tag with n
   });
 });
 
-describe("RoutingRunStore.clearIdempotencyKey — predicate routes NEW when the env mints new", () => {
-  it("routes a byPredicate reset to NEW only when residency is NEW (no legacy fan-out)", async () => {
-    const { router, newStore, legacyStore } = buildRouter();
+describe("RoutingRunStore.clearIdempotencyKey — predicate routes NEW-first when the env mints new", () => {
+  it("clears on NEW and does NOT touch legacy when NEW matches (post-flip key)", async () => {
+    const { router, newStore, legacyStore } = buildRouter(1, 0);
     const result = await router.clearIdempotencyKey({
       byPredicate: {
         idempotencyKey: "k",
@@ -68,6 +70,23 @@ describe("RoutingRunStore.clearIdempotencyKey — predicate routes NEW when the
     expect(result.count).toBe(1);
   });
 
+  it("falls back to LEGACY when NEW matches 0 (a key held on a pre-flip legacy run)", async () => {
+    // The env mints new now, but this key was created before the flip → its run lives on LEGACY.
+    const { router, newStore, legacyStore } = buildRouter(0, 1);
+    const result = await router.clearIdempotencyKey({
+      byPredicate: {
+        idempotencyKey: "k",
+        taskIdentifier: "task",
+        runtimeEnvironmentId: "env",
+        residency: "NEW",
+      },
+    });
+    // NEW checked first (0 rows), then LEGACY cleared the stale key — so the reset actually works.
+    expect(newStore.calls).toHaveLength(1);
+    expect(legacyStore.calls).toHaveLength(1);
+    expect(result.count).toBe(1);
+  });
+
   it("still fans out a byPredicate reset with no residency (mixed residency)", async () => {
     const { router, newStore, legacyStore } = buildRouter();
     await router.clearIdempotencyKey({

internal-packages/run-store/src/runOpsStore.ts

@@ -503,7 +503,7 @@ export class RoutingRunStore implements RunStore {
     return (await this.#routeOrNewForWrite(runId)).updateMetadata(runId, data, options);
   }
 
-  clearIdempotencyKey(
+  async clearIdempotencyKey(
     params: ClearIdempotencyKeyInput,
     tx?: PrismaClientOrTransaction
   ): Promise<{ count: number }> {
@@ -513,11 +513,17 @@ export class RoutingRunStore implements RunStore {
       const store = this.#route(params.byId.runId);
       return store.clearIdempotencyKey(params, undefined);
     }
-    // A `byPredicate` whose env mints run-ops ids has its matching runs on NEW, so route to NEW only
-    // and skip the wrong-DB (0-row) write to the draining legacy DB. Without that hint (or byFriendlyIds)
-    // the predicate can span mixed residency — fan out and sum.
+    // A `byPredicate` whose env mints run-ops ids has NEW-born runs, so check NEW first. But a key
+    // minted BEFORE the org flipped still lives on a LEGACY-resident run (idempotency TTL up to 30d),
+    // so fall back to LEGACY when NEW matched nothing — otherwise the reset 404s and the stale legacy
+    // key keeps deduping. In the steady (fully-drained) state NEW matches and legacy is never touched.
     if ("byPredicate" in params && params.byPredicate?.residency === "NEW") {
-      return this.#new.clearIdempotencyKey(params, undefined);
+      const fromNew = await this.#new.clearIdempotencyKey(params, undefined);
+      if (fromNew.count > 0) {
+        return fromNew;
+      }
+      const fromLegacy = await this.#legacy.clearIdempotencyKey(params);
+      return { count: fromNew.count + fromLegacy.count };
     }
     return Promise.all([
       this.#new.clearIdempotencyKey(params),