|
39 | 39 | */ |
40 | 40 |
|
41 | 41 | import { describe, it, expect, beforeEach } from 'vitest'; |
42 | | -import { AutomationEngine, InMemorySuspendedRunStore, resolveRunDataContext } from '@objectstack/service-automation'; |
| 42 | +import { AutomationEngine, InMemorySuspendedRunStore, ObjectStoreSuspendedRunStore, resolveRunDataContext } from '@objectstack/service-automation'; |
43 | 43 | // [#4550] The engine double below routes its write verbs through ObjectQL's OWN |
44 | 44 | // dispatch predicates rather than a hand-mirrored copy. |
45 | 45 | import { assertEngineDeleteDispatch, assertEngineUpdateDispatch } from '@objectstack/objectql'; |
@@ -1035,3 +1035,157 @@ describe('ruling A — a decision resumes the run as the person who decided', () |
1035 | 1035 | expect(seen.after_reject, 'the replay fell back to the trigger user').toEqual({ userId: 'u1', permissions: ['deal_manager'] }); |
1036 | 1036 | }); |
1037 | 1037 | }); |
| 1038 | + |
| 1039 | +/** |
| 1040 | + * GUARD 3 proves where the run is parked past the first page of parked runs |
| 1041 | + * (#28271). |
| 1042 | + * |
| 1043 | + * It used to find the run in `listSuspendedRunsDurable()`. A DB-backed store |
| 1044 | + * lists one 1000-row page of `paused` rows in driver order, and a process adds |
| 1045 | + * only what it parked itself. So when another process (a replica, or this one |
| 1046 | + * before a restart) parked the run behind more than 1000 others, the run was |
| 1047 | + * not on the listing, and every re-issue was refused with "parked at no node |
| 1048 | + * this engine can see" — the retry the refusal promises read the same page. |
| 1049 | + * Both replicas below share one store and one table set: `a` strands and |
| 1050 | + * restores the run, `b` re-issues it. |
| 1051 | + */ |
| 1052 | +describe('#28271 — GUARD 3 reads where the run is parked by its id, not from one page', () => { |
| 1053 | + let data: ReturnType<typeof makeFakeEngine>; |
| 1054 | + let marks: string[]; |
| 1055 | + let rejectBranchThrows: string | undefined; |
| 1056 | + let a: ReturnType<typeof wire>; |
| 1057 | + let b: ReturnType<typeof wire>; |
| 1058 | + |
| 1059 | + /** One process: a real engine over `store`, the approval node, and its own approvals service. */ |
| 1060 | + function wire(store: ObjectStoreSuspendedRunStore) { |
| 1061 | + const service = new ApprovalService({ engine: data as any, logger: noopLogger }); |
| 1062 | + const automation = new AutomationEngine(noopLogger as any, store); |
| 1063 | + registerApprovalNode(automation, service, noopLogger as any); |
| 1064 | + automation.registerNodeExecutor({ |
| 1065 | + type: 'mark', |
| 1066 | + async execute(node: any) { |
| 1067 | + if (node.id === 'mark_rejected' && rejectBranchThrows) throw new Error(rejectBranchThrows); |
| 1068 | + marks.push(node.id); |
| 1069 | + return { success: true }; |
| 1070 | + }, |
| 1071 | + } as never); |
| 1072 | + automation.registerFlow('deal_approval', DEAL_APPROVAL as never); |
| 1073 | + automation.registerFlow('revise_flow', REVISE_FLOW as never); |
| 1074 | + service.attachAutomation(automation); |
| 1075 | + return { automation, service }; |
| 1076 | + } |
| 1077 | + |
| 1078 | + /** `count` parked runs of some other flow, stored ahead of everything this test parks. */ |
| 1079 | + const parkOthers = (count: number) => { |
| 1080 | + const rows = data.tables.get('sys_automation_run') ?? (data.tables.set('sys_automation_run', []), data.tables.get('sys_automation_run')!); |
| 1081 | + for (let i = 0; i < count; i++) { |
| 1082 | + rows.push({ id: `other-${String(i).padStart(4, '0')}`, flow_name: 'other_flow', node_id: 'wait', status: 'paused' }); |
| 1083 | + } |
| 1084 | + }; |
| 1085 | + |
| 1086 | + /** `b`'s engine with one member taken away, everything else delegated. */ |
| 1087 | + const without = (member: string) => new Proxy(b.automation, { |
| 1088 | + get(target, prop) { |
| 1089 | + if (prop === member) return undefined; |
| 1090 | + const value = Reflect.get(target, prop, target); |
| 1091 | + return typeof value === 'function' ? value.bind(target) : value; |
| 1092 | + }, |
| 1093 | + }); |
| 1094 | + |
| 1095 | + beforeEach(() => { |
| 1096 | + marks = []; rejectBranchThrows = undefined; |
| 1097 | + data = makeFakeEngine(); |
| 1098 | + const store = new ObjectStoreSuspendedRunStore(data as any, noopLogger as any); |
| 1099 | + a = wire(store); |
| 1100 | + b = wire(store); |
| 1101 | + }); |
| 1102 | + |
| 1103 | + const rowOf = async (id: string) => |
| 1104 | + (await data.find('sys_approval_request', { where: { id } }))[0]; |
| 1105 | + |
| 1106 | + /** On `a`: park, strand a reject decision, and (unless told not to) re-arm its pause. */ |
| 1107 | + async function strandOnA({ restore = true } = {}) { |
| 1108 | + await a.automation.execute('deal_approval', { |
| 1109 | + object: 'crm_deal', record: { id: 'd1', amount: 100 }, userId: 'submitter', |
| 1110 | + } as never); |
| 1111 | + const req = (await data.find('sys_approval_request', { where: { status: 'pending' } }))[0]; |
| 1112 | + rejectBranchThrows = DOWNSTREAM_FAILURE; |
| 1113 | + const err = await a.service |
| 1114 | + .decide(req.id, { decision: 'reject', actorId: 'u1', comment: 'no' }, SYSTEM_CTX) |
| 1115 | + .then(() => null, (e: Error) => e); |
| 1116 | + expect(err?.message).toMatch(/^RESUME_FAILED/); |
| 1117 | + if (restore) { |
| 1118 | + expect((await a.automation.restoreConsumedSuspension(req.flow_run_id, { requestedBy: 'ops' })).restored).toBe(true); |
| 1119 | + } |
| 1120 | + rejectBranchThrows = undefined; |
| 1121 | + return req; |
| 1122 | + } |
| 1123 | + |
| 1124 | + it('past the first page, the pause another process re-armed is proved and the decision is re-issued', async () => { |
| 1125 | + parkOthers(1000); |
| 1126 | + const req = await strandOnA(); |
| 1127 | + // The premise: `b` holds nothing in memory, and the run is not on the page. |
| 1128 | + expect(await b.automation.hasSuspendedRun(req.flow_run_id), 'the pause is back, in the store').toBe(true); |
| 1129 | + expect((await b.automation.listSuspendedRunsDurable()).some(r => r.runId === req.flow_run_id), |
| 1130 | + 'and not on the listing\'s one page').toBe(false); |
| 1131 | + |
| 1132 | + const out = await b.service.continueRestoredRun(req.id, { requestedBy: 'ops', reason: 'record recreated' }); |
| 1133 | + expect(out.resumed, JSON.stringify(out)).toBe(true); |
| 1134 | + expect(out.decision).toBe('reject'); |
| 1135 | + expect(out.source).toBe('journal'); |
| 1136 | + expect(marks, 'the reject branch ran').toEqual(['mark_rejected']); |
| 1137 | + expect(await b.automation.hasSuspendedRun(req.flow_run_id), 'the continuation consumed the pause').toBe(false); |
| 1138 | + }); |
| 1139 | + |
| 1140 | + it('CONTROL: within the first page it is re-issued as before', async () => { |
| 1141 | + parkOthers(998); |
| 1142 | + const req = await strandOnA(); |
| 1143 | + expect((await b.automation.listSuspendedRunsDurable()).some(r => r.runId === req.flow_run_id)).toBe(true); |
| 1144 | + |
| 1145 | + const out = await b.service.continueRestoredRun(req.id, { requestedBy: 'ops' }); |
| 1146 | + expect(out.resumed, JSON.stringify(out)).toBe(true); |
| 1147 | + expect(marks).toEqual(['mark_rejected']); |
| 1148 | + }); |
| 1149 | + |
| 1150 | + it('CONTROL: past the first page, a run parked at another node is still refused, and the refusal names that node', async () => { |
| 1151 | + parkOthers(1000); |
| 1152 | + await a.automation.execute('revise_flow', { |
| 1153 | + object: 'crm_deal', record: { id: 'd7', amount: 3 }, userId: 'submitter', |
| 1154 | + } as never); |
| 1155 | + const rev = (await data.find('sys_approval_request', { where: { status: 'pending' } }))[0]; |
| 1156 | + await a.service.sendBack(rev.id, { actorId: 'u1', comment: 'fix it' } as any, SYSTEM_CTX); |
| 1157 | + expect((await rowOf(rev.id)).status).toBe('returned'); |
| 1158 | + |
| 1159 | + const refused = await b.service.continueRestoredRun(rev.id).then(() => null, (e: Error) => e); |
| 1160 | + expect(refused?.message).toMatch( |
| 1161 | + /^INVALID_STATE: run '.*' is parked at node 'wait_revision', but the send-back on request .* was issued from its own approval node 'review'/, |
| 1162 | + ); |
| 1163 | + expect(await data.find('sys_approval_request', { where: { flow_run_id: rev.flow_run_id } }), |
| 1164 | + 'no new pending round was opened').toHaveLength(1); |
| 1165 | + }); |
| 1166 | + |
| 1167 | + it('CONTROL: past the first page, a run that is not parked is still refused — by GUARD 3 too, where GUARD 2 cannot ask', async () => { |
| 1168 | + parkOthers(1000); |
| 1169 | + const req = await strandOnA({ restore: false }); |
| 1170 | + |
| 1171 | + const byGuard2 = await b.service.continueRestoredRun(req.id).then(() => null, (e: Error) => e); |
| 1172 | + expect(byGuard2?.message).toMatch(/is not suspended/); |
| 1173 | + |
| 1174 | + // An engine that cannot answer `hasSuspendedRun` skips GUARD 2; GUARD 3's |
| 1175 | + // keyed read finds no pause and refuses. |
| 1176 | + b.service.attachAutomation(without('hasSuspendedRun') as any); |
| 1177 | + const byGuard3 = await b.service.continueRestoredRun(req.id).then(() => null, (e: Error) => e); |
| 1178 | + expect(byGuard3?.message).toMatch(/^INVALID_STATE: run '.*' is parked at no node this engine can see/); |
| 1179 | + expect(marks, 'nothing ran').toEqual([]); |
| 1180 | + }); |
| 1181 | + |
| 1182 | + it('CONTROL: an engine that cannot say where a run is parked is refused', async () => { |
| 1183 | + const req = await strandOnA(); |
| 1184 | + b.service.attachAutomation(without('suspensionOf') as any); |
| 1185 | + |
| 1186 | + const refused = await b.service.continueRestoredRun(req.id).then(() => null, (e: Error) => e); |
| 1187 | + expect(refused?.message).toMatch(/^INVALID_STATE: this automation engine cannot report WHERE run '.*' is parked \(no suspensionOf\)/); |
| 1188 | + expect(marks, 'nothing ran').toEqual([]); |
| 1189 | + expect(await b.automation.hasSuspendedRun(req.flow_run_id), 'the pause is left as it was').toBe(true); |
| 1190 | + }); |
| 1191 | +}); |
0 commit comments