From 56d37b34631c6bdc6a33281501c0148183d61afc Mon Sep 17 00:00:00 2001 From: Vishnu R Date: Tue, 22 Sep 2026 16:40:11 +0530 Subject: [PATCH] fix: fail validation when a runtime operation is undocumented An implemented route missing from OpenAPI was a warning, so flexdoc validate exited successfully unless CI opted into --fail-on warning. That disagreement is the check this cycle exists to run. It is now an error, the derived status is fail, and the default CLI policy exits 1 while naming the method and path. Duplicate registrations stay warnings. Signed-off-by: Vishnu R --- docs/runtime-intelligence.md | 2 +- .../backend/src/contract-validation.test.ts | 8 ++++--- packages/backend/src/contract-validation.ts | 2 +- .../backend/src/runtime-intelligence.test.ts | 3 ++- .../RuntimeIntelligencePanel.test.tsx | 6 ++--- tools/flexdoc-cli/README.md | 2 +- tools/flexdoc-cli/test/validation.test.mjs | 24 +++++++++++++++++++ 7 files changed, 37 insertions(+), 10 deletions(-) diff --git a/docs/runtime-intelligence.md b/docs/runtime-intelligence.md index 16bc7fdf..c84636e5 100644 --- a/docs/runtime-intelligence.md +++ b/docs/runtime-intelligence.md @@ -148,7 +148,7 @@ The Node Runtime Intelligence snapshot can include one backend-produced `validat The four finding codes in this cut are: -- `runtime.operation-undocumented` — the running backend positively exposes an operation that OpenAPI does not document. Severity: `warning`. This is runtime evidence only and is not clickable as a spec operation. +- `runtime.operation-undocumented` — the running backend positively exposes an operation that OpenAPI does not document. Severity: `error`. The finding names the method and path. This is runtime evidence only and is not clickable as a spec operation. Because it is an error, the derived status is `fail`, and `flexdoc validate` exits `1` under the default policy. - `runtime.operation-unobserved` — OpenAPI documents an operation that runtime discovery did not observe. Severity: `error` when discovery is complete, `info` when discovery is partial. - `runtime.method-mismatch` — the same wire-equivalent path exists in both sources but under different HTTP methods. Severity: `error` when discovery is complete, `warning` when partial. The finding carries an expected documented method so the renderer can navigate to the spec operation. - `runtime.duplicate-operation` — the host exposes multiple registrations for one wire-equivalent method/path. Severity: `warning`. In 3.1 this is an explicitly supported/verified claim for Express (including Nest-on-Express) and Hono. Fastify participates in the other validation checks, but exact duplicate-registration reporting is not claimed for Fastify in this cut. diff --git a/packages/backend/src/contract-validation.test.ts b/packages/backend/src/contract-validation.test.ts index 6cf0805e..a3107f38 100644 --- a/packages/backend/src/contract-validation.test.ts +++ b/packages/backend/src/contract-validation.test.ts @@ -56,7 +56,7 @@ describe('runtime contract validation', () => { })); }); - it('reports undocumented runtime operations as warnings', () => { + it('fails the default validation status when a runtime operation is undocumented', () => { const result = validateRuntimeContract({ documentedRoutes: [{ method: 'GET', path: '/pets' }], runtimeRoutes: [ @@ -66,11 +66,13 @@ describe('runtime contract validation', () => { discoveryComplete: true, }); - expect(result.status).toBe('warn'); + expect(result.status).toBe('fail'); + expect(result.summary).toEqual({ total: 1, errors: 1, warnings: 0, info: 0 }); expect(result.findings).toEqual([expect.objectContaining({ code: 'runtime.operation-undocumented', - severity: 'warning', + severity: 'error', location: { kind: 'operation', method: 'POST', path: '/internal/reindex' }, + message: 'Runtime implements POST /internal/reindex, but OpenAPI does not document that operation.', })]); }); diff --git a/packages/backend/src/contract-validation.ts b/packages/backend/src/contract-validation.ts index 9056ef21..fabe2f1c 100644 --- a/packages/backend/src/contract-validation.ts +++ b/packages/backend/src/contract-validation.ts @@ -149,7 +149,7 @@ export function validateRuntimeContract(options: ValidateRuntimeContractOptions) findings.push({ id: findingId('runtime.operation-undocumented', route.path, route.method), code: 'runtime.operation-undocumented', - severity: 'warning', + severity: 'error', location: { kind: 'operation', method: route.method.toUpperCase(), path: route.path }, message: `Runtime implements ${route.method.toUpperCase()} ${route.path}, but OpenAPI does not document that operation.`, expected: 'Operation is represented in OpenAPI', diff --git a/packages/backend/src/runtime-intelligence.test.ts b/packages/backend/src/runtime-intelligence.test.ts index 9e6018f8..6388e25a 100644 --- a/packages/backend/src/runtime-intelligence.test.ts +++ b/packages/backend/src/runtime-intelligence.test.ts @@ -130,7 +130,8 @@ describe('runtime intelligence', () => { expect(snapshot.summary).toEqual({ documented: 2, runtime: 2, matched: 1, runtimeOnly: 1, documentedOnly: 1 }); expect(snapshot.runtimeOnly).toEqual([{ method: 'POST', path: '/internal/reindex' }]); expect(snapshot.documentedOnly).toEqual([{ method: 'GET', path: '/pets/{petId}' }]); - expect(snapshot.validation.summary).toEqual({ total: 2, errors: 1, warnings: 1, info: 0 }); + expect(snapshot.validation.status).toBe('fail'); + expect(snapshot.validation.summary).toEqual({ total: 2, errors: 2, warnings: 0, info: 0 }); expect(snapshot.serverOrigin).toBe('https://api.example.com'); expect(snapshot.server).toEqual({ localPort: 8443 }); expect(snapshot.environment).toEqual({ name: 'production' }); diff --git a/packages/client/src/components/RuntimeIntelligencePanel.test.tsx b/packages/client/src/components/RuntimeIntelligencePanel.test.tsx index 899d6158..4530c5c2 100644 --- a/packages/client/src/components/RuntimeIntelligencePanel.test.tsx +++ b/packages/client/src/components/RuntimeIntelligencePanel.test.tsx @@ -29,14 +29,14 @@ const snapshot = { { id: 'runtime.operation-undocumented:POST:/internal/reindex', code: 'runtime.operation-undocumented' as const, - severity: 'warning' as const, + severity: 'error' as const, location: { kind: 'operation' as const, method: 'POST', path: '/internal/reindex' }, message: 'Runtime implements POST /internal/reindex, but OpenAPI does not document that operation.', expected: 'Operation is represented in OpenAPI', observed: 'Operation exists only in the running backend', }, ], - summary: { total: 2, errors: 1, warnings: 1, info: 0 }, + summary: { total: 2, errors: 2, warnings: 0, info: 0 }, }, }; @@ -48,7 +48,7 @@ describe('RuntimeIntelligencePanel', () => { it('shows runtime metadata and structured 3.1 contract findings', () => { render( undefined} snapshot={snapshot} />); expect(screen.getByText('FAIL · 2 findings')).toBeInTheDocument(); - expect(screen.getByText('1 errors · 1 warnings · 0 info')).toBeInTheDocument(); + expect(screen.getByText('2 errors · 0 warnings · 0 info')).toBeInTheDocument(); expect(screen.getByText('runtime.operation-unobserved')).toBeInTheDocument(); expect(screen.getByText('runtime.operation-undocumented')).toBeInTheDocument(); expect(screen.getByText('Expected: Operation is exposed by the running backend')).toBeInTheDocument(); diff --git a/tools/flexdoc-cli/README.md b/tools/flexdoc-cli/README.md index 3bed6aec..71e83a1a 100644 --- a/tools/flexdoc-cli/README.md +++ b/tools/flexdoc-cli/README.md @@ -65,7 +65,7 @@ npx @prauga/flexdoc-cli validate "$FLEXDOC_RUNTIME_URL" \ `--header ` is repeatable. Do not combine an explicit `Authorization` header with `--bearer` or `--basic`, and do not use bearer and basic together. CLI arguments can be visible to local process inspection, so supply credentials from your CI secret facility rather than committing them to scripts or repository configuration. -By default, `validate` exits `1` only when the backend validation status is `fail` (or when the endpoint/payload cannot be consumed). `warn` and `partial` remain successful so the backend's severity model is preserved without surprising existing CI. +By default, `validate` exits `1` only when the backend validation status is `fail` (or when the endpoint/payload cannot be consumed). An implemented-but-undocumented operation is an error, so it fails that default policy and the log names the method and path. Other warnings, and `partial` discovery, remain successful unless `--fail-on` asks for them. To make CI stricter, add a fail policy: diff --git a/tools/flexdoc-cli/test/validation.test.mjs b/tools/flexdoc-cli/test/validation.test.mjs index c8dd548b..2600f5e8 100644 --- a/tools/flexdoc-cli/test/validation.test.mjs +++ b/tools/flexdoc-cli/test/validation.test.mjs @@ -144,6 +144,30 @@ test('formats actionable human output for CI logs', () => { assert.match(output, /Observed: 2 runtime registrations/); }); +test('default validate policy fails an undocumented runtime operation and names it', async () => { + const output = []; + const undocumented = { + status: 'fail', + complete: true, + findings: [{ + id: 'runtime.operation-undocumented:POST:/internal/reindex', + code: 'runtime.operation-undocumented', + severity: 'error', + location: { kind: 'operation', method: 'POST', path: '/internal/reindex' }, + message: 'Runtime implements POST /internal/reindex, but OpenAPI does not document that operation.', + expected: 'Operation is represented in OpenAPI', + observed: 'Operation exists only in the running backend', + }], + summary: { total: 1, errors: 1, warnings: 0, info: 0 }, + }; + const exitCode = await runValidationCli(['https://api.example.test/docs/__flexdoc/runtime'], { + fetchImpl: async () => ({ ok: true, status: 200, json: async () => ({ validation: undocumented }) }), + log: (value) => output.push(value), + }); + assert.equal(exitCode, 1); + assert.match(output.join('\n'), /runtime\.operation-undocumented · POST \/internal\/reindex/); +}); + test('default validate policy exits 1 only for backend fail status', async () => { const output = []; const failureCode = await runValidationCli(['https://api.example.test/docs/__flexdoc/runtime', '--json'], {