From b06568a203d43a77d305e58b478a86e99b30564f Mon Sep 17 00:00:00 2001 From: Michael Dailey Date: Mon, 11 May 2026 16:34:10 -0500 Subject: [PATCH 1/3] PDX-466: feat(mcp): add AJV JSON schema validation alongside hardcoded NX rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RCA: provar_nitrox_validate only ran hardcoded NX001–NX010 semantic rules; structural errors (wrong types, extra properties, enum violations) encoded in FactComponent.schema.json were never caught at validation time. Fix: Added Ajv2020 as a runtime dependency; schema is lazily loaded from lib/mcp/rules/FactComponent.schema.json on first call and validated in parallel with existing rules. Violations are returned as NX_SCHEMA_ issues (ERROR for type/required, WARNING for additionalProperties/pattern/enum). Falls back to hardcoded-rules-only when schema is unavailable. --- docs/mcp.md | 22 ++++++++- package.json | 3 +- server.json | 4 +- src/mcp/tools/nitroXTools.ts | 66 +++++++++++++++++++++++--- test/unit/mcp/nitroXTools.test.ts | 78 +++++++++++++++++++++++++++++++ 5 files changed, 161 insertions(+), 12 deletions(-) diff --git a/docs/mcp.md b/docs/mcp.md index 82627ac..bcec07e 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -1643,7 +1643,10 @@ Path policy is enforced per-file. A missing or unparseable file returns an `erro ### `provar_nitrox_validate` -Validate a NitroX `.po.json` (Hybrid Model component page object) against the FACT schema rules. Returns a quality score (0–100) and a list of issues. +Validate a NitroX `.po.json` (Hybrid Model component page object) against the FACT schema rules. Returns a quality score (0–100) and a combined list of issues from two validation passes that run in parallel: + +1. **Hardcoded semantic rules (NX001–NX010)** — always run +2. **JSON schema validation (NX*SCHEMA*\*)** — runs when the bundled `FactComponent.schema.json` is available; falls back to hardcoded-rules-only if the schema cannot be loaded Score formula: `100 − (20 × errors) − (5 × warnings) − (1 × infos)`, minimum 0. @@ -1659,7 +1662,7 @@ Score formula: `100 − (20 × errors) − (5 × warnings) − (1 × infos)`, mi | `issue_count` | Total issues | | `issues` | Array of `ValidationIssue` (see below) | -**Validation rules:** +**Hardcoded rules:** | Rule | Severity | Description | | ----- | -------- | -------------------------------------------------------------------------------------------------------------------------------------------- | @@ -1675,6 +1678,21 @@ Score formula: `100 − (20 × errors) − (5 × warnings) − (1 × infos)`, mi | NX009 | INFO | Interaction `name` contains characters outside `[A-Za-z0-9 ]` | | NX010 | INFO | `bodyTagName` contains whitespace | +**JSON schema rules (NX*SCHEMA*\*):** + +Rule IDs follow the pattern `NX_SCHEMA_` where `` is the AJV validation keyword in `SCREAMING_SNAKE_CASE`. Common rule IDs: + +| Rule ID | Severity | Description | +| --------------------------------- | -------- | ------------------------------------------------------------------------------ | +| `NX_SCHEMA_TYPE` | ERROR | Property has the wrong JSON type (e.g. string where boolean expected) | +| `NX_SCHEMA_REQUIRED` | ERROR | Required property missing (per JSON schema `required` array) | +| `NX_SCHEMA_MIN_ITEMS` | ERROR | Array has fewer items than `minItems` requires | +| `NX_SCHEMA_ADDITIONAL_PROPERTIES` | WARNING | Property not defined in the schema (schema uses `additionalProperties: false`) | +| `NX_SCHEMA_PATTERN` | WARNING | String value does not match the schema `pattern` | +| `NX_SCHEMA_ENUM` | WARNING | Value not in the allowed `enum` list | + +Schema issues complement — and may overlap with — the hardcoded NX rules. When overlap occurs, both rule IDs appear in the `issues` array. + **Error codes:** `MISSING_INPUT`, `NX000`, `FILE_NOT_FOUND`, `PATH_NOT_ALLOWED` --- diff --git a/package.json b/package.json index f1f95d1..90c374d 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@provartesting/provardx-cli", "description": "A plugin for the Salesforce CLI to orchestrate testing activities and report quality metrics to Provar Quality Hub", - "version": "1.5.0-beta.18", + "version": "1.5.0-beta.19", "mcpName": "io.github.ProvarTesting/provar", "license": "BSD-3-Clause", "plugins": [ @@ -11,6 +11,7 @@ "dependencies": { "@modelcontextprotocol/sdk": "^1.8.0", "@oclif/core": "^3.27.0", + "ajv": "^8.17.1", "@provartesting/provardx-plugins-automation": "1.2.2", "@provartesting/provardx-plugins-manager": "1.3.2", "@provartesting/provardx-plugins-utils": "1.3.3", diff --git a/server.json b/server.json index 76faa4b..f7d1f22 100644 --- a/server.json +++ b/server.json @@ -14,12 +14,12 @@ "url": "https://github.com/ProvarTesting/provardx-cli", "source": "github" }, - "version": "1.5.0-beta.18", + "version": "1.5.0-beta.19", "packages": [ { "registryType": "npm", "identifier": "@provartesting/provardx-cli", - "version": "1.5.0-beta.18", + "version": "1.5.0-beta.19", "transport": { "type": "stdio" }, diff --git a/src/mcp/tools/nitroXTools.ts b/src/mcp/tools/nitroXTools.ts index 1a8821b..d94986a 100644 --- a/src/mcp/tools/nitroXTools.ts +++ b/src/mcp/tools/nitroXTools.ts @@ -10,6 +10,8 @@ import fs from 'node:fs'; import path from 'node:path'; import os from 'node:os'; import { randomUUID } from 'node:crypto'; +import { fileURLToPath } from 'node:url'; +import { Ajv2020, type ValidateFunction, type ErrorObject } from 'ajv/dist/2020.js'; import { z } from 'zod'; import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import type { ServerConfig } from '../server.js'; @@ -36,6 +38,49 @@ function isObj(v: unknown): v is JsonObj { return typeof v === 'object' && v !== null && !Array.isArray(v); } +// ── AJV Schema Validator ────────────────────────────────────────────────────── + +const RULES_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'rules'); + +let cachedFactComponentValidator: ValidateFunction | null | undefined; + +function getFactComponentValidator(): ValidateFunction | null { + if (cachedFactComponentValidator !== undefined) return cachedFactComponentValidator; + + const schemaPath = path.join(RULES_DIR, 'FactComponent.schema.json'); + try { + // Fix known broken $ref in the bundled schema (#/defs/ → #/$defs/) + const patched = fs.readFileSync(schemaPath, 'utf-8').replace(/"#\/defs\//g, '"#/$defs/'); + const schema = JSON.parse(patched) as Record; + const ajv = new Ajv2020({ allErrors: true, strict: false, validateFormats: false }); + cachedFactComponentValidator = ajv.compile(schema); + } catch (e) { + log('warn', 'provar_nitrox_validate: FactComponent schema unavailable, using hardcoded rules only', { + error: String(e), + }); + cachedFactComponentValidator = null; + } + return cachedFactComponentValidator; +} + +function ajvErrorToIssue(err: ErrorObject): NitroXIssue { + const keyword = err.keyword.replace(/([a-z])([A-Z])/g, '$1_$2').toUpperCase(); + const instancePath = err.instancePath; + const appliesTo = instancePath ? instancePath.replace(/^\//, '').replace(/\//g, '.') : 'root'; + const pathParts = instancePath.split('/').filter(Boolean); + const severity: 'ERROR' | 'WARNING' = ['REQUIRED', 'TYPE', 'MIN_ITEMS', 'MINIMUM', 'MAXIMUM'].includes(keyword) + ? 'ERROR' + : 'WARNING'; + const issue: NitroXIssue = { + rule_id: `NX_SCHEMA_${keyword}`, + severity, + message: `Schema: ${instancePath || 'root'} — ${err.message ?? 'validation failed'}`, + applies_to: appliesTo, + }; + if (pathParts.length > 0) issue.field = pathParts[pathParts.length - 1]; + return issue; +} + // ── Directory Utilities ─────────────────────────────────────────────────────── const SKIP_DIRS = new Set(['node_modules', '.git']); @@ -168,40 +213,45 @@ function validateRootProperties(obj: JsonObj, issues: NitroXIssue[]): void { } } -/** Validate a parsed NitroX .po.json object against schema-derived rules. */ -export function validateNitroXContent(obj: JsonObj): NitroXValidationResult { +/** Validate a parsed NitroX .po.json against hardcoded NX rules and the FactComponent JSON schema. */ +export function validateNitroXContent(obj: JsonObj, schemaOverride?: ValidateFunction | null): NitroXValidationResult { const issues: NitroXIssue[] = []; validateRootProperties(obj, issues); - // Validate root-level parameters if (Array.isArray(obj['parameters'])) { for (const param of obj['parameters']) { if (isObj(param)) validateParameter(param, 'root', issues); } } - // Validate root-level interactions if (Array.isArray(obj['interactions'])) { for (const interaction of obj['interactions']) { if (isObj(interaction)) validateInteraction(interaction, 'root', issues); } } - // Validate root-level selectors if (Array.isArray(obj['selectors'])) { for (const sel of obj['selectors']) { if (isObj(sel)) validateSelector(sel, issues); } } - // Validate elements recursively if (Array.isArray(obj['elements'])) { for (const el of obj['elements']) { if (isObj(el)) validateElement(el, issues); } } + // AJV schema validation runs additively alongside NX001–NX010 + const validator = schemaOverride === undefined ? getFactComponentValidator() : schemaOverride; + if (validator) { + validator(obj); + for (const err of validator.errors ?? []) { + issues.push(ajvErrorToIssue(err)); + } + } + const errorCount = issues.filter((i) => i.severity === 'ERROR').length; const warningCount = issues.filter((i) => i.severity === 'WARNING').length; const infoCount = issues.filter((i) => i.severity === 'INFO').length; @@ -636,7 +686,9 @@ export function registerNitroXValidate(server: McpServer, config: ServerConfig): description: [ 'Validate a NitroX .po.json (Hybrid Model component page object) against schema rules.', 'Works for any NitroX-mapped component type: LWC, Screen Flow, Industry Components, Experience Cloud, HTML5.', - 'Returns a quality score (0–100) and a list of issues with rule IDs (NX001–NX010), severity, and suggestions.', + 'Runs two passes in parallel: hardcoded semantic rules (NX001–NX010) and JSON schema validation (NX_SCHEMA_* rule IDs).', + 'Schema issues catch structural errors not covered by NX rules: wrong property types, extra properties, enum violations.', + 'Returns a quality score (0–100) and a combined list of issues with rule IDs, severity, and suggestions.', 'Score formula: 100 − (20 × errors) − (5 × warnings) − (1 × infos).', ].join(' '), inputSchema: { diff --git a/test/unit/mcp/nitroXTools.test.ts b/test/unit/mcp/nitroXTools.test.ts index 5ae0b22..0c43332 100644 --- a/test/unit/mcp/nitroXTools.test.ts +++ b/test/unit/mcp/nitroXTools.test.ts @@ -408,6 +408,84 @@ describe('nitroXTools', () => { }); }); + // ── NX_SCHEMA_ rules (AJV schema validation) ───────────────────────────────── + + describe('NX_SCHEMA_ rules (AJV schema override)', () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let validateFn: ( + obj: Record, + v?: any + ) => { + issues: Array<{ rule_id: string; severity: string; message: string; applies_to: string; field?: string }>; + valid: boolean; + score: number; + issue_count: number; + }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let extraPropsValidator: any; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let typeViolationValidator: any; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let permissiveValidator: any; + + before(async () => { + const mod = await import('../../../src/mcp/tools/nitroXTools.js'); + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-explicit-any + validateFn = mod.validateNitroXContent as any; + + const { Ajv2020: AjvClass } = await import('ajv/dist/2020.js'); + const ajv = new AjvClass({ allErrors: true, strict: false }); + + extraPropsValidator = ajv.compile({ + type: 'object', + additionalProperties: false, + properties: { componentId: { type: 'string' } }, + }); + + typeViolationValidator = ajv.compile({ + type: 'object', + properties: { pageStructureElement: { type: 'boolean' } }, + }); + + permissiveValidator = ajv.compile({ + type: 'object', + additionalProperties: false, + properties: { + componentId: { type: 'string' }, + name: { type: 'string' }, + type: { type: 'string' }, + pageStructureElement: { type: 'boolean' }, + fieldDetailsElement: { type: 'boolean' }, + }, + }); + }); + + it('NX_SCHEMA_ADDITIONAL_PROPERTIES: extra property surfaces as WARNING', () => { + // Schema only allows componentId; passing an extra field should produce a schema issue + const result = validateFn({ componentId: VALID_UUID, _extraProp: true }, extraPropsValidator); + assert.ok(result.issues.some((i) => i.rule_id === 'NX_SCHEMA_ADDITIONAL_PROPERTIES')); + assert.equal(result.issues.find((i) => i.rule_id === 'NX_SCHEMA_ADDITIONAL_PROPERTIES')?.severity, 'WARNING'); + }); + + it('NX_SCHEMA_TYPE: wrong property type surfaces as ERROR', () => { + // Schema expects pageStructureElement to be boolean; passing a string should produce a type error + const result = validateFn({ ...VALID_ROOT, pageStructureElement: 'yes' }, typeViolationValidator); + assert.ok(result.issues.some((i) => i.rule_id === 'NX_SCHEMA_TYPE' && i.severity === 'ERROR')); + }); + + it('valid object matching schema produces no NX_SCHEMA_ issues', () => { + const result = validateFn(VALID_ROOT, permissiveValidator); + assert.ok(!result.issues.some((i) => i.rule_id.startsWith('NX_SCHEMA_'))); + }); + + it('null schema override: hardcoded rules still run; valid object scores 100', () => { + const result = validateFn(VALID_ROOT, null); + assert.equal(result.valid, true); + assert.equal(result.score, 100); + assert.equal(result.issue_count, 0); + }); + }); + // ── provar_nitrox_generate ───────────────────────────────────────────────── describe('provar_nitrox_generate', () => { From 11e1dbdabfb083b847da49ec6ecce7714afdf7ca Mon Sep 17 00:00:00 2001 From: Michael Dailey Date: Mon, 11 May 2026 16:37:04 -0500 Subject: [PATCH 2/3] PDX-466: fix(test): replace no-explicit-any with typed ValidateFunction import in NX_SCHEMA tests RCA: ESLint no-explicit-any rule rejected the any parameter type used for the schemaOverride parameter in AJV schema override tests; the eslint-disable comment was positioned on the wrong line. Fix: Added import type { ValidateFunction } from ajv/dist/2020.js and replaced all any usages with properly typed ValidateFunction and a narrow IssueShape type alias for the return value. --- test/unit/mcp/nitroXTools.test.ts | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/test/unit/mcp/nitroXTools.test.ts b/test/unit/mcp/nitroXTools.test.ts index 0c43332..514f0bf 100644 --- a/test/unit/mcp/nitroXTools.test.ts +++ b/test/unit/mcp/nitroXTools.test.ts @@ -10,6 +10,7 @@ import { strict as assert } from 'node:assert'; import fs from 'node:fs'; import path from 'node:path'; import os from 'node:os'; +import type { ValidateFunction } from 'ajv/dist/2020.js'; import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; // ── Minimal mock server ─────────────────────────────────────────────────────── @@ -411,27 +412,21 @@ describe('nitroXTools', () => { // ── NX_SCHEMA_ rules (AJV schema validation) ───────────────────────────────── describe('NX_SCHEMA_ rules (AJV schema override)', () => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - let validateFn: ( + type IssueShape = { rule_id: string; severity: string; message: string; applies_to: string; field?: string }; + type ValidateFnType = ( obj: Record, - v?: any - ) => { - issues: Array<{ rule_id: string; severity: string; message: string; applies_to: string; field?: string }>; - valid: boolean; - score: number; - issue_count: number; - }; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - let extraPropsValidator: any; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - let typeViolationValidator: any; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - let permissiveValidator: any; + v?: ValidateFunction | null + ) => { issues: IssueShape[]; valid: boolean; score: number; issue_count: number }; + + let validateFn!: ValidateFnType; + let extraPropsValidator!: ValidateFunction; + let typeViolationValidator!: ValidateFunction; + let permissiveValidator!: ValidateFunction; before(async () => { const mod = await import('../../../src/mcp/tools/nitroXTools.js'); - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-explicit-any - validateFn = mod.validateNitroXContent as any; + // Cast through unknown: the private NitroXValidationResult is structurally compatible with IssueShape[] + validateFn = mod.validateNitroXContent as unknown as ValidateFnType; const { Ajv2020: AjvClass } = await import('ajv/dist/2020.js'); const ajv = new AjvClass({ allErrors: true, strict: false }); From 8b60e3d353ea1de7473ac81dab5aa727d492092a Mon Sep 17 00:00:00 2001 From: Michael Dailey Date: Tue, 12 May 2026 07:12:11 -0500 Subject: [PATCH 3/3] PDX-466: fix(mcp): address Copilot review comments on nitrox-ajv-schema-validation RCA: Copilot flagged incorrect 'in parallel' wording (validation is synchronous/sequential), an overly broad ERROR severity mapping in ajvErrorToIssue (MIN_ITEMS/MINIMUM/MAXIMUM should be WARNING), and broken markdown rendering of NX_SCHEMA_* in docs (underscores parsed as italic markers). Fix: Reworded tool description and docs to 'sequential' passes; narrowed ERROR set to REQUIRED and TYPE only; fixed NX_SCHEMA_* heading and inline text with backtick quoting; updated docs table to show MIN_ITEMS as WARNING. --- docs/mcp.md | 8 ++++---- src/mcp/tools/nitroXTools.ts | 6 ++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/docs/mcp.md b/docs/mcp.md index bcec07e..577f225 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -1643,10 +1643,10 @@ Path policy is enforced per-file. A missing or unparseable file returns an `erro ### `provar_nitrox_validate` -Validate a NitroX `.po.json` (Hybrid Model component page object) against the FACT schema rules. Returns a quality score (0–100) and a combined list of issues from two validation passes that run in parallel: +Validate a NitroX `.po.json` (Hybrid Model component page object) against the FACT schema rules. Returns a quality score (0–100) and a combined list of issues from two sequential validation passes: 1. **Hardcoded semantic rules (NX001–NX010)** — always run -2. **JSON schema validation (NX*SCHEMA*\*)** — runs when the bundled `FactComponent.schema.json` is available; falls back to hardcoded-rules-only if the schema cannot be loaded +2. **JSON schema validation (`NX_SCHEMA_*`)** — runs when the bundled `FactComponent.schema.json` is available; falls back to hardcoded-rules-only if the schema cannot be loaded Score formula: `100 − (20 × errors) − (5 × warnings) − (1 × infos)`, minimum 0. @@ -1678,7 +1678,7 @@ Score formula: `100 − (20 × errors) − (5 × warnings) − (1 × infos)`, mi | NX009 | INFO | Interaction `name` contains characters outside `[A-Za-z0-9 ]` | | NX010 | INFO | `bodyTagName` contains whitespace | -**JSON schema rules (NX*SCHEMA*\*):** +**JSON schema rules (`NX_SCHEMA_*`):** Rule IDs follow the pattern `NX_SCHEMA_` where `` is the AJV validation keyword in `SCREAMING_SNAKE_CASE`. Common rule IDs: @@ -1686,7 +1686,7 @@ Rule IDs follow the pattern `NX_SCHEMA_` where `` is the AJV v | --------------------------------- | -------- | ------------------------------------------------------------------------------ | | `NX_SCHEMA_TYPE` | ERROR | Property has the wrong JSON type (e.g. string where boolean expected) | | `NX_SCHEMA_REQUIRED` | ERROR | Required property missing (per JSON schema `required` array) | -| `NX_SCHEMA_MIN_ITEMS` | ERROR | Array has fewer items than `minItems` requires | +| `NX_SCHEMA_MIN_ITEMS` | WARNING | Array has fewer items than `minItems` requires | | `NX_SCHEMA_ADDITIONAL_PROPERTIES` | WARNING | Property not defined in the schema (schema uses `additionalProperties: false`) | | `NX_SCHEMA_PATTERN` | WARNING | String value does not match the schema `pattern` | | `NX_SCHEMA_ENUM` | WARNING | Value not in the allowed `enum` list | diff --git a/src/mcp/tools/nitroXTools.ts b/src/mcp/tools/nitroXTools.ts index d94986a..febddb2 100644 --- a/src/mcp/tools/nitroXTools.ts +++ b/src/mcp/tools/nitroXTools.ts @@ -68,9 +68,7 @@ function ajvErrorToIssue(err: ErrorObject): NitroXIssue { const instancePath = err.instancePath; const appliesTo = instancePath ? instancePath.replace(/^\//, '').replace(/\//g, '.') : 'root'; const pathParts = instancePath.split('/').filter(Boolean); - const severity: 'ERROR' | 'WARNING' = ['REQUIRED', 'TYPE', 'MIN_ITEMS', 'MINIMUM', 'MAXIMUM'].includes(keyword) - ? 'ERROR' - : 'WARNING'; + const severity: 'ERROR' | 'WARNING' = ['REQUIRED', 'TYPE'].includes(keyword) ? 'ERROR' : 'WARNING'; const issue: NitroXIssue = { rule_id: `NX_SCHEMA_${keyword}`, severity, @@ -686,7 +684,7 @@ export function registerNitroXValidate(server: McpServer, config: ServerConfig): description: [ 'Validate a NitroX .po.json (Hybrid Model component page object) against schema rules.', 'Works for any NitroX-mapped component type: LWC, Screen Flow, Industry Components, Experience Cloud, HTML5.', - 'Runs two passes in parallel: hardcoded semantic rules (NX001–NX010) and JSON schema validation (NX_SCHEMA_* rule IDs).', + 'Runs two validation passes sequentially: hardcoded semantic rules (NX001–NX010) then JSON schema validation (NX_SCHEMA_* rule IDs).', 'Schema issues catch structural errors not covered by NX rules: wrong property types, extra properties, enum violations.', 'Returns a quality score (0–100) and a combined list of issues with rule IDs, severity, and suggestions.', 'Score formula: 100 − (20 × errors) − (5 × warnings) − (1 × infos).',