diff --git a/docs/mcp.md b/docs/mcp.md index 82627ac..577f225 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -1643,7 +1643,10 @@ Path policy is enforced per-file. A missing or unparseable file returns an `erro ### `provar_nitrox_validate` -Validate a NitroX `.po.json` (Hybrid Model component page object) against the FACT schema rules. Returns a quality score (0–100) and a list of issues. +Validate a NitroX `.po.json` (Hybrid Model component page object) against the FACT schema rules. Returns a quality score (0–100) and a combined list of issues from two sequential validation passes: + +1. **Hardcoded semantic rules (NX001–NX010)** — always run +2. **JSON schema validation (`NX_SCHEMA_*`)** — runs when the bundled `FactComponent.schema.json` is available; falls back to hardcoded-rules-only if the schema cannot be loaded Score formula: `100 − (20 × errors) − (5 × warnings) − (1 × infos)`, minimum 0. @@ -1659,7 +1662,7 @@ Score formula: `100 − (20 × errors) − (5 × warnings) − (1 × infos)`, mi | `issue_count` | Total issues | | `issues` | Array of `ValidationIssue` (see below) | -**Validation rules:** +**Hardcoded rules:** | Rule | Severity | Description | | ----- | -------- | -------------------------------------------------------------------------------------------------------------------------------------------- | @@ -1675,6 +1678,21 @@ Score formula: `100 − (20 × errors) − (5 × warnings) − (1 × infos)`, mi | NX009 | INFO | Interaction `name` contains characters outside `[A-Za-z0-9 ]` | | NX010 | INFO | `bodyTagName` contains whitespace | +**JSON schema rules (`NX_SCHEMA_*`):** + +Rule IDs follow the pattern `NX_SCHEMA_` where `` is the AJV validation keyword in `SCREAMING_SNAKE_CASE`. Common rule IDs: + +| Rule ID | Severity | Description | +| --------------------------------- | -------- | ------------------------------------------------------------------------------ | +| `NX_SCHEMA_TYPE` | ERROR | Property has the wrong JSON type (e.g. string where boolean expected) | +| `NX_SCHEMA_REQUIRED` | ERROR | Required property missing (per JSON schema `required` array) | +| `NX_SCHEMA_MIN_ITEMS` | WARNING | Array has fewer items than `minItems` requires | +| `NX_SCHEMA_ADDITIONAL_PROPERTIES` | WARNING | Property not defined in the schema (schema uses `additionalProperties: false`) | +| `NX_SCHEMA_PATTERN` | WARNING | String value does not match the schema `pattern` | +| `NX_SCHEMA_ENUM` | WARNING | Value not in the allowed `enum` list | + +Schema issues complement — and may overlap with — the hardcoded NX rules. When overlap occurs, both rule IDs appear in the `issues` array. + **Error codes:** `MISSING_INPUT`, `NX000`, `FILE_NOT_FOUND`, `PATH_NOT_ALLOWED` --- diff --git a/package.json b/package.json index f1f95d1..90c374d 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@provartesting/provardx-cli", "description": "A plugin for the Salesforce CLI to orchestrate testing activities and report quality metrics to Provar Quality Hub", - "version": "1.5.0-beta.18", + "version": "1.5.0-beta.19", "mcpName": "io.github.ProvarTesting/provar", "license": "BSD-3-Clause", "plugins": [ @@ -11,6 +11,7 @@ "dependencies": { "@modelcontextprotocol/sdk": "^1.8.0", "@oclif/core": "^3.27.0", + "ajv": "^8.17.1", "@provartesting/provardx-plugins-automation": "1.2.2", "@provartesting/provardx-plugins-manager": "1.3.2", "@provartesting/provardx-plugins-utils": "1.3.3", diff --git a/server.json b/server.json index 76faa4b..f7d1f22 100644 --- a/server.json +++ b/server.json @@ -14,12 +14,12 @@ "url": "https://github.com/ProvarTesting/provardx-cli", "source": "github" }, - "version": "1.5.0-beta.18", + "version": "1.5.0-beta.19", "packages": [ { "registryType": "npm", "identifier": "@provartesting/provardx-cli", - "version": "1.5.0-beta.18", + "version": "1.5.0-beta.19", "transport": { "type": "stdio" }, diff --git a/src/mcp/tools/nitroXTools.ts b/src/mcp/tools/nitroXTools.ts index 1a8821b..febddb2 100644 --- a/src/mcp/tools/nitroXTools.ts +++ b/src/mcp/tools/nitroXTools.ts @@ -10,6 +10,8 @@ import fs from 'node:fs'; import path from 'node:path'; import os from 'node:os'; import { randomUUID } from 'node:crypto'; +import { fileURLToPath } from 'node:url'; +import { Ajv2020, type ValidateFunction, type ErrorObject } from 'ajv/dist/2020.js'; import { z } from 'zod'; import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import type { ServerConfig } from '../server.js'; @@ -36,6 +38,47 @@ function isObj(v: unknown): v is JsonObj { return typeof v === 'object' && v !== null && !Array.isArray(v); } +// ── AJV Schema Validator ────────────────────────────────────────────────────── + +const RULES_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'rules'); + +let cachedFactComponentValidator: ValidateFunction | null | undefined; + +function getFactComponentValidator(): ValidateFunction | null { + if (cachedFactComponentValidator !== undefined) return cachedFactComponentValidator; + + const schemaPath = path.join(RULES_DIR, 'FactComponent.schema.json'); + try { + // Fix known broken $ref in the bundled schema (#/defs/ → #/$defs/) + const patched = fs.readFileSync(schemaPath, 'utf-8').replace(/"#\/defs\//g, '"#/$defs/'); + const schema = JSON.parse(patched) as Record; + const ajv = new Ajv2020({ allErrors: true, strict: false, validateFormats: false }); + cachedFactComponentValidator = ajv.compile(schema); + } catch (e) { + log('warn', 'provar_nitrox_validate: FactComponent schema unavailable, using hardcoded rules only', { + error: String(e), + }); + cachedFactComponentValidator = null; + } + return cachedFactComponentValidator; +} + +function ajvErrorToIssue(err: ErrorObject): NitroXIssue { + const keyword = err.keyword.replace(/([a-z])([A-Z])/g, '$1_$2').toUpperCase(); + const instancePath = err.instancePath; + const appliesTo = instancePath ? instancePath.replace(/^\//, '').replace(/\//g, '.') : 'root'; + const pathParts = instancePath.split('/').filter(Boolean); + const severity: 'ERROR' | 'WARNING' = ['REQUIRED', 'TYPE'].includes(keyword) ? 'ERROR' : 'WARNING'; + const issue: NitroXIssue = { + rule_id: `NX_SCHEMA_${keyword}`, + severity, + message: `Schema: ${instancePath || 'root'} — ${err.message ?? 'validation failed'}`, + applies_to: appliesTo, + }; + if (pathParts.length > 0) issue.field = pathParts[pathParts.length - 1]; + return issue; +} + // ── Directory Utilities ─────────────────────────────────────────────────────── const SKIP_DIRS = new Set(['node_modules', '.git']); @@ -168,40 +211,45 @@ function validateRootProperties(obj: JsonObj, issues: NitroXIssue[]): void { } } -/** Validate a parsed NitroX .po.json object against schema-derived rules. */ -export function validateNitroXContent(obj: JsonObj): NitroXValidationResult { +/** Validate a parsed NitroX .po.json against hardcoded NX rules and the FactComponent JSON schema. */ +export function validateNitroXContent(obj: JsonObj, schemaOverride?: ValidateFunction | null): NitroXValidationResult { const issues: NitroXIssue[] = []; validateRootProperties(obj, issues); - // Validate root-level parameters if (Array.isArray(obj['parameters'])) { for (const param of obj['parameters']) { if (isObj(param)) validateParameter(param, 'root', issues); } } - // Validate root-level interactions if (Array.isArray(obj['interactions'])) { for (const interaction of obj['interactions']) { if (isObj(interaction)) validateInteraction(interaction, 'root', issues); } } - // Validate root-level selectors if (Array.isArray(obj['selectors'])) { for (const sel of obj['selectors']) { if (isObj(sel)) validateSelector(sel, issues); } } - // Validate elements recursively if (Array.isArray(obj['elements'])) { for (const el of obj['elements']) { if (isObj(el)) validateElement(el, issues); } } + // AJV schema validation runs additively alongside NX001–NX010 + const validator = schemaOverride === undefined ? getFactComponentValidator() : schemaOverride; + if (validator) { + validator(obj); + for (const err of validator.errors ?? []) { + issues.push(ajvErrorToIssue(err)); + } + } + const errorCount = issues.filter((i) => i.severity === 'ERROR').length; const warningCount = issues.filter((i) => i.severity === 'WARNING').length; const infoCount = issues.filter((i) => i.severity === 'INFO').length; @@ -636,7 +684,9 @@ export function registerNitroXValidate(server: McpServer, config: ServerConfig): description: [ 'Validate a NitroX .po.json (Hybrid Model component page object) against schema rules.', 'Works for any NitroX-mapped component type: LWC, Screen Flow, Industry Components, Experience Cloud, HTML5.', - 'Returns a quality score (0–100) and a list of issues with rule IDs (NX001–NX010), severity, and suggestions.', + 'Runs two validation passes sequentially: hardcoded semantic rules (NX001–NX010) then JSON schema validation (NX_SCHEMA_* rule IDs).', + 'Schema issues catch structural errors not covered by NX rules: wrong property types, extra properties, enum violations.', + 'Returns a quality score (0–100) and a combined list of issues with rule IDs, severity, and suggestions.', 'Score formula: 100 − (20 × errors) − (5 × warnings) − (1 × infos).', ].join(' '), inputSchema: { diff --git a/test/unit/mcp/nitroXTools.test.ts b/test/unit/mcp/nitroXTools.test.ts index 5ae0b22..514f0bf 100644 --- a/test/unit/mcp/nitroXTools.test.ts +++ b/test/unit/mcp/nitroXTools.test.ts @@ -10,6 +10,7 @@ import { strict as assert } from 'node:assert'; import fs from 'node:fs'; import path from 'node:path'; import os from 'node:os'; +import type { ValidateFunction } from 'ajv/dist/2020.js'; import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; // ── Minimal mock server ─────────────────────────────────────────────────────── @@ -408,6 +409,78 @@ describe('nitroXTools', () => { }); }); + // ── NX_SCHEMA_ rules (AJV schema validation) ───────────────────────────────── + + describe('NX_SCHEMA_ rules (AJV schema override)', () => { + type IssueShape = { rule_id: string; severity: string; message: string; applies_to: string; field?: string }; + type ValidateFnType = ( + obj: Record, + v?: ValidateFunction | null + ) => { issues: IssueShape[]; valid: boolean; score: number; issue_count: number }; + + let validateFn!: ValidateFnType; + let extraPropsValidator!: ValidateFunction; + let typeViolationValidator!: ValidateFunction; + let permissiveValidator!: ValidateFunction; + + before(async () => { + const mod = await import('../../../src/mcp/tools/nitroXTools.js'); + // Cast through unknown: the private NitroXValidationResult is structurally compatible with IssueShape[] + validateFn = mod.validateNitroXContent as unknown as ValidateFnType; + + const { Ajv2020: AjvClass } = await import('ajv/dist/2020.js'); + const ajv = new AjvClass({ allErrors: true, strict: false }); + + extraPropsValidator = ajv.compile({ + type: 'object', + additionalProperties: false, + properties: { componentId: { type: 'string' } }, + }); + + typeViolationValidator = ajv.compile({ + type: 'object', + properties: { pageStructureElement: { type: 'boolean' } }, + }); + + permissiveValidator = ajv.compile({ + type: 'object', + additionalProperties: false, + properties: { + componentId: { type: 'string' }, + name: { type: 'string' }, + type: { type: 'string' }, + pageStructureElement: { type: 'boolean' }, + fieldDetailsElement: { type: 'boolean' }, + }, + }); + }); + + it('NX_SCHEMA_ADDITIONAL_PROPERTIES: extra property surfaces as WARNING', () => { + // Schema only allows componentId; passing an extra field should produce a schema issue + const result = validateFn({ componentId: VALID_UUID, _extraProp: true }, extraPropsValidator); + assert.ok(result.issues.some((i) => i.rule_id === 'NX_SCHEMA_ADDITIONAL_PROPERTIES')); + assert.equal(result.issues.find((i) => i.rule_id === 'NX_SCHEMA_ADDITIONAL_PROPERTIES')?.severity, 'WARNING'); + }); + + it('NX_SCHEMA_TYPE: wrong property type surfaces as ERROR', () => { + // Schema expects pageStructureElement to be boolean; passing a string should produce a type error + const result = validateFn({ ...VALID_ROOT, pageStructureElement: 'yes' }, typeViolationValidator); + assert.ok(result.issues.some((i) => i.rule_id === 'NX_SCHEMA_TYPE' && i.severity === 'ERROR')); + }); + + it('valid object matching schema produces no NX_SCHEMA_ issues', () => { + const result = validateFn(VALID_ROOT, permissiveValidator); + assert.ok(!result.issues.some((i) => i.rule_id.startsWith('NX_SCHEMA_'))); + }); + + it('null schema override: hardcoded rules still run; valid object scores 100', () => { + const result = validateFn(VALID_ROOT, null); + assert.equal(result.valid, true); + assert.equal(result.score, 100); + assert.equal(result.issue_count, 0); + }); + }); + // ── provar_nitrox_generate ───────────────────────────────────────────────── describe('provar_nitrox_generate', () => {