diff --git a/package-lock.json b/package-lock.json index b357c14..4846868 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "apiops-cycles-method-data", - "version": "3.4.1", + "version": "3.4.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "apiops-cycles-method-data", - "version": "3.4.1", + "version": "3.4.2", "license": "Apache-2.0", "workspaces": [ "packages/*" diff --git a/package.json b/package.json index 841fd16..4c07c35 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "apiops-cycles-method-data", - "version": "3.4.1", + "version": "3.4.2", "description": "APIOps Cycles Method data and canvases", "license": "Apache-2.0", "type": "module", @@ -35,7 +35,7 @@ "release:create-apiops:publish": "npm publish --workspace packages/create-apiops --access public", "check:packaging:skills": "node scripts/check-packaging-skills.mjs", "check:package:contents": "node scripts/check-package-contents.mjs", - "test:create-apiops": "node scripts/test-create-apiops-scaffold.mjs && node scripts/test-method-cli.mjs" + "test:create-apiops": "node scripts/test-create-apiops-scaffold.mjs && node scripts/test-method-cli.mjs && node scripts/test-run-design-audit.mjs" }, "devDependencies": { "@changesets/cli": "^2.29.7", diff --git a/packages/create-apiops/package.json b/packages/create-apiops/package.json index fc8755a..9dd365a 100644 --- a/packages/create-apiops/package.json +++ b/packages/create-apiops/package.json @@ -1,6 +1,6 @@ { "name": "create-apiops", - "version": "1.1.1", + "version": "1.1.2", "description": "Scaffold a new APIOps project with an OpenAPI spec, audit scaffolding, and APIOps documentation templates.", "license": "Apache-2.0", "type": "module", diff --git a/packages/create-apiops/template/package.json b/packages/create-apiops/template/package.json index e7f2527..7898c3c 100644 --- a/packages/create-apiops/template/package.json +++ b/packages/create-apiops/template/package.json @@ -6,7 +6,7 @@ "node": ">=22" }, "dependencies": { - "apiops-cycles-method-data": "^3.4.1", + "apiops-cycles-method-data": "^3.4.2", "canvascreator": "^1.7.3" }, "devDependencies": { diff --git a/packages/create-apiops/template/scripts/run-design-audit.js b/packages/create-apiops/template/scripts/run-design-audit.js index 07988a0..7c33913 100644 --- a/packages/create-apiops/template/scripts/run-design-audit.js +++ b/packages/create-apiops/template/scripts/run-design-audit.js @@ -2,28 +2,40 @@ import fs from "node:fs"; import { createRequire } from "node:module"; import path from "node:path"; import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; import YAML from "yaml"; -const root = process.cwd(); const require = createRequire(import.meta.url); -const profile = (process.argv[2] || "read-only").trim(); - -if (!["read-only", "full-crud"].includes(profile)) { - console.error(`Unknown audit profile: ${profile}`); - process.exit(1); +const __filename = fileURLToPath(import.meta.url); + +let root = process.cwd(); +let profile = "read-only"; +let auditSlug = profile; +let openApiPath = path.join(root, "specs", "openapi", "api.yaml"); +let reportJsonPath = path.join(root, "specs", "audit", `design-audit.${auditSlug}.json`); +let reportMarkdownPath = path.join(root, "specs", "audit", `design-audit.${auditSlug}.md`); +let reportDocsPath = path.join(root, "docs", "api", "audit", `design-audit.${auditSlug}.md`); +let reportHtmlPath = path.join(root, "docs", "api", "audit", `design-audit.${auditSlug}.html`); +let reportJUnitPath = path.join(root, "reports", "junit", `design-audit.${auditSlug}.xml`); +let legacyMarkdownPath = path.join(root, "specs", "audit", "design-audit.md"); +let legacyDocsPath = path.join(root, "docs", "api", "audit", "design-audit.md"); +let legacyHtmlPath = path.join(root, "docs", "api", "audit", "index.html"); + +function configureRun(rootDir, selectedProfile) { + root = rootDir; + profile = selectedProfile; + auditSlug = profile; + openApiPath = path.join(root, "specs", "openapi", "api.yaml"); + reportJsonPath = path.join(root, "specs", "audit", `design-audit.${auditSlug}.json`); + reportMarkdownPath = path.join(root, "specs", "audit", `design-audit.${auditSlug}.md`); + reportDocsPath = path.join(root, "docs", "api", "audit", `design-audit.${auditSlug}.md`); + reportHtmlPath = path.join(root, "docs", "api", "audit", `design-audit.${auditSlug}.html`); + reportJUnitPath = path.join(root, "reports", "junit", `design-audit.${auditSlug}.xml`); + legacyMarkdownPath = profile === "read-only" ? path.join(root, "specs", "audit", "design-audit.md") : null; + legacyDocsPath = profile === "read-only" ? path.join(root, "docs", "api", "audit", "design-audit.md") : null; + legacyHtmlPath = profile === "read-only" ? path.join(root, "docs", "api", "audit", "index.html") : null; } -const auditSlug = profile; -const openApiPath = path.join(root, "specs", "openapi", "api.yaml"); -const reportJsonPath = path.join(root, "specs", "audit", `design-audit.${auditSlug}.json`); -const reportMarkdownPath = path.join(root, "specs", "audit", `design-audit.${auditSlug}.md`); -const reportDocsPath = path.join(root, "docs", "api", "audit", `design-audit.${auditSlug}.md`); -const reportHtmlPath = path.join(root, "docs", "api", "audit", `design-audit.${auditSlug}.html`); -const reportJUnitPath = path.join(root, "reports", "junit", `design-audit.${auditSlug}.xml`); -const legacyMarkdownPath = profile === "read-only" ? path.join(root, "specs", "audit", "design-audit.md") : null; -const legacyDocsPath = profile === "read-only" ? path.join(root, "docs", "api", "audit", "design-audit.md") : null; -const legacyHtmlPath = profile === "read-only" ? path.join(root, "docs", "api", "audit", "index.html") : null; - function resolveChecklistSource() { const localOverridePath = path.join(root, "specs", "audit", "api-audit-checklist.json"); if (fs.existsSync(localOverridePath)) { @@ -315,10 +327,11 @@ function evaluateCheck(check, spec, item) { return checkStandardizedValues(spec); case "avoidAcronyms": return checkAvoidAcronyms(spec); - case "sectionCoverage": { + case "sectionCoverage": + case "stageCoverage": { return { ok: true, - details: [check.sectionId] + details: [check.stageId || check.sectionId] }; } case "pathDepthMax": @@ -346,6 +359,15 @@ function evaluateCheck(check, spec, item) { } } +const DEFAULT_LIFECYCLE_STAGES = Object.freeze([ + { id: "strategy", title: "Strategy", order: 1, readinessLabel: "Concept is Ready When..." }, + { id: "architecture", title: "Architecture", order: 2, readinessLabel: "Architecture is Ready When..." }, + { id: "design", title: "Design", order: 3, readinessLabel: "Design Prototype is Ready When..." }, + { id: "delivery", title: "Delivery", order: 4, readinessLabel: "Delivery is Ready When..." }, + { id: "publishing", title: "Publishing", order: 5, readinessLabel: "Production Ready for Publishing When..." }, + { id: "improving", title: "Improving", order: 6, readinessLabel: "Improvement Loops are Ready When..." } +]); + function normalizeStatus(item, evaluation) { if (item.defaultStatus === "na") return "na"; if (item.kind === "manual") return item.defaultStatus || "partial"; @@ -367,9 +389,82 @@ function formatStatus(status) { } } +function summarizeStatuses(items) { + const summary = { pass: 0, partial: 0, gap: 0, na: 0 }; + for (const item of items) { + summary[item.currentStatus] += 1; + } + summary.total = summary.pass + summary.partial + summary.gap + summary.na; + return summary; +} + +function formatStageSummary(summary) { + const parts = [`${summary.pass} pass`, `${summary.partial} partial`, `${summary.gap} gap`]; + if (summary.na > 0) { + parts.push(`${summary.na} n/a`); + } + return parts.join(" | "); +} + +function normalizeChecklistItem(item, primaryStage) { + return { + ...item, + primaryStage: item.primaryStage || primaryStage, + producedByStation: item.producedByStation || item.stationSource || [], + producedByStationCriteria: item.producedByStationCriteria || [], + guidelines: item.guidelines || item.guidelineRef || [], + expectedEvidenceTags: item.expectedEvidenceTags || item.evidenceType || [], + expectedEvidence: item.expectedEvidence || item.evidence || [] + }; +} + +function normalizeChecklist(checklist) { + const lifecycleStages = (Array.isArray(checklist.lifecycleStages) && checklist.lifecycleStages.length + ? checklist.lifecycleStages + : DEFAULT_LIFECYCLE_STAGES + ) + .slice() + .sort((left, right) => (left.order || 0) - (right.order || 0)); + + if (Array.isArray(checklist.stages)) { + const itemsByStage = new Map(checklist.stages.map((stage) => [stage.id, stage.items || []])); + return lifecycleStages.map((stage) => ({ + ...stage, + items: (itemsByStage.get(stage.id) || []).map((item) => normalizeChecklistItem(item, stage.id)) + })); + } + + const legacyStageBySectionId = { + "concept-ready": "strategy", + "design-prototype-ready": "design", + "production-ready": "publishing" + }; + const itemsByStage = new Map(lifecycleStages.map((stage) => [stage.id, []])); + + for (const section of checklist.sections || []) { + const stageId = legacyStageBySectionId[section.id] || "design"; + const items = itemsByStage.get(stageId) || []; + for (const item of section.items || []) { + items.push(normalizeChecklistItem(item, stageId)); + } + itemsByStage.set(stageId, items); + } + + return lifecycleStages.map((stage) => ({ + ...stage, + items: itemsByStage.get(stage.id) || [] + })); +} + +function findExistingEvidence(paths) { + return (paths || []) + .filter(Boolean) + .filter((entry) => fs.existsSync(path.join(root, entry))); +} + function buildChecklistResults(spec, checklist) { - const sections = checklist.sections.map((section) => { - const items = section.items + return normalizeChecklist(checklist).map((stage) => { + const items = stage.items .filter((item) => item.applicableTo.includes(profile)) .map((item) => { if (item.defaultStatus === "na") { @@ -377,47 +472,56 @@ function buildChecklistResults(spec, checklist) { id: item.id, label: item.label, kind: item.kind, - status: "na", - reason: item.reason || "Not applicable", - evidence: item.evidence || [] + primaryStage: item.primaryStage || stage.id, + producedByStation: item.producedByStation, + producedByStationCriteria: item.producedByStationCriteria, + guidelines: item.guidelines, + expectedEvidenceTags: item.expectedEvidenceTags, + expectedEvidence: item.expectedEvidence, + actualEvidenceFound: findExistingEvidence(item.expectedEvidence), + currentStatus: "na", + reason: item.reason || "Not applicable" }; } const evaluation = item.kind === "openapi" || item.kind === "aggregate" ? evaluateCheck(item.check, spec, item) : { ok: false, details: [] }; + const currentStatus = normalizeStatus(item, evaluation); + const actualEvidenceFound = evaluation.details?.length + ? evaluation.details + : ( + findExistingEvidence(item.expectedEvidence).length + ? findExistingEvidence(item.expectedEvidence) + : (item.kind === "openapi" ? [path.relative(root, openApiPath)] : []) + ); - const status = normalizeStatus(item, evaluation); return { id: item.id, label: item.label, kind: item.kind, - status, - evidence: evaluation.details?.length ? evaluation.details : (item.evidence || []), + primaryStage: item.primaryStage || stage.id, + producedByStation: item.producedByStation, + producedByStationCriteria: item.producedByStationCriteria, + guidelines: item.guidelines, + expectedEvidenceTags: item.expectedEvidenceTags, + expectedEvidence: item.expectedEvidence, + actualEvidenceFound, + currentStatus, reason: item.reason || "", evaluation: item.kind === "openapi" || item.kind === "aggregate" ? evaluation.ok : undefined }; }); return { - id: section.id, - title: section.title, + id: stage.id, + title: stage.title, + readinessLabel: stage.readinessLabel || "", + order: stage.order || 0, + summary: summarizeStatuses(items), items }; }); - - return sections; -} - -function summarizeChecklist(sections) { - const summary = { pass: 0, partial: 0, gap: 0, na: 0 }; - for (const section of sections) { - for (const item of section.items) { - summary[item.status] += 1; - } - } - summary.total = summary.pass + summary.partial + summary.gap + summary.na; - return summary; } function buildReport() { @@ -425,8 +529,8 @@ function buildReport() { const checklist = readJson(checklistSource.filePath); const spec = YAML.parse(readText(openApiPath)); const spectral = runSpectral(); - const sections = buildChecklistResults(spec, checklist); - const summary = summarizeChecklist(sections); + const stages = buildChecklistResults(spec, checklist); + const summary = summarizeStatuses(stages.flatMap((stage) => stage.items)); return { profile, @@ -442,7 +546,12 @@ function buildReport() { findings: spectral.findings }, summary, - sections + stageSummary: stages.map((stage) => ({ + id: stage.id, + title: stage.title, + ...stage.summary + })), + stages }; } @@ -461,16 +570,36 @@ function renderMarkdown(report) { lines.push(`- Gaps: ${report.summary.gap}`); lines.push(`- Not applicable: ${report.summary.na}`); lines.push(""); - lines.push("## Checklist Results"); + lines.push("## Lifecycle Summary"); + lines.push(""); + for (const stage of report.stageSummary) { + lines.push(`- ${stage.title}: ${formatStageSummary(stage)}`); + } + lines.push(""); + lines.push("## Stage Results"); lines.push(""); - for (const section of report.sections) { - lines.push(`### ${section.title}`); + for (const stage of report.stages) { + lines.push(`### ${stage.title}`); lines.push(""); - for (const item of section.items) { - lines.push(`- [${formatStatus(item.status)}] ${item.label}`); - if (item.evidence && item.evidence.length) { - lines.push(` - Evidence: ${item.evidence.slice(0, 5).join(", ")}`); + if (stage.readinessLabel) { + lines.push(stage.readinessLabel); + lines.push(""); + } + lines.push(`Summary: ${formatStageSummary(stage.summary)}`); + lines.push(""); + for (const item of stage.items) { + lines.push(`- [${formatStatus(item.currentStatus)}] ${item.label}`); + if (item.producedByStation.length) { + lines.push(` - Stations: ${item.producedByStation.join(", ")}`); + } + if (item.guidelines.length) { + lines.push(` - Guidelines: ${item.guidelines.join(", ")}`); + } + if (item.actualEvidenceFound.length) { + lines.push(` - Actual evidence found: ${item.actualEvidenceFound.slice(0, 5).join(", ")}`); + } else if (item.expectedEvidence.length) { + lines.push(` - Expected evidence: ${item.expectedEvidence.slice(0, 5).join(", ")}`); } if (item.reason) { lines.push(` - Reason: ${item.reason}`); @@ -479,11 +608,6 @@ function renderMarkdown(report) { lines.push(""); } - lines.push("## Summary"); - lines.push(""); - lines.push("The current contract and canvases support a storefront product search API well enough for an initial design review."); - lines.push("The remaining audit gaps are mostly operational: gateway policy, publishing integration, and production controls."); - lines.push(""); return `${lines.join("\n")}\n`; } @@ -492,17 +616,31 @@ function statusClass(status) { } function renderHtml(report) { - const itemsHtml = report.sections - .map((section) => { - const rows = section.items.map((item) => { - const evidence = item.evidence && item.evidence.length ? `
${xmlEscape(item.evidence.slice(0, 5).join(", "))}
` : ""; - const reason = item.reason ? `
${xmlEscape(item.reason)}
` : ""; + const stageSummaryHtml = report.stageSummary + .map((stage) => ` +
+
${xmlEscape(stage.title)}
+
${xmlEscape(formatStageSummary(stage))}
+
`) + .join("\n"); + + const itemsHtml = report.stages + .map((stage) => { + const rows = stage.items.map((item) => { + const stations = item.producedByStation.length ? `
Stations: ${xmlEscape(item.producedByStation.join(", "))}
` : ""; + const guidelines = item.guidelines.length ? `
Guidelines: ${xmlEscape(item.guidelines.join(", "))}
` : ""; + const evidence = item.actualEvidenceFound.length + ? `
Actual evidence: ${xmlEscape(item.actualEvidenceFound.slice(0, 5).join(", "))}
` + : (item.expectedEvidence.length ? `
Expected evidence: ${xmlEscape(item.expectedEvidence.slice(0, 5).join(", "))}
` : ""); + const reason = item.reason ? `
Reason: ${xmlEscape(item.reason)}
` : ""; return ` -
  • +
  • - ${xmlEscape(formatStatus(item.status))} + ${xmlEscape(formatStatus(item.currentStatus))} ${xmlEscape(item.label)}
    + ${stations} + ${guidelines} ${evidence} ${reason}
  • `; @@ -510,7 +648,13 @@ function renderHtml(report) { return `
    -

    ${xmlEscape(section.title)}

    +
    +
    +

    ${xmlEscape(stage.title)}

    +

    ${xmlEscape(stage.readinessLabel || "")}

    +
    +
    ${xmlEscape(formatStageSummary(stage.summary))}
    +
    `; }) @@ -533,7 +677,6 @@ function renderHtml(report) { --partial: #fff0b5; --gap: #ffcdcd; --na: #e8e4d2; - --accent: #7dc9e7; } body { margin: 0; @@ -542,25 +685,37 @@ function renderHtml(report) { color: var(--text); } .wrap { - max-width: 1100px; + max-width: 1180px; margin: 0 auto; padding: 32px 20px 56px; } - .hero { + .hero, .section { background: var(--card); border: 1px solid var(--border); border-radius: 18px; - padding: 24px; box-shadow: 0 12px 40px rgba(31, 41, 55, 0.08); + } + .hero { + padding: 24px; margin-bottom: 24px; } - .meta { + .meta, .coverage, .stage-grid { display: grid; - grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); gap: 12px; + } + .meta { + grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); + margin-top: 16px; + } + .coverage { + grid-template-columns: repeat(auto-fit, minmax(170px, 1fr)); + margin-top: 16px; + } + .stage-grid { + grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); margin-top: 16px; } - .metric { + .metric, .stage-summary { border: 1px solid var(--border); border-radius: 14px; padding: 14px; @@ -571,12 +726,6 @@ function renderHtml(report) { font-weight: 700; margin-top: 6px; } - .coverage { - display: grid; - grid-template-columns: repeat(auto-fit, minmax(170px, 1fr)); - gap: 12px; - margin: 20px 0 0; - } .pill { border-radius: 999px; padding: 10px 14px; @@ -589,17 +738,31 @@ function renderHtml(report) { .pill.partial { background: var(--partial); } .pill.gap { background: var(--gap); } .pill.na { background: var(--na); } + .stage-title { + font-weight: 700; + margin-bottom: 6px; + } + .stage-stats, .section-summary, .meta-row, .links a, .section-head p { + color: var(--muted); + } .section { - background: var(--card); - border: 1px solid var(--border); - border-radius: 18px; padding: 20px 22px; margin: 0 0 18px; } + .section-head { + display: flex; + justify-content: space-between; + gap: 16px; + align-items: flex-start; + margin-bottom: 14px; + } .section h2 { - margin: 0 0 12px; + margin: 0 0 6px; font-size: 1.2rem; } + .section-head p { + margin: 0; + } ul { list-style: none; padding: 0; @@ -620,7 +783,7 @@ function renderHtml(report) { display: flex; align-items: center; gap: 12px; - margin-bottom: 6px; + margin-bottom: 8px; } .badge { display: inline-block; @@ -636,8 +799,7 @@ function renderHtml(report) { .label { font-weight: 600; } - .evidence, .reason { - color: var(--muted); + .meta-row { font-size: 0.95rem; margin-top: 6px; } @@ -665,6 +827,9 @@ function renderHtml(report) {
    Gap ${report.summary.gap}
    NA ${report.summary.na}
    +
    + ${stageSummaryHtml} +