fix(cicd): exclude spec- and docs-only PRs from the Release QA report

.github/scripts/release-qa-status/src/exclusions.test.ts

@@ -1,4 +1,4 @@
-import { classifyExclusion } from './exclusions';
+import { classifyExclusion, isDocumentationPath } from './exclusions';
 import { PRDetails } from './types';
 
 function pr(overrides: Partial<PRDetails> = {}): PRDetails {
@@ -77,3 +77,112 @@ describe('classifyExclusion', () => {
     });
   });
 });
+
+describe('isDocumentationPath', () => {
+  it.each([
+    'specs/37376-uve-edit-pencil-permission/spec.md',
+    'specs/36985-block-editor-selection-guard/contracts/report.schema.json',
+    'docs/backend/JAVA_STANDARDS.md',
+    'docs/images/architecture.png',
+    'README.md',
+    'CONTRIBUTING.md',
+    'core-web/libs/sdk/react/CHANGELOG.MD',
+    'examples/nextjs/docs.mdx',
+  ])('treats %s as documentation', (path) => {
+    expect(isDocumentationPath(path)).toBe(true);
+  });
+
+  it.each([
+    'dotCMS/src/main/java/com/dotcms/rest/Foo.java',
+    'core-web/libs/dotcms-ui/src/lib/foo.component.html',
+    'bom/application/pom.xml',
+    '.github/workflows/cicd_6-release.yml',
+  ])('treats %s as implementation', (path) => {
+    expect(isDocumentationPath(path)).toBe(false);
+  });
+
+  // Agent/dev tooling ships as markdown in this repo but is the deliverable,
+  // not prose about one — PR #37309 added a whole Claude skill in markdown alone.
+  it.each([
+    '.claude/skills/dot-test-plan/SKILL.md',
+    '.claude/commands/create-issue.md',
+    '.agents/skills/angular-developer/references/signals.md',
+    '.cursor/rules/java.mdc',
+    '.specify/memory/constitution.md',
+    '.specify/templates/tasks-template.md',
+    'CLAUDE.md',
+    'core-web/CLAUDE.md',
+    'dotCMS/src/main/java/com/dotcms/rest/CLAUDE.md',
+    'core-web/apps/dotcms-ui/AGENTS.md',
+  ])('keeps agent tooling %s in QA scope', (path) => {
+    expect(isDocumentationPath(path)).toBe(false);
+  });
+});
+
+describe('classifyExclusion — path heuristics', () => {
+  it('excludes a Spec-Kit PR 1 as spec-only', () => {
+    expect(
+      classifyExclusion(
+        pr({
+          title: 'Spec: UVE contentlet permission gating',
+          changedFiles: [
+            'specs/37376-uve-edit-pencil-permission/spec.md',
+            'specs/37376-uve-edit-pencil-permission/data-model.md',
+          ],
+        })
+      )
+    ).toEqual({ excluded: true, reason: 'spec-only' });
+  });
+
+  it('excludes a pure docs PR as docs-only', () => {
+    expect(
+      classifyExclusion(pr({ changedFiles: ['docs/core/SPEC_KIT_QUICK_START.md', 'README.md'] }))
+    ).toEqual({ excluded: true, reason: 'docs-only' });
+  });
+
+  it('reports specs mixed with other docs as docs-only, not spec-only', () => {
+    expect(
+      classifyExclusion(
+        pr({ changedFiles: ['specs/37376-uve-edit-pencil-permission/spec.md', 'README.md'] })
+      )
+    ).toEqual({ excluded: true, reason: 'docs-only' });
+  });
+
+  it('keeps a PR that ships a spec alongside implementation', () => {
+    expect(
+      classifyExclusion(
+        pr({
+          changedFiles: [
+            'specs/37376-uve-edit-pencil-permission/spec.md',
+            'dotCMS/src/main/java/com/dotcms/rest/ContentResource.java',
+          ],
+        })
+      )
+    ).toEqual({ excluded: false });
+  });
+
+  it('keeps a markdown-only PR that ships agent tooling', () => {
+    // PR #37309: `feat(skills): add dot-pr-spec-summary` — markdown, but a feature.
+    expect(
+      classifyExclusion(
+        pr({ changedFiles: ['.claude/skills/dot-pr-spec-summary/SKILL.md', 'CLAUDE.md'] })
+      )
+    ).toEqual({ excluded: false });
+  });
+
+  it('keeps a PR whose file list is unknown', () => {
+    // Fetch failed or the PR has more files than one page holds. Over-report
+    // rather than silently hide what might be a code change.
+    expect(classifyExclusion(pr({ changedFiles: undefined }))).toEqual({ excluded: false });
+    expect(classifyExclusion(pr({ changedFiles: [] }))).toEqual({ excluded: false });
+  });
+
+  it('prefers an earlier reason over the path check', () => {
+    // A bot that only writes specs is still excluded as a bot.
+    expect(
+      classifyExclusion(
+        pr({ authorType: 'Bot', changedFiles: ['specs/37376-foo/spec.md'] })
+      )
+    ).toEqual({ excluded: true, reason: 'bot-author' });
+  });
+});

.github/scripts/release-qa-status/src/exclusions.ts

@@ -1,6 +1,7 @@
 /**
  * Determine whether a PR should be excluded from QA evaluation because it's
- * a bot, dependency bump, or release-machinery change.
+ * a bot, dependency bump, release-machinery change, or carries no
+ * implementation at all (spec / documentation only).
  */
 
 import { ExclusionReason, PRDetails } from './types';
@@ -39,6 +40,42 @@ const RELEASE_MACHINERY_PATTERNS: RegExp[] = [
   /^merge pull request/i,
 ];
 
+/**
+ * Agent/dev tooling that happens to ship as markdown. It reads like
+ * documentation but *is* the deliverable — PR #37309 added a whole Claude
+ * skill without touching a single non-markdown file — so it stays in QA scope.
+ * Checked before the documentation patterns, or `.claude/skills/x/SKILL.md`
+ * would match DOC_FILE_PATTERN and drop out of the report.
+ */
+const IMPLEMENTATION_PREFIXES = ['.claude/', '.agents/', '.cursor/', '.specify/'];
+const IMPLEMENTATION_BASENAMES = ['claude.md', 'agents.md'];
+
+/** Spec-Kit feature directories: `specs/<issue>-<slug>/{spec,plan,tasks}.md`, contracts, etc. */
+const SPEC_PREFIX = 'specs/';
+
+/** Documentation trees — every file under them is prose, whatever the extension. */
+const DOC_PREFIXES = ['docs/'];
+
+/** Loose prose anywhere else in the tree: README.md, CONTRIBUTING.md, a library's *.mdx. */
+const DOC_FILE_PATTERN = /\.mdx?$/i;
+
+/**
+ * True when a path carries no implementation — nothing QA could exercise.
+ * Case-insensitive: paths come straight from the GitHub API and casing varies
+ * (`CLAUDE.md`, `SKILL.md`, `README.MD`).
+ */
+export function isDocumentationPath(path: string): boolean {
+  const lower = path.toLowerCase();
+
+  if (IMPLEMENTATION_PREFIXES.some((p) => lower.startsWith(p))) return false;
+  const basename = lower.slice(lower.lastIndexOf('/') + 1);
+  if (IMPLEMENTATION_BASENAMES.includes(basename)) return false;
+
+  if (lower.startsWith(SPEC_PREFIX)) return true;
+  if (DOC_PREFIXES.some((p) => lower.startsWith(p))) return true;
+  return DOC_FILE_PATTERN.test(lower);
+}
+
 export interface ExclusionResult {
   excluded: boolean;
   reason?: ExclusionReason;
@@ -71,5 +108,21 @@ export function classifyExclusion(pr: PRDetails): ExclusionResult {
     return { excluded: true, reason: 'release-machinery' };
   }
 
+  // 4) Path heuristics — last, so a bot-authored spec PR still reads
+  //    `bot-author` rather than `spec-only`.
+  //
+  //    Spec-Kit ships every feature as two PRs, and PR 1 carries spec.md alone.
+  //    It has nothing runnable, so it never earns a QA label, and without this
+  //    it lands in `missing`/`unlinked` and pages its author on every release.
+  //
+  //    Fail safe: only exclude on a file list we actually have in full. An
+  //    absent list (fetch failed, or too many files to page) leaves the PR in
+  //    QA scope — over-reporting beats silently hiding a code change.
+  const files = pr.changedFiles;
+  if (files && files.length > 0 && files.every(isDocumentationPath)) {
+    const allSpecs = files.every((f) => f.toLowerCase().startsWith(SPEC_PREFIX));
+    return { excluded: true, reason: allSpecs ? 'spec-only' : 'docs-only' };
+  }
+
   return { excluded: false };
 }

.github/scripts/release-qa-status/src/format.test.ts

@@ -1,4 +1,4 @@
-import { renderMarkdown, renderSlack } from './format';
+import { renderMarkdown, renderSlack, renderText } from './format';
 import { PRQAResult, ReleaseQAReport, SlackMapping } from './types';
 
 function pr(
@@ -219,4 +219,45 @@ describe('renderMarkdown', () => {
     const out = renderMarkdown(report());
     expect(out).toContain('All non-excluded PRs have a recognized QA verdict');
   });
-});
+});
+
+describe('spec/docs-only exclusions', () => {
+  // The whole point of the path heuristics: a release whose only "gaps" were
+  // Spec-Kit PR 1s must not page anyone. renderSlack keys off
+  // failed+missing+unlinked+external, so excluded PRs drop out for free —
+  // this locks that in against a future refactor folding excluded into flagged.
+  const specOnlyRelease = () =>
+    report({
+      summary: { failed: 0, missing: 0, unlinked: 0, external: 0, passed: 1, excluded: 2 },
+      passed: [pr(37423, 'Dojo to Angular: dotAI Portlet', 'passed')],
+      excluded: [
+        pr(37404, 'Spec: UVE contentlet permission gating', 'excluded', {
+          exclusionReason: 'spec-only',
+        }),
+        pr(37115, 'docs(speckit): link the quick start walkthrough video', 'excluded', {
+          exclusionReason: 'docs-only',
+        }),
+      ],
+    });
+
+  it('stays silent on Slack', () => {
+    expect(renderSlack(specOnlyRelease(), { mappings: [] })).toBe('');
+  });
+
+  it('lists the reasons in the markdown excluded table', () => {
+    const out = renderMarkdown(specOnlyRelease());
+    expect(out).toContain('## Excluded (2)');
+    expect(out).toContain('`spec-only`');
+    expect(out).toContain('`docs-only`');
+    // The blurb above the table must name them, or a reader sees an
+    // unexplained reason in a section that only ever meant "bot / bump".
+    expect(out).toMatch(/spec-only.*docs-only/s);
+  });
+
+  it('lists the reasons in the text excluded section', () => {
+    const out = renderText(specOnlyRelease());
+    expect(out).toContain('Excluded (2)');
+    expect(out).toContain('#37404 Spec: UVE contentlet permission gating — @alice [spec-only]');
+    expect(out).toContain('[docs-only]');
+  });
+});

.github/scripts/release-qa-status/src/format.ts

@@ -81,9 +81,13 @@ export function renderText(report: ReleaseQAReport): string {
     report.summary.missing +
     report.summary.unlinked +
     report.summary.external;
+  // The Excluded section still renders below on a clean release: it is the only
+  // record of which PRs were skipped and why, and after the spec/docs-only rules
+  // the common case is exactly this one — nothing flagged *because* the gaps were
+  // spec PRs. Returning early here would leave that drop unexplained.
   if (totalFlagged === 0) {
     lines.push(':white_check_mark: All non-excluded PRs have a recognized QA verdict.');
-    return lines.join('\n');
+    lines.push('');
   }
 
   for (const key of ['failed', 'missing', 'unlinked', 'external'] as const) {
@@ -99,7 +103,8 @@ export function renderText(report: ReleaseQAReport): string {
   if (report.excluded.length > 0) {
     lines.push(`Excluded (${report.excluded.length})`);
     lines.push(
-      '  Bot / dependency-bump / version-bump / release-machinery PRs (skipped before QA check).'
+      '  Bot / dependency-bump / version-bump / release-machinery / spec-only / docs-only PRs\n' +
+        '  (skipped before QA check).'
     );
     for (const pr of report.excluded) lines.push(renderExcludedTextLine(pr));
     lines.push('');
@@ -161,10 +166,12 @@ export function renderMarkdown(report: ReleaseQAReport): string {
   out.push(`| Excluded | ${s.excluded} |`);
   out.push('');
 
+  // See renderText: the Excluded section below must survive a clean release,
+  // so this reports the all-clear instead of returning on it.
   const flagged = s.failed + s.missing + s.unlinked + s.external;
   if (flagged === 0) {
     out.push(':white_check_mark: All non-excluded PRs have a recognized QA verdict.');
-    return out.join('\n');
+    out.push('');
   }
 
   const sections: Array<{ bucket: PRQAResult[]; key: BucketKey }> = [
@@ -200,7 +207,11 @@ export function renderMarkdown(report: ReleaseQAReport): string {
   if (report.excluded.length > 0) {
     out.push(`## Excluded (${report.excluded.length})`);
     out.push('');
-    out.push('Bot / dependency-bump / version-bump / release-machinery PRs.');
+    out.push(
+      'Bot / dependency-bump / version-bump / release-machinery PRs, plus PRs that ' +
+        'change only specs or documentation (`spec-only` / `docs-only`) and so carry ' +
+        'nothing for QA to exercise.'
+    );
     out.push('');
     out.push('| PR | Title | Author | Reason |');
     out.push('|---|---|---|---|');

.github/scripts/release-qa-status/src/github.test.ts

@@ -1,4 +1,4 @@
-import { findPreviousTag } from './github';
+import { findPreviousTag, parseChangedFilesResponse } from './github';
 
 // Guards against drift from gather-release-data/src/github.ts, which resolves the
 // same release boundary. If these two disagree, the QA status and the changelog
@@ -30,3 +30,37 @@ describe('findPreviousTag', () => {
     expect(findPreviousTag(releases, 'v26.08.19-01')).toBeUndefined();
   });
 });
+
+describe('parseChangedFilesResponse', () => {
+  const files = (paths: string[], hasNextPage = false) => ({
+    files: { nodes: paths.map((path) => ({ path })), pageInfo: { hasNextPage } },
+  });
+
+  it('maps each alias to its paths', () => {
+    const data = { repository: { pr1: files(['specs/foo/spec.md', 'README.md']) } };
+    expect(parseChangedFilesResponse(data, [1])).toEqual(
+      new Map([[1, ['specs/foo/spec.md', 'README.md']]])
+    );
+  });
+
+  // Everything below must yield `undefined`, never `[]`. An empty array reads as
+  // "changed nothing", which classifyExclusion would be free to exclude on; the
+  // point of these cases is that we do not know, so the PR stays in QA scope.
+  it('returns undefined when the list is truncated', () => {
+    const spy = jest.spyOn(process.stderr, 'write').mockImplementation(() => true);
+    const data = { repository: { pr2: files(['a.md'], true) } };
+    expect(parseChangedFilesResponse(data, [2]).get(2)).toBeUndefined();
+    expect(spy).toHaveBeenCalledWith(expect.stringContaining('#2'));
+    spy.mockRestore();
+  });
+
+  it('returns undefined for a missing alias, a null PR, and a null files connection', () => {
+    const data = { repository: { pr4: null, pr5: { files: null } } };
+    const out = parseChangedFilesResponse(data, [3, 4, 5]);
+    expect(out.get(3)).toBeUndefined();
+    expect(out.get(4)).toBeUndefined();
+    expect(out.get(5)).toBeUndefined();
+    // Present-but-unknown, not absent: classifyExclusion reads every PR.
+    expect(out.size).toBe(3);
+  });
+});