feat(accessibility): give a report's score, and send report questions to BrowserStack AI

src/tools/accessibility.ts

@@ -16,6 +16,8 @@ import {
   ScanNotInListError,
   ScanRunNotFoundError,
   type IssueSource,
+  type ReportScore,
+  type SeverityScore,
 } from "./accessiblity-utils/report-fetcher.js";
 import {
   compareAppIssues,
@@ -526,6 +528,30 @@ function webSummaryNote(
   ];
 }
 
+/**
+ * The report's accessibility score, as the dashboard shows it, with what lost the points and
+ * how it is worked out, so a question about the score — or a plan to raise it — is answered
+ * from the report rather than guessed. The weights are the backend's (DedupedUrl
+ * IMPACT_SCORE_WEIGHTS, scoring version 2: a failed check counts double).
+ */
+function scoreNote(score: ReportScore | undefined): string[] {
+  if (!score) return [];
+  const by = (part: (s: SeverityScore) => string) =>
+    score.severities.map((s) => `${s.severity} ${part(s)}`).join(", ");
+  return [
+    `Accessibility score of the whole report: ${score.score} out of 100, as the dashboard shows it. Share of points lost by severity: ${by((s) => String(s.lost))}. Checks passed/failed by severity: ${by((s) => `${s.passed}/${s.failed}`)}. How it is worked out: a check is one rule on one page (best practices are not scored), and it fails when any element on that page fails the rule; each check is worth points by its severity (critical 10, serious 7, moderate 3, minor 1), a failed check counts double, and the score is the passed points as a share of all points. So a rule's check passes only once every failing element on that page is fixed, and fixing the rules in the severities that lost the most points raises it most.`,
+  ];
+}
+
+/**
+ * Where a question about a web report beyond its issue list goes: BrowserStack AI, which
+ * answers from the whole report and the product's own knowledge, rather than a plan the
+ * agent works out for itself from one page of issues.
+ */
+function askAiNote(source: IssueSource): string {
+  return `For anything about this report beyond listing its issues, such as a plan to fix them or raise the score, what to fix first, or why it scored as it did, call askBrowserStackAI with product a11y, the user's question and this report (${describeSource(source)}), and give the user its answer. BrowserStack AI answers from the whole report, so do not work out the plan yourself. If askBrowserStackAI says it is not enabled for this account, tell the user, then answer from the issues and score above.`;
+}
+
 const WEB_ISSUES_NOTE =
   "Each issue names its rule (issue_type): the rule's name, description, WCAG criteria and usual fix are under rules, and an issue has its own how_to_fix only when it differs. An issue with needs_review is one the engine could not decide, so a person confirms it: the value is its review status, and until it is confirmed it is not a confirmed failure. An issue with no needs_review, whose rule is not a best practice (best_practice under rules), is a confirmed WCAG failure.";
 
@@ -1085,23 +1111,30 @@ async function fetchAccessibilityIssues(
     throw error;
   }
 
-  const {
-    records,
-    rules,
-    summary,
-    page_length,
-    total_issues,
-    next_page,
-    remaining,
-    duplicate_issues,
-    unlinked_duplicates,
-    hidden_issues,
-    page,
-    error_pages = [],
-  } = await parseAccessibilityReportFromCSV(reportLink, {
-    nextPage: cursor,
-    pageUrl,
-  });
+  // The score, like the summary, belongs to the whole report: read on its first page only.
+  const [
+    {
+      records,
+      rules,
+      summary,
+      page_length,
+      total_issues,
+      next_page,
+      remaining,
+      duplicate_issues,
+      unlinked_duplicates,
+      hidden_issues,
+      page,
+      error_pages = [],
+    },
+    score,
+  ] = await Promise.all([
+    parseAccessibilityReportFromCSV(reportLink, {
+      nextPage: cursor,
+      pageUrl,
+    }),
+    cursor ? undefined : reportFetcher.score(source),
+  ]);
   // A redirected page was scanned, where it redirected to; a page that showed an error page
   // is said once, with the error page's issues.
   const errorPageUrls = new Set(error_pages.map((p) => samePage(p.url)));
@@ -1129,12 +1162,14 @@ async function fetchAccessibilityIssues(
     // A page that did not load has no issues of its own to count.
     ...(page && error_pages.length === 0 ? pageNote(page) : []),
     `Retrieved ${page_length} accessibility issues (Total: ${total_issues})`,
-    // The whole report's counts, said on its first page.
+    // The whole report's score and counts, said on its first page.
+    ...scoreNote(score),
     ...(cursor ? [] : webSummaryNote(summary, total_issues)),
     ...excludedIssuesNote(duplicate_issues, unlinked_duplicates, hidden_issues),
     fullReportLine(reportLink),
     WEB_ISSUES_NOTE,
     `Issues: ${wrapUntrusted("accessibility scan results", JSON.stringify({ issues: records, rules }))}`,
+    ...(cursor ? [] : [askAiNote(source)]),
     closingMessage(
       next_page,
       remaining,
@@ -1656,6 +1691,7 @@ function createScanSuccessResponse(
   dashboardLink: string | undefined,
   settings: ScanSettings = {},
   applied?: ScanSettings,
+  score?: ReportScore,
 ): CallToolResult {
   const cursor = report.next_page;
   const messages = [
@@ -1666,6 +1702,7 @@ function createScanSuccessResponse(
     scanSettingsNote(settings, applied),
     fullReportLine(reportUrl),
     `We found ${report.total_issues} issues. Below are the details of the ${report.page_length} most critical issues.`,
+    ...scoreNote(score),
     ...webSummaryNote(report.summary, report.total_issues),
     ...excludedIssuesNote(
       report.duplicate_issues,
@@ -1674,6 +1711,7 @@ function createScanSuccessResponse(
     ),
     WEB_ISSUES_NOTE,
     `Scan results: ${wrapUntrusted("accessibility scan results", JSON.stringify({ issues: report.records, rules: report.rules }))}`,
+    askAiNote({ kind: "scan", scanId, scanRunId }),
   ];
 
   if (cursor !== null) {
@@ -1774,16 +1812,21 @@ async function scanOutcome(
     throw error;
   }
 
+  const [report, score] = await Promise.all([
+    parseAccessibilityReportFromCSV(reportLink),
+    reportFetcher.score({ kind: "scan", scanId, scanRunId }),
+  ]);
   return [
     createScanSuccessResponse(
       scanName,
-      await parseAccessibilityReportFromCSV(reportLink),
+      report,
       scanId,
       scanRunId,
       reportLink,
       dashboardLink,
       settings,
       applied,
+      score,
     ),
     "completed",
   ];

src/tools/accessiblity-utils/report-fetcher.ts

@@ -102,6 +102,13 @@ export function scanRunLink(source: ScanBuildSource): string {
  * at limit=50 or 100 the cursor skips scans, so a larger page cannot be used to go faster.
  */
 export const MAX_SCAN_LIST_PAGES = 25;
+/**
+ * How far the score looks for a report link's scan. The score is read alongside the issues,
+ * after the export's wait, so a full walk would hold the issues up by ~30 s. A link to an
+ * earlier run is never found (the list has each scan's latest run only), so it would pay that
+ * in full on every first page. Two pages still find a scan someone just ran.
+ */
+export const SCORE_SCAN_LIST_PAGES = 2;
 /**
  * Runs already found by their build id. A build id names one fixed run, so the mapping never
  * changes: a link read again, or both sides of a compare, skip the scan-list walk. Keyed by
@@ -309,6 +316,46 @@ function comparedBuild(report: any): WebBuild {
   };
 }
 
+/** One severity's part in a report's score. */
+export interface SeverityScore {
+  severity: string;
+  /** The share of all the report's points (0–100) that this severity's failed checks lost. */
+  lost: number;
+  passed: number;
+  failed: number;
+}
+
+/** The dashboard's accessibility score for a report (0–100), and how each severity shaped it. */
+export interface ReportScore {
+  score: number;
+  severities: SeverityScore[];
+}
+
+const SCORE_SEVERITIES = ["critical", "serious", "moderate", "minor"];
+
+/**
+ * A report summary's `scoreData` — `{score, impact: {critical, criticalPassed,
+ * criticalFailed, serious, …}}`, where the bare severity is the share of points it lost — or
+ * undefined when the summary has no score (`{}` when the plan has none or the report is not
+ * finished).
+ */
+export function parseScore(data: any): ReportScore | undefined {
+  const score = data?.score;
+  if (typeof score !== "number" || !Number.isFinite(score)) return undefined;
+  const impact = data.impact ?? {};
+  const count = (value: unknown) =>
+    typeof value === "number" && Number.isFinite(value) ? value : 0;
+  return {
+    score,
+    severities: SCORE_SEVERITIES.map((severity) => ({
+      severity,
+      lost: count(impact[severity]),
+      passed: count(impact[`${severity}Passed`]),
+      failed: count(impact[`${severity}Failed`]),
+    })),
+  };
+}
+
 /** The thBuildId a dashboard report link carries, if any. */
 function thBuildIdOf(link: string | undefined): string | null {
   try {
@@ -674,9 +721,13 @@ export class AccessibilityReportFetcher {
   /**
    * A run named by its build id, as its scan's ids: the newest scans are walked for the one
    * whose own link carries this thBuildId. What needs the ids (comparing runs) uses this, as
-   * does a link the API does not know by its build id.
+   * does a link the API does not know by its build id. `maxPages` caps the walk below
+   * MAX_SCAN_LIST_PAGES where a miss costs more than it is worth (the score).
    */
-  async findScanRun({ thBuildId, scanName }: ScanBuildSource): Promise<{
+  async findScanRun(
+    { thBuildId, scanName }: ScanBuildSource,
+    maxPages: number = MAX_SCAN_LIST_PAGES,
+  ): Promise<{
     kind: "scan";
     scanId: string;
     scanRunId: string;
@@ -686,7 +737,7 @@ export class AccessibilityReportFetcher {
     if (found) return { kind: "scan", ...found };
     let nextPage: string | undefined;
     let sameName: number | undefined;
-    for (let page = 0; page < MAX_SCAN_LIST_PAGES; page++) {
+    for (let page = 0; page < maxPages; page++) {
       const resp = await apiClient.get({
         url: `${apiBase()}/website-scanner/v1/scans${
           nextPage ? `?nextPage=${encodeURIComponent(nextPage)}` : ""
@@ -727,7 +778,7 @@ export class AccessibilityReportFetcher {
       );
     }
     // nextPage is still set only when the walk stopped at the page cap, not at the end.
-    throw new ScanNotInListError(nextPage !== undefined, MAX_SCAN_LIST_PAGES);
+    throw new ScanNotInListError(nextPage !== undefined, maxPages);
   }
 
   /**
@@ -869,6 +920,45 @@ export class AccessibilityReportFetcher {
     return changes ?? undefined;
   }
 
+  /**
+   * The report's accessibility score as the dashboard shows it, from the report's own
+   * summary. Undefined when it has none (the plan has no scores, or the report is not
+   * finished), when the summary cannot be read, and for app builds and reports. A scanner
+   * run known only by its build id is found in the newest SCORE_SCAN_LIST_PAGES of the scan
+   * list first, which knows a scan's latest run only: an earlier run read by its link, or a
+   * scan older than those pages, has no score here.
+   */
+  async score(source: IssueSource): Promise<ReportScore | undefined> {
+    const q = encodeURIComponent;
+    let url: string;
+    switch (source.kind) {
+      case "scanBuild":
+        try {
+          return await this.score(
+            await this.findScanRun(source, SCORE_SCAN_LIST_PAGES),
+          );
+        } catch {
+          return undefined;
+        }
+      case "scan":
+        url = `${apiBase()}/website-scanner/v1/scans/${q(source.scanId)}/scan_runs/${q(source.scanRunId)}`;
+        break;
+      case "build":
+        url = `${apiBase()}/automated-tests/v1/builds/${q(source.buildId)}`;
+        break;
+      case "workflow":
+        url = `${apiBase()}/workflow-analyzer/v1/reports/${q(source.reportId)}`;
+        break;
+      default:
+        return undefined;
+    }
+    try {
+      return parseScore((await this.getData(url))?.scoreData);
+    } catch {
+      return undefined;
+    }
+  }
+
   /**
    * Where the report is in its processing, as the report's own status reads it: `completed`,
    * or `queued` / `running` / `processing` / `finalizing` while issues are still being added

tests/tools/accessibilityFailedAndSkippedPages.test.ts

@@ -30,6 +30,7 @@ vi.mock("../../src/tools/accessiblity-utils/report-fetcher.js", () => ({
     previousScanRun = m.previousScanRun;
     changesSinceLastRun = m.changesSinceLastRun;
     appBuildReport = m.appBuildReport;
+    score = vi.fn().mockResolvedValue(undefined);
   },
   EXPORT_WAIT_MS: 45_000,
   ExportPendingError: class extends Error {},

tests/tools/accessibilityFetchPagesAndCompare.test.ts

@@ -27,6 +27,7 @@ vi.mock("../../src/tools/accessiblity-utils/report-fetcher.js", () => ({
     scanRuns = m.scanRuns;
     changesSinceLastRun = m.changesSinceLastRun;
     findScanRun = m.findScanRun;
+    score = vi.fn().mockResolvedValue(undefined);
   },
   EXPORT_WAIT_MS: 45_000,
   ExportPendingError: class extends Error {