From 05352e0913fe527686861d279c5a1a22a0fc5114 Mon Sep 17 00:00:00 2001 From: Aaryan Choudhary Date: Tue, 10 Nov 2026 12:53:11 +0530 Subject: [PATCH 1/2] feat(accessibility): give a report's score, and send report questions to BrowserStack AI - fetchAccessibilityIssues (first page) and startAccessibilityScan now give the report's accessibility score as the dashboard shows it: the share of points each severity lost, the passed/failed checks per severity, and how the score is worked out (a check is one rule on one page; critical 10, serious 7, moderate 3, minor 1; a failed check counts double). - Read from the report's own summary for Website Scanner runs, Automated Tests builds and Workflow Analyzer reports. A scanner run read by its build id is found in the scan list first. App builds and reports have no score here. - The same results tell the agent to call askBrowserStackAI (product a11y) for anything about the report beyond listing its issues, such as a plan or what to fix first, instead of working out a plan itself. Co-Authored-By: Claude Opus 5.5 --- src/tools/accessibility.ts | 81 ++++++++++--- .../accessiblity-utils/report-fetcher.ts | 76 ++++++++++++ ...accessibilityFailedAndSkippedPages.test.ts | 1 + .../accessibilityFetchPagesAndCompare.test.ts | 1 + .../tools/accessibilityReportFetcher.test.ts | 80 +++++++++++++ tests/tools/accessibilityRescan.test.ts | 1 + tests/tools/accessibilityScanOutput.test.ts | 111 ++++++++++++++++++ tests/tools/accessibilityScanSettings.test.ts | 1 + 8 files changed, 333 insertions(+), 19 deletions(-) diff --git a/src/tools/accessibility.ts b/src/tools/accessibility.ts index e69af3de..840fb519 100644 --- a/src/tools/accessibility.ts +++ b/src/tools/accessibility.ts @@ -16,6 +16,8 @@ import { ScanNotInListError, ScanRunNotFoundError, type IssueSource, + type ReportScore, + type SeverityScore, } from "./accessiblity-utils/report-fetcher.js"; import { compareAppIssues, @@ -526,6 +528,30 @@ function webSummaryNote( ]; } +/** + * The report's accessibility score, as the dashboard shows it, with what lost the points and + * how it is worked out, so a question about the score — or a plan to raise it — is answered + * from the report rather than guessed. The weights are the backend's (DedupedUrl + * IMPACT_SCORE_WEIGHTS, scoring version 2: a failed check counts double). + */ +function scoreNote(score: ReportScore | undefined): string[] { + if (!score) return []; + const by = (part: (s: SeverityScore) => string) => + score.severities.map((s) => `${s.severity} ${part(s)}`).join(", "); + return [ + `Accessibility score of the whole report: ${score.score} out of 100, as the dashboard shows it. Share of points lost by severity: ${by((s) => String(s.lost))}. Checks passed/failed by severity: ${by((s) => `${s.passed}/${s.failed}`)}. How it is worked out: a check is one rule on one page (best practices are not scored), and it fails when any element on that page fails the rule; each check is worth points by its severity (critical 10, serious 7, moderate 3, minor 1), a failed check counts double, and the score is the passed points as a share of all points. So a rule's check passes only once every failing element on that page is fixed, and fixing the rules in the severities that lost the most points raises it most.`, + ]; +} + +/** + * Where a question about a web report beyond its issue list goes: BrowserStack AI, which + * answers from the whole report and the product's own knowledge, rather than a plan the + * agent works out for itself from one page of issues. + */ +function askAiNote(source: IssueSource): string { + return `For anything about this report beyond listing its issues, such as a plan to fix them or raise the score, what to fix first, or why it scored as it did, call askBrowserStackAI with product a11y, the user's question and this report (${describeSource(source)}), and give the user its answer. BrowserStack AI answers from the whole report, so do not work out the plan yourself. If askBrowserStackAI says it is not enabled for this account, tell the user, then answer from the issues and score above.`; +} + const WEB_ISSUES_NOTE = "Each issue names its rule (issue_type): the rule's name, description, WCAG criteria and usual fix are under rules, and an issue has its own how_to_fix only when it differs. An issue with needs_review is one the engine could not decide, so a person confirms it: the value is its review status, and until it is confirmed it is not a confirmed failure. An issue with no needs_review, whose rule is not a best practice (best_practice under rules), is a confirmed WCAG failure."; @@ -1085,23 +1111,30 @@ async function fetchAccessibilityIssues( throw error; } - const { - records, - rules, - summary, - page_length, - total_issues, - next_page, - remaining, - duplicate_issues, - unlinked_duplicates, - hidden_issues, - page, - error_pages = [], - } = await parseAccessibilityReportFromCSV(reportLink, { - nextPage: cursor, - pageUrl, - }); + // The score, like the summary, belongs to the whole report: read on its first page only. + const [ + { + records, + rules, + summary, + page_length, + total_issues, + next_page, + remaining, + duplicate_issues, + unlinked_duplicates, + hidden_issues, + page, + error_pages = [], + }, + score, + ] = await Promise.all([ + parseAccessibilityReportFromCSV(reportLink, { + nextPage: cursor, + pageUrl, + }), + cursor ? undefined : reportFetcher.score(source), + ]); // A redirected page was scanned, where it redirected to; a page that showed an error page // is said once, with the error page's issues. const errorPageUrls = new Set(error_pages.map((p) => samePage(p.url))); @@ -1129,12 +1162,14 @@ async function fetchAccessibilityIssues( // A page that did not load has no issues of its own to count. ...(page && error_pages.length === 0 ? pageNote(page) : []), `Retrieved ${page_length} accessibility issues (Total: ${total_issues})`, - // The whole report's counts, said on its first page. + // The whole report's score and counts, said on its first page. + ...scoreNote(score), ...(cursor ? [] : webSummaryNote(summary, total_issues)), ...excludedIssuesNote(duplicate_issues, unlinked_duplicates, hidden_issues), fullReportLine(reportLink), WEB_ISSUES_NOTE, `Issues: ${wrapUntrusted("accessibility scan results", JSON.stringify({ issues: records, rules }))}`, + ...(cursor ? [] : [askAiNote(source)]), closingMessage( next_page, remaining, @@ -1656,6 +1691,7 @@ function createScanSuccessResponse( dashboardLink: string | undefined, settings: ScanSettings = {}, applied?: ScanSettings, + score?: ReportScore, ): CallToolResult { const cursor = report.next_page; const messages = [ @@ -1666,6 +1702,7 @@ function createScanSuccessResponse( scanSettingsNote(settings, applied), fullReportLine(reportUrl), `We found ${report.total_issues} issues. Below are the details of the ${report.page_length} most critical issues.`, + ...scoreNote(score), ...webSummaryNote(report.summary, report.total_issues), ...excludedIssuesNote( report.duplicate_issues, @@ -1674,6 +1711,7 @@ function createScanSuccessResponse( ), WEB_ISSUES_NOTE, `Scan results: ${wrapUntrusted("accessibility scan results", JSON.stringify({ issues: report.records, rules: report.rules }))}`, + askAiNote({ kind: "scan", scanId, scanRunId }), ]; if (cursor !== null) { @@ -1774,16 +1812,21 @@ async function scanOutcome( throw error; } + const [report, score] = await Promise.all([ + parseAccessibilityReportFromCSV(reportLink), + reportFetcher.score({ kind: "scan", scanId, scanRunId }), + ]); return [ createScanSuccessResponse( scanName, - await parseAccessibilityReportFromCSV(reportLink), + report, scanId, scanRunId, reportLink, dashboardLink, settings, applied, + score, ), "completed", ]; diff --git a/src/tools/accessiblity-utils/report-fetcher.ts b/src/tools/accessiblity-utils/report-fetcher.ts index a4d60f0e..2097807a 100644 --- a/src/tools/accessiblity-utils/report-fetcher.ts +++ b/src/tools/accessiblity-utils/report-fetcher.ts @@ -309,6 +309,46 @@ function comparedBuild(report: any): WebBuild { }; } +/** One severity's part in a report's score. */ +export interface SeverityScore { + severity: string; + /** The share of all the report's points (0–100) that this severity's failed checks lost. */ + lost: number; + passed: number; + failed: number; +} + +/** The dashboard's accessibility score for a report (0–100), and how each severity shaped it. */ +export interface ReportScore { + score: number; + severities: SeverityScore[]; +} + +const SCORE_SEVERITIES = ["critical", "serious", "moderate", "minor"]; + +/** + * A report summary's `scoreData` — `{score, impact: {critical, criticalPassed, + * criticalFailed, serious, …}}`, where the bare severity is the share of points it lost — or + * undefined when the summary has no score (`{}` when the plan has none or the report is not + * finished). + */ +export function parseScore(data: any): ReportScore | undefined { + const score = data?.score; + if (typeof score !== "number" || !Number.isFinite(score)) return undefined; + const impact = data.impact ?? {}; + const count = (value: unknown) => + typeof value === "number" && Number.isFinite(value) ? value : 0; + return { + score, + severities: SCORE_SEVERITIES.map((severity) => ({ + severity, + lost: count(impact[severity]), + passed: count(impact[`${severity}Passed`]), + failed: count(impact[`${severity}Failed`]), + })), + }; +} + /** The thBuildId a dashboard report link carries, if any. */ function thBuildIdOf(link: string | undefined): string | null { try { @@ -869,6 +909,42 @@ export class AccessibilityReportFetcher { return changes ?? undefined; } + /** + * The report's accessibility score as the dashboard shows it, from the report's own + * summary. Undefined when it has none (the plan has no scores, or the report is not + * finished), when the summary cannot be read, and for app builds and reports. A scanner + * run known only by its build id is found in the scan list first, which knows a scan's + * latest run only: an earlier run read by its link has no score here. + */ + async score(source: IssueSource): Promise { + const q = encodeURIComponent; + let url: string; + switch (source.kind) { + case "scanBuild": + try { + return await this.score(await this.findScanRun(source)); + } catch { + return undefined; + } + case "scan": + url = `${apiBase()}/website-scanner/v1/scans/${q(source.scanId)}/scan_runs/${q(source.scanRunId)}`; + break; + case "build": + url = `${apiBase()}/automated-tests/v1/builds/${q(source.buildId)}`; + break; + case "workflow": + url = `${apiBase()}/workflow-analyzer/v1/reports/${q(source.reportId)}`; + break; + default: + return undefined; + } + try { + return parseScore((await this.getData(url))?.scoreData); + } catch { + return undefined; + } + } + /** * Where the report is in its processing, as the report's own status reads it: `completed`, * or `queued` / `running` / `processing` / `finalizing` while issues are still being added diff --git a/tests/tools/accessibilityFailedAndSkippedPages.test.ts b/tests/tools/accessibilityFailedAndSkippedPages.test.ts index 39828ea4..2cfb6fb0 100644 --- a/tests/tools/accessibilityFailedAndSkippedPages.test.ts +++ b/tests/tools/accessibilityFailedAndSkippedPages.test.ts @@ -30,6 +30,7 @@ vi.mock("../../src/tools/accessiblity-utils/report-fetcher.js", () => ({ previousScanRun = m.previousScanRun; changesSinceLastRun = m.changesSinceLastRun; appBuildReport = m.appBuildReport; + score = vi.fn().mockResolvedValue(undefined); }, EXPORT_WAIT_MS: 45_000, ExportPendingError: class extends Error {}, diff --git a/tests/tools/accessibilityFetchPagesAndCompare.test.ts b/tests/tools/accessibilityFetchPagesAndCompare.test.ts index b7255c06..915082d2 100644 --- a/tests/tools/accessibilityFetchPagesAndCompare.test.ts +++ b/tests/tools/accessibilityFetchPagesAndCompare.test.ts @@ -27,6 +27,7 @@ vi.mock("../../src/tools/accessiblity-utils/report-fetcher.js", () => ({ scanRuns = m.scanRuns; changesSinceLastRun = m.changesSinceLastRun; findScanRun = m.findScanRun; + score = vi.fn().mockResolvedValue(undefined); }, EXPORT_WAIT_MS: 45_000, ExportPendingError: class extends Error { diff --git a/tests/tools/accessibilityReportFetcher.test.ts b/tests/tools/accessibilityReportFetcher.test.ts index 114aaaa4..3bd337f2 100644 --- a/tests/tools/accessibilityReportFetcher.test.ts +++ b/tests/tools/accessibilityReportFetcher.test.ts @@ -622,3 +622,83 @@ describe("AccessibilityReportFetcher scan run history", () => { expect(get.mock.calls[0][0].url).toBe(`${API}/website-scanner/v1/scans/1234/scan_runs/30`); }); }); + +describe("AccessibilityReportFetcher.score", () => { + beforeEach(() => vi.clearAllMocks()); + + // As the API returns it (scan 4320391 run 2442237): the bare severity is the share of points lost. + const SCORE_DATA = { + score: 52, + impact: { + critical: 19, criticalPassed: 5, criticalFailed: 3, + serious: 27, seriousPassed: 14, seriousFailed: 6, + moderate: 2, moderatePassed: 5, moderateFailed: 1, + minor: 0, minorPassed: 2, minorFailed: 0, + }, + }; + const SCORE = { + score: 52, + severities: [ + { severity: "critical", lost: 19, passed: 5, failed: 3 }, + { severity: "serious", lost: 27, passed: 14, failed: 6 }, + { severity: "moderate", lost: 2, passed: 5, failed: 1 }, + { severity: "minor", lost: 0, passed: 2, failed: 0 }, + ], + }; + + it.each<[string, IssueSource, string]>([ + ["a Website Scanner run", { kind: "scan", scanId: "1234", scanRunId: "30" }, `${API}/website-scanner/v1/scans/1234/scan_runs/30`], + ["an Automated Tests build", { kind: "build", buildId: BUILD_ID }, `${API}/automated-tests/v1/builds/${BUILD_ID}`], + ["a Workflow Analyzer report", { kind: "workflow", reportId: "278684" }, `${API}/workflow-analyzer/v1/reports/278684`], + ])("reads %s's score from its summary", async (_name, source, url) => { + get.mockResolvedValueOnce(ok({ scoreData: SCORE_DATA })); + + await expect(fetcher().score(source)).resolves.toEqual(SCORE); + expect(get.mock.calls[0][0].url).toBe(url); + }); + + it("has no score when the summary carries none: a plan without scores, or an unfinished report", async () => { + get.mockResolvedValueOnce(ok({ scoreData: {} })); + await expect(fetcher().score({ kind: "workflow", reportId: "1" })).resolves.toBeUndefined(); + + get.mockResolvedValueOnce(ok({ status: "completed" })); + await expect(fetcher().score({ kind: "scan", scanId: "1", scanRunId: "2" })).resolves.toBeUndefined(); + }); + + it("has no score when the summary cannot be read", async () => { + get.mockResolvedValueOnce(failed(404, ["Scan run not found"])); + await expect(fetcher().score({ kind: "scan", scanId: "1", scanRunId: "2" })).resolves.toBeUndefined(); + + get.mockRejectedValueOnce(new Error("socket hang up")); + await expect(fetcher().score({ kind: "build", buildId: BUILD_ID })).resolves.toBeUndefined(); + }); + + it("finds a scanner run known only by its build id in the scan list, then reads its score", async () => { + get + .mockResolvedValueOnce( + ok({ + scans: [{ id: 4320391, name: "home", scanUrl: `https://accessibility.browserstack.com/site-scanner/projects/home/builds?thBuildId=${BUILD_ID}`, lastScanDetails: { scanRunId: 2442237 } }], + pagination: { hasNext: false }, + }), + ) + .mockResolvedValueOnce(ok({ scoreData: SCORE_DATA })); + + await expect(fetcher().score({ kind: "scanBuild", thBuildId: BUILD_ID })).resolves.toEqual(SCORE); + expect(get.mock.calls[1][0].url).toBe(`${API}/website-scanner/v1/scans/4320391/scan_runs/2442237`); + }); + + it("has no score for a scanner build id the scan list does not have", async () => { + get.mockResolvedValueOnce(ok({ scans: [], pagination: { hasNext: false } })); + + await expect(fetcher().score({ kind: "scanBuild", thBuildId: BUILD_ID })).resolves.toBeUndefined(); + expect(get).toHaveBeenCalledTimes(1); + }); + + it.each<[string, IssueSource]>([ + ["an app build", { kind: "appBuild", buildId: "app-1" }], + ["an app report", { kind: "appReport", reportId: "9" }], + ])("does not look for a score on %s", async (_name, source) => { + await expect(fetcher().score(source)).resolves.toBeUndefined(); + expect(get).not.toHaveBeenCalled(); + }); +}); diff --git a/tests/tools/accessibilityRescan.test.ts b/tests/tools/accessibilityRescan.test.ts index 79cca8a9..011c589f 100644 --- a/tests/tools/accessibilityRescan.test.ts +++ b/tests/tools/accessibilityRescan.test.ts @@ -314,6 +314,7 @@ describe("startAccessibilityScan with scanId", () => { AccessibilityReportFetcher: class { setAuth = vi.fn(); getReportLink = m.getReportLink; + score = vi.fn().mockResolvedValue(undefined); }, })); vi.doMock("../../src/tools/accessiblity-utils/report-parser.js", () => ({ diff --git a/tests/tools/accessibilityScanOutput.test.ts b/tests/tools/accessibilityScanOutput.test.ts index 416e9b48..845f615d 100644 --- a/tests/tools/accessibilityScanOutput.test.ts +++ b/tests/tools/accessibilityScanOutput.test.ts @@ -14,6 +14,7 @@ const m = vi.hoisted(() => ({ appBuildReport: vi.fn(), appReportIssues: vi.fn(), findScanRun: vi.fn(), + score: vi.fn(), parse: vi.fn(), dashboardLink: vi.fn(), failureDetails: vi.fn(async () => ({ pages: [] })), @@ -49,6 +50,7 @@ vi.mock("../../src/tools/accessiblity-utils/report-fetcher.js", async () => ({ appBuildReport = m.appBuildReport; appReportIssues = m.appReportIssues; findScanRun = m.findScanRun; + score = m.score; }, EXPORT_WAIT_MS: 45_000, ExportPendingError: class extends Error { @@ -1210,3 +1212,112 @@ describe("what a web report's answer says about its issues", () => { expect(m.getReportLink).not.toHaveBeenCalled(); }); }); + +describe("the report's accessibility score", () => { + let handlers: Record any>; + const SCORE = { + score: 52, + severities: [ + { severity: "critical", lost: 19, passed: 5, failed: 3 }, + { severity: "serious", lost: 27, passed: 14, failed: 6 }, + { severity: "moderate", lost: 2, passed: 5, failed: 1 }, + { severity: "minor", lost: 0, passed: 2, failed: 0 }, + ], + }; + + beforeEach(() => { + vi.clearAllMocks(); + // Earlier tests leave resolveBuild turning sources into scanner build ids; ids stay ids. + m.resolveBuild.mockImplementation(async (source: unknown) => source); + m.startScan.mockResolvedValue({ data: { id: "scan-42", scanRunId: "run-7" } }); + m.waitUntilComplete.mockResolvedValue("completed"); + m.getReportLink.mockResolvedValue("https://csv"); + m.parse.mockResolvedValue(GROUPED); + handlers = register(); + }); + + it("fetchAccessibilityIssues gives the score, what lost the points and how it is worked out, on the first page", async () => { + m.score.mockResolvedValue(SCORE); + const body = text( + await handlers.fetchAccessibilityIssues({ scanId: "scan-42", scanRunId: "run-7" }, CONTEXT), + ); + + expect(m.score).toHaveBeenCalledWith({ kind: "scan", scanId: "scan-42", scanRunId: "run-7" }); + expect(body).toContain("Accessibility score of the whole report: 52 out of 100, as the dashboard shows it."); + expect(body).toContain("Share of points lost by severity: critical 19, serious 27, moderate 2, minor 0."); + expect(body).toContain("Checks passed/failed by severity: critical 5/3, serious 14/6, moderate 5/1, minor 2/0."); + expect(body).toContain("a check is one rule on one page (best practices are not scored)"); + expect(body).toContain("(critical 10, serious 7, moderate 3, minor 1), a failed check counts double"); + }); + + it("does not read the score again for a later page", async () => { + m.score.mockResolvedValue(SCORE); + const body = text( + await handlers.fetchAccessibilityIssues({ scanId: "scan-42", scanRunId: "run-7", cursor: 140 }, CONTEXT), + ); + + expect(m.score).not.toHaveBeenCalled(); + expect(body).not.toContain("Accessibility score"); + }); + + it("says nothing about a score when the report has none", async () => { + m.score.mockResolvedValue(undefined); + const body = text( + await handlers.fetchAccessibilityIssues({ scanId: "scan-42", scanRunId: "run-7" }, CONTEXT), + ); + + expect(body).toContain("Retrieved 1 accessibility issues (Total: 79)"); + expect(body).not.toContain("Accessibility score"); + }); + + it("startAccessibilityScan gives the finished scan's score", async () => { + m.score.mockResolvedValue(SCORE); + const body = text( + await handlers.startAccessibilityScan({ name: "score", pageURL: "https://site.example" }, CONTEXT), + ); + + expect(m.score).toHaveBeenCalledWith({ kind: "scan", scanId: "scan-42", scanRunId: "run-7" }); + expect(body).toContain("Accessibility score of the whole report: 52 out of 100"); + }); +}); + +describe("questions about a report beyond its issues", () => { + let handlers: Record any>; + const ASK = "call askBrowserStackAI with product a11y, the user's question and this report"; + + beforeEach(() => { + vi.clearAllMocks(); + m.resolveBuild.mockImplementation(async (source: unknown) => source); + m.startScan.mockResolvedValue({ data: { id: "scan-42", scanRunId: "run-7" } }); + m.waitUntilComplete.mockResolvedValue("completed"); + m.getReportLink.mockResolvedValue("https://csv"); + m.parse.mockResolvedValue(GROUPED); + handlers = register(); + }); + + it("fetchAccessibilityIssues sends them to BrowserStack AI, naming the report, on the first page", async () => { + const body = text( + await handlers.fetchAccessibilityIssues({ scanId: "scan-42", scanRunId: "run-7" }, CONTEXT), + ); + + expect(body).toContain(`${ASK} (Website Scanner scan scan-42, run run-7)`); + expect(body).toContain("do not work out the plan yourself"); + expect(body).toContain("If askBrowserStackAI says it is not enabled for this account, tell the user"); + }); + + it("does not repeat it on a later page", async () => { + const body = text( + await handlers.fetchAccessibilityIssues({ scanId: "scan-42", scanRunId: "run-7", cursor: 140 }, CONTEXT), + ); + + expect(body).not.toContain("askBrowserStackAI"); + }); + + it("startAccessibilityScan sends them to BrowserStack AI too", async () => { + const body = text( + await handlers.startAccessibilityScan({ name: "ask", pageURL: "https://site.example" }, CONTEXT), + ); + + expect(body).toContain(`${ASK} (Website Scanner scan scan-42, run run-7)`); + }); +}); diff --git a/tests/tools/accessibilityScanSettings.test.ts b/tests/tools/accessibilityScanSettings.test.ts index e6cb5773..56d7a2b1 100644 --- a/tests/tools/accessibilityScanSettings.test.ts +++ b/tests/tools/accessibilityScanSettings.test.ts @@ -215,6 +215,7 @@ describe("startAccessibilityScan scan settings", () => { AccessibilityReportFetcher: class { setAuth = vi.fn(); getReportLink = m.getReportLink; + score = vi.fn().mockResolvedValue(undefined); }, })); vi.doMock("../../src/tools/accessiblity-utils/report-parser.js", () => ({ From 1db6c5c6ca1ad22199cbcb5385050d741c4f9bcf Mon Sep 17 00:00:00 2001 From: Aaryan Choudhary Date: Wed, 11 Nov 2026 18:44:43 +0530 Subject: [PATCH 2/2] fix(accessibility): look for a report link's score in 2 scan-list pages only For a Website Scanner link, the score finds the scan's ids by walking the scan list, up to 25 pages at ~1.2 s a page. The list has only each scan's latest run, so a link to an earlier run is never found: the walk ran in full on every first page, after the export's wait of up to 45 s, and could push the issue list past the client's 60 s timeout. The score now searches the newest 2 pages, which still find a scan that was just run. Reading issues and comparing runs keep the full 25-page walk. Co-Authored-By: Claude Opus 5.5 --- .../accessiblity-utils/report-fetcher.ts | 28 ++++++++++++++----- .../tools/accessibilityReportFetcher.test.ts | 16 +++++++++++ 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/src/tools/accessiblity-utils/report-fetcher.ts b/src/tools/accessiblity-utils/report-fetcher.ts index 2097807a..9358f29f 100644 --- a/src/tools/accessiblity-utils/report-fetcher.ts +++ b/src/tools/accessiblity-utils/report-fetcher.ts @@ -102,6 +102,13 @@ export function scanRunLink(source: ScanBuildSource): string { * at limit=50 or 100 the cursor skips scans, so a larger page cannot be used to go faster. */ export const MAX_SCAN_LIST_PAGES = 25; +/** + * How far the score looks for a report link's scan. The score is read alongside the issues, + * after the export's wait, so a full walk would hold the issues up by ~30 s. A link to an + * earlier run is never found (the list has each scan's latest run only), so it would pay that + * in full on every first page. Two pages still find a scan someone just ran. + */ +export const SCORE_SCAN_LIST_PAGES = 2; /** * Runs already found by their build id. A build id names one fixed run, so the mapping never * changes: a link read again, or both sides of a compare, skip the scan-list walk. Keyed by @@ -714,9 +721,13 @@ export class AccessibilityReportFetcher { /** * A run named by its build id, as its scan's ids: the newest scans are walked for the one * whose own link carries this thBuildId. What needs the ids (comparing runs) uses this, as - * does a link the API does not know by its build id. + * does a link the API does not know by its build id. `maxPages` caps the walk below + * MAX_SCAN_LIST_PAGES where a miss costs more than it is worth (the score). */ - async findScanRun({ thBuildId, scanName }: ScanBuildSource): Promise<{ + async findScanRun( + { thBuildId, scanName }: ScanBuildSource, + maxPages: number = MAX_SCAN_LIST_PAGES, + ): Promise<{ kind: "scan"; scanId: string; scanRunId: string; @@ -726,7 +737,7 @@ export class AccessibilityReportFetcher { if (found) return { kind: "scan", ...found }; let nextPage: string | undefined; let sameName: number | undefined; - for (let page = 0; page < MAX_SCAN_LIST_PAGES; page++) { + for (let page = 0; page < maxPages; page++) { const resp = await apiClient.get({ url: `${apiBase()}/website-scanner/v1/scans${ nextPage ? `?nextPage=${encodeURIComponent(nextPage)}` : "" @@ -767,7 +778,7 @@ export class AccessibilityReportFetcher { ); } // nextPage is still set only when the walk stopped at the page cap, not at the end. - throw new ScanNotInListError(nextPage !== undefined, MAX_SCAN_LIST_PAGES); + throw new ScanNotInListError(nextPage !== undefined, maxPages); } /** @@ -913,8 +924,9 @@ export class AccessibilityReportFetcher { * The report's accessibility score as the dashboard shows it, from the report's own * summary. Undefined when it has none (the plan has no scores, or the report is not * finished), when the summary cannot be read, and for app builds and reports. A scanner - * run known only by its build id is found in the scan list first, which knows a scan's - * latest run only: an earlier run read by its link has no score here. + * run known only by its build id is found in the newest SCORE_SCAN_LIST_PAGES of the scan + * list first, which knows a scan's latest run only: an earlier run read by its link, or a + * scan older than those pages, has no score here. */ async score(source: IssueSource): Promise { const q = encodeURIComponent; @@ -922,7 +934,9 @@ export class AccessibilityReportFetcher { switch (source.kind) { case "scanBuild": try { - return await this.score(await this.findScanRun(source)); + return await this.score( + await this.findScanRun(source, SCORE_SCAN_LIST_PAGES), + ); } catch { return undefined; } diff --git a/tests/tools/accessibilityReportFetcher.test.ts b/tests/tools/accessibilityReportFetcher.test.ts index 3bd337f2..27bf8a88 100644 --- a/tests/tools/accessibilityReportFetcher.test.ts +++ b/tests/tools/accessibilityReportFetcher.test.ts @@ -12,6 +12,7 @@ import { ExportPendingError, IssueSource, MAX_SCAN_LIST_PAGES, + SCORE_SCAN_LIST_PAGES, reportExports, MAX_REPORT_EXPORTS, foundScanRuns, @@ -694,6 +695,21 @@ describe("AccessibilityReportFetcher.score", () => { expect(get).toHaveBeenCalledTimes(1); }); + it("looks for a scanner run's score in the newest 2 pages of the scan list only", async () => { + // A link to an earlier run is never in the list, which shows each scan's latest run only: + // a full walk (~1.2 s a page) would hold up the issues read with it, on every first page. + const page = (n: number) => + ok({ + scans: [{ id: n, name: `scan ${n}`, scanUrl: `https://accessibility.browserstack.com/site-scanner/projects/p/builds?thBuildId=latest-${n}`, lastScanDetails: { scanRunId: n } }], + pagination: { hasNext: true, nextPage: `p${n + 1}` }, + }); + get.mockResolvedValueOnce(page(1)).mockResolvedValueOnce(page(2)); + + await expect(fetcher().score({ kind: "scanBuild", thBuildId: "an-earlier-run" })).resolves.toBeUndefined(); + expect(get).toHaveBeenCalledTimes(SCORE_SCAN_LIST_PAGES); + expect(SCORE_SCAN_LIST_PAGES).toBeLessThan(MAX_SCAN_LIST_PAGES); + }); + it.each<[string, IssueSource]>([ ["an app build", { kind: "appBuild", buildId: "app-1" }], ["an app report", { kind: "appReport", reportId: "9" }],