Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -307,6 +307,65 @@ describe("FairnessReportPage (#fairness-analytics)", () => {
);
});

it("#9673: renders the fleet-methodology paragraph when basis is a genuine fleet aggregate", async () => {
apiFetch.mockResolvedValue({ ok: true, data: FIXTURE, status: 200, durationMs: 10 });
renderWithClient(<FairnessReportPage />);

await waitFor(() => expect(screen.getByText("Decision accuracy")).toBeTruthy());
expect(screen.getByText(/Why the fleet number, not just our own repos/)).toBeTruthy();
});

it("#9673: hides the fleet-methodology paragraph when basis is one instance's own self-report", async () => {
apiFetch.mockResolvedValue({
ok: true,
data: {
...FIXTURE,
fleetAccuracy: {
...FIXTURE.fleetAccuracy,
instanceCount: 1,
basis: "single_instance_self_report",
},
},
status: 200,
durationMs: 10,
});
renderWithClient(<FairnessReportPage />);

await waitFor(() => expect(screen.getByText("Decision accuracy")).toBeTruthy());
expect(screen.queryByText(/Why the fleet number, not just our own repos/)).toBeNull();
// The card caption also drops the fleet framing rather than reporting "1 self-hosted instance" as corroboration.
expect(
screen.getByText(
"self-reported by one self-hosted instance, not corroborated across operators, last 90 days",
),
).toBeTruthy();
});

it("REGRESSION #9673: a weekly trend row with merged/closed/reversed all null still renders as — without throwing", async () => {
apiFetch.mockResolvedValue({
ok: true,
data: {
...FIXTURE,
accuracyTrend: [
{
weekStart: "2026-01-05",
merged: null,
closed: null,
reversed: null,
accuracyPct: null,
},
],
},
status: 200,
durationMs: 10,
});
renderWithClient(<FairnessReportPage />);

await waitFor(() => expect(screen.getByText("Weekly trend")).toBeTruthy());
const row = screen.getByText("2026-01-05").closest("tr")!;
expect(row.textContent).toBe("2026-01-05————");
});

it("#9068: renders the insufficient-instances state (not a fabricated zero) when gamingFlagsCaught is null", async () => {
apiFetch.mockResolvedValue({
ok: true,
Expand Down
28 changes: 18 additions & 10 deletions apps/loopover-ui/src/components/site/fairness-report-page.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import { TableScroll } from "@/components/site/data-table";
import { Card, Section } from "@/components/site/primitives";
import { StateBoundary } from "@/components/site/state-views";
import { Skeleton } from "@/components/ui/skeleton";
import type { PublicStats } from "@/components/site/proof-of-power-stats-model";
import { isFleetBasis, type PublicStats } from "@/components/site/proof-of-power-stats-model";

// Fairness report (#fairness-analytics): a deeper, linkable page behind the homepage's "Decision accuracy" tile.
// Reads the SAME /v1/public/stats payload proof-of-power-stats.tsx does (no new endpoint) -- this page just
Expand Down Expand Up @@ -126,15 +126,19 @@ export function FairnessReportPage() {
{headlineAccuracyPct != null ? `${pctFmt.format(headlineAccuracyPct)}%` : "—"}
</div>
<p className="mt-2 text-token-sm text-muted-foreground">
{/* #9673: `fleetEligible` only gates volume; `basis` says whether this number is
corroborated across operators or one instance's own self-report -- say which. */}
{fleetEligible
? `across ${intFmt.format(data.fleetAccuracy.instanceCount)} self-hosted instance${data.fleetAccuracy.instanceCount === 1 ? "" : "s"}, last ${data.fleetAccuracy.windowDays} days`
? isFleetBasis(data.fleetAccuracy)
? `across ${intFmt.format(data.fleetAccuracy.instanceCount)} self-hosted instance${data.fleetAccuracy.instanceCount === 1 ? "" : "s"}, last ${data.fleetAccuracy.windowDays} days`
: `self-reported by one self-hosted instance, not corroborated across operators, last ${data.fleetAccuracy.windowDays} days`
: data.totals.reversed > 0
? `${intFmt.format(data.totals.reversed)} human-reversed, lifetime`
: "reversal-grounded, lifetime"}
</p>
{/* #9168 computes `basis` precisely so this number is not read as corroborated-across-operators
when it is one operator's own disclosed outcomes; the page used to drop the field entirely. */}
{fleetEligible && data.fleetAccuracy.basis === "single_instance_self_report" ? (
{fleetEligible && !isFleetBasis(data.fleetAccuracy) ? (
<p className="mt-2 text-token-xs text-muted-foreground">
Self-reported by that single instance, not corroborated across operators
{data.fleetAccuracy.decidedCount != null
Expand Down Expand Up @@ -189,13 +193,17 @@ export function FairnessReportPage() {
or a self-assessment; both are counted after the fact from what actually happened on
GitHub, which is also why the two can differ.
</p>
<p>
<span className="font-medium text-foreground">
Why the fleet number, not just our own repos:
</span>{" "}
the self-hosted instance count above reflects the live fleet running ORB today, not
a historical snapshot of LoopOver's own repos alone.
</p>
{/* #9673: this paragraph asserts fleet corroboration -- render it only when `basis` says the
headline actually IS a fleet aggregate, not one instance's own self-report. */}
{isFleetBasis(data.fleetAccuracy) ? (
<p>
<span className="font-medium text-foreground">
Why the fleet number, not just our own repos:
</span>{" "}
the self-hosted instance count above reflects the live fleet running ORB today,
not a historical snapshot of LoopOver's own repos alone.
</p>
) : null}
</div>

{data.byProject.length > 0 ? (
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,13 @@ import type { PublicStatsSchema } from "@loopover/contract/public-api";
export type { PublicRulePrecision } from "@loopover/contract/public-api";
export type PublicStats = z.infer<typeof PublicStatsSchema>;

/** Whether `fleetAccuracy` is genuine cross-operator corroboration rather than one instance's own
* self-report (#9673) -- the distinction `basis` exists to publish (src/orb/analytics.ts). Both public
* surfaces that read `fleetAccuracy` call this rather than re-deriving the check inline. */
export function isFleetBasis(fleetAccuracy: PublicStats["fleetAccuracy"] | undefined): boolean {
return fleetAccuracy?.basis === "fleet";
}

/** Relative "updated Ns ago" label from the payload's updatedAt (mirrors MetaStrip's freshness logic). */
export function formatStatsAgo(updatedAt: string | null, nowMs: number): string {
if (!updatedAt) return "just now";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import { ProofOfPowerStats } from "@/components/site/proof-of-power-stats";
import {
formatStatsAgo,
formatTimeSaved,
isFleetBasis,
type PublicStats,
} from "@/components/site/proof-of-power-stats-model";

Expand Down Expand Up @@ -150,6 +151,24 @@ describe("formatStatsAgo", () => {
});
});

describe("isFleetBasis", () => {
it('is true only for basis: "fleet"', () => {
expect(isFleetBasis({ ...PAYLOAD.fleetAccuracy, basis: "fleet" })).toBe(true);
});
it('is false for basis: "single_instance_self_report"', () => {
expect(isFleetBasis({ ...PAYLOAD.fleetAccuracy, basis: "single_instance_self_report" })).toBe(
false,
);
});
it("is false when fleetAccuracy has no basis field (deployment predating #9168)", () => {
const { basis: _omitted, ...withoutBasis } = PAYLOAD.fleetAccuracy;
expect(isFleetBasis(withoutBasis as PublicStats["fleetAccuracy"])).toBe(false);
});
it("is false when fleetAccuracy itself is undefined (deployment predating #9168 entirely)", () => {
expect(isFleetBasis(undefined)).toBe(false);
});
});

describe("formatTimeSaved", () => {
it("uses days at scale, hours below 2 days, minutes below an hour", () => {
expect(formatTimeSaved(54160)).toEqual({ value: 38, unit: "days" }); // 2708 × 20 min
Expand Down Expand Up @@ -237,6 +256,7 @@ describe("ProofOfPowerStats", () => {
instanceCount: 3,
windowDays: 90,
gamingFlagsCaught: 2,
basis: "fleet",
},
},
});
Expand All @@ -255,6 +275,35 @@ describe("ProofOfPowerStats", () => {
expect(screen.getAllByRole("img", { name: "Trend over the last 8 weeks" })).toHaveLength(3);
});

it("REGRESSION #9673: discloses a single-instance self-report rather than presenting it as fleet corroboration", async () => {
apiFetch.mockResolvedValue({
ok: true,
status: 200,
durationMs: 1,
data: {
...PAYLOAD,
fleetAccuracy: {
...PAYLOAD.fleetAccuracy,
accuracyPct: 92,
instanceCount: 1,
coveragePct: null,
basis: "single_instance_self_report",
},
},
});
renderWithClient(<ProofOfPowerStats />);
await screen.findByText("Decision accuracy");
// The number itself is still shown -- basis changes the label, never the figure.
expect(screen.getByText("92%")).toBeTruthy();
expect(
screen.getByText(
"merge/close calls confirmed by outcome · self-reported by one self-hosted instance, not corroborated across operators",
),
).toBeTruthy();
// Never the fleet-count phrasing that would overclaim cross-operator corroboration.
expect(screen.queryByText(/1 self-hosted instance$/)).toBeNull();
});

it("#9050: disambiguates the published guarantee's coverage as the AI-judged sub-population, with the backfilled-vs-live split", async () => {
apiFetch.mockResolvedValue({
ok: true,
Expand All @@ -268,6 +317,7 @@ describe("ProofOfPowerStats", () => {
instanceCount: 3,
windowDays: 90,
gamingFlagsCaught: null, // #9068: below the fleet's own eligibility floor
basis: "fleet",
guaranteed: {
close: {
alpha: 0.05,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import { Sparkline } from "@/components/site/sparkline";
import {
formatStatsAgo,
formatTimeSaved,
isFleetBasis,
toTrendPoints,
type PublicStats,
} from "@/components/site/proof-of-power-stats-model";
Expand Down Expand Up @@ -148,6 +149,10 @@ export function ProofOfPowerStats({ className }: { className?: string }) {
const fleetEligible =
(data.fleetAccuracy?.instanceCount ?? 0) > 0 && data.fleetAccuracy?.accuracyPct != null;
const displayedAccuracyPct = fleetEligible ? data.fleetAccuracy!.accuracyPct : totals.accuracyPct;
// #9673: `fleetEligible` only gates volume (is there enough to show a number); `basis` says whether that
// number is corroborated across operators or one instance's own self-report -- the hint below must say
// which, without changing whether or what number is shown.
const accuracyIsFleetBasis = fleetEligible && isFleetBasis(data.fleetAccuracy);
const latestReuseRatePct =
data.reuseRateTrend.length > 0
? data.reuseRateTrend[data.reuseRateTrend.length - 1]!.reuseRatePct
Expand Down Expand Up @@ -207,7 +212,7 @@ export function ProofOfPowerStats({ className }: { className?: string }) {
// signals) -- plus the backfilled-vs-live split behind the evidence, when supplied.
// #9068: gamingFlagsCaught is null (not 0) below the fleet's own eligibility floor, so the
// suffix is omitted entirely rather than rendering a structural zero as "checked, found none".
`merge/close calls confirmed by outcome${data.fleetAccuracy.coveragePct != null ? ` · at ${data.fleetAccuracy.coveragePct}% coverage` : ""}${data.fleetAccuracy.guaranteed?.close ? ` · closes ≥${Math.round((1 - data.fleetAccuracy.guaranteed.close.alpha) * 1000) / 10}% guaranteed at ${data.fleetAccuracy.guaranteed.close.aiJudgedCoveragePct}% coverage of AI-judged closes${data.fleetAccuracy.guaranteed.close.backfilledPct != null ? ` (${data.fleetAccuracy.guaranteed.close.backfilledPct}% backfilled)` : ""}` : ""} · ${intFmt.format(data.fleetAccuracy.instanceCount)} self-hosted instance${data.fleetAccuracy.instanceCount === 1 ? "" : "s"}${data.fleetAccuracy.gamingFlagsCaught != null && data.fleetAccuracy.gamingFlagsCaught > 0 ? ` · ${intFmt.format(data.fleetAccuracy.gamingFlagsCaught)} gaming pattern${data.fleetAccuracy.gamingFlagsCaught === 1 ? "" : "s"} flagged` : ""}`
`merge/close calls confirmed by outcome${data.fleetAccuracy.coveragePct != null ? ` · at ${data.fleetAccuracy.coveragePct}% coverage` : ""}${data.fleetAccuracy.guaranteed?.close ? ` · closes ≥${Math.round((1 - data.fleetAccuracy.guaranteed.close.alpha) * 1000) / 10}% guaranteed at ${data.fleetAccuracy.guaranteed.close.aiJudgedCoveragePct}% coverage of AI-judged closes${data.fleetAccuracy.guaranteed.close.backfilledPct != null ? ` (${data.fleetAccuracy.guaranteed.close.backfilledPct}% backfilled)` : ""}` : ""}${accuracyIsFleetBasis ? ` · ${intFmt.format(data.fleetAccuracy.instanceCount)} self-hosted instance${data.fleetAccuracy.instanceCount === 1 ? "" : "s"}` : " · self-reported by one self-hosted instance, not corroborated across operators"}${data.fleetAccuracy.gamingFlagsCaught != null && data.fleetAccuracy.gamingFlagsCaught > 0 ? ` · ${intFmt.format(data.fleetAccuracy.gamingFlagsCaught)} gaming pattern${data.fleetAccuracy.gamingFlagsCaught === 1 ? "" : "s"} flagged` : ""}`
: totals.reversed > 0
? `${intFmt.format(totals.reversed)} human-reversed`
: "reversal-grounded"
Expand Down
Loading