Show how each client fares with the servers in the matrix

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## main (unreleased)
 
+### Features
+
+- [#23](https://github.com/nrepl/proof/pull/23): A clients tab on the compatibility matrix page, with how many of the checks each client depends on every server passes.
+
 ### Changes
 
 - [#22](https://github.com/nrepl/proof/pull/22): The bundled profiles have short names, with versions from their `version` snippets, so the servers in the compatibility matrix get columns of the same width.

cmd/proof/main.go

@@ -217,6 +217,9 @@ func run(args []string) int {
 	env := &check.Env{Profile: p, Addr: addr}
 	started := time.Now()
 	results := check.Run(env, selected, checks.WireRules())
+	for i, res := range results {
+		results[i].Clients = checks.ClientsOf(res.Severity, res.Refs)
+	}
 	outcome := check.Apply(results, p.ExpectedFailures, opts.only == "")
 
 	r := report.Run{Proof: proofVersion(), Server: p.Name, ServerVersion: ver, Homepage: p.Homepage,

doc/usage.md

@@ -236,6 +236,12 @@ shows why it matters and hovering over a verdict shows its details:
 $ proof matrix -html clojure.json my-server.json > matrix.html
 ```
 
+The page has a second tab for the clients proof knows about, with one
+row per client. There each server gets the number of checks it passes out
+of the ones that would break the client if they failed: the client's own
+checks (e.g. `cider.connect`), the checks that link to its code, and the
+ones every client depends on (e.g. `eval.value`).
+
 > [!TIP]
 > proof's own CI builds such a page for all the profiles in the
 > repository on a daily basis. You can find it at

internal/check/check.go

@@ -286,6 +286,9 @@ type Result struct {
 	// Expected holds the reason from the profile's expected failures when
 	// the check failed as expected.
 	Expected string
+	// Clients names the clients that break when the server fails the
+	// check.
+	Clients []string
 }
 
 // Run runs checks one at a time, then grades the wire traffic they

internal/checks/client_profiles.go

@@ -231,3 +231,24 @@ func nonDictField(resp nrepl.Response, path []string) any {
 	}
 	return v
 }
+
+// ClientsOf names the clients that break when a server fails a check or
+// rule with the given severity and refs: the ones with a client profile
+// whose code the refs point at, or every one of them when the refs don't
+// point at any (e.g. a value that never comes). A warning breaks nobody.
+func ClientsOf(severity check.Severity, refs []check.Ref) []string {
+	if severity != check.Fail {
+		return nil
+	}
+	var cited, all []string
+	for _, p := range clientProfiles() {
+		all = append(all, p.Name)
+		if slices.ContainsFunc(refs, func(r check.Ref) bool { return strings.HasPrefix(r.URL, p.Code) }) {
+			cited = append(cited, p.Name)
+		}
+	}
+	if len(cited) == 0 {
+		return all
+	}
+	return cited
+}

internal/checks/client_profiles_test.go

@@ -2,6 +2,7 @@ package checks
 
 import (
 	"regexp"
+	"slices"
 	"strings"
 	"testing"
 
@@ -20,9 +21,10 @@ func TestClientProfiles(t *testing.T) {
 		ids[c.ID] = true
 	}
 	pinned := regexp.MustCompile(`^https://github\.com/[^/]+/[^/]+/blob/[0-9a-f]{40}/`)
-	// The links in refs.go to the same clients.
+	// The links in refs.go to the same clients, which is how a check knows
+	// the clients it breaks.
 	bases := map[string]string{"CIDER": ciderBase, "Calva": calvaSrc, "Conjure": conjureSrc, "vim-fireplace": fireplaceBase,
-		"REPLy": strings.TrimSuffix(replyBase, "eval_modes/nrepl.clj")}
+		"REPLy": strings.TrimSuffix(replyBase, "eval_modes/nrepl.clj"), "neorepl": neoreplBase}
 	for _, p := range clientProfiles() {
 		if p.Name == "" || !pinned.MatchString(p.Code) {
 			t.Errorf("%q needs a name and links pinned to a commit, got %q", p.Name, p.Code)
@@ -56,6 +58,56 @@ func TestClientProfiles(t *testing.T) {
 	}
 }
 
+// A check breaks the clients whose code it links to, or all of them when
+// it links to none, unless it's only a warning. Every client has checks
+// of its own.
+func TestChecksNameTheClientsTheyBreak(t *testing.T) {
+	var all []string
+	for _, p := range clientProfiles() {
+		all = append(all, p.Name)
+	}
+	clients := map[string][]string{}
+	var everyClient []string
+	note := func(id string, severity check.Severity, refs []check.Ref) {
+		clients[id] = ClientsOf(severity, refs)
+		if slices.Equal(clients[id], all) {
+			everyClient = append(everyClient, id)
+		}
+	}
+	for _, c := range All() {
+		note(c.ID, c.Severity, c.Refs)
+	}
+	for _, r := range WireRules() {
+		note(r.ID, r.Severity, r.Refs)
+	}
+	for id, want := range map[string][]string{
+		"cider.connect":     {"CIDER"},
+		"eval.error-status": {"Calva", "CIDER", "REPLy"},
+		// rebel-readline has no client profile.
+		"wire.error-terminal": {"REPLy"},
+		"describe.versions":   nil,
+	} {
+		if !slices.Equal(clients[id], want) {
+			t.Errorf("%s breaks %v, not %v", id, clients[id], want)
+		}
+	}
+	// A check gets here by linking to no client's code, which is also what
+	// happens when it links to a client without a profile, so each one has
+	// to be listed.
+	want := []string{"session.persistent", "eval.value", "eval.stdout", "eval.stderr", "eval.survives-error",
+		"eval.multiple-forms", "robust.bad-frames", "wire.bencode", "wire.dict"}
+	if !slices.Equal(everyClient, want) {
+		t.Errorf("the checks that break every client are %v, not %v", everyClient, want)
+	}
+	for _, p := range clientProfiles() {
+		for _, c := range p.Checks {
+			if !slices.Equal(clients[c.ID], []string{p.Name}) {
+				t.Errorf("%s breaks %v, not just %s", c.ID, clients[c.ID], p.Name)
+			}
+		}
+	}
+}
+
 // Replies the client can live with still get a note when they say
 // something went wrong.
 func TestClientChecksNoteErrors(t *testing.T) {

internal/report/matrix.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"html"
 	"io"
+	"maps"
 	"os"
 	"slices"
 	"strings"
@@ -77,8 +78,14 @@ func (t table) place(id string) int {
 // passed returns how many checks the server of the i-th report passed,
 // out of the ones that ran.
 func (t table) passed(i int) string {
+	return score(slices.Collect(maps.Values(t.results[i])))
+}
+
+// score says how many of the results are passes, out of the ones that
+// ran.
+func score(results []jsonResult) string {
 	passed, ran := 0, 0
-	for _, res := range t.results[i] {
+	for _, res := range results {
 		if res.Verdict == check.Skipped {
 			continue
 		}
@@ -90,6 +97,62 @@ func (t table) passed(i int) string {
 	return fmt.Sprintf("%d/%d", passed, ran)
 }
 
+// clients are the clients that the checks say they break, the ones the
+// most checks break first. Every report has a say, as the ones made
+// before proof named clients don't name any.
+func (t table) clients() []string {
+	var names []string
+	counts := map[string]int{}
+	for _, id := range t.ids {
+		var named []string
+		for _, results := range t.results {
+			for _, name := range results[id].Clients {
+				if !slices.Contains(named, name) {
+					named = append(named, name)
+				}
+			}
+		}
+		for _, name := range named {
+			if counts[name] == 0 {
+				names = append(names, name)
+			}
+			counts[name]++
+		}
+	}
+	slices.SortStableFunc(names, func(a, b string) int { return counts[b] - counts[a] })
+	return names
+}
+
+// client is how the server of the i-th report fares with a client: the
+// score of the checks that break the client, and the ones it didn't
+// pass. The verdict is the worst of theirs, or skip when none ran.
+func (t table) client(i int, name string) (check.Verdict, string, []string) {
+	verdict := check.Pass
+	var ran []jsonResult
+	var missed []string
+	for _, id := range t.ids {
+		res, ok := t.results[i][id]
+		if !ok || res.Verdict == check.Skipped || !slices.Contains(res.Clients, name) {
+			continue
+		}
+		ran = append(ran, res)
+		switch res.Verdict {
+		case check.Failed:
+			verdict = check.Failed
+			missed = append(missed, res.ID+": "+res.Title)
+		case check.Errored:
+			if verdict != check.Failed {
+				verdict = check.Errored
+			}
+			missed = append(missed, res.ID+": "+res.Title+" (error)")
+		}
+	}
+	if len(ran) == 0 {
+		return check.Skipped, "skip", nil
+	}
+	return verdict, score(ran), missed
+}
+
 var cells = map[check.Verdict]string{
 	check.Pass:    "pass",
 	check.Failed:  "**fail**",
@@ -129,32 +192,33 @@ func escape(s string) string {
 	return strings.ReplaceAll(s, "|", `\|`)
 }
 
-// The page comes from matrix.html, with the table and the versions of
+// The page comes from matrix.html, with the tables and the versions of
 // proof where its markers are.
 //
 //go:embed matrix.html
 var matrixPage string
 
 // HTML writes the same matrix as Matrix as a web page, with the checks
 // grouped like in the text report. The why of a check and the details of
-// a verdict show up when hovering over them.
+// a verdict show up when hovering over them. Another tab of the page has
+// how each client fares with the servers.
 func HTML(w io.Writer, reports []Report) error {
 	t := newTable(reports)
 	esc := html.EscapeString
-	var b strings.Builder
-	b.WriteString("<thead>\n<tr><th>Check</th>")
+	var servers strings.Builder
 	for _, r := range reports {
 		name := esc(r.Server)
 		if r.Homepage != "" {
 			name = fmt.Sprintf(`<a href="%s">%s</a>`, esc(r.Homepage), name)
 		}
-		fmt.Fprintf(&b, "<th>%s", name)
+		fmt.Fprintf(&servers, "<th>%s", name)
 		if r.ServerVersion != "" {
-			fmt.Fprintf(&b, `<span class="version">%s</span>`, esc(r.ServerVersion))
+			fmt.Fprintf(&servers, `<span class="version">%s</span>`, esc(r.ServerVersion))
 		}
-		b.WriteString("</th>")
+		servers.WriteString("</th>")
 	}
-	b.WriteString("</tr>\n</thead>\n<tbody>\n")
+	var b strings.Builder
+	fmt.Fprintf(&b, "<thead>\n<tr><th>Check</th>%s</tr>\n</thead>\n<tbody>\n", servers.String())
 	group := ""
 	for _, id := range t.ids {
 		if g, _, _ := strings.Cut(id, "."); g != group {
@@ -180,6 +244,18 @@ func HTML(w io.Writer, reports []Report) error {
 	}
 	b.WriteString("</tr>\n</tfoot>\n")
 
+	var c strings.Builder
+	fmt.Fprintf(&c, "<thead>\n<tr><th>Client</th>%s</tr>\n</thead>\n<tbody>\n", servers.String())
+	for _, client := range t.clients() {
+		fmt.Fprintf(&c, `<tr><th scope="row">%s</th>`, esc(client))
+		for i := range reports {
+			verdict, score, missed := t.client(i, client)
+			fmt.Fprintf(&c, `<td class="%s"%s>%s</td>`, verdict, titleAttr(strings.Join(missed, "\n")), score)
+		}
+		c.WriteString("</tr>\n")
+	}
+	c.WriteString("</tbody>\n")
+
 	var proofs []string
 	var latest time.Time
 	for _, r := range reports {
@@ -197,7 +273,8 @@ func HTML(w io.Writer, reports []Report) error {
 	if !latest.IsZero() {
 		made += " on " + latest.UTC().Format(time.DateOnly)
 	}
-	_, err := strings.NewReplacer("<!-- table -->\n", b.String(), "<!-- proof -->", esc(made)).WriteString(w, matrixPage)
+	_, err := strings.NewReplacer("<!-- checks -->\n", b.String(), "<!-- clients -->\n", c.String(),
+		"<!-- proof -->", esc(made)).WriteString(w, matrixPage)
 	return err
 }
 

internal/report/matrix.html

@@ -46,12 +46,28 @@
   font: 15px/1.5 system-ui, -apple-system, "Segoe UI", sans-serif;
 }
 a { color: var(--link); }
-header, footer { max-width: 60rem; }
+header, footer, .tab > p, dl { max-width: 60rem; }
 h1 { margin: 0 0 8px; font-size: 1.75rem; }
 p { margin: 0 0 12px; }
 dl { display: grid; grid-template-columns: auto 1fr; gap: 4px 12px; margin: 0 0 20px; }
 dt { font-weight: 600; }
 dd { margin: 0; }
+nav { display: flex; gap: 4px; margin: 0 0 16px; border-bottom: 1px solid var(--line); }
+nav a {
+  margin-bottom: -1px;
+  padding: 6px 14px;
+  border-bottom: 2px solid transparent;
+  color: var(--muted);
+  font-weight: 600;
+  text-decoration: none;
+}
+nav a:hover { color: var(--fg); }
+main:not(:has(#clients:target)) a[href="#checks"],
+main:has(#clients:target) a[href="#clients"] { color: var(--fg); border-bottom-color: var(--link); }
+/* The checks are the first tab. The margin keeps a link to a tab from
+   scrolling the page. */
+.tab { display: none; scroll-margin-top: 100vh; }
+.tab:target, main:not(:has(.tab:target)) #checks { display: block; }
 .scroll { overflow: auto; max-height: 85vh; border: 1px solid var(--line); border-radius: 6px; }
 table { border-collapse: collapse; width: 100%; }
 th, td { padding: 4px 10px; border-bottom: 1px solid var(--line); text-align: left; }
@@ -97,6 +113,13 @@ <h1>nREPL Compatibility Matrix</h1>
 <a href="https://github.com/nrepl/proof">proof</a>, which are mostly about
 what the clients people use (CIDER, Calva, Conjure, vim-fireplace, REPLy and
 so on) need from a server.</p>
+</header>
+<main>
+<nav>
+<a href="#checks">Checks</a>
+<a href="#clients">Clients</a>
+</nav>
+<section id="checks" class="tab">
 <dl>
 <dt><span class="pass">pass</span></dt><dd>The server does what the clients need.</dd>
 <dt><span class="fail">fail</span></dt><dd>Some client won't work properly with the server, or users lose data.</dd>
@@ -105,12 +128,26 @@ <h1>nREPL Compatibility Matrix</h1>
 <dt><span class="error">error</span></dt><dd>proof couldn't run the check.</dd>
 </dl>
 <p>Hover over a check to see why it matters, and over a verdict for the details.</p>
-</header>
 <div class="scroll">
 <table>
-<!-- table -->
+<!-- checks -->
+</table>
+</div>
+</section>
+<section id="clients" class="tab">
+<p>Each cell says how many of the checks a client depends on the server
+passes, out of the ones that apply to it. Those are the client's own checks
+(e.g. <code>cider.connect</code>), which send what the client sends, the
+checks that link to its code, and the ones every client depends on (e.g.
+<code>eval.value</code>).</p>
+<p>Hover over a number to see the checks the server didn't pass.</p>
+<div class="scroll">
+<table>
+<!-- clients -->
 </table>
 </div>
+</section>
+</main>
 <footer>
 <p>Made with <!-- proof -->.</p>
 </footer>

internal/report/report.go

@@ -141,6 +141,7 @@ type jsonResult struct {
 	Severity string        `json:"severity"`
 	Why      string        `json:"why"`
 	Refs     []check.Ref   `json:"refs"`
+	Clients  []string      `json:"clients,omitempty"`
 	Details  []string      `json:"details,omitempty"`
 	Notes    []string      `json:"notes,omitempty"`
 	Expected string        `json:"expected,omitempty"`
@@ -158,7 +159,7 @@ func JSON(w io.Writer, run Run) error {
 	for _, r := range run.Results {
 		out.Results = append(out.Results, jsonResult{
 			ID: r.ID, Title: r.Title, Verdict: r.Verdict, Severity: r.Severity.String(),
-			Why: r.Why, Refs: r.Refs, Details: r.Details, Notes: r.Notes, Expected: r.Expected,
+			Why: r.Why, Refs: r.Refs, Clients: r.Clients, Details: r.Details, Notes: r.Notes, Expected: r.Expected,
 			Millis: r.Duration.Milliseconds(),
 		})
 	}