From 25949f291a4c158d4556bb87e1d3c3475db110fd Mon Sep 17 00:00:00 2001 From: Bozhidar Batsov Date: Sat, 7 Nov 2026 09:44:34 +0200 Subject: [PATCH 1/3] Check that definitions survive an eval that doesn't use them session.persistent evaluated a definition and then its use, which passes on a server that keeps only the definitions each eval touches. repartee did just that (evals ran with Elixir's prune_binding), so after `x = 1` and `y = 2` its sessions had lost `x`. The check evaluates the value snippet in between now. --- CHANGELOG.md | 1 + internal/checks/checks_test.go | 1 + internal/checks/fake_test.go | 7 +++++++ internal/checks/session.go | 8 ++++++-- 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e08e44d..f789149 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Changes - [#22](https://github.com/nrepl/proof/pull/22): The bundled profiles have short names, with versions from their `version` snippets, so the servers in the compatibility matrix get columns of the same width. +- [#24](https://github.com/nrepl/proof/pull/24): `session.persistent` evaluates something else between a definition and its use, so it catches a server that keeps only what each eval uses. ## 0.2.0 (2026-11-03) diff --git a/internal/checks/checks_test.go b/internal/checks/checks_test.go index 1a68347..125f304 100644 --- a/internal/checks/checks_test.go +++ b/internal/checks/checks_test.go @@ -119,6 +119,7 @@ func TestChecksCatchMisbehaviour(t *testing.T) { "interrupt.other-connection": S, "robust.disconnect": S}}, {"no ephemeral sessions", quirks{noEphemeral: true}, map[string]check.Verdict{"session.ephemeral": F, "fireplace.connect": F}}, {"last value only", quirks{lastValueOnly: true}, map[string]check.Verdict{"eval.multiple-forms": F, "fireplace.connect": F}}, + {"unused definitions forgotten", quirks{forgetUnused: true}, map[string]check.Verdict{"session.persistent": F}}, {"stderr dropped", quirks{dropErr: true}, map[string]check.Verdict{"eval.stderr": F}}, {"output after value", quirks{outAfterValue: true}, map[string]check.Verdict{"eval.stdout-order": W}}, {"no eval-error", quirks{noEvalError: true}, map[string]check.Verdict{"eval.error-status": F, "load-file.error": F}}, diff --git a/internal/checks/fake_test.go b/internal/checks/fake_test.go index a5ed6d5..ec0c79b 100644 --- a/internal/checks/fake_test.go +++ b/internal/checks/fake_test.go @@ -27,6 +27,7 @@ type quirks struct { socketSessions bool // sessions only exist on the connection that made them noEphemeral bool // requests without a session get unknown-session lastValueOnly bool + forgetUnused bool // an eval keeps only the definitions it uses, like repartee did dropErr bool outAfterValue bool noEvalError bool // failed evals get no eval-error status @@ -566,6 +567,7 @@ func (s *fakeServer) eval(c net.Conn, req nrepl.Message, local map[string]bool) } } var values []string + used := false for _, form := range forms(code) { value, out := "nil", "" switch form { @@ -610,8 +612,10 @@ func (s *fakeServer) eval(c net.Conn, req nrepl.Message, local map[string]bool) case "define": sess.vars["answer"] = "42" value = "#'user/answer" + used = true case "use": value = sess.vars["answer"] + used = true case "read": if !q.noNeedInput { sess.pending = req @@ -666,5 +670,8 @@ func (s *fakeServer) eval(c net.Conn, req nrepl.Message, local map[string]bool) if q.lastValueOnly && len(values) > 0 { s.send(c, req, map[string]any{"value": values[len(values)-1], "ns": "user"}) } + if q.forgetUnused && !used { + sess.vars = map[string]string{} + } finish() } diff --git a/internal/checks/session.go b/internal/checks/session.go index 8cae259..37d4f55 100644 --- a/internal/checks/session.go +++ b/internal/checks/session.go @@ -34,16 +34,20 @@ func sessionChecks() []*check.Check { Severity: check.Fail, Why: "Every REPL workflow depends on later evals seeing earlier definitions.", Refs: []check.Ref{specClone}, - Snippets: []string{"define"}, + Snippets: []string{"define", "value"}, Requires: []string{"session.clone"}, Run: func(t *check.T) { s := t.Snippet("define") + other := t.Snippet("value") c := t.Connect() session := t.Session(c) t.Eval(c, s.Code, session) + // An eval in between that doesn't use the definition, as a + // server that keeps only what each eval touches loses it there. + t.Eval(c, other.Code, session) resp := t.Eval(c, s.Use, session) if vs := resp.Values(); !slices.Equal(vs, []string{s.Value}) { - t.Violatef("after evaluating %q, %q gave %q, want [%q]", s.Code, s.Use, vs, s.Value) + t.Violatef("after evaluating %q and then %q, %q gave %q, want [%q]", s.Code, other.Code, s.Use, vs, s.Value) } }, }, From 668459a1f69ad32b05203cb133627f09b55f3663 Mon Sep 17 00:00:00 2001 From: Bozhidar Batsov Date: Sat, 7 Nov 2026 09:44:34 +0200 Subject: [PATCH 2/3] Check repartee for a value per expression The profile had no multiple snippet, so eval.multiple-forms always skipped, and repartee sending only the last value went unnoticed. --- CHANGELOG.md | 1 + profiles/repartee.toml | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f789149..26ecc9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - [#22](https://github.com/nrepl/proof/pull/22): The bundled profiles have short names, with versions from their `version` snippets, so the servers in the compatibility matrix get columns of the same width. - [#24](https://github.com/nrepl/proof/pull/24): `session.persistent` evaluates something else between a definition and its use, so it catches a server that keeps only what each eval uses. +- [#24](https://github.com/nrepl/proof/pull/24): The profile of repartee checks that each expression in an eval gets its own value. ## 0.2.0 (2026-11-03) diff --git a/profiles/repartee.toml b/profiles/repartee.toml index c72af20..b828c52 100644 --- a/profiles/repartee.toml +++ b/profiles/repartee.toml @@ -34,6 +34,10 @@ code = "proof_marker = :proof_marker" use = "proof_marker" value = ":proof_marker" +[snippets.multiple] +code = "1\n2" +values = ["1", "2"] + # Reads a line from stdin (newline included). [snippets.read-line] code = 'IO.gets("")' From bd45f7a7779eb49a37afc00253ae33ffa9256e0e Mon Sep 17 00:00:00 2001 From: Bozhidar Batsov Date: Sat, 7 Nov 2026 09:44:34 +0200 Subject: [PATCH 3/3] Act like dialtone and repartee as they are now nrepl-beam fixed close, unknown-op and multiple values in October, and stderr and repartee's values in November, but proof serve still did all of it for -like dialtone and -like repartee, and the test that keeps the presets in line with the matrix expected it too. Neither server has a scenario left, which proof list shows as none, and the docs and the notes on the spec say what they do now. --- CHANGELOG.md | 1 + README.md | 4 ++-- cmd/proof/main.go | 3 +++ doc/design.md | 2 +- doc/spec-changes.md | 10 +++++----- doc/usage.md | 12 ++++++------ internal/serve/matrix_test.go | 6 ++---- internal/serve/scenarios.go | 16 ++++++++-------- 8 files changed, 28 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26ecc9c..c9d4ee7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - [#22](https://github.com/nrepl/proof/pull/22): The bundled profiles have short names, with versions from their `version` snippets, so the servers in the compatibility matrix get columns of the same width. - [#24](https://github.com/nrepl/proof/pull/24): `session.persistent` evaluates something else between a definition and its use, so it catches a server that keeps only what each eval uses. - [#24](https://github.com/nrepl/proof/pull/24): The profile of repartee checks that each expression in an eval gets its own value. +- [#24](https://github.com/nrepl/proof/pull/24): `proof serve -like dialtone` and `-like repartee` act like nREPL, as neither server has the quirks of its scenarios anymore. ## 0.2.0 (2026-11-03) diff --git a/README.md b/README.md index afc832f..a660c71 100644 --- a/README.md +++ b/README.md @@ -111,8 +111,8 @@ $ proof serve -listen 127.0.0.1:7888 last-value no-err ``` Here it sends only the value of the last form (like Basilisp, jank and -dialtone) and drops what the code prints to stderr (like Basilisp, -dialtone and repartee). `proof list` shows all the scenarios, and every +let-go) and drops what the code prints to stderr (like Basilisp, nbb +and let-go). `proof list` shows all the scenarios, and every one of them is something a real server does (or something TCP can do to the replies). `-like jank` turns on all of jank's scenarios at once, and `proof list` shows the other servers it can behave like. diff --git a/cmd/proof/main.go b/cmd/proof/main.go index ae00dd3..b3ef5f1 100644 --- a/cmd/proof/main.go +++ b/cmd/proof/main.go @@ -426,6 +426,9 @@ func list(w io.Writer) { } line += " " + s } + if len(p.Scenarios) == 0 { + line += " none" + } fmt.Fprintln(w, line) } } diff --git a/doc/design.md b/doc/design.md index 96942e5..ad8be28 100644 --- a/doc/design.md +++ b/doc/design.md @@ -187,7 +187,7 @@ different problem, which `proof serve` takes care of. own it behaves like nREPL, and scenarios make it behave like other servers where they differ from nREPL. The scenarios aren't made up. Every one of them is something a server in the compatibility matrix does -(e.g. `last-value` is what Basilisp, jank and dialtone do), or something +(e.g. `last-value` is what Basilisp, jank and let-go do), or something TCP can do to any server's replies (e.g. deliver a message in pieces). That keeps the list down to the differences a client will actually run into, and the tests make sure it stays that way. proof runs its own diff --git a/doc/spec-changes.md b/doc/spec-changes.md index 8cf0d59..e5dd9ab 100644 --- a/doc/spec-changes.md +++ b/doc/spec-changes.md @@ -220,16 +220,16 @@ failed evaluation doesn't include it either. ### `close` -nREPL, Babashka, nbb, ClojureCLR and let-go respond with -`session-closed`, while dialtone, Basilisp and jank respond with just -`done` (so does the spec's example). vim-fireplace forgets a session only +nREPL, Babashka, nbb, ClojureCLR, let-go and dialtone respond with +`session-closed`, while Basilisp and jank respond with just `done` (so +does the spec's example). vim-fireplace forgets a session only when it sees `session-closed`, and nREPL's own documentation tells clients to check for it, so it should be required. ### Closing an Unknown Session -nREPL responds with `unknown-session`, dialtone with a plain `done`, and -Babashka, nbb, ClojureCLR and let-go with `session-closed`. No client +nREPL responds with `unknown-session`, and Babashka, nbb, ClojureCLR, +let-go and dialtone with `session-closed`. No client cares about this, so we just need to pick one of them. ### Unknown Sessions diff --git a/doc/usage.md b/doc/usage.md index d98ef3b..7327669 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -345,8 +345,8 @@ Running proof serve (last-value, no-err) on 127.0.0.1:7888. Connect your client ``` With these two scenarios it sends only the value of the last form (like -Basilisp, jank and dialtone) and drops whatever the code prints to -stderr (like Basilisp, dialtone and repartee). `proof list` shows all +Basilisp, jank and let-go) and drops whatever the code prints to +stderr (like Basilisp, nbb and let-go). `proof list` shows all the scenarios, and every one of them is something a real server does (or something TCP can do to the replies): @@ -354,12 +354,12 @@ the scenarios, and every one of them is something a real server does |---|---|---| | `split-output` | Output comes one character per message | nREPL splits long output, Basilisp sends `println`'s newline on its own | | `empty-messages` | Replies to `eval` include messages with nothing but `id` and `session` | jank | -| `last-value` | Only the value of the last form is sent | Basilisp, jank, let-go, dialtone | -| `no-err` | What the code prints to stderr never reaches the client | Basilisp, nbb, let-go, dialtone, repartee | +| `last-value` | Only the value of the last form is sent | Basilisp, jank, let-go | +| `no-err` | What the code prints to stderr never reaches the client | Basilisp, nbb, let-go | | `error-with-done` | `eval-error` comes in the same message as `done` | jank | | `no-eval-error` | An error comes with `err` and `ex`, but without the `eval-error` status | nbb, let-go | | `ansi-errors` | Error messages are colored with ANSI escape codes | let-go | -| `no-op-echo` | Replies to unknown ops don't say which op it was | ClojureCLR, Basilisp, jank, nbb, let-go, dialtone, repartee | +| `no-op-echo` | Replies to unknown ops don't say which op it was | ClojureCLR, Basilisp, jank, nbb, let-go | | `no-close-op` | `describe` doesn't list `close`, even though `close` works | jank | | `no-interrupt` | There's no `interrupt` op | ClojureCLR, Basilisp, jank, nbb, and let-go doesn't list it, nor stop an eval with it | | `no-stdin` | There's no `stdin` op, and reading input gets `nil` right away | ClojureCLR and Basilisp started in the background, as they read their own stdin, where Basilisp gets an empty string | @@ -367,7 +367,7 @@ the scenarios, and every one of them is something a real server does | `read-line-waits` | There's no `stdin` op, and reading input waits until the eval is interrupted or its session closed | let-go, which reads the terminal it runs in, where its own REPL takes the lines | | `string-versions` | `versions.proof` is a plain string rather than a dict | Babashka for `versions.babashka`, ClojureCLR for `versions.clojure.tools.nrepl` | | `no-version-string` | `versions` has `proof` and `nrepl` with the numbers of their versions as strings, but no `version-string` | let-go for `versions.let-go` and `versions.nrepl` | -| `no-session-closed` | `close` replies with `done` alone | Basilisp, jank, dialtone, repartee | +| `no-session-closed` | `close` replies with `done` alone | Basilisp, jank | | `shared-state` | Sessions on the same connection share `*1`, `*e` and the current namespace | ClojureCLR, Basilisp, jank, nbb, and let-go for the current namespace | | `socket-sessions` | A session only exists on the connection that cloned it | ClojureCLR, Basilisp, jank | | `any-session` | Requests for sessions that don't exist run in a new session | ClojureCLR, Basilisp, jank, nbb, let-go | diff --git a/internal/serve/matrix_test.go b/internal/serve/matrix_test.go index 77e04f3..4a1b22e 100644 --- a/internal/serve/matrix_test.go +++ b/internal/serve/matrix_test.go @@ -166,10 +166,8 @@ func TestPresetsGetTheColumnsOfTheirServers(t *testing.T) { "op.unknown-echo": W, "session.isolated": F, "session.unknown": F, "session.closed": F, "eval.stderr": F, "eval.error-status": F, "eval.error-report": W, "eval.multiple-forms": F, "eval.unknown-ns": F, "load-file.error": F}), - "dialtone": checktest.Merged(noJava, noLanguage, map[string]check.Verdict{"op.unknown-echo": W, "session.close": F, - "eval.stderr": F, "eval.multiple-forms": F}), - "repartee": checktest.Merged(noJava, noLanguage, map[string]check.Verdict{"op.unknown-echo": W, "session.close": F, - "eval.stderr": F}), + "dialtone": checktest.Merged(noJava, noLanguage), + "repartee": checktest.Merged(noJava, noLanguage), } for name, want := range columns { t.Run(name, func(t *testing.T) { diff --git a/internal/serve/scenarios.go b/internal/serve/scenarios.go index 91df8bf..6c0617c 100644 --- a/internal/serve/scenarios.go +++ b/internal/serve/scenarios.go @@ -57,10 +57,10 @@ var catalog = []scenario{ func(b *behavior) { b.splitOutput = true }, []string{"basilisp"}}, {Scenario{"empty-messages", "Replies to eval include messages with nothing but id and session", "jank"}, func(b *behavior) { b.emptyMessages = true }, []string{"jank"}}, - {Scenario{"last-value", "Only the value of the last form is sent", "Basilisp, jank, let-go, dialtone"}, - func(b *behavior) { b.lastValue = true }, []string{"basilisp", "jank", "let-go", "dialtone"}}, - {Scenario{"no-err", "What the code prints to stderr never reaches the client", "Basilisp, nbb, let-go, dialtone, repartee"}, - func(b *behavior) { b.noErr = true }, []string{"basilisp", "nbb", "let-go", "dialtone", "repartee"}}, + {Scenario{"last-value", "Only the value of the last form is sent", "Basilisp, jank, let-go"}, + func(b *behavior) { b.lastValue = true }, []string{"basilisp", "jank", "let-go"}}, + {Scenario{"no-err", "What the code prints to stderr never reaches the client", "Basilisp, nbb, let-go"}, + func(b *behavior) { b.noErr = true }, []string{"basilisp", "nbb", "let-go"}}, {Scenario{"error-with-done", "eval-error comes in the same message as done", "jank"}, func(b *behavior) { b.errorWithDone = true }, []string{"jank"}}, {Scenario{"no-eval-error", "An error comes with err and ex, but without the eval-error status", "nbb, let-go"}, @@ -68,8 +68,8 @@ var catalog = []scenario{ {Scenario{"ansi-errors", "Error messages are colored with ANSI escape codes", "let-go"}, func(b *behavior) { b.ansiErrors = true }, []string{"let-go"}}, {Scenario{"no-op-echo", "Replies to unknown ops don't say which op it was", - "ClojureCLR, Basilisp, jank, nbb, let-go, dialtone, repartee"}, - func(b *behavior) { b.noOpEcho = true }, []string{"clojure-clr", "basilisp", "jank", "nbb", "let-go", "dialtone", "repartee"}}, + "ClojureCLR, Basilisp, jank, nbb, let-go"}, + func(b *behavior) { b.noOpEcho = true }, []string{"clojure-clr", "basilisp", "jank", "nbb", "let-go"}}, {Scenario{"no-close-op", "describe doesn't list close, even though close works", "jank"}, func(b *behavior) { b.noCloseOp = true }, []string{"jank"}}, {Scenario{"no-interrupt", "There's no interrupt op", @@ -89,8 +89,8 @@ var catalog = []scenario{ {Scenario{"no-version-string", "versions has proof and nrepl with the numbers of their versions as strings, but no version-string", "let-go for versions.let-go and versions.nrepl"}, func(b *behavior) { b.noVersionString = true }, []string{"let-go"}}, - {Scenario{"no-session-closed", "close replies with done alone, without session-closed", "Basilisp, jank, dialtone, repartee"}, - func(b *behavior) { b.noSessionClosed = true }, []string{"basilisp", "jank", "dialtone", "repartee"}}, + {Scenario{"no-session-closed", "close replies with done alone, without session-closed", "Basilisp, jank"}, + func(b *behavior) { b.noSessionClosed = true }, []string{"basilisp", "jank"}}, {Scenario{"shared-state", "Sessions on the same connection share *1, *e and the current namespace", "ClojureCLR, Basilisp, jank, nbb, and let-go for the current namespace"}, func(b *behavior) { b.sharedState = true }, []string{"clojure-clr", "basilisp", "jank", "nbb", "let-go"}},