fix(cli): no panic on a malformed strand header; explain lists wrapped readings

CHANGELOG.md

@@ -10,7 +10,9 @@ All notable changes to this project are documented here. The format follows
 - `probbit explain STRAND --event N [--json]`: a read-only explanation of one strand event, with logged inputs and
   flags, elapsed time, previous and recomputed stance/odds, binding habits and their constraints, chain hashes and
   `recomputes` / `differs`. Uses the same replay and turn paths as `live verify`, through the selected event only.
-  Exit 0 recomputes, 1 differs, 2 bad input, 5 unsupported; existing commands and strand digests are unchanged.
+  Exit 0: the selected event recomputes; 1: the selected event, or a line before it, does not recompute (as `live verify`
+  reports it, including a damaged copy of the sealing line in a segment file); 2: a bad flag, an unreadable file, a
+  line that is not JSON or an event line without its fields, or bad inputs in the selected event; 5: unsupported. Existing commands and strand digests are unchanged.
 - [From SOUL.md to stance habits](docs/soul-to-habits.md): the bench-assistant worked example, rules-file mappings
   and the boundaries between stance constraints, model behavior and host action enforcement.
 
@@ -55,6 +57,13 @@ All notable changes to this project are documented here. The format follows
   call returns, that turn gets no stance either (its event is in the engine's record); a seal that lands after that
   leaves the turn its stance, with the right number. A retried message that was answered before the seal gets the stance of its own event again. The
   shadow gate is not affected and keeps logging.
+- `probbit live verify` and `probbit explain` panicked (exit 101) on a strand whose header state has a non-ASCII
+  `persona.digest`: the error message cut the value at a byte that was not a character boundary. The message now cuts
+  by characters, and both commands report the mismatch with exit 1 (`live verify` and `explain --json` print it as a
+  JSON document). The defect is in 0.14.0 too.
+- `probbit explain` (new in this release) listed a Boolean sense reading under `flags` only when it was bare
+  (`"tool_fail": true`); a wrapped reading (`{"v": true, "src": ...}`, docs/persona.md section 2.11) was left out. Both
+  forms are listed now.
 
 ## 0.14.1 - Unreleased
 

docs/CHEATSHEET.md

@@ -131,7 +131,7 @@ The exact tiers run before the sampler:
 - `check`: 0 every rule held and every strand replays; 1 a rule broken, an unresolved contradiction, a strand that fails or one
   this engine cannot verify (unsupported) (1 wins); 2 a bad persona, rule file or flag; 3 the only problem is an unproven rule.
 - `monitor`: 0 every line replays; 1 a line differs; 2 a bad flag, a port that cannot be had, or a file that is not a strand.
-- `explain` (unreleased): 0 the event recomputes; 1 a line differs in its prefix; 2 bad flags, missing file, out-of-range event or malformed JSON/event envelope; 5 unsupported field or line kind.
+- `explain` (unreleased): 0 the selected event recomputes; 1 the selected event, or a line before it, does not recompute (as `live verify` reports it: a changed digest, a broken chain, or inputs of an earlier event that the engine rejects); 2 a bad flag or out-of-range `--event`, an unreadable file, a line that is not JSON or an event line without its fields, or inputs of the selected event that the engine rejects; 5 unsupported field or line kind. In a segment file after the first, line 2 is the copy of the sealing line and is checked against the header's link, so a damaged copy is 1.
 
 ## Errors
 One channel, the same for every command (0.8.2):

probbit-cli/src/explain.rs

@@ -106,7 +106,7 @@ fn flags(inputs: &Json) -> Json {
         }
         if key == "senses" {
             for (id, reading) in value.as_obj().unwrap_or(&[]) {
-                let raw = reading.get("value").unwrap_or(reading);
+                let raw = reading.get("v").unwrap_or(reading);
                 if matches!(raw, Json::Bool(_)) {
                     flags.push((format!("senses.{id}"), raw.clone()));
                 }

probbit-cli/tests/damaged.rs

@@ -0,0 +1,169 @@
+//! Damaged strands: `live verify` and `explain` answer a broken file with an exit code (0, 1, 2 or 5) and a JSON document,
+//! never a panic. A small strand is written by the engine (3 events of the sensed example, a checkpoint after the 2nd),
+//! then damaged one place at a time.
+use probbit_persona::json::{self, Json};
+use probbit_persona::persona;
+use std::path::{Path, PathBuf};
+use std::process::{Command, Output, Stdio};
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::time::{Duration, Instant};
+
+const SENSED: &str = "examples/persona/sensed.yaml";
+
+fn root() -> PathBuf { Path::new(env!("CARGO_MANIFEST_DIR")).join("..") }
+fn tmp(name: &str) -> PathBuf {
+    static ID: AtomicUsize = AtomicUsize::new(0);
+    PathBuf::from(env!("CARGO_TARGET_TMPDIR")).join(format!("damaged-{}-{}-{name}", std::process::id(), ID.fetch_add(1, Ordering::Relaxed)))
+}
+/// `probbit ARGS` from the repository root, killed after 20 s: a hang shows as a failed run (no exit code), not a stuck
+/// test. Output goes to files, so a large document cannot fill a pipe while the test waits.
+fn probbit(args: &[&str]) -> Output {
+    let (out, err) = (tmp("stdout"), tmp("stderr"));
+    let mut child = Command::new(env!("CARGO_BIN_EXE_probbit")).args(args).current_dir(root()).stdin(Stdio::null())
+        .stdout(std::fs::File::create(&out).unwrap()).stderr(std::fs::File::create(&err).unwrap()).spawn().unwrap();
+    let start = Instant::now();
+    let mut timed_out = false;
+    let status = loop {
+        if let Some(s) = child.try_wait().unwrap() { break s; }
+        if start.elapsed() > Duration::from_secs(20) { timed_out = true; let _ = child.kill(); break child.wait().unwrap(); }
+        std::thread::sleep(Duration::from_millis(1));
+    };
+    let mut stderr = std::fs::read(&err).unwrap();
+    if timed_out { stderr.extend_from_slice(b"\ntimed out after 20 s"); }
+    let o = Output { status, stdout: std::fs::read(&out).unwrap(), stderr };
+    let _ = std::fs::remove_file(&out);
+    let _ = std::fs::remove_file(&err);
+    o
+}
+
+/// The strand's text: header, events 1 and 2, the checkpoint after event 2, event 3.
+fn small_strand() -> String {
+    let day = std::fs::read_to_string(root().join("examples/persona/golden/sensed/day.json")).unwrap();
+    let events: String = json::parse(&day).unwrap().as_arr().unwrap()[..3].iter().map(|e| json::write(e, false) + "\n").collect();
+    let (ev, st, strand) = (tmp("events.jsonl"), tmp("state.json"), tmp("small.strand"));
+    std::fs::write(&ev, events).unwrap();
+    let o = probbit(&["persona", "init", SENSED, "--seed", "3", "--out", st.to_str().unwrap()]);
+    assert_eq!(o.status.code(), Some(0), "{o:?}");
+    let o = probbit(&["live", SENSED, "--state", st.to_str().unwrap(), "--clock", "fixed", "--events", ev.to_str().unwrap(),
+        "--strand", strand.to_str().unwrap(), "--checkpoint-every", "2"]);
+    assert_eq!(o.status.code(), Some(0), "{o:?}");
+    let text = std::fs::read_to_string(&strand).unwrap().replace("\r\n", "\n");
+    for f in [&ev, &st, &strand] { let _ = std::fs::remove_file(f); }
+    let _ = std::fs::remove_file(format!("{}.lock", strand.display()));
+    assert_eq!(text.lines().count(), 5, "{text}");
+    assert!(text.lines().nth(3).unwrap().starts_with("{\"checkpoint\":2"), "{text}");
+    text
+}
+
+/// Both readers on `text`: (`live verify` output, `explain --event 1 --json` output).
+fn read_both(text: &str) -> (Output, Output) {
+    let [verify, explain, _] = read_all(text);
+    (verify, explain)
+}
+
+/// `live verify`, `explain --event 1 --json` and `explain --event 3 --json` (the last event, so explain reads every line).
+fn read_all(text: &str) -> [Output; 3] {
+    let p = tmp("variant.strand");
+    std::fs::write(&p, text).unwrap();
+    let path = p.to_str().unwrap();
+    let out = [
+        probbit(&["live", "verify", path]),
+        probbit(&["explain", path, "--event", "1", "--json"]),
+        probbit(&["explain", path, "--event", "3", "--json"]),
+    ];
+    let _ = std::fs::remove_file(&p);
+    out
+}
+
+/// The number of string values in `j`, depth first.
+fn strings(j: &Json) -> usize {
+    match j {
+        Json::Str(_) => 1,
+        Json::Arr(v) => v.iter().map(strings).sum(),
+        Json::Obj(kv) => kv.iter().map(|(_, v)| strings(v)).sum(),
+        _ => 0,
+    }
+}
+
+/// `j` with its `i`-th string value (depth first) replaced by `with`.
+fn replace(j: &Json, i: &mut usize, with: &str) -> Json {
+    match j {
+        Json::Str(s) => { let out = if *i == 0 { with.to_string() } else { s.clone() }; *i = i.wrapping_sub(1); Json::Str(out) }
+        Json::Arr(v) => Json::Arr(v.iter().map(|x| replace(x, i, with)).collect()),
+        Json::Obj(kv) => Json::Obj(kv.iter().map(|(k, v)| (k.clone(), replace(v, i, with))).collect()),
+        other => other.clone(),
+    }
+}
+
+fn header_with_persona_digest(text: &str, digest: &str) -> String {
+    let mut lines: Vec<String> = text.lines().map(str::to_owned).collect();
+    let mut h = json::parse(&lines[0]).unwrap();
+    if let Json::Obj(kv) = &mut h {
+        let st = &mut kv.iter_mut().find(|(k, _)| k == "state").unwrap().1;
+        if let Json::Obj(skv) = st {
+            let p = &mut skv.iter_mut().find(|(k, _)| k == "persona").unwrap().1;
+            if let Json::Obj(pkv) = p { pkv.iter_mut().find(|(k, _)| k == "digest").unwrap().1 = Json::Str(digest.into()); }
+        }
+    }
+    lines[0] = persona::canon(&h);
+    lines.join("\n") + "\n"
+}
+
+#[test]
+fn a_non_ascii_persona_digest_in_the_header_is_a_divergence_not_a_panic() {
+    let text = header_with_persona_digest(&small_strand(), &"é".repeat(10));
+    let (verify, explain) = read_both(&text);
+    for (name, o) in [("live verify", &verify), ("explain", &explain)] {
+        assert_eq!(o.status.code(), Some(1), "{name}: {o:?}");
+        let d = json::parse(std::str::from_utf8(&o.stdout).unwrap().trim()).unwrap_or_else(|e| panic!("{name}: not JSON ({})", e.msg));
+        let why = d.get("diverges").and_then(Json::as_str).unwrap_or_else(|| panic!("{name}: no diverges: {o:?}"));
+        assert!(why.contains("state.persona.digest") && why.contains("éééééééééé"), "{name}: {why}");
+    }
+}
+
+/// Every string value of every line set to ten `é`, to "" and to "x", one at a time, and every line cut in the middle
+/// once (at a character boundary): each variant through `live verify` and `explain --event 1` and `--event 3`. Exit 0, 1,
+/// 2 or 5 only, each with one JSON document on stdout; a run that panics, dies of a signal or takes over 20 s fails.
+#[test]
+fn no_panic_from_verify_or_explain_on_damaged_strand_lines() {
+    let text = small_strand();
+    let lines: Vec<&str> = text.lines().collect();
+    let mut variants: Vec<(String, String)> = vec![];
+    for (li, line) in lines.iter().enumerate() {
+        let j = json::parse(line).unwrap();
+        for si in 0..strings(&j) {
+            for with in ["éééééééééé", "", "x"] {
+                let mut v: Vec<String> = lines.iter().map(|l| l.to_string()).collect();
+                v[li] = persona::canon(&replace(&j, &mut si.clone(), with));
+                variants.push((format!("line {} string {} = {:?}", li + 1, si, with), v.join("\n") + "\n"));
+            }
+        }
+        let mid = line.char_indices().map(|(i, _)| i).nth(line.chars().count() / 2).unwrap_or(0);
+        let mut v: Vec<String> = lines.iter().map(|l| l.to_string()).collect();
+        v[li] = line[..mid].to_string();
+        variants.push((format!("line {} cut at byte {mid}", li + 1), v.join("\n") + "\n"));
+    }
+    // Bound the test time: at most MAX variants, taken at an even stride, so every line keeps some.
+    const MAX: usize = 600;
+    let total = variants.len();
+    let stride = total.div_ceil(MAX).max(1);
+    let mut bad = vec![];
+    let mut run = 0;
+    for (name, v) in variants.iter().step_by(stride) {
+        run += 1;
+        let [verify, explain1, explain3] = read_all(v);
+        for (cmd, o) in [("live verify", &verify), ("explain --event 1", &explain1), ("explain --event 3", &explain3)] {
+            match o.status.code() {
+                Some(0 | 1 | 2 | 5) => {
+                    if let Err(e) = json::parse(String::from_utf8_lossy(&o.stdout).trim()) {
+                        bad.push(format!("{name}: {cmd} exit {:?}: stdout is not one JSON document ({})", o.status.code(), e.msg));
+                    }
+                }
+                code => bad.push(format!("{name}: {cmd} exit {code:?}: {}", String::from_utf8_lossy(&o.stderr).lines().find(|l| l.contains("panicked")).unwrap_or(""))),
+            }
+        }
+    }
+    eprintln!("damaged-strand sweep: {run} of {total} variants run (stride {stride}), {} bad", bad.len());
+    assert!(run >= 100, "{run} variants");
+    assert!(bad.is_empty(), "{}", bad.join("\n"));
+}

probbit-cli/tests/explain.rs

@@ -103,6 +103,41 @@ fn normal_event_matches_the_frozen_checkpoint_and_has_prior_odds() {
     assert_eq!(std::fs::read_to_string(p).unwrap(), original);
 }
 
+#[test]
+fn bare_and_wrapped_boolean_sense_readings_are_both_flags() {
+    // docs/persona.md 2.11: a bare value takes the event's src, `{v, src}` carries its own; both are logged as sent.
+    let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("..");
+    let run = |args: &[&str]| {
+        let o = Command::new(env!("CARGO_BIN_EXE_probbit")).args(args).current_dir(&root).output().unwrap();
+        assert_eq!(o.status.code(), Some(0), "{args:?}: {o:?}");
+    };
+    let dir = PathBuf::from(env!("CARGO_TARGET_TMPDIR"));
+    let [ev, st, strand] = ["events.jsonl", "state.json", "wrapped.strand"].map(|n| dir.join(format!("explain-{}-{n}", std::process::id())));
+    for f in [&st, &strand] {
+        let _ = std::fs::remove_file(f);
+    }
+    std::fs::write(
+        &ev,
+        concat!(
+            r#"{"elapsed_hours":0.5,"senses":{"tool_fail":{"v":true,"src":"env:runner"}},"src":"env:host"}"#, "\n",
+            r#"{"elapsed_hours":0.5,"senses":{"tool_fail":true},"src":"env:host"}"#, "\n",
+        ),
+    )
+    .unwrap();
+    let (ev_s, st_s, strand_s) = (ev.to_str().unwrap(), st.to_str().unwrap(), strand.to_str().unwrap());
+    run(&["persona", "init", "examples/persona/sensed.yaml", "--seed", "3", "--out", st_s]);
+    run(&["live", "examples/persona/sensed.yaml", "--state", st_s, "--clock", "fixed", "--events", ev_s, "--strand", strand_s]);
+    for event in ["1", "2"] {
+        let o = explain(&strand, event, true);
+        assert_eq!(o.status.code(), Some(0), "{event}: {:?}", o);
+        assert_eq!(at(&doc(&o), &["flags", "senses.tool_fail"]), &json::Json::Bool(true), "event {event}");
+    }
+    for f in [&ev, &st, &strand] {
+        let _ = std::fs::remove_file(f);
+    }
+    let _ = std::fs::remove_file(format!("{strand_s}.lock"));
+}
+
 #[test]
 fn first_event_has_initial_levels_but_no_invented_odds() {
     let o = explain(&fixture("strands/agentdojo-0.11.0.strand"), "1", true);

probbit-persona/src/persona.rs

@@ -2436,7 +2436,7 @@ impl State {
         let kv = j.as_obj().ok_or_else(|| perr("state", "must be a JSON object"))?;
         need(matches!(get(kv, "probbit_persona_state"), Some(Json::Num(x)) if *x == 1.0), "state", "not a probbit persona state (format 1)")?;
         let pd = get(kv, "persona").and_then(|x| x.get("digest")).and_then(Json::as_str).ok_or_else(|| perr("state.persona.digest", "missing"))?;
-        if pd != p.digest { return Err(perr("state.persona.digest", format!("the persona file changed since this state was made ({} != {}); re-init", &pd[..pd.len().min(19)], &p.digest[..19]))); }
+        if pd != p.digest { return Err(perr("state.persona.digest", format!("the persona file changed since this state was made ({} != {}); re-init", pd.chars().take(19).collect::<String>(), &p.digest[..19]))); }
         let body = Json::Obj(kv.iter().filter(|(k, _)| k != "digest").cloned().collect());
         need(get(kv, "digest").and_then(Json::as_str) == Some(sha(&body).as_str()), "state.digest", "state was edited or corrupted")?;
         need(p.drives.is_none() || get(kv, "drives").is_some(), "state.drives", "required with a drives block")?;