From 2b23de1c219d6fed29265ef821a9b2eb9b18f649 Mon Sep 17 00:00:00 2001 From: "Flemming N. Larsen" Date: Tue, 29 Sep 2026 22:25:50 +0200 Subject: [PATCH 1/5] Propose supported HARN-008 integration evidence --- .clue/id-ledger.yaml | 4 +++ .../proposal.md | 27 +++++++++++++++++++ .../tasks.md | 18 +++++++++++++ 3 files changed, 49 insertions(+) create mode 100644 changes/CH-019-harn008-supported-evidence/proposal.md create mode 100644 changes/CH-019-harn008-supported-evidence/tasks.md diff --git a/.clue/id-ledger.yaml b/.clue/id-ledger.yaml index 7c7ba7a..35c2b97 100644 --- a/.clue/id-ledger.yaml +++ b/.clue/id-ledger.yaml @@ -246,6 +246,10 @@ events: - {id: M-148, kind: numeric, state: live, prefix: M, component: "148"} - {id: AN-016, kind: numeric, state: live, prefix: AN, component: "16"} - {id: TASKS-011, kind: numeric, state: live, prefix: TASKS, component: "11"} + - {id: CH-019, kind: numeric, state: reserved, prefix: CH, component: "19"} + - {id: TASKS-012, kind: numeric, state: reserved, prefix: TASKS, component: "12"} + - {id: CH-019, kind: numeric, state: live, prefix: CH, component: "19"} + - {id: TASKS-012, kind: numeric, state: live, prefix: TASKS, component: "12"} high-water: - {id: ADR-002, kind: numeric, state: reserved, prefix: ADR, component: "2"} - {id: AN-014, kind: numeric, state: reserved, prefix: AN, component: "14"} diff --git a/changes/CH-019-harn008-supported-evidence/proposal.md b/changes/CH-019-harn008-supported-evidence/proposal.md new file mode 100644 index 0000000..6c38f1d --- /dev/null +++ b/changes/CH-019-harn008-supported-evidence/proposal.md @@ -0,0 +1,27 @@ +--- +id: CH-019 +type: change +status: open +links: [P-001, CAP-007, CRIT-007, AN-003, AN-016] +title: Add supported integration evidence for skipped-turn telemetry +--- + +# CH-019 — Add supported integration evidence for skipped-turn telemetry + +## Problem + +CH-018 added opt-in skipped-turn telemetry and local probe evidence, but HARN-008 remains draft because its current checked-in tests are Python unit tests and the local runs are not a supported acceptance carrier. The callback-to-observation path needs an attributable JVM integration test before the evidence door M-148 can close. + +## Change + +Extend the existing JVM conformance test bed to run the compatibility harness with skipped-turn capture enabled and assert the result it produces. Reuse its existing staging and runner path; do not create a second battle implementation or change the evidence policy. Prove exact `(bot ID, round, turn)` correspondence to callback output, preservation of early turns, completed empty capture, and distinct disabled, unavailable, and incomplete outcomes. Activate HARN-008 and close M-148 only if the supported tests cover its full scenario. + +## Plan and acceptance + +This change serves `P-001/M-006` through evidence door `M-148`. M-006 remains open; one telemetry criterion does not complete the parity campaign. + +## Challenge to the commitment + +The main assumption is that a JUnit integration test which launches the existing Python compatibility harness and makes its own assertions over the returned observation is an honest supported JVM evidence carrier for the end-to-end contract. A credible alternative is to keep HARN-008 draft until telemetry aggregation moves to a supported implementation or Python becomes an accepted carrier. The cheapest useful test is a local JUnit probe that invokes the existing `--conformance` path with capture enabled and compares its structured telemetry with the probe's callback markers. Revise or stop if that path cannot expose the observation that HARN-008 promises, if it requires copying the Python parsing logic into Java, or if the test cannot reliably cover the warm-up events and status distinctions. + +An implementation could pass by comparing callback markers to a transient JSON response while the compatibility checkpoint or registry still drops the telemetry, leaving the person using M-006 without durable evidence. The integration test must assert the same persisted observation path used by compatibility measurements, and the digest must state that this tier remains local and may skip where the two-engine environment is absent. diff --git a/changes/CH-019-harn008-supported-evidence/tasks.md b/changes/CH-019-harn008-supported-evidence/tasks.md new file mode 100644 index 0000000..7ec4751 --- /dev/null +++ b/changes/CH-019-harn008-supported-evidence/tasks.md @@ -0,0 +1,18 @@ +--- +id: TASKS-012 +type: tasks +status: open +links: [CH-019] +title: Tasks for CH-019 +--- + +# Tasks + +- [ ] Add a JVM Integration path that invokes the compatibility harness with skipped-turn capture enabled and exposes its structured observation to JUnit. +- [ ] Add `HARN-008` Integration-positive evidence matching captured `(bot ID, round, turn)` tuples to callback output, including early and warm-up turns. +- [ ] Add `HARN-008` Integration-positive evidence for a completed capture with an empty event list, without making the result depend on a timing-sensitive no-skip battle. +- [ ] Add `HARN-008` Integration-negative evidence that disabled, unavailable, and incomplete capture remain distinct and never claim zero events. +- [ ] Verify the JUnit assertions cover the durable observation path used by compatibility measurements rather than only a transient conformance response. +- [ ] Activate `HARN-008` and mark `M-148` done only if every clause has supported positive and negative evidence; otherwise keep both states honest and record the exact gap. +- [ ] Update the capability design, telemetry analysis evidence, and `P-001` bookkeeping without changing M-006's open campaign scope. +- [ ] Run the supported local conformance and relevant unit checks, validate the digest, and publish the ready candidate under the full review workflow. From 3f049da2282f5d9259682605b7ea1324c417a327 Mon Sep 17 00:00:00 2001 From: "Flemming N. Larsen" Date: Tue, 29 Sep 2026 23:13:33 +0200 Subject: [PATCH 2/5] Add durable HARN-008 integration evidence --- compat-test/compat_test.py | 16 +- .../probes/NoSkippedTurnProbe.java | 14 ++ .../bridge/conformance/BattleOutcome.java | 8 +- .../conformance/ConformanceHarness.java | 83 +++++++- .../conformance/ConformanceTestBase.java | 20 ++ .../tankroyale/bridge/conformance/Json.java | 45 ++++ .../SkippedTurnTelemetryConformanceTest.java | 201 ++++++++++++++++++ 7 files changed, 379 insertions(+), 8 deletions(-) create mode 100644 compat-test/conformance-robots/conformance/probes/NoSkippedTurnProbe.java create mode 100644 conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/SkippedTurnTelemetryConformanceTest.java diff --git a/compat-test/compat_test.py b/compat-test/compat_test.py index 470981b..28c41d9 100644 --- a/compat-test/compat_test.py +++ b/compat-test/compat_test.py @@ -121,9 +121,10 @@ def local_bot_api_jar(): # by the classic client, so an unrecognised install must not create evidence. LITERUMBLE_ALLOWED_ROBOCODE_CLIENTS = ("1.10.3", "1.11.0", "1.11.1") -STATE_FILE = BASE_DIR / "test_progress.json" -REPORT_FILE = BASE_DIR / "compatibility_report.md" -ERRORS_DIR = BASE_DIR / "errors" +DATA_DIR = Path(os.environ.get("COMPAT_DATA_DIR", BASE_DIR)) +STATE_FILE = DATA_DIR / "test_progress.json" +REPORT_FILE = DATA_DIR / "compatibility_report.md" +ERRORS_DIR = DATA_DIR / "errors" WORK_DIR = Path(os.environ.get("COMPAT_WORK_DIR", BASE_DIR / "work")) RC_WORKER = BASE_DIR / "RcBattleWorker.java" @@ -150,8 +151,8 @@ def local_bot_api_jar(): # verdict is stated as movement from a recorded baseline rather than as an absolute delta: # a bot that has always differed by a given margin and still does has not regressed. REGRESSION_SET_FILE = BASE_DIR / "regression-set.json" -PARITY_REGISTRY_FILE = BASE_DIR / "parity-registry.json" -PARITY_REGISTRY_REPORT = BASE_DIR / "parity-registry.md" +PARITY_REGISTRY_FILE = DATA_DIR / "parity-registry.json" +PARITY_REGISTRY_REPORT = DATA_DIR / "parity-registry.md" MELEE_OPPONENTS_FILE = BASE_DIR / "melee-opponents.json" DEFAULT_BATCH_SIZE = 25 REGRESSION_REPEATS = 5 @@ -1139,6 +1140,8 @@ def run_tr_battle(jar_path: Path, classname, version, opts, setup, rc_signatures "--out", str(out_file), "--timeout", str(max(30, opts.timeout - 15)), ] + if opts.turn_timeout_micros is not None: + cmd.extend(["--turn-timeout", str(opts.turn_timeout_micros)]) started = time.monotonic() watcher = None if rc_signatures is not None: @@ -1461,6 +1464,8 @@ def parse_args(): "(auto-detected when not given)") p.add_argument("--capture-skipped-turns", action="store_true", help="record Tank Royale bridge skipped-turn events, including warm-up turns") + p.add_argument("--turn-timeout-micros", type=int, default=None, + help="override Tank Royale's turn timeout in microseconds for local probes") gate = p.add_argument_group("regression gate (C-004)") gate.add_argument("--regression", action="store_true", @@ -2082,6 +2087,7 @@ def main(): if opts.trace: return run_trace(opts) + DATA_DIR.mkdir(parents=True, exist_ok=True) state = load_state() state["settings"] = {"rounds": opts.rounds, "threshold": opts.threshold, "divisions": DIVISIONS} diff --git a/compat-test/conformance-robots/conformance/probes/NoSkippedTurnProbe.java b/compat-test/conformance-robots/conformance/probes/NoSkippedTurnProbe.java new file mode 100644 index 0000000..f641305 --- /dev/null +++ b/compat-test/conformance-robots/conformance/probes/NoSkippedTurnProbe.java @@ -0,0 +1,14 @@ +package conformance.probes; + +import robocode.AdvancedRobot; + +/** Responds to turns immediately so a completed capture can exercise the empty-event case. */ +public class NoSkippedTurnProbe extends AdvancedRobot { + + @Override + public void run() { + while (true) { + execute(); + } + } +} diff --git a/conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/BattleOutcome.java b/conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/BattleOutcome.java index 8c608d7..5057129 100644 --- a/conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/BattleOutcome.java +++ b/conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/BattleOutcome.java @@ -19,14 +19,16 @@ final class BattleOutcome { private final List errors; private final Double score; private final String detail; + private final String skippedTurnTelemetry; BattleOutcome(boolean completed, List consoles, List errors, - Double score, String detail) { + Double score, String detail, String skippedTurnTelemetry) { this.completed = completed; this.consoles = List.copyOf(consoles); this.errors = List.copyOf(errors); this.score = score; this.detail = detail; + this.skippedTurnTelemetry = skippedTurnTelemetry; } boolean completed() { @@ -49,6 +51,10 @@ String detail() { return detail; } + String skippedTurnTelemetry() { + return skippedTurnTelemetry; + } + /** True when any participant printed the marker. */ boolean anyConsoleContains(String marker) { return consoles.stream().anyMatch(c -> c.contains(marker)); diff --git a/conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/ConformanceHarness.java b/conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/ConformanceHarness.java index 71b69df..7f645fe 100644 --- a/conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/ConformanceHarness.java +++ b/conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/ConformanceHarness.java @@ -48,6 +48,11 @@ static Path repoRoot() { return REPO_ROOT; } + /** Jar produced by a conformance-source run, ready for the normal parity-sweep path. */ + static Path conformanceRobotJar(String robotClass) { + return WORK_DIR.resolve("conformance").resolve(robotClass + "_1.0.jar"); + } + /** The number of rounds every battle this harness runs is configured for. */ int rounds() { return rounds; @@ -112,6 +117,66 @@ BattleOutcome run(Engine engine, String robotClass, Path source, int participant return run(engine, robotClass, source, null, participants); } + BattleOutcome run(Engine engine, String robotClass, Path source, boolean captureSkippedTurns) { + return run(engine, robotClass, source, null, null, captureSkippedTurns, null); + } + + BattleOutcome run(Engine engine, String robotClass, Path source, boolean captureSkippedTurns, + int timeoutSeconds) { + return run(engine, robotClass, source, null, null, captureSkippedTurns, timeoutSeconds); + } + + /** Runs the ordinary two-engine sweep for one fixture and isolates its durable output. */ + String runCompatibilityMeasurement(Path collectionDir, Path dataDir, Path workDir, + String selectedRobot, boolean captureSkippedTurns, + Integer turnTimeoutMicros) { + List command = new ArrayList<>(List.of( + python, + HARNESS.toString(), + "--collections", "roborumble", + "--collection-dir", collectionDir.toString(), + "--only", selectedRobot, + "--limit", "1", + "--rounds", String.valueOf(rounds), + "--timeout", "300", + "--robocode-home", robocodeHome.toString())); + addBridgeArtifacts(command); + if (captureSkippedTurns) { + command.add("--capture-skipped-turns"); + } + if (turnTimeoutMicros != null) { + command.add("--turn-timeout-micros"); + command.add(String.valueOf(turnTimeoutMicros)); + } + + try { + ProcessBuilder builder = new ProcessBuilder(command) + .directory(HARNESS.getParent().toFile()) + .redirectErrorStream(true); + builder.environment().put("COMPAT_WORK_DIR", workDir.toString()); + builder.environment().put("COMPAT_DATA_DIR", dataDir.toString()); + Process process = builder.start(); + + AtomicReference output = new AtomicReference<>(""); + Thread pumpOutput = pump(process.getInputStream(), output); + if (!process.waitFor(20, TimeUnit.MINUTES)) { + process.destroyForcibly(); + join(pumpOutput); + return "the compatibility measurement did not finish within 20 minutes: " + + trim(output.get()); + } + join(pumpOutput); + return process.exitValue() == 0 ? null + : "the compatibility measurement exited " + process.exitValue() + ": " + + trim(output.get()); + } catch (IOException e) { + return "could not start the compatibility measurement: " + e.getMessage(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return "interrupted while waiting for the compatibility measurement"; + } + } + /** Runs a bridge-owned team fixture on one engine. */ BattleOutcome runTeam(Engine engine, String teamClass, Path source) { List command = new ArrayList<>(List.of( @@ -159,6 +224,12 @@ BattleOutcome runTeam(Engine engine, String teamClass, Path source) { private BattleOutcome run(Engine engine, String robotClass, Path source, String enemyClass, Integer participants) { + return run(engine, robotClass, source, enemyClass, participants, false, null); + } + + private BattleOutcome run(Engine engine, String robotClass, Path source, String enemyClass, + Integer participants, boolean captureSkippedTurns, + Integer timeoutSeconds) { List command = new ArrayList<>(List.of( python, HARNESS.toString(), @@ -180,6 +251,13 @@ private BattleOutcome run(Engine engine, String robotClass, Path source, String command.add("--participants"); command.add(String.valueOf(participants)); } + if (captureSkippedTurns) { + command.add("--capture-skipped-turns"); + } + if (timeoutSeconds != null) { + command.add("--timeout"); + command.add(String.valueOf(timeoutSeconds)); + } try { Process process = startHarness(command); @@ -219,7 +297,7 @@ private BattleOutcome run(Engine engine, String robotClass, Path source, String } private static BattleOutcome failed(String detail) { - return new BattleOutcome(false, List.of(), List.of(), null, detail); + return new BattleOutcome(false, List.of(), List.of(), null, detail, null); } /** Starts a harness process with workspace isolation for this Gradle test worker. */ @@ -266,7 +344,8 @@ private static BattleOutcome parse(String json) { Json.stringArray(json, "consoles"), Json.stringArray(json, "errors"), score, - Json.scalar(json, "fatal")); + Json.scalar(json, "fatal"), + Json.object(json, "skipped_turn_telemetry")); } /** Starts a daemon thread that drains a pipe to EOF into {@code sink}. */ diff --git a/conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/ConformanceTestBase.java b/conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/ConformanceTestBase.java index 2253ca6..8c7e1f4 100644 --- a/conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/ConformanceTestBase.java +++ b/conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/ConformanceTestBase.java @@ -131,6 +131,26 @@ BattleOutcome outcomeFor(Engine engine, String robotClass, Path source, int part return outcomeForFixture(engine, robotClass, source, null, participants); } + /** Runs one compatibility-harness battle with the opt-in telemetry setting under test. */ + BattleOutcome telemetryOutcome(Engine engine, String robotClass, Path source, + boolean captureSkippedTurns) { + return harness.run(engine, robotClass, source, captureSkippedTurns); + } + + /** Runs a telemetry battle with a short per-engine timeout for the incomplete path. */ + BattleOutcome telemetryOutcome(Engine engine, String robotClass, Path source, + boolean captureSkippedTurns, int timeoutSeconds) { + return harness.run(engine, robotClass, source, captureSkippedTurns, timeoutSeconds); + } + + /** Runs one standard parity measurement with isolated checkpoint and registry files. */ + String runRecordedMeasurement(Path collectionDir, Path dataDir, Path workDir, + String selectedRobot, boolean captureSkippedTurns, + Integer turnTimeoutMicros) { + return harness.runCompatibilityMeasurement(collectionDir, dataDir, workDir, + selectedRobot, captureSkippedTurns, turnTimeoutMicros); + } + private BattleOutcome outcomeForFixture(Engine engine, String robotClass, Path source, String enemyClass) { return outcomeForFixture(engine, robotClass, source, enemyClass, null); diff --git a/conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/Json.java b/conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/Json.java index 84119ca..b58b49a 100644 --- a/conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/Json.java +++ b/conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/Json.java @@ -45,6 +45,51 @@ static String scalar(String json, String field) { return json.substring(cursor, end).trim(); } + /** The raw object value for a field, or null when the field is absent or not an object. */ + static String object(String json, String field) { + int at = indexOfKey(json, field); + if (at < 0) { + return null; + } + int cursor = at; + while (cursor < json.length() && json.charAt(cursor) != ':') { + cursor++; + } + cursor++; + while (cursor < json.length() && Character.isWhitespace(json.charAt(cursor))) { + cursor++; + } + if (cursor >= json.length() || json.charAt(cursor) != '{') { + return null; + } + + int start = cursor; + int depth = 0; + boolean quoted = false; + boolean escaped = false; + for (; cursor < json.length(); cursor++) { + char c = json.charAt(cursor); + if (quoted) { + if (escaped) { + escaped = false; + } else if (c == '\\') { + escaped = true; + } else if (c == '"') { + quoted = false; + } + continue; + } + if (c == '"') { + quoted = true; + } else if (c == '{') { + depth++; + } else if (c == '}' && --depth == 0) { + return json.substring(start, cursor + 1); + } + } + return null; + } + /** The elements of an array of strings, or an empty list when the field is absent. */ static List stringArray(String json, String field) { List values = new ArrayList<>(); diff --git a/conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/SkippedTurnTelemetryConformanceTest.java b/conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/SkippedTurnTelemetryConformanceTest.java new file mode 100644 index 0000000..96ad4c7 --- /dev/null +++ b/conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/SkippedTurnTelemetryConformanceTest.java @@ -0,0 +1,201 @@ +package dev.robocode.tankroyale.bridge.conformance; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** Supported JVM integration evidence for HARN-008's skipped-turn observation contract. */ +class SkippedTurnTelemetryConformanceTest extends ConformanceTestBase { + + private static final String SKIPPING_ROBOT = "conformance.probes.SkippedTurnProbe"; + private static final String QUIET_ROBOT = "conformance.probes.NoSkippedTurnProbe"; + private static final Path SKIPPING_SOURCE = ConformanceHarness.repoRoot().resolve(Path.of( + "compat-test", "conformance-robots", "conformance", "probes", "SkippedTurnProbe.java")); + private static final Path QUIET_SOURCE = ConformanceHarness.repoRoot().resolve(Path.of( + "compat-test", "conformance-robots", "conformance", "probes", "NoSkippedTurnProbe.java")); + private static final Pattern READY = Pattern.compile( + "BRIDGE_SKIPPED_TURN_TELEMETRY_READY botId=(\\d+)"); + private static final Pattern CALLBACK_EVENT = Pattern.compile("SkippedTurnReported:(\\d+):(\\d+)"); + private static final Pattern TELEMETRY_EVENT = Pattern.compile( + "\\{\\s*\"bot_id\"\\s*:\\s*(\\d+)\\s*,\\s*" + + "\"round\"\\s*:\\s*(\\d+)\\s*,\\s*" + + "\"turn\"\\s*:\\s*(\\d+)\\s*}"); + private static final Pattern EMPTY_EVENTS = Pattern.compile("\"events\"\\s*:\\s*\\[\\s*]"); + private static final Pattern NULL_EVENTS = Pattern.compile("\"events\"\\s*:\\s*null"); + + @Test + @DisplayName("HARN-008 positive: the observation preserves callbacks and a completed zero capture") + void testHARN008_IntegrationPositive_PreservesCallbacksAndCompletedZeroCapture( + @TempDir Path temporaryDirectory) throws IOException { + Path collectionDir = temporaryDirectory.resolve("collection"); + Path skippedWork = temporaryDirectory.resolve("skipped-work"); + Path skippedData = temporaryDirectory.resolve("skipped-data"); + Path skippedJar = prepareFixtureJar( + SKIPPING_ROBOT, SKIPPING_SOURCE, collectionDir); + assertNull(runCompatibilityMeasurement(collectionDir, skippedData, skippedWork, + "SkippedTurnProbe", true, null)); + + String skippedTelemetry = persistedTelemetry(skippedData, skippedJar); + assertEquals("captured", Json.scalar(skippedTelemetry, "status")); + Set recorded = telemetryEvents(skippedTelemetry); + assertFalse(recorded.isEmpty(), "the deliberately slow probe must produce skipped turns"); + assertTrue(recorded.stream().anyMatch(SkippedTurnTelemetryConformanceTest::isWarmupTurn), + "events from the first round and its initial turns must be retained: " + recorded); + assertEquals(callbackEventsByBot(readBotConsoles(skippedWork)), recorded, + "each bridge tuple must match the callback that reached that bot"); + + Path quietWork = temporaryDirectory.resolve("quiet-work"); + Path quietData = temporaryDirectory.resolve("quiet-data"); + Path quietJar = prepareFixtureJar(QUIET_ROBOT, QUIET_SOURCE, collectionDir); + assertNull(runCompatibilityMeasurement(collectionDir, quietData, quietWork, + "NoSkippedTurnProbe", true, 10_000_000)); + + String quietTelemetry = persistedTelemetry(quietData, quietJar); + assertEquals("captured", Json.scalar(quietTelemetry, "status")); + assertTrue(EMPTY_EVENTS.matcher(quietTelemetry).find(), + "a completed enabled run with no events must persist an empty list: " + quietTelemetry); + } + + @Test + @DisplayName("HARN-008 negative: disabled, unavailable, and incomplete capture are not zero") + void testHARN008_IntegrationNegative_DistinguishesMissingCaptureStates() { + BattleOutcome disabled = telemetryOutcome( + Engine.BRIDGE, SKIPPING_ROBOT, SKIPPING_SOURCE, false); + assertTrue(disabled.completed(), disabled.summary()); + assertTelemetryState(disabled, "disabled"); + + BattleOutcome unavailable = telemetryOutcome( + Engine.CLASSIC, SKIPPING_ROBOT, SKIPPING_SOURCE, true); + assertTelemetryState(unavailable, "unavailable"); + + BattleOutcome incomplete = telemetryOutcome( + Engine.BRIDGE, SKIPPING_ROBOT, SKIPPING_SOURCE, true, 1); + assertTelemetryState(incomplete, "incomplete"); + } + + private static void assertTelemetryState(BattleOutcome outcome, String status) { + String telemetry = skippedTurnTelemetry(outcome); + assertEquals(status, Json.scalar(telemetry, "status"), outcome.summary()); + assertTrue(NULL_EVENTS.matcher(telemetry).find(), + () -> status + " capture must not claim an empty event list: " + telemetry); + } + + private static String skippedTurnTelemetry(BattleOutcome outcome) { + String telemetry = outcome.skippedTurnTelemetry(); + assertNotNull(telemetry, "the compatibility result omitted skipped-turn telemetry: " + + outcome.summary()); + return telemetry; + } + + private Path prepareFixtureJar(String robotClass, Path source, Path collectionDir) + throws IOException { + BattleOutcome packaged = telemetryOutcome(Engine.CLASSIC, robotClass, source, false); + assertTrue(packaged.completed(), packaged.summary()); + + Path generatedJar = ConformanceHarness.conformanceRobotJar(robotClass); + assertTrue(Files.isRegularFile(generatedJar), "the compatibility harness did not package " + robotClass); + Path destinationDir = collectionDir.resolve("roborumble"); + Files.createDirectories(destinationDir); + Path destination = destinationDir.resolve(generatedJar.getFileName()); + Files.copy(generatedJar, destination, StandardCopyOption.REPLACE_EXISTING); + return destination; + } + + private String runCompatibilityMeasurement(Path collectionDir, Path dataDir, Path workDir, + String selectedRobot, boolean captureSkippedTurns, + Integer turnTimeoutMicros) { + try { + Files.createDirectories(dataDir); + Files.createDirectories(workDir); + } catch (IOException e) { + return "could not create isolated measurement directories: " + e.getMessage(); + } + return runRecordedMeasurement(collectionDir, dataDir, workDir, selectedRobot, + captureSkippedTurns, turnTimeoutMicros); + } + + private static String persistedTelemetry(Path dataDir, Path jar) throws IOException { + String subject = "roborumble/" + jar.getFileName(); + String registry = Files.readString(dataDir.resolve("parity-registry.json")); + assertTrue(registry.contains("\"" + subject + "\""), + "the parity registry omitted the measured subject " + subject); + String state = Files.readString(dataDir.resolve("test_progress.json")); + String stateTelemetry = Json.object(state, "skipped_turn_telemetry"); + String registryTelemetry = Json.object(registry, "skipped_turn_telemetry"); + assertNotNull(stateTelemetry, "the checkpoint omitted skipped-turn telemetry"); + assertNotNull(registryTelemetry, "the parity observation omitted skipped-turn telemetry"); + assertEquals(Json.scalar(stateTelemetry, "status"), Json.scalar(registryTelemetry, "status"), + "the registry changed the checkpoint telemetry status"); + assertEquals(telemetryEvents(stateTelemetry), telemetryEvents(registryTelemetry), + "the registry changed the checkpoint event tuples"); + return registryTelemetry; + } + + private static List readBotConsoles(Path workDir) throws IOException { + Path botsDirectory = workDir.resolve("tr-bots"); + List consoles = new ArrayList<>(); + try (var paths = Files.walk(botsDirectory)) { + paths.filter(path -> path.getFileName().toString().equals("stdout.log")) + .forEach(path -> { + try { + consoles.add(Files.readString(path)); + } catch (IOException e) { + throw new IllegalStateException("could not read bot console " + path, e); + } + }); + } + return consoles; + } + + private static Set telemetryEvents(String telemetry) { + Set events = new HashSet<>(); + Matcher matcher = TELEMETRY_EVENT.matcher(telemetry); + while (matcher.find()) { + String event = matcher.group(1) + ":" + matcher.group(2) + ":" + matcher.group(3); + assertTrue(events.add(event), "duplicate telemetry tuple: " + event); + } + return events; + } + + private static Set callbackEventsByBot(List consoles) { + Set events = new HashSet<>(); + Set botIds = new HashSet<>(); + for (String console : consoles) { + Matcher ready = READY.matcher(console); + assertTrue(ready.find(), "bot telemetry readiness marker is missing from " + console); + int botId = Integer.parseInt(ready.group(1)); + assertTrue(botIds.add(botId), "duplicate bot readiness marker for bot " + botId); + Matcher callback = CALLBACK_EVENT.matcher(console); + while (callback.find()) { + // Classic StatusEvent rounds are zero-based; Tank Royale telemetry rounds are one-based. + int round = Integer.parseInt(callback.group(1)) + 1; + String event = botId + ":" + round + ":" + callback.group(2); + assertTrue(events.add(event), "duplicate callback tuple: " + event); + } + } + assertEquals(2, botIds.size(), "the persisted measurement must run two bridge bots"); + return events; + } + + private static boolean isWarmupTurn(String event) { + String[] fields = event.split(":"); + return Integer.parseInt(fields[1]) == 1 && Integer.parseInt(fields[2]) <= 10; + } +} From 2dae5dc9f49cb1221f57e255ec39f6f8d9e384c3 Mon Sep 17 00:00:00 2001 From: "Flemming N. Larsen" Date: Tue, 29 Sep 2026 23:13:55 +0200 Subject: [PATCH 3/5] Activate HARN-008 with registry evidence --- .clue/id-ledger.yaml | 1 + CHANGELOG.md | 1 + .../proposal.md | 27 ------------------- .../tasks.md | 18 ------------- compat-test/README.md | 6 ++--- .../AN-016-skipped-turn-telemetry-source.md | 4 +++ .../CAP-007-compatibility-harness/criteria.md | 7 +++-- .../CAP-007-compatibility-harness/design.md | 4 ++- docs/plans/P-001-bridge-parity-campaign.md | 2 +- 9 files changed, 16 insertions(+), 54 deletions(-) delete mode 100644 changes/CH-019-harn008-supported-evidence/proposal.md delete mode 100644 changes/CH-019-harn008-supported-evidence/tasks.md diff --git a/.clue/id-ledger.yaml b/.clue/id-ledger.yaml index 35c2b97..f76aa79 100644 --- a/.clue/id-ledger.yaml +++ b/.clue/id-ledger.yaml @@ -250,6 +250,7 @@ events: - {id: TASKS-012, kind: numeric, state: reserved, prefix: TASKS, component: "12"} - {id: CH-019, kind: numeric, state: live, prefix: CH, component: "19"} - {id: TASKS-012, kind: numeric, state: live, prefix: TASKS, component: "12"} + - {id: M-148, kind: numeric, state: retired, prefix: M, component: "148"} high-water: - {id: ADR-002, kind: numeric, state: reserved, prefix: ADR, component: "2"} - {id: AN-014, kind: numeric, state: reserved, prefix: AN, component: "14"} diff --git a/CHANGELOG.md b/CHANGELOG.md index dd6a31b..48349b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,3 +13,4 @@ - Preserved classic handler-exception reporting by surfacing legacy callback failures at the bridge boundary while retaining Tank Royale's event queue. - Added grouped team-robot staging and two-engine conformance evidence for team membership, teammate messages, directed-recipient isolation, droid no-scan behavior, and arbitrary serializable message payloads. - Restored classic team names, including battle-wide duplicate suffixes, through the Tank Royale name map for team identity, teammate lookup, directed messaging, and message-event senders when paired with the matching Tank Royale Bot API and runner. +- Added supported integration evidence that checks skipped-turn telemetry, including warm-up events and completed empty captures, in the persisted parity registry. diff --git a/changes/CH-019-harn008-supported-evidence/proposal.md b/changes/CH-019-harn008-supported-evidence/proposal.md deleted file mode 100644 index 6c38f1d..0000000 --- a/changes/CH-019-harn008-supported-evidence/proposal.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -id: CH-019 -type: change -status: open -links: [P-001, CAP-007, CRIT-007, AN-003, AN-016] -title: Add supported integration evidence for skipped-turn telemetry ---- - -# CH-019 — Add supported integration evidence for skipped-turn telemetry - -## Problem - -CH-018 added opt-in skipped-turn telemetry and local probe evidence, but HARN-008 remains draft because its current checked-in tests are Python unit tests and the local runs are not a supported acceptance carrier. The callback-to-observation path needs an attributable JVM integration test before the evidence door M-148 can close. - -## Change - -Extend the existing JVM conformance test bed to run the compatibility harness with skipped-turn capture enabled and assert the result it produces. Reuse its existing staging and runner path; do not create a second battle implementation or change the evidence policy. Prove exact `(bot ID, round, turn)` correspondence to callback output, preservation of early turns, completed empty capture, and distinct disabled, unavailable, and incomplete outcomes. Activate HARN-008 and close M-148 only if the supported tests cover its full scenario. - -## Plan and acceptance - -This change serves `P-001/M-006` through evidence door `M-148`. M-006 remains open; one telemetry criterion does not complete the parity campaign. - -## Challenge to the commitment - -The main assumption is that a JUnit integration test which launches the existing Python compatibility harness and makes its own assertions over the returned observation is an honest supported JVM evidence carrier for the end-to-end contract. A credible alternative is to keep HARN-008 draft until telemetry aggregation moves to a supported implementation or Python becomes an accepted carrier. The cheapest useful test is a local JUnit probe that invokes the existing `--conformance` path with capture enabled and compares its structured telemetry with the probe's callback markers. Revise or stop if that path cannot expose the observation that HARN-008 promises, if it requires copying the Python parsing logic into Java, or if the test cannot reliably cover the warm-up events and status distinctions. - -An implementation could pass by comparing callback markers to a transient JSON response while the compatibility checkpoint or registry still drops the telemetry, leaving the person using M-006 without durable evidence. The integration test must assert the same persisted observation path used by compatibility measurements, and the digest must state that this tier remains local and may skip where the two-engine environment is absent. diff --git a/changes/CH-019-harn008-supported-evidence/tasks.md b/changes/CH-019-harn008-supported-evidence/tasks.md deleted file mode 100644 index 7ec4751..0000000 --- a/changes/CH-019-harn008-supported-evidence/tasks.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -id: TASKS-012 -type: tasks -status: open -links: [CH-019] -title: Tasks for CH-019 ---- - -# Tasks - -- [ ] Add a JVM Integration path that invokes the compatibility harness with skipped-turn capture enabled and exposes its structured observation to JUnit. -- [ ] Add `HARN-008` Integration-positive evidence matching captured `(bot ID, round, turn)` tuples to callback output, including early and warm-up turns. -- [ ] Add `HARN-008` Integration-positive evidence for a completed capture with an empty event list, without making the result depend on a timing-sensitive no-skip battle. -- [ ] Add `HARN-008` Integration-negative evidence that disabled, unavailable, and incomplete capture remain distinct and never claim zero events. -- [ ] Verify the JUnit assertions cover the durable observation path used by compatibility measurements rather than only a transient conformance response. -- [ ] Activate `HARN-008` and mark `M-148` done only if every clause has supported positive and negative evidence; otherwise keep both states honest and record the exact gap. -- [ ] Update the capability design, telemetry analysis evidence, and `P-001` bookkeeping without changing M-006's open campaign scope. -- [ ] Run the supported local conformance and relevant unit checks, validate the digest, and publish the ready candidate under the full review workflow. diff --git a/compat-test/README.md b/compat-test/README.md index 66b2b6b..301e37f 100644 --- a/compat-test/README.md +++ b/compat-test/README.md @@ -48,9 +48,7 @@ the failure. > deferred same-priority events, e.g. every other scan event for robots that call a > blocking method such as `fire()` inside `onScannedRobot`. -All paths are defaults only — override with CLI flags (`--collection-dir`, -`--robocode-home`, `--runner-jar`, `--bridge-api-jar`, `--wrapper-jar`, `--bot-api-jar`) -or the corresponding `COMPAT_*` environment variables. +All paths are defaults only — override with CLI flags (`--collection-dir`, `--robocode-home`, `--runner-jar`, `--bridge-api-jar`, `--wrapper-jar`, `--bot-api-jar`) or the corresponding `COMPAT_*` environment variables. Set `COMPAT_DATA_DIR` to redirect the checkpoint, report, error logs, and parity registry to an isolated directory; set `COMPAT_WORK_DIR` to isolate temporary engine and bot files. ## Division setups @@ -208,6 +206,8 @@ Full error details land in `errors/robocode/.log` and Pass `--capture-skipped-turns` to enable opt-in records for Tank Royale bridge callbacks; capture is off by default. Each registry observation stores `tank_royale.skipped_turn_telemetry` with a status and, only when capture is complete, an `events` list of unique `{bot_id, round, turn}` values sorted by round and turn, including warm-up. `disabled`, `unavailable`, and `incomplete` use `events: null`; `captured` with `events: []` means the completed run delivered no skipped-turn callbacks to the bridge. Per-bot readiness and completion-count markers let the harness detect an older bridge jar or a truncated capture. The data describes callbacks delivered to the bridge and does not claim to record server detections that never reach it. +`--turn-timeout-micros` overrides Tank Royale's per-turn timeout for a local probe. A large value makes a completed no-event capture practical to verify without depending on a bot narrowly meeting the default turn deadline. + Use the conformance probe to check a forced-skip run locally: ```bash diff --git a/docs/analysis/AN-016-skipped-turn-telemetry-source.md b/docs/analysis/AN-016-skipped-turn-telemetry-source.md index 97c64f7..65abf06 100644 --- a/docs/analysis/AN-016-skipped-turn-telemetry-source.md +++ b/docs/analysis/AN-016-skipped-turn-telemetry-source.md @@ -37,6 +37,10 @@ After implementation, three one-round local Tank Royale runs of the existing `Sk A three-member team probe completed with two runner score groups and six bot-process logs. Counting scores as bots first produced `unavailable`; the harness now validates readiness against the expanded staged member directories and reported a completed capture with zero events. This confirms that team captures count bridge processes rather than runner score groups. +## Supported integration evidence + +On 2026-09-29, `SkippedTurnTelemetryConformanceTest` passed on the prepared Windows environment with classic Robocode 1.11.1 and Tank Royale runner/Bot API 1.4.0 from local revision `21324d6b7c4433e3fc1ba2a0e29c691fc3565e47`. Its forced-skip case ran the ordinary two-engine compatibility sweep for five rounds, compared every persisted registry tuple with the callbacks in that same run's two bot logs, and confirmed that round-one warm-up events survived into both the checkpoint and registry. A quiet fixture completed through the same path with a 10-second turn deadline and persisted `captured` with an empty event list. The integration suite also distinguished `disabled`, `unavailable`, and `incomplete` as `events: null`. This is prepared local conformance evidence; the tier skips when its external installations are absent and makes no cross-machine timing or performance claim. + The first end-to-end attempt also showed that `bot.getMyId()` is unavailable during `BotPeer` construction. The readiness marker now comes from `GameStartedEvent`, after the Bot API has assigned the ID; the forced-skip run passed after that correction. This source measures skipped-turn events delivered to the bridge. It does not claim to observe a server-detected event that never reaches the bridge. Disabled capture, an incomplete battle, and a completed capture with no events must remain distinguishable. Historical registry observations stay unmeasured. diff --git a/docs/capabilities/CAP-007-compatibility-harness/criteria.md b/docs/capabilities/CAP-007-compatibility-harness/criteria.md index 2876238..320714c 100644 --- a/docs/capabilities/CAP-007-compatibility-harness/criteria.md +++ b/docs/capabilities/CAP-007-compatibility-harness/criteria.md @@ -11,7 +11,7 @@ reversal-cost: low # CAP-007 — acceptance criteria -Every criterion is `@draft` against `M-001`. The first three describe behaviour that already works; they are draft because Python is not a supported evidence carrier, not because the behaviour is missing. `AN-003` records the distinction. +`HARN-001` through `HARN-007` remain `@draft` against `M-001` because their evidence is not yet supported; `AN-003` records why Python-only tests do not prove them. `HARN-008` is active with the JVM integration evidence required by `M-006`. ```gherkin Feature: The compatibility harness @@ -77,7 +77,7 @@ Feature: The compatibility harness Then the battlefield, round count, and participant count are that division's official values # The harness currently uses one setup for every division. Plan door: M-001. - @HARN-008 @draft + @HARN-008 Scenario: Skipped-turn telemetry preserves events and distinguishes missing capture from zero Test-type: Integration Given a completed Tank Royale compatibility run with telemetry enabled @@ -85,6 +85,5 @@ Feature: The compatibility harness Then the observation records one event per distinct bot ID, round, and skipped turn number, including warm-up turns And a completed enabled run with no events is captured with an empty event list And disabled, unavailable, or incomplete capture has a distinct status and does not claim zero events - # Callback-level telemetry is opt-in and does not claim to see server events that never reach the bridge. Plan door: M-006. - # The harness is Python and is not yet a supported evidence carrier. + # Callback-level telemetry is opt-in and does not claim to see server events that never reach the bridge. The JVM integration bed checks the normal parity sweep's persisted checkpoint and registry against the bot callback logs. Plan door: M-006. ``` diff --git a/docs/capabilities/CAP-007-compatibility-harness/design.md b/docs/capabilities/CAP-007-compatibility-harness/design.md index 5326ddb..316ae45 100644 --- a/docs/capabilities/CAP-007-compatibility-harness/design.md +++ b/docs/capabilities/CAP-007-compatibility-harness/design.md @@ -2,7 +2,7 @@ id: DES-007 type: design status: draft -links: [CAP-007, ARCH-003, C-003, C-004, C-007, AN-003] +links: [CAP-007, ARCH-003, C-003, C-004, C-007, AN-003, AN-016, CRIT-007] title: The compatibility harness — design provenance: inferred reversal-cost: low @@ -36,6 +36,8 @@ The harness imports an interrupted checkpoint with `--sync-registry`, runs new f Skipped-turn capture is opt-in with `--capture-skipped-turns`. For a completed Tank Royale run, `tank_royale.skipped_turn_telemetry` in each registry observation has `status: captured` and an `events` list containing distinct `{bot_id, round, turn}` records in round/turn order; warm-up turns are included. `disabled`, `unavailable`, and `incomplete` statuses carry `events: null`, so only a completed capture can report an empty list. The bridge buffers each bot's readiness and event markers until `GameEnded`, then writes a completion marker with its distinct-event count; the harness checks both markers against the registry events before reporting capture as complete. These records cover `SkippedTurnEvent` callbacks delivered to bridge bots and do not claim to observe server detections that never reach a bot. +The supported JVM integration test runs the ordinary two-engine compatibility sweep with isolated `COMPAT_DATA_DIR` and `COMPAT_WORK_DIR` paths. It compares the persisted checkpoint and append-only registry records with the same run's bot callback logs, verifies warm-up events, and records a completed empty capture using a probe run with a longer turn deadline. The test belongs to the local conformance tier and skips when its classic and Tank Royale installations are unavailable. + ## What changes for the milestone **Division setups.** One named constant per division carrying the official battlefield, rounds, and participant count, and one constant for the collection root — `C-003` and `C-007`. The single hard-coded setup goes away. diff --git a/docs/plans/P-001-bridge-parity-campaign.md b/docs/plans/P-001-bridge-parity-campaign.md index 839e918..f1f6ff8 100644 --- a/docs/plans/P-001-bridge-parity-campaign.md +++ b/docs/plans/P-001-bridge-parity-campaign.md @@ -83,7 +83,7 @@ They are bookkeeping rather than a second plan. A door closes when its criterion | M-141 | `EVT-013` | `EVT-013` is active, with evidence attributable to it. Successor to `EVT-003` (`M-103`). Work lands under M-001. | done | | M-142 | `EVT-014` | `EVT-014` is active, with evidence attributable to it. Successor to `EVT-007` (`M-107`). Work lands under M-001. | done | | M-143 | `EVT-015` | `EVT-015` is active, with evidence attributable to it. Successor to `EVT-001` (`M-101`). Work lands under M-001. | done | -| M-148 | `HARN-008` | `HARN-008` is active, with evidence attributable to it. Work lands under M-006. | todo | +| M-148 | `HARN-008` | `HARN-008` is active, with evidence attributable to it. Work lands under M-006. | done | ## Why this order From b88a1bd352143704af56d5835f301766a4785845 Mon Sep 17 00:00:00 2001 From: "Flemming N. Larsen" Date: Wed, 30 Sep 2026 20:48:45 +0200 Subject: [PATCH 4/5] Verify skipped-turn failure states persist --- compat-test/compat_test.py | 7 +++ .../conformance/ConformanceHarness.java | 10 ++++ .../conformance/ConformanceTestBase.java | 10 +++- .../SkippedTurnTelemetryConformanceTest.java | 46 ++++++++++++++++++- 4 files changed, 71 insertions(+), 2 deletions(-) diff --git a/compat-test/compat_test.py b/compat-test/compat_test.py index 28c41d9..31ccaa2 100644 --- a/compat-test/compat_test.py +++ b/compat-test/compat_test.py @@ -1182,6 +1182,13 @@ def run_tr_battle(jar_path: Path, classname, version, opts, setup, rc_signatures result["skipped_turn_telemetry"] = skipped_turn_telemetry( bot_dirs, capture_skipped_turns, result.get("completed", False), len(staged_log_dirs(bot_dirs))) + test_status = os.environ.get("COMPAT_TEST_SKIPPED_TURN_STATUS") + if test_status is not None: + if not capture_skipped_turns or test_status not in {"unavailable", "incomplete"}: + raise ValueError("invalid compatibility-test skipped-turn telemetry status") + # Let the JVM integration test send a synthetic status through the same checkpoint + # and registry write path after separately checking how the status is produced. + result["skipped_turn_telemetry"] = {"status": test_status, "events": None} return result diff --git a/conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/ConformanceHarness.java b/conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/ConformanceHarness.java index 7f645fe..93d6057 100644 --- a/conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/ConformanceHarness.java +++ b/conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/ConformanceHarness.java @@ -130,6 +130,13 @@ BattleOutcome run(Engine engine, String robotClass, Path source, boolean capture String runCompatibilityMeasurement(Path collectionDir, Path dataDir, Path workDir, String selectedRobot, boolean captureSkippedTurns, Integer turnTimeoutMicros) { + return runCompatibilityMeasurement(collectionDir, dataDir, workDir, selectedRobot, + captureSkippedTurns, turnTimeoutMicros, null); + } + + String runCompatibilityMeasurement(Path collectionDir, Path dataDir, Path workDir, + String selectedRobot, boolean captureSkippedTurns, + Integer turnTimeoutMicros, String testTelemetryStatus) { List command = new ArrayList<>(List.of( python, HARNESS.toString(), @@ -155,6 +162,9 @@ String runCompatibilityMeasurement(Path collectionDir, Path dataDir, Path workDi .redirectErrorStream(true); builder.environment().put("COMPAT_WORK_DIR", workDir.toString()); builder.environment().put("COMPAT_DATA_DIR", dataDir.toString()); + if (testTelemetryStatus != null) { + builder.environment().put("COMPAT_TEST_SKIPPED_TURN_STATUS", testTelemetryStatus); + } Process process = builder.start(); AtomicReference output = new AtomicReference<>(""); diff --git a/conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/ConformanceTestBase.java b/conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/ConformanceTestBase.java index 8c7e1f4..f182d06 100644 --- a/conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/ConformanceTestBase.java +++ b/conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/ConformanceTestBase.java @@ -147,8 +147,16 @@ BattleOutcome telemetryOutcome(Engine engine, String robotClass, Path source, String runRecordedMeasurement(Path collectionDir, Path dataDir, Path workDir, String selectedRobot, boolean captureSkippedTurns, Integer turnTimeoutMicros) { + return runRecordedMeasurement(collectionDir, dataDir, workDir, selectedRobot, + captureSkippedTurns, turnTimeoutMicros, null); + } + + /** Injects a non-captured test status after the ordinary runner result and before persistence. */ + String runRecordedMeasurement(Path collectionDir, Path dataDir, Path workDir, + String selectedRobot, boolean captureSkippedTurns, + Integer turnTimeoutMicros, String testTelemetryStatus) { return harness.runCompatibilityMeasurement(collectionDir, dataDir, workDir, - selectedRobot, captureSkippedTurns, turnTimeoutMicros); + selectedRobot, captureSkippedTurns, turnTimeoutMicros, testTelemetryStatus); } private BattleOutcome outcomeForFixture(Engine engine, String robotClass, Path source, diff --git a/conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/SkippedTurnTelemetryConformanceTest.java b/conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/SkippedTurnTelemetryConformanceTest.java index 96ad4c7..a6c829e 100644 --- a/conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/SkippedTurnTelemetryConformanceTest.java +++ b/conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/SkippedTurnTelemetryConformanceTest.java @@ -90,6 +90,43 @@ void testHARN008_IntegrationNegative_DistinguishesMissingCaptureStates() { assertTelemetryState(incomplete, "incomplete"); } + @Test + @DisplayName("HARN-008 negative: non-captured states survive checkpoint and registry persistence") + void testHARN008_IntegrationNegative_PersistsNonCapturedStates( + @TempDir Path temporaryDirectory) throws IOException { + Path collectionDir = temporaryDirectory.resolve("collection"); + Path fixtureJar = prepareFixtureJar(QUIET_ROBOT, QUIET_SOURCE, collectionDir); + + assertPersistedNonCapturedState(collectionDir, temporaryDirectory, fixtureJar, + "disabled", false, null); + assertPersistedNonCapturedState(collectionDir, temporaryDirectory, fixtureJar, + "unavailable", true, "unavailable"); + assertPersistedNonCapturedState(collectionDir, temporaryDirectory, fixtureJar, + "incomplete", true, "incomplete"); + } + + private void assertPersistedNonCapturedState(Path collectionDir, Path temporaryDirectory, + Path fixtureJar, String status, + boolean captureSkippedTurns, + String testTelemetryStatus) throws IOException { + Path dataDir = temporaryDirectory.resolve(status + "-data"); + Path workDir = temporaryDirectory.resolve(status + "-work"); + assertNull(runCompatibilityMeasurement(collectionDir, dataDir, workDir, + "NoSkippedTurnProbe", captureSkippedTurns, 10_000_000, testTelemetryStatus)); + + String registryTelemetry = persistedTelemetry(dataDir, fixtureJar); + String checkpoint = Files.readString(dataDir.resolve("test_progress.json")); + String checkpointTelemetry = Json.object(checkpoint, "skipped_turn_telemetry"); + assertEquals(status, Json.scalar(checkpointTelemetry, "status")); + assertTrue(NULL_EVENTS.matcher(checkpointTelemetry).find(), + status + " checkpoint capture must not claim an empty event list: " + + checkpointTelemetry); + assertEquals(status, Json.scalar(registryTelemetry, "status")); + assertTrue(NULL_EVENTS.matcher(registryTelemetry).find(), + status + " registry capture must not claim an empty event list: " + + registryTelemetry); + } + private static void assertTelemetryState(BattleOutcome outcome, String status) { String telemetry = skippedTurnTelemetry(outcome); assertEquals(status, Json.scalar(telemetry, "status"), outcome.summary()); @@ -121,6 +158,13 @@ private Path prepareFixtureJar(String robotClass, Path source, Path collectionDi private String runCompatibilityMeasurement(Path collectionDir, Path dataDir, Path workDir, String selectedRobot, boolean captureSkippedTurns, Integer turnTimeoutMicros) { + return runCompatibilityMeasurement(collectionDir, dataDir, workDir, selectedRobot, + captureSkippedTurns, turnTimeoutMicros, null); + } + + private String runCompatibilityMeasurement(Path collectionDir, Path dataDir, Path workDir, + String selectedRobot, boolean captureSkippedTurns, + Integer turnTimeoutMicros, String testTelemetryStatus) { try { Files.createDirectories(dataDir); Files.createDirectories(workDir); @@ -128,7 +172,7 @@ private String runCompatibilityMeasurement(Path collectionDir, Path dataDir, Pat return "could not create isolated measurement directories: " + e.getMessage(); } return runRecordedMeasurement(collectionDir, dataDir, workDir, selectedRobot, - captureSkippedTurns, turnTimeoutMicros); + captureSkippedTurns, turnTimeoutMicros, testTelemetryStatus); } private static String persistedTelemetry(Path dataDir, Path jar) throws IOException { From f4b9e915a361c060f6f354a6f654e3871eb779c2 Mon Sep 17 00:00:00 2001 From: "Flemming N. Larsen" Date: Wed, 30 Sep 2026 20:58:07 +0200 Subject: [PATCH 5/5] Scope telemetry test injection to CLI --- compat-test/compat_test.py | 6 ++++-- .../tankroyale/bridge/conformance/ConformanceHarness.java | 7 ++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/compat-test/compat_test.py b/compat-test/compat_test.py index 31ccaa2..636abf6 100644 --- a/compat-test/compat_test.py +++ b/compat-test/compat_test.py @@ -1182,9 +1182,9 @@ def run_tr_battle(jar_path: Path, classname, version, opts, setup, rc_signatures result["skipped_turn_telemetry"] = skipped_turn_telemetry( bot_dirs, capture_skipped_turns, result.get("completed", False), len(staged_log_dirs(bot_dirs))) - test_status = os.environ.get("COMPAT_TEST_SKIPPED_TURN_STATUS") + test_status = opts.test_skipped_turn_status if test_status is not None: - if not capture_skipped_turns or test_status not in {"unavailable", "incomplete"}: + if not capture_skipped_turns: raise ValueError("invalid compatibility-test skipped-turn telemetry status") # Let the JVM integration test send a synthetic status through the same checkpoint # and registry write path after separately checking how the status is produced. @@ -1473,6 +1473,8 @@ def parse_args(): help="record Tank Royale bridge skipped-turn events, including warm-up turns") p.add_argument("--turn-timeout-micros", type=int, default=None, help="override Tank Royale's turn timeout in microseconds for local probes") + p.add_argument("--test-skipped-turn-status", choices=("unavailable", "incomplete"), + default=None, help=argparse.SUPPRESS) gate = p.add_argument_group("regression gate (C-004)") gate.add_argument("--regression", action="store_true", diff --git a/conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/ConformanceHarness.java b/conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/ConformanceHarness.java index 93d6057..a9336ba 100644 --- a/conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/ConformanceHarness.java +++ b/conformance-test/src/test/java/dev/robocode/tankroyale/bridge/conformance/ConformanceHarness.java @@ -155,6 +155,10 @@ String runCompatibilityMeasurement(Path collectionDir, Path dataDir, Path workDi command.add("--turn-timeout-micros"); command.add(String.valueOf(turnTimeoutMicros)); } + if (testTelemetryStatus != null) { + command.add("--test-skipped-turn-status"); + command.add(testTelemetryStatus); + } try { ProcessBuilder builder = new ProcessBuilder(command) @@ -162,9 +166,6 @@ String runCompatibilityMeasurement(Path collectionDir, Path dataDir, Path workDi .redirectErrorStream(true); builder.environment().put("COMPAT_WORK_DIR", workDir.toString()); builder.environment().put("COMPAT_DATA_DIR", dataDir.toString()); - if (testTelemetryStatus != null) { - builder.environment().put("COMPAT_TEST_SKIPPED_TURN_STATUS", testTelemetryStatus); - } Process process = builder.start(); AtomicReference output = new AtomicReference<>("");