Add RLE to protobuf snapshots

src/Debug/debug.proto

@@ -264,6 +264,13 @@ message Checkpoint {
 
 message TableState { uint32 initial = 1; uint32 maximum = 2; repeated uint32 entries = 3; }
 
-message MemoryState { uint32 initial = 1; uint32 maximum = 2; uint32 pages = 3; bytes bytes = 4; }
+message MemoryState {
+  uint32 initial = 1;
+  uint32 maximum = 2;
+  uint32 pages = 3;
+  // RLE records: [unsigned LEB128 run length][repeated byte value]. Zero-page
+  // memory is represented by an empty byte string.
+  bytes bytes = 4;
+}
 
 message IOState { string key = 1; bool output = 2; sint32 value = 3; }

src/Debug/debugger-snapshot.cpp

@@ -1,3 +1,5 @@
+#include <limits>
+
 #include "debugger-decode.h"
 #include "debugger-encode.h"
 
@@ -139,7 +141,7 @@ class SnapshotEncodingContext {
                     state->memory.maximum = module->memory.maximum;
                     state->memory.pages = module->memory.pages;
                     state->memory.bytes.funcs.encode =
-                        nanopb_encoder::encode_bytes;
+                        nanopb_encoder::encode_rle_bytes;
                     state->memory.bytes.arg = &memory;
                     break;
                 case debug_SnapshotSection_SNAPSHOT_SECTION_BRANCH_TABLE:
@@ -323,7 +325,9 @@ bool collect_snapshot_override(pb_istream_t *stream, const pb_field_iter_t *,
 
 bool decode_snapshot(const std::vector<uint8_t> &payload,
                      DecodedSnapshot *out) {
+    // decode full snapshot
     debug_Snapshot snapshot = debug_Snapshot_init_zero;
+    std::vector<uint8_t> compressedMemoryBytes;
     snapshot.breakpoints.funcs.decode = collect_varints;
     snapshot.breakpoints.arg = &out->breakpoints;
     snapshot.functions.funcs.decode = collect_snapshot_function;
@@ -338,7 +342,7 @@ bool decode_snapshot(const std::vector<uint8_t> &payload,
     snapshot.locals.values.arg = &out->locals;
     snapshot.table.entries.funcs.decode = collect_varints;
     snapshot.table.entries.arg = &out->tableEntries;
-    set_decode_callback(&snapshot.memory.bytes, &out->memoryBytes);
+    set_decode_callback(&snapshot.memory.bytes, &compressedMemoryBytes);
     snapshot.branch_table.funcs.decode = collect_varints;
     snapshot.branch_table.arg = &out->branchTable;
     snapshot.callbacks.entries.funcs.decode = collect_callback_entries;
@@ -353,8 +357,7 @@ bool decode_snapshot(const std::vector<uint8_t> &payload,
     if (!decode_payload(payload, debug_Snapshot_fields, &snapshot))
         return false;
 
-    // Nested messages have presence bits. Together with the exact-size checks
-    // below, these reject selected/partial snapshots.
+    // check for missing fields
     if (!snapshot.has_locals || !snapshot.has_queue ||
         !snapshot.has_callbacks || !snapshot.has_table ||
         !snapshot.has_memory || !snapshot.queue.has_range ||
@@ -363,6 +366,15 @@ bool decode_snapshot(const std::vector<uint8_t> &payload,
         snapshot.queue.total_count != out->events.size())
         return false;
 
+    // check for possible memory out-of-bounds
+    if (snapshot.memory.pages > std::numeric_limits<size_t>::max() / PAGE_SIZE)
+        return false;
+    if (!decode_rle_exact(
+            compressedMemoryBytes.data(), compressedMemoryBytes.size(),
+            static_cast<size_t>(snapshot.memory.pages) * PAGE_SIZE,
+            &out->memoryBytes))
+        return false;
+
     out->programCounter = snapshot.program_counter;
     out->state = snapshot.state;
     out->tableInitial = snapshot.table.initial;

src/Debug/nanopb_encoder.cpp

@@ -9,6 +9,35 @@ bool encode_bytes(pb_ostream_t *stream, const pb_field_t *field,
            pb_encode_string(stream, view->data, view->size);
 }
 
+namespace {
+
+bool encode_rle_runs(pb_ostream_t *stream, const ByteView &view) {
+    for (size_t position = 0; position < view.size;) {
+        const uint8_t value = view.data[position];
+        size_t count = 1;
+        while (++position < view.size && view.data[position] == value) ++count;
+
+        if (!pb_encode_varint(stream, count) || !pb_write(stream, &value, 1))
+            return false;
+    }
+    return true;
+}
+
+}  // namespace
+
+bool encode_rle_bytes(pb_ostream_t *stream, const pb_field_t *field,
+                      void *const *arg) {
+    const auto *view = static_cast<const ByteView *>(*arg);
+    if (view == nullptr || (view->data == nullptr && view->size != 0))
+        return false;
+
+    pb_ostream_t sizing = PB_OSTREAM_SIZING;
+    return encode_rle_runs(&sizing, *view) &&
+           pb_encode_tag_for_field(stream, field) &&
+           pb_encode_varint(stream, sizing.bytes_written) &&
+           encode_rle_runs(stream, *view);
+}
+
 bool encode_varints(pb_ostream_t *stream, const pb_field_t *field,
                     void *const *arg) {
     const auto *view = static_cast<const Uint32View *>(*arg);

src/Debug/nanopb_encoder.h

@@ -16,6 +16,8 @@ struct Uint32View {
 
 bool encode_bytes(pb_ostream_t *stream, const pb_field_t *field,
                   void *const *arg);
+bool encode_rle_bytes(pb_ostream_t *stream, const pb_field_t *field,
+                      void *const *arg);
 bool encode_varints(pb_ostream_t *stream, const pb_field_t *field,
                     void *const *arg);
 bool encode_fixed32s(pb_ostream_t *stream, const pb_field_t *field,

src/Utils/util.cpp

@@ -2,6 +2,8 @@
 
 #include <cmath>
 #include <cstring>
+#include <limits>
+#include <utility>
 
 #include "macros.h"
 
@@ -80,6 +82,47 @@ uint64_t read_LEB_signed(uint8_t **pos, uint32_t maxbits) {
     return read_LEB_(pos, maxbits, true);
 }
 
+bool decode_rle_exact(const uint8_t *encoded, const size_t encodedSize,
+                      const size_t expectedSize,
+                      std::vector<uint8_t> *decoded) {
+    if (decoded == nullptr || (encoded == nullptr && encodedSize != 0))
+        return false;
+    if (encodedSize == 0) {
+        if (expectedSize != 0) return false;
+        decoded->clear();
+        return true;
+    }
+
+    std::vector<uint8_t> result(expectedSize);
+    const uint8_t *position = encoded;
+    const uint8_t *const end = encoded + encodedSize;
+    size_t offset = 0;
+
+    while (position < end) {
+        size_t count = 0;
+        unsigned int shift = 0;
+        uint8_t byte;
+        do {
+            if (position == end || shift >= std::numeric_limits<size_t>::digits)
+                return false;
+            byte = *position++;
+            const size_t payload = byte & 0x7fU;
+            if (payload > (SIZE_MAX >> shift)) return false;
+            count |= payload << shift;
+            shift += 7;
+        } while ((byte & 0x80U) != 0);
+
+        if (count == 0 || position == end || count > expectedSize - offset)
+            return false;
+        memset(result.data() + offset, *position++, count);
+        offset += count;
+    }
+
+    if (offset != expectedSize) return false;
+    *decoded = std::move(result);
+    return true;
+}
+
 uint32_t read_uint32(uint8_t **pos) {
     *pos += 4;
     return ((uint32_t *)(*pos - 4))[0];

src/Utils/util.h

@@ -2,9 +2,11 @@
 #define UTIL_H
 
 #include <climits>
+#include <cstddef>
 #include <cstdint>
 #include <cstdlib>
 #include <string>
+#include <vector>
 
 #include "../WARDuino.h"
 
@@ -45,6 +47,10 @@ uint64_t read_LEB(uint8_t **pos, uint32_t maxbits);
  */
 uint64_t read_LEB_signed(uint8_t **pos, uint32_t maxbits);
 
+/** Decode count-first unsigned-LEB128 RLE to exactly expectedSize bytes. */
+bool decode_rle_exact(const uint8_t *encoded, size_t encodedSize,
+                      size_t expectedSize, std::vector<uint8_t> *decoded);
+
 uint32_t read_uint32(uint8_t **pos);
 
 /**

tests/unit/parsing.cpp

@@ -2,6 +2,7 @@
 #include <doctest/doctest.h>
 
 #include <cmath>
+#include <vector>
 
 #include "../../src/Utils/util.h"
 
@@ -43,6 +44,82 @@ TEST_CASE("Test: leb128 signed encoding") {
     }
 }
 
+namespace {
+
+std::vector<uint8_t> encode_rle_for_test(const std::vector<uint8_t> &input) {
+    std::vector<uint8_t> encoded;
+    for (size_t position = 0; position < input.size();) {
+        const uint8_t value = input[position];
+        size_t count = 1;
+        while (++position < input.size() && input[position] == value) ++count;
+
+        do {
+            uint8_t byte = count & 0x7fU;
+            count >>= 7U;
+            if (count != 0) byte |= 0x80U;
+            encoded.push_back(byte);
+        } while (count != 0);
+        encoded.push_back(value);
+    }
+    return encoded;
+}
+
+}  // namespace
+
+TEST_CASE("Test: byte RLE run formation and exact decoding") {
+    SUBCASE("Empty memory has an empty representation") {
+        std::vector<uint8_t> decoded{1, 2, 3};
+        CHECK(decode_rle_exact(nullptr, 0, 0, &decoded));
+        CHECK(decoded.empty());
+    }
+
+    SUBCASE("A zero page is one legacy-compatible run") {
+        std::vector<uint8_t> page(PAGE_SIZE, 0);
+        const std::vector<uint8_t> encoded = encode_rle_for_test(page);
+        CHECK(encoded == std::vector<uint8_t>{0x80, 0x80, 0x04, 0x00});
+
+        std::vector<uint8_t> decoded;
+        CHECK(decode_rle_exact(encoded.data(), encoded.size(), page.size(),
+                               &decoded));
+        CHECK(decoded == page);
+    }
+
+    SUBCASE("Mixed and alternating runs retain their order") {
+        const std::vector<uint8_t> mixed{4, 4, 4, 9, 9, 2, 7, 7, 7, 7};
+        const std::vector<uint8_t> alternating{0, 1, 0, 1, 0, 1};
+        for (const auto *input : {&mixed, &alternating}) {
+            const std::vector<uint8_t> encoded = encode_rle_for_test(*input);
+            std::vector<uint8_t> decoded;
+            CHECK(decode_rle_exact(encoded.data(), encoded.size(),
+                                   input->size(), &decoded));
+            CHECK(decoded == *input);
+        }
+        CHECK(encode_rle_for_test(alternating).size() ==
+              alternating.size() * 2);
+    }
+}
+
+TEST_CASE("Test: malformed byte RLE is rejected atomically") {
+    const std::vector<uint8_t> original{8, 8};
+    std::vector<uint8_t> decoded = original;
+
+    const std::vector<std::vector<uint8_t>> malformed{
+        {0x00, 0x2a},
+        {0x80},
+        {0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x00},
+        {0x01},
+        {0x03, 0x2a},
+        {0x01, 0x2a},
+    };
+
+    for (const auto &encoded : malformed) {
+        CAPTURE(encoded);
+        CHECK_FALSE(
+            decode_rle_exact(encoded.data(), encoded.size(), 2, &decoded));
+        CHECK(decoded == original);
+    }
+}
+
 TEST_CASE("Test: wasm args parsing") {
     uint32_t params[] = {F32, F32};
     uint32_t results[] = {I32};