From 67259bfa968867da556c6437ee804e7dc2a86d3a Mon Sep 17 00:00:00 2001 From: Sunny Luo Date: Sat, 12 Sep 2026 14:20:23 +0800 Subject: [PATCH 1/4] fix(hermes-base): audit lossless operands and bound verification processes Preserve quoted whitespace and branch targets. Add a second raw operand audit backed by binary HBC strings, function identities, literal buffers, switch tables and exact double bits. Fail closed on unsupported or incomplete verification instead of treating two decoding failures as equal. Bound compiler probes, compilation and both disassembly passes; reap terminated processes and handle debug stream errors. Attach speculative sourcemap rejection handlers immediately so rejected base output cannot interrupt the plain fallback. Add negative bytecode-mutation regressions, real v96/v98 compiler CI and updated documentation covering verification scope and resource costs. Validated locally: 532 tests with HBC v96 and 532 with HBC v98, lint, typecheck, package build and Node smoke checks. --- .github/workflows/test.yml | 56 +++ README.md | 2 +- README.zh-CN.md | 2 +- docs/hermes-base-verification.md | 43 ++- src/bundle-runner.ts | 31 +- src/utils/hermes-base.ts | 192 ++++++++-- src/utils/hermes-literals.ts | 20 +- src/utils/hermes-raw.ts | 603 +++++++++++++++++++++++++++++++ src/utils/hermes-timeout.ts | 7 + tests/hermes-base-safety.test.ts | 33 ++ tests/hermes-base.test.ts | 17 +- tests/hermes-compile.test.ts | 112 ++++++ tests/hermes-literals.test.ts | 22 +- tests/hermes-raw.test.ts | 255 +++++++++++++ tests/hermes-timeout.test.ts | 102 ++++++ 15 files changed, 1419 insertions(+), 78 deletions(-) create mode 100644 src/utils/hermes-raw.ts create mode 100644 src/utils/hermes-timeout.ts create mode 100644 tests/hermes-base-safety.test.ts create mode 100644 tests/hermes-raw.test.ts create mode 100644 tests/hermes-timeout.test.ts diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 86b7149..5cf9b82 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -134,3 +134,59 @@ jobs: env: NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: npm publish --dry-run --access public --tag dry-run + + hermes-integration: + name: hermes-hbc-${{ matrix.hbc }} + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - hbc: 96 + package: react-native@0.77.3 + directory: react-native + executable: sdks/hermesc/linux64-bin/hermesc + - hbc: 98 + package: hermes-compiler@250829098.0.16 + directory: hermes-compiler + executable: hermesc/linux64-bin/hermesc + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@v2 + - uses: actions/setup-node@v7 + with: + node-version: '24.x' + - run: bun install --frozen-lockfile + - name: Install the pinned real compiler + shell: bash + env: + COMPILER_PACKAGE: ${{ matrix.package }} + COMPILER_DIRECTORY: ${{ matrix.directory }} + COMPILER_EXECUTABLE: ${{ matrix.executable }} + EXPECTED_HBC: ${{ matrix.hbc }} + run: | + set -euo pipefail + root="$RUNNER_TEMP/hermes-tests/$COMPILER_DIRECTORY" + mkdir -p "$root" + archive=$(npm pack "$COMPILER_PACKAGE" --pack-destination "$RUNNER_TEMP" --silent) + tar -xzf "$RUNNER_TEMP/$archive" --strip-components=1 -C "$root" \ + "package/$COMPILER_EXECUTABLE" package/package.json + export HERMESC="$root/$COMPILER_EXECUTABLE" + test -x "$HERMESC" + "$HERMESC" -version + bun -e 'import {probeHbcVersion} from "./src/utils/hermes-base"; if (probeHbcVersion(process.env.HERMESC) !== Number(process.env.EXPECTED_HBC)) throw new Error("unexpected HBC version");' + echo "HERMESC=$HERMESC" >> "$GITHUB_ENV" + - name: Run real compiler and fallback regressions + run: bun test tests/hermes-*.test.ts + - name: Run seeded differential fuzzing + run: bun run fuzz:hermes-base --rounds 50 --seed ${{ matrix.hbc }} --out "${{ runner.temp }}/hermes-fuzz" + - name: Preserve failing fuzz cases + if: failure() + uses: actions/upload-artifact@v7 + with: + name: hermes-fuzz-hbc-${{ matrix.hbc }} + path: ${{ runner.temp }}/hermes-fuzz + if-no-files-found: ignore diff --git a/README.md b/README.md index 44392d5..99a1d09 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ const publishResult = await provider.publish({ Hermes projects: `bundle` always runs hermesc with `-output-source-map`, so the debug info section is stripped from the bytecode (15–40% smaller, same as React Native's own release builds). The Hermes sourcemap stays in the intermediate directory (`.pushy/intermedia//.map`, never packed into the ppk) and is composed with the packager map — `--sourcemap` is on by default since 2.23 (`--no-sourcemap` opts out). When `bundle` publishes, that final map is uploaded and archived with the version (`sourceMapKey`), so `pushy symbolicate` can map crash stacks — including Hermes `address at` frames — back to source later. `pushy publish --sourcemap ` archives a map for a ppk built elsewhere; publishing without a map prints a warning. -Hermes delta mode (`-base-bytecode`): by default (`--hermesBase auto`) `bundle` compiles against the previous HBC of the same app, which keeps Hermes string IDs stable and makes hot-update patches 5–30× smaller. The base comes from the server (`GET /app/:id/hermesBase`), verified by sha256 and kept in a local cache (`.pushy/cache/`, 500 MB / 20 files, `PUSHY_CACHE_DIR` / `--cacheMaxMb` to tune, `pushy cache [clean]` to inspect or clear). `--hermesBase none` disables it; `--hermesBase ` uses a local artifact (for example the store build). `--verifyHermesBase` (default on) additionally compiles without the base (concurrently with the base compile) and compares both disassemblies — literal buffers by content, then function by function, with only representation differences (string ids, operand widths, jump distances, buffer offsets) folded away; on any mismatch or failure the CLI falls back to the plain compile, so the feature can never block a release. The log names the first difference (function, line, both sides) or, separately, a dump that could not be read; set `PUSHY_HERMES_BASE_DEBUG=1` to keep both disassemblies (`hermes-base-dump-base.txt` / `hermes-base-dump-plain.txt` next to the intermediate directory) for a bug report. What the check covers, what it does not yet, and how to triage a rejection: [docs/hermes-base-verification.md](docs/hermes-base-verification.md). The result is reported at publish (`hermesBaseOutcome`: `used` / `rejected` / `dump-failed` / `none`, plus the first difference) so the server can watch the rejection rate across apps. `HERMESC= bun run fuzz:hermes-base --rounds 300` compiles random programs against random bases and reports any build the check would wrongly reject (differential fuzzing of the normalization rules). Only hermesc builds that include the upstream delta-mode fix are used (classic `react-native/sdks/hermesc`, or `hermes-compiler` ≥ 250829098). If a base compile fails, the full hermesc output is written to `hermes-base-error.log` next to the intermediate directory. `--resetCache false` skips Metro's `--reset-cache` and reuses its transform cache, which makes repeated bundles much faster. +Hermes delta mode (`-base-bytecode`): by default (`--hermesBase auto`) `bundle` compiles against the previous HBC of the same app, which keeps Hermes string IDs stable and makes hot-update patches 5–30× smaller. The base comes from the server (`GET /app/:id/hermesBase`), verified by sha256 and kept in a local cache (`.pushy/cache/`, 500 MB / 20 files, `PUSHY_CACHE_DIR` / `--cacheMaxMb` to tune, `pushy cache [clean]` to inspect or clear). `--hermesBase none` disables it; `--hermesBase ` uses a local artifact (for example the store build). `--verifyHermesBase` (default on) additionally compiles without the base (concurrently with the base compile) and checks both artifacts in two passes: readable disassembly followed by raw operands resolved against complete binary strings, constants, function references and control-flow targets. Quoted whitespace and branch destinations are preserved; unsupported or unreadable layouts fail closed. On a mismatch, verification failure or compiler timeout the CLI falls back to the plain compile; failures of the plain compiler or the final sourcemap still fail the build. The raw pass adds verification time and holds both HBC files in memory, but does not add another compile. The log names the first difference (function, line, both sides) or, separately, a dump that could not be read; set `PUSHY_HERMES_BASE_DEBUG=1` to keep both disassemblies (`hermes-base-dump-base.txt` / `hermes-base-dump-plain.txt` next to the intermediate directory) for a bug report. Probe, verification and compile/sourcemap process deadlines default to 30/120/300 seconds respectively, configurable in milliseconds via `PUSHY_HERMES_PROBE_TIMEOUT_MS`, `PUSHY_HERMES_VERIFY_TIMEOUT_MS` and `PUSHY_HERMES_COMPILE_TIMEOUT_MS`. What the check covers, what it does not yet, and how to triage a rejection: [docs/hermes-base-verification.md](docs/hermes-base-verification.md). The result is reported at publish (`hermesBaseOutcome`: `used` / `rejected` / `dump-failed` / `none`, plus the first difference) so the server can watch the rejection rate across apps. `HERMESC= bun run fuzz:hermes-base --rounds 300` compiles random programs against random bases and reports any build the check would wrongly reject (differential fuzzing of the normalization rules). Only hermesc builds that include the upstream delta-mode fix are used (classic `react-native/sdks/hermesc`, or `hermes-compiler` ≥ 250829098). If a base compile fails, the full hermesc output is written to `hermes-base-error.log` next to the intermediate directory. `--resetCache false` skips Metro's `--reset-cache` and reuses its transform cache, which makes repeated bundles much faster. ### Version diff --git a/README.zh-CN.md b/README.zh-CN.md index 14411b2..a2d5630 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -78,7 +78,7 @@ const publishResult = await provider.publish({ Hermes 工程：`bundle` 调用 hermesc 时始终带 `-output-source-map`，因此字节码不含 debug info 段（小 15%～40%，与 React Native 自身 release 构建一致）。Hermes sourcemap 保留在中间目录（`.pushy/intermedia//.map`，不会打进 ppk），并与 packager map 合成——自 2.23 起 `--sourcemap` 默认开启（`--no-sourcemap` 关闭）。`bundle` 发布时会把这份最终 map 上传并随版本归档（`sourceMapKey`），之后用 `pushy symbolicate` 即可把崩溃堆栈（含 Hermes 的 `address at` 帧）还原到源码。别处打好的 ppk 可用 `pushy publish --sourcemap ` 归档；不带 map 发布会打印警告。 -Hermes delta 模式（`-base-bytecode`）：默认 `--hermesBase auto`，`bundle` 会以同一应用上一版的 HBC 为 base 编译，让 Hermes 字符串 ID 跨版本稳定，热更 patch 可缩小 5～30 倍。base 由服务端（`GET /app/:id/hermesBase`）给出、按 sha256 校验并存入本地缓存（`.pushy/cache/`，默认 500 MB / 20 个，可用 `PUSHY_CACHE_DIR` / `--cacheMaxMb` 调整，`pushy cache [clean]` 查看或清空）。`--hermesBase none` 关闭；`--hermesBase ` 指定本地文件（比如商店包）作 base。`--verifyHermesBase`（默认开）会并行再做一次普通编译并比对两份反汇编——先按内容比对字面量缓冲区，再逐函数比对，只折叠纯表示层差异（字符串 id、操作数宽度、跳转距离、缓冲区偏移）；任何不一致或失败都回退到普通编译，不会阻塞发版。日志会给出第一处差异（函数、行、两侧内容），dump 读取失败会单独说明；设置 `PUSHY_HERMES_BASE_DEBUG=1` 可把两份反汇编保留在中间目录旁（`hermes-base-dump-base.txt` / `hermes-base-dump-plain.txt`）用于提 issue。校验覆盖什么、还缺什么、如何排查一次拒绝：见 [docs/hermes-base-verification.md](docs/hermes-base-verification.md)。校验结果会随发布上报（`hermesBaseOutcome`：`used` / `rejected` / `dump-failed` / `none`，附第一处差异），服务端可据此观察全体应用的拒绝率。`HERMESC= bun run fuzz:hermes-base --rounds 300` 会用随机程序配随机 base 编译并报告校验误杀的构建（对归一化规则做差分模糊测试）。只有包含上游 delta 模式修复的 hermesc 才会启用（经典 `react-native/sdks/hermesc`，或 `hermes-compiler` ≥ 250829098）。base 编译失败时，完整的 hermesc 输出会写到中间目录旁边的 `hermes-base-error.log`。`--resetCache false` 可跳过 Metro 的 `--reset-cache`，复用其转换缓存，重复打包会快很多。 +Hermes delta 模式（`-base-bytecode`）：默认 `--hermesBase auto`，`bundle` 会以同一应用上一版的 HBC 为 base 编译，让 Hermes 字符串 ID 跨版本稳定，热更 patch 可缩小 5～30 倍。base 由服务端（`GET /app/:id/hermesBase`）给出、按 sha256 校验并存入本地缓存（`.pushy/cache/`，默认 500 MB / 20 个，可用 `PUSHY_CACHE_DIR` / `--cacheMaxMb` 调整，`pushy cache [clean]` 查看或清空）。`--hermesBase none` 关闭；`--hermesBase ` 指定本地文件（比如商店包）作 base。`--verifyHermesBase`（默认开）会并行再做一次普通编译并进行两遍校验：先比较易读反汇编，再将 raw 操作数与二进制中的完整字符串、常量、函数引用和控制流目标核对。字符串内部空白和分支目的地不会被抹掉；不支持或无法解析的布局不能判为等价。发现差异、无法完成校验或 base 编译超时会回退到普通编译；真正的 plain 编译或最终 sourcemap 失败仍会使构建失败。raw 核对增加验证时间并在内存中持有两份 HBC，但不增加编译次数。日志会给出第一处差异（函数、行、两侧内容），dump 读取失败会单独说明；设置 `PUSHY_HERMES_BASE_DEBUG=1` 可把两份反汇编保留在中间目录旁（`hermes-base-dump-base.txt` / `hermes-base-dump-plain.txt`）用于提 issue。版本探测、完整验证、单个编译/sourcemap 子进程的期限默认分别为 30/120/300 秒，可用 `PUSHY_HERMES_PROBE_TIMEOUT_MS`、`PUSHY_HERMES_VERIFY_TIMEOUT_MS`、`PUSHY_HERMES_COMPILE_TIMEOUT_MS`（毫秒）调整。校验覆盖什么、还缺什么、如何排查一次拒绝：见 [docs/hermes-base-verification.md](docs/hermes-base-verification.md)。校验结果会随发布上报（`hermesBaseOutcome`：`used` / `rejected` / `dump-failed` / `none`，附第一处差异），服务端可据此观察全体应用的拒绝率。`HERMESC= bun run fuzz:hermes-base --rounds 300` 会用随机程序配随机 base 编译并报告校验误杀的构建（对归一化规则做差分模糊测试）。只有包含上游 delta 模式修复的 hermesc 才会启用（经典 `react-native/sdks/hermesc`，或 `hermes-compiler` ≥ 250829098）。base 编译失败时，完整的 hermesc 输出会写到中间目录旁边的 `hermes-base-error.log`。`--resetCache false` 可跳过 Metro 的 `--reset-cache`，复用其转换缓存，重复打包会快很多。 ### Version diff --git a/docs/hermes-base-verification.md b/docs/hermes-base-verification.md index 3e73db8..ab769f4 100644 --- a/docs/hermes-base-verification.md +++ b/docs/hermes-base-verification.md @@ -2,7 +2,7 @@ `pushy bundle` 在 `--hermesBase auto`（默认）下用上一版 HBC 作 base 编译，让字符串 id 跨版本稳定、热更 patch 变小。`--verifyHermesBase`（默认开）并行再做一次普通编译，把两份产物反汇编后比对；任何不一致或失败都回退到普通编译。 -本文记录这套校验**防什么、怎么比、还缺什么**，供后续维护。代码在 `src/utils/hermes-base.ts`（`compareHermesBytecode`）、`src/utils/hermes-literals.ts`（二进制字面量缓冲区解码）与 `src/bundle-runner.ts`（`compileHermesByteCode`）。 +本文记录这套校验**防什么、怎么比、还缺什么**，供后续维护。代码在 `src/utils/hermes-base.ts`（`compareHermesBytecode`）、`src/utils/hermes-literals.ts`（二进制字面量缓冲区解码）、`src/utils/hermes-raw.ts`（原始操作数与完整二进制数据核对）与 `src/bundle-runner.ts`（`compileHermesByteCode`）。 ## 1. 防什么 @@ -12,14 +12,14 @@ ## 2. 怎么比（当前实现） -两侧各起一个 `hermesc -b -dump-bytecode -pretty-disassemble`，流式读取（不落盘，`PUSHY_HERMES_BASE_DEBUG=1` 时例外）： +校验分两遍：先各起一个 `hermesc -b -dump-bytecode -pretty-disassemble`，逐函数比较并提供易读诊断；这一步没有发现差异后，再各起一个带 `-pretty-disassemble=false` 的 raw dump 核对完整操作数。两遍顺序执行，不增加编译次数，同时最多两个 dump 进程。反汇编文本流式读取（不落盘，`PUSHY_HERMES_BASE_DEBUG=1` 时保留第一遍的 pretty 文本）。 | 区段 | 处理 | |---|---| -| `Bytecode File Information` 等头部 | 忽略 | -| `Global String Table` | 只用于解析 id，不比较（delta 会保留 base 的死字符串） | -| `Array Buffer` / `Object Key Buffer` / `Object Value Buffer`（v98：`Literal Value Buffer` / `Object Key Buffer`） | 文本段本身**不比较**（见下）；只在无法读二进制缓冲区时作为回退按整段内容比较 | -| `Function<…>` 各函数 | 头文本必须相同；正文逐行经 `normalizeDisassemblyLine` 归一化后比较；**按函数分段**，错位不级联 | +| `Bytecode File Information` 等头部 | 文本头不比较；raw 核对从二进制比较运行时选项、全局函数索引、segment ID、模块与函数源码表 | +| `Global String Table` | 不比较死字符串；有效 HBC 的字符串按二进制表恢复完整内容，而非截断或转义后的显示文本 | +| `Array Buffer` / `Object Key Buffer` / `Object Value Buffer`（v98：`Literal Value Buffer` / `Object Key Buffer`） | 文本段本身**不比较**（见下）；无法读二进制缓冲区时整段文本仅用于诊断，不能据此判定等价 | +| `Function<…>` / `NCFunction<…>` / `Constructor<…>` 各函数 | 头文本必须相同；正文逐行经 `normalizeDisassemblyLine` 归一化后比较；**按函数分段**，错位不级联 | | `Debug *` / `Textified callees table` | 排掉（只服务调试器/性能工具） | **字面量按指令比较（`src/utils/hermes-literals.ts`）**：`NewArrayWithBuffer rX, sizeHint, count, offset` / `NewObjectWithBuffer rX, sizeHint, count, keyOffset, valueOffset` 的 offset 是序列化字面量缓冲区里的**字节偏移**。校验先从两份 HBC 文件读出头部和三段缓冲区（`hbcTransform.ts` 的布局表），在每条指令处按 SLP 格式解码 `count` 个条目（tag 字节：bit 6..4 类型、bit 7 长度续字节、bit 3..0 长度低 4 位；值按类型定宽：double 8 字节、字符串 id 4/2/1 字节、int32 4 字节），字符串 id 经各自字符串表解析成文本后写进归一化后的指令行，两侧逐条比较。 @@ -35,13 +35,17 @@ v98 的 shape 索引和 offset 一样只用于定位（delta 可能重排 shape 表），归一化后只留 shape 指向的键与值；`AndParent` 的父对象寄存器原样保留比较。 -为什么不能按 dump 的整段文本比：Hermes 的缓冲区构建器会**重叠/去重**序列化后的字面量——一个字面量的最后一个值字节可以同时是下一个字面量的 tag 字节（模糊测试实测：`61 52 | cd 09 b3 05 11`，前一段以 `[String 82]` 结尾，后一条指令的 offset 正指向 `52`）。顺序解析整段缓冲区（hermesc 的 dump 就是这么打印的）从这里开始失步，之后的条目全是噪声；delta 构建的 id 宽度不同，重叠位置也不同，于是两段"噪声"在某处不一致就被判为差异。2026-09-10 的 20 轮冒烟模糊测试里 3 次误杀全部源于此，改按指令比较后全部等价。无法读二进制缓冲区（文件结构不识别）时两侧一起回退到整段文本比较，结果里 `literals: 'buffer'` 标明这一点。 +为什么不能按 dump 的整段文本比：Hermes 的缓冲区构建器会**重叠/去重**序列化后的字面量——一个字面量的最后一个值字节可以同时是下一个字面量的 tag 字节（模糊测试实测：`61 52 | cd 09 b3 05 11`，前一段以 `[String 82]` 结尾，后一条指令的 offset 正指向 `52`）。顺序解析整段缓冲区（hermesc 的 dump 就是这么打印的）从这里开始失步，之后的条目全是噪声；delta 构建的 id 宽度不同，重叠位置也不同，于是两段"噪声"在某处不一致就被判为差异。2026-09-10 的 20 轮冒烟模糊测试里 3 次误杀全部源于此，改按指令比较后全部等价。无法读二进制缓冲区（文件结构不识别）时两侧一起比较整段文本，仅辅助诊断；即使文本相等也返回 `dump-failed` 并回退 plain，不能以丢失 offset/count 的文本确认等价。结果里 `literals: 'buffer'` 标明这一点。 -`normalizeDisassemblyLine` 折叠的编码层差异：`New*WithBuffer` 的 offset（回退模式下只留 size）、`J*` 跳转目标、`DefineOwnById*` 的原始 string id → 字面量、`Long/LongIndex/Short` 宽度后缀与列对齐空白、`StringSwitchImm`/`UIntSwitchImm` 跳转表偏移、`offset N` 行、`Offset in debug table` 行。 +`normalizeDisassemblyLine` 只折叠表示层差异：按指令解析后的字面量地址、已知宽度后缀、引号外的列对齐空白、switch 表的物理偏移、debug 偏移。字符串内部的连续空格、跳转目标标签、寄存器均保留。未知 string ID、无法解码的字面量、未知 buffer 操作数形态直接失败；两侧都无法解析也不等价。 -结果三态：`equivalent` / `different`（带第一处差异：函数、行号、两侧内容，或缓冲区条目）/ `dump-failed`（dump 进程退出码非 0、无法启动、提前结束；带 stderr 末行）。后两种都放弃 base，但日志分开。 +**原始操作数核对**：`hermes-raw.ts` 从 raw dump 读取指令起点和操作数类型，并检查操作数与 HBC 字节一致、指令覆盖完整函数体。字符串从 small/overflow string table 与 string storage 按完整 ASCII/UTF-16 code unit 解码；BigInt、正则和 double 读取真实字节（保留 `-0` 和尾部精度）；函数引用保留索引，与顺序对齐的函数表共同检查，同名函数不能互换。地址映射为目标指令序号；整数和字符串 switch 从二进制恢复 case 值与目的地。函数运行时 flags 和参数/寄存器等字段也参与比较，剔除的仅是物理地址、debug presence 与 compact/overflow 表示。 -已知会在 `-pretty-disassemble` 下打印**原始数字 id** 而非字面量的指令目前只有 `DefineOwnById*`；这类指令是误杀的主要来源，出现新的形态就补一条规则 + 一个用例（`tests/hermes-base.test.ts` 的 `PLAIN_DUMP`/`DELTA_DUMP` 对，或 `tests/hermes-switch-normalization.test.ts`）。 +这不是一个可以忽略所有新指令的通用语义证明器。支持范围限定为已实现的 HBC v87–96、v98；升级布局或引入新的字符串/shape 引用指令时，需要复核 `hermes-raw.ts` 的解码规则及测试，不可仅扩展 diff-transform 的布局表。 + +结果三态：`equivalent` / `different`（带第一处差异：函数、行号、两侧内容，或缓冲区条目）/ `dump-failed`（无法解析完整语义数据、dump 进程失败/超时/提前结束；带原因或 stderr 末行）。后两种都放弃 base，但日志分开。 + +pretty 输出本身会截断长字符串与 BigInt、用函数名替代函数索引，并可能把 `-0` 显示为 `0`，因此不再以 pretty 相等作为最终结论。新增归一化规则时必须同时添加真实 HBC 负例，确保没有把语义差异折叠掉。 ## 3. 已完成与待办 @@ -66,25 +70,30 @@ SELECT hermesBaseOutcome, COUNT(*) FROM versions `scripts/fuzz-hermes-base.ts`（`HERMESC= bun run fuzz:hermes-base --rounds N --seed S [--out DIR] [--verbose]`）：按种子生成随机 JS（大量标识符/字符串字面量、嵌套数组对象字面量、整数与字符串 switch、闭包、try/catch、正则、模板字符串、Babel 形态的类、解构、寄存器压力大的函数），base 取池中另一份或当前程序的变异副本，真实 hermesc 编三次（base、plain、delta），跑 `compareHermesBytecode`；`different` 的 detail 去数字/寄存器/函数名后去重输出，失败轮次的源码与 HBC 留在 `--out`。每 10 轮另植入一处字面量改动并断言校验能抓到（防止归一化折叠过头）。首次运行（2026-09-10，20 轮）找到的三处误杀就是 §2 的缓冲区重叠问题，已由 3.2 修复；修复后种子 2（300 轮）、种子 3（200 轮）、种子 7（40 轮）共 519 轮有效编译：0 次差异、0 次 dump 失败、53 处植入差异全部抓到（生成器早期版本另有 21 轮 hermesc 拒绝编译，属生成器问题，已修）。生成器产出 hermesc 不接受的程序时按"生成器 bug"计数并保留现场，不算校验结果。植入的字面量若落在被优化掉的代码里（`!'x'`、不可达的 switch 分支——Static Hermes 的常量折叠比经典 hermesc 激进得多），产物里根本没有这个字符串，两侧确实等价；这类轮次按产物字符串存储里是否出现新字符串判定（不经被测校验），记为"optimized away"，不计入检出统计。 -### 3.4 函数身份 —— 未做 +### 3.4 函数身份与无损操作数 —— 已补充 + +raw 核对保留函数引用索引，与出现顺序对齐的函数体及二进制函数头联合比较。定向测试会只修改 HBC 中的引用，将两个同名闭包之一指向另一个：pretty 文本完全相同，但 raw 核对必须拒绝。另有 `-0`/`+0`、长字符串、UTF-16、长 BigInt 和 strict-mode flags 的回归用例。 -按 `Function(N params, M registers, K symbols)` 头 + 出现顺序对齐。模糊测试 500 余轮未观察到 delta 模式重排函数顺序；若将来观察到，改用 `-output-source-map` 给出的源位置作键。 +仍未实现任意函数重排下的身份映射；遇到合法的函数重排会保守回退 plain，不通过删除函数索引来规避。 ### 3.5 其它 - **`hermesBasePlainCompileFailed` 的语义** —— 已统一（`914514f`）：用于校验的普通编译失败时放弃 base、重编 plain 作为产物，上报 `dump-failed` + `plain compile failed: …`；真正的编译器故障会在重编时以构建错误暴露。 -- **内存峰值**：两个 hermesc + 两个 dump 并发。若 CI 机器吃紧，可把两个 dump 改成串行（多几秒）。 +- **资源开销**：base/plain 编译并发完成后，执行 pretty 和 raw 两遍 dump；每遍两个进程，raw 不增加编译。二进制元数据读取目前持有两份 HBC、完整字符串映射及字面量缓冲区，反汇编仅保留当前函数；这是完整数据核对的额外内存和时间开销。不要以删除验证数据来优化内存，可后续改为按段读取或降低并行度。 +- **进程期限**：版本探测默认 30 秒（`PUSHY_HERMES_PROBE_TIMEOUT_MS`），完整校验两遍合计默认 120 秒（`PUSHY_HERMES_VERIFY_TIMEOUT_MS`），单个编译/源码映射子进程默认 300 秒（`PUSHY_HERMES_COMPILE_TIMEOUT_MS`）。环境变量单位均为毫秒，必须是 1–2147483647 的整数，否则用默认值。超时终止子进程，优化失败回退 plain；真正的 plain 编译或最终 sourcemap 失败仍使构建失败。校验函数还接受 `AbortSignal`；base 下载任务的取消传播尚未统一。 +- **源码映射竞态**：推测执行的 base sourcemap 合成任务启动时立即观察拒绝，之后再根据最终采用哪份字节码决定抛出错误还是为 plain 重做合成。 +- **CI**：`hermes-hbc-96` / `hermes-hbc-98` job 分别安装固定的 `react-native@0.77.3` / `hermes-compiler@250829098.0.16`，校验可执行文件与真实 HBC 版本后运行 Hermes 回归和 50 轮固定种子 fuzz；缺少编译器会失败，不静默跳过。 - **`hbcdump`/`hbc-diff`**：Hermes 仓库自带的工具，RN 的 hermesc 不随附；如果将来 hermes-compiler 包里带上，可替代文本 dump 解析。 ## 4. 明确接受的剩余风险 -即使全部待办完成，这仍是"没找到差异"而非"证明等价"。覆盖不到：identifier hash 存储错误（hash 由内容确定性生成，概率极低）、调试信息层错误（不影响执行）、两侧被同一个 hermesc bug 同时影响。这些是选择 `--verifyHermesBase`（默认）与 `--hermesBase none` 之间时应知道的边界。 +即使全部待办完成，这仍是"没找到差异"而非"证明等价"。覆盖不到：identifier hash 存储错误（hash 由内容确定性生成，概率极低）、调试信息层错误（可能影响符号化，但不属于这里的执行等价检查）、两侧被同一个 hermesc bug 同时影响。这些是选择 `--verifyHermesBase`（默认）与 `--hermesBase none` 之间时应知道的边界。 ## 5. 排查一次拒绝 1. 看日志里的 detail：函数 + 行号 + 两侧内容，或缓冲区条目，或 dump 失败原因。 -2. 两行只差宽度后缀/数字 id/偏移 → 误杀，补归一化规则和用例；先用 `bun run fuzz:hermes-base` 复现一遍，看是否还有同类。 -3. 指令数量/顺序/opcode 不同，或某条 `New*WithBuffer` 解析后的字面量不同（detail 形如 `NewArrayWithBuffer r2 size=44 n=44 entry 7: [String "a"] vs [String "b"]`）→ 真差异，回退正确，向 Hermes 上游报告。detail 带 `` 说明偏移超出缓冲区或 tag 非法，同样是真问题。 +2. 两行只差宽度后缀/数字 id/偏移：先核对解析后的完整值与控制流目标。只有证明是表示差异后才能补归一化规则，不能直接删除操作数；同时添加负例与 fuzz。 +3. 指令数量/顺序/opcode 不同，或某条 `New*WithBuffer` 解析后的字面量不同（detail 形如 `NewArrayWithBuffer r2 size=44 n=44 entry 7: [String "a"] vs [String "b"]`）→ 真差异，回退正确，向 Hermes 上游报告。detail 带 `undecodable` 说明无法解码，归为 `dump-failed`；可能是损坏的 HBC，也可能是验证器尚不支持的布局，不能直接断言上游错误。 若日志/元数据里 `literals` 为 `buffer`（文件结构不识别），缓冲区是按整段文本比的，`Array Buffer entry N` 形式的差异可能是 §2 的重叠误杀，需人工用二进制解码核对。 -4. `dump-failed` → 与字节码无关，看 stderr（内存、被 kill、hermesc 版本）。 +4. `dump-failed` → 校验无法完成；检查解析原因、编译器版本、超时与 stderr。它不是等价证据，也不必然是与字节码无关的进程错误。 5. 需要完整现场：`PUSHY_HERMES_BASE_DEBUG=1 pushy bundle …`，两份反汇编在中间目录旁 `hermes-base-dump-{base,plain}.txt`。 diff --git a/src/bundle-runner.ts b/src/bundle-runner.ts index 5f37ed7..65085b7 100644 --- a/src/bundle-runner.ts +++ b/src/bundle-runner.ts @@ -16,6 +16,7 @@ import { probeHbcVersion, resolveHermesBase, } from './utils/hermes-base'; +import { hermesTimeout } from './utils/hermes-timeout'; import { t } from './utils/i18n'; import { getJavaScriptRuntime, @@ -906,6 +907,11 @@ function runProcess( return new Promise((resolve) => { const child = spawn(command, args, { stdio: ['ignore', 'ignore', captureStderr ? 'pipe' : 'ignore'], + timeout: hermesTimeout( + process.env.PUSHY_HERMES_COMPILE_TIMEOUT_MS, + 300_000, + ), + killSignal: 'SIGKILL', }); // hermesc echoes whole (minified) source lines per diagnostic: keep the // head and the tail, never an unbounded transcript @@ -971,8 +977,10 @@ const HERMES_BASE_WAIT_MS = 60_000; async function awaitPendingBase( pending: Promise, ): Promise { - const waitMs = - Number(process.env.PUSHY_HERMES_BASE_WAIT_MS) || HERMES_BASE_WAIT_MS; + const waitMs = hermesTimeout( + process.env.PUSHY_HERMES_BASE_WAIT_MS, + HERMES_BASE_WAIT_MS, + ); let timer: NodeJS.Timeout | undefined; const gaveUp = new Promise((resolve) => { timer = setTimeout( @@ -1129,7 +1137,11 @@ export async function compileHermesByteCode({ const fullStderr = attempt.error ? String(attempt.error.message ?? attempt.error) : attempt.stderr; - const reason = summarizeHermescStderr(fullStderr); + const reason = + summarizeHermescStderr(fullStderr) || + (attempt.signal + ? `signal ${attempt.signal} (compiler deadline or external termination)` + : ''); console.warn( t('hermesBaseCompileFailed', { reason: reason || `exit ${attempt.status}`, @@ -1151,7 +1163,10 @@ export async function compileHermesByteCode({ // redone from the plain map in the rare case the base is rejected const speculativeCompose = usedBase && sourcemapOutput - ? composeSourceMaps(packagerMap, hermesMap, sourcemapOutput) + ? composeSourceMaps(packagerMap, hermesMap, sourcemapOutput).then( + (value) => ({ ok: true as const, value }), + (error: unknown) => ({ ok: false as const, error }), + ) : null; if (usedBase && wantPlain) { if (!plainOk) { @@ -1222,11 +1237,9 @@ export async function compileHermesByteCode({ } } if (speculativeCompose) { - const done = await speculativeCompose.catch((error) => { - if (usedBase) throw error; - return false; - }); - composed = usedBase && done; + const done = await speculativeCompose; + if (!done.ok && usedBase) throw done.error; + composed = usedBase && done.ok && done.value; } if (!usedBase) { if (plainOk) { diff --git a/src/utils/hermes-base.ts b/src/utils/hermes-base.ts index 07e64ff..54ff683 100644 --- a/src/utils/hermes-base.ts +++ b/src/utils/hermes-base.ts @@ -8,6 +8,8 @@ // server lookup, a sha256-named local cache, download + verification, and the // optional disassembly equivalence check. Every failure degrades to the plain // compile — nothing here may block a release. + +import { StringDecoder } from 'node:string_decoder'; import { spawn, spawnSync } from 'child_process'; import { createHash } from 'crypto'; import fs from 'fs-extra'; @@ -22,6 +24,8 @@ import { LiteralResolver, readLiteralBuffers, } from './hermes-literals'; +import { auditRawHermesBytecode, readHermesSemanticData } from './hermes-raw'; +import { hermesTimeout } from './hermes-timeout'; import { t } from './i18n'; import { webFetch } from './runtime'; import { enumZipEntries, readEntry } from './zip-entries'; @@ -288,7 +292,14 @@ function compileProbe(hermesCommand: string): number | null { const result = spawnSync( hermesCommand, ['-emit-binary', '-out', output, input, '-O', '-w'], - { stdio: 'ignore' }, + { + stdio: 'ignore', + timeout: hermesTimeout( + process.env.PUSHY_HERMES_PROBE_TIMEOUT_MS, + 30_000, + ), + killSignal: 'SIGKILL', + }, ); if (result.status !== 0 || !fs.existsSync(output)) { return null; @@ -1014,8 +1025,7 @@ export const LITERAL_SEPARATOR = '\u001f'; * those offsets, so two builds that lay their buffers out differently still * compare by what each instruction builds. The shape index is dropped like an * offset: it only locates the keys. Null when the operands do not fit the - * layout; an offset that cannot be decoded is spelled out (and so never - * equals a decoded one). + * layout; an offset or reference that cannot be decoded fails closed. */ function renderBufferInstruction( opcode: string, @@ -1026,7 +1036,9 @@ function renderBufferInstruction( if (opcode.startsWith('NewArray')) { if (operands.length !== 3) return null; const entries = literals.array(operands[2], count); - return `size=${sizeHint} n=${count} [${entries ? entries.join(LITERAL_SEPARATOR) : ``}]`; + if (!entries) + throw new Error(`undecodable array literal at ${operands[2]}`); + return `size=${sizeHint} n=${count} [${entries.join(LITERAL_SEPARATOR)}]`; } if (literals.layout === 'shaped') { if (operands.length !== 2) return null; @@ -1035,7 +1047,9 @@ function renderBufferInstruction( const keys = shape && literals.objectKeys(shape.keyOffset, shape.count); const values = shape && literals.objectValues(valueOffset, shape.count); if (!shape || !keys || !values) { - return `n=${shape?.count ?? '?'} {}`; + throw new Error( + `undecodable object literal at shape ${shapeIndex}/${valueOffset}`, + ); } const pairs = keys.map((k, i) => `${k}: ${values[i]}`); return `n=${shape.count} {${pairs.join(LITERAL_SEPARATOR)}}`; @@ -1044,7 +1058,9 @@ function renderBufferInstruction( const keys = literals.objectKeys(operands[2], count); const values = literals.objectValues(operands[3], count); if (!keys || !values) { - return `size=${sizeHint} n=${count} {}`; + throw new Error( + `undecodable object literal at ${operands[2]}/${operands[3]}`, + ); } const pairs = keys.map((k, i) => `${k}: ${values[i]}`); return `size=${sizeHint} n=${count} {${pairs.join(LITERAL_SEPARATOR)}}`; @@ -1084,19 +1100,24 @@ export function normalizeDisassemblyLine( literals, ); if (rendered) return `${op} ${regs} ${rendered}`; + throw new Error(`unsupported literal operands: ${line.trim()}`); } // no binary buffers: only the size hint survives; the buffer content - // is compared as a whole instead (compareBuffers) + // is compared as a whole for diagnostics only (never accepted as equivalent) return `${op} ${regs} sizes=${nums.slice(0, 1).join(',')}`; } } if (opcode.charCodeAt(0) === 0x4a /* J */) { m = /^(\s*J[A-Za-z]+?)(Long)?\s+(L\d+|\d+)(.*)$/.exec(line); - if (m) return `${m[1]} ${m[4]}`; + if (m) return `${m[1]} ${m[3]}${normalizeOperandSpacing(m[4])}`; } if (opcode.startsWith('DefineOwnById')) { m = /^(\s*DefineOwnById\w*\s+r\d+, r\d+, \d+, )(\d+)$/.exec(line); - if (m) line = `${m[1]}"${strings.get(Number(m[2])) ?? `?${m[2]}`}"`; + if (m) { + const text = strings.get(Number(m[2])); + if (text === undefined) throw new Error(`unresolved string id ${m[2]}`); + line = `${m[1]}${JSON.stringify(text)}`; + } } // Operand-width variants of one instruction (GetByIdShort/GetById/GetByIdLong, // LoadConstString/LoadConstStringLongIndex, ...) only differ by how wide a @@ -1108,7 +1129,7 @@ export function normalizeDisassemblyLine( (opEnd === line.length || isSpace(line.charCodeAt(opEnd))) ) { const folded = foldWidthSuffix(opcode); - line = `${line.slice(0, indent)}${folded}${line.slice(opEnd).replace(/\s+/g, ' ')}`; + line = `${line.slice(0, indent)}${folded}${normalizeOperandSpacing(line.slice(opEnd))}`; // Switch jump tables sit after the instructions; their relative offset (and // the table header hermesc prints for them) moves with instruction widths. // The two switch instructions carry that offset in different operands: @@ -1129,6 +1150,32 @@ export function normalizeDisassemblyLine( return line; } +/** Only formatting outside quoted operands may be collapsed. */ +function normalizeOperandSpacing(text: string): string { + let result = ''; + let quoted = false; + let escaped = false; + let spacing = false; + for (const char of text) { + if (quoted) { + result += char; + if (escaped) escaped = false; + else if (char === '\\') escaped = true; + else if (char === '"') quoted = false; + continue; + } + if (/\s/.test(char)) { + if (!spacing) result += ' '; + spacing = true; + } else { + result += char; + spacing = false; + if (char === '"') quoted = true; + } + } + return result; +} + function isSpace(code: number): boolean { return code === 0x20 || code === 0x09 || code === 0x0d; } @@ -1151,9 +1198,10 @@ function foldWidthSuffix(opcode: string): string { async function* streamLines( stream: NodeJS.ReadableStream, ): AsyncGenerator { + const decoder = new StringDecoder('utf8'); let rest = ''; for await (const chunk of stream as AsyncIterable) { - rest += typeof chunk === 'string' ? chunk : chunk.toString('utf8'); + rest += typeof chunk === 'string' ? chunk : decoder.write(chunk); let index = rest.indexOf('\n'); while (index >= 0) { yield rest.slice(0, index); @@ -1161,6 +1209,7 @@ async function* streamLines( index = rest.indexOf('\n'); } } + rest += decoder.end(); if (rest) yield rest; } @@ -1190,6 +1239,9 @@ export interface HermesEquivalenceResult { export interface HermesEquivalenceOptions { /** write each side's raw disassembly here (bug reports) */ dumpTo?: { withBase: string; plain: string }; + /** Total deadline for both verification passes. */ + timeoutMs?: number; + signal?: AbortSignal; } interface DumpFunction { @@ -1216,7 +1268,7 @@ const DUMP_STDERR_KEEP = 4 * 1024; const DETAIL_LINE_MAX = 120; function isFunctionHeader(line: string): boolean { - return line.startsWith('Function<') || line.startsWith('NCFunction<'); + return /^(?:Function|NCFunction|Constructor); + private debugFinished: Promise = Promise.resolve(); + private readonly pass: PassThrough; constructor( readonly proc: ReturnType, dumpTo?: string, buffers?: LiteralBuffers | null, + private readonly binaryStrings?: Map, ) { // the string table is filled while the preamble streams by, before the // first instruction needs it + if (binaryStrings) { + for (const [id, value] of binaryStrings) this.strings.set(id, value); + } if (buffers) this.literals = new LiteralResolver(buffers, this.strings); - const pass = new PassThrough(); + const pass = (this.pass = new PassThrough()); proc.stdout!.pipe(pass); - if (dumpTo) proc.stdout!.pipe(fs.createWriteStream(dumpTo)); + if (dumpTo) { + const output = (this.debugOutput = fs.createWriteStream(dumpTo)); + this.debugFinished = new Promise((resolve) => { + output.once('finish', resolve); + output.once('error', (error) => { + this.dumpError = error; + proc.kill('SIGKILL'); + resolve(); + }); + }); + proc.stdout!.pipe(output); + } this.iterator = streamLines(pass); let stderr = ''; proc.stderr?.on('data', (chunk: Buffer | string) => { stderr = (stderr + chunk.toString()).slice(-DUMP_STDERR_KEEP); }); this.exit = new Promise((resolve) => { + let processError: Error | undefined; // a spawn failure (ENOENT) may leave stdout open and never 'close' proc.on('error', (error) => { + proc.stdout?.unpipe(pass); + proc.stdout?.destroy(); pass.end(); - resolve({ code: null, signal: null, error, stderr }); + processError = error; + if (!proc.pid) resolve({ code: null, signal: null, error, stderr }); }); - proc.on('close', (code, signal) => resolve({ code, signal, stderr })); + proc.on('close', (code, signal) => + resolve({ code, signal, error: processError, stderr }), + ); }); } @@ -1298,8 +1375,12 @@ class DumpReader { } if (this.section === 'Global String Table') { const m = STRING_TABLE_LINE.exec(line); - if (m) this.strings.set(Number(m[1]), m[2]); - } else if (this.section?.endsWith('Buffer') && line.trim() !== '') { + if (m && !this.binaryStrings) this.strings.set(Number(m[1]), m[2]); + } else if ( + !this.literals && + this.section?.endsWith('Buffer') && + line.trim() !== '' + ) { const entry = line.trim(); const ref = BUFFER_STRING_ENTRY.exec(entry); const text = ref ? this.strings.get(Number(ref[1])) : undefined; @@ -1344,11 +1425,18 @@ class DumpReader { /** drain whatever is left and report how the process ended */ async finish(): Promise { while ((await this.nextLine()) !== null) {} - return this.exit; + const exit = await this.exit; + await this.debugFinished; + return this.dumpError ? { ...exit, error: this.dumpError } : exit; } - kill() { - this.proc.kill(); + async kill() { + this.proc.stdout?.unpipe(); + this.pass.destroy(); + this.proc.stdout?.destroy(); + this.proc.kill('SIGKILL'); + this.debugOutput?.end(); + await Promise.all([this.exit, this.debugFinished]); } } @@ -1426,8 +1514,8 @@ function compareFunctions(a: DumpFunction, b: DumpFunction): string | null { } /** - * Compare two HBC files by disassembly (see normalizeDisassemblyLine): the - * literal buffers by content, then function by function. `-b` forces hermesc + * Compare two HBC files by pretty disassembly, then audit the raw operands + * against full binary strings, literals, function identities and addresses. `-b` forces hermesc * to treat inputs as bytecode whatever their extension. Both dumps are * consumed as streams so the ~100 MB of text never touches the disk (unless * `dumpTo` asks for it). A dump process that fails or ends early is reported @@ -1441,19 +1529,41 @@ export async function compareHermesBytecode( plain: string, options: HermesEquivalenceOptions = {}, ): Promise { + const controller = new AbortController(); + const timeoutMs = hermesTimeout( + options.timeoutMs ?? process.env.PUSHY_HERMES_VERIFY_TIMEOUT_MS, + 120_000, + ); + const abort = () => controller.abort(options.signal?.reason); + if (options.signal?.aborted) abort(); + else options.signal?.addEventListener('abort', abort, { once: true }); + const timer = setTimeout( + () => + controller.abort( + new Error(`Hermes verification timed out after ${timeoutMs}ms`), + ), + timeoutMs, + ); + timer.unref(); const spawnDump = (file: string) => spawn( hermesCommand, ['-b', '-dump-bytecode', '-pretty-disassemble', file], - { stdio: ['ignore', 'pipe', 'pipe'] }, + { + stdio: ['ignore', 'pipe', 'pipe'], + signal: controller.signal, + killSignal: 'SIGKILL', + }, ); // The literal buffers are read from both files up front (header + three // byte ranges). Only when both sides can be read do the instructions // compare decoded literals; otherwise both fall back to the dumped buffers // as a whole, so the two sides always normalize the same way. - const [buffersA, buffersB] = await Promise.all([ + const [buffersA, buffersB, dataA, dataB] = await Promise.all([ readLiteralBuffers(withBase).catch(() => null), readLiteralBuffers(plain).catch(() => null), + readHermesSemanticData(withBase).catch(() => null), + readHermesSemanticData(plain).catch(() => null), ]); const binary = buffersA && buffersB && buffersA.version === buffersB.version; const literals = binary ? 'instruction' : 'buffer'; @@ -1461,11 +1571,13 @@ export async function compareHermesBytecode( spawnDump(withBase), options.dumpTo?.withBase, binary ? buffersA : null, + dataA?.strings, ); const b = new DumpReader( spawnDump(plain), options.dumpTo?.plain, binary ? buffersB : null, + dataB?.strings, ); let functions = 0; const different = (detail: string): HermesEquivalenceResult => ({ @@ -1510,7 +1622,23 @@ export async function compareHermesBytecode( literals, }; } - return { status: 'equivalent', functions, literals }; + if (!binary || !dataA || !dataB) { + return { + status: 'dump-failed', + detail: + 'unsupported or unreadable HBC layout; text-only comparison cannot verify equivalence', + functions, + literals, + }; + } + const audit = await auditRawHermesBytecode( + hermesCommand, + [withBase, plain], + [dataA, dataB], + [buffersA, buffersB], + controller.signal, + ); + return { ...audit, functions, literals }; } if (functions === 0 && !binary) { // buffers precede the functions, so both are complete by now. This @@ -1524,9 +1652,17 @@ export async function compareHermesBytecode( if (detail) return different(detail); functions++; } + } catch (error) { + return { + status: 'dump-failed', + detail: error instanceof Error ? error.message : String(error), + functions, + literals, + }; } finally { - a.kill(); - b.kill(); + clearTimeout(timer); + options.signal?.removeEventListener('abort', abort); + await Promise.all([a.kill(), b.kill()]); } } diff --git a/src/utils/hermes-literals.ts b/src/utils/hermes-literals.ts index 9677ca8..68c3901 100644 --- a/src/utils/hermes-literals.ts +++ b/src/utils/hermes-literals.ts @@ -133,7 +133,7 @@ export async function readLiteralBuffers( /** * Decode `count` literal values starting at byte `offset`. Null when the * offset or a run reaches outside the buffer or a tag type is unknown; the - * caller reports that as a difference rather than guessing. `layout` decides + * caller fails verification rather than treating two failures as equivalent. `layout` decides * what type 6 means (see the file comment). */ export function decodeSerializedLiterals( @@ -142,6 +142,14 @@ export function decodeSerializedLiterals( count: number, layout: LiteralBuffers['layout'] = 'split', ): LiteralValue[] | null { + if ( + !Number.isSafeInteger(offset) || + !Number.isSafeInteger(count) || + offset < 0 || + count < 0 || + count > 1_000_000 + ) + return null; const values: LiteralValue[] = []; let pos = offset; while (values.length < count) { @@ -152,6 +160,7 @@ export function decodeSerializedLiterals( if (pos >= buffer.length) return null; length = (length << 8) | buffer[pos++]; } + if (length === 0) return null; const type = tag & TAG_TYPE_MASK; for (let i = 0; i < length && values.length < count; i++) { switch (type) { @@ -203,7 +212,7 @@ export function decodeSerializedLiterals( /** * One literal as text, string ids resolved through the dump's string table - * (an unknown id stays visible as `?id`). Doubles keep their exact bits in + * (an unknown id fails closed). Doubles keep their exact bits in * the text so -0 and 0, or two NaNs, never collide by accident. */ export function renderLiteral( @@ -225,7 +234,9 @@ export function renderLiteral( } case 'string': { const text = strings.get(value.id); - return `[String ${text === undefined ? `?${value.id}` : JSON.stringify(text)}]`; + if (text === undefined) + throw new Error(`unresolved string id ${value.id}`); + return `[String ${JSON.stringify(text)}]`; } } } @@ -283,7 +294,8 @@ export class LiteralResolver { /** Shape table entry (HBC v98); null out of range or in the split layout. */ shape(index: number): { keyOffset: number; count: number } | null { const b = this.buffers; - if (b.layout !== 'shaped' || index < 0) return null; + if (b.layout !== 'shaped' || !Number.isSafeInteger(index) || index < 0) + return null; const at = index * SHAPE_ENTRY_SIZE; if (at + SHAPE_ENTRY_SIZE > b.shapes.length) return null; return { diff --git a/src/utils/hermes-raw.ts b/src/utils/hermes-raw.ts new file mode 100644 index 0000000..33e00af --- /dev/null +++ b/src/utils/hermes-raw.ts @@ -0,0 +1,603 @@ +/** + * Lossless operand audit accompanying the human-readable Hermes comparison. + * Pretty disassembly truncates strings and BigInts, prints function names in + * place of IDs, and renders -0 as 0. Never use those spellings as identities. + * + * The raw dump supplies instruction/operand boundaries; referenced data and + * doubles come from the HBC itself. Bytecode layouts are intentionally bounded + * to the layouts already supported by hbcTransform. Unknown data fails closed. + */ + +import { readFile } from 'node:fs/promises'; +import { StringDecoder } from 'node:string_decoder'; +import { spawn } from 'child_process'; +import { pickLayout, resolveHbcSections } from './hbcTransform'; +import { type LiteralBuffers, LiteralResolver } from './hermes-literals'; + +export class UnverifiableHermesBytecode extends Error {} + +function requireValue(value: T | null | undefined, what: string): T { + if (value === null || value === undefined) { + throw new UnverifiableHermesBytecode(what); + } + return value; +} + +function checkedSlice(data: Buffer, start: number, length: number): Buffer { + if ( + !Number.isSafeInteger(start) || + !Number.isSafeInteger(length) || + start < 0 || + length < 0 || + start + length > data.length + ) { + throw new UnverifiableHermesBytecode('HBC reference outside its section'); + } + return data.subarray(start, start + length); +} + +interface FunctionData { + offset: number; + size: number; + metadata: string; +} + +export interface HermesSemanticData { + bytes: Buffer; + version: number; + strings: Map; + functions: FunctionData[]; + bigints: Buffer[]; + regexps: Buffer[]; + metadata: string; +} + +/** Read exact strings (including UTF-16 code units) and function identities. */ +export async function readHermesSemanticData( + file: string, +): Promise { + const bytes = await readFile(file); + const resolved = requireValue( + resolveHbcSections(bytes, bytes.length), + 'unsupported or unreadable HBC layout', + ); + // A newly added diff-transform layout must not silently opt into semantic + // parsing: function header and operand schemas need an independent review. + if ( + !( + (resolved.version >= 87 && resolved.version <= 96) || + resolved.version === 98 + ) + ) { + throw new UnverifiableHermesBytecode('unsupported semantic HBC version'); + } + const layout = requireValue(pickLayout(bytes), 'unknown HBC layout'); + const section = (name: string) => { + const r = requireValue(resolved.sections.get(name), `missing ${name}`); + return checkedSlice(bytes, r.start, r.size); + }; + const strings = new Map(); + const small = section('smallStringTable'); + const overflow = section('overflowStringTable'); + const storage = section('stringStorage'); + for (let at = 0; at < small.length; at += 4) { + const word = small.readUInt32LE(at); + const utf16 = (word & 1) !== 0; + let offset = (word >>> 1) & 0x7fffff; + let length = word >>> 24; + if (length === 255) { + const entry = checkedSlice(overflow, offset * 8, 8); + offset = entry.readUInt32LE(0); + length = entry.readUInt32LE(4); + } + const value = checkedSlice(storage, offset, length * (utf16 ? 2 : 1)); + strings.set(at / 4, value.toString(utf16 ? 'utf16le' : 'latin1')); + } + const string = (id: number) => + requireValue(strings.get(id), `unresolved string id ${id}`); + const shaped = resolved.version === 98; + const entrySize = shaped ? 12 : 16; + const headers = section('functionHeaders'); + const functions: FunctionData[] = []; + for (let at = 0; at < headers.length; at += entrySize) { + const word0 = headers.readUInt32LE(at); + const word1 = headers.readUInt32LE(at + 4); + let flags = headers[at + entrySize - 1]; + let offset = word0 & 0x1ffffff; + let size: number; + let name: number; + let fields: number[]; + if (flags & 0x20) { + // SmallFuncHeader::getLargeHeaderOffset, classic and Static Hermes. + const largeOffset = shaped + ? ((word1 >>> 14) & 0xff) * 0x1000000 + offset + : (headers.readUInt32LE(at + 8) & 0x1ffffff) * 0x10000 + offset; + const large = checkedSlice(bytes, largeOffset, shaped ? 37 : 31); + offset = large.readUInt32LE(0); + size = large.readUInt32LE(shaped ? 12 : 8); + name = large.readUInt32LE(shaped ? 16 : 12); + flags = large[shaped ? 36 : 30]; + fields = shaped + ? [4, 8, 20, 24, 28].map((p) => large.readUInt32LE(p)) + : [4, 20, 24].map((p) => large.readUInt32LE(p)); + fields.push(...large.subarray(shaped ? 32 : 28, shaped ? 36 : 30)); + } else if (shaped) { + size = word1 & 0x3fff; + name = (word1 >>> 14) & 0xff; + fields = [ + (word0 >>> 25) & 31, + word0 >>> 30, + (word1 >>> 22) & 31, + word1 >>> 27, + headers[at + 8], + headers[at + 9], + headers[at + 10] & 63, + (headers[at + 10] >>> 6) & 1, + headers[at + 10] >>> 7, + ]; + } else { + size = word1 & 0x7fff; + name = word1 >>> 15; + fields = [ + word0 >>> 25, + headers.readUInt32LE(at + 8) >>> 25, + headers[at + 12], + headers[at + 13], + headers[at + 14], + ]; + } + checkedSlice(bytes, offset, size); + functions.push({ + offset, + size, + // Debug presence and compact/overflow encoding are not runtime semantics. + metadata: JSON.stringify([string(name), flags & ~0x30, fields]), + }); + } + const pairedStorage = (tableName: string, storageName: string) => { + const table = section(tableName); + const data = section(storageName); + const entries: Buffer[] = []; + for (let at = 0; at < table.length; at += 8) { + entries.push( + checkedSlice(data, table.readUInt32LE(at), table.readUInt32LE(at + 4)), + ); + } + return entries; + }; + const counts = Object.fromEntries( + layout.headerFields.map((name, i) => [ + name, + bytes.readUInt32LE(32 + i * 4), + ]), + ); + const options = bytes[32 + layout.headerFields.length * 4]; + const moduleTable = section('cjsModuleTable'); + const modules: unknown[] = []; + for (let at = 0; at < moduleTable.length; at += 8) { + const id = moduleTable.readUInt32LE(at); + modules.push([ + options & 2 ? id : string(id), + moduleTable.readUInt32LE(at + 4), + ]); + } + const sourceTable = section('functionSourceTable'); + const sources: unknown[] = []; + for (let at = 0; at < sourceTable.length; at += 8) { + sources.push([ + sourceTable.readUInt32LE(at), + string(sourceTable.readUInt32LE(at + 4)), + ]); + } + return { + bytes, + version: resolved.version, + strings, + functions, + bigints: pairedStorage('bigIntTable', 'bigIntStorage'), + regexps: pairedStorage('regExpTable', 'regExpStorage'), + metadata: JSON.stringify([ + resolved.version, + counts.globalCodeIndex, + counts.segmentID, + options, + modules, + sources, + ]), + }; +} + +const OPERAND_BYTES: Record = { + Reg8: 1, + Reg32: 4, + UInt8: 1, + UInt16: 2, + UInt32: 4, + Addr8: 1, + Addr32: 4, + Imm32: 4, + Double: 8, +}; + +// Zero-based string operand positions from BytecodeList.def, with the missing +// DefineOwnById annotation supplied explicitly. Keep classic and v98 variants. +const STRING_OPERANDS: Record = { + DeclareGlobalVar: [0], + GetById: [3], + GetByIdWithReceiver: [4], + TryGetById: [3], + PutById: [3], + TryPutById: [3], + PutNewOwnById: [2], + PutNewOwnNEById: [2], + PutByIdLoose: [3], + PutByIdStrict: [3], + TryPutByIdLoose: [3], + TryPutByIdStrict: [3], + DefineOwnById: [3], + DelById: [2], + LoadConstString: [1], + CreatePrivateName: [1], + CreateRegExp: [1, 2], +}; + +interface RawInstruction { + offset: number; + opcode: string; + operands: { type: string; value: number; bits?: string }[]; + size: number; +} + +function foldWidth(opcode: string): string { + return opcode.replace(/(?:LongIndex|Long|Short)$/, ''); +} + +function parseRawInstruction( + line: string, + data: HermesSemanticData, + fn: FunctionData, +): RawInstruction { + const m = /^\[@ (\d+)\] ([A-Za-z][A-Za-z0-9]*)(.*)$/.exec(line); + if (!m) + throw new UnverifiableHermesBytecode( + `unrecognized raw instruction: ${line.slice(0, 100)}`, + ); + const offset = Number(m[1]); + let size = 1; + const operands = + m[3].trim() === '' + ? [] + : m[3] + .trim() + .split(/,\s*/) + .map((text) => { + const operand = /^([^<>]+)<([A-Za-z0-9]+)>$/.exec(text); + if (!operand || !Object.hasOwn(OPERAND_BYTES, operand[2])) { + throw new UnverifiableHermesBytecode( + `unknown raw operand: ${text}`, + ); + } + const type = operand[2]; + const width = OPERAND_BYTES[type]; + const raw = checkedSlice( + checkedSlice(data.bytes, fn.offset, fn.size), + offset + size, + width, + ); + size += width; + // Raw dump doubles are also rounded! Compare the actual IEEE-754 bits. + if (type === 'Double') + return { type, value: 0, bits: raw.toString('hex') }; + const value = Number(operand[1]); + const binaryValue = + type === 'Addr8' + ? raw.readInt8(0) + : type === 'Addr32' || type === 'Imm32' + ? raw.readInt32LE(0) + : width === 1 + ? raw.readUInt8(0) + : width === 2 + ? raw.readUInt16LE(0) + : raw.readUInt32LE(0); + if (!Number.isSafeInteger(value) || value !== binaryValue) { + throw new UnverifiableHermesBytecode( + 'raw operand does not match the HBC bytes', + ); + } + return { type, value }; + }); + return { offset, opcode: m[2], operands, size }; +} + +/** Canonicalize one complete function; addresses become instruction ordinals. */ +export function normalizeRawHermesFunction( + lines: string[], + data: HermesSemanticData, + functionIndex: number, + buffers: LiteralBuffers, +): string[] { + const fn = requireValue( + data.functions[functionIndex], + 'function index outside HBC table', + ); + const instructions = lines + .filter((line) => line.startsWith('[@ ')) + .map((line) => parseRawInstruction(line, data, fn)); + const targets = new Map(); + let end = 0; + for (const [index, inst] of instructions.entries()) { + if (inst.offset !== end) + throw new UnverifiableHermesBytecode( + 'incomplete or unordered raw instruction stream', + ); + targets.set(inst.offset, index); + end += inst.size; + } + if (end !== fn.size || instructions.length === 0) { + throw new UnverifiableHermesBytecode( + 'raw dump ended before the function body', + ); + } + const target = (offset: number) => + requireValue( + targets.get(offset), + `branch target ${offset} is not an instruction`, + ); + const string = (id: number) => + requireValue(data.strings.get(id), `unresolved string id ${id}`); + const literal = new LiteralResolver(buffers, data.strings); + const output = [fn.metadata]; + for (const inst of instructions) { + const op = foldWidth(inst.opcode); + const values = inst.operands.map((o) => o.value); + const refs = STRING_OPERANDS[op] ?? []; + const operands: unknown[] = inst.operands.map((operand, i) => { + if (refs.includes(i)) return ['string', string(operand.value)]; + if (operand.type.startsWith('Addr')) + return ['target', target(inst.offset + operand.value)]; + if (operand.type === 'Double') return ['double', operand.bits]; + return [ + operand.type.startsWith('Reg') ? 'register' : 'integer', + operand.value, + ]; + }); + if (op === 'LoadConstBigInt') { + operands[1] = [ + 'bigint', + requireValue(data.bigints[values[1]], 'unresolved BigInt').toString( + 'hex', + ), + ]; + } else if (op === 'CreateRegExp') { + operands[3] = [ + 'regexp', + requireValue(data.regexps[values[3]], 'unresolved RegExp').toString( + 'hex', + ), + ]; + } else if (op === 'NewArrayWithBuffer') { + if (values.length !== 4) + throw new UnverifiableHermesBytecode('unknown array buffer operands'); + operands[3] = requireValue( + literal.array(values[3], values[2]), + 'undecodable array literal', + ); + } else if ( + op === 'NewObjectWithBuffer' || + op === 'NewObjectWithBufferAndParent' + ) { + if (buffers.layout === 'split') { + if (values.length !== 5 || op !== 'NewObjectWithBuffer') + throw new UnverifiableHermesBytecode('unknown split object operands'); + operands[3] = requireValue( + literal.objectKeys(values[3], values[2]), + 'undecodable object keys', + ); + operands[4] = requireValue( + literal.objectValues(values[4], values[2]), + 'undecodable object values', + ); + } else { + const shapeAt = op === 'NewObjectWithBufferAndParent' ? 2 : 1; + if (values.length !== shapeAt + 2) + throw new UnverifiableHermesBytecode( + 'unknown shaped object operands', + ); + const shape = requireValue( + literal.shape(values[shapeAt]), + 'unknown object shape', + ); + operands[shapeAt] = requireValue( + literal.objectKeys(shape.keyOffset, shape.count), + 'undecodable shape keys', + ); + operands[shapeAt + 1] = requireValue( + literal.objectValues(values[shapeAt + 1], shape.count), + 'undecodable shape values', + ); + } + } else if (op === 'CacheNewObject') { + if (buffers.layout !== 'shaped' || values.length !== 4) { + throw new UnverifiableHermesBytecode('unknown cached object operands'); + } + const shape = requireValue( + literal.shape(values[2]), + 'unknown cached object shape', + ); + operands[2] = requireValue( + literal.objectKeys(shape.keyOffset, shape.count), + 'undecodable cached object keys', + ); + } else if ( + op === 'UIntSwitchImm' || + op === 'SwitchImm' || + op === 'StringSwitchImm' + ) { + const strings = op === 'StringSwitchImm'; + const tableAt = strings ? 2 : 1; + const count = strings ? values[4] : values[4] - values[3] + 1; + // The table offset is relative to the instruction, rounded up to 4-byte alignment. + const start = + Math.ceil((fn.offset + inst.offset + values[tableAt]) / 4) * 4; + const table = checkedSlice(data.bytes, start, count * (strings ? 8 : 4)); + const entries: unknown[] = []; + for (let i = 0; i < count; i++) { + const at = i * (strings ? 8 : 4); + entries.push([ + strings ? string(table.readUInt32LE(at)) : values[3] + i, + target(inst.offset + table.readInt32LE(at + (strings ? 4 : 0))), + ]); + } + operands[tableAt] = entries; + } + output.push(JSON.stringify([op, operands])); + } + return output; +} + +export interface RawAuditResult { + status: 'equivalent' | 'different' | 'dump-failed'; + detail?: string; +} + +async function* linesOf(stream: NodeJS.ReadableStream): AsyncGenerator { + const decoder = new StringDecoder('utf8'); + let rest = ''; + for await (const chunk of stream as AsyncIterable) { + rest += decoder.write(chunk); + let at = rest.indexOf('\n'); + while (at >= 0) { + yield rest.slice(0, at).replace(/\r$/, ''); + rest = rest.slice(at + 1); + at = rest.indexOf('\n'); + } + } + rest += decoder.end(); + if (rest) yield rest; +} + +/** A second, raw pass: no additional compile, and only one function in memory. */ +export async function auditRawHermesBytecode( + command: string, + files: [string, string], + data: [HermesSemanticData, HermesSemanticData], + buffers: [LiteralBuffers, LiteralBuffers], + signal: AbortSignal, +): Promise { + if (data[0].metadata !== data[1].metadata) + return { status: 'different', detail: 'HBC runtime metadata differs' }; + const children = files.map((file) => + spawn( + command, + ['-b', '-dump-bytecode', '-pretty-disassemble=false', file], + { + stdio: ['ignore', 'pipe', 'pipe'], + signal, + killSignal: 'SIGKILL', + }, + ), + ); + const exits = children.map( + (child) => + new Promise((resolve) => { + let stderr = ''; + let processError: Error | undefined; + child.stderr?.on('data', (chunk: Buffer) => { + stderr = (stderr + chunk.toString()).slice(-4096); + }); + child.once('error', (error) => { + child.stdout?.destroy(); + processError = error; + if (!child.pid) resolve(error.message); + }); + child.once('close', (code, reason) => + resolve( + processError + ? processError.message + : code === 0 + ? null + : `${reason ?? `exit ${code}`}: ${stderr.trim().slice(-300)}`, + ), + ); + }), + ); + const functions = async function* (side: number): AsyncGenerator { + let lines: string[] | null = null; + let index = 0; + for await (const line of linesOf(children[side].stdout!)) { + if (/^(?:Function|NCFunction|Constructor) 0 && parsed <= 0x7fffffff + ? parsed + : fallback; +} diff --git a/tests/hermes-base-safety.test.ts b/tests/hermes-base-safety.test.ts new file mode 100644 index 0000000..9fc3ba1 --- /dev/null +++ b/tests/hermes-base-safety.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, test } from 'bun:test'; +import { normalizeDisassemblyLine } from '../src/utils/hermes-base'; + +const normalize = (line: string) => + normalizeDisassemblyLine(line, new Map()); + +describe('Hermes-base normalization must retain semantic differences', () => { + test('does not collapse spaces inside a string literal', () => { + expect(normalize(' LoadConstString r0, "a b"')).not.toBe( + normalize(' LoadConstString r0, "a b"'), + ); + }); + + test('does not erase a changed branch destination', () => { + // In a whole-function regression both labels should already exist and + // retain their locations; retarget a later branch between those labels. + expect(normalize(' JmpTrue L1, r0')).not.toBe( + normalize(' JmpTrue L2, r0'), + ); + }); + + test('negative control: different register operands remain different', () => { + expect(normalize(' Add r0, r1, r2')).not.toBe( + normalize(' Add r0, r1, r3'), + ); + }); + + test('positive control: column padding outside literals may be folded', () => { + expect(normalize(' LoadConstString r0, "same"')).toBe( + normalize(' LoadConstString r0, "same"'), + ); + }); +}); diff --git a/tests/hermes-base.test.ts b/tests/hermes-base.test.ts index 02e9064..78a49ad 100644 --- a/tests/hermes-base.test.ts +++ b/tests/hermes-base.test.ts @@ -661,7 +661,7 @@ describe('helpers', () => { ).toBe(' NewObjectWithBuffer r5 sizes=386'); expect( normalizeDisassemblyLine(' JStrictEqualLong L12, r1, r2', strings), - ).toBe(' JStrictEqual , r1, r2'); + ).toBe(' JStrictEqual L12, r1, r2'); expect( normalizeDisassemblyLine(' DefineOwnById r7, r8, 2, 11591', strings), ).toBe(' DefineOwnById r7, r8, 2, "foo"'); @@ -846,7 +846,7 @@ function legacyNormalize( return `${m[1]}${m[2] ?? ''} ${regs} sizes=${nums.slice(0, 1).join(',')}`; } m = /^(\s*J[A-Za-z]+?)(Long)?\s+(L\d+|\d+)(.*)$/.exec(line); - if (m) return `${m[1]} ${m[4]}`; + if (m) return `${m[1]} ${m[3]}${m[4]}`; m = /^(\s*DefineOwnById\w*\s+r\d+, r\d+, \d+, )(\d+)$/.exec(line); if (m) line = `${m[1]}"${strings.get(Number(m[2])) ?? `?${m[2]}`}"`; m = /^(\s*)([A-Za-z]+?)(?:LongIndex|Long|Short)?(\s+.*|)$/.exec(line); @@ -864,6 +864,7 @@ describe('normalizeDisassemblyLine fast path', () => { const strings = new Map([ [3, 'foo'], [42, 'bar'], + [7, 'known'], ]); const corpus = [ 'Offset in debug table: source 0x0, lexical 0x0', @@ -1082,15 +1083,17 @@ describe.if(os.platform() !== 'win32')( }); afterEach(() => fs.removeSync(dir)); - test('a foreign-base compile is equivalent: ids, widths, buffers and debug tables differ only in representation', async () => { + test('matching text dumps without readable HBC metadata are unverifiable', async () => { const result = await compareHermesBytecode( fakeHermesc, write('delta.hbc', DELTA_DUMP), write('plain.hbc', PLAIN_DUMP), ); - // fake dumps without HBC files behind them: buffers compared as a whole + // Equal human-readable text cannot establish bytecode equivalence. expect(result).toEqual({ - status: 'equivalent', + status: 'dump-failed', + detail: + 'unsupported or unreadable HBC layout; text-only comparison cannot verify equivalence', functions: 2, literals: 'buffer', }); @@ -1100,7 +1103,7 @@ describe.if(os.platform() !== 'win32')( write('d2.hbc', DELTA_DUMP), write('p2.hbc', PLAIN_DUMP), ), - ).toBe(true); + ).toBe(false); }); test("literal buffer content is compared through each side's string table", async () => { @@ -1228,7 +1231,7 @@ describe.if(os.platform() !== 'win32')( write('plain.hbc', PLAIN_DUMP), { dumpTo }, ); - expect(result.status).toBe('equivalent'); + expect(result.status).toBe('dump-failed'); // the tee streams close on their own once the processes exit await new Promise((r) => setTimeout(r, 100)); expect(fs.readFileSync(dumpTo.withBase, 'utf8')).toBe(DELTA_DUMP); diff --git a/tests/hermes-compile.test.ts b/tests/hermes-compile.test.ts index 8b7439b..e27e62b 100644 --- a/tests/hermes-compile.test.ts +++ b/tests/hermes-compile.test.ts @@ -199,6 +199,118 @@ exec "${hermesc}" "$@" expect(leftovers()).toEqual([]); }); + test('a hanging base compile times out and reuses the successful plain compile', async () => { + const wrapperDir = path.join( + dir, + 'node_modules/react-native/sdks/hermesc/linux64-bin', + ); + fs.ensureDirSync(wrapperDir); + const wrapper = path.join(wrapperDir, 'hermesc'); + const calls = path.join(dir, 'deadline-calls'); + fs.writeFileSync( + wrapper, + `#!/bin/sh +echo "$*" >> "${calls}" +case "$*" in *-base-bytecode=*) exec "${process.execPath}" -e 'setInterval(() => {}, 1000)';; esac +exec "${hermesc}" "$@" +`, + { mode: 0o755 }, + ); + const previous = process.env.PUSHY_HERMES_COMPILE_TIMEOUT_MS; + process.env.PUSHY_HERMES_COMPILE_TIMEOUT_MS = '250'; + try { + const result = await compileHermesByteCode({ + bundleName, + outputFolder, + sourcemapOutput: '', + shouldCleanSourcemap: true, + baseRequest: { option: baseHbc, verify: true }, + hermesCommand: wrapper, + }); + expect(result.base).toBeNull(); + expect(result.outcomeDetail).toContain('base compile failed'); + expect( + getHbcVersion(fs.readFileSync(path.join(outputFolder, bundleName))), + ).toBe(probeHbcVersion(hermesc!)!); + const compiles = fs + .readFileSync(calls, 'utf8') + .split('\n') + .filter( + (line) => + line.includes('-emit-binary') && line.includes(outputFolder), + ); + expect(compiles).toHaveLength(2); + expect(leftovers()).toEqual([]); + } finally { + if (previous === undefined) + delete process.env.PUSHY_HERMES_COMPILE_TIMEOUT_MS; + else process.env.PUSHY_HERMES_COMPILE_TIMEOUT_MS = previous; + } + }, 5000); + + test('an early speculative sourcemap rejection is observed before verification finishes', async () => { + const wrapperDir = path.join( + dir, + 'node_modules/react-native/sdks/hermesc/linux64-bin', + ); + const scripts = path.join(dir, 'node_modules/react-native/scripts'); + fs.ensureDirSync(wrapperDir); + fs.ensureDirSync(scripts); + fs.writeJsonSync(path.join(dir, 'node_modules/react-native/package.json'), { + name: 'react-native', + version: '0.77.3', + }); + const marker = path.join(dir, 'compose-attempted'); + // The discarded base's composer fails immediately. After verification + // fails, composing the retained plain output must still succeed. + fs.writeFileSync( + path.join(scripts, 'compose-source-maps.js'), + ` +const fs = require('fs'); +if (!fs.existsSync(${JSON.stringify(marker)})) { + fs.writeFileSync(${JSON.stringify(marker)}, 'first'); + throw new Error('speculative composer failed'); +} +fs.writeFileSync(process.argv[process.argv.indexOf('-o') + 1], '{}'); +`, + ); + const wrapper = path.join(wrapperDir, 'hermesc'); + fs.writeFileSync( + wrapper, + `#!/bin/sh +case "$*" in *-dump-bytecode*) sleep 0.25; exit 3;; esac +exec "${hermesc}" "$@" +`, + { mode: 0o755 }, + ); + const map = path.join(outputFolder, `${bundleName}.map`); + fs.writeFileSync(map, '{}'); + const cwd = process.cwd(); + const unhandled: unknown[] = []; + const onUnhandled = (error: unknown) => unhandled.push(error); + process.on('unhandledRejection', onUnhandled); + process.chdir(dir); + try { + const result = await compileHermesByteCode({ + bundleName, + outputFolder, + sourcemapOutput: map, + shouldCleanSourcemap: true, + baseRequest: { option: baseHbc, verify: true }, + hermesCommand: wrapper, + }); + expect(fs.existsSync(marker)).toBe(true); + expect(result.outcome).toBe('dump-failed'); + expect(result.base).toBeNull(); + expect(unhandled).toEqual([]); + expect(fs.readFileSync(map, 'utf8')).toBe('{}'); + expect(leftovers()).toEqual([]); + } finally { + process.chdir(cwd); + process.off('unhandledRejection', onUnhandled); + } + }, 5000); + test('a selection started ahead of time is consumed by the compile', async () => { const pending = startHermesBaseSelection({ option: baseHbc, diff --git a/tests/hermes-literals.test.ts b/tests/hermes-literals.test.ts index 12a864f..33ec9ca 100644 --- a/tests/hermes-literals.test.ts +++ b/tests/hermes-literals.test.ts @@ -139,12 +139,12 @@ describe('decodeSerializedLiterals', () => { describe('renderLiteral', () => { const strings = new Map([[82, 'color']]); - test('strings resolve through the table; unknown ids stay visible', () => { + test('strings resolve through the table; unknown ids fail closed', () => { expect(renderLiteral({ kind: 'string', id: 82 }, strings)).toBe( '[String "color"]', ); - expect(renderLiteral({ kind: 'string', id: 5 }, strings)).toBe( - '[String ?5]', + expect(() => renderLiteral({ kind: 'string', id: 5 }, strings)).toThrow( + 'unresolved string id 5', ); }); test('doubles keep their bits: -0 and 0 differ, two NaNs agree', () => { @@ -213,14 +213,14 @@ describe('normalizeDisassemblyLine with binary literals', () => { ).toBe(' NewObjectWithBuffer r1 size=1 n=1 {[String "k"]: true}'); }); - test('an undecodable offset is spelled out so it never matches a decoded one', () => { - expect( + test('an undecodable offset fails closed even when both sides are broken', () => { + expect(() => normalizeDisassemblyLine( ' NewArrayWithBuffer r4, 2, 2, 9', strings, resolver, ), - ).toBe(' NewArrayWithBuffer r4 size=2 n=2 []'); + ).toThrow('undecodable'); }); test('without buffers only the size hint survives (whole-buffer fallback)', () => { @@ -305,21 +305,21 @@ describe('normalizeDisassemblyLine with v98 shaped literals', () => { ); }); - test('a shape or key offset out of range is spelled out', () => { - expect( + test('a shape or key offset out of range fails closed', () => { + expect(() => normalizeDisassemblyLine( ' NewObjectWithBuffer r2, 5, 0', strings, resolver, ), - ).toBe(' NewObjectWithBuffer r2 n=? {}'); - expect( + ).toThrow('undecodable'); + expect(() => normalizeDisassemblyLine( ' NewObjectWithBuffer r2, 1, 0', strings, resolver, ), - ).toBe(' NewObjectWithBuffer r2 n=1 {}'); + ).toThrow('undecodable'); }); test('the shape index itself is not compared, only what it points at', () => { diff --git a/tests/hermes-raw.test.ts b/tests/hermes-raw.test.ts new file mode 100644 index 0000000..1e9aff5 --- /dev/null +++ b/tests/hermes-raw.test.ts @@ -0,0 +1,255 @@ +import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; +import { spawnSync } from 'child_process'; +import { createHash } from 'crypto'; +import fs from 'fs-extra'; +import os from 'os'; +import path from 'path'; +import { compareHermesBytecode } from '../src/utils/hermes-base'; +import { readHermesSemanticData } from '../src/utils/hermes-raw'; + +const hermesc = process.env.HERMESC; +const hasHermesc = Boolean(hermesc && fs.existsSync(hermesc)); +const widths: Record = { + Reg8: 1, + Reg32: 4, + UInt8: 1, + UInt16: 2, + UInt32: 4, + Addr8: 1, + Addr32: 4, + Imm32: 4, + Double: 8, +}; + +// Unlike a source-only mutation, changing an HBC operand leaves every function +// body/string table entry in place. This directly tests reference integrity. +describe.if(hasHermesc)('lossless Hermes operand audit (real compiler)', () => { + let dir: string; + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'rnu-hermes-raw-')); + }); + afterEach(() => fs.removeSync(dir)); + + const compile = (name: string, source: string, base?: string) => { + const input = path.join(dir, `${name}.js`); + const output = path.join(dir, `${name}.hbc`); + fs.writeFileSync(input, source); + const result = spawnSync( + hermesc!, + [ + '-emit-binary', + '-O', + '-w', + '-output-source-map', + '-out', + output, + input, + ...(base ? [`-base-bytecode=${base}`] : []), + ], + { encoding: 'utf8', timeout: 10_000 }, + ); + expect(result.status, result.stderr).toBe(0); + return output; + }; + const dump = (file: string, pretty: boolean) => { + const result = spawnSync( + hermesc!, + ['-b', '-dump-bytecode', `-pretty-disassemble=${pretty}`, file], + { encoding: 'utf8', maxBuffer: 10 * 1024 * 1024, timeout: 10_000 }, + ); + expect(result.status, result.stderr).toBe(0); + return result.stdout; + }; + const rewrite = (file: string, bytes: Buffer) => { + // Preserve the HBC integrity footer; only the selected semantic value changes. + createHash('sha1') + .update(bytes.subarray(0, -20)) + .digest() + .copy(bytes, bytes.length - 20); + const changed = `${file}.changed`; + fs.writeFileSync(changed, bytes); + return changed; + }; + const operands = async (file: string, matches: (op: string) => boolean) => { + const data = await readHermesSemanticData(file); + let functionIndex = -1; + const found: { + opcode: string; + positions: number[]; + values: number[]; + types: string[]; + }[] = []; + for (const line of dump(file, false).split('\n')) { + if (/^(?:Function|NCFunction|Constructor)]+)<(\w+)>/g)) { + positions.push(position); + values.push(Number(operand[1].trim())); + types.push(operand[2]); + position += widths[operand[2]]; + } + found.push({ opcode: m[2], positions, values, types }); + } + return { data, found }; + }; + + test.each([ + [ + 'long ASCII', + 'common-prefix-longer-than-the-pretty-limit-A', + 'common-prefix-longer-than-the-pretty-limit-B', + ], + ['UTF-16', '中华人民共和国教育科学研究甲', '中华人民共和国教育科学研究乙'], + ['literal escape versus control byte', '\\x00', '\u0000'], + ['significant whitespace', 'a b', 'a b'], + ['escaped quote and whitespace', 'prefix"a b', 'prefix"a b'], + ])('%s strings must compare by full value', async (_name, left, right) => { + const a = compile('a', `globalThis.value = ${JSON.stringify(left)};`); + const b = compile('b', `globalThis.value = ${JSON.stringify(right)};`); + expect((await compareHermesBytecode(hermesc!, a, b)).status).toBe( + 'different', + ); + }); + + test('same-name closures cannot hide a changed function reference', async () => { + const file = compile( + 'closures', + 'globalThis.fs = [function same(){print(1);}, function same(){print(2);}];', + ); + const { data, found } = await operands(file, (op) => + op.startsWith('CreateClosure'), + ); + const candidates = found.filter( + (inst) => + JSON.parse(data.functions[inst.values[2]].metadata)[0] === 'same', + ); + expect(candidates).toHaveLength(2); + const [first, second] = candidates; + const bytes = Buffer.from(data.bytes); + bytes.writeUIntLE( + second.values[2], + first.positions[2], + widths[first.types[2]], + ); + const changed = rewrite(file, bytes); + expect(dump(changed, true)).toBe(dump(file, true)); + const result = await compareHermesBytecode(hermesc!, changed, file); + expect(result.status).toBe('different'); + expect(result.detail).toContain('raw instruction'); + }); + + test('the IEEE-754 sign of zero is retained even when both dumps print 0', async () => { + const file = compile('zero', 'globalThis.x = -0;'); + const { data, found } = await operands( + file, + (op) => op === 'LoadConstDouble', + ); + expect(found.length).toBeGreaterThan(0); + const bytes = Buffer.from(data.bytes); + bytes.writeDoubleLE(0, found[0].positions[1]); + const changed = rewrite(file, bytes); + expect(dump(changed, true)).toBe(dump(file, true)); + const result = await compareHermesBytecode(hermesc!, changed, file); + expect(result.status).toBe('different'); + expect(result.detail).toContain('raw instruction'); + }); + + test('long BigInts differ after the human-readable prefix', async () => { + const prefix = '1234567890'.repeat(30); + const a = compile('big-a', `globalThis.x = ${prefix}1n;`); + const b = compile('big-b', `globalThis.x = ${prefix}2n;`); + expect((await compareHermesBytecode(hermesc!, a, b)).status).toBe( + 'different', + ); + }); + + test('header runtime flags are checked rather than discarded with debug metadata', async () => { + const file = compile( + 'strictness', + 'globalThis.x = function f(){return this;};', + ); + const data = await readHermesSemanticData(file); + const bytes = Buffer.from(data.bytes); + const entrySize = data.version === 98 ? 12 : 16; + bytes[128 + entrySize - 1] ^= 4; // global function strictMode + const result = await compareHermesBytecode( + hermesc!, + rewrite(file, bytes), + file, + ); + expect(result.status).toBe('different'); + }); + + test('switch tables remain equivalent against a foreign base', async () => { + const base = compile( + 'base', + `globalThis.strings = ${JSON.stringify(Array.from({ length: 400 }, (_, i) => `foreign${i}`))};`, + ); + const numbers = Array.from( + { length: 150 }, + (_, i) => `case ${i}: return ${i * i + 19};`, + ).join('\n'); + const strings = Array.from( + { length: 100 }, + (_, i) => `case 'common-prefix-long-string-${i}': return ${i * i + 31};`, + ).join('\n'); + const source = `globalThis.n = function(x){switch(x){${numbers} default: return -1;}};\nglobalThis.s = function(x){switch(x){${strings} default: return -2;}};`; + const plain = compile('plain', source); + const delta = compile('delta', source, base); + expect((await compareHermesBytecode(hermesc!, delta, plain)).status).toBe( + 'equivalent', + ); + }); + + test('overflow function headers retain the same runtime fields', async () => { + const base = compile('header-base', 'globalThis.old = "older strings";'); + const params = Array.from({ length: 140 }, (_, i) => `p${i}`).join(','); + const source = `globalThis.f = function many(${params}){print(p0, p139);};`; + const plain = compile('header-plain', source); + const delta = compile('header-delta', source, base); + expect((await compareHermesBytecode(hermesc!, delta, plain)).status).toBe( + 'equivalent', + ); + }); + + test('truncated raw output is unverifiable, not equivalent', async () => { + const file = compile('short', 'globalThis.x = "hello";'); + const wrapper = path.join(dir, 'short-dump'); + fs.writeFileSync( + wrapper, + `#!/bin/sh\ncase "$*" in *pretty-disassemble=false*) echo 'Function(1 params, 3 registers):'; exit 0;; esac\nexec "${hermesc}" "$@"\n`, + { mode: 0o755 }, + ); + const result = await compareHermesBytecode(wrapper, file, file); + expect(result.status).toBe('dump-failed'); + expect(result.detail).toContain('raw dump ended before'); + }); + + test('the deadline also covers the second, raw dump pass', async () => { + const file = compile('raw-timeout', 'print(1);'); + const wrapper = path.join(dir, 'raw-hangs'); + fs.writeFileSync( + wrapper, + `#!/bin/sh\ncase "$*" in *pretty-disassemble=false*) exec "${process.execPath}" -e 'setInterval(() => {}, 1000)';; esac\nexec "${hermesc}" "$@"\n`, + { mode: 0o755 }, + ); + const result = await compareHermesBytecode(wrapper, file, file, { + timeoutMs: 100, + }); + expect(result.status).toBe('dump-failed'); + }, 2000); + + test('an unreadable debug-dump destination fails safely', async () => { + const file = compile('debug-output', 'print(1);'); + const result = await compareHermesBytecode(hermesc!, file, file, { + dumpTo: { withBase: dir, plain: dir }, + timeoutMs: 1000, + }); + expect(result.status).toBe('dump-failed'); + }); +}); diff --git a/tests/hermes-timeout.test.ts b/tests/hermes-timeout.test.ts new file mode 100644 index 0000000..e390d8b --- /dev/null +++ b/tests/hermes-timeout.test.ts @@ -0,0 +1,102 @@ +import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; +import fs from 'fs-extra'; +import os from 'os'; +import path from 'path'; +import { + compareHermesBytecode, + probeHbcVersion, +} from '../src/utils/hermes-base'; +import { hermesTimeout } from '../src/utils/hermes-timeout'; + +const keys = ['PUSHY_CACHE_DIR', 'PUSHY_HERMES_PROBE_TIMEOUT_MS'] as const; +describe('Hermes deadlines', () => { + test('invalid or overflowing values cannot turn deadlines into a 1ms timer', () => { + for (const value of [ + undefined, + '', + '0', + '-1', + 'NaN', + 'Infinity', + '1.5', + '2147483648', + ]) { + expect(hermesTimeout(value, 2000)).toBe(2000); + } + expect(hermesTimeout('50', 2000)).toBe(50); + }); +}); + +describe.if(process.platform !== 'win32')('Hermes subprocess deadlines', () => { + let dir: string; + let hanging: string; + const saved = Object.fromEntries(keys.map((key) => [key, process.env[key]])); + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'rnu-hermes-timeout-')); + process.env.PUSHY_CACHE_DIR = path.join(dir, 'cache'); + hanging = path.join(dir, 'hermesc'); + // exec, not a shell child: the killed process owns stdout and cannot leave + // an inherited pipe open to prevent close/finish from settling. + fs.writeFileSync( + hanging, + `#!/bin/sh\nexec "${process.execPath}" -e 'setInterval(() => {}, 1000)'\n`, + { mode: 0o755 }, + ); + }); + afterEach(() => { + for (const key of keys) { + if (saved[key] === undefined) delete process.env[key]; + else process.env[key] = saved[key]; + } + fs.removeSync(dir); + }); + + test('a hung synchronous version probe times out', () => { + process.env.PUSHY_HERMES_PROBE_TIMEOUT_MS = '50'; + expect(probeHbcVersion(hanging)).toBeNull(); + }, 2000); + + test('hung pretty dumps time out and both children are reaped', async () => { + const result = await compareHermesBytecode( + hanging, + 'missing-a', + 'missing-b', + { timeoutMs: 50 }, + ); + expect(result.status).toBe('dump-failed'); + }, 2000); + + test('external cancellation aborts active dump processes', async () => { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 50); + try { + const result = await compareHermesBytecode( + hanging, + 'missing-a', + 'missing-b', + { + signal: controller.signal, + timeoutMs: 1000, + }, + ); + expect(result.status).toBe('dump-failed'); + } finally { + clearTimeout(timer); + } + }, 2000); + + test('an already-aborted request fails without an unhandled spawn error', async () => { + const controller = new AbortController(); + controller.abort(); + const result = await compareHermesBytecode( + hanging, + 'missing-a', + 'missing-b', + { + signal: controller.signal, + timeoutMs: 500, + }, + ); + expect(result.status).toBe('dump-failed'); + }, 2000); +}); From 20eaecb4b95a0f1b33a7d7ee322a89d1eb4407fe Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 12 Sep 2026 07:16:06 +0000 Subject: [PATCH 2/4] test(hermes-base): isolate late errors and preserve spacing semantics --- src/utils/hermes-base.ts | 17 ++++++-- src/utils/hermes-raw.ts | 13 +++++- tests/fixtures/hermes-async-check.cjs | 61 +++++++++++++++++++++++++++ tests/hermes-base-safety.test.ts | 58 +++++++++++++++++++++++++ tests/hermes-compile.test.ts | 58 ++++++++++++++----------- tests/hermes-timeout.test.ts | 52 +++++++++++++++++------ 6 files changed, 219 insertions(+), 40 deletions(-) create mode 100644 tests/fixtures/hermes-async-check.cjs diff --git a/src/utils/hermes-base.ts b/src/utils/hermes-base.ts index 54ff683..2128861 100644 --- a/src/utils/hermes-base.ts +++ b/src/utils/hermes-base.ts @@ -1150,13 +1150,18 @@ export function normalizeDisassemblyLine( return line; } -/** Only formatting outside quoted operands may be collapsed. */ +/** + * Collapse formatting outside quoted operands without altering literal code units. + * ASCII dump spacing takes the char-code fast path; uncommon Unicode characters + * retain the complete runtime `\s` semantics instead of a narrower space list. + */ function normalizeOperandSpacing(text: string): string { let result = ''; let quoted = false; let escaped = false; let spacing = false; - for (const char of text) { + for (let i = 0; i < text.length; i++) { + const char = text[i]; if (quoted) { result += char; if (escaped) escaped = false; @@ -1164,7 +1169,12 @@ function normalizeOperandSpacing(text: string): string { else if (char === '"') quoted = false; continue; } - if (/\s/.test(char)) { + const code = text.charCodeAt(i); + const whitespace = + code <= 0x7f + ? isSpace(code) || (code >= 0x0a && code <= 0x0c) + : /\s/.test(char); + if (whitespace) { if (!spacing) result += ' '; spacing = true; } else { @@ -1343,6 +1353,7 @@ class DumpReader { }); } + /** Consume lookahead first, then the stream; null is reserved for EOF. */ private async nextLine(): Promise { if (this.pending !== null) { const line = this.pending; diff --git a/src/utils/hermes-raw.ts b/src/utils/hermes-raw.ts index 33e00af..3239ed6 100644 --- a/src/utils/hermes-raw.ts +++ b/src/utils/hermes-raw.ts @@ -16,6 +16,7 @@ import { type LiteralBuffers, LiteralResolver } from './hermes-literals'; export class UnverifiableHermesBytecode extends Error {} +/** Fail closed on missing references rather than comparing equal placeholders. */ function requireValue(value: T | null | undefined, what: string): T { if (value === null || value === undefined) { throw new UnverifiableHermesBytecode(what); @@ -23,6 +24,7 @@ function requireValue(value: T | null | undefined, what: string): T { return value; } +/** Bound a reference to its owning section before creating a zero-copy view. */ function checkedSlice(data: Buffer, start: number, length: number): Buffer { if ( !Number.isSafeInteger(start) || @@ -248,10 +250,15 @@ interface RawInstruction { size: number; } +/** Remove encoding-width suffixes; operand values remain part of the audit. */ function foldWidth(opcode: string): string { return opcode.replace(/(?:LongIndex|Long|Short)$/, ''); } +/** + * Read raw operand boundaries and verify integer values against the HBC bytes. + * Doubles use their exact bits, not the rounded number printed by hermesc. + */ function parseRawInstruction( line: string, data: HermesSemanticData, @@ -460,6 +467,7 @@ export interface RawAuditResult { detail?: string; } +/** Decode split UTF-8 sequences without buffering the whole raw dump. */ async function* linesOf(stream: NodeJS.ReadableStream): AsyncGenerator { const decoder = new StringDecoder('utf8'); let rest = ''; @@ -476,7 +484,10 @@ async function* linesOf(stream: NodeJS.ReadableStream): AsyncGenerator { if (rest) yield rest; } -/** A second, raw pass: no additional compile, and only one function in memory. */ +/** + * Audit a pair of raw dumps while retaining one function per side plus HBC data. + * Cancellation/errors fail closed; both subprocesses are terminated and reaped. + */ export async function auditRawHermesBytecode( command: string, files: [string, string], diff --git a/tests/fixtures/hermes-async-check.cjs b/tests/fixtures/hermes-async-check.cjs new file mode 100644 index 0000000..fcdc283 --- /dev/null +++ b/tests/fixtures/hermes-async-check.cjs @@ -0,0 +1,61 @@ +/** + * Run async-error regressions outside the test runner under either Bun or Node. + * Listeners remain installed until natural exit: even errors after the result + * marker make the child fail. A parent deadline also detects leaked handles. + */ +const assert = require('node:assert/strict'); +const path = require('node:path'); + +const unexpected = []; +/** Record both Promise rejections and uncaught ChildProcess error events. */ +function recordUnexpected(kind, error) { + unexpected.push(kind); + process.exitCode = 1; + console.error(`HERMES_ASYNC_ERROR ${kind}: ${String(error)}`); +} +process.on('unhandledRejection', (error) => + recordUnexpected('unhandledRejection', error), +); +process.on('uncaughtException', (error) => + recordUnexpected('uncaughtException', error), +); + +/** Exercise the actual API, then allow timers and immediates to report errors. */ +async function main() { + const config = JSON.parse(process.argv[2]); + if (config.cwd) process.chdir(config.cwd); + let result; + if (config.operation === 'abort') { + const { compareHermesBytecode } = require(path.resolve(config.modulePath)); + const controller = new AbortController(); + controller.abort(); + result = await compareHermesBytecode( + config.command || process.execPath, + 'missing-a', + 'missing-b', + { signal: controller.signal, timeoutMs: 500 }, + ); + assert.equal(result.status, 'dump-failed'); + assert.match(result.detail, /abort/i); + } else if (config.operation === 'compile') { + const { compileHermesByteCode } = require(path.resolve(config.modulePath)); + result = await compileHermesByteCode(config.options); + } else if (config.operation === 'control-rejection') { + setImmediate(() => Promise.reject(new Error('intentional rejection'))); + } else if (config.operation === 'control-exception') { + setImmediate(() => { + throw new Error('intentional exception'); + }); + } else { + throw new Error(`Unknown operation: ${config.operation}`); + } + // Checking only immediately after await can miss later-turn error delivery. + await new Promise((resolve) => setTimeout(resolve, 0)); + await new Promise((resolve) => setImmediate(resolve)); + assert.deepEqual(unexpected, []); + console.log(`HERMES_ASYNC_RESULT ${JSON.stringify(result)}`); +} +main().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/tests/hermes-base-safety.test.ts b/tests/hermes-base-safety.test.ts index 9fc3ba1..0497b12 100644 --- a/tests/hermes-base-safety.test.ts +++ b/tests/hermes-base-safety.test.ts @@ -31,3 +31,61 @@ describe('Hermes-base normalization must retain semantic differences', () => { ); }); }); + +/** Previous spacing implementation, kept as an independent compatibility oracle. */ +function referenceSpacing(text: string): string { + let result = ''; + let quoted = false; + let escaped = false; + let spacing = false; + for (const char of text) { + if (quoted) { + result += char; + if (escaped) escaped = false; + else if (char === '\\') escaped = true; + else if (char === '"') quoted = false; + continue; + } + if (/\s/.test(char)) { + if (!spacing) result += ' '; + spacing = true; + } else { + result += char; + spacing = false; + if (char === '"') quoted = true; + } + } + return result; +} + +describe('Hermes operand spacing compatibility', () => { + test('matches the previous whitespace behavior for every UTF-16 code unit', () => { + const mismatches: number[] = []; + for (let code = 0; code <= 0xffff; code++) { + const operands = ` r0, ${String.fromCharCode(code)} r1`; + if ( + normalize(` Mov${operands}`) !== + ` Mov${referenceSpacing(operands)}` + ) { + mismatches.push(code); + } + } + expect(mismatches).toEqual([]); + }); + + test('preserves whitespace, escapes and surrogate pairs inside quotes', () => { + const whitespace = + '\t\n\v\f\r \u00a0\u1680\u2000\u2028\u2029\u202f\u205f\u3000\ufeff'; + const operands = [ + ` r0, "a${whitespace}b"`, + String.raw` r0, "a\" b\\ c", r1`, + ` r0, "😀 𠮷\ud800\udfff",\u00a0\ufeffr1`, + ' r0, \u0085\u180e\u200b r1', + ]; + for (const operand of operands) { + expect(normalize(` Mov${operand}`)).toBe( + ` Mov${referenceSpacing(operand)}`, + ); + } + }); +}); diff --git a/tests/hermes-compile.test.ts b/tests/hermes-compile.test.ts index e27e62b..b0a3467 100644 --- a/tests/hermes-compile.test.ts +++ b/tests/hermes-compile.test.ts @@ -285,30 +285,40 @@ exec "${hermesc}" "$@" ); const map = path.join(outputFolder, `${bundleName}.map`); fs.writeFileSync(map, '{}'); - const cwd = process.cwd(); - const unhandled: unknown[] = []; - const onUnhandled = (error: unknown) => unhandled.push(error); - process.on('unhandledRejection', onUnhandled); - process.chdir(dir); - try { - const result = await compileHermesByteCode({ - bundleName, - outputFolder, - sourcemapOutput: map, - shouldCleanSourcemap: true, - baseRequest: { option: baseHbc, verify: true }, - hermesCommand: wrapper, - }); - expect(fs.existsSync(marker)).toBe(true); - expect(result.outcome).toBe('dump-failed'); - expect(result.base).toBeNull(); - expect(unhandled).toEqual([]); - expect(fs.readFileSync(map, 'utf8')).toBe('{}'); - expect(leftovers()).toEqual([]); - } finally { - process.chdir(cwd); - process.off('unhandledRejection', onUnhandled); - } + const child = spawnSync( + process.execPath, + [ + path.join(__dirname, 'fixtures/hermes-async-check.cjs'), + JSON.stringify({ + operation: 'compile', + modulePath: require.resolve('../src/bundle-runner'), + cwd: dir, + options: { + bundleName, + outputFolder, + sourcemapOutput: map, + shouldCleanSourcemap: true, + baseRequest: { option: baseHbc, verify: true }, + hermesCommand: wrapper, + }, + }), + ], + { encoding: 'utf8', timeout: 4000 }, + ); + expect(child.error).toBeUndefined(); + expect(child.signal).toBeNull(); + expect(child.status).toBe(0); + expect(child.stderr).not.toContain('HERMES_ASYNC_ERROR'); + const line = child.stdout + .split('\n') + .find((value) => value.startsWith('HERMES_ASYNC_RESULT ')); + expect(line).toBeDefined(); + const result = JSON.parse(line!.slice('HERMES_ASYNC_RESULT '.length)); + expect(fs.existsSync(marker)).toBe(true); + expect(result.outcome).toBe('dump-failed'); + expect(result.base).toBeNull(); + expect(fs.readFileSync(map, 'utf8')).toBe('{}'); + expect(leftovers()).toEqual([]); }, 5000); test('a selection started ahead of time is consumed by the compile', async () => { diff --git a/tests/hermes-timeout.test.ts b/tests/hermes-timeout.test.ts index e390d8b..91ffa96 100644 --- a/tests/hermes-timeout.test.ts +++ b/tests/hermes-timeout.test.ts @@ -1,4 +1,5 @@ import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; +import { spawnSync } from 'child_process'; import fs from 'fs-extra'; import os from 'os'; import path from 'path'; @@ -85,18 +86,45 @@ describe.if(process.platform !== 'win32')('Hermes subprocess deadlines', () => { } }, 2000); - test('an already-aborted request fails without an unhandled spawn error', async () => { - const controller = new AbortController(); - controller.abort(); - const result = await compareHermesBytecode( - hanging, - 'missing-a', - 'missing-b', - { - signal: controller.signal, - timeoutMs: 500, - }, + test('an already-aborted request has no late unhandled errors in an isolated process', () => { + const child = spawnSync( + process.execPath, + [ + path.join(__dirname, 'fixtures/hermes-async-check.cjs'), + JSON.stringify({ + operation: 'abort', + modulePath: require.resolve('../src/utils/hermes-base'), + command: hanging, + }), + ], + { encoding: 'utf8', timeout: 1500 }, ); - expect(result.status).toBe('dump-failed'); + expect(child.error).toBeUndefined(); + expect(child.signal).toBeNull(); + expect(child.status).toBe(0); + expect(child.stderr).not.toContain('HERMES_ASYNC_ERROR'); + expect(child.stdout).toContain('"status":"dump-failed"'); }, 2000); }); + +describe('isolated async-error observer negative controls', () => { + for (const [operation, event] of [ + ['control-rejection', 'unhandledRejection'], + ['control-exception', 'uncaughtException'], + ]) { + test(`fails for a late ${event}`, () => { + const child = spawnSync( + process.execPath, + [ + path.join(__dirname, 'fixtures/hermes-async-check.cjs'), + JSON.stringify({ operation }), + ], + { encoding: 'utf8', timeout: 1500 }, + ); + expect(child.error).toBeUndefined(); + expect(child.signal).toBeNull(); + expect(child.status).toBe(1); + expect(child.stderr).toContain(`HERMES_ASYNC_ERROR ${event}`); + }, 2000); + } +}); From 5e1943d7e4c44d1d76fb032ddc5b2ddb951c965b Mon Sep 17 00:00:00 2001 From: Sunny Luo Date: Sat, 12 Sep 2026 15:16:53 +0800 Subject: [PATCH 3/4] ci(hermes-base): check late abort errors on Node 18 --- .github/workflows/test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5cf9b82..34da7f4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -102,6 +102,11 @@ jobs: - name: Load every built module and run the offline commands run: node scripts/smoke-lib.js + - name: Check late abort errors on the oldest supported Node.js + run: >- + node tests/fixtures/hermes-async-check.cjs + '{"operation":"abort","modulePath":"./lib/utils/hermes-base.js"}' + publish-dry-run: runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 10 From fa8dcf22d64f76f0ab8878081f9a6aef33961386 Mon Sep 17 00:00:00 2001 From: Sunny Luo Date: Sat, 12 Sep 2026 21:51:28 +0800 Subject: [PATCH 4/4] fix(hermes-base): handle empty functions and debug dump failures Accept declared zero-byte functions without weakening non-empty body or function-count completeness checks. End every debug sink on spawn errors and cancellation before waiting for stream completion. Resolve restricted-global string operands and normalize classic SwitchImm physical table offsets while preserving control-flow targets. Document the specific v98 function-header snapshot rather than assuming all HBC 98 compilers share its layout. Add isolated debug-output timeout/abort/pre-abort/ENOENT regressions, real classic lexical-declaration coverage, and pinned real Metro self and base/plain comparisons. Extend CI to hermes-compiler 250829098.0.17 and run debug failure checks on Node 18. Validated full suites with real Metro fixtures: HBC96 548 passed; HBC98 .16/.17 each 547 passed, one classic-only case skipped, zero failures. Lint, typecheck, build and Node 22 debug/smoke checks passed. --- .github/workflows/test.yml | 30 ++++- docs/hermes-base-verification.md | 12 +- src/utils/hermes-base.ts | 13 ++- src/utils/hermes-raw.ts | 14 ++- tests/fixtures/hermes-async-check.cjs | 19 ++++ tests/hermes-blockers.test.ts | 153 ++++++++++++++++++++++++++ tests/hermes-metro.test.ts | 84 ++++++++++++++ tests/hermes-raw.test.ts | 34 +++++- 8 files changed, 347 insertions(+), 12 deletions(-) create mode 100644 tests/hermes-blockers.test.ts create mode 100644 tests/hermes-metro.test.ts diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 34da7f4..6dec9d1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -107,6 +107,11 @@ jobs: node tests/fixtures/hermes-async-check.cjs '{"operation":"abort","modulePath":"./lib/utils/hermes-base.js"}' + - name: Check debug-output failure cleanup on Node 18 + env: + HERMES_TEST_NODE: node + run: bun test tests/hermes-blockers.test.ts + publish-dry-run: runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 10 @@ -141,7 +146,7 @@ jobs: run: npm publish --dry-run --access public --tag dry-run hermes-integration: - name: hermes-hbc-${{ matrix.hbc }} + name: hermes-hbc-${{ matrix.hbc }}${{ matrix.suffix }} runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 15 strategy: @@ -149,13 +154,20 @@ jobs: matrix: include: - hbc: 96 + suffix: "" package: react-native@0.77.3 directory: react-native executable: sdks/hermesc/linux64-bin/hermesc - hbc: 98 + suffix: "" package: hermes-compiler@250829098.0.16 directory: hermes-compiler executable: hermesc/linux64-bin/hermesc + - hbc: 98 + suffix: -patch17 + package: hermes-compiler@250829098.0.17 + directory: hermes-compiler + executable: hermesc/linux64-bin/hermesc steps: - uses: actions/checkout@v7 with: @@ -184,6 +196,20 @@ jobs: "$HERMESC" -version bun -e 'import {probeHbcVersion} from "./src/utils/hermes-base"; if (probeHbcVersion(process.env.HERMESC) !== Number(process.env.EXPECTED_HBC)) throw new Error("unexpected HBC version");' echo "HERMESC=$HERMESC" >> "$GITHUB_ENV" + - name: Fetch pinned real Metro fixtures + shell: bash + run: | + set -euo pipefail + root="$RUNNER_TEMP/hermes-metro-fixtures" + mkdir -p "$root" + base="https://raw.githubusercontent.com/sunnylqm/hbc-diff-benchmark/e6a870a1c26c4b64c7860d7e1aa575707d22ad88" + for file in base.jsbundle s3-medium-feature.jsbundle; do + curl --fail --location --retry 2 --max-time 60 "$base/fixtures/$file" -o "$root/$file" + done + curl --fail --location --retry 2 --max-time 60 "$base/LICENSE" -o "$root/LICENSE" + echo "11c8ad8f7e8c7c59ee45582c77d896a35fa646617f3ba0f5b338a425a7c93b7d $root/base.jsbundle" | sha256sum --check + echo "a693e68254b6c13fae8f839d20c14f9d11c5ab98d4be1b8d13ba1e929a12d752 $root/s3-medium-feature.jsbundle" | sha256sum --check + echo "HERMES_METRO_FIXTURES=$root" >> "$GITHUB_ENV" - name: Run real compiler and fallback regressions run: bun test tests/hermes-*.test.ts - name: Run seeded differential fuzzing @@ -192,6 +218,6 @@ jobs: if: failure() uses: actions/upload-artifact@v7 with: - name: hermes-fuzz-hbc-${{ matrix.hbc }} + name: hermes-fuzz-hbc-${{ matrix.hbc }}${{ matrix.suffix }} path: ${{ runner.temp }}/hermes-fuzz if-no-files-found: ignore diff --git a/docs/hermes-base-verification.md b/docs/hermes-base-verification.md index ab769f4..8a4d93b 100644 --- a/docs/hermes-base-verification.md +++ b/docs/hermes-base-verification.md @@ -37,14 +37,18 @@ v98 的 shape 索引和 offset 一样只用于定位（delta 可能重排 shape 为什么不能按 dump 的整段文本比：Hermes 的缓冲区构建器会**重叠/去重**序列化后的字面量——一个字面量的最后一个值字节可以同时是下一个字面量的 tag 字节（模糊测试实测：`61 52 | cd 09 b3 05 11`，前一段以 `[String 82]` 结尾，后一条指令的 offset 正指向 `52`）。顺序解析整段缓冲区（hermesc 的 dump 就是这么打印的）从这里开始失步，之后的条目全是噪声；delta 构建的 id 宽度不同，重叠位置也不同，于是两段"噪声"在某处不一致就被判为差异。2026-09-10 的 20 轮冒烟模糊测试里 3 次误杀全部源于此，改按指令比较后全部等价。无法读二进制缓冲区（文件结构不识别）时两侧一起比较整段文本，仅辅助诊断；即使文本相等也返回 `dump-failed` 并回退 plain，不能以丢失 offset/count 的文本确认等价。结果里 `literals: 'buffer'` 标明这一点。 -`normalizeDisassemblyLine` 只折叠表示层差异：按指令解析后的字面量地址、已知宽度后缀、引号外的列对齐空白、switch 表的物理偏移、debug 偏移。字符串内部的连续空格、跳转目标标签、寄存器均保留。未知 string ID、无法解码的字面量、未知 buffer 操作数形态直接失败；两侧都无法解析也不等价。 +`normalizeDisassemblyLine` 只折叠表示层差异：按指令解析后的字面量地址、已知宽度后缀、引号外的列对齐空白、switch 表的物理偏移（含经典 `SwitchImm`）、debug 偏移。字符串内部的连续空格、跳转目标标签、寄存器均保留。未知 string ID、无法解码的字面量、未知 buffer 操作数形态直接失败；两侧都无法解析也不等价。 **原始操作数核对**：`hermes-raw.ts` 从 raw dump 读取指令起点和操作数类型，并检查操作数与 HBC 字节一致、指令覆盖完整函数体。字符串从 small/overflow string table 与 string storage 按完整 ASCII/UTF-16 code unit 解码；BigInt、正则和 double 读取真实字节（保留 `-0` 和尾部精度）；函数引用保留索引，与顺序对齐的函数表共同检查，同名函数不能互换。地址映射为目标指令序号；整数和字符串 switch 从二进制恢复 case 值与目的地。函数运行时 flags 和参数/寄存器等字段也参与比较，剔除的仅是物理地址、debug presence 与 compact/overflow 表示。 -这不是一个可以忽略所有新指令的通用语义证明器。支持范围限定为已实现的 HBC v87–96、v98；升级布局或引入新的字符串/shape 引用指令时，需要复核 `hermes-raw.ts` 的解码规则及测试，不可仅扩展 diff-transform 的布局表。 +这不是一个可以忽略所有新指令的通用语义证明器。已验证的函数头范围为 HBC v87–96，以及 `hermes-compiler@250829098.0.16/.17` 的 v98 快照；升级布局或引入新的字符串/shape 引用指令时，需要复核 `hermes-raw.ts` 的解码规则及测试，不可仅扩展 diff-transform 的布局表。 结果三态：`equivalent` / `different`（带第一处差异：函数、行号、两侧内容，或缓冲区条目）/ `dump-failed`（无法解析完整语义数据、dump 进程失败/超时/提前结束；带原因或 stderr 末行）。后两种都放弃 base，但日志分开。 +**零长度函数**：Static Hermes 的真实 Metro 产物可能保留 `bytecodeSizeInBytes == 0` 的死函数。raw 校验要求实际指令字节总长等于函数头声明长度，而不是要求每个函数至少有一条指令；空函数仍比较运行时元数据，遗漏整个函数头仍由总函数数检查拒绝。非空函数的指令被截断仍为 `dump-failed`。 + +**v98 函数头快照边界**：当前大头按 37 字节、flags 位于 `[36]` 读取，小头 cache 位域为 6/1/1，绑定 `250829098` 稳定快照。上游 [7193d4485b](https://github.com/facebook/hermes/commit/7193d4485beeb87cd7a3b6ca8b6b5d97a1a433c4) 删除 `NumCacheNewObject` 后，仍报 v98 的构建曾使用 36 字节 / flags `[35]`、cache 位域 7/1。`hbcTransform` 的两套 v98 文件头布局不能识别这次**函数头**变化；本轮不宣称支持该后续快照，也不能仅凭文件头或版本号选择它。更换编译器时必须补对应的大头、小头和真实 Metro 测试；未审核的 v98 构建应使用 `--hermesBase none`。 + pretty 输出本身会截断长字符串与 BigInt、用函数名替代函数索引，并可能把 `-0` 显示为 `0`，因此不再以 pretty 相等作为最终结论。新增归一化规则时必须同时添加真实 HBC 负例，确保没有把语义差异折叠掉。 ## 3. 已完成与待办 @@ -82,7 +86,9 @@ raw 核对保留函数引用索引，与出现顺序对齐的函数体及二进 - **资源开销**：base/plain 编译并发完成后，执行 pretty 和 raw 两遍 dump；每遍两个进程，raw 不增加编译。二进制元数据读取目前持有两份 HBC、完整字符串映射及字面量缓冲区，反汇编仅保留当前函数；这是完整数据核对的额外内存和时间开销。不要以删除验证数据来优化内存，可后续改为按段读取或降低并行度。 - **进程期限**：版本探测默认 30 秒（`PUSHY_HERMES_PROBE_TIMEOUT_MS`），完整校验两遍合计默认 120 秒（`PUSHY_HERMES_VERIFY_TIMEOUT_MS`），单个编译/源码映射子进程默认 300 秒（`PUSHY_HERMES_COMPILE_TIMEOUT_MS`）。环境变量单位均为毫秒，必须是 1–2147483647 的整数，否则用默认值。超时终止子进程，优化失败回退 plain；真正的 plain 编译或最终 sourcemap 失败仍使构建失败。校验函数还接受 `AbortSignal`；base 下载任务的取消传播尚未统一。 - **源码映射竞态**：推测执行的 base sourcemap 合成任务启动时立即观察拒绝，之后再根据最终采用哪份字节码决定抛出错误还是为 plain 重做合成。 -- **CI**：`hermes-hbc-96` / `hermes-hbc-98` job 分别安装固定的 `react-native@0.77.3` / `hermes-compiler@250829098.0.16`，校验可执行文件与真实 HBC 版本后运行 Hermes 回归和 50 轮固定种子 fuzz；缺少编译器会失败，不静默跳过。 +- **CI**：`hermes-hbc-96` / `hermes-hbc-98` / `hermes-hbc-98-patch17` 分别安装固定的 `react-native@0.77.3` / `hermes-compiler@250829098.0.16` / `.17`。除小程序回归与 50 轮固定种子 fuzz 外，三组都执行真实 Metro bundle 的自比和 base/plain 比较；v98 测试断言产物确实包含零长度函数。缺编译器或指定的 fixture 缺失/哈希不符会失败，不静默跳过。 +- **真实 Metro 数据**：[hbc-diff-benchmark](https://github.com/sunnylqm/hbc-diff-benchmark/tree/e6a870a1c26c4b64c7860d7e1aa575707d22ad88) 的 `base.jsbundle` 和 `s3-medium-feature.jsbundle`（MIT，保留其 LICENSE；来源与生成步骤见该仓库 `fixtures/GENERATION.md`）。CI 固定提交和 SHA256，不运行下载的 JS，仅交给固定编译器。离线运行：`HERMESC= HERMES_METRO_FIXTURES= bun test tests/hermes-metro.test.ts`；普通单元测试不联网下载。 +- **调试输出异常回归**：`tests/hermes-blockers.test.ts` 在独立进程覆盖带 `dumpTo` 的超时、运行中取消、预取消、ENOENT，同时检查返回标记、退出状态和晚到的异步错误。错误处理立即 unpipe 全部目的地并结束 debug 文件，避免 `finish()` 等待一个只会在后续 `kill()` 中结束的流；Node 18 CI 也执行相同场景。 - **`hbcdump`/`hbc-diff`**：Hermes 仓库自带的工具，RN 的 hermesc 不随附；如果将来 hermes-compiler 包里带上，可替代文本 dump 解析。 ## 4. 明确接受的剩余风险 diff --git a/src/utils/hermes-base.ts b/src/utils/hermes-base.ts index 2128861..93971a8 100644 --- a/src/utils/hermes-base.ts +++ b/src/utils/hermes-base.ts @@ -1134,14 +1134,16 @@ export function normalizeDisassemblyLine( // the table header hermesc prints for them) moves with instruction widths. // The two switch instructions carry that offset in different operands: // StringSwitchImm rX, , , , - // UIntSwitchImm rX, , , , + // UIntSwitchImm (classic: SwitchImm) rX, , , , // Folding only the first shape let a shifted UIntSwitchImm offset read as a // real difference and drop an otherwise good delta build. if (folded === 'StringSwitchImm') { m = /^(\s*StringSwitchImm r\d+, \d+, )\d+(, L\d+, \d+)$/.exec(line); if (m) line = `${m[1]}${m[2]}`; - } else if (folded === 'UIntSwitchImm') { - m = /^(\s*UIntSwitchImm r\d+, )\d+(, L\d+, \d+, \d+)$/.exec(line); + } else if (folded === 'UIntSwitchImm' || folded === 'SwitchImm') { + m = /^(\s*(?:UIntSwitchImm|SwitchImm) r\d+, )\d+(, L\d+, \d+, \d+)$/.exec( + line, + ); if (m) line = `${m[1]}${m[2]}`; } else if (folded === 'offset' && /^\s*offset \d+$/.test(line)) { line = line.replace(/\d+$/, ''); @@ -1341,9 +1343,12 @@ class DumpReader { let processError: Error | undefined; // a spawn failure (ENOENT) may leave stdout open and never 'close' proc.on('error', (error) => { - proc.stdout?.unpipe(pass); + // destroy() need not emit end, so pipe() will not finish its sinks. + // End the debug file here: finish() awaits it before finally/kill(). + proc.stdout?.unpipe(); proc.stdout?.destroy(); pass.end(); + this.debugOutput?.end(); processError = error; if (!proc.pid) resolve({ code: null, signal: null, error, stderr }); }); diff --git a/src/utils/hermes-raw.ts b/src/utils/hermes-raw.ts index 3239ed6..c230319 100644 --- a/src/utils/hermes-raw.ts +++ b/src/utils/hermes-raw.ts @@ -97,6 +97,14 @@ export async function readHermesSemanticData( } const string = (id: number) => requireValue(strings.get(id), `unresolved string id ${id}`); + // Audited v98 function schema: hermes-compiler 250829098.0.16/.17 + // (250829098 stable snapshot), NOT every static_h build reporting HBC 98. + // Large headers: 37 bytes, flags[36]; small cache bits: 6/1/1. Upstream + // 7193d4485beeb87cd7a3b6ca8b6b5d97a1a433c4 removed NumCacheNewObject + // without immediately bumping HBC: 36 bytes/flags[35], cache bits 7/1. + // hbcTransform's file-header variants do not distinguish that function + // schema change. A compiler upgrade needs independent large/small fixtures; + // do not infer either schema from numStringSwitchImms or HBC version alone. const shaped = resolved.version === 98; const entrySize = shaped ? 12 : 16; const headers = section('functionHeaders'); @@ -225,6 +233,7 @@ const OPERAND_BYTES: Record = { // DefineOwnById annotation supplied explicitly. Keep classic and v98 variants. const STRING_OPERANDS: Record = { DeclareGlobalVar: [0], + ThrowIfHasRestrictedGlobalProperty: [0], GetById: [3], GetByIdWithReceiver: [4], TryGetById: [3], @@ -340,7 +349,10 @@ export function normalizeRawHermesFunction( targets.set(inst.offset, index); end += inst.size; } - if (end !== fn.size || instructions.length === 0) { + // Static Hermes retains legal zero-byte (dead) functions in the table. + // Their metadata is still audited, and the raw reader still requires every + // function header. Only a body shorter/longer than its declared size fails. + if (end !== fn.size) { throw new UnverifiableHermesBytecode( 'raw dump ended before the function body', ); diff --git a/tests/fixtures/hermes-async-check.cjs b/tests/fixtures/hermes-async-check.cjs index fcdc283..36bfbbb 100644 --- a/tests/fixtures/hermes-async-check.cjs +++ b/tests/fixtures/hermes-async-check.cjs @@ -37,6 +37,25 @@ async function main() { ); assert.equal(result.status, 'dump-failed'); assert.match(result.detail, /abort/i); + } else if (config.operation === 'verify') { + const { compareHermesBytecode } = require(path.resolve(config.modulePath)); + const controller = new AbortController(); + let timer; + if (config.abortAfterMs === 0) controller.abort(); + else if (config.abortAfterMs !== undefined) { + timer = setTimeout(() => controller.abort(), config.abortAfterMs); + } + try { + result = await compareHermesBytecode( + config.command, + 'missing-a', + 'missing-b', + { ...config.options, signal: controller.signal }, + ); + assert.equal(result.status, 'dump-failed'); + } finally { + clearTimeout(timer); + } } else if (config.operation === 'compile') { const { compileHermesByteCode } = require(path.resolve(config.modulePath)); result = await compileHermesByteCode(config.options); diff --git a/tests/hermes-blockers.test.ts b/tests/hermes-blockers.test.ts new file mode 100644 index 0000000..c66c888 --- /dev/null +++ b/tests/hermes-blockers.test.ts @@ -0,0 +1,153 @@ +import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; +import { spawnSync } from 'child_process'; +import fs from 'fs-extra'; +import os from 'os'; +import path from 'path'; +import { normalizeDisassemblyLine } from '../src/utils/hermes-base'; +import type { LiteralBuffers } from '../src/utils/hermes-literals'; +import { + type HermesSemanticData, + normalizeRawHermesFunction, +} from '../src/utils/hermes-raw'; + +const buffers: LiteralBuffers = { + layout: 'shaped', + version: 98, + values: Buffer.alloc(0), + objectKeys: Buffer.alloc(0), + shapes: Buffer.alloc(0), +}; + +/** Minimal HBC data isolates completeness checks from compiler optimization. */ +function functionData(size: number): HermesSemanticData { + return { + bytes: Buffer.alloc(size), + version: 98, + strings: new Map(), + functions: [{ offset: 0, size, metadata: '["dead",2,[]]' }], + bigints: [], + regexps: [], + metadata: '[]', + }; +} + +test('classic switch offsets fold but default destinations remain significant', () => { + const normalize = (line: string) => normalizeDisassemblyLine(line, new Map()); + expect(normalize(' SwitchImm r2, 2339, L7, 0, 31')).toBe( + normalize(' SwitchImm r2, 2340, L7, 0, 31'), + ); + expect(normalize(' SwitchImm r2, 2339, L7, 0, 31')).not.toBe( + normalize(' SwitchImm r2, 2340, L8, 0, 31'), + ); +}); + +describe('raw function completeness', () => { + test('a declared zero-length function retains metadata without instructions', () => { + const data = functionData(0); + expect(normalizeRawHermesFunction([], data, 0, buffers)).toEqual([ + data.functions[0].metadata, + ]); + }); + + test('missing instructions in a non-empty function still fail closed', () => { + expect(() => + normalizeRawHermesFunction([], functionData(1), 0, buffers), + ).toThrow('raw dump ended before the function body'); + }); + + test('instructions cannot be injected into a declared empty function', () => { + expect(() => + normalizeRawHermesFunction( + ['[@ 0] Unreachable'], + functionData(0), + 0, + buffers, + ), + ).toThrow('raw dump ended before the function body'); + }); + + test('restricted global properties compare by string value, not string ID', () => { + const normalize = (id: number, value: string) => { + const data = functionData(5); + data.version = 96; + data.bytes.writeUInt32LE(id, 1); + data.strings.set(id, value); + return normalizeRawHermesFunction( + [`[@ 0] ThrowIfHasRestrictedGlobalProperty ${id}`], + data, + 0, + { + layout: 'split', + version: 96, + array: Buffer.alloc(0), + objectKeys: Buffer.alloc(0), + objectValues: Buffer.alloc(0), + }, + ); + }; + expect(normalize(1, 'shared-prefix-global-A')).toEqual( + normalize(4096, 'shared-prefix-global-A'), + ); + expect(normalize(1, 'shared-prefix-global-A')).not.toEqual( + normalize(4096, 'shared-prefix-global-B'), + ); + }); +}); + +// Isolate the API call: a pending Promise can let Node/Bun exit without a +// result, and a leaked child can instead keep it alive. Check marker AND exit. +describe.if(process.platform !== 'win32')('debug dump failure cleanup', () => { + let dir: string; + let command: string; + // CI also exercises these child-process paths against the built Node 18 API. + const runtime = process.env.HERMES_TEST_NODE || process.execPath; + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'hermes-debug-exit-')); + command = path.join(dir, 'hermesc'); + fs.writeFileSync( + command, + `#!/bin/sh\nexec "${runtime}" -e 'setInterval(() => {}, 1000)'\n`, + { mode: 0o755 }, + ); + }); + afterEach(() => fs.removeSync(dir)); + + for (const scenario of ['timeout', 'abort', 'pre-abort', 'ENOENT'] as const) { + test(`${scenario} with dumpTo returns and closes debug streams`, () => { + const dumpTo = { + withBase: path.join(dir, 'base.txt'), + plain: path.join(dir, 'plain.txt'), + }; + const child = spawnSync( + runtime, + [ + path.join(__dirname, 'fixtures/hermes-async-check.cjs'), + JSON.stringify({ + operation: 'verify', + modulePath: process.env.HERMES_TEST_NODE + ? path.resolve(__dirname, '../lib/utils/hermes-base.js') + : require.resolve('../src/utils/hermes-base'), + command: scenario === 'ENOENT' ? `${command}-missing` : command, + abortAfterMs: + scenario === 'pre-abort' + ? 0 + : scenario === 'abort' + ? 100 + : undefined, + options: { dumpTo, timeoutMs: scenario === 'timeout' ? 100 : 1000 }, + }), + ], + { encoding: 'utf8', timeout: 2500 }, + ); + expect(child.error, child.stderr).toBeUndefined(); + expect(child.signal).toBeNull(); + expect(child.status, child.stderr).toBe(0); + expect(child.stderr).not.toContain('HERMES_ASYNC_ERROR'); + expect(child.stdout).toContain('HERMES_ASYNC_RESULT'); + expect(child.stdout).toContain('"status":"dump-failed"'); + for (const file of Object.values(dumpTo)) { + expect(fs.readFileSync(file, 'utf8')).toBe(''); + } + }, 4000); + } +}); diff --git a/tests/hermes-metro.test.ts b/tests/hermes-metro.test.ts new file mode 100644 index 0000000..ebe0a1d --- /dev/null +++ b/tests/hermes-metro.test.ts @@ -0,0 +1,84 @@ +import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; +import { spawnSync } from 'child_process'; +import { createHash } from 'crypto'; +import fs from 'fs-extra'; +import os from 'os'; +import path from 'path'; +import { compareHermesBytecode } from '../src/utils/hermes-base'; +import { readHermesSemanticData } from '../src/utils/hermes-raw'; + +// CI explicitly supplies the pinned, hash-checked real Metro fixture directory. +// Offline unit runs do not download application bundles as a side effect. +const fixtures = process.env.HERMES_METRO_FIXTURES; +const hermesc = process.env.HERMESC; +const sources = { + 'base.jsbundle': + '11c8ad8f7e8c7c59ee45582c77d896a35fa646617f3ba0f5b338a425a7c93b7d', + 's3-medium-feature.jsbundle': + 'a693e68254b6c13fae8f839d20c14f9d11c5ab98d4be1b8d13ba1e929a12d752', +}; + +describe.if(Boolean(fixtures))('real Metro bundle verification', () => { + let dir: string; + let plain: string; + let delta: string; + beforeAll(() => { + // Missing compiler/fixture is a failure once the integration job opts in. + expect(hermesc && fs.existsSync(hermesc)).toBe(true); + for (const [name, hash] of Object.entries(sources)) { + const bytes = fs.readFileSync(path.join(fixtures!, name)); + expect(createHash('sha256').update(bytes).digest('hex')).toBe(hash); + } + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'hermes-metro-')); + const compile = (source: string, name: string, base?: string) => { + const out = path.join(dir, name); + const result = spawnSync( + hermesc!, + [ + '-emit-binary', + '-O', + '-w', + '-output-source-map', + '-out', + out, + path.join(fixtures!, source), + ...(base ? [`-base-bytecode=${base}`] : []), + ], + { encoding: 'utf8', timeout: 30_000 }, + ); + expect(result.error, result.stderr).toBeUndefined(); + expect(result.status, result.stderr).toBe(0); + return out; + }; + const base = compile('base.jsbundle', 'base.hbc'); + plain = compile('s3-medium-feature.jsbundle', 'plain.hbc'); + delta = compile('s3-medium-feature.jsbundle', 'delta.hbc', base); + }, 120_000); + afterAll(() => { + if (dir) fs.removeSync(dir); + }); + + test('self-comparison includes legal zero-byte Static Hermes functions', async () => { + const data = await readHermesSemanticData(plain); + expect(data.functions.length).toBeGreaterThan(10_000); + if (data.version === 98) { + // This assertion prevents an unrelated small fixture from replacing the + // production-shaped regression that exposed the empty-function bug. + expect( + data.functions.filter((fn) => fn.size === 0).length, + ).toBeGreaterThan(0); + } + const result = await compareHermesBytecode(hermesc!, plain, plain); + expect(result.status, result.detail).toBe('equivalent'); + expect(result.functions).toBe(data.functions.length); + console.log( + `Metro HBC ${data.version}: ${data.functions.length} functions, ${data.functions.filter((fn) => fn.size === 0).length} empty`, + ); + }, 60_000); + + test('a real base/plain release pair remains equivalent', async () => { + const result = await compareHermesBytecode(hermesc!, delta, plain); + expect(result.status, result.detail).toBe('equivalent'); + expect(result.functions).toBeGreaterThan(10_000); + }, 60_000); +}); diff --git a/tests/hermes-raw.test.ts b/tests/hermes-raw.test.ts index 1e9aff5..c835c3f 100644 --- a/tests/hermes-raw.test.ts +++ b/tests/hermes-raw.test.ts @@ -4,7 +4,10 @@ import { createHash } from 'crypto'; import fs from 'fs-extra'; import os from 'os'; import path from 'path'; -import { compareHermesBytecode } from '../src/utils/hermes-base'; +import { + compareHermesBytecode, + probeHbcVersion, +} from '../src/utils/hermes-base'; import { readHermesSemanticData } from '../src/utils/hermes-raw'; const hermesc = process.env.HERMESC; @@ -30,7 +33,12 @@ describe.if(hasHermesc)('lossless Hermes operand audit (real compiler)', () => { }); afterEach(() => fs.removeSync(dir)); - const compile = (name: string, source: string, base?: string) => { + const compile = ( + name: string, + source: string, + base?: string, + extra: string[] = [], + ) => { const input = path.join(dir, `${name}.js`); const output = path.join(dir, `${name}.hbc`); fs.writeFileSync(input, source); @@ -45,6 +53,7 @@ describe.if(hasHermesc)('lossless Hermes operand audit (real compiler)', () => { output, input, ...(base ? [`-base-bytecode=${base}`] : []), + ...extra, ], { encoding: 'utf8', timeout: 10_000 }, ); @@ -116,6 +125,27 @@ describe.if(hasHermesc)('lossless Hermes operand audit (real compiler)', () => { ); }); + test.skipIf(!hasHermesc || probeHbcVersion(hermesc!) === 98)( + 'classic global lexical declarations resolve restricted-property string IDs', + async () => { + const base = compile('lexical-base', 'print("old-base-string");'); + const source = + 'let sharedPrefixGlobalLexical = "value"; print(sharedPrefixGlobalLexical);'; + const plain = compile('lexical-plain', source, undefined, [ + '-block-scoping', + ]); + const delta = compile('lexical-delta', source, base, ['-block-scoping']); + const op = /ThrowIfHasRestrictedGlobalProperty (\d+)/; + const plainId = op.exec(dump(plain, false)); + const deltaId = op.exec(dump(delta, false)); + expect(plainId).not.toBeNull(); + expect(deltaId).not.toBeNull(); + expect(plainId![1]).not.toBe(deltaId![1]); + const result = await compareHermesBytecode(hermesc!, delta, plain); + expect(result.status, result.detail).toBe('equivalent'); + }, + ); + test('same-name closures cannot hide a changed function reference', async () => { const file = compile( 'closures',