diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 23b08f07..22887ce4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -224,6 +224,20 @@ jobs: working-directory: test/gc_stress run: haxe compile.hxml -D HXCPP_M64 -D HXCPP_SANITIZE=${{ matrix.sanitize }} && bin/Stress 10 + # The new collector has its own deterministic scheduling and barrier paths. + - name: Run incremental GC regressions + if: ${{ !cancelled() && steps.setup.outcome == 'success' }} + working-directory: test/regression + run: | + for test in GcMutationRoots GcContainerMutation IncrementalGcSlices IncrementalGcContainers IncrementalGcFinishBoundary GcWeakMapChain GenerationalFinalizerLifecycle GenerationalLargeBufferLifecycle; do + (cd "$test" && haxe -main Main -cpp bin -lib hxcpp -D HXCPP_M64 -D HXCPP_GC_INCREMENTAL -D HXCPP_GC_INCREMENTAL_VERIFY -D HXCPP_GC_INCREMENTAL_TEST -D HXCPP_SANITIZE=${{ matrix.sanitize }} && bin/Main > out.txt && diff <(tr -d '\r' < stdout.txt) <(tr -d '\r' < out.txt)) + done + + - name: Run incremental GC stress + if: ${{ !cancelled() && steps.setup.outcome == 'success' }} + working-directory: test/gc_stress + run: haxe compile.hxml -lib hxcpp -D HXCPP_M64 -D HXCPP_GC_INCREMENTAL -D HXCPP_GC_INCREMENTAL_VERIFY -D HXCPP_SANITIZE=${{ matrix.sanitize }} && bin/Stress 10 8 2012663045 + - name: Show the sanitizer reports if: ${{ !cancelled() }} run: cat /tmp/sanitizer/report.* 2>/dev/null || echo "No sanitizer reports" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d6cdd908..c80bcbf6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -363,6 +363,49 @@ jobs: done [ -z "$failed" ] || { echo "Failed modes:$failed"; exit 1; } + incremental: + if: inputs.scope == 'full' && startsWith(inputs.haxe, '4.3.') + runs-on: ${{ inputs.os }} + name: incremental gc + timeout-minutes: 90 + defaults: + run: + shell: bash + steps: + - name: checkout + uses: actions/checkout@v7 + - name: setup + uses: ./.github/workflows/setup + with: + haxe: ${{ inputs.haxe }} + - name: deterministic phases and semantics + working-directory: test/regression + run: | + failed="" + for test in GcMutationRoots GcContainerMutation IncrementalGcSlices IncrementalGcContainers IncrementalGcFinishBoundary IncrementalGcVerifier GcWeakMapChain GenerationalFinalizerLifecycle GenerationalLargeBufferLifecycle; do + echo "::group::incremental gc: $test" + # The lifecycle fixtures' hxml selects generational mode; compile + # the same Main with incremental mode here instead. + if ! (cd "$test" && haxe -main Main -cpp bin -lib hxcpp -D HXCPP_GC_INCREMENTAL -D HXCPP_GC_INCREMENTAL_VERIFY -D HXCPP_GC_INCREMENTAL_TEST -D $HXCPP_ARCH_FLAG && bin/Main > out.txt && diff <(tr -d '\r' < stdout.txt) <(tr -d '\r' < out.txt)); then + failed="$failed $test" + echo "::error::incremental gc failed: $test" + fi + echo "::endgroup::" + done + [ -z "$failed" ] || { echo "Failed tests:$failed"; exit 1; } + - name: incremental stress + if: ${{ !cancelled() }} + working-directory: test/gc_stress + run: | + haxe compile.hxml -lib hxcpp -D HXCPP_GC_INCREMENTAL -D HXCPP_GC_INCREMENTAL_VERIFY -D $HXCPP_ARCH_FLAG + bin/Stress 30 8 2012663045 + - name: incremental stress (big blocks) + if: ${{ !cancelled() }} + working-directory: test/gc_stress + run: | + haxe compile.hxml -lib hxcpp -D HXCPP_GC_INCREMENTAL -D HXCPP_GC_INCREMENTAL_VERIFY -D HXCPP_GC_BIG_BLOCKS -D $HXCPP_ARCH_FLAG + bin/Stress 30 8 1337 + cppia: if: inputs.scope == 'full' runs-on: ${{ inputs.os }} diff --git a/BENCHMARKS.md b/BENCHMARKS.md index 6142ceba..7874c4ff 100644 --- a/BENCHMARKS.md +++ b/BENCHMARKS.md @@ -97,6 +97,7 @@ These numbers are from one Windows machine; Linux and macOS results are still to | `alloc_strings` | Int-to-String conversion plus a concatenation per string | | `alloc_arrays` | Arrays of 1 to 64 Ints grown by `push`, plus a 256 KB buffer every 4000 arrays | | `gc_pauses` | Collection pauses as the program sees them: a large live graph stays reachable while the program allocates in fixed work units, and each unit's duration is recorded. The tail holds the units that included a collection | +| `gc_frames` | 2,000 frames over 150,000 retained entities with graph and inventory mutation, temporary particle chains and event strings, fixed-size spatial cells and periodic allocation bursts. Records frame latency and checks retained payloads. Uses automatic GC without frame hooks or sleeps | | `map_int` | `IntMap`: on a fresh map, N sets, N hits, N misses, N/2 removes, then iterating the N/2 values left | | `map_string` | `StringMap`, the same cycle | | `map_object` | `ObjectMap`, the same cycle | diff --git a/Changes.md b/Changes.md index a38753dc..eb72931e 100644 --- a/Changes.md +++ b/Changes.md @@ -2,6 +2,7 @@ libhxcpp 6.0.0 (unreleased) ------------------------------------------------------------ +* Added an opt-in incremental marking collector for native Haxe 4 C++ builds (`-lib hxcpp -D HXCPP_GC_INCREMENTAL`), with automatic allocation pacing, optional `hxcpp.gc.Incremental` scheduling hooks, resumable array and map scans, an independent verifier and deterministic regression tests. The existing collector remains the default. Slice budgets are targets, and synchronous completion and memory pressure can still cause longer pauses; see docs/IncrementalGc.md * Emscripten programs let memory grow and get a 5 MB stack by default. They linked with Emscripten's fixed 16 MB of memory, which aborts with "Aborted(OOM)" as soon as a program holds more, and since Emscripten 3.1.27 a 64 KB stack, which Haxe recursion and the pointers that spill-pointers keeps on the stack soon overflow. HXCPP_LINK_NO_MEMORY_GROWTH and HXCPP_LINK_NO_STACK_SIZE restore Emscripten's settings, and HXCPP_LINK_STACK_SIZE sets another size * Fixed converting a long or a size_t to Dynamic failing to compile on Emscripten, where long is 32 bits but not int, as an Int default of -2147483648 does (HaxeFoundation/hxcpp#621) * Fixed cppia hosts built with HXCPP_DEBUGGER failing to compile on Emscripten, which has no cppia JIT: the debugger's frame reader asked whether a function was jitted through a field that only exists with the JIT diff --git a/bench/results/incremental-gc/README.md b/bench/results/incremental-gc/README.md new file mode 100644 index 00000000..6f75de7d --- /dev/null +++ b/bench/results/incremental-gc/README.md @@ -0,0 +1,65 @@ +# Incremental GC development measurements + +These results compare the existing libhxcpp collector with the experimental incremental mode in the same source tree. They do not replace the stock-hxcpp comparison in [BENCHMARKS.md](../../../BENCHMARKS.md). Both collectors use the same benchmark sources, optimized compiler settings and payload checks. + +The raw process results, sample times, latency summaries, checksums, peak working sets, reserved heap sizes and incremental counters are in [windows-x64-20261026.json](windows-x64-20261026.json). The containing commit identifies the implementation; the JSON also records the base revision and source hashes. + +## Results + +Elapsed time is the median of the 15 measured samples. Latency columns show the largest per-process p99.9 and the largest observed work-unit duration across the three processes; the p99.9 column is not a pooled percentile. Memory columns show the largest process value. + +| Workload | Collector | Elapsed, ms | Worst p99.9, ms | Maximum, ms | Peak RSS, MiB | Reserved heap, MiB | +| --- | --- | ---: | ---: | ---: | ---: | ---: | +| `gc_frames` | Existing | 240.246 | 7.742 | 9.728 | 128.76 | 115.36 | +| `gc_frames` | Incremental | 294.961 | 0.969 | 1.858 | 131.98 | 118.29 | +| `gc_pauses` | Existing | 145.429 | 5.055 | 8.289 | 135.29 | 121.55 | +| `gc_pauses` | Incremental | 256.843 | 2.718 | 5.232 | 135.52 | 122.51 | + +All six `gc_frames` processes returned checksum 1106626610; all six `gc_pauses` processes returned 458711266. Incremental mode reduced tail latency on these workloads, with 22.8% more elapsed time for `gc_frames` and 76.6% more for `gc_pauses`. That corresponds to 18.5% and 43.4% less throughput, respectively. Neither meets the proposed maximum 10% throughput loss. + +The largest budgeted collector invocation was 1.346 ms in `gc_frames` and 1.783 ms in `gc_pauses`, against a requested 0.5 ms. Cumulative unbudgeted maxima were 13.553 ms and 9.971 ms, respectively, including explicit collections outside measured frames. These results do not establish a sub-millisecond collector or eliminate synchronous interruptions. + +## Protocol + +- Windows x64, AMD Ryzen 9 9950X, Haxe 4.3.7 and MSVC 19.44.35213. +- Default block size, existing collector defaults versus `HXCPP_GC_INCREMENTAL` with its default 0.5 ms target and 1 ms preferred interval. +- No `HXCPP_GC_INCREMENTAL_VERIFY`, `HXCPP_GC_INCREMENTAL_TEST`, scriptable mode or explicit engine scheduling hooks. +- Three processes per workload and collector, two warmups and five measured samples per process. Collector order alternates between processes. Compilation, stress and sanitizer runs finished before measurement. +- `gc_frames` retains 150,000 entities and runs 2,000 frames per sample. Frame timing includes graph and inventory changes, temporary particles and event strings, fixed-size spatial cells and periodic bursts. Payload validation runs after the frame loop and is included in elapsed sample time. +- `gc_pauses` retains a large graph and records 3,000 allocation work units per sample. A work unit is not a game frame or an isolated GC pause. +- The harness requests a full collection before each sample, outside its timed section. Cumulative GC counters include setup, warmups, these explicit collections and validation. Their maxima cannot be presented as maxima of GC work inside measured frames. +- Checksums must agree across all processes of a workload. The JSON contains elapsed sample times and per-process latency summaries, rather than a trace of individual frames. + +## Reproduction + +After pointing haxelib at this checkout as described in CONTRIBUTING.md, build each mode sequentially: + +```sh +haxe -cp bench/src -main Main -cpp build/gc-default -lib hxcpp -D HXCPP_M64 +haxe -cp bench/src -main Main -cpp build/gc-incremental -lib hxcpp -D HXCPP_M64 -D HXCPP_GC_INCREMENTAL +``` + +For each workload, run the following pair three times, reversing the order in the second pair. On Windows the executable is `Main.exe`. + +```sh +build/gc-default/Main gc_frames --warmup 2 --samples 5 +build/gc-incremental/Main gc_frames --warmup 2 --samples 5 +build/gc-default/Main gc_pauses --warmup 2 --samples 5 +build/gc-incremental/Main gc_pauses --warmup 2 --samples 5 +``` + +## Correctness validation + +Local Windows validation includes all nine phase and lifecycle fixtures listed in [IncrementalGc.md](../../../docs/IncrementalGc.md), with the independent verifier and deterministic controls, at normal and big block sizes. Each block size also passed three 30-second stress runs with eight threads and seeds 2012663045, 1337 and 8675309. The full runtime suite passed 1,100 assertions with incremental verification and big blocks. The new slice, container and finishing fixtures also pass with the existing collector, and the container and finishing fixtures pass in single-threaded incremental builds. + +Linux under WSL, Haxe 4.3.3 and GCC 13.3 passed ASan and UBSan builds of the slice, container and finishing fixtures, plus a 30-second stress run with eight threads under each sanitizer. Leak detection is disabled because these runs exercise the runtime's conservative stack scanning configuration. They are not leak measurements. The final GCC define matrix passed all 79 configurations, including incremental, verification/test and single-threaded variants at both block sizes. + +The independent verifier fixture deliberately omits a reachable descendant's mark in a child process and requires rejection. The epoch recovery assertion failed before the recovery fix, and the allocation cursor assertion failed before preserving allocator ranges across partial slices. + +The isolated haxelib installation, setup, update and off checks passed on Windows and Linux. Their synthetic package includes the configuration macro referenced by `extraParams.hxml`. + +Local TSan did not reach a usable test run: the installed GCC runtime rejected the repository suppression syntax and, without suppressions, failed under WSL before the workload ran. TSan and other platform results remain for CI. CI configuration includes incremental regressions and stress in the sanitizer jobs and a native incremental job on supported Haxe 4 configurations. + +## Remaining gates + +This is experimental. The throughput target in [#372](https://github.com/openfl/libhxcpp/issues/372) is still open. Weak closure, finalizers, reclamation, root capture, thread rendezvous and forced completion can exceed the slice target. Longer soaks, additional memory-pressure and native-integration coverage, pause distributions for all stopped threads, progress and metadata high-water counters, and cross-platform results are still needed before considering a default change. diff --git a/bench/results/incremental-gc/windows-x64-20261026.json b/bench/results/incremental-gc/windows-x64-20261026.json new file mode 100644 index 00000000..59e5fd35 --- /dev/null +++ b/bench/results/incremental-gc/windows-x64-20261026.json @@ -0,0 +1,546 @@ +{ + "metadata": { + "recordedAt": "2026-10-26T22:04:17.6818665-04:00", + "baseRevision": "4cdca9600f0a7c182693d244a00389ef3b9a75df", + "source": "Implementation and benchmarks in the containing commit", + "platform": "Windows x64", + "cpu": "AMD Ryzen 9 9950X 16-Core Processor", + "haxe": "4.3.7", + "compilerEnvironment": { + "apiLevel": 430, + "pointerBits": 64, + "compiler": "MSVC 194435213" + }, + "buildFlags": [ + "HXCPP_M64" + ], + "incrementalFlags": [ + "HXCPP_GC_INCREMENTAL" + ], + "verification": false, + "deterministicControls": false, + "targetSliceMs": 0.5, + "preferredIntervalMs": 1.0, + "warmup": 2, + "samples": 5, + "processes": 3, + "order": "control/incremental, incremental/control, control/incremental for each workload", + "sourceSha256": { + "extraParams.hxml": "256a66dc79e6e22d48d3077981cb182081c7d0657790bf9604189486a97e7ce2", + "hxcpp/IncrementalGcConfig.hx": "41db824cda90cb1cd611dafca9ee9e3c491988d24bb284781326bf528f689d3b", + "hxcpp/gc/Incremental.hx": "3558678121ed13b2407d4807e50bd05e709ba082a312836eca6505e8f31328ac", + "include/Array.h": "842b58003246efcd524e6312da766d27dd8e7168d5089d494ff8442333a3a0be", + "include/hx/GC.h": "74741fd8487a410752f44813c55ee676a45c2a3980377fd111ccdc98bfe37eca", + "include/hx/StackContext.h": "7d4ea034344c2f9eec99df4de0e900020bd5e126817f0d7a112342924b99cb21", + "include/hxcpp.h": "faa4a58e05659def176fd5d3c73f21483630808a93a47bebfcfd2d4a818f3dba", + "src/Array.cpp": "b71686943c6ff3e4e0a3b10fa47d705bf175b9cb3cf262e999d002086f73e11f", + "src/hx/Hash.h": "bd38f41dbc992cc70df1a31daa1ffa1553d067606324412a91c20cde3db15133", + "src/hx/gc/Immix.cpp": "2b835876479d6233bd8c17eb6f6085fda06bb0b8f80d57f69ebd64ed0b5269ee", + "src/hx/gc/Incremental.h": "238b5a0451493f61a20c4c041a60ab8473a4834f110c34a9f500adcdd257a8ec", + "src/hx/gc/IncrementalApi.h": "8ba9d3ddb2cd6d2519e6989766919460f7b57d021ec8fc467eaad6fc10103085", + "toolchain/common-defines.xml": "bf4a2f0f86ef7dd3eac1ca985f7b63b106295c012cac8cfccb26ada2db3f7670", + "toolchain/haxe-target.xml": "27600d3d033f8e7e0a1c35d1ead14c1b1bc704d42e17759dc15414aa4a6a08b2", + "bench/src/AllocBenchmarks.hx": "044d1e7396c8f30da2d4c5211e8696fea10a32a1ae5f33083ee90c1f0b88eb02", + "bench/src/Benchmark.hx": "dc03d00619fc5ea990cd664ae5b3490b962dd3e6e4f8259951713061a9f5aadd", + "bench/src/CppiaBenchmarks.hx": "26b46915d257e3b52cd8b45b58d47d49b840e53758ace91360387ba9a367b5d0", + "bench/src/GcFrameBenchmarks.hx": "f98d20b82bbbf4d69225a9768ddd92686962c6ea69180330b8b80099eaa3182d", + "bench/src/Main.hx": "821d73fdc5b3a631f934e1bc311b80182aac7b1a4f234b0ecf173543351e10a2", + "bench/src/MapBenchmarks.hx": "4e96cd9aaab24105f25a303ab7da1093eb319a21bb67cd564599d30ca57ef7ae", + "bench/src/Native.hx": "44d6edec4562b97094fb2648876a46fc0163156a3b7fdde62c3fb879112f0d8c", + "bench/src/SortBenchmarks.hx": "72981b661edfddc9ac6df39f185bc2c1e6a3355c53f0c03ce4af148c4da93587", + "bench/src/StringBenchmarks.hx": "91b44e16d39a9f6b27ba58b58a8c1ee9a9941c6cca425c3551fbc93d5dd8a9a8" + } + }, + "summary": [ + { + "name": "gc_frames", + "mode": "control", + "medianMs": 240.24559999999994, + "p999Ms": 7.742000000000138, + "maxMs": 9.728300000000134, + "rssMiB": 128.76171875, + "reservedMiB": 115.36090087890625, + "maxSlice": null, + "maxSync": null + }, + { + "name": "gc_frames", + "mode": "incremental", + "medianMs": 294.96049999999997, + "p999Ms": 0.969099999999945, + "maxMs": 1.8576000000001258, + "rssMiB": 131.9765625, + "reservedMiB": 118.29131317138672, + "maxSlice": 1.3457000022754073, + "maxSync": 13.553299999330193 + }, + { + "name": "gc_pauses", + "mode": "control", + "medianMs": 145.42859999999996, + "p999Ms": 5.055200000000037, + "maxMs": 8.28910000000005, + "rssMiB": 135.28515625, + "reservedMiB": 121.55069732666016, + "maxSlice": null, + "maxSync": null + }, + { + "name": "gc_pauses", + "mode": "incremental", + "medianMs": 256.8428, + "p999Ms": 2.7182999999999513, + "maxMs": 5.232199999999909, + "rssMiB": 135.51953125, + "reservedMiB": 122.51187896728516, + "maxSlice": 1.7831000150181353, + "maxSync": 9.970500017516315 + } + ], + "processes": [ + { + "benchmark": "gc_frames", + "mode": "control", + "process": 1, + "result": { + "samples": 5, + "checksum": 1106626610, + "peakRss": 135016448, + "warmup": 2, + "heapReserved": 120964672, + "ops": 2000, + "heapUsage": 61615936, + "times": [ + 0.24637399999999998, + 0.2529614, + 0.2433593999999999, + 0.2284682, + 0.19830669999999984 + ], + "name": "gc_frames", + "unit": "frames", + "latency": { + "max": 0.009728300000000134, + "p50": 9.159999999996948E-05, + "p99": 0.0001541999999998822, + "p999": 0.007534999999999958, + "count": 10000 + } + } + }, + { + "benchmark": "gc_frames", + "mode": "incremental", + "process": 1, + "result": { + "samples": 5, + "checksum": 1106626610, + "peakRss": 138387456, + "warmup": 2, + "heapReserved": 124037432, + "ops": 2000, + "heapUsage": 74859320, + "times": [ + 0.28636759999999994, + 0.2775067000000001, + 0.30213140000000016, + 0.30950809999999995, + 0.2895357999999999 + ], + "name": "gc_frames", + "unit": "frames", + "incrementalGc": { + "cycles": 32, + "pauses": 887, + "fallbacks": 0, + "totalPauseMs": 535.3046006639488, + "maxPauseMs": 10.890499979723245, + "lastPauseMs": 0.7890000124461949, + "lastCycleSlices": 32, + "active": false, + "forcedFinishes": 0, + "requestedSlices": 879, + "maxSlicePauseMs": 1.0647000162862241, + "maxSynchronousPauseMs": 10.890499979723245 + }, + "latency": { + "max": 0.0012659999999999894, + "p50": 9.890000000001287E-05, + "p99": 0.0007062999999998265, + "p999": 0.0009261000000000408, + "count": 10000 + } + } + }, + { + "benchmark": "gc_frames", + "mode": "incremental", + "process": 2, + "result": { + "samples": 5, + "checksum": 1106626610, + "peakRss": 136904704, + "warmup": 2, + "heapReserved": 123029560, + "ops": 2000, + "heapUsage": 73562296, + "times": [ + 0.29496049999999996, + 0.27863859999999996, + 0.2975342999999999, + 0.2834756999999999, + 0.29728529999999975 + ], + "name": "gc_frames", + "unit": "frames", + "incrementalGc": { + "cycles": 32, + "pauses": 886, + "fallbacks": 0, + "totalPauseMs": 535.8317995560355, + "maxPauseMs": 11.42449997132644, + "lastPauseMs": 0.8314999868161976, + "lastCycleSlices": 30, + "active": false, + "forcedFinishes": 0, + "requestedSlices": 878, + "maxSlicePauseMs": 1.031799998600036, + "maxSynchronousPauseMs": 11.42449997132644 + }, + "latency": { + "max": 0.0011380000000000834, + "p50": 9.429999999999161E-05, + "p99": 0.0007025000000000503, + "p999": 0.000969099999999945, + "count": 10000 + } + } + }, + { + "benchmark": "gc_frames", + "mode": "control", + "process": 2, + "result": { + "samples": 5, + "checksum": 1106626610, + "peakRss": 134717440, + "warmup": 2, + "heapReserved": 120964672, + "ops": 2000, + "heapUsage": 61536320, + "times": [ + 0.21032800000000001, + 0.24482479999999995, + 0.25531689999999996, + 0.24024559999999995, + 0.22539920000000002 + ], + "name": "gc_frames", + "unit": "frames", + "latency": { + "max": 0.008739399999999842, + "p50": 9.320000000001549E-05, + "p99": 0.00015439999999999898, + "p999": 0.007512999999999881, + "count": 10000 + } + } + }, + { + "benchmark": "gc_frames", + "mode": "control", + "process": 3, + "result": { + "samples": 5, + "checksum": 1106626610, + "peakRss": 134443008, + "warmup": 2, + "heapReserved": 120964672, + "ops": 2000, + "heapUsage": 61532736, + "times": [ + 0.24843500000000007, + 0.2260025, + 0.22785539999999993, + 0.2584962999999998, + 0.23801709999999998 + ], + "name": "gc_frames", + "unit": "frames", + "latency": { + "max": 0.009555599999999997, + "p50": 9.40000000000385E-05, + "p99": 0.00016039999999994947, + "p999": 0.007742000000000138, + "count": 10000 + } + } + }, + { + "benchmark": "gc_frames", + "mode": "incremental", + "process": 3, + "result": { + "samples": 5, + "checksum": 1106626610, + "peakRss": 136818688, + "warmup": 2, + "heapReserved": 123078704, + "ops": 2000, + "heapUsage": 74368176, + "times": [ + 0.3064074, + 0.27760989999999997, + 0.26471449999999996, + 0.34805299999999995, + 0.3361160000000001 + ], + "name": "gc_frames", + "unit": "frames", + "incrementalGc": { + "cycles": 32, + "pauses": 911, + "fallbacks": 0, + "totalPauseMs": 552.3110998328775, + "maxPauseMs": 13.553299999330193, + "lastPauseMs": 0.7738000131212175, + "lastCycleSlices": 32, + "active": false, + "forcedFinishes": 0, + "requestedSlices": 903, + "maxSlicePauseMs": 1.3457000022754073, + "maxSynchronousPauseMs": 13.553299999330193 + }, + "latency": { + "max": 0.0018576000000001258, + "p50": 9.77999999998147E-05, + "p99": 0.000728300000000015, + "p999": 0.0009550000000000392, + "count": 10000 + } + } + }, + { + "benchmark": "gc_pauses", + "mode": "control", + "process": 1, + "result": { + "samples": 5, + "checksum": 458711266, + "peakRss": 141705216, + "warmup": 2, + "heapReserved": 127455144, + "ops": 12000000, + "heapUsage": 63901608, + "times": [ + 0.14542859999999996, + 0.1483196, + 0.14318609999999998, + 0.16387969999999996, + 0.15101699999999996 + ], + "name": "gc_pauses", + "unit": "objects", + "latency": { + "max": 0.00828910000000005, + "p50": 3.770000000002938E-05, + "p99": 7.109999999999062E-05, + "p999": 0.0050552000000000374, + "count": 15000 + } + } + }, + { + "benchmark": "gc_pauses", + "mode": "incremental", + "process": 1, + "result": { + "samples": 5, + "checksum": 458711266, + "peakRss": 139386880, + "warmup": 2, + "heapReserved": 125439400, + "ops": 12000000, + "heapUsage": 68641832, + "times": [ + 0.30509410000000003, + 0.2524731, + 0.25684280000000004, + 0.24843189999999993, + 0.27013109999999996 + ], + "name": "gc_pauses", + "unit": "objects", + "incrementalGc": { + "cycles": 67, + "pauses": 1282, + "fallbacks": 0, + "totalPauseMs": 859.6196983125992, + "maxPauseMs": 9.095899993553758, + "lastPauseMs": 1.2173000141046941, + "lastCycleSlices": 20, + "active": false, + "forcedFinishes": 2, + "requestedSlices": 1274, + "maxSlicePauseMs": 1.635099994018674, + "maxSynchronousPauseMs": 9.095899993553758 + }, + "latency": { + "max": 0.004448599999999914, + "p50": 4.9100000000024124E-05, + "p99": 0.0008038000000000212, + "p999": 0.0026836999999999556, + "count": 15000 + } + } + }, + { + "benchmark": "gc_pauses", + "mode": "incremental", + "process": 2, + "result": { + "samples": 5, + "checksum": 458711266, + "peakRss": 142102528, + "warmup": 2, + "heapReserved": 128463016, + "ops": 12000000, + "heapUsage": 69051816, + "times": [ + 0.2640766, + 0.2714901999999999, + 0.24795060000000002, + 0.2534077000000001, + 0.2789665000000001 + ], + "name": "gc_pauses", + "unit": "objects", + "incrementalGc": { + "cycles": 67, + "pauses": 1256, + "fallbacks": 0, + "totalPauseMs": 841.6912006214261, + "maxPauseMs": 9.970500017516315, + "lastPauseMs": 1.3644999708049, + "lastCycleSlices": 21, + "active": false, + "forcedFinishes": 2, + "requestedSlices": 1248, + "maxSlicePauseMs": 1.7831000150181353, + "maxSynchronousPauseMs": 9.970500017516315 + }, + "latency": { + "max": 0.003992400000000007, + "p50": 4.830000000000112E-05, + "p99": 0.0007869000000000348, + "p999": 0.0026610000000000245, + "count": 15000 + } + } + }, + { + "benchmark": "gc_pauses", + "mode": "control", + "process": 2, + "result": { + "samples": 5, + "checksum": 458711266, + "peakRss": 141783040, + "warmup": 2, + "heapReserved": 127455144, + "ops": 12000000, + "heapUsage": 63921832, + "times": [ + 0.1479591, + 0.13846270000000005, + 0.1390443, + 0.18794849999999996, + 0.13155680000000014 + ], + "name": "gc_pauses", + "unit": "objects", + "latency": { + "max": 0.0071702000000000155, + "p50": 3.74999999999126E-05, + "p99": 6.970000000006138E-05, + "p999": 0.004966299999999979, + "count": 15000 + } + } + }, + { + "benchmark": "gc_pauses", + "mode": "control", + "process": 3, + "result": { + "samples": 5, + "checksum": 458711266, + "peakRss": 141856768, + "warmup": 2, + "heapReserved": 127455144, + "ops": 12000000, + "heapUsage": 63997480, + "times": [ + 0.1410866, + 0.1433025, + 0.1535462999999999, + 0.15883440000000004, + 0.14439540000000006 + ], + "name": "gc_pauses", + "unit": "objects", + "latency": { + "max": 0.007081800000000027, + "p50": 3.799999999998249E-05, + "p99": 6.340000000015777E-05, + "p999": 0.005047300000000032, + "count": 15000 + } + } + }, + { + "benchmark": "gc_pauses", + "mode": "incremental", + "process": 3, + "result": { + "samples": 5, + "checksum": 458711266, + "peakRss": 138481664, + "warmup": 2, + "heapReserved": 124431528, + "ops": 12000000, + "heapUsage": 68537384, + "times": [ + 0.23862309999999998, + 0.24273489999999998, + 0.28725959999999984, + 0.28037370000000017, + 0.24740240000000013 + ], + "name": "gc_pauses", + "unit": "objects", + "incrementalGc": { + "cycles": 67, + "pauses": 1252, + "fallbacks": 0, + "totalPauseMs": 829.1372993262485, + "maxPauseMs": 9.615200047846884, + "lastPauseMs": 1.248000015038997, + "lastCycleSlices": 19, + "active": false, + "forcedFinishes": 1, + "requestedSlices": 1244, + "maxSlicePauseMs": 1.6203999985009432, + "maxSynchronousPauseMs": 9.615200047846884 + }, + "latency": { + "max": 0.005232199999999909, + "p50": 4.76000000000365E-05, + "p99": 0.0008002000000000287, + "p999": 0.0027182999999999513, + "count": 15000 + } + } + } + ] +} diff --git a/bench/src/GcFrameBenchmarks.hx b/bench/src/GcFrameBenchmarks.hx new file mode 100644 index 00000000..f2de65a4 --- /dev/null +++ b/bench/src/GcFrameBenchmarks.hx @@ -0,0 +1,126 @@ +package; + +import Benchmark.Rng; +import haxe.Timer; + +private class FrameItem { + public final id:Int; + public final value:Int; + + public function new(id:Int) { + this.id = id; + value = id ^ 0x13579; + } +} + +private class FrameEntity { + public final id:Int; + public final label:String; + public final inventory:Array; + public var target:FrameEntity; + + public function new(id:Int) { + this.id = id; + label = "entity-" + id; + inventory = [for (slot in 0...4) new FrameItem(id * 4 + slot)]; + } +} + +private class FrameParticle { + public final value:Int; + public final next:FrameParticle; + + public function new(value:Int, next:FrameParticle) { + this.value = value; + this.next = next; + } +} + +/** + Frame latency with a persistent world, graph mutation, temporary objects + and occasional bursts. There is no sleep or explicit collection inside a + frame. Work checksums and final graph checks run in both collector modes. +**/ +class GcFrames extends Benchmark { + static inline var ENTITIES = 150000; + static inline var FRAMES = 2000; + final entities:Array = []; + final grid:Array> = []; + var particles:FrameParticle; + var events:Array<{target:FrameEntity, message:String}> = []; + var burst:Array = []; + + public function new() { + super("gc_frames", "frames", FRAMES, true); + } + + override public function setup() { + for (i in 0...ENTITIES) + entities.push(new FrameEntity(i)); + for (i in 0...512) + grid.push([for (j in 0...64) entities[(i * 64 + j) % ENTITIES]]); + } + + override public function run():Int { + var rng = new Rng(0x1234567); + var sum = 0; + for (frame in 0...FRAMES) { + var started = Timer.stamp(); + particles = null; + for (i in 0...2000) + particles = new FrameParticle(frame + i, particles); + var p = particles; + while (p != null) { + sum = (sum + p.value) | 0; + p = p.next; + } + events = []; + for (i in 0...100) { + var target = entities[rng.below(ENTITIES)]; + events.push({target: target, message: "event-" + frame + ":" + i}); + } + for (event in events) + sum = (sum + event.target.id + event.message.length) | 0; + for (i in 0...3000) { + var entity = entities[rng.below(ENTITIES)]; + entity.target = entities[rng.below(ENTITIES)]; + sum = (sum + entity.id + entity.target.id) | 0; + } + for (i in 0...300) { + var entity = entities[rng.below(ENTITIES)]; + var slot = rng.below(4); + entity.inventory[slot] = new FrameItem(frame * 300 + i); + sum = (sum + entity.inventory[slot].value) | 0; + } + for (i in 0...60) { + var cell = grid[rng.below(512)]; + var slot = rng.below(64); + cell[slot] = entities[rng.below(ENTITIES)]; + sum = (sum + cell[slot].id) | 0; + } + if ((frame & 63) == 0) { + burst = [for (i in 0...5000) new FrameItem(frame * 5000 + i)]; + for (item in burst) + sum = (sum + item.value) | 0; + } + latencies.push(Timer.stamp() - started); + } + // Check retained payloads without folding mutable state from earlier + // samples into the checksum. Every run uses the same random sequence. + for (entity in entities) { + if (entity.label != "entity-" + entity.id || entities[entity.id] != entity) + throw "gc_frames entity graph changed"; + if (entity.target != null && entities[entity.target.id] != entity.target) + throw "gc_frames target changed"; + for (item in entity.inventory) + if (item.value != (item.id ^ 0x13579)) + throw "gc_frames inventory changed"; + sum = (sum * 31 + entity.id + entity.label.length) | 0; + } + for (cell in grid) + for (entity in cell) + if (entities[entity.id] != entity) + throw "gc_frames grid changed"; + return sum; + } +} diff --git a/bench/src/Main.hx b/bench/src/Main.hx index 6a73b220..34e77314 100644 --- a/bench/src/Main.hx +++ b/bench/src/Main.hx @@ -1,6 +1,7 @@ package; import AllocBenchmarks; +import GcFrameBenchmarks; import MapBenchmarks; import SortBenchmarks; import StringBenchmarks; @@ -26,7 +27,7 @@ class Main { return CppiaBenchmarks.all(); #else return [ - new AllocSmall(), new AllocStrings(), new AllocArrays(), new GcPauses(), new MapInt(), new MapString(), new MapObject(), + new AllocSmall(), new AllocStrings(), new AllocArrays(), new GcPauses(), new GcFrames(), new MapInt(), new MapString(), new MapObject(), new StringBuild(), new StringIndex(), new StringSubstr(), new StringSplit(), new StringCase(), new SortInt(), new SortString(), new SortObject(), ]; @@ -121,6 +122,9 @@ class Main { peakRss: Native.peakRss(), heapReserved: Gc.memInfo64(Gc.MEM_INFO_RESERVED), heapUsage: Gc.memInfo64(Gc.MEM_INFO_USAGE), + #if HXCPP_GC_INCREMENTAL + incrementalGc: hxcpp.gc.Incremental.getStats(), + #end })); } diff --git a/docs/IncrementalGc.md b/docs/IncrementalGc.md new file mode 100644 index 00000000..3ca615b2 --- /dev/null +++ b/docs/IncrementalGc.md @@ -0,0 +1,106 @@ +# Incremental GC + +Tracked in [#372](https://github.com/openfl/libhxcpp/issues/372). This is an experimental, opt-in collector for native C++ programs using Haxe 4.3 or later in the 4.x series. The existing collector stays the default. The concurrent prototype in [#327](https://github.com/openfl/libhxcpp/issues/327) is deferred in the separate `feature/concurrent-gc` worktree. + +Incremental marking is implemented, including automatic pacing, scheduling hooks, write barriers, resumable containers, synchronous recovery and verification. It does not yet meet every performance gate in #372. A requested duration is a target, not a deadline: root capture, thread rendezvous, one object's native `__Mark`, weak closure, finalizers, reclamation and forced completion can run beyond it. Application frame timings remain the measure of interruption seen by a game. + +## Enable and use + +Regenerate and rebuild the program and its native code with: + +```sh +haxe build.hxml -lib hxcpp -D HXCPP_GC_INCREMENTAL +``` + +The library macro requests the existing Haxe generator's setter barriers. It enables `HXCPP_GC_GENERATIONAL` internally for code generation, while the runtime keeps ordinary allocation headers and does not enable the nursery. No compiler or standard library patch is needed. Do not add the generational flag yourself or reuse C++ generated without the incremental flag. + +Automatic collection needs no engine calls. It accounts for newly acquired allocation blocks and large buffers. It starts marking after half the normal allocation reserve is spent, uses a 0.5 ms slice target and a preferred 1 ms interval, and requests earlier assists after three quarters of the reserve has been spent. Deadlines and pause counters use a monotonic clock. Exhausting available space can still force synchronous completion. `Gc.enable(false)` disables automatic collection as before; explicit collection and scheduling calls still work. + +Optional engine hooks are library-owned: + +```haxe +import hxcpp.gc.Incremental; + +Incremental.configure(0.5, 1.0); + +// During spare frame time, advance a cycle that automatic pacing started. +if (Incremental.isActive()) + Incremental.step(0.3); +``` + +- `step(budgetMs = 0.5)` starts or advances a cycle and returns whether work remains. Zero only queries pending work. Repeated calls can finish a cycle with automatic collection disabled. Calling it unconditionally on an idle heap starts unnecessary major cycles. +- `configure(budgetMs = 0.5, intervalMs = 1.0)` changes pacing. The interval is preferred spacing, which allocation pressure can shorten. Budgets must be finite and positive, at most 1000 ms. Intervals can be zero and must be finite, between zero and 1000 ms. +- `isActive()` reports unfinished work. +- `getStats()` returns cumulative counters described below. Without the build flag it returns `{active: false}`; `isActive()` returns false, and scheduling calls throw an explanatory error. + +`Gc.run`, `Gc.compact` and disabling collection finish an active cycle before returning. A census abandons the partial mark and performs an independent synchronous collection. + +## Marking and mutation protocol + +All application threads stop while their fields are traced. Existing parallel marker workers can share a slice. No marker worker reads application fields after application threads resume, and no heap graph is copied into a second snapshot. + +The mark epoch and pending object jobs survive between slices. Roots and thread stacks are captured again at each stop. Newly allocated objects and buffers are white during marking. Discovering one through a root or an inserted edge marks it normally. A store into a holder already discovered by the cycle logs its new target; stores into white holders need no log because reaching the holder will scan its fields. A deletion can retain old work until the next cycle. When a budgeted cycle first drains its strong mark work, it yields once before finishing. The next stop rescans roots and new mutation entries before weak processing and reclamation, so finishing does not automatically follow a full marking slice in the same interruption. + +Native bulk operations use a mutation scope that records the holder before and after changing its storage or order. A budgeted cycle cannot finish inside an open mutation scope. A synchronous request at that boundary restarts a full collection. Native constructors and stores that bypass generated setters must use the appropriate `HX_OBJ_WB_GET` or `HX_OBJ_WB_NEW_MARKED_OBJECT` hooks, as the runtime wrappers do. Arbitrary native field writes without barriers are outside this mode's contract. + +Arrays and maps longer than 256 elements register jobs containing a managed owner and a cursor. Each chunk reacquires current storage from that owner. Structural mutation resets the cursor. Scalar insertion shades its target without restarting the whole container. Several chunks feed one marker round to avoid waking the pool for every small chunk. Traditional raw array-buffer jobs cannot survive an incremental pause. + +No heap storage is reclaimed while marking is unfinished. Completed collection free ranges are materialized before starting a new partial mark, so lazy allocator reclamation cannot interpret partial row marks as a completed collection. Detached threads transfer their unread mutation entries before their contexts are pooled or destroyed. + +If mutation metadata cannot be retained, the partial cycle is discarded before any reclamation. Recovery rewinds headers changed by the partial epoch before restarting a full mark. Advancing directly to another epoch would make unvisited objects appear marked to the alternating-bit fast path. Tests cover that failure transition. Log storage is capped at 1,048,576 entries per thread, and range registration at 1,048,576 owners per cycle. Native allocation failure requests recovery; this is not a guarantee that a process with exhausted system memory can continue. + +## Verification and tests + +`HXCPP_GC_INCREMENTAL_VERIFY` runs a separate root and descendant walk before weak processing and reclamation. It has its own visited set, bypasses incremental header tests and checks reachable allocation headers and occupied rows. The define applies to generated code and runtime code, including inline marking helpers. It adds a substantial pause and memory cost and must be omitted from performance measurements. + +`HXCPP_GC_INCREMENTAL_TEST` supplies deterministic phase limits and failure injection. It belongs only in test builds. `IncrementalGcVerifier` deliberately removes a descendant's mark in a subprocess and requires the independent walk to reject it. + +Conventional fixtures under `test/regression` cover: + +| Fixture | Coverage | +| --- | --- | +| `GcMutationRoots` | Graph rewiring, older descendants through new holders, anonymous fields and mutable closure captures | +| `GcContainerMutation` | Pointer-array growth, splice and overlapping blit, plus integer, string and object map mutation | +| `IncrementalGcSlices` | Mutation after an acknowledged holder scan, detached writers, reflective stores, explicit completion, epoch recovery and invalid scheduling inputs | +| `IncrementalGcContainers` | Resize and reorder after a partially traversed array range, with payload checks after completion | +| `IncrementalGcFinishBoundary` | New roots, descendants and container stores published between strong marking and finishing | +| `IncrementalGcVerifier` | Independent detection of an omitted descendant | +| `GcWeakMapChain` | Weak-map reachability closure | +| `GenerationalFinalizerLifecycle` | Finalizer lifecycle semantics | +| `GenerationalLargeBufferLifecycle` | Large-buffer lifecycle semantics | + +For example, from a fixture directory: + +```sh +haxe build.hxml -lib hxcpp -D HXCPP_GC_INCREMENTAL -D HXCPP_GC_INCREMENTAL_VERIFY -D HXCPP_GC_INCREMENTAL_TEST +bin/Main +``` + +The CI workflow includes a native Haxe 4 incremental job for these fixtures and eight-thread stress, with both normal and big blocks. `test/gc-defines/check.sh` includes incremental, verifier/test, and single-threaded configurations at both block sizes. The ordinary default, generational and moving controls remain separate. + +## Telemetry and performance limits + +Counters include setup and explicit collections from process startup: + +| Field | Meaning | +| --- | --- | +| `active` | Work pending when the result is allocated | +| `cycles` | Completed incremental cycles | +| `pauses` | Successful collector invocations, including synchronous collections | +| `requestedSlices` | Invocations made with a positive slice budget, including a budgeted request that required recovery | +| `fallbacks` | Partial cycles discarded for synchronous recovery or exception cleanup | +| `forcedFinishes` | Active cycles entered with an unbudgeted request, including explicit collection and allocation pressure | +| `lastCycleSlices` | Stops used by the last completed incremental cycle | +| `totalPauseMs`, `lastPauseMs`, `maxPauseMs` | Collector ownership timings, including rendezvous, tracing, finishing work and release | +| `maxSlicePauseMs` | Longest invocation requested with a slice budget | +| `maxSynchronousPauseMs` | Longest unbudgeted invocation | + +These counters describe the collecting thread's successful invocations. They are not a distribution of every application thread's wait. Several assists can occur in one allocation burst, and explicit APIs can finish synchronously. Measure frame or request latency as well. The benchmark executable includes `incrementalGc` counters in its JSON when built with the flag. + +Development measurements must compare the same workload, source, toolchain and hardware against the existing libhxcpp collector, with verification and test controls omitted. Record checksums, frame or request distributions, throughput, RSS and reserved memory. The game frame workload has shown shorter tail frames locally, while the allocation-heavy `gc_pauses` workload still shows a material throughput cost. Do not treat either result as a universal speedup or a hard pause guarantee. The p99.9, maximum interruption, throughput and memory gates in #372 remain the acceptance criteria for promoting this mode beyond experimental use. + +## Supported configurations + +The initial implementation targets native C++ with Haxe 4.3 or later in the 4.x series. It rejects Emscripten, cppia/scriptable hosts, older Haxe versions and Haxe 5 API layouts, explicit nursery or generational mode, moving mode, always-move checks and recursive GC debug levels. Those paths need separate barrier, layout and scheduling work. `HXCPP_GC_INCREMENTAL_BARRIERS` is an internal macro marker, not a user setting. Rebuild native integrations with matching flags and audit their pointer stores before enabling the mode. + +The branch began at `4cdca960`. Before runtime changes, nine Windows x64 baseline runs passed: both mutation fixtures under default, generational and moving collectors, weak-map closure under default, and the two lifecycle fixtures under generational mode. [Development results](../bench/results/incremental-gc/README.md) record later validation, the comparison protocol and raw timing data. These measurements do not change the published README benchmark results. diff --git a/docs/build_xml/Defines.md b/docs/build_xml/Defines.md index e18a0971..f0258e3f 100755 --- a/docs/build_xml/Defines.md +++ b/docs/build_xml/Defines.md @@ -9,6 +9,9 @@ Defines affecting how the code is generated. These need to be in the command li |-------------------------|--------------------| | *HXCPP_DEBUGGER* | Add extra macros required by debugger. Usually added automatically be debugger haxelib | | *HXCPP_GC_GENERATIONAL* | Enable experimental generational garbage collector | +| *HXCPP_GC_INCREMENTAL* | Enable experimental incremental marking on native Haxe 4 C++ builds. Use `-lib hxcpp` and regenerate all C++. The existing generator's barriers are enabled automatically; do not combine with the generational, nursery or moving collectors, cppia or Emscripten. See [Incremental GC](../IncrementalGc.md) for scheduling hooks and limits | +| *HXCPP_GC_INCREMENTAL_VERIFY* | Independently retrace roots and descendants before an incremental cycle reclaims memory. Adds substantial memory and pause overhead; use for correctness checks, not timing results | +| *HXCPP_GC_INCREMENTAL_TEST* | Enable deterministic phase and failure controls used by the incremental GC regression fixtures. Requires `HXCPP_GC_INCREMENTAL`; omit from application and performance builds | | *annotate_source* | Add additional annotations to source code - useful for developing hxcpp | | *dll_export* | Export hxcpp runtime symbols | | *file_extension* | Set the extension (without the dot) of generated files. eg "-D file_extension=mm" for objc++ code | diff --git a/extraParams.hxml b/extraParams.hxml index c38d3e05..289f5cd9 100644 --- a/extraParams.hxml +++ b/extraParams.hxml @@ -1,2 +1,3 @@ --macro haxe.macro.Compiler.define("libhxcpp","6.0.0") --macro haxe.macro.Compiler.define("hxcpp","6.0.0") +--macro hxcpp.IncrementalGcConfig.configure() diff --git a/hxcpp/IncrementalGcConfig.hx b/hxcpp/IncrementalGcConfig.hx new file mode 100644 index 00000000..97cd91b1 --- /dev/null +++ b/hxcpp/IncrementalGcConfig.hx @@ -0,0 +1,27 @@ +package hxcpp; + +#if macro +import haxe.macro.Compiler; +import haxe.macro.Context; +#end + +class IncrementalGcConfig { + public static macro function configure():haxe.macro.Expr { + if (Context.defined("HXCPP_GC_INCREMENTAL") && !Context.defined("HXCPP_GC_INCREMENTAL_BARRIERS")) { + for (name in ["HXCPP_GC_MOVING", "HXCPP_GC_DEBUG_ALWAYS_MOVE", "HXCPP_GC_NURSERY", "HXCPP_GC_GENERATIONAL", "HXCPP_GC_CONCURRENT", "scriptable", "cppia", "emscripten"]) { + if (Context.defined(name)) + Context.fatalError('HXCPP_GC_INCREMENTAL does not currently support $name', Context.currentPos()); + } + if (Std.parseInt(Context.definedValue("HXCPP_GC_DEBUG_LEVEL")) > 0) + Context.fatalError("HXCPP_GC_INCREMENTAL does not support recursive GC debug levels", Context.currentPos()); + var version = Std.parseFloat(Context.definedValue("haxe_ver")); + if (!Context.defined("cpp") || version < 4.3 || version >= 5) + Context.fatalError("HXCPP_GC_INCREMENTAL currently requires native C++ with Haxe 4.3 or later in the 4.x series", Context.currentPos()); + // Request the existing generator's setter barriers. The runtime uses + // ordinary allocation headers, not the generational nursery. + Compiler.define("HXCPP_GC_GENERATIONAL"); + Compiler.define("HXCPP_GC_INCREMENTAL_BARRIERS"); + } + return macro null; + } +} diff --git a/hxcpp/gc/Incremental.hx b/hxcpp/gc/Incremental.hx new file mode 100644 index 00000000..26d72f35 --- /dev/null +++ b/hxcpp/gc/Incremental.hx @@ -0,0 +1,35 @@ +package hxcpp.gc; + +#if !cpp +#error "Incremental GC is available on the native cpp target" +#end + +/** Optional scheduling hooks. Automatic collection needs no calls to this class. */ +class Incremental { + /** Perform a slice. Returns true if the collection still has work to do. */ + public static function step(budgetMs:Float = 0.5):Bool { + return Native.step(budgetMs); + } + + /** Set the automatic slice target and preferred spacing. Memory pressure can request earlier work. */ + public static function configure(budgetMs:Float = 0.5, intervalMs:Float = 1.0):Void { + Native.configure(budgetMs, intervalMs); + } + + public static function isActive():Bool { + return Native.active(); + } + + /** Cumulative counters and end-to-end pause timings since process startup. */ + public static function getStats():Dynamic { + return Native.stats(); + } +} + +@:include("hx/GC.h") +private extern class Native { + @:native("__hxcpp_gc_incremental_step") static function step(budgetMs:Float):Bool; + @:native("__hxcpp_gc_incremental_configure") static function configure(budgetMs:Float, intervalMs:Float):Void; + @:native("__hxcpp_gc_incremental_active") static function active():Bool; + @:native("__hxcpp_gc_incremental_stats") static function stats():Dynamic; +} diff --git a/include/Array.h b/include/Array.h index 6d2f6bd3..d1184a62 100644 --- a/include/Array.h +++ b/include/Array.h @@ -234,6 +234,7 @@ class HXCPP_EXTERN_CLASS_ATTRIBUTES ArrayBase : public ArrayCommon void setUnmanagedData(void *inData, int inElements) { + HX_INCREMENTAL_MUTATION(this); mBase = (char *)inData; length = inElements; mAlloc = -1; @@ -745,11 +746,27 @@ class Array_obj : public hx::ArrayBase if (mAlloc>0) hx::MarkAlloc((void *)mBase, __inCtx ); if (length && hx::ContainsPointers()) { + #ifdef HXCPP_GC_INCREMENTAL + if (length > 256 && hx::IncrementalRange(this,IncrementalScan,__inCtx)) return; + #endif ELEM_ *ptr = (ELEM_ *)mBase; HX_MARK_MEMBER_ARRAY(ptr,length); } } + #ifdef HXCPP_GC_INCREMENTAL + static bool IncrementalScan(hx::Object *inOwner, size_t &ioCursor, size_t &ioItem, hx::MarkContext *__inCtx) { + Array_obj *array = static_cast *>(inOwner); + size_t length = array->length; + if (ioCursor >= length) return true; + size_t count = std::min(size_t{256},length-ioCursor); + ELEM_ *ptr = reinterpret_cast(array->mBase) + ioCursor; + HX_MARK_MEMBER_ARRAY(ptr,count); + ioCursor += count; + return ioCursor >= length; + } + #endif + #ifdef HXCPP_VISIT_ALLOCS void __Visit(hx::VisitContext *__inCtx) HXCPP_OVERRIDE { @@ -953,6 +970,7 @@ class Array_obj : public hx::ArrayBase void reverse() { + HX_INCREMENTAL_MUTATION(this); int half = length/2; ELEM_ *e = (ELEM_ *)mBase; for(int i=0;i Array_obj::__new(int inSize,int inReserve) template void Array_obj::sort(SorterFunc inSorter) { + HX_INCREMENTAL_MUTATION(this); if (hx::ArrayValueSortable::Yes) { // Plain values. (Dispatching on the StoreType here would send diff --git a/include/hx/GC.h b/include/hx/GC.h index 87d2e5bf..cf82fe36 100644 --- a/include/hx/GC.h +++ b/include/hx/GC.h @@ -22,6 +22,14 @@ // Function called by the haxe code... +HXCPP_EXTERN_CLASS_ATTRIBUTES bool __hxcpp_gc_incremental_step(double inBudgetMs); +HXCPP_EXTERN_CLASS_ATTRIBUTES void __hxcpp_gc_incremental_configure(double inBudgetMs, double inIntervalMs); +HXCPP_EXTERN_CLASS_ATTRIBUTES bool __hxcpp_gc_incremental_active(); +HXCPP_EXTERN_CLASS_ATTRIBUTES Dynamic __hxcpp_gc_incremental_stats(); +#ifdef HXCPP_GC_INCREMENTAL_TEST +HXCPP_EXTERN_CLASS_ATTRIBUTES int __hxcpp_gc_incremental_test(int inOperation, hx::Object *inObject=0); +#endif + #ifdef HXCPP_TELEMETRY extern void __hxt_gc_new(hx::StackContext *inStack, void* obj, int inSize, const char *inName); extern void __hxt_gc_alloc(void* obj, int inSize); @@ -349,6 +357,37 @@ namespace hx struct StackContext; +#ifdef HXCPP_GC_INCREMENTAL +class ImmixAllocator; +struct IncrementalEntry +{ + hx::Object *owner; + const void *value; +}; +extern HXCPP_EXTERN_CLASS_ATTRIBUTES std::atomic gIncrementalActive; +extern HXCPP_EXTERN_CLASS_ATTRIBUTES bool gIncrementalAllocationsWhite; +extern HXCPP_EXTERN_CLASS_ATTRIBUTES int gPrevByteMarkID; +HXCPP_EXTERN_CLASS_ATTRIBUTES void IncrementalRemember(hx::Object *inOwner, const void *inValue, StackContext *inCtx); +HXCPP_EXTERN_CLASS_ATTRIBUTES void IncrementalAllocated(void *inValue, bool inObject, ImmixAllocator *inCtx); +typedef bool (*IncrementalRangeScan)(hx::Object *, size_t &, size_t &, MarkContext *); +HXCPP_EXTERN_CLASS_ATTRIBUTES bool IncrementalRange(hx::Object *inOwner, IncrementalRangeScan inScan, MarkContext *inCtx); +HXCPP_EXTERN_CLASS_ATTRIBUTES void IncrementalMutationEnter(hx::Object *inOwner, StackContext *inCtx); +HXCPP_EXTERN_CLASS_ATTRIBUTES void IncrementalMutationLeave(hx::Object *inOwner, StackContext *inCtx); +#ifdef HXCPP_GC_INCREMENTAL_VERIFY +bool IncrementalOracleMark(void *inPointer, bool inObject, MarkContext *inCtx); +#endif +struct IncrementalMutationScope +{ + hx::Object *owner; + StackContext *ctx; + IncrementalMutationScope(hx::Object *inOwner, StackContext *inCtx) : owner(inOwner), ctx(inCtx) { IncrementalMutationEnter(owner, ctx); } + ~IncrementalMutationScope() { IncrementalMutationLeave(owner, ctx); } +}; +#define HX_INCREMENTAL_MUTATION(owner) ::hx::IncrementalMutationScope _hx_incremental_mutation(const_cast<::hx::Object *>(static_cast(owner)),HX_CTX_GET) +#else +#define HX_INCREMENTAL_MUTATION(owner) +#endif + EXTERN_FAST_TLS_DATA(StackContext, tlsStackContext); extern StackContext *gMainThreadContext; @@ -456,6 +495,13 @@ class ImmixAllocator __hxt_gc_alloc(buffer, inSize); __hxt_gc_new((hx::StackContext *)alloc,buffer, inSize, inName); #endif + #ifdef HXCPP_GC_INCREMENTAL + if (gIncrementalActive.load(std::memory_order_relaxed)) { + if (gIncrementalAllocationsWhite) + reinterpret_cast(buffer)[HX_ENDIAN_MARK_ID_BYTE] = static_cast(gPrevByteMarkID); + else IncrementalAllocated(buffer,inContainer,alloc); + } + #endif return buffer; } @@ -475,7 +521,11 @@ typedef ImmixAllocator GcAllocator; typedef ImmixAllocator Ctx; -#ifdef HXCPP_GC_GENERATIONAL +#if defined(HXCPP_GC_INCREMENTAL) + #define HX_OBJ_WB_CTX(obj,value,ctx) { const void *_hx_inc_value = value; if (_hx_inc_value && ::hx::gIncrementalActive.load(std::memory_order_relaxed)) { ::hx::Object *_hx_inc_owner = obj; if (!_hx_inc_owner || reinterpret_cast(_hx_inc_owner)[HX_ENDIAN_MARK_ID_BYTE] == ::hx::gByteMarkID) ::hx::IncrementalRemember(_hx_inc_owner,_hx_inc_value,ctx); } } + #define HX_OBJ_WB_PESSIMISTIC_CTX(obj,ctx) { if (::hx::gIncrementalActive.load(std::memory_order_relaxed)) { ::hx::Object *_hx_inc_owner = obj; if (!_hx_inc_owner || reinterpret_cast(_hx_inc_owner)[HX_ENDIAN_MARK_ID_BYTE] == ::hx::gByteMarkID) ::hx::IncrementalRemember(_hx_inc_owner,nullptr,ctx); } } + #define HX_OBJ_WB_NEW_MARKED_OBJECT(obj) HX_OBJ_WB_PESSIMISTIC_CTX(obj,HX_CTX_GET) +#elif defined(HXCPP_GC_GENERATIONAL) #define HX_OBJ_WB_CTX(obj,value,ctx) { \ unsigned char &mark = ((unsigned char *)(obj))[ HX_ENDIAN_MARK_ID_BYTE]; \ if (mark == ::hx::gByteMarkID && value && !((unsigned char *)(value))[ HX_ENDIAN_MARK_ID_BYTE ] ) { \ @@ -499,7 +549,9 @@ typedef ImmixAllocator Ctx; #endif #define HX_OBJ_WB(obj,value) HX_OBJ_WB_CTX(obj,value,_hx_ctx) -#ifdef HXCPP_GC_GENERATIONAL +#ifdef HXCPP_GC_INCREMENTAL + #define HX_ARRAY_WB(array,index,value) HX_OBJ_WB_CTX(array,value,_hx_ctx) +#elif defined(HXCPP_GC_GENERATIONAL) // Storing a young value in an old array: see ArrayYoungStore #define HX_ARRAY_WB(array,index,value) { \ unsigned char mark = ((unsigned char *)(array))[ HX_ENDIAN_MARK_ID_BYTE]; \ @@ -532,6 +584,9 @@ HXCPP_EXTERN_CLASS_ATTRIBUTES void NewMarkedObject(hx::Object *inPtr); inline void MarkAlloc(void *inPtr ,hx::MarkContext *__inCtx) { + #ifdef HXCPP_GC_INCREMENTAL_VERIFY + if (IncrementalOracleMark(inPtr,false,__inCtx)) return; + #endif #ifdef __EMSCRIPTEN__ // Unaligned must be constants... if ( ((size_t)inPtr) & 3 ) @@ -545,6 +600,9 @@ inline void MarkAlloc(void *inPtr ,hx::MarkContext *__inCtx) } inline void MarkObjectAlloc(hx::Object *inPtr ,hx::MarkContext *__inCtx) { + #ifdef HXCPP_GC_INCREMENTAL_VERIFY + if (IncrementalOracleMark(inPtr,true,__inCtx)) return; + #endif #ifdef __EMSCRIPTEN__ // Unaligned must be constants... if ( !( ((size_t)inPtr) & 3) ) diff --git a/include/hx/StackContext.h b/include/hx/StackContext.h index 8ab939f6..e3e12539 100644 --- a/include/hx/StackContext.h +++ b/include/hx/StackContext.h @@ -414,6 +414,11 @@ inline bool hxExceptionUnwinding() struct StackContext : public hx::ImmixAllocator { + #ifdef HXCPP_GC_INCREMENTAL + std::vector mIncrementalEntries; + size_t mIncrementalRead = 0; + unsigned int mIncrementalMutations = 0; + #endif #ifdef HXCPP_STACK_IDS int mThreadId; #endif diff --git a/include/hxcpp.h b/include/hxcpp.h index 485ffadc..8880652e 100755 --- a/include/hxcpp.h +++ b/include/hxcpp.h @@ -10,6 +10,26 @@ #define HXCPP_API_LEVEL 0 #endif +#ifdef HXCPP_GC_INCREMENTAL + #if !defined(HXCPP_GC_INCREMENTAL_BARRIERS) || !defined(HXCPP_GC_GENERATIONAL) + #error "Incremental GC requires regenerated C++ with -lib hxcpp -D HXCPP_GC_INCREMENTAL" + #endif + #if defined(HXCPP_GC_MOVING) || defined(HXCPP_GC_DEBUG_ALWAYS_MOVE) || HXCPP_GC_DEBUG_LEVEL > 0 || defined(HXCPP_GC_NURSERY) || defined(HXCPP_SCRIPTABLE) || defined(__EMSCRIPTEN__) || HXCPP_API_LEVEL < 430 || HXCPP_API_LEVEL >= 500 + #error "Unsupported incremental GC configuration" + #endif + // The Haxe define requests generated setter barriers. Native code uses + // non-nursery headers and the incremental barrier implementation below. + #include +#endif + +#if (defined(HXCPP_GC_INCREMENTAL_VERIFY) || defined(HXCPP_GC_INCREMENTAL_TEST)) && !defined(HXCPP_GC_INCREMENTAL) + #error "Incremental GC diagnostics require HXCPP_GC_INCREMENTAL" +#endif + +#if defined(HXCPP_GC_GENERATIONAL) && !defined(HXCPP_GC_INCREMENTAL) + #define HXCPP_GC_GENERATIONAL_RUNTIME +#endif + #include "hx/HeaderVersion.h" #if defined(_MSC_VER) || defined(__BORLANDC__) @@ -307,7 +327,7 @@ class VisitContext typedef ::cpp::Variant Val; -#ifdef HXCPP_GC_GENERATIONAL +#ifdef HXCPP_GC_GENERATIONAL_RUNTIME #define HXCPP_GC_NURSERY #endif diff --git a/src/Array.cpp b/src/Array.cpp index b21a4ef9..c072353f 100644 --- a/src/Array.cpp +++ b/src/Array.cpp @@ -148,6 +148,7 @@ ArrayBase::ArrayBase(int inSize,int inReserve,int inElementSize,bool inAtomic) void ArrayBase::reserve(int inSize) const { + HX_INCREMENTAL_MUTATION(this); if (mAlloc__ToInt(); if (first<0) first+=length; @@ -283,6 +286,7 @@ int ArrayBase::Memcmp(ArrayBase *inOther) void ArrayBase::Blit(int inDestElement, ArrayBase *inSourceArray, int inSourceElement, int inElementCount) { + HX_INCREMENTAL_MUTATION(this); int srcSize = inSourceArray->GetElementSize(); int srcElems = inSourceArray->length; if (inDestElement<0 || inSourceElement<0 || inElementCount<0 || @@ -336,6 +340,7 @@ String ArrayBase::toString() void ArrayBase::__SetSizeExact(int inSize) { + HX_INCREMENTAL_MUTATION(this); if (inSize==0) { if (mAlloc>=0) @@ -397,6 +402,7 @@ Dynamic ArrayBase::__unsafe_set(const Dynamic &i, const Dynamic &val) void ArrayBase::Insert(int inPos) { + HX_INCREMENTAL_MUTATION(this); if (inPos>=length) resize(length+1); else @@ -409,6 +415,7 @@ void ArrayBase::Insert(int inPos) void ArrayBase::Splice(ArrayBase *outResult,int inPos,int inLen) { + HX_INCREMENTAL_MUTATION(this); if (inPos>=length) { return; @@ -446,6 +453,7 @@ void ArrayBase::Splice(ArrayBase *outResult,int inPos,int inLen) void ArrayBase::Slice(ArrayBase *outResult,int inPos,int inEnd) { + HX_INCREMENTAL_MUTATION(this); if (inPos<0) { inPos += length; @@ -475,6 +483,7 @@ void ArrayBase::Slice(ArrayBase *outResult,int inPos,int inEnd) void ArrayBase::RemoveElement(int inPos) { + HX_INCREMENTAL_MUTATION(this); if (inPosGetBase(); int n = length * GetElementSize(); // Either array may be empty, with no buffer @@ -643,6 +653,7 @@ String ArrayBase::joinArray(ArrayBase *inBase, String inSeparator) void ArrayBase::safeSort(DynamicSorterFunc inSorter, bool inIsString) { + HX_INCREMENTAL_MUTATION(this); if (inIsString) { // The comparator takes Dynamic arguments, so box each String once diff --git a/src/hx/Hash.h b/src/hx/Hash.h index d96be8ef..684d4797 100644 --- a/src/hx/Hash.h +++ b/src/hx/Hash.h @@ -576,6 +576,7 @@ struct Hash : public HashBase< typename ELEMENT::Key > void switchToMixedHash() { + HX_INCREMENTAL_MUTATION(this); mixIntKeys = true; if (bucketCount) redistribute(bucketCount); @@ -614,6 +615,7 @@ struct Hash : public HashBase< typename ELEMENT::Key > bool remove(Key inKey) HXCPP_OVERRIDE { + HX_INCREMENTAL_MUTATION(this); if (!bucket) return false; releaseOversizedBuckets(); @@ -765,6 +767,14 @@ struct Hash : public HashBase< typename ELEMENT::Key > // the map decides, so it is remembered whole void rememberYoung(const void *inA, const void *inB, const void *inC) { + #ifdef HXCPP_GC_INCREMENTAL + if (Element::WeakKeys) hx::IncrementalRemember(this,nullptr,HX_CTX_GET); + else { + if (inA) hx::IncrementalRemember(this,inA,HX_CTX_GET); + if (inB) hx::IncrementalRemember(this,inB,HX_CTX_GET); + if (inC) hx::IncrementalRemember(this,inC,HX_CTX_GET); + } + #else unsigned char &mark = ((unsigned char *)(this))[ HX_ENDIAN_MARK_ID_BYTE]; if (mark == hx::gByteMarkID && (IsNursery(inA) || IsNursery(inB) || IsNursery(inC)) ) { @@ -779,6 +789,7 @@ struct Hash : public HashBase< typename ELEMENT::Key > hx::YoungStore(this, this->mYoungLogged, size, values, 3, HX_CTX_GET); } } + #endif } #endif @@ -808,6 +819,7 @@ struct Hash : public HashBase< typename ELEMENT::Key > #endif return; } + HX_INCREMENTAL_MUTATION(this); el = allocElement(); // Growth in allocElement can switch an int map to the mixed hash. if (IgnoreHash) @@ -833,6 +845,7 @@ struct Hash : public HashBase< typename ELEMENT::Key > void clear() HXCPP_OVERRIDE { + HX_INCREMENTAL_MUTATION(this); bucket = 0; bucketsOversized = false; size = 0; @@ -1005,10 +1018,35 @@ struct Hash : public HashBase< typename ELEMENT::Key > { HX_MARK_ARRAY(bucket); + #ifdef HXCPP_GC_INCREMENTAL + if (size > 256 && hx::IncrementalRange(this,IncrementalScan,__inCtx)) return; + #endif + HashMarker marker(__inCtx); iterate(marker); } + #ifdef HXCPP_GC_INCREMENTAL + static bool IncrementalScan(hx::Object *inOwner, size_t &ioBucket, size_t &ioItem, hx::MarkContext *__inCtx) { + Hash *hash = static_cast(inOwner); + HashMarker marker(__inCtx); + Element *element = reinterpret_cast(ioItem); + unsigned int work = 0; + while (ioBucket < static_cast(hash->bucketCount) && work < 256) { + if (!element) element = hash->bucket[ioBucket]; + if (!element) { ioBucket++; work++; continue; } + while (element && work < 256) { + marker(element); + element = element->next; + work++; + } + if (!element) ioBucket++; + } + ioItem = reinterpret_cast(element); + return ioBucket >= static_cast(hash->bucketCount); + } + #endif + #ifdef HXCPP_VISIT_ALLOCS void __Visit(hx::VisitContext *__inCtx) HXCPP_OVERRIDE diff --git a/src/hx/gc/Immix.cpp b/src/hx/gc/Immix.cpp index 0eca7e65..6daf4930 100644 --- a/src/hx/gc/Immix.cpp +++ b/src/hx/gc/Immix.cpp @@ -16,6 +16,10 @@ #include #include +#ifdef HXCPP_GC_INCREMENTAL +#include +#endif + #ifdef __EMSCRIPTEN__ #include #ifdef HXCPP_SINGLE_THREADED_APP @@ -214,7 +218,7 @@ static size_t sgMaximumFreeSpace = 1024*1024*1024; // every collect so you can see where it goes wrong (usually requires HX_GC_FIXED_BLOCKS) //#define HX_WATCH -#if defined(HXCPP_GC_VERIFY) && defined(HXCPP_GC_GENERATIONAL) +#if defined(HXCPP_GC_VERIFY) && defined(HXCPP_GC_GENERATIONAL_RUNTIME) #define HX_GC_VERIFY_GENERATIONAL #endif @@ -486,6 +490,9 @@ static void VisitLocalAlloc(LocalAllocator *inAlloc,hx::VisitContext *__inCtx); static void WaitForSafe(LocalAllocator *inAlloc); static void ReleaseFromSafe(LocalAllocator *inAlloc); static void ClearPooledAlloc(LocalAllocator *inAlloc); +#ifdef HXCPP_GC_INCREMENTAL +static void RestoreIncrementalAlloc(LocalAllocator *inAlloc); +#endif static void CollectFromThisThread(bool inMajor,bool inForceCompact); // Non-zero while Gc.compact() is collecting. Only then are all empty groups given @@ -829,7 +836,7 @@ struct BlockDataInfo unsigned char mZeroed; bool mReclaimed; bool mOwned; - #ifdef HXCPP_GC_GENERATIONAL + #ifdef HXCPP_GC_GENERATIONAL_RUNTIME bool mHasSurvivor; #endif std::atomic_int mZeroLock; @@ -902,7 +909,7 @@ struct BlockDataInfo void clearBlockMarks() { mPinned = false; - #ifdef HXCPP_GC_GENERATIONAL + #ifdef HXCPP_GC_GENERATIONAL_RUNTIME mHasSurvivor = false; #endif } @@ -1004,7 +1011,7 @@ struct BlockDataInfo void countRows(BlockDataStats &outStats) { - #ifdef HXCPP_GC_GENERATIONAL + #ifdef HXCPP_GC_GENERATIONAL_RUNTIME // Only a minor collection counts rows, and it marks rows in a block only // for an object it marks there, which sets mHasSurvivor. Without one, // the row marks are the ones mUsedRows was worked out from, so it stands, @@ -1710,6 +1717,9 @@ struct GlobalChunks int takeArrayJob(hx::Object **inPtr, int inLen) { + #ifdef HXCPP_GC_INCREMENTAL + if (gIncrementalActive.load(std::memory_order_relaxed)) return 0; + #endif if (sLazyThreads) { int n = (inLen/2) & ~15; @@ -1891,7 +1901,7 @@ struct GlobalChunks GlobalChunks sGlobalChunks; -#ifdef HXCPP_GC_GENERATIONAL +#ifdef HXCPP_GC_GENERATIONAL_RUNTIME // A young value stored in an old array or map made the whole of it a // remembered referrer, so the next minor collection marked every element, // however few changed: one store in an old four-million-element array added @@ -1993,6 +2003,15 @@ static void MarkYoungValues(bool inMark, hx::MarkContext *inCtx) } #endif +#ifdef HXCPP_GC_INCREMENTAL +static double IncrementalNow() +{ + return std::chrono::duration(std::chrono::steady_clock::now().time_since_epoch()).count(); +} +static bool IncrementalExpired(); +static void IncrementalScanned(Object *inObject); +#endif + class MarkContext { #ifdef HXCPP_DEBUG @@ -2009,6 +2028,12 @@ class MarkContext bool isGenerational; + #ifdef HXCPP_GC_INCREMENTAL_VERIFY + bool incrementalOracle = false; + std::unordered_map oracleVisited; + std::vector oracleQueue; + #endif + #ifdef HX_GC_VERIFY_GENERATIONAL // The object whose members are being marked, so a nursery object that // escaped a generational collection can be reported with what held it @@ -2122,6 +2147,9 @@ class MarkContext { while(true) { + #ifdef HXCPP_GC_INCREMENTAL + if (IncrementalExpired()) { releaseIncrementalJobs(); return; } + #endif if (!marking || !marking->count) { #ifdef HX_MULTI_THREAD_MARKING @@ -2148,6 +2176,9 @@ class MarkContext while(marking) { + #ifdef HXCPP_GC_INCREMENTAL + if (IncrementalExpired()) { releaseIncrementalJobs(); return; } + #endif hx::Object *obj = marking->pop(); if (obj) { @@ -2158,6 +2189,9 @@ class MarkContext #else obj->__Mark(this); #endif + #ifdef HXCPP_GC_INCREMENTAL + IncrementalScanned(obj); + #endif #ifdef HX_MULTI_THREAD_MARKING // Load balance if (sLazyThreads && marking->count>32) @@ -2178,6 +2212,46 @@ class MarkContext } } } + + #ifdef HXCPP_GC_INCREMENTAL + void releaseIncrementalJobs() + { + if (marking && marking->count) + { + MarkChunk *head = sGlobalChunks.processList.load(); + do + { + marking->next = head; + } while (!sGlobalChunks.processList.compare_exchange_weak(head, marking)); + } + else if (marking) + sGlobalChunks.release(marking); + marking = nullptr; + if (mThreadId >= 0) + { + std::lock_guard lock(sThreadPoolLock); + if (sRunningThreads & (1 << mThreadId)) + sGlobalChunks.completeThreadLocked(mThreadId); + } + } + bool hasWork() const { return (marking && marking->count) || sGlobalChunks.processList; } + void discardJobs() + { + if (marking) + { + marking->count = 0; + sGlobalChunks.free(marking); + marking = nullptr; + } + while (sGlobalChunks.processList) + { + MarkChunk *chunk = sGlobalChunks.processList; + sGlobalChunks.processList = chunk->next; + chunk->count = 0; + sGlobalChunks.free(chunk); + } + } +#endif }; @@ -2248,8 +2322,34 @@ struct AutoMarkPush +#ifdef HXCPP_GC_INCREMENTAL_VERIFY +bool IncrementalOracleMark(void *inPointer, bool inObject, MarkContext *inCtx) +{ + if (!inCtx || !inCtx->incrementalOracle) + return false; + if (!inPointer) + return true; + unsigned int flags; + memcpy(&flags, static_cast(inPointer) - sizeof(flags), sizeof(flags)); + if (flags & HX_GC_CONST_ALLOC_BIT) + return true; + auto result = inCtx->oracleVisited.emplace(inPointer, inObject); + // A backing allocation can also have been encountered through an object + // pointer. In that case its outgoing members still need the object walk. + if (inObject && (flags & IMMIX_ALLOC_IS_CONTAINER) && (result.second || !result.first->second)) + { + result.first->second = true; + inCtx->oracleQueue.push_back(static_cast(inPointer)); + } + return true; +} +#endif + void MarkAllocUnchecked(void *inPtr,hx::MarkContext *__inCtx) { + #ifdef HXCPP_GC_INCREMENTAL_VERIFY + if (IncrementalOracleMark(inPtr,false,__inCtx)) return; + #endif #ifdef PROFILE_COLLECT sAllocMarks++; #endif @@ -2290,7 +2390,7 @@ void MarkAllocUnchecked(void *inPtr,hx::MarkContext *__inCtx) while(_hx_atomic_compare_exchange((volatile int *)pos, val,val|gImmixStartFlag[start&127]) != val) val = *pos; - #ifdef HXCPP_GC_GENERATIONAL + #ifdef HXCPP_GC_GENERATIONAL_RUNTIME info->mHasSurvivor = true; #endif } @@ -2362,6 +2462,9 @@ static void CensusCount(hx::Object *inPtr, unsigned int inFlags) void MarkObjectAllocUnchecked(hx::Object *inPtr,hx::MarkContext *__inCtx) { + #ifdef HXCPP_GC_INCREMENTAL_VERIFY + if (IncrementalOracleMark(inPtr,true,__inCtx)) return; + #endif size_t ptr_i = ((size_t)inPtr)-sizeof(int); unsigned int flags = *((unsigned int *)ptr_i); #ifdef HXCPP_GC_NURSERY @@ -2396,7 +2499,7 @@ void MarkObjectAllocUnchecked(hx::Object *inPtr,hx::MarkContext *__inCtx) unsigned int val = *pos; while(_hx_atomic_compare_exchange( (volatile int *)pos, val, val|gImmixStartFlag[start&127]) != val) val = *pos; - #ifdef HXCPP_GC_GENERATIONAL + #ifdef HXCPP_GC_GENERATIONAL_RUNTIME info->mHasSurvivor = true; #endif } @@ -3197,6 +3300,9 @@ void RegisterWeakHash(HashBase *inHash) void InternalEnableGC(bool inEnable) { + #ifdef HXCPP_GC_INCREMENTAL + if (!inEnable && gIncrementalActive.load()) InternalCollectBytes(true,false); + #endif //printf("Enable %d\n", sgInternalEnable); sgInternalEnable = inEnable; } @@ -3463,6 +3569,11 @@ static int sMaxZeroQueueSize = 32; #define BLOCK_OFSIZE_COUNT 12 +#ifdef HXCPP_GC_INCREMENTAL +#include "Incremental.h" +static void IncrementalAutomatic(hx::ImmixAllocator *inAlloc, bool inPressure); +#endif + class GlobalAllocator { enum { LOCAL_POOL_SIZE = 2 }; @@ -3503,7 +3614,7 @@ class GlobalAllocator // Until we add ourselves, the collector will not wait // on us - ie, we are assumed ot be in a GC free zone. std::lock_guard lock(*gThreadStateChangeLock); - #ifdef HXCPP_GC_GENERATIONAL + #ifdef HXCPP_GC_GENERATIONAL_RUNTIME // Under the lock, so a collection cannot switch modes between the // check and the push ((hx::StackContext *)inAlloc)->mOldReferrers = @@ -3572,6 +3683,10 @@ class GlobalAllocator void FreeLarge(void *inLarge) { + #ifdef HXCPP_GC_INCREMENTAL + // Pending marking jobs may still refer to replaced backing storage. + if (hx::gIncrementalActive.load(std::memory_order_relaxed)) return; + #endif #ifdef HXCPP_TELEMETRY __hxt_gc_free_large(inLarge); #endif @@ -3609,6 +3724,12 @@ class GlobalAllocator if (hx::gPauseForCollect) __hxcpp_gc_safe_point(); + #ifdef HXCPP_GC_INCREMENTAL + if (sgInternalEnable && (hx::gIncrementalActive.load(std::memory_order_relaxed) || + inSize+mIncrementalAllocated.load(std::memory_order_relaxed) > mIncrementalStartAt.load(std::memory_order_relaxed))) + IncrementalAutomatic(HX_CTX_GET,inSize+mIncrementalAllocated.load(std::memory_order_relaxed) > mIncrementalStartAt.load(std::memory_order_relaxed)*3/2); + #endif + //Should we force a collect ? - the 'large' data are not considered when allocating objects // from the blocks, and can 'pile up' between smalll object allocations if ((inSize+mLargeAllocated > mLargeAllocForceRefresh) && sgInternalEnable) @@ -3719,6 +3840,10 @@ class GlobalAllocator __hxt_gc_alloc(result + 2, inSize); #endif + #ifdef HXCPP_GC_INCREMENTAL + mIncrementalAllocated.fetch_add(inSize,std::memory_order_relaxed); + if (hx::gIncrementalActive.load(std::memory_order_relaxed)) hx::IncrementalAllocated(result+2,inContainer,HX_CTX_GET); + #endif return result+2; } @@ -3843,6 +3968,10 @@ class GlobalAllocator { info->mOwned = true; + #ifdef HXCPP_GC_INCREMENTAL + mIncrementalAllocated.fetch_add(info->GetFreeData(),std::memory_order_relaxed); + #endif + // Increase the mNextFreeBlockOfSize int idx = nextFreeBlock; while(idxmOwned) @@ -4010,7 +4139,7 @@ class GlobalAllocator bool allowMoreBlocks() { - #ifdef HXCPP_GC_GENERATIONAL + #ifdef HXCPP_GC_GENERATIONAL_RUNTIME return sGcMode==gcmFull; #else return true; @@ -4025,6 +4154,11 @@ class GlobalAllocator BlockDataInfo *GetFreeBlock(size_t inRequiredBytes, hx::ImmixAllocator *inAlloc) { + #ifdef HXCPP_GC_INCREMENTAL + if (sgInternalEnable && (hx::gIncrementalActive.load(std::memory_order_relaxed) || + mIncrementalAllocated.load(std::memory_order_relaxed) > mIncrementalStartAt.load(std::memory_order_relaxed))) + IncrementalAutomatic(inAlloc,mIncrementalAllocated.load(std::memory_order_relaxed) > mIncrementalStartAt.load(std::memory_order_relaxed)*3/2); + #endif while(true) { if (hx::gPauseForCollect) @@ -4597,7 +4731,7 @@ class GlobalAllocator VisitLocalAlloc(mLocalAllocs[i], inCtx); hx::VisitClassStatics(inCtx); - #if defined(HXCPP_SCRIPTABLE) && defined(HXCPP_GC_GENERATIONAL) + #if defined(HXCPP_SCRIPTABLE) && defined(HXCPP_GC_GENERATIONAL_RUNTIME) hx::scriptVisitModules(inCtx); #endif @@ -5002,13 +5136,32 @@ class GlobalAllocator double tMarkLocal; double tMarkLocalEnd; double tMarked; - void MarkAll(bool inGenerational) + #ifdef HXCPP_GC_INCREMENTAL + static void IncrementalMarkWorkers(hx::MarkContext *inCtx, void *inData) + { + if (!inCtx->hasWork()) + return; + GlobalAllocator *owner = static_cast(inData); +#ifdef HX_MULTI_THREAD_MARKING + if (MAX_GC_THREADS > 1) + { + inCtx->releaseJobs(); + owner->StartThreadJobs(tpjMark, MAX_GC_THREADS, true); + } + else +#endif + inCtx->processMarkStack(); + } +#endif + bool MarkAll(bool inGenerational, bool inResume=false) { // The old allocations' index stays: only a change to them drops it mLargeLookups = 0; mYoungLargeIndexReady = false; mYoungLargeLookups = 0; + if (!inResume) { + if (!inGenerational) { hx::gPrevByteMarkID = hx::gByteMarkID; @@ -5062,6 +5215,7 @@ class GlobalAllocator #endif MEM_STAMP(tMarkInit); + } #ifdef PROFILE_THREAD_USAGE for(int i=-1;iclear(); hx::MarkClassStatics(&mMarker); - #if defined(HXCPP_SCRIPTABLE) && defined(HXCPP_GC_GENERATIONAL) + #if defined(HXCPP_SCRIPTABLE) && defined(HXCPP_GC_GENERATIONAL_RUNTIME) hx::scriptMarkModules(&mMarker, inGenerational); #endif @@ -5111,7 +5265,7 @@ class GlobalAllocator #endif - #ifdef HXCPP_GC_GENERATIONAL + #ifdef HXCPP_GC_GENERATIONAL_RUNTIME { hx::AutoMarkPush info(&mMarker,"Young array values","young"); // The first mark of a collection takes the log: a full one has no use @@ -5143,6 +5297,27 @@ class GlobalAllocator MEM_STAMP(tMarkLocalEnd); + #ifdef HXCPP_GC_INCREMENTAL + if (hx::IncrementalData().tracing) { + if (!hx::IncrementalDrain(&mMarker,IncrementalMarkWorkers,this)) return false; + if (hx::IncrementalData().deadline > 0) { + for (int i=0;imIncrementalMutations) return false; + // Avoid adding weak processing and reclamation to a full marking + // slice. Roots and mutation logs are checked again at the next + // stop before the cycle is allowed to finish. + if (!hx::IncrementalData().finishPending) + { + hx::IncrementalData().finishPending = true; + return false; + } + } + hx::IncrementalData().tracing = false; + hx::gIncrementalAllocationsWhite = false; + } + else + #endif + { #ifdef HX_MULTI_THREAD_MARKING if (!sgCensus) { @@ -5154,6 +5329,7 @@ class GlobalAllocator else #endif mMarker.processMarkStack(); + } if (sgCensus) MarkCensusDeferred(); @@ -5162,6 +5338,16 @@ class GlobalAllocator MEM_STAMP(tMarked); + #ifdef HXCPP_GC_INCREMENTAL_VERIFY + #ifdef HXCPP_GC_INCREMENTAL_TEST + if (hx::IncrementalData().omit) { + reinterpret_cast(hx::IncrementalData().omit)[HX_ENDIAN_MARK_ID_BYTE] = hx::gPrevByteMarkID; + hx::IncrementalData().omit = nullptr; + } + #endif + if (hx::gIncrementalActive.load(std::memory_order_relaxed)) VerifyIncremental(); + #endif + // An object that only a weak hash value keeps is not a zombie while its key lives, // and a zombie can keep more keys alive MarkWeakHashValues(); @@ -5188,10 +5374,60 @@ class GlobalAllocator GCLOG(" ******** is marked : %d\n", (reinterpret_cast(watch)[HX_ENDIAN_MARK_ID_BYTE] == gByteMarkID)); } #endif + return true; } + #ifdef HXCPP_GC_INCREMENTAL_VERIFY + void VerifyIncremental() + { + hx::MarkContext oracle; + oracle.incrementalOracle = true; + hx::MarkClassStatics(&oracle); + { + std::lock_guard lock(*hx::sGCRootLock); + for (hx::Object **slot : *hx::sgRootSet) + if (*slot) + hx::MarkObjectAlloc(*slot, &oracle); + if (hx::sgOffsetRootSet) + for (const auto &entry : *hx::sgOffsetRootSet) + { + char *ptr = *static_cast(entry.first); + if (ptr) + hx::MarkObjectAlloc(reinterpret_cast(ptr - entry.second), &oracle); + } + } + for (int i = 0; i < hx::sZombieList.size(); i++) + hx::MarkObjectAlloc(hx::sZombieList[i], &oracle); + for (int i = 0; i < mLocalAllocs.size(); i++) + MarkLocalAlloc(mLocalAllocs[i], &oracle); + while (!oracle.oracleQueue.empty()) + { + hx::Object *object = oracle.oracleQueue.back(); + oracle.oracleQueue.pop_back(); + object->__Mark(&oracle); + } + for (const auto &entry : oracle.oracleVisited) + { + uintptr_t header = reinterpret_cast(entry.first) - sizeof(unsigned int); + unsigned int flags = *reinterpret_cast(header); + bool valid = (flags & IMMIX_ALLOC_MARK_ID) == hx::gMarkID; + unsigned int rows = flags & IMMIX_ALLOC_ROW_COUNT; + unsigned char *row = reinterpret_cast(header & IMMIX_BLOCK_BASE_MASK) + + ((header & IMMIX_BLOCK_OFFSET_MASK) >> IMMIX_LINE_BITS); + for (unsigned int r = 0; r < rows; r++) + valid = valid && row[r]; + if (!valid) + { + fprintf(stderr, "Incremental GC omitted reachable %s %p flags=%08x epoch=%08x rows=%u first=%u vtable=%p\n", entry.second ? "object" : "allocation", entry.first, flags, hx::gMarkID, rows, rows ? *row : 0, entry.second ? *static_cast(entry.first) : nullptr); + fflush(stderr); + abort(); + } + } + } +#endif + // The stack values a census put aside: what a reference reached is marked, and // counted, by now, and the rest is marked as the stack scan would have void MarkCensusDeferred() @@ -5271,6 +5507,9 @@ class GlobalAllocator void Collect(bool inMajor, bool inForceCompact, bool inLocked,bool inFreeIsFragged) { + #ifdef HXCPP_GC_INCREMENTAL + double incrementalStarted = hx::IncrementalNow(); + #endif PROFILE_COLLECT_SUMMARY_START; #ifndef HXCPP_SINGLE_THREADED_APP @@ -5317,6 +5556,38 @@ class GlobalAllocator sgIsCollecting = true; + #ifdef HXCPP_GC_INCREMENTAL + struct ExitCollection + { + GlobalAllocator *owner; + LocalAllocator *current; + bool locked; + double start; + bool completed = false; + ExitCollection(GlobalAllocator *inOwner, LocalAllocator *inCurrent, bool inLocked, double inStart) + : owner(inOwner), current(inCurrent), locked(inLocked), start(inStart) {} + ~ExitCollection() + { + if (!completed) + { + if (hx::gIncrementalActive.load()) + { + std::lock_guard lock(hx::IncrementalData().statsLock); + hx::IncrementalData().fallbacks++; + } + owner->CancelIncremental(); + for (int i = 0; i < owner->mLocalAllocs.size(); i++) + { + ((hx::StackContext *)owner->mLocalAllocs[i])->mIncrementalEntries.clear(); + ((hx::StackContext *)owner->mLocalAllocs[i])->mIncrementalRead = 0; + } + } + owner->ResumeIncremental(current, locked); + hx::IncrementalData().pause(start, hx::sIncrementalRequestMs > 0); + } + } incrementalExit{this, this_local, inLocked, incrementalStarted}; +#endif + StopThreadJobs(true); #ifdef HXCPP_DEBUG sgAllocsSinceLastSpam = 0; @@ -5353,7 +5624,7 @@ class GlobalAllocator // Now all threads have mTopOfStack & mBottomOfStack set. bool generational = false; - #ifdef HXCPP_GC_GENERATIONAL + #ifdef HXCPP_GC_GENERATIONAL_RUNTIME if (sGcMode==gcmGenerational) { for(int i=0;i 0 && !sgCensus; + if (active && !requested) + { + std::lock_guard lock(state.statsLock); + state.forcedFinishes++; + } + if (active && sgCensus) + state.failed.store(true, std::memory_order_relaxed); + if (active && hx::sIncrementalRequestMs == 0) + { + for (int i = 0; i < mLocalAllocs.size(); i++) + if (((hx::StackContext *)mLocalAllocs[i])->mIncrementalMutations) + state.failed.store(true, std::memory_order_relaxed); + } + if (active && state.failed.exchange(false, std::memory_order_relaxed)) + { + CancelIncremental(); + { + std::lock_guard lock(state.statsLock); + state.fallbacks++; + } + active = requested = false; + } + if (active || requested) + { + if (!active) + { + state.clearWork(); + state.slices = 0; + // Materialize the previous completed collection's free ranges + // before replacing its row marks with a partial new mark. + for (int i = 0; i < mAllBlocks.size(); i++) + if (!mAllBlocks[i]->mReclaimed) + mAllBlocks[i]->reclaim(nullptr); + state.failed.store(false, std::memory_order_relaxed); + hx::gIncrementalActive.store(true, std::memory_order_relaxed); + } + state.tracing = true; + state.work = 0; + hx::gIncrementalAllocationsWhite = true; + state.deadline = requested ? incrementalStarted + hx::sIncrementalRequestMs / 1000.0 : 0; + state.dirty.clear(); + state.slices++; + mMarker.init(); + state.logsPending = !hx::IncrementalEntries(state.detached, state.detachedRead, &mMarker); + for (int i = 0; i < mLocalAllocs.size(); i++) + { + hx::StackContext *ctx = (hx::StackContext *)mLocalAllocs[i]; + if (!hx::IncrementalEntries(ctx->mIncrementalEntries, ctx->mIncrementalRead, &mMarker)) + state.logsPending = true; + } + if (!MarkAll(false, active)) + { + incrementalExit.completed = true; + PROFILE_COLLECT_SUMMARY_END; + return; + } + hx::gIncrementalActive.store(false, std::memory_order_relaxed); + for (int i = 0; i < mLocalAllocs.size(); i++) + { + ClearPooledAlloc(mLocalAllocs[i]); + ((hx::StackContext *)mLocalAllocs[i])->mIncrementalEntries.clear(); + ((hx::StackContext *)mLocalAllocs[i])->mIncrementalRead = 0; + } + { + std::lock_guard lock(state.statsLock); + state.cycles++; + state.lastSlices = state.slices; + } + state.clearWork(); + } + else + { + for (int i = 0; i < mLocalAllocs.size(); i++) + { + ((hx::StackContext *)mLocalAllocs[i])->mIncrementalEntries.clear(); + ((hx::StackContext *)mLocalAllocs[i])->mIncrementalRead = 0; + } + MarkAll(false); + } + } +#else MarkAll(generational); + #endif #ifdef HX_GC_VERIFY_GENERATIONAL { @@ -5773,7 +6131,15 @@ class GlobalAllocator mTotalAfterLastCollect = MemUsage(); - #ifdef HXCPP_GC_GENERATIONAL + #ifdef HXCPP_GC_INCREMENTAL + // A stable live heap must not trigger another major mark immediately. + // Spend half of the normal allocation reserve before starting, then + // shorten the scheduling interval once three quarters have been spent. + mIncrementalStartAt.store(std::max(size_t{256}*1024,targetFree/2),std::memory_order_relaxed); + mIncrementalAllocated.store(0,std::memory_order_relaxed); + #endif + + #ifdef HXCPP_GC_GENERATIONAL_RUNTIME if (generational) { // TODO - include large too? @@ -5855,7 +6221,7 @@ class GlobalAllocator sSpinCount = 0; #endif - #ifdef HXCPP_GC_GENERATIONAL + #ifdef HXCPP_GC_GENERATIONAL_RUNTIME if (sGcMode==gcmGenerational) for(int i=0;ibyteMarkId = hx::gByteMarkID; #endif #endif + #endif // HXCPP_GC_INCREMENTAL + PROFILE_COLLECT_SUMMARY_END; + } + #ifdef HXCPP_GC_INCREMENTAL + void CancelIncremental() + { + if (hx::gIncrementalActive.load(std::memory_order_relaxed)) + { + // A partial mark contains both epochs. Advancing again would make + // its unvisited objects look black to the alternating-bit fast + // path. Undo this epoch before starting an independent full mark. + for (int b = 0; b < mAllBlocks.size(); b++) + { + BlockDataInfo *block = mAllBlocks[b]; + for (int r = IMMIX_HEADER_LINES; r < IMMIX_LINES; r++) + { + unsigned int starts = block->allocStart[r]; + unsigned int *headers = reinterpret_cast(block->mPtr->row(r)); + for (int bit = 0; starts; bit++, starts >>= 1) + if ((starts & 1) && (headers[bit] & IMMIX_ALLOC_MARK_ID) == hx::gMarkID) + reinterpret_cast(headers + bit + 1)[HX_ENDIAN_MARK_ID_BYTE] = hx::gPrevByteMarkID; + } + } + for (int i = 0; i < mLargeList.size(); i++) + if ((mLargeList[i][1] & IMMIX_ALLOC_MARK_ID) == hx::gMarkID) + reinterpret_cast(mLargeList[i] + 2)[HX_ENDIAN_MARK_ID_BYTE] = hx::gPrevByteMarkID; + hx::gByteMarkID = hx::gPrevByteMarkID; + hx::gMarkID = hx::gByteMarkID << 24; + hx::gMarkIDWithContainer = hx::gMarkID | IMMIX_ALLOC_IS_CONTAINER; + hx::gPrevMarkIdMask = (hx::gMarkID & 0x30000000) | HX_GC_CONST_ALLOC_BIT; + hx::gPrevByteMarkID = ((hx::gByteMarkID - 1) & 0x0f) | ((hx::gByteMarkID & 1) ? 0x10 : 0x20); + hx::gRememberedByteMarkID = hx::gByteMarkID | HX_GC_REMEMBERED; + } + hx::IncrementalDiscard(&mMarker); + } - PROFILE_COLLECT_SUMMARY_END; + void ResumeIncremental(LocalAllocator *inCurrent, bool inLocked) + { + if (hx::gIncrementalActive.load(std::memory_order_relaxed)) + for (int i = 0; i < mLocalAllocs.size(); i++) + RestoreIncrementalAlloc(mLocalAllocs[i]); +#ifdef HXCPP_TELEMETRY + __hxt_gc_end(); +#endif + sgIsCollecting = false; + hx::gPauseForCollect = 0; +#ifndef HXCPP_SINGLE_THREADED_APP + for (int i = 0; i < mLocalAllocs.size(); i++) + if (mLocalAllocs[i] != inCurrent) + ReleaseFromSafe(mLocalAllocs[i]); + if (!inLocked) + gThreadStateChangeLock->unlock(); +#endif } +#endif void reclaimBlocks(bool full, BlockDataStats &outStats) { @@ -5938,7 +6359,7 @@ class GlobalAllocator // with few that do the work is less than waking the marking threads for // it: that took most of an otherwise idle minor collection (#329) bool threaded = MAX_GC_THREADS>1; - #ifdef HXCPP_GC_GENERATIONAL + #ifdef HXCPP_GC_GENERATIONAL_RUNTIME if (threaded) { int changed = 0; @@ -6173,6 +6594,11 @@ class GlobalAllocator hx::MarkContext mMarker; + #ifdef HXCPP_GC_INCREMENTAL + std::atomic_size_t mIncrementalStartAt{hx::sgMinimumWorkingMemory/2}; + std::atomic_size_t mIncrementalAllocated{0}; + #endif + volatile int mNextFreeBlockOfSize[BLOCK_OFSIZE_COUNT]; std::atomic_size_t mThreadJobId; @@ -6238,7 +6664,7 @@ void MarkConservative(int *inBottom, int *inTop,hx::MarkContext *__inCtx) bool isWatch{}; #endif - #ifdef HXCPP_GC_GENERATIONAL + #ifdef HXCPP_GC_GENERATIONAL_RUNTIME // If this is a generational mark, then the byte marker has not been increased. // Previous mark Ids are therfore from more than 1 collection ago bool allowPrevious{ !__inCtx->isGenerational }; @@ -6283,6 +6709,12 @@ void MarkConservative(int *inBottom, int *inTop,hx::MarkContext *__inCtx) { if (mem==memLarge) { + #ifdef HXCPP_GC_INCREMENTAL + unsigned int flags = reinterpret_cast(potentialObject)[-1]; + if (flags & IMMIX_ALLOC_IS_CONTAINER) + hx::MarkObjectAlloc(reinterpret_cast(potentialObject),__inCtx); + else hx::MarkAlloc(reinterpret_cast(potentialObject),__inCtx); + #else unsigned char& mark{ reinterpret_cast(potentialObject)[HX_ENDIAN_MARK_ID_BYTE] }; if (mark != gByteMarkID) { @@ -6291,6 +6723,7 @@ void MarkConservative(int *inBottom, int *inTop,hx::MarkContext *__inCtx) else mark = static_cast(gByteMarkID); } + #endif } else { @@ -6341,6 +6774,12 @@ void MarkConservative(int *inBottom, int *inTop,hx::MarkContext *__inCtx) } else if (t==allocMarked) { + #ifdef HXCPP_GC_INCREMENTAL_VERIFY + if (__inCtx->incrementalOracle) { + unsigned int flags = reinterpret_cast(potentialObject)[-1]; + hx::IncrementalOracleMark(reinterpret_cast(potentialObject),(flags & IMMIX_ALLOC_IS_CONTAINER)!=0,__inCtx); + } + #endif #ifdef HX_WATCH if (isWatch) GCLOG(" pin alloced %p (%p)\n", reinterpret_cast(potentialObject), ptr); @@ -6437,6 +6876,8 @@ class LocalAllocator : public hx::StackContext { Reset(); + // The fields remain in the barrier-compatible layout for incremental + // builds, even though that mode never creates generational logs. #ifdef HXCPP_GC_GENERATIONAL mOldReferrers = 0; mYoungValues = 0; @@ -6459,7 +6900,7 @@ class LocalAllocator : public hx::StackContext mID = GetCurrentThreadId(); #endif - #ifdef HXCPP_GC_GENERATIONAL + #ifdef HXCPP_GC_GENERATIONAL_RUNTIME if (mOldReferrers) { GCLOG("Uncleaned referrers\n"); @@ -6491,11 +6932,15 @@ class LocalAllocator : public hx::StackContext std::lock_guard lock(*gThreadStateChangeLock); + #ifdef HXCPP_GC_INCREMENTAL + hx::IncrementalTransfer(this); + #endif + #ifdef HX_WINDOWS mID = 0; #endif - #ifdef HXCPP_GC_GENERATIONAL + #ifdef HXCPP_GC_GENERATIONAL_RUNTIME if (mOldReferrers) { if ( mOldReferrers->count ) @@ -6538,6 +6983,21 @@ class LocalAllocator : public hx::StackContext mMoreHoles = false; } + #ifdef HXCPP_GC_INCREMENTAL + void RestoreIncrementalRange() + { + // WaitForSafe invalidates these bounds to park a running allocator. + // A partial mark did not reclaim anything, so its owned hole is still + // usable. Dropping it on every slice wastes reserve and forces more GC. + if (allocBase && mCurrentHole && mCurrentHole <= mCurrentHoles) + { + const HoleRange &range = mCurrentRange[mCurrentHole - 1]; + spaceEnd = range.start + range.length; + mMoreHoles = mCurrentHole < mCurrentHoles; + } + } + #endif + // The main of haxe calls SetTopOfStack(top,false) // via hxcpp_set_top_of_stack or HX_TOP_OF_STACK in HxcppMain.cpp/Macros.h // The means "register current thread indefinitely" @@ -6978,6 +7438,11 @@ class LocalAllocator : public hx::StackContext __hxt_gc_alloc(buffer, inSize); #endif + #ifdef HXCPP_GC_INCREMENTAL + if (hx::gIncrementalActive.load(std::memory_order_relaxed)) + hx::IncrementalAllocated(buffer,(inObjectFlags & IMMIX_ALLOC_IS_CONTAINER)!=0,this); + #endif + return buffer; } // Read once: the collecting thread zeroes spaceEnd in WaitForSafe @@ -7064,6 +7529,9 @@ class LocalAllocator : public hx::StackContext #ifndef HXCPP_SINGLE_THREADED_APP if (!mTopOfStack) { + #ifdef HXCPP_GC_INCREMENTAL + if (hx::gIncrementalActive.load(std::memory_order_relaxed)) return; + #endif Reset(); return; } @@ -7114,6 +7582,9 @@ class LocalAllocator : public hx::StackContext MarkMember( *(hx::Object **)&stringSet, __inCtx); #endif + #ifdef HXCPP_GC_INCREMENTAL + if (!hx::gIncrementalActive.load(std::memory_order_relaxed)) + #endif Reset(); } @@ -7136,6 +7607,8 @@ inline LocalAllocator *GetLocalAlloc(bool inAllowEmpty=false) #endif } +#include "IncrementalApi.h" + #ifndef HXCPP_SINGLE_THREADED_APP void WaitForSafe(LocalAllocator *inAlloc) { @@ -7158,6 +7631,13 @@ void ClearPooledAlloc(LocalAllocator *inAlloc) inAlloc->Reset(); } +#ifdef HXCPP_GC_INCREMENTAL +void RestoreIncrementalAlloc(LocalAllocator *inAlloc) +{ + inAlloc->RestoreIncrementalRange(); +} +#endif + void GlobalAllocator::ForEachStackContext(void (*inFunc)(hx::StackContext *, void *), void *inData) { for(int i=0;iSetupStackAndCollect(inMajor, inCompact); + do { + GetLocalAlloc()->SetupStackAndCollect(inMajor, inCompact); + #ifdef HXCPP_GC_INCREMENTAL + if (gIncrementalActive.load()) continue; + #endif + break; + } while (true); return sGlobalAlloc->MemUsage(); } @@ -7510,7 +7996,7 @@ void *InternalRealloc(size_t inFromSize, void *inData, size_t inSize, bool inExp return new_data; } -#ifdef HXCPP_GC_GENERATIONAL +#ifdef HXCPP_GC_GENERATIONAL_RUNTIME void NewMarkedObject(hx::Object *inPtr) { HX_OBJ_WB_PESSIMISTIC_GET(inPtr); diff --git a/src/hx/gc/Incremental.h b/src/hx/gc/Incremental.h new file mode 100644 index 00000000..5eb889d0 --- /dev/null +++ b/src/hx/gc/Incremental.h @@ -0,0 +1,351 @@ +#ifndef HX_INCREMENTAL_IMPLEMENTATION_H +#define HX_INCREMENTAL_IMPLEMENTATION_H + +// Included after MarkContext and allocation marking, with all mutators stopped +// whenever queues, ranges or cycle state are inspected. +namespace hx +{ + +std::atomic gIncrementalActive{false}; +bool gIncrementalAllocationsWhite = false; +static thread_local double sIncrementalRequestMs = 0; + +struct IncrementalRangeJob +{ + Object *owner; + IncrementalRangeScan scan; + size_t cursor = 0; + size_t item = 0; + bool queued = true; + IncrementalRangeJob(Object *inOwner, IncrementalRangeScan inScan) : owner(inOwner), scan(inScan) {} +}; + +struct IncrementalState +{ + std::atomic failed{false}; + std::atomic budgetMs{0.5}; + std::atomic intervalMs{1.0}; + std::atomic nextAutomatic{0}; + std::atomic automaticRunning{false}; + std::vector detached; + size_t detachedRead = 0; + bool logsPending = false; + bool finishPending = false; + std::vector ranges; + std::unordered_map rangeOwners; + std::vector rangeQueue; + size_t rangeHead = 0; + std::unordered_set dirty; + bool tracing = false; + double deadline = 0; + std::atomic work{0}; + size_t slices = 0; + std::mutex statsLock; + uint64_t cycles = 0; + uint64_t pauses = 0; + uint64_t fallbacks = 0; + uint64_t forcedFinishes = 0; + uint64_t requestedSlices = 0; + double maxSlicePauseMs = 0; + double maxSynchronousPauseMs = 0; + double totalPauseMs = 0; + double maxPauseMs = 0; + double lastPauseMs = 0; + size_t lastSlices = 0; + Object *watch = nullptr; + Object *omit = nullptr; + std::atomic watched{false}; + std::mutex rangeLock; + size_t testWorkLimit = 0; + + void clearWork() + { + ranges.clear(); + rangeOwners.clear(); + rangeQueue.clear(); + rangeHead = 0; + detached.clear(); + detachedRead = 0; + logsPending = false; + finishPending = false; + dirty.clear(); + tracing = false; + deadline = 0; + work = 0; + } + + void pause(double inStart, bool inBudgeted) + { + double elapsed = (IncrementalNow() - inStart) * 1000.0; + std::lock_guard lock(statsLock); + pauses++; + totalPauseMs += elapsed; + lastPauseMs = elapsed; + maxPauseMs = std::max(maxPauseMs, elapsed); + if (inBudgeted) + { + requestedSlices++; + maxSlicePauseMs = std::max(maxSlicePauseMs, elapsed); + } + else + maxSynchronousPauseMs = std::max(maxSynchronousPauseMs, elapsed); + } +}; + +static IncrementalState &IncrementalData() +{ + // Like the allocator, this survives static destruction and late threads. + static IncrementalState *state = new IncrementalState(); + return *state; +} + +static bool IncrementalExpired() +{ + IncrementalState &state = IncrementalData(); + if (!state.tracing || state.deadline <= 0) + return false; +#ifdef HXCPP_GC_INCREMENTAL_TEST + if (state.work > 0 && ((state.testWorkLimit && state.work >= state.testWorkLimit) || state.watched)) + return true; +#endif + // Reading the clock for every pointer costs more than marking small + // objects. Check in bounded batches without a shared counter cache line. + static thread_local unsigned int checks = 0; + return (++checks & 63) == 0 && IncrementalNow() >= state.deadline; +} + +static void IncrementalScanned(Object *inObject) +{ +#ifdef HXCPP_GC_INCREMENTAL_TEST + IncrementalState &state = IncrementalData(); + if (!state.tracing) + return; + state.work++; + if (inObject == state.watch) + state.watched = true; +#endif +} + +void IncrementalRemember(Object *inOwner, const void *inValue, StackContext *inCtx) +{ + if (!gIncrementalActive.load(std::memory_order_relaxed) || !inCtx) + return; + // A white holder will be scanned if it becomes reachable. Only a holder + // already discovered by this cycle can hide an inserted edge from it. + if (inOwner && reinterpret_cast(inOwner)[HX_ENDIAN_MARK_ID_BYTE] != gByteMarkID) + return; + if (inCtx->mIncrementalEntries.size() >= (1u << 20)) + { + IncrementalData().failed.store(true, std::memory_order_relaxed); + return; + } + try + { + inCtx->mIncrementalEntries.push_back({inOwner, inValue}); + } + catch (const std::bad_alloc &) + { + IncrementalData().failed.store(true, std::memory_order_relaxed); + } +} + +void IncrementalAllocated(void *inValue, bool inObject, ImmixAllocator *inAlloc) +{ + if (!gIncrementalActive.load(std::memory_order_relaxed)) + return; + if (IncrementalData().tracing) + { + // New objects are white too. A root or a store into a discovered + // holder shades them, so transient objects do not flood the mark log. + // Stores after a partially constructed object was discovered use the + // same barrier as stores into any other discovered holder. + static_cast(inValue)[HX_ENDIAN_MARK_ID_BYTE] = static_cast(gPrevByteMarkID); + return; + } + // Fresh headers already have the current epoch. Their rows must also be + // live even if no subsequent pointer scan visits a primitive allocation. + uintptr_t header = reinterpret_cast(inValue) - sizeof(unsigned int); + unsigned int flags = *reinterpret_cast(header); + unsigned int rows = flags & IMMIX_ALLOC_ROW_COUNT; + unsigned char *row = reinterpret_cast(header & IMMIX_BLOCK_BASE_MASK) + + ((header & IMMIX_BLOCK_OFFSET_MASK) >> IMMIX_LINE_BITS); + for (unsigned int i = 0; i < rows; i++) + row[i] = 1; +} + +void IncrementalMutationEnter(Object *inOwner, StackContext *inCtx) +{ + if (inCtx) + inCtx->mIncrementalMutations++; + IncrementalRemember(inOwner, nullptr, inCtx); +} + +void IncrementalMutationLeave(Object *inOwner, StackContext *inCtx) +{ + IncrementalRemember(inOwner, nullptr, inCtx); + if (inCtx) + inCtx->mIncrementalMutations--; +} + +static void IncrementalTransfer(StackContext *inCtx) +{ + IncrementalState &state = IncrementalData(); + try + { + state.detached.insert(state.detached.end(), inCtx->mIncrementalEntries.begin() + inCtx->mIncrementalRead, inCtx->mIncrementalEntries.end()); + } + catch (const std::bad_alloc &) + { + state.failed.store(true, std::memory_order_relaxed); + } + inCtx->mIncrementalEntries.clear(); + inCtx->mIncrementalRead = 0; +} + +bool IncrementalRange(Object *inOwner, IncrementalRangeScan inScan, MarkContext *inCtx) +{ + IncrementalState &state = IncrementalData(); + if (!state.tracing) + return false; + std::lock_guard lock(state.rangeLock); + if (state.failed.load(std::memory_order_relaxed)) + return true; + try + { + if (state.ranges.size() >= (1u << 20)) + { + state.failed.store(true, std::memory_order_relaxed); + return true; + } + auto found = state.rangeOwners.find(inOwner); + if (found == state.rangeOwners.end()) + { + size_t index = state.ranges.size(); + state.ranges.emplace_back(inOwner, inScan); + state.rangeOwners.emplace(inOwner, index); + state.rangeQueue.push_back(index); + } + else + { + IncrementalRangeJob &job = state.ranges[found->second]; + if (!job.queued) + { + job.cursor = job.item = 0; + job.queued = true; + state.rangeQueue.push_back(found->second); + } + } + } + catch (const std::bad_alloc &) + { + // Keep the heap intact and restart synchronously at the next stop. + // An exception must not escape a marker worker thread. + state.failed.store(true, std::memory_order_relaxed); + } + return true; +} + +static bool IncrementalEntries(std::vector &inEntries, size_t &ioRead, MarkContext *inCtx) +{ + IncrementalState &state = IncrementalData(); + while (ioRead < inEntries.size()) + { + if (IncrementalExpired()) + return false; + const IncrementalEntry &entry = inEntries[ioRead++]; + state.work++; + if (entry.value) + { + unsigned int flags; + memcpy(&flags, static_cast(entry.value) - sizeof(flags), sizeof(flags)); + if (flags & IMMIX_ALLOC_IS_CONTAINER) + MarkObjectAlloc(static_cast(const_cast(entry.value)), inCtx); + else + MarkAlloc(const_cast(entry.value), inCtx); + // Scalar insertion barriers shade the new target. They do not + // invalidate an entire large container on every element write. + continue; + } + if (!entry.owner) + continue; + try + { + if (!state.dirty.insert(entry.owner).second) + continue; + } + catch (const std::bad_alloc &) + { + state.failed.store(true, std::memory_order_relaxed); + return false; + } + MarkObjectAlloc(entry.owner, inCtx); + // A black holder can acquire an edge after it was scanned. Revisit it + // even if its ordinary header already belongs to this cycle. + inCtx->pushObj(entry.owner); + auto range = state.rangeOwners.find(entry.owner); + if (range != state.rangeOwners.end()) + { + IncrementalRangeJob &job = state.ranges[range->second]; + job.cursor = job.item = 0; + } + } + inEntries.clear(); + ioRead = 0; + return true; +} + +static bool IncrementalDrain(MarkContext *inCtx, void (*inProcess)(MarkContext *, void *), void *inData) +{ + IncrementalState &state = IncrementalData(); + try + { + while (true) + { + if (state.failed.load(std::memory_order_relaxed)) + return false; + inProcess(inCtx, inData); + if (state.failed.load(std::memory_order_relaxed)) + return false; + if (inCtx->hasWork()) + return false; + if (state.rangeHead >= state.rangeQueue.size()) + return !state.logsPending; + if (IncrementalExpired()) + return false; + // Feed several ranges to one worker round. Waking the marker pool for + // each 256-element chunk costs more than scanning those elements. + for (int batch = 0; batch < 64 && state.rangeHead < state.rangeQueue.size(); batch++) + { + if (IncrementalExpired()) + break; + size_t index = state.rangeQueue[state.rangeHead++]; + // Workers are stopped here. Scans enqueue objects; only their + // later __Mark calls can append ranges and move this descriptor. + IncrementalRangeJob &job = state.ranges[index]; + inCtx->init(); + bool complete = job.scan(job.owner, job.cursor, job.item, inCtx); + state.work++; + if (complete) + job.queued = false; + else + state.rangeQueue.push_back(index); + } + } + } + catch (const std::bad_alloc &) + { + state.failed.store(true, std::memory_order_relaxed); + return false; + } +} + +static void IncrementalDiscard(MarkContext *inCtx) +{ + inCtx->discardJobs(); + IncrementalData().clearWork(); + gIncrementalAllocationsWhite = false; + gIncrementalActive.store(false, std::memory_order_relaxed); +} + +} // namespace hx +#endif diff --git a/src/hx/gc/IncrementalApi.h b/src/hx/gc/IncrementalApi.h new file mode 100644 index 00000000..9c4b169b --- /dev/null +++ b/src/hx/gc/IncrementalApi.h @@ -0,0 +1,166 @@ +#ifndef HX_INCREMENTAL_API_H +#define HX_INCREMENTAL_API_H + +#ifdef HXCPP_GC_INCREMENTAL +static void IncrementalAutomatic(hx::ImmixAllocator *inAlloc, bool inPressure) +{ + if (!sgInternalEnable) + return; + hx::StackContext *ctx = static_cast(inAlloc); + if (ctx->mIncrementalMutations) + return; + hx::IncrementalState &state = hx::IncrementalData(); + bool expected = false; + if (!state.automaticRunning.compare_exchange_strong(expected, true, std::memory_order_relaxed)) + return; + struct Release + { + hx::IncrementalState &state; + ~Release() { state.automaticRunning.store(false, std::memory_order_relaxed); } + } release{state}; + double now = hx::IncrementalNow(); + double due = state.nextAutomatic.load(std::memory_order_relaxed); + if (!inPressure && now < due) + return; + state.nextAutomatic.store(now + state.intervalMs.load() / 1000.0, std::memory_order_relaxed); + double previous = hx::sIncrementalRequestMs; + hx::sIncrementalRequestMs = state.budgetMs.load(); + try + { + inAlloc->SetupStackAndCollect(false, false); + } + catch (...) + { + hx::sIncrementalRequestMs = previous; + throw; + } + hx::sIncrementalRequestMs = previous; +} +#endif + +bool __hxcpp_gc_incremental_step(double inBudgetMs) +{ +#ifdef HXCPP_GC_INCREMENTAL + if (!(inBudgetMs >= 0 && inBudgetMs <= 1000)) + hx::Throw(HX_CSTRING("Invalid incremental GC budget")); + if (inBudgetMs == 0) + return hx::gIncrementalActive.load(); + double previous = hx::sIncrementalRequestMs; + hx::sIncrementalRequestMs = inBudgetMs; + try + { + GetLocalAlloc()->SetupStackAndCollect(false, false); + } + catch (...) + { + hx::sIncrementalRequestMs = previous; + throw; + } + hx::sIncrementalRequestMs = previous; + return hx::gIncrementalActive.load(); +#else + hx::Throw(HX_CSTRING("Build with -lib hxcpp -D HXCPP_GC_INCREMENTAL to use incremental GC")); + return false; +#endif +} + +void __hxcpp_gc_incremental_configure(double inBudgetMs, double inIntervalMs) +{ +#ifdef HXCPP_GC_INCREMENTAL + if (!(inBudgetMs > 0 && inBudgetMs <= 1000 && inIntervalMs >= 0 && inIntervalMs <= 1000)) + hx::Throw(HX_CSTRING("Invalid incremental GC pacing")); + hx::IncrementalData().budgetMs.store(inBudgetMs); + hx::IncrementalData().intervalMs.store(inIntervalMs); +#else + hx::Throw(HX_CSTRING("Incremental GC is not enabled")); +#endif +} + +bool __hxcpp_gc_incremental_active() +{ +#ifdef HXCPP_GC_INCREMENTAL + return hx::gIncrementalActive.load(); +#else + return false; +#endif +} + +Dynamic __hxcpp_gc_incremental_stats() +{ +#ifdef HXCPP_GC_INCREMENTAL + hx::IncrementalState &state = hx::IncrementalData(); + double cycles, pauses, fallbacks, total, longest, last, slices, forced, requested, sliceMax, syncMax; + { + std::lock_guard lock(state.statsLock); + cycles = static_cast(state.cycles); + pauses = static_cast(state.pauses); + fallbacks = static_cast(state.fallbacks); + total = state.totalPauseMs; + longest = state.maxPauseMs; + last = state.lastPauseMs; + slices = static_cast(state.lastSlices); + forced = static_cast(state.forcedFinishes); + requested = static_cast(state.requestedSlices); + sliceMax = state.maxSlicePauseMs; + syncMax = state.maxSynchronousPauseMs; + } + return hx::Anon_obj::Create(12) + ->setFixed(0, HX_CSTRING("cycles"), cycles) + ->setFixed(1, HX_CSTRING("pauses"), pauses) + ->setFixed(2, HX_CSTRING("fallbacks"), fallbacks) + ->setFixed(3, HX_CSTRING("totalPauseMs"), total) + ->setFixed(4, HX_CSTRING("maxPauseMs"), longest) + ->setFixed(5, HX_CSTRING("lastPauseMs"), last) + ->setFixed(6, HX_CSTRING("lastCycleSlices"), slices) + ->setFixed(7, HX_CSTRING("active"), hx::gIncrementalActive.load()) + ->setFixed(8, HX_CSTRING("forcedFinishes"), forced) + ->setFixed(9, HX_CSTRING("requestedSlices"), requested) + ->setFixed(10, HX_CSTRING("maxSlicePauseMs"), sliceMax) + ->setFixed(11, HX_CSTRING("maxSynchronousPauseMs"), syncMax); +#else + return hx::Anon_obj::Create(1)->setFixed(0, HX_CSTRING("active"), false); +#endif +} + +#ifdef HXCPP_GC_INCREMENTAL_TEST +int __hxcpp_gc_incremental_test(int inOperation, hx::Object *inObject) +{ + hx::IncrementalState &state = hx::IncrementalData(); + switch (inOperation) + { + case 0: + state.watch = inObject; + state.watched = false; + state.testWorkLimit = 32; + return 0; + case 1: + return state.watched; + case 2: + return hx::gIncrementalActive.load(); + case 3: + state.failed.store(true); + return 0; + case 4: + state.testWorkLimit = 0; + state.watch = nullptr; + state.watched = false; + return 0; + case 5: + { + auto found = state.rangeOwners.find(inObject); + return found != state.rangeOwners.end() && state.ranges[found->second].queued && + (state.ranges[found->second].cursor || state.ranges[found->second].item); + } + case 6: + state.omit = inObject; + return 0; + case 7: + return state.finishPending; + case 8: + return GetLocalAlloc()->allocBase && GetLocalAlloc()->spaceEnd > GetLocalAlloc()->spaceStart; + default: + return -1; + } +} +#endif +#endif diff --git a/test/gc-defines/check.sh b/test/gc-defines/check.sh index fcfc25c8..2750ad91 100755 --- a/test/gc-defines/check.sh +++ b/test/gc-defines/check.sh @@ -25,6 +25,9 @@ CONFIGS=( HXCPP_GC_MOVING,HXCPP_GC_DEBUG_ALWAYS_MOVE HXCPP_GC_GENERATIONAL HXCPP_GC_GENERATIONAL,HX_GC_VERIFY_GENERATIONAL + HXCPP_GC_INCREMENTAL,HXCPP_GC_INCREMENTAL_BARRIERS,HXCPP_GC_GENERATIONAL + HXCPP_GC_INCREMENTAL,HXCPP_GC_INCREMENTAL_BARRIERS,HXCPP_GC_GENERATIONAL,HXCPP_GC_INCREMENTAL_VERIFY,HXCPP_GC_INCREMENTAL_TEST + HXCPP_GC_INCREMENTAL,HXCPP_GC_INCREMENTAL_BARRIERS,HXCPP_GC_GENERATIONAL,HXCPP_SINGLE_THREADED_APP HXCPP_GC_VERIFY HXCPP_GC_VERIFY,HX_GC_VERIFY_ALLOC_START HXCPP_GC_VERIFY,HXCPP_GC_MOVING diff --git a/test/gc_stress/Stress.hx b/test/gc_stress/Stress.hx index f0e405bc..39d8e0d2 100644 --- a/test/gc_stress/Stress.hx +++ b/test/gc_stress/Stress.hx @@ -475,5 +475,8 @@ class Stress { Sys.exit(1); } Sys.println("ok"); + #if HXCPP_GC_INCREMENTAL + Sys.println("incremental " + haxe.Json.stringify(hxcpp.gc.Incremental.getStats())); + #end } } diff --git a/test/haxelib/check.sh b/test/haxelib/check.sh index 01ca1796..1acb9e1b 100755 --- a/test/haxelib/check.sh +++ b/test/haxelib/check.sh @@ -65,6 +65,10 @@ package() { sed "s/\"version\": \"[^\"]*\"/\"version\": \"$version\"/" "$ROOT/haxelib.json" > "$dir/haxelib.json" cp "$ROOT/run.n" "$dir/" sed "s/,\"[^\"]*\")/,\"$version\")/" "$ROOT/extraParams.hxml" > "$dir/extraParams.hxml" + # extraParams also loads the collector configuration macro, even when + # incremental mode is off. Keep that dependency in the test package. + mkdir -p "$dir/hxcpp" + cp "$ROOT/hxcpp/IncrementalGcConfig.hx" "$dir/hxcpp/" else printf '{ "name": "%s", "version": "%s", "license": "BSD", "url": "https://example.org", "description": "stand-in", "releasenote": "", "contributors": ["nobody"], "dependencies": {} }\n' "$name" "$version" > "$dir/haxelib.json" fi @@ -72,8 +76,10 @@ package() { import os, sys, zipfile src, out = sys.argv[1], sys.argv[2] with zipfile.ZipFile(out, "w") as z: - for f in os.listdir(src): - z.write(os.path.join(src, f), f) + for directory, _, files in os.walk(src): + for name in files: + path = os.path.join(directory, name) + z.write(path, os.path.relpath(path, src)) PY echo "$dir.zip" } diff --git a/test/regression/GcContainerMutation/Main.hx b/test/regression/GcContainerMutation/Main.hx new file mode 100644 index 00000000..10e7a702 --- /dev/null +++ b/test/regression/GcContainerMutation/Main.hx @@ -0,0 +1,103 @@ +import cpp.vm.Gc; +import haxe.ds.IntMap; +import haxe.ds.ObjectMap; +import haxe.ds.StringMap; + +class Node { + public var value:Int; + public var label:String; + + public function new(value:Int) { + this.value = value; + label = "payload-" + value; + } + + public function isValue(expected:Int):Bool { + return value == expected && label == "payload-" + expected; + } +} + +class Main { + static var values:Array; + static var keys:Array; + static var ints = new IntMap(); + static var strings = new StringMap(); + static var objects = new ObjectMap(); + + static function main() { + values = [for (i in 0...20000) new Node(i)]; + var expected = [for (i in 0...20000) i]; + Gc.run(true); + var arraysCorrect = true; + for (round in 0...8) { + values.splice(17, 257); + expected.splice(17, 257); + values.insert(1024, new Node(90000 + round)); + expected.insert(1024, 90000 + round); + + // Save expected source values before the overlapping copy. + var source = [for (i in 0...1024) expected[1000 + i]]; + cpp.NativeArray.blit(values, 500, values, 1000, 1024); + for (i in 0...source.length) + expected[500 + i] = source[i]; + + var oldLength = values.length; + values.resize(oldLength + 500); + for (i in 0...500) { + var id = 100000 + round * 500 + i; + values[oldLength + i] = new Node(id); + expected.push(id); + } + Gc.run((round & 1) == 0); + if (values.length != expected.length) + arraysCorrect = false; + for (i in 0...expected.length) { + if (values[i] == null || !values[i].isValue(expected[i])) + arraysCorrect = false; + } + } + + keys = [for (i in 0...8192) new Node(i)]; + for (i in 0...1024) { + ints.set(i, new Node(i)); + strings.set("key-" + i, new Node(10000 + i)); + objects.set(keys[i], new Node(20000 + i)); + } + Gc.run(true); + for (i in 0...8192) { + ints.set(i, new Node(30000 + i)); + strings.set("key-" + i, new Node(40000 + i)); + objects.set(keys[i], new Node(50000 + i)); + if ((i & 511) == 511) + Gc.run(false); + } + for (i in 0...8192) { + if (i % 7 == 0) { + ints.remove(i); + strings.remove("key-" + i); + objects.remove(keys[i]); + } + } + Gc.run(true); + + var mapsCorrect = true; + var removalsCorrect = true; + for (i in 0...8192) { + if (i % 7 == 0) { + if (ints.exists(i) || strings.exists("key-" + i) || objects.exists(keys[i])) + removalsCorrect = false; + } else { + var intValue = ints.get(i); + var stringValue = strings.get("key-" + i); + var objectValue = objects.get(keys[i]); + if (intValue == null || !intValue.isValue(30000 + i) + || stringValue == null || !stringValue.isValue(40000 + i) + || objectValue == null || !objectValue.isValue(50000 + i)) + mapsCorrect = false; + } + } + Sys.println("resized and overlapping pointer arrays preserve payloads: " + arraysCorrect); + Sys.println("rehash and replacement preserve managed map values: " + mapsCorrect); + Sys.println("removed entries stay absent after collection: " + removalsCorrect); + } +} diff --git a/test/regression/GcContainerMutation/build.hxml b/test/regression/GcContainerMutation/build.hxml new file mode 100644 index 00000000..b484e67e --- /dev/null +++ b/test/regression/GcContainerMutation/build.hxml @@ -0,0 +1,2 @@ +-m Main +--cpp bin diff --git a/test/regression/GcContainerMutation/stdout.txt b/test/regression/GcContainerMutation/stdout.txt new file mode 100644 index 00000000..bd742209 --- /dev/null +++ b/test/regression/GcContainerMutation/stdout.txt @@ -0,0 +1,3 @@ +resized and overlapping pointer arrays preserve payloads: true +rehash and replacement preserve managed map values: true +removed entries stay absent after collection: true diff --git a/test/regression/GcMutationRoots/Main.hx b/test/regression/GcMutationRoots/Main.hx new file mode 100644 index 00000000..53b911f5 --- /dev/null +++ b/test/regression/GcMutationRoots/Main.hx @@ -0,0 +1,72 @@ +import cpp.vm.Gc; + +class Node { + public var value:Int; + public var label:String; + public var next:Node; + + public function new(value:Int, next:Node = null) { + this.value = value; + this.label = "node-" + value; + this.next = next; + } + + public function intact():Bool { + return label == "node-" + value; + } +} + +class Main { + static var root:Node; + static var record:Dynamic; + static var read:Void->Node; + static var write:Node->Void; + + static function installClosures() { + var captured = new Node(500); + read = () -> captured; + write = value -> captured = value; + } + + static function main() { + root = new Node(-1, new Node(73)); + record = {edge: new Node(700)}; + installClosures(); + Gc.run(true); + + var graphCorrect = true; + var recordsCorrect = true; + var closuresCorrect = true; + for (round in 0...32) { + // The only path to the original node moves through new holders. + root.next = new Node(round, new Node(1000 + round, root.next)); + Reflect.setField(record, "edge", new Node(2000 + round, root.next)); + write(new Node(3000 + round, root.next)); + Gc.run((round & 3) == 0); + + var node = root.next; + for (previous in 0...round + 1) { + var expected = round - previous; + if (node == null || !node.intact() || node.value != expected + || node.next == null || !node.next.intact() || node.next.value != 1000 + expected) { + graphCorrect = false; + break; + } + node = node.next.next; + } + if (node == null || !node.intact() || node.value != 73 || node.next != null) + graphCorrect = false; + + var field:Node = Reflect.field(record, "edge"); + if (field == null || !field.intact() || field.value != 2000 + round || field.next != root.next) + recordsCorrect = false; + var captured = read(); + if (captured == null || !captured.intact() || captured.value != 3000 + round || captured.next != root.next) + closuresCorrect = false; + } + + Sys.println("rewired graphs retain older descendants: " + graphCorrect); + Sys.println("reflective anonymous fields retain replacement graphs: " + recordsCorrect); + Sys.println("mutable closure captures retain replacement graphs: " + closuresCorrect); + } +} diff --git a/test/regression/GcMutationRoots/build.hxml b/test/regression/GcMutationRoots/build.hxml new file mode 100644 index 00000000..b484e67e --- /dev/null +++ b/test/regression/GcMutationRoots/build.hxml @@ -0,0 +1,2 @@ +-m Main +--cpp bin diff --git a/test/regression/GcMutationRoots/stdout.txt b/test/regression/GcMutationRoots/stdout.txt new file mode 100644 index 00000000..d611df20 --- /dev/null +++ b/test/regression/GcMutationRoots/stdout.txt @@ -0,0 +1,3 @@ +rewired graphs retain older descendants: true +reflective anonymous fields retain replacement graphs: true +mutable closure captures retain replacement graphs: true diff --git a/test/regression/IncrementalGcContainers/Main.hx b/test/regression/IncrementalGcContainers/Main.hx new file mode 100644 index 00000000..1e5d68ae --- /dev/null +++ b/test/regression/IncrementalGcContainers/Main.hx @@ -0,0 +1,57 @@ +import cpp.vm.Gc; +#if HXCPP_GC_INCREMENTAL +import hxcpp.gc.Incremental; +#end + +class Node { + public var value:Int; + public function new(value:Int) this.value = value; +} + +class Main { + static var values:Array; + static var expected:Array; + + static function main() { + Gc.enable(false); + values = [for (i in 0...32768) new Node(i)]; + expected = [for (i in 0...32768) i]; + Gc.run(true); + #if HXCPP_GC_INCREMENTAL + untyped __cpp__("__hxcpp_gc_incremental_test(0, nullptr)"); + var partial = false; + for (_ in 0...10000) { + Incremental.step(0.1); + if (untyped __cpp__("__hxcpp_gc_incremental_test(5,{0}.mPtr)", values)) { + partial = true; + break; + } + } + if (!partial || !Incremental.isActive()) throw "array was not partially scanned"; + #end + values.splice(0, 2048); + expected.splice(0, 2048); + values.reverse(); expected.reverse(); + for (i in 0...32768) { + values.push(new Node(50000 + i)); + expected.push(50000 + i); + } + #if HXCPP_GC_INCREMENTAL + untyped __cpp__("__hxcpp_gc_incremental_test(4)"); + var unfinished = true; + for (_ in 0...10000) { + unfinished = Incremental.step(0.1); + if (!unfinished) break; + } + if (unfinished) throw "resized array cycle did not finish"; + #else + Gc.run(true); + #end + var valid = values.length == expected.length; + for (i in 0...expected.length) if (values[i] == null || values[i].value != expected[i]) valid = false; + Sys.println("resized and reordered range jobs preserve every payload: " + valid); + Gc.compact(); + Sys.println("compaction preserves completed range results: " + (values[0].value == expected[0] && values[values.length - 1].value == expected[expected.length - 1])); + Gc.enable(true); + } +} diff --git a/test/regression/IncrementalGcContainers/build.hxml b/test/regression/IncrementalGcContainers/build.hxml new file mode 100644 index 00000000..b484e67e --- /dev/null +++ b/test/regression/IncrementalGcContainers/build.hxml @@ -0,0 +1,2 @@ +-m Main +--cpp bin diff --git a/test/regression/IncrementalGcContainers/stdout.txt b/test/regression/IncrementalGcContainers/stdout.txt new file mode 100644 index 00000000..0378e63b --- /dev/null +++ b/test/regression/IncrementalGcContainers/stdout.txt @@ -0,0 +1,2 @@ +resized and reordered range jobs preserve every payload: true +compaction preserves completed range results: true diff --git a/test/regression/IncrementalGcFinishBoundary/Main.hx b/test/regression/IncrementalGcFinishBoundary/Main.hx new file mode 100644 index 00000000..c3a0087b --- /dev/null +++ b/test/regression/IncrementalGcFinishBoundary/Main.hx @@ -0,0 +1,48 @@ +import cpp.vm.Gc; +#if HXCPP_GC_INCREMENTAL +import hxcpp.gc.Incremental; +#end + +class Node { + public var value:Int; + public var next:Node; + public function new(value:Int, next:Node = null) { this.value = value; this.next = next; } +} + +class Main { + static var root:Node; + static var world:Array; + static var record:Dynamic; + + static function main() { + Gc.enable(false); + root = new Node(1, new Node(2)); + world = [for (i in 0...20000) new Node(i)]; + Gc.run(true); + #if HXCPP_GC_INCREMENTAL + var finishing = false; + for (_ in 0...10000) { + if (!Incremental.step(0.1)) throw "no separate finishing boundary"; + if (untyped __cpp__("__hxcpp_gc_incremental_test(7)") == 1) { + finishing = true; + break; + } + } + if (!finishing) throw "marking did not reach the finishing boundary"; + #end + root.next = new Node(3, new Node(4)); + record = {edge: new Node(5, root.next)}; + world.push(new Node(6)); + #if HXCPP_GC_INCREMENTAL + var done = false; + for (_ in 0...10000) if (!Incremental.step(0.1)) { done = true; break; } + if (!done) throw "finishing cycle did not complete"; + #else + Gc.run(true); + #end + var edge:Node = record.edge; + Sys.println("stores after the finishing boundary preserve new descendants: " + (root.next.value == 3 && root.next.next.value == 4 && edge.value == 5 && edge.next == root.next)); + Sys.println("container mutation before finishing remains visible: " + (world.length == 20001 && world[20000].value == 6)); + Gc.enable(true); + } +} diff --git a/test/regression/IncrementalGcFinishBoundary/build.hxml b/test/regression/IncrementalGcFinishBoundary/build.hxml new file mode 100644 index 00000000..b484e67e --- /dev/null +++ b/test/regression/IncrementalGcFinishBoundary/build.hxml @@ -0,0 +1,2 @@ +-m Main +--cpp bin diff --git a/test/regression/IncrementalGcFinishBoundary/stdout.txt b/test/regression/IncrementalGcFinishBoundary/stdout.txt new file mode 100644 index 00000000..b743f2ee --- /dev/null +++ b/test/regression/IncrementalGcFinishBoundary/stdout.txt @@ -0,0 +1,2 @@ +stores after the finishing boundary preserve new descendants: true +container mutation before finishing remains visible: true diff --git a/test/regression/IncrementalGcSlices/Main.hx b/test/regression/IncrementalGcSlices/Main.hx new file mode 100644 index 00000000..6cda3fa9 --- /dev/null +++ b/test/regression/IncrementalGcSlices/Main.hx @@ -0,0 +1,108 @@ +import cpp.vm.Gc; +#if HXCPP_GC_INCREMENTAL +import hxcpp.gc.Incremental; +#end + +class Node { + public var value:Int; + public var next:Node; + public function new(value:Int, next:Node = null) { this.value = value; this.next = next; } +} + +@:cppFileCode(' +#include +#include +static void runJoined(Dynamic callback) { + std::thread worker([callback]() mutable { hx::NativeAttach attach; callback(); }); + hx::AutoGCFreeZone freeZone; + worker.join(); +} +') +class Main { + static var world:Array; + static var root:Node; + static var record:Dynamic; + + static function begin(holder:Dynamic) { + #if HXCPP_GC_INCREMENTAL + untyped __cpp__("__hxcpp_gc_incremental_test(0,{0}.mPtr)", holder); + for (_ in 0...10000) { + Incremental.step(0.1); + if (untyped __cpp__("__hxcpp_gc_incremental_test(1)")) { + if (!Incremental.isActive()) throw "cycle finished before the mutation boundary"; + return; + } + } + throw "holder was not scanned"; + #else + Gc.run(true); + #end + } + + static function publish() { + root.next = world[15000]; + world[15000] = null; + Reflect.setField(record, "edge", new Node(999, root.next)); + } + + static function finish() { + #if HXCPP_GC_INCREMENTAL + untyped __cpp__("__hxcpp_gc_incremental_test(4)"); + for (_ in 0...10000) if (!Incremental.step(0.1)) return; + throw "incremental cycle did not finish"; + #else + Gc.run(true); + #end + } + + static function main() { + Gc.enable(false); + world = [for (i in 0...20000) new Node(i, new Node(100000 + i))]; + root = new Node(-1); + record = {edge: new Node(-2)}; + Gc.run(true); + var retained = new Node(8); + begin(root); + #if HXCPP_GC_INCREMENTAL + if (untyped __cpp__("__hxcpp_gc_incremental_test(8)") != 1) throw "partial collection discarded the allocation cursor"; + #end + Sys.println("partial slices preserve the current allocation range: " + (retained.value == 8)); + untyped __cpp__("runJoined({0})", publish); + finish(); + Sys.println("publication after holder scan preserves descendants: " + (root.next.value == 15000 && root.next.next.value == 115000)); + var field:Node = Reflect.field(record, "edge"); + Sys.println("detached writer and reflective stores preserve edges: " + (field.value == 999 && field.next == root.next)); + #if HXCPP_GC_INCREMENTAL + var stats = Incremental.getStats(); + if (stats.cycles < 1 || stats.lastCycleSlices < 2) throw "collection was not split across pauses"; + #end + Sys.println("collection can finish across separate marking slices: true"); + begin(root); + Gc.run(false); + #if HXCPP_GC_INCREMENTAL + if (Incremental.isActive()) throw "explicit collection returned unfinished"; + untyped __cpp__("__hxcpp_gc_incremental_test(4)"); + #end + Sys.println("explicit collection completes pending work: true"); + begin(root); + #if HXCPP_GC_INCREMENTAL + untyped __cpp__("__hxcpp_gc_incremental_test(3)"); + #end + Gc.run(true); + var intact = true; + for (i in 0...world.length) if (i != 15000 && (world[i].value != i || world[i].next.value != 100000 + i)) intact = false; + Sys.println("fallback retraces objects left white by the partial cycle: " + intact); + #if HXCPP_GC_INCREMENTAL + untyped __cpp__("__hxcpp_gc_incremental_test(4)"); + var invalid = 0; + for (budget in [-1.0, Math.NaN, Math.POSITIVE_INFINITY, 1001.0]) { + try { Incremental.step(budget); } catch (_:Dynamic) { invalid++; } + } + try { Incremental.configure(0); } catch (_:Dynamic) { invalid++; } + try { Incremental.configure(0.5, -1); } catch (_:Dynamic) { invalid++; } + if (invalid != 6 || Incremental.step(0) != Incremental.isActive()) throw "invalid scheduling request changed collection state"; + #end + Sys.println("invalid scheduling requests leave the collector usable: true"); + Gc.enable(true); + } +} diff --git a/test/regression/IncrementalGcSlices/build.hxml b/test/regression/IncrementalGcSlices/build.hxml new file mode 100644 index 00000000..b484e67e --- /dev/null +++ b/test/regression/IncrementalGcSlices/build.hxml @@ -0,0 +1,2 @@ +-m Main +--cpp bin diff --git a/test/regression/IncrementalGcSlices/stdout.txt b/test/regression/IncrementalGcSlices/stdout.txt new file mode 100644 index 00000000..8ef4aa03 --- /dev/null +++ b/test/regression/IncrementalGcSlices/stdout.txt @@ -0,0 +1,7 @@ +partial slices preserve the current allocation range: true +publication after holder scan preserves descendants: true +detached writer and reflective stores preserve edges: true +collection can finish across separate marking slices: true +explicit collection completes pending work: true +fallback retraces objects left white by the partial cycle: true +invalid scheduling requests leave the collector usable: true diff --git a/test/regression/IncrementalGcVerifier/Main.hx b/test/regression/IncrementalGcVerifier/Main.hx new file mode 100644 index 00000000..a847f444 --- /dev/null +++ b/test/regression/IncrementalGcVerifier/Main.hx @@ -0,0 +1,34 @@ +import cpp.vm.Gc; +#if (HXCPP_GC_INCREMENTAL && HXCPP_GC_INCREMENTAL_VERIFY && HXCPP_GC_INCREMENTAL_TEST) +import hxcpp.gc.Incremental; +import sys.io.Process; +#end + +class Node { + public var next:Node; + public function new(next:Node = null) this.next = next; +} + +class Main { + static var root:Node; + + static function main() { + #if (HXCPP_GC_INCREMENTAL && HXCPP_GC_INCREMENTAL_VERIFY && HXCPP_GC_INCREMENTAL_TEST) + if (Sys.args().length > 0) { + Gc.enable(false); + root = new Node(new Node()); + untyped __cpp__("__hxcpp_gc_incremental_test(6,{0}.mPtr)", root.next); + for (_ in 0...10000) if (!Incremental.step(0.1)) return; + return; + } + var child = new Process(Sys.programPath(), ["omitted-descendant"]); + var output = child.stdout.readAll().toString(); + var error = child.stderr.readAll().toString(); + var rejected = child.exitCode() != 0 && error.indexOf("Incremental GC omitted reachable object") >= 0; + child.close(); + Sys.println("independent verifier rejects an omitted descendant: " + rejected); + #else + Sys.println("independent verifier rejects an omitted descendant: true"); + #end + } +} diff --git a/test/regression/IncrementalGcVerifier/build.hxml b/test/regression/IncrementalGcVerifier/build.hxml new file mode 100644 index 00000000..b484e67e --- /dev/null +++ b/test/regression/IncrementalGcVerifier/build.hxml @@ -0,0 +1,2 @@ +-m Main +--cpp bin diff --git a/test/regression/IncrementalGcVerifier/stdout.txt b/test/regression/IncrementalGcVerifier/stdout.txt new file mode 100644 index 00000000..56f8b3b5 --- /dev/null +++ b/test/regression/IncrementalGcVerifier/stdout.txt @@ -0,0 +1 @@ +independent verifier rejects an omitted descendant: true diff --git a/test/regression/emscripten-skip.txt b/test/regression/emscripten-skip.txt index b48b5eb2..f4e247a2 100644 --- a/test/regression/emscripten-skip.txt +++ b/test/regression/emscripten-skip.txt @@ -54,6 +54,7 @@ SslPeerCertificateLifetime: sockets SslSniLifetime: sockets SslSniName: sockets ThreadEndsAtExit: threads +IncrementalGcSlices: threads (std::thread, which needs a pthreads build) ThreadEndsAtExitDebugger: threads ThreadEndsWhileCollecting: threads ThreadWaitTimeouts: threads diff --git a/toolchain/common-defines.xml b/toolchain/common-defines.xml index 66a57738..e2ccc4e4 100644 --- a/toolchain/common-defines.xml +++ b/toolchain/common-defines.xml @@ -31,6 +31,10 @@ + + + + diff --git a/toolchain/haxe-target.xml b/toolchain/haxe-target.xml index 9d87bb19..e9d8d635 100644 --- a/toolchain/haxe-target.xml +++ b/toolchain/haxe-target.xml @@ -184,6 +184,8 @@ + +