From 1c57baa788ce5adbf0efe56900276a1ee0ab0dba Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Fri, 25 Sep 2026 17:43:02 +0800 Subject: [PATCH 1/4] Label measured slow CTest rows and verify PR exclusion counts --- scripts/ci_verify.py | 72 +++++++++++++++++++++++++++++++++++---- scripts/test_ci_verify.py | 58 +++++++++++++++++++++++++------ tests/CMakeLists.txt | 43 +++++++++++++++++++++++ 3 files changed, 156 insertions(+), 17 deletions(-) diff --git a/scripts/ci_verify.py b/scripts/ci_verify.py index d974b72f..1ce2a52b 100644 --- a/scripts/ci_verify.py +++ b/scripts/ci_verify.py @@ -330,6 +330,10 @@ # B-C-SURFACE's new witnesses are rows inside test_native_c_api. No release row # skips, so 672 registered is 672 run. RELEASE_MIN_TESTS = 672 +# PR-only registration floors. These are the complete CTest populations at +# 91d65ad6 (INT24); the ordinary full-run floors above remain unchanged. +# An excluded run must still discover at least this many rows before -LE. +EXCLUDED_REGISTERED_MIN = {'debug': 653, 'sanitizers': 653, 'native': 662} # CTest's closing summary: '100% tests passed out of N' when nothing failed, # '97% tests passed, 3 tests failed out of N' otherwise. N includes a skipped # row (counted as passed) and a row CTest could not start (counted as @@ -342,6 +346,7 @@ CTEST_LISTED_ROW = re.compile(r'^\s*\d+ - (.+) \(([^()\n]+)\)\s*$', re.MULTILINE) # A skipped row's own result line: ' 4/10 Test #3: name .....***Skipped 0.01 sec'. CTEST_SKIPPED_RESULT = re.compile(r'^\s*\d+/\d+ Test\s+#\d+: .*\*\*\*Skipped\b', re.MULTILINE) +CTEST_LIST_COUNT = re.compile(r'^Total Tests:\s*(\d+)\s*$', re.MULTILINE) # LeakSanitizer is unavailable in Apple's ASan runtime. Keep the Linux CI # lane strict, while allowing the local macOS ASan/UBSan profile to execute # its actual instrumented tests instead of failing during runtime startup. @@ -473,6 +478,12 @@ def ctest_row_count(output: bytes) -> int | None: return int(match.group('total')) if match else None +def ctest_list_count(output: bytes) -> int | None: + """Read CTest's own `ctest -N` discovery count, failing closed on ambiguity.""" + counts = CTEST_LIST_COUNT.findall(output.decode('utf-8', 'replace')) + return int(counts[0]) if len(counts) == 1 else None + + @dataclass(frozen=True) class CTestRows: """The rows of one CTest run: its own count, and those that did not run.""" @@ -619,7 +630,8 @@ def parse_args(argv: list[str] | None, *, source: Path = ROOT) -> argparse.Names parser.add_argument('--require-websocket', action='store_true', help='native only: execute test_native_live_websocket and refuse skip (77)') parser.add_argument('--exclude-label', default=None, - help='exclude one CTest label from this local verification run') + help='exclude one CTest label; verify the run count against ' + 'CTest discovery with and without -LE') parser.add_argument('--min-tests', type=int, default=None, help='fail the ctest-floor stage unless at least N CTest rows ran ' '(a skipped or not-run row is listed, never counted); ' @@ -948,6 +960,9 @@ def __init__(self, cfg: VerifyConfig): 'requireWebsocket': cfg.require_websocket, 'curlDir': str(cfg.curl_dir) if cfg.curl_dir else None, 'minTests': cfg.min_tests, + 'excludeLabel': cfg.exclude_label, + 'ctestRegistered': None, + 'ctestSelected': None, # Rows that ran, the floor's count; CTest's own count; the rows # it did not run, listed beside ctestRows and never counted. 'ctestRows': None, @@ -986,7 +1001,8 @@ def write_summary(self) -> None: self.summary_path.write_text(json.dumps(self.summary, indent=2, sort_keys=True) + '\n') def invoke(self, name: str, argv: list[str], *, extra_env: dict[str, str] | None = None, - timeout: int = 600, combine_stderr: bool = True) -> Completed: + timeout: int = 600, combine_stderr: bool = True, + stream_output: bool | None = None) -> Completed: if self.cfg.ccache_path: extra_env = {**(extra_env or {}), 'CCACHE_COMPILERCHECK': 'content'} log = self.logs / f'{name}.log' @@ -1001,12 +1017,13 @@ def invoke(self, name: str, argv: list[str], *, extra_env: dict[str, str] | None self.stages.append(stage) log.write_text(header) self.write_summary() - if self.cfg.stream_output: + stream = self.cfg.stream_output if stream_output is None else stream_output + if stream: print(f'ci_verify: {name}: {shlex.join(map(str, argv))}', flush=True) try: result = call_runner( self.cfg.runner, list(map(str, argv)), extra_env=extra_env, timeout=timeout, - combine_stderr=combine_stderr, stream_output=self.cfg.stream_output) + combine_stderr=combine_stderr, stream_output=stream) except Exception as error: result = Completed(1, b'', str(error).encode()) body = result.stdout + (b'' if not result.stderr else b'\n' + result.stderr) @@ -1418,6 +1435,21 @@ def run(self) -> int: apple_asan = (self.cfg.profile.sanitizers and sys.platform == 'darwin' and not cxx_name.startswith('g++')) ctest_jobs = 1 if apple_asan else self.cfg.jobs + registered = selected = None + if self.cfg.exclude_label: + listing = ['ctest', '--test-dir', str(self.cfg.build_dir), '-N'] + all_rows = self.invoke('ctest-list-all', listing, timeout=120, + stream_output=False) + selected_rows = self.invoke('ctest-list-selected', + listing + ['-LE', self.cfg.exclude_label], + timeout=120, stream_output=False) + if all_rows.returncode == 0: + registered = ctest_list_count(all_rows.stdout + all_rows.stderr) + if selected_rows.returncode == 0: + selected = ctest_list_count(selected_rows.stdout + selected_rows.stderr) + self.summary['ctestRegistered'] = registered + self.summary['ctestSelected'] = selected + self.write_summary() ctest = ['ctest', '--test-dir', str(self.cfg.build_dir), '--output-on-failure', '--no-tests=error', '--parallel', str(ctest_jobs)] if self.cfg.exclude_label: @@ -1425,7 +1457,7 @@ def run(self) -> int: if ctest_supports_junit(self.cfg.runner): ctest += ['--output-junit', str(self.cfg.build_dir / 'ctest-junit.xml')] ran = self.invoke('ctest', ctest, extra_env=self.sanitizer_env(), timeout=1800) - self.enforce_test_floor(ran) + self.enforce_test_floor(ran, registered=registered, selected=selected) installed = self.invoke( 'install', @@ -1456,7 +1488,8 @@ def run(self) -> int: status = 'passed' if not self.failures else 'failed' return self.finish(status, 0 if status == 'passed' else 1) - def enforce_test_floor(self, ran: Completed) -> None: + def enforce_test_floor(self, ran: Completed, *, registered: int | None = None, + selected: int | None = None) -> None: """Refuse a CTest run in which fewer rows ran than the profile's floor. The count is the rows whose test ran to a verdict, pass or fail. A row @@ -1477,6 +1510,33 @@ def enforce_test_floor(self, ran: Completed) -> None: self.summary['ctestSkipped'] = list(rows.skipped) if rows else None self.summary['ctestNotRun'] = list(rows.not_run) if rows else None self.summary['ctestDisabled'] = list(rows.disabled) if rows else None + if self.cfg.exclude_label: + minimum = max(EXCLUDED_REGISTERED_MIN.get(self.cfg.profile.name, 0), + self.cfg.min_tests or 0) + if unreadable or rows is None or registered is None or selected is None: + self.fail_stage('ctest-exclusion', + 'CTest discovery or run count was unreadable; cannot prove ' + 'registered - labelled = ran' + + (f' ({unreadable})' if unreadable else '')) + elif registered < minimum: + self.fail_stage('ctest-exclusion', + f'CTest registered {registered} rows, below the PR ' + f'registration floor of {minimum}') + elif not 0 < selected < registered: + self.fail_stage('ctest-exclusion', + f'CTest registered {registered} rows and selected {selected}; ' + f'expected a nonempty {self.cfg.exclude_label} exclusion') + elif rows.ran != selected: + self.fail_stage('ctest-exclusion', + f'CTest registered {registered}, labelled ' + f'{registered - selected}, selected {selected}, but ran ' + f'{rows.ran}{rows.not_counted()}') + else: + self.pass_stage('ctest-exclusion', + f'ctest registered {registered}, labelled ' + f'{registered - selected}, ran {rows.ran} ' + f'(registered - labelled){rows.not_counted()}') + return floor = self.cfg.min_tests if floor is None: self.write_summary() diff --git a/scripts/test_ci_verify.py b/scripts/test_ci_verify.py index 498b8e8d..d56346e4 100644 --- a/scripts/test_ci_verify.py +++ b/scripts/test_ci_verify.py @@ -215,6 +215,18 @@ def __call__(self, argv, *, extra_env=None, timeout=600, combine_stderr=True, if self.exits.get('actual_empty_ctest'): return default_runner(argv, extra_env=extra_env, timeout=timeout, combine_stderr=combine_stderr, stream_output=False) + registered = self.exits.get( + 'ctest_registered', + ci_verify.EXCLUDED_REGISTERED_MIN.get( + self.profile, ci_verify.PROFILE[self.profile].min_tests or KERNEL_MIN_TESTS)) + labelled = self.exits.get('ctest_labelled', 5) + if '-N' in argv: + stage = 'ctest-list-selected' if '-LE' in argv else 'ctest-list-all' + if self.exits.get(stage) == 'unreadable': + return Completed(0, b'CTest list has no total\n', b'') + count = registered - labelled if '-LE' in argv else registered + return Completed(int(self.exits.get(stage, 0)), + f'Total Tests: {count}\n'.encode(), b'') env_ok = True if self.profile == 'sanitizers': env_ok = extra_env == SANITIZER_RUN_ENV @@ -227,8 +239,9 @@ def __call__(self, argv, *, extra_env=None, timeout=600, combine_stderr=True, # 'ctest_raw' scripts the output verbatim. if 'ctest_raw' in self.exits: return Completed(int(self.exits.get('ctest', 0)), self.exits['ctest_raw'], b'') - rows = self.exits.get( - 'ctest_rows', ci_verify.PROFILE[self.profile].min_tests or KERNEL_MIN_TESTS) + default_rows = (registered - labelled if '-LE' in argv else + ci_verify.PROFILE[self.profile].min_tests or KERNEL_MIN_TESTS) + rows = self.exits.get('ctest_rows', default_rows) if rows == 'absent': return Completed(int(self.exits.get('ctest', 0)), b'tests\n', b'') summary = ctest_output(rows, self.exits.get('ctest_skipped', ()), @@ -1295,14 +1308,36 @@ def test_successful_scripted_release_exit_zero(self): self.assertEqual(Path(ctest_argv[ctest_argv.index('--output-junit') + 1]).resolve(), (build_dir / 'ctest-junit.xml').resolve()) - def test_ctest_label_exclusion_is_forwarded(self): - code, summary, scripted, _ = self.run_profile( - extra=['--exclude-label', 'l4-pending']) - self.assertEqual(code, 0, summary['failures']) - ctest_argv = next( - argv for argv in scripted.calls if argv[0] == 'ctest' and '--test-dir' in argv) - self.assertIn('-LE', ctest_argv) - self.assertEqual(ctest_argv[ctest_argv.index('-LE') + 1], 'l4-pending') + def test_pr_exclusion_proves_registered_minus_labelled_equals_ran(self): + self.assertEqual(ci_verify.EXCLUDED_REGISTERED_MIN, + {'debug': 653, 'sanitizers': 653, 'native': 662}) + for profile, registered in ci_verify.EXCLUDED_REGISTERED_MIN.items(): + with self.subTest(profile=profile): + code, summary, scripted, _ = self.run_profile( + profile, extra=['--exclude-label', 'slow']) + self.assertEqual(code, 0, summary['failures']) + self.assertEqual(summary['ctestRegistered'], registered) + self.assertEqual(summary['ctestSelected'], registered - 5) + self.assertEqual(summary['ctestRows'], registered - 5) + self.assertIn('ctest-exclusion', stage_names(summary)) + self.assertNotIn('ctest-floor', stage_names(summary)) + self.assertIn('ctest-list-all', stage_names(summary)) + self.assertIn('ctest-list-selected', stage_names(summary)) + + def test_ctest_discovery_count_rejects_missing_or_ambiguous_summary(self): + self.assertEqual(ci_verify.ctest_list_count(b'Test #1: a\nTotal Tests: 1\n'), 1) + self.assertIsNone(ci_verify.ctest_list_count(b'No tests were found\n')) + self.assertIsNone(ci_verify.ctest_list_count(b'Total Tests: 1\nTotal Tests: 2\n')) + + def test_pr_exclusion_refuses_lost_registration_or_label(self): + for exits in ({'ctest_registered': 652}, {'ctest_labelled': 0}, + {'ctest_rows': 647}, {'ctest_skipped': ['silent_skip']}, + {'ctest-list-selected': 'unreadable'}): + with self.subTest(exits=exits): + code, summary, _, _ = self.run_profile( + 'debug', extra=['--exclude-label', 'slow'], **exits) + self.assertEqual(code, 1) + self.assertIn('ctest-exclusion', failure_stages(summary)) def test_junit_flag_omitted_when_unsupported(self): code, summary, scripted, build_dir = self.run_profile(junit_help='absent') @@ -1314,7 +1349,8 @@ def test_ctest_label_exclusion_is_forwarded(self): code, summary, scripted, _ = self.run_profile(extra=['--exclude-label', 'l4-pending']) self.assertEqual(code, 0, summary['failures']) ctest_argv = next( - argv for argv in scripted.calls if argv[0] == 'ctest' and '--test-dir' in argv) + argv for argv in scripted.calls + if argv[0] == 'ctest' and '--test-dir' in argv and '-N' not in argv) self.assertIn('-LE', ctest_argv) self.assertEqual(ctest_argv[ctest_argv.index('-LE') + 1], 'l4-pending') diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 76d3e1f3..fd0103a0 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -2366,3 +2366,46 @@ if(PINEFORGE_BUILD_SOURCE_LAYER AND TARGET test_adapter_live_state_equivalence) add_test(NAME test_adapter_margin_revival_erasure COMMAND test_adapter_live_state_equivalence --margin-revival) endif() + +# PR CI-LITE timing selection: INT23 sanitizer job 107895095058 and INT24 +# sanitizer/Debug jobs 107996509646, 107996511076, 107996511419. +# A row is slow when it exceeded 60 s in either sanitizer run or 30 s in +# either Debug run. This is the single home of the PR exclusion population; +# full push, dispatch, and maintainer verification still register and run it. +set(PINEFORGE_PR_SLOW_TESTS + test_adapter_live_state_equivalence + test_adapter_live_state_scaling + test_adapter_reissue_binding + test_aggregate_cpp_versions + test_broker_state_hash_coverage_mutations + test_chart_day_memo + test_ci_verify + test_l10recon1_merge_fill_liveness + test_local_time_fields + test_magnifier_endpoints4 + test_native_bar_open_hook + test_native_command_after + test_native_cpp_abi + test_native_cpp_versions + test_native_definition_index + test_native_direct_mutation + test_native_event_retention + test_native_handle_stable_replace + test_native_host_reads + test_native_lookup_memos + test_native_match_band_precheck + test_native_match_row_reuse + test_native_precommit_hook + test_native_quiet_point + test_native_runtime_ambient + test_native_script_bucket_completions + test_native_settlement_carry +) +foreach(_pf_slow_test IN LISTS PINEFORGE_PR_SLOW_TESTS) + if(PINEFORGE_BUILD_SOURCE_LAYER AND NOT TEST ${_pf_slow_test}) + message(FATAL_ERROR "PR slow row is no longer registered: ${_pf_slow_test}") + endif() + if(TEST ${_pf_slow_test}) + set_property(TEST ${_pf_slow_test} APPEND PROPERTY LABELS slow) + endif() +endforeach() From f55a8bf11ed28606255d9512c044a5e0ca4d15a4 Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Fri, 25 Sep 2026 17:43:10 +0800 Subject: [PATCH 2/4] Run advisory PR CI in parallel and honor maintainer merge statuses --- .github/workflows/ci.yml | 33 +++----- .github/workflows/corpus-parity.yml | 17 ++--- .github/workflows/native-live.yml | 7 +- .github/workflows/promote-baseline.yml | 34 +++++---- scripts/ci_preflight.py | 100 ++++++++++++++++++++++++- scripts/test_ci_preflight.py | 42 ++++++++++- 6 files changed, 187 insertions(+), 46 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 44ba07d2..03272cc6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,7 @@ on: branches: [main] pull_request: branches: [main] + workflow_dispatch: permissions: contents: read @@ -66,7 +67,6 @@ jobs: if-no-files-found: warn build: - needs: preflight timeout-minutes: 45 strategy: fail-fast: false @@ -121,7 +121,7 @@ jobs: # ratio -- for a runner whose CPU accounting is unusable; record the # measured reason here and in docs/ci.md before setting it. PINEFORGE_RUNTIME_BUDGET_CANDIDATE_ONLY: '0' - run: python3 scripts/ci_verify.py ${{ matrix.build_type == 'Release' && 'release' || 'debug' }} --build-dir build --jobs 4 --ccache + run: python3 scripts/ci_verify.py ${{ matrix.build_type == 'Release' && 'release' || 'debug' }} --build-dir build --jobs 4 --ccache ${{ github.event_name == 'pull_request' && matrix.build_type == 'Debug' && '--exclude-label slow' || '' }} - name: Stage and summarize diagnostics if: always() @@ -142,7 +142,6 @@ jobs: # instrumentation propagates into every test binary (CMakeLists.txt option # PINEFORGE_ENABLE_SANITIZERS). sanitizers: - needs: preflight runs-on: ubuntu-24.04 timeout-minutes: 90 env: @@ -172,7 +171,7 @@ jobs: ccache-${{ runner.os }}-${{ runner.arch }}-sanitizers- - name: Verify (sanitizers) - run: python3 scripts/ci_verify.py sanitizers --build-dir build-asan --jobs 4 --ccache + run: python3 scripts/ci_verify.py sanitizers --build-dir build-asan --jobs 4 --ccache ${{ github.event_name == 'pull_request' && '--exclude-label slow' || '' }} - name: Stage and summarize diagnostics if: always() @@ -193,7 +192,6 @@ jobs: # modules are built from the kernel alone, and every examples/native host # runs as an example_* row asserting both its exit code and its summary line. kernel-only: - needs: preflight runs-on: ubuntu-24.04 timeout-minutes: 45 steps: @@ -234,30 +232,23 @@ jobs: path: ci-diagnostics/ if-no-files-found: warn - # Reuse the full native proof once per CI run. The reusable workflow also - # retains manual dispatch for explicitly requested standalone verification. + # Reuse native proof once per CI run. Its manual dispatch runs every row. native-live: - needs: preflight uses: ./.github/workflows/native-live.yml + with: + exclude_slow: ${{ github.event_name == 'pull_request' }} - # TradingView parity, on the population a pull request can wait for. No - # ci_verify.py profile compiles a corpus strategy, so without this job - # nothing between a src/** change and a merge re-derives the engine's - # parity: the whole sweep is ~32 min and can only run nightly - # (corpus-parity.yml). This one re-runs the 30 probes - # scripts/corpus_parity_subset.txt names against the same pinned sha256 - # rows, and feeds the required `build` context below — which is what makes - # a trade-moving change unmergeable rather than caught the next night. - # Deliberately unfiltered by path: `build` requires success, and a skipped - # dependency is not success. + # Advisory TradingView parity subset on every CI run. The maintainers post + # the required pineforge/parity status from their separate parity verdict. + # Keep this job unfiltered so build-gate reports its result on every run. corpus-parity-subset: - needs: preflight uses: ./.github/workflows/corpus-parity.yml with: mode: subset - # Keep the required `build` context while making it cover every proof lane. - # Failure, cancellation or skipping in any dependency must fail this gate. + # Advisory aggregate context covering every GitHub Actions lane. The merge + # ruleset requires pineforge/verify and pineforge/parity, posted by the + # maintainers, rather than this build context. build-gate: name: build needs: [preflight, build, sanitizers, native-live, kernel-only, corpus-parity-subset] diff --git a/.github/workflows/corpus-parity.yml b/.github/workflows/corpus-parity.yml index 827b1571..9ef63432 100644 --- a/.github/workflows/corpus-parity.yml +++ b/.github/workflows/corpus-parity.yml @@ -23,17 +23,17 @@ name: Corpus parity # runs nightly and on demand, plus on any pull request that touches the pin or # the parity tooling itself. # -# AND HOW PARITY STILL BLOCKS A MERGE. The `corpus-parity-subset` job below +# PR EARLY SIGNAL. The `corpus-parity-subset` job below # re-runs the 30 probes scripts/corpus_parity_subset.txt names -- in parallel, # they write into disjoint directories -- and judges them against the same # pinned sha256 rows the nightly uses. Measured on the same laptop at JOBS=8, # under sibling load: derive 2 s, build the runtime + the 30 strategy .so 68 s # from clean, run 45 s wall for 80 s of probe CPU, judge <1 s; 94 s end to end # over an up-to-date build directory, under 3 min cold. -# .github/workflows/ci.yml calls it as a reusable workflow and -# makes it a dependency of the required `build` context, so a src/** change that -# moves one of those 30 probes by one byte cannot merge. Nothing about the -# nightly job changes. +# .github/workflows/ci.yml calls it as a reusable workflow and includes it in +# the advisory `build` aggregate. The maintainers' separate full parity +# verdict posts the required pineforge/parity commit status on the PR head. +# Nothing about the nightly job changes. on: schedule: @@ -44,7 +44,7 @@ on: workflow_call: inputs: mode: - description: 'full (all 312 probes) or subset (the pull-request gate)' + description: 'full (all 312 probes) or subset (PR early signal)' type: string default: full pull_request: @@ -156,9 +156,8 @@ jobs: path: corpus-parity-diagnostics/ if-no-files-found: warn - # The blocking half: the 30 probes scripts/corpus_parity_subset.txt names, - # judged against the same pinned sha256 rows. Reached only through - # .github/workflows/ci.yml, which needs it for the required `build` context. + # The advisory PR subset: the 30 probes scripts/corpus_parity_subset.txt + # names, judged against the same pinned sha256 rows. Reached through CI. corpus-parity-subset: if: inputs.mode == 'subset' runs-on: ubuntu-24.04 diff --git a/.github/workflows/native-live.yml b/.github/workflows/native-live.yml index ab19f10a..9249e9f6 100644 --- a/.github/workflows/native-live.yml +++ b/.github/workflows/native-live.yml @@ -2,6 +2,11 @@ name: Native live runner on: workflow_call: + inputs: + exclude_slow: + description: Exclude measured slow CTest rows in the caller's PR run + type: boolean + default: false workflow_dispatch: permissions: @@ -100,7 +105,7 @@ jobs: cmake --install curl-build 2>&1 | tee curl-install.log - name: Verify native live - run: python3 scripts/ci_verify.py native --build-dir build-live --jobs 4 --generator Ninja --curl-dir "${{ github.workspace }}/build-native-deps/curl-install/lib/cmake/CURL" --ccache --require-websocket + run: python3 scripts/ci_verify.py native --build-dir build-live --jobs 4 --generator Ninja --curl-dir "${{ github.workspace }}/build-native-deps/curl-install/lib/cmake/CURL" --ccache --require-websocket ${{ inputs.exclude_slow && '--exclude-label slow' || '' }} - name: Stage and summarize diagnostics if: always() diff --git a/.github/workflows/promote-baseline.yml b/.github/workflows/promote-baseline.yml index 71fc42a3..523b0d22 100644 --- a/.github/workflows/promote-baseline.yml +++ b/.github/workflows/promote-baseline.yml @@ -7,11 +7,12 @@ # On a merged PR it advances the campaign baseline one axis to the merged # composite, reusing the snapshot from the pr-gate run that already cleared # it. It promotes ONLY when the merge is exact-head (the gated commit's tree is -# now on the base branch), CI is green on that commit, and a PASS verdict binds +# now on the base branch), both maintainer verification statuses are green on +# that commit, and a PASS verdict binds # the full (engine, codegen) pair. Otherwise it exits green without promoting. # -# The gate logic (exact-head + CI-green + baseline-promote.mjs) is unchanged; -# only the transport to the campaign plane changed. The plane is now GCP: the +# The gate logic is exact-head + pineforge/verify + pineforge/parity + +# baseline-promote.mjs. The plane is now GCP: the # promotion reads the active baseline and the gate ledger from the Postgres # registry through a cloud-sql-proxy, streams the pinned documents from the # GCS evidence bucket, and appends the new baseline row — authenticated by @@ -72,10 +73,9 @@ on: permissions: contents: read - # "Confirm CI is green" reads check-runs, which is `checks`. The older - # combined-status API is not called, so no `statuses: read` is granted -- - # this block is exactly what the steps below do, and nothing spare. - checks: read + # The two required merge-gate verdicts are commit statuses, not Actions + # check-runs. Inspect the PR head commit's statuses before promotion. + statuses: read # Workload identity federation mints the GCP credential from the job's own # OIDC token; nothing here can promote without this repo's identity. id-token: write @@ -121,7 +121,7 @@ jobs: fi echo "ok=true" >> "$GITHUB_OUTPUT" - - name: Confirm CI is green on the merged commit + - name: Confirm the merge gate on the merged PR head id: ci if: steps.exacthead.outputs.ok == 'true' env: @@ -129,10 +129,18 @@ jobs: HEAD_SHA: ${{ github.event.pull_request.head.sha || github.event.inputs.merge_commit }} run: | set -euo pipefail - bad=$(gh api "repos/${{ github.repository }}/commits/$HEAD_SHA/check-runs" \ - --jq '[.check_runs[] | select(.conclusion=="failure" or .conclusion=="cancelled" or .conclusion=="timed_out" or .conclusion=="action_required")] | length') - if [ "$bad" != "0" ]; then - echo "::notice::$bad failing check-run(s) on $HEAD_SHA — not green; skipping" + # GitHub lists newest statuses first. Keep the first occurrence of + # each context across pages, so an older success cannot mask a + # newer failure or pending verdict on this exact commit. + statuses=$(gh api --paginate \ + "repos/${{ github.repository }}/commits/$HEAD_SHA/statuses?per_page=100") + if ! jq -es ' + reduce (.[][]) as $s ({}; + if has($s.context) then . else .[$s.context] = $s.state end) + | .["pineforge/verify"] == "success" + and .["pineforge/parity"] == "success" + ' <<< "$statuses" >/dev/null; then + echo "::notice::pineforge/verify and pineforge/parity must both be success on $HEAD_SHA — skipping" echo "green=false" >> "$GITHUB_OUTPUT"; exit 0 fi echo "green=true" >> "$GITHUB_OUTPUT" @@ -188,7 +196,7 @@ jobs: chmod +x /tmp/cloud-sql-proxy /tmp/cloud-sql-proxy --port 5433 \ gen-lang-client-0864094636:asia-east1:pineforge-workflow-pg & - for i in $(seq 1 30); do + for _ in $(seq 1 30); do if (exec 3<>/dev/tcp/127.0.0.1/5433) 2>/dev/null; then exit 0; fi sleep 1 done diff --git a/scripts/ci_preflight.py b/scripts/ci_preflight.py index 87699e9c..d1c85fea 100644 --- a/scripts/ci_preflight.py +++ b/scripts/ci_preflight.py @@ -20,6 +20,7 @@ from __future__ import annotations import argparse +import hashlib import json from pathlib import Path import re @@ -30,6 +31,88 @@ from ci_verify import ROOT, source_guard_commands ACTIONLINT_VERSION = '1.7.12' +# The names live only in tests/CMakeLists.txt. A digest pins that measured +# population without maintaining a second row list in this Python guard. +PR_SLOW_ROWS_SHA256 = '90f8932f921579c55561e1af962610e0e311d1f93f28722a4c5931d7b6f7fb53' + + +def _jobs(workflow: str) -> dict[str, str]: + body = workflow.split('\njobs:\n', 1) + if len(body) != 2: + return {} + matches = list(re.finditer(r'^ ([a-z][a-z0-9-]*):\s*$', body[1], re.MULTILINE)) + return {match.group(1): body[1][match.end(): + matches[index + 1].start() if index + 1 < len(matches) + else len(body[1])] + for index, match in enumerate(matches)} + + +def ci_workflow_findings(ci: str, native: str, promote: str, cmake: str) -> list[str]: + """Pin the PR-light/full-event split, parallel start, merge statuses and row home.""" + findings = [] + events = ci.split('\non:\n', 1) + events = events[1].split('\npermissions:', 1)[0] if len(events) == 2 else '' + for trigger in ('push:\n branches: [main]', + 'pull_request:\n branches: [main]', 'workflow_dispatch:'): + if ' ' + trigger not in events: + findings.append(f'ci.yml must retain {trigger.split(":", 1)[0]}') + jobs = _jobs(ci) + proof_jobs = ('build', 'sanitizers', 'kernel-only', 'native-live', + 'corpus-parity-subset') + for job in ('preflight', *proof_jobs, 'build-gate'): + if job not in jobs: + findings.append(f'ci.yml is missing job {job}') + for job in proof_jobs: + if re.search(r'^ needs:', jobs.get(job, ''), re.MULTILINE): + findings.append(f'{job} must start alongside preflight') + gate = jobs.get('build-gate', '') + if ('needs: [preflight, build, sanitizers, native-live, kernel-only, ' + 'corpus-parity-subset]' not in gate or ' if: always()' not in gate + or ' name: build' not in gate): + findings.append('build-gate must aggregate every proof job and preflight') + for job, variable in (('preflight', 'PREFLIGHT_RESULT'), ('build', 'BUILD_RESULT'), + ('sanitizers', 'SANITIZER_RESULT'), ('native-live', 'NATIVE_RESULT'), + ('kernel-only', 'KERNEL_RESULT'), + ('corpus-parity-subset', 'PARITY_RESULT')): + if (f'${{{{ needs.{job}.result }}}}' not in gate + or f'"${variable}" == "success"' not in gate): + findings.append(f'build-gate must require {job} success') + debug_flag = "${{ github.event_name == 'pull_request' && matrix.build_type == 'Debug' && '--exclude-label slow' || '' }}" + pr_flag = "${{ github.event_name == 'pull_request' && '--exclude-label slow' || '' }}" + if debug_flag not in jobs.get('build', '') or jobs.get('build', '').count('--exclude-label slow') != 1: + findings.append('only PR Debug matrix jobs may exclude slow rows') + if pr_flag not in jobs.get('sanitizers', '') or jobs.get('sanitizers', '').count('--exclude-label slow') != 1: + findings.append('only PR sanitizers may exclude slow rows') + if 'exclude_slow: ${{ github.event_name == \'pull_request\' }}' not in jobs.get('native-live', ''): + findings.append('native-live must receive the PR-only exclusion input') + for job in ('kernel-only', 'corpus-parity-subset'): + if '--exclude-label' in jobs.get(job, ''): + findings.append(f'{job} must keep its full population') + if (' workflow_call:\n inputs:\n exclude_slow:' not in native + or ' type: boolean\n default: false' not in native + or ' workflow_dispatch:' not in native + or "${{ inputs.exclude_slow && '--exclude-label slow' || '' }}" not in native): + findings.append('native-live must default to full rows for dispatch and push') + if (' statuses: read' not in promote or ' checks: read' in promote + or '/commits/$HEAD_SHA/statuses?per_page=100' not in promote + or 'gh api --paginate' not in promote + or 'reduce (.[][]) as $s' not in promote + or 'if has($s.context) then . else .[$s.context] = $s.state end' not in promote + or '.["pineforge/verify"] == "success"' not in promote + or '.["pineforge/parity"] == "success"' not in promote + or '/check-runs' in promote): + findings.append('baseline promotion must require the latest two head statuses') + blocks = re.findall(r'^set\(PINEFORGE_PR_SLOW_TESTS\n(.*?)^\)', cmake, + re.MULTILINE | re.DOTALL) + names = re.findall(r'^ (test_[A-Za-z0-9_]+)$', blocks[0], re.MULTILINE) if len(blocks) == 1 else [] + canonical = '\n'.join(names) + '\n' + if (len(blocks) != 1 or len(names) != 27 or len(set(names)) != 27 + or hashlib.sha256(canonical.encode()).hexdigest() != PR_SLOW_ROWS_SHA256 + or cmake.count('APPEND PROPERTY LABELS slow') != 1 + or 'set_property(TEST ${_pf_slow_test} APPEND PROPERTY LABELS slow)' not in cmake + or 'if(PINEFORGE_BUILD_SOURCE_LAYER AND NOT TEST ${_pf_slow_test})' not in cmake): + findings.append('measured slow rows must keep one pinned CMake label list') + return findings def docs_workflow_findings(workflow: str) -> list[str]: @@ -70,7 +153,10 @@ def check_commands(source: Path) -> list[tuple]: str(source / '.github/workflows/ci.yml'), str(source / '.github/workflows/native-live.yml'), str(source / '.github/workflows/corpus-parity.yml'), - str(source / '.github/workflows/docs.yml')]), + str(source / '.github/workflows/docs.yml'), + str(source / '.github/workflows/promote-baseline.yml')]), + ('ci-workflow-contract', [sys.executable, str(source / 'scripts/ci_preflight.py'), + '--check-ci-workflow']), ('docs-workflow-contract', [sys.executable, str(source / 'scripts/ci_preflight.py'), '--check-docs-workflow']), ('docs-workflow-contract-tests', @@ -179,7 +265,19 @@ def main() -> int: parser.add_argument('--output-dir', type=Path, default=ROOT / 'build-ci-preflight') parser.add_argument('--check-docs-workflow', action='store_true') parser.add_argument('--self-test-docs-workflow', action='store_true') + parser.add_argument('--check-ci-workflow', action='store_true') args = parser.parse_args() + if args.check_ci_workflow: + findings = ci_workflow_findings( + (ROOT / '.github/workflows/ci.yml').read_text(), + (ROOT / '.github/workflows/native-live.yml').read_text(), + (ROOT / '.github/workflows/promote-baseline.yml').read_text(), + (ROOT / 'tests/CMakeLists.txt').read_text()) + for finding in findings: + print(finding) + if not findings: + print('CI workflow contract: PR exclusions, full events, statuses and slow rows OK') + return 1 if findings else 0 if args.check_docs_workflow or args.self_test_docs_workflow: workflow = (ROOT / '.github/workflows/docs.yml').read_text() if args.self_test_docs_workflow: diff --git a/scripts/test_ci_preflight.py b/scripts/test_ci_preflight.py index 7ded5d29..9f0ac389 100644 --- a/scripts/test_ci_preflight.py +++ b/scripts/test_ci_preflight.py @@ -8,10 +8,50 @@ import tempfile import unittest -from ci_preflight import check_commands, run_checks +from ci_preflight import check_commands, ci_workflow_findings, run_checks class PreflightFailures(unittest.TestCase): + def ci_sources(self): + root = Path(__file__).resolve().parents[1] + return [(root / path).read_text() for path in ( + '.github/workflows/ci.yml', '.github/workflows/native-live.yml', + '.github/workflows/promote-baseline.yml', 'tests/CMakeLists.txt')] + + def test_ci_contract_pins_parallel_pr_light_and_full_events(self): + original = self.ci_sources() + self.assertEqual(ci_workflow_findings(*original), []) + mutations = ( + (0, ' workflow_dispatch:\n', ''), + (0, ' native-live:\n', ' missing-native-live:\n'), + (0, ' sanitizers:\n', ' missing-sanitizers:\n'), + (0, ' build:\n', ' missing-build:\n'), + (0, ' kernel-only:\n', ' missing-kernel:\n'), + (0, ' corpus-parity-subset:\n', ' missing-subset:\n'), + (0, ' build-gate:\n', ' missing-gate:\n'), + (0, '"$SANITIZER_RESULT" == "success"', '"$SANITIZER_RESULT" != "success"'), + (0, ' sanitizers:\n', ' sanitizers:\n needs: preflight\n'), + (0, "matrix.build_type == 'Debug' && '--exclude-label slow'", "'--exclude-label slow'"), + (0, "github.event_name == 'pull_request' && '--exclude-label slow'", "'--exclude-label slow'"), + (0, "exclude_slow: ${{ github.event_name == 'pull_request' }}", 'exclude_slow: true'), + (1, ' default: false', ' default: true'), + (1, "${{ inputs.exclude_slow && '--exclude-label slow' || '' }}", ''), + (2, ' statuses: read', ' checks: read'), + (2, '/commits/$HEAD_SHA/statuses?per_page=100', '/commits/$HEAD_SHA/check-runs'), + (2, '.["pineforge/parity"] == "success"', 'true'), + (2, 'if has($s.context) then . else .[$s.context] = $s.state end', + '.[$s.context] = $s.state'), + (3, ' test_chart_day_memo\n test_ci_verify\n', + ' test_chart_day_memo\n'), + (3, 'APPEND PROPERTY LABELS slow', 'APPEND PROPERTY LABELS other'), + ) + for index, before, after in mutations: + with self.subTest(index=index, before=before): + changed = original.copy() + self.assertIn(before, changed[index]) + changed[index] = changed[index].replace(before, after, 1) + self.assertNotEqual(ci_workflow_findings(*changed), []) + def run_preflight(self, commands): directory = tempfile.TemporaryDirectory() self.addCleanup(directory.cleanup) From 8557a0f0d41a953ac182f3a73918ce4181d2e222 Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Fri, 25 Sep 2026 17:44:07 +0800 Subject: [PATCH 3/4] Document advisory PR CI and required verification statuses --- AGENTS.md | 39 +++++----- README.md | 2 +- docs/ci.md | 103 ++++++++++++++++++++------- docs/design/native-feature-parity.md | 4 +- docs/native-refactor-progress.md | 4 +- 5 files changed, 102 insertions(+), 50 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ead3ffe3..9497ac86 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,6 +6,13 @@ Both C++ unit tests and full corpus verification must pass. +The merge ruleset requires `pineforge/verify` and `pineforge/parity` commit +statuses on the PR head's exact tree. The maintainers' `lab verify` tooling +posts them after full verification on their x86-64 build hosts and a parity +verdict. GitHub Actions is advisory: PR Debug, sanitizers and native jobs +exclude the measured `slow` CTest rows; push-to-main and manual CI dispatch +run every row. The commands below run the full sets. + ```bash # Fast workflow/source checks first (requires actionlint 1.7.12 + ShellCheck). # This does not replace either verification step below. @@ -122,23 +129,17 @@ dispatches, reviews and measures; this section is what binds you here. engine closer to an independent backtest + forward-execution state machine, and Pine-parity behaviour kept in codegen with this kernel clean. -## Parity campaign gate (applies on EVERY harness) - -Pushes and PRs from this repo are gated by the PineForge parity campaign: a -fresh (≤6h) PASS verdict must bind the exact (engine, codegen) HEADs, recorded -on the campaign registry. Under Claude Code a PreToolUse hook -(`.claude/settings.json`, calls `pineforge-workflow/campaign/hooks/pr-gate.mjs`) -enforces this on `git push` / `gh pr create|ready|merge`. Codex, OpenCode, and -other harnesses run NO hook — the discipline is exactly as binding there: before -any push, run the gate and record the verdict (see the `pr-gate` skill in -`pineforge-workflow/.claude/skills/` — plain markdown, readable anywhere): - -```sh -gcloud run jobs execute pineforge-pr-gate --project gen-lang-client-0864094636 \ - --region asia-east1 --args '^|^--pipeline|pr_gate|--conf|' -lab gate record --verdict --engine --codegen -``` +## Merge gate and parity campaign + +The `PineForge strict CI base` ruleset requires `pineforge/verify` (full +`ci_verify.py` profiles on the PR head's exact tree) and `pineforge/parity` +(no parity regression, or no engine behaviour change). The maintainers' +`lab verify` tooling posts both commit statuses; GitHub Actions does not post them. +Its PR jobs provide faster advisory feedback, while push-to-main and manual +dispatch run the full CI profiles. A campaign PASS verdict still binds the +exact engine and codegen HEADs for baseline promotion. -Merged single-axis PRs advance the campaign baseline automatically -(`.github/workflows/promote-baseline.yml`); a squash/rebase that rewrites the -sha defers and must be re-gated. +Merged single-axis PRs advance the campaign baseline automatically through +`.github/workflows/promote-baseline.yml` only when the exact-head guard, both +required statuses on that head, and the campaign verdict pass. A squash or +rebase that rewrites the SHA defers promotion and needs new verification. diff --git a/README.md b/README.md index c450c018..a436b44a 100644 --- a/README.md +++ b/README.md @@ -260,7 +260,7 @@ Every script is exported from TradingView as-is (its own inputs, its own default - **strong** — ≥ 95% matched, trade count within 6%, entries within 0.1% and exits within 0.5% at p90; - **moderate / weak** — ≥ 75% coverage, or less. -Published parity results use a fixed population and reproducible Cloud Run measurements. The formal gate requires **no hard-surface regression** and strictly positive pooled movement across the target excellent and excellent+strong bands. A documented native-correctness exception permits exactly zero target-band movement with no individual regression, after full comparison, independent review and CI; its actual FAIL remains recorded and baseline promotion is deferred. Negative movement is outside this exception. Baseline promotion requires a recorded PASS and an exact-head merge with green CI. +Published parity results use a fixed population and reproducible Cloud Run measurements. The formal gate requires **no hard-surface regression** and strictly positive pooled movement across the target excellent and excellent+strong bands. A documented native-correctness exception permits exactly zero target-band movement with no individual regression, after full comparison and independent review; its actual FAIL remains recorded and baseline promotion is deferred. Negative movement is outside this exception. The merge ruleset requires the maintainers' `pineforge/verify` and `pineforge/parity` commit statuses on the exact PR head; GitHub Actions CI is advisory. Baseline promotion also requires a recorded PASS and an exact-head merge. ### What the closed test taught the engine diff --git a/docs/ci.md b/docs/ci.md index af6168df..47b038d7 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -1,6 +1,6 @@ # Local verification and CI -Start with the fast preflight used by GitHub Actions. Install actionlint 1.7.12 +Start with the preflight used by GitHub Actions. Install actionlint 1.7.12 and ShellCheck, then run: ```sh @@ -9,7 +9,8 @@ python3 scripts/ci_preflight.py This checks the CI/native workflow syntax, expressions and shell commands, source ABI/hash/schema guards, and the verifier's failure-handling tests. A -failure blocks all compilation lanes and retains logs in `build-ci-preflight/`. +failure fails the advisory aggregate `build` job and retains logs in +`build-ci-preflight/`. The compilation lanes start alongside preflight. It does not compile the engine or replace any complete verification profile. Then use the same verification entrypoint as GitHub Actions before publishing: @@ -93,6 +94,17 @@ release rows. Raise the constant when a row lands, and pass `--min-tests N` to override it for one run (the flag gates any profile; `kernel` and `release` have a default). +For PR runs with `--exclude-label slow`, the verifier asks CTest to enumerate +the registered rows both with `ctest -N` and with `ctest -N -LE slow`. It then +requires the rows that actually ran to equal the second count, and records +`registered`, `labelled = registered - selected`, and `ran` in +`ctest-exclusion.log` and `ci-summary.json`. A skip, missing executable, lost +registration below the PR registration floor, unreadable enumeration, or +missing label fails. The PR registration floors at the INT24 base are 653 +for Debug and sanitizers and 662 for native. The existing full-run release and +kernel floors remain 672 and 271 rows that ran; full runs do not exclude a +label. + Preflight also runs the detached-comment census of the kernel compile closure (`detached-comments`: `scripts/measure_detached_comments.py --check-ceiling`) against the checked-in `DETACHED_LINE_CEILING`, which is 0: R5 lane B-DOCS @@ -316,18 +328,19 @@ probe, machine load average 100–160 from sibling builds). Judging the result and the dominant term is single-core serial work that a 4-vCPU hosted runner runs slower, not faster. -So the whole population cannot be held to the ~25 min a required pull-request -check is budgeted for, and it runs nightly (01:30 UTC), on `workflow_dispatch`, +So the whole population cannot be held to the ~25 min pull-request CI target, +and it runs nightly (01:30 UTC), on `workflow_dispatch`, and on pull requests that touch the corpus pin or the parity tooling. ccache (2 GB, keyed per commit with a prefix restore) makes a re-run at an unchanged pin almost entirely cache hits; the Git-LFS chart feed (~176 MB, one full-history 1m CSV) is pulled once per job. -### The subset that does block a merge +### The advisory pull-request subset -A nightly gate cannot stop a merge: a `src/**` change that moves a TradingView -trade merges green and is caught the next night at the earliest. The blocking -half is a named population, not a weaker oracle: +The GitHub Actions subset gives early feedback on a `src/**` change that moves +a TradingView trade. The maintainers' separate parity verdict posts the +required `pineforge/parity` status on the exact PR head. The subset is a named +population using the same byte oracle: ```sh ./scripts/check_corpus_parity.sh --subset @@ -365,19 +378,18 @@ judge <1 s — **94 s** end to end over an up-to-date build directory, against 1792 s for the run phase alone in full mode. `corpus-parity.yml` exposes it as a reusable workflow (`mode: subset`) and -`ci.yml` calls it on every CI run and lists it in `build-gate`'s `needs`. The -"PineForge strict CI base" ruleset requires the contexts `build` and -`sanitizers`; `build` **is** `build-gate`, so the parity subset blocks a merge -through the context that is already required, with no ruleset change. A -skipped dependency is not a success, so the job is deliberately unfiltered by -path. +`ci.yml` calls it on every CI run and lists it in `build-gate`'s `needs`. A +skipped dependency fails that advisory aggregate job, so the subset remains +unfiltered by path. The merge ruleset requires `pineforge/verify` and +`pineforge/parity`, posted by the maintainers' `lab verify` tooling after +full verification on the PR head's exact tree. What the subset does not prove, and what therefore stays with the nightly sweep: the other 282 probes, and the tier headline — `scripts/verify_corpus.py` grades the whole population, so 30 re-runs cannot print its line, and grading the 282 untouched tapes beside them would judge the -corpus's own older generation rather than this engine. The subset is a blocking -floor, not a replacement. +corpus's own older generation rather than this engine. The subset is an early +signal alongside the full parity verdict. The subset job checks the submodule out with `GIT_LFS_SKIP_SMUDGE=1` and restores `corpus/data` from a cache keyed by the gitlink, so the ~176 MB feed @@ -504,14 +516,51 @@ identity are retained even if preparation fails before the verifier starts. Compiler objects, binaries and dependency caches are not uploaded as diagnostics. -Superseded pull-request runs are canceled. CI/native main/post-merge and manual proof runs -use distinct concurrency groups and remain uncanceled. Native verification is a -reusable workflow called once by CI, with a separate concurrency namespace; it -also supports manual dispatch. The required `build` check passes only when -preflight, all four standard builds, sanitizers, native verification and the -TradingView parity subset succeed. A failed, canceled or skipped dependency -cannot produce a green `build` check. -The separate required `sanitizers` status remains available. - -These checks do not run the parity campaign. Fixed-population Cloud measurement, -the actual gate and post-merge evidence remain separate acceptance steps. +Superseded pull-request runs are canceled. Main/post-merge and manual CI runs +use distinct concurrency groups and remain uncanceled. A PR runs preflight, +both Release jobs, kernel-only and the parity subset with their full sets; +both Debug jobs, sanitizers and native-live run the registered set excluding +the 27 CTest rows labelled `slow` in `tests/CMakeLists.txt`. These rows were +chosen from INT23/INT24 job logs: over 60 seconds in either sanitizer run or +over 30 seconds in either Debug run. Preflight and all proof jobs start in +parallel. The advisory `build` aggregate succeeds only if every job succeeds. + +A push to `main` and a manual dispatch of `ci.yml` run every profile without +the exclusion. Standalone manual dispatch of `native-live.yml` is also full. +The maintainers run the full `ci_verify.py` profiles on their x86-64 build +hosts before the PR merge gate and post `pineforge/verify` to the PR head. +Their parity verdict posts `pineforge/parity`, reporting no regression or that +the PR changed no engine behaviour. The `PineForge strict CI base` ruleset +requires these two commit statuses. GitHub Actions jobs, including `build`, +`sanitizers`, docs and the parity subset, are advisory. Baseline promotion +requires both statuses to be successful on the exact merged PR head, as well +as its existing exact-head and campaign verdict guards. + +The full corpus sweep remains a separate acceptance step. The nightly and +manual corpus workflow, and the maintainers' full parity verification, keep +the broader population running. + +### CI-LITE timing estimate + +The INT24 PR run `36105656107` measured 9.1 min preflight, 30.2/27.8 min +macOS/Ubuntu Debug, 25.1 min native, 20.5/16.5 min Ubuntu/macOS Release, +15.8 min kernel, and 3.8 min parity subset. Its sanitizer job ran 61.5 min +and hit the verifier's 30 min CTest timeout. The complete INT23 sanitizer +job `107895095058` measured 42.1 min: 20.0 min CTest and 22.1 min outside +CTest, including a 21.1 min build. The 27 labelled rows consumed 34.8 and +42.3 min of summed test time in the INT24 Ubuntu and macOS Debug jobs. The +26 rows then present in INT23 sanitizers consumed 57.5 of its 76.7 min of +summed test time; the 27th row landed by INT24. + +At four concurrent CTest jobs, the remaining rows have scheduling lower +bounds of 1.1 min Ubuntu Debug, 1.6 min macOS Debug, 0.5 min native, and +4.8 min INT23 sanitizers. Adding observed non-CTest time and a small CTest +scheduling allowance estimates about 19 min Ubuntu Debug, 20 min macOS +Debug, 16 min native, and 28 min sanitizer on the complete INT23 run. Release +and kernel keep their measured full durations. With preflight running in +parallel, the modeled PR wall time is about 28 min on INT23 conditions. The +INT24 sanitizer build-to-CTest interval alone was 30.3 min, so its conditions +imply a wall time above 35 min even after excluding the rows. The 25 min +target needs a measured sanitizer build improvement; the current job logs do +not separate compile from link time or report a ccache hit rate, so CI-LITE +does not assume a build optimization without evidence. diff --git a/docs/design/native-feature-parity.md b/docs/design/native-feature-parity.md index 78b87d12..dffbb28c 100644 --- a/docs/design/native-feature-parity.md +++ b/docs/design/native-feature-parity.md @@ -345,7 +345,7 @@ the campaign's own anchor audit found that the first thing to rot: ### 3.1 Parity discipline for every lane (R5-10) — stated once, referenced per lane -- **(a) NEUTRALITY.** Every new behaviour is opt-in by a spec field or a new request kind. The adapter's `project()` (pine_adapter.cpp:2141-2278) never sets the field and never emits the kind, so adapter runs are byte-identical by construction. The four feed-shape fields are folded into `hash_spec` unconditionally today, including defaults (native_execution_consumer.cpp:238-260), so even a defaulted new field would move every continuation hash; the precedent for a conditional digest is `precommit_digest_` (native_execution_consumer.hpp:1620-1624). Each lane adds a test that a default spec hashes to the pre-change constant. Every new durable field is hashed (`check_broker_state_hash_coverage.py`, fail-closed, waivers in `broker_state_hash_waivers.txt`). Proof per PR: `scripts/run_corpus.sh` + `scripts/verify_corpus.py` locally, then ONE Cloud Run campaign sweep on the final tree (fixed population, 4190 probes, zero tolerance on the hard band — native-refactor-progress.md:41-47; a fresh exact-HEAD verdict is required, `sha256:0178f02a2a0cd9a592d5182a4b0c6823e9914fccdb8d3dc00efc4d0d9fe9fcc1` `sha256:2dc825fd690aa47e3b969c26093e95f3cacfe56ff250d80bc72394d4fe92e02a` AGENTS.md:124-143). A lane that cannot be shown neutral by construction does not merge. +- **(a) NEUTRALITY.** Every new behaviour is opt-in by a spec field or a new request kind. The adapter's `project()` (pine_adapter.cpp:2141-2278) never sets the field and never emits the kind, so adapter runs are byte-identical by construction. The four feed-shape fields are folded into `hash_spec` unconditionally today, including defaults (native_execution_consumer.cpp:238-260), so even a defaulted new field would move every continuation hash; the precedent for a conditional digest is `precommit_digest_` (native_execution_consumer.hpp:1620-1624). Each lane adds a test that a default spec hashes to the pre-change constant. Every new durable field is hashed (`check_broker_state_hash_coverage.py`, fail-closed, waivers in `broker_state_hash_waivers.txt`). Proof per PR: `scripts/run_corpus.sh` + `scripts/verify_corpus.py` locally, then ONE Cloud Run campaign sweep on the final tree (fixed population, 4190 probes, zero tolerance on the hard band — native-refactor-progress.md:41-47; a fresh exact-HEAD verdict is required, `sha256:0178f02a2a0cd9a592d5182a4b0c6823e9914fccdb8d3dc00efc4d0d9fe9fcc1` `sha256:e3aa487ce8d69db9054576f75839345c0acaab55c3fc0a374c2f9c8c071559ec` AGENTS.md:132-145). A lane that cannot be shown neutral by construction does not merge. - **(b) TWIN.** One chosen probe per lane runs through the adapter AND a hand-written `NativeStrategyHost` on the same bars and spec. Trade lists are diffed **by identity** (entry / exit time, price, qty, pnl, commission), never by trade number; then event, ownership and hash detail (S's D → E order). A difference is allowed only where a named TV-quirk row of §1 is active, and each one is itemized. Because 430 test files drive the adapter and 14 the native host (O), every lane also adds native-only tests: a lane is not accepted on the corpus alone, nor on a unit test alone (S). - **(c) EPOCH BUDGET.** Three inline-namespace epochs move — the request epoch, the host epoch and (*fresh*, not in the inputs) the run-spec epoch, which every new spec field moves. They stood at `native_order_v5`, `engine_script_run_v17` and `native_run_spec_v2` when this plan was written; on this tree they are `native_order_v7` (native_order.hpp:25), `engine_script_run_v19` (native_host.hpp:20) and `native_run_spec_v3` (native_run_spec.hpp:15): the budget below was spent exactly once, and the v19 value epoch (R5 lane V19-A) took the host epoch one step past it. One bump each per release batch: `native_order_v6` = L3 + L7 + L4's origin / event; `engine_script_run_v18` = L4's host hooks + L5 + L6 virtuals + L2's `hash_host_extension` + §2.ii a, f; `native_run_spec_v3` = the spec blocks of L2, L4, L5, L6, L8 (L9 if it ships in the batch). A lane that lands before its batch closes stages behind the open epoch, so codegen-built strategies rebuild once per batch. Epoch lanes are not neutral-refactor lanes: the brief lists the header extension explicitly; the frozen C++ ABI fixtures (`tests/fixtures/native_cpp_abi/host-*`) and the ABI checkers (`check_native_cpp_abi.py`, `check_settlement_cpp_abi.py`) move with them; `variant_size` pins change (`OrderIntent` native_order.hpp:229, `CommandEvent` native_order.hpp:1262). Every new test unit compiles first against the frozen previous-epoch header closure (fail-before, `CLAUDE.md`). - **Worker verification** (`CLAUDE.md`): `ci_preflight.py`, then `ci_verify.py release`; plus `check_c_abi_runtime.py` and `check_native_include_independence.py`. No campaign sweep from a worker. @@ -381,7 +381,7 @@ on `main` — `git log --grep "lane "` finds the rest of a lane's commits: |---|---|---| | **P2** | `scripts/check_kernel_residuals.py`: `strings`/`nm` over `libpineforge_kernel.a` held against ADR-0001's residual tables, as a `kernel`-profile stage and a CTest row in every profile. Ruled the eighty-nine pending-row names the first probe was blind to (`coof_*`, `pooc_*`, `market_admission_*`, `tv_carry_qty`) by family, and the ambient EMA seeding default by mechanism — `ta::EmaSeeding` became a named per-instance option | `c421b7a9` | | **P2b** | Made that gate profile-independent: it reads the linkable surface (`nm` over the archive, `strings` over a debug-stripped copy) rather than debug info, and carves out the source-path literals a sanitizer build writes into rodata | `8e099884` | -| **P3** | TradingView parity can block a merge: `scripts/check_corpus_parity.sh --subset` runs the same byte oracle over a 30-probe subset a pull request can afford, through a status context the ruleset already requires | `9b831117` | +| **P3** | `scripts/check_corpus_parity.sh --subset` runs the same byte oracle over a 30-probe subset a pull request can afford. Since the 2026-09-25 merge-gate change, this GitHub Actions run is advisory; the maintainers' full parity verdict posts the required `pineforge/parity` status | `9b831117` | | **P4** | The C surface's asymmetries: `PF_NATIVE_MARGIN_CHECK_FX_ROLL` under its own C name, a C host's recorded route to a closed row's exit ticket, and the calculation cadence on the migration page | `98b94672` | | **P5** | The untested capabilities: the market-if-touched geometry of `Limit{price, fill_through}` kernel-only, `closed_trade(i)` as the closed row by reference, and a kernel liquidation seen through `native_open_lots()` under `NativeRunSpec::margin` | `41ca2ee8` | | **P6** | The standing question "is a spec field the adapter never sets a decision or dead weight?" answered for all ten: `scripts/check_native_feature_rulings.py` and ADR-0001's ruling table, with three native examples added | `ceb07e4a` | diff --git a/docs/native-refactor-progress.md b/docs/native-refactor-progress.md index 31e8fede..54a931a0 100644 --- a/docs/native-refactor-progress.md +++ b/docs/native-refactor-progress.md @@ -165,6 +165,8 @@ Publish reviewed increments in an evolving draft PR so implementation and open decisions are reviewable. Draft and CI-first publication may precede the full compatibility sweep. Before merge, require native proof, unchanged fixed-population compatibility evidence, the actual gate result, independent -review of the final candidate, and green CI. Squash merge and verify the +review of the final candidate, and successful `pineforge/verify` and +`pineforge/parity` statuses on the exact PR head. GitHub Actions CI is an +advisory signal. Squash merge and verify the resulting tree and post-merge checks separately. Never translate a neutral gate failure into PASS or force baseline promotion. From 53d36551ba38549d4edccbc80d5bc587bbff6cfc Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Fri, 25 Sep 2026 18:56:10 +0800 Subject: [PATCH 4/4] Preserve existing verifier test bodies during CI enumeration --- scripts/test_ci_verify.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/scripts/test_ci_verify.py b/scripts/test_ci_verify.py index d56346e4..97f65f89 100644 --- a/scripts/test_ci_verify.py +++ b/scripts/test_ci_verify.py @@ -131,7 +131,10 @@ def __init__(self, build_dir: Path, source: Path, profile: str = 'release', **ex def __call__(self, argv, *, extra_env=None, timeout=600, combine_stderr=True, stream_output=False) -> Completed: argv = list(map(str, argv)) - self.calls.append(argv) + # Discovery invocations are asserted through Driver stages. Keep the + # existing execution-call fixture stable for pre-existing tests. + if argv[0] != 'ctest' or '-N' not in argv: + self.calls.append(argv) extra_env = extra_env or {} joined = ' '.join(argv) secret = os.environ.get('CI_VERIFY_TEST_SECRET') @@ -1308,6 +1311,15 @@ def test_successful_scripted_release_exit_zero(self): self.assertEqual(Path(ctest_argv[ctest_argv.index('--output-junit') + 1]).resolve(), (build_dir / 'ctest-junit.xml').resolve()) + def test_ctest_label_exclusion_is_forwarded(self): + code, summary, scripted, _ = self.run_profile( + extra=['--exclude-label', 'l4-pending']) + self.assertEqual(code, 0, summary['failures']) + ctest_argv = next( + argv for argv in scripted.calls if argv[0] == 'ctest' and '--test-dir' in argv) + self.assertIn('-LE', ctest_argv) + self.assertEqual(ctest_argv[ctest_argv.index('-LE') + 1], 'l4-pending') + def test_pr_exclusion_proves_registered_minus_labelled_equals_ran(self): self.assertEqual(ci_verify.EXCLUDED_REGISTERED_MIN, {'debug': 653, 'sanitizers': 653, 'native': 662}) @@ -1349,8 +1361,7 @@ def test_ctest_label_exclusion_is_forwarded(self): code, summary, scripted, _ = self.run_profile(extra=['--exclude-label', 'l4-pending']) self.assertEqual(code, 0, summary['failures']) ctest_argv = next( - argv for argv in scripted.calls - if argv[0] == 'ctest' and '--test-dir' in argv and '-N' not in argv) + argv for argv in scripted.calls if argv[0] == 'ctest' and '--test-dir' in argv) self.assertIn('-LE', ctest_argv) self.assertEqual(ctest_argv[ctest_argv.index('-LE') + 1], 'l4-pending')