From 604e5f28e307d836b90887d39101d6a60957f859 Mon Sep 17 00:00:00 2001 From: J <315733358+raiseCatError@users.noreply.github.com> Date: Mon, 5 Oct 2026 00:21:18 +0530 Subject: [PATCH] Speed up local verification and shard platform CI --- .github/actions/linux-shells/action.yml | 30 +++ .github/workflows/ci.yml | 241 ++++++++++++------ AGENTS.md | 34 ++- CONTRIBUTING.md | 21 +- docs/development-verification.md | 99 +++++++ .../plans/2026-10-04-ci-verification.md | 21 ++ package.json | 10 +- scripts/check-process-leaks.sh | 8 + scripts/test-selection.mjs | 82 ++++++ scripts/test.mjs | 19 +- scripts/timing-smoke.mjs | 13 + tests/testSharding.test.ts | 37 +++ 12 files changed, 502 insertions(+), 113 deletions(-) create mode 100644 .github/actions/linux-shells/action.yml create mode 100644 docs/development-verification.md create mode 100644 docs/superpowers/plans/2026-10-04-ci-verification.md create mode 100644 scripts/check-process-leaks.sh create mode 100644 scripts/test-selection.mjs create mode 100644 scripts/timing-smoke.mjs create mode 100644 tests/testSharding.test.ts diff --git a/.github/actions/linux-shells/action.yml b/.github/actions/linux-shells/action.yml new file mode 100644 index 00000000..fb0f1abe --- /dev/null +++ b/.github/actions/linux-shells/action.yml @@ -0,0 +1,30 @@ +name: Linux shell prerequisites +description: Install missing shell/build tools and secure system completion paths +runs: + using: composite + steps: + - shell: bash + run: | + # Ubuntu images supply native build tools; probe rather than reinstall. + packages=() + command -v zsh >/dev/null || packages+=(zsh) + command -v fish >/dev/null || packages+=(fish) + command -v g++ >/dev/null || packages+=(g++) + command -v make >/dev/null || packages+=(make) + command -v python3 >/dev/null || packages+=(python3) + if (( ${#packages[@]} )); then + sudo apt-get update + sudo apt-get install -y "${packages[@]}" + fi + insecure="$(zsh -fc 'autoload -Uz compaudit; compaudit' 2>/dev/null || true)" + while IFS= read -r path; do + [ -z "$path" ] && continue + case "$path" in + /usr/share/zsh|/usr/share/zsh/*|/usr/local/share/zsh|/usr/local/share/zsh/*) + sudo chown root:root "$path" + sudo chmod go-w "$path" + ;; + *) echo "::error::Unexpected insecure completion path: $path"; exit 1 ;; + esac + done <<< "$insecure" + zsh -fc 'autoload -Uz compaudit; compaudit' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0cd41764..ca532e20 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,8 +4,6 @@ on: push: branches: [master, dev] pull_request: - # Stacked PRs need the same gates as integration PRs. - workflow_dispatch: permissions: @@ -16,109 +14,184 @@ concurrency: cancel-in-progress: true jobs: - verify: - name: Verify (${{ matrix.os }}, Node ${{ matrix.node-version }}) + changes: + name: Detect code changes + runs-on: ubuntu-24.04 + outputs: + code: ${{ steps.changes.outputs.code }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - id: changes + env: + EVENT: ${{ github.event_name }} + BASE: ${{ github.event.pull_request.base.sha }} + HEAD: ${{ github.event.pull_request.head.sha }} + shell: bash + run: | + # Never path-skip the workflow: the final CI check must always exist. + # Push/manual runs always collect comprehensive evidence. + if [[ "$EVENT" != pull_request ]]; then + echo 'code=true' >> "$GITHUB_OUTPUT" + else + git diff --check "$BASE" "$HEAD" + git diff --no-renames --name-only -z "$BASE" "$HEAD" > /tmp/changed-files + python3 - <<'PYTHON' >> "$GITHUB_OUTPUT" + from pathlib import Path + files = Path('/tmp/changed-files').read_bytes().split(b'\0') + docs_only = all(f.startswith(b'docs/') or f.endswith(b'.md') for f in files if f) + print('code=' + str(not docs_only).lower()) + PYTHON + fi + + quality: + name: Quality (Node 26) + needs: changes + if: needs.changes.outputs.code == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 26.x + cache: npm + - run: npm ci + - run: npm run build + - run: npm run typecheck:bench + - run: npm run test:fast + - run: git diff --check + + tests: + name: Tests (${{ matrix.os }}, ${{ matrix.shard }}/2) + needs: changes + if: needs.changes.outputs.code == 'true' runs-on: ${{ matrix.os }} timeout-minutes: 10 strategy: fail-fast: false matrix: os: [macos-latest, ubuntu-24.04] - node-version: [22.x, 26.x] - + shard: [1, 2] steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 with: - node-version: ${{ matrix.node-version }} - cache: 'npm' - - - name: Install Linux shell and native build prerequisites - if: runner.os == 'Linux' - run: sudo apt-get update && sudo apt-get install -y zsh fish build-essential python3 - - - name: Verify secure Linux system completion paths + node-version: 26.x + cache: npm + - uses: ./.github/actions/linux-shells if: runner.os == 'Linux' - run: | - # Hosted image completion directories can be group-writable. Global - # compinit then prompts before NMSh's isolated startup files run. - insecure="$(zsh -fc 'autoload -Uz compaudit; compaudit' 2>/dev/null || true)" - while IFS= read -r path; do - [ -z "$path" ] && continue - case "$path" in - /usr/share/zsh|/usr/share/zsh/*|/usr/local/share/zsh|/usr/local/share/zsh/*) - sudo chown root:root "$path" - sudo chmod go-w "$path" - ;; - *) echo "::error::Unexpected insecure completion path: $path"; exit 1 ;; - esac - done <<< "$insecure" - zsh -fc 'autoload -Uz compaudit; compaudit' - - - name: Install dependencies - run: npm ci - - - name: Build - run: npm run build - - - name: Typecheck - run: npm run typecheck - - - name: Typecheck benchmark script - run: npx tsc --ignoreConfig --noEmit --types node --target ES2022 --module NodeNext --moduleResolution NodeNext --esModuleInterop --skipLibCheck scripts/benchmarks.ts scripts/platform-benchmarks.ts scripts/idle-benchmarks.ts - - - name: Bounded platform timing smoke - run: | - node --import=tsx scripts/platform-benchmarks.ts - NMSH_BENCH_SAMPLES=5 NMSH_BENCH_WARMUP=1 npm run bench -- completion/configured-cold completion/configured-warm composer/screen-plan transcript/wrap-present-10000 - - - name: Run tests - run: npm test -- --test-timeout=120000 + - run: npm ci + # Built-launcher tests must execute rather than silently skip on fresh runners. + - run: npm run build + - run: npm test -- --shard=${{ matrix.shard }}/2 --test-timeout=120000 + - name: Process leak check + if: always() + run: bash scripts/check-process-leaks.sh - - name: Verify working tree clean - run: git diff --check + node22: + name: Node 22 compatibility + needs: changes + if: needs.changes.outputs.code == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22.x + cache: npm + - uses: ./.github/actions/linux-shells + - run: npm ci + - run: npm run build + - run: npm run test:node22 -- --test-timeout=120000 + - name: Process leak check + if: always() + run: bash scripts/check-process-leaks.sh + timings: + name: Timing smoke (${{ matrix.os }}, Node 26) + if: github.event_name != 'pull_request' + runs-on: ${{ matrix.os }} + timeout-minutes: 5 + strategy: + fail-fast: false + matrix: + os: [macos-latest, ubuntu-24.04] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 26.x + cache: npm + - uses: ./.github/actions/linux-shells + if: runner.os == 'Linux' + - run: npm ci + - run: npm run bench:smoke - name: Process leak check - run: | - if ps -axo pid,ppid,pgid,command | grep -E '[z]sh.*nmsh-semantic|[n]msh-semantic|[c]onfigured-completion\.zsh|[c]apture\.zsh|[v]iewportSyntax' | grep -v grep; then - echo "::error::NMSh helper or test processes leaked!" - exit 1 - fi - echo "No processes leaked." + if: always() + run: bash scripts/check-process-leaks.sh - # One additional distribution for package-manager and libc/shell-packaging diversity (DNF, Fedora zsh/fish). - # A focused subset, not a matrix: the full suite runs on Ubuntu and macOS above. fedora: name: Linux portability subset (Fedora) + if: github.event_name != 'pull_request' runs-on: ubuntu-24.04 timeout-minutes: 15 - container: fedora:42 # clipboard process-lifetime tests need a real init to reap children; they run on Ubuntu and macOS + container: fedora:42 steps: - name: Install system prerequisites run: dnf install -y git zsh fish gcc-c++ make python3 which lsof procps-ng - - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup Node.js 22.x - uses: actions/setup-node@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 with: node-version: 22.x + cache: npm + - run: npm ci + - run: npm run build + - run: npm run test:fedora -- --test-timeout=120000 + - name: Process leak check + if: always() + run: bash scripts/check-process-leaks.sh - - name: Install dependencies - run: npm ci - - - name: Build - run: npm run build - - - name: Typecheck - run: npm run typecheck - - - name: Platform, package-manager, updater and shell-backend tests - run: node --import=tsx --test --test-timeout=120000 tests/linuxPlatform.test.ts tests/tools.test.ts tests/update.test.ts tests/hostProfiles.test.ts tests/portabilityUninstall.test.ts + status: + name: CI + if: always() + needs: [changes, quality, tests, node22, timings, fedora] + runs-on: ubuntu-24.04 + steps: + - name: Require all applicable gates + env: + RESULTS: ${{ toJSON(needs) }} + run: | + python3 - <<'PYTHON' + import json, os + jobs = json.loads(os.environ['RESULTS']) + code = jobs['changes']['outputs'].get('code') + required = ['changes'] + if code == 'true': + required += ['quality', 'tests', 'node22'] + elif code != 'false': + raise SystemExit('Missing change classification') + if '${{ github.event_name }}' != 'pull_request': + required += ['timings', 'fedora'] + failed = [name for name in required if jobs[name]['result'] != 'success'] + if failed: + raise SystemExit('Failed or skipped required gates: ' + ', '.join(failed)) + print('All applicable CI gates passed') + PYTHON + + # Preserve the exact check names required by the existing master ruleset. + required-checks: + name: Verify (${{ matrix.node-version }}) + needs: status + if: always() + runs-on: ubuntu-24.04 + strategy: + matrix: + node-version: [22.x, 26.x] + steps: + - name: Require aggregate CI success env: - NMSH_DISABLE_UPDATES: '1' - COLORTERM: truecolor + RESULT: ${{ needs.status.result }} + run: test "$RESULT" = success diff --git a/AGENTS.md b/AGENTS.md index 81b06251..cd5a47cb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -53,12 +53,24 @@ Submitted commands retain their semantic presentation in the NMSh output history Released (0.16.0): a real ShellAdapter with zsh, Fish and Bash 4.4+ backends ([docs/architecture/shell-adapter.md](docs/architecture/shell-adapter.md)). Nushell and PowerShell remain future. ## Testing / verification -Canonical verification commands: +During implementation, run focused affected tests. Use `npm run verify:fast` for +ordinary iteration (build, an explicit core test subset, and diff checks); it is +not the final gate. Before pushing a meaningful checkpoint, run `npm run verify` +(build, the full canonical suite, and diff checks). Build already checks the +source TypeScript; `npm run typecheck` remains available for direct use. + +For release-sensitive changes run `npm run verify:release`, which adds benchmark +script typechecking and bounded timing smoke. These commands reuse local +node_modules; use `npm ci` for clean CI/release environments. Batch coherent +changes and avoid pushing tiny or known-broken edits to use Actions as a test +runner. GitHub CI provides independent platform verification, not a replacement +for local checks. See [development verification](docs/development-verification.md) +for sharding, platform gates and exact-release evidence requirements. + ```bash -npm run build -npm run typecheck -npm test -git diff --check +npm run verify:fast +npm run verify +npm run verify:release ``` When writing tests involving `TerminalApp`, you must carefully tear down child processes and temp ZDOTDIRs: @@ -127,12 +139,9 @@ When given a task such as "work on the next Ready NMSh issue", follow this workf 9. **Add or update automated tests for behavior changes** where appropriate. -10. **Run the canonical verification suite:** +10. **Run canonical local verification before pushing a coherent checkpoint:** ```bash - npm run build - npm run typecheck - npm test - git diff --check + npm run verify ``` 11. **Commit and push the feature branch.** @@ -330,10 +339,7 @@ Cloud agents must detect their actual environment. Do not assume a cloud VM is m The project requires Node >=22. Prefer `npm ci` then run supported canonical verification: ```bash -npm run build -npm run typecheck -npm test -git diff --check +npm run verify ``` GitHub CI remains an integration gate. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 05cf972c..14205686 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -38,13 +38,24 @@ Then submit a Pull Request from `feature/example` to `dev`. ## Canonical verification -Before submitting a pull request, ensure that your changes pass the canonical verification suite: +During implementation, run focused affected tests. Use `npm run verify:fast` for +ordinary iteration (build, an explicit core test subset, and diff checks); it is +not the final gate. Before pushing a meaningful checkpoint, run `npm run verify` +(build, the full canonical suite, and diff checks). Build already checks the +source TypeScript; `npm run typecheck` remains available for direct use. + +For release-sensitive changes run `npm run verify:release`, which adds benchmark +script typechecking and bounded timing smoke. These commands reuse local +node_modules; use `npm ci` for clean CI/release environments. Batch coherent +changes and avoid pushing tiny or known-broken edits to use Actions as a test +runner. GitHub CI provides independent platform verification, not a replacement +for local checks. See [development verification](docs/development-verification.md) +for sharding, platform gates and exact-release evidence requirements. ```bash -npm run build -npm run typecheck -npm test -git diff --check +npm run verify:fast +npm run verify +npm run verify:release ``` *Note: When writing tests involving `TerminalApp`, you must carefully tear down child processes and temp ZDOTDIRs using `app['stop'](0)` and `app['session'].kill()` to prevent zombie processes.* diff --git a/docs/development-verification.md b/docs/development-verification.md new file mode 100644 index 00000000..3e60ae5d --- /dev/null +++ b/docs/development-verification.md @@ -0,0 +1,99 @@ +# Development verification + +## Local feedback + +- During implementation: focused affected tests via `node --import=tsx --test tests/.test.ts`, or `npm run verify:fast`. +- Before pushing a coherent checkpoint: `npm run verify`. +- Release-sensitive work: `npm run verify:release`. + +`verify:fast` builds and checks a curated core subset (editor, key decoding, +highlighting, protocol framing, completion model, path display, Linux helpers, +and test selection) plus `git diff --check`. It is not a full-suite gate. +`verify` builds, runs canonical `npm test`, and checks diffs. `verify:release` +adds `typecheck:bench` and `bench:smoke`. No aggregate recompiles the same source +twice. The test runner caps default file workers at four (or the lower Node default +on small runners), since each PTY worker also owns helper processes. Explicit +Node `--test-concurrency` arguments override the cap. Assertions and budgets +remain unchanged. Local commands reuse node_modules. Clean CI/release environments use +`npm ci`; setup-node caches npm downloads, never node_modules. + +## Canonical suite and sharding + +The only shard interface is `npm test -- --shard=1/2` (or `2/2`). No arguments +still runs every recursively discovered `tests/**/*.test.ts` file. File selection +lives in `scripts/test-selection.mjs`; its tests verify deterministic assignment, +exact union, no overlap, invalid-input rejection, and curated-list existence. +Runtime files are partitioned as whole files by deterministic greedy scheduling. +Rounded measured weights for slow files balance PTY work; ordinary/new files +default to one. Reprofile weights only when actual shard imbalance warrants it. `suggestionRanking.test.ts` runs +separately after runtime files, only in shard one, preserving uncontested latency +measurement. New files automatically enter the canonical suite and a shard. + +The fast, Node 22 and Fedora lists are explicit, reviewed subsets. They do not +replace the full canonical suite. Node 22 covers build identity/version, actual +built/source startup launch paths, real zsh/Fish/Bash session adapters, service +compatibility, wire protocols, Linux helpers and host profiles. Long detached +session scenarios run fully on Node 26 on both macOS and Ubuntu. + +## CI gates + +Code PRs run quality on Ubuntu/Node 26, two macOS/Node 26 shards, two Ubuntu/Node +26 shards, and curated Ubuntu/Node 22 compatibility. Each shard builds because +built-launcher tests otherwise skip on a clean checkout. Each integration shard +and smoke job checks helper-process leaks, including after a test failure. +Existing hard latency assertions remain in the canonical tests. + +Timing smoke runs on macOS and Ubuntu/Node 26 on dev/master pushes and manual +workflow dispatch. These measure platform-dependent shell/configuration startup +as well as bounded completion, composer and transcript samples. Fedora 42 runs +its existing portability subset on those same events. Benchmarks remain +informational regression evidence, not new timing thresholds. + +Documentation-only PRs (`*.md` or `docs/**`, including deleted files) skip costly +jobs at runtime. The workflow always starts and completes its aggregate `CI` +check; detection failures fail closed. `Verify (22.x)` and `Verify (26.x)` remain +aggregate aliases for the existing master ruleset. All applicable gates must +succeed; skipped required code or release gates cannot pass the aggregate. +Concurrency remains workflow + ref, cancelling superseded runs of the same PR +or branch independently. + +## Exact release commit + +Before tagging v0.17.0, run `verify:release` locally and require a successful +manual CI dispatch on the exact candidate commit (or its dev/master push run). +The SHA must match the release commit: both complete Node 26 platform suites, +Node 22 runtime compatibility, Fedora subset, both platform timing smokes, +build correctness, benchmark typechecking and diff checks must be green. +An ordinary PR run does not supply Fedora or benchmark release evidence. +Automated checks do not substitute for physical terminal validation. + +## Measurements + +Baseline: v0.16 dev run 37219429715 completed in 5m02s. Slowest job was +Ubuntu/Node 22, 4m59s, of which tests took 4m07s. The four OS/Node legs each ran +the full suite, two identical source compilations and timing smoke. +v0.17 PR run 37223805076 completed in 5m09s; slowest was macOS/Node 22, 4m57s. +It also ran Fedora on every PR. + +After: two full-suite equivalents per code PR (one per OS), distributed over +four parallel shards, plus curated Node 22 compatibility. Dev/manual runs add +Fedora and platform timing evidence in parallel. Measured local and Actions +results are recorded after validation; runner timing varies and 2–3 minutes is +a target rather than a correctness gate. + +Local profiling uses the unchanged canonical runner on macOS arm64/Node 26.8.1 +with real PTY/process access and the inherited `NO_COLOR=1` removed for existing +presentation fixtures. It passed in 82.9s, including the four new selection +regression cases. The fast aggregate passed in about 4s (63 tests). Measured +test work used for shard weights totals 225.7s versus 220.8s; these sums are not +wall times because files execute concurrently. + +Local final verification: `verify:fast` 2.1s; `verify` passed, and the subsequent +`verify:release` passed its nested canonical gate (1,514 tests, zero skips), +benchmark-script typecheck and timing smoke in 169.0s. Canonical test processes +in that release run took 158.4s combined. The Node 22 curated list passed on the +available Node 26 runtime in 33.9s; actual Node 22 validation is delegated to CI. +Final shard runs passed in 52.2s and 79.9s with four workers. Every shard and release run retains +all assertions and budgets. Higher local fan-out exposed intermittent existing +completion, resume-browser latency and terminal-mode timing failures; focused +reruns and final bounded-worker verification passed without assertion changes. diff --git a/docs/superpowers/plans/2026-10-04-ci-verification.md b/docs/superpowers/plans/2026-10-04-ci-verification.md new file mode 100644 index 00000000..f370ef15 --- /dev/null +++ b/docs/superpowers/plans/2026-10-04-ci-verification.md @@ -0,0 +1,21 @@ +# CI verification implementation plan + +Goal: shorten local and PR feedback without reducing canonical platform coverage. +Architecture: preserve the canonical runner and ranking isolation; partition whole +runtime files using measured weights. Separate quality, minimum-runtime smoke, +and dev/manual release evidence. Keep required aggregate checks always present. +Scope: active feature/v017-compatibility-discovery branch and PR #309 only. + +- [x] Inspect runner, workflow, package scripts, docs and observed Actions timings. +- [x] Measure the unchanged canonical suite and inspect slow files. +- [x] Test selection invariants before implementing the shard helper. +- [x] Add explicit fast/Node 22/Fedora lists and local verification commands. +- [x] Replace duplicated full-suite matrix with platform shards and separate gates. +- [x] Preserve required status aliases and test docs-only/failure behavior. +- [x] Update development and exact-release instructions. +- [x] Run local aggregates, both shards, benchmark smoke and workflow validation. +- [ ] Commit coherently as J, push and compare actual Actions timings. + +Review focus: exact suite union, ranking contention, invalid shard arguments, +code-to-doc renames, skipped/failed required gates, built-launcher test execution, +Linux prerequisite/security checks and helper-process cleanup. diff --git a/package.json b/package.json index c476569b..f98e445a 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,15 @@ "bench": "node --import=tsx scripts/benchmarks.ts", "dev": "tsx src/index.ts", "build": "tsc -p tsconfig.json && node scripts/write-build-info.mjs", - "typecheck": "tsc -p tsconfig.json --noEmit" + "typecheck": "tsc -p tsconfig.json --noEmit", + "test:fast": "node scripts/test.mjs --suite=fast", + "test:node22": "node scripts/test.mjs --suite=node22", + "test:fedora": "node scripts/test.mjs --suite=fedora", + "typecheck:bench": "tsc --ignoreConfig --noEmit --types node --target ES2022 --module NodeNext --moduleResolution NodeNext --esModuleInterop --skipLibCheck scripts/benchmarks.ts scripts/platform-benchmarks.ts scripts/idle-benchmarks.ts", + "bench:smoke": "node scripts/timing-smoke.mjs", + "verify:fast": "npm run build && npm run test:fast && git diff --check", + "verify": "npm run build && npm test && git diff --check", + "verify:release": "npm run verify && npm run typecheck:bench && npm run bench:smoke" }, "keywords": [ "terminal", diff --git a/scripts/check-process-leaks.sh b/scripts/check-process-leaks.sh new file mode 100644 index 00000000..468185eb --- /dev/null +++ b/scripts/check-process-leaks.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +processes="$(ps -axo pid,ppid,pgid,command)" +if grep -E '[z]sh.*nmsh-semantic|[n]msh-semantic|[c]onfigured-completion\.zsh|[c]apture\.zsh|[v]iewportSyntax' <<< "$processes" | grep -v grep; then + echo "::error::NMSh helper or test processes leaked!" + exit 1 +fi +echo "No processes leaked." diff --git a/scripts/test-selection.mjs b/scripts/test-selection.mjs new file mode 100644 index 00000000..02ed2157 --- /dev/null +++ b/scripts/test-selection.mjs @@ -0,0 +1,82 @@ +import {readdirSync} from 'node:fs'; +import {join} from 'node:path'; + +// Explicit reviewed subsets, never inferred from file names or execution time. +// Fast: pure editor/protocol/presentation/platform logic; no real PTY fixtures. +const suites = { + fast: ['testSharding', 'input', 'keys', 'highlighter', 'shellProtocol', 'sessionProtocol', 'completionModel', 'pathDisplay', 'linuxPlatform'], + // Minimum-runtime gate: actual launcher, persistent shell backends, protocol, + // platform discovery and build/version behavior without long detach scenarios. + node22: ['testSharding', 'buildInfo', 'startupLaunch', 'shellAdapters', 'sessionProtocol', 'shellProtocol', 'linuxPlatform', 'hostProfiles', 'serviceCompat'], + fedora: ['linuxPlatform', 'tools', 'update', 'hostProfiles', 'portabilityUninstall'], +}; + +// Rounded seconds from the macOS Node 26 canonical baseline (2026-10-04). +// Only slow files need weights; new/ordinary files default to one. Greedy +// whole-file scheduling avoids clustering the costly PTY files in one shard. +const runtimeWeights = { + "bundledCatalog.test.ts": 5, + "compatibilityHarness.test.ts": 28, + "configuredCompletion.test.ts": 6, + "ctrlZJobControl.test.ts": 12, + "detachedOutput.test.ts": 22, + "fullChromaQa.test.ts": 5, + "hostProfiles.test.ts": 10, + "interactiveCli.test.ts": 14, + "liveHardening.test.ts": 45, + "liveStatus.test.ts": 5, + "muxInterop.test.ts": 31, + "nativeCaptureLifecycle.test.ts": 6, + "sessionLifecycle.test.ts": 28, + "sessionPresets.test.ts": 20, + "shellAdapters.test.ts": 35, + "shellSwitchApp.test.ts": 23, + "startupBlocked.test.ts": 29, + "startupDiscovery.test.ts": 22, + "startupLaunch.test.ts": 8 +}; + +export function discoverTestFiles() { + return readdirSync('tests', {recursive: true}).filter(name => name.endsWith('.test.ts')).map(name => join('tests', name)).sort(); +} + +export function parseTestOptions(argv) { + const options = {args: []}; + for (const arg of argv) { + if (arg.startsWith('--shard=')) { + if (options.shard) throw new Error('Specify --shard only once'); + const match = /^--shard=([1-9]\d*)\/([1-9]\d*)$/u.exec(arg); + if (!match) throw new Error('Expected --shard=index/count (1-based)'); + const index = Number(match[1]), count = Number(match[2]); + if (!Number.isSafeInteger(count) || !Number.isSafeInteger(index) || index > count) throw new Error('Invalid shard range'); + options.shard = {index, count}; + } else if (arg.startsWith('--suite=')) { + const suite = arg.slice('--suite='.length); + if (options.suite || !Object.hasOwn(suites, suite)) throw new Error('Unknown or repeated test suite'); + options.suite = suite; + } else options.args.push(arg); + } + if (options.shard && options.suite) throw new Error('Curated suites cannot be sharded'); + return options; +} + +export function selectTestGroups(files, {shard, suite} = {}) { + let selected = [...files].sort(); + if (suite) { + selected = suites[suite].map(name => `tests/${name}.test.ts`).sort(); + for (const file of selected) if (!files.includes(file)) throw new Error(`Missing curated test: ${file}`); + } + const ranking = selected.filter(file => file.endsWith('/suggestionRanking.test.ts')); + let runtime = selected.filter(file => !ranking.includes(file)); + if (shard) { + const weight = file => runtimeWeights[file.slice(file.lastIndexOf('/') + 1)] ?? 1; + const buckets = Array.from({length: shard.count}, () => ({files: [], weight: 0})); + for (const file of [...runtime].sort((a, b) => weight(b) - weight(a) || (a < b ? -1 : a > b ? 1 : 0))) { + const bucket = buckets.reduce((best, candidate) => candidate.weight < best.weight ? candidate : best); + bucket.files.push(file); + bucket.weight += weight(file); + } + runtime = buckets[shard.index - 1].files.sort(); + } + return [runtime, !shard || shard.index === 1 ? ranking : []]; +} diff --git a/scripts/test.mjs b/scripts/test.mjs index c7f9b479..2993a0ac 100644 --- a/scripts/test.mjs +++ b/scripts/test.mjs @@ -1,8 +1,9 @@ import {spawn} from 'node:child_process'; import {mkdtempSync, readdirSync, realpathSync, rmSync} from 'node:fs'; -import {tmpdir} from 'node:os'; +import {availableParallelism, tmpdir} from 'node:os'; import {join} from 'node:path'; import {pathToFileURL} from 'node:url'; +import {discoverTestFiles, parseTestOptions, selectTestGroups} from './test-selection.mjs'; /** Short private temp roots keep Unix sockets below their path length limit. */ export async function runTestFiles(files, args = [], {cwd = process.cwd(), stdio = 'inherit', report = message => console.error(message)} = {}) { @@ -14,7 +15,10 @@ export async function runTestFiles(files, args = [], {cwd = process.cwd(), stdio const env = {...process.env, COLORTERM: process.env.COLORTERM ?? 'truecolor', TMPDIR: root, TMP: root, TEMP: root, XDG_CONFIG_HOME: join(root, 'config'), NMSH_DISABLE_UPDATES: '1'}; // A nested runner must not impersonate its parent's test worker. delete env.NODE_TEST_CONTEXT; - const child = spawn(process.execPath, ['--import=tsx', '--test', ...args, ...files], { + // PTY workers also own shell/helper processes. Bound fan-out on larger + // development hosts; explicit Node --test-concurrency options still win. + const concurrency = Math.max(1, Math.min(4, availableParallelism() - 1)); + const child = spawn(process.execPath, ['--import=tsx', '--test', `--test-concurrency=${concurrency}`, ...args, ...files], { cwd, stdio, env, }); let interrupted = false; @@ -43,15 +47,12 @@ export async function runTestFiles(files, args = [], {cwd = process.cwd(), stdio } if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - const files = readdirSync('tests', {recursive: true}).filter(name => name.endsWith('.test.ts')).map(name => join('tests', name)).sort(); - // Preserve the history latency budget without measuring competing PTY/render - // fixtures on shared runners. All ranking tests still run canonically. - const ranking = files.filter(file => file.endsWith('suggestionRanking.test.ts')); - const runtime = files.filter(file => !ranking.includes(file)); try { + const {args, ...selection} = parseTestOptions(process.argv.slice(2)); + const groups = selectTestGroups(discoverTestFiles(), selection); process.exitCode = 0; - for (const group of [runtime, ranking]) { - if (group.length) process.exitCode = Math.max(process.exitCode, (await runTestFiles(group, process.argv.slice(2))).code); + for (const group of groups) { + if (group.length) process.exitCode = Math.max(process.exitCode, (await runTestFiles(group, args)).code); } } catch (error) { console.error(error); process.exitCode = 1; } diff --git a/scripts/timing-smoke.mjs b/scripts/timing-smoke.mjs new file mode 100644 index 00000000..6668bcd2 --- /dev/null +++ b/scripts/timing-smoke.mjs @@ -0,0 +1,13 @@ +import {spawnSync} from 'node:child_process'; + +for (const args of [ + ['--import=tsx', 'scripts/platform-benchmarks.ts'], + ['--import=tsx', 'scripts/benchmarks.ts', 'completion/configured-cold', 'completion/configured-warm', 'composer/screen-plan', 'transcript/wrap-present-10000'], +]) { + const result = spawnSync(process.execPath, args, { + stdio: 'inherit', timeout: 120000, + env: {...process.env, NMSH_BENCH_SAMPLES: '5', NMSH_BENCH_WARMUP: '1'}, + }); + if (result.error) console.error(result.error); + if (result.status !== 0) { process.exitCode = result.status ?? 1; break; } +} diff --git a/tests/testSharding.test.ts b/tests/testSharding.test.ts new file mode 100644 index 00000000..0c9241df --- /dev/null +++ b/tests/testSharding.test.ts @@ -0,0 +1,37 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +// @ts-ignore standalone runner outside product tsconfig +import {discoverTestFiles, selectTestGroups, parseTestOptions} from '../scripts/test-selection.mjs'; + +test('shards are deterministic, disjoint and cover the exact canonical suite', () => { + const files = discoverTestFiles(); + for (const count of [1, 2, 3, 7]) { + const shards = Array.from({length: count}, (_, i) => selectTestGroups(files, {shard: {index: i + 1, count}}).flat()); + assert.deepEqual(shards.flat().sort(), files); + assert.equal(new Set(shards.flat()).size, files.length); + for (let i = 0; i < count; i++) assert.deepEqual(selectTestGroups([...files].reverse(), {shard: {index: i + 1, count}}).flat(), shards[i]); + } +}); +test('ranking runs separately and only in shard one', () => { + const files = discoverTestFiles(); + assert.deepEqual(selectTestGroups(files)[1], ['tests/suggestionRanking.test.ts']); + assert.deepEqual(selectTestGroups(files, {shard: {index: 1, count: 2}})[1], ['tests/suggestionRanking.test.ts']); + assert.deepEqual(selectTestGroups(files, {shard: {index: 2, count: 2}})[1], []); +}); +test('invalid selection fails instead of silently omitting tests', () => { + for (const value of ['0/2', '3/2', '1/0', '1.5/2', '1', 'x/2', '1/2/3']) assert.throws(() => parseTestOptions([`--shard=${value}`])); + assert.throws(() => parseTestOptions(['--shard=1/2', '--shard=2/2'])); + assert.throws(() => parseTestOptions(['--suite=typo'])); + assert.throws(() => parseTestOptions(['--suite=fast', '--shard=1/2'])); + assert.deepEqual(parseTestOptions(['--shard=2/3', '--test-timeout=120000']), {shard: {index: 2, count: 3}, args: ['--test-timeout=120000']}); +}); +test('curated suites exist, contain no duplicates and are proper canonical subsets', () => { + const files = discoverTestFiles(); + for (const suite of ['fast', 'node22', 'fedora']) { + const selected = selectTestGroups(files, {suite}).flat(); + assert.ok(selected.length > 0 && selected.length < files.length); + assert.equal(new Set(selected).size, selected.length); + assert.ok(selected.every((file: string) => files.includes(file))); + assert.throws(() => selectTestGroups(files.filter((file: string) => file !== selected[0]), {suite})); + } +});