diff --git a/.bazelrc b/.bazelrc index cd653bdc43..192f4e3aef 100644 --- a/.bazelrc +++ b/.bazelrc @@ -8,21 +8,12 @@ common --show_progress_rate_limit=15 # The shared fleet CAS uses size-based eviction, so it cannot guarantee # Bazel's default three-hour retention. Revalidate remote outputs before reuse. build --experimental_remote_cache_ttl=0s -# crate_universe re-fetches git dependencies into an empty CARGO_HOME on every -# splice. libgit2 indexes asupersync's ~1 GB history on one thread for over 25 -# minutes; the git CLI indexes it in about 2. -common --repo_env=CARGO_NET_GIT_FETCH_WITH_CLI=true -# Fetch only the pinned commit of each git dependency: 76 MB instead of 1.3 GB, -# and a cold splice drops from about 6 minutes to 1. The splice runs rules_rust's -# stable host cargo, which reads -Z options only under RUSTC_BOOTSTRAP. A nightly -# host cargo would download another 418 MB toolchain on every CI job. -common --repo_env=CARGO_UNSTABLE_GIT=shallow-deps +# Cargo reads the unstable acquisition policy during repository evaluation. common --repo_env=RUSTC_BOOTSTRAP=1 -# MODULE.bazel registers only nightly/2026-10-25. Without this, rules_rust -# resolves the stable channel and the build fails closed. +# MODULE.bazel registers the pinned stable compiler. # Hosts can extend this repository profile with their private fleet settings. build --config=tracedecay -build:tracedecay --@rules_rust//rust/toolchain/channel=nightly +build:tracedecay --@rules_rust//rust/toolchain/channel=stable build --incompatible_enable_cc_toolchain_resolution build --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 build --workspace_status_command=scripts/bazel/workspace_status.sh @@ -72,11 +63,17 @@ build:clippy --@rules_rust//rust/settings:pipelined_compilation=true build:clippy --@rules_rust//rust/settings:clippy_flag=-Dwarnings build:clippy --@rules_rust//rust/settings:clippy.toml=//:clippy.toml +# Default tests and their build-time tools use the assertion-enabled perf levels. +build --compilation_mode=fastbuild +build --host_compilation_mode=fastbuild + # Externals return to opt-level 3 over the perf overrides below. build:release --compilation_mode=opt +build:release --host_compilation_mode=opt build:release --@rules_rust//rust/settings:per_crate_rustc_flag=external/@-Copt-level=3 build:dist --compilation_mode=opt +build:dist --host_compilation_mode=opt build:dist --@rules_rust//rust/settings:per_crate_rustc_flag=external/@-Copt-level=3 build:dist --@rules_rust//rust/settings:lto=thin build:dist --@rules_rust//rust/settings:extra_rustc_flag=-Ccodegen-units=1 diff --git a/.cargo/acquisition.toml b/.cargo/acquisition.toml new file mode 100644 index 0000000000..24fd00740d --- /dev/null +++ b/.cargo/acquisition.toml @@ -0,0 +1,5 @@ +[net] +git-fetch-with-cli = true + +[unstable] +git = "shallow-deps" diff --git a/.cargo/config.toml b/.cargo/config.toml index df8547a14f..4b29f2ad3e 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -101,20 +101,9 @@ linker = "scripts/macos-sign-linker.sh" [alias] # Full Rust test suite. Requires cargo-nextest: # https://nexte.st/docs/installation/pre-built-binaries/ -# Both aliases build with the perf cargo profile, which keeps the -# critical-path kernels (code index, graph, sha2, …) optimized for the -# generation-scale suites; see [profile.perf] in the root Cargo.toml. -# -# `test-ci` is the hosted acceptance selection: nextest's `ci` policy -# (.config/nextest.toml: fail-fast off, one retry that still fails flaky -# results, 8 threads, slow-timeout termination), the root fixture feature and -# `search-eval`, which is what compiles the evaluator acceptance suites (the -# root crate's dependency on the evaluator is optional so the Linux transport -# partition does not link it; a `--workspace` run resolves the evaluator -# anyway). `.github/workflows/ci.yml` runs this -# exact alias, and its support binaries/archives are built with `--profile -# perf` so they share artifacts with it. Extra flags append: `cargo test-ci -# --locked -E 'binary(=x)'`. +# Full-suite aliases share the assertion-enabled perf profile. Hosted CI +# runs Bazel targets generated from the same per-crate optimization levels. +# Extra filters append, for example: cargo test-ci --locked -E 'binary(=x)'. test-ci = "nextest run --workspace --profile ci --features tracedecay/test-helpers,tracedecay/search-eval --cargo-profile perf" # Deliberately broader than CI: every optional feature (test-transport # acceptance suites, …) under the same cargo profile, with the diff --git a/.github/actions/build-cli/action.yml b/.github/actions/build-cli/action.yml index d84ddf0b24..e8790bd152 100644 --- a/.github/actions/build-cli/action.yml +++ b/.github/actions/build-cli/action.yml @@ -9,7 +9,8 @@ inputs: Bazel config. `dist` is the shipping-tuned compile (thin LTO, strip, codegen-units=1). `release` is opt-level 3 without LTO, the CI / cargo `--release` equivalent; tracedecay-cli's default features are - `production`, the release source profile. `ci` is a dev-profile build. + `production`, the release source profile. `ci` uses the assertion-enabled + per-crate optimization levels from Cargo's `perf` profile. required: true dest: description: Directory the binary is copied into, as tracedecay[.exe]. @@ -17,6 +18,9 @@ inputs: cache-key: description: Disk-cache name (see the setup-bazel action). required: true + prepared-build: + description: Whether the caller already configured Bazel for this build. + default: "false" tailscale-client-id: description: Restricted OIDC client for compatible trusted CI builds. default: "" @@ -31,6 +35,7 @@ runs: steps: - uses: ./.github/actions/setup-bazel id: bazel + if: inputs.prepared-build != 'true' with: cache-key: ${{ inputs.cache-key }} tailscale-client-id: ${{ inputs.tailscale-client-id }} diff --git a/.github/actions/setup-bazel/action.yml b/.github/actions/setup-bazel/action.yml index 713bf164f7..b5b43ae6a1 100644 --- a/.github/actions/setup-bazel/action.yml +++ b/.github/actions/setup-bazel/action.yml @@ -27,8 +27,8 @@ inputs: outputs: fleet-enabled: - description: Whether this job uses private workers and dynamic execution. - value: ${{ inputs.tailscale-client-id != '' }} + description: Whether Tailscale enrollment and executor connectivity enabled the fleet. + value: ${{ steps.fleet.outputs.enabled }} disk-cache-path: description: Action-cache directory to save after the caller's build. value: ${{ steps.disk-cache-config.outputs.path }} @@ -88,17 +88,6 @@ runs: # D:/_bazel) keeps MSVC's header paths under MAX_PATH. output-base: ${{ runner.os == 'Windows' && 'D:/b' || '' }} - # Bazel validates action contents itself. A fresh save key retains new - # source-build outputs even when MODULE.bazel and BUILD files are unchanged. - - name: Restore master action cache - id: disk-cache - if: inputs.tailscale-client-id == '' - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ steps.disk-cache-config.outputs.path }} - key: ${{ steps.disk-cache-config.outputs.prefix }}run-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} - restore-keys: ${{ steps.disk-cache-config.outputs.restore-keys }} - - name: Preserve the runner network environment for Tailscale if: runner.os == 'Linux' && inputs.tailscale-client-id != '' shell: bash @@ -118,7 +107,10 @@ runs: sudo visudo -cf /etc/sudoers.d/tailscale-ci-network - name: Connect to the private Bazel fleet + id: tailscale if: inputs.tailscale-client-id != '' + # Private infrastructure is optional; compilation and test failures are not. + continue-on-error: true uses: tailscale/github-action@99a8333667fff04fdd6cffb46dfd409f8329a7bf # v4 with: oauth-client-id: ${{ inputs.tailscale-client-id }} @@ -126,49 +118,28 @@ runs: tags: tag:bazel-ci ping: ubuntu-main - - name: Share compilation and tests with private workers - if: inputs.tailscale-client-id != '' + - name: Select private executor or hosted execution + id: fleet shell: bash + env: + TAILSCALE_OUTCOME: ${{ steps.tailscale.outcome }} run: | - case "$RUNNER_OS/$RUNNER_ARCH" in - Linux/X64) - profile=rbe-linux - mkdir -p /tmp/zig-cache - ;; - macOS/ARM64) profile=rbe-macos ;; - *) - echo "No compatible fleet worker for $RUNNER_OS/$RUNNER_ARCH" >&2 - exit 1 - ;; - esac - # Use the checkout's existing local rc import for fleet settings. - # Tests carry their tools in runfiles, so both branches use identical inputs. - printf 'build --config=%s\n' "$profile" > .bazelrc.local - cat >> .bazelrc.local <<'RC' - build --remote_executor=grpc://ubuntu-main:50051 - build --remote_cache=grpc://ubuntu-main:50051 - build --remote_instance_name=main - build --remote_timeout=600 - # Survive brief coordinator restarts; five retries expire in about 3s. - build --remote_retries=10 - # Fleet compilation must not fall back onto the smaller hosted VM. - # Dynamic tests still have their explicitly scheduled local branch. - build --noremote_local_fallback - build --remote_download_outputs=toplevel - build --jobs=40 - build --local_resources=cpu=HOST_CPUS - build --local_resources=memory=HOST_RAM*.5 - build --internal_spawn_scheduler - build --dynamic_local_strategy=sandboxed - build --dynamic_remote_strategy=remote - # Large Rust links raced on the hosted VM while the same actions ran - # remotely. local_resources only schedules estimates; it does not cap - # rustc memory. Keep compilation on the fleet and tests hybrid. - build --strategy=Rustc=remote - build --strategy=TestRunner=dynamic - build --bes_backend=grpc://ubuntu-main:1985 - build --bes_results_url=https://ubuntu-main.tail039078.ts.net/invocation/ - build --bes_timeout=30s - build --bes_upload_mode=fully_async - RC + python3 "$GITHUB_ACTION_PATH/configure_fleet.py" + - name: Prepare the Linux fleet toolchain cache + if: steps.fleet.outputs.enabled == 'true' && runner.os == 'Linux' + shell: bash + run: mkdir -p /tmp/zig-cache + + # Bazel validates action contents itself. A fresh save key retains new + # source-build outputs even when MODULE.bazel and BUILD files are unchanged. + # Restore after enrollment so an offline private executor gets the same + # hosted cache as a job without a fleet identity. + - name: Restore master action cache + id: disk-cache + if: steps.fleet.outputs.enabled != 'true' + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ steps.disk-cache-config.outputs.path }} + key: ${{ steps.disk-cache-config.outputs.prefix }}run-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} + restore-keys: ${{ steps.disk-cache-config.outputs.restore-keys }} diff --git a/.github/actions/setup-bazel/configure_fleet.py b/.github/actions/setup-bazel/configure_fleet.py new file mode 100644 index 0000000000..3b67a24865 --- /dev/null +++ b/.github/actions/setup-bazel/configure_fleet.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Select the private executor for the invoking CI checkout when reachable.""" + +from __future__ import annotations + +import os +import socket +from pathlib import Path + + +def configure_fleet( + workspace: Path, + output: Path, + connection_outcome: str, + runner_os: str, + runner_arch: str, + host: str = "ubuntu-main", + port: int = 50051, +) -> bool: + profile = {("Linux", "X64"): "rbe-linux", ("macOS", "ARM64"): "rbe-macos"}.get( + (runner_os, runner_arch) + ) + enabled = False + if connection_outcome == "success" and profile: + try: + # This checks the coordinator, not the number or capacity of workers. + with socket.create_connection((host, port), timeout=5): + pass + enabled = True + except OSError as error: + print( + f"::warning::Private Bazel executor is unreachable: {error}; " + "using hosted execution." + ) + elif connection_outcome == "failure": + print("::warning::Private Bazel enrollment failed; using hosted execution.") + elif connection_outcome == "success": + print( + f"::warning::No fleet profile for {runner_os}/{runner_arch}; " + "using hosted execution." + ) + + if enabled: + rc = f"""# Generated by CI setup; workstation configuration is unchanged. +build --config={profile} +build --remote_executor=grpc://{host}:{port} +build --remote_cache=grpc://{host}:{port} +build --remote_instance_name=main +build --remote_timeout=600 +# Survive brief coordinator restarts; five retries expire in about 3s. +build --remote_retries=10 +# Fleet compilation must not fall back onto the smaller hosted VM. +# Dynamic tests still have their explicitly scheduled local branch. +build --noremote_local_fallback +build --remote_download_outputs=toplevel +build --jobs=40 +build --local_resources=cpu=HOST_CPUS +build --local_resources=memory=HOST_RAM*.5 +build --internal_spawn_scheduler +build --dynamic_local_strategy=sandboxed +build --dynamic_remote_strategy=remote +# asupersync and other Rust crates use the fleet first. Do not race large +# compilations on the hosted VM; local_resources does not cap rustc memory. +build --strategy=Rustc=remote +# Tag `local` requires the local strategy; dynamic-only cannot spawn those tests. +build --strategy=TestRunner=dynamic,local +build --bes_backend=grpc://ubuntu-main:1985 +build --bes_results_url=https://ubuntu-main.tail039078.ts.net/invocation/ +build --bes_timeout=30s +build --bes_upload_mode=fully_async +""" + else: + rc = "# Private executor unavailable or not configured; use hosted execution.\n" + + # This action owns the CI checkout's local rc. Clear a previous fleet + # selection on retry, before advertising that hosted execution is selected. + (workspace / ".bazelrc.local").write_text(rc) + with output.open("a") as handle: + handle.write(f"enabled={str(enabled).lower()}\n") + return enabled + + +if __name__ == "__main__": + configure_fleet( + Path.cwd(), + Path(os.environ["GITHUB_OUTPUT"]), + os.environ.get("TAILSCALE_OUTCOME", ""), + os.environ["RUNNER_OS"], + os.environ["RUNNER_ARCH"], + ) diff --git a/.github/workflows/blaze.yml b/.github/workflows/blaze.yml new file mode 100644 index 0000000000..23d3195c3a --- /dev/null +++ b/.github/workflows/blaze.yml @@ -0,0 +1,307 @@ +name: Blaze + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + push: + branches: [master] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: blaze-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} + +jobs: + workspace: + name: Blaze workspace (Linux x86_64) + runs-on: ubuntu-24.04 + timeout-minutes: 90 + env: + RUSTUP_TOOLCHAIN: 1.99.0 + BLAZE_REVISION: d44ef985e45e69661fb44696d61a18a251a39b08 + BLAZE_TESTS: all + BLAZE_CACHE: ${{ github.workspace }}/target/blaze/store + NATIVE_BLAZE_BIN_DIR: ${{ github.workspace }}/target/blaze/tool/bin + CARGO_TARGET_DIR: ${{ github.workspace }}/target/blaze + CARGO_TERM_COLOR: always + # Bound concurrent cold rustc jobs; the launcher also disables spare warming. + CARGO_BUILD_JOBS: '2' + WORKSPACE_FEATURES: tracedecay/test-helpers,tracedecay/test-transport,tracedecay/search-eval,tracedecay-cli/test-transport,tracedecay-query/production + steps: + # Search-eval fixtures resolve historical source commits locally. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: '26' + + - uses: ./.github/actions/setup-pnpm + + - name: Install native build prerequisites + run: | + sudo apt-get update + sudo apt-get install --yes build-essential clang pkg-config libssl-dev + rustup toolchain install 1.99.0 --profile minimal \ + --component rust-src --component rust-analyzer --component rustfmt --component clippy + rustc --version --verbose + + - name: Resolve cache identity and persistent paths + id: cache-config + env: + CONFIG_HASH: ${{ hashFiles('rust-toolchain.toml', '.cargo/config.toml', 'Cargo.toml', 'crates/*/Cargo.toml', 'MODULE.bazel', '.bazelrc', 'scripts/bazel/blaze/**', '.github/workflows/blaze.yml', 'pnpm-lock.yaml') }} + LOCK_HASH: ${{ hashFiles('Cargo.lock') }} + BOOTSTRAP_HASH: ${{ hashFiles('scripts/bazel/blaze/repositories.bzl', 'MODULE.bazel', 'MODULE.bazel.lock', '.bazelversion', '.bazelrc', 'rust-toolchain.toml', '.github/workflows/blaze.yml') }} + run: | + set -euo pipefail + # runner.temp is unavailable in job-level env. Resolve it on the runner + # and persist these paths for subsequent build and cache steps. + export BLAZE_HOME="$RUNNER_TEMP/blaze-runtime" + export CARGO_HOME="$RUNNER_TEMP/blaze-cargo" + printf 'BLAZE_HOME=%s\nCARGO_HOME=%s\n' "$BLAZE_HOME" "$CARGO_HOME" >> "$GITHUB_ENV" + mkdir -p "$BLAZE_HOME" "$BLAZE_CACHE" "$CARGO_HOME" + chmod 700 "$BLAZE_HOME" + { + echo "prefix=blaze-1-${RUNNER_OS}-${RUNNER_ARCH}-rust-1.99.0-${BLAZE_REVISION}-perf-workspace-${CONFIG_HASH}" + echo "lock=${LOCK_HASH}" + echo "source=$(git rev-parse HEAD)-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + echo "tool-key=blaze-native-1-${RUNNER_OS}-${RUNNER_ARCH}-ubuntu-24.04-rust-1.99.0-${BLAZE_REVISION}-${BOOTSTRAP_HASH}" + echo 'store-paths<> "$GITHUB_OUTPUT" + + # GitHub first searches this ref (refs/pull/N/merge on PRs), then its + # base/default branch. PR saves cannot be read by master or sibling PRs. + # The lock-specific prefix is preferred; source SHAs occur only in saves. + - name: Restore Blaze code store + id: store + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ steps.cache-config.outputs.store-paths }} + key: ${{ steps.cache-config.outputs.prefix }}-store-${{ steps.cache-config.outputs.lock }}-${{ steps.cache-config.outputs.source }} + restore-keys: | + ${{ steps.cache-config.outputs.prefix }}-store-${{ steps.cache-config.outputs.lock }}- + ${{ steps.cache-config.outputs.prefix }}-store- + + - name: Restore compiled workspace artifacts + id: artifacts + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ steps.cache-config.outputs.artifact-paths }} + key: ${{ steps.cache-config.outputs.prefix }}-artifacts-${{ steps.cache-config.outputs.lock }}-${{ steps.cache-config.outputs.source }} + restore-keys: | + ${{ steps.cache-config.outputs.prefix }}-artifacts-${{ steps.cache-config.outputs.lock }}- + ${{ steps.cache-config.outputs.prefix }}-artifacts- + + - name: Restore Cargo dependency acquisition + id: acquisition + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ steps.cache-config.outputs.acquisition-paths }} + key: ${{ steps.cache-config.outputs.prefix }}-acquisition-${{ steps.cache-config.outputs.lock }}-${{ steps.cache-config.outputs.source }} + restore-keys: | + ${{ steps.cache-config.outputs.prefix }}-acquisition-${{ steps.cache-config.outputs.lock }}- + ${{ steps.cache-config.outputs.prefix }}-acquisition- + + # No fleet credentials or remote executors. This action cache covers + # the launcher, not compilation performed inside repository rules. + # Disable automatic post-job saves on failed runs. + - uses: ./.github/actions/setup-bazel + id: bazel + with: + cache-key: ${{ steps.cache-config.outputs.prefix }}-launcher + cache-save: 'false' + + # Executables produced by a repository rule are not Bazel action-cache + # entries. Reuse only this exact source/toolchain/bootstrap configuration. + - name: Restore pinned native Blaze binaries + id: tool + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ env.NATIVE_BLAZE_BIN_DIR }} + key: ${{ steps.cache-config.outputs.tool-key }} + + - name: Provision pinned native Blaze binaries on cache miss + if: steps.tool.outputs.cache-hit != 'true' + run: | + set -euo pipefail + bazel build @blaze//:runtime + bazel cquery @blaze//:runtime --output=files > "$RUNNER_TEMP/blaze-runtime-files.txt" + BLAZE_EXECUTION_ROOT="$(bazel info execution_root)" + export BLAZE_EXECUTION_ROOT + python3 - <<'PY' + import os + from pathlib import Path + import shutil + + expected = {'blaze', 'ra-jit', 'ra-jit-rustc', 'ra-jit-rustc-first'} + execution_root = Path(os.environ['BLAZE_EXECUTION_ROOT']) + listing = Path(os.environ['RUNNER_TEMP']) / 'blaze-runtime-files.txt' + declared = {} + for line in listing.read_text().splitlines(): + artifact = Path(line) + if artifact.name not in expected or artifact.name in declared: + raise SystemExit(f'Unexpected native runtime artifact: {line}') + source = artifact if artifact.is_absolute() else execution_root / artifact + declared[artifact.name] = source.resolve(strict=True) + if declared.keys() != expected: + raise SystemExit(f'Incomplete native runtime artifacts: {sorted(declared)}') + destination = Path(os.environ['NATIVE_BLAZE_BIN_DIR']) + destination.mkdir(parents=True, exist_ok=True) + for name, source in declared.items(): + shutil.copy2(source, destination / name) + PY + + - name: Validate the cached native tool layout + id: runtime + run: | + python3 - <<'PY' + import os + from pathlib import Path + + directory = Path(os.environ['NATIVE_BLAZE_BIN_DIR']) + expected = {'blaze', 'ra-jit', 'ra-jit-rustc', 'ra-jit-rustc-first'} + if {path.name for path in directory.iterdir()} != expected: + raise SystemExit(f'Unexpected native tool cache layout: {directory}') + for name in expected: + binary = directory / name + if binary.is_symlink() or not binary.is_file() or not os.access(binary, os.X_OK): + raise SystemExit(f'Native tool cache entry is not a portable executable file: {binary}') + PY + + - name: Build workspace binaries and host fixture with Blaze + id: build + run: | + bazel run --repo_env=TRACEDECAY_BLAZE_BIN_DIR="$NATIVE_BLAZE_BIN_DIR" \ + //scripts/bazel/blaze:blaze -- build \ + --workspace --bins --locked --profile perf --features "$WORKSPACE_FEATURES" + bazel run --repo_env=TRACEDECAY_BLAZE_BIN_DIR="$NATIVE_BLAZE_BIN_DIR" \ + //scripts/bazel/blaze:blaze -- build \ + --package tracedecay-cli --example tracedecay-host-cli-fixture \ + --locked --profile perf --features "$WORKSPACE_FEATURES" + + - name: Run the full workspace test selection with Blaze + id: test + run: | + bazel run --repo_env=TRACEDECAY_BLAZE_BIN_DIR="$NATIVE_BLAZE_BIN_DIR" \ + //scripts/bazel/blaze:blaze -- test \ + --workspace --locked --profile perf --no-fail-fast --features "$WORKSPACE_FEATURES" + + - name: Report native engine and compiler fallback status + id: status + if: always() && steps.build.outcome != 'skipped' + run: | + set -euo pipefail + bazel run --repo_env=TRACEDECAY_BLAZE_BIN_DIR="$NATIVE_BLAZE_BIN_DIR" \ + //scripts/bazel/blaze:blaze -- status 2>&1 | tee "$RUNNER_TEMP/blaze-status.txt" + + - name: Stop the daemon before saving persistent artifacts + if: always() && steps.runtime.outcome == 'success' + run: >- + bazel run --repo_env=TRACEDECAY_BLAZE_BIN_DIR="$NATIVE_BLAZE_BIN_DIR" + //scripts/bazel/blaze:blaze -- stop + + # All saves require successful build, tests, status, and shutdown. A + # restore never bypasses compilation or test execution, even on exact hits. + - name: Save validated pinned native Blaze binaries + id: save-tool + if: success() && steps.tool.outputs.cache-hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ env.NATIVE_BLAZE_BIN_DIR }} + key: ${{ steps.tool.outputs.cache-primary-key }} + + - name: Save validated Blaze code store + id: save-store + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ steps.cache-config.outputs.store-paths }} + key: ${{ steps.store.outputs.cache-primary-key }} + + - name: Save validated compiled workspace artifacts + id: save-artifacts + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ steps.cache-config.outputs.artifact-paths }} + key: ${{ steps.artifacts.outputs.cache-primary-key }} + + - name: Save validated dependency acquisition + id: save-acquisition + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ steps.cache-config.outputs.acquisition-paths }} + key: ${{ steps.acquisition.outputs.cache-primary-key }} + + - name: Save validated Bazel launcher actions + id: save-launcher + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ steps.bazel.outputs.disk-cache-path }} + key: ${{ steps.bazel.outputs.disk-cache-key }}-${{ github.sha }} + + - name: Summarize cache reuse and workspace validation + if: always() + env: + BUILD_OUTCOME: ${{ steps.build.outcome }} + TEST_OUTCOME: ${{ steps.test.outcome }} + TOOL_REQUESTED: ${{ steps.tool.outputs.cache-primary-key }} + TOOL_RESTORED: ${{ steps.tool.outputs.cache-matched-key }} + TOOL_SAVED: ${{ steps.save-tool.outcome }} + STORE_REQUESTED: ${{ steps.store.outputs.cache-primary-key }} + STORE_RESTORED: ${{ steps.store.outputs.cache-matched-key }} + STORE_SAVED: ${{ steps.save-store.outcome }} + ARTIFACTS_REQUESTED: ${{ steps.artifacts.outputs.cache-primary-key }} + ARTIFACTS_RESTORED: ${{ steps.artifacts.outputs.cache-matched-key }} + ARTIFACTS_SAVED: ${{ steps.save-artifacts.outcome }} + ACQUISITION_REQUESTED: ${{ steps.acquisition.outputs.cache-primary-key }} + ACQUISITION_RESTORED: ${{ steps.acquisition.outputs.cache-matched-key }} + ACQUISITION_SAVED: ${{ steps.save-acquisition.outcome }} + LAUNCHER_SAVED: ${{ steps.save-launcher.outcome }} + run: | + python3 - <<'PY' + import os + from pathlib import Path + + with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as summary: + summary.write(f"Build: {os.environ['BUILD_OUTCOME']}; tests: {os.environ['TEST_OUTCOME']}.\n\n") + summary.write('Every restored cache still runs the workspace build and tests with `BLAZE_TESTS=all`.\n\n') + summary.write('| Cache | Requested save key | Restored key | Save step |\n| --- | --- | --- | --- |\n') + for name in ('TOOL', 'STORE', 'ARTIFACTS', 'ACQUISITION'): + requested = os.environ[f'{name}_REQUESTED'] or '(not reached)' + restored = os.environ[f'{name}_RESTORED'] or '(none)' + saved = os.environ[f'{name}_SAVED'] + summary.write(f'| {name.lower()} | `{requested}` | `{restored}` | {saved} |\n') + summary.write(f"\nBazel launcher action cache save step: {os.environ['LAUNCHER_SAVED']}.\n") + summary.write('\nCaches save in the current GitHub ref after successful validation; master pushes prime master.\n\n') + summary.write('Compiler fallback is workspace validation, not proof that Blaze compiled a program. ' + 'The command logs report fallback reasons; daemon status follows.\n\n') + status = Path(os.environ['RUNNER_TEMP']) / 'blaze-status.txt' + if status.is_file(): + summary.write('```text\n' + status.read_text() + '\n```\n') + PY diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c80a229eb..196f98a411 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -149,9 +149,6 @@ jobs: - name: Test Bazel workspace provenance run: python3 scripts/test-bazel-workspace-status.py - - name: Test Bazel default-feature alias collapse - run: python3 scripts/test-gen-builds-default-alias.py - - name: Test the real-repository hot-path benchmark harness run: python3 scripts/test-bench-hot-paths.py @@ -173,6 +170,7 @@ jobs: //scripts:ambient_tokio_check_test //scripts:rusqlite_storage_boundary_test //scripts:packaged_lsp_bridge_test + //scripts:configure_ci_fleet_test - name: Test the ranked-tail trim replay against the Rust rule cases run: bazel run //scripts:test-measure-ranked-tail-trim.py @@ -195,7 +193,7 @@ jobs: python3 scripts/test-check-release-artifacts.py python3 scripts/test-generate-release-inventory.py python3 scripts/test-update-release-checksums.py - python3 scripts/test-neutralize-release-pr-closing-keywords.py + bazel test --config=ci --test_output=errors //scripts:release_pr_refresh_test bash tests/macos_stable_codesign_test.sh # Exercise the release guard scripts when admitting proposed changes. @@ -214,8 +212,8 @@ jobs: # Manifest edits that skip a gen_builds.py rerun leave stale BUILD # files; compare the committed files against a fresh generation. - - name: Check generated Bazel BUILD files are current - run: bazel run //scripts:bazel/gen_builds.py -- --check + - name: Check generated Bazel BUILD files and production feature isolation + run: bazel run //scripts:gen_builds -- --check - name: Test and check dev skill host copies run: | @@ -542,8 +540,13 @@ jobs: # Matrix shards share github.job, so they would race the same save key. # One writer keeps the 10 GiB Actions cache from holding several copies # that would evict this prefix (observed: 6.9 GiB workspace-only save). + # An offline fleet matrix has one job, which becomes that writer. - name: Save compiled Bazel actions - if: always() && !cancelled() && github.ref == 'refs/heads/master' && matrix.shard == 'root-b' && steps.bazel.outputs.disk-cache-key != '' + if: >- + always() && !cancelled() && github.ref == 'refs/heads/master' + && (matrix.shard == 'root-b' + || (matrix.shard == 'fleet' && steps.bazel.outputs.fleet-enabled != 'true')) + && steps.bazel.outputs.disk-cache-key != '' uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ steps.bazel.outputs.disk-cache-path }} @@ -588,7 +591,8 @@ jobs: - uses: ./.github/actions/setup-bazel id: bazel with: - # PDB-free actions must not restore the former oversized cache. + # The previous PDB-producing cache restored an 11 GiB archive. + # Start a new generation for the changed compile/link configuration. cache-key: ci-windows-test-v2 - name: Build and type-check the Windows workspace @@ -716,10 +720,7 @@ jobs: path: ${{ steps.bazel.outputs.disk-cache-path }} key: ${{ steps.bazel.outputs.disk-cache-key }} - # The production feature graph must exclude test-only surfaces. The - # feature-gated variants themselves (every `test-transport` library and the - # lean `--no-default-features` root, emitted by scripts/bazel/gen_builds.py) - # are ordinary `//...` targets that the Clippy job compiles. + # Feature gates remains a required check and exercises the parser and policy. feature-gates: name: Feature gates if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }} @@ -728,10 +729,12 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - # One graph walk per push. The release resolver repeats it on the tag it - # builds, since a release is dispatched without waiting on this run. - - name: Check the production feature graph - run: python3 scripts/check-production-feature-profile.py + - uses: ./.github/actions/setup-bazel + with: + cache-key: ci-feature-gates + + - name: Test generator feature policy and Cargo tree parsing + run: bazel test --config=ci --test_output=errors //scripts:gen_builds_test # The configuration users actually receive: the release build of # `tracedecay-cli`, whose default features are `production`. Nothing else on diff --git a/.github/workflows/release-beta.yml b/.github/workflows/release-beta.yml index f37ce60cb5..bd8961037c 100644 --- a/.github/workflows/release-beta.yml +++ b/.github/workflows/release-beta.yml @@ -154,13 +154,13 @@ jobs: persist-credentials: false sparse-checkout: | .github/actions/package-release + .github/actions/build-cli .github/actions/setup-bazel scripts/generate-release-inventory.py sparse-checkout-cone-mode: false - # Fleet setup comes from this workflow revision even when rebuilding an - # older tag whose build-cli action predates macOS remote execution. - name: Set up Bazel for release builds + id: bazel uses: ./.release-automation/.github/actions/setup-bazel with: cache-key: release-${{ matrix.target }} @@ -178,11 +178,12 @@ jobs: - name: Build release binary for packaging if: matrix.inventory_only != true id: cli - uses: ./.github/actions/build-cli + uses: ./.release-automation/.github/actions/build-cli with: config: dist dest: target/${{ matrix.target }}/release cache-key: release-${{ matrix.target }} + prepared-build: 'true' - name: Package and verify native archive and MCPB if: matrix.inventory_only != true @@ -232,6 +233,13 @@ jobs: path: tracedecay-beta-${{ env.RELEASE_TAG }}-${{ matrix.name }}.${{ matrix.archive }} if-no-files-found: error + - name: Save compiled Bazel actions + if: always() && !cancelled() && github.ref == 'refs/heads/master' && matrix.inventory_only != true && steps.bazel.outputs.disk-cache-key != '' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ steps.bazel.outputs.disk-cache-path }} + key: ${{ steps.bazel.outputs.disk-cache-key }} + publish: needs: [validate, build] # Publish only a whole release. beta.86 shipped without macOS because a diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ccc3143471..775290214b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -155,13 +155,13 @@ jobs: persist-credentials: false sparse-checkout: | .github/actions/package-release + .github/actions/build-cli .github/actions/setup-bazel scripts/generate-release-inventory.py sparse-checkout-cone-mode: false - # Fleet setup comes from this workflow revision even when rebuilding an - # older tag whose build-cli action predates macOS remote execution. - name: Set up Bazel for release builds + id: bazel uses: ./.release-automation/.github/actions/setup-bazel with: cache-key: release-${{ matrix.target }} @@ -180,11 +180,12 @@ jobs: - name: Build release binary for packaging if: matrix.inventory_only != true id: cli - uses: ./.github/actions/build-cli + uses: ./.release-automation/.github/actions/build-cli with: config: dist dest: target/${{ matrix.target }}/release cache-key: release-${{ matrix.target }} + prepared-build: 'true' # verify-release requires every target to embed the same bundle. - name: Record the embedded dashboard digest @@ -253,6 +254,13 @@ jobs: path: tracedecay-${{ needs.validate-release.outputs.tag }}-${{ matrix.name }}.${{ matrix.archive }} if-no-files-found: error + - name: Save compiled Bazel actions + if: always() && !cancelled() && github.ref == 'refs/heads/master' && matrix.inventory_only != true && steps.bazel.outputs.disk-cache-key != '' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ steps.bazel.outputs.disk-cache-path }} + key: ${{ steps.bazel.outputs.disk-cache-key }} + verify-release: name: Verify immutable GitHub release needs: [validate-release, build] diff --git a/AGENTS.md b/AGENTS.md index b7afcf7714..1b9c384712 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -94,7 +94,7 @@ unauthorized external action after completing independent, authorized work. resolves Rust dependencies from the workspace manifests and `Cargo.lock`. - After changing Rust dependencies, run `CARGO_BAZEL_REPIN=workspace bazel fetch --repo=@c`. - Then run `bazel run //scripts:bazel/gen_builds.py`. + Then run `bazel run //scripts:gen_builds`. Use `CARGO_BAZEL_REPIN=` for a targeted dependency update. Commit the refreshed lockfile and generated targets together. - Code generation: `bazel run //sdks/codegen:generate -- ` diff --git a/BUILD.bazel b/BUILD.bazel index ac204e0138..5b250aaac5 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -11,6 +11,8 @@ npm_link_all_packages(name = "node_modules") exports_files( [ + ".cargo/acquisition.toml", + ".github/actions/setup-bazel/configure_fleet.py", ".github/release-targets.json", ".npmrc", "MODULE.bazel", @@ -18,6 +20,7 @@ exports_files( "Cargo.toml", "clippy.toml", "rustfmt.toml", + "rust-toolchain.toml", "pnpm-lock.yaml", ], visibility = ["//visibility:public"], diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 78f51249e5..7e47a568c0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -26,11 +26,24 @@ lockfile and generated Bazel targets: ```bash CARGO_BAZEL_REPIN=workspace bazel fetch --repo=@c -bazel run //scripts:bazel/gen_builds.py +bazel run //scripts:gen_builds ``` For a targeted dependency update, use `CARGO_BAZEL_REPIN=` instead. Review and commit `Cargo.lock` and any generated target changes together. +Use `bazel run //scripts:gen_builds -- --check` to compare generated targets +without writing. The generator uses Bazel-declared Cargo and rustc matching +`rust-toolchain.toml`. Git and network credentials remain host prerequisites. + +`.cargo/acquisition.toml` configures Git acquisition for the generator +and crate_universe. The generator passes it through Cargo's `--config`, which +takes precedence over environment configuration. crate_universe copies it +into its isolated splice, where environment configuration can override it. +The splice receives only this acquisition file. The generator also discovers +the checkout's `.cargo/config.toml` normally. Ordinary validation leaves +`CARGO_BAZEL_REPIN` and `REPIN` unset. Investigate unexpected lockfile changes +before repinning. + Dashboard `contracts:generate` and `contracts:check` run the `//sdks/codegen:generate` and `//sdks/codegen:dashboard_schema` Bazel targets. diff --git a/Cargo.toml b/Cargo.toml index 2638e7ff20..a72a3c1b15 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,9 +67,8 @@ edition = "2024" version = "1.0.0-beta.92" [workspace.dependencies] -# nightly-outcome-try is an asupersync default (Outcome's Try impls). It -# compiles only on the pinned nightly. native-runtime stays explicit because -# it is not part of that default set. +# asupersync probes support for Outcome's Try implementations and leaves them +# inactive on stable. native-runtime remains explicit because defaults are off. # Switch back to `version = "=0.6.0"` once 0.6.0 is published to crates.io (#3874). asupersync = { git = "https://github.com/Dicklesworthstone/asupersync", rev = "432e2c3b2cf2bad02fc88e380d29a6b0b5c01422", default-features = false, features = ["proc-macros", "native-runtime", "nightly-outcome-try"] } # grafeo 0.5.43 from our fork (ScriptedAlchemy/grafeo, main). The @@ -167,16 +166,15 @@ strip = "symbols" # debug=1 against 45.7 MiB without debuginfo, which was blowing the shared # build-cache budget and turning evictions into repeat 5-minute misses. # -# `test` (built-in, inherits `dev`) and `perf` (below) inherit every setting -# and package override declared here, so per-package overrides live only on -# `dev`. +# Ordinary development and focused tests remain unoptimized. Full CI uses +# the assertion-enabled perf profile so production-scale fixtures exercise +# the same bounded readiness contracts as the optimized product. [profile.dev] debug = "line-tables-only" incremental = true -opt-level = 1 +opt-level = 0 -# Focused root unit tests retain test assertions while keeping rustc's -# code-generation working set bounded on developer hosts. +# Focused root unit tests retain assertions with a bounded codegen working set. [profile.bounded-test] inherits = "test" debug = 0 @@ -187,44 +185,29 @@ codegen-units = 1 [profile.dev.package."*"] debug = 0 -# Wire-protocol crate: opt-level=1 + thin-local-LTO spends ~86s LLVM on a -# 130MiB rlib / 2.2M instructions of serde+DTO glue (measured). This is not a -# kernel; keep it unoptimized so rmeta/rlib stay cheap for the root crate. -[profile.dev.package.tracedecay-daemon-protocol] +[profile.perf] +inherits = "dev" +opt-level = 1 +debug-assertions = true +overflow-checks = true + +# Serde/DTO glue spends most compilation time in LLVM and does not benefit +# from the kernel optimization used by the generation-scale suites. +[profile.perf.package.tracedecay-daemon-protocol] opt-level = 0 -# Same shape as the protocol crate: transport-neutral DTOs whose serde and -# schemars glue is almost all LLVM time. Measured on a workspace-cold -# `tracedecay-cli` build: ~205s at opt-level 1 against ~79s at opt-level 0, -# ~40s off wall time since the crate is on the critical path. -[profile.dev.package.tracedecay-contracts] +[profile.perf.package.tracedecay-contracts] opt-level = 0 -# The first token count pays a one-time BPE model load (embedded-vocabulary -# base64 decode + BPE regex compile) that costs seconds in unoptimized -# builds; the dashboard token-count warm task and its tests hit it on every -# test-process start, and `context_scout_model` gives its proposal one second -# in total. Optimize the loader wherever tests run, not only under `perf`. -[profile.dev.package.tiktoken-rs] +# A cold tokenizer loads its embedded vocabulary and compiles BPE regexes. +# At opt-level 0 that startup alone exceeds the proposal journey's budget. +[profile.perf.package.tiktoken-rs] opt-level = 2 -[profile.dev.package.regex] +[profile.perf.package.regex] opt-level = 2 -# The full-suite profile. Ordinary `cargo build` / `cargo test` (dev/test) -# optimize no specific workspace package so edit-compile loops stay cheap; -# this profile inherits dev and additionally keeps the critical-path kernels -# below optimized so the generation-scale code-index and graph-publication -# suites run at production-like speed. Hosted CI (`.github/workflows/ci.yml`) -# and the local full-suite aliases (`cargo test-ci` / `cargo test-all` in -# .cargo/config.toml) all build support binaries, archives and tests with -# this one profile, so they share artifacts and assertions stay on. -[profile.perf] -inherits = "dev" - -# Code-index lifecycle tests exercise six-figure production generations under -# the same bounded readiness contracts as the daemon. Keep the owning kernels -# optimized without hiding assertions or optimizing unrelated workspace code. +# Code-index and graph-publication suites build production-scale generations. [profile.perf.package.tracedecay-code-extraction] opt-level = 2 @@ -267,9 +250,8 @@ opt-level = 2 [profile.perf.package.serde_json] opt-level = 2 -# Graph publication digest checks hash multi-megabyte canonical replay -# sources on every durable read/write; unoptimized sha2 alone exceeds the -# exact-SQL test transaction budget. +# Canonical replay integrity hashes multi-megabyte sources inside the exact +# SQL transaction budget; unoptimized hashing exceeds that budget. [profile.perf.package.sha2] opt-level = 3 @@ -293,255 +275,3 @@ tree-sitter-perl = { git = "https://github.com/ganezdragon/tree-sitter-perl", re # copied byte for byte from the published crate. Drop it once a release # ships #368 (DerekStride/tree-sitter-sql#384). tree-sitter-sequel = { git = "https://github.com/ScriptedAlchemy/tree-sitter-sql", rev = "52fb886f900022d459df9f87cecb3d5033ce7268" } - -# Dev opt-level 0 for workspace glue crates (every tracedecay-* crate that is -# not a profile.perf kernel below). A `cargo build -p tracedecay-cli --timings` -# audit on 8 cores found the wall-time critical path is a serial chain of -# workspace crates whose cost is ~85% LLVM codegen at dev opt-level 1: -# rusqlite-runtime (96s codegen), tracedecay (69s), daemon-service (49s), -# mcp (35s), application (142s, off-path but fan-out). Compiling dispatch, -# DTO, and plumbing crates unoptimized cut a cold build from 461s to 345s -# wall (-25%) and workspace CPU by ~45% (measured). The perf-pinned kernels -# (code-index, rusqlite-runtime, store, grafeo-*, ...) keep dev opt-level 1 so -# local tests keep optimized engine code; the perf mirror pins preserve every -# crate's previous perf level since `perf` inherits dev overrides. - -[profile.dev.package.tracedecay] -opt-level = 0 - -[profile.perf.package.tracedecay] -opt-level = 1 - -[profile.dev.package.tracedecay-agent-hosts] -opt-level = 0 - -[profile.perf.package.tracedecay-agent-hosts] -opt-level = 1 - -[profile.dev.package.tracedecay-api] -opt-level = 0 - -[profile.perf.package.tracedecay-api] -opt-level = 1 - -[profile.dev.package.tracedecay-application] -opt-level = 0 - -[profile.perf.package.tracedecay-application] -opt-level = 1 - -[profile.dev.package.tracedecay-automation] -opt-level = 0 - -[profile.perf.package.tracedecay-automation] -opt-level = 1 - -[profile.dev.package.tracedecay-automation-runtime] -opt-level = 0 - -[profile.perf.package.tracedecay-automation-runtime] -opt-level = 1 - -[profile.dev.package.tracedecay-capture] -opt-level = 0 - -[profile.perf.package.tracedecay-capture] -opt-level = 1 - -[profile.dev.package.tracedecay-cli] -opt-level = 0 - -[profile.perf.package.tracedecay-cli] -opt-level = 1 - -[profile.dev.package.tracedecay-code-index-retention] -opt-level = 0 - -[profile.perf.package.tracedecay-code-index-retention] -opt-level = 1 - -[profile.dev.package.tracedecay-configuration] -opt-level = 0 - -[profile.perf.package.tracedecay-configuration] -opt-level = 1 - -[profile.dev.package.tracedecay-daemon-control] -opt-level = 0 - -[profile.perf.package.tracedecay-daemon-control] -opt-level = 1 - -[profile.dev.package.tracedecay-daemon-identity] -opt-level = 0 - -[profile.perf.package.tracedecay-daemon-identity] -opt-level = 1 - -[profile.dev.package.tracedecay-daemon-service] -opt-level = 0 - -[profile.perf.package.tracedecay-daemon-service] -opt-level = 1 - -[profile.dev.package.tracedecay-dashboard-api] -opt-level = 0 - -[profile.perf.package.tracedecay-dashboard-api] -opt-level = 1 - -[profile.dev.package.tracedecay-framing] -opt-level = 0 - -[profile.perf.package.tracedecay-framing] -opt-level = 1 - -[profile.dev.package.tracedecay-global-db] -opt-level = 0 - -[profile.perf.package.tracedecay-global-db] -opt-level = 1 - -[profile.dev.package.tracedecay-graph-query] -opt-level = 0 - -[profile.perf.package.tracedecay-graph-query] -opt-level = 1 - -[profile.dev.package.tracedecay-hooks] -opt-level = 0 - -[profile.perf.package.tracedecay-hooks] -opt-level = 1 - -[profile.dev.package.tracedecay-host-admission] -opt-level = 0 - -[profile.perf.package.tracedecay-host-admission] -opt-level = 1 - -[profile.dev.package.tracedecay-host-integration] -opt-level = 0 - -[profile.perf.package.tracedecay-host-integration] -opt-level = 1 - -[profile.dev.package.tracedecay-lcm] -opt-level = 0 - -[profile.perf.package.tracedecay-lcm] -opt-level = 1 - -[profile.dev.package.tracedecay-lsp] -opt-level = 0 - -[profile.perf.package.tracedecay-lsp] -opt-level = 1 - -[profile.dev.package.tracedecay-maintenance] -opt-level = 0 - -[profile.perf.package.tracedecay-maintenance] -opt-level = 1 - -[profile.dev.package.tracedecay-mcp] -opt-level = 0 - -[profile.perf.package.tracedecay-mcp] -opt-level = 1 - -[profile.dev.package.tracedecay-mcp-catalog] -opt-level = 0 - -[profile.perf.package.tracedecay-mcp-catalog] -opt-level = 1 - -[profile.dev.package.tracedecay-policy] -opt-level = 0 - -[profile.perf.package.tracedecay-policy] -opt-level = 1 - -[profile.dev.package.tracedecay-privacy] -opt-level = 0 - -[profile.perf.package.tracedecay-privacy] -opt-level = 1 - -[profile.dev.package.tracedecay-private-fs] -opt-level = 0 - -[profile.perf.package.tracedecay-private-fs] -opt-level = 1 - -[profile.dev.package.tracedecay-project] -opt-level = 0 - -[profile.perf.package.tracedecay-project] -opt-level = 1 - -[profile.dev.package.tracedecay-query] -opt-level = 0 - -[profile.perf.package.tracedecay-query] -opt-level = 1 - -[profile.dev.package.tracedecay-sdk] -opt-level = 0 - -[profile.perf.package.tracedecay-sdk] -opt-level = 1 - -[profile.dev.package.tracedecay-search-eval] -opt-level = 0 - -[profile.perf.package.tracedecay-search-eval] -opt-level = 1 - -[profile.dev.package.tracedecay-session-memory] -opt-level = 0 - -[profile.perf.package.tracedecay-session-memory] -opt-level = 1 - -[profile.dev.package.tracedecay-session-runtime] -opt-level = 0 - -[profile.perf.package.tracedecay-session-runtime] -opt-level = 1 - -[profile.dev.package.tracedecay-session-temporal-store] -opt-level = 0 - -[profile.perf.package.tracedecay-session-temporal-store] -opt-level = 1 - -[profile.dev.package.tracedecay-sessions] -opt-level = 0 - -[profile.perf.package.tracedecay-sessions] -opt-level = 1 - -[profile.dev.package.tracedecay-source-edit] -opt-level = 0 - -[profile.perf.package.tracedecay-source-edit] -opt-level = 1 - -[profile.dev.package.tracedecay-store-runtime] -opt-level = 0 - -[profile.perf.package.tracedecay-store-runtime] -opt-level = 1 - -[profile.dev.package.tracedecay-temporal-query] -opt-level = 0 - -[profile.perf.package.tracedecay-temporal-query] -opt-level = 1 - -[profile.dev.package.tracedecay-tool-catalog] -opt-level = 0 - -[profile.perf.package.tracedecay-tool-catalog] -opt-level = 1 diff --git a/MODULE.bazel b/MODULE.bazel index e30399e489..dbef6d1e81 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -39,41 +39,57 @@ bazel_dep(name = "hermetic_cc_toolchain", version = "4.3.0") zig = use_extension("@hermetic_cc_toolchain//toolchain:ext.bzl", "toolchains") use_repo(zig, "zig_sdk") +RUST_VERSION = "1.99.0" + rust = use_extension("@rules_rust//rust:extensions.bzl", "rust") rust.toolchain( edition = "2024", - # Cargo's dev profile compiles every crate at opt-level 1 and release at - # 3; generated targets and the .bazelrc block refine per crate from the - # perf profile. + # Bazel's default fastbuild mirrors the assertion-enabled Cargo perf + # profile. Generated targets and the .bazelrc block refine its opt-level + # per crate; releases use opt-level 3. opt_level = { "dbg": "0", "fastbuild": "1", "opt": "3", }, - # fastbuild emits no Rust debug information. Avoid declaring redundant - # Windows PDB outputs and caching a second copy for every executable. + # fastbuild already emits no Rust debug information. Tell rules_rust not + # to declare a Windows PDB for each binary or copy it into the action cache. + # Explicit debug builds retain their symbols. strip_level = { "dbg": "none", "fastbuild": "debuginfo", "opt": "debuginfo", }, - versions = ["nightly/2026-10-25"], - rustfmt_version = "nightly/2026-10-25", - rust_analyzer_version = "nightly/2026-10-25", + versions = [RUST_VERSION], + rustfmt_version = RUST_VERSION, + rust_analyzer_version = RUST_VERSION, ) use_repo( rust, "rust_toolchains", - rust_analyzer_tools = "rust_analyzer_nightly-2026-10-25_tools", + rust_analyzer_tools = "rust_analyzer_" + RUST_VERSION + "_tools", ) register_toolchains("@rust_toolchains//:all") +host = use_extension("@rules_rust//rust:extensions.bzl", "rust_host_tools") +host.host_tools(name = "cargo_host_tools", version = RUST_VERSION) +use_repo(host, "cargo_host_tools") + +# Provision the native Cargo driver and its sibling wrappers from one upstream pin. +blaze_repository = use_repo_rule("//scripts/bazel/blaze:repositories.bzl", "blaze_repository") +blaze_repository( + name = "blaze", + host_tools = "@cargo_host_tools//:BUILD.bazel", +) + crate = use_extension("@rules_rust//crate_universe:extensions.bzl", "crate") crate.from_cargo( # One letter: Windows build-script paths repeat the hub's repository # name, and MSVC cannot open headers past MAX_PATH (gen_builds.py). name = "c", + cargo_config = "//:.cargo/acquisition.toml", cargo_lockfile = "//:Cargo.lock", + host_tools = "@cargo_host_tools", manifests = ["//:Cargo.toml"], ) diff --git a/benchmark_data/BUILD.bazel b/benchmark_data/BUILD.bazel index 34029c7d21..473ff10808 100644 --- a/benchmark_data/BUILD.bazel +++ b/benchmark_data/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds # BEGIN generated by scripts/bazel/gen_builds.py filegroup( diff --git a/crates/tracedecay-agent-hosts/BUILD.bazel b/crates/tracedecay-agent-hosts/BUILD.bazel index 425dfdc231..37b5b97776 100644 --- a/crates/tracedecay-agent-hosts/BUILD.bazel +++ b/crates/tracedecay-agent-hosts/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-api/BUILD.bazel b/crates/tracedecay-api/BUILD.bazel index 2568704d59..e0ba6c6208 100644 --- a/crates/tracedecay-api/BUILD.bazel +++ b/crates/tracedecay-api/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-application/BUILD.bazel b/crates/tracedecay-application/BUILD.bazel index 6c83d7e155..4132335dc7 100644 --- a/crates/tracedecay-application/BUILD.bazel +++ b/crates/tracedecay-application/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-automation-runtime/BUILD.bazel b/crates/tracedecay-automation-runtime/BUILD.bazel index af5d30ba01..e64cef019e 100644 --- a/crates/tracedecay-automation-runtime/BUILD.bazel +++ b/crates/tracedecay-automation-runtime/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-automation/BUILD.bazel b/crates/tracedecay-automation/BUILD.bazel index 47561a4189..f72467d8fc 100644 --- a/crates/tracedecay-automation/BUILD.bazel +++ b/crates/tracedecay-automation/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-capture/BUILD.bazel b/crates/tracedecay-capture/BUILD.bazel index 91c502cc6a..8ecf26fe2f 100644 --- a/crates/tracedecay-capture/BUILD.bazel +++ b/crates/tracedecay-capture/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-cli/BUILD.bazel b/crates/tracedecay-cli/BUILD.bazel index 54557ecf2a..4a771bf949 100644 --- a/crates/tracedecay-cli/BUILD.bazel +++ b/crates/tracedecay-cli/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-cli/build.rs b/crates/tracedecay-cli/build.rs index 8161dfad62..1c3d79f849 100644 --- a/crates/tracedecay-cli/build.rs +++ b/crates/tracedecay-cli/build.rs @@ -282,6 +282,8 @@ fn run_pnpm(dir: &Path, args: &[&str], envs: &[(&str, &OsStr)]) -> io::Result<() .args(args) .envs(envs.iter().copied()) .current_dir(dir) + // Build-script stdout is Cargo's directive stream; frontend logs belong on stderr. + .stdout(io::stderr()) .status() }; // Standalone pnpm installs `pnpm.exe`; npm and Corepack install a diff --git a/crates/tracedecay-code-extraction/BUILD.bazel b/crates/tracedecay-code-extraction/BUILD.bazel index 129f933b88..238c27bbbe 100644 --- a/crates/tracedecay-code-extraction/BUILD.bazel +++ b/crates/tracedecay-code-extraction/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-code-index-retention/BUILD.bazel b/crates/tracedecay-code-index-retention/BUILD.bazel index 5d76d69e5d..4b7fca3c8f 100644 --- a/crates/tracedecay-code-index-retention/BUILD.bazel +++ b/crates/tracedecay-code-index-retention/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-code-index-runtime/BUILD.bazel b/crates/tracedecay-code-index-runtime/BUILD.bazel index d0626bbbb5..99a07c7b7c 100644 --- a/crates/tracedecay-code-index-runtime/BUILD.bazel +++ b/crates/tracedecay-code-index-runtime/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-code-index-runtime/src/git_transactions/owner.rs b/crates/tracedecay-code-index-runtime/src/git_transactions/owner.rs index ce2454c21b..a7c4ccd30a 100644 --- a/crates/tracedecay-code-index-runtime/src/git_transactions/owner.rs +++ b/crates/tracedecay-code-index-runtime/src/git_transactions/owner.rs @@ -556,14 +556,14 @@ impl DaemonGitIndexTransactionServiceRegistry { let native = FixedDaemonGitIndexExecutor::new( DaemonProjectGitIndexPreviewAssembler::new(native_root, project_id.clone()), ); - DaemonGitIndexTransactionService::start( + Box::pin(DaemonGitIndexTransactionService::start( store, native, GitEffectClassifierV1::default(), DaemonGitIndexPolicyRecheck::new(service_authority), mutation_queue, observed_at, - ) + )) .await .map(|service| (project_id, service)) }) diff --git a/crates/tracedecay-code-index/BUILD.bazel b/crates/tracedecay-code-index/BUILD.bazel index 8fd723a9d3..4c6b9717b3 100644 --- a/crates/tracedecay-code-index/BUILD.bazel +++ b/crates/tracedecay-code-index/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-configuration/BUILD.bazel b/crates/tracedecay-configuration/BUILD.bazel index 8d5a9e436e..8e072250ae 100644 --- a/crates/tracedecay-configuration/BUILD.bazel +++ b/crates/tracedecay-configuration/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-contracts/BUILD.bazel b/crates/tracedecay-contracts/BUILD.bazel index afd6b7f2ee..fbf4207d7a 100644 --- a/crates/tracedecay-contracts/BUILD.bazel +++ b/crates/tracedecay-contracts/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-daemon-control/BUILD.bazel b/crates/tracedecay-daemon-control/BUILD.bazel index 762491c6cd..04caea2f93 100644 --- a/crates/tracedecay-daemon-control/BUILD.bazel +++ b/crates/tracedecay-daemon-control/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-daemon-identity/BUILD.bazel b/crates/tracedecay-daemon-identity/BUILD.bazel index efc21e4ca4..e1787bca22 100644 --- a/crates/tracedecay-daemon-identity/BUILD.bazel +++ b/crates/tracedecay-daemon-identity/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-daemon-protocol/BUILD.bazel b/crates/tracedecay-daemon-protocol/BUILD.bazel index 81a71fdad0..2065de454b 100644 --- a/crates/tracedecay-daemon-protocol/BUILD.bazel +++ b/crates/tracedecay-daemon-protocol/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-daemon-service/BUILD.bazel b/crates/tracedecay-daemon-service/BUILD.bazel index 872fc1faf7..82a5916c20 100644 --- a/crates/tracedecay-daemon-service/BUILD.bazel +++ b/crates/tracedecay-daemon-service/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-dashboard-api/BUILD.bazel b/crates/tracedecay-dashboard-api/BUILD.bazel index 609662f08f..2aef6b2247 100644 --- a/crates/tracedecay-dashboard-api/BUILD.bazel +++ b/crates/tracedecay-dashboard-api/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-domain/BUILD.bazel b/crates/tracedecay-domain/BUILD.bazel index 3b36489adc..9fa68a88c0 100644 --- a/crates/tracedecay-domain/BUILD.bazel +++ b/crates/tracedecay-domain/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-framing/BUILD.bazel b/crates/tracedecay-framing/BUILD.bazel index f6f8feba71..674dc9fc89 100644 --- a/crates/tracedecay-framing/BUILD.bazel +++ b/crates/tracedecay-framing/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-global-db/BUILD.bazel b/crates/tracedecay-global-db/BUILD.bazel index 8064e12258..f611b9e94e 100644 --- a/crates/tracedecay-global-db/BUILD.bazel +++ b/crates/tracedecay-global-db/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-graph-db/BUILD.bazel b/crates/tracedecay-graph-db/BUILD.bazel index 27205813c5..147410a5db 100644 --- a/crates/tracedecay-graph-db/BUILD.bazel +++ b/crates/tracedecay-graph-db/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-graph-query/BUILD.bazel b/crates/tracedecay-graph-query/BUILD.bazel index fd4d499193..6319b428e7 100644 --- a/crates/tracedecay-graph-query/BUILD.bazel +++ b/crates/tracedecay-graph-query/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-hooks/BUILD.bazel b/crates/tracedecay-hooks/BUILD.bazel index 7c6be16012..e144c111ca 100644 --- a/crates/tracedecay-hooks/BUILD.bazel +++ b/crates/tracedecay-hooks/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-host-admission/BUILD.bazel b/crates/tracedecay-host-admission/BUILD.bazel index 97fa40d19d..01831693bf 100644 --- a/crates/tracedecay-host-admission/BUILD.bazel +++ b/crates/tracedecay-host-admission/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-host-integration/BUILD.bazel b/crates/tracedecay-host-integration/BUILD.bazel index c5bb3acc46..0a58ff4b83 100644 --- a/crates/tracedecay-host-integration/BUILD.bazel +++ b/crates/tracedecay-host-integration/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-lcm/BUILD.bazel b/crates/tracedecay-lcm/BUILD.bazel index 34f131d431..8571a6d369 100644 --- a/crates/tracedecay-lcm/BUILD.bazel +++ b/crates/tracedecay-lcm/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-lsp/BUILD.bazel b/crates/tracedecay-lsp/BUILD.bazel index 44dc56ddd6..f4a6ebc631 100644 --- a/crates/tracedecay-lsp/BUILD.bazel +++ b/crates/tracedecay-lsp/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-maintenance/BUILD.bazel b/crates/tracedecay-maintenance/BUILD.bazel index 669ee3c71e..e888a32ce8 100644 --- a/crates/tracedecay-maintenance/BUILD.bazel +++ b/crates/tracedecay-maintenance/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-mcp-catalog/BUILD.bazel b/crates/tracedecay-mcp-catalog/BUILD.bazel index 6714552a93..c5824c1f91 100644 --- a/crates/tracedecay-mcp-catalog/BUILD.bazel +++ b/crates/tracedecay-mcp-catalog/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-mcp/BUILD.bazel b/crates/tracedecay-mcp/BUILD.bazel index fbdf0dfeb3..06b3fc7d7b 100644 --- a/crates/tracedecay-mcp/BUILD.bazel +++ b/crates/tracedecay-mcp/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-policy/BUILD.bazel b/crates/tracedecay-policy/BUILD.bazel index a071601739..6159ec7c50 100644 --- a/crates/tracedecay-policy/BUILD.bazel +++ b/crates/tracedecay-policy/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-privacy/BUILD.bazel b/crates/tracedecay-privacy/BUILD.bazel index e0137acbc4..c7130dc4d9 100644 --- a/crates/tracedecay-privacy/BUILD.bazel +++ b/crates/tracedecay-privacy/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-private-fs/BUILD.bazel b/crates/tracedecay-private-fs/BUILD.bazel index 179ba3b0c6..b40493ae65 100644 --- a/crates/tracedecay-private-fs/BUILD.bazel +++ b/crates/tracedecay-private-fs/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-project/BUILD.bazel b/crates/tracedecay-project/BUILD.bazel index 85ec01f168..4399e856fc 100644 --- a/crates/tracedecay-project/BUILD.bazel +++ b/crates/tracedecay-project/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-query/BUILD.bazel b/crates/tracedecay-query/BUILD.bazel index e1499a70d6..77a562baa4 100644 --- a/crates/tracedecay-query/BUILD.bazel +++ b/crates/tracedecay-query/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-runtime-core/BUILD.bazel b/crates/tracedecay-runtime-core/BUILD.bazel index 0ed5da707b..148a2deb5b 100644 --- a/crates/tracedecay-runtime-core/BUILD.bazel +++ b/crates/tracedecay-runtime-core/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-rusqlite-runtime/BUILD.bazel b/crates/tracedecay-rusqlite-runtime/BUILD.bazel index 3d63e07c43..4523831b08 100644 --- a/crates/tracedecay-rusqlite-runtime/BUILD.bazel +++ b/crates/tracedecay-rusqlite-runtime/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-rusqlite-runtime/src/native_runtime.rs b/crates/tracedecay-rusqlite-runtime/src/native_runtime.rs index e9299cd8df..b1b379bf84 100644 --- a/crates/tracedecay-rusqlite-runtime/src/native_runtime.rs +++ b/crates/tracedecay-rusqlite-runtime/src/native_runtime.rs @@ -266,6 +266,8 @@ impl Drop for Settlement { /// Completes at `deadline` on a native timer task, so a foreign executor never /// polls an Asupersync timer: its wait wakes through the join's `Waker`. +/// An elapsed deadline completes on its first poll; it must not wait for a +/// worker before a caller can prefer it over starting already-expired work. /// Dropping the returned future cancels the timer. /// /// Boxed so every timed caller proves this future `Send` once, here, instead @@ -273,6 +275,9 @@ impl Drop for Settlement { pub fn sleep_until(deadline: Instant) -> BoxFuture<'static, Result<(), String>> { Box::pin(async move { let cx = cx().ok_or("native execution context unavailable")?; + if Instant::now() >= deadline { + return Ok(()); + } let mut timer = cx .spawn(move |cx| async move { // Measured before the timer clock is read, so the wake never @@ -1120,10 +1125,11 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn a_past_deadline_takes_a_ready_value_and_elapses_a_pending_one() { let past = Instant::now() - Duration::from_millis(1); - assert_eq!(timeout_at(past, async { 7 }).await, Ok(7)); + assert_eq!(sleep_until(past).now_or_never(), Some(Ok(()))); + assert_eq!(timeout_at(past, async { 7 }).now_or_never(), Some(Ok(7))); assert_eq!( - timeout_at(past, std::future::pending::<()>()).await, - Err(TimeoutError::Elapsed) + timeout_at(past, std::future::pending::<()>()).now_or_never(), + Some(Err(TimeoutError::Elapsed)) ); } diff --git a/crates/tracedecay-sdk/BUILD.bazel b/crates/tracedecay-sdk/BUILD.bazel index 012b6a99e0..8bacfecee7 100644 --- a/crates/tracedecay-sdk/BUILD.bazel +++ b/crates/tracedecay-sdk/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-search-eval/BUILD.bazel b/crates/tracedecay-search-eval/BUILD.bazel index e5a972b333..e09aae133c 100644 --- a/crates/tracedecay-search-eval/BUILD.bazel +++ b/crates/tracedecay-search-eval/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-session-memory/BUILD.bazel b/crates/tracedecay-session-memory/BUILD.bazel index c0b2299ef6..8b73cf0a94 100644 --- a/crates/tracedecay-session-memory/BUILD.bazel +++ b/crates/tracedecay-session-memory/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-session-runtime/BUILD.bazel b/crates/tracedecay-session-runtime/BUILD.bazel index d18cd3cb9e..c63058c622 100644 --- a/crates/tracedecay-session-runtime/BUILD.bazel +++ b/crates/tracedecay-session-runtime/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-session-temporal-store/BUILD.bazel b/crates/tracedecay-session-temporal-store/BUILD.bazel index 979c328a61..9115bc930c 100644 --- a/crates/tracedecay-session-temporal-store/BUILD.bazel +++ b/crates/tracedecay-session-temporal-store/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-sessions/BUILD.bazel b/crates/tracedecay-sessions/BUILD.bazel index 80c44aeb64..4c18689377 100644 --- a/crates/tracedecay-sessions/BUILD.bazel +++ b/crates/tracedecay-sessions/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-source-edit/BUILD.bazel b/crates/tracedecay-source-edit/BUILD.bazel index 54f2fa57c0..b4923fd412 100644 --- a/crates/tracedecay-source-edit/BUILD.bazel +++ b/crates/tracedecay-source-edit/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-store-runtime/BUILD.bazel b/crates/tracedecay-store-runtime/BUILD.bazel index 25b917b209..758283fce3 100644 --- a/crates/tracedecay-store-runtime/BUILD.bazel +++ b/crates/tracedecay-store-runtime/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-store/BUILD.bazel b/crates/tracedecay-store/BUILD.bazel index 181be22b74..4ded0bc74b 100644 --- a/crates/tracedecay-store/BUILD.bazel +++ b/crates/tracedecay-store/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-temporal-query/BUILD.bazel b/crates/tracedecay-temporal-query/BUILD.bazel index 3a8eed7358..81cd5b1a5d 100644 --- a/crates/tracedecay-temporal-query/BUILD.bazel +++ b/crates/tracedecay-temporal-query/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-tokenizer/BUILD.bazel b/crates/tracedecay-tokenizer/BUILD.bazel index 69cbb5c132..6fea40a96f 100644 --- a/crates/tracedecay-tokenizer/BUILD.bazel +++ b/crates/tracedecay-tokenizer/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay-tool-catalog/BUILD.bazel b/crates/tracedecay-tool-catalog/BUILD.bazel index 5174fe20e9..d2f9c8c20c 100644 --- a/crates/tracedecay-tool-catalog/BUILD.bazel +++ b/crates/tracedecay-tool-catalog/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay/BUILD.bazel b/crates/tracedecay/BUILD.bazel index 1bbfc4a2ad..5748e8d7a0 100644 --- a/crates/tracedecay/BUILD.bazel +++ b/crates/tracedecay/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds package(default_visibility = ["//visibility:public"]) diff --git a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay/pending.rs b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay/pending.rs index 4a0037b9ca..0e811dc67e 100644 --- a/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay/pending.rs +++ b/crates/tracedecay/src/daemon/project_open_owners/advisory_runtime/scout_journey_tests/replay/pending.rs @@ -29,22 +29,22 @@ use super::{mount_intent, registered_fixture}; #[tokio::test] async fn pending_stop_replay_claims_its_mounted_intent_before_feedback_settles() { - pending_stop_delivers_once(false, None).await; + Box::pin(pending_stop_delivers_once(false, None)).await; } #[tokio::test] async fn pending_stop_retry_reuses_its_retained_claim_before_feedback_settles() { - pending_stop_delivers_once(true, None).await; + Box::pin(pending_stop_delivers_once(true, None)).await; } #[tokio::test] async fn pending_stop_rejects_a_retained_claim_after_configuration_advances() { - pending_stop_delivers_once(true, Some(false)).await; + Box::pin(pending_stop_delivers_once(true, Some(false))).await; } #[tokio::test] async fn completed_stop_rejects_a_retained_claim_after_configuration_advances() { - pending_stop_delivers_once(true, Some(true)).await; + Box::pin(pending_stop_delivers_once(true, Some(true))).await; } async fn pending_stop_delivers_once( diff --git a/docs/ci-remote-builds.md b/docs/ci-remote-builds.md index 4d5a5c2c26..6077cf1ac4 100644 --- a/docs/ci-remote-builds.md +++ b/docs/ci-remote-builds.md @@ -16,18 +16,40 @@ and macOS CI retain their existing hosted execution. The Mac currently has a newer Xcode/SDK than the macos-26 image; matching OS and CPU alone does not make native compilation compatible. -Rustc and TestRunner actions use Bazel dynamic execution with sandboxed local -execution and remote execution. Both may start the same action; Bazel keeps the -first result and cancels the other. Local work respects the runner CPU count -and half its RAM. The shared action cache remains enabled. Tests carry Node, Rust Analyzer, -and their compiler tools as declared runfiles. +Rustc actions, including asupersync, prefer remote execution. Large Rust +compilations do not race a second copy on the smaller hosted runner. +TestRunner actions use Bazel dynamic execution with sandboxed local execution +and remote execution: both may start the same test, and Bazel keeps the first +result and cancels the other. Local work is admitted against the runner CPU +count and half its RAM; these scheduler estimates do not cap a compiler's +actual memory use. The shared action cache remains enabled. Tests carry Node, +Rust Analyzer, and their compiler tools as declared runfiles. Each remote test reserves four worker CPU units and the Rust harness uses four threads. The local branch reserves the hosted runner CPU budget so suites do not compete with one another for local cores. Remote infrastructure failures allow -local fallback; an actual test failure still fails the test. BuildBuddy events +sandboxed local fallback. Bazel's fallback flag applies to the CI command, +including asupersync and other remote actions. Actual compiler errors, failed +tests, and remote action execution timeouts still fail the build. BuildBuddy events upload asynchronously, so a reporting outage does not fail a successful build. -A terminated runner can leave an incomplete report. Fleet jobs skip restoring -and saving the multi-GB Actions disk cache; other jobs retain it. +A terminated runner can leave an incomplete report. + +CI checks Tailscale enrollment and TCP connectivity to the executor before +selecting the private fleet. If either fails, it uses the hosted runner and +restores the usual master Actions disk cache. `fleet-enabled` reports that +observed selection; passing a client ID alone does not enable it. Jobs with a +reachable executor skip the multi-GB Actions disk-cache restore. The existing +Linux matrix is chosen before this check, so a fleet job that falls back still +builds and tests `//...` in that job rather than expanding into hosted shards. + +The connectivity check proves that the coordinator accepts connections, not +that compatible workers are online. A reachable coordinator can queue an +action while its worker pool is offline. Local fallback follows infrastructure +errors from Bazel; it does not guarantee immediate recovery from a queue with +no workers. The checked-in execution platforms select OS and CPU architecture, +with CPU admission requirements. They do not define a high-memory worker +class for asupersync. Worker memory capacity and scheduling belong to the +private fleet configuration and must be verified from executed-action worker +identities and metrics. ## Trust configuration @@ -58,8 +80,10 @@ The action registers ephemeral nodes and logs them out during job cleanup. ## Verification -An authenticated hosted run must show Tailscale OIDC enrollment, remote cache -or execution activity in Bazel's action summary, and the BuildBuddy invocation -URL. A cache hit alone does not prove remote execution. Cold dynamic builds may -show both sandboxed and remote actions; warm builds may use the cache entirely. +An authenticated hosted run must show Tailscale OIDC enrollment, successful +executor connectivity, remote cache or execution activity in Bazel's action +summary, and the BuildBuddy invocation URL. A cache hit alone does not prove +remote execution. Cold builds may show both sandboxed and remote test actions; +warm builds may use the cache entirely. An offline setup must report +`fleet-enabled=false` and restore the hosted action cache. Actionlint checks workflow syntax but does not establish network or OIDC access. diff --git a/docs/development/runtime-performance.md b/docs/development/runtime-performance.md index 707a58cf1a..f637f750d3 100644 --- a/docs/development/runtime-performance.md +++ b/docs/development/runtime-performance.md @@ -373,8 +373,9 @@ while the lane runs. For OS-level counters, read The existing `large_repos` benchmark also provides a finite audit through the production composition. Build it with -`cargo bench -p tracedecay --profile perf --features test-helpers,test-transport --bench large_repos --no-run`, -then pass the executable printed by Cargo explicitly to the wrapper. +`bazel build //crates/tracedecay:large_repos`, then pass +`$(bazel info bazel-bin)/crates/tracedecay/large_repos` explicitly to the wrapper. +This uses the assertion-enabled optimization levels from Cargo's `perf` profile. ```sh TRACEDECAY_BENCH_PROFILE=perf scripts/run-tool-performance.sh \ diff --git a/docs/remote-execution.md b/docs/remote-execution.md index 583199a83f..452d7a24ad 100644 --- a/docs/remote-execution.md +++ b/docs/remote-execution.md @@ -1,7 +1,7 @@ # Private Bazel execution The repository always expands its `tracedecay` Bazel configuration, which -selects the pinned nightly toolchain. Add the following to the host's +selects the pinned Rust toolchain. Add the following to the host's `~/.bazelrc` to extend that profile with private execution settings: ```bazelrc @@ -15,7 +15,8 @@ build:fleet --remote_local_fallback Every new TraceDecay worktree inherits this host profile automatically. Other repositories do not select it. Without a host extension, including under -`--nohome_rc` in CI, the repository still selects nightly and builds locally. +`--nohome_rc` in CI, the repository still selects the pinned toolchain and +builds locally. The ignored `.bazelrc.local` remains available for later per-checkout overrides; command-line options take precedence over rc settings. @@ -83,12 +84,27 @@ including the action UUID, temporary fixture home, and `.tracedecay/daemon.sock` must be shorter than 104 bytes on macOS and 108 bytes on Linux. Validate the full path from an executed test; checking only the work directory misses this limit. Worker sandboxes must allow Unix socket bind/connect inside the owned -action directories. Keep action `TMPDIR` on the same writable filesystem and -mount as +action directories. On macOS, give the CAS and action trees matching sandbox +rights, including Unix socket permissions: macOS rejects hardlinks between +paths with different sandbox rights even when both allow file reads and writes. +Verify a CAS-to-action hardlink under the actual worker profile before starting +the worker. Scope these permissions to the worker-owned trees. + +Keep action `TMPDIR` on the same writable filesystem and mount as compiler outputs: a separate temporary mount can make Rust's artifact renames fail with `EXDEV`. The action root may be separate from the CAS cache, but must remain on the intended build volume. +NativeLink's macOS directory cache uses APFS `clonefile` automatically, with a +hardlink fallback. For many actions sharing a large toolchain, enable +`directory_cache.experimental_subtree_caching` so unchanged subtrees can be +prepared once and cloned across different action roots. Size `max_entries` and +`max_size_bytes` for subtree entries as well as roots; nested entries count their +logical sizes even when APFS shares their physical blocks. Confirm nonzero +`clonefile_hits` and `subtree_hits` in the directory-cache summary on a real run. +When input preparation stalls with little network activity, inspect local disk +throughput and the external drive's negotiated USB speed before tuning RPC limits. + See [CI fleet configuration](ci-remote-builds.md) for hosted runners, dynamic execution, and restricted Tailscale enrollment. diff --git a/renovate.json b/renovate.json index d18c91f19e..da51c42e4f 100644 --- a/renovate.json +++ b/renovate.json @@ -20,17 +20,18 @@ "matchStrings": ["# renovate: datasource=(?\\S+) depName=(?\\S+)\\s+[A-Z_]+: \"?(?[^\\s\"]+)\"?"] }, { - "description": "rules_rust's nightly pin, and the .bazelrc note that names it, must match rust-toolchain.toml", + "description": "rules_rust's compiler, formatter, and analyzer pins must match rust-toolchain.toml", "customType": "regex", - "managerFilePatterns": ["/^MODULE\\.bazel$/", "/^\\.bazelrc$/"], + "managerFilePatterns": ["/^MODULE\\.bazel$/"], "matchStrings": [ - "nightly/(?\\d{4}-\\d{2}-\\d{2})", - "rust_analyzer_nightly-(?\\d{4}-\\d{2}-\\d{2})_tools" + "versions\\s*=\\s*\\[\\s*\"(?\\d+\\.\\d+\\.\\d+)\"", + "rustfmt_version\\s*=\\s*\"(?\\d+\\.\\d+\\.\\d+)\"", + "rust_analyzer_version\\s*=\\s*\"(?\\d+\\.\\d+\\.\\d+)\"", + "rust_analyzer_(?\\d+\\.\\d+\\.\\d+)_tools" ], "depNameTemplate": "rust", "datasourceTemplate": "rust-version", - "extractVersionTemplate": "^nightly-(?\\d{4}-\\d{2}-\\d{2})$", - "versioningTemplate": "regex:^(?\\d{4})-(?\\d{2})-(?\\d{2})$" + "versioningTemplate": "semver" } ], "packageRules": [ @@ -40,10 +41,11 @@ "enabled": false }, { - "description": "One PR moves every Rust nightly pin; a year rollover reads as major to the date versioning, so keep it in the same branch", + "description": "Keep every Rust pin on Blaze's supported compiler until its standard-library ABI support changes", "matchDepNames": ["rust"], "matchDatasources": ["rust-version"], - "groupName": "rust nightly", + "groupName": "rust toolchain", + "allowedVersions": "1.99.0", "separateMajorMinor": false } ] diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 27982fcd2f..5101448bae 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,6 +1,5 @@ [toolchain] -# Dated nightly: asupersync's nightly-outcome-try default needs try_trait_v2. -# 2026-09-27 includes rust-lang/rust#162275; 2026-08-25 does not. -channel = "nightly-2026-10-25" +# Match Blaze's supported compiler and standard-library ABI. +channel = "1.99.0" # LSP acceptance negotiates capabilities with the pinned upstream analyzer. components = ["clippy", "rustfmt", "rust-analyzer", "rust-src"] diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 1045906590..a5a420707a 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -61,6 +61,66 @@ py_test( main = "test-bazel-dependency-patches.py", ) +_CARGO_QUERY_DATA = [ + "//:.cargo/acquisition.toml", + "@cargo_host_tools//:cargo", + "@cargo_host_tools//:rustc", + "@cargo_host_tools//:rustc_lib", +] + +_CARGO_QUERY_ENV = { + "GEN_BUILDS_CARGO": "$(rlocationpath @cargo_host_tools//:cargo)", + "GEN_BUILDS_RUSTC": "$(rlocationpath @cargo_host_tools//:rustc)", + "GEN_BUILDS_POLICY": "$(rlocationpath //:.cargo/acquisition.toml)", +} + +py_binary( + name = "gen_builds", + srcs = ["bazel/gen_builds.py"], + data = _CARGO_QUERY_DATA, + env = _CARGO_QUERY_ENV, + main = "bazel/gen_builds.py", + deps = ["@rules_python//python/runfiles"], +) + +py_test( + name = "gen_builds_test", + size = "small", + srcs = [ + "bazel/gen_builds.py", + "test-gen-builds-default-alias.py", + ], + data = _CARGO_QUERY_DATA + ["//:rust-toolchain.toml"], + env = _CARGO_QUERY_ENV | { + "GEN_BUILDS_TOOLCHAIN": "$(rlocationpath //:rust-toolchain.toml)", + }, + main = "test-gen-builds-default-alias.py", + deps = ["@rules_python//python/runfiles"], +) + +py_test( + name = "configure_ci_fleet_test", + size = "small", + srcs = ["test-configure-ci-fleet.py"], + data = ["//:.github/actions/setup-bazel/configure_fleet.py"], + main = "test-configure-ci-fleet.py", +) + +py_test( + name = "release_pr_refresh_test", + size = "small", + srcs = [ + "neutralize-release-pr-closing-keywords.py", + "test-neutralize-release-pr-closing-keywords.py", + ], + data = [ + "neutralize-release-pr-closing-keywords.sh", + "update-release-pr-lockfile.sh", + ], + main = "test-neutralize-release-pr-closing-keywords.py", + tags = ["local", "manual"], +) + py_test( name = "ambient_tokio_check_test", size = "small", @@ -113,7 +173,6 @@ exports_files( "generate-release-inventory.py", "test-generate-release-inventory.py", "test-update-release-checksums.py", - "bazel/gen_builds.py", "bazel/rust_test_environment_probe.txt", "check-dashboard-bundle.py", "check-github-hosted-runners.py", diff --git a/scripts/bazel/blaze/BUILD.bazel b/scripts/bazel/blaze/BUILD.bazel new file mode 100644 index 0000000000..bf29846ba1 --- /dev/null +++ b/scripts/bazel/blaze/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_python//python:defs.bzl", "py_binary") + +exports_files(["repositories.bzl"]) + +py_binary( + name = "blaze", + srcs = ["blaze.py"], + data = ["@blaze//:driver", "@blaze//:runtime"], + env = {"TRACEDECAY_BLAZE_EXECUTABLE": "$(rlocationpath @blaze//:driver)"}, + legacy_create_init = 0, + main = "blaze.py", + deps = ["@rules_python//python/runfiles"], + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/scripts/bazel/blaze/README.md b/scripts/bazel/blaze/README.md new file mode 100644 index 0000000000..5218ff3126 --- /dev/null +++ b/scripts/bazel/blaze/README.md @@ -0,0 +1,172 @@ +# Native Blaze through Bazel + +For a normal edit loop, keep the package, features and command stable so the +same daemon worker can reuse its compiler state: + +```sh +bazel run //scripts/bazel/blaze:blaze -- test -p tracedecay-private-fs --lib +bazel run //scripts/bazel/blaze:blaze -- build -p tracedecay-cli --bin tracedecay +bazel run //scripts/bazel/blaze:blaze -- run -p tracedecay-cli --bin tracedecay -- --version +bazel run //scripts/bazel/blaze:blaze -- status +bazel run //scripts/bazel/blaze:blaze -- stop +``` + +Full workspace validation loads much more state. Use a host with measured memory +headroom, especially when it also runs NativeLink workers: + +```sh +features=tracedecay/test-helpers,tracedecay/test-transport,tracedecay/search-eval,tracedecay-cli/test-transport,tracedecay-query/production +bazel run //scripts/bazel/blaze:blaze -- build --workspace --bins --features "$features" +bazel run //scripts/bazel/blaze:blaze -- build -p tracedecay-cli --example tracedecay-host-cli-fixture --features "$features" +bazel run --run_env=BLAZE_TESTS=all //scripts/bazel/blaze:blaze -- test --workspace --features "$features" --no-fail-fast +``` + +The target launches upstream's native Cargo driver from the checkout root. Cargo +owns workspace membership, features, build scripts, test selection and generated +artifacts; this integration has no second package list or compiler graph. Arguments +pass through unchanged except that `build`, `test`, `run` and `check` default to +`--locked` and load the existing `.cargo/acquisition.toml` with `--config`. The +launcher also exports that file's acquisition settings for the pinned engine's +metadata subprocesses, which do not forward `--config`. This does not force the +Cargo-only path. Native diagnostics, +fallback explanations and exit status pass through. +These commands also use Bazel's `RUSTC_BOOTSTRAP=1` environment so pinned stable +Cargo honors that policy's unstable shallow-dependency setting. +Use Cargo's package/target/feature flags for focused runs. Required integration-test +features still apply; `--workspace` alone does not enable them. + +The `@blaze` repository provisions `blaze`, `ra-jit`, `ra-jit-rustc` and +`ra-jit-rustc-first` together from upstream commit +`d44ef985e45e69661fb44696d61a18a251a39b08`. Its source archive is SHA-256 pinned. +The rules_rust Cargo bootstrap helper builds the four binaries with `--locked +--release`, shared artifacts, and Bazel's pinned host Rust compiler. This first +provisioning compiles upstream and requires its native build prerequisites and +registry access. It does not use the older nightly release asset. The optional +`blaze-proc-macro-srv` is not provisioned: upstream uses rustc to compile procedural +macros and the Rust toolchain server to expand them in this configuration. + +For local validation against an already built checkout of **that exact commit**, +provide an absolute directory containing all four binaries: + +```sh +bazel run --repo_env=TRACEDECAY_BLAZE_BIN_DIR="$BLAZE_CHECKOUT/target/release" \ + //scripts/bazel/blaze:blaze -- test -p tracedecay-private-fs --lib +``` + +This explicit repository override skips provisioning, declares and watches those +four binaries, and makes no provenance assertion about the supplied directory. +Omit it to use the pinned source build. Keep the same override on subsequent +invocations to use the same tool identity. + +## Runtime and CI caches + +Defaults are absolute paths rooted in the checkout: + +| Purpose | Directory | Configuration | +| --- | --- | --- | +| Cargo artifacts and native build metadata | `target/blaze/cargo` | `CARGO_TARGET_DIR`, or Cargo `--target-dir` | +| Compiled code store | `target/blaze/store` | `BLAZE_CACHE` / `RA_JIT_CACHE` | +| Daemon socket, locks, logs, workers, downloaded toolchain | `target/blaze/home` | `BLAZE_HOME` | + +Upstream translates `BLAZE_HOME` to `RA_JIT_HOME`, overriding a separately supplied +`RA_JIT_HOME`. The launcher therefore uses one home for both purposes; an explicit +`RA_JIT_HOME` alone also becomes this shared home. Keep Unix socket paths short +enough for the host. `BLAZE_CACHE` takes precedence over `RA_JIT_CACHE`, as upstream +defines. Cargo's ordinary `target/` artifacts remain separate by default. + +CI can restore/save `target/blaze/cargo` and `target/blaze/store` between matching +toolchain, upstream pin, platform, dependency and feature configurations. Stop the +daemon before saving caches. Do not cache live sockets, locks or workers from +`target/blaze/home`; an upstream-downloaded toolchain can be cached specifically +under `target/blaze/home/toolchains`. A daemon restart loses its in-memory state; +cross-job disk-cache reuse and same-daemon edit reuse are distinct measurements. +The Bazel repository download cache retains the source archive, not the compiled +tool. The bootstrapped binaries persist only with that Bazel output base; a fresh +CI output base rebuilds upstream unless CI deliberately restores a compatible +compiled tool directory and selects it with `TRACEDECAY_BLAZE_BIN_DIR`. Neither +Bazel's action disk cache nor the workspace code store avoids this bootstrap. +Cargo's registry cache is separate. No TraceDecay speedup is claimed by this +launcher. + +Blaze resolves a complete Rust 1.99.0 toolchain with `rust-src` from `RUSTC`, rustup, +or its own checksum-pinned download. `bazel run //scripts/bazel/blaze:blaze -- +toolchain` reports the selection. Bazel's `cargo_host_tools` provisions the compiler +used to build Blaze; it lacks `rust-src` and is not passed off as the complete +runtime toolchain. An explicit incomplete `RUSTC` is rejected by upstream. +When `CARGO` names an absolute executable, its directory also leads the launched +process's `PATH`: upstream's unit-graph loader invokes bare `cargo`, and a +different shell broker can otherwise discard its JSON output and force fallback. + +## Correctness and workspace prerequisites + +Normal Bazel Rust compilation and tests remain the correctness authority and the +source of canonical artifacts. Blaze's incremental diagnostics do not fully +replace rustc's borrow checking. This target does not replace existing correctness +jobs, release builds or published artifacts. + +The full workspace CI run selects Cargo's `perf` profile, which retains debug +assertions and overflow checks while optimizing the tokenizer, graph and hash +kernels used by production-scale fixtures. Ordinary development builds keep +the unoptimized `dev` profile. CI stores its compiled artifacts under +`target/blaze/perf` and includes the profile in its cache identity. + +Native Cargo uses manifest sources and does **not** apply `MODULE.bazel`'s +asupersync scheduler and blocking-pool stack patches, AWS include-path patch, or +arborium runtime-manifest-directory patches. A native Cargo result is not evidence +of behavior under those patched Bazel dependencies. Keep this distinction explicit +when comparing failures, performance and artifacts. + +The existing `.cargo/config.toml` remains active, including its forced workspace +test profile and global-database isolation. Build scripts still require their +real inputs and native tools. The CLI dashboard embedder either builds the +dashboard (requiring the JavaScript prerequisites) or accepts a prebuilt bundle +only with `TRACEDECAY_SKIP_DASHBOARD_BUILD=1` and a valid +`TRACEDECAY_DASHBOARD_BUNDLE_SHA256`. The launcher does not fabricate generated +inputs, weaken bundle checks, suppress failures or change test policy. + +## Memory, edit loops, and upstream recipes + +The launcher defaults `BLAZE_SPARE=0`: Cargo finishes its cold build before Blaze +loads the compiler database. An explicit `BLAZE_SPARE` or legacy `RA_JIT_SPARE` +overrides it. This reduces overlapping cold-build memory; it does not reduce the +memory needed by a fully loaded workspace. CI also limits concurrent Cargo jobs +to two. Neither setting is an overall memory cap. + +`BLAZE_MEMORY_MAX` evicts eligible idle workers. Upstream never evicts a loading, +busy, or most-recently-used worker to meet that budget. `BLAZE_OWN_BASE_MEMORY` +only bounds the optional background base compilation. For actual containment, +use the host's cgroup limits and budget agents, Blaze, NativeLink and the desktop +together. A 16 GiB host cannot safely promise independent 11.5 GiB agent and +10 GiB worker allowances. Use focused packages there; measure the complete CLI's +working set before choosing it for a full-workspace daemon. + +Daemon settings (home, store, spare, memory and idle timeout) are read at startup. +Stop the existing daemon before changing them, once its active work finishes. +For worktrees that should share a daemon and code store, set the same absolute +`BLAZE_HOME` and `BLAZE_CACHE` in host Bazel run settings. Keep `CARGO_TARGET_DIR` +separate per checkout unless Cargo artifact sharing has been explicitly arranged. +The Mac's configured paths stay on `bigssd`; no storage relocation is required. + +For rebuild-on-save, an existing watcher can run the same focused Bazel command. +Do not enable Blaze as a global Cargo wrapper: the explicit driver preserves our +Bazel entrypoint and avoids upstream's slower global-wrapper path. Live process +patching requires application integration with `ra-jit-live`; it is separate from +incremental build support and is not enabled in TraceDecay's daemon. + +The pinned source bootstrap is upstream's ordinary release build, not its +PGO-trained distribution recipe. Benchmark that exact tool build before claiming +a speedup. Keep the ordinary Bazel correctness jobs: Blaze's checks do not replace +rustc's full borrow checking. `BLAZE_TESTS=all` remains the CI policy; affected-only +tests are an optional local experiment, not an acceptance shortcut. + +Blaze's [CI runner recipe](https://github.com/indexable-inc/blaze/blob/d44ef985e45e69661fb44696d61a18a251a39b08/docs/ci-runner.md) +is a design document, not a shipped NativeLink execution backend. Wrapping Blaze +in `bazel run` does not send its internal Cargo/rustc or JIT work to our fleet. +Ordinary Bazel builds retain that remote execution path. Disk caches also cannot +restore a daemon's in-memory analysis on a fresh GitHub runner. + +Upstream references for the pinned implementation: +[usage](https://github.com/indexable-inc/blaze/blob/d44ef985e45e69661fb44696d61a18a251a39b08/docs/usage.md), +[Cargo and memory controls](https://github.com/indexable-inc/blaze/blob/d44ef985e45e69661fb44696d61a18a251a39b08/docs/cargo.md), +[cache formats and trust](https://github.com/indexable-inc/blaze/blob/d44ef985e45e69661fb44696d61a18a251a39b08/docs/cache.md), +and [installation](https://github.com/indexable-inc/blaze/blob/d44ef985e45e69661fb44696d61a18a251a39b08/docs/install.md). diff --git a/scripts/bazel/blaze/blaze.py b/scripts/bazel/blaze/blaze.py new file mode 100644 index 0000000000..a8cab70f5b --- /dev/null +++ b/scripts/bazel/blaze/blaze.py @@ -0,0 +1,80 @@ +"""Run the declared native Blaze Cargo driver against the calling checkout.""" + +import os +from pathlib import Path +import sys +import tomllib + +from python.runfiles import runfiles + + +def main(): + workspace = os.environ.get("BUILD_WORKSPACE_DIRECTORY") + if not workspace or not Path(workspace).is_absolute(): + raise ValueError("use bazel run //scripts/bazel/blaze:blaze -- [flags]") + workspace = Path(workspace).resolve(strict=True) + if not (workspace / "Cargo.toml").is_file(): + raise ValueError(f"workspace has no Cargo.toml: {workspace}") + files = runfiles.Create() + location = os.environ.get("TRACEDECAY_BLAZE_EXECUTABLE") + if files is None or not location: + raise ValueError("Bazel has not supplied the native Blaze runfiles") + resolved = files.Rlocation(location) + if resolved is None: + raise ValueError(f"native Blaze runfile is unavailable: {location}") + executable = Path(resolved).resolve(strict=True) + suffix = executable.suffix if os.name == "nt" else "" + for companion in ("ra-jit", "ra-jit-rustc", "ra-jit-rustc-first"): + path = executable.with_name(companion + suffix) + if not path.is_file(): + raise ValueError(f"native Blaze companion is unavailable: {path}") + + state = workspace / "target" / "blaze" + env = os.environ.copy() + # Upstream maps every BLAZE_* to RA_JIT_*, including HOME. Consequently + # BLAZE_HOME is both the toolchain home and the daemon's runtime home. + env.setdefault("BLAZE_HOME", env.get("RA_JIT_HOME") or str(state / "home")) + if "BLAZE_CACHE" not in env: + env.setdefault("RA_JIT_CACHE", str(state / "store")) + env.setdefault("CARGO_TARGET_DIR", str(state / "cargo")) + # A spare loads another compiler database while Cargo builds. Shared hosts + # need the sequential cold start unless the operator explicitly opts in. + env.setdefault("BLAZE_SPARE", env.get("RA_JIT_SPARE", "0")) + # The pinned engine also launches bare `cargo` for its unit graph. Honor + # the explicit toolchain there instead of a different PATH broker/shim. + cargo = env.get("CARGO") + if cargo and Path(cargo).is_absolute(): + env["PATH"] = os.pathsep.join((str(Path(cargo).parent), env.get("PATH", os.defpath))) + + args = sys.argv[1:] + if args and args[0] in {"build", "test", "run", "check"}: + policy = workspace / ".cargo" / "acquisition.toml" + if not policy.is_file(): + raise ValueError(f"Cargo acquisition policy is unavailable: {policy}") + # Match Bazel's repository environment: stable Cargo otherwise ignores + # the shared policy's unstable shallow-dependency acquisition setting. + env["RUSTC_BOOTSTRAP"] = "1" + # The pinned engine's workspace loader launches additional Cargo + # metadata commands without forwarding --config. Keep acquisition + # consistent for those children using values from the same policy. + acquisition = tomllib.loads(policy.read_text()) + env["CARGO_NET_GIT_FETCH_WITH_CLI"] = str( + acquisition["net"]["git-fetch-with-cli"] + ).lower() + env["CARGO_UNSTABLE_GIT"] = acquisition["unstable"]["git"] + options = args[1:args.index("--")] if "--" in args else args[1:] + if "--locked" not in options and "--frozen" not in options: + args.insert(1, "--locked") + args[1:1] = ["--config", str(policy)] + os.chdir(workspace) + # Native output, Cargo fallback explanations, signals and exit status pass + # directly through; the launcher neither retries nor substitutes Cargo. + os.execve(executable, [str(executable), *args], env) + + +if __name__ == "__main__": + try: + main() + except (OSError, ValueError) as error: + print(f"blaze: {error}", file=sys.stderr) + sys.exit(2) diff --git a/scripts/bazel/blaze/repositories.bzl b/scripts/bazel/blaze/repositories.bzl new file mode 100644 index 0000000000..0e49d71436 --- /dev/null +++ b/scripts/bazel/blaze/repositories.bzl @@ -0,0 +1,78 @@ +"""Bazel-owned provisioning of the pinned upstream native Blaze Cargo driver.""" + +load("@rules_rust//cargo/private:cargo_bootstrap.bzl", "cargo_bootstrap") + +_COMMIT = "d44ef985e45e69661fb44696d61a18a251a39b08" +_SHA256 = "ba3b5b7f8aad697750ff6820483fbde641be839adc84f4adeb566c8c89a1f37d" +_BINARIES = ["ra-jit", "blaze", "ra-jit-rustc", "ra-jit-rustc-first"] + +def _blaze_repository_impl(ctx): + suffix = ".exe" if "windows" in ctx.os.name.lower() else "" + override = ctx.getenv("TRACEDECAY_BLAZE_BIN_DIR") + if override: + absolute = override.startswith("/") or (suffix and ( + override[1:3] in [":\\", ":/"] or override.startswith("\\\\") + )) + if not absolute: + fail("TRACEDECAY_BLAZE_BIN_DIR must be an absolute directory") + directory = ctx.path(override) + for binary in _BINARIES: + path = directory.get_child(binary + suffix) + if not path.exists: + fail("TRACEDECAY_BLAZE_BIN_DIR is missing {}".format(path)) + ctx.watch(path) + ctx.symlink(path, "bin/" + binary + suffix) + else: + ctx.download_and_extract( + url = "https://github.com/indexable-inc/blaze/archive/{}.tar.gz".format(_COMMIT), + sha256 = _SHA256, + stripPrefix = "blaze-" + _COMMIT, + ) + tools = ctx.path(ctx.attr.host_tools).dirname.get_child("bin") + # All four builds share Cargo's artifacts. The maintained bootstrap helper + # supplies --locked and --release; upstream retains its own dependency graph. + for binary in _BINARIES: + built = cargo_bootstrap( + repository_ctx = ctx, + cargo_bin = tools.get_child("cargo" + suffix), + rustc_bin = tools.get_child("rustc" + suffix), + binary = binary, + cargo_manifest = ctx.path("crates/ra-jit/Cargo.toml"), + # A cold build compiles rust-analyzer, Cranelift, and the native + # driver. CI was still compiling new dependencies at the + # helper's generic 600-second cutoff; workspace tests keep + # their own deadlines after this compiler is provisioned. + timeout = 1800, + environment = { + "RA_JIT_GIT_SHA": _COMMIT, + "RUSTC_WORKSPACE_WRAPPER": "", + "RUSTC_WRAPPER": "", + }, + ) + ctx.symlink(built, "bin/" + binary + suffix) + + ctx.file("BUILD.bazel", """ +package(default_visibility = ["//visibility:public"]) + +filegroup( + name = "driver", + srcs = ["bin/blaze{suffix}"], +) + +filegroup( + name = "runtime", + srcs = {binaries}, +) +""".format( + suffix = suffix, + binaries = repr(["bin/" + binary + suffix for binary in _BINARIES]), + )) + +blaze_repository = repository_rule( + implementation = _blaze_repository_impl, + attrs = { + "host_tools": attr.label(mandatory = True, allow_single_file = True), + }, + environ = ["TRACEDECAY_BLAZE_BIN_DIR"], + doc = "Build the native Blaze driver with Bazel's pinned host Rust tools.", +) diff --git a/scripts/bazel/gen_builds.py b/scripts/bazel/gen_builds.py old mode 100755 new mode 100644 index 366be494d0..489e3cbeaf --- a/scripts/bazel/gen_builds.py +++ b/scripts/bazel/gen_builds.py @@ -30,19 +30,23 @@ import hashlib import json +import os import re import subprocess import sys +import time import tomllib from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass from functools import lru_cache -from pathlib import Path +from pathlib import Path, PurePosixPath, PureWindowsPath +from typing import NamedTuple -REPO = Path(__file__).resolve().parent.parent.parent +from python.runfiles import runfiles BANNER = ( "# Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock.\n" - "# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py\n" + "# Regenerate instead of editing. bazel run //scripts:gen_builds\n" ) # Generated filegroups embedded in a handwritten BUILD.bazel live inside @@ -209,6 +213,7 @@ # Marks a resolution context that Cargo builds with `--no-default-features`. NO_DEFAULT_FEATURES = "!no-default-features" +PRODUCTION_CONTEXT = ("tracedecay", "normal,build", frozenset()) def normalize_default_alias(feats, feature_table): @@ -234,25 +239,95 @@ def normalize_default_alias(feats, feature_table): _ANSI = re.compile(r"\x1b\[[0-9;]*m") -def run(cmd, cwd=REPO): - # Cargo emits UTF-8; the Windows console default (cp1252) cannot decode it. - out = subprocess.run( - cmd, cwd=cwd, capture_output=True, text=True, - encoding="utf-8", - ) +def run(cmd, *, cwd, env=None): + command = " ".join(cmd) + started = time.monotonic() + print(f"gen_builds: start {command}", file=sys.stderr, flush=True) + try: + out = subprocess.run( + cmd, cwd=cwd, env=env, stdout=subprocess.PIPE, text=True, encoding="utf-8", + ) + finally: + print( + f"gen_builds: end {command} ({time.monotonic() - started:.2f}s)", + file=sys.stderr, flush=True, + ) if out.returncode != 0: - sys.exit(f"{' '.join(cmd)} failed:\n{out.stderr}") + print(f"{command} failed (exit {out.returncode})", file=sys.stderr, flush=True) + sys.exit(out.returncode) # CARGO_TERM_COLOR=always CI shells would otherwise bake color escapes # into parsed feature names; --color never covers Cargo, this covers the # rest. return _ANSI.sub("", out.stdout) -def perf_opt_levels(): +@dataclass(frozen=True) +class CargoQuery: + workspace: Path + cargo: Path + rustc: Path + policy: Path + + @classmethod + def from_bazel(cls): + workspace = os.environ.get("BUILD_WORKSPACE_DIRECTORY") + if not workspace or not Path(workspace).is_absolute(): + raise ValueError("run from the checkout with bazel run //scripts:gen_builds [-- --check]") + workspace = Path(workspace).resolve(strict=True) + if not (workspace / "Cargo.toml").is_file(): + raise ValueError(f"BUILD_WORKSPACE_DIRECTORY has no Cargo.toml: {workspace}") + files = runfiles.Create() + if files is None: + raise ValueError("Bazel runfiles are unavailable; use bazel run //scripts:gen_builds") + + def declared_file(variable): + location = os.environ.get(variable) + if not location: + raise ValueError(f"missing {variable}; use bazel run //scripts:gen_builds") + resolved = files.Rlocation(location, source_repo="") + if resolved is None or not Path(resolved).is_file(): + raise ValueError(f"missing declared runfile for {variable}: {location}") + return Path(resolved).resolve(strict=True) + + query = cls( + workspace, + declared_file("GEN_BUILDS_CARGO"), + declared_file("GEN_BUILDS_RUSTC"), + declared_file("GEN_BUILDS_POLICY"), + ) + query.validate_toolchain() + return query + + def validate_toolchain(self): + toolchain = tomllib.loads( + (self.workspace / "rust-toolchain.toml").read_text(encoding="utf-8") + ) + channel = toolchain["toolchain"]["channel"] + for name, identity in ( + ("cargo", self.run(["--version"]).strip()), + ("rustc", run([str(self.rustc), "--version"], cwd=self.workspace).strip()), + ): + if identity.split()[:2] != [name, channel]: + raise ValueError( + f"declared {name} must match rust-toolchain.toml channel {channel!r}; got {identity!r}" + ) + print(f"gen_builds: {identity}", file=sys.stderr, flush=True) + + def run(self, args): + env = os.environ.copy() + env.update(RUSTC=str(self.rustc), RUSTC_BOOTSTRAP="1") + return run( + [str(self.cargo), "--config", str(self.policy), *args], + cwd=self.workspace, + env=env, + ) + + +def perf_opt_levels(workspace): """Cargo `perf` opt-level per package name, the profile hosted CI tests with. `perf` inherits `dev`, so a package resolves to its perf override, then its dev override, then the perf and dev profile defaults.""" - profiles = tomllib.loads((REPO / "Cargo.toml").read_text(encoding="utf-8"))["profile"] + profiles = tomllib.loads((workspace / "Cargo.toml").read_text(encoding="utf-8"))["profile"] perf = profiles.get("perf", {}) dev = profiles.get("dev", {}) default = perf.get("opt-level", dev.get("opt-level", 0)) @@ -267,55 +342,116 @@ def perf_opt_levels(): return default, levels -def cargo_metadata(): +def cargo_metadata(query): return json.loads( - run(["cargo", "metadata", "--format-version", "1", "--no-deps", "-q", + query.run(["metadata", "--locked", "--format-version", "1", "--no-deps", "-q", "--color", "never"]) ) +def validate_production_features(members): + product = members.get("tracedecay") + if product is None: + raise ValueError("missing tracedecay product package declaration") + features = product.get("features") or {} + if features.get("default") != ["production"]: + raise ValueError("tracedecay default feature must delegate only to production") + production = set(features.get("production", [])) + if not {"token-counting", "lite", "full"}.issubset(production): + raise ValueError("tracedecay production feature set lost a required member") + if "test-transport" in production: + raise ValueError("tracedecay production feature directly enables test-transport") + + _TREE_LINE = re.compile( - r"^(?P\S+) v(?P\S+)(?: \((?P[^)]+)\))?\|(?P.*)$" + r"(?P[\w-]+) v(?P[0-9]+\.[0-9]+\.[0-9]+" + r"(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?)" + r"(?: \(proc-macro\))?(?: \((?P[^\r\n]+)\))?" + r"\|(?P[^|\r\n]*)" +) +_TREE_EXTERNAL_SOURCE = re.compile( + r"(?:[A-Za-z][A-Za-z0-9+.-]*://\S+|registry `[^`]+`|dir .+|builtin .+)" ) -def feature_map(pkg_name, edges, features=()): +class _TreeRow(NamedTuple): + package_id: str + name: str + version: str + path: str | None + features: frozenset[str] + + +def _parse_tree_row(line): + # Cargo appends the duplicate marker after the entire formatted row. + row = line.removesuffix(" (*)") + match = _TREE_LINE.fullmatch(row) + if match is None: + raise ValueError(f"cargo tree emitted an invalid package row: {line!r}") + source = match.group("source") + path = None + if source is not None: + if PurePosixPath(source).is_absolute() or PureWindowsPath(source).is_absolute(): + path = source + elif _TREE_EXTERNAL_SOURCE.fullmatch(source) is None: + raise ValueError(f"cargo tree emitted an invalid package source: {line!r}") + raw = match.group("feats") + features = frozenset(raw.split(",")) if raw else frozenset() + if any(not feature or re.search(r"[\s|()*]", feature) for feature in features): + raise ValueError(f"cargo tree emitted an invalid feature list: {line!r}") + return _TreeRow( + row.rsplit("|", 1)[0], match.group("name"), match.group("ver"), path, features, + ) + + +def _parse_feature_map(output, context): + production = context == PRODUCTION_CONTEXT + fmap = {} + for line in output.splitlines(): + # Cargo separates workspace roots with an empty line. + if not line: + continue + try: + row = _parse_tree_row(line) + except ValueError as error: + raise ValueError(f"{context}: {error}") from error + if production and "test-transport" in row.features: + raise ValueError( + f"{context}: production graph enables test-transport for {row.package_id}" + ) + # Cargo renders host-native paths; source URLs are not member paths. + if row.path is not None and "/crates/" in row.path.replace("\\", "/"): + fmap.setdefault(row.name, set()).update(row.features) + if production and "tracedecay" not in fmap: + raise ValueError(f"{context}: cargo tree omitted the tracedecay root package") + return fmap + + +def feature_map(query, pkg_name, edges, features=()): """cargo tree resolution for one root context, or for the whole workspace when `pkg_name` is None (features then carry their `pkg/` prefix). Maps member -> feature set.""" + context = (pkg_name, edges, frozenset(features)) root = ["--workspace"] if pkg_name is None else ["-p", pkg_name] cmd = [ - "cargo", "tree", *root, "-e", edges, + "tree", "--locked", *root, "-e", edges, "-f", "{p}|{f}", "--prefix", "none", "--color", "never", ] - features = set(features) + features = set(context[2]) if NO_DEFAULT_FEATURES in features: features.discard(NO_DEFAULT_FEATURES) cmd.append("--no-default-features") if features: cmd += ["--features", ",".join(sorted(features))] - fmap = {} - for line in run(cmd).splitlines(): - m = _TREE_LINE.match(line.strip()) - if not m or not m.group("path"): - continue - # cargo tree renders host-native separators; labels below assume / - if "/crates/" not in m.group("path").replace("\\", "/"): - continue - # cargo tree marks repeated subtrees by appending ' (*)' inside the - # feature field; strip the marker before splitting. - raw = m.group("feats").replace("(*)", "") - feats = {f.strip() for f in raw.split(",") if f.strip()} - fmap.setdefault(m.group("name"), set()).update(feats) - return fmap + return _parse_feature_map(query.run(cmd), context) -def target_feature_annotations(locked): +def target_feature_annotations(query, locked): """crate.annotation()s rebuilding each TARGET_FEATURE_CRATES library in its own repository from Cargo's target resolution. The override keeps the generated deps and build script; only the feature set narrows.""" full = json.loads( - run(["cargo", "metadata", "--format-version", "1", "-q", "--color", "never", + query.run(["metadata", "--locked", "--format-version", "1", "-q", "--color", "never", "--filter-platform", TARGET_TRIPLE]) ) blocks = [] @@ -325,15 +461,17 @@ def target_feature_annotations(locked): sys.exit(f"{name}: expected one locked version, found {versions}") version = versions[0] tree = [ - m for line in run([ - "cargo", "tree", "-p", f"{name}@{version}", "-e", "normal", + _parse_tree_row(line) for line in query.run([ + "tree", "--locked", "-p", f"{name}@{version}", "-e", "normal", "--depth", "1", "--target", TARGET_TRIPLE, "-f", "{p}|{f}", "--prefix", "none", "--color", "never", ]).splitlines() - if (m := _TREE_LINE.match(line.strip())) + if line ] + if not tree or tree[0].name != name or tree[0].version != version: + raise ValueError(f"cargo tree omitted the {name}@{version} annotation root") root, deps = tree[0], tree[1:] - features = sorted(f for f in root.group("feats").split(",") if f) + features = sorted(root.features) package = next( p for p in full["packages"] if p["name"] == name and p["version"] == version ) @@ -349,7 +487,7 @@ def lib_label(dep_name, dep_version): dep_lib = next(t for t in dep["targets"] if "lib" in t["kind"]) return f"@{CRATE_HUB}__{dep_name}-{dep_version}//:{dep_lib['name'].replace('-', '_')}" - dep_labels = sorted(lib_label(m.group("name"), m.group("ver")) for m in deps) + dep_labels = sorted(lib_label(row.name, row.version) for row in deps) if has_build: dep_labels.append(f"@{repo}//:build_script_build") crate_root = Path(lib["src_path"]).relative_to(Path(package["manifest_path"]).parent).as_posix() @@ -585,13 +723,16 @@ def test_resource_globs(p, t, extra_dirs=()): def main(): if sys.argv[1:] not in ([], ["--check"]): - sys.exit("usage: gen_builds.py [--check]") + sys.exit("usage: bazel run //scripts:gen_builds [-- --check]") check_only = sys.argv[1:] == ["--check"] - meta = cargo_metadata() - opt_default, opt_levels = perf_opt_levels() + query = CargoQuery.from_bazel() + workspace = query.workspace + meta = cargo_metadata(query) members = {p["name"]: p for p in meta["packages"]} + validate_production_features(members) + opt_default, opt_levels = perf_opt_levels(workspace) dir_of = { - p["name"]: Path(p["manifest_path"]).parent.relative_to(REPO).as_posix() + p["name"]: Path(p["manifest_path"]).parent.relative_to(workspace).as_posix() for p in meta["packages"] } rel_src = lambda p, t: Path(t["src_path"]).relative_to(Path(p["manifest_path"]).parent).as_posix() @@ -653,7 +794,7 @@ def resolution(key): # Each distinct resolution is an independent `cargo tree` walk. walks = list(dict.fromkeys(resolution(key) for key in contexts)) with ThreadPoolExecutor() as pool: - walked = dict(zip(walks, pool.map(lambda key: feature_map(*key), walks))) + walked = dict(zip(walks, pool.map(lambda key: feature_map(query, *key), walks))) contexts = {key: walked[resolution(key)] for key in contexts} feature_tables = {p["name"]: p.get("features") or {} for p in meta["packages"]} @@ -806,7 +947,7 @@ def srcs_for(p, t): crate_dir = Path(p["manifest_path"]).parent rel = rel_src(p, t) if rel.startswith(".."): - repo_rel = Path(t["src_path"]).resolve().relative_to(REPO) + repo_rel = Path(t["src_path"]).resolve().relative_to(workspace) glabel, pkg_dir, pkg_rel = group_label(repo_rel, sources=True) return (f'["{glabel}"]', f'"//{pkg_dir}:{pkg_rel}"') d = Path(rel).parent @@ -835,7 +976,7 @@ def extra_mod_srcs(p, t, covered=()): for m in RE_PATH_ATTR.finditer(f.read_text(encoding="utf-8")): resolved = (f.parent / m.group(1)).resolve() try: - repo_rel = resolved.relative_to(REPO) + repo_rel = resolved.relative_to(workspace) except ValueError: continue if resolved.parent not in pulled_dirs: @@ -877,7 +1018,7 @@ def extra_compile_data(p, t, extra_dirs=(), resource_dirs=RESOURCE_DIRS): ] for resolved in include_hits: try: - repo_rel = resolved.relative_to(REPO) + repo_rel = resolved.relative_to(workspace) except ValueError: continue if crate_dir in resolved.parents or resolved.parent == crate_dir: @@ -1460,7 +1601,7 @@ def splice_generated(existing, block, path): return existing.rstrip("\n") + "\n\n" + block + "\n" end = existing.find(GEN_END, begin) if end == -1: - sys.exit(f"{path.relative_to(REPO)}: unterminated {GEN_BEGIN!r} block") + sys.exit(f"{path.relative_to(workspace)}: unterminated {GEN_BEGIN!r} block") end += len(GEN_END) if existing[end : end + 1] == "\n": end += 1 @@ -1492,7 +1633,7 @@ def render_shared(path, pkg_dir): # rules_rust matches per-crate flags on a prefix of the crate root's exec # path, which carries the canonical name of the crate_universe repository # (`rules_rust++crate+__-`). - locked = tomllib.loads((REPO / "Cargo.lock").read_text(encoding="utf-8"))["package"] + locked = tomllib.loads((workspace / "Cargo.lock").read_text(encoding="utf-8"))["package"] external_flags = [] for package in sorted(locked, key=lambda pkg: (pkg["name"], pkg["version"])): level = opt_levels.get(package["name"]) @@ -1502,7 +1643,7 @@ def render_shared(path, pkg_dir): external_flags.append( f'build "--@rules_rust//rust/settings:per_crate_rustc_flag={prefix}@-Copt-level={level}"' ) - bazelrc = REPO / ".bazelrc" + bazelrc = workspace / ".bazelrc" rc_block = ( GEN_BEGIN + "\n# Cargo perf opt-levels for external crates (see [profile.perf]).\n" @@ -1511,9 +1652,9 @@ def render_shared(path, pkg_dir): + GEN_END ) outputs[bazelrc] = splice_generated(bazelrc.read_text(encoding="utf-8"), rc_block, bazelrc) - module = REPO / "MODULE.bazel" + module = workspace / "MODULE.bazel" outputs[module] = splice_generated( - module.read_text(encoding="utf-8"), target_feature_annotations(locked), module + module.read_text(encoding="utf-8"), target_feature_annotations(query, locked), module ) if unknown := sorted(set(TEST_DATA) - test_data_used): @@ -1525,7 +1666,7 @@ def render_shared(path, pkg_dir): member_dirs = set(dir_of.values()) obsolete = [] for pkg_dir in sorted(by_pkg): - path = REPO / pkg_dir / "BUILD.bazel" + path = workspace / pkg_dir / "BUILD.bazel" if pkg_dir in member_dirs: # Member BUILDs are fully rendered above; append the filegroups # other members reach into this one's subtree for. @@ -1542,11 +1683,11 @@ def render_shared(path, pkg_dir): "scripts": {"2024": {'["bazel/rust_test_environment_probe.rs"]'}}, "sdks/codegen": {"2024": {'["dashboard_schema.rs"]'}}, }.items(): - path = REPO / package / "BUILD.bazel" + path = workspace / package / "BUILD.bazel" rules = formatting_rules(package, sources) block = GEN_BEGIN + "\n" + "\n\n".join(rules) + "\n" + GEN_END outputs[path] = splice_generated(path.read_text(encoding="utf-8"), block, path) - root_build = REPO / "BUILD.bazel" + root_build = workspace / "BUILD.bazel" format_aggregate = ( GEN_BEGIN + '\nfilegroup(\n name = "rustfmt_check",\n' + " srcs = [" + q(sorted(formatting_targets)) + "],\n" @@ -1555,7 +1696,7 @@ def render_shared(path, pkg_dir): outputs[root_build] = splice_generated(root_build.read_text(encoding="utf-8"), format_aggregate, root_build) # A generated block or fully generated file can also go stale when its # last Cargo reference disappears; re-render the marked ones too. - for path in sorted(REPO.glob("*/BUILD.bazel")): + for path in sorted(workspace.glob("*/BUILD.bazel")): if path.resolve() in outputs or not path.exists(): continue text = path.read_text(encoding="utf-8") @@ -1576,15 +1717,17 @@ def render_shared(path, pkg_dir): ] # A removed member leaves its generated BUILD behind. stale += [ - path for path in sorted(REPO.glob("crates/*/BUILD.bazel")) - if path not in outputs and path.read_text(encoding="utf-8").startswith(BANNER) + path for path in sorted(workspace.glob("crates/*/BUILD.bazel")) + if path not in outputs and path.read_text(encoding="utf-8").startswith( + "# Generated by scripts/bazel/gen_builds.py" + ) ] stale += sorted(obsolete) if stale: print("generated BUILD.bazel files are out of date:") for path in stale: - print(f" {path.relative_to(REPO)}") - print("rerun: bazel run //scripts:bazel/gen_builds.py") + print(f" {path.relative_to(workspace)}") + print("rerun: bazel run //scripts:gen_builds") # Show the first diffs so CI output explains the divergence. import difflib shown = 0 @@ -1592,12 +1735,12 @@ def render_shared(path, pkg_dir): if shown >= 60: break if path not in outputs: - print(f"--- {path.relative_to(REPO)}: generated file with no source member") + print(f"--- {path.relative_to(workspace)}: generated file with no source member") continue diff = list(difflib.unified_diff( path.read_text(encoding="utf-8").splitlines(), outputs[path].splitlines(), - str(path.relative_to(REPO)), "regenerated", lineterm="", + str(path.relative_to(workspace)), "regenerated", lineterm="", )) for line in diff[:60 - shown]: print(line) @@ -1615,7 +1758,7 @@ def render_shared(path, pkg_dir): # reference is gone; remove them so --check stops flagging the same # stale file. Handwritten files keep their (now empty-free) content. path.unlink() - print(f"removed obsolete generated {path.relative_to(REPO)}") + print(f"removed obsolete generated {path.relative_to(workspace)}") counts = { m: sorted(tname(m, kind, c) for kind, c in v) @@ -1627,4 +1770,7 @@ def render_shared(path, pkg_dir): if __name__ == "__main__": - main() + try: + main() + except (OSError, ValueError) as error: + sys.exit(str(error)) diff --git a/scripts/check-production-feature-profile.py b/scripts/check-production-feature-profile.py deleted file mode 100755 index ea12e82500..0000000000 --- a/scripts/check-production-feature-profile.py +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env python3 -"""Prove the production Cargo graph does not enable test-transport.""" - -from __future__ import annotations - -import argparse -from pathlib import Path -import subprocess -import tomllib - - -def resolved_features(repo: Path, *arguments: str) -> dict[str, set[str]]: - completed = subprocess.run( - [ - "cargo", - "tree", - "--locked", - "--package", - "tracedecay", - "--edges", - "normal,build", - "--prefix", - "none", - "--format", - "{p}|{f}", - *arguments, - ], - cwd=repo, - check=True, - capture_output=True, - encoding="utf-8", - ) - result: dict[str, set[str]] = {} - for line in completed.stdout.splitlines(): - package_id, separator, feature_list = line.partition("|") - if not separator: - raise SystemExit(f"cargo tree emitted an invalid package row: {line}") - package_id = package_id.removesuffix(" (*)") - features = {feature for feature in feature_list.split(",") if feature} - result.setdefault(package_id, set()).update(features) - return result - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument( - "--repo", - type=Path, - default=Path(__file__).resolve().parent.parent, - ) - arguments = parser.parse_args() - repo = arguments.repo.resolve() - - # The product package moved to `crates/tracedecay`; the repository root is - # now a virtual workspace manifest that declares no features at all. - package_manifest = repo.joinpath("crates", "tracedecay", "Cargo.toml") - if not package_manifest.is_file(): - raise SystemExit(f"missing product package manifest at {package_manifest}") - with package_manifest.open("rb") as handle: - manifest = tomllib.load(handle) - features = manifest.get("features", {}) - if features.get("default") != ["production"]: - raise SystemExit("default feature must delegate only to production") - required_production = {"token-counting", "lite", "full"} - if not required_production.issubset(set(features.get("production", []))): - raise SystemExit("production feature set lost a required member") - if "test-transport" in features["production"]: - raise SystemExit("production feature directly enables test-transport") - # `default` is exactly `["production"]`, so a second default-feature walk - # repeats this graph. `cargo metadata` unifies dev-dependency features - # across the workspace and makes test-only transports look reachable; - # this normal/build tree is the artifact `cargo build` produces. - production_graph = resolved_features( - repo, "--no-default-features", "--features", "production" - ) - root_id = next( - ( - package_id - for package_id in production_graph - if package_id.startswith("tracedecay ") - ), - None, - ) - if root_id is None: - raise SystemExit("cargo tree omitted the tracedecay root package") - contaminated = [ - package_id - for package_id, package_features in production_graph.items() - if "test-transport" in package_features - ] - if contaminated: - raise SystemExit( - "production graph enables test-transport for: " + ", ".join(contaminated) - ) - - print("production Cargo feature graph excludes test-transport") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/test-configure-ci-fleet.py b/scripts/test-configure-ci-fleet.py new file mode 100644 index 0000000000..483ecaacac --- /dev/null +++ b/scripts/test-configure-ci-fleet.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Exercise CI executor selection against real reachable and closed sockets.""" + +from __future__ import annotations + +import importlib.util +import os +import shlex +import socket +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +HELPER = Path(__file__).parent.parent / ".github/actions/setup-bazel/configure_fleet.py" +SPEC = importlib.util.spec_from_file_location("configure_ci_fleet", HELPER) +assert SPEC is not None and SPEC.loader is not None +FLEET = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(FLEET) + + +class ConfigureCiFleetTest(unittest.TestCase): + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.workspace = Path(self.temp.name) + self.output = self.workspace / "output" + self.rc = self.workspace / ".bazelrc.local" + self.endpoint = socket.socket() + self.addCleanup(self.endpoint.close) + self.endpoint.bind(("127.0.0.1", 0)) + + def configure( + self, outcome: str = "success", runner_os: str = "Linux", arch: str = "X64" + ) -> bool: + host, port = self.endpoint.getsockname() + return FLEET.configure_fleet( + self.workspace, self.output, outcome, runner_os, arch, host, port + ) + + def options(self) -> dict[str, str]: + return { + option.partition("=")[0]: option.partition("=")[2] + for line in self.rc.read_text().splitlines() + for option in shlex.split(line, comments=True)[1:] + } + + def test_reachable_executor_prefers_remote_rust_without_hosted_fallback(self) -> None: + self.endpoint.listen() + self.assertTrue(self.configure()) + options = self.options() + self.assertEqual(options["--config"], "rbe-linux") + self.assertIn("--noremote_local_fallback", options) + self.assertEqual(options["--remote_retries"], "10") + strategies = [ + line for line in self.rc.read_text().splitlines() if "--strategy=" in line + ] + self.assertIn("build --strategy=Rustc=remote", strategies) + self.assertIn("build --strategy=TestRunner=dynamic,local", strategies) + self.assertEqual(self.output.read_text(), "enabled=true\n") + + def test_offline_executor_clears_previous_fleet_selection(self) -> None: + self.rc.write_text("build --remote_executor=grpc://old-executor:50051\n") + self.assertFalse(self.configure()) + self.assertEqual(self.options(), {}) + self.assertEqual(self.output.read_text(), "enabled=false\n") + + def test_failed_enrollment_does_not_contact_executor(self) -> None: + self.endpoint.listen() + self.endpoint.setblocking(False) + self.assertFalse(self.configure(outcome="failure")) + self.assertEqual(self.options(), {}) + self.assertEqual(self.output.read_text(), "enabled=false\n") + with self.assertRaises(BlockingIOError): + self.endpoint.accept() + + def test_no_identity_keeps_hosted_configuration(self) -> None: + self.assertFalse(self.configure(outcome="skipped")) + self.assertEqual(self.options(), {}) + self.assertEqual(self.output.read_text(), "enabled=false\n") + + def test_configuration_write_failure_is_not_treated_as_fleet_outage(self) -> None: + self.endpoint.listen() + self.rc.mkdir() + with self.assertRaises(OSError): + self.configure() + self.assertFalse(self.output.exists()) + + def test_action_helper_runs_from_a_separate_release_checkout(self) -> None: + result = subprocess.run( + [sys.executable, str(HELPER.absolute())], + cwd=self.workspace, + env={ + **os.environ, + "GITHUB_OUTPUT": str(self.output), + "TAILSCALE_OUTCOME": "skipped", + "RUNNER_OS": "Linux", + "RUNNER_ARCH": "X64", + }, + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.options(), {}) + self.assertEqual(self.output.read_text(), "enabled=false\n") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test-gen-builds-default-alias.py b/scripts/test-gen-builds-default-alias.py index 5dadadb25b..1d4f9c623b 100755 --- a/scripts/test-gen-builds-default-alias.py +++ b/scripts/test-gen-builds-default-alias.py @@ -1,12 +1,20 @@ #!/usr/bin/env python3 -"""Behavioral tests for Bazel feature aliases and test input ownership.""" - from __future__ import annotations import importlib.util +import io +import json +import os +import subprocess +import sys import tempfile import unittest +from contextlib import redirect_stderr +from dataclasses import replace from pathlib import Path +from unittest import mock + +from python.runfiles import runfiles SCRIPT = Path(__file__).resolve().parent / "bazel" / "gen_builds.py" @@ -20,6 +28,337 @@ def load_gen_builds(): return module +class ProductionFeatureResolutionTests(unittest.TestCase): + ROOT = ( + "tracedecay v0.1.0 (/workspace/crates/tracedecay)|" + "default,full,lite,production,token-counting" + ) + ROOT_FEATURES = {"default", "full", "lite", "production", "token-counting"} + + def setUp(self) -> None: + self.gen = load_gen_builds() + + def resolve(self, output, package="tracedecay", edges="normal,build", features=()): + query = mock.Mock(spec=self.gen.CargoQuery) + query.run.return_value = output + return self.gen.feature_map(query, package, edges, features) + + def test_clean_graph_projects_members_and_unions_repeated_rows(self) -> None: + output = "\n".join([ + self.ROOT, + "tracedecay-code-index v0.1.0 (/workspace/crates/tracedecay-code-index)|lite", + "serde v1.0.0|default,derive", + "syn v2.0.0 (registry `alternate`)|full", + "git-dep v1.2.3-beta.1+build (https://example.invalid/crates/git-dep?rev=main#01234567)|std", + "path-dep v0.2.0 (/vendor/path-dep)|", + "derive-dep v1.0.0 (proc-macro)|", + "git-derive v1.0.0 (proc-macro) (file:///vendor/crates/git-derive#01234567)|derive", + "path-derive v1.0.0 (proc-macro) (/vendor/path-derive)|derive", + "registry-derive v1.0.0 (proc-macro) (registry `https://example.invalid/crates/index`)|derive", + "tracedecay-macros v0.1.0 (proc-macro) (/workspace/crates/tracedecay-macros)|expand", + "tracedecay-code-index v0.1.0 (/workspace/crates/tracedecay-code-index)|full (*)", + "tracedecay-empty v0.1.0 (/workspace/crates/tracedecay-empty)| (*)", + "", + ]) + self.assertEqual(self.resolve(output), { + "tracedecay": self.ROOT_FEATURES, + "tracedecay-code-index": {"lite", "full"}, + "tracedecay-macros": {"expand"}, + "tracedecay-empty": set(), + }) + + def test_production_rejects_transport_including_external_repeated_rows(self) -> None: + packages = [ + "tracedecay v0.1.0 (/workspace/crates/tracedecay)", + "member v0.1.0 (/workspace/crates/member)", + "registry-dep v1.0.0", + "alt-dep v1.0.0 (registry `alternate`)", + "git-dep v1.0.0 (https://example.invalid/dependency?branch=main#01234567)", + "path-dep v1.0.0 (/vendor/path-dep)", + "derive-dep v1.0.0 (proc-macro)", + "alt-derive v1.0.0 (proc-macro) (registry `alternate`)", + "git-derive v1.0.0 (proc-macro) (https://example.invalid/derive#01234567)", + "path-derive v1.0.0 (proc-macro) (/vendor/path-derive)", + ] + for package in packages: + with self.subTest(package=package): + output = f"{self.ROOT}\n{package}|std\n{package}|std,test-transport (*)\n" + with self.assertRaises(ValueError) as raised: + self.resolve(output) + self.assertIn("test-transport", str(raised.exception)) + self.assertIn(package, str(raised.exception)) + + def test_dev_and_explicit_test_contexts_keep_transport_features(self) -> None: + output = "\n".join([ + self.ROOT + ",test-transport", + "external v1.0.0|test-transport (*)", + "", + "member v0.1.0 (/workspace/crates/member)|test-transport", + "", + ]) + contexts = [ + (None, "normal,build,dev", ("tracedecay/test-transport",)), + ("tracedecay", "normal,build,dev", ()), + ("tracedecay", "normal,build", ("test-transport",)), + ("member", "normal,build", ()), + ] + for context in contexts: + with self.subTest(context=context): + self.assertEqual(self.resolve(output, *context), { + "tracedecay": self.ROOT_FEATURES | {"test-transport"}, + "member": {"test-transport"}, + }) + with self.assertRaises(ValueError): + self.resolve(output) + + def test_nonrepeated_external_transport_is_rejected(self) -> None: + with self.assertRaises(ValueError): + self.resolve(f"{self.ROOT}\nexternal v1.0.0|test-transport\n") + + def test_lean_context_preserves_empty_features_and_locked_request(self) -> None: + output = "tracedecay v0.1.0 (/workspace/crates/tracedecay)|\n" + query = mock.Mock(spec=self.gen.CargoQuery) + query.run.return_value = output + self.assertEqual(self.gen.feature_map( + query, "tracedecay", "normal,build", (self.gen.NO_DEFAULT_FEATURES,), + ), {"tracedecay": set()}) + command = query.run.call_args.args[0] + self.assertIn("--locked", command) + self.assertIn("--no-default-features", command) + self.assertNotIn("--features", command) + + def test_missing_production_root_is_rejected(self) -> None: + for output in ( + "", "\n\n", "serde v1.0.0|default\n", + "member v0.1.0 (/workspace/crates/member)|\n", + "tracedecay v0.1.0|production\n", + "tracedecay v0.1.0 (https://example.invalid/crates/tracedecay#01234567)|production\n", + ): + with self.subTest(output=output), self.assertRaises(ValueError): + self.resolve(output) + + def test_malformed_external_rows_after_valid_root_are_rejected(self) -> None: + for row in ( + "not a package row", + "external v1.0.0", + "external vgarbage|std", + "external v1.0.0 (proc-macro) (/vendor/external|std", + "external v1.0.0 (unknown source)|std", + "external v1.0.0|std|test-transport", + "external v1.0.0|std,,test-transport", + "external v1.0.0|std,", + "external v1.0.0|std (*) ,test-transport", + "external v1.0.0|test-(*)transport", + "external v1.0.0 (*)|std", + "external v1.0.0|std (*) (*)", + ): + with self.subTest(row=row), self.assertRaises(ValueError): + self.resolve(f"{self.ROOT}\n{row}\n") + + def test_native_paths_and_parentheses_remain_member_paths(self) -> None: + for path in ( + "/workspace (local)/crates/tracedecay", + "/workspace|local/crates/tracedecay", + r"C:\workspace\crates\tracedecay", + "C:/workspace/crates/tracedecay", + r"\\server\workspace\crates\tracedecay", + ): + with self.subTest(path=path): + self.assertEqual(self.resolve( + f"tracedecay v0.1.0 ({path})|production\n", + ), {"tracedecay": {"production"}}) + + +class ProductionDeclarationTests(unittest.TestCase): + def setUp(self) -> None: + self.gen = load_gen_builds() + self.features = { + "default": ["production"], + "production": ["token-counting", "lite", "full"], + "token-counting": [], + "lite": [], + "full": [], + } + + def test_invalid_declarations_fail_before_resolution_in_both_cli_modes(self) -> None: + invalid_tables = [ + {}, + self.features | {"default": []}, + self.features | {"default": ["production", "test-transport"]}, + self.features | {"production": []}, + self.features | {"production": ["token-counting", "lite", "full", "test-transport"]}, + ] + invalid_tables.extend( + self.features | {"production": [f for f in self.features["production"] if f != missing]} + for missing in ("token-counting", "lite", "full") + ) + packages = [[], [{"name": "tracedecay"}]] + [ + [{"name": "tracedecay", "features": table}] for table in invalid_tables + ] + for arguments in ([], ["--check"]): + for declaration in packages: + with self.subTest(arguments=arguments, packages=declaration): + query = mock.Mock(spec=self.gen.CargoQuery, workspace=Path("/unused")) + query.run.return_value = json.dumps({ + "packages": declaration, + }) + with mock.patch.object(self.gen.CargoQuery, "from_bazel", return_value=query), \ + mock.patch.object(sys, "argv", [str(SCRIPT), *arguments]): + with self.assertRaises(ValueError): + self.gen.main() + self.assertEqual(query.run.call_count, 1) + command = query.run.call_args.args[0] + self.assertEqual(command[0], "metadata") + self.assertIn("--locked", command) + + def test_valid_declaration_reaches_graph_policy_before_any_output_write(self) -> None: + for arguments in ([], ["--check"]): + with self.subTest(arguments=arguments), tempfile.TemporaryDirectory() as scratch: + root = Path(scratch).resolve() + product = root / "crates/tracedecay" + product.mkdir(parents=True) + build = product / "BUILD.bazel" + build.write_text("existing output\n", encoding="utf-8") + (root / "Cargo.toml").write_text("[profile.dev]\n", encoding="utf-8") + metadata = json.dumps({"packages": [{ + "name": "tracedecay", + "features": self.features, + "manifest_path": str(product / "Cargo.toml"), + "dependencies": [], + "targets": [], + }]}) + + def cargo_output(command, **kwargs): + self.assertIn("--locked", command) + if command[0] == "metadata": + return metadata + self.assertEqual(command[0], "tree") + return ( + f"tracedecay v0.1.0 ({product})|production\n" + "external v1.0.0 (proc-macro)|test-transport (*)\n" + ) + + query = mock.Mock(spec=self.gen.CargoQuery, workspace=root) + query.run.side_effect = cargo_output + with mock.patch.object(self.gen.CargoQuery, "from_bazel", return_value=query), \ + mock.patch.object(sys, "argv", [str(SCRIPT), *arguments]): + with self.assertRaises(ValueError) as raised: + self.gen.main() + self.assertIn("test-transport", str(raised.exception)) + self.assertEqual(build.read_text(encoding="utf-8"), "existing output\n") + self.assertEqual(set(root.rglob("BUILD.bazel")), {build}) + + +class CargoAcquisitionTests(unittest.TestCase): + def setUp(self) -> None: + self.gen = load_gen_builds() + scratch = tempfile.TemporaryDirectory(prefix="cargo query ", dir=os.environ["TEST_TMPDIR"]) + self.addCleanup(scratch.cleanup) + root = Path(scratch.name).resolve() + self.workspace = root / "workspace with spaces" + self.workspace.mkdir() + (self.workspace / "Cargo.toml").write_text("[workspace]\n", encoding="utf-8") + files = runfiles.Create() + self.assertIsNotNone(files) + toolchain = files.Rlocation(os.environ["GEN_BUILDS_TOOLCHAIN"], source_repo="") + self.assertIsNotNone(toolchain) + (self.workspace / "rust-toolchain.toml").write_bytes(Path(toolchain).read_bytes()) + env = { + key: value for key, value in os.environ.items() + if not key.startswith(("CARGO", "RUSTC", "RUSTUP", "__CARGO_TEST")) + } + for variable, directory in ( + ("CARGO_HOME", "cargo home"), + ("RUSTUP_HOME", "rustup home"), + ("HOME", "home"), + ): + path = root / directory + path.mkdir() + env[variable] = str(path) + env.update( + BUILD_WORKSPACE_DIRECTORY=str(self.workspace), + USERPROFILE=env["HOME"], + PATH="", + ) + environment = mock.patch.dict(os.environ, env, clear=True) + environment.start() + self.addCleanup(environment.stop) + + def test_declared_tools_load_acquisition_policy_with_origin(self) -> None: + query = self.gen.CargoQuery.from_bazel() + for key, value in ( + ("unstable.git", '"shallow-deps"'), + ("net.git-fetch-with-cli", "true"), + ): + with self.subTest(key=key): + output = query.run(["-Zunstable-options", "config", "get", key, "--show-origin"]) + setting, origin = output.strip().split(" # ", 1) + self.assertEqual(setting, f"{key} = {value}") + self.assertIn(str(query.policy), origin) + self.assertNotIn("RUSTC_BOOTSTRAP", os.environ) + self.assertNotIn("RUSTC", os.environ) + + def test_cargo_rejects_array_policy_and_unactivated_inspection(self) -> None: + query = self.gen.CargoQuery.from_bazel() + args = ["-Zunstable-options", "config", "get", "unstable.git", "--show-origin"] + query.run(args) + malformed = self.workspace / "invalid-acquisition.toml" + malformed.write_text('[unstable]\ngit = ["shallow-deps"]\n', encoding="utf-8") + with self.assertRaises(SystemExit) as raised: + replace(query, policy=malformed).run(args) + self.assertNotEqual(raised.exception.code, 0) + unactivated = subprocess.run( + [str(query.cargo), "--config", str(query.policy), *args], + cwd=query.workspace, + env=os.environ.copy(), + capture_output=True, + text=True, + encoding="utf-8", + ) + self.assertNotEqual(unactivated.returncode, 0) + self.assertIn("nightly", unactivated.stderr.lower()) + + def test_missing_declared_tool_and_version_mismatch_fail(self) -> None: + with mock.patch.dict(os.environ, {"GEN_BUILDS_CARGO": ""}): + with self.assertRaisesRegex(ValueError, "GEN_BUILDS_CARGO"): + self.gen.CargoQuery.from_bazel() + (self.workspace / "rust-toolchain.toml").write_text( + '[toolchain]\nchannel = "0.0.0"\n', encoding="utf-8", + ) + with self.assertRaisesRegex(ValueError, "must match rust-toolchain.toml"): + self.gen.CargoQuery.from_bazel() + + def test_cli_requires_workspace_and_rejects_unknown_arguments(self) -> None: + with mock.patch.dict(os.environ, {"BUILD_WORKSPACE_DIRECTORY": ""}): + for args in ([], ["--check"]): + with self.subTest(args=args), mock.patch.object(sys, "argv", [str(SCRIPT), *args]): + with self.assertRaisesRegex(ValueError, "bazel run //scripts:gen_builds"): + self.gen.main() + with mock.patch.object(sys, "argv", [str(SCRIPT), "--unknown"]): + with self.assertRaises(SystemExit) as raised: + self.gen.main() + self.assertIn("usage:", raised.exception.code) + + +class ResolverProcessTests(unittest.TestCase): + def test_stdout_remains_protocol_data(self) -> None: + gen = load_gen_builds() + diagnostics = io.StringIO() + with tempfile.TemporaryDirectory() as scratch, redirect_stderr(diagnostics): + output = gen.run([ + sys.executable, "-c", "print('package v1.0.0|std')", + ], cwd=scratch) + self.assertEqual(output, "package v1.0.0|std\n") + self.assertTrue(diagnostics.getvalue()) + + def test_nonzero_exit_remains_failure(self) -> None: + gen = load_gen_builds() + with tempfile.TemporaryDirectory() as scratch, redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as raised: + gen.run([sys.executable, "-c", "raise SystemExit(23)"], cwd=scratch) + self.assertEqual(raised.exception.code, 23) + + class DefaultAliasNormalizationTests(unittest.TestCase): @classmethod def setUpClass(cls) -> None: diff --git a/scripts/test-neutralize-release-pr-closing-keywords.py b/scripts/test-neutralize-release-pr-closing-keywords.py index 936fd203ec..1c6cc784c6 100644 --- a/scripts/test-neutralize-release-pr-closing-keywords.py +++ b/scripts/test-neutralize-release-pr-closing-keywords.py @@ -179,8 +179,7 @@ def git(*args: str, cwd: Path = main) -> None: "neutralize-release-pr-closing-keywords.py", ): shutil.copy2(ROOT / "scripts" / name, main / "scripts" / name) - (main / "scripts" / "bazel").mkdir() - generator = main / "scripts" / "bazel" / "gen_builds.py" + generator = main / "scripts" / "generate-fixture-builds.py" generator.write_text( bazel_generator or """from pathlib import Path @@ -244,9 +243,9 @@ def git(*args: str, cwd: Path = main) -> None: ' "fetch --repo=@c")\n' ' [ "${CARGO_BAZEL_REPIN:-}" = workspace ] || exit 65\n' " printf 'version = 4\\n# refreshed by Bazel\\n' > Cargo.lock ;;\n" - ' "run //scripts:bazel/gen_builds.py")\n' + ' "run //scripts:gen_builds")\n' ' grep -q "# refreshed by Bazel" Cargo.lock || exit 66\n' - ' exec python3 scripts/bazel/gen_builds.py ;;\n' + ' exec python3 scripts/generate-fixture-builds.py ;;\n' ' *) exit 64 ;;\n' 'esac\n' ), diff --git a/scripts/update-release-pr-lockfile.sh b/scripts/update-release-pr-lockfile.sh index 7993284818..58536e38f0 100755 --- a/scripts/update-release-pr-lockfile.sh +++ b/scripts/update-release-pr-lockfile.sh @@ -20,7 +20,7 @@ RELEASE_PR_BRANCH=$( ) pnpm install --frozen-lockfile CARGO_BAZEL_REPIN=workspace bazel fetch --repo=@c -bazel run //scripts:bazel/gen_builds.py +bazel run //scripts:gen_builds pnpm --dir sdks/typescript run build pnpm --dir plugin/chatgpt-extension run build diff --git a/tests/BUILD.bazel b/tests/BUILD.bazel index ddda3cb21f..175bfff9e1 100644 --- a/tests/BUILD.bazel +++ b/tests/BUILD.bazel @@ -1,5 +1,5 @@ # Generated by scripts/bazel/gen_builds.py from Cargo.toml + Cargo.lock. -# Regenerate instead of editing. bazel run //scripts:bazel/gen_builds.py +# Regenerate instead of editing. bazel run //scripts:gen_builds # BEGIN generated by scripts/bazel/gen_builds.py filegroup(