diff --git a/.changes/preserve-unconditional-risc-v-1146.md b/.changes/preserve-unconditional-risc-v-1146.md new file mode 100644 index 00000000..4910c33a --- /dev/null +++ b/.changes/preserve-unconditional-risc-v-1146.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Preserve unconditional RISC-V ECDSA generator-table loads when LLVM optimizes masked selection. diff --git a/.config/criterion.json b/.config/criterion.json index d9ba70bd..734da31e 100644 --- a/.config/criterion.json +++ b/.config/criterion.json @@ -6,5 +6,5 @@ "confidence_level": 0.95, "significance_level": 0.05, "noise_threshold": 0.01, - "max_run_seconds": 3600 + "max_run_seconds": 5400 } diff --git a/.config/tooling.toml b/.config/tooling.toml index b5d756d9..188ded65 100644 --- a/.config/tooling.toml +++ b/.config/tooling.toml @@ -43,12 +43,12 @@ packages = ["build-essential", "ca-certificates", "curl", "git", "jq", "pkgconf" [ci] cargo = ["just", "cargo-nextest"] -[ci-riscv-build] +[ci-cross-build] cargo = ["just", "cargo-nextest"] -packages = ["build-essential", "ca-certificates", "curl", "git", "jq", "python3", "cmake", "clang", "libclang-dev", "perl", "gcc-riscv64-linux-gnu", "g++-riscv64-linux-gnu", "libc6-dev-riscv64-cross"] +packages = ["build-essential", "ca-certificates", "curl", "git", "jq", "python3", "cmake", "clang", "libclang-dev", "perl"] -[ci-riscv-run] -cargo = ["just", "cargo-nextest"] +[ci-cross-run] +cargo = [] packages = ["ca-certificates", "curl", "git", "python3"] [ci-compat] diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 00000000..15e7440d --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(git rev-parse --show-toplevel)" +# Test the staged source, not a different working-tree version. +git diff --quiet || { echo 'Stage or set aside unstaged tracked changes before committing.' >&2; exit 1; } +if [[ -n "$(git ls-files --others --exclude-standard)" ]]; then + echo 'Stage intended new files or move unrelated untracked files outside the checkout before committing.' >&2 + exit 1 +fi +before=$(git write-tree) +just check-macos +git diff --quiet && [[ "$(git write-tree)" == "$before" && -z "$(git ls-files --others --exclude-standard)" ]] || { + echo 'Source changed during macOS validation; review and retry the commit.' >&2 + exit 1 +} diff --git a/.githooks/pre-merge-commit b/.githooks/pre-merge-commit new file mode 100755 index 00000000..7fc385e4 --- /dev/null +++ b/.githooks/pre-merge-commit @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +set -euo pipefail +exec "$(git rev-parse --show-toplevel)/.githooks/pre-commit" diff --git a/.github/runs-on.yml b/.github/runs-on.yml index 5e7aefad..a7d21bd5 100644 --- a/.github/runs-on.yml +++ b/.github/runs-on.yml @@ -1,23 +1,75 @@ -# AWS runners. Other providers are selected directly in workflows. +# AWS runner shapes. Workflows reference these names; other providers keep their own labels. runners: + plan: + family: [c7a.large, c7i.large, c8a.large, c8i.large] + cpu: 2 + image: ubuntu24-minimal-x64 + spot: price-capacity-optimized + volume: 40gb:gp3 + miri: + family: [c7a.xlarge, c7i.xlarge, c8a.xlarge, c8i.xlarge] + cpu: 4 + image: ubuntu24-minimal-x64 + spot: price-capacity-optimized + volume: 150gb:gp3 + package: + family: [c7a.2xlarge, c7i.2xlarge, c8a.2xlarge, c8i.2xlarge] + cpu: 8 + image: ubuntu24-minimal-x64 + spot: price-capacity-optimized + volume: 100gb:gp3 + cross-build: + family: [c7a.8xlarge, c7i.8xlarge, c8a.8xlarge, c8i.8xlarge] + cpu: 32 + image: ubuntu24-minimal-x64 + spot: price-capacity-optimized + volume: 150gb:gp3 x86_64-linux: - family: [c7a, c7i, c8a, c8i, m7a, m7i, m8a, m8i] - cpu: [16, 32] - ram: [32, 128] - image: ubuntu24-full-x64 + family: [c7a.4xlarge, c7i.4xlarge, c8a.4xlarge, c8i.4xlarge] + cpu: 16 + image: ubuntu24-minimal-x64 spot: price-capacity-optimized volume: 150gb:gp3 aarch64-linux: - family: [c7g, c8g, m7g, m8g] - cpu: [16, 32] - ram: [32, 128] - image: ubuntu24-full-arm64 + family: [c7g.4xlarge, c8g.4xlarge, c9g.4xlarge] + cpu: 16 + image: ubuntu24-minimal-arm64 spot: price-capacity-optimized volume: 150gb:gp3 x86_64-win: - family: [c7a, c7i, c8a, c8i, m7a, m7i, m8a, m8i] - cpu: [16, 32] - ram: [32, 128] + family: [c7a.4xlarge, c7i.4xlarge, c8a.4xlarge, c8i.4xlarge] + cpu: 16 image: windows25-full-x64 spot: price-capacity-optimized volume: 200gb:gp3 + # Fixed, non-Flex, On-Demand hardware for timing and benchmark evidence. + measure-x86_64-linux-intel: + family: [c8i.2xlarge] + cpu: 8 + image: ubuntu24-minimal-x64 + spot: false + volume: 100gb:gp3 + measure-x86_64-linux-amd: + family: [c8a.2xlarge] + cpu: 8 + image: ubuntu24-minimal-x64 + spot: false + volume: 100gb:gp3 + measure-x86_64-win-intel: + family: [c8i.2xlarge] + cpu: 8 + image: windows25-full-x64 + spot: false + volume: 100gb:gp3 + measure-x86_64-win-amd: + family: [c8a.2xlarge] + cpu: 8 + image: windows25-full-x64 + spot: false + volume: 100gb:gp3 + measure-aarch64-linux: + family: [c9g.2xlarge] + cpu: 8 + image: ubuntu24-minimal-arm64 + spot: false + volume: 100gb:gp3 diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 4169ed94..ffd0babf 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: architectures: - description: all, or comma-separated x86_64-linux,aarch64-linux,x86_64-win,s390x-linux,powerpc64le-linux,riscv64-linux + description: "x86-64 runs Intel + AMD; all, or comma-separated x86_64-linux,aarch64-linux,x86_64-win,s390x-linux,powerpc64le-linux,riscv64-linux" type: string required: true default: x86_64-linux @@ -49,10 +49,12 @@ defaults: jobs: plan: - runs-on: runs-on=${{ github.run_id }}/family=c7a+c7i+c8a+c8i+m7a+m7i+m8a+m8i/cpu=2/image=ubuntu24-minimal-x64/spot=price-capacity-optimized/env=production + runs-on: runs-on=${{ github.run_id }}/runner=plan/env=production timeout-minutes: 5 outputs: matrix: ${{ steps.plan.outputs.matrix }} + cross: ${{ steps.plan.outputs.cross }} + builds: ${{ steps.plan.outputs.builds }} steps: - uses: "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1" # v7.0.1 with: @@ -60,9 +62,49 @@ jobs: - id: plan run: python3 scripts/bench/ci.py plan - bench: + cross-build: needs: plan - name: ${{ matrix.platform }} + if: needs.plan.outputs.cross == 'true' + name: Prepare ${{ matrix.target }} benchmarks + runs-on: runs-on=${{ github.run_id }}/runner=cross-build/env=production + timeout-minutes: 120 + strategy: + fail-fast: true + matrix: ${{ fromJSON(needs.plan.outputs.builds) }} + env: + TARGET: ${{ matrix.target }} + steps: + - uses: "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1" # v7.0.1 + with: + persist-credentials: false + - name: Cross-build selected benchmark configurations + run: | + scripts/tooling/x86_64-linux.sh --ci-cross-build "$TARGET" + source "$HOME/.local/share/rscrypto-tooling/environment.sh" + python3 scripts/bench/ci.py prepare "$TARGET" "target/$TARGET-bench.tar.gz" + git diff --exit-code HEAD + - name: Transfer exact benchmark binaries and runner tools + uses: "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a" # v7.0.1 + with: + name: ${{ matrix.target }}-bench-prepared + path: | + target/${{ matrix.target }}-bench.tar.gz + target/${{ matrix.target }}-tools.tar.gz + if-no-files-found: error + retention-days: 2 + - name: Retain benchmark preparation evidence + if: ${{ !cancelled() }} + uses: "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a" # v7.0.1 + with: + name: bench-prepare-${{ matrix.target }}-${{ github.run_id }}-${{ github.run_attempt }} + path: target/bench/preparation/ + if-no-files-found: warn + retention-days: 30 + + bench: + needs: [plan, cross-build] + if: ${{ !cancelled() && needs.plan.result == 'success' && (needs.cross-build.result == 'success' || needs.cross-build.result == 'skipped') }} + name: ${{ matrix.name }} strategy: fail-fast: true matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} @@ -73,7 +115,7 @@ jobs: with: persist-credentials: false - name: Install tooling and measure (Linux) - if: runner.os == 'Linux' + if: runner.os == 'Linux' && !matrix.target env: PLATFORM: ${{ matrix.platform }} run: | @@ -81,6 +123,22 @@ jobs: source "$HOME/.local/share/rscrypto-tooling/environment.sh" python3 scripts/bench/ci.py run git diff --exit-code HEAD + - name: Download prepared benchmark binaries and runner tools + if: matrix.target + uses: "actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c" # v8.0.1 + with: + name: ${{ matrix.target }}-bench-prepared + path: target/bench-download + - name: Discover and measure on native hardware + if: matrix.target + env: + PLATFORM: ${{ matrix.platform }} + TARGET: ${{ matrix.target }} + run: | + scripts/tooling/"$PLATFORM.sh" --ci-cross-run "target/bench-download/$TARGET-tools.tar.gz" + source "$HOME/.local/share/rscrypto-tooling/environment.sh" + python3 scripts/bench/ci.py measure "$TARGET" "target/bench-download/$TARGET-bench.tar.gz" + git diff --exit-code HEAD - name: Install tooling and measure (Windows) if: runner.os == 'Windows' shell: powershell @@ -94,7 +152,7 @@ jobs: if: ${{ always() }} uses: "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a" # v7.0.1 with: - name: bench-${{ matrix.platform }}-${{ github.run_id }}-${{ github.run_attempt }} + name: bench-${{ matrix.name }}-${{ github.run_id }}-${{ github.run_attempt }} path: target/bench/ if-no-files-found: warn retention-days: 30 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 70fbf0d2..5b96ee5f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,62 +22,69 @@ defaults: shell: bash jobs: - riscv-build: - name: Prepare RISC-V tests - runs-on: runs-on=${{ github.run_id }}/family=c7a+c7i+c8a+c8i+m7a+m7i+m8a+m8i/cpu=32/ram=64+128/image=ubuntu24-minimal-x64/spot=price-capacity-optimized/volume=150gb:gp3/env=production + cross-build: + name: Prepare ${{ matrix.target }} tests + runs-on: runs-on=${{ github.run_id }}/runner=cross-build/env=production timeout-minutes: 45 + strategy: + fail-fast: true + matrix: + target: [riscv64gc-unknown-linux-gnu, powerpc64le-unknown-linux-gnu, s390x-unknown-linux-gnu] + env: + TARGET: ${{ matrix.target }} steps: - uses: "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1" # v7.0.1 with: persist-credentials: false - - name: Cross-check and prepare complete RISC-V suites + - name: Cross-check and prepare complete target suites run: | - scripts/tooling/x86_64-linux.sh --ci-riscv-build + scripts/tooling/x86_64-linux.sh --ci-cross-build "$TARGET" source "$HOME/.local/share/rscrypto-tooling/environment.sh" just test-transfer - just test-riscv prepare target/riscv-tests.tar.gz + just test-cross prepare "$TARGET" "target/$TARGET-tests.tar.gz" - uses: "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a" # v7.0.1 with: - name: riscv-tests - path: target/riscv-tests.tar.gz + name: ${{ matrix.target }}-tests + path: | + target/${{ matrix.target }}-tests.tar.gz + target/${{ matrix.target }}-tools.tar.gz if-no-files-found: error retention-days: 2 native: - needs: riscv-build + needs: cross-build name: ${{ matrix.platform }} strategy: fail-fast: true matrix: include: - platform: compatibility - runner: runs-on=${{ github.run_id }}/family=c7a+c7i+c8a+c8i+m7a+m7i+m8a+m8i/cpu=32/ram=64+128/image=ubuntu24-minimal-x64/spot=price-capacity-optimized/volume=150gb:gp3/env=production + runner: runs-on=${{ github.run_id }}/runner=cross-build/env=production timeout: 60 - platform: package - runner: runs-on=${{ github.run_id }}/family=c7a+c7i+c8a+c8i+m7a+m7i+m8a+m8i/cpu=8/image=ubuntu24-minimal-x64/spot=price-capacity-optimized/volume=100gb:gp3/env=production + runner: runs-on=${{ github.run_id }}/runner=package/env=production timeout: 60 - platform: x86_64-linux - runner: runs-on=${{ github.run_id }}/family=c7a+c7i+c8a+c8i+m7a+m7i+m8a+m8i/cpu=16+32/ram=32+128/image=ubuntu24-minimal-x64/spot=price-capacity-optimized/volume=150gb:gp3/env=production + runner: runs-on=${{ github.run_id }}/runner=x86_64-linux/env=production timeout: 60 - platform: aarch64-linux - runner: runs-on=${{ github.run_id }}/family=c7g+c8g+m7g+m8g/cpu=16+32/ram=32+128/image=ubuntu24-minimal-arm64/spot=price-capacity-optimized/volume=150gb:gp3/env=production + runner: runs-on=${{ github.run_id }}/runner=aarch64-linux/env=production timeout: 60 - platform: s390x-linux + target: s390x-unknown-linux-gnu runner: ubuntu-24.04-s390x - timeout: 90 + timeout: 30 - platform: powerpc64le-linux + target: powerpc64le-unknown-linux-gnu runner: ubuntu-24.04-ppc64le-p10 - # Cold native release compilation took 41 minutes before the portable build. - timeout: 180 + timeout: 30 - platform: riscv64-linux + target: riscv64gc-unknown-linux-gnu runner: ubuntu-24.04-riscv # Compilation runs on x86-64; this budget covers native execution and bootstrap. timeout: 30 - - platform: aarch64-macos - runner: macos-15 - timeout: 90 - platform: x86_64-win - runner: runs-on=${{ github.run_id }}/family=c7a+c7i+c8a+c8i+m7a+m7i+m8a+m8i/cpu=16+32/ram=32+128/image=windows25-full-x64/spot=price-capacity-optimized/volume=200gb:gp3/env=production + runner: runs-on=${{ github.run_id }}/runner=x86_64-win/env=production timeout: 60 runs-on: ${{ matrix.runner }} timeout-minutes: ${{ matrix.timeout }} @@ -86,7 +93,7 @@ jobs: with: persist-credentials: false - name: Install tooling, check, and test (Linux) - if: runner.os == 'Linux' && matrix.platform != 'compatibility' && matrix.platform != 'package' && matrix.platform != 'riscv64-linux' + if: runner.os == 'Linux' && matrix.platform != 'compatibility' && matrix.platform != 'package' && !matrix.target env: PLATFORM: ${{ matrix.platform }} run: | @@ -105,30 +112,31 @@ jobs: if [[ "$PLATFORM" == x86_64-linux || "$PLATFORM" == aarch64-linux ]]; then just test-musl; fi git diff --exit-code HEAD - - name: Download RISC-V suites from this workflow run - if: matrix.platform == 'riscv64-linux' + - name: Download cross-compiled suites from this workflow run + if: matrix.target uses: "actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c" # v8.0.1 with: - name: riscv-tests - path: target/riscv-download - - name: Execute complete RISC-V suites - if: matrix.platform == 'riscv64-linux' + name: ${{ matrix.target }}-tests + path: target/cross-download + - name: Execute complete cross-compiled suites + if: matrix.target run: | - scripts/tooling/riscv64-linux.sh --ci-riscv-run + scripts/tooling/"${{ matrix.platform }}.sh" --ci-cross-run "target/cross-download/${{ matrix.target }}-tools.tar.gz" source "$HOME/.local/share/rscrypto-tooling/environment.sh" - just test-riscv run target/riscv-download/riscv-tests.tar.gz + just test-cross run "${{ matrix.target }}" "target/cross-download/${{ matrix.target }}-tests.tar.gz" git diff --exit-code HEAD - - name: Package RISC-V execution evidence - if: ${{ !cancelled() && matrix.platform == 'riscv64-linux' }} + - name: Package cross-compiled execution evidence + if: ${{ !cancelled() && matrix.target }} run: | - if [[ -d target/riscv-results ]]; then - tar -czf target/riscv-results.tar.gz -C target riscv-results + if [[ -d target/cross-results ]]; then + tar -czf target/cross-results.tar.gz -C target cross-results fi - - name: Retain RISC-V execution evidence - if: ${{ !cancelled() && matrix.platform == 'riscv64-linux' }} + - name: Retain cross-compiled execution evidence + if: ${{ !cancelled() && matrix.target }} uses: "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a" # v7.0.1 with: - path: target/riscv-results.tar.gz + path: target/cross-results.tar.gz + name: ${{ matrix.target }}-results archive: false if-no-files-found: warn retention-days: 7 @@ -149,15 +157,6 @@ jobs: just ci-compat git diff --exit-code HEAD - - name: Install tooling, check, and test (macOS ARM64) - if: runner.os == 'macOS' - run: | - scripts/tooling/aarch64-macos.sh - just ci-check - just test --all --release - just test --all --release --portable - git diff --exit-code HEAD - - name: Install tooling, check, and test (Windows) if: runner.os == 'Windows' shell: powershell diff --git a/.github/workflows/ct.yml b/.github/workflows/ct.yml index ad7bc42b..3780d5e2 100644 --- a/.github/workflows/ct.yml +++ b/.github/workflows/ct.yml @@ -10,6 +10,14 @@ on: default: all workflow_dispatch: inputs: + diagnose_p384: + description: Diagnostic only; measure the current P-384 binary once on RISC-V (overrides architectures) + type: boolean + default: false + replay_p384: + description: Diagnostic only; replay the P-384 binary from run 34672864167 three times on RISC-V + type: boolean + default: false architectures: description: all, or comma-separated x86_64-linux,aarch64-linux,x86_64-win,s390x-linux,powerpc64le-linux,riscv64-linux type: string @@ -21,7 +29,7 @@ permissions: env: CARGO_RAIL_CACHE: "off" - INPUT_ARCHITECTURES: ${{ inputs.architectures || 'all' }} + INPUT_ARCHITECTURES: ${{ inputs.diagnose_p384 && 'riscv64-linux' || inputs.architectures || 'all' }} concurrency: group: ct-${{ github.workflow }}-${{ github.ref }} @@ -32,12 +40,57 @@ defaults: shell: bash jobs: + replay: + if: github.event_name == 'workflow_dispatch' && inputs.replay_p384 + name: Replay P-384 baseline (diagnostic) + runs-on: ubuntu-24.04-riscv + timeout-minutes: 360 + permissions: + contents: read + actions: read + steps: + - uses: "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1" # v7.0.1 + with: + persist-credentials: false + - uses: "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1" # v7.0.1 + with: + ref: 32734d2dfac7a8015e3410c32b4db5e07484e496 + path: target/ct-replay-source + persist-credentials: false + - uses: "actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c" # v8.0.1 + with: + name: riscv64gc-unknown-linux-gnu-ct-prepared + run-id: 34672864167 + github-token: ${{ github.token }} + path: target/ct-replay-download + - name: Install tools bound to the original source + working-directory: target/ct-replay-source + run: scripts/tooling/riscv64-linux.sh --ci-cross-run "$GITHUB_WORKSPACE/target/ct-replay-download/riscv64gc-unknown-linux-gnu-tools.tar.gz" + - name: Verify original archive and repeat the full sample case + run: | + echo 'c551b7e2b4928e4153fd864bed860d4d414a88bdccedf8be8110d4da07377b24 target/ct-replay-download/riscv64gc-unknown-linux-gnu-ct.tar.gz' | sha256sum --check + source "$HOME/.local/share/rscrypto-tooling/environment.sh" + just ct-replay --source-root target/ct-replay-source --archive target/ct-replay-download/riscv64gc-unknown-linux-gnu-ct.tar.gz --out target/ct-replay-results --case ecdsa_p384_public_key_fixed_vs_random_secret + - name: Retain every repetition and original preparation evidence + if: ${{ !cancelled() }} + uses: "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a" # v7.0.1 + with: + name: ct-p384-replay-${{ github.run_id }}-${{ github.run_attempt }} + path: | + target/ct-replay-results/ + target/ct-replay-download/ + if-no-files-found: error + retention-days: 30 + select: - runs-on: runs-on=${{ github.run_id }}/family=c7a+c7i+c8a+c8i+m7a+m7i+m8a+m8i/cpu=2/image=ubuntu24-minimal-x64/spot=price-capacity-optimized/env=production + if: ${{ !inputs.replay_p384 }} + # Public-repository named runners resolve from main, not this workflow's ref. + runs-on: runs-on=${{ github.run_id }}/cpu=2/family=c7a.large+c7i.large+c8a.large+c8i.large/image=ubuntu24-minimal-x64/spot=price-capacity-optimized/volume=40gb:gp3/env=production timeout-minutes: 5 outputs: matrix: ${{ steps.select.outputs.matrix }} - riscv: ${{ steps.select.outputs.riscv }} + cross: ${{ steps.select.outputs.cross }} + builds: ${{ steps.select.outputs.builds }} steps: - uses: "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1" # v7.0.1 with: @@ -45,32 +98,40 @@ jobs: - id: select run: python3 scripts/ct/ci.py plan - riscv-build: + cross-build: needs: select - if: needs.select.outputs.riscv == 'true' - runs-on: runs-on=${{ github.run_id }}/family=c7a+c7i+c8a+c8i+m7a+m7i+m8a+m8i/cpu=32/ram=64+128/image=ubuntu24-minimal-x64/spot=price-capacity-optimized/volume=150gb:gp3/env=production + if: needs.select.outputs.cross == 'true' + runs-on: runs-on=${{ github.run_id }}-${{ matrix.target }}/cpu=32/family=c7a.8xlarge+c7i.8xlarge+c8a.8xlarge+c8i.8xlarge/image=ubuntu24-minimal-x64/spot=price-capacity-optimized/volume=150gb:gp3/env=production timeout-minutes: 45 + name: Prepare ${{ matrix.target }} CT + strategy: + fail-fast: true + matrix: ${{ fromJSON(needs.select.outputs.builds) }} + env: + TARGET: ${{ matrix.target }} steps: - uses: "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1" # v7.0.1 with: persist-credentials: false - - name: Prepare and validate exact RISC-V CT binaries + - name: Prepare and validate exact target CT binaries run: | - scripts/tooling/x86_64-linux.sh --ci-riscv-build + scripts/tooling/x86_64-linux.sh --ci-cross-build "$TARGET" source "$HOME/.local/share/rscrypto-tooling/environment.sh" just test-transfer - just ct-full --target riscv64gc-unknown-linux-gnu --prepare-archive target/riscv-ct.tar.gz + just ct-full --target "$TARGET" --prepare-archive "target/$TARGET-ct.tar.gz" - uses: "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a" # v7.0.1 with: - name: riscv-ct-prepared - path: target/riscv-ct.tar.gz + name: ${{ matrix.target }}-ct-prepared + path: | + target/${{ matrix.target }}-ct.tar.gz + target/${{ matrix.target }}-tools.tar.gz if-no-files-found: error retention-days: 2 ct: - needs: [select, riscv-build] - if: ${{ !cancelled() && needs.select.result == 'success' && (needs.riscv-build.result == 'success' || needs.riscv-build.result == 'skipped') }} - name: ${{ matrix.platform }} (full) + needs: [select, cross-build] + if: ${{ !cancelled() && needs.select.result == 'success' && (needs.cross-build.result == 'success' || needs.cross-build.result == 'skipped') }} + name: ${{ matrix.platform }} (${{ inputs.diagnose_p384 && 'P-384 diagnostic' || 'full' }}) strategy: fail-fast: true matrix: ${{ fromJSON(needs.select.outputs.matrix) }} @@ -78,29 +139,34 @@ jobs: timeout-minutes: ${{ matrix.timeout }} env: PLATFORM: ${{ matrix.platform }} + DIAGNOSE_P384: ${{ inputs.diagnose_p384 && 'true' || 'false' }} steps: - uses: "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1" # v7.0.1 with: persist-credentials: false - name: Install tooling and run CT (Linux) - if: runner.os == 'Linux' && matrix.platform != 'riscv64-linux' + if: runner.os == 'Linux' && !matrix.target run: | scripts/tooling/"$PLATFORM.sh" --ci-ct-full source "$HOME/.local/share/rscrypto-tooling/environment.sh" if [[ "$PLATFORM" == x86_64-linux ]]; then just ct-test; fi just ct-full - - name: Download prepared RISC-V CT evidence from this workflow run - if: matrix.platform == 'riscv64-linux' + - name: Download prepared target CT evidence from this workflow run + if: matrix.target uses: "actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c" # v8.0.1 with: - name: riscv-ct-prepared - path: target/riscv-download - - name: Measure the prepared binaries on physical RISC-V - if: matrix.platform == 'riscv64-linux' + name: ${{ matrix.target }}-ct-prepared + path: target/cross-download + - name: Measure the prepared binaries on native hardware + if: matrix.target run: | - scripts/tooling/riscv64-linux.sh --ci-riscv-run + scripts/tooling/"$PLATFORM.sh" --ci-cross-run "target/cross-download/${{ matrix.target }}-tools.tar.gz" source "$HOME/.local/share/rscrypto-tooling/environment.sh" - just ct-full --target riscv64gc-unknown-linux-gnu --run-archive target/riscv-download/riscv-ct.tar.gz + if [[ "$DIAGNOSE_P384" == true ]]; then + just ct-replay --source-root . --archive "target/cross-download/${{ matrix.target }}-ct.tar.gz" --out target/ct/p384-candidate --case ecdsa_p384_public_key_fixed_vs_random_secret --repetitions 1 + else + just ct-full --target "${{ matrix.target }}" --run-archive "target/cross-download/${{ matrix.target }}-ct.tar.gz" + fi - name: Install tooling and run CT (Windows) if: runner.os == 'Windows' shell: powershell diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 9289b52e..e5fec7b3 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -58,7 +58,7 @@ jobs: include: - platform: x86_64-linux task: miri - runs-on: runs-on=${{ github.run_id }}/family=${{ matrix.platform == 'aarch64-linux' && 'c7g+c8g+m7g+m8g' || 'c7a+c7i+c8a+c8i+m7a+m7i+m8a+m8i' }}/cpu=${{ matrix.task == 'miri' && '4' || '16' }}/image=${{ matrix.platform == 'aarch64-linux' && 'ubuntu24-minimal-arm64' || 'ubuntu24-minimal-x64' }}/spot=price-capacity-optimized/volume=150gb:gp3/env=production + runs-on: runs-on=${{ github.run_id }}/runner=${{ matrix.task == 'miri' && 'miri' || matrix.platform }}/env=production timeout-minutes: ${{ (matrix.task == 'miri' || github.event_name == 'pull_request') && 60 || 90 }} steps: - uses: "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1" # v7.0.1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e4ed29ef..1ee180a6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -49,7 +49,7 @@ jobs: publish: needs: [ci, ct, fuzz] - runs-on: runs-on=${{ github.run_id }}/family=c7a+c7i+c8a+c8i+m7a+m7i+m8a+m8i/cpu=8/image=ubuntu24-minimal-x64/spot=price-capacity-optimized/volume=100gb:gp3/env=production + runs-on: runs-on=${{ github.run_id }}/runner=package/env=production timeout-minutes: 60 environment: release permissions: diff --git a/.rumdl.toml b/.rumdl.toml new file mode 100644 index 00000000..68a974f1 --- /dev/null +++ b/.rumdl.toml @@ -0,0 +1,56 @@ +[global] +line-length = 100 +cache-dir = ".cache/rumdl" +extend-enable = ["MD060", "MD070", "MD073", "MD080", "MD082", "MD084"] +exclude = [] + +[MD003] +style = "atx" + +[MD004] +style = "dash" + +[MD013] +code-blocks = false +code-spans = false +math-blocks = false +reflow = true +reflow-mode = "semantic-line-breaks" +reflow-length-exemptions = true + +[MD029] +style = "one" + +[MD035] +style = "---" + +[MD040] +style = "consistent" +preferred-aliases = { Shell = "bash" } +unknown-language-action = "error" + +[MD046] +style = "fenced" + +[MD048] +style = "backtick" + +[MD049] +style = "asterisk" + +[MD050] +style = "asterisk" + +[MD051] +ignore-case = false + +[MD055] +style = "leading-and-trailing" + +[MD060] +style = "aligned" +max-width = 100 +loose-last-column = true + +[MD082] +allow-parent-headings = true diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c299026e..ebfa5fb8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -53,6 +53,17 @@ including Miri and machine-code zeroization evidence. ## Validate +macOS ARM64 qualification runs on the maintainer's physical Apple Silicon Mac, +before every commit, including documentation and tooling changes. Run +`just install-hooks` once per checkout. The pre-commit and pre-merge-commit hooks +run `just check-macos`: native checks, complete release tests with native and +portable dispatch (including doctests), and the Apple Silicon RSA assembly gate. +Install prerequisites with `scripts/tooling/aarch64-macos.sh` when needed. +The hooks reject unstaged tracked changes and untracked files so the tested +source matches the staged commit. Do not bypass the hooks. Git hooks are local; +GitHub does not enforce this qualification and remote-created commits must not +replace the locally validated submission path. + Run `just --list` to discover the current recipes. Start with: ```bash @@ -168,7 +179,7 @@ Add the risk-specific evidence reached by the change: Cross-compilation proves compilation, not runtime behavior, constant-time execution, or performance. Record target lanes that cannot run. -RISC-V CI separates cross-compilation from native execution to avoid long builds +RISC-V, POWER, and IBM Z CI separate cross-compilation from native execution to avoid long builds on the physical runner. Both native-dispatch and portable release suites, doctests, and the full CT campaign remain required. The transfer commands and integrity requirements are documented in [scripts/README.md](scripts/README.md). @@ -213,8 +224,8 @@ cargo rail release run rscrypto --bump auto --skip-tag --allow-non-default-branc Review the generated diff, including manifests and lockfiles in independent workspaces, validate it, and merge through a PR. Complete physical Apple Silicon -RSA assembly and timing qualification locally before submission; hosted macOS -CI does not replace that evidence. +RSA assembly and timing qualification locally before submission. macOS does +not run in hosted CI. For the one-time publishing setup, create a GitHub environment named `release` restricted to `main`. Configure rscrypto's crates.io Trusted Publisher for @@ -224,7 +235,7 @@ See the [crates.io setup instructions](https://crates.io/docs/trusted-publishing To deploy, select **Actions → Release → Run workflow → main**. No version input is needed. The workflow rejects unconsumed change files, a version/changelog -mismatch, or a tag pointing elsewhere. CI (including hosted macOS ARM64), full +mismatch, or a tag pointing elsewhere. CI (with macOS ARM64 qualified locally before committing), full CT on all configured CI architectures, and both fuzz architectures plus Miri run concurrently against the triggering commit. Publication requires all three workflows to succeed. Benchmarks are separate. diff --git a/benches/common/criterion.rs b/benches/common/criterion.rs index ce7ba244..74d29c79 100644 --- a/benches/common/criterion.rs +++ b/benches/common/criterion.rs @@ -10,8 +10,8 @@ fn criterion() -> criterion::Criterion { let count = |name: &str| usize::try_from(integer(name)).expect("Criterion count fits usize"); let budget = integer("max_run_seconds"); assert!( - (1..=3600).contains(&budget), - "benchmark budget must be at most one hour" + (1..=5400).contains(&budget), + "benchmark budget must be at most 90 minutes" ); // The orchestration watchdog bounds the entire multi-binary run. This also // bounds an individual harness invoked directly through Cargo. diff --git a/docs/benchmarking.md b/docs/benchmarking.md index 7c41660e..6ef554dd 100644 --- a/docs/benchmarking.md +++ b/docs/benchmarking.md @@ -79,7 +79,7 @@ request. Select the revision with GitHub's branch selector, then choose: | Input | Examples | Meaning | | --- | --- | --- | -| `architectures` | `x86_64-linux` | One native platform. | +| `architectures` | `x86_64-linux` | One native platform; x86-64 runs both Intel and AMD. | | `architectures` | `s390x-linux,powerpc64le-linux,riscv64-linux` | Any subset, separated by commas or spaces. | | `architectures` | `all` | Linux x86-64, Linux ARM64, Windows x86-64, IBM Z, IBM POWER, and RISC-V. | | `selection` | `sha256` | One catalog algorithm. | @@ -101,21 +101,42 @@ features for the selected targets; it does not select separate targets. A small planning job validates the request and creates the exact runner matrix. AWS provides fixed on-demand instance types for Linux x86-64/ARM64 and Windows -x86-64. IBM and RISE provide their existing native runners. The selected +x86-64. Both x86-64 operating systems run separate Intel and AMD jobs with +distinct artifact names. Machine shapes live in [runs-on.yml](../.github/runs-on.yml); +these use current-generation processors without an AVX2-only baseline. macOS +benchmarks run locally on the Apple Silicon Mac. +IBM and RISE provide their existing native runners. The selected architectures run concurrently; benchmark configurations run sequentially on -each machine. Installers use `--ci-bench` on Linux and `-CiBench` on Windows. +each machine. RISC-V, POWER, and IBM Z compile the selected configurations on +x86-64 using `--ci-cross-build TARGET`. Their native jobs use `--ci-cross-run` +with the verified tools archive, then discover and measure the transferred binaries. +Other Linux jobs use `--ci-bench`; Windows uses `-CiBench`. No caches or speed-regression gates are enabled. Donated hosts may be shared, -and fixed AWS instance types do not eliminate host noise. Inspect uncertainty +and fixed AWS instance types do not eliminate host noise. Equal vCPU counts +do not imply equal physical core counts; interpret parallel results with the +recorded CPU topology. Inspect uncertainty and repeat matched measurements before making performance claims. Each job retains `target/bench/` as a GitHub artifact, including failed-run evidence, source and machine identity, the resolved case plan, logs, and raw -Criterion results. The benchmark runner keeps its one-hour pipeline budget; +Criterion results. Preparation and native measurement each have a separate +90-minute invocation budget; `all` is a selection, not a guarantee that every case will fit that budget. Narrow large runs by algorithm, group, target, or case filter. The workflow allows additional provisioning time, especially on RISC-V. Manual dispatch becomes available after the workflow reaches the default branch. +Cross-build preparation never executes target code. It seals each unique catalog +build configuration, source identity, compiler/linker evidence, exact binary hash, +and requested sampling settings. Native consumption rejects changed sources, +settings, configurations, or ELF architectures before discovery. It uses the same +case filtering, measurement, and result verification as an ordinary run, without +compiling again. The retained input manifest identifies the build host; result +compatibility records the measurement host and its runtime settings. Cross-built +and native-built results have distinct build identities for baseline comparisons. +The two invocations use `just bench ... target=TARGET prepare_archive=ARCHIVE` and +`just bench ... target=TARGET run_archive=ARCHIVE` with matching selections and settings. + The same granular selections work locally: ```sh @@ -255,9 +276,9 @@ window to collect the requested samples for slow operations. `argon2id` includes small, OWASP, and parallel workloads; no expensive-workload opt-in is required. `just bench` bounds the whole pipeline—build, discovery, measurement, analysis, -and result verification—to at most one hour. `just profile` uses the same limit for build, +and result verification—to at most 90 minutes. `just profile` uses the same limit for build, discovery, and capture. The configured limit may be lowered but cannot exceed -3,600 seconds. Shutdown starts before the deadline, reserving up to five seconds +5,400 seconds. Shutdown starts before the deadline, reserving up to five seconds to retain failed-run evidence before stopping surviving child processes. A timed out run exits with status 124; partial results do not constitute a complete run. Plans whose requested sampling windows alone exhaust the budget are rejected diff --git a/docs/platforms.md b/docs/platforms.md index 3f1404be..8c8c4e86 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -59,15 +59,17 @@ and [repository recipes](../scripts/README.md): | Check | Scope | | --- | --- | | `just check` | Host and catalogued cross-target compilation and lint checks. | -| Native CI | Native and portable suites plus doctests on Linux x86-64, AArch64, POWER, IBM Z, and RISC-V, macOS AArch64, and Windows x86-64. RISC-V builds on x86-64 and executes transferred artifacts on native hardware. | +| Native CI | Native and portable suites plus doctests on Linux x86-64, AArch64, POWER, IBM Z, and RISC-V, and Windows x86-64. RISC-V, POWER, and IBM Z build on x86-64 and execute transferred artifacts on native hardware. | +| `just check-macos` | Local Apple Silicon checks, native and portable release suites plus doctests, and physical RSA assembly qualification before every commit. | | `just test-musl` | Native and portable suites plus doctests on matching x86-64 or AArch64 Linux hosts. | | `just ci-compat` | Feature/MSRV and bare-metal compilation; scalar and SIMD vector execution for `wasm32-unknown-unknown` and `wasm32-wasip1` in Wasmtime. | A configured check is not a passing result for the current revision. Inspect matching run artifacts before qualifying a release. Bare-metal checks do not execute on devices, and Wasmtime results do not establish browser-engine -behavior. Windows AArch64 runtime CI remains deferred. Hosted macOS CI does -not replace physical Apple Silicon RSA assembly and timing qualification. +behavior. Windows AArch64 runtime CI remains deferred. macOS checks and tests run +locally before commits; physical Apple Silicon timing qualification remains +a separate local requirement. Performance and constant-time claims require retained evidence for the exact operation and configuration. Neither a target's presence in the catalog nor a diff --git a/justfile b/justfile index 0d1db9b5..c0b7499c 100644 --- a/justfile +++ b/justfile @@ -113,13 +113,22 @@ check: ci-check: @scripts/check/check.sh native -# Cross-check the complete RISC-V native CI compilation surface. +# Validate Apple Silicon checks, native/portable release tests, and RSA assembly locally. +check-macos: + @scripts/check/macos.sh + +# Enable mandatory local macOS validation for commits and merge commits in this checkout. +[group('tooling')] +install-hooks: + @git config --local core.hooksPath .githooks + +# Cross-check the complete target native CI compilation surface. ci-check-target target: @scripts/check/check.sh target {{quote(target)}} -# Prepare or execute complete, source-bound RISC-V test artifacts. -test-riscv operation archive: - @scripts/lib/python.sh scripts/test/riscv.py {{quote(operation)}} {{quote(archive)}} +# Prepare or execute complete, source-bound cross-compiled test artifacts. +test-cross operation target archive: + @scripts/lib/python.sh scripts/test/cross.py {{quote(operation)}} {{quote(target)}} {{quote(archive)}} # Check dependency policy for every supported target, once per CI workflow. ci-policy: @@ -152,6 +161,7 @@ test-scripts: @scripts/lib/python.sh scripts/tooling/install_test.py @scripts/lib/python.sh scripts/bench/ci_test.py @scripts/lib/python.sh scripts/test/fuzz_features_test.py + @scripts/lib/python.sh scripts/check/macos_test.py @scripts/lib/python.sh scripts/check/check_runner_test.py @scripts/lib/python.sh scripts/check/compat_test.py @scripts/lib/python.sh scripts/test/fuzz_runner_test.py @@ -212,6 +222,11 @@ ct-dudect *args: ct-full *args: @scripts/lib/python.sh scripts/ct/full.py "$@" +# Measure one prepared RISC-V CT case on a pinned CPU; diagnostic evidence only. +[group('constant-time')] +ct-replay *args: + @scripts/lib/python.sh scripts/ct/replay.py "$@" + # Run BINSEC; Manifest-Declared Binary CT Kernels [group('constant-time')] ct-binsec *args: diff --git a/scripts/README.md b/scripts/README.md index 060f3e50..a503ec55 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -11,6 +11,7 @@ benchmark commands. User-facing entry points are the recipes reported by | `check/check.sh` | `just check`, `just ci-check`, `just ci-check-target` | | `check/compat.py` | `just ci-compat` | | `test/test-musl.sh` | `just test-musl` | +| `check/macos.sh` | `just check-macos`, local commit hooks | | `check/dependencies.sh` | `just ci-policy`, dependency checks within `just check` | | `check/lint-independent-workspaces.sh` | `check/check.sh` | @@ -23,7 +24,7 @@ failure propagation with substitute executors. Run it with | Script | Caller | | --- | --- | | `test/test.sh` | `just test` | -| `test/riscv.py` | `just test-riscv prepare ARCHIVE`, `just test-riscv run ARCHIVE` | +| `test/cross.py` | `just test-cross prepare TARGET ARCHIVE`, `just test-cross run TARGET ARCHIVE` | | `test/doctest_bundle.py` | RISC-V doctest compilation and target execution | | `test/test-examples.sh` | `just test-examples` | | `test/test-miri.sh` | `just test-miri` | @@ -40,13 +41,25 @@ exporter self-tests; `just ct-test` includes them. and the pinned rustdoc compile/run contract, including deliberate failures. It requires the repository-pinned nightly and runs a small Rust fixture. -RISC-V CI builds on Ubuntu x86-64 using `--ci-riscv-build` tooling. It runs the +RISC-V, POWER, and IBM Z CI build on Ubuntu x86-64 using `--ci-cross-build TARGET` tooling. It runs the same target-specific checks and builds all release tests in both dispatch modes, including all doctest compilation checks. Nextest archives and persisted doctest programs -are transferred to the physical RISC-V runner, whose `--ci-riscv-run` tooling +are transferred to the matching native runner, whose `--ci-cross-run TOOLS_ARCHIVE` tooling only executes them. Preparation is not a runtime pass. The Rust release profile, target compiler, feature sets, and test assertions remain unchanged. +Each preparation job also cross-builds the pinned `just` and Nextest tools into a +separate source-bound archive. The native bootstrap verifies that archive and each +tool's ELF architecture before adding its directory to PATH. No Cargo tools compile +on the execution runner. Both Nextest builds use the same locked crate release; +producer and consumer identities must match except for their host architecture. +GNU cross-compilers and target libc development packages use the same Ubuntu CI +snapshot as native provisioning. No compiler cache is used. + +Cross-builds exercise dependency build scripts and procedural macros on x86-64. +They preserve target runtime evidence but do not qualify those tools running as +native POWER, IBM Z, or RISC-V host programs. + The archive records the Git revision, effective source digest, compiler, Nextest, release settings, and every file's digest and executable bit. Execution rejects different sources, missing or changed files, inherited selection overrides, and @@ -60,7 +73,7 @@ Doctests use rustdoc's extraction inventory and compilation checks, preserving `compile_fail`, error-code checks, `no_run`, and `should_panic`. Transfer preparation disables merging because the pinned rustdoc's merged runner executes despite global `--no-run`. Each runnable standalone program must subsequently execute on -RISC-V. Ordinary `just test` doctests retain rustdoc's default merging behavior. +the target hardware. Ordinary `just test` doctests retain rustdoc's default merging behavior. Example names and feature requirements come from Cargo metadata. Use `just test-miri --rsa` for the focused RSA scope and `just test-fuzz --targets A,B` @@ -87,15 +100,15 @@ latest report summarizes the selected cases and their requested budgets. `ct/manifest.py` owns shared target and measurement selection. `ct/provenance.py` owns shared file hashing and build identity. -RISC-V CT uses `just ct-full --target riscv64gc-unknown-linux-gnu --prepare-archive ARCHIVE` -on the x86-64 build host and the corresponding `--run-archive ARCHIVE` on physical -RISC-V. Preparation retains strict API/artifact validation, generated-code checks, +These targets use `just ct-full --target TARGET --prepare-archive ARCHIVE` +on the x86-64 build host and the corresponding `--run-archive ARCHIVE` on the matching +native hardware. Preparation retains strict API/artifact validation, generated-code checks, and the cleanup sentinel. It also compiles and disassembles the exact DudeCT executable that will be timed. The consumer verifies the source and artifacts, then runs the existing full manifest campaign with unchanged sampling, threshold, and per-case timeouts. No target code is rebuilt during measurement. Reports distinguish build and measurement hosts and retain the original preparation bundle. -This transfer mode is restricted to RISC-V; it cannot bypass native BINSEC on +This transfer mode is restricted to the three targets in `lib/cross_build.py`; it cannot bypass native BINSEC on targets that require it. `ct/full.py`, `ct/binsec.py`, and `ct/validate.py` back `just ct-full`, @@ -113,6 +126,20 @@ with a small latest report at `dudect/dudect-report.json`. Failed preparation or execution cannot reuse a previous run's measurements. Historical runs remain on disk until explicitly removed; full reports inventory only their current run. +`just ct-replay --source-root SOURCE --archive ARCHIVE --out OUTPUT --case CASE` +repeats one prepared RISC-V case three times on one allowed CPU. It validates the +original source and transferred binary, preserves the manifest sample count and +timeout, and retains all results at threshold 10. Timing failures do not shorten +the planned campaign; execution failures do. Host snapshots record affinity, +frequency settings where exposed, load, and processes. They do not guarantee an +otherwise idle machine. Replay is diagnostic evidence, not full qualification. +Use `--repetitions 1` for one candidate measurement at the same sample count. +The CT workflow's `replay_p384` input selects the original run 34672864167 and +commit 32734d2d. It requires that run's prepared artifact to remain available. +With `replay_p384` disabled, `diagnose_p384` instead prepares the current commit +and measures its P-384 public-key derivation case once on RISC-V. It overrides +the architecture selection. Neither diagnostic mode qualifies a release. + ## Benchmarks and updates | Script | Caller | @@ -123,6 +150,7 @@ disk until explicitly removed; full reports inventory only their current run. | `bench/profile.py` | runner: exact-case Samply capture | | `bench/evidence.py` | shared build/runtime environment collector | | `bench/settings.py` | measurement, profiling, and watchdog | +| `bench/transfer.py` | Compile-only preparation and verified native consumption for RISC-V, POWER, and IBM Z | | `bench/bounded.py` | `just bench`, `just profile`: process-tree deadline | | `update-all.sh` | `just update` | @@ -159,8 +187,9 @@ fuzz commands run independently of that plan. | `lib/fuzz-packages.sh` | Fuzz scripts | | `lib/python.sh` | Python-backed check, test, CT, and benchmark scripts | | `lib/toolchain.py`, `lib/toolchain.sh` | Shared toolchain selection for installers, builds, checks, tests, and benchmarks | -| `lib/evidence_bundle.py` | Source binding, sealing, and transfer integrity for RISC-V tests and CT | -| `lib/riscv_build.py` | Pinned RISC-V cross-compiler environment for test and CT preparation | +| `tooling/transfer.py` | Cross-build and verify the pinned native runner tools | +| `lib/evidence_bundle.py` | Source binding, sealing, and transfer integrity for cross-compiled tests, tools, and CT | +| `lib/cross_build.py` | Explicit target identities and cross-compiler environment for test, tool, and CT preparation | Python tooling requires Python 3.11 or newer. The updater installs its catalog-pinned Python libraries into a temporary virtual environment; checks and benchmarks use @@ -178,8 +207,8 @@ Run `scripts/tooling/.sh` on the native Ubuntu version pinned in `x86_64-linux`, `riscv64-linux`, `s390x-linux`, and `powerpc64le-linux`. The installers use sudo when needed. Windows uses the corresponding `aarch64-win.ps1` or `x86_64-win.ps1` in an elevated PowerShell session. -Local macOS tools remain locally managed; hosted macOS CI uses -`scripts/tooling/aarch64-macos.sh`. +Local macOS tools remain locally managed; `scripts/tooling/aarch64-macos.sh` +can provision the pinned prerequisites for `just check-macos`. CI calls these same installers with `--ci` on Linux or `-Ci` on Windows. The catalog's `ci` section selects the Cargo tools needed by `just ci-check`, @@ -296,8 +325,8 @@ results. The library also receives broad feature builds for both WASM targets. The x86-64 and ARM64 Linux rows install native musl build prerequisites and run `just test-musl`: the complete native and portable test suites plus doctests, -compiled and executed for the matching musl target. Apple ARM64 executes in -the hosted macOS CI row; Windows ARM64 execution remains deferred. No +compiled and executed for the matching musl target. Apple ARM64 checks and tests execute locally through +`just check-macos` before commits; Windows ARM64 execution remains deferred. No compatibility lane enables persistent caches. ## Release orchestration @@ -309,7 +338,8 @@ Its failure/recovery tests run through `just test-scripts`. Maintainer setup, preparation, deployment, and retry instructions live in [CONTRIBUTING.md](../CONTRIBUTING.md#release). -`scripts/tooling/aarch64-macos.sh` provisions the hosted ARM64 CI lane from the -repository's Rust and Cargo tool pins. That lane shares CI's native fail-fast -matrix and runs native/portable tests and checks. Physical Apple Silicon RSA -assembly and timing evidence remain local pre-submit requirements. +`scripts/check/macos.sh` owns `just check-macos`, which replaces hosted macOS +checks and tests with local Apple Silicon validation. Install `.githooks` with +`just install-hooks` in each maintainer checkout. macOS remains a supported +release target; physical Apple Silicon timing qualification remains a separate +local pre-submit requirement. diff --git a/scripts/bench/benchmark_catalog_test.py b/scripts/bench/benchmark_catalog_test.py index f491d28d..dd110678 100755 --- a/scripts/bench/benchmark_catalog_test.py +++ b/scripts/bench/benchmark_catalog_test.py @@ -109,6 +109,7 @@ def main() -> None: subprocess.run([sys.executable, str(ROOT / "scripts/bench/bounded_test.py")], check=True) subprocess.run([sys.executable, str(ROOT / "scripts/bench/run_test.py")], check=True) subprocess.run([sys.executable, str(ROOT / "scripts/bench/profile_test.py")], check=True) + subprocess.run([sys.executable, str(ROOT / "scripts/bench/transfer_test.py")], check=True) print("benchmark catalog and orchestration tests passed") diff --git a/scripts/bench/bounded.py b/scripts/bench/bounded.py index 33c0c663..7e06fbc4 100644 --- a/scripts/bench/bounded.py +++ b/scripts/bench/bounded.py @@ -35,8 +35,8 @@ def stop(process: subprocess.Popen, sig: int) -> None: def run(command: list[str], seconds: float) -> int: - if not 0 < seconds <= 3600: - raise ValueError("run budget must be positive and at most one hour") + if not 0 < seconds <= 5400: + raise ValueError("run budget must be positive and at most 90 minutes") grace = min(5.0, seconds / 10) with subprocess.Popen(command, start_new_session=os.name == "posix") as process: try: diff --git a/scripts/bench/bounded_test.py b/scripts/bench/bounded_test.py index 1baf0dbd..4437cfb7 100644 --- a/scripts/bench/bounded_test.py +++ b/scripts/bench/bounded_test.py @@ -16,7 +16,8 @@ class BudgetTests(unittest.TestCase): def test_exit_status_and_limit_validation(self): self.assertEqual(run([sys.executable, "-c", "raise SystemExit(7)"], 2), 7) - for limit in (0, -1, 3601): + self.assertEqual(run([sys.executable, "-c", "pass"], 5400), 0) + for limit in (0, -1, 5401): with self.assertRaises(ValueError): run([sys.executable, "-c", "pass"], limit) diff --git a/scripts/bench/ci.py b/scripts/bench/ci.py index 5aeb9cfb..3270667b 100644 --- a/scripts/bench/ci.py +++ b/scripts/bench/ci.py @@ -17,7 +17,27 @@ import settings sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'lib')) -from ci_platforms import PLATFORMS, platforms +from ci_platforms import platforms as native_platforms +from cross_build import TARGETS + + +def platforms(value: str, run_id: str) -> dict: + rows = [] + for row in native_platforms(value, run_id)['include']: + if row['platform'].startswith('x86_64-'): + for vendor in ('intel', 'amd'): + rows.append({**row, 'name': row['platform'] + '-' + vendor, + 'runner': row['runner'].replace('-intel/', '-' + vendor + '/')}) + else: + rows.append({**row, 'name': row['platform']}) + for row in rows: + row['timeout'] = max(row['timeout'], 120) + target = row['platform'].removesuffix('-linux') + '-unknown-linux-gnu' + if row['platform'] == 'riscv64-linux': + target = 'riscv64gc-unknown-linux-gnu' + if target in TARGETS: + row['target'] = target + return {'include': rows} def arguments(env: dict) -> list[str]: @@ -49,12 +69,21 @@ def main() -> int: matrix = platforms(os.environ.get('INPUT_ARCHITECTURES', ''), os.environ['GITHUB_RUN_ID']) with Path(os.environ['GITHUB_OUTPUT']).open('a', encoding='utf-8') as output: output.write('matrix=' + json.dumps(matrix, separators=(',', ':')) + '\n') - print('Platforms: ' + ', '.join(row['platform'] for row in matrix['include'])) + builds = [{'target': row['target']} for row in matrix['include'] if 'target' in row] + output.write('cross=' + str(bool(builds)).lower() + '\n') + output.write('builds=' + json.dumps({'include': builds}, separators=(',', ':')) + '\n') + print('Platforms: ' + ', '.join(row['name'] for row in matrix['include'])) print('Benchmark arguments: ' + json.dumps(args)) return 0 + if len(sys.argv) == 4 and sys.argv[1] in ('prepare', 'measure'): + operation, target, archive = sys.argv[1:] + if target not in TARGETS: + raise ValueError('unsupported cross-build target') + args += ['target=' + target, ('prepare_archive=' if operation == 'prepare' else 'run_archive=') + archive] + return subprocess.run(['just', 'bench', *args], check=False).returncode if sys.argv[1:] == ['run']: return subprocess.run(['just', 'bench', *args], check=False).returncode - raise ValueError('usage: scripts/bench/ci.py plan|run') + raise ValueError('usage: scripts/bench/ci.py plan|run|{prepare|measure} TARGET ARCHIVE') if __name__ == '__main__': diff --git a/scripts/bench/ci_test.py b/scripts/bench/ci_test.py index ab9aea81..b12e7253 100644 --- a/scripts/bench/ci_test.py +++ b/scripts/bench/ci_test.py @@ -17,9 +17,11 @@ class ManualBench(unittest.TestCase): def test_one_many_all_platforms(self): self.assertEqual([r['platform'] for r in ci.platforms('riscv64-linux', '42')['include']], ['riscv64-linux']) rows = ci.platforms('s390x-linux, x86_64-win s390x-linux', '42')['include'] - self.assertEqual([r['platform'] for r in rows], ['s390x-linux', 'x86_64-win']) - self.assertIn('windows25-full-x64', rows[1]['runner']) - self.assertEqual(len(ci.platforms('all', '42')['include']), 6) + self.assertEqual([r['platform'] for r in rows], ['s390x-linux', 'x86_64-win', 'x86_64-win']) + self.assertIn('measure-x86_64-win-intel', rows[1]['runner']) + self.assertIn('measure-x86_64-win-amd', rows[2]['runner']) + self.assertNotEqual(rows[1]['name'], rows[2]['name']) + self.assertEqual(len(ci.platforms('all', '42')['include']), 8) for bad in ('', 'all,x86_64-linux', 'apple-arm64', 'x86_64-linux,typo'): with self.subTest(bad=bad), self.assertRaises(ValueError): ci.platforms(bad, '42') @@ -51,6 +53,24 @@ def test_run_forwards_regex_literally_and_propagates_failure(self): self.assertIn('filter=' + env['INPUT_FILTER'], command) self.assertNotIn('shell', execute.call_args.kwargs) + def test_transfer_dispatch_keeps_selection_and_separate_invocations(self): + env = {'INPUT_SELECTION': 'sha256,sha512', 'INPUT_FILTER': '^sha256/rscrypto/64$', 'INPUT_SAMPLE_SIZE': '24'} + for operation, flag in (('prepare', 'prepare_archive'), ('measure', 'run_archive')): + with self.subTest(operation=operation), patch.dict(os.environ, env, clear=True), \ + patch.object(sys, 'argv', ['ci.py', operation, 's390x-unknown-linux-gnu', 'archive with spaces.tar.gz']), \ + patch.object(ci.subprocess, 'run', return_value=subprocess.CompletedProcess([], 9)) as execute: + self.assertEqual(ci.main(), 9) + command = execute.call_args.args[0] + self.assertEqual(command[:4], ['just', 'bench', 'sha256', 'sha512']) + self.assertIn(flag + '=archive with spaces.tar.gz', command) + self.assertIn('sample_size=24', command) + self.assertIn('target=s390x-unknown-linux-gnu', command) + for name in ('riscv64-linux', 's390x-linux', 'powerpc64le-linux'): + row, = ci.platforms(name, '42')['include'] + self.assertIn('target', row) + self.assertGreaterEqual(row['timeout'], 120) + self.assertTrue(all('target' not in row for row in ci.platforms('x86_64-linux', '42')['include'])) + def test_isolated_python_entry_points(self): root = Path(__file__).resolve().parents[2] for command in ([sys.executable, '-I', str(root / 'scripts/bench/runner.py'), '--help'], @@ -64,7 +84,11 @@ def test_isolated_python_entry_points(self): result = subprocess.run([sys.executable, '-I', str(root / 'scripts/bench/ci.py'), 'plan'], env=env, capture_output=True, text=True) self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(len(json.loads(output.read_text().removeprefix('matrix='))['include']), 6) + values = dict(line.split('=', 1) for line in output.read_text().splitlines()) + self.assertEqual(len(json.loads(values['matrix'])['include']), 8) + self.assertEqual(values['cross'], 'true') + self.assertEqual({row['target'] for row in json.loads(values['builds'])['include']}, + {'riscv64gc-unknown-linux-gnu', 'powerpc64le-unknown-linux-gnu', 's390x-unknown-linux-gnu'}) if __name__ == '__main__': diff --git a/scripts/bench/execution.py b/scripts/bench/execution.py index 869ec643..22c22bfa 100644 --- a/scripts/bench/execution.py +++ b/scripts/bench/execution.py @@ -89,7 +89,7 @@ def hardware() -> dict: if platform.system() == "Darwin": host["model"] = subprocess.check_output(["sysctl", "-n", "hw.model", "machdep.cpu.brand_string"], text=True).strip() elif Path("/proc/cpuinfo").is_file(): - fields = {"vendor_id", "model name", "cpu family", "model", "stepping", "Features", "flags", "CPU architecture", "CPU implementer", "CPU part", "machine", "processor"} + fields = {"vendor_id", "model name", "cpu family", "model", "stepping", "siblings", "cpu cores", "Features", "flags", "CPU architecture", "CPU implementer", "CPU part", "machine", "processor"} host["cpu"] = sorted(set(line.strip() for line in Path("/proc/cpuinfo").read_text().splitlines() if ":" in line and line.split(":", 1)[0].strip() in fields)) else: diff --git a/scripts/bench/runner.py b/scripts/bench/runner.py index 71f451a4..67bdfc36 100644 --- a/scripts/bench/runner.py +++ b/scripts/bench/runner.py @@ -44,6 +44,10 @@ def parse(arguments: list[str]): bench.add_argument("--diag", nargs="?", const=True, type=boolean, default=False) bench.add_argument("--output-dir", default="benchmark_results") bench.add_argument("--baseline", type=Path) + transfer = bench.add_mutually_exclusive_group() + transfer.add_argument("--prepare-archive", type=Path) + transfer.add_argument("--run-archive", type=Path) + bench.add_argument("--target") for name in ("warmup-ms", "measure-ms", "sample-size"): bench.add_argument("--" + name, type=int) profile = commands.add_parser("profile") @@ -69,7 +73,7 @@ def parse(arguments: list[str]): normalized.append("--" + key.replace("_", "-") + "=" + value) else: normalized.append(token) - value_follows = token in {"--bench", "--filter", "--output-dir", "--baseline", "--warmup-ms", "--measure-ms", "--sample-size"} + value_follows = token in {"--bench", "--filter", "--output-dir", "--baseline", "--prepare-archive", "--run-archive", "--target", "--warmup-ms", "--measure-ms", "--sample-size"} if arguments[:1] in (["codegen"], ["llvm-lines"]): boundary = normalized.index("--") if "--" in normalized else len(normalized) args = parser.parse_args(normalized[:boundary]) @@ -133,8 +137,8 @@ def requests(args, catalog) -> list[dict]: for name, scope, pattern in dict.fromkeys(rows)] -def resolve(rows, effective, log, env) -> list[dict]: - execution = build_identity() +def resolve(rows, effective, log, env, prepared=None) -> list[dict]: + execution = prepared[1] if prepared is not None else build_identity() builds = {} matches = {} matched = dict.fromkeys((row["pattern"] for row in rows), False) @@ -142,7 +146,7 @@ def resolve(rows, effective, log, env) -> list[dict]: command = build_command(row["binary"], row["features"]) key = tuple(command) if key not in builds: - artifact = build(command, log, env) + artifact = prepared[0][key] if prepared is not None else build(command, log, env) cases = discover(artifact, "", log, env) patterns = [pattern for request in rows if build_command(request["binary"], request["features"]) == command for pattern in (request["scope"], request["pattern"])] @@ -188,6 +192,16 @@ def bench(args, catalog) -> None: effective = settings.load({name: value for name in ("warmup_ms", "measure_ms", "sample_size") if (value := getattr(args, name)) is not None}) effective.pop("max_run_seconds") + if args.prepare_archive or args.run_archive or args.target: + from cross_build import TARGETS + if args.target not in TARGETS or not (args.prepare_archive or args.run_archive) or args.list: + raise ValueError('benchmark transfer requires a supported target and prepare/run archive, without --list') + if args.prepare_archive: + if args.baseline: + raise ValueError('baselines apply to native measurement, not preparation') + from transfer import prepare + prepare(ROOT, args.target, args.prepare_archive.resolve(), Path(args.output_dir).resolve(), rows, effective) + return if args.list: with tempfile.TemporaryDirectory(prefix="rscrypto-list-") as directory: root = Path(directory) @@ -218,11 +232,20 @@ def bench(args, catalog) -> None: write_json(root / "requests.json", {"configurations": rows, "budget_seconds": settings.load()["max_run_seconds"]}) source_evidence(root) env = dict(os.environ) | {"CRITERION_HOME": str(root / "discovery")} - plan = resolve(rows, effective, root / "output.txt", env) + prepared = None + if args.run_archive: + from transfer import consume + prepared = consume(ROOT, args.target, args.run_archive.resolve(), root / 'input', rows, effective) + write_json(root / 'transfer.json', {'archive_sha256': digest(args.run_archive), 'target': args.target, + 'manifest': 'input/bundle.json'}) + plan = resolve(rows, effective, root / "output.txt", env, prepared) shutil.rmtree(root / "discovery", ignore_errors=True) write_json(root / "plan.json", plan) measure(root, plan, baseline) verify(root) + if args.run_archive: + from transfer import bundle, KIND + bundle.verify(ROOT, root / 'input', KIND, args.target) status = 0 except BaseException as error: status = exit_code(error) diff --git a/scripts/bench/settings.py b/scripts/bench/settings.py index 2a1552cb..afc801ce 100644 --- a/scripts/bench/settings.py +++ b/scripts/bench/settings.py @@ -16,8 +16,8 @@ def load(overrides=None) -> dict: for key, minimum in integers.items(): if type(settings[key]) is not int or settings[key] < minimum: raise ValueError(f"{CONFIG}: {key} must be an integer >= {minimum}") - if settings["max_run_seconds"] > 3600: - raise ValueError("benchmark run budget cannot exceed one hour") + if settings["max_run_seconds"] > 5400: + raise ValueError("benchmark run budget cannot exceed 90 minutes") for key in ("confidence_level", "significance_level", "noise_threshold"): if type(settings[key]) not in (int, float) or not 0 < settings[key] < 1: raise ValueError(f"{CONFIG}: {key} must be between zero and one") diff --git a/scripts/bench/transfer.py b/scripts/bench/transfer.py new file mode 100644 index 00000000..44098c8b --- /dev/null +++ b/scripts/bench/transfer.py @@ -0,0 +1,92 @@ +"""Transfer benchmark executables; discover and measure only on native hardware.""" + +import copy +import os +from pathlib import Path +import platform +import shutil +import subprocess +import tempfile + +from execution import build, build_command, build_identity, digest, exit_code, hardware +from evidence import collect +from cross_build import TARGETS, environment, require_host, verify_elf +import evidence_bundle as bundle + +KIND = 'rscrypto.cross.bench' + + +def prepare(root, target, archive, output, rows, settings): + if (platform.system(), platform.machine()) != ('Linux', 'x86_64'): + raise ValueError('benchmark preparation requires Linux x86-64') + if archive.exists(): + raise ValueError(f'refusing to overwrite benchmark evidence: {archive}') + os.environ.update(environment(target)) + source = bundle.source_identity(root) + parent = output / 'preparation' / target + parent.mkdir(parents=True, exist_ok=True) + directory = Path(tempfile.mkdtemp(prefix='build-', dir=parent)) + print(f'Preparation directory: {directory}', flush=True) + try: + compiler_path = shutil.which(TARGETS[target][1] + '-gcc') + if compiler_path is None: + raise FileNotFoundError('install ci-cross-build tooling before preparing benchmarks') + compiler = Path(compiler_path).resolve(strict=True) + metadata = {'rows': rows, 'settings': settings, 'build': build_identity(), 'artifacts': [], + 'linker': {'path': str(compiler), 'sha256': digest(compiler), + 'version': subprocess.check_output([str(compiler), '--version'], text=True)}} + seen = set() + for row in rows: + command = build_command(row['binary'], row['features']) + if tuple(command) in seen: + continue + seen.add(tuple(command)) + artifact = build([*command, '--target', target], directory / 'output.txt', dict(os.environ)) + binary = directory / 'bin' / str(len(seen)) / row['binary'] + binary.parent.mkdir(parents=True) + shutil.copy2(artifact['path'], binary) + verify_elf(binary, target) + if digest(binary) != artifact['sha256']: + raise ValueError('benchmark executable changed during preparation') + metadata['artifacts'].append({'configuration': command, 'artifact': artifact, + 'binary': binary.relative_to(directory).as_posix()}) + (directory / 'status.txt').write_text('state=prepared\n') + bundle.seal(root, directory, KIND, target, source, metadata) + bundle.pack(directory, archive) + except BaseException as error: + (directory / 'status.txt').write_text(f'state=failed\nexit_code={exit_code(error)}\n') + raise + print(f'Benchmark preparation complete; native measurement remains required: {archive}', flush=True) + + +def consume(root, target, archive, directory, rows, settings): + require_host(target) + environment(target) # Reject inherited compiler/runner overrides before discovery. + bundle.unpack(archive, directory) + manifest = bundle.verify(root, directory, KIND, target) + metadata = manifest['metadata'] + if metadata['rows'] != rows or metadata['settings'] != settings: + raise ValueError('prepared benchmark selection or sampling settings differ') + artifacts = {} + for row in metadata['artifacts']: + key = tuple(row['configuration']) + if key in artifacts: + raise ValueError('duplicate prepared benchmark configuration') + binary = directory / bundle.relative_path(row['binary']) + verify_elf(binary, target) + artifact = row['artifact'] + if (digest(binary) != artifact['sha256'] + or artifact['command'] != [*row['configuration'], '--target', target]): + raise ValueError('prepared benchmark binary or build command differs') + artifacts[key] = {**artifact, 'path': str(binary)} + expected = {tuple(build_command(row['binary'], row['features'])) for row in rows} + if set(artifacts) != expected: + raise ValueError('prepared benchmark configurations are incomplete or unexpected') + # Build provenance remains unchanged in the input manifest. Compatibility + # uses the measurement host and runtime environment, never the x86 builder. + compatibility = copy.deepcopy(metadata['build']) + compatibility['host'] = hardware() + compatibility['environment']['runtime'] = collect()['runtime'] + compatibility['target'] = target + compatibility['linker'] = metadata['linker'] + return artifacts, compatibility diff --git a/scripts/bench/transfer_test.py b/scripts/bench/transfer_test.py new file mode 100644 index 00000000..04e2afd4 --- /dev/null +++ b/scripts/bench/transfer_test.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Verify compile-only preparation and complete native measurement after transfer.""" + +import contextlib +import copy +import io +import json +import os +from pathlib import Path +import shutil +import unittest +from unittest.mock import patch + +import runner +import settings +import transfer +import run_test + + +class TransferTests(unittest.TestCase): + def setUp(self): + self.fixture = run_test.RunnerTests() + self.fixture.setUp() + self.addCleanup(self.fixture.doCleanups) + self.root = self.fixture.root + (self.root / 'benches').unlink() + shutil.copytree(run_test.ROOT / 'benches', self.root / 'benches') + (self.root / '.gitignore').write_text('/target/\n/benchmark_results/\n/bin/\n/*.jsonl\n') + self.archive = self.root / 'target/bench.tar.gz' + self.archive.parent.mkdir() + self.fixture.tool('s390x-linux-gnu-gcc', "print('gcc fixture')") + self.target = 's390x-unknown-linux-gnu' + self.selectors = ['sha256', 'sha512', 'crc32-ieee'] + self.previous = Path.cwd() + os.chdir(self.root) + self.addCleanup(os.chdir, self.previous) + + def invoke(self, operation, *extra): + args = runner.parse(['bench', *self.selectors, '--target', self.target, + '--' + operation + '-archive', str(self.archive), *extra]) + # These are host-executable external-tool fixtures, not IBM ISA evidence. + # Real ELF identity/rejection is covered by test-transfer; CI must execute + # the actual cross-built binaries on each native architecture. + machine = 'x86_64' if operation == 'prepare' else 's390x' + with patch.dict(os.environ, self.fixture.env, clear=True), patch.object(runner, 'ROOT', self.root), \ + patch.object(settings, 'CONFIG', self.root / '.config/criterion.json'), \ + patch('platform.system', return_value='Linux'), patch('platform.machine', return_value=machine), \ + patch.object(transfer, 'verify_elf'), contextlib.redirect_stdout(io.StringIO()): + runner.bench(args, runner.load_catalog()) + + def test_preparation_never_discovers_and_consumer_never_builds(self): + with patch.object(runner, 'discover', side_effect=AssertionError('foreign discovery')): + self.invoke('prepare') + builds = (self.root / 'builds.jsonl').read_text().splitlines() + self.assertEqual(len(builds), 2) # sha256 and sha512 share the sha2 build. + self.assertTrue(all('--target' in json.loads(row) for row in builds)) + self.assertFalse(self.fixture.calls(listing=True)) + self.assertFalse(self.fixture.calls()) + self.assertFalse(self.fixture.runs()) + # Remove the original executable tree and reject any attempted rebuild. + for binary in (self.root / 'bin').glob('*--*'): + binary.unlink() + with patch.object(runner, 'build', side_effect=AssertionError('native compilation')): + self.invoke('run') + self.assertEqual(len(self.fixture.calls(listing=True)), 2) + self.assertEqual(len(self.fixture.calls()), 2) + plan = self.fixture.plan() + self.assertEqual(set(plan[0]['cases']), {'sha256/rscrypto/64', 'sha256/other/64', 'sha512/rscrypto/64'}) + self.assertEqual(plan[1]['cases'], ['crc32/rscrypto/64']) + self.assertTrue(all(entry['compatibility']['host']['machine'] == 's390x' for entry in plan)) + run = self.fixture.runs()[0] + original = json.loads((run / 'input/bundle.json').read_text()) + self.assertEqual(original['metadata']['build']['host']['machine'], 'x86_64') + self.assertEqual((run / 'status.txt').read_text(), 'state=complete\nexit_code=0\n') + self.assertEqual(len(runner.verify(run)), 2) + self.assertEqual((self.root / 'builds.jsonl').read_text().splitlines(), builds) + + def test_changed_request_sampling_source_or_binary_never_measures(self): + self.invoke('prepare') + for extra in (['--sample-size', '24'], ['--filter', '^sha256/rscrypto/64$']): + with self.subTest(extra=extra), self.assertRaisesRegex(ValueError, 'selection or sampling'): + self.invoke('run', *extra) + source = self.root / 'Cargo.toml' + source.write_text(source.read_text() + '\n# changed source\n') + with self.assertRaisesRegex(ValueError, 'source does not match'): + self.invoke('run') + source.write_text(source.read_text().removesuffix('\n# changed source\n')) + incoming = self.root / 'target/edit' + transfer.bundle.unpack(self.archive, incoming) + binary = next((incoming / 'bin').rglob('sha2')) + binary.write_bytes(b'changed binary') + self.archive.unlink() + transfer.bundle.pack(incoming, self.archive) + with self.assertRaisesRegex(ValueError, 'files changed'): + self.invoke('run') + self.assertFalse(self.fixture.calls(listing=True)) + self.assertFalse(self.fixture.calls()) + + def test_missing_configuration_is_rejected_even_in_a_sealed_bundle(self): + self.invoke('prepare') + incoming = self.root / 'target/edit' + transfer.bundle.unpack(self.archive, incoming) + manifest = json.loads((incoming / 'bundle.json').read_text()) + metadata = copy.deepcopy(manifest['metadata']) + metadata['artifacts'].pop() + transfer.bundle.seal(self.root, incoming, transfer.KIND, self.target, manifest['source'], metadata) + self.archive.unlink() + transfer.bundle.pack(incoming, self.archive) + with self.assertRaisesRegex(ValueError, 'configurations are incomplete'): + self.invoke('run') + self.assertFalse(self.fixture.calls(listing=True)) + self.assertFalse(self.fixture.calls()) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/check/check.sh b/scripts/check/check.sh index 9d682988..754b4ff5 100755 --- a/scripts/check/check.sh +++ b/scripts/check/check.sh @@ -6,7 +6,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")/../.." mode=$1 case "$mode" in check|fix|local|native) [[ $# -eq 1 ]] || exit 2 ;; - target) [[ $# -eq 2 && "$2" == riscv64gc-unknown-linux-gnu ]] || exit 2 ;; + target) [[ $# -eq 2 ]] || exit 2; python3 scripts/lib/cross_build.py "$2" >/dev/null ;; *) echo 'usage: scripts/check/check.sh {check|fix|local|native}' >&2; exit 2 ;; esac @@ -14,7 +14,7 @@ host=$(scripts/lib/toolchain.sh --print-host) if [[ "$mode" == target ]]; then host=$2 # Cargo keeps build scripts/proc macros on the build host; all checked product - # targets and independent workspaces use the same RISC-V cfg as native CI. + # targets and independent workspaces use the same target cfg as native CI. export CARGO_BUILD_TARGET="$host" fi [[ -n "$host" ]] || { echo 'cannot determine Rust host' >&2; exit 1; } diff --git a/scripts/check/check_runner_test.py b/scripts/check/check_runner_test.py index d8922606..be4e1e42 100755 --- a/scripts/check/check_runner_test.py +++ b/scripts/check/check_runner_test.py @@ -85,7 +85,7 @@ def main(): with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) for name in ('scripts/check/check.sh', 'scripts/check/dependencies.sh', 'scripts/lib/toolchain.sh', - 'scripts/lib/toolchain.py', 'scripts/lib/python.sh', 'Cargo.toml', + 'scripts/lib/toolchain.py', 'scripts/lib/cross_build.py', 'scripts/lib/python.sh', 'Cargo.toml', 'rust-toolchain.toml', '.config/toolchains.toml', '.config/target-matrix.json'): destination = root / name destination.parent.mkdir(parents=True, exist_ok=True) @@ -183,15 +183,16 @@ def expanded_features(selected): if deny: assert '--target' not in deny[0] assert (['lint-independent-workspaces.sh'] in commands) == (mode != 'fix') - result, commands = run('target', 'riscv64gc-unknown-linux-gnu') - assert result.returncode == 0, result.stderr - cross = [c for c in commands if c[0] == 'cargo' and 'clippy' in c] - assert len(cross) == 2 - assert all('--all-targets' in c and c[c.index('--target') + 1] == 'riscv64gc-unknown-linux-gnu' for c in cross) - assert all(c[1] == '+' + nightly for c in cross) - assert sum('--release' in c for c in cross) == 1 - assert ['lint-independent-workspaces.sh'] in commands - assert not any('--fix' in c for c in commands) + for target in ('riscv64gc-unknown-linux-gnu', 'powerpc64le-unknown-linux-gnu', 's390x-unknown-linux-gnu'): + result, commands = run('target', target) + assert result.returncode == 0, result.stderr + cross = [c for c in commands if c[0] == 'cargo' and 'clippy' in c] + assert len(cross) == 2 + assert all('--all-targets' in c and c[c.index('--target') + 1] == target for c in cross) + assert all(c[1] == '+' + nightly for c in cross) + assert sum('--release' in c for c in cross) == 1 + assert ['lint-independent-workspaces.sh'] in commands + assert not any('--fix' in c for c in commands) result, commands = run('check') assert result.returncode == 0, result.stderr inventories = [c for c in commands if c[0] == 'rustup'] diff --git a/scripts/check/macos.sh b/scripts/check/macos.sh new file mode 100755 index 00000000..cf69b978 --- /dev/null +++ b/scripts/check/macos.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/../.." +[[ "$(uname -s)" == Darwin && "$(uname -m)" == arm64 ]] || { + echo 'macOS validation requires the local Apple Silicon Mac.' >&2 + exit 1 +} +[[ "$(scripts/lib/toolchain.sh --print-host)" == aarch64-apple-darwin ]] || { + echo 'macOS validation requires an aarch64-apple-darwin Rust toolchain.' >&2 + exit 1 +} +just ci-check +just test --all --release +just test --all --release --portable +just test-rsa-macos-asm diff --git a/scripts/check/macos_test.py b/scripts/check/macos_test.py new file mode 100644 index 00000000..06ef4ede --- /dev/null +++ b/scripts/check/macos_test.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Commits must fail when macOS validation fails or the staged source differs.""" +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[2] + + +class MacOSCommit(unittest.TestCase): + def test_gate_preserves_ci_modes_and_stops_on_failure(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'scripts/check').mkdir(parents=True) + (root / 'scripts/lib').mkdir() + script = root / 'scripts/check/macos.sh' + shutil.copy2(ROOT / 'scripts/check/macos.sh', script) + for name, body in { + 'scripts/lib/toolchain.sh': 'echo aarch64-apple-darwin', + 'uname': 'if [ "$1" = -s ]; then echo Darwin; else echo arm64; fi', + 'just': 'echo "$*" >> calls; [ "$*" != "${FAIL_COMMAND:-}" ]', + }.items(): + tool = root / name + tool.write_text('#!/bin/sh\n' + body + '\n') + tool.chmod(0o755) + env = {**os.environ, 'BASH_ENV': '/dev/null', + 'PATH': str(root) + os.pathsep + os.environ['PATH']} + self.assertEqual(subprocess.run([str(script)], env=env).returncode, 0) + self.assertEqual((root / 'calls').read_text().splitlines(), [ + 'ci-check', 'test --all --release', 'test --all --release --portable', + 'test-rsa-macos-asm']) + (root / 'calls').unlink() + result = subprocess.run([str(script)], env={**env, 'FAIL_COMMAND': 'test --all --release'}) + self.assertNotEqual(result.returncode, 0) + self.assertEqual((root / 'calls').read_text().splitlines(), + ['ci-check', 'test --all --release']) + + def test_hook_checks_source_and_propagates_validation_failure(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + subprocess.run(['git', 'init', '-q', directory], check=True) + hook = root / '.git/hooks/pre-commit' + shutil.copy2(ROOT / '.githooks/pre-commit', hook) + tool = root / '.git/just' + tool.write_text('#!/bin/sh\necho "$*" >> .git/calls\nexit "${CHECK_STATUS:-0}"\n') + tool.chmod(0o755) + env = {**os.environ, 'BASH_ENV': '/dev/null', 'PATH': str(root / '.git') + os.pathsep + os.environ['PATH']} + source = root / 'source' + source.write_text('staged') + subprocess.run(['git', '-C', directory, 'add', 'source'], check=True) + + def run(**extra): + return subprocess.run([str(hook)], cwd=root, env={**env, **extra}, + capture_output=True, text=True).returncode + + self.assertEqual(run(), 0) + self.assertEqual((root / '.git/calls').read_text(), 'check-macos\n') + self.assertEqual(run(CHECK_STATUS='7'), 7) + source.write_text('unstaged') + self.assertNotEqual(run(), 0) + source.write_text('staged') + (root / 'untracked').write_text('new source') + self.assertNotEqual(run(), 0) + self.assertEqual((root / '.git/calls').read_text().splitlines(), + ['check-macos', 'check-macos']) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/ct/ci.py b/scripts/ct/ci.py index d2ed8142..96dfaf3a 100644 --- a/scripts/ct/ci.py +++ b/scripts/ct/ci.py @@ -7,6 +7,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'lib')) from ci_platforms import platforms +from cross_build import TARGETS def main(): @@ -14,9 +15,18 @@ def main(): matrix = platforms(os.environ['INPUT_ARCHITECTURES'], os.environ['GITHUB_RUN_ID']) for row in matrix['include']: row['timeout'] = 360 + builds = [] + for row in matrix['include']: + target = row['platform'].removesuffix('-linux') + '-unknown-linux-gnu' + if row['platform'] == 'riscv64-linux': + target = 'riscv64gc-unknown-linux-gnu' + if target in TARGETS: + row['target'] = target + builds.append({'target': target}) with Path(os.environ['GITHUB_OUTPUT']).open('a', encoding='utf-8') as output: output.write('matrix=' + json.dumps(matrix, separators=(',', ':')) + '\n') - output.write('riscv=' + str(any(row['platform'] == 'riscv64-linux' for row in matrix['include'])).lower() + '\n') + output.write('cross=' + str(bool(builds)).lower() + '\n') + output.write('builds=' + json.dumps({'include': builds}, separators=(',', ':')) + '\n') return raise ValueError('usage: scripts/ct/ci.py plan') diff --git a/scripts/ct/ci_test.py b/scripts/ct/ci_test.py index 7cb76c18..e4cddb35 100644 --- a/scripts/ct/ci_test.py +++ b/scripts/ct/ci_test.py @@ -33,7 +33,7 @@ def test_riscv_job_outlasts_required_case_budget(self): def test_one_many_all_selection(self): with tempfile.TemporaryDirectory() as temporary: output = Path(temporary) / 'output' - for selection, count in (('x86_64-linux', 1), ('x86_64-win,riscv64-linux', 2), ('all', 6)): + for selection, count in (('x86_64-linux', 1), ('powerpc64le-linux', 1), ('s390x-linux', 1), ('x86_64-win,riscv64-linux', 2), ('all', 6)): output.write_text('') with patch.dict(os.environ, INPUT_ARCHITECTURES=selection, GITHUB_RUN_ID='123', GITHUB_OUTPUT=str(output)), patch.object(sys, 'argv', ['ci.py', 'plan']): @@ -42,7 +42,15 @@ def test_one_many_all_selection(self): rows = json.loads(values['matrix'])['include'] self.assertEqual(len(rows), count) self.assertTrue(all(row['timeout'] == 360 for row in rows)) - self.assertEqual(values['riscv'], str(any(row['platform'] == 'riscv64-linux' for row in rows)).lower()) + expected = { + 'riscv64-linux': 'riscv64gc-unknown-linux-gnu', + 'powerpc64le-linux': 'powerpc64le-unknown-linux-gnu', + 's390x-linux': 's390x-unknown-linux-gnu', + } + builds = [{'target': expected[row['platform']]} for row in rows if row['platform'] in expected] + self.assertEqual(json.loads(values['builds']), {'include': builds}) + self.assertEqual(values['cross'], str(bool(builds)).lower()) + self.assertEqual([row['target'] for row in rows if 'target' in row], [row['target'] for row in builds]) if __name__ == '__main__': diff --git a/scripts/ct/dudect_execute.py b/scripts/ct/dudect_execute.py index db7ac9b6..1d974fa4 100644 --- a/scripts/ct/dudect_execute.py +++ b/scripts/ct/dudect_execute.py @@ -7,7 +7,6 @@ import json import os import subprocess -import platform from pathlib import Path # Embedded Windows Python omits the script directory. @@ -22,18 +21,15 @@ def verify_transferred(prepared): metadata = prepared['metadata'] if 'transfer' not in metadata: return - if (platform.system(), platform.machine(), metadata['target']) != ( - 'Linux', 'riscv64', 'riscv64gc-unknown-linux-gnu'): - raise ValueError('transferred timing requires physical RISC-V Linux') + sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'lib')) + from cross_build import require_host, verify_elf + require_host(metadata['target']) for key in ('binary', 'binary_disassembly', 'binary_symbols', 'linker_command_log'): row = metadata[key] path = Path(row['path']) if sha256_file(path) != row['sha256'] or path.stat().st_size != row['bytes']: raise ValueError(f'transferred timing evidence changed: {key}') - with Path(metadata['binary']['path']).open('rb') as source: - header = source.read(20) - if header[:6] != b'\x7fELF\x02\x01' or header[18:20] != b'\xf3\x00': - raise ValueError('timed executable is not a little-endian RISC-V ELF64 binary') + verify_elf(Path(metadata['binary']['path']), metadata['target']) def main() -> int: diff --git a/scripts/ct/full.py b/scripts/ct/full.py index 54f919e6..35b3ef07 100755 --- a/scripts/ct/full.py +++ b/scripts/ct/full.py @@ -1059,8 +1059,8 @@ def write_full_report(out_dir: Path, report: dict[str, Any]) -> tuple[Path, Path def main() -> int: parser = argparse.ArgumentParser(description=__doc__) transfer_args = parser.add_mutually_exclusive_group() - transfer_args.add_argument("--prepare-archive", type=Path, help="prepare sealed RISC-V CT evidence without timing") - transfer_args.add_argument("--run-archive", type=Path, help="measure sealed RISC-V CT evidence without rebuilding") + transfer_args.add_argument("--prepare-archive", type=Path, help="prepare sealed cross-compiled CT evidence without timing") + transfer_args.add_argument("--run-archive", type=Path, help="measure sealed cross-compiled CT evidence without rebuilding") parser.add_argument("--target", default=None) parser.add_argument("--profile", default="release") parser.add_argument("--threshold", type=float, default=float(os.environ.get("RSCRYPTO_CT_DUDECT_THRESHOLD", "10.0"))) @@ -1094,14 +1094,15 @@ def main() -> int: transferred = None transfer_identity = None if args.prepare_archive or args.run_archive: - from transfer import TARGET, bundle - if target != TARGET or args.profile != "release" or args.dudect_filter or args.dudect_gate != "required": - parser.error("CT transfer requires the complete RISC-V release lane") + from transfer import bundle + from cross_build import TARGETS + if target not in TARGETS or args.profile != "release" or args.dudect_filter or args.dudect_gate != "required": + parser.error("CT transfer requires the complete supported cross-compiled release lane") if args.threshold != 10.0 or "RSCRYPTO_CT_DUDECT_SAMPLES" in os.environ: parser.error("CT transfer requires unchanged manifest sampling and threshold") if args.prepare_archive: - from riscv_build import environment - os.environ.update(environment()) + from cross_build import environment + os.environ.update(environment(target)) transfer_identity = bundle.source_identity(root) host = host_target(root) if not args.prepare_archive and not is_host_executable_target(target, host): @@ -1146,7 +1147,7 @@ def main() -> int: if args.run_archive: from transfer import consume - steps, transferred = consume(root, out_dir, args.run_archive.resolve()) + steps, transferred = consume(root, out_dir, args.run_archive.resolve(), target) else: artifacts_result = run_command( root, @@ -1278,7 +1279,7 @@ def main() -> int: if preparation.status != "pass": print(f"DudeCT preparation failed: {preparation.stderr_path}", file=sys.stderr) return 1 - export(root, out_dir, shared, steps, transfer_identity, args.prepare_archive.resolve()) + export(root, out_dir, shared, steps, transfer_identity, args.prepare_archive.resolve(), target) print(f"CT preparation complete; native timing remains required: {args.prepare_archive}") return 0 diff --git a/scripts/ct/replay.py b/scripts/ct/replay.py new file mode 100644 index 00000000..5d787180 --- /dev/null +++ b/scripts/ct/replay.py @@ -0,0 +1,88 @@ +"""Repeat one prepared CT case on a pinned native CPU, retaining every result.""" + +import argparse +import json +import os +from pathlib import Path +import subprocess +import sys +from datetime import datetime, timezone + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from transfer import consume +from dudect_execute import measure +from dudect_report import write_report +from manifest import dudect_sample_count + + +def snapshot(cpu): + paths = [Path('/proc/cpuinfo'), Path('/proc/loadavg'), Path('/proc/stat')] + paths += sorted(Path(f'/sys/devices/system/cpu/cpu{cpu}/cpufreq').glob('*')) + values = {} + for path in paths: + if path.is_file(): + try: + values[str(path)] = path.read_text() + except OSError as error: + values[str(path)] = str(error) + return {'utc': datetime.now(timezone.utc).isoformat(), 'cpu': cpu, + 'affinity': sorted(os.sched_getaffinity(0)), 'files': values, + 'processes': subprocess.check_output( + ['ps', '-eo', 'pid,ppid,psr,pcpu,comm'], text=True)} + + +def repeat(prepared, args, cpu): + case = prepared['manifest_cases'][args.case] + samples = dudect_sample_count(case) + results = [] + for index in range(args.repetitions): + directory = args.out / f'repetition-{index + 1}' + directory.mkdir() + write_report(directory / 'before.json', snapshot(cpu)) + invocation = argparse.Namespace( + prepared=args.prepared, evidence_dir=directory, samples=samples, + smoke=False, threshold=10.0, filter=args.case, + timeout=case.get('timeout_seconds', 300), latest=None) + print(f'Repetition {index + 1}/{args.repetitions}: {args.case}, {samples} samples, CPU {cpu}', flush=True) + status = measure(prepared, invocation) + write_report(directory / 'after.json', snapshot(cpu)) + results.append({'repetition': index + 1, 'exit_code': status}) + write_report(args.out / 'replay.json', { + 'diagnostic_only': True, 'case': args.case, 'repetitions': results, + 'planned_repetitions': args.repetitions, + 'source': prepared['metadata']['transfer']['source'], + 'binary': prepared['metadata']['binary'], + 'note': 'All planned repetitions run even after timing failures; no result is discarded.'}) + if status not in (0, 1): + return status + return int(any(row['exit_code'] for row in results)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--source-root', type=Path, required=True) + parser.add_argument('--archive', type=Path, required=True) + parser.add_argument('--out', type=Path, required=True) + parser.add_argument('--case', required=True) + parser.add_argument('--repetitions', type=int, choices=(1, 3), default=3, + help='one candidate measurement or three baseline repetitions') + args = parser.parse_args() + args.out = args.out.resolve() + args.out.mkdir(parents=True, exist_ok=False) + _, args.prepared = consume(args.source_root.resolve(), args.out, + args.archive.resolve(), 'riscv64gc-unknown-linux-gnu') + prepared = json.loads(args.prepared.read_text()) + if args.case not in prepared['manifest_cases']: + raise ValueError('case is absent from the prepared manifest') + # Pin the controller and its measurement child; leave system policy unchanged. + allowed = os.sched_getaffinity(0) + cpu = min(allowed) + os.sched_setaffinity(0, {cpu}) + try: + return repeat(prepared, args, cpu) + finally: + os.sched_setaffinity(0, allowed) + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/scripts/ct/replay_test.py b/scripts/ct/replay_test.py new file mode 100644 index 00000000..79e73ddc --- /dev/null +++ b/scripts/ct/replay_test.py @@ -0,0 +1,69 @@ +"""A diagnostic replay must retain failures and never stop at the first pass.""" + +import argparse +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import replay + + +class Replay(unittest.TestCase): + def test_fixed_campaign_retains_failure_and_original_budget(self): + with tempfile.TemporaryDirectory() as temporary: + out = Path(temporary) + args = argparse.Namespace(out=out, prepared=out / 'prepared.json', case='p384', repetitions=3) + prepared = {'manifest_cases': {'p384': {'samples': 20000, 'timeout_seconds': 7200}}, + 'metadata': {'transfer': {'source': {'commit': 'original'}}, + 'binary': {'sha256': 'original-binary'}}} + # Substitute the external measurement boundary; this is controller evidence only. + with patch.object(replay, 'snapshot', return_value={'cpu': 2}), \ + patch.object(replay, 'measure', side_effect=[1, 0, 0]) as measure: + self.assertEqual(replay.repeat(prepared, args, 2), 1) + self.assertEqual(measure.call_count, 3) + for index, call in enumerate(measure.call_args_list, 1): + invocation = call.args[1] + self.assertEqual((invocation.samples, invocation.threshold, invocation.timeout), + (20000, 10.0, 7200)) + self.assertEqual(invocation.evidence_dir, out / f'repetition-{index}') + self.assertTrue((invocation.evidence_dir / 'after.json').is_file()) + report = json.loads((out / 'replay.json').read_text()) + self.assertEqual([row['exit_code'] for row in report['repetitions']], [1, 0, 0]) + self.assertEqual(report['source'], {'commit': 'original'}) + self.assertTrue(report['diagnostic_only']) + + def test_tooling_failure_stops_and_remains_recorded(self): + with tempfile.TemporaryDirectory() as temporary: + out = Path(temporary) + args = argparse.Namespace(out=out, prepared=out / 'prepared.json', case='p384', repetitions=3) + prepared = {'manifest_cases': {'p384': {'samples': 20000}}, + 'metadata': {'transfer': {'source': {}}, 'binary': {}}} + with patch.object(replay, 'snapshot', return_value={}), \ + patch.object(replay, 'measure', return_value=124) as measure: + self.assertEqual(replay.repeat(prepared, args, 0), 124) + self.assertEqual(measure.call_count, 1) + report = json.loads((out / 'replay.json').read_text()) + self.assertEqual(report['repetitions'], [{'repetition': 1, 'exit_code': 124}]) + + def test_single_candidate_retains_failure_without_extra_measurements(self): + with tempfile.TemporaryDirectory() as temporary: + out = Path(temporary) + args = argparse.Namespace(out=out, prepared=out / 'prepared.json', case='p384', repetitions=1) + prepared = {'manifest_cases': {'p384': {'samples': 20000, 'timeout_seconds': 7200}}, + 'metadata': {'transfer': {'source': {'commit': 'candidate'}}, 'binary': {}}} + with patch.object(replay, 'snapshot', return_value={}), \ + patch.object(replay, 'measure', return_value=1) as measure: + self.assertEqual(replay.repeat(prepared, args, 0), 1) + self.assertEqual(measure.call_count, 1) + invocation = measure.call_args.args[1] + self.assertEqual((invocation.samples, invocation.threshold, invocation.timeout), (20000, 10.0, 7200)) + report = json.loads((out / 'replay.json').read_text()) + self.assertEqual(report['planned_repetitions'], 1) + self.assertEqual(report['repetitions'], [{'repetition': 1, 'exit_code': 1}]) + self.assertEqual(report['source'], {'commit': 'candidate'}) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/ct/test.sh b/scripts/ct/test.sh index 7522283d..fb8cb946 100755 --- a/scripts/ct/test.sh +++ b/scripts/ct/test.sh @@ -10,4 +10,5 @@ PYTHON="$("$ROOT/scripts/lib/python.sh" --print)" "$PYTHON" "$ROOT/scripts/ct/smoke_test.py" "$PYTHON" "$ROOT/scripts/ct/zeroization_test.py" "$PYTHON" "$ROOT/scripts/ct/ci_test.py" +"$PYTHON" "$ROOT/scripts/ct/replay_test.py" just --justfile "$ROOT/justfile" test-harnesses diff --git a/scripts/ct/transfer.py b/scripts/ct/transfer.py index f9b66e9d..4e9ce0f9 100644 --- a/scripts/ct/transfer.py +++ b/scripts/ct/transfer.py @@ -1,4 +1,4 @@ -"""Transfer RISC-V CT preparation without rebuilding the measured executable.""" +"""Transfer cross-compiled CT preparation without rebuilding the measured executable.""" import json import platform @@ -10,12 +10,12 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'lib')) import evidence_bundle as bundle -TARGET = 'riscv64gc-unknown-linux-gnu' -KIND = 'rscrypto.riscv.ct' +from cross_build import require_host +KIND = 'rscrypto.cross.ct' GATES = ('ct-artifacts', 'ct-validate-artifacts', 'ct-zeroization-sentinel') -def export(root, out_dir, shared, steps, identity, archive): +def export(root, out_dir, shared, steps, identity, archive, target): if [step['name'] for step in steps] != list(GATES) or any(step['status'] != 'pass' for step in steps): raise ValueError('every CT preparation gate must pass before export') with tempfile.TemporaryDirectory(prefix='ct-export-', dir=root / 'target') as temporary: @@ -26,23 +26,22 @@ def export(root, out_dir, shared, steps, identity, archive): 'asm-heuristics.json', 'asm-heuristics.md', 'zeroization.json'): shutil.copy2(out_dir / name, directory / name) shutil.copytree(shared, directory / 'shared') - bundle.seal(root, directory, KIND, TARGET, identity, + bundle.seal(root, directory, KIND, target, identity, {'steps': steps, 'build_root': str(root), 'out_dir': str(out_dir)}) bundle.pack(directory, archive) -def consume(root, out_dir, archive): - if platform.system() != 'Linux' or platform.machine() != 'riscv64': - raise ValueError('transferred CT measurements require physical RISC-V Linux') +def consume(root, out_dir, archive, target): + require_host(target) run = Path(tempfile.mkdtemp(prefix='import-', dir=out_dir)) directory = run / 'original' bundle.unpack(archive, directory) - manifest = bundle.verify(root, directory, KIND, TARGET) + manifest = bundle.verify(root, directory, KIND, target) steps = manifest['metadata']['steps'] if [step['name'] for step in steps] != list(GATES) or any(step['status'] != 'pass' for step in steps): raise ValueError('transferred CT preparation gates are incomplete or failed') provenance = json.loads((directory / 'provenance.json').read_text()) - if provenance['target'] != TARGET or provenance['profile'] != 'release': + if provenance['target'] != target or provenance['profile'] != 'release': raise ValueError('wrong CT artifact target/profile') # Retain the producer bundle unchanged; only path-bearing runtime copies move. for name in ('provenance.json', 'evidence-index.json', 'artifact-hashes.txt', @@ -62,7 +61,7 @@ def consume(root, out_dir, archive): shutil.copytree(directory / 'shared', shared) prepared = json.loads((shared / 'prepared.json').read_text()) metadata = prepared['metadata'] - if metadata['target'] != TARGET or metadata['profile'] != 'release': + if metadata['target'] != target or metadata['profile'] != 'release': raise ValueError('wrong timed binary target/profile') for key in ('binary', 'binary_disassembly', 'binary_symbols', 'linker_command_log'): row = metadata[key] diff --git a/scripts/lib/ci_platforms.py b/scripts/lib/ci_platforms.py index 6cf2478e..b0fe0324 100644 --- a/scripts/lib/ci_platforms.py +++ b/scripts/lib/ci_platforms.py @@ -2,12 +2,12 @@ # Measurement hardware is fixed where the provider allows it. Donated runners # retain their provider labels; their actual machine identity is retained with the measurement evidence. PLATFORMS = { - 'x86_64-linux': ('c8i.2xlarge', 'ubuntu24-minimal-x64', 90), - 'aarch64-linux': ('c8g.2xlarge', 'ubuntu24-minimal-arm64', 90), - 'x86_64-win': ('c8i.2xlarge', 'windows25-full-x64', 90), - 's390x-linux': ('', 'ubuntu-24.04-s390x', 90), - 'powerpc64le-linux': ('', 'ubuntu-24.04-ppc64le-p10', 90), - 'riscv64-linux': ('', 'ubuntu-24.04-riscv', 180), + 'x86_64-linux': ('measure-x86_64-linux-intel', 90), + 'aarch64-linux': ('measure-aarch64-linux', 90), + 'x86_64-win': ('measure-x86_64-win-intel', 90), + 's390x-linux': ('ubuntu-24.04-s390x', 90), + 'powerpc64le-linux': ('ubuntu-24.04-ppc64le-p10', 90), + 'riscv64-linux': ('ubuntu-24.04-riscv', 180), } @@ -19,8 +19,8 @@ def platforms(value: str, run_id: str) -> dict: raise ValueError('GITHUB_RUN_ID must be numeric') rows = [] for name in names: - family, image, timeout = PLATFORMS[name] - label = (f'runs-on={run_id}/family={family}/cpu=8/image={image}/spot=false/volume=100gb:gp3/env=production' - if family else image) + runner, timeout = PLATFORMS[name] + label = (f'runs-on={run_id}/runner={runner}/env=production' + if runner.startswith('measure-') else runner) rows.append({'platform': name, 'runner': label, 'timeout': timeout}) return {'include': rows} diff --git a/scripts/lib/cross_build.py b/scripts/lib/cross_build.py new file mode 100644 index 00000000..ffbc20ef --- /dev/null +++ b/scripts/lib/cross_build.py @@ -0,0 +1,51 @@ +"""Cross-build configuration for the three Linux execution-only CI runners.""" + +import os +import platform +import sys +import toolchain + +# Target triple: (uname machine, GNU tool prefix, ELF byte order, ELF machine). +TARGETS = { + 'riscv64gc-unknown-linux-gnu': ('riscv64', 'riscv64-linux-gnu', 'little', 243), + 'powerpc64le-unknown-linux-gnu': ('ppc64le', 'powerpc64le-linux-gnu', 'little', 21), + 's390x-unknown-linux-gnu': ('s390x', 's390x-linux-gnu', 'big', 22), +} + + +def require_host(target): + machine, _, _, _ = TARGETS[target] + if (platform.system(), platform.machine()) != ('Linux', machine): + raise ValueError(f'transferred evidence requires native Linux {machine}') + + +def verify_elf(path, target): + _, _, endian, machine = TARGETS[target] + with path.open('rb') as source: + header = source.read(20) + if (len(header) != 20 or header[:5] != b'\x7fELF\x02' + or header[5] != (1 if endian == 'little' else 2) + or int.from_bytes(header[18:20], endian) != machine): + raise ValueError(f'not a {target} ELF64 executable: {path}') + + +def environment(target): + _, compiler, _, _ = TARGETS[target] + conflicting = [key for key in os.environ if key in { + 'RUSTFLAGS', 'CARGO_ENCODED_RUSTFLAGS', 'RUSTDOCFLAGS', 'CARGO_ENCODED_RUSTDOCFLAGS', 'RUSTC', 'RUSTDOC', + 'RUSTC_WRAPPER', 'RUSTC_WORKSPACE_WRAPPER', 'CARGO_BUILD_RUSTFLAGS', 'CARGO_BUILD_TARGET', + } or key.startswith(('CARGO_PROFILE_', 'CARGO_TARGET_', 'NEXTEST_'))] + if conflicting: + raise ValueError(f'unreviewed build/test overrides: {sorted(conflicting)}') + normalized = target.replace('-', '_') + return {**os.environ, 'RUSTUP_TOOLCHAIN': toolchain.for_target(target), 'CARGO_RAIL_CACHE': 'off', + f'CARGO_TARGET_{normalized.upper()}_LINKER': compiler + '-gcc', + f'CC_{normalized}': compiler + '-gcc', + f'CXX_{normalized}': compiler + '-g++', + f'AR_{normalized}': compiler + '-ar'} + + +if __name__ == '__main__': + # The shell installer uses the same closed target set as the transfer code. + target, = sys.argv[1:] + print(TARGETS[target][1]) diff --git a/scripts/lib/riscv_build.py b/scripts/lib/riscv_build.py deleted file mode 100644 index 06f83b75..00000000 --- a/scripts/lib/riscv_build.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Pinned RISC-V cross-build environment shared by tests and CT.""" - -import os -import toolchain - -TARGET = 'riscv64gc-unknown-linux-gnu' - - -def environment(): - conflicting = [key for key in os.environ if key in { - 'RUSTFLAGS', 'CARGO_ENCODED_RUSTFLAGS', 'RUSTDOCFLAGS', 'CARGO_ENCODED_RUSTDOCFLAGS', 'RUSTC', 'RUSTDOC', - 'RUSTC_WRAPPER', 'RUSTC_WORKSPACE_WRAPPER', 'CARGO_BUILD_RUSTFLAGS', - } or key.startswith(('CARGO_PROFILE_', 'CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_', 'NEXTEST_'))] - if conflicting: - raise ValueError(f'unreviewed build/test overrides: {sorted(conflicting)}') - return {**os.environ, 'RUSTUP_TOOLCHAIN': toolchain.for_target(TARGET), 'CARGO_RAIL_CACHE': 'off', - 'CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_LINKER': 'riscv64-linux-gnu-gcc', - 'CC_riscv64gc_unknown_linux_gnu': 'riscv64-linux-gnu-gcc', - 'CXX_riscv64gc_unknown_linux_gnu': 'riscv64-linux-gnu-g++', - 'AR_riscv64gc_unknown_linux_gnu': 'riscv64-linux-gnu-ar'} diff --git a/scripts/test/riscv.py b/scripts/test/cross.py similarity index 78% rename from scripts/test/riscv.py rename to scripts/test/cross.py index 3baffebe..a34147f3 100755 --- a/scripts/test/riscv.py +++ b/scripts/test/cross.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Build complete RISC-V suites on a fast host; run sealed suites on RISC-V.""" +"""Build complete target suites on x86-64; execute sealed suites on native hardware.""" import argparse import json @@ -14,10 +14,9 @@ ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / 'scripts/lib')) import evidence_bundle as bundle -from riscv_build import environment +from cross_build import TARGETS, environment, require_host import doctest_bundle -TARGET = 'riscv64gc-unknown-linux-gnu' def features(mode): graph = tomllib.loads((ROOT / 'Cargo.toml').read_text())['features'] @@ -43,51 +42,50 @@ def nextest_version(): return actual -def prepare(archive): +def prepare(target, archive): if archive.exists(): raise ValueError(f'refusing to overwrite existing evidence: {archive}') if platform.system() != 'Linux' or platform.machine() != 'x86_64': - raise ValueError('RISC-V CI preparation requires the Ubuntu x86-64 cross-toolchain host') - env = environment() - compiler_path = shutil.which('riscv64-linux-gnu-gcc') + raise ValueError('cross-compiled CI preparation requires the Ubuntu x86-64 cross-toolchain host') + env = environment(target) + compiler_path = shutil.which(TARGETS[target][1] + '-gcc') if compiler_path is None: - raise FileNotFoundError('install the ci-riscv-build tooling before preparing RISC-V tests') + raise FileNotFoundError('install the ci-cross-build tooling before preparing cross-compiled tests') compiler = Path(compiler_path).resolve(strict=True) identity = bundle.source_identity(ROOT) version = nextest_version() - directory = ROOT / 'target/riscv-transfer' + directory = ROOT / 'target/cross-transfer' / target directory.mkdir(parents=True, exist_ok=True) with tempfile.TemporaryDirectory(prefix='build-', dir=directory) as temporary: out = Path(temporary) # Same all-targets native/portable Clippy, independent workspaces, and docs. - subprocess.run(['just', 'ci-check-target', TARGET], cwd=ROOT, env=env, check=True) + subprocess.run(['just', 'ci-check-target', target], cwd=ROOT, env=env, check=True) metadata = {'nextest': version, 'rustc': subprocess.check_output(['rustc', '-vV'], env=env, text=True), 'profile': tomllib.loads((ROOT / 'Cargo.toml').read_text())['profile']['release'], 'checks': 'ci-check-target', 'modes': {}} metadata['linker'] = {'path': str(compiler), 'sha256': bundle.digest(compiler), 'version': subprocess.check_output([str(compiler), '--version'], text=True)} for mode in ('native', 'portable'): - args = ['--target', TARGET, *features(mode)] + args = ['--target', target, *features(mode)] command = ['cargo', 'nextest', 'archive', '--locked', '--workspace', '--release', *args, '--archive-file', str(out / f'{mode}.tar.zst')] subprocess.run(command, cwd=ROOT, env=env, check=True) plan = doctest_bundle.prepare(ROOT, out / f'{mode}-docs', args, env) metadata['modes'][mode] = {'command': command, 'doctests': plan['total']} - bundle.seal(ROOT, out, 'rscrypto.riscv.tests', TARGET, identity, metadata) + bundle.seal(ROOT, out, 'rscrypto.cross.tests', target, identity, metadata) bundle.pack(out, archive) -def execute(archive): - if platform.system() != 'Linux' or platform.machine() != 'riscv64': - raise ValueError('RISC-V tests must execute on the physical RISC-V Linux runner') - environment() # Reject inherited filters/profile overrides on the consumer too. +def execute(target, archive): + require_host(target) + environment(target) # Reject inherited filters/profile overrides on the consumer too. version = nextest_version() - directory = ROOT / 'target/riscv-results' + directory = ROOT / 'target/cross-results' / target directory.mkdir(parents=True, exist_ok=True) out = Path(tempfile.mkdtemp(prefix='run-', dir=directory)) incoming = out / 'input' bundle.unpack(archive, incoming) - manifest = bundle.verify(ROOT, incoming, 'rscrypto.riscv.tests', TARGET) + manifest = bundle.verify(ROOT, incoming, 'rscrypto.cross.tests', target) if nextest_identity(manifest['metadata']['nextest']) != nextest_identity(version): raise ValueError('producer and consumer Nextest versions differ') if set(manifest['metadata']['modes']) != {'native', 'portable'}: @@ -100,19 +98,20 @@ def execute(archive): '--no-tests', 'fail'], cwd=ROOT, stdout=log, stderr=subprocess.STDOUT, check=True) results[mode] = doctest_bundle.execute(ROOT, incoming / f'{mode}-docs', out / f'{mode}-docs') # Detect accidental changes to inputs throughout execution as well as before it. - bundle.verify(ROOT, incoming, 'rscrypto.riscv.tests', TARGET) + bundle.verify(ROOT, incoming, 'rscrypto.cross.tests', target) (out / 'summary.json').write_text(json.dumps({'status': 'pass', 'source': manifest['source'], 'archive_sha256': bundle.digest(archive), 'host': platform.uname()._asdict(), 'nextest': version, 'modes': results}, indent=2) + '\n') - print(f'RISC-V native/portable suites and doctests passed: {out}', flush=True) + print(f'{target} native/portable suites and doctests passed: {out}', flush=True) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('operation', choices=('prepare', 'run')) + parser.add_argument('target', choices=TARGETS) parser.add_argument('archive', type=Path) args = parser.parse_args() - (prepare if args.operation == 'prepare' else execute)(args.archive.resolve()) + (prepare if args.operation == 'prepare' else execute)(args.target, args.archive.resolve()) if __name__ == '__main__': diff --git a/scripts/test/just_arguments_test.py b/scripts/test/just_arguments_test.py index d052f1d2..a84ac501 100644 --- a/scripts/test/just_arguments_test.py +++ b/scripts/test/just_arguments_test.py @@ -57,6 +57,7 @@ def run(recipe, args, expected): 'test': [], 'test-miri': [], 'test-fuzz': [], 'test-fuzz-asan': [], 'ct-dudect': [], 'ct-artifacts': [], 'update': [], 'ct-full': ['scripts/ct/full.py'], 'ct-binsec': ['scripts/ct/binsec.py'], + 'ct-replay': ['scripts/ct/replay.py'], 'ct-validate': ['scripts/ct/validate.py'], 'bench': ['scripts/bench/bounded.py', str(recorder), 'scripts/bench/runner.py', 'bench'], 'profile': ['scripts/bench/bounded.py', str(recorder), 'scripts/bench/runner.py', 'profile'], diff --git a/scripts/test/transfer_test.py b/scripts/test/transfer_test.py index e54b6836..e95989ab 100755 --- a/scripts/test/transfer_test.py +++ b/scripts/test/transfer_test.py @@ -2,6 +2,7 @@ """Reject mixed-source, corrupt, incomplete, and unsafe transferred evidence.""" import io +import importlib.util import json import os from pathlib import Path @@ -20,8 +21,48 @@ import doctest_bundle import dudect_execute import transfer -import riscv -from riscv_build import environment +import cross +from cross_build import TARGETS, environment, require_host, verify_elf + +TARGET = "riscv64gc-unknown-linux-gnu" +spec = importlib.util.spec_from_file_location('runner_tools', ROOT / 'scripts/tooling/transfer.py') +runner_tools = importlib.util.module_from_spec(spec) +spec.loader.exec_module(runner_tools) + + +# Independent ELF identities from the ABI: EM_RISCV=243, EM_PPC64=21, EM_S390=22. +def elf_header(target): + return { + 'riscv64gc-unknown-linux-gnu': b'\x7fELF\x02\x01' + bytes(12) + b'\xf3\x00', + 'powerpc64le-unknown-linux-gnu': b'\x7fELF\x02\x01' + bytes(12) + b'\x15\x00', + 's390x-unknown-linux-gnu': b'\x7fELF\x02\x02' + bytes(12) + b'\x00\x16', + }[target] + + +class TargetIdentity(unittest.TestCase): + def test_wrong_machine_endian_and_host_are_rejected(self): + with tempfile.TemporaryDirectory() as temporary: + binary = Path(temporary) / 'binary' + for target, machine in (('riscv64gc-unknown-linux-gnu', 'riscv64'), + ('powerpc64le-unknown-linux-gnu', 'ppc64le'), + ('s390x-unknown-linux-gnu', 's390x')): + with self.subTest(target=target): + binary.write_bytes(elf_header(target)) + verify_elf(binary, target) + for other in set(TARGETS) - {target}: + with self.assertRaises(ValueError): verify_elf(binary, other) + header = bytearray(elf_header(target)) + header[5] = 3 - header[5] + binary.write_bytes(header) + with self.assertRaises(ValueError): verify_elf(binary, target) + binary.write_bytes(elf_header(target)[:19]) + with self.assertRaises(ValueError): verify_elf(binary, target) + with patch('platform.system', return_value='Linux'), patch('platform.machine', return_value=machine): + require_host(target) + for other in set(TARGETS) - {target}: + with self.assertRaises(ValueError): require_host(other) + with patch('platform.system', return_value='Linux'), patch('platform.machine', return_value='x86_64'): + with self.assertRaises(ValueError): require_host(target) class NextestIdentity(unittest.TestCase): @@ -34,14 +75,14 @@ def test_cross_architecture_reports_match_without_weakening_revision_check(self) 'commit-date: 2026-08-04\n' 'host: x86_64-unknown-linux-gnu') consumer = report.replace('x86_64-unknown-linux-gnu', 'riscv64gc-unknown-linux-gnu') - self.assertEqual(riscv.nextest_identity(report), riscv.nextest_identity(consumer)) + self.assertEqual(cross.nextest_identity(report), cross.nextest_identity(consumer)) changed = consumer.replace('60fa45f638ffc3f35e74afa65737f45fcd32db2a', '0' * 40) - self.assertNotEqual(riscv.nextest_identity(report), riscv.nextest_identity(changed)) + self.assertNotEqual(cross.nextest_identity(report), cross.nextest_identity(changed)) incomplete = '\n'.join(line for line in consumer.splitlines() if not line.startswith('commit-hash:')) - self.assertNotEqual(riscv.nextest_identity(report), riscv.nextest_identity(incomplete)) + self.assertNotEqual(cross.nextest_identity(report), cross.nextest_identity(incomplete)) for invalid in ('', f'not-nextest {pin}', consumer.replace(pin, f'{pin}-mismatch')): with self.subTest(report=invalid), self.assertRaises(ValueError): - riscv.nextest_identity(invalid) + cross.nextest_identity(invalid) class Bundles(unittest.TestCase): @@ -101,7 +142,17 @@ def test_archive_rejects_traversal_links_and_duplicates(self): with self.assertRaises(ValueError): bundle.unpack(archive, self.root / f'target/unpack-{index}') def test_ct_transfer_relocates_but_preserves_producer_identity(self): - ct = self.root / 'target/ct/riscv64gc-unknown-linux-gnu/release' + for target, (machine, _, _, _) in TARGETS.items(): + with self.subTest(target=target): + case = Bundles() + case.setUp() + try: + case.check_test_ct_transfer_relocates_but_preserves_producer_identity(target, machine) + finally: + case.doCleanups() + + def check_test_ct_transfer_relocates_but_preserves_producer_identity(self, target, machine): + ct = self.root / 'target/ct' / target / 'release' (ct / 'artifacts').mkdir(parents=True) (ct / 'artifacts/evidence').write_bytes(b'exact code') logs = ct / 'full/logs'; logs.mkdir(parents=True) @@ -112,36 +163,83 @@ def test_ct_transfer_relocates_but_preserves_producer_identity(self): for name in ('evidence-index.json', 'artifact-hashes.txt', 'asm-heuristics.json', 'asm-heuristics.md', 'zeroization.json'): (ct / name).write_text('{}') - (ct / 'provenance.json').write_text(json.dumps({'target': transfer.TARGET, 'profile': 'release'})) + (ct / 'provenance.json').write_text(json.dumps({'target': target, 'profile': 'release'})) shared = ct / 'build/shared'; shared.mkdir(parents=True) - metadata = {'target': transfer.TARGET, 'profile': 'release', 'host': {'machine': 'x86_64'}} + metadata = {'target': target, 'profile': 'release', 'host': {'machine': 'x86_64'}} for key in ('binary', 'binary_disassembly', 'binary_symbols', 'linker_command_log'): path = shared / key - path.write_bytes(b'\x7fELF\x02\x01' + bytes(12) + b'\xf3\x00' if key == 'binary' else b'evidence') + path.write_bytes(elf_header(target) if key == 'binary' else b'evidence') metadata[key] = {'path': str(path), 'bytes': path.stat().st_size, 'sha256': bundle.digest(path)} (shared / 'prepared.json').write_text(json.dumps({'metadata': metadata, 'manifest_cases': {}})) archive = self.root / 'target/ct.tar.gz' - transfer.export(self.root, ct, shared, steps, self.identity, archive) - with patch('platform.system', return_value='Linux'), patch('platform.machine', return_value='riscv64'): - imported_steps, path = transfer.consume(self.root, ct, archive) + transfer.export(self.root, ct, shared, steps, self.identity, archive, target) + with patch('platform.system', return_value='Linux'), patch('platform.machine', return_value=machine): + imported_steps, path = transfer.consume(self.root, ct, archive, target) prepared = json.loads(path.read_text()) self.assertEqual(prepared['metadata']['build_host']['machine'], 'x86_64') - self.assertEqual(prepared['metadata']['host']['machine'], 'riscv64') + self.assertEqual(prepared['metadata']['host']['machine'], machine) self.assertTrue(all(Path(s['stdout']).is_file() for s in imported_steps)) dudect_execute.verify_transferred(prepared) binary = Path(prepared['metadata']['binary']['path']) binary.write_bytes(b'changed executable') with self.assertRaises(ValueError): dudect_execute.verify_transferred(prepared) with patch('platform.machine', return_value='x86_64'): - with self.assertRaises(ValueError): transfer.consume(self.root, ct, archive) - with self.assertRaises(ValueError): transfer.export(self.root, ct, shared, steps[:-1], self.identity, archive) + with self.assertRaises(ValueError): transfer.consume(self.root, ct, archive, target) + with self.assertRaises(ValueError): transfer.export(self.root, ct, shared, steps[:-1], self.identity, archive, target) + + def test_runner_tools_are_source_bound_and_verified_before_installation(self): + pins = tomllib.loads((ROOT / '.config/tooling.toml').read_text())['cargo'] + path = self.root / '.config/tooling.toml' + path.parent.mkdir() + shutil.copy2(ROOT / '.config/tooling.toml', path) + identity = bundle.source_identity(self.root) + for index, target in enumerate(TARGETS): + directory = self.root / 'target' / str(index) + (directory / 'bin').mkdir(parents=True) + for name in ('just', 'cargo-nextest'): + tool = directory / 'bin' / name + tool.write_bytes(elf_header(target)) + tool.chmod(0o755) + metadata = {name: pins[name] for name in ('just', 'cargo-nextest')} + for failure in ('none', 'version', 'elf', 'source'): + with self.subTest(target=target, failure=failure): + if failure == 'version': metadata['just'] = 'wrong' + else: metadata['just'] = pins['just'] + (directory / 'bin/just').write_bytes(b'not target code' if failure == 'elf' else elf_header(target)) + bundle.seal(self.root, directory, runner_tools.KIND, target, identity, metadata) + archive = self.root / 'target' / f'{index}-{failure}.tar.gz' + bundle.pack(directory, archive) + if failure == 'source': (self.root / 'source.rs').write_text('changed') + destination = self.root / 'target' / f'{index}-{failure}-installed' + with patch.object(runner_tools, 'ROOT', self.root), patch('platform.system', return_value='Linux'), \ + patch('platform.machine', return_value=TARGETS[target][0]), patch('sys.stdout', new=io.StringIO()) as output: + if failure == 'none': + runner_tools.install(target, archive, destination) + self.assertEqual(output.getvalue().strip(), str(destination / 'bin')) + else: + with self.assertRaises(ValueError): runner_tools.install(target, archive, destination) + self.assertEqual(output.getvalue(), '') + (self.root / 'source.rs').write_text('original\n') def test_overrides_are_rejected(self): - for key in ('RUSTFLAGS', 'CARGO_PROFILE_RELEASE_LTO', 'NEXTEST_FILTERSET'): - with patch.dict(os.environ, {key: 'override'}): - with self.assertRaises(ValueError): environment() + for target in TARGETS: + for key in ('RUSTFLAGS', 'CARGO_PROFILE_RELEASE_LTO', 'NEXTEST_FILTERSET', + 'CARGO_TARGET_' + target.replace('-', '_').upper() + '_RUSTFLAGS', + 'CARGO_TARGET_' + target.replace('-', '_').upper() + '_RUNNER'): + with self.subTest(target=target, key=key), patch.dict(os.environ, {key: 'override'}): + with self.assertRaises(ValueError): environment(target) def test_prepare_keeps_both_full_release_modes_and_fails_closed(self): + for target, (machine, _, _, _) in TARGETS.items(): + with self.subTest(target=target): + case = Bundles() + case.setUp() + try: + case.check_test_prepare_keeps_both_full_release_modes_and_fails_closed(target, machine) + finally: + case.doCleanups() + + def check_test_prepare_keeps_both_full_release_modes_and_fails_closed(self, target, machine): for name in ('Cargo.toml', '.config/tooling.toml'): destination = self.root / name destination.parent.mkdir(parents=True, exist_ok=True) @@ -165,7 +263,7 @@ def test_prepare_keeps_both_full_release_modes_and_fails_closed(self): elif name.endswith('gcc'): print('gcc pinned fixture') ''') tool.chmod(0o755) - for name in ('just', 'cargo', 'rustc', 'riscv64-linux-gnu-gcc'): + for name in ('just', 'cargo', 'rustc', TARGETS[target][1] + '-gcc'): (commands / name).symlink_to(tool) # Do not include fixture executable symlinks in the effective source digest. with (self.root / '.gitignore').open('a') as output: output.write('commands/\n') @@ -174,24 +272,24 @@ def docs(root, directory, args, env): directory.mkdir() (directory / 'fixture').write_text('prepared') return {'total': 232} - with patch.object(riscv, 'ROOT', self.root), patch('platform.system', return_value='Linux'), \ + with patch.object(cross, 'ROOT', self.root), patch('platform.system', return_value='Linux'), \ patch('platform.machine', return_value='x86_64'), patch.object(doctest_bundle, 'prepare', side_effect=docs) as prepare_docs, \ patch.dict(os.environ, PATH=str(commands) + os.pathsep + os.environ['PATH'], TRANSFER_TEST_LOG=str(log), TRANSFER_NEXTEST=pin): archive = self.root / 'target/test-suites.tar.gz' - riscv.prepare(archive) + cross.prepare(target, archive) calls = [json.loads(row) for row in log.read_text().splitlines()] - self.assertIn(['just', 'ci-check-target', riscv.TARGET], calls) + self.assertIn(['just', 'ci-check-target', target], calls) builds = [row for row in calls if row[:3] == ['cargo', 'nextest', 'archive']] self.assertEqual(len(builds), 2) self.assertTrue(all('--workspace' in row and '--locked' in row and '--release' in row for row in builds)) - self.assertTrue(all(row[row.index('--target') + 1] == riscv.TARGET for row in builds)) + self.assertTrue(all(row[row.index('--target') + 1] == target for row in builds)) self.assertNotIn('portable-only', builds[0][builds[0].index('--features') + 1].split(',')) self.assertIn('--all-features', builds[1]) self.assertEqual(prepare_docs.call_count, 2) with patch.dict(os.environ, TRANSFER_FAIL='1'): failed = self.root / 'target/failed.tar.gz' - with self.assertRaises(subprocess.CalledProcessError): riscv.prepare(failed) + with self.assertRaises(subprocess.CalledProcessError): cross.prepare(target, failed) self.assertFalse(failed.exists()) @@ -257,7 +355,7 @@ def test_real_nextest_archive_executes_after_transfer(self): subprocess.run(['git', '-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.invalid', 'commit', '-qm', 'fixture'], cwd=root, check=True) directory = root / 'target/transfer'; directory.mkdir(parents=True) - consumer_version = riscv.nextest_version() + consumer_version = cross.nextest_version() metadata = {'nextest': consumer_version, 'modes': {}} # Real cross-architecture reports have the same release/commit but # different host lines, as in CI run 34527080605. Preserve that @@ -273,14 +371,14 @@ def test_real_nextest_archive_executes_after_transfer(self): self.assertEqual(built.returncode, 0, built.stderr) plan = doctest_bundle.prepare(root, directory / (mode + '-docs'), [], env) metadata['modes'][mode] = {'doctests': plan['total']} - bundle.seal(root, directory, 'rscrypto.riscv.tests', riscv.TARGET, bundle.source_identity(root), metadata) + bundle.seal(root, directory, 'rscrypto.cross.tests', TARGET, bundle.source_identity(root), metadata) archive = root / 'target/transfer.tar.gz'; bundle.pack(directory, archive) # Removing the original build tree catches hidden dependencies on it. shutil.rmtree(root / 'target/release') - with patch.object(riscv, 'ROOT', root), patch('platform.system', return_value='Linux'), \ + with patch.object(cross, 'ROOT', root), patch('platform.system', return_value='Linux'), \ patch('platform.machine', return_value='riscv64'), patch('sys.stdout', new=io.StringIO()): - riscv.execute(archive) - summaries = list((root / 'target/riscv-results').glob('*/summary.json')) + cross.execute(TARGET, archive) + summaries = list((root / 'target/cross-results' / TARGET).glob('*/summary.json')) self.assertEqual(len(summaries), 1) summary = json.loads(summaries[0].read_text()) self.assertEqual(summary['status'], 'pass') diff --git a/scripts/tooling/catalog.py b/scripts/tooling/catalog.py index 6dff6705..40506a3e 100644 --- a/scripts/tooling/catalog.py +++ b/scripts/tooling/catalog.py @@ -95,8 +95,8 @@ def validate(data): if any(asset not in data['x86_64-linux']['assets'] for asset in data['ci-compat']['assets']): raise ValueError('ci-compat: missing pinned archive') for profile, required in (('ci', {'just', 'cargo-nextest'}), - ('ci-riscv-build', {'just', 'cargo-nextest'}), - ('ci-riscv-run', {'just', 'cargo-nextest'}), + ('ci-cross-build', {'just', 'cargo-nextest'}), + ('ci-cross-run', set()), ('ci-policy', {'cargo-deny', 'cargo-audit'}), ('ci-compat', {'just'}), ('ci-fuzz', {'just', 'cargo-fuzz'}), diff --git a/scripts/tooling/install_test.py b/scripts/tooling/install_test.py index dd324dc0..62be615e 100755 --- a/scripts/tooling/install_test.py +++ b/scripts/tooling/install_test.py @@ -54,6 +54,8 @@ executable.write_text('#!/bin/sh\nexit 0\n') executable.chmod(0o755) print(directory) + elif script == 'transfer.py': + if args[1] == 'install': print(str(pathlib.Path(args[-1]) / 'bin')) elif script in ('compat.py', 'package.py') and '--install' in args: pass else: @@ -62,7 +64,7 @@ class LinuxInstall(unittest.TestCase): - def provision(self, platform, fail=False, real_apt=False, without_preference=False, profile='ci'): + def provision(self, platform, fail=False, real_apt=False, without_preference=False, profile='ci', target=None): temporary = tempfile.TemporaryDirectory(prefix='rscrypto installer ') self.addCleanup(temporary.cleanup) root = Path(temporary.name) @@ -107,7 +109,8 @@ def package(name, version, installed=False, depends=''): INSTALL_APT_ARCH=subprocess.check_output(['dpkg', '--print-architecture'], text=True).strip()) if without_preference: env['INSTALL_WITHOUT_PREFERENCE'] = '1' - result = subprocess.run([BASH, str(ROOT / 'scripts/tooling/linux.sh'), platform, '--' + profile], + extra = [target] if profile == 'ci-cross-build' else ['tools.tar.gz'] if profile == 'ci-cross-run' else [] + result = subprocess.run([BASH, str(ROOT / 'scripts/tooling/linux.sh'), platform, '--' + profile, *extra], env=env, capture_output=True, text=True) calls = [json.loads(line) for line in (root / 'commands.jsonl').read_text().splitlines()] return result, calls, root @@ -202,23 +205,37 @@ def test_package_profile_installs_consumer_prerequisites_once(self): self.assertTrue(compiler_installs[0][1].endswith('scripts/check/package.py')) self.assertFalse(any('musl-tools=1.0' in c for c in calls)) - def test_riscv_build_and_execution_tooling_are_separate(self): + def test_cross_build_and_execution_tooling_are_separate(self): nightly = tomllib.loads((ROOT / '.config/toolchains.toml').read_text())['nightly'] - for platform, profile in (('x86_64-linux', 'ci-riscv-build'), ('riscv64-linux', 'ci-riscv-run')): - with self.subTest(profile=profile): - result, calls, _ = self.provision(platform, profile=profile) - self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - apt = next(c for c in calls if c[0] == 'apt-get' and '--allow-downgrades' in c) - self.assertEqual([a for a in apt if a.endswith('=1.0')], - [p + '=1.0' for p in CATALOG[profile]['packages']]) - installs = [c for c in calls if c[:3] == ['rustup', 'toolchain', 'install']] - self.assertIn(nightly, [c[3] for c in installs]) - components = [c[i + 1] for c in installs for i, arg in enumerate(c) if arg == '--component'] - self.assertEqual(components, ['rustfmt', 'clippy', 'llvm-tools'] if profile.endswith('build') else []) - if profile.endswith('build'): - self.assertIn(['rustup', 'target', 'add', '--toolchain', nightly, 'riscv64gc-unknown-linux-gnu'], calls) - else: - self.assertFalse(any(c[0] == 'cargo' and ('build' in c or 'install' in c) for c in calls)) + for platform, target, prefix, libc in ( + ('riscv64-linux', 'riscv64gc-unknown-linux-gnu', 'riscv64-linux-gnu', 'riscv64'), + ('powerpc64le-linux', 'powerpc64le-unknown-linux-gnu', 'powerpc64le-linux-gnu', 'ppc64el'), + ('s390x-linux', 's390x-unknown-linux-gnu', 's390x-linux-gnu', 's390x'), + ): + for profile in ('ci-cross-build', 'ci-cross-run'): + with self.subTest(target=target, profile=profile): + result, calls, _ = self.provision('x86_64-linux' if profile.endswith('build') else platform, + profile=profile, target=target) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + apt = next(c for c in calls if c[0] == 'apt-get' and '--allow-downgrades' in c) + packages = list(CATALOG[profile]['packages']) + if profile.endswith('build'): + packages += ['gcc-' + prefix, 'g++-' + prefix, 'libc6-dev-' + libc + '-cross'] + self.assertEqual([a for a in apt if a.endswith('=1.0')], [p + '=1.0' for p in packages]) + installs = [c for c in calls if c[:3] == ['rustup', 'toolchain', 'install']] + self.assertIn(nightly, [c[3] for c in installs]) + components = [c[i + 1] for c in installs for i, arg in enumerate(c) if arg == '--component'] + self.assertEqual(components, ['rustfmt', 'clippy', 'llvm-tools'] if profile.endswith('build') else []) + if profile.endswith('build'): + self.assertIn(['rustup', 'target', 'add', '--toolchain', nightly, target], calls) + nextest = [c for c in calls if c[0] == 'cargo' and c[-1] == 'cargo-nextest'] + self.assertEqual(len(nextest), 1) + self.assertIn('install', nextest[0]) + else: + self.assertFalse(any(c[0] == 'cargo' and ('build' in c or 'install' in c or 'binstall' in c) for c in calls)) + transfers = [c for c in calls if c[0] == 'python3' and c[1].endswith('/tooling/transfer.py')] + self.assertEqual(len(transfers), 1) + self.assertEqual(transfers[0][2:4], ['prepare' if profile.endswith('build') else 'install', target]) def test_proof_tools_only_on_supported_full_ct_hosts(self): for platform in ('x86_64-linux', 'aarch64-linux', 's390x-linux', 'powerpc64le-linux', 'riscv64-linux'): diff --git a/scripts/tooling/linux.sh b/scripts/tooling/linux.sh index 2a8bd27f..974abb50 100755 --- a/scripts/tooling/linux.sh +++ b/scripts/tooling/linux.sh @@ -11,15 +11,24 @@ proof=false case "${1:-}" in --ci-ct-full) ci=true; profile=ci-ct; proof=true; shift ;; --ci) ci=true; shift ;; - --ci-compat|--ci-package|--ci-fuzz|--ci-miri|--ci-ct|--ci-bench|--ci-riscv-build|--ci-riscv-run) ci=true; profile="${1#--}"; shift ;; + --ci-compat|--ci-package|--ci-fuzz|--ci-miri|--ci-ct|--ci-bench|--ci-cross-build|--ci-cross-run) ci=true; profile="${1#--}"; shift ;; esac case "$profile:$platform" in - ci-riscv-run:riscv64-linux) ;; - ci-riscv-run:*|ci-riscv-build:riscv64-linux) echo "invalid RISC-V tooling host" >&2; exit 64 ;; + ci-cross-run:riscv64-linux|ci-cross-run:powerpc64le-linux|ci-cross-run:s390x-linux|ci-cross-build:x86_64-linux) ;; + ci-cross-run:*|ci-cross-build:*) echo "invalid cross-build tooling host" >&2; exit 64 ;; ci:*|ci-bench:*|ci-ct:*|*:x86_64-linux|ci-fuzz:aarch64-linux) ;; *) echo "$profile tooling is unsupported on $platform" >&2; exit 64 ;; esac -[[ "$#" -eq 0 ]] || { echo "usage: scripts/tooling/$platform.sh [--ci|--ci-compat|--ci-package|--ci-fuzz|--ci-miri|--ci-ct|--ci-ct-full|--ci-bench|--ci-riscv-build|--ci-riscv-run]" >&2; exit 64; } +cross_target="" +tools_archive="" +if [[ "$profile" == ci-cross-build ]]; then + cross_target="${1:?cross-build target is required}" + shift +elif [[ "$profile" == ci-cross-run ]]; then + tools_archive="${1:?runner tools archive is required}" + shift +fi +[[ "$#" -eq 0 ]] || { echo "usage: scripts/tooling/$platform.sh [--ci|--ci-compat|--ci-package|--ci-fuzz|--ci-miri|--ci-ct|--ci-ct-full|--ci-bench|--ci-cross-build TARGET|--ci-cross-run TOOLS_ARCHIVE]" >&2; exit 64; } machine="${platform%-linux}" [[ "$machine" != powerpc64le ]] || machine=ppc64le case "$platform" in @@ -79,6 +88,12 @@ python3 "$SCRIPT_DIR/catalog.py" validate package_section="$linux_section" [[ "$profile" == ci || "$profile" == ci-bench || "$profile" == ci-package ]] || package_section="$profile" mapfile -t packages < <(catalog_get "$package_section" packages) +if [[ "$profile" == ci-cross-build ]]; then + cross_prefix="$(python3 scripts/lib/cross_build.py "$cross_target")" + cross_arch="${cross_prefix%-linux-gnu}" + [[ "$cross_arch" != powerpc64le ]] || cross_arch=ppc64el + packages+=("gcc-$cross_prefix" "g++-$cross_prefix" "libc6-dev-$cross_arch-cross") +fi if [[ "$ci" == false ]]; then mapfile -t native_packages < <(catalog_get "$platform" packages) packages+=("${native_packages[@]}") @@ -115,12 +130,12 @@ components=() if [[ "$ci" == false ]]; then mapfile -t components < <(catalog_get "$platform" components); fi component_args=() for component in "${components[@]}"; do component_args+=(--component "$component"); done -if [[ "$profile" == ci-riscv-build ]]; then - nightly="$(python3 "$SCRIPT_DIR/../lib/toolchain.py" --target riscv64gc-unknown-linux-gnu)" +if [[ "$profile" == ci-cross-build ]]; then + nightly="$(python3 "$SCRIPT_DIR/../lib/toolchain.py" --target "$cross_target")" rustup toolchain install "$channel" --profile minimal --component rustfmt rustup toolchain install "$nightly" --profile minimal --component clippy --component llvm-tools - rustup target add --toolchain "$nightly" riscv64gc-unknown-linux-gnu -elif [[ "$profile" == ci-riscv-run ]]; then + rustup target add --toolchain "$nightly" "$cross_target" +elif [[ "$profile" == ci-cross-run ]]; then # No compiler workloads run here; Rust supplies the pinned Nextest launcher # and host identity used by the existing CT orchestrator. stable="$(python3 "$SCRIPT_DIR/../lib/toolchain.py")" @@ -156,7 +171,7 @@ if catalog_get "$platform" assets cargo-binstall >/dev/null 2>&1; then binstall= # Archive tools retain their complete directory layouts, including LLVM and Zig libraries. if [[ "$ci" == true ]]; then : > "$temporary/archives" - if [[ "$binstall" == true ]]; then + if [[ "$binstall" == true && "$profile" != ci-cross-run ]]; then directory="$(python3 "$SCRIPT_DIR/catalog.py" install-archive "$platform" cargo-binstall "$prefix")" printf 'cargo-binstall\t%s\n' "$directory" > "$temporary/archives" fi @@ -198,7 +213,7 @@ fi for tool in "${cargo_tools[@]}"; do version="$(catalog_get cargo "$tool")" # Cargo's install registry verifies exact installed package versions on reruns. - if [[ "$binstall" == true ]]; then + if [[ "$binstall" == true && ( "$profile" != ci-cross-build || "$tool" != cargo-nextest ) ]]; then env -u RUSTC_WRAPPER -u CARGO_ENCODED_RUSTFLAGS \ cargo +"$channel" binstall --locked --no-confirm --targets "$host" --targets "${host%-gnu}-musl" "$tool@$version" else @@ -211,6 +226,15 @@ if [[ "$ci" == false && "$platform" != aarch64-linux && "$platform" != x86_64-li cargo +"$channel" install --locked --target "$(catalog_get "$platform" rust-host)" --version "$(catalog_get versions cargo-rail)" cargo-rail fi +if [[ "$profile" == ci-cross-build ]]; then + python3 "$SCRIPT_DIR/transfer.py" prepare "$cross_target" "target/$cross_target-tools.tar.gz" +elif [[ "$profile" == ci-cross-run ]]; then + tools_parent="$(mktemp -d "$prefix/runner-tools.XXXXXX")" + tools_bin="$(python3 "$SCRIPT_DIR/transfer.py" install "$host" "$tools_archive" "$tools_parent/verified")" + path_prefix="$tools_bin:$path_prefix" + export PATH="$path_prefix:$PATH" +fi + # Persistent paths are shared by interactive shells and non-interactive Bash recipes. environment="$prefix/environment.sh" { @@ -251,6 +275,6 @@ case "$profile" in ci-compat) wasmtime --version ;; ci-fuzz) cargo fuzz --version ;; ci-ct|ci-miri|ci-package|ci-bench) just --version ;; - ci|ci-riscv-build|ci-riscv-run) cargo nextest --version ;; + ci|ci-cross-build|ci-cross-run) cargo nextest --version ;; esac printf 'Installed %s tooling. Load with: source "%s"\n' "$platform" "$environment" diff --git a/scripts/tooling/transfer.py b/scripts/tooling/transfer.py new file mode 100644 index 00000000..1af1b523 --- /dev/null +++ b/scripts/tooling/transfer.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Build pinned runner tools on x86-64 and verify them before native installation.""" + +import argparse +from pathlib import Path +import platform +import subprocess +import sys +import tempfile +import tomllib + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / 'scripts/lib')) +import evidence_bundle as bundle +from cross_build import TARGETS, environment, require_host, verify_elf + +KIND = 'rscrypto.cross.tools' + + +def prepare(target, archive): + if (platform.system(), platform.machine()) != ('Linux', 'x86_64'): + raise ValueError('runner tools must be cross-built on Linux x86-64') + env = environment(target) + identity = bundle.source_identity(ROOT) + pins = tomllib.loads((ROOT / '.config/tooling.toml').read_text())['cargo'] + directory = ROOT / 'target/cross-tools' / target + directory.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix='build-', dir=directory) as temporary: + out = Path(temporary) + for name in ('just', 'cargo-nextest'): + subprocess.run(['cargo', 'install', '--locked', '--version', pins[name], '--target', target, + '--root', str(out), name], cwd=ROOT, env=env, check=True) + verify_elf(out / 'bin' / name, target) + bundle.seal(ROOT, out, KIND, target, identity, {name: pins[name] for name in ('just', 'cargo-nextest')}) + bundle.pack(out, archive) + + +def install(target, archive, destination): + require_host(target) + pins = tomllib.loads((ROOT / '.config/tooling.toml').read_text())['cargo'] + bundle.unpack(archive, destination) + manifest = bundle.verify(ROOT, destination, KIND, target) + if manifest['metadata'] != {name: pins[name] for name in ('just', 'cargo-nextest')}: + raise ValueError('runner tool versions do not match this checkout') + for name in ('just', 'cargo-nextest'): + verify_elf(destination / 'bin' / name, target) + # Only verified tool directories enter PATH, after the caller sees success. + print(destination / 'bin') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('operation', choices=('prepare', 'install')) + parser.add_argument('target', choices=TARGETS) + parser.add_argument('archive', type=Path) + parser.add_argument('destination', nargs='?', type=Path) + args = parser.parse_args() + if args.operation == 'prepare': + if args.destination: + parser.error('prepare does not take a destination') + prepare(args.target, args.archive.resolve()) + else: + if not args.destination: + parser.error('install requires a fresh destination') + install(args.target, args.archive.resolve(), args.destination.resolve()) + + +if __name__ == '__main__': + main() diff --git a/src/auth/ecdsa.rs b/src/auth/ecdsa.rs index 7764c6b4..85335979 100644 --- a/src/auth/ecdsa.rs +++ b/src/auth/ecdsa.rs @@ -3874,6 +3874,11 @@ fn select_signing_generator_affine_ct(curve: &Curve, digit: u .enumerate() { let mask = mask_eq_usize(digit, index); + #[cfg(target_arch = "riscv64")] + // SECURITY: LLVM 23 otherwise branches on the secret digit and loads only + // the matching table entry. Keep the mask opaque so every entry is read. + // Target-specific generated-code and native timing evidence remain required. + let mask = core::hint::black_box(mask); x = Uint::select(x, candidate_x, mask); y = Uint::select(y, candidate_y, mask); }