diff --git a/.github/EXTERNAL_RPC_CHECKS_ISSUE_TEMPLATE.md b/.github/EXTERNAL_RPC_CHECKS_ISSUE_TEMPLATE.md new file mode 100644 index 00000000000..162c901975c --- /dev/null +++ b/.github/EXTERNAL_RPC_CHECKS_ISSUE_TEMPLATE.md @@ -0,0 +1,8 @@ +--- +title: "[automated] External dataset RPC checks failure" +labels: ["Bug"] +--- + +## Description + +Latest RPC checks against the external `data.riba.plus` dataset failed. Please [check the logs]({{ env.WORKFLOW_URL }}) for more information. diff --git a/.github/workflows/external-rpc-checks.yml b/.github/workflows/external-rpc-checks.yml new file mode 100644 index 00000000000..aa954427ff1 --- /dev/null +++ b/.github/workflows/external-rpc-checks.yml @@ -0,0 +1,40 @@ +name: data.riba.plus RPC checks +permissions: + contents: read + issues: write + +concurrency: + group: ${{ github.workflow }} + +on: + workflow_dispatch: + schedule: + - cron: "0 13 * * *" # Runs every day at 13:00 UTC + +jobs: + rpc-checks: + name: RPC checks against the data.riba.plus dataset + runs-on: warp-ubuntu-2404-x64-8x + defaults: + run: + working-directory: scripts/tests/external-rpc-checks + steps: + - name: Checkout Sources + uses: actions/checkout@v7 + - name: Run the RPC checks + run: ./setup.sh + - name: Dump docker logs + if: always() + uses: jwalton/gh-docker-logs@v2 + - name: Set WORKFLOW_URL + if: always() + run: | + export WORKFLOW_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + echo "${WORKFLOW_URL}" + echo "WORKFLOW_URL=${WORKFLOW_URL}" >> "$GITHUB_ENV" + - uses: JasonEtco/create-an-issue@v2 + if: github.ref == 'refs/heads/main' && failure() + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + filename: .github/EXTERNAL_RPC_CHECKS_ISSUE_TEMPLATE.md diff --git a/docs/docs/developers/guides/rpc_api_compatibility.md b/docs/docs/developers/guides/rpc_api_compatibility.md index 470d22ab7a1..032b83c14c5 100644 --- a/docs/docs/developers/guides/rpc_api_compatibility.md +++ b/docs/docs/developers/guides/rpc_api_compatibility.md @@ -50,6 +50,12 @@ Forest. Forest does not yet support mining and none of the mining-related RPC calls will be implemented in the foreseeable future. +## External dataset checks + +Forest is also checked against the external `data.riba.plus` dataset of recorded +RPC responses, see `.github/workflows/external-rpc-checks.yml` and +`scripts/tests/external-rpc-checks`. + ## Gateway The `lotus-gateway` executable is a reverse-proxy that sanitizes RPC calls diff --git a/scripts/tests/external-rpc-checks/.env b/scripts/tests/external-rpc-checks/.env new file mode 100644 index 00000000000..bd2497d7964 --- /dev/null +++ b/scripts/tests/external-rpc-checks/.env @@ -0,0 +1,2 @@ +FOREST_RPC_PORT=2345 +FOREST_HEALTHZ_RPC_PORT=2346 diff --git a/scripts/tests/external-rpc-checks/docker-compose.yaml b/scripts/tests/external-rpc-checks/docker-compose.yaml new file mode 100644 index 00000000000..ad49d91976d --- /dev/null +++ b/scripts/tests/external-rpc-checks/docker-compose.yaml @@ -0,0 +1,68 @@ +name: external-rpc-checks + +x-forest-service: &forest-service + image: ghcr.io/chainsafe/forest:edge-fat + volumes: + - node-data:/data + environment: + # keeps the database in the container's volume, shared across containers + - FOREST_PATH=/data + user: 0:0 + +services: + # Resolves yesterday's snapshot, imports it and back-fills the chain index. + init: + <<: *forest-service + volumes: + - node-data:/data + - ./init.sh:/init.sh:ro + entrypoint: ["/bin/bash", "/init.sh"] + + forest: + <<: *forest-service + environment: + - FOREST_PATH=/data + - FOREST_CHAIN_INDEXER_ENABLED=1 + command: + - --chain=calibnet + - --encrypt-keystore=false + - --no-gc + - --rpc-address=0.0.0.0:${FOREST_RPC_PORT} + - --healthcheck-address=0.0.0.0:${FOREST_HEALTHZ_RPC_PORT} + healthcheck: + # `/livez` from inside the container: RPC answering and peers connected. + test: + [ + "CMD", + "forest-cli", + "healthcheck", + "live", + "--healthcheck-port", + "${FOREST_HEALTHZ_RPC_PORT}", + ] + interval: 30s + timeout: 10s + retries: 3 + start_period: 5m + start_interval: 5s + + # Probes the node for the epochs the checks are about to query. + verify: + image: ghcr.io/chainsafe/forest-rpc-checks:latest + depends_on: + forest: + condition: service_healthy + volumes: + - ./verify.rb:/verify.rb:ro + entrypoint: ["ruby", "/verify.rb"] + environment: + FOREST_RPC_URL: forest:${FOREST_RPC_PORT}/rpc/v1 + + # The image entrypoints into `bundle exec ruby check_rpc.rb`, so only the epochs are appended. + rpc-checks: + image: ghcr.io/chainsafe/forest-rpc-checks:latest + environment: + FOREST_RPC_URL: forest:${FOREST_RPC_PORT}/rpc/v1 + +volumes: + node-data: diff --git a/scripts/tests/external-rpc-checks/init.sh b/scripts/tests/external-rpc-checks/init.sh new file mode 100755 index 00000000000..1d8339d43be --- /dev/null +++ b/scripts/tests/external-rpc-checks/init.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Runs in the `init` service: resolves the snapshot to test against, imports it +# and back-fills the chain index, all before the daemon starts. + +set -euo pipefail + +# The Forest image ships neither curl nor jq. +apt-get update -qq +apt-get install -y -qq --no-install-recommends curl jq + +# The dataset lags the chain, so yesterday's snapshot is the one to test against. +day=$(date -u -d '1 day ago' +%F) +url=$(curl --silent --show-error --fail --retry 3 --connect-timeout 10 --max-time 60 \ + "https://forest-archive.chainsafe.dev/list/calibnet/latest-v2?format=json" | + jq --raw-output --arg day "${day}" '[.items[].url | select(contains("_" + $day + "_"))] | first') +[[ ${url} == https* ]] || { + echo "no calibnet snapshot published for ${day}" + exit 1 +} + +# Snapshot names end in the epoch of their head tipset. ./setup.sh reads it back +# to pick the range to check. +epoch=${url##*_height_} +epoch=${epoch%%.*} +printf '%s\n' "${epoch}" > /data/snapshot-epoch + +forest --chain=calibnet --encrypt-keystore=false --import-snapshot="${url}" --halt-after-import + +# Indexes the 1000 epochs below the snapshot head, inclusive on both ends +forest-tool index backfill --chain=calibnet --from="${epoch}" --to="$((epoch - 1000))" diff --git a/scripts/tests/external-rpc-checks/setup.sh b/scripts/tests/external-rpc-checks/setup.sh new file mode 100755 index 00000000000..de63c8d5260 --- /dev/null +++ b/scripts/tests/external-rpc-checks/setup.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Runs the RPC checks against the external data.riba.plus dataset. +# Needs docker only; everything else happens in containers. + +set -euo pipefail + +PARENT_PATH=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P) +pushd "${PARENT_PATH}" + +# This should not be needed in GH. It is useful for running locally. +docker compose down --remove-orphans --volumes + +# Imports the snapshot and back-fills the index, recording the head epoch. +docker compose run --rm init +SNAPSHOT_EPOCH="$(docker compose run --rm --no-TTY --entrypoint cat init /data/snapshot-epoch)" +START=$((SNAPSHOT_EPOCH - 1000)) +END=$((SNAPSHOT_EPOCH - 1)) + +docker compose up --detach --wait forest + +docker compose run --rm verify "${START}" "${END}" + +docker compose run --rm rpc-checks "${START}" "${END}" + +popd diff --git a/scripts/tests/external-rpc-checks/verify.rb b/scripts/tests/external-rpc-checks/verify.rb new file mode 100644 index 00000000000..4dc1bbe7158 --- /dev/null +++ b/scripts/tests/external-rpc-checks/verify.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +# Confirms the back-fill indexed the epochs the checks query. Runs in a container +# so the node's RPC port never has to be published. + +require 'json' +require 'net/http' + +url = URI("http://#{ENV.fetch('FOREST_RPC_URL')}") + +ARGV.each do |argument| + epoch = Integer(argument) + request = Net::HTTP::Post.new(url, 'Content-Type' => 'application/json') + request.body = { + jsonrpc: '2.0', + id: 1, + method: 'eth_getBlockByNumber', + params: ["0x#{epoch.to_s(16)}", false] + }.to_json + + response = Net::HTTP.start(url.hostname, url.port) { |http| http.request(request) } + body = JSON.parse(response.body) + abort "epoch #{epoch} is not indexed: #{response.code} #{response.body}" unless body.dig('result', 'number') + + puts "epoch #{epoch} is indexed" +end diff --git a/src/tool/offline_server/server.rs b/src/tool/offline_server/server.rs index ddadde4a3d2..dfb530319f0 100644 --- a/src/tool/offline_server/server.rs +++ b/src/tool/offline_server/server.rs @@ -199,7 +199,7 @@ pub async fn start_offline_server( } // Validate tipsets since the {height} EPOCH when `height >= 0`, - // or valiadte the last {-height} EPOCH(s) when `height < 0` + // or validate the last {-height} EPOCH(s) when `height < 0` let validate_until_epoch = if height > 0 { height } else {