diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b3dbfb99..025c8a32 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -37,7 +37,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v7 with: - node-version: 24.22.1 + node-version: 24.22.3 - name: Enable Corepack run: corepack enable diff --git a/.nvmrc b/.nvmrc index ce62013c..229b3ce8 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -24.22.1 +24.22.3 diff --git a/Dockerfile b/Dockerfile index 07fb3425..34543349 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG DOCKER_IMAGE=node:24.22.1-alpine3.24 +ARG DOCKER_IMAGE=node:24.22.3-alpine3.24 # Build FROM ${DOCKER_IMAGE} AS build-stage diff --git a/Dockerfile.brain.dev b/Dockerfile.brain.dev index 9dca83a3..35c4e936 100644 --- a/Dockerfile.brain.dev +++ b/Dockerfile.brain.dev @@ -1,4 +1,4 @@ -ARG DOCKER_IMAGE=node:24.22.1-alpine3.24 +ARG DOCKER_IMAGE=node:24.22.3-alpine3.24 FROM ${DOCKER_IMAGE} AS build-stage RUN apk add --no-cache --upgrade ca-certificates ca-certificates-bundle && update-ca-certificates diff --git a/docs/gloas-builder-config.md b/docs/gloas-builder-config.md index 608185f0..337cade6 100644 --- a/docs/gloas-builder-config.md +++ b/docs/gloas-builder-config.md @@ -1,5 +1,7 @@ # Gloas / ePBS validator configuration +> **Disabled for now.** This feature is switched off with `featureFlags.builderConfig` in `packages/common/src/featureFlags.ts`. While it is off, the UI hides builder settings, the builder RPCs and builder fields in updates are rejected, reload does not apply stored settings, and `BUILDER_IDS` is ignored (its one-time import still runs once the feature is enabled). Settings already saved stay in the database unchanged, and overrides already applied to a validator client by 0.2.0-rc.0 are not removed. + Brain manages configuration. The validator client signs slot-specific proposer preferences and builder authorizations and supplies the resolved configuration in proposal requests. The beacon node solicits and selects bids. Web3Signer diff --git a/package.json b/package.json index 49f6ea2d..a1660b17 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "type": "module", "private": true, "engines": { - "node": ">=24.22.1" + "node": ">=24.22.3" }, "workspaces": [ "packages/*" diff --git a/packages/brain/package.json b/packages/brain/package.json index a135c294..20c19d45 100644 --- a/packages/brain/package.json +++ b/packages/brain/package.json @@ -6,8 +6,9 @@ "main": "dist/index.js", "types": "dist/types.d.ts", "scripts": { - "test": "yarn run test:unit", + "test": "yarn run test:unit && yarn run test:e2e", "test:unit": "mocha --exit --recursive 'test/**/*.unit.test.ts'", + "test:e2e": "mocha --exit --recursive 'test/**/*.e2e.test.ts'", "dev": "yarn run link:ui && tsx --watch src/index.ts", "link:ui": "ln -s ../ui/build/ ./uiBuild", "build": "yarn run clean:dist && yarn run clean:ln && tsc -p tsconfig.json", @@ -33,6 +34,7 @@ "mocha": "^12.0.3", "rimraf": "^6.1.3", "sinon": "^22.1.0", + "socket.io-client": "^4.8.4", "typescript": "~6.0.3" } } diff --git a/packages/brain/src/index.ts b/packages/brain/src/index.ts index bc66f94d..58d9c4c1 100644 --- a/packages/brain/src/index.ts +++ b/packages/brain/src/index.ts @@ -9,6 +9,8 @@ import { params } from "./params.js"; import { getCrons } from "./modules/cron/index.js"; import { brainConfig } from "./modules/config/index.js"; import { importBuilderIds } from "./modules/config/importBuilderIds.js"; +import { BeaconStatusStore } from "./modules/beaconStatus/index.js"; +import { featureFlags } from "@stakingbrain/common"; logger.info(`Starting brain...`); @@ -34,22 +36,30 @@ await brainDb.initialize(signerApi, validatorApi); logger.debug(`Loaded ${Object.keys(brainDb.getData()).length} validators`); // Import builder choices once before cron/API writes can begin. -try { - const imported = importBuilderIds({ - builderIds: process.env.BUILDER_IDS, - network: config.chain.network, - brainDb, - markerPath: path.join(path.dirname(brainDbPath), "builder-config-imported") - }); - if (imported) - logger.info(`Builder settings imported: ${imported.imported} validators; ${imported.skipped} preserved`); -} catch (error) { - // A failed optional handoff must not interrupt ordinary key/fee reconciliation. - logger.error("Unable to import BUILDER_IDS; import will retry at next startup", error); +if (featureFlags.builderConfig) { + try { + const imported = importBuilderIds({ + builderIds: process.env.BUILDER_IDS, + network: config.chain.network, + brainDb, + markerPath: path.join(path.dirname(brainDbPath), "builder-config-imported") + }); + if (imported) + logger.info(`Builder settings imported: ${imported.imported} validators; ${imported.skipped} preserved`); + } catch (error) { + // A failed optional handoff must not interrupt ordinary key/fee reconciliation. + logger.error("Unable to import BUILDER_IDS; import will retry at next startup", error); + } +} else if (process.env.BUILDER_IDS) { + // Leave the one-time marker unwritten so the import still runs once the feature is enabled. + logger.warn("BUILDER_IDS is set but builder configuration is disabled in this version; ignoring it"); } +// Beacon chain details (index, withdrawal credentials) for listing validators. +const beaconStatus = new BeaconStatusStore(brainDb, beaconchainApi); + // CRON -const { reloadValidatorsCronTask } = getCrons({ +const { reloadValidatorsCronTask, beaconStatusCronTask } = getCrons({ sendNotification: true, postgresClient, prometheusApi, @@ -59,9 +69,13 @@ const { reloadValidatorsCronTask } = getCrons({ brainDb, chainConfig: config.chain, beaconchainApi, - dappmanagerApi + dappmanagerApi, + beaconStatus }); reloadValidatorsCronTask.start(); +beaconStatusCronTask.start(); +// Reads only: load beacon details now rather than after the first interval. +void beaconStatusCronTask.runNow(); //trackValidatorsPerformanceCronTask.start(); // Start server APIs @@ -74,13 +88,15 @@ const { uiServer, launchpadServer, brainApiServer } = getServers({ beaconchainApi, brainDb, reloadValidatorsCronTask, - allowedOriginsFromEnv: config.apis.cors + allowedOriginsFromEnv: config.apis.cors, + beaconStatus }); // Graceful shutdown function handle(signal: string): void { logger.info(`${signal} received. Shutting down...`); reloadValidatorsCronTask.stop(); + beaconStatusCronTask.stop(); //trackValidatorsPerformanceCronTask.stop(); brainDb.close(); postgresClient.close().catch((err) => logger.error(`Error closing postgres client`, err)); // postgresClient db connection is the only external resource that needs to be closed diff --git a/packages/brain/src/modules/apiClients/beaconchain/index.ts b/packages/brain/src/modules/apiClients/beaconchain/index.ts index 2ea4881f..28ba3bdf 100644 --- a/packages/brain/src/modules/apiClients/beaconchain/index.ts +++ b/packages/brain/src/modules/apiClients/beaconchain/index.ts @@ -166,16 +166,19 @@ export class BeaconchainApi extends StandardApi { */ public async postStateValidators({ stateId, - body + body, + timeout }: { stateId: BlockId; body: { ids: string[]; statuses: ValidatorStatus[] }; + timeout?: number; }): Promise { try { return await this.request({ method: "POST", endpoint: path.join(this.beaconchainEndpoint, "states", stateId, "validators"), - body: JSON.stringify(body) + body: JSON.stringify(body), + timeout }); } catch (e) { throw new BeaconchainApiError(`Error getting (POST) state validators from beaconchain. ${e.message}`); diff --git a/packages/brain/src/modules/apiClients/signer/index.ts b/packages/brain/src/modules/apiClients/signer/index.ts index 8ed5d405..16ab82aa 100644 --- a/packages/brain/src/modules/apiClients/signer/index.ts +++ b/packages/brain/src/modules/apiClients/signer/index.ts @@ -15,6 +15,9 @@ import path from "node:path"; import { prefix0xPubkey } from "../prefix0xPubkey.js"; import { SignerApiError } from "./error.js"; +/** Longer than the UI's RPC timeout: an import already sent should be recorded even if the UI gave up. */ +export const SIGNER_IMPORT_TIMEOUT_MS = 10 * 60 * 1000; + /** * Key Manager API standard * https://ethereum.github.io/keymanager-APIs/ @@ -121,7 +124,10 @@ export class Web3SignerApi extends StandardApi { method: "POST", endpoint: this.localKeymanagerEndpoint, body: JSON.stringify(postRequest), - headers: this.originHeader + headers: this.originHeader, + // Keystore decryption is slow. Dropping the connection does not stop the signer, it only + // discards the result, so wait long enough to record the outcome of large imports. + timeout: SIGNER_IMPORT_TIMEOUT_MS }); } catch (e) { throw new SignerApiError(`Error importing (POST) keystores to remote signer. ${e.message}`); diff --git a/packages/brain/src/modules/apiServers/index.ts b/packages/brain/src/modules/apiServers/index.ts index dcdcf754..d3c40ea6 100644 --- a/packages/brain/src/modules/apiServers/index.ts +++ b/packages/brain/src/modules/apiServers/index.ts @@ -9,6 +9,7 @@ import { BeaconchainApi } from "../apiClients/beaconchain/index.js"; import { CronJob } from "../cron/cron.js"; import { startLaunchpadApi } from "./launchpad/index.js"; import http from "http"; +import { BeaconStatusStore } from "../beaconStatus/index.js"; export const getServers = ({ brainConfig, @@ -19,7 +20,8 @@ export const getServers = ({ beaconchainApi, brainDb, reloadValidatorsCronTask, - allowedOriginsFromEnv + allowedOriginsFromEnv, + beaconStatus }: { brainConfig: BrainConfig; uiBuildPath: string; @@ -30,6 +32,7 @@ export const getServers = ({ brainDb: BrainDataBase; reloadValidatorsCronTask: CronJob; allowedOriginsFromEnv: string | string[] | null; + beaconStatus: BeaconStatusStore; }): { uiServer: http.Server; launchpadServer: http.Server; @@ -45,13 +48,13 @@ export const getServers = ({ validatorApi, beaconchainApi, postgresClient, - allowedOriginsFromEnv + allowedOriginsFromEnv, + beaconStatus }), launchpadServer: startLaunchpadApi({ brainDb, signerApi, validatorApi, - beaconchainApi, reloadValidatorsCronTask, network: brainConfig.chain.network, signerUrl: brainConfig.apis.signerUrl, diff --git a/packages/brain/src/modules/apiServers/launchpad/routes/feeRecipients/route.ts b/packages/brain/src/modules/apiServers/launchpad/routes/feeRecipients/route.ts index 89e60cd0..f07a95f3 100644 --- a/packages/brain/src/modules/apiServers/launchpad/routes/feeRecipients/route.ts +++ b/packages/brain/src/modules/apiServers/launchpad/routes/feeRecipients/route.ts @@ -1,16 +1,17 @@ import express from "express"; import { isValidBlsPubkey } from "@stakingbrain/common"; import { updateValidators } from "../../../ui/calls/updateValidators.js"; -import { getValidators } from "../../../ui/calls/getValidators.js"; import { BrainPubkeysFeeRecipients } from "../../types.js"; import { validateUpdateFeeRecipientRequestBody } from "./validation.js"; -import type { CustomValidatorGetResponse } from "../../../ui/calls/types.js"; import logger from "../../../../logger/index.js"; import { BrainDataBase } from "../../../../db/index.js"; import { CronJob } from "../../../../cron/cron.js"; -import { BeaconchainApi } from "../../../../apiClients/beaconchain/index.js"; -import { Web3SignerApi } from "../../../../apiClients/signer/index.js"; import { ValidatorApi } from "../../../../apiClients/validator/index.js"; +import { ValidatorOperationBusyError } from "../../../../db/withValidatorOperationLock.js"; + +/** Stored fee recipients; client status is not needed, so reading them makes no client calls. */ +const storedFeeRecipients = (brainDb: BrainDataBase) => + Object.entries(brainDb.getData()).map(([pubkey, { feeRecipient }]) => ({ pubkey, feeRecipient })); /** * Retrieves fee recipient information for specified validators via GET request. @@ -40,17 +41,13 @@ import { ValidatorApi } from "../../../../apiClients/validator/index.js"; export const createFeeRecipientsRouter = ({ brainDb, - signerApi, signerUrl, validatorApi, - beaconchainApi, reloadValidatorsCronTask }: { brainDb: BrainDataBase; - signerApi: Web3SignerApi; signerUrl: string; validatorApi: ValidatorApi; - beaconchainApi: BeaconchainApi; reloadValidatorsCronTask: CronJob; }) => { const feeRecipientsRouter = express.Router(); @@ -58,12 +55,7 @@ export const createFeeRecipientsRouter = ({ feeRecipientsRouter.get(feeRecipientsEndpoint, async (req, res) => { try { - const validators: CustomValidatorGetResponse[] = await getValidators({ - brainDb, - signerApi, - validatorApi, - beaconchainApi - }); + const validators = storedFeeRecipients(brainDb); const pubkeysParam = req.query.pubkeys as string | undefined; if (!pubkeysParam) { @@ -143,12 +135,7 @@ export const createFeeRecipientsRouter = ({ } try { - const currentValidators: CustomValidatorGetResponse[] = await getValidators({ - brainDb, - signerApi, - validatorApi, - beaconchainApi - }); + const currentValidators = storedFeeRecipients(brainDb); const validatorsToUpdate = currentValidators.filter((validator) => requestBody.validators.some( @@ -177,7 +164,8 @@ export const createFeeRecipientsRouter = ({ res.status(200).send(); } catch (error) { logger.error("Failed to update validators:", error); - res.status(500).send({ message: "Internal server error" }); + if (error instanceof ValidatorOperationBusyError) res.status(503).send({ message: error.message }); + else res.status(500).send({ message: "Internal server error" }); } }); diff --git a/packages/brain/src/modules/apiServers/launchpad/routes/keystores/route.ts b/packages/brain/src/modules/apiServers/launchpad/routes/keystores/route.ts index 32085794..c8afcd37 100644 --- a/packages/brain/src/modules/apiServers/launchpad/routes/keystores/route.ts +++ b/packages/brain/src/modules/apiServers/launchpad/routes/keystores/route.ts @@ -1,4 +1,5 @@ import { Network, Tag } from "@stakingbrain/common"; +import { ValidatorOperationBusyError } from "../../../../db/withValidatorOperationLock.js"; import express from "express"; import { deleteValidators } from "../../../ui/calls/deleteValidators.js"; import { importValidators } from "../../../ui/calls/importValidators.js"; @@ -11,6 +12,13 @@ import { BrainDataBase } from "../../../../db/index.js"; import { Web3SignerApi } from "../../../../apiClients/signer/index.js"; import { ValidatorApi } from "../../../../apiClients/validator/index.js"; +/** Busy is temporary and safe to retry: nothing was changed. Other failures stay opaque. */ +function sendOperationError(res: express.Response, error: Error): void { + logger.error(error); + if (error instanceof ValidatorOperationBusyError) res.status(503).send({ message: error.message }); + else res.status(500).send({ message: "Internal server error" }); +} + export const createKeystoresRouter = ({ signerApi, reloadValidatorsCronTask, @@ -62,8 +70,7 @@ export const createKeystoresRouter = ({ res.status(200).send(importResponse); } catch (e) { - logger.error(e); - res.status(500).send({ message: "Internal server error" }); + sendOperationError(res, e); } }); @@ -89,8 +96,7 @@ export const createKeystoresRouter = ({ res.status(200).send(deleteResponse); } catch (e) { - logger.error(e); - res.status(500).send({ message: "Internal server error" }); + sendOperationError(res, e); } }); diff --git a/packages/brain/src/modules/apiServers/launchpad/startLaunchpadApi.ts b/packages/brain/src/modules/apiServers/launchpad/startLaunchpadApi.ts index 853ef314..c4f7b5f2 100644 --- a/packages/brain/src/modules/apiServers/launchpad/startLaunchpadApi.ts +++ b/packages/brain/src/modules/apiServers/launchpad/startLaunchpadApi.ts @@ -8,14 +8,12 @@ import { createKeystoresRouter, createFeeRecipientsRouter } from "./routes/index import { CronJob } from "../../cron/cron.js"; import { BrainDataBase } from "../../db/index.js"; import { Network, RPC_MAX_PAYLOAD_BYTES } from "@stakingbrain/common"; -import { BeaconchainApi } from "../../apiClients/beaconchain/index.js"; import { Web3SignerApi } from "../../apiClients/signer/index.js"; import { ValidatorApi } from "../../apiClients/validator/index.js"; export function startLaunchpadApi({ signerApi, validatorApi, - beaconchainApi, reloadValidatorsCronTask, brainDb, network, @@ -24,7 +22,6 @@ export function startLaunchpadApi({ }: { signerApi: Web3SignerApi; validatorApi: ValidatorApi; - beaconchainApi: BeaconchainApi; reloadValidatorsCronTask: CronJob; brainDb: BrainDataBase; network: Network; @@ -35,14 +32,21 @@ export function startLaunchpadApi({ app.use(express.json({ limit: RPC_MAX_PAYLOAD_BYTES, inflate: false })); app.use(cors({ origin: allowedOriginsFromEnv ?? allowedOrigins })); - app.use(createKeystoresRouter({ reloadValidatorsCronTask, brainDb, network, validatorApi, signerApi, signerUrl })); app.use( - createFeeRecipientsRouter({ + createKeystoresRouter({ + reloadValidatorsCronTask, brainDb, + network, + validatorApi, signerApi, + signerUrl + }) + ); + app.use( + createFeeRecipientsRouter({ + brainDb, signerUrl, validatorApi, - beaconchainApi, reloadValidatorsCronTask }) ); diff --git a/packages/brain/src/modules/apiServers/ui/calls/deleteValidators.ts b/packages/brain/src/modules/apiServers/ui/calls/deleteValidators.ts index 2aade46f..154a96ff 100644 --- a/packages/brain/src/modules/apiServers/ui/calls/deleteValidators.ts +++ b/packages/brain/src/modules/apiServers/ui/calls/deleteValidators.ts @@ -1,4 +1,4 @@ -import { withValidatorOperationLock } from "../../../db/withValidatorOperationLock.js"; +import { CLIENT_OPERATION_MAX_WAIT_MS, withValidatorOperationLock } from "../../../db/withValidatorOperationLock.js"; import { Web3SignerApi, ValidatorApi } from "../../../apiClients/index.js"; import { Web3signerDeleteRequest, Web3signerDeleteResponse } from "../../../apiClients/types.js"; import { CronJob } from "../../../cron/cron.js"; @@ -25,47 +25,51 @@ export async function deleteValidators({ brainDb: BrainDataBase; deleteRequest: Web3signerDeleteRequest; }): Promise { - return withValidatorOperationLock(brainDb, async () => { - try { - if ( - !deleteRequest || - !Array.isArray(deleteRequest.pubkeys) || - deleteRequest.pubkeys.length === 0 || - Array.from(deleteRequest.pubkeys).some( - (key) => typeof key !== "string" || !isValidBlsPubkey(prefix0xPubkey(key)) + return withValidatorOperationLock( + brainDb, + async () => { + try { + if ( + !deleteRequest || + !Array.isArray(deleteRequest.pubkeys) || + deleteRequest.pubkeys.length === 0 || + Array.from(deleteRequest.pubkeys).some( + (key) => typeof key !== "string" || !isValidBlsPubkey(prefix0xPubkey(key)) + ) ) - ) - throw new Error("Select valid validator public keys to delete"); - deleteRequest = { pubkeys: deleteRequest.pubkeys.map(prefix0xPubkey) }; - // Pause future ticks while this operation holds the shared validator lock. - reloadValidatorsCronTask.stop(); + throw new Error("Select valid validator public keys to delete"); + deleteRequest = { pubkeys: deleteRequest.pubkeys.map(prefix0xPubkey) }; + // Pause future ticks while this operation holds the shared validator lock. + reloadValidatorsCronTask.stop(); - // Delete keystores on web3signer API - const web3signerDeleteResponse = await signerApi.deleteRemoteKeys(deleteRequest); + // Delete keystores on web3signer API + const web3signerDeleteResponse = await signerApi.deleteRemoteKeys(deleteRequest); - const deletedPubkeys = deleteRequest.pubkeys.filter((_, index) => - ["deleted", "not_found", "not_active"].includes(web3signerDeleteResponse.data[index]?.status) - ); - // Persist before client cleanup so reload can finish an interrupted deletion. - brainDb.deleteValidators(deletedPubkeys); - if (deletedPubkeys.length > 0) { - const clientResponse = await validatorApi - .deleteRemoteKeys({ pubkeys: deletedPubkeys }) - .catch((err) => logger.error(`Error deleting validator pubkeys`, err)); - for (const [index, pubkey] of deletedPubkeys.entries()) { - if (!clientResponse || !["deleted", "not_found"].includes(clientResponse.data[index]?.status)) continue; - await validatorApi - .deleteFeeRecipient(pubkey) - .catch((err) => logger.error(`Error deleting validator feeRecipient`, err)); + const deletedPubkeys = deleteRequest.pubkeys.filter((_, index) => + ["deleted", "not_found", "not_active"].includes(web3signerDeleteResponse.data[index]?.status) + ); + // Persist before client cleanup so reload can finish an interrupted deletion. + brainDb.deleteValidators(deletedPubkeys); + if (deletedPubkeys.length > 0) { + const clientResponse = await validatorApi + .deleteRemoteKeys({ pubkeys: deletedPubkeys }) + .catch((err) => logger.error(`Error deleting validator pubkeys`, err)); + for (const [index, pubkey] of deletedPubkeys.entries()) { + if (!clientResponse || !["deleted", "not_found"].includes(clientResponse.data[index]?.status)) continue; + await validatorApi + .deleteFeeRecipient(pubkey) + .catch((err) => logger.error(`Error deleting validator feeRecipient`, err)); + } } - } - // IMPORTANT: start the cron - reloadValidatorsCronTask.start(); - return web3signerDeleteResponse; - } catch (e) { - reloadValidatorsCronTask.restart(); - throw e; - } - }); + // IMPORTANT: start the cron + reloadValidatorsCronTask.start(); + return web3signerDeleteResponse; + } catch (e) { + reloadValidatorsCronTask.restart(); + throw e; + } + }, + { maxWaitMs: CLIENT_OPERATION_MAX_WAIT_MS } + ); } diff --git a/packages/brain/src/modules/apiServers/ui/calls/exitValidators.ts b/packages/brain/src/modules/apiServers/ui/calls/exitValidators.ts index 0035750b..ca2d4641 100644 --- a/packages/brain/src/modules/apiServers/ui/calls/exitValidators.ts +++ b/packages/brain/src/modules/apiServers/ui/calls/exitValidators.ts @@ -4,6 +4,7 @@ import { ValidatorExitExecute, ValidatorExitGet } from "../../../apiClients/types.js"; +import { BEACON_CHUNK_SIZE } from "../../../beaconStatus/index.js"; import logger from "../../../logger/index.js"; /** @@ -96,15 +97,19 @@ async function _getExitValidators({ // Get the genesis from the beaconchain API to sign the voluntary exit const genesis = await beaconchainApi.getGenesis(); - // Get the validators indexes from the validator API - const validatorPubkeysIndexes = ( - await Promise.all(pubkeys.map((pubkey) => beaconchainApi.getStateValidator({ state: "head", pubkey }))) - ).map((validator) => { - return { - pubkey: validator.data.validator.pubkey, - index: validator.data.index - }; - }); + // Look up indexes with bulk requests: one request per lookup would flood the beacon node for large batches. + const found = new Map(); + for (let start = 0; start < pubkeys.length; start += BEACON_CHUNK_SIZE) { + const { data } = await beaconchainApi.postStateValidators({ + stateId: "head", + body: { ids: pubkeys.slice(start, start + BEACON_CHUNK_SIZE), statuses: [] } + }); + for (const validator of data) found.set(validator.validator.pubkey.toLowerCase(), validator.index); + } + const missing = pubkeys.filter((pubkey) => !found.has(pubkey.toLowerCase())); + if (missing.length > 0) + throw new Error(`Validators not found on the beacon chain, so they cannot exit: ${missing.join(", ")}`); + const validatorPubkeysIndexes = pubkeys.map((pubkey) => ({ pubkey, index: found.get(pubkey.toLowerCase())! })); const validatorsExit: ValidatorExitGet[] = []; for (const validatorIndex of validatorPubkeysIndexes) { diff --git a/packages/brain/src/modules/apiServers/ui/calls/getBuilderConfigs.ts b/packages/brain/src/modules/apiServers/ui/calls/getBuilderConfigs.ts index 1fe50854..5caf7041 100644 --- a/packages/brain/src/modules/apiServers/ui/calls/getBuilderConfigs.ts +++ b/packages/brain/src/modules/apiServers/ui/calls/getBuilderConfigs.ts @@ -1,5 +1,6 @@ import { isValidBlsPubkey, prefix0xPubkey } from "@stakingbrain/common"; import { BrainDataBase } from "../../../db/index.js"; +import { storedPubkeys } from "../../../db/utils.js"; import { ValidatorApi } from "../../../apiClients/validator/index.js"; import { ValidatorBuilderConfigResponse } from "./types.js"; @@ -21,9 +22,10 @@ export async function getBuilderConfigs({ ) throw new Error("pubkeys must be an array of validator BLS public keys"); const dbData = brainDb.getData(); + const storedKeys = storedPubkeys(dbData); const results: ValidatorBuilderConfigResponse[] = pubkeys.map((key) => { - const pubkey = prefix0xPubkey(key); - if (!dbData[pubkey]) throw new Error(`Validator ${pubkey} is not in the database`); + const pubkey = storedKeys.get(prefix0xPubkey(key).toLowerCase()); + if (!pubkey) throw new Error(`Validator ${prefix0xPubkey(key)} is not in the database`); const builderConfig = dbData[pubkey].builderConfig; return { pubkey, diff --git a/packages/brain/src/modules/apiServers/ui/calls/getValidators.ts b/packages/brain/src/modules/apiServers/ui/calls/getValidators.ts index 840fff89..5887a8ec 100644 --- a/packages/brain/src/modules/apiServers/ui/calls/getValidators.ts +++ b/packages/brain/src/modules/apiServers/ui/calls/getValidators.ts @@ -1,130 +1,180 @@ -import { isValidWithdrawableBlsAddress, isValidNonWithdrawableBlsAddress } from "@stakingbrain/common"; +import { featureFlags } from "@stakingbrain/common"; import { CustomValidatorGetResponse } from "./types.js"; import { BrainDataBase } from "../../../db/index.js"; -import { ValidatorApi, Web3SignerApi, BeaconchainApi } from "../../../apiClients/index.js"; +import { ValidatorApi, Web3SignerApi } from "../../../apiClients/index.js"; +import { BeaconStatusStore } from "../../../beaconStatus/index.js"; +import { forEachConcurrently } from "../../../utils/forEachConcurrently.js"; import logger from "../../../logger/index.js"; -type BeaconDetails = Pick; -type ListProgress = { - beaconNext: number; +/** Bound on one list request; unfinished reads are reported as pending for the UI to request again. */ +export const LIST_DEADLINE_MS = 10_000; +const READ_CONCURRENCY = 8; +const FEE_CHECK_TTL_MS = 60_000; + +interface FeeCheck { + expected: string; + correct: boolean; + expires: number; +} +interface ReadState { + /** Where the next request resumes fee reads, so successive requests cover different validators. */ feeNext: number; - beacon: Map; -}; -const listProgress = new WeakMap(); -const BEACON_CACHE_MS = 60_000; + fee: Map; + /** Bumped on every validator change; a read that overlaps one is not cached. */ + generation: number; +} +const readStates = new WeakMap(); + +function readStateFor(brainDb: BrainDataBase): ReadState { + let state = readStates.get(brainDb); + if (!state) { + const created: ReadState = { feeNext: 0, fee: new Map(), generation: 0 }; + // Change notifications arrive after an operation has configured the validator client. A check read + // mid-operation saw the new expected address but the old client value, so only mismatches can be + // stale; drop them, and refuse to cache reads that straddle the change. + brainDb.onChange(() => { + for (const [pubkey, check] of created.fee) if (!check.correct) created.fee.delete(pubkey); + created.generation++; + }); + readStates.set(brainDb, created); + state = created; + } + return state; +} + +const message = (error: unknown) => (error instanceof Error ? error.message : String(error)); -/** Read a stable snapshot; independent queues rotate unfinished work across bounded refreshes. */ +/** + * List validators with their signer and validator client status, read within one bounded request. + * Reads cut off by the deadline are marked `detailsPending` (request again); failed reads are reported + * in `detailsError` rather than replaced by an earlier result. Beacon details come from the beacon + * store, which announces its updates. + */ export async function getValidators({ brainDb, validatorApi, signerApi, - beaconchainApi + beaconStatus }: { brainDb: BrainDataBase; validatorApi: ValidatorApi; signerApi: Web3SignerApi; - beaconchainApi: BeaconchainApi; + beaconStatus: BeaconStatusStore; }): Promise { const entries = Object.entries(brainDb.getData()); - const progress = listProgress.get(brainDb) ?? { beaconNext: 0, feeNext: 0, beacon: new Map() }; - listProgress.set(brainDb, progress); - const currentKeys = new Set(entries.map(([pubkey]) => pubkey)); - for (const [pubkey, cached] of progress.beacon) { - if (!currentKeys.has(pubkey) || cached.expires <= Date.now()) progress.beacon.delete(pubkey); - } if (entries.length === 0) return []; - const results: CustomValidatorGetResponse[] = entries.map(([pubkey, details]) => ({ - pubkey, - index: details.index?.toString() ?? "", - tag: details.tag, - feeRecipient: details.feeRecipient, - builderConfig: details.builderConfig, - builderConfigStatus: - validatorApi.getBuilderConfigStatus(pubkey) ?? - (details.builderConfig === undefined ? undefined : { status: "pending" }), - withdrawalCredentials: { address: "", format: "error" }, - ...progress.beacon.get(pubkey)?.details - // Membership and fee checks stay undefined until a successful read establishes their value. - })); - const controller = new AbortController(); - const deadline = setTimeout(() => controller.abort(), 10000); + const state = readStateFor(brainDb); + const now = Date.now(); + const present = new Set(entries.map(([pubkey]) => pubkey)); + for (const [pubkey, check] of state.fee) if (!present.has(pubkey) || check.expires <= now) state.fee.delete(pubkey); - async function readQueue( - cursor: "beaconNext" | "feeNext", - read: (result: CustomValidatorGetResponse) => Promise - ) { - const start = progress[cursor] % results.length; - let next = 0; - await Promise.all( - Array.from({ length: Math.min(8, results.length) }, async () => { - while (next < results.length && !controller.signal.aborted) { - const index = (start + next++) % results.length; - progress[cursor] = (index + 1) % results.length; - try { - await read(results[index]); - } catch (error) { - logger.error(error); - } - } + const rows: CustomValidatorGetResponse[] = entries.map(([pubkey, details]) => { + const { beaconError, ...beacon } = beaconStatus.view(pubkey); + return { + pubkey, + tag: details.tag, + feeRecipient: details.feeRecipient, + ...beacon, + index: details.index?.toString() ?? beacon.index ?? "", + detailsPending: false, + detailsError: beaconError, + // Stored settings are kept while the feature is off, but not surfaced. + ...(featureFlags.builderConfig && { + builderConfig: details.builderConfig, + builderConfigStatus: + validatorApi.getBuilderConfigStatus(pubkey) ?? + (details.builderConfig === undefined ? undefined : { status: "pending" }) }) - ); - } + }; + }); + const errors = rows.map((row) => (row.detailsError ? [row.detailsError] : [])); + const pending = new Set(); + + const controller = new AbortController(); + const deadline = setTimeout(() => controller.abort(), LIST_DEADLINE_MS); + const { signal } = controller; + /** Cut off by the deadline: pending. Anything else: a definite failure to report. */ + const failed = (indexes: Iterable, what: string, error: unknown) => { + for (const index of indexes) { + if (signal.aborted) pending.add(index); + else errors[index].push(`${what}: ${message(error)}`); + } + }; try { await Promise.all([ (async () => { try { - const remoteKeys = await validatorApi.getRemoteKeys(controller.signal); - const keys = new Set(remoteKeys.data.map(({ pubkey }) => pubkey.toLowerCase())); - for (const result of results) result.validatorImported = keys.has(result.pubkey.toLowerCase()); - await readQueue("feeNext", async (result) => { - if (!result.validatorImported) return; - const response = await validatorApi.getFeeRecipient(result.pubkey, controller.signal); - result.validatorFeeRecipientCorrect = - response.data.ethaddress.toLowerCase() === result.feeRecipient.toLowerCase(); - }); + const keys = new Set( + (await signerApi.listRemoteKeys(signal)).data.map(({ validating_pubkey }) => + validating_pubkey.toLowerCase() + ) + ); + for (const row of rows) row.signerImported = keys.has(row.pubkey.toLowerCase()); } catch (error) { logger.error(error); + failed(rows.keys(), "Could not read the signer's keys", error); } })(), (async () => { + let keys: Set; try { - const signerKeys = await signerApi.listRemoteKeys(controller.signal); - const keys = new Set(signerKeys.data.map(({ validating_pubkey }) => validating_pubkey.toLowerCase())); - for (const result of results) result.signerImported = keys.has(result.pubkey.toLowerCase()); + keys = new Set((await validatorApi.getRemoteKeys(signal)).data.map(({ pubkey }) => pubkey.toLowerCase())); } catch (error) { logger.error(error); - } - })(), - readQueue("beaconNext", async (result) => { - const cached = progress.beacon.get(result.pubkey); - if (cached) { - Object.assign(result, cached.details); + failed(rows.keys(), "Could not read the validator client's keys", error); return; } - const { data } = await beaconchainApi.getStateValidator({ - state: "head", - pubkey: result.pubkey, - signal: controller.signal + // Fee recipients are only checked for keys the client has; cached checks need a current + // membership read, so an outage is never hidden behind an earlier success. + const toRead: number[] = []; + rows.forEach((row, index) => { + row.validatorImported = keys.has(row.pubkey.toLowerCase()); + // A check made before the key left the client says nothing about it once it returns. + if (!row.validatorImported) { + state.fee.delete(row.pubkey); + return; + } + const cached = state.fee.get(row.pubkey); + if (cached?.expected === row.feeRecipient.toLowerCase()) row.validatorFeeRecipientCorrect = cached.correct; + else toRead.push(index); }); - const address = data.validator.withdrawal_credentials; - const details: BeaconDetails = { - index: data.index, - withdrawalCredentials: { - address, - format: isValidWithdrawableBlsAddress(address) - ? "ecdsa" - : isValidNonWithdrawableBlsAddress(address) - ? "bls" - : "unknown" + // Resume where the previous request stopped, so a large list converges over a few requests. + const start = toRead.findIndex((index) => index >= state.feeNext % rows.length); + const order = start <= 0 ? toRead : [...toRead.slice(start), ...toRead.slice(0, start)]; + order.forEach((index) => pending.add(index)); + await forEachConcurrently(order, READ_CONCURRENCY, async (index) => { + if (signal.aborted) return; + // Advance on dispatch: reads that hang until the deadline must not hold the next request back. + state.feeNext = index + 1; + const row = rows[index]; + const expected = row.feeRecipient.toLowerCase(); + const generation = state.generation; + try { + const { data } = await validatorApi.getFeeRecipient(row.pubkey, signal); + row.validatorFeeRecipientCorrect = data.ethaddress.toLowerCase() === expected; + pending.delete(index); + if (state.generation === generation) + state.fee.set(row.pubkey, { + expected, + correct: row.validatorFeeRecipientCorrect, + expires: Date.now() + FEE_CHECK_TTL_MS + }); + } catch (error) { + if (!signal.aborted) pending.delete(index); + logger.error(error); + failed([index], "Could not read the fee recipient", error); } - }; - Object.assign(result, details); - progress.beacon.set(result.pubkey, { details, expires: Date.now() + BEACON_CACHE_MS }); - }) + }); + })() ]); - return results; } finally { clearTimeout(deadline); } + + rows.forEach((row, index) => { + row.detailsPending = pending.has(index); + row.detailsError = errors[index].length > 0 ? errors[index].join("; ") : undefined; + }); + return rows; } diff --git a/packages/brain/src/modules/apiServers/ui/calls/importValidators.ts b/packages/brain/src/modules/apiServers/ui/calls/importValidators.ts index 14b6dd78..fadf115e 100644 --- a/packages/brain/src/modules/apiServers/ui/calls/importValidators.ts +++ b/packages/brain/src/modules/apiServers/ui/calls/importValidators.ts @@ -1,4 +1,4 @@ -import { withValidatorOperationLock } from "../../../db/withValidatorOperationLock.js"; +import { CLIENT_OPERATION_MAX_WAIT_MS, withValidatorOperationLock } from "../../../db/withValidatorOperationLock.js"; import { isFeeRecipientEditable, isValidBlsPubkey, @@ -49,157 +49,166 @@ export async function importValidators({ signerUrl: string; brainDb: BrainDataBase; }): Promise { - return withValidatorOperationLock(brainDb, async () => { - reloadValidatorsCronTask.stop(); - try { - if ( - !postRequest || - !Array.isArray(postRequest.validatorsImportRequest) || - postRequest.validatorsImportRequest.length === 0 - ) - throw new Error("Select at least one validator to import"); - if (!["ui", "api"].includes(postRequest.importFrom)) throw new Error("Invalid import origin"); - const existingData = brainDb.getData(); - const existing = new Map(Object.keys(existingData).map((pubkey) => [pubkey.toLowerCase(), pubkey])); - const validators: ValidatorImportRequest[] = []; - const validatorsToPost: ValidatorImportRequest[] = []; - const results: Web3signerPostResponse = { data: [] }; - const seen = new Set(); - for (const [requestIndex, validator] of postRequest.validatorsImportRequest.entries()) { - try { - if (!validator || (typeof validator.keystore !== "string" && !Buffer.isBuffer(validator.keystore))) - throw new Error("Keystore must contain JSON text"); - const keystore = validator.keystore.toString(); - let parsed: { pubkey?: string } | null; + return withValidatorOperationLock( + brainDb, + async () => { + reloadValidatorsCronTask.stop(); + try { + if ( + !postRequest || + !Array.isArray(postRequest.validatorsImportRequest) || + postRequest.validatorsImportRequest.length === 0 + ) + throw new Error("Select at least one validator to import"); + if (!["ui", "api"].includes(postRequest.importFrom)) throw new Error("Invalid import origin"); + const existingData = brainDb.getData(); + const existing = new Map(Object.keys(existingData).map((pubkey) => [pubkey.toLowerCase(), pubkey])); + const validators: ValidatorImportRequest[] = []; + const validatorsToPost: ValidatorImportRequest[] = []; + const results: Web3signerPostResponse = { data: [] }; + const seen = new Set(); + for (const [requestIndex, validator] of postRequest.validatorsImportRequest.entries()) { try { - parsed = JSON.parse(keystore); - } catch { - throw new Error("Invalid keystore JSON"); + if (!validator || (typeof validator.keystore !== "string" && !Buffer.isBuffer(validator.keystore))) + throw new Error("Keystore must contain JSON text"); + const keystore = validator.keystore.toString(); + let parsed: { pubkey?: string } | null; + try { + parsed = JSON.parse(keystore); + } catch { + throw new Error("Invalid keystore JSON"); + } + if (typeof parsed?.pubkey !== "string" || !isValidBlsPubkey(prefix0xPubkey(parsed.pubkey))) + throw new Error("Keystore must contain a valid BLS public key"); + const pubkey = prefix0xPubkey(parsed.pubkey).toLowerCase(); + if (seen.has(pubkey)) throw new Error(`Duplicate public key in import request: ${shortenPubkey(pubkey)}`); + if (!tags.includes(validator.tag)) throw new Error("Invalid validator tag"); + if (typeof validator.password !== "string") throw new Error("Keystore password must be a string"); + const feeRecipient = + !["gnosis", "lukso"].includes(network) && !isFeeRecipientEditable(validator.tag, postRequest.importFrom) + ? getNonEditableFeeRecipient( + validator.tag as NonEditableFeeRecipientTag, + network, + validator.feeRecipient + ) + : validator.feeRecipient; + if (typeof feeRecipient !== "string" || !isValidEcdsaPubkey(feeRecipient)) + throw new Error(`Invalid fee recipient for ${shortenPubkey(pubkey)}`); + seen.add(pubkey); + logger.info(`Setting ${feeRecipient} as fee recipient for ${pubkey}`); + validators.push({ + requestIndex, + keystore, + password: validator.password, + tag: validator.tag, + feeRecipient: feeRecipient, + pubkey + }); + } catch (e) { + results.data[requestIndex] = { + status: "error", + message: e.message + }; } - if (typeof parsed?.pubkey !== "string" || !isValidBlsPubkey(prefix0xPubkey(parsed.pubkey))) - throw new Error("Keystore must contain a valid BLS public key"); - const pubkey = prefix0xPubkey(parsed.pubkey).toLowerCase(); - if (seen.has(pubkey)) throw new Error(`Duplicate public key in import request: ${shortenPubkey(pubkey)}`); - if (!tags.includes(validator.tag)) throw new Error("Invalid validator tag"); - if (typeof validator.password !== "string") throw new Error("Keystore password must be a string"); - const feeRecipient = - !["gnosis", "lukso"].includes(network) && !isFeeRecipientEditable(validator.tag, postRequest.importFrom) - ? getNonEditableFeeRecipient(validator.tag as NonEditableFeeRecipientTag, network, validator.feeRecipient) - : validator.feeRecipient; - if (typeof feeRecipient !== "string" || !isValidEcdsaPubkey(feeRecipient)) - throw new Error(`Invalid fee recipient for ${shortenPubkey(pubkey)}`); - seen.add(pubkey); - logger.info(`Setting ${feeRecipient} as fee recipient for ${pubkey}`); - validators.push({ - requestIndex, - keystore, - password: validator.password, - tag: validator.tag, - feeRecipient: feeRecipient, - pubkey - }); - } catch (e) { - results.data[requestIndex] = { - status: "error", - message: e.message - }; } - } - if (validators.length === 0) return results; - const toDbRecords = (entries: ValidatorImportRequest[]): StakingBrainDb => - Object.fromEntries( - entries.map(({ pubkey, tag, feeRecipient }) => [ - pubkey, - { - tag, - feeRecipient, - automaticImport: postRequest.importFrom !== "ui", - importPending: true + if (validators.length === 0) return results; + const toDbRecords = (entries: ValidatorImportRequest[]): StakingBrainDb => + Object.fromEntries( + entries.map(({ pubkey, tag, feeRecipient }) => [ + pubkey, + { + tag, + feeRecipient, + automaticImport: postRequest.importFrom !== "ui", + importPending: true + } + ]) + ); + // Check all database constraints, including size, before changing signer storage. + brainDb.validateNewValidators(toDbRecords(validators)); + // Journal new keys before the signer can change state. Keep uncertain outcomes across restarts. + brainDb.addValidators({ validators: toDbRecords(validators) }); + + // Import keystores and passwords on web3signer API + const web3signerPostResponse = await signerApi.importRemoteKeys({ + keystores: validators.map((validator) => validator.keystore), + passwords: validators.map((validator) => validator.password), + slashing_protection: postRequest.slashing_protection ? postRequest.slashing_protection.toString() : undefined + }); + + logger.debug(`Imported keystores into web3signer API: ${JSON.stringify(web3signerPostResponse.data)}`); + + // Signer API import keystore may fail for some keystores, but not all + // @see https://github.com/ConsenSys/web3signer/issues/713 + // Remove explicit new-key failures, but retain intent when a result is missing. + const rejectedNewKeys: string[] = []; + const confirmedKeys: string[] = []; + for (const [index, validator] of validators.entries()) { + const { pubkey, requestIndex } = validator; + const response = web3signerPostResponse.data[index] ?? { + status: "error", + message: "Import outcome unknown: signer did not return a result. Import intent retained for recovery." + }; + results.data[requestIndex] = response; + const postStatus = response.status; + if (postStatus === "error") { + // A missing result is uncertain, unlike an explicit per-key rejection. + if (web3signerPostResponse.data[index]?.status === "error") { + if (!existing.has(pubkey)) rejectedNewKeys.push(pubkey); + response.message = + (response.message ?? "") + + ". Check that the keystore file format is valid and the password is correct."; } - ]) - ); - // Check all database constraints, including size, before changing signer storage. - brainDb.validateNewValidators(toDbRecords(validators)); - // Journal new keys before the signer can change state. Keep uncertain outcomes across restarts. - brainDb.addValidators({ validators: toDbRecords(validators) }); - - // Import keystores and passwords on web3signer API - const web3signerPostResponse = await signerApi.importRemoteKeys({ - keystores: validators.map((validator) => validator.keystore), - passwords: validators.map((validator) => validator.password), - slashing_protection: postRequest.slashing_protection ? postRequest.slashing_protection.toString() : undefined - }); - - logger.debug(`Imported keystores into web3signer API: ${JSON.stringify(web3signerPostResponse.data)}`); - - // Signer API import keystore may fail for some keystores, but not all - // @see https://github.com/ConsenSys/web3signer/issues/713 - // Remove explicit new-key failures, but retain intent when a result is missing. - const rejectedNewKeys: string[] = []; - const confirmedKeys: string[] = []; - for (const [index, validator] of validators.entries()) { - const { pubkey, requestIndex } = validator; - const response = web3signerPostResponse.data[index] ?? { - status: "error", - message: "Import outcome unknown: signer did not return a result. Import intent retained for recovery." - }; - results.data[requestIndex] = response; - const postStatus = response.status; - if (postStatus === "error") { - // A missing result is uncertain, unlike an explicit per-key rejection. - if (web3signerPostResponse.data[index]?.status === "error") { - if (!existing.has(pubkey)) rejectedNewKeys.push(pubkey); - response.message = - (response.message ?? "") + ". Check that the keystore file format is valid and the password is correct."; + logger.error(`Error importing keystore for pubkey ${shortenPubkey(pubkey)}: ${response.message}`); + } else if (postStatus === "duplicate" || postStatus === "imported") { + if (postStatus === "duplicate") logger.warn(`Duplicate keystore for pubkey ${shortenPubkey(pubkey)}`); + const storedKey = existing.get(pubkey); + confirmedKeys.push(storedKey ?? pubkey); + if (!storedKey || existingData[storedKey].importPending) + validatorsToPost.push({ + ...validator, + pubkey: storedKey ?? pubkey, + feeRecipient: storedKey ? existingData[storedKey].feeRecipient : validator.feeRecipient + }); } - logger.error(`Error importing keystore for pubkey ${shortenPubkey(pubkey)}: ${response.message}`); - } else if (postStatus === "duplicate" || postStatus === "imported") { - if (postStatus === "duplicate") logger.warn(`Duplicate keystore for pubkey ${shortenPubkey(pubkey)}`); - const storedKey = existing.get(pubkey); - confirmedKeys.push(storedKey ?? pubkey); - if (!storedKey || existingData[storedKey].importPending) - validatorsToPost.push({ - ...validator, - pubkey: storedKey ?? pubkey, - feeRecipient: storedKey ? existingData[storedKey].feeRecipient : validator.feeRecipient - }); } - } - - brainDb.deleteValidators(rejectedNewKeys); - if (confirmedKeys.length > 0) - brainDb.updateValidators(confirmedKeys.map((pubkey) => ({ pubkey, importPending: undefined }))); - - if (validatorsToPost.length === 0) return results; - // Import pubkeys on validator API - await validatorApi - .postRemoteKeys({ - remote_keys: validatorsToPost.map((validator) => ({ - pubkey: validator.pubkey, - url: signerUrl - })) - }) - .catch((err) => logger.error(`Error setting validator pubkeys`, err)); + brainDb.deleteValidators(rejectedNewKeys); + if (confirmedKeys.length > 0) + brainDb.updateValidators(confirmedKeys.map((pubkey) => ({ pubkey, importPending: undefined }))); - logger.debug(`Added pubkeys to validator API`); + if (validatorsToPost.length === 0) return results; - // Import feeRecipient on Validator API - for (const validator of validatorsToPost) { + // Import pubkeys on validator API await validatorApi - .setFeeRecipient(validator.feeRecipient, validator.pubkey) - .catch((err) => logger.error(`Error setting validator feeRecipient for pubkey ${validator.pubkey} :`, err)); - } + .postRemoteKeys({ + remote_keys: validatorsToPost.map((validator) => ({ + pubkey: validator.pubkey, + url: signerUrl + })) + }) + .catch((err) => logger.error(`Error setting validator pubkeys`, err)); + + logger.debug(`Added pubkeys to validator API`); + + // Import feeRecipient on Validator API + for (const validator of validatorsToPost) { + await validatorApi + .setFeeRecipient(validator.feeRecipient, validator.pubkey) + .catch((err) => logger.error(`Error setting validator feeRecipient for pubkey ${validator.pubkey} :`, err)); + } - logger.debug(`Added fee recipients to validator API`); + logger.debug(`Added fee recipients to validator API`); - logger.debug(`Written on db: ${validatorsToPost.map((v) => v.pubkey).join(", ")}`); - return results; - } finally { - reloadValidatorsCronTask.start(); - } - }); + logger.debug(`Written on db: ${validatorsToPost.map((v) => v.pubkey).join(", ")}`); + return results; + } finally { + reloadValidatorsCronTask.start(); + } + }, + { maxWaitMs: CLIENT_OPERATION_MAX_WAIT_MS } + ); } function getNonEditableFeeRecipient( diff --git a/packages/brain/src/modules/apiServers/ui/calls/index.ts b/packages/brain/src/modules/apiServers/ui/calls/index.ts index fe59f2a1..881c29a5 100644 --- a/packages/brain/src/modules/apiServers/ui/calls/index.ts +++ b/packages/brain/src/modules/apiServers/ui/calls/index.ts @@ -15,6 +15,8 @@ import { NumberOfDaysToQuery } from "../../../validatorsDataIngest/types.js"; import { BeaconchainApi, PostgresClient, ValidatorApi, Web3SignerApi } from "../../../apiClients/index.js"; import { BrainConfig } from "../../../config/types.js"; import { getBuilderRelayPresets } from "../../../config/builderRelayPresets.js"; +import { featureFlags, FeatureDisabledError } from "@stakingbrain/common"; +import { BeaconStatusStore } from "../../../beaconStatus/index.js"; export const createRpcMethods = ({ postgresClient, @@ -23,7 +25,8 @@ export const createRpcMethods = ({ signerApi, brainDb, reloadValidatorsCronTask, - brainConfig + brainConfig, + beaconStatus }: { postgresClient: PostgresClient; beaconchainApi: BeaconchainApi; @@ -32,15 +35,28 @@ export const createRpcMethods = ({ brainDb: BrainDataBase; reloadValidatorsCronTask: CronJob; brainConfig: BrainConfig; + beaconStatus: BeaconStatusStore; }): RpcMethods => { const { beaconchainUrl, executionClientUrl, signerUrl, validatorUrl } = brainConfig.apis; const { minGenesisTime, secondsPerSlot, network, isMevBoostSet, executionClient, consensusClient } = brainConfig.chain; return { - getBuilderRelayPresets: async () => getBuilderRelayPresets(network), - getBuilderConfigs: async (pubkeys: string[]) => await getBuilderConfigs({ pubkeys, brainDb, validatorApi }), + getBuilderRelayPresets: async () => { + if (!featureFlags.builderConfig) throw new FeatureDisabledError("Builder configuration"); + return getBuilderRelayPresets(network); + }, + getBuilderConfigs: async (pubkeys: string[]) => { + if (!featureFlags.builderConfig) throw new FeatureDisabledError("Builder configuration"); + return await getBuilderConfigs({ pubkeys, brainDb, validatorApi }); + }, deleteValidators: async (deleteRequest: Web3signerDeleteRequest) => - await deleteValidators({ brainDb, reloadValidatorsCronTask, validatorApi, signerApi, deleteRequest }), + await deleteValidators({ + brainDb, + reloadValidatorsCronTask, + validatorApi, + signerApi, + deleteRequest + }), exitValidators: async ({ pubkeys }: { pubkeys: string[] }) => await exitValidators({ pubkeys, beaconchainApi, signerApi }), getExitValidators: async ({ pubkeys }: { pubkeys: string[] }) => @@ -67,13 +83,7 @@ export const createRpcMethods = ({ signerUrl, requestFrom }), - getValidators: async () => - await getValidators({ - brainDb, - validatorApi, - signerApi, - beaconchainApi - }), + getValidators: async () => getValidators({ brainDb, validatorApi, signerApi, beaconStatus }), signerGetStatus: async () => await signerGetStatus({ signerApi }), getStakerConfig: async () => await getStakerConfig({ diff --git a/packages/brain/src/modules/apiServers/ui/calls/types.ts b/packages/brain/src/modules/apiServers/ui/calls/types.ts index 183c29cf..27693453 100644 --- a/packages/brain/src/modules/apiServers/ui/calls/types.ts +++ b/packages/brain/src/modules/apiServers/ui/calls/types.ts @@ -1,3 +1,4 @@ +import type { BeaconStatusView } from "../../../beaconStatus/index.js"; import { BuilderConfig, BuilderConfigStatus, BuilderRelayPresets, StakerConfig, Tag } from "@stakingbrain/common"; import type { Web3signerDeleteRequest, @@ -60,19 +61,19 @@ export interface ValidatorBuilderConfigResponse { message?: string; } -export type WithdrawalCredentialsFormat = "ecdsa" | "bls" | "unknown" | "error"; -export interface CustomValidatorGetResponse { +export type { WithdrawalCredentialsFormat } from "../../../beaconStatus/index.js"; +export interface CustomValidatorGetResponse extends Omit { pubkey: string; index: string; tag: Tag; feeRecipient: string; - withdrawalCredentials: { - format: WithdrawalCredentialsFormat; - address: string; - }; signerImported?: boolean; validatorImported?: boolean; validatorFeeRecipientCorrect?: boolean; + /** A client read was cut off by the list deadline; request the list again to continue. */ + detailsPending: boolean; + /** Why a check failed, when it failed rather than being cut off. */ + detailsError?: string; builderConfig?: BuilderConfig | null; builderConfigStatus?: BuilderConfigStatus; } diff --git a/packages/brain/src/modules/apiServers/ui/calls/updateValidators.ts b/packages/brain/src/modules/apiServers/ui/calls/updateValidators.ts index 73e92ece..93a9917f 100644 --- a/packages/brain/src/modules/apiServers/ui/calls/updateValidators.ts +++ b/packages/brain/src/modules/apiServers/ui/calls/updateValidators.ts @@ -1,8 +1,15 @@ -import { prefix0xPubkey, isFeeRecipientEditable, ActionRequestOrigin } from "@stakingbrain/common"; -import { withValidatorOperationLock } from "../../../db/withValidatorOperationLock.js"; +import { + prefix0xPubkey, + isFeeRecipientEditable, + ActionRequestOrigin, + featureFlags, + FeatureDisabledError +} from "@stakingbrain/common"; +import { CLIENT_OPERATION_MAX_WAIT_MS, withValidatorOperationLock } from "../../../db/withValidatorOperationLock.js"; import logger from "../../../logger/index.js"; import { CustomValidatorUpdateRequest } from "./types.js"; import { StakingBrainDb } from "../../../db/types.js"; +import { storedPubkeys } from "../../../db/utils.js"; import { CronJob } from "../../../cron/cron.js"; import { BrainDataBase } from "../../../db/index.js"; import { ValidatorApi } from "../../../apiClients/validator/index.js"; @@ -26,20 +33,24 @@ export async function updateValidators({ customValidatorUpdateRequest: requests, requestFrom }: UpdateValidatorsOptions): Promise { - return withValidatorOperationLock(brainDb, async () => { - reloadValidatorsCronTask.stop(); - try { - const currentValidators = brainDb.getData(); - const updates = prepareUpdates(requests, currentValidators, requestFrom); + return withValidatorOperationLock( + brainDb, + async () => { + reloadValidatorsCronTask.stop(); + try { + const currentValidators = brainDb.getData(); + const updates = prepareUpdates(requests, currentValidators, requestFrom); - // The database validates the entire batch before persisting any settings. - brainDb.updateValidators(updates); - const importedKeys = await importMissingBuilderKeys(validatorApi, signerUrl, updates); - await applyUpdates(validatorApi, updates, currentValidators, importedKeys); - } finally { - reloadValidatorsCronTask.start(); - } - }); + // The database validates the entire batch before persisting any settings. + brainDb.updateValidators(updates); + const importedKeys = await importMissingBuilderKeys(validatorApi, signerUrl, updates); + await applyUpdates(validatorApi, updates, currentValidators, importedKeys); + } finally { + reloadValidatorsCronTask.start(); + } + }, + { maxWaitMs: CLIENT_OPERATION_MAX_WAIT_MS } + ); } function prepareUpdates( @@ -51,11 +62,12 @@ function prepareUpdates( const updates: CustomValidatorUpdateRequest[] = []; const seen = new Set(); + const storedKeys = storedPubkeys(currentValidators); for (const request of requests) { if (!request || typeof request.pubkey !== "string") throw new Error("Validator public key is required"); - const pubkey = prefix0xPubkey(request.pubkey); + const pubkey = storedKeys.get(prefix0xPubkey(request.pubkey).toLowerCase()); + if (!pubkey) throw new Error(`Validator ${prefix0xPubkey(request.pubkey)} is not in the database`); const current = currentValidators[pubkey]; - if (!current) throw new Error(`Validator ${pubkey} is not in the database`); if (seen.has(pubkey)) throw new Error(`Duplicate validator ${pubkey}`); if (Object.keys(request).some((field) => !["pubkey", "feeRecipient", "builderConfig"].includes(field))) throw new Error("Unsupported validator setting"); @@ -64,7 +76,10 @@ function prepareUpdates( const update: CustomValidatorUpdateRequest = { pubkey }; if (feeRecipient !== undefined && isFeeRecipientEditable(current.tag, requestFrom)) update.feeRecipient = feeRecipient; - if (builderConfig !== undefined) update.builderConfig = builderConfig; + if (builderConfig !== undefined) { + if (!featureFlags.builderConfig) throw new FeatureDisabledError("Builder configuration"); + update.builderConfig = builderConfig; + } if (update.feeRecipient !== undefined || update.builderConfig !== undefined) { updates.push(update); seen.add(pubkey); diff --git a/packages/brain/src/modules/apiServers/ui/startUiServer.ts b/packages/brain/src/modules/apiServers/ui/startUiServer.ts index b17d2dfa..05c8f477 100644 --- a/packages/brain/src/modules/apiServers/ui/startUiServer.ts +++ b/packages/brain/src/modules/apiServers/ui/startUiServer.ts @@ -1,4 +1,6 @@ -import { Network } from "@stakingbrain/common"; +import { Network, VALIDATORS_CHANGED_EVENT } from "@stakingbrain/common"; + +const VALIDATORS_CHANGED_DEBOUNCE_MS = 500; import cors from "cors"; import express from "express"; import path from "path"; @@ -14,6 +16,7 @@ import { BrainDataBase } from "../../db/index.js"; import { BrainConfig } from "../../config/types.js"; import { BeaconchainApi, PostgresClient, ValidatorApi, Web3SignerApi } from "../../apiClients/index.js"; import { allowedOrigins, uiSocketOptions } from "./config.js"; +import { BeaconStatusStore } from "../../beaconStatus/index.js"; export function startUiServer({ brainDb, @@ -24,7 +27,9 @@ export function startUiServer({ uiBuildPath, brainConfig, reloadValidatorsCronTask, - allowedOriginsFromEnv + allowedOriginsFromEnv, + beaconStatus, + port = params.uiPort }: { brainDb: BrainDataBase; beaconchainApi: BeaconchainApi; @@ -35,6 +40,9 @@ export function startUiServer({ brainConfig: BrainConfig; reloadValidatorsCronTask: CronJob; allowedOriginsFromEnv: string | string[] | null; + beaconStatus: BeaconStatusStore; + /** Defaults to the production UI port; tests pass 0 for an ephemeral port. */ + port?: number; }): http.Server { const { network } = brainConfig.chain; // create index.html modified with network @@ -48,7 +56,8 @@ export function startUiServer({ signerApi, brainDb, reloadValidatorsCronTask, - brainConfig + brainConfig, + beaconStatus }); const app = express(); @@ -57,6 +66,17 @@ export function startUiServer({ // Socket io const io = new Server(server, uiSocketOptions(uiOrigins)); + // One import or reload writes several times; coalesce them into a single refresh hint. + let changeTimer: NodeJS.Timeout | undefined; + brainDb.onChange(() => { + if (changeTimer) return; + changeTimer = setTimeout(() => { + changeTimer = undefined; + io.emit(VALIDATORS_CHANGED_EVENT); + }, VALIDATORS_CHANGED_DEBOUNCE_MS); + changeTimer.unref(); + }); + io.on("connection", (socket) => { logger.debug("A user connected"); @@ -87,8 +107,8 @@ export function startUiServer({ res.sendFile(path.join(uiBuildPath, "index.html")); }); - server.listen(params.uiPort, () => { - logger.info(`Server listening on *:${params.uiPort}`); + server.listen(port, () => { + logger.info(`Server listening on *:${port}`); }); return server; diff --git a/packages/brain/src/modules/beaconStatus/index.ts b/packages/brain/src/modules/beaconStatus/index.ts new file mode 100644 index 00000000..357b8d17 --- /dev/null +++ b/packages/brain/src/modules/beaconStatus/index.ts @@ -0,0 +1,152 @@ +import { isValidNonWithdrawableBlsAddress, isValidWithdrawableBlsAddress } from "@stakingbrain/common"; +import type { BeaconchainApi } from "../apiClients/index.js"; +import type { BrainDataBase } from "../db/index.js"; +import { storedPubkeys } from "../db/utils.js"; +import logger from "../logger/index.js"; + +/** Beacon lookups per request; large key sets need only a few requests per refresh. */ +export const BEACON_CHUNK_SIZE = 500; +/** A slow or syncing beacon node must not hold up a refresh for long. */ +export const BEACON_TIMEOUT_MS = 30_000; + +export type WithdrawalCredentialsFormat = "ecdsa" | "bls" | "unknown" | "error"; + +export interface BeaconStatusView { + index?: string; + withdrawalCredentials: { format: WithdrawalCredentialsFormat; address: string }; + /** Not looked up yet; the Brain pushes validatorsChanged when the lookup completes. */ + beaconPending: boolean; + /** Why the lookup failed, while no earlier result is available. */ + beaconError?: string; +} + +interface Entry { + /** null: unknown to the beacon chain (e.g. deposit not processed yet). */ + beacon?: { index: string; withdrawalCredentials: string } | null; + error?: string; +} + +const key = (pubkey: string) => pubkey.toLowerCase(); + +/** + * Beacon chain details (index, withdrawal credentials) for every validator, fetched with bulk requests + * by its own periodic job, so neither listing validators nor reconciling clients waits on the beacon + * node. Validators added to the database are looked up as soon as the change is announced, and every + * change here is announced in turn so open UIs refresh. Indexes are persisted for the Brain API. + */ +export class BeaconStatusStore { + private entries = new Map(); + private queue?: Set | "all"; + private run?: Promise; + + constructor( + private readonly brainDb: BrainDataBase, + private readonly beaconchainApi: BeaconchainApi + ) { + brainDb.onChange(() => { + // Only keys never attempted: failed lookups wait for the periodic job, so a failure that is + // itself announced cannot trigger another lookup. + const unseen = Object.keys(this.brainDb.getData()).filter((pubkey) => !this.entries.has(key(pubkey))); + if (unseen.length > 0) + this.refresh(unseen).catch((e) => logger.error("Error looking up new validators on the beacon chain", e)); + }); + } + + /** + * Look up the given validators, or all of them. Calls while a lookup runs merge into one follow-up; + * the periodic job skips ticks while it is still running, so slow lookups never pile up. + */ + refresh(pubkeys?: readonly string[]): Promise { + if (pubkeys === undefined || this.queue === "all") this.queue = "all"; + else this.queue = new Set([...(this.queue ?? []), ...pubkeys.map(key)]); + this.run ??= this.drain().finally(() => (this.run = undefined)); + return this.run; + } + + /** Resolves once no lookup is running, e.g. before closing the database. */ + async whenIdle(): Promise { + while (this.run) await this.run.catch(() => undefined); + } + + view(pubkey: string): BeaconStatusView { + const entry = this.entries.get(key(pubkey)); + const address = entry?.beacon?.withdrawalCredentials ?? ""; + return { + index: entry?.beacon?.index, + withdrawalCredentials: { + address, + format: entry?.beacon + ? isValidWithdrawableBlsAddress(address) + ? "ecdsa" + : isValidNonWithdrawableBlsAddress(address) + ? "bls" + : "unknown" + : "error" + }, + beaconPending: entry === undefined, + beaconError: entry?.beacon === undefined ? entry?.error : undefined + }; + } + + private async drain(): Promise { + while (this.queue) { + const queued = this.queue; + this.queue = undefined; + const stored = storedPubkeys(this.brainDb.getData()); + // Forget validators that left the database, so a later re-import starts fresh. + for (const pubkey of this.entries.keys()) if (!stored.has(pubkey)) this.entries.delete(pubkey); + const targets = + queued === "all" ? [...stored.values()] : [...queued].flatMap((pubkey) => stored.get(pubkey) ?? []); + for (let start = 0; start < targets.length; start += BEACON_CHUNK_SIZE) + await this.lookUp(targets.slice(start, start + BEACON_CHUNK_SIZE)); + } + } + + private async lookUp(pubkeys: string[]): Promise { + let changed = false; + try { + const { data } = await this.beaconchainApi.postStateValidators({ + stateId: "head", + body: { ids: pubkeys, statuses: [] }, + timeout: BEACON_TIMEOUT_MS + }); + const found = new Map(data.map((validator) => [key(validator.validator.pubkey), validator])); + const stored = this.brainDb.getData(); + const indexUpdates: { pubkey: string; index: number }[] = []; + for (const pubkey of pubkeys) { + const validator = found.get(key(pubkey)); + const beacon = validator + ? { index: validator.index, withdrawalCredentials: validator.validator.withdrawal_credentials } + : null; + const current = this.entries.get(key(pubkey)); + const same = + current?.error === undefined && + current !== undefined && + (current.beacon === beacon || + (!!current.beacon && + !!beacon && + current.beacon.index === beacon.index && + current.beacon.withdrawalCredentials === beacon.withdrawalCredentials)); + if (!same) { + this.entries.set(key(pubkey), { beacon }); + changed = true; + } + // Persist indexes: the Brain API serves them and other tools (e.g. Lido CSM) rely on them. + if (validator && stored[pubkey] && stored[pubkey].index !== Number(validator.index)) + indexUpdates.push({ pubkey, index: Number(validator.index) }); + } + if (indexUpdates.length > 0) this.brainDb.updateValidators(indexUpdates); + } catch (e) { + logger.error("Unable to look up validators on the beacon chain", e); + const error = `Could not reach the beacon node: ${e.message}`; + // Keep earlier results; rows never looked up report the failure instead of waiting forever. + for (const pubkey of pubkeys) { + const current = this.entries.get(key(pubkey)); + if (current?.error === error) continue; + this.entries.set(key(pubkey), { beacon: current?.beacon, error }); + changed = true; + } + } + if (changed) this.brainDb.notifyChanged(); + } +} diff --git a/packages/brain/src/modules/cron/cron.ts b/packages/brain/src/modules/cron/cron.ts index 2b6af3df..cf35c6a5 100644 --- a/packages/brain/src/modules/cron/cron.ts +++ b/packages/brain/src/modules/cron/cron.ts @@ -17,21 +17,23 @@ export class CronJob { public start(): void { if (this.task === null) { logger.info(`Starting cron job with interval ${this.interval}ms`); - this.task = setInterval(async () => { - // A slow reload must not queue another copy on every interval. - if (this.running) return; - this.running = true; - try { - await this.jobFunction(); - } catch (error) { - logger.error("Error running cron job", error); - } finally { - this.running = false; - } - }, this.interval); + this.task = setInterval(() => void this.runNow(), this.interval); } else logger.warn("Task is already running. Use restart to restart the job."); } + /** Run the job now unless a run is already in progress; a slow run must not queue more copies. */ + public async runNow(): Promise { + if (this.running) return; + this.running = true; + try { + await this.jobFunction(); + } catch (error) { + logger.error("Error running cron job", error); + } finally { + this.running = false; + } + } + public stop(): void { if (this.task !== null) { logger.info("Stopping cron job."); diff --git a/packages/brain/src/modules/cron/index.ts b/packages/brain/src/modules/cron/index.ts index f55c5a80..f20e78d9 100644 --- a/packages/brain/src/modules/cron/index.ts +++ b/packages/brain/src/modules/cron/index.ts @@ -9,6 +9,7 @@ import { PrometheusApi } from "../apiClients/prometheus/index.js"; import { PostgresClient } from "../apiClients/postgres/index.js"; import { BeaconchainApi } from "../apiClients/beaconchain/index.js"; import { DappmanagerApi } from "../apiClients/dappmanager/index.js"; +import { BeaconStatusStore } from "../beaconStatus/index.js"; export const getCrons = ({ sendNotification, @@ -20,7 +21,8 @@ export const getCrons = ({ brainDb, chainConfig, beaconchainApi, - dappmanagerApi + dappmanagerApi, + beaconStatus }: { sendNotification: boolean; postgresClient: PostgresClient; @@ -32,6 +34,7 @@ export const getCrons = ({ chainConfig: ChainConfig; beaconchainApi: BeaconchainApi; dappmanagerApi: DappmanagerApi; + beaconStatus: BeaconStatusStore; }) => { const { executionClient, consensusClient, secondsPerSlot, slotsPerEpoch } = chainConfig; return { @@ -50,6 +53,9 @@ export const getCrons = ({ }), reloadValidatorsCronTask: new CronJob(60 * 1000, () => reloadValidators(signerApi, signerUrl, validatorApi, brainDb) - ) + ), + // Beacon details for listing, on their own schedule: a slow beacon node must not delay + // reconciliation, and a run still in progress makes later ticks skip rather than queue. + beaconStatusCronTask: new CronJob(60 * 1000, () => beaconStatus.refresh()) }; }; diff --git a/packages/brain/src/modules/cron/reloadValidators/deleteSignerPubkeysNotInDb.ts b/packages/brain/src/modules/cron/reloadValidators/deleteSignerPubkeysNotInDb.ts index 3659d964..7e3c5739 100644 --- a/packages/brain/src/modules/cron/reloadValidators/deleteSignerPubkeysNotInDb.ts +++ b/packages/brain/src/modules/cron/reloadValidators/deleteSignerPubkeysNotInDb.ts @@ -25,12 +25,13 @@ export async function deleteSignerPubkeysNotInDb({ }); for (const [index, pubkeyToRemove] of signerPubkeysToRemove.entries()) { - const signerDeleteStatus = signerDeleteResponse.data[index].status; - if (signerDeleteStatus === "deleted" || signerDeleteStatus === "not_found") + // A missing result is uncertain; keep the key listed so later steps treat it as still present. + const result = signerDeleteResponse.data[index]; + if (result?.status === "deleted" || result?.status === "not_found") signerPubkeys.splice(signerPubkeys.indexOf(pubkeyToRemove), 1); else logger.error( - `${logPrefix}Error deleting pubkey ${pubkeyToRemove} from signer API: ${signerDeleteResponse.data[index].message}` + `${logPrefix}Error deleting pubkey ${pubkeyToRemove} from signer API: ${result?.message ?? "signer did not return a result"}` ); } } diff --git a/packages/brain/src/modules/cron/reloadValidators/deleteValidatorPubkeysNotInDb.ts b/packages/brain/src/modules/cron/reloadValidators/deleteValidatorPubkeysNotInDb.ts index fbf6b2cd..590c5b37 100644 --- a/packages/brain/src/modules/cron/reloadValidators/deleteValidatorPubkeysNotInDb.ts +++ b/packages/brain/src/modules/cron/reloadValidators/deleteValidatorPubkeysNotInDb.ts @@ -22,13 +22,13 @@ export async function deleteValidatorPubkeysNotInDB({ }); for (const [index, pubkeyToRemove] of validatorPubkeysToRemove.entries()) { - const deleteValidatorKeyStatus = deleteValidatorKeysResponse.data[index].status; - - if (deleteValidatorKeyStatus === "deleted" || deleteValidatorKeyStatus === "not_found") + // A missing result is uncertain; keep the key listed so the next reload tries again. + const result = deleteValidatorKeysResponse.data[index]; + if (result?.status === "deleted" || result?.status === "not_found") validatorPubkeys.splice(validatorPubkeys.indexOf(pubkeyToRemove), 1); else logger.error( - `${logPrefix}Error deleting pubkey ${pubkeyToRemove} from validator API: ${deleteValidatorKeyStatus} ${deleteValidatorKeysResponse.data[index].message}` + `${logPrefix}Error deleting pubkey ${pubkeyToRemove} from validator API: ${result?.status ?? "no result"} ${result?.message ?? ""}` ); } } diff --git a/packages/brain/src/modules/cron/reloadValidators/index.ts b/packages/brain/src/modules/cron/reloadValidators/index.ts index b456b5af..9f00086b 100644 --- a/packages/brain/src/modules/cron/reloadValidators/index.ts +++ b/packages/brain/src/modules/cron/reloadValidators/index.ts @@ -1,3 +1,4 @@ +import { featureFlags } from "@stakingbrain/common"; import { withValidatorOperationLock } from "../../db/withValidatorOperationLock.js"; import { Web3SignerApi, ValidatorApi } from "../../apiClients/index.js"; import { BrainDataBase } from "../../db/index.js"; @@ -60,19 +61,14 @@ export async function reloadValidators( const dbKeys = new Set(dbPubkeys.map((pubkey) => pubkey.toLowerCase())); // 4. POST to validator API pubkeys that are in DB and not in validator API - await postValidatorPubkeysFromDb({ - validatorApi, - signerUrl, - brainDbPubkeysToAdd: Object.keys(activeData).filter((pubkey) => !validatorKeys.has(pubkey.toLowerCase())), - validatorPubkeys - }); + const brainDbPubkeysToAdd = Object.keys(activeData).filter((pubkey) => !validatorKeys.has(pubkey.toLowerCase())); + await postValidatorPubkeysFromDb({ validatorApi, signerUrl, brainDbPubkeysToAdd, validatorPubkeys }); // 5. DELETE to validator API pubkeys that are in validator API and not in DB - await deleteValidatorPubkeysNotInDB({ - validatorApi, - validatorPubkeys, - validatorPubkeysToRemove: validatorPubkeys.filter((pubkey) => !dbKeys.has(pubkey.toLowerCase())) - }); + const validatorPubkeysToRemove = validatorPubkeys.filter((pubkey) => !dbKeys.has(pubkey.toLowerCase())); + await deleteValidatorPubkeysNotInDB({ validatorApi, validatorPubkeys, validatorPubkeysToRemove }); + // Client membership is shown in the UI; repairs here change it without touching the database. + if (brainDbPubkeysToAdd.length > 0 || validatorPubkeysToRemove.length > 0) brainDb.notifyChanged(); // 6. POST to validator API fee recipients that are in DB in validator API await postValidatorsFeeRecipientsFromDb({ @@ -81,7 +77,7 @@ export async function reloadValidators( }); // Independently retry managed configuration after key imports and fee destinations. - await postValidatorsBuilderConfigsFromDb({ validatorApi, dbData: activeData }); + if (featureFlags.builderConfig) await postValidatorsBuilderConfigsFromDb({ validatorApi, dbData: activeData }); logger.debug(`${logPrefix}Finished reloading data`); } catch (e) { logger.error(`${logPrefix}Error reloading data`, e); diff --git a/packages/brain/src/modules/cron/reloadValidators/postValidatorPubkeysFromDb.ts b/packages/brain/src/modules/cron/reloadValidators/postValidatorPubkeysFromDb.ts index 9a5c4f19..0e619b2e 100644 --- a/packages/brain/src/modules/cron/reloadValidators/postValidatorPubkeysFromDb.ts +++ b/packages/brain/src/modules/cron/reloadValidators/postValidatorPubkeysFromDb.ts @@ -26,11 +26,12 @@ export async function postValidatorPubkeysFromDb({ }); for (const [index, pubkeyToAdd] of brainDbPubkeysToAdd.entries()) { - const postKeyStatus = postKeysResponse.data[index].status; - if (postKeyStatus === "imported" || postKeyStatus === "duplicate") validatorPubkeys.push(pubkeyToAdd); + // A missing result is uncertain; leave the key out so the next reload tries again. + const result = postKeysResponse.data[index]; + if (result?.status === "imported" || result?.status === "duplicate") validatorPubkeys.push(pubkeyToAdd); else logger.error( - `${logPrefix}Error adding pubkey ${pubkeyToAdd} to validator API: ${postKeyStatus} ${postKeysResponse.data[index].message}` + `${logPrefix}Error adding pubkey ${pubkeyToAdd} to validator API: ${result?.status ?? "no result"} ${result?.message ?? ""}` ); } } diff --git a/packages/brain/src/modules/db/index.ts b/packages/brain/src/modules/db/index.ts index fbf9fddb..c3ed5120 100644 --- a/packages/brain/src/modules/db/index.ts +++ b/packages/brain/src/modules/db/index.ts @@ -5,7 +5,7 @@ import fs from "fs"; import logger from "../logger/index.js"; import { Web3SignerApi, ValidatorApi } from "../apiClients/index.js"; import { params } from "../../params.js"; -import { isValidTag } from "./utils.js"; +import { isValidTag, storedPubkeys } from "./utils.js"; import { shortenPubkey, isValidBlsPubkey, isValidEcdsaPubkey, validateBuilderConfig } from "@stakingbrain/common"; import { BrainDbError } from "./error.js"; @@ -33,11 +33,56 @@ export class BrainDataBase extends LowSync { this.dbName = dbName; } + private changeListeners = new Set<() => void>(); + private notificationHolds = 0; + private changedWhileHeld = false; + public override write(): void { super.write(); // The JSON adapter replaces the file atomically; restore the private mode on // the replacement too, since records may contain builder authorization data. this.setOwnerWriteRead(); + this.notifyChanged(); + } + + /** Announce a validator change, including client-side changes that involve no database write. */ + public notifyChanged(): void { + if (this.notificationHolds > 0) { + this.changedWhileHeld = true; + return; + } + for (const listener of this.changeListeners) { + try { + listener(); + } catch (e) { + logger.error("Database change listener failed", e); + } + } + } + + /** + * Hold change notifications until the returned release runs, then send at most one. An operation + * writes the database before configuring clients, so notifying mid-operation would let UIs read + * client state that is about to change and never hear about the final result. + */ + public holdChangeNotifications(): () => void { + this.notificationHolds++; + let released = false; + return () => { + if (released) return; + released = true; + this.notificationHolds--; + if (this.notificationHolds === 0 && this.changedWhileHeld) { + this.changedWhileHeld = false; + this.notifyChanged(); + } + }; + } + + /** Notify after each persisted change, e.g. to push live updates to open UIs. Returns an unsubscribe. */ + public onChange(listener: () => void): () => void { + this.changeListeners.add(listener); + return () => this.changeListeners.delete(listener); } /** @@ -120,12 +165,14 @@ export class BrainDataBase extends LowSync { this.validateDb(); this.validateUpdateValidators(validators); const updatedData = { ...this.data }; + const storedKeys = storedPubkeys(this.data); for (const { pubkey, ...changes } of validators) { - if (!this.data[pubkey]) { + const storedKey = storedKeys.get(pubkey.toLowerCase()); + if (!storedKey) { logger.warn(`Pubkey ${pubkey} not found in the database`); continue; } - updatedData[pubkey] = { ...this.data[pubkey], ...changes }; + updatedData[storedKey] = { ...this.data[storedKey], ...changes }; } this.ensureDbMaxSize(updatedData, true); this.data = updatedData; @@ -145,7 +192,7 @@ export class BrainDataBase extends LowSync { try { this.validateDb(); if (!this.data) return; - const storedKeys = new Map(Object.keys(this.data).map((key) => [key.toLowerCase(), key])); + const storedKeys = storedPubkeys(this.data); let changed = false; for (const pubkey of pubkeys) { const storedKey = storedKeys.get(pubkey.toLowerCase()); @@ -296,8 +343,9 @@ export class BrainDataBase extends LowSync { // Validate Ethereum address if (!isValidBlsPubkey(pubkey)) errors.push(`\n pubkey ${pubkeySubstr}: bls is invalid`); - if (seen.has(pubkey)) errors.push(`\n pubkey ${pubkeySubstr}: duplicate update`); - seen.add(pubkey); + const key = pubkey.toLowerCase(); + if (seen.has(key)) errors.push(`\n pubkey ${pubkeySubstr}: duplicate update`); + seen.add(key); // FeeRecipient if ("feeRecipient" in validator) { diff --git a/packages/brain/src/modules/db/utils.ts b/packages/brain/src/modules/db/utils.ts index 3f3910b8..d0f482ff 100644 --- a/packages/brain/src/modules/db/utils.ts +++ b/packages/brain/src/modules/db/utils.ts @@ -3,3 +3,8 @@ import { Tag, tags } from "@stakingbrain/common"; export function isValidTag(tag: Tag): boolean { return tags.includes(tag); } + +/** Map lowercase pubkeys to their stored spelling; clients do not agree on hex casing. */ +export function storedPubkeys(data: object): Map { + return new Map(Object.keys(data).map((pubkey) => [pubkey.toLowerCase(), pubkey])); +} diff --git a/packages/brain/src/modules/db/withValidatorOperationLock.ts b/packages/brain/src/modules/db/withValidatorOperationLock.ts index b181fdf0..79fda0ad 100644 --- a/packages/brain/src/modules/db/withValidatorOperationLock.ts +++ b/packages/brain/src/modules/db/withValidatorOperationLock.ts @@ -2,19 +2,60 @@ import type { BrainDataBase } from "./index.js"; const pendingOperations = new WeakMap>(); -/** Keep database snapshots and their client writes ordered, including across awaits. */ -export async function withValidatorOperationLock(brainDb: BrainDataBase, operation: () => Promise): Promise { +export class ValidatorOperationBusyError extends Error { + constructor() { + super("Brain is busy with another validator operation. Nothing was changed; try again shortly."); + this.name = "ValidatorOperationBusyError"; + } +} + +/** + * Keep database snapshots and their client writes ordered, including across awaits. + * + * With `maxWaitMs`, an operation that cannot start in time is abandoned without running, so a + * caller that has already given up (for example a timed-out UI request) never changes state later. + * The abandoned slot still waits for its predecessor, so later operations keep their order. + */ +export async function withValidatorOperationLock( + brainDb: BrainDataBase, + operation: () => Promise, + { maxWaitMs }: { maxWaitMs?: number } = {} +): Promise { const previous = pendingOperations.get(brainDb) ?? Promise.resolve(); let release!: () => void; const current = new Promise((resolve) => { release = resolve; }); pendingOperations.set(brainDb, current); - await previous; + const cleanup = (): void => { + release(); + if (pendingOperations.get(brainDb) === current) pendingOperations.delete(brainDb); + }; + + if (maxWaitMs !== undefined) { + let timer: NodeJS.Timeout | undefined; + const acquired = await Promise.race([ + previous.then(() => true), + new Promise((resolve) => { + timer = setTimeout(() => resolve(false), maxWaitMs); + }) + ]); + clearTimeout(timer); + if (!acquired) { + void previous.then(cleanup); + throw new ValidatorOperationBusyError(); + } + } else await previous; + + // Announce the operation's changes once it has finished configuring clients, not per write. + const releaseNotifications = brainDb.holdChangeNotifications(); try { return await operation(); } finally { - release(); - if (pendingOperations.get(brainDb) === current) pendingOperations.delete(brainDb); + releaseNotifications(); + cleanup(); } } + +/** Leaves most of the UI's RPC_TIMEOUT_MS for the operation itself once it starts. */ +export const CLIENT_OPERATION_MAX_WAIT_MS = 30 * 1000; diff --git a/packages/brain/src/modules/utils/forEachConcurrently.ts b/packages/brain/src/modules/utils/forEachConcurrently.ts new file mode 100644 index 00000000..09777326 --- /dev/null +++ b/packages/brain/src/modules/utils/forEachConcurrently.ts @@ -0,0 +1,16 @@ +/** Run `task` for every item with at most `limit` in flight; per-item errors are the task's to handle. */ +export async function forEachConcurrently( + items: readonly T[], + limit: number, + task: (item: T, index: number) => Promise +): Promise { + let next = 0; + await Promise.all( + Array.from({ length: Math.min(limit, items.length) }, async () => { + while (next < items.length) { + const index = next++; + await task(items[index], index); + } + }) + ); +} diff --git a/packages/brain/test/e2e/fakeClients.ts b/packages/brain/test/e2e/fakeClients.ts new file mode 100644 index 00000000..278bd475 --- /dev/null +++ b/packages/brain/test/e2e/fakeClients.ts @@ -0,0 +1,217 @@ +import express from "express"; +import http from "node:http"; +import type { AddressInfo } from "node:net"; + +/** Minimal in-memory stand-ins for web3signer, a validator client and a beacon node, served over real HTTP. */ + +type Started = { url: string; close: () => Promise }; + +async function serve(app: express.Express): Promise { + const server = http.createServer(app); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const { port } = server.address() as AddressInfo; + return { + url: `http://127.0.0.1:${port}`, + close: () => + new Promise((resolve) => { + server.closeAllConnections(); + server.close(() => resolve()); + }) + }; +} + +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); +const prefixed = (pubkey: string) => (pubkey.startsWith("0x") ? pubkey : `0x${pubkey}`).toLowerCase(); + +export class FakeSigner { + keys = new Set(); + importRequests: { pubkeys: string[]; slashingPubkeys?: string[] }[] = []; + /** Simulated keystore decryption time. */ + importDelayPerKeyMs = 0; + /** 1-based import call number that answers HTTP 500, as a crashed or overloaded signer would. */ + failImportCall?: number; + + async start(): Promise { + const app = express(); + app.use(express.json({ limit: "64mb" })); + app.get("/upcheck", (_req, res) => { + res.type("text/plain").send("OK"); + }); + app.get("/eth/v1/keystores", (_req, res) => { + res.json({ + data: [...this.keys].map((validating_pubkey) => ({ validating_pubkey, derivation_path: "", readonly: false })) + }); + }); + app.post("/eth/v1/keystores", async (req, res) => { + const { keystores, passwords, slashing_protection } = req.body as { + keystores: string[]; + passwords: string[]; + slashing_protection?: string; + }; + const pubkeys = keystores.map((keystore) => prefixed(JSON.parse(keystore).pubkey)); + this.importRequests.push({ + pubkeys, + slashingPubkeys: slashing_protection + ? JSON.parse(slashing_protection).data.map(({ pubkey }: { pubkey: string }) => prefixed(pubkey)) + : undefined + }); + await delay(this.importDelayPerKeyMs * keystores.length); + if (this.importRequests.length === this.failImportCall) { + res.status(500).json({ message: "Internal signer error" }); + return; + } + res.json({ + data: pubkeys.map((pubkey, index) => { + if (passwords[index] !== "password") return { status: "error", message: "Invalid password" }; + if (this.keys.has(pubkey)) return { status: "duplicate", message: "" }; + this.keys.add(pubkey); + return { status: "imported", message: "" }; + }) + }); + }); + app.delete("/eth/v1/keystores", (req, res) => { + const pubkeys = (req.body.pubkeys as string[]).map(prefixed); + res.json({ + data: pubkeys.map((pubkey) => ({ status: this.keys.delete(pubkey) ? "deleted" : "not_found", message: "" })), + slashing_protection: JSON.stringify({ metadata: {}, data: [] }) + }); + }); + return serve(app); + } +} + +export class FakeValidatorClient { + remoteKeys = new Map(); + feeRecipients = new Map(); + /** Every request path, to assert what the Brain did (and did not) call. */ + paths: string[] = []; + /** Simulated latency of a busy validator client on every write. */ + writeDelayMs = 0; + /** Answer every request with 503, as a restarting client would. */ + unavailable = false; + + async start(): Promise { + const app = express(); + app.use(express.json({ limit: "64mb" })); + app.use(async (req, res, next) => { + this.paths.push(`${req.method} ${req.path}`); + if (this.unavailable) { + res.status(503).json({ message: "Validator client is restarting" }); + return; + } + if (req.method !== "GET") await delay(this.writeDelayMs); + next(); + }); + app.get("/eth/v1/remotekeys", (_req, res) => { + res.json({ data: [...this.remoteKeys].map(([pubkey, url]) => ({ pubkey, url, readonly: false })) }); + }); + app.post("/eth/v1/remotekeys", (req, res) => { + const remoteKeys = req.body.remote_keys as { pubkey: string; url: string }[]; + res.json({ + data: remoteKeys.map(({ pubkey, url }) => { + const key = prefixed(pubkey); + const status = this.remoteKeys.has(key) ? "DUPLICATE" : "IMPORTED"; + this.remoteKeys.set(key, url); + return { status, message: "" }; + }) + }); + }); + app.delete("/eth/v1/remotekeys", (req, res) => { + res.json({ + data: (req.body.pubkeys as string[]).map((pubkey) => ({ + status: this.remoteKeys.delete(prefixed(pubkey)) ? "DELETED" : "NOT_FOUND", + message: "" + })) + }); + }); + app.get("/eth/v1/validator/:pubkey/feerecipient", (req, res) => { + const ethaddress = this.feeRecipients.get(prefixed(req.params.pubkey)); + if (!ethaddress) res.status(404).json({ message: "Validator not found" }); + else res.json({ data: { pubkey: req.params.pubkey, ethaddress } }); + }); + app.post("/eth/v1/validator/:pubkey/feerecipient", (req, res) => { + this.feeRecipients.set(prefixed(req.params.pubkey), req.body.ethaddress); + res.status(202).end(); + }); + app.delete("/eth/v1/validator/:pubkey/feerecipient", (req, res) => { + this.feeRecipients.delete(prefixed(req.params.pubkey)); + res.status(204).end(); + }); + return serve(app); + } +} + +export class FakeBeaconNode { + indexes = new Map(); + /** Simulated per-validator lookup latency of a busy or syncing beacon node. */ + lookupDelayMs = 0; + lookups = 0; + /** Simulated latency of one bulk lookup, whatever its size. */ + bulkDelayMs = 0; + bulkLookups: number[] = []; + unavailable = false; + + async start(): Promise { + const app = express(); + app.use(express.json({ limit: "64mb" })); + app.post("/eth/v1/beacon/states/head/validators", async (req, res) => { + const ids = (req.body.ids as string[]).map(prefixed); + this.bulkLookups.push(ids.length); + await delay(this.bulkDelayMs); + if (this.unavailable) { + res.status(503).json({ message: "Beacon node is syncing" }); + return; + } + res.json({ + execution_optimistic: false, + finalized: false, + data: ids.flatMap((pubkey) => { + const index = this.indexes.get(pubkey); + return index === undefined + ? [] + : [ + { + index: String(index), + balance: "32000000000", + status: "active_ongoing", + validator: { pubkey, withdrawal_credentials: "0x01" + "00".repeat(11) + "ab".repeat(20) } + } + ]; + }) + }); + }); + app.get("/eth/v1/beacon/states/head/validators/:pubkey", async (req, res) => { + this.lookups++; + // Stop early when the Brain aborts the request at its deadline. + let closed = false; + req.on("close", () => (closed = true)); + await delay(this.lookupDelayMs); + if (closed) return; + const pubkey = prefixed(req.params.pubkey); + const index = this.indexes.get(pubkey); + if (index === undefined) { + res.status(404).json({ message: "Validator not found" }); + return; + } + res.json({ + execution_optimistic: false, + data: { + index: String(index), + balance: "32000000000", + status: "active_ongoing", + validator: { + pubkey, + withdrawal_credentials: "0x01" + "00".repeat(11) + "ab".repeat(20), + effective_balance: "32000000000", + slashed: false, + activation_eligibility_epoch: "0", + activation_epoch: "0", + exit_epoch: "18446744073709551615", + withdrawable_epoch: "18446744073709551615" + } + } + }); + }); + return serve(app); + } +} diff --git a/packages/brain/test/e2e/largeBatches.e2e.test.ts b/packages/brain/test/e2e/largeBatches.e2e.test.ts new file mode 100644 index 00000000..d6b86165 --- /dev/null +++ b/packages/brain/test/e2e/largeBatches.e2e.test.ts @@ -0,0 +1,366 @@ +import { expect } from "chai"; +import sinon from "sinon"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { once } from "node:events"; +import type http from "node:http"; +import type { AddressInfo } from "node:net"; +import { io, Socket } from "socket.io-client"; +import { Network, VALIDATORS_CHANGED_EVENT } from "@stakingbrain/common"; +import { BeaconchainApi, ValidatorApi, Web3SignerApi } from "../../src/modules/apiClients/index.js"; +import { BrainDataBase } from "../../src/modules/db/index.js"; +import { CronJob } from "../../src/modules/cron/cron.js"; +import { reloadValidators } from "../../src/modules/cron/reloadValidators/index.js"; +import { startUiServer } from "../../src/modules/apiServers/ui/startUiServer.js"; +import type { BrainConfig } from "../../src/modules/config/types.js"; +import logger from "../../src/modules/logger/index.js"; +// The UI's own client code, so the test drives the Brain exactly as the browser does. +import { RpcClient } from "../../../ui/src/socket/rpcClient.js"; +import { + notAttempted, + planImportBatches, + retryableIndexes, + runImportBatches, + summarizeImport +} from "../../../ui/src/utils/importBatches.js"; +import type { CustomImportRequest } from "../../src/modules/apiServers/ui/calls/types.js"; +import { BeaconStatusStore } from "../../src/modules/beaconStatus/index.js"; +import { FakeBeaconNode, FakeSigner, FakeValidatorClient } from "./fakeClients.js"; + +const feeRecipient = "0x" + "ab".repeat(20); +const pubkeyAt = (index: number) => "0x" + (index + 1).toString(16).padStart(96, "0"); + +function importRequest(pubkeys: string[], password = "password"): CustomImportRequest { + return { + importFrom: "ui", + // One interchange covering every key, as users export it from their previous setup. + slashing_protection: JSON.stringify({ + metadata: { interchange_format_version: "5", genesis_validators_root: "0x" + "00".repeat(32) }, + data: pubkeys.map((pubkey) => ({ pubkey, signed_blocks: [], signed_attestations: [] })) + }), + validatorsImportRequest: pubkeys.map((pubkey) => ({ + keystore: JSON.stringify({ version: 4, pubkey: pubkey.slice(2), crypto: { kdf: { function: "scrypt" } } }), + password, + tag: "solo", + feeRecipient + })) + }; +} + +async function waitFor(condition: () => boolean, timeoutMs = 5000): Promise { + const deadline = Date.now() + timeoutMs; + while (!condition()) { + if (Date.now() > deadline) throw new Error("Timed out waiting for condition"); + await new Promise((resolve) => setTimeout(resolve, 20)); + } +} + +describe("E2E: large validator batches through the UI socket", function () { + this.timeout(60_000); + + let directory: string; + let signer: FakeSigner; + let validator: FakeValidatorClient; + let beacon: FakeBeaconNode; + let closeFakes: (() => Promise)[]; + let signerApi: Web3SignerApi; + let validatorApi: ValidatorApi; + let signerUrl: string; + let brainDb: BrainDataBase; + let cron: CronJob; + let beaconStatus: BeaconStatusStore; + let server: http.Server; + let sockets: Socket[]; + let client: RpcClient; + + const connect = async (): Promise<{ socket: Socket; client: RpcClient }> => { + const { port } = server.address() as AddressInfo; + const socket = io(`http://127.0.0.1:${port}`, { transports: ["websocket"], reconnection: false }); + sockets.push(socket); + const rpc = new RpcClient(socket); + if (!socket.connected) await new Promise((resolve) => socket.once("connect", () => resolve())); + return { socket, client: rpc }; + }; + + const importAll = ( + request: CustomImportRequest, + pubkeys: string[], + only?: number[], + earlier?: { status: string }[] + ) => + runImportBatches({ + batches: planImportBatches(request, pubkeys, { only }), + results: earlier ?? pubkeys.map(notAttempted), + importBatch: (batch) => client.call("importValidators", batch), + onProgress: () => undefined, + shouldStop: () => false + }); + + beforeEach(async () => { + for (const level of ["debug", "info", "warn", "error"] as const) sinon.stub(logger, level); + sinon.stub(console, "log"); + directory = fs.mkdtempSync(path.join(os.tmpdir(), "stakingbrain-e2e-")); + signer = new FakeSigner(); + validator = new FakeValidatorClient(); + beacon = new FakeBeaconNode(); + const [signerServer, validatorServer, beaconServer] = await Promise.all([ + signer.start(), + validator.start(), + beacon.start() + ]); + closeFakes = [signerServer.close, validatorServer.close, beaconServer.close]; + signerUrl = signerServer.url; + signerApi = new Web3SignerApi({ baseUrl: signerServer.url, authToken: "token" }, Network.Hoodi); + validatorApi = new ValidatorApi({ baseUrl: validatorServer.url, authToken: "token" }, Network.Hoodi); + const beaconchainApi = new BeaconchainApi({ baseUrl: beaconServer.url }, Network.Hoodi); + + brainDb = new BrainDataBase(path.join(directory, "brain-db.json")); + await brainDb.initialize(signerApi, validatorApi); + // As in production: reconciliation and beacon details are separate jobs. + beaconStatus = new BeaconStatusStore(brainDb, beaconchainApi); + cron = new CronJob(60_000, () => reloadValidators(signerApi, signerUrl, validatorApi, brainDb)); + cron.start(); + + const uiBuildPath = path.join(directory, "ui"); + fs.mkdirSync(uiBuildPath); + fs.writeFileSync(path.join(uiBuildPath, "index.html"), ""); + server = startUiServer({ + brainDb, + beaconchainApi, + validatorApi, + signerApi, + postgresClient: {} as never, + uiBuildPath, + brainConfig: { + chain: { network: Network.Hoodi, minGenesisTime: 0, secondsPerSlot: 12, isMevBoostSet: false }, + apis: { signerUrl, validatorUrl: validatorServer.url, beaconchainUrl: beaconServer.url } + } as unknown as BrainConfig, + reloadValidatorsCronTask: cron, + allowedOriginsFromEnv: null, + beaconStatus, + port: 0 + }); + if (!server.listening) await once(server, "listening"); + sockets = []; + ({ client } = await connect()); + }); + + afterEach(async () => { + for (const socket of sockets) socket.disconnect(); + cron.stop(); + await beaconStatus.whenIdle(); + server.closeAllConnections(); + await new Promise((resolve) => server.close(resolve)); + await Promise.all(closeFakes.map((close) => close())); + sinon.restore(); + fs.rmSync(directory, { recursive: true, force: true }); + }); + + it("imports 120 keystores in batches and configures every client", async () => { + const pubkeys = Array.from({ length: 120 }, (_, index) => pubkeyAt(index)); + signer.importDelayPerKeyMs = 5; + const { results, stopped } = await importAll(importRequest(pubkeys), pubkeys); + + expect(stopped).to.equal(undefined); + expect(summarizeImport(results)).to.deep.equal({ imported: 120 }); + // Six bounded calls, each carrying only its own keys' slashing history. + expect(signer.importRequests.map(({ pubkeys }) => pubkeys.length)).to.deep.equal([20, 20, 20, 20, 20, 20]); + for (const { pubkeys: batch, slashingPubkeys } of signer.importRequests) + expect(slashingPubkeys).to.deep.equal(batch); + expect(signer.keys.size).to.equal(120); + expect(validator.remoteKeys.size).to.equal(120); + expect([...validator.remoteKeys.values()].every((url) => url === signerUrl)).to.equal(true); + expect(validator.feeRecipients.size).to.equal(120); + expect([...validator.feeRecipients.values()].every((address) => address === feeRecipient)).to.equal(true); + const records = Object.values(brainDb.getData()); + expect(records).to.have.length(120); + expect(records.some(({ importPending }) => importPending)).to.equal(false); + + // The reload job agrees with the result and removes nothing. + await reloadValidators(signerApi, signerUrl, validatorApi, brainDb); + expect(Object.keys(brainDb.getData())).to.have.length(120); + expect(validator.remoteKeys.size).to.equal(120); + }); + + it("stops at a failed batch, keeps the journal, and a retry finishes the import", async () => { + const pubkeys = Array.from({ length: 100 }, (_, index) => pubkeyAt(index)); + signer.failImportCall = 3; + const first = await importAll(importRequest(pubkeys), pubkeys); + + expect(first.stopped).to.equal("error"); + expect(summarizeImport(first.results)).to.deep.equal({ imported: 40, error: 20, not_attempted: 40 }); + expect(first.results[45].message).to.include("500"); + // The failed batch stays journaled as pending, and reload must not delete those records. + await reloadValidators(signerApi, signerUrl, validatorApi, brainDb); + const pending = Object.values(brainDb.getData()).filter(({ importPending }) => importPending); + expect(pending).to.have.length(20); + + const retry = retryableIndexes(first.results); + expect(retry).to.have.length(60); + const second = await importAll(importRequest(pubkeys), pubkeys, retry, first.results); + expect(second.stopped).to.equal(undefined); + expect(summarizeImport(second.results)).to.deep.equal({ imported: 100 }); + expect(signer.keys.size).to.equal(100); + expect(validator.remoteKeys.size).to.equal(100); + expect(validator.feeRecipients.size).to.equal(100); + expect(Object.values(brainDb.getData()).some(({ importPending }) => importPending)).to.equal(false); + }); + + it("reports per-key signer rejections without failing the rest of the batch", async () => { + const pubkeys = Array.from({ length: 30 }, (_, index) => pubkeyAt(index)); + const request = importRequest(pubkeys); + request.validatorsImportRequest[25].password = "wrong"; + const { results, stopped } = await importAll(request, pubkeys); + expect(stopped).to.equal(undefined); + expect(summarizeImport(results)).to.deep.equal({ imported: 29, error: 1 }); + expect(results[25].message).to.include("password"); + expect(brainDb.getData()[pubkeys[25]]).to.equal(undefined); + expect(retryableIndexes(results)).to.deep.equal([25]); + }); + + it("pushes validatorsChanged to other open UIs on import and delete", async () => { + const watcher = await connect(); + let events = 0; + watcher.client.on(VALIDATORS_CHANGED_EVENT, () => events++); + const pubkeys = Array.from({ length: 45 }, (_, index) => pubkeyAt(index)); + await importAll(importRequest(pubkeys), pubkeys); + await waitFor(() => events > 0); + // Many database writes are coalesced into few hints rather than one per write. + expect(events).to.be.lessThan(6); + expect(await watcher.client.call("getValidators", undefined)).to.have.length(45); + + const before = events; + await client.call("deleteValidators", { pubkeys: pubkeys.slice(0, 5) }); + await waitFor(() => events > before); + expect(await watcher.client.call("getValidators", undefined)).to.have.length(40); + expect(validator.remoteKeys.size).to.equal(40); + }); + + it("pushes a final update after slow validator-client writes so other UIs end up current", async () => { + validator.writeDelayMs = 1500; + const watcher = await connect(); + // Behave like the validator list: refresh on every push and keep the latest rows. + let latest: Awaited>> = []; + let refreshes = Promise.resolve(); + watcher.client.on(VALIDATORS_CHANGED_EVENT, () => { + refreshes = refreshes.then(async () => { + latest = await watcher.client.call("getValidators", undefined); + }); + }); + // Writes are sequential, so a few keys are enough to outlast the notification debounce. + const pubkeys = Array.from({ length: 4 }, (_, index) => pubkeyAt(index)); + await importAll(importRequest(pubkeys), pubkeys); + // Let the last push and its refresh land. + await new Promise((resolve) => setTimeout(resolve, 1000)); + await refreshes; + expect(latest).to.have.length(4); + expect(latest.every(({ validatorImported }) => validatorImported === true)).to.equal(true); + expect(latest.every(({ validatorFeeRecipientCorrect }) => validatorFeeRecipientCorrect === true)).to.equal(true); + }); + + it("lists 600 validators at once and fills in beacon data with a push, without polling", async () => { + const pubkeys = Array.from({ length: 600 }, (_, index) => pubkeyAt(index)); + pubkeys.forEach((pubkey, index) => beacon.indexes.set(pubkey, 1000 + index)); + // A slow beacon node must not slow the import or the list down. + beacon.bulkDelayMs = 1500; + const watcher = await connect(); + // Behave like the validator list: refresh on every push and keep the latest rows. + let latest: Awaited>> = []; + let refreshes = Promise.resolve(); + watcher.client.on(VALIDATORS_CHANGED_EVENT, () => { + refreshes = refreshes.then(async () => { + latest = await watcher.client.call("getValidators", undefined); + }); + }); + + await importAll(importRequest(pubkeys), pubkeys); + const started = Date.now(); + let rows = await watcher.client.call("getValidators", undefined); + expect(Date.now() - started).to.be.lessThan(5000); + expect(rows).to.have.length(600); + // Signer and client membership and fee recipients are read live, within one bounded request. + expect(rows.every(({ signerImported, validatorImported }) => signerImported && validatorImported)).to.equal(true); + expect(rows.every(({ validatorFeeRecipientCorrect }) => validatorFeeRecipientCorrect === true)).to.equal(true); + + await waitFor( + () => + latest.length === 600 && latest.every(({ detailsPending, beaconPending }) => !detailsPending && !beaconPending), + 20_000 + ); + await refreshes; + rows = latest; + expect(rows.some(({ detailsPending }) => detailsPending)).to.equal(false); + expect(rows.map(({ index }) => index)).to.deep.equal(pubkeys.map((_, index) => String(1000 + index))); + expect(rows.every(({ withdrawalCredentials }) => withdrawalCredentials.format === "ecdsa")).to.equal(true); + // Lookups for the 30 import batches merge, and each covers many keys at once. + expect(beacon.bulkLookups.length).to.be.lessThan(10); + expect(beacon.lookups).to.equal(0); + // Indexes are stored, so the Brain API and the explorer link have them. + expect(Object.values(brainDb.getData()).every(({ index }) => index !== undefined)).to.equal(true); + }); + + it("reports unreachable clients as errors instead of loading forever, then recovers", async () => { + const pubkeys = Array.from({ length: 5 }, (_, index) => pubkeyAt(index)); + pubkeys.forEach((pubkey, index) => beacon.indexes.set(pubkey, index)); + beacon.unavailable = true; + await importAll(importRequest(pubkeys), pubkeys); + await beaconStatus.refresh(); + let rows = await client.call("getValidators", undefined); + expect(rows.every(({ detailsPending, beaconPending }) => !detailsPending && !beaconPending)).to.equal(true); + expect(rows[0].detailsError).to.include("beacon node"); + + // A validator client outage after a successful check is reported, not hidden by the earlier result. + validator.unavailable = true; + rows = await client.call("getValidators", undefined); + expect( + rows.every( + ({ validatorImported, validatorFeeRecipientCorrect }) => + validatorImported === undefined && validatorFeeRecipientCorrect === undefined + ) + ).to.equal(true); + expect(rows[0].detailsError).to.include("validator client"); + + validator.unavailable = false; + beacon.unavailable = false; + await beaconStatus.refresh(); + rows = await client.call("getValidators", undefined); + expect( + rows.every( + ({ detailsError, detailsPending, beaconPending }) => !detailsError && !detailsPending && !beaconPending + ) + ).to.equal(true); + }); + + it("repairs a validator client that lost its keys and shows the repair", async () => { + const pubkeys = Array.from({ length: 20 }, (_, index) => pubkeyAt(index)); + await importAll(importRequest(pubkeys), pubkeys); + // The validator client restarted without its remote keys or fee recipients. + validator.remoteKeys.clear(); + validator.feeRecipients.clear(); + await reloadValidators(signerApi, signerUrl, validatorApi, brainDb); + expect(validator.remoteKeys.size).to.equal(20); + expect(validator.feeRecipients.size).to.equal(20); + const rows = await client.call("getValidators", undefined); + expect( + rows.every( + ({ validatorImported, validatorFeeRecipientCorrect }) => validatorImported && validatorFeeRecipientCorrect + ) + ).to.equal(true); + }); + + it("keeps builder configuration switched off end to end", async () => { + const pubkeys = [pubkeyAt(0)]; + await importAll(importRequest(pubkeys), pubkeys); + const error = await client.call("getBuilderRelayPresets", undefined).catch((e: Error) => e); + expect(error).to.be.instanceOf(Error).and.have.property("message").that.includes("disabled"); + const update = await client + .call("updateValidators", [{ pubkey: pubkeys[0], builderConfig: null }]) + .catch((e: Error) => e); + expect(update).to.be.instanceOf(Error).and.have.property("message").that.includes("disabled"); + await reloadValidators(signerApi, signerUrl, validatorApi, brainDb); + expect(validator.paths.filter((p) => p.includes("builder_config"))).to.deep.equal([]); + const [row] = await client.call("getValidators", undefined); + expect(row).to.not.have.property("builderConfig"); + }); +}); diff --git a/packages/brain/test/helpers/beacon.ts b/packages/brain/test/helpers/beacon.ts new file mode 100644 index 00000000..24554fc9 --- /dev/null +++ b/packages/brain/test/helpers/beacon.ts @@ -0,0 +1,7 @@ +import type { BeaconchainApi } from "../../src/modules/apiClients/index.js"; + +/** A beacon node that knows none of the validators, for tests that do not exercise beacon data. */ +export const emptyBeacon = (): BeaconchainApi => + ({ + postStateValidators: async () => ({ execution_optimistic: false, finalized: true, data: [] }) + }) as unknown as BeaconchainApi; diff --git a/packages/brain/test/unit/modules/beaconStatus.unit.test.ts b/packages/brain/test/unit/modules/beaconStatus.unit.test.ts new file mode 100644 index 00000000..4835007b --- /dev/null +++ b/packages/brain/test/unit/modules/beaconStatus.unit.test.ts @@ -0,0 +1,130 @@ +import { expect } from "chai"; +import sinon from "sinon"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { BrainDataBase } from "../../../src/modules/db/index.js"; +import type { BeaconchainApi } from "../../../src/modules/apiClients/index.js"; +import { BEACON_CHUNK_SIZE, BeaconStatusStore } from "../../../src/modules/beaconStatus/index.js"; +import logger from "../../../src/modules/logger/index.js"; + +const pubkeyAt = (index: number) => "0x" + (index + 1).toString(16).padStart(96, "0"); +const credentials = "0x01" + "00".repeat(11) + "cd".repeat(20); +const response = (pubkeys: string[]) => ({ + execution_optimistic: false, + finalized: true, + data: pubkeys.map((pubkey) => ({ + index: String(parseInt(pubkey.slice(-6), 16) + 1000), + status: "active_ongoing", + validator: { pubkey, withdrawal_credentials: credentials } + })) +}); + +describe("Beacon details store", () => { + let directory: string; + let brainDb: BrainDataBase; + let beacon: { postStateValidators: sinon.SinonStub }; + let store: BeaconStatusStore; + let notifications: sinon.SinonSpy; + const keys = Array.from({ length: 3 }, (_, index) => pubkeyAt(index)); + const addKeys = (pubkeys: string[]) => + brainDb.addValidators({ + validators: Object.fromEntries( + pubkeys.map((key) => [key, { tag: "solo", feeRecipient: "0x" + "ab".repeat(20), automaticImport: false }]) + ) + }); + const settle = () => new Promise((resolve) => setImmediate(resolve)); + + beforeEach(() => { + sinon.stub(logger, "error"); + sinon.stub(logger, "warn"); + directory = fs.mkdtempSync(path.join(os.tmpdir(), "stakingbrain-beacon-")); + brainDb = new BrainDataBase(path.join(directory, "db.json")); + addKeys(keys); + beacon = { postStateValidators: sinon.stub().callsFake(async ({ body }) => response(body.ids)) }; + store = new BeaconStatusStore(brainDb, beacon as unknown as BeaconchainApi); + notifications = sinon.spy(); + brainDb.onChange(notifications); + }); + afterEach(() => { + sinon.restore(); + fs.rmSync(directory, { recursive: true, force: true }); + }); + + it("reports validators as pending until looked up, then their index and credentials", async () => { + expect(keys.map((key) => store.view(key).beaconPending)).to.deep.equal([true, true, true]); + await store.refresh(); + const view = store.view(keys[0]); + expect(view).to.deep.include({ index: "1001", beaconPending: false, beaconError: undefined }); + expect(view.withdrawalCredentials).to.deep.equal({ address: credentials, format: "ecdsa" }); + expect(notifications.called).to.equal(true); + }); + + it("persists indexes so the Brain API can serve them", async () => { + await store.refresh(); + expect(Object.values(brainDb.getData()).map(({ index }) => index)).to.deep.equal([1001, 1002, 1003]); + }); + + it("looks up large key sets in chunks", async () => { + addKeys(Array.from({ length: 1200 }, (_, index) => pubkeyAt(index + 10))); + await settle(); + beacon.postStateValidators.resetHistory(); + await store.refresh(); + expect(beacon.postStateValidators.getCalls().map(({ args }) => args[0].body.ids.length)).to.deep.equal([ + BEACON_CHUNK_SIZE, + BEACON_CHUNK_SIZE, + 1203 - 2 * BEACON_CHUNK_SIZE + ]); + }); + + it("treats validators unknown to the beacon chain as complete", async () => { + beacon.postStateValidators.callsFake(async ({ body }) => response(body.ids.slice(1))); + await store.refresh(); + expect(store.view(keys[0])).to.deep.include({ index: undefined, beaconPending: false, beaconError: undefined }); + expect(store.view(keys[0]).withdrawalCredentials.format).to.equal("error"); + }); + + it("reports a failed lookup, keeps earlier results, and recovers", async () => { + beacon.postStateValidators.rejects(new Error("socket hang up")); + await store.refresh(); + expect(store.view(keys[0])).to.deep.include({ beaconPending: false }); + expect(store.view(keys[0]).beaconError).to.include("socket hang up"); + + beacon.postStateValidators.callsFake(async ({ body }) => response(body.ids)); + await store.refresh(); + expect(store.view(keys[0])).to.deep.include({ index: "1001", beaconError: undefined }); + + beacon.postStateValidators.rejects(new Error("socket hang up")); + await store.refresh(); + expect(store.view(keys[0])).to.deep.include({ index: "1001", beaconError: undefined }); + }); + + it("looks up validators added to the database, without looping on failures", async () => { + await store.refresh(); + beacon.postStateValidators.resetHistory(); + beacon.postStateValidators.rejects(new Error("syncing")); + addKeys([pubkeyAt(50)]); + await settle(); + await settle(); + // One lookup for the new key; its announced failure must not trigger another. + expect(beacon.postStateValidators.getCalls().map(({ args }) => args[0].body.ids)).to.deep.equal([[pubkeyAt(50)]]); + expect(store.view(pubkeyAt(50)).beaconError).to.include("syncing"); + }); + + it("merges overlapping refreshes into one follow-up", async () => { + let release!: () => void; + beacon.postStateValidators.onFirstCall().callsFake( + ({ body }) => + new Promise((resolve) => { + release = () => resolve(response(body.ids)); + }) + ); + const runs = [store.refresh([keys[0]]), store.refresh([keys[1]]), store.refresh([keys[2]])]; + release(); + await Promise.all(runs); + expect(beacon.postStateValidators.getCalls().map(({ args }) => args[0].body.ids)).to.deep.equal([ + [keys[0]], + [keys[1], keys[2]] + ]); + }); +}); diff --git a/packages/brain/test/unit/modules/builderConfig/builderConfig.unit.test.ts b/packages/brain/test/unit/modules/builderConfig/builderConfig.unit.test.ts index 371636dd..98bb20a1 100644 --- a/packages/brain/test/unit/modules/builderConfig/builderConfig.unit.test.ts +++ b/packages/brain/test/unit/modules/builderConfig/builderConfig.unit.test.ts @@ -3,7 +3,15 @@ import sinon from "sinon"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { BuilderConfig, Network, MEV_SP_ADDRESS_MAINNET, ConsensusClient, ExecutionClient } from "@stakingbrain/common"; +import { + BuilderConfig, + Network, + MEV_SP_ADDRESS_MAINNET, + ConsensusClient, + ExecutionClient, + featureFlags, + FeatureDisabledError +} from "@stakingbrain/common"; import { ValidatorApi, Web3SignerApi, BeaconchainApi } from "../../../../src/modules/apiClients/index.js"; import { AllowedMethods } from "../../../../src/modules/apiClients/types.js"; import { HttpApiError } from "../../../../src/modules/apiClients/error.js"; @@ -13,9 +21,12 @@ import { reloadValidators } from "../../../../src/modules/cron/reloadValidators/ import { updateValidators } from "../../../../src/modules/apiServers/ui/calls/updateValidators.js"; import { getBuilderConfigs } from "../../../../src/modules/apiServers/ui/calls/getBuilderConfigs.js"; import { getValidators } from "../../../../src/modules/apiServers/ui/calls/getValidators.js"; +import { createRpcMethods } from "../../../../src/modules/apiServers/ui/calls/index.js"; import { setActiveValidatorsLoadedInBrain } from "../../../../src/modules/cron/trackValidatorsPerformance/setActiveValidatorsLoadedInBrain.js"; import { ValidatorStatus } from "../../../../src/modules/apiClients/types.js"; import logger from "../../../../src/modules/logger/index.js"; +import { BeaconStatusStore } from "../../../../src/modules/beaconStatus/index.js"; +import { emptyBeacon } from "../../../helpers/beacon.js"; function deferred(): { promise: Promise; resolve: () => void } { let resolve!: () => void; @@ -86,8 +97,11 @@ describe("Builder configuration management", () => { api.remoteKeys = [...keys]; cron = sinon.createStubInstance(CronJob); sinon.stub(logger, "error"); + // The feature is switched off in this version; this suite covers it for when it returns. + featureFlags.builderConfig = true; }); afterEach(() => { + featureFlags.builderConfig = false; sinon.restore(); fs.rmSync(directory, { recursive: true, force: true }); }); @@ -545,13 +559,11 @@ describe("Builder configuration management", () => { signer.listRemoteKeys.resolves({ data: keys.map((validating_pubkey) => ({ validating_pubkey, derivation_path: "", readonly: false })) }); - const beacon = sinon.createStubInstance(BeaconchainApi); - beacon.getStateValidator.rejects(new Error("not deposited")); const result = await getValidators({ brainDb: db, validatorApi: validator, signerApi: signer, - beaconchainApi: beacon + beaconStatus: new BeaconStatusStore(db, emptyBeacon()) }); expect(result.map(({ validatorFeeRecipientCorrect }) => validatorFeeRecipientCorrect)).to.deep.equal([ true, @@ -562,3 +574,73 @@ describe("Builder configuration management", () => { }); } }); + +describe("Builder configuration while disabled", () => { + let directory: string; + let db: BrainDataBase; + let api: RecordingValidator; + const config: BuilderConfig = { builders: [{ url: "https://hoodi.aestus.live" }] }; + beforeEach(() => { + expect(featureFlags.builderConfig).to.equal(false); + directory = fs.mkdtempSync(path.join(os.tmpdir(), "stakingbrain-builders-off-")); + db = new BrainDataBase(path.join(directory, "db.json")); + // A record saved while the feature was on must survive untouched. + db.addValidators({ + validators: { [keys[0]]: { tag: "solo", feeRecipient: recipient, automaticImport: false, builderConfig: config } } + }); + api = new RecordingValidator(); + api.remoteKeys = [keys[0]]; + sinon.stub(logger, "error"); + }); + afterEach(() => { + sinon.restore(); + fs.rmSync(directory, { recursive: true, force: true }); + }); + + it("rejects builder updates without touching stored settings or the client", async () => { + const error = await updateValidators({ + reloadValidatorsCronTask: sinon.createStubInstance(CronJob), + brainDb: db, + validatorApi: api, + signerUrl: "http://signer", + customValidatorUpdateRequest: [{ pubkey: keys[0], builderConfig: null }] + }).catch((e: Error) => e); + expect(error).to.be.instanceOf(FeatureDisabledError); + expect(db.getData()[keys[0]].builderConfig).to.deep.equal(config); + expect(api.calls.filter(({ endpoint }) => endpoint.includes("builder_config"))).to.deep.equal([]); + }); + + it("does not replay stored settings on reload and hides them from the list", async () => { + const signer = sinon.createStubInstance(Web3SignerApi); + signer.upcheck.resolves(); + signer.listRemoteKeys.resolves({ data: [{ validating_pubkey: keys[0], derivation_path: "", readonly: false }] }); + await reloadValidators(signer, "http://signer", api, db); + expect(api.calls.filter(({ endpoint }) => endpoint.includes("builder_config"))).to.deep.equal([]); + expect(db.getData()[keys[0]].builderConfig).to.deep.equal(config); + + const [row] = await getValidators({ + brainDb: db, + validatorApi: api, + signerApi: signer, + beaconStatus: new BeaconStatusStore(db, emptyBeacon()) + }); + expect(row).to.not.have.property("builderConfig"); + expect(row).to.not.have.property("builderConfigStatus"); + }); + + it("refuses the builder RPCs", async () => { + const methods = createRpcMethods({ + postgresClient: {} as never, + beaconchainApi: {} as never, + validatorApi: api, + signerApi: {} as never, + brainDb: db, + reloadValidatorsCronTask: {} as never, + brainConfig: { apis: {}, chain: { network: Network.Hoodi } } as never, + beaconStatus: new BeaconStatusStore(db, emptyBeacon()) + }); + for (const call of [() => methods.getBuilderRelayPresets(), () => methods.getBuilderConfigs([keys[0]])]) + expect(await call().catch((e: Error) => e)).to.be.instanceOf(FeatureDisabledError); + expect(api.calls).to.deep.equal([]); + }); +}); diff --git a/packages/brain/test/unit/modules/db/onChange.unit.test.ts b/packages/brain/test/unit/modules/db/onChange.unit.test.ts new file mode 100644 index 00000000..fb763419 --- /dev/null +++ b/packages/brain/test/unit/modules/db/onChange.unit.test.ts @@ -0,0 +1,75 @@ +import { expect } from "chai"; +import sinon from "sinon"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { BrainDataBase } from "../../../../src/modules/db/index.js"; +import logger from "../../../../src/modules/logger/index.js"; +import { withValidatorOperationLock } from "../../../../src/modules/db/withValidatorOperationLock.js"; + +describe("Database change notifications", () => { + let directory: string; + let brainDb: BrainDataBase; + const pubkey = "0x" + "11".repeat(48); + const record = { tag: "solo" as const, feeRecipient: "0x" + "ab".repeat(20), automaticImport: false }; + + beforeEach(() => { + directory = fs.mkdtempSync(path.join(os.tmpdir(), "stakingbrain-change-")); + brainDb = new BrainDataBase(path.join(directory, "db.json")); + sinon.stub(logger, "warn"); + sinon.stub(logger, "error"); + }); + afterEach(() => { + sinon.restore(); + fs.rmSync(directory, { recursive: true, force: true }); + }); + + it("notifies after persisted changes only, until unsubscribed", () => { + const listener = sinon.spy(); + const unsubscribe = brainDb.onChange(listener); + brainDb.addValidators({ validators: { [pubkey]: { ...record } } }); + expect(listener.callCount).to.equal(1); + // A delete of an unknown key writes nothing, so it must not trigger UI refreshes. + brainDb.deleteValidators(["0x" + "22".repeat(48)]); + expect(listener.callCount).to.equal(1); + unsubscribe(); + brainDb.deleteValidators([pubkey]); + expect(listener.callCount).to.equal(1); + }); + + it("sends one notification after a held operation, and none if nothing changed", () => { + const listener = sinon.spy(); + brainDb.onChange(listener); + const release = brainDb.holdChangeNotifications(); + const nested = brainDb.holdChangeNotifications(); + brainDb.addValidators({ validators: { [pubkey]: { ...record } } }); + brainDb.updateValidators([{ pubkey, feeRecipient: "0x" + "cd".repeat(20) }]); + nested(); + expect(listener.callCount).to.equal(0); + release(); + release(); + expect(listener.callCount).to.equal(1); + brainDb.holdChangeNotifications()(); + expect(listener.callCount).to.equal(1); + }); + + it("defers notifications for the whole validator operation", async () => { + const listener = sinon.spy(); + brainDb.onChange(listener); + await withValidatorOperationLock(brainDb, async () => { + brainDb.addValidators({ validators: { [pubkey]: { ...record } } }); + await new Promise((resolve) => setTimeout(resolve, 5)); + // Clients are configured after the write; nobody should have been told yet. + expect(listener.callCount).to.equal(0); + }); + expect(listener.callCount).to.equal(1); + }); + + it("keeps writing when a listener throws", () => { + brainDb.onChange(() => { + throw new Error("socket gone"); + }); + brainDb.addValidators({ validators: { [pubkey]: { ...record } } }); + expect(Object.keys(brainDb.getData())).to.deep.equal([pubkey]); + }); +}); diff --git a/packages/brain/test/unit/modules/db/withValidatorOperationLock.unit.test.ts b/packages/brain/test/unit/modules/db/withValidatorOperationLock.unit.test.ts new file mode 100644 index 00000000..1aa60804 --- /dev/null +++ b/packages/brain/test/unit/modules/db/withValidatorOperationLock.unit.test.ts @@ -0,0 +1,44 @@ +import { expect } from "chai"; +import sinon from "sinon"; +import type { BrainDataBase } from "../../../../src/modules/db/index.js"; +import { + ValidatorOperationBusyError, + withValidatorOperationLock +} from "../../../../src/modules/db/withValidatorOperationLock.js"; + +const fakeDb = () => ({ holdChangeNotifications: () => () => undefined }) as unknown as BrainDataBase; + +describe("Validator operation lock", () => { + afterEach(() => sinon.restore()); + + it("keeps later operations behind the running one after a waiter is abandoned", async () => { + const clock = sinon.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + const brainDb = fakeDb(); + const events: string[] = []; + let finishFirst!: () => void; + + const first = withValidatorOperationLock(brainDb, async () => { + events.push("first:start"); + await new Promise((resolve) => (finishFirst = resolve)); + events.push("first:end"); + }); + const abandoned = withValidatorOperationLock(brainDb, async () => events.push("abandoned"), { + maxWaitMs: 1000 + }).catch((error: Error) => error); + const third = withValidatorOperationLock(brainDb, async () => events.push("third")); + + await clock.tickAsync(1000); + expect(await abandoned).to.be.instanceOf(ValidatorOperationBusyError); + expect(events).to.deep.equal(["first:start"]); + + clock.restore(); + finishFirst(); + await Promise.all([first, third]); + expect(events).to.deep.equal(["first:start", "first:end", "third"]); + }); + + it("runs a bounded operation that acquires the lock in time", async () => { + const brainDb = fakeDb(); + expect(await withValidatorOperationLock(brainDb, async () => "done", { maxWaitMs: 1000 })).to.equal("done"); + }); +}); diff --git a/packages/brain/test/unit/modules/exitValidators.unit.test.ts b/packages/brain/test/unit/modules/exitValidators.unit.test.ts new file mode 100644 index 00000000..0c13df28 --- /dev/null +++ b/packages/brain/test/unit/modules/exitValidators.unit.test.ts @@ -0,0 +1,67 @@ +import { expect } from "chai"; +import sinon from "sinon"; +import { BeaconchainApi, Web3SignerApi } from "../../../src/modules/apiClients/index.js"; +import { getExitValidators } from "../../../src/modules/apiServers/ui/calls/exitValidators.js"; +import { BEACON_CHUNK_SIZE } from "../../../src/modules/beaconStatus/index.js"; +import logger from "../../../src/modules/logger/index.js"; + +const pubkeyAt = (index: number) => "0x" + (index + 1).toString(16).padStart(96, "0"); + +describe("Validator exits for large selections", () => { + let beacon: sinon.SinonStubbedInstance; + let signer: sinon.SinonStubbedInstance; + beforeEach(() => { + sinon.stub(logger, "debug"); + beacon = sinon.createStubInstance(BeaconchainApi); + signer = sinon.createStubInstance(Web3SignerApi); + beacon.getEpochHeader.resolves(10); + beacon.getForkFromState.resolves({ + data: { previous_version: "0x00", current_version: "0x01", epoch: "1" } + } as never); + beacon.getGenesis.resolves({ + data: { genesis_time: "0", genesis_validators_root: "0x" + "00".repeat(32), genesis_fork_version: "0x00" } + } as never); + signer.signVoluntaryExit.callsFake(async ({ pubkey }) => ({ signature: `sig-${pubkey}` })); + }); + afterEach(() => sinon.restore()); + + it("looks up indexes in bulk chunks and keeps the selection order", async () => { + const pubkeys = Array.from({ length: 1200 }, (_, index) => pubkeyAt(index)); + beacon.postStateValidators.callsFake( + async ({ body }) => + ({ + execution_optimistic: false, + finalized: true, + // The beacon API does not promise any order. + data: [...body.ids].reverse().map((pubkey) => ({ + index: String(pubkeys.indexOf(pubkey)), + balance: "0", + status: "active_ongoing", + validator: { pubkey: pubkey.toUpperCase().replace("0X", "0x") } + })) + }) as never + ); + const exits = await getExitValidators({ beaconchainApi: beacon, signerApi: signer, pubkeys }); + expect(beacon.postStateValidators.getCalls().map(({ args }) => args[0].body.ids.length)).to.deep.equal([ + BEACON_CHUNK_SIZE, + BEACON_CHUNK_SIZE, + 200 + ]); + expect(beacon.getStateValidator.called).to.equal(false); + // Signing follows the selection order, whatever order the beacon node answered in. + expect(signer.signVoluntaryExit.getCalls().map(({ args }) => args[0].pubkey)).to.deep.equal(pubkeys); + expect(exits.map(({ message }) => message.validator_index)).to.deep.equal(pubkeys.map((_, index) => String(index))); + }); + + it("refuses to exit when a validator is unknown to the beacon chain", async () => { + const pubkeys = [pubkeyAt(0), pubkeyAt(1)]; + beacon.postStateValidators.resolves({ + execution_optimistic: false, + finalized: true, + data: [{ index: "0", validator: { pubkey: pubkeys[0] } }] + } as never); + const error = await getExitValidators({ beaconchainApi: beacon, signerApi: signer, pubkeys }).catch((e) => e); + expect(error).to.be.instanceOf(Error).and.have.property("message").that.includes(pubkeys[1]); + expect(signer.signVoluntaryExit.called).to.equal(false); + }); +}); diff --git a/packages/brain/test/unit/modules/jsonBodyEncoding.unit.test.ts b/packages/brain/test/unit/modules/jsonBodyEncoding.unit.test.ts index 906ff49e..b4a8e00c 100644 --- a/packages/brain/test/unit/modules/jsonBodyEncoding.unit.test.ts +++ b/packages/brain/test/unit/modules/jsonBodyEncoding.unit.test.ts @@ -15,6 +15,7 @@ import { BeaconchainApi, PostgresClient, ValidatorApi, Web3SignerApi } from "../ import { BrainDataBase } from "../../../src/modules/db/index.js"; import { CronJob } from "../../../src/modules/cron/cron.js"; import { params } from "../../../src/params.js"; +import { BeaconStatusStore } from "../../../src/modules/beaconStatus/index.js"; function postJson(server: http.Server, body: Buffer, encoding?: string): Promise { const address = server.address() as AddressInfo; @@ -57,6 +58,7 @@ describe("JSON request content encoding", () => { const reloadValidatorsCronTask = sinon.createStubInstance(CronJob); const signerUrl = "http://127.0.0.1:1"; const allowedOriginsFromEnv = null; + const beaconStatus = new BeaconStatusStore(brainDb, beaconchainApi); servers.set("Brain", startBrainApi({ brainDb, allowedOriginsFromEnv })); servers.set( "Launchpad", @@ -64,7 +66,6 @@ describe("JSON request content encoding", () => { brainDb, validatorApi, signerApi, - beaconchainApi, reloadValidatorsCronTask, signerUrl, network: Network.Hoodi, @@ -82,6 +83,7 @@ describe("JSON request content encoding", () => { postgresClient: sinon.createStubInstance(PostgresClient), uiBuildPath: directory, allowedOriginsFromEnv, + beaconStatus, brainConfig: { chain: { network: Network.Hoodi, diff --git a/packages/brain/test/unit/modules/keyManagement.unit.test.ts b/packages/brain/test/unit/modules/keyManagement.unit.test.ts index 356c41f7..6f0d57b5 100644 --- a/packages/brain/test/unit/modules/keyManagement.unit.test.ts +++ b/packages/brain/test/unit/modules/keyManagement.unit.test.ts @@ -13,6 +13,11 @@ import { updateValidators } from "../../../src/modules/apiServers/ui/calls/updat import { reloadValidators } from "../../../src/modules/cron/reloadValidators/index.js"; import type { CustomImportRequest } from "../../../src/modules/apiServers/ui/calls/types.js"; import logger from "../../../src/modules/logger/index.js"; +import { + CLIENT_OPERATION_MAX_WAIT_MS, + ValidatorOperationBusyError, + withValidatorOperationLock +} from "../../../src/modules/db/withValidatorOperationLock.js"; const keys = ["11", "22", "33"].map((byte) => "0x" + byte.repeat(48)); const recipient = "0x" + "ab".repeat(20); @@ -278,4 +283,71 @@ describe("Key management partial failures", () => { expect(validatorApi.postRemoteKeys.called).to.equal(false); expect(Object.keys(brainDb.getData())).to.deep.equal([pubkey]); }); + + it("updates the stored record when the request uses different hex casing", async () => { + const pubkey = "0x" + "ab".repeat(48); + const other = "0x" + "cd".repeat(20); + brainDb.addValidators({ validators: { [pubkey]: { ...record } } }); + await updateValidators({ + brainDb, + validatorApi, + signerUrl: "http://signer:9000", + reloadValidatorsCronTask, + customValidatorUpdateRequest: [{ pubkey: "0x" + "AB".repeat(48), feeRecipient: other }] + }); + expect(brainDb.getData()).to.deep.equal({ [pubkey]: { ...record, feeRecipient: other } }); + expect(validatorApi.setFeeRecipient.calledOnceWith(other, pubkey)).to.equal(true); + }); + + it("rejects the same validator twice in one update even with different hex casing", async () => { + const pubkey = "0x" + "ab".repeat(48); + brainDb.addValidators({ validators: { [pubkey]: { ...record } } }); + const result = await updateValidators({ + brainDb, + validatorApi, + signerUrl: "http://signer:9000", + reloadValidatorsCronTask, + customValidatorUpdateRequest: [ + { pubkey, feeRecipient: recipient }, + { pubkey: "0x" + "AB".repeat(48), feeRecipient: recipient } + ] + }).catch((error: Error) => error); + expect(result).to.be.instanceOf(Error).and.have.property("message").that.includes("Duplicate validator"); + expect(validatorApi.setFeeRecipient.called).to.equal(false); + }); + + it("keeps a signer key listed when its deletion result is missing", async () => { + brainDb.addValidators({ validators: { [keys[0]]: { ...record } } }); + signerApi.upcheck.resolves(); + signerApi.listRemoteKeys.resolves({ + data: [keys[0], keys[1]].map((validating_pubkey) => ({ validating_pubkey, derivation_path: "", readonly: false })) + }); + signerApi.deleteRemoteKeys.resolves({ data: [] }); + validatorApi.getRemoteKeys.resolves({ data: [{ pubkey: keys[0], readonly: false, url: "http://signer" }] }); + await reloadValidators(signerApi, "http://signer", validatorApi, brainDb); + expect(signerApi.deleteRemoteKeys.calledOnceWith({ pubkeys: [keys[1]] })).to.equal(true); + // Reload continues past the short response instead of aborting. + expect(validatorApi.setFeeRecipient.calledOnceWith(recipient, keys[0])).to.equal(true); + }); + + it("abandons a client operation that cannot start before its wait budget", async () => { + const clock = sinon.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + let finishReload!: () => void; + const reload = withValidatorOperationLock(brainDb, () => new Promise((resolve) => (finishReload = resolve))); + const update = updateValidators({ + brainDb, + validatorApi, + signerUrl: "http://signer:9000", + reloadValidatorsCronTask, + customValidatorUpdateRequest: [{ pubkey: keys[0], feeRecipient: recipient }] + }).catch((error: Error) => error); + await clock.tickAsync(CLIENT_OPERATION_MAX_WAIT_MS); + expect(await update).to.be.instanceOf(ValidatorOperationBusyError); + clock.restore(); + finishReload(); + await reload; + // The abandoned call never touched clients or the cron, even after the lock was released. + expect(reloadValidatorsCronTask.stop.called).to.equal(false); + expect(validatorApi.setFeeRecipient.called).to.equal(false); + }); }); diff --git a/packages/brain/test/unit/modules/validatorList.unit.test.ts b/packages/brain/test/unit/modules/validatorList.unit.test.ts index 0f6ed551..af51020a 100644 --- a/packages/brain/test/unit/modules/validatorList.unit.test.ts +++ b/packages/brain/test/unit/modules/validatorList.unit.test.ts @@ -4,40 +4,28 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { BrainDataBase } from "../../../src/modules/db/index.js"; -import { ValidatorApi, Web3SignerApi, BeaconchainApi } from "../../../src/modules/apiClients/index.js"; -import { getValidators } from "../../../src/modules/apiServers/ui/calls/getValidators.js"; +import { ValidatorApi, Web3SignerApi } from "../../../src/modules/apiClients/index.js"; +import { getValidators, LIST_DEADLINE_MS } from "../../../src/modules/apiServers/ui/calls/getValidators.js"; +import { BeaconStatusStore } from "../../../src/modules/beaconStatus/index.js"; import logger from "../../../src/modules/logger/index.js"; -import type { BeaconchainValidatorFromStateGetResponse } from "../../../src/modules/apiClients/beaconchain/types.js"; -import { ValidatorStatus } from "../../../src/modules/apiClients/beaconchain/types.js"; +import { emptyBeacon } from "../../helpers/beacon.js"; const keys = Array.from({ length: 30 }, (_, index) => "0x" + (index + 1).toString(16).padStart(96, "0")); const feeRecipient = "0x" + "ab".repeat(20); -const beaconResponse = (pubkey: string, address: string): BeaconchainValidatorFromStateGetResponse => ({ - execution_optimistic: false, - data: { - index: String(keys.indexOf(pubkey)), - balance: "32000000000", - status: ValidatorStatus.ACTIVE_ONGOING, - validator: { - pubkey, - withdrawal_credentials: address, - effective_balance: "32000000000", - slashed: false, - activation_eligibility_epoch: "0", - activation_epoch: "0", - exit_epoch: "18446744073709551615", - withdrawable_epoch: "18446744073709551615" - } - } -}); +const hang = ({ signal }: { signal?: AbortSignal }) => + new Promise((_resolve, reject) => { + signal!.addEventListener("abort", () => reject(new Error("Aborted")), { once: true }); + }); -describe("Validator list bulk reads", () => { +describe("Validator list reads", () => { let directory: string; let brainDb: BrainDataBase; let validatorApi: sinon.SinonStubbedInstance; let signerApi: sinon.SinonStubbedInstance; - let beaconchainApi: sinon.SinonStubbedInstance; + let beaconStatus: BeaconStatusStore; + beforeEach(() => { + sinon.stub(logger, "error"); directory = fs.mkdtempSync(path.join(os.tmpdir(), "stakingbrain-list-")); brainDb = new BrainDataBase(path.join(directory, "db.json")); brainDb.addValidators({ @@ -45,124 +33,159 @@ describe("Validator list bulk reads", () => { }); validatorApi = sinon.createStubInstance(ValidatorApi); signerApi = sinon.createStubInstance(Web3SignerApi); - beaconchainApi = sinon.createStubInstance(BeaconchainApi); - validatorApi.getRemoteKeys.resolves({ - data: keys.map((pubkey) => ({ pubkey, readonly: false, url: "http://signer" })) - }); + beaconStatus = new BeaconStatusStore(brainDb, emptyBeacon()); + validatorApi.getRemoteKeys.resolves({ data: keys.map((pubkey) => ({ pubkey, readonly: false, url: "" })) }); signerApi.listRemoteKeys.resolves({ data: keys.map((validating_pubkey) => ({ validating_pubkey, derivation_path: "", readonly: false })) }); - validatorApi.getFeeRecipient.resolves({ data: { ethaddress: feeRecipient } }); - sinon.stub(logger, "error"); + validatorApi.getFeeRecipient.resolves({ data: { ethaddress: feeRecipient.toUpperCase().replace("0X", "0x") } }); }); afterEach(() => { sinon.restore(); fs.rmSync(directory, { recursive: true, force: true }); }); - const read = () => getValidators({ brainDb, validatorApi, signerApi, beaconchainApi }); + const list = () => getValidators({ brainDb, validatorApi, signerApi, beaconStatus }); - it("keeps successful fee checks when a different validator fails", async () => { - validatorApi.getFeeRecipient.withArgs(keys[0]).rejects(new Error("key missing")); - beaconchainApi.getStateValidator.rejects(new Error("not deposited")); - const results = await read(); - expect(results.map(({ pubkey }) => pubkey)).to.deep.equal(keys); - expect(results[0].validatorFeeRecipientCorrect).to.equal(undefined); - expect(results.slice(1).every(({ validatorFeeRecipientCorrect }) => validatorFeeRecipientCorrect)).to.equal(true); + it("reads membership and fee recipients from the clients", async () => { + validatorApi.getFeeRecipient.withArgs(keys[1]).resolves({ data: { ethaddress: "0x" + "00".repeat(20) } }); + signerApi.listRemoteKeys.resolves({ data: [] }); + const rows = await list(); + expect(rows.every(({ signerImported }) => signerImported === false)).to.equal(true); + expect(rows.every(({ validatorImported }) => validatorImported)).to.equal(true); + expect(rows.map(({ validatorFeeRecipientCorrect }) => validatorFeeRecipientCorrect).slice(0, 3)).to.deep.equal([ + true, + false, + true + ]); + expect(rows.every(({ detailsPending, detailsError }) => !detailsPending && !detailsError)).to.equal(true); }); - it("bounds concurrent beacon reads and stops the whole list at its deadline", async () => { + it("bounds reads by concurrency and deadline and reports cut-off rows as pending", async () => { const clock = sinon.useFakeTimers(); - beaconchainApi.getStateValidator.callsFake( - async ({ signal }) => - new Promise((_resolve, reject) => { - if (signal?.aborted) reject(new Error("Aborted")); - signal!.addEventListener("abort", () => reject(new Error("Aborted")), { once: true }); - }) - ); - const pending = read(); - await clock.tickAsync(9999); - expect(beaconchainApi.getStateValidator.callCount).to.equal(8); + validatorApi.getFeeRecipient.callsFake(((_pubkey: string, signal?: AbortSignal) => hang({ signal })) as never); + const rows = list(); + await clock.tickAsync(LIST_DEADLINE_MS - 1); + expect(validatorApi.getFeeRecipient.callCount).to.equal(8); await clock.tickAsync(1); - const results = await pending; - expect(results.map(({ pubkey }) => pubkey)).to.deep.equal(keys); - expect(results.every(({ withdrawalCredentials }) => withdrawalCredentials.format === "error")).to.equal(true); - expect(beaconchainApi.getStateValidator.callCount).to.equal(8); + const result = await rows; + expect(result.every(({ detailsPending, detailsError }) => detailsPending && !detailsError)).to.equal(true); expect(clock.countTimers()).to.equal(0); }); - it("finishes later beacon reads on refresh and retains earlier results without delaying fee checks", async () => { + it("converges over follow-up requests: only unread rows are pending", async () => { const clock = sinon.useFakeTimers(); - const address = "0x01" + "00".repeat(11) + "ab".repeat(20); - beaconchainApi.getStateValidator.callsFake( - ({ pubkey, signal }) => + validatorApi.getFeeRecipient.callsFake( + ((_pubkey: string, signal?: AbortSignal) => new Promise((resolve, reject) => { - const abort = () => { + const timer = setTimeout(() => resolve({ data: { ethaddress: feeRecipient } }), 4000); + signal?.addEventListener("abort", () => { clearTimeout(timer); reject(new Error("Aborted")); - }; - const timer = setTimeout(() => { - signal!.removeEventListener("abort", abort); - resolve(beaconResponse(pubkey, address)); - }, 4000); - signal!.addEventListener("abort", abort, { once: true }); - }) - ); - const first = read(); - await clock.tickAsync(10000); - const firstResults = await first; - expect(firstResults.filter(({ index }) => index !== "").length).to.equal(16); - expect(firstResults.every(({ validatorFeeRecipientCorrect }) => validatorFeeRecipientCorrect)).to.equal(true); - const second = read(); - await clock.tickAsync(10000); - const secondResults = await second; - expect(secondResults.map(({ index }) => index)).to.deep.equal(keys.map((_, index) => String(index))); - expect(secondResults.every(({ withdrawalCredentials }) => withdrawalCredentials.address === address)).to.equal( - true + }); + })) as never ); - // Expiration forces fresh reads so changed withdrawal credentials are not cached forever. - await clock.tickAsync(60000); - beaconchainApi.getStateValidator.resetHistory(); - const third = read(); - await clock.tickAsync(10000); - await third; - expect(beaconchainApi.getStateValidator.callCount).to.be.greaterThan(0); - expect(clock.countTimers()).to.equal(0); + const pendingByRound: number[] = []; + for (let round = 0; round < 3; round++) { + const rows = list(); + await clock.tickAsync(LIST_DEADLINE_MS); + const result = await rows; + const pending = result.filter(({ detailsPending }) => detailsPending); + // Pending means exactly "not read yet". + expect(pending.every(({ validatorFeeRecipientCorrect }) => validatorFeeRecipientCorrect === undefined)).to.equal( + true + ); + pendingByRound.push(pending.length); + } + expect(pendingByRound).to.deep.equal([14, 0, 0]); }); - it("rotates stalled fee reads and keeps unverified results unknown", async () => { + it("moves past reads that time out, so responsive validators are not starved", async () => { const clock = sinon.useFakeTimers(); - beaconchainApi.getStateValidator.rejects(new Error("not deposited")); - validatorApi.getFeeRecipient.callsFake( - async (_pubkey, signal) => - new Promise((_resolve, reject) => { - signal!.addEventListener("abort", () => reject(new Error("Aborted")), { once: true }); - }) - ); - const first = read(); - await clock.tickAsync(10000); + const stuck = new Set(keys.slice(0, 8)); + validatorApi.getFeeRecipient.callsFake(((pubkey: string, signal?: AbortSignal) => + stuck.has(pubkey) ? hang({ signal }) : Promise.resolve({ data: { ethaddress: feeRecipient } })) as never); + for (let round = 0; round < 2; round++) { + const rows = list(); + await clock.tickAsync(LIST_DEADLINE_MS); + await rows; + } + const rows = list(); + await clock.tickAsync(LIST_DEADLINE_MS); + const result = await rows; expect( - (await first).every(({ validatorFeeRecipientCorrect }) => validatorFeeRecipientCorrect === undefined) + result + .slice(8) + .every(({ detailsPending, validatorFeeRecipientCorrect }) => !detailsPending && validatorFeeRecipientCorrect) ).to.equal(true); - expect(validatorApi.getFeeRecipient.args.map(([pubkey]) => pubkey)).to.deep.equal(keys.slice(0, 8)); - validatorApi.getFeeRecipient.resetHistory(); - const second = read(); - await clock.tickAsync(10000); - await second; - expect(validatorApi.getFeeRecipient.args.map(([pubkey]) => pubkey)).to.deep.equal(keys.slice(8, 16)); - expect(clock.countTimers()).to.equal(0); + expect(result.slice(0, 8).every(({ detailsPending }) => detailsPending)).to.equal(true); }); - it("keeps membership unknown on list failures while independently reading beacon data", async () => { - validatorApi.getRemoteKeys.rejects(new Error("unavailable")); - signerApi.listRemoteKeys.rejects(new Error("unavailable")); - beaconchainApi.getStateValidator.callsFake(async ({ pubkey }) => beaconResponse(pubkey, "0x01" + "00".repeat(31))); - const results = await read(); - expect( - results.every( - ({ signerImported, validatorImported, validatorFeeRecipientCorrect }) => - signerImported === undefined && validatorImported === undefined && validatorFeeRecipientCorrect === undefined - ) - ).to.equal(true); - expect(results.map(({ index }) => index)).to.deep.equal(keys.map((_, index) => String(index))); + it("forgets a cached fee check when the key leaves the validator client", async () => { + await list(); + validatorApi.getRemoteKeys.resolves({ + data: keys.slice(1).map((pubkey) => ({ pubkey, readonly: false, url: "" })) + }); + await list(); + // The key returns, but its fee recipient was not restored. + validatorApi.getRemoteKeys.resolves({ data: keys.map((pubkey) => ({ pubkey, readonly: false, url: "" })) }); + validatorApi.getFeeRecipient.withArgs(keys[0]).resolves({ data: { ethaddress: "0x" + "00".repeat(20) } }); + brainDb.notifyChanged(); + expect((await list())[0].validatorFeeRecipientCorrect).to.equal(false); + }); + + it("reports an unreachable client instead of an earlier success", async () => { + await list(); + validatorApi.getRemoteKeys.rejects(new Error("ECONNREFUSED")); + const rows = await list(); + expect(rows.every(({ validatorImported }) => validatorImported === undefined)).to.equal(true); + expect(rows.every(({ validatorFeeRecipientCorrect }) => validatorFeeRecipientCorrect === undefined)).to.equal(true); + expect(rows.every(({ detailsPending }) => !detailsPending)).to.equal(true); + expect(rows[0].detailsError).to.include("validator client").and.include("ECONNREFUSED"); + }); + + it("reports a failed fee read for that row only", async () => { + validatorApi.getFeeRecipient.withArgs(keys[0]).rejects(new Error("HTTP 404")); + const rows = await list(); + expect(rows[0].detailsPending).to.equal(false); + expect(rows[0].validatorFeeRecipientCorrect).to.equal(undefined); + expect(rows[0].detailsError).to.include("HTTP 404"); + expect(rows.slice(1).every(({ detailsError }) => !detailsError)).to.equal(true); + }); + + it("re-reads a fee recipient checked mid-update once the update announces itself", async () => { + const other = "0x" + "cd".repeat(20); + brainDb.updateValidators([{ pubkey: keys[0], feeRecipient: other }]); + let finishRead!: () => void; + validatorApi.getFeeRecipient.withArgs(keys[0]).callsFake( + () => + new Promise((resolve) => { + finishRead = () => resolve({ data: { ethaddress: feeRecipient } }); + }) as never + ); + const during = list(); + await new Promise((resolve) => setImmediate(resolve)); + validatorApi.getFeeRecipient.withArgs(keys[0]).resolves({ data: { ethaddress: other } }); + brainDb.notifyChanged(); + finishRead(); + expect((await during)[0].validatorFeeRecipientCorrect).to.equal(false); + expect((await list())[0].validatorFeeRecipientCorrect).to.equal(true); + }); + + it("caches matching fee checks between requests and re-checks after an edit", async () => { + await list(); + expect(validatorApi.getFeeRecipient.callCount).to.equal(keys.length); + await list(); + expect(validatorApi.getFeeRecipient.callCount).to.equal(keys.length); + brainDb.updateValidators([{ pubkey: keys[0], feeRecipient: "0x" + "cd".repeat(20) }]); + const rows = await list(); + expect(validatorApi.getFeeRecipient.callCount).to.equal(keys.length + 1); + expect(rows[0].validatorFeeRecipientCorrect).to.equal(false); + }); + + it("takes beacon details from the beacon store, pending until it has looked them up", async () => { + const rows = await list(); + expect(rows.every(({ beaconPending, detailsPending }) => beaconPending && !detailsPending)).to.equal(true); + await beaconStatus.refresh(); + expect((await list()).every(({ beaconPending }) => !beaconPending)).to.equal(true); }); }); diff --git a/packages/brain/tsconfig.test.json b/packages/brain/tsconfig.test.json index d992a247..fee679a2 100644 --- a/packages/brain/tsconfig.test.json +++ b/packages/brain/tsconfig.test.json @@ -1,7 +1,8 @@ { "extends": "../../tsconfig.json", "compilerOptions": { - "rootDir": "." // Root directory of input files + // The e2e tests import the UI's client modules to drive the Brain exactly as the browser does. + "rootDir": ".." }, "include": ["test"], "exclude": ["node_modules", "dist"] diff --git a/packages/common/src/featureFlags.ts b/packages/common/src/featureFlags.ts new file mode 100644 index 00000000..cfe34132 --- /dev/null +++ b/packages/common/src/featureFlags.ts @@ -0,0 +1,16 @@ +/** + * Features shipped in code but switched off for now. Turning one back on needs no data migration: + * the Brain keeps persisted settings (e.g. each validator's `builderConfig`) while a feature is off. + * The object is mutable only so tests can exercise disabled features; application code must not write it. + */ +export const featureFlags = { + /** Gloas/ePBS per-validator builder configuration: UI, RPCs, reload re-apply and BUILDER_IDS import. */ + builderConfig: false +}; + +export class FeatureDisabledError extends Error { + constructor(feature: string) { + super(`${feature} is disabled in this version of the Staking Brain`); + this.name = "FeatureDisabledError"; + } +} diff --git a/packages/common/src/index.ts b/packages/common/src/index.ts index 94830437..55200214 100644 --- a/packages/common/src/index.ts +++ b/packages/common/src/index.ts @@ -4,3 +4,4 @@ export * from "./params.js"; export * from "./builderConfig.js"; export * from "./builderRelayPresets.js"; export * from "./rpc.js"; +export * from "./featureFlags.js"; diff --git a/packages/common/src/rpc.ts b/packages/common/src/rpc.ts index 34dc908c..2d36714b 100644 --- a/packages/common/src/rpc.ts +++ b/packages/common/src/rpc.ts @@ -1,3 +1,5 @@ // The database allows 6 MiB. Leave room for RPC framing and bulk settings requests. export const RPC_MAX_PAYLOAD_BYTES = 8 * 1024 * 1024; export const RPC_TIMEOUT_MS = 120 * 1000; +/** Server-to-UI push: validator records changed; clients should refresh their list. Carries no data. */ +export const VALIDATORS_CHANGED_EVENT = "validatorsChanged"; diff --git a/packages/ui/src/ImportScreen.tsx b/packages/ui/src/ImportScreen.tsx index caa4bc56..ff73bcf7 100644 --- a/packages/ui/src/ImportScreen.tsx +++ b/packages/ui/src/ImportScreen.tsx @@ -18,7 +18,7 @@ import { Stack } from "@mui/material"; import { Link } from "react-router-dom"; -import { useEffect, useState } from "react"; +import { useEffect, useRef, useState } from "react"; import BackupIcon from "@mui/icons-material/Backup"; import { ImportStatus, KeystoreInfo, TagSelectOption } from "./types"; import FileCardList from "./components/FileCards/FileCardList"; @@ -28,10 +28,13 @@ import CloseIcon from "@mui/icons-material/Close"; import { rpcClient } from "./socket"; import ArrowCircleLeftOutlinedIcon from "@mui/icons-material/ArrowCircleLeftOutlined"; import { extractPubkey } from "./utils/dataUtils"; -import type { Web3signerPostResponse } from "@stakingbrain/brain"; import { createImportRequest } from "./utils/createImportRequest"; -import { appendImportFiles, isImportFormValid, ImportFormState } from "./utils/importForm"; +import { appendImportFiles, incompleteImportIndexes, isImportFormValid, ImportFormState } from "./utils/importForm"; import PageLayout, { PageHeader } from "./components/Layout/PageLayout"; +import { ImportProgress, ImportResult, notAttempted, planImportBatches, runImportBatches } from "./utils/importBatches"; + +// Each card holds up to three inputs; rendering hundreds at once makes typing laggy. +const FILE_CARDS_PAGE = 25; export default function ImportScreen({ network, @@ -40,7 +43,11 @@ export default function ImportScreen({ network: Network; isMevBoostSet: boolean; }): JSX.Element { - const [keystoresPostResponse, setKeystoresPostResponse] = useState(); + const [importResults, setImportResults] = useState(); + const [importProgress, setImportProgress] = useState(); + const [importStopped, setImportStopped] = useState<"user" | "error">(); + const [stopRequested, setStopRequested] = useState(false); + const stopRef = useRef(false); const [keystoresPostError, setKeystoresPostError] = useState(); const [openDialog, setOpenDialog] = useState(false); const [importForm, setImportForm] = useState({ @@ -63,6 +70,8 @@ export default function ImportScreen({ const [slashingFile, setSlashingFile] = useState(); const [showMevWarning, setShowMevWarning] = useState(false); const [isButtonDisabled, setIsButtonDisabled] = useState(true); + const [visibleFiles, setVisibleFiles] = useState(FILE_CARDS_PAGE); + const incomplete = incompleteImportIndexes(importForm); // If the user has selected the 'lido' tag and MEV-Boost is not set, show the warning useEffect(() => { @@ -137,11 +146,18 @@ export default function ImportScreen({ setIsButtonDisabled(disable); // Set the state based on the conditions }, [passwords, tags, feeRecipients, acceptedFiles, slashingFile, slashingProtectionIncluded, showMevWarning]); - async function importKeystores() { + /** Import in batches with live progress. `only` retries a subset while keeping earlier results. */ + async function importKeystores(only?: number[]) { try { setImportStatus(ImportStatus.Importing); setKeystoresPostError(undefined); - setKeystoresPostResponse(undefined); + setImportStopped(undefined); + stopRef.current = false; + setStopRequested(false); + if (!only) { + setImportResults(undefined); + setImportProgress(undefined); + } handleClickOpenDialog(); const importRequest = await createImportRequest({ @@ -151,11 +167,24 @@ export default function ImportScreen({ feeRecipients, slashingFile: slashingProtectionIncluded ? slashingFile : undefined }); + const batches = planImportBatches( + importRequest, + acceptedFiles.map(({ pubkey }) => pubkey), + { only } + ); + const { results, stopped } = await runImportBatches({ + batches, + results: (only && importResults) || acceptedFiles.map(notAttempted), + importBatch: (request) => rpcClient.call("importValidators", request), + onProgress: (progress, results) => { + setImportProgress(progress); + setImportResults(results); + }, + shouldStop: () => stopRef.current + }); - const response = await rpcClient.call("importValidators", importRequest); - - setKeystoresPostResponse(response); - setKeystoresPostError(undefined); + setImportResults(results); + setImportStopped(stopped); setImportStatus(ImportStatus.Imported); } catch (e) { console.error(e); @@ -328,6 +357,23 @@ export default function ImportScreen({ )} + {acceptedFiles.length > 1 && ( + + {acceptedFiles.length} keystores selected + + + )} + {FileCardList( acceptedFiles, setAcceptedFiles, @@ -342,7 +388,35 @@ export default function ImportScreen({ useSameFeerecipient, getFeeRecipientFieldHelperText, isFeeRecipientFieldWrong, - tagSelectOptions + tagSelectOptions, + visibleFiles + )} + + {acceptedFiles.length > visibleFiles && ( + + + + Showing {visibleFiles} of {acceptedFiles.length} keystores. + {!(useSamePassword && useSameTag && useSameFeerecipient) && + " Hidden keystores still need their own values; turn on shared settings to fill them all at once."} + + + + + + + )} @@ -397,6 +471,21 @@ export default function ImportScreen({ ) : null} + {incomplete.length > 0 && acceptedFiles.length > 1 && ( + = visibleFiles ? ( + + ) : undefined + } + > + {incomplete.length} of {acceptedFiles.length} keystores still need a password, protocol or valid fee recipient + (first: #{incomplete[0] + 1}, {acceptedFiles[incomplete[0]]?.file.name}). + + )} {showMevWarning && ( } disabled={isButtonDisabled || importStatus === ImportStatus.Importing} - onClick={importKeystores} + onClick={() => importKeystores()} sx={{ borderRadius: 2 }} > Import validators @@ -448,10 +537,18 @@ export default function ImportScreen({ { + stopRef.current = true; + setStopRequested(true); + }} + onRetry={(indexes) => importKeystores(indexes)} /> ); diff --git a/packages/ui/src/components/Dialogs/DeletionWarning.tsx b/packages/ui/src/components/Dialogs/DeletionWarning.tsx index d01cf23a..3017a2d2 100644 --- a/packages/ui/src/components/Dialogs/DeletionWarning.tsx +++ b/packages/ui/src/components/Dialogs/DeletionWarning.tsx @@ -1,21 +1,14 @@ import type { JSX } from "react"; -import type { GridRowId } from "@mui/x-data-grid"; import { shortenPubkey } from "@stakingbrain/common"; import type { CustomValidatorGetResponse } from "@stakingbrain/brain"; -export default function DeletionWarning({ - selectedRows, - rows -}: { - selectedRows: GridRowId[]; - rows: CustomValidatorGetResponse[]; -}): JSX.Element { +export default function DeletionWarning({ validators }: { validators: CustomValidatorGetResponse[] }): JSX.Element { return ( <> Are you sure you want to delete these keystores?
    - {selectedRows.map((row, i) => ( -
  • {shortenPubkey(rows[parseInt(row.toString())].pubkey)}
  • + {validators.map(({ pubkey }) => ( +
  • {shortenPubkey(pubkey)}
  • ))}
After deletion, these keystores won't be used for signing anymore and your slashing protection data will be diff --git a/packages/ui/src/components/Dialogs/EditFeesDialog.tsx b/packages/ui/src/components/Dialogs/EditFeesDialog.tsx index 84adbaba..d9c30150 100644 --- a/packages/ui/src/components/Dialogs/EditFeesDialog.tsx +++ b/packages/ui/src/components/Dialogs/EditFeesDialog.tsx @@ -22,7 +22,6 @@ import { import InfoOutlinedIcon from "@mui/icons-material/InfoOutlined"; import EditOutlinedIcon from "@mui/icons-material/EditOutlined"; import DialogHeader from "./DialogHeader"; -import type { GridRowId } from "@mui/x-data-grid"; import { BURN_ADDRESS, isValidEcdsaPubkey, areAllFeeRecipientsEditable, Network } from "@stakingbrain/common"; import React from "react"; import type { @@ -47,15 +46,14 @@ import { useTheme, useMediaQuery } from "@mui/material"; export default function FeeRecipientDialog({ open, setOpen, - rows, - selectedRows, + validators, network, isMevBoostSet }: { open: boolean; setOpen: (open: boolean) => void; - rows: CustomValidatorGetResponse[]; - selectedRows: GridRowId[]; + /** The selected validators, resolved by pubkey. */ + validators: CustomValidatorGetResponse[]; network: Network; isMevBoostSet: boolean; }): JSX.Element { @@ -75,7 +73,7 @@ export default function FeeRecipientDialog({ useEffect(() => { isAnyWithdrawalCredentialsDiff("ecdsa") && getNonEcdsaValidatorsData(); getSmoothValidatorsSelected(); - }, [selectedRows]); + }, [validators]); const mevSpAddress = getSmoothAddressByNetwork(network); const smoothUrl = getSmoothUrlByNetwork(network); @@ -141,9 +139,7 @@ export default function FeeRecipientDialog({ async function updateValidators() { const validatorsData: CustomValidatorUpdateRequest[] = []; - selectedRows.forEach((rowId) => { - const row = rows[+rowId]; - + validators.forEach((row) => { if (row) { validatorsData.push({ pubkey: row.pubkey, @@ -170,18 +166,18 @@ export default function FeeRecipientDialog({ }; function areAllSelectedFeeRecipientsEditable() { - const selectedTags = selectedRows.map((rowId) => rows[+rowId].tag).flat(); + const selectedTags = validators.map((row) => row.tag).flat(); return areAllFeeRecipientsEditable(selectedTags); } function areAllOldFrsSameAsGiven(givenFr: string) { - const oldFeeRecipients = selectedRows.map((rowId) => rows[+rowId].feeRecipient).flat(); + const oldFeeRecipients = validators.map((row) => row.feeRecipient).flat(); return oldFeeRecipients.every((fr) => fr === givenFr); } function isRemovingMevSpFr() { - const oldFeeRecipients = selectedRows.map((rowId) => rows[+rowId].feeRecipient).flat(); + const oldFeeRecipients = validators.map((row) => row.feeRecipient).flat(); return ( mevSpAddress !== null && @@ -198,8 +194,7 @@ export default function FeeRecipientDialog({ // Given a possible withdrawal credentials format, checks if any of the selected validators has it const isAnyWithdrawalCredentialsEqual = (givenFormat: WithdrawalCredentialsFormat): boolean => { let isAnyAsGivenFormat = false; - for (const row of selectedRows) { - const rowData = rows[+row]; + for (const rowData of validators) { const withdrawalFormat = rowData.withdrawalCredentials.format; if (withdrawalFormat === givenFormat) { isAnyAsGivenFormat = true; @@ -212,8 +207,7 @@ export default function FeeRecipientDialog({ // Given a possible withdrawal credentials format, checks if any of the selected validators has a different format const isAnyWithdrawalCredentialsDiff = (givenFormat: WithdrawalCredentialsFormat): boolean => { let isAnyAsGivenFormat = false; - for (const row of selectedRows) { - const rowData = rows[+row]; + for (const rowData of validators) { const withdrawalFormat = rowData.withdrawalCredentials.format; if (withdrawalFormat !== givenFormat) { isAnyAsGivenFormat = true; @@ -225,9 +219,8 @@ export default function FeeRecipientDialog({ // Update the smoothValidatorsPubkeys state with the selected validators that have Smooth's fee recipient const getSmoothValidatorsSelected = (): void => { - const smoothValidatorsPubkeys: string[] = selectedRows - .map((row) => { - const rowData = rows[+row]; + const smoothValidatorsPubkeys: string[] = validators + .map((rowData) => { return rowData.feeRecipient === mevSpAddress ? rowData.pubkey : null; }) .filter((pubkey): pubkey is string => pubkey !== null); @@ -237,9 +230,8 @@ export default function FeeRecipientDialog({ // Update the nonEcdsaValidatorsData state with the selected validators that have a withdrawal address format different from ecdsa const getNonEcdsaValidatorsData = (): void => { - const filteredValidators = selectedRows - .map((row) => { - const rowData = rows[+row]; + const filteredValidators = validators + .map((rowData) => { return rowData.withdrawalCredentials.format !== "ecdsa" ? { pubkey: rowData.pubkey, @@ -620,7 +612,7 @@ export default function FeeRecipientDialog({ title="Fee recipient" subtitle="Choose where validator proposal fees are sent." icon={} - count={selectedRows.length} + count={validators.length} onClose={handleClose} busy={loading} /> diff --git a/packages/ui/src/components/Dialogs/ExitWarning.tsx b/packages/ui/src/components/Dialogs/ExitWarning.tsx index 8e1e9abc..d7e96645 100644 --- a/packages/ui/src/components/Dialogs/ExitWarning.tsx +++ b/packages/ui/src/components/Dialogs/ExitWarning.tsx @@ -1,22 +1,15 @@ import type { JSX } from "react"; import { Alert } from "@mui/material"; -import type { GridRowId } from "@mui/x-data-grid"; import { shortenPubkey } from "@stakingbrain/common"; import type { CustomValidatorGetResponse } from "@stakingbrain/brain"; -export default function ExitWarning({ - selectedRows, - rows -}: { - selectedRows: GridRowId[]; - rows: CustomValidatorGetResponse[]; -}): JSX.Element { +export default function ExitWarning({ validators }: { validators: CustomValidatorGetResponse[] }): JSX.Element { return ( <> Are you sure you want to exit these validators?
    - {selectedRows.map((row, i) => ( -
  • {shortenPubkey(rows[parseInt(row.toString())].pubkey)}
  • + {validators.map(({ pubkey }) => ( +
  • {shortenPubkey(pubkey)}
  • ))}
diff --git a/packages/ui/src/components/Dialogs/ImportDialog.tsx b/packages/ui/src/components/Dialogs/ImportDialog.tsx index dcc29bcb..06c553cd 100644 --- a/packages/ui/src/components/Dialogs/ImportDialog.tsx +++ b/packages/ui/src/components/Dialogs/ImportDialog.tsx @@ -1,12 +1,19 @@ import type { JSX } from "react"; +import { useMemo, useState } from "react"; import { Alert, Dialog, DialogContent, Box, Button, + Chip, DialogActions, Divider, + FormControlLabel, + LinearProgress, + Stack, + Switch, + Typography, useMediaQuery, useTheme } from "@mui/material"; @@ -18,30 +25,70 @@ import { SlideTransition } from "./Transitions"; import UploadFileIcon from "@mui/icons-material/UploadFile"; import DialogHeader from "./DialogHeader"; import ValidatorResult from "./ValidatorResult"; -import type { Web3signerPostResponse } from "@stakingbrain/brain"; +import { + ImportProgress, + ImportResult, + NOT_ATTEMPTED, + retryableIndexes, + summarizeImport +} from "../../utils/importBatches"; + +// Rendering hundreds of result cards at once makes the dialog sluggish; reveal more on demand. +const RESULTS_PAGE = 100; + +const summaryLabels: [status: string, label: string, color: "success" | "default" | "error"][] = [ + ["imported", "imported", "success"], + ["duplicate", "already present", "default"], + ["error", "failed", "error"], + [NOT_ATTEMPTED, "not attempted", "default"] +]; export default function ImportDialog({ open, setOpen, - keystoresPostResponse, - keystoresPostError, + results, + progress, + error, + stopped, + stopRequested, importStatus, - acceptedFiles + acceptedFiles, + onStop, + onRetry }: { open: boolean; setOpen: (open: boolean) => void; - keystoresPostResponse: Web3signerPostResponse | undefined; - keystoresPostError: string | undefined; + results: ImportResult[] | undefined; + progress: ImportProgress | undefined; + error: string | undefined; + stopped: "user" | "error" | undefined; + stopRequested: boolean; importStatus: string; acceptedFiles: KeystoreInfo[]; + onStop: () => void; + onRetry: (indexes: number[]) => void; }): JSX.Element { const navigate = useNavigate(); const isMobile = useMediaQuery(useTheme().breakpoints.down("sm")); + const importing = importStatus === ImportStatus.Importing; + const [onlyProblems, setOnlyProblems] = useState(true); + const [visible, setVisible] = useState(RESULTS_PAGE); const handleClose = () => { setOpen(false); navigate({ pathname: "/", search: window.location.search }); }; + const summary = useMemo(() => summarizeImport(results ?? []), [results]); + const retry = useMemo(() => retryableIndexes(results ?? []), [results]); + const problems = (summary.error ?? 0) + (summary[NOT_ATTEMPTED] ?? 0); + // While importing, untouched rows would only be noise; afterwards show problems first when there are any. + const filtering = (results?.length ?? 0) > 10 && onlyProblems && (problems > 0 || importing); + const shown = (results ?? []) + .map((result, index) => ({ result, index })) + .filter(({ result }) => + filtering ? (importing ? result.status === "error" : !["imported", "duplicate"].includes(result.status)) : true + ); + return ( { - if (!reason) { - handleClose(); - } + if (!reason && !importing) handleClose(); }} aria-labelledby="alert-dialog-title" aria-describedby="alert-dialog-description" TransitionComponent={SlideTransition} > } count={acceptedFiles.length} onClose={handleClose} - busy={importStatus === ImportStatus.Importing} + busy={importing} /> - {keystoresPostError ? ( - {keystoresPostError} - ) : keystoresPostResponse ? ( -
- {keystoresPostResponse.data.map((result, index) => ( - - ))} -
- ) : ( + {error ? ( + {error} + ) : !results || !progress ? ( + ) : ( + + + + + {progress.done} of {progress.total} validators processed + + {progress.batches > 1 && ( + + Batch {Math.min(progress.batch + (importing ? 1 : 0), progress.batches)} of {progress.batches} + + )} + + + {importing && ( + + Decrypting keystores takes time. Keep this window open; you can stop after the current batch. + + )} + + + + {summaryLabels + .filter(([status]) => summary[status]) + .map(([status, label, color]) => ( + + ))} + + + {stopped === "error" && ( + + The import stopped because a batch failed. Validators not yet imported can be retried; keys that did + reach the signer will show as already present. + + )} + {stopped === "user" && ( + Import stopped. You can retry the validators that were not imported. + )} + + {results.length > 10 && (problems > 0 || importing) && ( + setOnlyProblems(checked)} />} + label="Show only problems" + /> + )} + +
+ {shown.slice(0, visible).map(({ result, index }) => ( + + ))} + {shown.length > visible && ( + + )} +
+
)}
- + ) : ( + retry.length > 0 && + !error && ( + + ) + )} + diff --git a/packages/ui/src/components/Dialogs/KeystoresDeleteDialog.tsx b/packages/ui/src/components/Dialogs/KeystoresDeleteDialog.tsx index e23415d4..81b0345d 100644 --- a/packages/ui/src/components/Dialogs/KeystoresDeleteDialog.tsx +++ b/packages/ui/src/components/Dialogs/KeystoresDeleteDialog.tsx @@ -14,14 +14,13 @@ import type { CustomValidatorGetResponse, Web3signerDeleteResponse } from "@stak import { useTheme, useMediaQuery } from "@mui/material"; export default function KeystoresDeleteDialog({ - rows, - selectedRows, + validators, setSelectedRows, open, setOpen }: { - rows: CustomValidatorGetResponse[]; - selectedRows: GridRowId[]; + /** The selected validators, resolved by pubkey. */ + validators: CustomValidatorGetResponse[]; setSelectedRows: (selectedRows: GridRowId[]) => void; open: boolean; setOpen: (open: boolean) => void; @@ -37,7 +36,7 @@ export default function KeystoresDeleteDialog({ try { setKeystoresDelete(undefined); setLoading(true); - const pubkeysToDelete = selectedRows.map((row) => rows[parseInt(row.toString())]?.pubkey); + const pubkeysToDelete = validators.map(({ pubkey }) => pubkey); setDeletedPubkeys(pubkeysToDelete); setKeystoresDelete( await rpcClient.call("deleteValidators", { @@ -76,7 +75,7 @@ export default function KeystoresDeleteDialog({ title={keystoresDelete ? "Deletion results" : "Delete keystores"} subtitle="Remove validator keys from this signer." icon={} - count={keystoresDelete ? deletedPubkeys.length : selectedRows.length} + count={keystoresDelete ? deletedPubkeys.length : validators.length} onClose={handleClose} busy={loading} /> @@ -121,7 +120,7 @@ export default function KeystoresDeleteDialog({ ) : ( - + )} diff --git a/packages/ui/src/components/Dialogs/KeystoresExitDialog.tsx b/packages/ui/src/components/Dialogs/KeystoresExitDialog.tsx index b88c4cac..7f69c28e 100644 --- a/packages/ui/src/components/Dialogs/KeystoresExitDialog.tsx +++ b/packages/ui/src/components/Dialogs/KeystoresExitDialog.tsx @@ -24,14 +24,13 @@ import type { CustomValidatorGetResponse, ValidatorExitExecute } from "@stakingb import { useTheme, useMediaQuery } from "@mui/material"; export default function KeystoresExitDialog({ - rows, - selectedRows, + validators, setSelectedRows, open, setOpen }: { - rows: CustomValidatorGetResponse[]; - selectedRows: GridRowId[]; + /** The selected validators, resolved by pubkey. */ + validators: CustomValidatorGetResponse[]; setSelectedRows: (selectedRows: GridRowId[]) => void; open: boolean; setOpen: (open: boolean) => void; @@ -49,7 +48,7 @@ export default function KeystoresExitDialog({ try { setLoading(true); const exitKeysores = await rpcClient.call("getExitValidators", { - pubkeys: selectedRows.map((row) => rows[parseInt(row.toString())].pubkey) + pubkeys: validators.map(({ pubkey }) => pubkey) }); exitKeysores.forEach((validator) => { @@ -75,7 +74,7 @@ export default function KeystoresExitDialog({ setLoading(true); setValidatorsExitResponse( await rpcClient.call("exitValidators", { - pubkeys: selectedRows.map((row) => rows[parseInt(row.toString())].pubkey) + pubkeys: validators.map(({ pubkey }) => pubkey) }) ); setLoading(false); @@ -109,7 +108,7 @@ export default function KeystoresExitDialog({ title={validatorsExitResponse ? "Exit results" : "Exit validators"} subtitle="Review and confirm your validator exits." icon={} - count={validatorsExitResponse?.length ?? selectedRows.length} + count={validatorsExitResponse?.length ?? validators.length} onClose={handleClose} busy={loading} /> @@ -135,7 +134,7 @@ export default function KeystoresExitDialog({ ) : ( <> - +
diff --git a/packages/ui/src/components/Dialogs/ValidatorResult.tsx b/packages/ui/src/components/Dialogs/ValidatorResult.tsx index c4515541..b81962d5 100644 --- a/packages/ui/src/components/Dialogs/ValidatorResult.tsx +++ b/packages/ui/src/components/Dialogs/ValidatorResult.tsx @@ -21,7 +21,7 @@ export default function ValidatorResult({ {shortenPubkey(pubkey)} - + {message && ( diff --git a/packages/ui/src/components/FileCards/FileCardList.tsx b/packages/ui/src/components/FileCards/FileCardList.tsx index 2c1af428..ba7c3cf6 100644 --- a/packages/ui/src/components/FileCards/FileCardList.tsx +++ b/packages/ui/src/components/FileCards/FileCardList.tsx @@ -18,7 +18,9 @@ export default function FileCardList( useSameFeeRecipient: boolean, getFeeRecipientFieldHelperText: (index: number) => string, isFeeRecipientFieldWrong: (index: number) => boolean, - tagSelectOptions: TagSelectOption[] + tagSelectOptions: TagSelectOption[], + // Only the first `limit` cards render; indexes stay aligned because the slice starts at 0. + limit = Infinity ): JSX.Element[] { const removeFileFromList = ( fileInfo: KeystoreInfo, @@ -38,115 +40,119 @@ export default function FileCardList( setTags(tags.filter((f, index) => index !== indexToRemove)); }; - return Array.from(fileInfos).map((fileInfo, index) => ( - - - - - {fileInfo.file.name} - - ( + + + + + {fileInfo.file.name} + + + + + aria-label={`Remove ${fileInfo.file.name}`} + onClick={() => + removeFileFromList( + fileInfo, + fileInfos, + setAcceptedFiles, + passwords, + setPasswords, + feeRecipients, + setFeeRecipients, + tags, + setTags + ) + } + > + + - - removeFileFromList( - fileInfo, - fileInfos, - setAcceptedFiles, - passwords, - setPasswords, - feeRecipients, - setFeeRecipients, - tags, - setTags - ) - } - > - - - - {(!useSameTag || !useSameFeeRecipient || !useSamePassword) && ( - - {!useSamePassword && ( - { - const newPasswords = [...passwords]; - newPasswords[index] = event.target.value; - setPasswords(newPasswords); - }} - helperText="Password used to create this keystore." - /> - )} - {!useSameTag && ( - <> + {(!useSameTag || !useSameFeeRecipient || !useSamePassword) && ( + + {!useSamePassword && ( { - const newTags = [...tags]; - newTags[index] = event.target.value as Tag; - setTags(newTags); + const newPasswords = [...passwords]; + newPasswords[index] = event.target.value; + setPasswords(newPasswords); + }} + helperText="Password used to create this keystore." + /> + )} + {!useSameTag && ( + <> + { + const newTags = [...tags]; + newTags[index] = event.target.value as Tag; + setTags(newTags); - if (!useSameFeeRecipient && !isFeeRecipientEditable(event.target.value as Tag)) { - const newFeeRecipients = [...feeRecipients]; - newFeeRecipients[index] = ""; - setFeeRecipients(newFeeRecipients); - } + if (!useSameFeeRecipient && !isFeeRecipientEditable(event.target.value as Tag)) { + const newFeeRecipients = [...feeRecipients]; + newFeeRecipients[index] = ""; + setFeeRecipients(newFeeRecipients); + } + }} + > + {tagSelectOptions.map((option) => ( + + {option.label} + + ))} + + + )} + {!useSameFeeRecipient && ( + { + const newFeeRecipients = [...feeRecipients]; + newFeeRecipients[index] = event.target.value; + setFeeRecipients(newFeeRecipients); }} - > - {tagSelectOptions.map((option) => ( - - {option.label} - - ))} - - - )} - {!useSameFeeRecipient && ( - { - const newFeeRecipients = [...feeRecipients]; - newFeeRecipients[index] = event.target.value; - setFeeRecipients(newFeeRecipients); - }} - error={isFeeRecipientFieldWrong(index)} - helperText={getFeeRecipientFieldHelperText(index)} - disabled={!!tags[index] && !isFeeRecipientEditable(tags[index])} - /> - )} - - )} - - )); + error={isFeeRecipientFieldWrong(index)} + helperText={getFeeRecipientFieldHelperText(index)} + disabled={!!tags[index] && !isFeeRecipientEditable(tags[index])} + /> + )} + + )} + + )); } diff --git a/packages/ui/src/components/ValidatorList/KeystoresDataGrid.tsx b/packages/ui/src/components/ValidatorList/KeystoresDataGrid.tsx index 69916950..ea99857b 100644 --- a/packages/ui/src/components/ValidatorList/KeystoresDataGrid.tsx +++ b/packages/ui/src/components/ValidatorList/KeystoresDataGrid.tsx @@ -6,7 +6,8 @@ import { MAINNET_ORACLE_URL, TESTNET_ORACLE_URL, nonEditableFeeRecipientTags, - NonEditableFeeRecipientTag + NonEditableFeeRecipientTag, + featureFlags } from "@stakingbrain/common"; import SmoothStatus from "./SmoothStatus"; import { GridColDef } from "@mui/x-data-grid"; @@ -61,10 +62,6 @@ export default function KeystoresDataGrid({ const [paginationModel, setPaginationModel] = useState({ page: 0, pageSize: 10 }); const blockExplorerUrl = blockExplorerUrls.get(network); - useEffect(() => { - setSummaryUrlBuildingStatus(BeaconchaUrlBuildingStatus.NotStarted); - }, [selectedRows, network, rows]); - const [validatorsSubscriptionMap, setValidatorsSubscriptionMap] = useState(null); const [oracleCallError, setOracleCallError] = useState(); @@ -151,14 +148,25 @@ export default function KeystoresDataGrid({ setValidatorsSubscriptionMap(newValidatorSubscriptionStatus); setOracleCallError(undefined); } catch (e) { + // Record the attempt (all UNKNOWN) so live refresh rounds do not re-query an oracle that is down. + setValidatorsSubscriptionMap(newValidatorSubscriptionStatus); setOracleCallError("Error fetching subscription status. Oracle might be down: " + e.message); } }; + // The list refreshes live and in rounds; only query the external oracle when what it is asked + // about changes, and not on every round while details are still loading. + const oracleQueryKey = rows.map((row) => `${row.pubkey}:${row.tag}:${row.index}`).join(","); + const detailsLoading = rows.some((row) => row.detailsPending); + const selectedIds = new Set(selectedRows); + const selectedValidators = rows.filter((row) => selectedIds.has(row.pubkey)); + // Live refreshes replace `rows` often; only reset the dashboard hint when what it depends on changes. useEffect(() => { - if (rows.length > 0) { - fetchValidatorsData(rows); - } - }, [rows]); + setSummaryUrlBuildingStatus(BeaconchaUrlBuildingStatus.NotStarted); + }, [selectedRows, network, oracleQueryKey]); + useEffect(() => { + if (rows.length === 0 || (detailsLoading && validatorsSubscriptionMap !== null)) return; + fetchValidatorsData(rows); + }, [oracleQueryKey, detailsLoading, network]); const columns: GridColDef[] = [ { @@ -249,7 +257,7 @@ export default function KeystoresDataGrid({ } ]; - const customRows = rows.map((row, index) => ({ + const customRows = rows.map((row) => ({ pubkey: row.pubkey, explorerUrl: blockExplorerUrl ? blockExplorerUrl + "/validator/" + row.pubkey : "", feeRecipient: row.feeRecipient, @@ -261,27 +269,28 @@ export default function KeystoresDataGrid({ feeRecipientImported: row.validatorFeeRecipientCorrect, builderConfig: row.builderConfig, builderConfigStatus: row.builderConfigStatus, - id: index + // Pubkeys, not positions: live refreshes reorder and remove rows under a selection. + id: row.pubkey })); + const builderConfigColumn: GridColDef = { + field: "builderConfigStatus", + headerName: "Builder config", + description: "Last application attempt; open Builder config to inspect client values and refresh status", + width: 180, + sortable: false, + renderCell: ({ row }) => ( + + + {capitalize(row.builderConfigStatus?.status ?? (row.builderConfig === undefined ? "Unmanaged" : "Pending"))} + + + ) + }; + if (userMode === "advanced") columns.push( - { - field: "builderConfigStatus", - headerName: "Builder config", - description: "Last application attempt; open Builder config to inspect client values and refresh status", - width: 180, - sortable: false, - renderCell: ({ row }) => ( - - - {capitalize( - row.builderConfigStatus?.status ?? (row.builderConfig === undefined ? "Unmanaged" : "Pending") - )} - - - ) - }, + ...(featureFlags.builderConfig ? [builderConfigColumn] : []), { field: "explorerUrl", headerName: "URL", @@ -412,7 +421,7 @@ export default function KeystoresDataGrid({ } const summaryUrl = buildValidatorSummaryURL({ - validatorIndexes: selectedRows.map((row) => rows[row as number]?.index ?? ""), + validatorIndexes: selectedValidators.map((row) => row.index), network }); @@ -426,12 +435,12 @@ export default function KeystoresDataGrid({ } function areAllSelectedRowsExitable() { - return selectedRows.every((row) => rows[row as number].tag !== "rocketpool"); + return selectedValidators.every((row) => row.tag !== "rocketpool"); } function areAllSelectedRowsUneditable() { - return selectedRows.every((row) => { - const tag = rows[row as number].tag; + return selectedValidators.every((row) => { + const tag = row.tag; return nonEditableFeeRecipientTags.includes(tag as NonEditableFeeRecipientTag); }); } @@ -483,14 +492,16 @@ export default function KeystoresDataGrid({ - + {featureFlags.builderConfig && ( + + )} setDeleteOpen(true)}> diff --git a/packages/ui/src/components/ValidatorList/ValidatorList.tsx b/packages/ui/src/components/ValidatorList/ValidatorList.tsx index 1baa83f8..f9a86a84 100644 --- a/packages/ui/src/components/ValidatorList/ValidatorList.tsx +++ b/packages/ui/src/components/ValidatorList/ValidatorList.tsx @@ -3,10 +3,10 @@ import KeystoresDataGrid from "./KeystoresDataGrid"; import KeystoresDeleteDialog from "../Dialogs/KeystoresDeleteDialog"; import EditFeesDialog from "../Dialogs/EditFeesDialog"; import BuilderConfigDialog from "../Dialogs/BuilderConfigDialog"; -import { Alert, Box, CircularProgress } from "@mui/material"; +import { Alert, Box, Button, CircularProgress, LinearProgress, Typography } from "@mui/material"; import type { GridRowId } from "@mui/x-data-grid"; -import { StakerConfig as StakerConfigType } from "@stakingbrain/common"; -import { useEffect, useState } from "react"; +import { featureFlags, StakerConfig as StakerConfigType, VALIDATORS_CHANGED_EVENT } from "@stakingbrain/common"; +import { useEffect, useMemo, useRef, useState } from "react"; import { BeaconchaUrlBuildingStatus } from "../../types"; import { rpcClient } from "../../socket"; import StakerConfig from "../StakerConfig/StakerConfig"; @@ -14,6 +14,7 @@ import KeystoresExitDialog from "../Dialogs/KeystoresExitDialog"; import { getSmoothAddressByNetwork } from "../../utils/addresses"; import type { CustomValidatorGetResponse } from "@stakingbrain/brain"; import PageLayout from "../Layout/PageLayout"; +import { countUnfinished, FOLLOW_UP_MS, MAX_STALLED_FOLLOW_UPS, nextStallCount } from "../../utils/listFollowUp"; export default function ValidatorList({ stakerConfig, @@ -34,27 +35,103 @@ export default function ValidatorList({ const { network, isMevBoostSet } = stakerConfig; const smoothAddress = getSmoothAddressByNetwork(network); - // Re-render table after delete/update validators + const anyDialogOpen = deleteOpen || editFeesOpen || exitOpen || builderConfigOpen; + const dialogOpenRef = useRef(anyDialogOpen); + dialogOpenRef.current = anyDialogOpen; + const loaded = useRef(false); + const inFlight = useRef(false); + const queued = useRef(false); + const [refreshError, setRefreshError] = useState(); + const stalls = useRef(0); + const lastUnfinished = useRef(undefined); + const [stalled, setStalled] = useState(false); + + // Refresh when a dialog closes (its action may have changed validators) and on first load. useEffect(() => { - if (!deleteOpen && !editFeesOpen && !exitOpen && !builderConfigOpen) { - getValidators(); + if (!anyDialogOpen) { + // Only the first load blanks the page; later refreshes keep the table visible. + void getValidators(loaded.current); setSelectedRows([]); } - }, [deleteOpen, editFeesOpen, exitOpen, builderConfigOpen]); + }, [anyDialogOpen]); + + // Live updates: the Brain pushes a hint whenever a validator or its client status changes, including + // when pending checks complete, so the list never needs to poll. Refreshes wait while a dialog is + // open so its content does not shift; closing a dialog always refreshes. + useEffect(() => { + const refresh = () => { + if (!dialogOpenRef.current) void getValidators(true); + }; + const offChanged = rpcClient.on(VALIDATORS_CHANGED_EVENT, refresh); + // Catch up on hints missed while disconnected. + const offConnect = rpcClient.on("connect", refresh); + return () => { + offChanged(); + offConnect(); + }; + }, []); + + // Client reads cut off by the Brain's deadline only continue when the list is requested again, so keep + // requesting (one request at a time) while that makes progress. A failed request does not poll: it + // shows an error with Retry. Beacon details arrive by push and need no follow-up. + useEffect(() => { + if (anyDialogOpen || stalled || refreshError || !validatorsGet || countUnfinished(validatorsGet) === 0) return; + const timer = setTimeout(() => void getValidators(true), FOLLOW_UP_MS); + return () => clearTimeout(timer); + }, [validatorsGet, anyDialogOpen, stalled, refreshError]); - async function getValidators() { + function retry() { + stalls.current = 0; + lastUnfinished.current = undefined; + setStalled(false); + setRefreshError(undefined); + void getValidators(true); + } + + async function getValidators(background = false) { + // Coalesce overlapping triggers into one follow-up refresh. + if (inFlight.current) { + queued.current = true; + return; + } + inFlight.current = true; try { - setLoading(true); - setValidatorsGet(await rpcClient.call("getValidators", undefined)); + if (!background) setLoading(true); + const rows = await rpcClient.call("getValidators", undefined); + loaded.current = true; + // Rows are identified by pubkey, so a refresh only has to drop validators that are gone. + const present = new Set(rows.map(({ pubkey }) => pubkey)); + setSelectedRows((selected) => selected.filter((id) => present.has(String(id)))); + const unfinished = countUnfinished(rows); + stalls.current = nextStallCount(stalls.current, lastUnfinished.current, unfinished); + lastUnfinished.current = unfinished; + setStalled(stalls.current >= MAX_STALLED_FOLLOW_UPS); + setValidatorsGet(rows); setValidatorsGetError(undefined); - setLoading(false); + setRefreshError(undefined); } catch (e) { console.error(e); - setValidatorsGetError(e.message); - setLoading(false); + // A failed background refresh keeps the last good table and offers a retry. + if (!background || !loaded.current) setValidatorsGetError(e.message); + else setRefreshError(e.message); + } finally { + if (!background) setLoading(false); + inFlight.current = false; + if (queued.current) { + queued.current = false; + if (!dialogOpenRef.current) void getValidators(true); + } } } + const selectedValidators = useMemo(() => { + const selected = new Set(selectedRows.map(String)); + return (validatorsGet ?? []).filter(({ pubkey }) => selected.has(pubkey)); + }, [validatorsGet, selectedRows]); + const pendingCount = + validatorsGet?.filter(({ detailsPending, beaconPending }) => detailsPending || beaconPending).length ?? 0; + const detailsErrors = [...new Set((validatorsGet ?? []).flatMap(({ detailsError }) => detailsError ?? []))]; + const SmoothBanner = () => { return ( @@ -102,6 +179,50 @@ export default function ValidatorList({ ) : validatorsGet ? ( <> + {refreshError && ( + + Retry + + } + > + The validator list could not be refreshed: {refreshError} + + )} + {detailsErrors.length > 0 && ( + + Some validator details are unavailable: {detailsErrors.join("; ")} + + )} + {stalled && ( + + Retry + + } + > + Details for {validatorsGet && countUnfinished(validatorsGet)} validators are loading slowly. Your + validator client or signer may be busy. + + )} + {pendingCount > 0 && ( + + + Checking {pendingCount} of {validatorsGet.length} validators with your clients… + + + + )} )} - {builderConfigOpen && ( - validatorsGet[row as number])} - setOpen={setBuilderConfigOpen} - /> + {featureFlags.builderConfig && builderConfigOpen && ( + )} {exitOpen && ( void): () => void { + this.socket.on(event, handler); + return () => { + this.socket.off(event, handler); + }; + } + async call(method: T, params: RoutesArguments[T]): Promise { return new Promise((resolve, reject) => { const rpcRequest = { diff --git a/packages/ui/src/utils/importBatches.ts b/packages/ui/src/utils/importBatches.ts new file mode 100644 index 00000000..a05c287b --- /dev/null +++ b/packages/ui/src/utils/importBatches.ts @@ -0,0 +1,219 @@ +import type { CustomImportRequest, Web3signerPostResponse } from "@stakingbrain/brain"; +import { prefix0xPubkey, RPC_MAX_PAYLOAD_BYTES } from "@stakingbrain/common"; + +/** + * Keystore decryption in the signer is slow, so one RPC call per batch keeps every call well inside + * the RPC timeout, lets the UI report progress and lets the user stop between batches. + */ +export const IMPORT_BATCH_SIZE = 20; +// Sizes below are exact encoded bytes; keep a margin under the transport limit for RPC framing. +const IMPORT_BATCH_MAX_BYTES = Math.floor(RPC_MAX_PAYLOAD_BYTES * 0.9); +const encoder = new TextEncoder(); +const byteLength = (text: string) => encoder.encode(text).byteLength; + +export interface ImportResult { + status: string; + message?: string; +} + +export interface ImportBatch { + /** Positions in the original file list, so results stay aligned with the files the user picked. */ + indexes: number[]; + request: CustomImportRequest; + /** Set when a single key cannot fit one request; it is reported as an error without being sent. */ + tooLarge?: { bytes: number; maxBytes: number }; +} + +export interface ImportProgress { + done: number; + total: number; + batch: number; + batches: number; +} + +export const NOT_ATTEMPTED = "not_attempted"; + +export function notAttempted(): ImportResult { + return { status: NOT_ATTEMPTED, message: "Not imported yet: the import stopped before reaching this validator." }; +} + +type Interchange = { data: { pubkey?: unknown }[] }; + +function parseInterchange(interchange: string): Interchange | undefined { + try { + const parsed: unknown = JSON.parse(interchange); + if (parsed && typeof parsed === "object" && Array.isArray((parsed as { data?: unknown }).data)) + return parsed as Interchange; + } catch { + // Fall through: let the signer report the malformed file, as before. + } + return undefined; +} + +const entryPubkey = (entry: { pubkey?: unknown }) => + typeof entry?.pubkey === "string" ? prefix0xPubkey(entry.pubkey).toLowerCase() : undefined; + +/** Keep only the slashing history of the batch's keys; the signer only needs those. */ +export function filterInterchange(interchange: string, pubkeys: ReadonlySet): string { + const record = parseInterchange(interchange); + if (!record) return interchange; + return JSON.stringify({ + ...record, + data: record.data.filter((entry) => pubkeys.has(entryPubkey(entry) ?? "")) + }); +} + +/** + * Bytes each part adds to the serialized RPC request. `slashing_protection` travels as a JSON string + * inside JSON, so its entries are measured after a second round of escaping. + */ +function measureImport(request: CustomImportRequest, pubkeys: readonly string[]) { + const record = request.slashing_protection === undefined ? undefined : parseInterchange(request.slashing_protection); + const slashingBytes = new Map(); + for (const entry of record?.data ?? []) { + const key = entryPubkey(entry); + if (key) + slashingBytes.set(key, (slashingBytes.get(key) ?? 0) + byteLength(JSON.stringify(JSON.stringify(entry))) - 1); + } + const emptyRequest: CustomImportRequest = { + importFrom: request.importFrom, + slashing_protection: + request.slashing_protection === undefined + ? undefined + : record + ? JSON.stringify({ ...record, data: [] }) + : request.slashing_protection, + validatorsImportRequest: [] + }; + const base = byteLength(JSON.stringify({ jsonrpc: "2.0", method: "importValidators", params: emptyRequest, id: 0 })); + const perKey = (index: number) => + byteLength(JSON.stringify(request.validatorsImportRequest[index])) + + 1 + + (pubkeys[index] ? (slashingBytes.get(prefix0xPubkey(pubkeys[index]).toLowerCase()) ?? 0) : 0); + return { base, perKey }; +} + +/** Split an import into ordered batches by count and size. `only` limits a retry to some files. */ +export function planImportBatches( + request: CustomImportRequest, + pubkeys: readonly string[], + { + only, + maxKeystores = IMPORT_BATCH_SIZE, + maxBytes = IMPORT_BATCH_MAX_BYTES + }: { only?: readonly number[]; maxKeystores?: number; maxBytes?: number } = {} +): ImportBatch[] { + const selected = only ?? request.validatorsImportRequest.map((_, index) => index); + const { base, perKey } = measureImport(request, pubkeys); + const groups: number[][] = []; + const oversized = new Map(); + let group: number[] = []; + let bytes = base; + for (const index of selected) { + const size = perKey(index); + if (base + size > maxBytes) { + oversized.set(index, base + size); + continue; + } + if (group.length > 0 && (group.length >= maxKeystores || bytes + size > maxBytes)) { + groups.push(group); + group = []; + bytes = base; + } + group.push(index); + bytes += size; + } + if (group.length > 0) groups.push(group); + + const tooLarge: ImportBatch[] = [...oversized].map(([index, size]) => ({ + indexes: [index], + // Never sent: the run reports it as an error instead. + request: { importFrom: request.importFrom, validatorsImportRequest: [] }, + tooLarge: { bytes: size, maxBytes } + })); + return [ + ...tooLarge, + ...groups.map((indexes) => ({ + indexes, + request: { + importFrom: request.importFrom, + slashing_protection: + request.slashing_protection === undefined + ? undefined + : filterInterchange( + request.slashing_protection, + new Set(indexes.map((index) => prefix0xPubkey(pubkeys[index]).toLowerCase())) + ), + validatorsImportRequest: indexes.map((index) => request.validatorsImportRequest[index]) + } + })) + ]; +} + +/** + * Import batches in order. A failed batch stops the run: its error may be a timeout whose outcome is + * unknown, and the remaining files are left "not attempted" so the user can retry them safely + * (keys that did land are reported as duplicates). + */ +export async function runImportBatches({ + batches, + results: initial, + importBatch, + onProgress, + shouldStop +}: { + batches: readonly ImportBatch[]; + results: readonly ImportResult[]; + importBatch: (request: CustomImportRequest) => Promise; + onProgress: (progress: ImportProgress, results: ImportResult[]) => void; + shouldStop: () => boolean; +}): Promise<{ results: ImportResult[]; stopped?: "user" | "error" }> { + const results = [...initial]; + for (const { indexes } of batches) for (const index of indexes) results[index] = notAttempted(); + const total = batches.reduce((sum, { indexes }) => sum + indexes.length, 0); + let done = 0; + onProgress({ done, total, batch: 0, batches: batches.length }, [...results]); + + for (const [position, batch] of batches.entries()) { + if (shouldStop()) return { results, stopped: "user" }; + if (batch.tooLarge) { + const mb = (bytes: number) => (bytes / 1024 / 1024).toFixed(1); + results[batch.indexes[0]] = { + status: "error", + message: `This keystore and its slashing protection history are too large to import in one request (${mb(batch.tooLarge.bytes)} MB, limit ${mb(batch.tooLarge.maxBytes)} MB). Import it with a pruned slashing protection file.` + }; + done += 1; + onProgress({ done, total, batch: position + 1, batches: batches.length }, [...results]); + continue; + } + let failed = false; + try { + const response = await importBatch(batch.request); + batch.indexes.forEach((index, offset) => { + results[index] = response.data[offset] ?? { + status: "error", + message: "Import outcome unknown: Brain returned no result for this key. Retry to check." + }; + }); + } catch (error) { + failed = true; + const message = error instanceof Error ? error.message : String(error); + for (const index of batch.indexes) results[index] = { status: "error", message }; + } + done += batch.indexes.length; + onProgress({ done, total, batch: position + 1, batches: batches.length }, [...results]); + if (failed) return { results, stopped: "error" }; + } + return { results }; +} + +/** Files worth retrying: anything not confirmed imported or already present. */ +export function retryableIndexes(results: readonly ImportResult[]): number[] { + return results.flatMap((result, index) => (["imported", "duplicate"].includes(result?.status) ? [] : [index])); +} + +export function summarizeImport(results: readonly ImportResult[]): Record { + const summary: Record = {}; + for (const { status } of results) summary[status] = (summary[status] ?? 0) + 1; + return summary; +} diff --git a/packages/ui/src/utils/importForm.ts b/packages/ui/src/utils/importForm.ts index 6390db6a..d262c4b5 100644 --- a/packages/ui/src/utils/importForm.ts +++ b/packages/ui/src/utils/importForm.ts @@ -32,18 +32,22 @@ export function appendImportFiles( }; } -export function isImportFormValid({ acceptedFiles, passwords, tags, feeRecipients }: ImportFormState): boolean { +function isImportEntryValid({ passwords, tags, feeRecipients }: ImportFormState, index: number): boolean { + const tag = tags[index]; return ( - acceptedFiles.length > 0 && - acceptedFiles.every((_, index) => { - const tag = tags[index]; - return ( - typeof passwords[index] === "string" && - passwords[index].length > 0 && - !!tag && - (!isFeeRecipientEditable(tag) || - (isValidEcdsaPubkey(feeRecipients[index]) && feeRecipients[index] !== BURN_ADDRESS)) - ); - }) + typeof passwords[index] === "string" && + passwords[index].length > 0 && + !!tag && + (!isFeeRecipientEditable(tag) || + (isValidEcdsaPubkey(feeRecipients[index]) && feeRecipients[index] !== BURN_ADDRESS)) ); } + +/** Positions of files still missing a password, protocol or valid fee recipient. */ +export function incompleteImportIndexes(state: ImportFormState): number[] { + return state.acceptedFiles.flatMap((_, index) => (isImportEntryValid(state, index) ? [] : [index])); +} + +export function isImportFormValid(state: ImportFormState): boolean { + return state.acceptedFiles.length > 0 && incompleteImportIndexes(state).length === 0; +} diff --git a/packages/ui/src/utils/listFollowUp.ts b/packages/ui/src/utils/listFollowUp.ts new file mode 100644 index 00000000..f6e71938 --- /dev/null +++ b/packages/ui/src/utils/listFollowUp.ts @@ -0,0 +1,15 @@ +/** Delay before requesting the list again while client reads were cut off by the Brain's deadline. */ +export const FOLLOW_UP_MS = 3000; +/** Stop requesting after this many follow-ups in a row that did not reduce the pending rows. */ +export const MAX_STALLED_FOLLOW_UPS = 3; + +/** Rows whose client reads were cut off: requesting the list again continues them. */ +export function countUnfinished(rows: readonly { detailsPending: boolean }[]): number { + return rows.filter(({ detailsPending }) => detailsPending).length; +} + +/** Consecutive follow-ups that made no progress; resets whenever pending rows decrease or clear. */ +export function nextStallCount(stalls: number, previousPending: number | undefined, pending: number): number { + if (pending === 0) return 0; + return previousPending !== undefined && pending >= previousPending ? stalls + 1 : 0; +} diff --git a/packages/ui/test/importBatches.test.ts b/packages/ui/test/importBatches.test.ts new file mode 100644 index 00000000..a945cec1 --- /dev/null +++ b/packages/ui/test/importBatches.test.ts @@ -0,0 +1,198 @@ +import { expect } from "chai"; +import { RPC_MAX_PAYLOAD_BYTES } from "@stakingbrain/common"; +import type { CustomImportRequest, Web3signerPostResponse } from "@stakingbrain/brain"; +import { + filterInterchange, + ImportProgress, + notAttempted, + NOT_ATTEMPTED, + planImportBatches, + retryableIndexes, + runImportBatches, + summarizeImport +} from "../src/utils/importBatches.js"; + +const pubkeys = Array.from({ length: 45 }, (_, index) => "0x" + (index + 1).toString(16).padStart(96, "0")); +const interchange = JSON.stringify({ + metadata: { interchange_format_version: "5", genesis_validators_root: "0x" + "00".repeat(32) }, + data: pubkeys.map((pubkey) => ({ pubkey, signed_blocks: [], signed_attestations: [] })) +}); +const request = (slashing?: string): CustomImportRequest => ({ + importFrom: "ui", + slashing_protection: slashing, + validatorsImportRequest: pubkeys.map((pubkey) => ({ + keystore: JSON.stringify({ pubkey: pubkey.slice(2) }), + password: "password", + tag: "solo", + feeRecipient: "0x" + "ab".repeat(20) + })) +}); +const imported = (request: CustomImportRequest): Web3signerPostResponse => ({ + data: request.validatorsImportRequest.map(() => ({ status: "imported", message: "" })) +}); + +describe("Batched validator import", () => { + it("splits by count, keeps file order and sends each batch only its own slashing history", () => { + const batches = planImportBatches(request(interchange), pubkeys); + expect(batches.map(({ indexes }) => indexes.length)).to.deep.equal([20, 20, 5]); + expect(batches.flatMap(({ indexes }) => indexes)).to.deep.equal(pubkeys.map((_, index) => index)); + const second = JSON.parse(batches[1].request.slashing_protection!); + expect(second.metadata.interchange_format_version).to.equal("5"); + expect(second.data.map(({ pubkey }: { pubkey: string }) => pubkey)).to.deep.equal(pubkeys.slice(20, 40)); + }); + + it("splits by size so a batch stays under the transport limit", () => { + const batches = planImportBatches(request(), pubkeys, { maxBytes: 1000 }); + expect(batches.every(({ indexes }) => indexes.length < 20)).to.equal(true); + expect(batches.flatMap(({ indexes }) => indexes)).to.have.length(45); + }); + + it("counts each key's slashing history and JSON encoding toward the batch size", () => { + // 4,000 attestations per key: twenty of these do not fit one RPC payload. + const attestations = Array.from({ length: 4000 }, (_, epoch) => ({ + source_epoch: String(epoch), + target_epoch: String(epoch + 1), + signing_root: "0x" + "ab".repeat(32) + })); + const heavy = JSON.stringify({ + metadata: { interchange_format_version: "5", genesis_validators_root: "0x" + "00".repeat(32) }, + data: pubkeys.map((pubkey) => ({ pubkey, signed_blocks: [], signed_attestations: attestations })) + }); + const batches = planImportBatches(request(heavy), pubkeys); + expect(batches.length).to.be.greaterThan(Math.ceil(pubkeys.length / 20)); + expect(batches.flatMap(({ indexes }) => indexes)).to.deep.equal(pubkeys.map((_, index) => index)); + for (const { request: batch } of batches) { + // The same framing the RPC client measures before sending. + const rpcBytes = new TextEncoder().encode( + JSON.stringify({ jsonrpc: "2.0", method: "importValidators", params: batch, id: Date.now() }) + ).byteLength; + expect(rpcBytes + 64).to.be.at.most(RPC_MAX_PAYLOAD_BYTES); + } + }); + + it("reports a key too large for any request and imports the rest", async () => { + const huge = Array.from({ length: 60000 }, (_, epoch) => ({ + source_epoch: String(epoch), + target_epoch: String(epoch + 1), + signing_root: "0x" + "cd".repeat(32) + })); + const keys = pubkeys.slice(0, 3); + const slashing = JSON.stringify({ + metadata: { interchange_format_version: "5", genesis_validators_root: "0x" + "00".repeat(32) }, + data: keys.map((pubkey, index) => ({ + pubkey, + signed_blocks: [], + signed_attestations: index === 1 ? huge : [] + })) + }); + const full = request(slashing); + const batches = planImportBatches( + { ...full, validatorsImportRequest: full.validatorsImportRequest.slice(0, 3) }, + keys + ); + const sent: number[][] = []; + const { results, stopped } = await runImportBatches({ + batches, + results: keys.map(notAttempted), + importBatch: async (batch) => { + sent.push(batch.validatorsImportRequest.map((_, i) => i)); + return imported(batch); + }, + onProgress: () => undefined, + shouldStop: () => false + }); + expect(stopped).to.equal(undefined); + expect(results.map(({ status }) => status)).to.deep.equal(["imported", "error", "imported"]); + expect(results[1].message).to.include("too large"); + expect(sent).to.have.length(1); + }); + + it("leaves a malformed slashing file untouched for the signer to report", () => { + expect(filterInterchange("{not json", new Set(pubkeys))).to.equal("{not json"); + expect(planImportBatches(request(), pubkeys)[0].request.slashing_protection).to.equal(undefined); + }); + + it("matches slashing entries regardless of 0x prefix and hex casing", () => { + const data = JSON.stringify({ metadata: {}, data: [{ pubkey: pubkeys[0].slice(2).toUpperCase() }] }); + expect(JSON.parse(filterInterchange(data, new Set([pubkeys[0]]))).data).to.have.length(1); + }); + + it("reports progress after each batch and keeps results aligned with files", async () => { + const progress: ImportProgress[] = []; + const { results, stopped } = await runImportBatches({ + batches: planImportBatches(request(), pubkeys), + results: pubkeys.map(notAttempted), + importBatch: async (batch) => imported(batch), + onProgress: (update) => progress.push(update), + shouldStop: () => false + }); + expect(stopped).to.equal(undefined); + expect(progress.map(({ done }) => done)).to.deep.equal([0, 20, 40, 45]); + expect(results.every(({ status }) => status === "imported")).to.equal(true); + }); + + it("stops after a failed batch and leaves the rest retryable", async () => { + let calls = 0; + const { results, stopped } = await runImportBatches({ + batches: planImportBatches(request(), pubkeys), + results: pubkeys.map(notAttempted), + importBatch: async (batch) => { + if (++calls === 2) throw new Error("Connection lost or request timed out."); + return imported(batch); + }, + onProgress: () => undefined, + shouldStop: () => false + }); + expect(stopped).to.equal("error"); + expect(calls).to.equal(2); + expect(summarizeImport(results)).to.deep.equal({ imported: 20, error: 20, [NOT_ATTEMPTED]: 5 }); + expect(results[25].message).to.include("timed out"); + expect(retryableIndexes(results)).to.deep.equal(pubkeys.map((_, index) => index).slice(20)); + }); + + it("honours a stop request between batches", async () => { + let stop = false; + const { results, stopped } = await runImportBatches({ + batches: planImportBatches(request(), pubkeys), + results: pubkeys.map(notAttempted), + importBatch: async (batch) => { + stop = true; + return imported(batch); + }, + onProgress: () => undefined, + shouldStop: () => stop + }); + expect(stopped).to.equal("user"); + expect(summarizeImport(results)).to.deep.equal({ imported: 20, [NOT_ATTEMPTED]: 25 }); + }); + + it("retries only the requested files and keeps earlier results", async () => { + const earlier = pubkeys.map((_, index) => + index < 40 ? { status: "imported", message: "" } : { status: "error", message: "boom" } + ); + const sent: number[] = []; + const { results } = await runImportBatches({ + batches: planImportBatches(request(), pubkeys, { only: retryableIndexes(earlier) }), + results: earlier, + importBatch: async (batch) => { + sent.push(batch.validatorsImportRequest.length); + return { data: batch.validatorsImportRequest.map(() => ({ status: "duplicate", message: "" })) }; + }, + onProgress: () => undefined, + shouldStop: () => false + }); + expect(sent).to.deep.equal([5]); + expect(summarizeImport(results)).to.deep.equal({ imported: 40, duplicate: 5 }); + }); + + it("marks keys without a returned result as errors instead of success", async () => { + const { results } = await runImportBatches({ + batches: planImportBatches(request(), pubkeys.slice(0, 2), { only: [0, 1] }), + results: [notAttempted(), notAttempted()], + importBatch: async () => ({ data: [{ status: "imported", message: "" }] }), + onProgress: () => undefined, + shouldStop: () => false + }); + expect(results.map(({ status }) => status)).to.deep.equal(["imported", "error"]); + }); +}); diff --git a/packages/ui/test/importForm.test.ts b/packages/ui/test/importForm.test.ts index b502a21f..5b1be350 100644 --- a/packages/ui/test/importForm.test.ts +++ b/packages/ui/test/importForm.test.ts @@ -1,5 +1,10 @@ import { expect } from "chai"; -import { appendImportFiles, isImportFormValid, ImportFormState } from "../src/utils/importForm.js"; +import { + appendImportFiles, + incompleteImportIndexes, + isImportFormValid, + ImportFormState +} from "../src/utils/importForm.js"; import { createImportRequest } from "../src/utils/createImportRequest.js"; import { extractPubkey } from "../src/utils/dataUtils.js"; @@ -67,3 +72,17 @@ describe("Bulk import form", () => { } }); }); + +describe("Large import forms", () => { + it("points at the files that still need input", () => { + const acceptedFiles = ["a", "b", "c"].map((name) => ({ file: new File(["{}"], name), pubkey: "0x" + name })); + expect( + incompleteImportIndexes({ + acceptedFiles, + passwords: ["pw", "", "pw"], + tags: ["solo", "solo", ""], + feeRecipients: ["0x" + "ab".repeat(20), "0x" + "ab".repeat(20), ""] + }) + ).to.deep.equal([1, 2]); + }); +}); diff --git a/packages/ui/test/listFollowUp.test.ts b/packages/ui/test/listFollowUp.test.ts new file mode 100644 index 00000000..40bd57c8 --- /dev/null +++ b/packages/ui/test/listFollowUp.test.ts @@ -0,0 +1,18 @@ +import { expect } from "chai"; +import { countUnfinished, nextStallCount } from "../src/utils/listFollowUp.js"; + +describe("Validator list follow-up requests", () => { + it("counts only rows whose client reads were cut off", () => { + expect(countUnfinished([{ detailsPending: true }, { detailsPending: false }, { detailsPending: true }])).to.equal( + 2 + ); + }); + + it("stops after follow-ups that make no progress and resets on progress", () => { + expect(nextStallCount(0, undefined, 10)).to.equal(0); + expect(nextStallCount(0, 10, 6)).to.equal(0); + expect(nextStallCount(0, 6, 6)).to.equal(1); + expect(nextStallCount(2, 6, 7)).to.equal(3); + expect(nextStallCount(2, 6, 0)).to.equal(0); + }); +}); diff --git a/yarn.lock b/yarn.lock index afe3f120..bbbb3493 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1052,6 +1052,7 @@ __metadata: rimraf: "npm:^6.1.3" sinon: "npm:^22.1.0" socket.io: "npm:^4.8.4" + socket.io-client: "npm:^4.8.4" typescript: "npm:~6.0.3" languageName: unknown linkType: soft